tao.chen/model-platform
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

refactor: remove local fs, add config class to common pacakge

Browse Source
This commit is contained in:
tao.chen
2026-07-31 12:46:31 +08:00
parent 872388775a
commit f288c90d37
18 changed files with 192 additions and 343 deletions
Download Patch File Download Diff File Expand all files Collapse all files
schedule/src/schedule/execution.py
+6 -12
View File
@@ -36,7 +36,6 @@ async def _execute_notebook(
*,
artifact_name: str,
arguments: list[str],
workspace_root: Path,
timeout_seconds: int,
) -> ExecutionResult:
output = artifact.with_name(f"executed-{artifact_name}")
@@ -48,13 +47,11 @@ async def _execute_notebook(
str(artifact),
"--output",
str(output),
"--workspace",
str(workspace_root),
"--timeout",
str(max(1, timeout_seconds)),
"--arguments-json",
json.dumps(arguments, ensure_ascii=False),
cwd=str(workspace_root),
cwd=str(artifact.parent),
stdout=asyncio.subprocess.PIPE,
stderr=asyncio.subprocess.STDOUT,
)
@@ -109,14 +106,13 @@ async def _execute_python(
artifact: Path,
*,
arguments: list[str],
workspace_root: Path,
timeout_seconds: int,
) -> ExecutionResult:
process = await asyncio.create_subprocess_exec(
sys.executable,
str(artifact),
*arguments,
cwd=str(workspace_root),
cwd=str(artifact.parent),
stdout=asyncio.subprocess.PIPE,
stderr=asyncio.subprocess.STDOUT,
)
@@ -172,16 +168,16 @@ async def execute_artifact(
script_type: str,
artifact_path: str,
arguments: list[str],
workspace_root: Path,
timeout_seconds: int,
) -> ExecutionResult:
runtime_root = workspace_root / "runtime_tmp" / "schedule-runs"
runtime_root.mkdir(parents=True, exist_ok=True)
# Stage the artifact under Python's system temp dir (cleaned on context
# exit). No local-FS volume assumption; the bytes only live for the
# duration of the subprocess.
suffix = ".ipynb" if script_type == "notebook" else ".py"
raw_name = PurePosixPath(artifact_path.replace("\\", "/")).name
artifact_name = raw_name if raw_name.endswith(suffix) else f"artifact{suffix}"
prefix = f"{run_id[-6:]}-{node_run_id[-6:]}-"
with tempfile.TemporaryDirectory(prefix=prefix, dir=runtime_root) as directory:
with tempfile.TemporaryDirectory(prefix=prefix) as directory:
artifact = Path(directory) / artifact_name
artifact.write_bytes(source)
if script_type == "notebook":
@@ -189,14 +185,12 @@ async def execute_artifact(
artifact,
artifact_name=artifact_name,
arguments=arguments,
workspace_root=workspace_root,
timeout_seconds=timeout_seconds,
)
if script_type == "python":
return await _execute_python(
artifact,
arguments=arguments,
workspace_root=workspace_root,
timeout_seconds=timeout_seconds,
)
raise ValueError(f"unsupported script_type: {script_type}")
schedule/src/schedule/main.py
+4 -8
View File
@@ -1,10 +1,9 @@
from __future__ import annotations
import os
from contextlib import asynccontextmanager
from pathlib import Path
from typing import Any, AsyncIterator
from common.config import settings
from common.db import create_database_engine, create_session_factory
from common.service_app import create_service_app
from schedule.service import (
@@ -17,7 +16,7 @@ from schedule.storage_client import SchedulerStorageClient
@asynccontextmanager
async def lifespan(app: Any) -> AsyncIterator[None]:
engine = create_database_engine(os.environ["DATABASE_URL"])
engine = create_database_engine(settings.database_url)
session_factory = create_session_factory(engine)
backend_http_client = build_storage_http_client()
service = SchedulerService(
@@ -25,10 +24,7 @@ async def lifespan(app: Any) -> AsyncIterator[None]:
backend_http_client=backend_http_client,
object_store=build_object_store(),
storage_client=SchedulerStorageClient(backend_http_client),
workspace_root=Path(
os.getenv("WORKSPACE_ROOT", "/workspace/workspaces")
),
database_url=os.environ["DATABASE_URL"],
database_url=settings.database_url,
)
app.state.scheduler_service = service
await service.start()
@@ -41,6 +37,6 @@ async def lifespan(app: Any) -> AsyncIterator[None]:
app = create_service_app(
os.getenv("SERVICE_NAME", "scheduler-worker"),
settings.service_name,
lifespan=lifespan,
)
schedule/src/schedule/notebook_runner.py
+4 -3
View File
@@ -36,14 +36,12 @@ def main() -> None:
parser = argparse.ArgumentParser()
parser.add_argument("--input", required=True)
parser.add_argument("--output", required=True)
parser.add_argument("--workspace", required=True)
parser.add_argument("--timeout", required=True, type=int)
parser.add_argument("--arguments-json", default="[]")
args = parser.parse_args()
source = Path(args.input)
output = Path(args.output)
workspace = Path(args.workspace)
arguments = json.loads(args.arguments_json)
if not isinstance(arguments, list) or not all(
isinstance(item, str) for item in arguments
@@ -68,7 +66,10 @@ def main() -> None:
kernel_name="python3",
allow_errors=False,
)
client.execute(cwd=str(workspace))
# No explicit cwd — the kernel inherits the parent's cwd, which the
# scheduler sets to the staged artifact directory. Keeping it here
# avoids any "cwd must exist" requirement on the host.
client.execute()
except Exception as exc:
traceback.print_exc()
exit_code = 124 if "timeout" in type(exc).__name__.lower() else 1
schedule/src/schedule/service.py
+7 -14
View File
@@ -18,13 +18,13 @@ Backend that ``CronScheduler`` calls at every cron tick.
from __future__ import annotations
import logging
import os
from datetime import UTC, datetime
from typing import Any
import httpx
from sqlalchemy.ext.asyncio import AsyncSession, async_sessionmaker
from common.config import settings
from common.ids import new_ulid
from schedule.orchestrator import DispatchOrchestrator
@@ -52,14 +52,12 @@ class SchedulerService:
backend_http_client: httpx.AsyncClient,
object_store: Any,
storage_client: Any,
workspace_root: Any,
database_url: str,
) -> None:
self.session_factory = session_factory
self.backend_http_client = backend_http_client
self.object_store = object_store
self.storage_client = storage_client
self.workspace_root = workspace_root
self.database_url = database_url
# Wire worker BEFORE orchestrator: orchestrator's dispatch table
@@ -69,7 +67,6 @@ class SchedulerService:
session_factory=session_factory,
object_store=object_store,
storage_client=storage_client,
workspace_root=workspace_root,
)
self.orchestrator = DispatchOrchestrator(
session_factory=session_factory,
@@ -162,20 +159,16 @@ class SchedulerService:
def build_object_store() -> Any:
"""Construct a boto3 S3 client pointed at RustFS.
Reads ``RUSTFS_ENDPOINT`` (full URL), ``RUSTFS_ACCESS_KEY``, and
``RUSTFS_SECRET_KEY``. Falls back to ``http://rustfs:9000`` for the
endpoint — that's the default docker-compose service name.
Reads ``rustfs_endpoint`` / ``rustfs_access_key`` / ``rustfs_secret_key``
from :data:`common.config.settings`.
"""
import boto3
return boto3.client(
"s3",
endpoint_url=os.getenv(
"RUSTFS_ENDPOINT",
"http://rustfs:9000",
),
aws_access_key_id=os.environ["RUSTFS_ACCESS_KEY"],
aws_secret_access_key=os.environ["RUSTFS_SECRET_KEY"],
endpoint_url=settings.rustfs_endpoint,
aws_access_key_id=settings.rustfs_access_key,
aws_secret_access_key=settings.rustfs_secret_key,
region_name="us-east-1",
)
@@ -183,7 +176,7 @@ def build_object_store() -> Any:
def build_storage_http_client() -> httpx.AsyncClient:
"""Construct the httpx client that talks to Backend's HTTP API."""
return httpx.AsyncClient(
base_url=os.getenv("BACKEND_API_URL", "http://backend:8000"),
base_url=settings.backend_api_url,
timeout=httpx.Timeout(60.0),
)
schedule/src/schedule/worker.py
-7
View File
@@ -50,12 +50,10 @@ class NodeExecutor:
session_factory,
object_store: Any,
storage_client: Any,
workspace_root: Path,
) -> None:
self.session_factory = session_factory
self.object_store = object_store
self.storage_client = storage_client
self.workspace_root = workspace_root
async def handle_node_execute(
self,
@@ -76,10 +74,6 @@ class NodeExecutor:
object_key=context["object_key"],
content_hash=context["content_hash"],
)
workspace_root = (
self.workspace_root / context["workspace_code"]
).resolve()
workspace_root.mkdir(parents=True, exist_ok=True)
result = await execute_artifact(
content,
run_id=payload["run_id"],
@@ -87,7 +81,6 @@ class NodeExecutor:
script_type=payload["script_type"],
artifact_path=payload["artifact_path"],
arguments=[str(item) for item in payload.get("arguments", [])],
workspace_root=workspace_root,
timeout_seconds=int(payload["timeout_seconds"]),
)
except Exception as exc: