From 2c54f8818125c4996bdce6709ee135d3a4b9f971 Mon Sep 17 00:00:00 2001 From: "tao.chen" <93983997+taochen-ct@users.noreply.github.com> Date: Mon, 3 Aug 2026 19:12:40 +0800 Subject: [PATCH 01/19] fix: notebook-dir --- runtime/src/runtime/process.py | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/runtime/src/runtime/process.py b/runtime/src/runtime/process.py index 68d1b10..ccf40d9 100644 --- a/runtime/src/runtime/process.py +++ b/runtime/src/runtime/process.py @@ -160,7 +160,7 @@ def _bump_last_used(record: JupyterProcessRecord) -> None: async def start_workspace(ws_id: str) -> dict: async with get_workspace_lock(ws_id): - workspace_path = WORKSPACES_ROOT / ws_id + workspace_path = (WORKSPACES_ROOT / ws_id).resolve() workspace_path.mkdir(parents=True, exist_ok=True) if ws_id in JUPYTER_PROCESSES: @@ -202,7 +202,7 @@ async def start_workspace(ws_id: str) -> dict: "--allow-root", f"--ServerApp.token={token}", f"--ServerApp.base_url={base_path}", - "--notebook-dir=.", + f"--notebook-dir={workspace_path}", "--ServerApp.terminals_enabled=False", "--NotebookApp.terminals_enabled=False", "--ServerApp.allow_origin=*", @@ -212,7 +212,7 @@ async def start_workspace(ws_id: str) -> dict: ] try: - process, log_file = start_process(cmd, workspace_path) + process, log_file = start_process(cmd, workspace_path.as_posix()) except Exception as e: logger.error(f"Failed to start Jupyter for {ws_id}: {e}") raise HTTPException( From 6c4bf4748754541d5fce7d3f0836f2023b61f986 Mon Sep 17 00:00:00 2001 From: "tao.chen" <93983997+taochen-ct@users.noreply.github.com> Date: Mon, 3 Aug 2026 19:43:51 +0800 Subject: [PATCH 02/19] update: hide top-panel --- runtime/Dockerfile | 2 ++ runtime/overrides.json | 5 +++++ 2 files changed, 7 insertions(+) create mode 100644 runtime/overrides.json diff --git a/runtime/Dockerfile b/runtime/Dockerfile index 124aa7e..3ab9720 100644 --- a/runtime/Dockerfile +++ b/runtime/Dockerfile @@ -37,5 +37,7 @@ COPY common ./common COPY runtime ./runtime RUN UV_DEFAULT_INDEX="https://pypi.tuna.tsinghua.edu.cn/simple/" uv sync --frozen --no-dev --no-editable --package runtime +COPY overrides.json /app/.venv/share/jupyter/lab/settings/overrides.json + EXPOSE 8000 CMD ["uv", "run", "--frozen", "--package", "runtime", "gunicorn", "--config", "runtime/gunicorn.conf.py", "runtime.main:app"] diff --git a/runtime/overrides.json b/runtime/overrides.json new file mode 100644 index 0000000..30bbed3 --- /dev/null +++ b/runtime/overrides.json @@ -0,0 +1,5 @@ +{ + "@jupyterlab/apputils-extension:themes": { + "rawOverrides": "#top-panel { display: none !important; height: 0 !important; }" + } +} \ No newline at end of file From e19340332a30f008a213f5fa6b9479761bf4401e Mon Sep 17 00:00:00 2001 From: "tao.chen" <93983997+taochen-ct@users.noreply.github.com> Date: Mon, 3 Aug 2026 19:51:24 +0800 Subject: [PATCH 03/19] update: hide top-panel --- runtime/Dockerfile | 4 +++- runtime/overrides.json | 5 ----- 2 files changed, 3 insertions(+), 6 deletions(-) delete mode 100644 runtime/overrides.json diff --git a/runtime/Dockerfile b/runtime/Dockerfile index 3ab9720..b2df1d7 100644 --- a/runtime/Dockerfile +++ b/runtime/Dockerfile @@ -37,7 +37,9 @@ COPY common ./common COPY runtime ./runtime RUN UV_DEFAULT_INDEX="https://pypi.tuna.tsinghua.edu.cn/simple/" uv sync --frozen --no-dev --no-editable --package runtime -COPY overrides.json /app/.venv/share/jupyter/lab/settings/overrides.json +RUN mkdir -p /root/.jupyter/custom /app/.venv/share/jupyter/custom +RUN echo '#top-panel { display: none !important; height: 0 !important; }' > /root/.jupyter/custom/custom.css && \ + cp /root/.jupyter/custom/custom.css /app/.venv/share/jupyter/custom/custom.css EXPOSE 8000 CMD ["uv", "run", "--frozen", "--package", "runtime", "gunicorn", "--config", "runtime/gunicorn.conf.py", "runtime.main:app"] diff --git a/runtime/overrides.json b/runtime/overrides.json deleted file mode 100644 index 30bbed3..0000000 --- a/runtime/overrides.json +++ /dev/null @@ -1,5 +0,0 @@ -{ - "@jupyterlab/apputils-extension:themes": { - "rawOverrides": "#top-panel { display: none !important; height: 0 !important; }" - } -} \ No newline at end of file From 85791ab1eae591a13d8e602eda19489a9433a0cc Mon Sep 17 00:00:00 2001 From: "tao.chen" <93983997+taochen-ct@users.noreply.github.com> Date: Mon, 3 Aug 2026 19:57:29 +0800 Subject: [PATCH 04/19] update: rclone add remote control --- runtime/src/runtime/mount.py | 3 +++ 1 file changed, 3 insertions(+) diff --git a/runtime/src/runtime/mount.py b/runtime/src/runtime/mount.py index 97cfbff..5c07a5e 100644 --- a/runtime/src/runtime/mount.py +++ b/runtime/src/runtime/mount.py @@ -68,6 +68,9 @@ def start_rclone_mount() -> None: "--dir-cache-time", "30s", "--poll-interval", "30s", "--log-level", "INFO", + "--rc", + "--rc-addr 0.0.0.0:5572", + "--rc-no-auth", ] RCLONE_PROCESS = subprocess.Popen( cmd, From e1172a00917ca58ff47aaa6e357ede8dfe314ff1 Mon Sep 17 00:00:00 2001 From: "tao.chen" <93983997+taochen-ct@users.noreply.github.com> Date: Mon, 3 Aug 2026 20:02:42 +0800 Subject: [PATCH 05/19] update: rclone add remote control --- runtime/src/runtime/mount.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/runtime/src/runtime/mount.py b/runtime/src/runtime/mount.py index 5c07a5e..bfcbe34 100644 --- a/runtime/src/runtime/mount.py +++ b/runtime/src/runtime/mount.py @@ -69,7 +69,7 @@ def start_rclone_mount() -> None: "--poll-interval", "30s", "--log-level", "INFO", "--rc", - "--rc-addr 0.0.0.0:5572", + "--rc-addr", "0.0.0.0:5572", "--rc-no-auth", ] RCLONE_PROCESS = subprocess.Popen( From 134f8ca552f7abce903c5180351805ddbca1f1cd Mon Sep 17 00:00:00 2001 From: "tao.chen" <93983997+taochen-ct@users.noreply.github.com> Date: Mon, 3 Aug 2026 20:23:18 +0800 Subject: [PATCH 06/19] feat: refresh VFS --- .env.example | 7 +++++++ backend/src/backend/main.py | 8 ++++++++ backend/src/backend/scripts.py | 13 +++++++++++++ common/src/common/config.py | 4 ++++ 4 files changed, 32 insertions(+) diff --git a/.env.example b/.env.example index 2d6ab09..481b536 100644 --- a/.env.example +++ b/.env.example @@ -44,3 +44,10 @@ RUSTFS_VERSION_BUCKET=versions RUSTFS_RUN_LOG_BUCKET=run-logs RUSTFS_TRASH_BUCKET=trash RUSTFS_TRASH_RETENTION_DAYS=30 + +# rclone RC (HTTP control API). The runtime container starts rclone with +# `--rc --rc-addr 0.0.0.0:5572 --rc-no-auth` (see runtime/src/runtime/mount.py), +# so the backend can POST /vfs/refresh here to invalidate the FUSE dir-cache +# after writing new workspace files. Default points at the runtime service +# over the compose network. +RCLONE_RC_URL=http://runtime:5572 diff --git a/backend/src/backend/main.py b/backend/src/backend/main.py index c390cd4..6752caf 100644 --- a/backend/src/backend/main.py +++ b/backend/src/backend/main.py @@ -16,6 +16,7 @@ from backend.auth import router as auth_router from backend.jupyter import router as jupyter_router from backend.resources import router as resources_router from backend.runtime_client import RuntimeClient +from backend.rclone_rc_client import RcloneRCClient from backend.schedule_runs import router as schedule_runs_router from backend.schedules import router as schedules_router from backend.scripts import router as scripts_router @@ -59,9 +60,16 @@ async def lifespan(app: Any) -> AsyncIterator[None]: timeout=httpx.Timeout(30.0), ) app.state.runtime_client = RuntimeClient(runtime_http_client) + # Short timeout — refresh is best-effort and runs in a BackgroundTask. + rclone_http_client = httpx.AsyncClient( + base_url=settings.rclone_rc_url, + timeout=httpx.Timeout(30.0), + ) + app.state.rclone_rc_client = RcloneRCClient(rclone_http_client) try: yield finally: + await rclone_http_client.aclose() await runtime_http_client.aclose() await storage_http_client.aclose() await engine.dispose() diff --git a/backend/src/backend/scripts.py b/backend/src/backend/scripts.py index 5a1ace7..92e87f4 100644 --- a/backend/src/backend/scripts.py +++ b/backend/src/backend/scripts.py @@ -10,6 +10,7 @@ from typing import Any from fastapi import ( APIRouter, + BackgroundTasks, Depends, Header, HTTPException, @@ -344,6 +345,7 @@ async def create_script_record( async def create_script( payload: CreateScriptRequest, request: Request, + background_tasks: BackgroundTasks, context: RequestContext = Depends(request_context), session: AsyncSession = Depends(database_session), ) -> dict[str, Any]: @@ -359,6 +361,11 @@ async def create_script( context=context, session=session, ) + background_tasks.add_task( + request.app.state.rclone_rc_client.vfs_refresh, + dir_path=context.workspace.workspace_id, + recursive=True, + ) return { "request_id": context.request_id, "data": script_payload(script, storage_data), @@ -372,6 +379,7 @@ async def create_script( ) async def upload_script( request: Request, + background_tasks: BackgroundTasks, file_name: str = Query(min_length=1, max_length=255), parent_path: str = Query(default="", max_length=1024), visibility: str = Query( @@ -413,6 +421,11 @@ async def upload_script( context=context, session=session, ) + background_tasks.add_task( + request.app.state.rclone_rc_client.vfs_refresh, + dir_path=context.workspace.workspace_id, + recursive=True, + ) return { "request_id": context.request_id, "data": script_payload(script, storage_data), diff --git a/common/src/common/config.py b/common/src/common/config.py index 290c2e7..9a2eae3 100644 --- a/common/src/common/config.py +++ b/common/src/common/config.py @@ -54,6 +54,10 @@ class Settings(BaseSettings): default="http://runtime:8000", description="Backend → Runtime HTTP endpoint.", ) + rclone_rc_url: str = Field( + default="http://runtime:5572", + description="Backend → rclone RC HTTP endpoint (VFS cache invalidation).", + ) # ── RustFS object storage ──────────────────────────────────── rustfs_endpoint: str = Field( From 939a579b07c9abc53fb7a70d643ccc2f005ad6ea Mon Sep 17 00:00:00 2001 From: "tao.chen" <93983997+taochen-ct@users.noreply.github.com> Date: Mon, 3 Aug 2026 20:37:56 +0800 Subject: [PATCH 07/19] feat: rclone client --- backend/src/backend/rclone_rc_client.py | 92 +++++++++++++++++++++++++ 1 file changed, 92 insertions(+) create mode 100644 backend/src/backend/rclone_rc_client.py diff --git a/backend/src/backend/rclone_rc_client.py b/backend/src/backend/rclone_rc_client.py new file mode 100644 index 0000000..0be8d55 --- /dev/null +++ b/backend/src/backend/rclone_rc_client.py @@ -0,0 +1,92 @@ +"""rclone RC client. + +Used by the backend to invalidate the rclone FUSE directory cache after +writing a new workspace object (notebook / script / upload). The runtime +container already starts rclone with ``--rc --rc-addr 0.0.0.0:5572 +--rc-no-auth`` (see ``runtime/src/runtime/mount.py``), so we only need +an HTTP client here — no extra runtime plumbing. + +Failure is logged and swallowed: the VFS refresh is best-effort and +must never bubble up to the API caller. +""" + +from __future__ import annotations + +from dataclasses import dataclass +from typing import Any + +import httpx +from loguru import logger + + +@dataclass(frozen=True) +class RcloneRCError(Exception): + status_code: int + detail: Any + + +_RCLONE_RC_TRANSPORT_ERROR = RcloneRCError( + 503, + { + "code": "RCLONE_RC_UNAVAILABLE", + "message": "rclone RC 暂时不可用", + "retryable": True, + "details": {}, + }, +) + + +class RcloneRCClient: + def __init__(self, client: httpx.AsyncClient) -> None: + self.client = client + + async def _request( + self, + method: str, + path: str, + payload: dict[str, Any], + ) -> dict[str, Any]: + try: + response = await self.client.request(method, path, json=payload) + except httpx.RequestError as exc: + raise _RCLONE_RC_TRANSPORT_ERROR from exc + if response.is_error: + try: + detail = response.json().get("detail", response.text) + except ValueError: + detail = response.text + raise RcloneRCError(response.status_code, detail) + return response.json() + + async def vfs_refresh( + self, + dir_path: str, + *, + recursive: bool = True, + ) -> None: + """Invalidate the FUSE dir-cache for ``dir_path``. + + Fires ``POST /vfs/refresh`` against the rclone RC server with + ``_async=true`` so the call returns immediately while rclone + performs the directory walk in the background. Errors are + logged and swallowed — a refresh failure must never fail the + API call that triggered it. + """ + try: + await self._request( + "POST", + "/vfs/refresh", + { + "dir": dir_path, + "recursive": recursive, + "_async": True, + }, + ) + logger.info( + f"vfs_refresh dir={dir_path} recursive={recursive} ok" + ) + except Exception as exc: # noqa: BLE001 - best-effort + logger.warning(f"vfs_refresh dir={dir_path} failed: {exc}") + + +__all__ = ["RcloneRCClient", "RcloneRCError"] \ No newline at end of file From 1bd7da384ca890dfb0aaadfb98177b20c1a9943d Mon Sep 17 00:00:00 2001 From: "tao.chen" <93983997+taochen-ct@users.noreply.github.com> Date: Mon, 3 Aug 2026 20:45:00 +0800 Subject: [PATCH 08/19] feat: add loguru --- backend/pyproject.toml | 1 + schedule/pyproject.toml | 1 + uv.lock | 4 ++++ 3 files changed, 6 insertions(+) diff --git a/backend/pyproject.toml b/backend/pyproject.toml index 893bbda..f51f2ed 100644 --- a/backend/pyproject.toml +++ b/backend/pyproject.toml @@ -13,6 +13,7 @@ dependencies = [ "gunicorn>=26.0.0", "passlib==1.7.4", "bcrypt>=4.0,<4.1", + "loguru>=0.7.2", ] [tool.uv.sources] diff --git a/schedule/pyproject.toml b/schedule/pyproject.toml index ec08d53..4f5230b 100644 --- a/schedule/pyproject.toml +++ b/schedule/pyproject.toml @@ -13,6 +13,7 @@ dependencies = [ "nbformat==5.10.4", "ipykernel==6.29.5", "gunicorn>=26.0.0", + "loguru>=0.7.2", ] [tool.uv.sources] diff --git a/uv.lock b/uv.lock index e8943c4..f37b795 100644 --- a/uv.lock +++ b/uv.lock @@ -201,6 +201,7 @@ dependencies = [ { name = "fastapi" }, { name = "gunicorn" }, { name = "httpx" }, + { name = "loguru" }, { name = "passlib" }, { name = "uvicorn", extra = ["standard"] }, ] @@ -215,6 +216,7 @@ requires-dist = [ { name = "fastapi", specifier = "==0.116.1" }, { name = "gunicorn", specifier = ">=26.0.0" }, { name = "httpx", specifier = "==0.28.1" }, + { name = "loguru", specifier = ">=0.7.2" }, { name = "passlib", specifier = "==1.7.4" }, { name = "uvicorn", extras = ["standard"], specifier = "==0.35.0" }, ] @@ -1955,6 +1957,7 @@ dependencies = [ { name = "gunicorn" }, { name = "httpx" }, { name = "ipykernel" }, + { name = "loguru" }, { name = "nbclient" }, { name = "nbformat" }, { name = "pymysql" }, @@ -1969,6 +1972,7 @@ requires-dist = [ { name = "gunicorn", specifier = ">=26.0.0" }, { name = "httpx", specifier = "==0.28.1" }, { name = "ipykernel", specifier = "==6.29.5" }, + { name = "loguru", specifier = ">=0.7.2" }, { name = "nbclient", specifier = "==0.10.2" }, { name = "nbformat", specifier = "==5.10.4" }, { name = "pymysql", specifier = "==1.2.0" }, From 078d8f86878bca0b246291c1047644359cf03f85 Mon Sep 17 00:00:00 2001 From: "tao.chen" <93983997+taochen-ct@users.noreply.github.com> Date: Tue, 4 Aug 2026 09:37:14 +0800 Subject: [PATCH 09/19] update: delete vfs refresh --- backend/src/backend/scripts.py | 20 ++++++++++---------- 1 file changed, 10 insertions(+), 10 deletions(-) diff --git a/backend/src/backend/scripts.py b/backend/src/backend/scripts.py index 92e87f4..f326cb0 100644 --- a/backend/src/backend/scripts.py +++ b/backend/src/backend/scripts.py @@ -361,11 +361,11 @@ async def create_script( context=context, session=session, ) - background_tasks.add_task( - request.app.state.rclone_rc_client.vfs_refresh, - dir_path=context.workspace.workspace_id, - recursive=True, - ) + # background_tasks.add_task( + # request.app.state.rclone_rc_client.vfs_refresh, + # dir_path=context.workspace.workspace_id, + # recursive=True, + # ) return { "request_id": context.request_id, "data": script_payload(script, storage_data), @@ -421,11 +421,11 @@ async def upload_script( context=context, session=session, ) - background_tasks.add_task( - request.app.state.rclone_rc_client.vfs_refresh, - dir_path=context.workspace.workspace_id, - recursive=True, - ) + # background_tasks.add_task( + # request.app.state.rclone_rc_client.vfs_refresh, + # dir_path=context.workspace.workspace_id, + # recursive=True, + # ) return { "request_id": context.request_id, "data": script_payload(script, storage_data), From d45109916a76c41a2f939ae1b81b2cc87e4180ea Mon Sep 17 00:00:00 2001 From: "tao.chen" <93983997+taochen-ct@users.noreply.github.com> Date: Tue, 4 Aug 2026 10:30:24 +0800 Subject: [PATCH 10/19] chore: update docker-compose.yml --- docker-compose.yml | 4 +--- 1 file changed, 1 insertion(+), 3 deletions(-) diff --git a/docker-compose.yml b/docker-compose.yml index 117cec6..d08deb7 100644 --- a/docker-compose.yml +++ b/docker-compose.yml @@ -114,9 +114,7 @@ services: volumes: - ./runtime:/app/runtime healthcheck: - test: - - CMD-SHELL - - grep -q ' /app/workspaces .* - fuse.rclone ' /proc/self/mountinfo && curl -fsS http://127.0.0.1:8000/api/v1/health >/dev/null + test: ["CMD-SHELL", "grep -q ' /app/workspaces .* - fuse.rclone ' /proc/self/mountinfo && curl -fsS http://127.0.0.1:8000/api/v1/health >/dev/null"] interval: 10s timeout: 5s retries: 18 From 173d562bd11ba9ef452d10d94b5011d1337b7a15 Mon Sep 17 00:00:00 2001 From: "tao.chen" <93983997+taochen-ct@users.noreply.github.com> Date: Tue, 4 Aug 2026 11:06:52 +0800 Subject: [PATCH 11/19] update: debug create file by jupyter --- backend/src/backend/runtime_client.py | 130 +++++++++++++++++++++++++- backend/src/backend/scripts.py | 64 ++++++++++--- 2 files changed, 178 insertions(+), 16 deletions(-) diff --git a/backend/src/backend/runtime_client.py b/backend/src/backend/runtime_client.py index da3708e..7a13a6d 100644 --- a/backend/src/backend/runtime_client.py +++ b/backend/src/backend/runtime_client.py @@ -4,7 +4,7 @@ from dataclasses import dataclass from typing import Any import httpx - +from loguru import logger @dataclass(frozen=True) class RuntimeClientError(Exception): @@ -77,5 +77,133 @@ class RuntimeClient: {"action": "start", "workspace_id": workspace_id}, ) + async def _ensure_workspace( + self, + workspace_id: str, + ) -> dict[str, Any]: + """Return a running workspace descriptor, starting it if needed. + + Mirrors the lazy-start pattern used by + :func:`backend.jupyter.verify_jupyter_access`: try ``get`` + first, fall through to ``start`` if the workspace is not yet + running. Bumps ``last_used_at`` via the runtime registry on the + way in, so the idle reaper is satisfied for the duration of the + subsequent Jupyter call. + """ + ws = await self.get_workspace(workspace_id) + if not ws or ws.get("status") != "running": + ws = await self.start_workspace(workspace_id) + if not ws.get("port") or not ws.get("token"): + raise RuntimeClientError( + 500, + { + "code": "JUPYTER_DESCRIPTOR_INVALID", + "message": "Runtime returned no port/token for workspace", + "retryable": False, + "details": {"workspace_id": workspace_id}, + }, + ) + return ws + + async def _jupyter_contents_post( + self, + workspace_id: str, + ws: dict[str, Any], + body: dict[str, Any], + ) -> dict[str, Any]: + """POST to the workspace Jupyter's ``/api/contents/`` directly. + + Builds the URL from the per-workspace ``base_url`` + ``port`` + and authenticates with the runtime-issued token. We reuse the + existing ``httpx.AsyncClient`` (its ``base_url`` is overridden + by the absolute URL we hand it). + """ + url = ( + f"{ws['base_url']}:{ws['port']}" + f"/jupyter/{workspace_id}/api/contents/" + ) + logger.debug(url) + logger.debug(body) + headers = {"Authorization": f"token {ws['token']}"} + try: + response = await self.client.post( + url, json=body, headers=headers + ) + except httpx.RequestError as exc: + raise _RUNTIME_TRANSPORT_ERROR from exc + if response.is_error: + try: + detail = response.json() + except ValueError: + detail = response.text + raise RuntimeClientError(response.status_code, detail) + return response.json() + + async def create_notebook( + self, + workspace_id: str, + *, + name: str, + cells: list[dict[str, Any]] | None = None, + ) -> dict[str, Any]: + """Create a notebook in the workspace root. + + Auto-starts the workspace's Jupyter if it is not yet running, + then posts to the Jupyter contents API + (``/jupyter/{ws_id}/api/contents/``) directly using the + runtime-issued token. Returns the Jupyter contents descriptor + (``path``, ``type``, ``created``, ``writable``...). + + ``name`` must be a safe basename (no path separators); the + Jupyter side will reject anything that escapes the workspace + root. + """ + ws = await self._ensure_workspace(workspace_id) + logger.debug(ws) + body = { + "type": "notebook", + "name": name, + "content": { + "cells": cells if cells is not None else [], + "metadata": {}, + "nbformat": 4, + "nbformat_minor": 5, + }, + } + + return await self._jupyter_contents_post(workspace_id, ws, body) + + async def upload_file( + self, + workspace_id: str, + *, + name: str, + content: str, + content_type: str = "text/plain", + ) -> dict[str, Any]: + """Upload a text file to the workspace root. + + ``content_type`` selects the Jupyter ``format`` field: anything + under ``text/`` or ``application/json`` is sent as ``text``. + Binary uploads are not yet supported and raise ``ValueError`` + — the Jupyter contents API base64 pathway is not wired up here. + """ + if not ( + content_type.startswith("text/") + or content_type == "application/json" + ): + raise ValueError( + f"content_type '{content_type}' is not supported; " + "only text/* and application/json are accepted" + ) + ws = await self._ensure_workspace(workspace_id) + body = { + "type": "file", + "name": name, + "content": content, + "format": "text", + } + return await self._jupyter_contents_post(workspace_id, ws, body) + __all__ = ["RuntimeClient", "RuntimeClientError"] diff --git a/backend/src/backend/scripts.py b/backend/src/backend/scripts.py index f326cb0..2b17766 100644 --- a/backend/src/backend/scripts.py +++ b/backend/src/backend/scripts.py @@ -7,7 +7,7 @@ import mimetypes from datetime import UTC, datetime from pathlib import PurePosixPath from typing import Any - +from loguru import logger from fastapi import ( APIRouter, BackgroundTasks, @@ -32,6 +32,7 @@ from backend.dependencies import ( database_session, request_context, ) +from backend.runtime_client import RuntimeClientError from backend.schemas import ( CreateScriptRequest, CreateWorkspaceDirectoryRequest, @@ -279,7 +280,7 @@ async def create_script_record( ) if parent_path is None else normalize_user_path(parent_path) child_path = f"{folder}/{name}" if folder else name relative_path = user_relative_path(context, child_path) - + logger.debug(relative_path) existing_script = await session.scalar( select(Scripts) .join( @@ -290,6 +291,7 @@ async def create_script_record( Scripts.workspace_id == context.workspace.workspace_id, StorageObjects.relative_path == relative_path, Scripts.status == "active", + Scripts.is_deleted == 0, ) ) if existing_script is not None: @@ -298,19 +300,51 @@ async def create_script_record( "a file with the same path already exists", ) - storage_data = await request.app.state.storage_client.create_server_object( - workspace_id=context.workspace.workspace_id, - user_id=context.user.user_id, - usage_type="working_copy", - file_name=name, - content_type=mimetypes.guess_type(name)[0] - or "application/octet-stream", - content=content, - visibility=visibility, - is_immutable=False, - idempotency_key=f"script:{context.workspace.workspace_id}:{relative_path}", - relative_path=relative_path, - ) + # Push the file directly to the workspace's live Jupyter instance. + # Auto-starts the workspace if no Jupyter is running yet. Once + # Jupyter has the file in its local mount, the rclone VFS will + # eventually replicate it back to object storage. + runtime_client = request.app.state.runtime_client + workspace_id = context.workspace.workspace_id + content_hash = hashlib.sha256(content).hexdigest() + size_bytes = len(content) + try: + if script_type == "notebook": + notebook = json.loads(content.decode("utf-8")) + logger.debug(notebook) + jupyter_resp = await runtime_client.create_notebook( + workspace_id, + name=name, + cells=notebook.get("cells"), + ) + else: + jupyter_resp = await runtime_client.upload_file( + workspace_id, + name=name, + content=content.decode("utf-8"), + content_type=( + mimetypes.guess_type(name)[0] or "text/plain" + ), + ) + logger.debug(jupyter_resp) + except RuntimeClientError as exc: + raise HTTPException( + status_code=exc.status_code, + detail=exc.detail, + ) from exc + + # Synthesise a storage_data-shaped dict from the Jupyter response + # so the existing script_payload + response shape keep working. + # The storage_object_id is a fresh ULID — there is no real + # StorageObject row for this file; downstream list/get operations + # that JOIN StorageObjects will skip jupyter-only scripts. + storage_data = { + "storage_object_id": new_ulid(), + "relative_path": relative_path, + "object_key": f"{workspace_id}/{relative_path}", + "content_hash": content_hash, + "size_bytes": size_bytes, + } object_id = storage_data["storage_object_id"] script = await session.scalar( select(Scripts).where(Scripts.current_object_id == object_id) From ddf112034d9bbee9499d99d02506d9c795c7318a Mon Sep 17 00:00:00 2001 From: "tao.chen" <93983997+taochen-ct@users.noreply.github.com> Date: Tue, 4 Aug 2026 11:29:33 +0800 Subject: [PATCH 12/19] update: jupyter api payload, wire os.environ --- backend/src/backend/runtime_client.py | 4 ++-- common/src/common/utils.py | 8 ++++++++ runtime/src/runtime/process.py | 20 ++++++++++++-------- 3 files changed, 22 insertions(+), 10 deletions(-) diff --git a/backend/src/backend/runtime_client.py b/backend/src/backend/runtime_client.py index 7a13a6d..8f022cb 100644 --- a/backend/src/backend/runtime_client.py +++ b/backend/src/backend/runtime_client.py @@ -162,7 +162,7 @@ class RuntimeClient: logger.debug(ws) body = { "type": "notebook", - "name": name, + "path": name, "content": { "cells": cells if cells is not None else [], "metadata": {}, @@ -199,7 +199,7 @@ class RuntimeClient: ws = await self._ensure_workspace(workspace_id) body = { "type": "file", - "name": name, + "path": name, "content": content, "format": "text", } diff --git a/common/src/common/utils.py b/common/src/common/utils.py index 4631119..40725ec 100644 --- a/common/src/common/utils.py +++ b/common/src/common/utils.py @@ -12,6 +12,7 @@ specific concerns: from __future__ import annotations +import os import socket import subprocess import time @@ -30,6 +31,7 @@ def start_process( cmd: list[str], workspace_path: Path, log_dir: str | Path = "/tmp/process_logs", + env: dict[str, str] | None = None, ) -> tuple[subprocess.Popen, Path]: """Launch ``cmd`` as a subprocess and return ``(process, log_file)``. @@ -44,6 +46,11 @@ def start_process( start_time = time.strftime("%Y%m%d_%H%M%S") temp_log = log_dir_path / f"process_start_{start_time}.log" + # 构建合并后的环境变量 + full_env = os.environ.copy() + if env: + full_env.update(env) + with open(temp_log, "a", buffering=1) as log_file: process = subprocess.Popen( cmd, @@ -51,6 +58,7 @@ def start_process( stdout=log_file, stderr=subprocess.STDOUT, start_new_session=True, + env=full_env, ) final_log = log_dir_path / f"process_{process.pid}_{start_time}.log" diff --git a/runtime/src/runtime/process.py b/runtime/src/runtime/process.py index ccf40d9..5e73f34 100644 --- a/runtime/src/runtime/process.py +++ b/runtime/src/runtime/process.py @@ -195,7 +195,8 @@ async def start_workspace(ws_id: str) -> dict: base_path = f"/jupyter/{ws_id}/" cmd = [ - "jupyter", "notebook", + "jupyter", + "notebook", f"--port={port}", "--ip=0.0.0.0", "--no-browser", @@ -212,12 +213,14 @@ async def start_workspace(ws_id: str) -> dict: ] try: - process, log_file = start_process(cmd, workspace_path.as_posix()) + process, log_file = start_process( + cmd, + workspace_path.as_posix(), + env={"PATH": f"/app/.venv/bin:{os.environ.get('PATH', '')}"}, + ) except Exception as e: logger.error(f"Failed to start Jupyter for {ws_id}: {e}") - raise HTTPException( - status_code=500, detail=f"Failed to start Jupyter: {e}" - ) + raise HTTPException(status_code=500, detail=f"Failed to start Jupyter: {e}") full_url = f"{PUBLIC_BASE_URL}:{port}{base_path}?token={token}" meta_path = _meta_path(ws_id) @@ -341,8 +344,7 @@ async def get_workspace(ws_id: str) -> dict: raise HTTPException( status_code=404, detail=( - f"Jupyter process for workspace '{ws_id}' " - "has terminated unexpectedly." + f"Jupyter process for workspace '{ws_id}' " "has terminated unexpectedly." ), ) @@ -416,7 +418,9 @@ def reconcile_processes() -> dict[str, int]: except PermissionError: alive = True # someone else's process, leave alone if not alive: - logger.info(f"reconcile: dropping stale sidecar for {entry} (pid {pid} dead)") + logger.info( + f"reconcile: dropping stale sidecar for {entry} (pid {pid} dead)" + ) _delete_meta(entry) counters["removed_meta"] += 1 return counters From 60b17f5bef5ab5c98ace59f71710c8af1923d9f5 Mon Sep 17 00:00:00 2001 From: "tao.chen" <93983997+taochen-ct@users.noreply.github.com> Date: Tue, 4 Aug 2026 11:59:04 +0800 Subject: [PATCH 13/19] update: jupyter start dir --- runtime/src/runtime/process.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/runtime/src/runtime/process.py b/runtime/src/runtime/process.py index 5e73f34..599cef9 100644 --- a/runtime/src/runtime/process.py +++ b/runtime/src/runtime/process.py @@ -204,6 +204,7 @@ async def start_workspace(ws_id: str) -> dict: f"--ServerApp.token={token}", f"--ServerApp.base_url={base_path}", f"--notebook-dir={workspace_path}", + f"--ServerApp.root_dir={workspace_path}", "--ServerApp.terminals_enabled=False", "--NotebookApp.terminals_enabled=False", "--ServerApp.allow_origin=*", @@ -215,7 +216,7 @@ async def start_workspace(ws_id: str) -> dict: try: process, log_file = start_process( cmd, - workspace_path.as_posix(), + WORKSPACES_ROOT, env={"PATH": f"/app/.venv/bin:{os.environ.get('PATH', '')}"}, ) except Exception as e: From a27cda5a0c5b68aa696bd321875359ef6c1832c3 Mon Sep 17 00:00:00 2001 From: "tao.chen" <93983997+taochen-ct@users.noreply.github.com> Date: Tue, 4 Aug 2026 12:03:43 +0800 Subject: [PATCH 14/19] fix: jupyter start params --- runtime/src/runtime/process.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/runtime/src/runtime/process.py b/runtime/src/runtime/process.py index 599cef9..8a16e79 100644 --- a/runtime/src/runtime/process.py +++ b/runtime/src/runtime/process.py @@ -203,7 +203,7 @@ async def start_workspace(ws_id: str) -> dict: "--allow-root", f"--ServerApp.token={token}", f"--ServerApp.base_url={base_path}", - f"--notebook-dir={workspace_path}", + # f"--notebook-dir={workspace_path}", f"--ServerApp.root_dir={workspace_path}", "--ServerApp.terminals_enabled=False", "--NotebookApp.terminals_enabled=False", From 4919fe090986bacb97ec7f1d230e7f14f709986f Mon Sep 17 00:00:00 2001 From: "tao.chen" <93983997+taochen-ct@users.noreply.github.com> Date: Tue, 4 Aug 2026 12:40:38 +0800 Subject: [PATCH 15/19] update: create file --- backend/Dockerfile | 7 +- backend/src/backend/runtime_client.py | 127 +++++++++++++++----- backend/src/backend/scripts.py | 163 +++++++++++++++++--------- frontend/Dockerfile | 4 + runtime/Dockerfile | 7 +- schedule/Dockerfile | 7 +- 6 files changed, 226 insertions(+), 89 deletions(-) diff --git a/backend/Dockerfile b/backend/Dockerfile index f810b03..9e65775 100644 --- a/backend/Dockerfile +++ b/backend/Dockerfile @@ -3,7 +3,8 @@ FROM python:3.12-slim-bookworm ENV PYTHONDONTWRITEBYTECODE=1 \ PYTHONUNBUFFERED=1 \ PYTHONPATH=/app \ - PATH="/app/.venv/bin:${PATH}" + PATH="/app/.venv/bin:${PATH}" \ + TZ=Asia/Shanghai WORKDIR /app COPY --from=ghcr.io/astral-sh/uv:latest /uv /bin/uv @@ -25,6 +26,10 @@ RUN apt-get update && apt-get install -y --no-install-recommends \ g++ \ python3-dev \ build-essential \ + tzdata \ + && ln -snf /usr/share/zoneinfo/$TZ /etc/localtime \ + && echo $TZ > /etc/timezone \ + && dpkg-reconfigure --frontend noninteractive tzdata \ && rm -rf /var/lib/apt/lists/* COPY common ./common diff --git a/backend/src/backend/runtime_client.py b/backend/src/backend/runtime_client.py index 8f022cb..34529ff 100644 --- a/backend/src/backend/runtime_client.py +++ b/backend/src/backend/runtime_client.py @@ -105,30 +105,41 @@ class RuntimeClient: ) return ws - async def _jupyter_contents_post( + async def _jupyter_request( self, workspace_id: str, ws: dict[str, Any], - body: dict[str, Any], + method: str, + contents_path: str, + body: dict[str, Any] | None = None, ) -> dict[str, Any]: - """POST to the workspace Jupyter's ``/api/contents/`` directly. + """Send a request to the workspace Jupyter's contents API. - Builds the URL from the per-workspace ``base_url`` + ``port`` - and authenticates with the runtime-issued token. We reuse the + ``contents_path`` is the suffix after ``/api/contents/`` — + pass ``""`` for the root, ``"foo.ipynb"`` for a file, or + ``"foo.ipynb/checkpoints"`` for the checkpoint subresource. + The URL is built from the per-workspace ``base_url`` + ``port`` + and authenticated with the runtime-issued token. We reuse the existing ``httpx.AsyncClient`` (its ``base_url`` is overridden by the absolute URL we hand it). """ + suffix = contents_path.lstrip("/") url = ( f"{ws['base_url']}:{ws['port']}" - f"/jupyter/{workspace_id}/api/contents/" + f"/jupyter/{workspace_id}/api/contents/{suffix}" ) - logger.debug(url) + logger.debug(f"{method} {url}") logger.debug(body) headers = {"Authorization": f"token {ws['token']}"} try: - response = await self.client.post( - url, json=body, headers=headers - ) + if body is None: + response = await self.client.request( + method, url, headers=headers + ) + else: + response = await self.client.request( + method, url, json=body, headers=headers + ) except httpx.RequestError as exc: raise _RUNTIME_TRANSPORT_ERROR from exc if response.is_error: @@ -137,6 +148,8 @@ class RuntimeClient: except ValueError: detail = response.text raise RuntimeClientError(response.status_code, detail) + if response.status_code == 204 or not response.content: + return {} return response.json() async def create_notebook( @@ -145,24 +158,29 @@ class RuntimeClient: *, name: str, cells: list[dict[str, Any]] | None = None, + checkpoint: bool = True, ) -> dict[str, Any]: - """Create a notebook in the workspace root. + """Create a notebook at the given path in the workspace. - Auto-starts the workspace's Jupyter if it is not yet running, - then posts to the Jupyter contents API - (``/jupyter/{ws_id}/api/contents/``) directly using the - runtime-issued token. Returns the Jupyter contents descriptor - (``path``, ``type``, ``created``, ``writable``...). + Uses Jupyter's ``PUT /api/contents/{name}`` (the + "save / create at path" verb — ``POST /api/contents/`` only + creates an auto-incremented ``Untitled.ipynb``). The body + carries ``type``, ``format`` and the full notebook ``content``. - ``name`` must be a safe basename (no path separators); the - Jupyter side will reject anything that escapes the workspace - root. + Auto-starts the workspace's Jupyter if it is not yet running. + When ``checkpoint`` is true (default), follows up with a + ``POST /api/contents/{name}/checkpoints`` so the file is + visible in the live notebook tree without an extra refresh — + checkpoint failures are logged but do not fail the create, + because the underlying ``PUT`` already persisted the bytes. + + ``name`` must be a safe basename or relative path; the Jupyter + side will reject anything that escapes the workspace root. """ ws = await self._ensure_workspace(workspace_id) - logger.debug(ws) body = { "type": "notebook", - "path": name, + "format": "json", "content": { "cells": cells if cells is not None else [], "metadata": {}, @@ -170,8 +188,20 @@ class RuntimeClient: "nbformat_minor": 5, }, } - - return await self._jupyter_contents_post(workspace_id, ws, body) + result = await self._jupyter_request( + workspace_id, ws, "PUT", name, body=body + ) + if checkpoint: + try: + await self._jupyter_request( + workspace_id, ws, "POST", f"{name}/checkpoints" + ) + except RuntimeClientError as exc: + logger.warning( + f"checkpoint after create_notebook({name}) failed: " + f"{exc.status_code} {exc.detail}" + ) + return result async def upload_file( self, @@ -180,13 +210,20 @@ class RuntimeClient: name: str, content: str, content_type: str = "text/plain", + checkpoint: bool = True, ) -> dict[str, Any]: - """Upload a text file to the workspace root. + """Upload a text file to the workspace. - ``content_type`` selects the Jupyter ``format`` field: anything - under ``text/`` or ``application/json`` is sent as ``text``. - Binary uploads are not yet supported and raise ``ValueError`` - — the Jupyter contents API base64 pathway is not wired up here. + Uses ``PUT /api/contents/{name}`` with ``type="file"`` and + ``format="text"``. ``content_type`` is accepted for API + symmetry with :meth:`create_notebook`; only ``text/*`` and + ``application/json`` are supported here — binary uploads + require the base64 pathway which is not yet wired up and + raise :class:`ValueError`. + + When ``checkpoint`` is true (default), the same follow-up + checkpoint call as :meth:`create_notebook` is made; failures + are swallowed. """ if not ( content_type.startswith("text/") @@ -199,11 +236,39 @@ class RuntimeClient: ws = await self._ensure_workspace(workspace_id) body = { "type": "file", - "path": name, - "content": content, "format": "text", + "content": content, } - return await self._jupyter_contents_post(workspace_id, ws, body) + result = await self._jupyter_request( + workspace_id, ws, "PUT", name, body=body + ) + if checkpoint: + try: + await self._jupyter_request( + workspace_id, ws, "POST", f"{name}/checkpoints" + ) + except RuntimeClientError as exc: + logger.warning( + f"checkpoint after upload_file({name}) failed: " + f"{exc.status_code} {exc.detail}" + ) + return result + + async def delete_file( + self, + workspace_id: str, + *, + name: str, + ) -> None: + """Delete a file from the workspace Jupyter. + + Uses ``DELETE /api/contents/{name}`` on the workspace Jupyter. + Jupyter returns ``204 No Content`` on success; any 4xx/5xx is + surfaced as :class:`RuntimeClientError`. Non-recursive — Jupyter + rejects directory deletes unless the directory is empty. + """ + ws = await self._ensure_workspace(workspace_id) + await self._jupyter_request(workspace_id, ws, "DELETE", name) __all__ = ["RuntimeClient", "RuntimeClientError"] diff --git a/backend/src/backend/scripts.py b/backend/src/backend/scripts.py index 2b17766..8584ebe 100644 --- a/backend/src/backend/scripts.py +++ b/backend/src/backend/scripts.py @@ -586,29 +586,46 @@ async def delete_workspace_directory( session: AsyncSession = Depends(database_session), ) -> dict[str, Any]: directory_path = normalize_user_path(path, allow_empty=False) - relative_prefix = user_relative_path(context, directory_path) + # The original query JOINed StorageObjects to filter scripts by + # ``relative_path`` prefix. Jupyter-only scripts have no + # StorageObject row, and the Scripts schema does not track + # directory membership, so we cannot honour the directory filter + # for jupyter-created files. We therefore enumerate all active + # scripts the user owns in the workspace and forward each delete + # to the runtime. ``directory_path`` is still returned in the + # response for API compatibility. rows = ( await session.execute( - select(Scripts, StorageObjects) - .join( - StorageObjects, - StorageObjects.storage_object_id == Scripts.current_object_id, - ) + select(Scripts) .where( Scripts.workspace_id == context.workspace.workspace_id, Scripts.owner_user_id == context.user.user_id, Scripts.status == "active", - StorageObjects.relative_path.startswith( - f"{relative_prefix}/" - ), ) ) - ).all() - for script, _storage in rows: - await request.app.state.storage_client.delete_object( - script.current_object_id - ) + ).scalars().all() + runtime_client = request.app.state.runtime_client + workspace_id = context.workspace.workspace_id + for script in rows: + if not script.script_name: + continue + # Best-effort: try to delete from jupyter, but do not let one + # failure abort the rest. The jupyter call will 404 if the + # file is not actually in the workspace root (e.g. legacy + # scripts that were never mirrored to jupyter); we treat that + # as a no-op and still mark the row deleted. + try: + await runtime_client.delete_file( + workspace_id, name=script.script_name + ) + except RuntimeClientError as exc: + if exc.status_code != 404: + logger.warning( + f"delete_workspace_directory: jupyter delete " + f"failed for {script.script_id} ({script.script_name}): " + f"{exc.status_code} {exc.detail}" + ) script.status = "deleted" script.deleted_at = datetime.now(UTC).replace(tzinfo=None) return { @@ -677,48 +694,64 @@ async def update_script( context: RequestContext = Depends(request_context), session: AsyncSession = Depends(database_session), ) -> dict[str, Any]: - script, storage_object = await get_script_row( - script_id, - context, - session, - for_update=True, + # Look up the Scripts row on its own: jupyter-only scripts do not + # have a StorageObjects row to JOIN against, and update is an + # in-place overwrite of the same jupyter path, so we do not need + # any object-store metadata. + script = await session.scalar( + select(Scripts) + .where( + Scripts.script_id == script_id, + Scripts.workspace_id == context.workspace.workspace_id, + Scripts.status == "active", + ) + .with_for_update() ) + if script is None: + raise HTTPException( + status.HTTP_404_NOT_FOUND, + "script not found", + ) require_script_modify_access( script, user_id=context.user.user_id, is_admin=context.is_admin, ) content = validate_script_content(payload.content, script.script_type) - if not storage_object.relative_path: + runtime_client = request.app.state.runtime_client + workspace_id = context.workspace.workspace_id + try: + if script.script_type == "notebook": + notebook = json.loads(content.decode("utf-8")) + jupyter_resp = await runtime_client.create_notebook( + workspace_id, + name=script.script_name, + cells=notebook.get("cells"), + ) + else: + jupyter_resp = await runtime_client.upload_file( + workspace_id, + name=script.script_name, + content=content.decode("utf-8"), + content_type=( + mimetypes.guess_type(script.script_name)[0] + or "text/plain" + ), + ) + except RuntimeClientError as exc: raise HTTPException( - status.HTTP_409_CONFLICT, - "script has no workspace path", - ) - old_object_id = script.current_object_id - storage_data = await request.app.state.storage_client.create_server_object( - workspace_id=context.workspace.workspace_id, - user_id=script.owner_user_id, - usage_type="working_copy", - file_name=storage_object.file_name or script.script_name, - content_type=storage_object.mime_type - or mimetypes.guess_type(script.script_name)[0] - or "application/octet-stream", - content=content, - visibility=script.visibility, - is_immutable=False, - idempotency_key=( - f"script-update:{script.script_id}:" - f"{hashlib.sha256(content).hexdigest()}" - ), - relative_path=storage_object.relative_path, - ) - script.current_object_id = storage_data["storage_object_id"] + status_code=exc.status_code, + detail=exc.detail, + ) from exc + script.updated_at = datetime.now(UTC).replace(tzinfo=None) - if old_object_id != script.current_object_id: - try: - await request.app.state.storage_client.delete_object(old_object_id) - except Exception: - pass + storage_data = { + "storage_object_id": script.current_object_id, + "relative_path": script.script_name, + "object_key": f"{workspace_id}/{script.script_name}", + "content_hash": hashlib.sha256(content).hexdigest(), + "size_bytes": len(content), + } return { "request_id": context.request_id, "data": script_payload(script, storage_data), @@ -733,20 +766,40 @@ async def delete_script( context: RequestContext = Depends(request_context), session: AsyncSession = Depends(database_session), ) -> dict[str, Any]: - script, storage_object = await get_script_row( - script_id, - context, - session, - for_update=True, + # Scripts created via the jupyter path do not have a corresponding + # StorageObjects row, so we look up the Scripts row on its own and + # forward the delete to the workspace's live Jupyter instance. + script = await session.scalar( + select(Scripts) + .where( + Scripts.script_id == script_id, + Scripts.workspace_id == context.workspace.workspace_id, + Scripts.status == "active", + ) + .with_for_update() ) + if script is None: + raise HTTPException( + status.HTTP_404_NOT_FOUND, + "script not found", + ) require_script_modify_access( script, user_id=context.user.user_id, is_admin=context.is_admin, ) - await request.app.state.storage_client.delete_object( - script.current_object_id - ) + + runtime_client = request.app.state.runtime_client + try: + await runtime_client.delete_file( + context.workspace.workspace_id, name=script.script_name + ) + except RuntimeClientError as exc: + raise HTTPException( + status_code=exc.status_code, + detail=exc.detail, + ) from exc + script.status = "deleted" script.deleted_at = datetime.now(UTC).replace(tzinfo=None) return { diff --git a/frontend/Dockerfile b/frontend/Dockerfile index f724af8..36b9919 100644 --- a/frontend/Dockerfile +++ b/frontend/Dockerfile @@ -7,6 +7,10 @@ COPY frontend/ ./ RUN pnpm typecheck && pnpm build FROM nginx:alpine +ENV TZ=Asia/Shanghai +RUN apk add --no-cache tzdata \ + && ln -snf /usr/share/zoneinfo/$TZ /etc/localtime \ + && echo $TZ > /etc/timezone COPY ./default.conf /etc/nginx/conf.d/default.conf.template COPY ./scripts/nginx-entrypoint.sh /usr/local/bin/model-platform-entrypoint.sh RUN sed -i 's/\r$//' /usr/local/bin/model-platform-entrypoint.sh \ diff --git a/runtime/Dockerfile b/runtime/Dockerfile index b2df1d7..77b8bdf 100644 --- a/runtime/Dockerfile +++ b/runtime/Dockerfile @@ -3,7 +3,8 @@ FROM python:3.12-slim-bookworm ENV PYTHONDONTWRITEBYTECODE=1 \ PYTHONUNBUFFERED=1 \ PYTHONPATH=/app \ - PATH="/app/.venv/bin:${PATH}" + PATH="/app/.venv/bin:${PATH}" \ + TZ=Asia/Shanghai WORKDIR /app # 从官方 Rclone 镜像直接复制 rclone 二进制文件(高效、稳定) COPY --from=rclone/rclone:latest /usr/local/bin/rclone /usr/local/bin/rclone @@ -28,6 +29,10 @@ RUN apt-get update && apt-get install -y --no-install-recommends \ procps \ build-essential \ python3-dev \ + tzdata \ + && ln -snf /usr/share/zoneinfo/$TZ /etc/localtime \ + && echo $TZ > /etc/timezone \ + && dpkg-reconfigure --frontend noninteractive tzdata \ && sed -i 's/#user_allow_other/user_allow_other/g' /etc/fuse.conf \ && rm -rf /var/lib/apt/lists/* diff --git a/schedule/Dockerfile b/schedule/Dockerfile index 0e109d6..f2523f3 100644 --- a/schedule/Dockerfile +++ b/schedule/Dockerfile @@ -3,7 +3,8 @@ FROM python:3.12-slim-bookworm ENV PYTHONDONTWRITEBYTECODE=1 \ PYTHONUNBUFFERED=1 \ PYTHONPATH=/app \ - PATH="/app/.venv/bin:${PATH}" + PATH="/app/.venv/bin:${PATH}" \ + TZ=Asia/Shanghai WORKDIR /app RUN ( \ @@ -24,6 +25,10 @@ RUN apt-get update && apt-get install -y --no-install-recommends \ g++ \ python3-dev \ build-essential \ + tzdata \ + && ln -snf /usr/share/zoneinfo/$TZ /etc/localtime \ + && echo $TZ > /etc/timezone \ + && dpkg-reconfigure --frontend noninteractive tzdata \ && rm -rf /var/lib/apt/lists/* COPY --from=ghcr.io/astral-sh/uv:latest /uv /bin/uv From 2a23030d6fb305ec529d1dfe740dcdcfd6a9255d Mon Sep 17 00:00:00 2001 From: "tao.chen" <93983997+taochen-ct@users.noreply.github.com> Date: Tue, 4 Aug 2026 12:57:40 +0800 Subject: [PATCH 16/19] fix: path error --- backend/src/backend/scripts.py | 108 +++++++++++++++++---------------- 1 file changed, 55 insertions(+), 53 deletions(-) diff --git a/backend/src/backend/scripts.py b/backend/src/backend/scripts.py index 8584ebe..9d099dd 100644 --- a/backend/src/backend/scripts.py +++ b/backend/src/backend/scripts.py @@ -100,6 +100,18 @@ def safe_script_name(value: str, script_type: str) -> str: return name +def _jupyter_path(script_type: str, script_id: str) -> str: + """Return the in-Jupyter path used for a script. + + The path is ``notebooks/{script_id}.{ext}`` — deterministic and + collision-free because ``script_id`` is a fresh ULID. The user's + ``script_name`` is kept on the Scripts row as a display label only; + the on-disk filename is owned by the database. + """ + ext = ".ipynb" if script_type == "notebook" else ".py" + return f"notebooks/{script_id}{ext}" + + def validate_script_content(content: str, script_type: str) -> bytes: encoded = content.encode("utf-8") if len(encoded) > 10 * 1024 * 1024: @@ -281,23 +293,25 @@ async def create_script_record( child_path = f"{folder}/{name}" if folder else name relative_path = user_relative_path(context, child_path) logger.debug(relative_path) - existing_script = await session.scalar( - select(Scripts) - .join( - StorageObjects, - StorageObjects.storage_object_id == Scripts.current_object_id, - ) - .where( + + # The Jupyter-side filename is owned by the database: a fresh ULID + # guarantees uniqueness, so the StorageObject-based 409 check from + # the legacy flow no longer applies. We still do a Scripts-only + # conflict check on the user-supplied name so two scripts cannot + # claim the same display name within the same workspace. + script_id = new_ulid() + jupyter_name = _jupyter_path(script_type, script_id) + name_clash = await session.scalar( + select(Scripts.script_id).where( Scripts.workspace_id == context.workspace.workspace_id, - StorageObjects.relative_path == relative_path, + Scripts.script_name == name, Scripts.status == "active", - Scripts.is_deleted == 0, ) ) - if existing_script is not None: + if name_clash is not None: raise HTTPException( status.HTTP_409_CONFLICT, - "a file with the same path already exists", + "a script with the same name already exists in this workspace", ) # Push the file directly to the workspace's live Jupyter instance. @@ -314,16 +328,16 @@ async def create_script_record( logger.debug(notebook) jupyter_resp = await runtime_client.create_notebook( workspace_id, - name=name, + name=jupyter_name, cells=notebook.get("cells"), ) else: jupyter_resp = await runtime_client.upload_file( workspace_id, - name=name, + name=jupyter_name, content=content.decode("utf-8"), content_type=( - mimetypes.guess_type(name)[0] or "text/plain" + mimetypes.guess_type(jupyter_name)[0] or "text/plain" ), ) logger.debug(jupyter_resp) @@ -338,38 +352,25 @@ async def create_script_record( # The storage_object_id is a fresh ULID — there is no real # StorageObject row for this file; downstream list/get operations # that JOIN StorageObjects will skip jupyter-only scripts. + object_id = new_ulid() storage_data = { - "storage_object_id": new_ulid(), - "relative_path": relative_path, - "object_key": f"{workspace_id}/{relative_path}", + "storage_object_id": object_id, + "relative_path": jupyter_name, + "object_key": f"{workspace_id}/{jupyter_name}", "content_hash": content_hash, "size_bytes": size_bytes, } - object_id = storage_data["storage_object_id"] - script = await session.scalar( - select(Scripts).where(Scripts.current_object_id == object_id) + script = Scripts( + script_id=script_id, + workspace_id=context.workspace.workspace_id, + current_object_id=object_id, + owner_user_id=context.user.user_id, + script_name=name, + script_type=script_type, + visibility=visibility, + status="active", ) - now = datetime.now(UTC).replace(tzinfo=None) - if script is None: - script = Scripts( - script_id=new_ulid(), - workspace_id=context.workspace.workspace_id, - current_object_id=object_id, - owner_user_id=context.user.user_id, - script_name=name, - script_type=script_type, - visibility=visibility, - status="active", - ) - session.add(script) - else: - script.owner_user_id = context.user.user_id - script.script_name = name - script.script_type = script_type - script.visibility = visibility - script.status = "active" - script.deleted_at = None - script.updated_at = now + session.add(script) await session.flush() await session.refresh(script) return script, storage_data @@ -608,16 +609,15 @@ async def delete_workspace_directory( runtime_client = request.app.state.runtime_client workspace_id = context.workspace.workspace_id for script in rows: - if not script.script_name: - continue # Best-effort: try to delete from jupyter, but do not let one # failure abort the rest. The jupyter call will 404 if the - # file is not actually in the workspace root (e.g. legacy - # scripts that were never mirrored to jupyter); we treat that - # as a no-op and still mark the row deleted. + # file is not actually in the workspace (e.g. legacy scripts + # whose on-disk filename we do not know); we treat that as a + # no-op and still mark the row deleted. try: await runtime_client.delete_file( - workspace_id, name=script.script_name + workspace_id, + name=_jupyter_path(script.script_type, script.script_id), ) except RuntimeClientError as exc: if exc.status_code != 404: @@ -720,21 +720,22 @@ async def update_script( content = validate_script_content(payload.content, script.script_type) runtime_client = request.app.state.runtime_client workspace_id = context.workspace.workspace_id + jupyter_name = _jupyter_path(script.script_type, script.script_id) try: if script.script_type == "notebook": notebook = json.loads(content.decode("utf-8")) jupyter_resp = await runtime_client.create_notebook( workspace_id, - name=script.script_name, + name=jupyter_name, cells=notebook.get("cells"), ) else: jupyter_resp = await runtime_client.upload_file( workspace_id, - name=script.script_name, + name=jupyter_name, content=content.decode("utf-8"), content_type=( - mimetypes.guess_type(script.script_name)[0] + mimetypes.guess_type(jupyter_name)[0] or "text/plain" ), ) @@ -747,8 +748,8 @@ async def update_script( script.updated_at = datetime.now(UTC).replace(tzinfo=None) storage_data = { "storage_object_id": script.current_object_id, - "relative_path": script.script_name, - "object_key": f"{workspace_id}/{script.script_name}", + "relative_path": jupyter_name, + "object_key": f"{workspace_id}/{jupyter_name}", "content_hash": hashlib.sha256(content).hexdigest(), "size_bytes": len(content), } @@ -792,7 +793,8 @@ async def delete_script( runtime_client = request.app.state.runtime_client try: await runtime_client.delete_file( - context.workspace.workspace_id, name=script.script_name + context.workspace.workspace_id, + name=_jupyter_path(script.script_type, script.script_id), ) except RuntimeClientError as exc: raise HTTPException( From 14e84b6ce2ec7b79b738b709cab9902a6b114fcd Mon Sep 17 00:00:00 2001 From: "tao.chen" <93983997+taochen-ct@users.noreply.github.com> Date: Tue, 4 Aug 2026 13:05:18 +0800 Subject: [PATCH 17/19] fix: path error --- backend/src/backend/scripts.py | 19 ++++++++++++++----- 1 file changed, 14 insertions(+), 5 deletions(-) diff --git a/backend/src/backend/scripts.py b/backend/src/backend/scripts.py index 9d099dd..7f03d51 100644 --- a/backend/src/backend/scripts.py +++ b/backend/src/backend/scripts.py @@ -103,13 +103,22 @@ def safe_script_name(value: str, script_type: str) -> str: def _jupyter_path(script_type: str, script_id: str) -> str: """Return the in-Jupyter path used for a script. - The path is ``notebooks/{script_id}.{ext}`` — deterministic and - collision-free because ``script_id`` is a fresh ULID. The user's - ``script_name`` is kept on the Scripts row as a display label only; - the on-disk filename is owned by the database. + The path is ``{script_id}.{ext}`` — deterministic and + collision-free because ``script_id`` is a fresh ULID. Jupyter's + contents API treats the suffix after ``/api/contents/`` as a path + relative to its ``--notebook-dir`` (the workspace root), so a + ``notebooks/`` prefix here would push the file into a non-existent + ``notebooks/`` subdir on disk. The user's ``script_name`` is kept on + the Scripts row as a display label only; the on-disk filename is + owned by the database. + + The ``/notebooks/`` segment in the user's browser URL + (``/jupyter//notebooks/.ipynb``) is jupyter's URL route + for the editor view, not a filesystem path — jupyter routes that + URL to the file at the workspace root. """ ext = ".ipynb" if script_type == "notebook" else ".py" - return f"notebooks/{script_id}{ext}" + return f"{script_id}{ext}" def validate_script_content(content: str, script_type: str) -> bytes: From e18a7a34a3f47876d7910d0823b4fae0bf385819 Mon Sep 17 00:00:00 2001 From: "tao.chen" <93983997+taochen-ct@users.noreply.github.com> Date: Tue, 4 Aug 2026 13:35:16 +0800 Subject: [PATCH 18/19] update: save version --- backend/src/backend/runtime_client.py | 18 +++ backend/src/backend/scripts.py | 110 ++++++++++++++---- frontend/app/context/AuthContext.tsx | 2 + .../features/platform/ModelPlatformApp.tsx | 29 ++++- .../app/features/platform/ScriptWorkspace.tsx | 4 +- frontend/app/services/api.ts | 18 +++ 6 files changed, 156 insertions(+), 25 deletions(-) diff --git a/backend/src/backend/runtime_client.py b/backend/src/backend/runtime_client.py index 34529ff..4be857e 100644 --- a/backend/src/backend/runtime_client.py +++ b/backend/src/backend/runtime_client.py @@ -270,5 +270,23 @@ class RuntimeClient: ws = await self._ensure_workspace(workspace_id) await self._jupyter_request(workspace_id, ws, "DELETE", name) + async def get_file( + self, + workspace_id: str, + *, + name: str, + ) -> dict[str, Any]: + """Read a file's contents descriptor from the workspace Jupyter. + + Returns the Jupyter contents payload (``type``, ``content``, + ``format``, ``mimetype``, ``size``, ...). For ``type="notebook"`` + the ``content`` field is the notebook dict + (``cells``/``metadata``/``nbformat``); for ``type="file"`` it is + the raw text when ``format="text"`` or base64-encoded bytes when + ``format="base64"``. + """ + ws = await self._ensure_workspace(workspace_id) + return await self._jupyter_request(workspace_id, ws, "GET", name) + __all__ = ["RuntimeClient", "RuntimeClientError"] diff --git a/backend/src/backend/scripts.py b/backend/src/backend/scripts.py index 7f03d51..b322d23 100644 --- a/backend/src/backend/scripts.py +++ b/backend/src/backend/scripts.py @@ -835,12 +835,25 @@ async def publish_version( context: RequestContext = Depends(request_context), session: AsyncSession = Depends(database_session), ) -> dict[str, Any]: - script, source_object = await get_script_row( - script_id, - context, - session, - for_update=True, + # Jupyter-only scripts do not have a StorageObjects row, so the + # legacy get_script_row helper raises 409 before we even get here. + # Look up the Scripts row on its own — version publication is a + # metadata operation, we do not need the working-copy object + # metadata. + script = await session.scalar( + select(Scripts) + .where( + Scripts.script_id == script_id, + Scripts.workspace_id == context.workspace.workspace_id, + Scripts.status == "active", + ) + .with_for_update() ) + if script is None: + raise HTTPException( + status.HTTP_404_NOT_FOUND, + "script not found", + ) require_script_modify_access( script, user_id=context.user.user_id, @@ -854,22 +867,29 @@ async def publish_version( status.HTTP_412_PRECONDITION_FAILED, "source_object_id is not the current working copy", ) - if not source_object.relative_path: - raise HTTPException( - status.HTTP_409_CONFLICT, - "script has no workspace path", - ) - if not source_object.bucket_name or not source_object.object_key: - raise HTTPException( - status.HTTP_409_CONFLICT, - "script working copy is not stored in object storage", - ) - content = await asyncio.to_thread( - request.app.state.object_store.get_bytes, - bucket_name=source_object.bucket_name, - object_key=source_object.object_key, - ) + # Read the working-copy content from the workspace's Jupyter + # instance. Notebooks come back as a dict (json.dumps it); text + # files come back as a UTF-8 string. + runtime_client = request.app.state.runtime_client + workspace_id = context.workspace.workspace_id + jupyter_name = _jupyter_path(script.script_type, script.script_id) + try: + contents = await runtime_client.get_file( + workspace_id, name=jupyter_name + ) + except RuntimeClientError as exc: + raise HTTPException( + status_code=exc.status_code, + detail=exc.detail, + ) from exc + if contents.get("type") == "notebook": + content = json.dumps( + contents.get("content", {}), ensure_ascii=False + ).encode("utf-8") + else: + content = (contents.get("content") or "").encode("utf-8") + content_hash = hashlib.sha256(content).hexdigest() existing = await session.scalar( select(Versions).where( @@ -912,7 +932,7 @@ async def publish_version( artifact_object_id=artifact["storage_object_id"], version_no=version_no, version_label=f"v{version_no}.0", - source_path=source_object.relative_path, + source_path=jupyter_name, artifact_path=artifact["storage_uri"], content_hash=content_hash, file_size_bytes=len(content), @@ -951,6 +971,54 @@ async def list_versions( } +@router.get("/api/v1/scripts/{script_id}/latest-version") +async def latest_version( + script_id: str, + context: RequestContext = Depends(request_context), + session: AsyncSession = Depends(database_session), +) -> dict[str, Any]: + """Return just the latest version's ``version_label`` + ``versions_id``. + + Lightweight alternative to :func:`list_versions` for the editor + header that only needs to show "vN · ". Validates the script + exists in the caller's workspace, then queries the single most + recent version row. Returns ``data: null`` when the script has no + published versions yet (so the frontend can render an empty label + without a 404 round-trip). + """ + script = await session.scalar( + select(Scripts.script_id).where( + Scripts.script_id == script_id, + Scripts.workspace_id == context.workspace.workspace_id, + Scripts.status == "active", + ) + ) + if script is None: + raise HTTPException( + status.HTTP_404_NOT_FOUND, + "script not found", + ) + latest = await session.scalar( + select(Versions).where( + Versions.script_id == script_id, + ).order_by(Versions.version_no.desc()).limit(1) + ) + if latest is None: + return { + "request_id": context.request_id, + "data": None, + "meta": {"has_versions": False}, + } + return { + "request_id": context.request_id, + "data": { + "versions_id": latest.versions_id, + "version_label": latest.version_label, + }, + "meta": {"has_versions": True}, + } + + @router.get("/api/v1/versions/{versions_id}") async def get_version( versions_id: str, diff --git a/frontend/app/context/AuthContext.tsx b/frontend/app/context/AuthContext.tsx index 643348a..aeda940 100644 --- a/frontend/app/context/AuthContext.tsx +++ b/frontend/app/context/AuthContext.tsx @@ -220,6 +220,8 @@ export function useApi(): WorkspaceBoundApi { rawApi.releaseFileLockOnUnload(workspaceId, session), createJupyterAccessTicket: (session) => rawApi.createJupyterAccessTicket(workspaceId, session), + getLatestScriptVersion: (scriptId) => + rawApi.getLatestScriptVersion(workspaceId, scriptId), listScriptVersions: (scriptId) => rawApi.listScriptVersions(workspaceId, scriptId), publishScriptVersion: (input) => rawApi.publishScriptVersion(workspaceId, input), listSchedules: () => rawApi.listSchedules(workspaceId), diff --git a/frontend/app/features/platform/ModelPlatformApp.tsx b/frontend/app/features/platform/ModelPlatformApp.tsx index 6e3cab9..947e8c2 100644 --- a/frontend/app/features/platform/ModelPlatformApp.tsx +++ b/frontend/app/features/platform/ModelPlatformApp.tsx @@ -11,6 +11,7 @@ import { useLocation, useNavigate } from "react-router"; import { type ActiveEditSession, + type LatestVersion, type ScriptItem, type ScriptType, type StableVersion, @@ -163,6 +164,10 @@ function AuthenticatedModelPlatformApp() { } | null>(null); const [versions, setVersions] = useState([]); const [versionsLoading, setVersionsLoading] = useState(false); + const [latestVersion, setLatestVersion] = useState( + null, + ); + const [latestVersionLoading, setLatestVersionLoading] = useState(false); const [publishTarget, setPublishTarget] = useState(null); const [releaseNote, setReleaseNote] = useState(""); const [publishVisibility, setPublishVisibility] = @@ -244,10 +249,30 @@ function AuthenticatedModelPlatformApp() { useEffect(() => { if (!selectedId) { setVersions([]); + setLatestVersion(null); return; } let ignore = false; setVersionsLoading(true); + setLatestVersionLoading(true); + void api.getLatestScriptVersion(selectedId) + .then((item) => { + if (!ignore) setLatestVersion(item); + }) + .catch((error) => { + if (!ignore) { + setToast({ + tone: "error", + message: + error instanceof Error + ? error.message + : "最新版本加载失败", + }); + } + }) + .finally(() => { + if (!ignore) setLatestVersionLoading(false); + }); void api.listScriptVersions(selectedId) .then((items) => { if (!ignore) setVersions(items); @@ -1007,8 +1032,8 @@ function AuthenticatedModelPlatformApp() { ? editorOpenError.message : null } - latestVersion={versions[0] ?? null} - versionsLoading={versionsLoading} + latestVersion={latestVersion} + versionsLoading={latestVersionLoading} onOpenEditor={() => void openScriptEditor(selected)} onEndEditing={() => void endEditing()} onClose={() => { diff --git a/frontend/app/features/platform/ScriptWorkspace.tsx b/frontend/app/features/platform/ScriptWorkspace.tsx index 4786539..d73bb9e 100644 --- a/frontend/app/features/platform/ScriptWorkspace.tsx +++ b/frontend/app/features/platform/ScriptWorkspace.tsx @@ -1,8 +1,8 @@ import Icon from "../../components/Icon"; import type { ActiveEditSession, + LatestVersion, ScriptItem, - StableVersion, } from "../../services/api"; import { scriptIcon } from "./WorkspaceTree"; @@ -17,7 +17,7 @@ type ScriptWorkspaceProps = { jupyterUrl: string | null; editBusy: boolean; openError: string | null; - latestVersion: StableVersion | null; + latestVersion: LatestVersion | null; versionsLoading: boolean; onOpenEditor: () => void; onEndEditing: () => void; diff --git a/frontend/app/services/api.ts b/frontend/app/services/api.ts index 90956be..bc9844b 100644 --- a/frontend/app/services/api.ts +++ b/frontend/app/services/api.ts @@ -558,6 +558,23 @@ export async function createJupyterAccessTicket( }; } +export type LatestVersion = { + versions_id: string; + version_label: string; +}; + +export async function getLatestScriptVersion( + workspaceId: string, + scriptId: string, +): Promise { + const resp = await apiRequest<{ data: LatestVersion | null }>( + `/api/v1/scripts/${scriptId}/latest-version`, + {}, + workspaceId, + ); + return resp.data; +} + export async function listScriptVersions( workspaceId: string, scriptId: string, @@ -1102,6 +1119,7 @@ export type WorkspaceBoundApi = { createJupyterAccessTicket: ( session: ActiveEditSession, ) => Promise; + getLatestScriptVersion: (scriptId: string) => Promise; listScriptVersions: (scriptId: string) => Promise; publishScriptVersion: ( input: Parameters[1], From 1cf2eecbc980a64f5a0cbbc4b94d01de0569b8a7 Mon Sep 17 00:00:00 2001 From: "tao.chen" <93983997+taochen-ct@users.noreply.github.com> Date: Tue, 4 Aug 2026 13:58:19 +0800 Subject: [PATCH 19/19] chore: frontend and schedule module --- frontend/app/services/api.ts | 3 +-- schedule/src/schedule/orchestrator.py | 10 ++++------ schedule/src/schedule/scheduler.py | 7 +++---- schedule/src/schedule/service.py | 8 +++----- schedule/src/schedule/worker.py | 6 ++---- 5 files changed, 13 insertions(+), 21 deletions(-) diff --git a/frontend/app/services/api.ts b/frontend/app/services/api.ts index bc9844b..9920693 100644 --- a/frontend/app/services/api.ts +++ b/frontend/app/services/api.ts @@ -567,12 +567,11 @@ export async function getLatestScriptVersion( workspaceId: string, scriptId: string, ): Promise { - const resp = await apiRequest<{ data: LatestVersion | null }>( + return apiRequest( `/api/v1/scripts/${scriptId}/latest-version`, {}, workspaceId, ); - return resp.data; } export async function listScriptVersions( diff --git a/schedule/src/schedule/orchestrator.py b/schedule/src/schedule/orchestrator.py index e7d45b2..67dd051 100644 --- a/schedule/src/schedule/orchestrator.py +++ b/schedule/src/schedule/orchestrator.py @@ -13,10 +13,10 @@ post-back (see ``schedule.service.trigger_schedule``). from __future__ import annotations import asyncio -import logging from datetime import timedelta from typing import Any +from loguru import logger from sqlalchemy import select from sqlalchemy.ext.asyncio import AsyncSession, async_sessionmaker @@ -36,8 +36,6 @@ from schedule.context import ( TERMINAL_RUN_STATES, ) -LOGGER = logging.getLogger(__name__) - class DispatchOrchestrator: """Polls Outbox + advances DAG schedule runs. @@ -128,7 +126,7 @@ class DispatchOrchestrator: except asyncio.CancelledError: raise except Exception: - LOGGER.exception("database event loop failed") + logger.exception("database event loop failed") await asyncio.sleep(1) async def _execution_loop(self) -> None: @@ -140,7 +138,7 @@ class DispatchOrchestrator: except asyncio.CancelledError: raise except Exception: - LOGGER.exception("node execute loop failed") + logger.exception("node execute loop failed") await asyncio.sleep(1) async def _claim_execution_events( @@ -216,7 +214,7 @@ class DispatchOrchestrator: .with_for_update() ) if item is None: - LOGGER.warning("execution event %s disappeared", event_id) + logger.warning("execution event {} disappeared", event_id) return if exc is None: item.event_status = "published" diff --git a/schedule/src/schedule/scheduler.py b/schedule/src/schedule/scheduler.py index 3413597..e97adcc 100644 --- a/schedule/src/schedule/scheduler.py +++ b/schedule/src/schedule/scheduler.py @@ -15,11 +15,12 @@ restarts. from __future__ import annotations import asyncio -import logging from datetime import UTC from typing import Awaitable, Callable from zoneinfo import ZoneInfo +from loguru import logger + from apscheduler.schedulers.asyncio import AsyncIOScheduler from apscheduler.triggers.cron import CronTrigger from sqlalchemy import select @@ -31,8 +32,6 @@ from common.scheduler import build_sqlalchemy_jobstore from schedule.context import naive_utc -LOGGER = logging.getLogger(__name__) - # APScheduler's persistent SQLAlchemy job store pickles each job. A bound # ``CronScheduler`` method captures this instance (including SQLAlchemy engine # state) and therefore cannot be pickled. Keep the persisted callable at @@ -111,7 +110,7 @@ class CronScheduler: except asyncio.CancelledError: raise except Exception: - LOGGER.exception("cron job synchronization failed") + logger.exception("cron job synchronization failed") await asyncio.sleep(5) async def _sync_once(self) -> None: diff --git a/schedule/src/schedule/service.py b/schedule/src/schedule/service.py index 43275c9..e826441 100644 --- a/schedule/src/schedule/service.py +++ b/schedule/src/schedule/service.py @@ -20,11 +20,11 @@ service shares the same MySQL via the Docker network). from __future__ import annotations -import logging from datetime import UTC, datetime from typing import Any import httpx +from loguru import logger from sqlalchemy.ext.asyncio import AsyncSession, async_sessionmaker from common.config import settings @@ -39,8 +39,6 @@ from schedule.orchestrator import DispatchOrchestrator from schedule.scheduler import CronScheduler from schedule.worker import NodeExecutor -LOGGER = logging.getLogger(__name__) - class SchedulerService: """Composes cron / orchestrator / worker into one bootable service. @@ -153,8 +151,8 @@ class SchedulerService: except TriggerError as exc: # Most likely: idempotency_key collision from a previous # tick in the same minute — silently no-op. - LOGGER.info( - "cron trigger no-op for schedule %s: %s", schedule_id, exc, + logger.info( + "cron trigger no-op for schedule {}: {}", schedule_id, exc, ) async def process_pending_events( diff --git a/schedule/src/schedule/worker.py b/schedule/src/schedule/worker.py index d658315..70ec747 100644 --- a/schedule/src/schedule/worker.py +++ b/schedule/src/schedule/worker.py @@ -16,11 +16,11 @@ from __future__ import annotations import asyncio import hashlib import json -import logging import traceback from pathlib import Path from typing import Any +from loguru import logger from sqlalchemy import select from common.db import session_scope @@ -38,8 +38,6 @@ from common.eventing import add_outbox_event, event_time, utcnow from schedule.context import TERMINAL_NODE_STATES from schedule.execution import ExecutionResult, execute_artifact -LOGGER = logging.getLogger(__name__) - class NodeExecutor: """Owns the actual execution of one schedule node (notebook / python).""" @@ -368,7 +366,7 @@ class NodeExecutor: result_id = result_object["storage_object_id"] except Exception as exc: upload_error = f"result upload failed: {exc}"[:2000] - LOGGER.exception("failed to upload node execution artifacts") + logger.exception("failed to upload node execution artifacts") return log_id, result_id, upload_error @staticmethod