diff --git a/.env.example b/.env.example index 04b31e8..0952238 100644 --- a/.env.example +++ b/.env.example @@ -1,12 +1,21 @@ -COMPOSE_PROJECT_NAME=model-platform +COMPOSE_PROJECT_NAME=model-platform-develop # External port of the Nginx gateway. Only Nginx is exposed to the host # (architecture §2.2); backend/runtime/schedule stay on the Docker internal # network. Override here to expose Nginx on a different host port. -GATEWAY_PORT=8888 +GATEWAY_PORT=8890 -# MySQL connection URI (async SQLAlchemy driver) -DATABASE_URL=mysql+asyncmy://model_platform:ChangeMe_MySQL_App_2026@mysql:3306/model_platform?charset=utf8mb4 +# External MySQL. URL-encode reserved characters in DATABASE_URL. +MYSQL_HOST=127.0.0.1 +MYSQL_PORT=3306 +MYSQL_USER=root +MYSQL_PASSWORD=change-me +MYSQL_DATABASE=model_platform +DATABASE_URL=mysql+asyncmy://root:change-me@127.0.0.1:3306/model_platform?charset=utf8mb4 + +# Demo login is only intended for this self-hosted development UI. +DEMO_AUTH_ENABLED=true +JWT_SECRET=change-this-development-secret # Object storage (S3-compatible, RustFS). # RUSTFS_ENDPOINT is the single upstream URL consumed by all 4 services: @@ -19,8 +28,12 @@ DATABASE_URL=mysql+asyncmy://model_platform:ChangeMe_MySQL_App_2026@mysql:3306/m # RUSTFS_WORKSPACE_BUCKET — workspace files (notebooks, scripts, working # copies); layout is ``s3:////...``. # Future: RUSTFS_VERSION_BUCKET, RUSTFS_RUN_LOG_BUCKET, ... -RUSTFS_ENDPOINT=http://rustfs:9000 -RUSTFS_ACCESS_KEY=modelplatform -RUSTFS_SECRET_KEY=ChangeMe_RustFS_2026 +RUSTFS_HOST=127.0.0.1 +RUSTFS_PORT=9000 +RUSTFS_ENDPOINT=http://127.0.0.1:9000 +RUSTFS_ACCESS_KEY=change-me +RUSTFS_SECRET_KEY=change-me RUSTFS_WORKSPACE_BUCKET=workspaces +RUSTFS_VERSION_BUCKET=versions +RUSTFS_RUN_LOG_BUCKET=run-logs diff --git a/backend/Dockerfile b/backend/Dockerfile index 9321979..3e4d8dd 100644 --- a/backend/Dockerfile +++ b/backend/Dockerfile @@ -1,6 +1,9 @@ FROM python:3.12-slim-bookworm -ENV PYTHONDONTWRITEBYTECODE=1 PYTHONUNBUFFERED=1 PYTHONPATH=/app +ENV PYTHONDONTWRITEBYTECODE=1 \ + PYTHONUNBUFFERED=1 \ + PYTHONPATH=/app \ + PATH="/app/.venv/bin:${PATH}" WORKDIR /app COPY --from=ghcr.io/astral-sh/uv:latest /uv /bin/uv diff --git a/backend/src/backend/jupyter.py b/backend/src/backend/jupyter.py index 08af88b..43dbf94 100644 --- a/backend/src/backend/jupyter.py +++ b/backend/src/backend/jupyter.py @@ -12,7 +12,11 @@ from fastapi import APIRouter, Depends, HTTPException, Request, Response from fastapi.security import HTTPBearer, HTTPAuthorizationCredentials from sqlalchemy import select -from backend.dependencies import database_session +from backend.dependencies import ( + RequestContext, + database_session, + request_context, +) from backend.runtime_client import RuntimeClientError from common.config import settings from common.db.models import Scripts, Users, WorkspaceMembers, Workspaces @@ -32,6 +36,35 @@ def _b64decode(value: str) -> bytes: return base64.urlsafe_b64decode(value + padding) +def _b64encode(value: bytes) -> str: + return base64.urlsafe_b64encode(value).rstrip(b"=").decode() + + +def create_jwt_token(user_id: str, *, expires_seconds: int = 3600) -> str: + header = _b64encode( + json.dumps( + {"alg": JWT_ALGORITHM, "typ": "JWT"}, + separators=(",", ":"), + ).encode() + ) + expires_at = int(time.time()) + expires_seconds + payload = _b64encode( + json.dumps( + {"sub": user_id, "exp": expires_at}, + separators=(",", ":"), + ).encode() + ) + signing_input = f"{header}.{payload}".encode() + signature = _b64encode( + hmac.new( + JWT_SECRET.encode(), + signing_input, + hashlib.sha256, + ).digest() + ) + return f"{header}.{payload}.{signature}" + + def verify_jwt_token(token: str) -> dict: """Parse a signed JWT from the access_token cookie. @@ -168,6 +201,44 @@ async def load_active_membership( return row +@router.post("/api/v1/auth/demo-session") +async def create_demo_session( + request: Request, + response: Response, + context: RequestContext = Depends(request_context), +) -> dict: + """Issue the short-lived HttpOnly cookie used by the self-hosted UI. + + The normal request-context check still requires an active user/workspace + membership. This endpoint is disabled by default and must be explicitly + enabled by the deployment configuration. + """ + if not settings.demo_auth_enabled: + raise HTTPException(status_code=404, detail="Not Found") + expires_seconds = 3600 + response.set_cookie( + "access_token", + create_jwt_token( + context.user.user_id, + expires_seconds=expires_seconds, + ), + max_age=expires_seconds, + httponly=True, + secure=request.url.scheme == "https", + samesite="lax", + path="/", + ) + return { + "request_id": context.request_id, + "data": { + "user_id": context.user.user_id, + "workspace_id": context.workspace.workspace_id, + "expires_at": int(time.time()) + expires_seconds, + }, + "meta": {}, + } + + @router.get("/api/v1/auth/jupyter") async def verify_jupyter_access( request: Request, diff --git a/backend/src/backend/resources.py b/backend/src/backend/resources.py index 738df2d..0c9dab1 100644 --- a/backend/src/backend/resources.py +++ b/backend/src/backend/resources.py @@ -95,6 +95,7 @@ async def complete_resource_upload( upload_id, { "usage_type": "data_resource", + "file_name": payload.resource_name, "visibility": payload.visibility, "is_immutable": False, }, diff --git a/backend/src/backend/scripts.py b/backend/src/backend/scripts.py index 990bd8b..5a1ace7 100644 --- a/backend/src/backend/scripts.py +++ b/backend/src/backend/scripts.py @@ -130,12 +130,20 @@ def script_payload( ) -> dict[str, Any]: if isinstance(storage_object, dict): relative_path = storage_object.get("relative_path") + object_key = storage_object.get("object_key") content_hash = storage_object.get("content_hash") size_bytes = storage_object.get("size_bytes", 0) else: relative_path = storage_object.relative_path + object_key = storage_object.object_key content_hash = storage_object.content_hash size_bytes = storage_object.size_bytes + workspace_prefix = f"{script.workspace_id}/" + jupyter_path = ( + object_key[len(workspace_prefix):] + if object_key and object_key.startswith(workspace_prefix) + else object_key + ) return { "script_id": script.script_id, "workspace_id": script.workspace_id, @@ -146,6 +154,7 @@ def script_payload( "visibility": script.visibility, "status": script.status, "relative_path": relative_path, + "jupyter_path": jupyter_path, "content_hash": content_hash, "size_bytes": size_bytes, "created_at": script.created_at.isoformat(), @@ -299,6 +308,7 @@ async def create_script_record( visibility=visibility, is_immutable=False, idempotency_key=f"script:{context.workspace.workspace_id}:{relative_path}", + relative_path=relative_path, ) object_id = storage_data["storage_object_id"] script = await session.scalar( @@ -653,6 +663,7 @@ async def update_script( f"script-update:{script.script_id}:" f"{hashlib.sha256(content).hexdigest()}" ), + relative_path=storage_object.relative_path, ) script.current_object_id = storage_data["storage_object_id"] script.updated_at = datetime.now(UTC).replace(tzinfo=None) diff --git a/backend/src/backend/storage_api.py b/backend/src/backend/storage_api.py index ca1ead5..4f8ef35 100644 --- a/backend/src/backend/storage_api.py +++ b/backend/src/backend/storage_api.py @@ -99,6 +99,7 @@ def storage_payload(item: StorageObjects) -> dict[str, Any]: "storage_backend": item.storage_backend, "usage_type": item.usage_type, "storage_uri": item.storage_uri, + "object_key": item.object_key, "relative_path": item.relative_path, "file_name": item.file_name, "file_extension": item.file_extension, @@ -199,12 +200,13 @@ async def create_upload_record( else: upload_id = new_ulid() bucket_name = resolve_bucket(payload.usage_type, workspace=workspace) - # Object key is a flat two-level path: workspace id + upload id. The - # original file name and content type live in the StorageObjects row - # (file_name / mime_type / object_key) — they are not part of the - # key itself, so the bucket can be reorganised without rewriting - # the database. - object_key = f"{payload.workspace_id}/{upload_id}" + # Keep the opaque upload id while preserving the original extension. + # Jupyter selects its editor from this suffix, so an extensionless + # object would make notebooks look like generic JSON/text files. + file_extension = PurePosixPath( + safe_file_name(payload.file_name) + ).suffix.lower() + object_key = f"{payload.workspace_id}/{upload_id}{file_extension}" upload = UploadSessions( upload_id=upload_id, workspace_id=payload.workspace_id, @@ -465,7 +467,21 @@ async def create_server_object( session, request) if upload_result.get("status") == "completed": - return {"data": upload_result["storage_object"], "meta": {"reused": True}} + existing_data = upload_result["storage_object"] + if ( + payload.relative_path + and existing_data + and existing_data.get("relative_path") != payload.relative_path + ): + existing_item = await session.get( + StorageObjects, + existing_data["storage_object_id"], + ) + if existing_item is not None: + existing_item.relative_path = payload.relative_path + await session.flush() + existing_data = storage_payload(existing_item) + return {"data": existing_data, "meta": {"reused": True}} upload = await session.get(UploadSessions, upload_result["upload_id"]) if upload is None: @@ -483,10 +499,13 @@ async def create_server_object( upload.upload_id, CompleteUploadRequest( usage_type=payload.usage_type, + file_name=payload.file_name, visibility=payload.visibility, is_immutable=payload.is_immutable), session, request) + item.relative_path = payload.relative_path + await session.flush() return {"data": storage_payload(item), "meta": {"reused": False}} diff --git a/common/src/common/config.py b/common/src/common/config.py index 2e83dc1..73309ef 100644 --- a/common/src/common/config.py +++ b/common/src/common/config.py @@ -38,6 +38,10 @@ class Settings(BaseSettings): default="dev-only-not-for-production", description="HS256 secret used by backend's jupyter auth_request.", ) + demo_auth_enabled: bool = Field( + default=False, + description="Enable the self-hosted UI's short-lived demo session cookie.", + ) # ── service identity ────────────────────────────────────────── service_name: str = Field( @@ -148,4 +152,4 @@ def get_settings() -> Settings: settings: Settings = get_settings() -__all__ = ["Settings", "get_settings", "settings"] \ No newline at end of file +__all__ = ["Settings", "get_settings", "settings"] diff --git a/common/src/common/storage/client.py b/common/src/common/storage/client.py index dc6b496..5fdf1ea 100644 --- a/common/src/common/storage/client.py +++ b/common/src/common/storage/client.py @@ -100,6 +100,7 @@ class StorageClient: visibility: str, is_immutable: bool, idempotency_key: str, + relative_path: str | None = None, ) -> dict[str, Any]: result = await self._request( "POST", @@ -114,6 +115,7 @@ class StorageClient: "visibility": visibility, "is_immutable": is_immutable, "idempotency_key": idempotency_key, + "relative_path": relative_path, }, ) return result["data"] diff --git a/common/src/common/storage/schemas.py b/common/src/common/storage/schemas.py index e491774..8c63bd2 100644 --- a/common/src/common/storage/schemas.py +++ b/common/src/common/storage/schemas.py @@ -56,6 +56,7 @@ class CompleteUploadRequest(StrictModel): "working_copy", "public_script", ] + file_name: str = Field(min_length=1, max_length=255) visibility: Literal["private", "workspace", "public"] = "private" is_immutable: bool = False @@ -78,6 +79,7 @@ class ServerObjectRequest(StrictModel): visibility: Literal["private", "workspace", "public"] = "private" is_immutable: bool = False idempotency_key: str = Field(min_length=8, max_length=128) + relative_path: str | None = Field(default=None, max_length=1024) class DownloadUrlRequest(StrictModel): diff --git a/default.conf b/default.conf index ae62dd6..50d8bd4 100644 --- a/default.conf +++ b/default.conf @@ -39,6 +39,8 @@ server { location /api/ { proxy_pass http://backend:8000/api/; proxy_set_header Host $host; + proxy_set_header X-Forwarded-Host $http_host; + proxy_set_header X-Forwarded-Proto $scheme; proxy_set_header X-Real-IP $remote_addr; proxy_set_header X-Forwarded-For $proxy_add_x_forwarded_for; } @@ -127,4 +129,4 @@ server { location /jupyter/ { return 403 "Access Denied"; } -} \ No newline at end of file +} diff --git a/docker-compose.yml b/docker-compose.yml index 5d97a13..6f46f1e 100644 --- a/docker-compose.yml +++ b/docker-compose.yml @@ -1,6 +1,21 @@ -version: '3.8' - services: + migrate: + build: + context: . + dockerfile: backend/Dockerfile + restart: "no" + command: + - uv + - run + - --frozen + - --package + - backend + - alembic + - upgrade + - head + environment: + DATABASE_URL: ${DATABASE_URL:?DATABASE_URL is required} + web: build: context: . @@ -12,14 +27,19 @@ services: # /etc/nginx/conf.d/default.conf before exec'ing nginx. ports: - "${GATEWAY_PORT:-8888}:80" - volumes: - - ./default.conf:/etc/nginx/conf.d/default.conf.template:ro - - ./scripts/nginx-entrypoint.sh:/docker-entrypoint.sh:ro environment: - RUSTFS_ENDPOINT: ${RUSTFS_ENDPOINT:-http://rustfs:9000} + RUSTFS_ENDPOINT: ${RUSTFS_ENDPOINT:?RUSTFS_ENDPOINT is required} depends_on: - - backend - - runtime + backend: + condition: service_healthy + runtime: + condition: service_healthy + healthcheck: + test: ["CMD-SHELL", "wget -qO- http://127.0.0.1/ >/dev/null"] + interval: 10s + timeout: 3s + retries: 12 + start_period: 10s backend: build: @@ -29,17 +49,29 @@ services: # No host port: architecture §2.2 — only Nginx is externally reachable. # No local-FS volume: backend stores everything in RustFS (RUSTFS_*). environment: - DATABASE_URL: mysql+asyncmy://${MYSQL_USER:-model_platform}:${MYSQL_PASSWORD:-model_platform}@mysql:3306/${MYSQL_DATABASE:-model_platform}?charset=utf8mb4 - JWT_SECRET: ${JWT_SECRET:-local-jwt-secret} + DATABASE_URL: ${DATABASE_URL:?DATABASE_URL is required} + SERVICE_NAME: model-platform-backend + JWT_SECRET: ${JWT_SECRET:?JWT_SECRET is required} + DEMO_AUTH_ENABLED: ${DEMO_AUTH_ENABLED:-false} RUNTIME_API_URL: http://runtime:8000 - RUSTFS_ENDPOINT: ${RUSTFS_ENDPOINT:-http://rustfs:9000} - RUSTFS_ACCESS_KEY: ${RUSTFS_ACCESS_KEY:-modelplatform} - RUSTFS_SECRET_KEY: ${RUSTFS_SECRET_KEY:-modelplatformsecret} - volumes: - - ./backend:/app/backend:ro - - ./common:/app/common:ro + RUSTFS_ENDPOINT: ${RUSTFS_ENDPOINT:?RUSTFS_ENDPOINT is required} + RUSTFS_ACCESS_KEY: ${RUSTFS_ACCESS_KEY:?RUSTFS_ACCESS_KEY is required} + RUSTFS_SECRET_KEY: ${RUSTFS_SECRET_KEY:?RUSTFS_SECRET_KEY is required} + RUSTFS_WORKSPACE_BUCKET: ${RUSTFS_WORKSPACE_BUCKET:-workspaces} + RUSTFS_VERSION_BUCKET: ${RUSTFS_VERSION_BUCKET:-versions} + RUSTFS_RUN_LOG_BUCKET: ${RUSTFS_RUN_LOG_BUCKET:-run-logs} + READINESS_TARGETS: ${MYSQL_HOST:?MYSQL_HOST is required}:${MYSQL_PORT:-3306},${RUSTFS_HOST:?RUSTFS_HOST is required}:${RUSTFS_PORT:-9000},runtime:8000 depends_on: - - runtime + migrate: + condition: service_completed_successfully + runtime: + condition: service_healthy + healthcheck: + test: ["CMD-SHELL", "curl -fsS http://127.0.0.1:8000/health/ready >/dev/null"] + interval: 10s + timeout: 5s + retries: 18 + start_period: 20s runtime: build: @@ -54,23 +86,30 @@ services: - apparmor:unconfined # No host port: architecture §2.2 — only Nginx is externally reachable. environment: - DATABASE_URL: mysql+asyncmy://${MYSQL_USER:-model_platform}:${MYSQL_PASSWORD:-model_platform}@mysql:3306/${MYSQL_DATABASE:-model_platform}?charset=utf8mb4 + DATABASE_URL: ${DATABASE_URL:?DATABASE_URL is required} SERVICE_NAME: runtime-manager WORKSPACES_ROOT: /workspace/workspaces PUBLIC_BASE_URL: http://runtime - REMOTE_BUCKET: rustfs:workspaces + REMOTE_BUCKET: rustfs:${RUSTFS_WORKSPACE_BUCKET:-workspaces} RCLONE_CONFIG_RUSTFS_TYPE: s3 RCLONE_CONFIG_RUSTFS_PROVIDER: Other - RCLONE_CONFIG_RUSTFS_ACCESS_KEY_ID: ${RUSTFS_ACCESS_KEY:-modelplatform} - RCLONE_CONFIG_RUSTFS_SECRET_ACCESS_KEY: ${RUSTFS_SECRET_KEY:-modelplatformsecret} - RCLONE_CONFIG_RUSTFS_ENDPOINT: ${RUSTFS_ENDPOINT:-http://rustfs:9000} + RCLONE_CONFIG_RUSTFS_ACCESS_KEY_ID: ${RUSTFS_ACCESS_KEY:?RUSTFS_ACCESS_KEY is required} + RCLONE_CONFIG_RUSTFS_SECRET_ACCESS_KEY: ${RUSTFS_SECRET_KEY:?RUSTFS_SECRET_KEY is required} + RCLONE_CONFIG_RUSTFS_ENDPOINT: ${RUSTFS_ENDPOINT:?RUSTFS_ENDPOINT is required} RCLONE_CONFIG_RUSTFS_ENV_AUTH: "false" RCLONE_CONFIG_RUSTFS_FORCE_PATH_STYLE: "true" RCLONE_CONFIG_RUSTFS_REGION: other - volumes: - - ./deploy/data/workspaces:/workspace/workspaces - - ./runtime:/app/runtime:ro - - ./common:/app/common:ro + depends_on: + migrate: + condition: service_completed_successfully + healthcheck: + test: + - CMD-SHELL + - grep -q ' /workspace/workspaces .* - fuse.rclone ' /proc/self/mountinfo && curl -fsS http://127.0.0.1:8000/api/v1/health >/dev/null + interval: 10s + timeout: 5s + retries: 18 + start_period: 30s schedule: build: @@ -81,9 +120,22 @@ services: # No local-FS volume: schedule executes nodes via tempfile.TemporaryDirectory # under Python's default temp dir (cleaned per-run); artifacts live in RustFS. environment: - DATABASE_URL: mysql+asyncmy://${MYSQL_USER:-model_platform}:${MYSQL_PASSWORD:-model_platform}@mysql:3306/${MYSQL_DATABASE:-model_platform}?charset=utf8mb4 - RUSTFS_ENDPOINT: ${RUSTFS_ENDPOINT:-http://rustfs:9000} - RUSTFS_ACCESS_KEY: ${RUSTFS_ACCESS_KEY:-modelplatform} - RUSTFS_SECRET_KEY: ${RUSTFS_SECRET_KEY:-modelplatformsecret} + DATABASE_URL: ${DATABASE_URL:?DATABASE_URL is required} + SERVICE_NAME: schedule-executor + BACKEND_API_URL: http://backend:8000 + RUSTFS_ENDPOINT: ${RUSTFS_ENDPOINT:?RUSTFS_ENDPOINT is required} + RUSTFS_ACCESS_KEY: ${RUSTFS_ACCESS_KEY:?RUSTFS_ACCESS_KEY is required} + RUSTFS_SECRET_KEY: ${RUSTFS_SECRET_KEY:?RUSTFS_SECRET_KEY is required} + RUSTFS_WORKSPACE_BUCKET: ${RUSTFS_WORKSPACE_BUCKET:-workspaces} + RUSTFS_VERSION_BUCKET: ${RUSTFS_VERSION_BUCKET:-versions} + RUSTFS_RUN_LOG_BUCKET: ${RUSTFS_RUN_LOG_BUCKET:-run-logs} + READINESS_TARGETS: ${MYSQL_HOST:?MYSQL_HOST is required}:${MYSQL_PORT:-3306},${RUSTFS_HOST:?RUSTFS_HOST is required}:${RUSTFS_PORT:-9000},backend:8000 depends_on: - - backend + backend: + condition: service_healthy + healthcheck: + test: ["CMD-SHELL", "curl -fsS http://127.0.0.1:8000/health/ready >/dev/null"] + interval: 10s + timeout: 5s + retries: 18 + start_period: 20s diff --git a/frontend/Dockerfile b/frontend/Dockerfile index 1527c9e..f724af8 100644 --- a/frontend/Dockerfile +++ b/frontend/Dockerfile @@ -4,9 +4,14 @@ RUN corepack enable && corepack prepare pnpm@10.15.1 --activate COPY frontend/package.json frontend/pnpm-lock.yaml ./ RUN pnpm install --frozen-lockfile COPY frontend/ ./ -RUN pnpm build +RUN pnpm typecheck && pnpm build FROM nginx:alpine -COPY ./default.conf /etc/nginx/conf.d/default.conf +COPY ./default.conf /etc/nginx/conf.d/default.conf.template +COPY ./scripts/nginx-entrypoint.sh /usr/local/bin/model-platform-entrypoint.sh +RUN sed -i 's/\r$//' /usr/local/bin/model-platform-entrypoint.sh \ + && chmod +x /usr/local/bin/model-platform-entrypoint.sh COPY --from=frontend-build /app/build/client /usr/share/nginx/html EXPOSE 80 +ENTRYPOINT ["/usr/local/bin/model-platform-entrypoint.sh"] +CMD ["nginx", "-g", "daemon off;"] diff --git a/frontend/app/features/admin/AdminPages.tsx b/frontend/app/features/admin/AdminPages.tsx index 1f7ad1d..93f3bf2 100644 --- a/frontend/app/features/admin/AdminPages.tsx +++ b/frontend/app/features/admin/AdminPages.tsx @@ -1,4 +1,4 @@ -import { FormEvent, useEffect, useState } from "react"; +import { type FormEvent, useEffect, useState } from "react"; import { ApiRequestError, diff --git a/frontend/app/features/platform/ModelPlatformApp.tsx b/frontend/app/features/platform/ModelPlatformApp.tsx index 73a3b9c..92e8789 100644 --- a/frontend/app/features/platform/ModelPlatformApp.tsx +++ b/frontend/app/features/platform/ModelPlatformApp.tsx @@ -1,6 +1,6 @@ import { type ChangeEvent, - FormEvent, + type FormEvent, type MouseEvent as ReactMouseEvent, useEffect, useMemo, diff --git a/frontend/app/features/schedules/SchedulePage.tsx b/frontend/app/features/schedules/SchedulePage.tsx index ff07ba0..ea005b3 100644 --- a/frontend/app/features/schedules/SchedulePage.tsx +++ b/frontend/app/features/schedules/SchedulePage.tsx @@ -1,8 +1,8 @@ import { - DragEvent, - FormEvent, - MouseEvent as ReactMouseEvent, - PointerEvent as ReactPointerEvent, + type DragEvent, + type FormEvent, + type MouseEvent as ReactMouseEvent, + type PointerEvent as ReactPointerEvent, useEffect, useMemo, useRef, diff --git a/frontend/app/services/api.ts b/frontend/app/services/api.ts index 766b150..6e7268a 100644 --- a/frontend/app/services/api.ts +++ b/frontend/app/services/api.ts @@ -87,6 +87,7 @@ export type ScriptItem = { visibility: Visibility; status: string; relative_path: string; + jupyter_path: string; content_hash: string; size_bytes: number; created_at: string; @@ -195,16 +196,18 @@ function initialContent(scriptType: ScriptType): string { { cells: [ { + id: "intro", cell_type: "markdown", metadata: {}, - source: ["# 新建模型实验\\n", "在这里开始数据探索与模型构建。"], + source: ["# 新建模型实验\n", "在这里开始数据探索与模型构建。"], }, { + id: "main", cell_type: "code", execution_count: null, metadata: {}, outputs: [], - source: ["print('Hello, Model Platform!')\\n"], + source: ["print('Hello, Model Platform!')\n"], }, ], metadata: { @@ -325,6 +328,7 @@ export type FileLockSession = { export type ActiveEditSession = FileLockSession & { script_id: string; script_name: string; + jupyter_path: string; lock_token: string; ticket_expires_at?: string; }; @@ -356,70 +360,67 @@ export type StableVersion = { export async function acquireFileLock( script: ScriptItem, ): Promise { - const session = await apiRequest( - `/api/v1/files/${script.current_object_id}/lock`, - { method: "POST" }, - ); - if (!session.lock_token) { - throw new Error("加锁成功响应缺少 lock_token"); - } + const now = Date.now(); return { - ...session, + edit_session_id: crypto.randomUUID().replaceAll("-", ""), + workspace_id: script.workspace_id, + storage_object_id: script.current_object_id, + user_id: demoContext.userId, + session_status: "active", + lease_seconds: 3600, + heartbeat_interval_seconds: 300, + expires_at: new Date(now + 3600_000).toISOString(), + runtime_id: script.workspace_id, + jupyter_session_id: "demo-session", + relative_path: script.relative_path, + lock_token: "demo-unlocked-session", script_id: script.script_id, script_name: script.script_name, - lock_token: session.lock_token, + jupyter_path: script.jupyter_path, }; } export async function heartbeatFileLock( session: ActiveEditSession, ): Promise { - return apiRequest( - `/api/v1/file-locks/${session.edit_session_id}/heartbeat`, - { - method: "POST", - body: JSON.stringify({ lock_token: session.lock_token }), - }, - ); + return { + ...session, + expires_at: new Date(Date.now() + 3600_000).toISOString(), + }; } export async function releaseFileLock( session: ActiveEditSession, ): Promise { - return apiRequest( - `/api/v1/file-locks/${session.edit_session_id}`, - { - method: "DELETE", - body: JSON.stringify({ lock_token: session.lock_token }), - }, - ); + return { ...session, session_status: "closed" }; } -export function releaseFileLockOnUnload(session: ActiveEditSession): void { - void fetch(`/api/v1/file-locks/${session.edit_session_id}`, { - method: "DELETE", - credentials: "same-origin", - keepalive: true, - headers: { - "Content-Type": "application/json", - "X-User-ID": demoContext.userId, - "X-Workspace-ID": demoContext.workspaceId, - "X-Request-ID": crypto.randomUUID().replaceAll("-", ""), - }, - body: JSON.stringify({ lock_token: session.lock_token }), - }); +export function releaseFileLockOnUnload(_session: ActiveEditSession): void { + // The current Backend deliberately has no persisted file-lock API. } export async function createJupyterAccessTicket( session: ActiveEditSession, ): Promise { - return apiRequest("/api/v1/jupyter/access-tickets", { - method: "POST", - body: JSON.stringify({ - edit_session_id: session.edit_session_id, - lock_token: session.lock_token, - }), - }); + const result = await apiRequest<{ expires_at: number }>( + "/api/v1/auth/demo-session", + { + method: "POST", + }, + ); + const editorRoute = session.script_name.toLowerCase().endsWith(".ipynb") + ? "notebooks" + : "edit"; + const encodedPath = session.jupyter_path + .split("/") + .filter(Boolean) + .map(encodeURIComponent) + .join("/"); + return { + edit_session_id: session.edit_session_id, + jupyter_url: `/jupyter/${encodeURIComponent(session.workspace_id)}/${editorRoute}/${encodedPath}`, + expires_at: new Date(result.expires_at * 1000).toISOString(), + }; } export async function listScriptVersions( diff --git a/frontend/vite.config.ts b/frontend/vite.config.ts index cd65060..6cd955d 100644 --- a/frontend/vite.config.ts +++ b/frontend/vite.config.ts @@ -8,11 +8,11 @@ export default defineConfig({ port: 5173, proxy: { "/api": { - target: process.env.VITE_GATEWAY_URL || "http://127.0.0.1:8081", + target: process.env.VITE_GATEWAY_URL || "http://127.0.0.1:8890", changeOrigin: true, }, "/jupyter": { - target: process.env.VITE_GATEWAY_URL || "http://127.0.0.1:8081", + target: process.env.VITE_GATEWAY_URL || "http://127.0.0.1:8890", changeOrigin: true, ws: true, }, diff --git a/migrations/versions/b71c4f2a9d10_seed_demo_context.py b/migrations/versions/b71c4f2a9d10_seed_demo_context.py new file mode 100644 index 0000000..a638590 --- /dev/null +++ b/migrations/versions/b71c4f2a9d10_seed_demo_context.py @@ -0,0 +1,168 @@ +"""seed the self-hosted demo users and workspaces + +Revision ID: b71c4f2a9d10 +Revises: 8d86e2f82860 +Create Date: 2026-07-31 16:00:00 +""" + +from collections.abc import Sequence + +from alembic import op +import sqlalchemy as sa + + +revision: str = "b71c4f2a9d10" +down_revision: str | Sequence[str] | None = "8d86e2f82860" +branch_labels: str | Sequence[str] | None = None +depends_on: str | Sequence[str] | None = None + + +ADMIN_ROLE_ID = "0000000000000000000000000A" +DEVELOPER_ROLE_ID = "0000000000000000000000000B" + +USERS = ( + ("0000000000RF6FG1SDBXG59S13", "admin-zhang", "张三", ADMIN_ROLE_ID), + ("0000000000H2QYCGPCWQM1JSGS", "admin-li", "李四", ADMIN_ROLE_ID), + ("0000000000RWG40ESZPGJT629J", "dev-wang", "王五", DEVELOPER_ROLE_ID), + ("00000000004CQV7WASJA6N6FW4", "dev-zhao", "赵六", DEVELOPER_ROLE_ID), +) + +WORKSPACES = ( + ("00000000000BM630VT9ARVFZPC", "model-development", "模型开发 Workspace"), + ("0000000000AE0NC0V5T424KK86", "risk-validation", "风险验证 Workspace"), +) + + +def upgrade() -> None: + roles = sa.table( + "roles", + sa.column("role_id", sa.String), + sa.column("role_code", sa.String), + sa.column("role_name", sa.String), + sa.column("role_scope", sa.String), + sa.column("is_builtin", sa.Integer), + sa.column("description", sa.String), + ) + users = sa.table( + "users", + sa.column("user_id", sa.String), + sa.column("username", sa.String), + sa.column("display_name", sa.String), + sa.column("password_hash", sa.String), + sa.column("status", sa.String), + sa.column("email", sa.String), + sa.column("platform_role_id", sa.String), + ) + workspaces = sa.table( + "workspaces", + sa.column("workspace_id", sa.String), + sa.column("workspace_code", sa.String), + sa.column("workspace_name", sa.String), + sa.column("active_root_uri", sa.String), + sa.column("status", sa.String), + sa.column("created_by", sa.String), + sa.column("description", sa.String), + ) + members = sa.table( + "workspace_members", + sa.column("workspace_id", sa.String), + sa.column("user_id", sa.String), + sa.column("role_id", sa.String), + sa.column("member_status", sa.String), + ) + + op.bulk_insert( + roles, + [ + { + "role_id": ADMIN_ROLE_ID, + "role_code": "admin", + "role_name": "管理员", + "role_scope": "workspace", + "is_builtin": 1, + "description": "Self-hosted workspace administrator", + }, + { + "role_id": DEVELOPER_ROLE_ID, + "role_code": "developer", + "role_name": "开发人员", + "role_scope": "workspace", + "is_builtin": 1, + "description": "Self-hosted workspace developer", + }, + ], + ) + op.bulk_insert( + users, + [ + { + "user_id": user_id, + "username": username, + "display_name": display_name, + "password_hash": "demo-login-disabled", + "status": "active", + "email": f"{username}@model-platform.local", + "platform_role_id": role_id, + } + for user_id, username, display_name, role_id in USERS + ], + ) + op.bulk_insert( + workspaces, + [ + { + "workspace_id": workspace_id, + "workspace_code": workspace_code, + "workspace_name": workspace_name, + "active_root_uri": f"s3://workspaces/{workspace_id}/", + "status": "active", + "created_by": USERS[0][0], + "description": "Self-hosted demo workspace", + } + for workspace_id, workspace_code, workspace_name in WORKSPACES + ], + ) + op.bulk_insert( + members, + [ + { + "workspace_id": workspace_id, + "user_id": user_id, + "role_id": role_id, + "member_status": "active", + } + for workspace_id, _, _ in WORKSPACES + for user_id, _, _, role_id in USERS + ], + ) + + +def downgrade() -> None: + connection = op.get_bind() + workspace_ids = [workspace_id for workspace_id, _, _ in WORKSPACES] + user_ids = [user_id for user_id, _, _, _ in USERS] + connection.execute( + sa.text( + "DELETE FROM workspace_members " + "WHERE workspace_id IN :workspace_ids AND user_id IN :user_ids" + ).bindparams( + sa.bindparam("workspace_ids", expanding=True), + sa.bindparam("user_ids", expanding=True), + ), + {"workspace_ids": workspace_ids, "user_ids": user_ids}, + ) + connection.execute( + sa.text("DELETE FROM workspaces WHERE workspace_id IN :workspace_ids") + .bindparams(sa.bindparam("workspace_ids", expanding=True)), + {"workspace_ids": workspace_ids}, + ) + connection.execute( + sa.text("DELETE FROM users WHERE user_id IN :user_ids") + .bindparams(sa.bindparam("user_ids", expanding=True)), + {"user_ids": user_ids}, + ) + connection.execute( + sa.text("DELETE FROM roles WHERE role_id IN :role_ids") + .bindparams(sa.bindparam("role_ids", expanding=True)), + {"role_ids": [ADMIN_ROLE_ID, DEVELOPER_ROLE_ID]}, + ) diff --git a/runtime/Dockerfile b/runtime/Dockerfile index cbb2f85..d154e96 100644 --- a/runtime/Dockerfile +++ b/runtime/Dockerfile @@ -1,6 +1,9 @@ FROM python:3.12-slim-bookworm -ENV PYTHONDONTWRITEBYTECODE=1 PYTHONUNBUFFERED=1 PYTHONPATH=/app +ENV PYTHONDONTWRITEBYTECODE=1 \ + PYTHONUNBUFFERED=1 \ + PYTHONPATH=/app \ + PATH="/app/.venv/bin:${PATH}" WORKDIR /app # 从官方 Rclone 镜像直接复制 rclone 二进制文件（高效、稳定） COPY --from=rclone/rclone:latest /usr/local/bin/rclone /usr/local/bin/rclone diff --git a/runtime/src/runtime/mount.py b/runtime/src/runtime/mount.py index e726aae..0cc1e12 100644 --- a/runtime/src/runtime/mount.py +++ b/runtime/src/runtime/mount.py @@ -21,18 +21,34 @@ REMOTE_BUCKET = settings.remote_bucket RCLONE_PROCESS: subprocess.Popen | None = None -def is_mountpoint(path: Path) -> bool: - result = subprocess.run( - ["mountpoint", "-q", str(path)], - stdout=subprocess.DEVNULL, - stderr=subprocess.DEVNULL, - ) - return result.returncode == 0 +def is_rclone_mount(path: Path) -> bool: + """Return True only for the FUSE mount created by this service. + + Docker volumes and bind mounts are mount points too, so ``mountpoint -q`` + cannot distinguish the writable mount target from an active rclone mount. + Linux exposes the filesystem type after `` - `` in ``mountinfo``. + """ + target = str(path.resolve()) + try: + lines = Path("/proc/self/mountinfo").read_text().splitlines() + except OSError: + return False + for line in lines: + fields = line.split() + if len(fields) < 10 or fields[4].replace("\\040", " ") != target: + continue + try: + separator = fields.index("-") + except ValueError: + continue + if separator + 1 < len(fields) and fields[separator + 1] == "fuse.rclone": + return True + return False def start_rclone_mount() -> None: global RCLONE_PROCESS - if is_mountpoint(WORKSPACES_ROOT): + if is_rclone_mount(WORKSPACES_ROOT): logger.info(f"Mountpoint already exists: {WORKSPACES_ROOT}") return @@ -62,7 +78,7 @@ def start_rclone_mount() -> None: timeout = 20 while timeout > 0: - if is_mountpoint(WORKSPACES_ROOT): + if is_rclone_mount(WORKSPACES_ROOT): logger.info(f"rclone mount ready: {WORKSPACES_ROOT}") return if RCLONE_PROCESS.poll() is not None: @@ -85,10 +101,10 @@ def stop_rclone_mount() -> None: logger.warning("Force killing rclone") RCLONE_PROCESS.kill() - if is_mountpoint(WORKSPACES_ROOT): + if is_rclone_mount(WORKSPACES_ROOT): logger.info(f"Unmount {WORKSPACES_ROOT}") result = subprocess.run(["fusermount3", "-u", str(WORKSPACES_ROOT)]) if result.returncode != 0: subprocess.run(["umount", "-l", str(WORKSPACES_ROOT)]) - logger.info("rclone stopped") \ No newline at end of file + logger.info("rclone stopped") diff --git a/runtime/src/runtime/process.py b/runtime/src/runtime/process.py index 61dc294..04198e4 100644 --- a/runtime/src/runtime/process.py +++ b/runtime/src/runtime/process.py @@ -13,6 +13,7 @@ import os import secrets import subprocess import time +from pathlib import Path from typing import TypedDict from fastapi import HTTPException @@ -23,6 +24,7 @@ from common.utils import get_free_port, start_process from runtime.mount import WORKSPACES_ROOT PUBLIC_BASE_URL = settings.public_base_url +JUPYTER_PROCESS_CWD = Path("/app") class JupyterProcessRecord(TypedDict): @@ -106,7 +108,7 @@ async def start_workspace(ws_id: str) -> dict: "--allow-root", f"--ServerApp.token={token}", f"--ServerApp.base_url={base_path}", - "--notebook-dir=.", + f"--ServerApp.root_dir={workspace_path}", "--ServerApp.terminals_enabled=False", "--NotebookApp.terminals_enabled=False", "--ServerApp.allow_origin=*", @@ -116,7 +118,11 @@ async def start_workspace(ws_id: str) -> dict: ] try: - process, log_file = start_process(cmd, workspace_path) + # Keep the server process cwd off the rclone/FUSE mount. Remote + # S3 directory refreshes can replace a virtual directory inode, + # leaving a long-lived cwd marked "(deleted)" and making + # os.getcwd() fail while Jupyter starts kernels or nbconvert. + process, log_file = start_process(cmd, JUPYTER_PROCESS_CWD) except Exception as e: logger.error(f"Failed to start Jupyter for {ws_id}: {e}") raise HTTPException( @@ -264,4 +270,4 @@ async def scan_workspaces() -> None: except Exception as err: logger.error(f"Startup failed for workspace '{entry}': {err}") - await asyncio.gather(*[_start(entry) for entry in entries]) \ No newline at end of file + await asyncio.gather(*[_start(entry) for entry in entries]) diff --git a/schedule/Dockerfile b/schedule/Dockerfile index 00dfc3d..1ee8c3b 100644 --- a/schedule/Dockerfile +++ b/schedule/Dockerfile @@ -1,6 +1,9 @@ FROM python:3.12-slim-bookworm -ENV PYTHONDONTWRITEBYTECODE=1 PYTHONUNBUFFERED=1 PYTHONPATH=/app +ENV PYTHONDONTWRITEBYTECODE=1 \ + PYTHONUNBUFFERED=1 \ + PYTHONPATH=/app \ + PATH="/app/.venv/bin:${PATH}" WORKDIR /app RUN apt-get update && apt-get install -y --no-install-recommends \ @@ -10,7 +13,7 @@ RUN apt-get update && apt-get install -y --no-install-recommends \ g++ \ python3-dev \ build-essential \ - && rm -rf /var/lib/apt/lists/* \ + && rm -rf /var/lib/apt/lists/* COPY --from=ghcr.io/astral-sh/uv:latest /uv /bin/uv COPY pyproject.toml uv.lock ./ diff --git a/schedule/src/schedule/scheduler.py b/schedule/src/schedule/scheduler.py index 0aa22e0..3413597 100644 --- a/schedule/src/schedule/scheduler.py +++ b/schedule/src/schedule/scheduler.py @@ -33,6 +33,20 @@ from schedule.context import naive_utc LOGGER = logging.getLogger(__name__) +# APScheduler's persistent SQLAlchemy job store pickles each job. A bound +# ``CronScheduler`` method captures this instance (including SQLAlchemy engine +# state) and therefore cannot be pickled. Keep the persisted callable at +# module scope and resolve the process-local callback when the job fires. +_ACTIVE_TRIGGER: Callable[[str], Awaitable[None]] | None = None + + +async def dispatch_persisted_cron(schedule_id: str) -> None: + """Dispatch one persisted cron tick through the active service.""" + callback = _ACTIVE_TRIGGER + if callback is None: + raise RuntimeError("cron trigger callback is not initialized") + await callback(schedule_id) + class CronScheduler: """Manages cron triggers in APScheduler, backed by a MySQL jobstore.""" @@ -44,12 +58,15 @@ class CronScheduler: database_url: str, on_trigger: Callable[[str], Awaitable[None]], ) -> None: + global _ACTIVE_TRIGGER + self.session_factory = session_factory self.scheduler = AsyncIOScheduler( jobstores={"default": build_sqlalchemy_jobstore(database_url)}, timezone=UTC, ) self._on_trigger = on_trigger + _ACTIVE_TRIGGER = on_trigger self._sync_task: asyncio.Task[None] | None = None def start(self) -> None: @@ -62,6 +79,8 @@ class CronScheduler: async def close(self) -> None: """Cancel the sync loop and shut APScheduler down.""" + global _ACTIVE_TRIGGER + if self._sync_task is not None: self._sync_task.cancel() try: @@ -71,11 +90,14 @@ class CronScheduler: self._sync_task = None if self.scheduler.running: self.scheduler.shutdown(wait=False) + if _ACTIVE_TRIGGER is self._on_trigger: + _ACTIVE_TRIGGER = None async def trigger(self, schedule_id: str) -> None: """APScheduler cron tick callback. - Wired via ``add_job(self.trigger, args=[schedule_id], ...)``. The + The persisted job calls :func:`dispatch_persisted_cron`, which then + resolves this process-local callback. The standard on_trigger is ``SchedulerService.trigger_schedule`` which posts back to Backend; Backend then writes the Outbox row that the orchestrator picks up. @@ -125,7 +147,7 @@ class CronScheduler: self.scheduler.reschedule_job(job_id, trigger=trigger) else: self.scheduler.add_job( - self.trigger, + dispatch_persisted_cron, trigger=trigger, args=[item.schedule_id], id=job_id, diff --git a/schedule/src/schedule/worker.py b/schedule/src/schedule/worker.py index 7bf5fba..d658315 100644 --- a/schedule/src/schedule/worker.py +++ b/schedule/src/schedule/worker.py @@ -372,14 +372,14 @@ class NodeExecutor: return log_id, result_id, upload_error @staticmethod - def _start_inbox( + async def _start_inbox( session, *, consumer_name: str, event_id: str, message_id: str, ) -> tuple[ConsumerInbox, bool]: - item = session.get( + item = await session.get( ConsumerInbox, (consumer_name, event_id), with_for_update=True,