diff --git a/.dockerignore b/.dockerignore new file mode 100644 index 0000000..e336878 --- /dev/null +++ b/.dockerignore @@ -0,0 +1,13 @@ +.git +.gitignore +**/__pycache__ +**/*.py[cod] +**/.pytest_cache +**/.mypy_cache +**/.ruff_cache +**/.venv +frontend/node_modules +frontend/build +frontend/.react-router +deploy/data +*.zip diff --git a/.env.example b/.env.example new file mode 100644 index 0000000..3216d94 --- /dev/null +++ b/.env.example @@ -0,0 +1,85 @@ +COMPOSE_PROJECT_NAME=model-platform-develop +SCHEDULE_EVENT_NAMESPACE=model-platform-develop + +# External port of the Nginx gateway. Only Nginx is exposed to the host +# (architecture §2.2); backend/runtime/schedule stay on the Docker internal +# network. Override here to expose Nginx on a different host port. +GATEWAY_PORT=8890 + +# External MySQL. URL-encode reserved characters in DATABASE_URL. +MYSQL_HOST=127.0.0.1 +MYSQL_PORT=3306 +MYSQL_USER=root +MYSQL_PASSWORD=change-me +MYSQL_DATABASE=model_platform +DATABASE_URL=mysql+asyncmy://root:change-me@127.0.0.1:3306/model_platform?charset=utf8mb4 + +# Demo login is only intended for this self-hosted development UI. +DEMO_AUTH_ENABLED=true +JWT_SECRET=change-this-development-secret + +# ============================================================================ +# CRITICAL: must set BEFORE first run. The initial admin user is seeded by +# the deployment bootstrap. Never keep the development default in production. +# ============================================================================ +INITIAL_ADMIN_PASSWORD=admin12345 + +# Backend loguru stderr sink level. One of DEBUG / INFO / WARNING / ERROR +# / CRITICAL. Anything else (e.g. lowercase) falls back to INFO inside +# configure_logging(). Change to DEBUG to see request bodies in +# runtime_client._jupyter_request. +LOG_LEVEL=INFO + +# Object storage. Two modes are supported: +# STORAGE_BACKEND=s3 — connects to an S3-compatible service (MinIO, +# RustFS, SeaweedFS, AWS S3, …). Requires the +# S3_* block below. +# STORAGE_BACKEND=local — stores objects on the local filesystem under +# LOCAL_STORAGE_BASE_DIR. Backend and runtime +# share this directory via a Docker volume +# (docker-compose.yml mounts `local-storage`). +# Useful for dev, single-node, air-gapped. +STORAGE_BACKEND=s3 +LOCAL_STORAGE_BASE_DIR=/data + +# Object storage (S3-compatible). Only used when STORAGE_BACKEND=s3. +# S3_ENDPOINT is the single upstream URL consumed by all 4 services: +# - nginx (via scripts/nginx-entrypoint.sh, which parses host + port) +# - backend / runtime / schedule (passed through to boto3 / rclone) +# S3_ACCESS_KEY / S3_SECRET_KEY are read by Python code in +# backend/ and schedule/ (boto3 credentials). +# +# S3 buckets are purpose-named: +# S3_WORKSPACE_BUCKET — workspace files (notebooks, scripts, working +# copies); layout is ``s3:////...``. +# S3_VERSION_BUCKET — immutable script-version artifacts. +# S3_RUN_LOG_BUCKET — schedule run logs and execution results. +# S3_TRASH_BUCKET — soft-deleted objects; source bucket key is preserved +# as a prefix so restore is a same-key move. +S3_HOST=127.0.0.1 +S3_PORT=9000 +S3_ENDPOINT=http://127.0.0.1:9000 +S3_ACCESS_KEY=change-me +S3_SECRET_KEY=change-me +S3_WORKSPACE_BUCKET=workspace +S3_VERSION_BUCKET=version +S3_RUN_LOG_BUCKET=run-log +S3_TRASH_BUCKET=trash +S3_TRASH_RETENTION_DAYS=30 + +# rclone RC (HTTP control API). The runtime container starts rclone with +# `--rc --rc-addr 0.0.0.0:5572 --rc-no-auth` (see runtime/src/runtime/mount.py), +# so the backend can POST /vfs/refresh here to invalidate the FUSE dir-cache +# after writing new workspace files. Default points at the runtime service +# over the compose network. +RCLONE_RC_URL=http://runtime:5572 + +# ============================================================================ +# Service-to-service auth (P0-1 fix). +# Backend's /internal/v1/* storage control plane requires this shared secret. +# The schedule worker reads the same value and sends it as the +# ``X-Internal-Service-Token`` header. Value MUST match between backend and +# schedule. Generate a random 64-char string for any non-dev deployment: +# python -c "import secrets; print(secrets.token_urlsafe(48))" +# ============================================================================ +INTERNAL_SERVICE_TOKEN=change-me-internal-service-token diff --git a/.gitignore b/.gitignore new file mode 100644 index 0000000..eaf4562 --- /dev/null +++ b/.gitignore @@ -0,0 +1,10 @@ +.idea + +.env + +.venv + +__pycache__/ +*.pyc +.DS_Store + diff --git a/API.md b/API.md new file mode 100644 index 0000000..acad6a4 --- /dev/null +++ b/API.md @@ -0,0 +1,1138 @@ +# 模型平台接口文档 + +> 本文档面向**前端开发者与第三方集成方**。所有接口的入口是 Nginx +> 网关(默认 `http://localhost:8888`),除 `/api/v1/auth/jupyter` 由 +> Nginx `auth_request` 自动调用,其他接口都通过 `/api/v1/...` 同源访问。 +> +> 服务端基础 URL 示例: `http://localhost:8888` +> +> 通用响应外壳: +> ```json +> { +> "request_id": "01HXY...", +> "data": { ... }, +> "meta": {} +> } +> ``` +> 错误响应为标准 HTTP 4xx / 5xx,body 为 `{"detail": "..."}` 或 +> `{"code": "...", "message": "...", "details": {}}`。 + +## 目录 + +1. [鉴权](#一鉴权) +2. [统一约定](#二统一约定) +3. [脚本 / Notebook (`/api/v1/scripts/...`)](#三脚本--notebook) +4. [调度 (`/api/v1/schedules/...` + `/api/v1/schedule-runs/...`)](#四调度) +5. [数据资源 (`/api/v1/data-resources/...`)](#五数据资源) +6. [管理后台 (`/api/v1/admin/...`)](#六管理后台) +7. [系统管理 (`/api/v1/platform/...`)](#七系统管理-apiv1platform) +8. [Jupyter 路由 (Nginx `auth_request`)](#八jupyter-路由) +9. [对象存储控制面 (`/internal/v1/objects`,服务间 RPC + Token 鉴权)](#九对象存储控制面) +10. [健康检查](#十健康检查) + +--- + +## 一、鉴权 + +平台用 **JWT (HS256)**。登录后,前端在后续请求里**任选一种**携带方式: + +- **Cookie**(推荐用于浏览器):登录成功后后端种 `Authorization` 或自定义 cookie;前端无需手写。 +- **`Authorization: Bearer `**(推荐用于脚本与第三方)。 + +`JWT_SECRET` 由后端从 `Settings.jwt_secret` 读取,前端不需要知道,只需要保证登录态带过来即可。 + +- 当 `Authorization` 与 `Cookie` 同时存在时,后端**优先**使用 `Authorization`。 +- 缺失或过期 → HTTP `401`。 +- 有效但用户不在 workspace → HTTP `403`(由 Nginx `auth_request` 透传给客户端)。 + +> `/api/v1/auth/me` 与 `/api/v1/auth/login` 响应中的 `data.user` 对象额外携带以下三个字段: +> - `role_code: string | null` —— 用户的**平台角色**(`users.platform_role_id` 指向的 Roles 行的 `role_code`),取值为 `admin` / `developer` / `null`(未分配)。**注意:本字段同时也是该用户在所有 workspace 中的角色**——workspace 角色始终继承自平台角色,本端点不再返回 workspace 级独立角色码。 +> - `is_system_admin: bool` —— 派生自 `users.platform_role_id` 指向的角色 `role_code == 'admin'` 且用户状态为 `active`。前端据此决定是否渲染"系统管理"入口。 +> - `permissions: string[]` —— 当前用户通过其平台角色(`platform_role_id`)间接持有的菜单权限码列表(`permission_code`),按字典序排列;未分配平台角色时为空数组。前端据此过滤菜单与 `` 路由守卫。**仅控制前端展示,不参与后端 endpoint 鉴权**——后端鉴权继续由 `system_admin_context`(`role_code == 'admin'`)与 workspace membership 负责。详见 §7.12-7.14。 + +--- + +## 二、统一约定 + +| 类别 | 约定 | +|---|---| +| ID 格式 | 全部为 ULID(26 字符),如 `01HXY9C5B8N3K4P7Q6RT2V0J8D` | +| 时间戳 | ISO-8601 UTC,毫秒精度,如 `2026-07-31T11:23:45.123` | +| 时区 | 所有 `*_at` 字段均为 UTC,前端需自行转换显示 | +| 分页 | 大列表接口使用 `limit` (≤200) + 隐式 cursor,无 `offset` | +| 幂等键 | 上传类接口要求 `Idempotency-Key` 请求头,≥8 字符,≤128 字符 | +| 软删 | 删除操作走 `is_deleted` 软删,不返回 404;再次查询时已软删资源 `status="deleted"` | +| 排序 | 列表默认按业务键倒序(更新时间 / 入队时间等) | +| 鉴权头 | 见 §一 | + +--- + +## 三、脚本 / Notebook + +> 业务概念: `Scripts` 是用户工作区里的脚本或 notebook,`StorageObjects` +> 是它在对象存储里的"工作副本",`Versions` 是 immutable 的稳定版本。 +> 写操作受 **is_locked 门禁 + owner 校验** 保护(架构 V3.1 §4)。 + +### 3.1 `GET /api/v1/workspace-tree` (**legacy / 全量视图**) + +> **Deprecated**: 新代码请走 §3.3.1 (按 `parent_path` 单层)。该接口仍保留供调试 / 兼容使用, +> 谓词已统一为 `object_status='available' AND is_deleted=0`,与 §3.3.1 保持一致。 + +列出当前用户在 workspace 内的**整个目录树**(扁平数组)。 + +- **来源**: 显式 `StorageObjects` 行 (`object_type='directory'`,见 §3.2) **并入** 从 `Scripts.relative_path` 派生的祖先目录,**去重**。空目录(只有显式行、没有文件)也会出现。 +- **鉴权**: workspace 成员 +- **请求体**: 无 +- **响应**: + ```json + { + "request_id": "...", + "data": { + "directories": [ + {"path": "scripts", "name": "scripts", "parent_path": ""}, + {"path": "scripts/etl", "name": "etl", "parent_path": "scripts"} + ] + }, + "meta": {"directory_count": 2} + } + ``` + +### 3.2 `POST /api/v1/workspace-directories` + +创建一个**目录**。后端会落一行 `StorageObjects(object_type='directory', storage_backend='rustfs', storage_uri='inline://directory/{path}', visibility='private')`,因此空目录也能在 §3.1 树里出现并保留下来。 + +- **请求体**: + ```json + { + "directory_name": "etl", + "parent_path": "scripts" + } + ``` + | 字段 | 必填 | 说明 | + |---|---|---| + | `directory_name` | 是 | 目录名(单段,不能含 `/`) | + | `parent_path` | 否 | 父目录相对路径,空字符串或缺省表示用户根目录 | + +- **父目录存在性校验**: 必须在 `StorageObjects` 存在 `relative_path == scoped_prefix/{parent}` 的行,或 `relative_path` 以 `scoped_prefix/{parent}/` 开头。否则 **404**。 +- **同名冲突**: 已有 `relative_path` 完全相等的行(无论 file / directory) → **409**。冲突判定由应用层 `SELECT ... FOR UPDATE` 完成;`idx_storage_workspace_path(workspace_id, storage_backend, path_hash)` 仅作为查找索引,不再提供 DB 级唯一性兜底。 +- **响应 201**: + ```json + { + "request_id": "...", + "data": { + "storage_object_id": "01HXY...", + "path": "scripts/etl", + "name": "etl", + "parent_path": "scripts" + } + } + ``` + +### 3.3 `DELETE /api/v1/workspace-directories?path=...` + +删除一个目录(以及目录下当前用户拥有的所有 `Scripts`,**会触发 is_locked 校验**)。 +`StorageObjects` 行移到 trash bucket(`settings.s3_trash_bucket`),`object_status` 置为 `deleted`。 + +- **查询参数**: + | 名 | 类型 | 必填 | 说明 | + |---|---|---|---| + | `path` | string | 是 | 相对路径,例如 `scripts/etl` | +- **响应**: + ```json + { + "data": { + "path": "scripts/etl", + "status": "deleted", + "deleted_scripts": 3, + "versions_preserved": true + } + } + ``` + +### 3.3.1 `GET /api/v1/workspace-directories?parent_path=...` + +列出指定父目录下的**直接子目录**(单层),用于前端懒加载。**新代码请走本接口**;§3.1 仅作为 legacy / 全量保留。 + +- **行为**: `parent_path` 为空字符串或缺省 → 用户根目录;非空 → 该父目录的直接子目录(workspace-relative)。仅返回 `object_status='available' AND is_deleted=0` 的 `StorageObjects` 行。 +- **鉴权**: workspace 成员 +- **查询参数**: + | 名 | 类型 | 必填 | 说明 | + |---|---|---|---| + | `parent_path` | string | 否 | 父目录相对路径,空字符串或缺省表示用户根目录 | +- **谓词(SQL 等价)**: `relative_path LIKE '/%' AND relative_path NOT LIKE '/%/%'`,其中 `prefix = scoped_prefix/{parent_path}`,索引走 `idx_storage_workspace_relative_path(workspace_id, relative_path(255))`。 +- **响应**: + ```json + { + "request_id": "...", + "data": { + "directories": [ + {"path": "scripts", "name": "scripts", "parent_path": "", "has_children": true}, + {"path": "scripts/etl", "name": "etl", "parent_path": "scripts", "has_children": false}, + {"path": "scripts/etl/daily", "name": "daily", "parent_path": "scripts/etl", "has_children": false} + ] + }, + "meta": {"directory_count": 3} + } + ``` + - 字段表(继承 §3.1): + | 字段 | 类型 | 说明 | + |---|---|---| + | `path` | string | workspace 内相对路径 | + | `name` | string | `path` 的最后一段 | + | `parent_path` | string | 父目录相对路径,根目录用空串 | + | `has_children` | bool | 该目录下是否还有直接子目录(后端额外 `EXISTS` 查询,可为空目录为 `false`) | +- **空结果**: 不返回 404,空目录列表即 `directories: []`。 +- **错误**: 401(未登录)/ 403(非 workspace 成员)同其他接口。 + +### 3.4 `GET /api/v1/scripts` + +列出当前 workspace 内**全部 active 脚本**。不受 is_locked 影响(读路径不锁)。 + +- **响应**: `data` 为 `ScriptPayload` 数组(见 §3.10)。 + +### 3.5 `GET /api/v1/scripts/{script_id}` + +取单个脚本详情。 + +### 3.6 `POST /api/v1/scripts` + +创建一个脚本(直接走 `create_server_object` 上传)。 + +- **请求体**: + ```json + { + "script_name": "train.py", + "script_type": "python", + "content": "print('hello')", + "visibility": "workspace", + "parent_path": "scripts" + } + ``` + | 字段 | 必填 | 说明 | + |---|---|---| + | `script_name` | 是 | 文件名;后端按 `script_type` 补齐扩展名(`.py` / `.ipynb`) | + | `script_type` | 是 | `python` \| `notebook` | + | `content` | 是 | 文本内容(`.ipynb` 必须是合法 JSON,含 `cells` 数组) | + | `visibility` | 否 | `private` \| `workspace` (默认) \| `public` | + | `parent_path` | 否 | 父目录路径 | + +### 3.7 `POST /api/v1/scripts/upload?file_name=...&parent_path=...&visibility=...` + +multipart/binary 形式上传大文件(走 server-proxied PUT,详见 §九)。 + +- **查询参数**: `file_name`(必填)、`parent_path`、`visibility` +- **请求体**: 原始文件字节(`Content-Type` 必须与脚本类型匹配) +- **适用场景**: 大于 100 KiB 的 notebook / 资源文件 + +### 3.8 `PUT /api/v1/scripts/{script_id}` + +更新脚本**工作副本**。受 **owner + is_locked** 双重门禁: + +- admin 总是允许 +- owner 总是允许 +- 非 owner + `is_locked == 0` → 允许 +- 非 owner + `is_locked == 1` → **403** + +- **请求体**: `{"content": "..."}` + +### 3.9 `DELETE /api/v1/scripts/{script_id}` + +软删脚本。**版本**(`Versions`)会被保留以供审计。门禁同 §3.8。 +`StorageObjects` 行移到 trash bucket(`settings.s3_trash_bucket`),`object_status` 置为 `deleted`。 + +### 3.10 ScriptPayload 字段 + +| 字段 | 类型 | 说明 | +|---|---|---| +| `script_id` | ULID | | +| `workspace_id` | ULID | | +| `current_object_id` | ULID | 当前工作副本指向的 `StorageObjects.storage_object_id` | +| `owner_user_id` | ULID | | +| `script_name` | string | | +| `script_type` | `python` \| `notebook` | | +| `visibility` | enum | | +| `status` | `active` \| `deleted` | | +| `is_locked` | boolean | 是否锁定。`PUT /scripts/{id}`(§3.8) 受此字段门禁;可通过 §3.16 切换 | +| `relative_path` | string \| null | 例如 `workspace/01HXX.../scripts/etl/train.py`(以 `user_id` 为作用域) | +| `content_hash` | string \| null | SHA-256 十六进制 | +| `size_bytes` | int | | +| `created_at` / `updated_at` | ISO-8601 | | + +### 3.11 `POST /api/v1/scripts/{script_id}/versions` + +发布一个**稳定版本**(immutable,绑定到 `S3_VERSION_BUCKET`)。门禁同 §3.8。 + +- **请求体**: + ```json + { + "source_object_id": "01HXY...", + "release_note": "首次发布", + "visibility": "workspace" + } + ``` +- **行为**: + - 读 `source_object_id` 对应的工作副本内容,算 SHA-256 + - 同 `content_hash` 已存在则返回 200 + `meta.reused = true`(去重) + - 否则把副本内容上 `versions` 桶,创建 `Versions` 行 +- **响应**: + ```json + { + "data": { + "versions_id": "01HXY...", + "version_no": 3, + "version_label": "v3.0", + "content_hash": "...", + "file_size_bytes": 2048, + "artifact_path": "s3://versions//", + "...": "..." + }, + "meta": {"reused": false} + } + ``` + +### 3.12 `GET /api/v1/scripts/{script_id}/versions` + +列出该脚本的所有版本(倒序)。 + +### 3.13 `GET /api/v1/versions/{versions_id}` + +单版本详情。 + +### 3.14 `DELETE /api/v1/versions/{versions_id}` + +从调度候选中**隐藏**此版本(不删除对象存储里的对象)。门禁:**owner 校验基于所属 `Scripts` 的 owner**——即"按整本 script 判定",而非"按版本发布者判定"。 + +### 3.15 `POST /api/v1/versions/{versions_id}/download-url` + +生成对象存储的 presigned download URL(走 S3 兼容协议,local 模式下该 endpoint 在 s3 模式才生效)。 + +- **请求体**: + ```json + {"expires_seconds": 300} + ``` +- **响应**: + ```json + { + "data": { + "storage_object_id": "...", + "presigned_url": "https:///storage//?X-Amz-...", + "method": "GET", + "expires_in_seconds": 300 + } + } + ``` + +### 3.16 `PATCH /api/v1/scripts/{script_id}/lock` + +切换脚本的 `is_locked` 状态。**只切换锁**,不修改脚本内容。 + +- **鉴权**: admin 或 `owner_user_id == 当前用户`(沿用 `require_script_modify_access`)。其余一律 **404**。 +- **请求体**: + ```json + {"is_locked": true} + ``` + | 字段 | 必填 | 说明 | + |---|---|---| + | `is_locked` | 是 | 目标状态。`true` 锁定;`false` 解锁 | +- **响应 200**: `data` 为更新后的 `ScriptPayload`(§3.10)。 + ```json + { + "request_id": "...", + "data": { "...ScriptPayload 字段...": "is_locked: false" }, + "meta": {} + } + ``` +- **行为**: 行级锁 (`SELECT ... FOR UPDATE`) 防止并发切换;提交后立即生效,影响后续 §3.8 `PUT /scripts/{id}` 的门禁判定。 +- **错误码**: + | 码 | 含义 | + |---|---| + | 404 | 脚本不存在 / 非当前用户无权访问(不区分,避免暴露存在性) | + +--- + +## 四、调度 + +> 业务概念: `Schedules` 是 DAG 模板(nodes + edges),`ScheduleRuns` 是 +> 触发产生的一次执行实例,**自带 snapshot 锁住当时的 DAG**,`ScheduleNodeRuns` +> 是 run 里每个 node 每次尝试的记录。 +> +> 调度链路(架构 V3.1 §2.3): +> ``` +> 手动: POST /run ─→ schedule_runs (queued) + outbox_events +> cron: Executor APScheduler tick → POST /run ─→ 同上 +> │ +> Orchestrator (poll outbox 0.25s) │ +> → schedule_node_runs (queued) + outbox_events(job.node.execute) +> Worker (poll outbox) │ +> → 执行 → schedule_node_runs (succeeded/failed) + outbox_events(job.node.finished) +> Orchestrator 收 finished → 推进下一个 node / 终结 run +> ``` + +### 4.1 `GET /api/v1/schedule-artifacts` + +列出可绑定到节点的 `Versions`(DAG 画布下拉框的素材源)。 + +### 4.2 `POST /api/v1/cron/preview` + +预览一个 cron 表达式的未来 5 次触发时间。 + +- **请求体**: + ```json + {"expression": "0 0 * * *", "timezone": "Asia/Shanghai"} + ``` + +### 4.3 `POST /api/v1/schedules/{schedule_id}/validate` + +校验 DAG 拓扑(环路检测、孤立节点等)。 + +### 4.4 调度模板 CRUD + +| 方法 | 路径 | 说明 | +|---|---|---| +| `GET` | `/api/v1/schedules` | 列当前 workspace 的所有 schedule | +| `POST` | `/api/v1/schedules` | 创建(返回 201) | +| `GET` | `/api/v1/schedules/{id}` | 详情(含 nodes + edges) | +| `PUT` / `PATCH` | `/api/v1/schedules/{id}` | 改 cron / 时区 / 启用 / max_concurrency / failure_policy | +| `DELETE` | `/api/v1/schedules/{id}` | 软删 | + +`CreateScheduleRequest` 字段: + +```json +{ + "schedule_name": "nightly-train", + "trigger_type": "cron", + "cron_expression": "0 0 * * *", + "timezone": "Asia/Shanghai", + "enabled": true, + "max_concurrency": 3, + "failure_policy": "stop" +} +``` + +### 4.5 节点 CRUD + +| 方法 | 路径 | 说明 | +|---|---|---| +| `POST` | `/api/v1/schedules/{id}/nodes` | 加节点(必填 `versions_id` 绑定 Versions) | +| `PUT` | `/api/v1/schedules/{id}/nodes/{node_id}` | 改节点参数/版本引用/重试策略 | +| `DELETE` | `/api/v1/schedules/{id}/nodes/{node_id}` | 删节点 | + +节点 `arguments` / `env_refs` 只存引用,不存明文密钥。 + +### 4.6 边 CRUD + +| 方法 | 路径 | 说明 | +|---|---|---| +| `POST` | `/api/v1/schedules/{id}/edges` | 加边(`source_node_id` / `target_node_id`) | +| `PUT` | `/api/v1/schedules/{id}/edges/{edge_id}` | 改 `condition_expr` | +| `DELETE` | `/api/v1/schedules/{id}/edges/{edge_id}` | 删边 | + +> ⚠ `condition_expr` 字段当前**仅落库,不参与执行判定**。DAG 只能表示依赖, +> 不能表达"父 node value > 0 才走 A 分支"等条件分支。 + +### 4.7 触发与查询 + +#### `POST /api/v1/schedules/{schedule_id}/run` + +手动触发一次 run。 + +- **必填请求头**:`Idempotency-Key`(≥8 字符) +- **可选请求体**:`{"reason": "manual_run"}`(默认) | `{"reason": "cron"}` +- **响应 202**: + ```json + { + "data": { + "run_id": "01HXY...", + "schedule_id": "...", + "trigger_type": "manual", + "run_status": "queued", + "queued_at": "...", + "schedule_snapshot": {"nodes": [...], "edges": [...]} + }, + "meta": {"reused": false} + } + ``` + - 同 `Idempotency-Key` 已存在 → 返回原 run + `meta.reused = true` + - 同 key 但元数据不一致 → **409 Conflict** + - DAG 无效 / 节点 > 100 / 边 > 500 → **409** + 错误码 `SCHEDULE_DAG_INVALID` + +#### `GET /api/v1/schedule-runs` + +列出 run。可选 `?schedule_id=...` 与 `?status=queued|running|succeeded|failed|cancelled|timed_out` 过滤。 + +#### `GET /api/v1/schedule-runs/{run_id}` + +单 run 详情 + 所有 `node_run`。 + +### 4.8 run 状态机 + +``` +queued ──→ running ──┬─→ succeeded + ├─→ failed + ├─→ cancelled (未实现) + └─→ timed_out +``` + +--- + +## 五、数据资源 + +> 通用二进制资源(数据集、模型 checkpoint、任意文件)。Base 路径 +> 前缀是 `/api/v1/data-resources`,**不**带脚本/notebook 的 owner 锁。 + +| 方法 | 路径 | 说明 | +|---|---|---| +| `POST` | `/api/v1/data-resources/uploads` | 创建上传会话,返回 `upload_id` + `upload_path` | +| `PUT` | `/api/v1/data-resources/uploads/{upload_id}` | 上传字节(请求体即文件内容) | +| `POST` | `/api/v1/data-resources/uploads/{upload_id}/bind` | 绑定已完成上传为数据资源 | +| `GET` | `/api/v1/data-resources` | 列表(workspace 范围) | +| `GET` | `/api/v1/data-resources/{id}` | 详情 | +| `POST` | `/api/v1/data-resources/{id}/download-url` | 生成 presigned GET URL | +| `DELETE` | `/api/v1/data-resources/{id}` | 软删 | + +**同名冲突**:同一 `workspace` 内、同一目录(`target_path` 相等)、同一 `owner` 下,`resource_name` 重复提交绑定返回 `409`。不同目录或不同 `owner` 允许重名。重新绑定同一 `upload_id`(`storage_object_id` 已落库)走 idempotent 复用路径,不视为冲突。 + +字节归档到 trash bucket(`settings.s3_trash_bucket`)。 + +请求示例(上传):`POST /api/v1/data-resources/uploads` + +```json +{ + "file_name": "data.csv", + "content_type": "text/csv", + "expected_size_bytes": 1048576, + "expected_hash": "", + "idempotency_key": "client-uuid-or-similar" +} +``` + +**完整上传流程(前端应实现的模式)**: +``` +1. POST /uploads → {upload_id, upload_path, expires_at} +2. PUT upload_path with raw file bytes (Content-Type: application/octet-stream) +3. 服务器端走 backend.put() → 200 {data: StorageObjectPayload} +``` + +字节经过 backend 进程(server-proxied upload),最大 100 MiB,由 backend +直接调 `AsyncStorageBackend.put()` 写入存储(不再走 presigned PUT 直传)。 +前端无需关心 S3 协议或签名。 + +**小对象(<100 KiB)捷径**:直接调 `create_server_object` 把字节 base64 放进 +`content_base64` 字段(JSON 体里走),内部走同一条 `AsyncStorageBackend.put` +路径,前端无需分两步。 + +--- + +## 六、管理后台 + +Base 前缀 `/api/v1/admin`。 + +| 方法 | 路径 | 说明 | +|---|---|---| +| `GET` | `/api/v1/admin/employees` | 列员工(workspace 成员) | +| `POST` | `/api/v1/admin/employees` | 创建员工账号(返回 201) | +| `PATCH` | `/api/v1/admin/employees/{user_id}` | 改员工信息(角色/状态等) | +| `DELETE` | `/api/v1/admin/employees/{user_id}` | 软删员工 | + +> 当前所有 admin 端点要求 `is_admin` 上下文标志,具体 token 校验流程 +> 见 §一。 + +--- + +### 6.1 `POST /api/v1/admin/employees` + +创建员工账号。 + +- **请求体字段**: + +| 字段 | 类型 | 必填 | 限制 | 说明 | +|---|---|---|---|---| +| `username` | string | 是 | 2~64 字符 | 登录名,workspace 内唯一 | +| `display_name` | string | 是 | 1~100 字符 | 显示名称 | +| `email` | string | 否 | ≤255 字符 | 邮箱,全局唯一 | +| `role_code` | string | 否 | `admin` \| `developer` | 默认 `developer` | +| `password` | string | 是 | 8~72 字符 | 登录密码 | + +- **密码说明**: + - 密码明文**不会**存入数据库,后端使用 bcrypt 哈希后保存到 `password_hash`。 + - 请求体中 `password` 必填,长度必须在 8~72 字符之间,否则返回 `422`。 + - 创建成功后的响应**不**包含 `password` 或 `password_hash`。 + +- **响应 201**: + ```json + { + "request_id": "...", + "data": { + "user_id": "...", + "username": "...", + "display_name": "...", + "email": "...", + "status": "active", + "role_code": "developer", + "role_name": "...", + "created_at": "..." + }, + "meta": {} + } + ``` + +> `PATCH` / `DELETE` 员工接口**不**涉及密码字段,也不返回密码相关信息。 + +## 七、系统管理 (`/api/v1/platform/...`) + +平台级(跨 workspace)管理接口,用于管理员工、workspace 实体与 workspace 成员。 +所有端点要求调用者是**系统管理员**——其 `users.platform_role_id` 指向 +`role_code='admin'` 的角色行,且 `users.status == 'active'`。系统管理员判定 +通过 `GET /api/v1/auth/me` 响应中的 `data.user.is_system_admin` 字段(详见 §一)。 + +| 方法 | 路径 | 说明 | +|---|---|---| +| `GET` | `/api/v1/platform/employees` | 列全平台未软删员工;包含停用、锁定及无平台角色用户 | +| `POST` | `/api/v1/platform/employees` | 创建平台员工账号(返回 201);不自动加入任何 workspace | +| `PATCH` | `/api/v1/platform/employees/{user_id}` | 改员工资料/状态/平台角色(仅系统管理员) | +| `DELETE` | `/api/v1/platform/employees/{user_id}` | 软删员工;级联软删其 workspace 成员关系(仅系统管理员) | +| `GET` | `/api/v1/platform/workspaces` | 列 workspace(`active`/`archived`);已软删的过滤掉 | +| `POST` | `/api/v1/platform/workspaces` | 创建 workspace(返回 201);创建者自动成为 admin 成员 | +| `GET` | `/api/v1/platform/workspaces/{workspace_id}` | 单个 workspace(含已 disabled 的,用于恢复) | +| `PATCH` | `/api/v1/platform/workspaces/{workspace_id}` | 改 workspace 字段;`status` 仅允许 `active`/`archived` | +| `DELETE` | `/api/v1/platform/workspaces/{workspace_id}` | 软删 workspace;级联软删其成员 | +| `GET` | `/api/v1/platform/workspaces/{workspace_id}/members` | 列成员 | +| `POST` | `/api/v1/platform/workspaces/{workspace_id}/members` | 添加成员(返回 201) | +| `PATCH` | `/api/v1/platform/workspaces/{workspace_id}/members/{user_id}` | 改成员 `member_status`;**不能改 role_code**(workspace 角色继承自平台角色) | +| `DELETE` | `/api/v1/platform/workspaces/{workspace_id}/members/{user_id}` | 软删成员 | +| `GET` | `/api/v1/platform/roles` | 列全部 `role_scope='platform'` 的角色及其 `permission_codes`(仅系统管理员) | +| `GET` | `/api/v1/platform/roles/{role_code}/permissions` | 单个平台角色的 `permission_codes`(仅系统管理员) | +| `PATCH` | `/api/v1/platform/roles/{role_code}/permissions` | 整体替换角色权限集合;对 `admin` 角色保留 `system.view` + `system.manage` 的最后系统管理员保护(仅系统管理员) | + +> **不变量**: +> - 每个 workspace 必须始终保留至少一个 `admin` 角色的活跃成员;对最后 admin 做降级 / 停用 / 删除 → 409。 +> - `PATCH /employees/{user_id}` 降级 admin → developer 时同样触发 workspace last-admin 守卫(因为 workspace 角色继承自 platform 角色,降级会级联到所有活跃 membership);platform 必须始终保留至少一个 `active` 系统管理员;对最后系统管理员做降级 / 停用 / 删除 → 409。 +> - 系统管理员不能通过 `DELETE .../members/{self}` 把自己移除(403)。唯一退出方式是 `DELETE /workspaces/{id}` 软删整个 workspace,后者会级联软删所有成员。 +> - workspace 与成员列表接口静默 `pageSize=100` 上限,无客户端分页参数(YAGNI);`GET /employees` 按契约返回全部未软删员工,不设隐藏上限。 +> - 跨 workspace 操作**不**需要 `?workspace_id=` query 参数,与 `/api/v1/admin/...`(workspace 内成员管理)不要混淆。 + +### 7.1 `POST /api/v1/platform/workspaces` + +创建 workspace;创建者(当前系统管理员)自动成为该 workspace 的 `admin` 成员。 + +- **请求体字段**: + +| 字段 | 类型 | 必填 | 限制 | 说明 | +|---|---|---|---|---| +| `workspace_code` | string | 是 | regex `^[a-z0-9-]{3,32}$`(类似 git repo 名) | 创建后冻结,不可改 | +| `workspace_name` | string | 是 | 1~150 字符 | 显示名称 | +| `quota_bytes` | int | 否 | ≥0,默认 `0` | 配额字节数,`0` 表示无配额 | +| `description` | string | 否 | ≤1000 字符 | | + +- **服务端自动生成字段**(不接收): + - `workspace_id`(ULID) + - `active_root_uri`(`s3://workspaces/{workspace_id}/`) + - `status`(`"active"`) + - `created_by`(当前管理员 `user_id`) + - `created_at` / `updated_at`(DB 自动) + +- **响应 201**:见下 §7.2 `WorkspacePayload`。 + +### 7.2 `GET /api/v1/platform/workspaces/{workspace_id}` / `WorkspacePayload` + +- **响应 200**: + ```json + { + "request_id": "...", + "data": { + "workspace_id": "01HXY...", + "workspace_code": "model-development", + "workspace_name": "模型开发 Workspace", + "active_root_uri": "s3://workspaces/01HXY.../", + "quota_bytes": 0, + "status": "active", + "description": null, + "created_by": "01HXY...", + "created_at": "2026-08-04T12:00:00.000", + "updated_at": null + }, + "meta": {"count": ..., "page_size": 100} + } + ``` + +### 7.3 `PATCH /api/v1/platform/workspaces/{workspace_id}` + +部分更新。**不可改**:`workspace_id`、`workspace_code`、`active_root_uri`、`created_by`、时间戳、软删标记。 + +- **请求体字段**(全部可选): + +| 字段 | 类型 | 限制 | 说明 | +|---|---|---|---| +| `workspace_name` | string | 1~150 | | +| `quota_bytes` | int | ≥0 | | +| `description` | string | ≤1000 | | +| `status` | string | `active` \| `archived` | **不允许 `disabled`**——软删须走 DELETE | + +- 错误:`status="disabled"` → 422;已 disabled 的 workspace → 409。 + +### 7.4 `DELETE /api/v1/platform/workspaces/{workspace_id}` + +软删除。允许从 `active` 或 `archived` 状态调用。 + +- **副作用**: + - 该 workspace 行:`status='disabled'`、`is_deleted=1`、`deleted_at=NOW()` + - **级联**:所有未删除的 `workspace_members` 行同步 `is_deleted=1`、`deleted_at=NOW()` +- 已 disabled 的 workspace 再删 → 409。 + +### 7.5 `POST /api/v1/platform/workspaces/{workspace_id}/members` + +添加成员。 + +> **Workspace 角色继承平台角色**:本端点不接受 `role_code`,新成员的 `workspace_members.role_id` 始终等于其 `users.platform_role_id` 指向的角色行。**不可填** `system_admin`(那是用户级身份,不是 workspace 角色)。要改某成员的 workspace 角色,请改 `users.platform_role_id`,即 `PATCH /api/v1/platform/employees/{user_id}`。 + +- **请求体字段**: + +| 字段 | 类型 | 必填 | 限制 | 说明 | +|---|---|---|---|---| +| `user_id` | string | 是 | 26 字符 ULID | | + +- 服务端默认 `member_status='active'`。 +- 用户不存在或已软删除 → 404;用户状态不是 `active` → 409;用户**尚未分配平台角色**(`users.platform_role_id IS NULL`)→ 409 "目标用户尚未分配平台角色,无法加入 workspace"。 +- 用户已是该 workspace 成员 → 409 "用户已是该 workspace 成员;workspace 角色继承自平台角色,要变更请 PATCH /api/v1/platform/employees/{user_id} 修改 role_code"。 +- 新员工必须先通过 `POST /api/v1/platform/employees` 建立账号(可以同时传 `role_code=admin|developer`)。 + +### 7.6 `PATCH /api/v1/platform/workspaces/{workspace_id}/members/{user_id}` + +修改成员的状态。**不能通过本端点修改 role_code**——workspace 角色始终继承自 `users.platform_role_id`;要改角色请 `PATCH /api/v1/platform/employees/{user_id}`。 + +- **请求体字段**(全部可选): + +| 字段 | 类型 | 限制 | 说明 | +|---|---|---|---| +| `member_status` | string | `active` \| `disabled` \| `locked` | 停用 / 锁定最后 admin → 409 | + +- 提交 `role_code` 字段 → 422(Pydantic `extra='forbid'`),不是静默忽略。 + +### 7.7 `DELETE /api/v1/platform/workspaces/{workspace_id}/members/{user_id}` + +软删除成员。 + +- **自我移除保护**:`user_id == 当前管理员 user_id` → 403 "系统管理员不能把自己从 workspace 移除;如需退出,请删除整个 workspace" +- **末位 admin 保护**:若删除的是最后一个 `admin` 角色活跃成员 → 409 +- 不存在的成员 → 404 + +### 7.8 `GET /api/v1/platform/employees` + +返回全平台所有未软删除员工。调用者必须是系统管理员;已认证但不是系统管理员时返回 `403`。 + +- 无请求体或查询参数。 +- 包含 `active`、`disabled`、`locked` 状态及未分配平台角色的员工。 +- `users.is_deleted != 0` 的员工不会返回。 +- 不设隐藏数量上限,结果按 `created_at`、`user_id` 排序。 +- `role_code`、`role_name` 表示 `users.platform_role_id` 对应的平台角色;未分配时均为 `null`。 + +- **响应 200**: + ```json + { + "request_id": "...", + "data": [ + { + "user_id": "01HXY...", + "username": "developer", + "display_name": "开发人员", + "email": "developer@example.com", + "status": "active", + "role_code": null, + "role_name": null, + "created_at": "2026-08-06T12:00:00.000" + } + ], + "meta": {"count": 1} + } + ``` + +### 7.9 `POST /api/v1/platform/employees` + +只创建平台员工账号,不创建任何 `workspace_members` 记录。调用者必须是系统管理员;已认证但不是系统管理员时返回 `403`。 + +- **请求体字段**: + +| 字段 | 类型 | 必填 | 限制 | 说明 | +|---|---|---|---|---| +| `username` | string | 是 | 2~64 字符 | 全平台唯一登录名 | +| `display_name` | string | 是 | 1~100 字符 | 显示名称 | +| `email` | string | 否 | ≤255 字符 | 邮箱,全平台唯一 | +| `password` | string | 是 | 8~72 字符 | 登录密码 | +| `role_code` | string | 否 | `admin` \| `developer` | 平台角色;不传或 `null` 表示**不分配角色**(用户无法加入任何 workspace,见 §7.5) | + +- 新用户状态固定为 `active`;`role_code` 决定 `users.platform_role_id` 指向 `role_code='admin'` / `'developer'` 的 Roles 行,未传则 `platform_role_id=NULL`。 +- 平台**仅**有 `admin` / `developer` 两个平台角色;不存在第三个角色枚举值。 +- 密码使用 bcrypt 哈希保存;响应不包含 `password` 或 `password_hash`。 +- 用户名或邮箱重复 → 409;`role_code` 取值非法 → 422。 +- 创建成功后,可调用 `POST /api/v1/platform/workspaces/{workspace_id}/members` 将用户加入指定 workspace(会要求用户已有 `platform_role_id`,否则 409)。 + +- **响应 201**:字段与 §7.8 的员工元素一致;当 `role_code` 传入时,`role_code` / `role_name` 反映 `platform_role_id`;未传入时均为 `null`。`meta` 为空对象。 + +### 7.10 `PATCH /api/v1/platform/employees/{user_id}` + +修改平台员工的显示名、邮箱、状态或平台角色。调用者必须是系统管理员。 + +- **请求体字段**(全部可选): + +| 字段 | 类型 | 限制 | 说明 | +|---|---|---|---| +| `display_name` | string | 1~100 | trim 后写入 | +| `email` | string \| null | ≤255 | trim 后写入;空字符串归一为 `null` | +| `status` | string | `active` \| `disabled` \| `locked` | 直接写入 `users.status` | +| `role_code` | string | `admin` \| `developer` | 同步设置 `users.platform_role_id`;`admin` 指向 `role_code='admin'` 的 Roles 行,`developer` 指向 `role_code='developer'` 的 Roles 行 | + +- 禁止通过该端点修改:`username`、`password`、`password_hash`、`platform_role_id`;请求体中包含这些字段 → 422。 +- `role_code` 只能取 `admin` / `developer`;**不能通过本端点把 `platform_role_id` 置为 `null`**(降级为"无平台角色"须走单独的内部流程,前端不要尝试)。 +- `role_code` 改动会**立即级联同步**该用户**所有活跃 workspace 成员行**的 `workspace_members.role_id`(`PATCH /employees` 在事务内 `UPDATE workspace_members SET role_id=:new WHERE user_id=:uid AND is_deleted=0`)。否则 `/me` 返回的 `workspaces[].role_code` 与下游 `request_context` 都会读到过期角色。本端点是调整任何成员 workspace 角色的**唯一**入口。 +- 自保护: + - 修改自身 `status` 为非 `active` → 409 "不能停用当前登录账号"。 + - 修改自身 `role_code` 为 `developer`(即降级系统管理员身份)→ 409 "不能降级自身管理员角色"。 +- 最后系统管理员保护:当目标用户当前为 active 系统管理员,本次变更会让其离开"active 系统管理员"集合(降级角色 / 停用账号)时,平台必须仍保留至少一名 active 系统管理员,否则 → 409 "platform 必须保留至少一个 active 系统管理员"。**自保护在前、last-admin 计数在后**(参考 CLAUDE.md 工程笔记)。 +- **workspace last-admin 守卫**(降级路径):因为 `role_code` 改动会级联同步 `workspace_members.role_id`,把 `admin` 降级为 `developer` 时必须**额外**检查该用户在**每个**他是唯一 active admin 的 workspace 中仍有替补 admin,否则 → 409,错误信息列出将失去 admin 的 `workspace_code` 列表;提示"请先在这些 workspace 中指定其他 admin,再降级该用户"。`developer → admin` 升级路径不受此守卫约束(只增不减)。 +- 目标用户不存在或已软删除 → 404 "用户不存在";`role_code` 对应的角色行不存在 → 422。 + +- **响应 200**:返回更新后的 `PlatformEmployeePayload`,`role_code` / `role_name` 反映最新的 `platform_role_id`。 + +### 7.11 `DELETE /api/v1/platform/employees/{user_id}` + +软删除平台员工;级联软删其所有 `workspace_members` 行。调用者必须是系统管理员。 + +- 行为: + - 设置 `users.status='disabled'`、`users.is_deleted=1`、`users.deleted_at=NOW()`。 + - 同事务内 `UPDATE workspace_members SET is_deleted=1, deleted_at=NOW() WHERE user_id=:user_id AND is_deleted=0`。 + - 不级联修改 `workspaces` 记录,workspace 仍可被单独管理。 +- 保护: + - 删除自身 → 409 "不能删除当前登录账号"。 + - 目标为唯一 active 系统管理员 → 409 "platform 必须保留至少一个 active 系统管理员"。 + - 目标不存在或已软删除 → 404 "用户不存在"(与 §7.4 DELETE workspace 对已 disabled 返回 409 不同,本端点对已软删用户统一返回 404)。 +- 软删后行为: + - `GET /api/v1/platform/employees` 不再返回该用户。 + - `POST /api/v1/platform/workspaces/{id}/members` 用同一 `user_id` 重新加入 → 404。 + - `POST /api/v1/platform/employees` 用同 `username` 重新创建 → 409(唯一索引)。 +- 本端点不提供恢复接口,与其他 DELETE 端点行为一致。 + +- **响应 200**: + ```json + { + "request_id": "...", + "data": { "user_id": "01HXY...", "deleted": true }, + "meta": {} + } + ``` + +### 7.12 `GET /api/v1/platform/roles` + +列出所有 `role_scope='platform'` 的角色及其当前 `permission_codes`。调用者必须是系统管理员。 + +- 用于平台管理员配置页加载左侧"角色"下拉与权限矩阵。 +- 返回结果按 `role_code` 升序。 +- `permission_codes` 与 DB 中 `role_permissions` 关联表的活跃(`is_deleted = 0`)行一致,按字典序排列;空集合表示该角色当前没有任何菜单权限。 +- `is_builtin` 反映 `roles.is_builtin`;当前 seed 的 `admin` / `developer` 均为 `1`。 + +- **响应 200**: + ```json + { + "request_id": "...", + "data": [ + { + "role_id": "01HXY...", + "role_code": "admin", + "role_name": "管理员", + "is_builtin": true, + "permission_codes": [ + "dashboard.view", "experiment.all", "experiment.own", + "resource.personal", "resource.public.manage", + "resource.public.upload", "schedule.all", "schedule.own", + "script.build", "script.public.manage", "system.manage", + "system.view" + ] + }, + { + "role_id": "01HXY...", + "role_code": "developer", + "role_name": "开发人员", + "is_builtin": true, + "permission_codes": [ + "dashboard.view", "experiment.own", "resource.personal", + "schedule.own", "script.build", "script.public.manage" + ] + } + ], + "meta": { "count": 2 } + } + ``` + +### 7.13 `GET /api/v1/platform/roles/{role_code}/permissions` + +获取单个平台角色的当前 `permission_codes`。调用者必须是系统管理员。 + +- 用途:角色权限管理面板加载右侧"已分配"列;也用于前端做 diff 显示。 + +- 角色不存在 → 404;角色存在但 `role_scope != 'platform'`(例如只有 `workspace` 角色行匹配) → 404。 + +- **响应 200**: + ```json + { + "request_id": "...", + "data": { + "role_id": "01HXY...", + "role_code": "admin", + "role_name": "管理员", + "is_builtin": true, + "permission_codes": [ + "dashboard.view", "experiment.all", "experiment.own", + "resource.personal", "resource.public.manage", + "resource.public.upload", "schedule.all", "schedule.own", + "script.build", "script.public.manage", "system.manage", + "system.view" + ] + }, + "meta": {} + } + ``` + +### 7.14 `PATCH /api/v1/platform/roles/{role_code}/permissions` + +整体替换指定平台角色的 `permission_codes`(diff-based 写入,见下文)。调用者必须是系统管理员。 + +> **本端点只控制前端菜单可见性**——不修改 `system_admin_context` 的鉴权判定(`role_code == "admin"` 始终等价于"拥有所有平台菜单权限")。若需调整 API 鉴权,请改 `backend.platform.system_admin_context`,不要绕过本端点。 + +- **请求体字段**: + +| 字段 | 类型 | 必填 | 限制 | 说明 | +|---|---|---|---|---| +| `permission_codes` | string[] | 是 | 0~64 项;内部去重(首次出现优先);元素必须是现存且未软删的 `permissions.permission_code` | 完整替换集合(非 patch);传 `[]` 表示清空该角色的全部菜单权限 | + +- **守卫顺序(load-bearing,不可调换)**: + 1. 角色不存在 / `role_scope != 'platform'` → 404。 + 2. **`admin` 角色**:提交的 `permission_codes` 必须同时包含 `system.view` 与 `system.manage`,否则 → 409 "admin 角色必须保留 system.view 与 system.manage 权限"。 + 3. **非 `admin` 角色**:`permission_codes` 中**禁止**含 `system.*` 项 → 422 "非 admin 角色不能拥有 system.* 权限: [...]"。menu permission 只控前端展示,后端 `/api/v1/platform/*` 鉴权仍按 `role_code == "admin"`;若让 developer 拿到 `system.manage`,前端会渲染"系统管理"入口但所有 platform API 调用 403,UX 割裂。 + 4. 任意 `permission_code` 不在 `permissions` 表活跃行中 → 422 "未知的 permission_code: [...]"。 + 5. 写入策略:diff-based —— 只 soft-delete `current \ new` 的关联,只 INSERT `new \ current` 的关联。重复提交同 payload 是 no-op;包含原有 codes 的 patch 不会触发 `(role_id, permission_id)` 主键冲突。**严禁**先全量 soft-delete 再全量 INSERT(会 `IntegrityError`,因为软删行仍占主键 slot)。 +- 写入后,响应 `data.permission_codes` 为本次写入后的活跃集合,与再次 `GET §7.13` 完全一致。 + +- **响应 200**: + ```json + { + "request_id": "...", + "data": { + "role_id": "01HXY...", + "role_code": "admin", + "role_name": "管理员", + "is_builtin": true, + "permission_codes": ["dashboard.view", "system.manage", "system.view"] + }, + "meta": {} + } + ``` + +- **当前 seed 的 permission_code 全集**(来自迁移 `f6a7b8c9d0e1`,与 `migrations/data/migrate_system_json.py::PERMISSION_NAMES` 真值对齐,不要在客户端另造一份): + +| `permission_code` | `module_code` | admin | developer | +|---|---|:-:|:-:| +| `dashboard.view` | dashboard | ✓ | ✓ | +| `script.build` | script | ✓ | ✓ | +| `script.public.manage` | script | ✓ | ✓ | +| `schedule.own` | schedule | ✓ | ✓ | +| `schedule.all` | schedule | ✓ | | +| `experiment.own` | experiment | ✓ | ✓ | +| `experiment.all` | experiment | ✓ | | +| `resource.personal` | resource | ✓ | ✓ | +| `resource.public.upload` | resource | ✓ | | +| `resource.public.manage` | resource | ✓ | | +| `system.view` | system | ✓ | | +| `system.manage` | system | ✓ | | + +> **与现有 7.x 端点的语义差异**(避免 reviewer 误读): +> - 本端点不修改 `users.platform_role_id`,只调整 `role_permissions` 关联表。 +> - "清空 developer 的全部权限"是合法操作;只有 `admin` 受 `system.*` 强制约束。 +> - `role_code` 不是 `permission_code`,前端不要用前者去判断菜单可见性。 + +--- + +## 八、Jupyter 路由 + +> **本节是 Nginx 行为,不是直接 HTTP 端点**。前端**不要**直接调用。 + +### 8.1 浏览器 → 用户打开 notebook + +用户在前端点击某个 notebook,前端拼出 URL: +``` +GET /jupyter/{workspace_id}/notebooks/{相对路径}.ipynb +GET /jupyter/{workspace_id}/lab/tree/{相对路径}.ipynb +GET /jupyter/{workspace_id}/api/contents/{相对路径}.ipynb +WS /jupyter/{workspace_id}/api/kernels/... +``` + +### 8.2 Nginx `auth_request` 鉴权 + +Nginx 收到上述请求后,**先**发一个内部子请求: +``` +GET /internal-auth + ↓ +Nginx: 抽 X-Original-Workspace-Id + X-Original-URI + Cookie + Authorization + ↓ +GET /api/v1/auth/jupyter + ↓ +Backend 流程: + 1. verify_jwt_token (HS256, JWT_SECRET) + 2. require_workspace_member (WorkspaceMembers JOIN) + 3. extract_notebook_path (只对 /notebooks/*.ipynb 做 lock 校验) + 4. check_notebook_is_locked (Scripts.is_locked + owner) + 5. runtime_client.get_workspace / start_workspace (拿子进程地址 + token) + ↓ +Backend 响应 200 + 响应头: + x-upstream-addr: : + x-jupyter-internal-token: <子进程 token> + ↓ +Nginx: auth_request_set 捕获这两个变量,proxy_pass 到子进程并注入 + Authorization: token $jupyter_token + ↓ +浏览器收到响应,**自始至终未接触 Jupyter Token** +``` + +### 8.3 鉴权失败码 + +| 状态 | 触发条件 | +|---|---| +| 401 | JWT 缺失/过期/校验失败 | +| 403 | 用户不在 workspace / notebook 被锁定且非 owner | +| 404 | workspace 不存在 | +| 503 | runtime 容器不可达(子进程启动失败) | + +Nginx 把这些状态原样透传给浏览器,前端可在 `onerror` 里判断。 + +--- + +## 九、对象存储控制面 + +> **范围**:Schedule worker 调用 backend 写 `run_log` / `run_result` 用的 +> 单端点 RPC。**前端不要直接调用,也不要把这个路径用于任何用户输入**。 +> 历史上有 6 个 `/internal/v1/*` 端点,经过 P0-1 修复后只剩这 1 个; +> 删除的端点全部已迁移到 JWT 保护的 `/api/v1/data-resources/*` 与 +> `/api/v1/scripts/*` 路由(见 §五、§三)。 + +底层抽象:`common.storage.AsyncStorageBackend`(`put/get/delete/exists/stat/ +list/get_url/copy`)。按 `settings.storage_backend` 选实现:`"s3"` 走 +S3-兼容服务,`"local"` 走 `LOCAL_STORAGE_BASE_DIR` 子目录。 + +### 鉴权(P0-1 后) + +所有 `/internal/v1/*` 路由都会校验请求头里的 service token: + +``` +X-Internal-Service-Token: +``` + +- backend 与 schedule 必须把同一个值注入到 `INTERNAL_SERVICE_TOKEN` 环境变量。 +- 缺失或不相符 → `401`。 +- backend 配置为空 → `503`(`internal service token not configured`)。 + +### 9.1 `POST /internal/v1/objects` + +**单步创建**(不走两步上传,字节 base64 进 JSON 体)。适用 < 100 KiB +对象(避免 multipart/大请求体的前端复杂度)。内部直接调 +`AsyncStorageBackend.put(key, content, content_type=..., +metadata={"sha256": ...})`。 + +```json +{ + "workspace_id": "...", + "user_id": "...", + "usage_type": "run_log", + "file_name": "log.txt", + "content_type": "text/plain", + "content_base64": "PHN0ZXAtY29udGVudD4=", + "visibility": "private", + "is_immutable": true, + "idempotency_key": "...", + "relative_path": null +} +``` + +### 9.2 usage_type → 桶路由(自动) + +| usage_type | 实际桶(env var) | 默认桶名 | +|---|---|---| +| `working_copy`, `public_script`, `data_resource`, `snapshot` | `S3_WORKSPACE_BUCKET` | `workspaces` | +| `version_artifact` | `S3_VERSION_BUCKET` | `versions` | +| `run_log`, `run_result` | `S3_RUN_LOG_BUCKET` | `run-logs` | +| (soft-delete target) | `S3_TRASH_BUCKET` | `trash` | + +若 `Workspaces.artifact_bucket` 非空,优先用 per-workspace 桶(覆盖 usage_type 路由)。 + +桶在 `STORAGE_BACKEND=s3` 时是 4 个独立 S3 bucket,在 +`STORAGE_BACKEND=local` 时是 `LOCAL_STORAGE_BASE_DIR` 下的 4 个子目录。 + +### 9.3 已删除的端点(P0-1 收纳) + +| 旧端点 | 替代路由 | 说明 | +|---|---|---| +| `POST /internal/v1/uploads` | `POST /api/v1/data-resources/uploads`(JWT) | 前端走公开路径 | +| `PUT /internal/v1/uploads/{id}` | `PUT /api/v1/data-resources/uploads/{id}` | 同上 | +| `POST /internal/v1/uploads/{id}/abort` | (客户端取消即可) | 无后端状态 | +| `POST /internal/v1/objects/{id}/download-url` | 各自的公开路由生成 presigned URL | download URL 由公开路由返回 | +| `DELETE /internal/v1/objects/{id}` | 公开路由的删除操作 | 与 data-resource 联动 | + +--- + +## 十、健康检查 + +| 方法 | 路径 | 用途 | +|---|---|---| +| `GET` | `/` | 简单服务标识 | +| `GET` | `/health/live` | 进程存活(不检查依赖) | +| `GET` | `/health/ready` | 依赖就绪(可选 TCP 探测列表) | +| `GET` | `/api/v1/health` | 公开健康检查(前端可访问) | + +`/health/ready` 支持 `READINESS_TARGETS` 环境变量,逗号分隔的 `host:port` +列表,例如 `mysql:3306,s3:9000`,全部 TCP 通则返回 200,否则 503。 +`STORAGE_BACKEND=local` 模式下不需要 S3 host,列表里删掉即可。 + +--- + +## 附录 A — 错误码参考 + +| HTTP | 业务码 / 含义 | 触发场景 | +|---|---|---| +| 400 | 参数错误 | Pydantic 校验失败 | +| 401 | 未鉴权 | JWT 缺失/无效 | +| 403 | 鉴权失败 | 非 workspace 成员 / `is_locked` 阻写 / **非系统管理员访问 `/api/v1/platform/*`** / 系统管理员自我移除 workspace 成员 | +| 404 | 不存在 | resource_id / script_id / schedule_id 找不到 | +| 409 | 冲突 | DAG 无效 / 同 idempotency_key 不同元数据 / 目标已存在 / `is_immutable` 阻删 / **workspace 末位 admin 保护** | +| 412 | 条件失败 | `source_object_id` 与当前工作副本不一致 | +| 413 | 太大 | 内容超过 100 MiB / 10 MiB | +| 422 | 语义错误 | 文件名非法 / cron 表达式非法 / 路径逃逸 / **`workspace_code` 不匹配 `^[a-z0-9-]{3,32}$` / `status="disabled"` 走 PATCH** | +| 500 | 内部错误 | DB / 存储不可达 | + +## 附录 B — 状态枚举 + +| 类型 | 取值 | +|---|---| +| `Workspaces.status` | `active` / `archived` / `disabled`(`disabled` 由 DELETE 设置,PATCH 不允许设) | +| `WorkspaceMembers.member_status` | `active` / `disabled` / `locked` | +| `StorageObjects.usage_type` | `data_resource` / `version_artifact` / `snapshot` / `run_log` / `run_result` / `working_copy` / `public_script` | +| `StorageObjects.object_status` | `available` / `deleted` | +| `StorageObjects.visibility` | `private` / `workspace` / `public` | +| `Scripts.status` | `active` / `deleted` | +| `Schedules.trigger_type` | `manual` / `cron` / `api` | +| `Schedules.failure_policy` | `stop` / `continue` | +| `ScheduleRuns.run_status` | `queued` / `running` / `succeeded` / `failed` / `cancelled` / `timed_out` | +| `ScheduleNodeRuns.node_status` | `queued` / `running` / `succeeded` / `failed` / `skipped` / `cancelled` / `timed_out` | +| `UploadSessions.upload_status` | `created` / `uploading` / `completed` / `expired` / `aborted` / `failed` | + +## 附录 C — 通用枚举字段 + +| 字段 | 取值 | +|---|---| +| `visibility` | `private` / `workspace` / `public` | +| `is_locked` | `0` / `1`(`Scripts` 表 TINYINT) | + +--- + +## 附录 D — 跨域与 Cookie + +- Nginx 同源代理,前端与 API 同源,不需要 CORS 配置。 +- 鉴权通过 Cookie 或 `Authorization` 头携带(见 §一)。 +- 上传类接口要求 `Idempotency-Key`,前端应在请求构造时就生成稳定 UUID + 并缓存,失败重试时复用同一 key。 diff --git a/ARCHITECTURE.md b/ARCHITECTURE.md new file mode 100644 index 0000000..141ceec --- /dev/null +++ b/ARCHITECTURE.md @@ -0,0 +1,39 @@ +# 简化系统架构 + +```text +Browser + | + v +Nginx Gateway (静态 React Router SPA + /api + /jupyter 代理) + |-----------------------------| + | /api/v1 | /jupyter/ + v v +FastAPI Backend Shared Jupyter Server + | ^ + | Runtime HTTP | Runtime 管理会话/票据 + v | +Runtime Manager ---------------| + | + +------ MySQL（编辑租约、运行实例） + +FastAPI Backend + | 1. 写 schedule_runs + outbox_events + | 2. 尝试 HTTP 立即推送 + v +Schedule Executor（APScheduler） + |-- MySQL APSchedulerJobStore + |-- MySQL Outbox 轮询兜底 + |-- DAG 节点执行与重试 + |-- S3 日志/结果 + +-- Backend 内部 Storage API +``` + +## 关键简化 + +1. 删除 Redis 服务、Redis Streams 和 Redis 文件锁。 +2. 调度定义、运行记录、Outbox、Inbox、Cron JobStore 都由 MySQL 保存。 +3. 立即运行采用 Backend -> Schedule Executor 内部 HTTP 推送；推送失败由 MySQL Outbox 轮询兜底。 +4. Schedule Executor 自带 APScheduler，负责 Cron 触发和 DAG 执行。 +5. 文件编辑锁改为 MySQL 租约，Runtime 单副本运行。 +6. Jupyter 使用一个共享容器，工作区目录通过 Volume 挂载同步。 +7. 前端改为 React Router SPA，并按 feature / route / service / component 分层。 diff --git a/CLAUDE.md b/CLAUDE.md index 12a227d..2f00ba5 100644 --- a/CLAUDE.md +++ b/CLAUDE.md @@ -1,78 +1,97 @@ -# CLAUDE.md +# Repository Guide -This file provides guidance to Claude Code (claude.ai/code) when working with code in this repository. +## Architecture -## Repository overview +- `frontend` — React Router v8 SPA; production bundle built in `nginx/Dockerfile`. +- `backend` — public FastAPI API + internal S3 storage API in one process. +- `runtime` — Jupyter lifecycle, MySQL edit leases, short-lived in-memory access tickets. +- `schedule` — APScheduler, MySQL JobStore, MySQL Outbox polling and DAG execution. +- `common` — SQLAlchemy models, database/session helpers, IDs, object-store helpers. +- `migrations` — Alembic schema + seed migrations. +- `nginx` — static frontend, `/api/` proxy, authenticated `/jupyter/` proxy. -This is a Python 3.12 `uv` workspace for a small Jupyter workspace platform. The root `pyproject.toml` includes four workspace packages: +Redis and the former separate Storage API container are intentionally removed. -- `backend`: FastAPI service used by Nginx `auth_request`. It reads the original workspace/URI headers, performs the current authentication and notebook-lock checks, asks Runtime to find or start a workspace, and returns the selected Jupyter upstream and internal token in response headers for Nginx. -- `runtime`: FastAPI process manager. At startup it mounts `REMOTE_BUCKET` with `rclone`, scans `WORKSPACES_ROOT`, and auto-starts a Jupyter Notebook process for each workspace directory. It keeps process metadata (PID, port, token, URL, start time) in the in-memory `JUPYTER_PROCESSES` dictionary and exposes health and start/stop/list/get actions. -- `common`: Shared package scaffold for SQLAlchemy/Alembic database code and object-storage helpers. `common/src/common/migrations/env.py` uses `common.db.base.Base.metadata`; the model/session/storage layers are currently mostly placeholders. -- `schedule`: Package scaffold only; `main.py` and `executor.py` are currently empty apart from module headers. +## Commands -The request path in the deployed stack is **client → Nginx (`web`) → backend auth subrequest → runtime → per-workspace Jupyter**. `default.conf` also proxies `/storage/` to RustFS and configures WebSocket forwarding for Jupyter. The frontend is not implemented in this repository; `frontend/README.md` only identifies it as the frontend. - -## Common commands - -Run these from the repository root. Dependencies are managed by the committed `uv.lock` file. +From repo root: ```bash -# Install/synchronize all workspace dependencies -uv sync - -# Run the services locally -make runtime # Runtime on 0.0.0.0:8001 -make runtime-dev # Runtime with Uvicorn reload -make backend # Backend on 0.0.0.0:8000 -make backend-dev # Backend with Uvicorn reload - -# Equivalent direct commands -uv run --package runtime uvicorn runtime.main:app --host 0.0.0.0 --port 8001 -uv run --package backend uvicorn backend.main:app --host 0.0.0.0 --port 8000 - -# Run the containerized stack (Nginx is exposed on localhost:8888) -docker compose up --build -docker compose down +uv sync --all-packages +uv run python -m compileall common/src backend/src runtime/src schedule/src +uv run --package backend alembic upgrade head +uv run --package backend pytest backend/tests -q ``` -`Makefile` variables can be overridden on the command line, for example: +cp .env.example .env +docker compose config +docker compose up -d --build +``` + +Frontend: ```bash -make runtime-dev WORKSPACES_ROOT=./test/workspaces RUNTIME_PORT=8001 -make backend-dev RUNTIME_BASE_URL=http://127.0.0.1:8001 +cd frontend +pnpm install +pnpm dev +pnpm typecheck # runs `react-router typegen && tsc` +pnpm build ``` -The local Runtime defaults to `WORKSPACES_ROOT=/app/workspaces` in code, while the Makefile overrides it to `./test/workspaces`; make sure the directory exists and contains workspace directories before expecting auto-start behavior. Runtime also needs `rclone`, FUSE support, and a valid remote configuration when exercising the mount path. The Docker image provides `rclone` and `fuse3`; a local process does not. +## Service rules -### Tests, linting, and formatting +- Browser traffic enters through Gateway only; frontend uses same-origin `/api/v1/...`. +- Backend writes `schedule_runs` and `outbox_events`, then best‑effort HTTP-dispatches to Schedule Executor. The executor always polls pending MySQL Outbox rows, so dispatch failure does not lose a task. +- Cron jobs persisted in MySQL `apscheduler_jobs`. +- Runtime must stay single-replica while file leases and Jupyter tickets use the simplified implementation. +- Never expose the internal Jupyter token to the browser. +- Never delete Docker volumes when preserving MySQL or storage data. -There is currently no test suite, test configuration, or lint/format command in the repository. Do not assume `make test`, `pytest`, Ruff, or Black is configured. If tests are added, run a single test with the project’s chosen runner (the conventional pytest form is `uv run pytest path/to/test_file.py::test_name`), then add the corresponding dependency and documented command rather than silently relying on a globally installed tool. +## Engineering notes -A lightweight current smoke check is to compile the Python sources: +Hard-won lessons. Read the relevant bullet before touching the named area. -```bash -python -m compileall backend/src common/src runtime/src schedule/src -``` +### Python runtime — always go through `uv run` -### Database migrations +- Repo is a uv workspace; `common` / `backend` / `runtime` / `schedule` / `migrations` share one `.venv`. Bare `python` / `pytest` / `alembic` resolves to system Python and **all `from backend.X import ...` / `from common.X import ...` fail with ModuleNotFoundError**, or worse: an out-of-date venv silently runs stale code. +- Always prefix with `uv run [--package ] `: + - `uv run --package backend pytest backend/tests -q` + - `uv run --package backend alembic upgrade head` / `downgrade -1` + - `uv run python -m compileall common/src backend/src runtime/src schedule/src` + - `uv run python -c "from backend.foo import bar"` for one-shot inspection +- Migration files use **plain** `op.drop_index` / `op.create_index` — MySQL 8.0 does not support `IF EXISTS` / `IF NOT EXISTS` on `DROP INDEX` / `CREATE INDEX`, even though Alembic exposes the flag. -Alembic is configured in `common/alembic.ini`, with scripts under `common/src/common/migrations`. The checked-in `sqlalchemy.url` is the generated placeholder `driver://user:pass@localhost/dbname`, so replace/configure it for a real database before running migrations. From the root, the usual commands are: +### Platform auth / soft-delete (`/api/v1/platform/employees`) -```bash -uv run alembic -c common/alembic.ini current -uv run alembic -c common/alembic.ini upgrade head -uv run alembic -c common/alembic.ini revision --autogenerate -m "describe change" -``` +- **Reuse `system_admin_context` + `_*_admins` helpers** in `backend/src/backend/platform.py`. Self-protection and the last-admin guard for `Users.platform_role_id` mirror the workspace pattern. `_count_active_system_admins(session, exclude_user_id=...)` is already there; do not reinvent the count in the handler. +- **PATCH guard order is load-bearing.** Always check `is_current_system_admin` last-admin first, then `leaves_admin_pool`, then self-demotion. Reversing the order lets test mocks bypass the count helper. +- **404 vs 409 on already-soft-deleted:** `delete_platform_employee` returns a single 404 "用户不存在" for both `user is None` and `user.is_deleted != 0`. This intentionally differs from `DELETE /workspaces/{id}` which returns 409 for already-disabled. Document both in API.md. +- **PATCH cannot null-out `platform_role_id`.** `PlatformEmployeeUpdate.role_code: Literal["admin","developer"]` (not `Optional`). Add a separate endpoint if clearing is needed — never loosen the Literal. +- **DELETE cascade covers `WorkspaceMembers` only** — writes `is_deleted=1, deleted_at=now` on `WorkspaceMembers` rows for the user. Does not touch `Workspaces`. -## Important implementation details +### Backend test mocks (SQLAlchemy 2.0 / pytest-asyncio) -- `backend/src/backend/main.py` is the ASGI entrypoint (`backend.main:app`). Nginx sends `/internal-auth` to `GET /api/v1/auth/jupyter`; the backend expects `X-Original-Workspace-Id` and `X-Original-URI`, chooses a bearer token or `access_token` cookie, and returns `x-upstream-addr` plus `x-jupyter-internal-token` headers. The JWT validation and database lock lookup are currently mocked/commented, so treat this as demo behavior rather than complete production authentication. -- `runtime/src/runtime/main.py` is the ASGI entrypoint (`runtime.main:app`). Startup/shutdown is implemented through FastAPI lifespan: mount/scan/start on startup, stop Jupyter processes and unmount on shutdown. The API is `GET /api/v1/health` and `POST /api/v1/jupyter` with `action` values `start`, `stop`, `list`, or `get` (workspace ID is required for all except `list`). -- Runtime workspace IDs are directory names below `WORKSPACES_ROOT`. A started Jupyter server receives a dynamic port, a generated token, and base URL `/jupyter//`. Because process state is only in memory, restarting Runtime loses the registry and causes startup scanning to rediscover workspaces. -- `common` is a uv workspace dependency declared by backend/runtime/schedule, but the current service code does not yet contain substantial shared model or storage integration. Keep shared DB/storage behavior in `common` rather than duplicating it in services. -- `docker-compose.yml` builds backend and runtime from the root workspace, exposes backend on `8000`, runtime on `8001`, and Nginx on `8888`. Runtime requires Linux FUSE capabilities (`SYS_ADMIN`, `/dev/fuse`, and unconfined AppArmor). Its RustFS/rclone settings are supplied as compose environment variables; use deployment secrets or environment overrides instead of committing credentials. +- **`compile(literal_binds=True)` uppercases keywords.** `"from roles" in text` misses the table reference; match `text.lower()` or test for `"roles.role_id"` / `"roles.role_code"` directly. +- **`Depends`-style helpers are awaited.** `_load_role_by_code` and `_count_active_system_admins` need `async def` mocks — sync `lambda` raises `TypeError: object int can't be used in 'await' expression`. +- **Mock response ordering matters for re-reads.** `update_platform_employee` reads `current_role` before write then `response_role` after. A scalar mock returning a fixed value will return the pre-write role in the response — track call order or look up by `user.platform_role_id` post-write. +- **Mock users must declare every attribute the handler writes.** `Users.deleted_at` is not in column defaults; `SimpleNamespace(user_id=...)` raises `AttributeError` on `target.deleted_at = now`. Set `user.deleted_at = None` explicitly. -## Change boundaries +### Frontend state + routing (zustand + React Router v8) -Keep service entrypoints and the Nginx header contract compatible when changing backend/runtime behavior. If an API action or response field changes, update both the caller in `backend` and the proxy rules in `default.conf` together. For runtime process changes, preserve cleanup in the lifespan shutdown path and consider failures from missing mounts, dead subprocesses, dynamic port allocation, and remote storage separately. +Lessons from splitting `frontend/app/features/platform/ModelPlatformApp.tsx` (1057 → 121 lines) into zustand stores + nested routes. + +- **Barrel-file CSS imports vanish on refactor.** `features/admin/AdminPages.tsx` was a barrel that side-effect-imported `admin.css` + `dashboard.css`. When route files started importing `DashboardPage` / `SystemAdminPage` directly, the CSS disappeared silently. Fix: each page component imports its own CSS at the top — `DashboardPage` needs **both** `admin.css` (`.dashboard-page`, `.dashboard-hero`, `.dashboard-metrics`, `.dashboard-actions`) **and** `dashboard.css` (`.dashboard-grid`). `UserManagementPage` / `ProjectManagementPage` / `SystemAdminPage` only need `admin.css`. Don't put CSS imports in route files; let the components own their styles. `SchedulePage` already follows this pattern with `schedule.css`. +- **Module-level zustand store + `bindApi(api)`** for auth-dependent APIs. Keep a module-level `_api` ref; expose `bindScriptWorkspaceApi(api)`; layout calls it in **render body** (not `useEffect([api])`). Actions read `_api` internally — no api param on every call. Pair the render-body bind with a separate empty-deps `useEffect(() => () => bindScriptWorkspaceApi(null), [])` for unmount cleanup only. + - **Why render body, not `useEffect([api])`:** React effect order on deps change is *parent cleanup → child effect → parent effect*. With `useEffect([api])`, the parent's cleanup wipes `_api` to `null` *before* child effects (e.g. `ScriptsPage`'s `useEffect([workspaceId])` calling `load()`) run, producing the `script workspace API 未绑定` race whenever `currentWorkspace.workspace_id` changes. Render-body binding runs synchronously during the parent's render, which happens before the child's render and effects, so `_api` is always current by the time child code touches the store. +- **Lifecycle hooks belong in the layout, not in route components.** Heartbeats (active edit session + cached sessions), the 10-min cleanup timer, and `beforeunload` lock-release must mount at the layout level — navigating to `/schedules` otherwise unmounts them and cached locks expire. Pattern: store exposes `tickHeartbeats()` / `tickCleanup()` / `releaseActiveOnUnload()`; the layout hook just owns the `setInterval` and `addEventListener`. +- **Route components with their own internal state must remount on workspace/user change.** `SchedulesPage` / `SystemAdminPage` / `UserManagementPage` / `ProjectManagementPage` keep `useState` for fetched data (`schedules`, `artifacts`, `selectedSchedule`, employees, projects). When `currentWorkspace` or `user` changes, the `api` reference updates but the cached state does not — the UI shows the previous workspace's data. Pattern: route wrappers set `key={\`${user?.user_id ?? "anon"}-${currentWorkspace?.workspace_id ?? "none"}\`}` on the page component to force React to unmount and remount, resetting all internal state and re-running `useEffect` data fetches. `ScriptsPage` doesn't need this — its store-backed state is reset via `scriptWorkspaceStore.reset()` on workspace change. +- **`{ current: T | null }` module-level handle for non-subscribing consumers.** Sidebar reads "is there an active edit session?" without subscribing to the store — expose a plain `{ current: ... }` object at module scope and update it synchronously inside the store's `setEditSession` action. +- **zustand `StateCreator` enforces declared action signatures.** Declaring `loadLatestVersion: () => Promise` while accidentally returning a cleanup function from the implementation makes `tsc` reject the whole store with TS2345. Match the declared type exactly. +- **Nested routes in React Router v8.** Use `route("", "layout.tsx", [route("x", "x.tsx"), ...])` from `@react-router/dev/routes`. URLs stay flat; the layout renders ``. Don't use `route("*", ...)` as a wildcard — it skips the nested children config. +- **Route ids are derived from file paths.** The same file cannot be referenced by two route entries (`index("X.tsx")` + `route("y", "X.tsx")` → `duplicate route id`). To make `/` redirect to `/workbench`, create a tiny `RootIndex.tsx` that renders `` rather than reusing `DashboardRoute.tsx`. +- **An unmatched nested child leaves `` blank → white screen inside the layout.** Always cover `/` either with `index(...)` or by letting the parent layout `` on a location check. `/login` is the only top-level path that escapes this trap. +- **`pnpm typecheck` runs `react-router typegen && tsc`.** Type errors from the generated `+types/...` files surface here too. New route files must be registered in `routes.ts` first. + +### Frontend coupling + +- **`UserManagementPage.tsx` and `api.ts` still call `/api/v1/admin/employees`.** Don't migrate them in the same change as a `/api/v1/platform/employees` addition — the contract surface is intentionally duplicated. \ No newline at end of file diff --git a/CODE_REVIEW.md b/CODE_REVIEW.md new file mode 100644 index 0000000..5e1b250 --- /dev/null +++ b/CODE_REVIEW.md @@ -0,0 +1,202 @@ +# 代码审查报告 + +- 审查日期：2026-08-14 +- 审查范围：backend / common / runtime / schedule / frontend / migrations / 部署配置（约 1.5 万行 Python + 50 个前端文件） +- 审查方式：Ruff 静态扫描 + 逐模块人工逻辑审查 +- 严重度约定：🔴 高（功能错误 / 安全风险，应尽快修复）｜🟡 中（特定条件下出错或资源泄漏）｜🔵 低（健壮性 / 可维护性问题） + +--- + +# 第一部分：静态扫描结果 + +## 总体健康状况 + +| 检查项 | 结果 | +| --- | --- | +| `python -m compileall`（4 个包） | 通过 | +| `pytest backend/tests` | 37 passed（含 2026-08-14 数据资源链路新增用例） | +| 前端 `tsc --noEmit` | 零错误 | +| Ruff | 391 条（198 条可 `--fix` 自动修复） | +| 危险模式（eval / shell=True / dangerouslySetInnerHTML / SQL 拼接） | 未发现 | +| `.env` 入库 | 未入库（.gitignore 已覆盖） | + +## Ruff 规则分布（Top） + +| 规则 | 数量 | 说明 | +| --- | --- | --- | +| B008 | 147 | FastAPI `Depends()`/`Query()` 惯用法误报，建议配置忽略或 `extend-immutable-calls` | +| UP045 | 108 | `Optional[X]` → `X \| None`，可自动修复 | +| I001 | 38 | import 排序，可自动修复 | +| BLE001 | 19 | 盲捕 `Exception`（runtime/process.py 占 11 处），多数为兜底、可接受 | +| F401 | 16 | 未使用 import，可自动修复 | + +其余：F811 ×3（storage_api.py 本地 def 覆盖顶部同名 import）、DTZ003 ×4（platform.py 使用已弃用的 `datetime.utcnow()`）、F841 ×2（scripts.py 赋值未使用）等。 + +## 静态扫描安全发现 + +1. 🔴 **docker-compose 端口映射与"仅 Nginx 对外"架构矛盾**：`backend`（8891）和 `runtime`（8892）实际映射了宿主机端口，但注释声称 "No host port"。后果： + - backend `/internal/v1/*` 存储 API 无 JWT 认证，直接信任请求体中的 `user_id`/`workspace_id`（services/storage.py:215）； + - runtime `POST /api/v1/jupyter` 无认证，可对任意 workspace 启动/停止 Jupyter。 + 建议：删除 `ports` 映射或绑定 `127.0.0.1`；为 `/internal/*` 增加 service token。 +2. 🟡 `INITIAL_ADMIN_PASSWORD` 默认 `admin12345`，且当前 `.env` 实际使用该值。 +3. 🟡 `/api/v1/auth/login` 无速率限制 / 账户锁定，可在线爆破（401 统一文案防枚举做得对）。 +4. 🔵 `jwt_secret` 存在 `"dev-only-not-for-production"` 兜底默认值（当前 .env 已配置真随机值），建议生产启动时 fail-fast 校验。 + +## 测试覆盖 + +backend 14.4k 行代码仅 3 个测试文件 / 30 个用例；runtime、schedule、common 无测试。CLAUDE.md 中记录的 platform 员工管理 guard 顺序等 load-bearing 逻辑无回归测试兜底。 + +--- + +# 第二部分：逐模块逻辑审查 + +## 1. backend — 存储层（storage_api.py / services/storage.py） + +**🔴 B1. 上传失败/过期状态永远不落库** +`services/storage.py` `upload_bytes_to_session` 中 `upload.upload_status = "expired" / "failed"` 三处赋值后紧跟 `raise HTTPException`，而 `session_scope`（common/db/session.py）遇异常即回滚——这些状态迁移全部被撤销。失败的上传会话在库里永远停在 `created/uploading`。IntegrityError 路径更显眼：先置 `failed` 再显式 `rollback()`，把自己的赋值也回滚了。 +> 2026-08-14 部分处理：IntegrityError 分支的无效赋值与显式 `rollback()` 已删除（该分支现靠命名锁前置避免，见下方"数据资源链路复审"第 3 项）；"failed/expired 状态被 session_scope 回滚"的根因仍在，待修。 + +**🟡 B2. 软删除大对象整体读入内存** +`soft_delete_object` 用 `get()`（返回 `bytes`）把对象全量加载后复制到 trash 桶。nginx 允许 100G 上传，删除大文件会 OOM。S3 应走 server-side copy，local 后端已有 `copy()`。 + +**🔵 B3. `USAGE_TYPE_TO_PURPOSE[item.usage_type]` 直接索引** +未知 usage_type 抛 KeyError → 500。同文件 `_resolve_bucket_for_usage` 是优雅降级的，两处策略不一致。 +> ✅ 2026-08-14 已修复：改为 `.get(item.usage_type, "workspace")` 兜底。 + +**🔵 B4. 上传 IntegrityError 后 S3 对象成孤儿** +先 `put` 后 `flush`，flush 失败只回滚 DB，已写入的对象字节无补偿删除（scripts.py 的 Jupyter 文件有补偿清理，这里没有，不对称）。 + +## 2. backend — 脚本模块（scripts.py，1552 行） + +**🔴 B5. `update_script` 不回写 DB 元数据** +编辑保存后只更新 `script.updated_at`，新的 `content_hash`/`size_bytes` 只拼在响应 dict 里——`StorageObjects` 行永远保留创建时的旧哈希和大小。且响应里 `relative_path` 用了 `jupyter_path` 格式（无 `workspace/` 前缀），与库中存储格式不一致。 + +**🟡 B6. `delete_script` 不删 Jupyter 侧实际文件** +只软删 DB 行并把 S3 字节移入 trash，工作区挂载里的文件保留。创建路径的注释明确说 "rclone VFS 会把文件复制回对象存储"——残留文件存在被回写"复活"的风险，且用户在 Jupyter 里仍能看到/编辑已删除的脚本。 + +**🔵 B7. 日志泄露内容** +`logger.debug(notebook)` 把整本 notebook JSON 写进日志；`logger.debug(jupyter_resp)` 同理。`log_level` 默认即为 DEBUG。 + +**🔵 B8. `_derive_jupyter_path` 的 fallback 可疑** +无 StorageObjects 行时用 `script_id` 当文件名（`.ipynb`），与创建时按用户文件名命名的规则不匹配——"jupyter-only" 脚本的读写会打到不存在的文件上。若为 legacy 数据准备，需写清来源。 + +## 3. backend — 其余（auth / dependencies / jupyter / platform / admin / schedule_runs） + +- ✅ JWT 验签固定 HS256、防 alg 混淆、`compare_digest`；登录 401 统一文案防枚举；`request_context` 强制显式 `workspace_id`；platform.py 的 last-admin 守卫顺序与 CLAUDE.md 约定一致。 +- 🔵 `schedule_runs.py` 手动触发接口允许 `payload.reason == "cron"` 把 `trigger_type` 伪装成 cron，审计字段失真。 +- 🔵 手动 `run_schedule_now` 不校验 `schedule.enabled`，已禁用的调度仍可手动跑（需确认是否有意）。 +- 🔵 nginx 对 `/jupyter/` 响应 `add_header X-Debug-Full-Url ... always`，向浏览器暴露内部 upstream 地址，调试头应下线。 + +## 4. common（config / auth / db / eventing / storage / scheduler trigger） + +- ✅ jwt.py、passwords.py、local 后端 `_resolve` 路径穿越防护、eventing.py 均无问题。 +- 🟡 C1. `trigger.py` 模块 docstring 声称 "executor 会在执行前 re-verify `Users.status='active'`"，但 worker.py 全文没有任何 Users 校验——**被停用/删除用户的既有调度仍会照常执行**。文档与实现二选一。 +- 🔵 C2. bcrypt 72 字节上限：schema 限 `max_length=72` 字符，但按 UTF-8 字节算 24 个汉字就超限，`hash_password` 抛 ValueError → 注册 500。应按字节校验或截断。 +- 🔵 C3. local 后端 `get_url` 返回 `file://` URI，浏览器无法使用——local 模式下"下载链接"功能实际不可用。 +- 🔵 C4. 敏感配置均有默认值兜底，生产漏配会静默使用弱密钥（加固建议：默认值时启动 fail-fast）。 + +## 5. runtime（main / mount / process） + +**🟡 R1. reaper 清理死进程无锁，可误删新记录** +`_reap_loop` 基于循环开头的快照判断 `poll() != None` 后直接 `del JUPYTER_PROCESSES[ws_id]`。若此间 `start_workspace`（持锁）已删掉旧记录、启动新进程并写入新记录，reaper 这一 `del` 删掉的是**新记录**——新 Jupyter 脱离注册表，永不回收（端口/进程泄漏）。 + +**🟡 R2. `_drop_workspace_lock` 破坏互斥** +stop 后移除锁：正在 await 旧锁的协程仍持有旧锁对象，新调用者创建新锁——同一 workspace 出现两把锁，start 可并发。建议不删锁或引用计数。 + +**🟡 R3. rclone RC 无认证监听 0.0.0.0:5572** +`--rc --rc-no-auth`——Docker 网络内任何容器可读写整个 workspace bucket。建议绑定 127.0.0.1 或加 rc-user/rc-pass。 + +- 🔵 R4. `stop_workspace` 在事件循环里同步 `process.wait(timeout=3)`，逐 workspace 串行阻塞事件循环。 +- 🔵 R5. start 复用路径对超 `MAX_LIFETIME` 的进程只 warn 仍复用，而 reaper 会在 60s 内按 max-lifetime 杀掉它——两处策略矛盾。 +- 🔵 R6. `get_free_port()` 到 Jupyter bind 之间 TOCTOU；`JupyterProcessRecord` TypedDict 漏声明实际写入的 `workspace_path` 键。 +- 🔵 R7. `/tmp/rclone-mount.log` 追加写入无轮转。 + +## 6. schedule（service / scheduler / orchestrator / worker / execution） + +**🔴 S1. worker 下载产物忽略 bucket，与发布路径矛盾** +`_download_artifact` 只用 `object_key`，`build_object_store()` 固定指向 version bucket；但发布路径 `_resolve_bucket_for_usage` 尊重 `workspace.artifact_bucket` 覆盖。**凡是配置了 artifact_bucket 的 workspace，其调度运行必然找不到产物**。service.py 中 "regardless of the artifact's workspace 是正确解析" 的注释在该场景下是错的。 + +**🟡 S2. 重试耗尽后 run 永久卡死** +outbox 事件重试 5 次后标记 `failed`，但对应的 `node_run` 永远停在 `queued/running`，`_advance_run` 再无机会触发——run 永远 `running`，无 run 级超时或 janitor 兜底。 + +**🟡 S3. 单个坏 cron 表达式毒化整个同步循环** +`_sync_once` 遍历中直接 `CronTrigger.from_crontab(...)` / `ZoneInfo(item.timezone)`，任一 schedule 抛异常即中断整轮同步，所有 cron 调度停止 reconcile（外层 5s 重试同样失败）。需要 per-schedule try/except 隔离。 + +- 🔵 S4. 每 5 秒对所有 job 无条件 `reschedule_job`，churn 无谓；可比对表达式后再 reschedule。 +- 🔵 S5. `process_pending_events` 的 select 未加 `skip_locked`——单副本靠 `dispatch_lock` 没问题，多副本部署会重复认领（inbox 幂等兜底，但浪费执行）。 +- 🔵 S6. `notebook_runner.py` 用 `"timeout" in type(exc).__name__.lower()` 判定超时映射 exit 124，依赖 nbclient 异常类名，依赖升级会静默失效。 +- 🔵 S7. scheduler.py / service.py 模块 docstring 仍写 "cron tick posts back to Backend"，实际早已直写 DB——过时注释。 +- ✅ orchestrator 的 DAG 推进逻辑（重试次数语义、skip 条件、failure_policy、崩溃恢复、bootstrap 去重 flush）逐条推演未发现错误；租约绑定 `timeout_seconds + slack` 的设计正确。 + +## 7. frontend + +- ✅ `tsc` 零错误；无 `dangerouslySetInnerHTML`/`eval`；zustand 绑定模式按 CLAUDE.md 约定实现。 +- 🟡 F1. `api.ts` 的 `heartbeatFileLock`/`releaseFileLock` 是纯前端 stub，不触达任何后端接口——"编辑锁"实际只存在于当前标签页内存，刷新即失效。注释自称 compatibility mode，需确认是预期终态还是临时方案。 +- 🔵 F2. `waitForJupyterReady` 400ms×10 ≈ 4s 总超时，对冷启动 Jupyter（runtime 侧允许 30s）明显偏短，用户会偶发看到"启动超时"。 + +## 8. 部署与迁移 + +- 🔴 docker-compose 端口暴露（见第一部分安全发现 1），与 R3 叠加后攻击面从"内网"扩到"宿主机可达"，为全库最高优先项。 +- 🟡 `INITIAL_ADMIN_PASSWORD=admin12345` 同时出现在 compose 默认值和当前 `.env`。 +- ✅ migrations 单 baseline 文件结构干净；`.env` 未入库；JWT_SECRET 已是真随机值。 + +--- + +# 附：2026-08-14 数据资源上传/绑定/删除链路复审（已全部修复） + +针对 `POST /uploads` → `PUT /uploads/{id}` → `POST /uploads/{id}/bind` → `DELETE /{resource_id}` +链路的专项复审。起因：个人根目录已存在文件 `a` 时，子目录上传同名文件报 +"a data resource with this name already exists in this workspace"——原查重按 +workspace 全局 `resource_name` 唯一，不区分目录。本轮修复含语义调整与 10 个附带问题， +改动文件：`backend/resources.py`、`backend/services/storage.py`、`backend/storage_api.py`、 +`common/db/models/storage.py`（仅注释）、`backend/tests/test_resources.py`。 + +**语义调整（需求确认）** + +- 同名查重维度：`workspace + resource_name` → `owner + 目录 + resource_name`。 + 目录从绑定的 `StorageObjects.object_key`（`{ws}/{user}/{target_path}/{file_name}`） + 解析（`resources.py::resource_directory`），同目录同名才 409；不同目录、不同用户均可重名。 +- 回收站防覆盖：`trash_key` 由 `{purpose}/{object_key}` 改为 + `{purpose}/{object_key}-{storage_object_id}`，同名文件多次删除不再互相覆盖； + restore 时剥掉 id 后缀拷回原 key（`endswith` 判断，旧格式数据天然兼容）。 + +**修复清单（按审查时严重度排序）** + +1. 🔴 bind 不幂等：成功后的重试必被同名查重误伤 409。→ storage object 复用检查前置， + 已有 active 绑定行直接返回（`reused: true`）。 +2. 🔴 删除后重新 bind 复用 `status="deleted"` 的"尸体"行（无 status 过滤），200 返回已删除数据。 + → 复用查询加 `status == "active"`，旧删除行走新建流程。 +3. 🔴 PUT 先于 INSERT，同 key 并发上传互相覆盖字节，赢家的 `content_hash` 与实际字节不符。 + → `upload_bytes_to_session` 的「PUT + INSERT」临界区按 object_key 加 MySQL 命名锁 + （新增 `acquire_named_lock`/`release_named_lock`，锁名 sha256 压缩到 64 字符内， + finally 释放）；锁内复查占用并换 ULID 后缀新 key。 +4. 🟡 `delete_resource` 不检查共享引用，直接软删底层对象导致其他 active 资源行失效。 + → 删除前统计同 object 的其他 active 行，有引用则只删 DataResources 行。 +5. 🟡 `soft_delete_object` 未知 usage_type KeyError → 500。→ `.get` 兜底（即原 B3）。 +6. 🟡 幂等键元数据比对漏 file_name/target_path，同键不同路径会静默复用旧会话。 + → 比对加路径维度；已有会话的 key 先经 `_strip_uniqueness_suffix` 去唯一化后缀再比。 +7. 🔵 bind 不校验 `usage_type`，任意用途的 upload session 可 bind 成数据资源。 + → 非 `data_resource` 报 409。 +8. 🔵 bind 查重无并发保护。→ 查重 + 建行按 `(owner, 目录, 名称)` 加命名锁。 +9. 🔵 回收站 key 碰撞 + 移入回收站后 `object_key_hash` 过期。→ 见上方语义调整； + hash 随 trash key 同步更新。 +10. 🔵 IntegrityError 分支 `upload_status="failed"` 紧接着 `rollback()` 自我撤销。 + → 删除无效赋值与多余 rollback（锁内复查后该分支仅剩理论可能）。 + +**遗留说明** + +- 命名锁依赖 MySQL `GET_LOCK`/`RELEASE_LOCK`（锁绑定会话连接，连接归还连接池不自动释放， + 故必须 finally 显式释放）；更换数据库时需重审。 +- 测试 37 passed；新增用例覆盖：同目录同名 409、跨目录/跨 owner 同名放行、幂等重 bind、 + 非 data_resource 上传 bind 拒绝、目录/后缀解析等纯函数。 + +--- + +# 修复优先级建议 + +1. 端口暴露（第一部分安全发现 1）+ R3（rclone RC 无认证） +2. B5（update_script 元数据不落库）——数据正确性，每日发生 +3. S1（artifact_bucket 覆盖导致调度必然失败）——潜在功能断裂 +4. S2 + R1/R2（run 卡死、Jupyter 注册表竞态）——长跑后的资源/状态泄漏 +5. B1（根因仍在，IntegrityError 分支已随数据资源链路复审清理）/ B2 / C1 / F1（状态机失真、大文件 OOM、停用用户仍执行、锁 stub） +6. 其余低优项可并入一次 `ruff --fix` + 清理提交 diff --git a/DEVELOP.md b/DEVELOP.md new file mode 100644 index 0000000..2781ab5 --- /dev/null +++ b/DEVELOP.md @@ -0,0 +1,448 @@ +# DEVELOP.md — Developer Guide + +This guide is for engineers working on the model platform codebase. For +high-level design see `ARCHITECTURE.md`; for the current state of in-flight +refactors see `HANDOVER.md`. + +## Code layout + +``` +common/ Pure-Python shared library + config.py Settings (pydantic-settings, lru_cache singleton) + db/ SQLAlchemy 2.0 async engine, session_scope, Base + db/models/ 26 tables in 9 domain files (zero FK, zero relationship) + scheduler/ build_sqlalchemy_jobstore (delayed import) + storage/ AsyncStorageBackend abstraction (s3 + local impls) + Pydantic schemas + eventing.py add_outbox_event / utcnow / event_time + service_app.py /health/ready TCP probe, /api/v1/health + schemas.py StrictModel base + utils.py get_free_port, start_process + +backend/ Public FastAPI service + tiny /internal/v1/objects RPC + main.py lifespan + route registration + jupyter.py /api/v1/auth/jupyter — the ONLY auth entry + scripts.py CRUD for scripts/notebooks (object storage via AsyncStorageBackend) + schedules.py DAG CRUD: schedules, nodes, edges + schedule_runs.py Trigger / list / get runs + schedule_schemas.py Pydantic request/response models + admin.py Admin endpoints + resources.py Misc data resources + storage_api.py /internal/v1/objects — single token-guarded endpoint (P0-1) + storage_client.py Stub (HTTP client removed post-migration; rewrite pending) + schedule_client.py Placeholder module (was the HTTP-push executor client) + runtime_client.py Self-contained httpx wrapper for the runtime + jupyter.py auth_request handler + dependencies.py request_context, database_session + +schedule/ Schedule Executor (DAG worker) + context.py Constants + naive_utc + scheduler.py CronScheduler (APScheduler + 5s sync loop) + orchestrator.py DispatchOrchestrator (Outbox poll + DAG advance) + worker.py NodeExecutor (notebook / python execution) + service.py SchedulerService facade (composes the three) + main.py Lifespan + FastAPI app + storage_client.py SchedulerStorageClient — talks to backend /internal/v1/objects + execution.py execute_artifact (notebook + python paths) + notebook_runner.py Subprocess entry point (nbclient) + +runtime/ Jupyter Runtime + main.py FastAPI entry: jupyter action endpoints + process.py Per-workspace subprocess pool + asyncio locks + mount.py rclone FUSE mount lifecycle + +frontend/ React Router SPA (vite build → nginx) + app/ features/ routes/ services/ components/ + +migrations/ Alembic schema versions +docker-compose.yml 4 services +default.conf Nginx template +scripts/nginx-entrypoint.sh +.env.example +``` + +## Configuration system + +All env vars go through one place: `common/src/common/config.py`. + +```python +from common.config import settings + +settings.database_url # str +settings.storage_backend # str: "s3" (default) or "local" +settings.local_storage_base_dir # str: root dir for storage data (default "/data"); see "Storage" below for per-mode derivation +settings.s3_endpoint # str (full URL, e.g. "http://s3:9000"; s3 mode only) +settings.s3_access_key # str (s3 mode only) +settings.s3_secret_key # str (s3 mode only) +settings.s3_workspace_bucket # str (s3 mode only) +settings.s3_version_bucket # str (s3 mode only) +settings.s3_run_log_bucket # str (s3 mode only) +settings.s3_trash_bucket # str (s3 mode only) +settings.s3_trash_retention_days # int (s3 mode only) +settings.jwt_secret # HS256 secret for the auth_request handler +settings.backend_api_url # schedule → backend HTTP base +settings.runtime_api_url # backend → runtime HTTP base +settings.public_base_url # runtime public base URL +settings.service_name # surfaced in /health +settings.readiness_targets # CSV host:port list for /health/ready +``` + +`Settings` reads from process env first, then from a `.env` file at CWD +if present. `pydantic-settings` auto-loads. `case_sensitive=False` so +`DATABASE_URL` / `database_url` both work. + +### Adding a new env var + +1. Add the field to `Settings`: + ```python + new_var: str = Field(default="x", description="...") + ``` +2. Add the line to `.env.example` with a comment. +3. Use `settings.new_var` at the call site. + +Do **not** call `os.environ["NEW_VAR"]` or `os.getenv("NEW_VAR")` in +application code. The grep below should return zero hits: + +```bash +grep -rnE 'os\.(environ\[?["\x27][A-Z_]+|getenv\(["\x27][A-Z_]+)' --include="*.py" \ + backend/src/ schedule/src/ runtime/src/ common/src/ +``` + +## Conventions + +### Database / SQLAlchemy + +- **No foreign keys, no `relationship`** — every join is explicit. +- Every table has `is_deleted TINYINT(1) NOT NULL DEFAULT 0` and + `deleted_at DATETIME(3) NULL`; queries must filter `is_deleted == 0` + (or `deleted_at.is_(None)`) to avoid logical-deleted rows. +- Domain files under `common/src/common/db/models/` are split by + bounded context: `audit` / `events` / `experiments` / `identity` / + `runtime` / `schedules` / `scripts` / `storage` / `workspaces`. +- All models are `class X(Base)` SQLAlchemy 2.0 declarative-mapped. +- The full schema is in `migrations/versions/`. Apply with: + ```bash + uv run --frozen --package backend alembic upgrade head + ``` + +### Storage + +- All object bytes go through `common.storage.AsyncStorageBackend`, + created by `create_storage(config)` from `common.storage.factory`. +- Two backends are registered: `local` (filesystem, local mode) and + `s3` (S3-compatible service, s3 mode). Selection is per-deployment + via `settings.storage_backend` (`"s3"` default, `"local"` for + dev / single-node / air-gapped). +- The factory helper `build_storage_config(bucket_name)` returns the + right `create_storage` kwargs for each of the 4 purpose buckets + (`workspace`, `version`, `run_log`, `trash`). Use it in lifespan code; + route handlers don't see the difference. +- Bucket resolution from `usage_type` is in **one place** + (`backend/storage_api.py:resolve_bucket`); route handlers only know + about `app.state.object_stores[bucket_name]`. +- The runtime's view of the workspace bucket on disk is exposed by + `common.storage.workspaces_root()`: + - `s3` mode: `${settings.local_storage_base_dir}/workspace` + (default `/data/workspace`, the rclone FUSE mount target). + - `local` mode: `${settings.local_storage_base_dir}/workspace` + (default `/data/workspace`, a subdir of the shared local-storage + volume). + `settings.local_storage_base_dir` is the **only** path setting; the + helper handles the per-mode suffix. Don't read `settings.workspaces_root` + or any other path setting directly in runtime code — use this helper. +- The pre-2026 abstraction (`RustFSObjectStore` / `common.storage.client` + / `StorageClient` HTTP wrapper) is gone. Don't reintroduce it. + +### Auth + +- Browser → `/jupyter/{workspace_id}/...` → Nginx `auth_request` → + `GET /api/v1/auth/jupyter` (Backend). +- The handler: + 1. Parses cookie / Bearer JWT (HS256 + `settings.jwt_secret`). + 2. Verifies `WorkspaceMembers` for the workspace. + 3. Verifies `Scripts.is_locked` for the requested notebook path + (owner / unlocked → allow; otherwise 403). + 4. Calls `RuntimeClient.get_workspace` / `start_workspace`. + 5. Returns `x-upstream-addr` + `x-jupyter-internal-token` response + headers. **Browser never holds the runtime token.** + +### Service-to-service auth (P0-1 fix) + +- Schedule → Backend single endpoint ``POST /internal/v1/objects`` is + guarded by ``require_internal_service`` in ``backend.storage_api``. +- The token header is ``X-Internal-Service-Token`` (case-insensitive + on the wire because FastAPI ``Header`` lowercase-matches the name + ``x-internal-service-token``); the secret value comes from + ``settings.internal_service_token`` / env ``INTERNAL_SERVICE_TOKEN``. +- Comparison uses ``secrets.compare_digest`` — never equality. +- Backend and schedule must be configured with the same value; a + mismatch fails fast at the first notebook run (``401``) which is + intentional. ``.env.example`` ships a placeholder + ``change-me-internal-service-token`` and the docker-compose + ``${INTERNAL_SERVICE_TOKEN:?...}`` reference forces production + deployments to set a real value. +- Removing the legacy backend / runtime host-port mappings + (``8891:8000`` / ``8892:8000``) is part of the same fix — no service + is reachable from the host except Nginx anymore. +- Nginx captures the headers via `auth_request_set` and proxies to the + upstream sub-process with `Authorization: token $jupyter_token`. + +### Permission gates + +- For write operations on a script/notebook, call + `require_script_modify_access(script, user_id=..., is_admin=...)` + from `backend/scripts.py`. It enforces: + - admin or owner → allow + - non-owner, `is_locked == 0` → allow + - non-owner, `is_locked == 1` → 403 +- Read endpoints (`list_scripts`, `get_script`) intentionally do **not** + check `is_locked` — workspace members can see the script list. + +### Outbox events + +- The platform's only async-messaging fabric is the MySQL + `OutboxEvents` table. Producers (Backend) write rows in the same + transaction as the business state. Consumers (Schedule Executor) + poll every 250 ms and update `event_status` to `published` or + `failed` (with retry). +- Use `common.eventing.add_outbox_event` to write. +- `event_type` values currently in use: + - `schedule.run.requested` — produced by `schedule_runs.py` / cron post-back + - `job.node.execute` — produced by orchestrator when a node is ready + - `job.node.finished` — produced by worker after node execution +- `consumer_inbox` provides exactly-once delivery per + `(consumer_name, event_id)` (with `process_status: processing → + succeeded` lifecycle). + +### Async / sync signatures + +- `SchedulerService.start()` and `SchedulerService.close()` are + `async def` (so the FastAPI lifespan can `await` them). +- `CronScheduler.start()`, `DispatchOrchestrator.start()` are + **`def` (sync)** — they only `create_task(...)` and return. Don't + `await` them. +- `cron.start()`, `orchestrator.start()`, `worker.handle_node_execute` + are wired together in `SchedulerService.__init__`; their lifetime + is owned by the facade. + +## Local development + +### One-time setup + +```bash +# Python workspace (monorepo via uv workspaces) +uv sync --all-packages + +# Frontend deps +cd frontend && pnpm install && cd .. +``` + +### Per-service dev + +```bash +# Backend (terminal 1) +export DATABASE_URL="mysql+asyncmy://model_platform:model_platform@127.0.0.1:3306/model_platform?charset=utf8mb4" +export STORAGE_BACKEND=s3 +export S3_ACCESS_KEY=modelplatform +export S3_SECRET_KEY=modelplatformsecret +export S3_ENDPOINT=http://127.0.0.1:9000 +# Or for local mode: +# export STORAGE_BACKEND=local +# export LOCAL_STORAGE_BASE_DIR=/data +uv run --frozen --package backend uvicorn backend.main:app --host 0.0.0.0 --port 8000 --reload + +# Schedule Executor (terminal 2) +uv run --frozen --package schedule uvicorn schedule.main:app --host 0.0.0.0 --port 8001 --reload + +# Runtime (terminal 3 — needs SYS_ADMIN, FUSE, devmode) +uv run --frozen --package runtime python -m runtime.main +``` + +### Frontend dev + +```bash +cd frontend +pnpm dev # http://localhost:5173, proxies /api to backend +pnpm typecheck +pnpm build +``` + +### Static checks + +```bash +# Python compile +uv run --frozen --package backend python -m compileall -q backend/src common/src +uv run --frozen --package schedule python -m compileall -q schedule/src +uv run --frozen --package runtime python -m compileall -q runtime/src + +# Type check (frontend) +cd frontend && pnpm typecheck && cd .. + +# Docker compose config +docker compose config --quiet +``` + +### Smoke test + +```bash +# Run the full ORM import + Settings smoke test +PYTHONPATH="backend/src:common/src" uv run --frozen --package backend python -c " +from backend.main import app +from common.config import settings +print('backend:', len(app.routes), 'routes') +print('settings ok:', settings.s3_endpoint) +" +``` + +## Common tasks + +### Add a new DAG endpoint + +1. Add the route handler in `backend/schedules.py` (DAG template) or + `backend/schedule_runs.py` (run lifecycle). +2. Validate request via `backend/schedule_schemas.py`. +3. For mutations on nodes/edges/versions: route through + `create_script_record` / `get_script_row` and apply + `require_script_modify_access` if it touches a script. +4. If it produces an outbox event, use + `add_outbox_event(session, event_type="...", producer="...", ...)`. + +### Add a new env var + +See "Adding a new env var" above. + +### Add a new MySQL table + +1. Add a model class in `common/src/common/db/models/.py`. + Include `is_deleted TINYINT(1) NOT NULL DEFAULT 0` and + `deleted_at DATETIME(3) NULL`. +2. Export it from `common/src/common/db/models/__init__.py`. +3. Generate the migration: + ```bash + uv run --frozen --package backend alembic revision --autogenerate -m "add " + ``` +4. Review the generated `migrations/versions/*.py` — Alembic may + miss comments / server defaults. Manually fix the migration. +5. Apply locally: + ```bash + uv run --frozen --package backend alembic upgrade head + ``` + +### Wire a new storage bucket + +The current 4 buckets are wired in `backend/storage_api.py:resolve_bucket`: + +```python +BUCKET_FOR_USAGE: dict[str, str] = { + "working_copy": settings.s3_workspace_bucket, + "public_script": settings.s3_workspace_bucket, + "data_resource": settings.s3_workspace_bucket, + "snapshot": settings.s3_workspace_bucket, + "version_artifact": settings.s3_version_bucket, + "run_log": settings.s3_run_log_bucket, + "run_result": settings.s3_run_log_bucket, +} +``` + +The constant `PURPOSE_BUCKETS = ("workspace", "version", "run_log", "trash")` +in `common.storage.factory` enumerates the four backends built in the +backend lifespan. To add a fifth bucket: + +1. Add the env var to `Settings` (s3 mode only): + ```python + s3__bucket: str = Field(default="", description="...") + ``` +2. Add to `.env.example` with a one-line comment. +3. Append `""` to the `PURPOSE_BUCKETS` tuple in + `common/storage/factory.py`. `build_storage_config("")` + will then automatically read `settings.s3__bucket` (s3 + mode) or use `/` (local mode). +4. Extend the `Literal` in `common/storage/schemas.py` (in + `CreateUploadRequest.usage_type`, `ServerObjectRequest.usage_type`) + to include the new value. +5. Add an entry in `BUCKET_FOR_USAGE` mapping the new `usage_type` to + the new bucket env var. +6. Pre-create the bucket (s3 mode) or subdirectory (local mode) in the + deployment. The backend no longer auto-creates buckets. + +A workspace's `artifact_bucket` column (when non-null) overrides the +default for that workspace, regardless of `usage_type`. + +### Add a new schedule node type + +`schedule/execution.py` dispatches on `script_type` in +`execute_artifact`. Add a new branch + a new `_` function. +`worker.py` does not need to change — the dispatch happens inside +`execute_artifact`. + +## Tests + +There is **no formal test suite yet** (see HANDOVER §Pending Tasks +P1). A reasonable first test surface: + +- `require_script_modify_access` (admin / owner / non-owner-unlock / + non-owner-lock): pure-function unit test, no DB. +- `validate_dag` (cycle detection + orphan detection) in + `backend/schedules.py`. +- `execute_artifact` end-to-end with mocked `content_hash` and a + real `tempfile.TemporaryDirectory`. + +Test convention: pytest with `pytest-asyncio` for `async def` +handlers. Use SQLite in-memory (or a MySQL test container) for DB +integration. Use moto for S3. + +## Troubleshooting + +### "the greenlet library is required" + +SQLAlchemy 2.0 needs `greenlet` for `engine.dispose()` in async +contexts. Add `greenlet>=3.0.0` to `common/pyproject.toml` and +`uv sync --all-packages`. (Already present in this repo.) + +### "Can't connect to MySQL server" + +Either MySQL isn't running, or the network namespace doesn't allow +`mysql:3306` resolution. Inside the Docker network, services reach +each other by service name (`mysql`, `backend`, `runtime`, +`schedule`, `s3`). + +### Jupyter routing 401s + +Inspect `docker compose logs backend` — `jupyter.py:check_notebook_is_locked` +or `load_active_membership` will return an explicit reason. Then +check the JWT (use `JWT_SECRET` from `.env`). + +### Schedule run never advances + +`schedule_runs.run_status` is stuck at `queued`. Two likely causes: +- `outbox_events` is empty (Backend's `add_outbox_event` failed — + check `add_outbox_event` in `schedule_runs.py`). +- The orchestrator's polling loop is dead. Check + `docker compose logs schedule` and look for "database event loop + failed" exceptions. + +### "AttributeError: 'SchedulerService' object has no attribute 'worker'" + +`worker` must be constructed before `orchestrator` in +`SchedulerService.__init__`, because orchestrator's dispatch table +captures `self.worker.handle_node_execute` at construction time. +See `service.py` — the order is load-bearing. + +## Style + +- Type hints everywhere (this repo uses `from __future__ import + annotations`). +- 4-space indent, double quotes, no trailing whitespace. +- Comments are technical (explain *why*, not *what*). +- Module docstrings document non-obvious invariants. Don't add + docstrings to functions whose behavior is self-evident from the + name. +- 4 levels of indentation = "this function is doing too much; split + it". (Project convention; see e.g. `execute_artifact`.) + +## See also + +- `ARCHITECTURE.md` — design diagrams +- `HANDOVER.md` — current refactor state and pending work +- `CLAUDE.md` — agent-facing conventions for the repo +- `models / __init__.py` — exhaustive list of all 26 tables +- `common/config.py` — all env vars in one place diff --git a/HANDOVER.md b/HANDOVER.md new file mode 100644 index 0000000..bb56051 --- /dev/null +++ b/HANDOVER.md @@ -0,0 +1,498 @@ +# Handover — 近期变更与待办 + +面向接手运维 / 二次开发的人。记录最近几次 commit 的动机、改动范围、未尽事项， +以及「线上有旧数据时怎么过渡」。详细架构见 `ARCHITECTURE.md`；本仓库的开发规约 +见 `DEVELOP.md`。 + +--- + +## 0. 当前交接状态（2026-08-20） + +### 0.1 代码与服务 + +- 基线代码：`ae3b6d6`（`develop`）。本次改动尚未提交，修改范围见下方「0.4」。 +- Docker Compose 项目名：`model-platform-develop14`。 +- 主入口：；浏览器通过 web/Nginx 访问后端 API。 +- 当前启用的 `.env` 已配置 MySQL 连接和存储后端。**不要把 `.env` 中的连接串、密码或 + Token 提交到 Git。** + +检查服务： + +```powershell +docker compose -p model-platform-develop14 ps +Invoke-WebRequest http://127.0.0.1:9120/api/v1/health +``` + +完整重建并启动： + +```powershell +docker compose -p model-platform-develop14 up -d --build +``` + +仅后端 Python 代码变动时，使用下面命令即可；它不会删除数据库或 `data/`： + +```powershell +docker compose -p model-platform-develop14 build backend +docker compose -p model-platform-develop14 up -d --force-recreate --no-deps backend +``` + +### 0.2 调度模块本次行为 + +调度页面位于 `frontend/app/features/schedules/`，对应 API 位于 +`backend/src/backend/schedules.py`。 + +- 删除调度方案：删除该方案的节点、连线、运行记录，以及可删除的运行日志/结果产物；若方案 + 仍有运行中的任务，后端返回 409，避免删到一半。 +- 删除节点：运行中的节点不可删除；有历史运行记录时，前端会提示“该节点有运行日志，是否一并删除？”。 + 用户确认后会删除节点、关联连线、该节点的 `schedule_node_runs` 记录和可删除日志/结果。 +- `StorageObjects.is_immutable=1` 的审计原件受存储层保护，不能物理删除；删除节点时会删除其 + 运行记录引用，使其不再在调度页面展示，但保留原件，避免再次出现 + `immutable object cannot be deleted` 并导致节点删除回滚。 +- 运行记录列表已加入轮询：有运行中任务时约 1.5 秒刷新一次，空闲时约 5 秒刷新一次；右侧刷新 + 按钮仍可手动刷新。 + +### 0.3 关键排查位置 + +| 现象 | 首先查看 | +|---|---| +| 页面请求报错 | 浏览器 F12 → Network → Fetch/XHR，查看请求 URL、状态码和响应内容 | +| API / 数据库异常 | `docker compose -p model-platform-develop14 logs --tail=200 backend` | +| Cron 未触发 | `docker compose -p model-platform-develop14 logs --tail=200 schedule` | +| 脚本执行失败 | 调度页面“运行记录”→“查看结果”→ 节点日志 | +| 容器状态或端口问题 | `docker compose -p model-platform-develop14 ps` | + +### 0.4 未提交改动清单 + +当前工作区包含以下代码改动，提交前应按需执行 typecheck / 后端测试： + +| 文件 | 作用 | +|---|---| +| `backend/src/backend/schedule_schemas.py` | 节点删除请求支持历史记录确认参数 | +| `backend/src/backend/schedules.py` | 调度/节点删除、运行中保护、日志清理与不可变产物兼容 | +| `frontend/app/services/api.ts` | 节点删除 API 参数 | +| `frontend/app/context/AuthContext.tsx` | API 参数透传 | +| `frontend/app/features/schedules/state/schedulesStore.ts` | 删除确认及删除后刷新状态 | +| `frontend/app/features/schedules/SchedulePage.tsx` | 运行记录轮询 | + +`data/` 是本地运行数据，不应作为本次代码改动一起提交。 + +--- + +## 1. 最近 8 个 commit（按时间倒序） + +### `85b2916` — fix: local_storage_base_dir（bucket 标识符语义统一：最终 fix） + +**背景**：见 §3。「删除文件存在报错 `failed to move object to trash: 'workspace'`」 +的真实根因 —— `StorageObjects.bucket_name` 写的是裸字符串（`"workspace"`）， +`app.state.object_stores` 的 key 是 local 模式拼出来的路径（`"data/workspace"`）， +两边错位导致 KeyError。 + +**改动**（1 个文件，19+/15-）： + +| 文件 | 改动 | +|---|---| +| `common/src/common/storage/factory.py` | `actual_bucket_name(purpose)` 合并 local/s3 两个分支，单一返回 `settings.s3__bucket`；`build_storage_uri` local 分支从 `Path(bucket_name).absolute()` 改为 `(Path(local_storage_base_dir) / bucket_name).resolve()`；注释 / docstring 同步重写 | + +**重要语义区分**： + +- `actual_bucket_name(purpose)` → 返回 **s3 风格 bucket 标识符**（如 `"workspace"`、 + `"run-logs"`），local 和 s3 模式返回相同字符串。这是 `app.state.object_stores` + 的 dict key，也是 `StorageObjects.bucket_name` / `UploadSessions.bucket_name` + 存储的值。 +- `build_storage_config(purpose)` → 内部仍然 `Path(local_storage_base_dir) / purpose` + 拼 `base_dir`，所以 local backend 实例的磁盘根仍是 `./data//`。这层 + 与 `bucket_name` 字段解耦，是设计上正确的。 + +### `25e563d` — update: ruff check --fix + +`ruff` 自动格式化，13 个文件，0 逻辑改动。可视为「`25e563d` 之前最近一次大改 +（`5d49ff5` / `2e070fa` / `5c5dec9` / `d855912` / `85b2916`）的格式统一」。如果 +review 时看到大量 import 重排 / 引号风格变化，都是 ruff 干的。 + +### `d855912` — fix: soft delete helper（写入端 bucket_name 修复） + +**背景**：承接 `85b2916` 的语义重写。`85b2916` 之前 `scripts.py:482` 用的是 +`settings.s3_workspace_bucket`（裸字符串）；修完 `actual_bucket_name` 之后 +该函数返回的是 s3 风格标识符，**逻辑等价**，但 `scripts.py` 调用方式必须跟着改 +才能保持一致性。`_resolve_bucket_for_usage` 的 fallback 路径有同一类 bug。 + +**改动**（2 个文件，4+/2-）： + +| 文件 | 改动 | +|---|---| +| `backend/src/backend/scripts.py` | import 增加 `actual_bucket_name`；`bucket_name = settings.s3_workspace_bucket` → `bucket_name = actual_bucket_name("workspace")` | +| `backend/src/backend/services/storage.py` | `_resolve_bucket_for_usage` fallback 从 `BUCKET_FOR_USAGE.get(usage_type, settings.s3_workspace_bucket)` 改为 `BUCKET_FOR_USAGE.get(usage_type, actual_bucket_name(USAGE_TYPE_TO_PURPOSE.get(usage_type, "workspace")))` | + +### `139f2c0` — fix: delete error（trash 路径统一 + 删除后 DB 字段更新） + +**背景**：删除文件统一改为移动到 trash 桶后，存在三个连锁 bug： + +1. **local 模式 trash 路径双重 `data`**：`trash_key = f"{item.bucket_name}/{item.object_key}"`。 + local 模式下 `bucket_name` 是绝对路径（如 `/data/workspace`），最终落到 + `data/trash/data/workspace/{ws_id}/{user_id}/...`。 +2. **s3 模式桶名 / 路径名错位**：`.env` 之前是 `S3_WORKSPACE_BUCKET=workspaces`（复数）， + 代码 default / `PURPOSE_BUCKETS` / `USAGE_TYPE_TO_PURPOSE` 全部是单数 `workspace`。 + 跨环境部署产生 `KeyError: 'workspaces'`（软删时 `object_stores[item.bucket_name]` 抛错）。 +3. **软删后 `storage_objects` 路径字段未更新**：只翻 `trash_key` / `object_status` / + `deleted_at` / `is_deleted`，`bucket_name` / `object_key` / `storage_uri` 仍指向源， + 与物理位置不一致，`purge_trash_object` 必须读旧 `trash_key` 才能定位 trash 文件。 + +**改动**（5 个文件，44+/24-）： + +| 文件 | 改动 | +|---|---| +| `backend/src/backend/services/storage.py` | `trash_key = f"{source_purpose}/{item.object_key}"`；软删成功后更新 `bucket_name` / `object_key` / `storage_uri` 指向 trash 位置 | +| `backend/src/backend/storage_api.py` | `restore_object` 用 `item.object_key` 直接定位 trash，按 `/` 拆 `source_purpose`，写回源桶，更新 DB 字段指回 source；`purge_trash_object` 用 `(actual_bucket_name("trash"), item.object_key)` 删除，加 `bucket_name != trash` 的 409 防御 | +| `common/src/common/storage/factory.py` | 引入 `USAGE_TYPE_TO_PURPOSE` 字典（替代 `storage_api.py` 的本地定义） | +| `common/src/common/storage/__init__.py` | 导出 `USAGE_TYPE_TO_PURPOSE` | +| `.env.example` | `S3_WORKSPACE_BUCKET=workspaces` → `workspace`、`S3_VERSION_BUCKET=versions` → `version`、`S3_RUN_LOG_BUCKET=run-logs` → `run-log`（统一单数） | + +**注意点**： + +- `workspaces_root()` 函数名 / `WORKSPACES_ROOT` 环境变量名不动（这是 path 概念，不是 bucket 名）。 +- `Workspaces` 数据库表名 / `auth.py` 的 `"workspaces"` JSON key（API 响应）不动（这些不是桶名）。 +- `visibility="workspace"` 是 ACL Literal，不动。 + +### `2e070fa` — fix: pre check（前端 script 创建/上传前置校验） + +`CreateScriptModal.tsx`（+25 行）— 文件名 / workspace_id / 类型前置校验。 +`ScriptsPage.tsx`（+1）/ `scriptWorkspaceStore.ts`（+19/-10）— store action 增加 +`precheck` 步骤，提前拦截无效请求。 + +### `5c5dec9` — fix: allows_same_name_different_parent + +允许同名 script 出现在不同父目录下。`scripts.py`（+32/-?）、`services/storage.py`、 +`storage_api.py` 微调；`test_scripts.py` 新增 57 行回归测试。 + +### `5d49ff5` — fix: file upload error（大重写） + +修复 file upload 路径上一连串 schema / API / 上传流程的不一致。改动面最大 +（14 个文件，872+/604-）： + +- 新增索引 / 调整唯一约束（去掉了 scripts 和 data_resources 的旧 unique index） +- 把多个小 migration 合并进 `e1f2a3b4c5d6_rebuild_baseline.py` 一份大 baseline +- `resources.py` / `scripts.py` 上传流程对齐 +- `test_resources.py` / `test_scripts.py` 大量回归 +- `API.md` 增加 3 行 + +### `c65d6dc` — fix: build error + +`backend/Dockerfile` 单行修复（镜像构建报错）。 + +--- + +## 2. Trash 修复详解（commit `139f2c0`） + +### 2.1 修复后的语义 + +- **trash_key**：`f"{source_purpose}/{item.object_key}"`。`source_purpose` 来自 + `USAGE_TYPE_TO_PURPOSE[item.usage_type]`，与 `item.bucket_name` 解耦 — + 避免 local 模式绝对路径泄漏 + 命名错位风险。 +- **软删后 `storage_objects` 行的 7 个字段同步**： + + | 字段 | 软删前 | 软删后 | + |---|---|---| + | `bucket_name` | 源桶 | trash 桶 | + | `object_key` | 源 key | trash key | + | `trash_key` | NULL | trash key | + | `storage_uri` | 源 URI | trash URI（`build_storage_uri` 重建） | + | `object_status` | `available` | `deleted` | + | `is_deleted` | `0` | `1` | + | `deleted_at` | NULL | `` | + +- **物理文件落点**： + + | 模式 | 源文件（被删） | trash 文件（新落） | + |---|---|---| + | S3 | `s3:///` | `s3://trash//` | + | local | `//` | `/trash//` | + + `source_purpose` ∈ `{workspace, version, run_log}`。 + +### 2.2 `restore_object` 反向流程 + +```python +data = await object_stores[trash_bucket].get(item.object_key) # 从 trash 读 +source_purpose, _, source_key = item.object_key.partition("/") # 拆 source_purpose +target_bucket = actual_bucket_name(source_purpose) # 还原目标桶 +await object_stores[target_bucket].put(source_key, data) # 写回源 + +item.bucket_name = target_bucket +item.object_key = source_key +item.storage_uri = build_storage_uri(target_bucket, source_key) +# trash_key 保留不动 → 未来 reaper 可据此清理 trash 里的 orphan 副本 +item.object_status = "available" +item.deleted_at = None +``` + +**trash 文件不会被 restore 主动删除**（保留现有安全语义：restore 失败时 +数据仍在 trash 里有兜底）。未实现的 reaper 未来可扫 +`object_status='available' AND trash_key IS NOT NULL` 清理 orphan。 + +### 2.3 `purge_trash_object` 防御 + +加 409 防御：`if item.bucket_name != actual_bucket_name("trash"): raise 409`。 +避免误删源数据（用 `item.object_key` 直接删 trash 文件，不再读旧 `trash_key`）。 + +### 2.4 USAGE_TYPE_TO_PURPOSE 提升到 common 层 + +`backend/storage_api.py` 与 `backend/services/storage.py` 都依赖这个映射。 +提到 `common/storage/factory.py` 后两边都 `from common.storage import USAGE_TYPE_TO_PURPOSE`， +避免互相 import 循环。 + +--- + +## 3. Bucket 标识符语义统一（commits `d855912` + `85b2916`） + +### 3.1 错位现象 + +`actual_bucket_name(purpose)` 在 `85b2916` 之前有两个分支： + +```python +def actual_bucket_name(purpose: str) -> str: + if settings.storage_backend == "local": + return str(Path(settings.local_storage_base_dir) / purpose) # → "data/workspace" + return getattr(settings, f"s3_{purpose}_bucket") # → "workspace" +``` + +这条函数被三个地方用： + +| 消费方 | 期望的值 | 实际拿到的（local 模式，修前） | +|---|---|---| +| `app.state.object_stores` dict key | 与 DB `bucket_name` 一致 | `"data/workspace"` | +| `StorageObjects.bucket_name` 写入 | 与 dict key 一致 | `"data/workspace"`（来自 `scripts.py:482` 的旧 `settings.s3_workspace_bucket`）| +| `UploadSessions.bucket_name` 写入 | 与 dict key 一致 | `settings.s3_workspace_bucket` = `"workspace"`（来自 `_resolve_bucket_for_usage` 的 fallback）| + +表里第三行就是本次出问题的根本：`UploadSessions.bucket_name` 写的是裸字符串 `"workspace"`，但 +`app.state.object_stores` 的 key 是路径 `"data/workspace"`。两者 lookup 时必然 KeyError。 +**注意**：本部署所有 DB row 的 `bucket_name` 都是裸字符串（`"workspace"`），跟 +`object_stores` dict key 错位 — 软删时 `object_stores["workspace"]` 直接抛 +`KeyError: 'workspace'`，被 `soft_delete_object` 的 try/except 包成 +`HTTPException(502, "failed to move object to trash: 'workspace'")`。 + +`scripts.py:482` 写出的 row 反而是错的 `"data/workspace"`，**但**所有现存 row 写于本会话 +之前的旧版本，所以 DB 里没有这种 row。`scripts.py` 是用户从 Jupyter 创建新脚本时触发的 +路径，本次会话没有产生新 row。 + +### 3.2 `85b2916` 的最终语义 + +`actual_bucket_name(purpose)` 合并为单一返回： + +```python +def actual_bucket_name(purpose: str) -> str: + from common.config import settings # 延迟 import 避免循环 + return getattr(settings, f"s3_{purpose}_bucket") +``` + +返回值约定（local / s3 两种模式相同）： + +| purpose | 返回（与 `.env` 对齐） | +|---|---| +| `workspace` | `settings.s3_workspace_bucket` | +| `version` | `settings.s3_version_bucket` | +| `run_log` | `settings.s3_run_log_bucket` | +| `trash` | `settings.s3_trash_bucket` | + +这是**纯标识符**，不是文件系统路径。 + +`build_storage_config(purpose)` 内部仍然走 `Path(local_storage_base_dir) / purpose` 拼 +`base_dir`，所以 local backend 实例的磁盘根目录没变（仍是 `./data//`）。 +**这里有个两层抽象**： + +- `bucket_name` 字段（DB 存储 / dict key / URI 字符串）— s3 风格标识符，`actual_bucket_name` 负责 +- `base_dir`（local backend 实际写入的磁盘根）— 路径，由 `build_storage_config` 内部负责 + +两层解耦后，修改 `actual_bucket_name` 不会移动任何字节。 + +### 3.3 `build_storage_uri` 配套修复 + +旧版本 local 分支： + +```python +absolute_path = Path(bucket_name).absolute().as_posix() # 假设 bucket_name 是路径 +``` + +修后： + +```python +absolute_path = (Path(settings.local_storage_base_dir) / bucket_name).resolve().as_posix() +``` + +因为 `bucket_name` 不再是路径，必须显式拼上 `local_storage_base_dir`。 + +### 3.4 写入端对齐（commit `d855912`） + +| 文件 | 改动 | +|---|---| +| `backend/src/backend/scripts.py:482` | `bucket_name = settings.s3_workspace_bucket` → `bucket_name = actual_bucket_name("workspace")` | +| `backend/src/backend/services/storage.py:_resolve_bucket_for_usage` | fallback 从 `settings.s3_workspace_bucket` 改为 `actual_bucket_name(USAGE_TYPE_TO_PURPOSE.get(usage_type, "workspace"))` | + +**为什么 `d855912` 是必要的**：`85b2916` 改了 `actual_bucket_name` 的返回语义， +但 `scripts.py:482` 直接读 `settings.s3_workspace_bucket`、`_resolve_bucket_for_usage` +直接读 `settings.s3_workspace_bucket` —— 这两处必须改成 `actual_bucket_name(...)` ， +否则它们绕过了新语义，写出的 row 又是裸字符串，新 dict key 是 s3 风格标识符， +两边重新错位。**两 commit 必须一起部署**，单独 `85b2916` 会让 dict key 是 `"workspace"， +DB 仍是 `settings.s3_workspace_bucket` 写出的 `"workspace"` —— 看似 OK 但万一某天 +`.env` 改了 `S3_WORKSPACE_BUCKET=workspaces`，就立刻重新炸。 + +--- + +## 4. 线上数据兼容（重要） + +### 4.1 当前部署（local 模式 + 单数桶名） + +`d855912` + `85b2916` 之后，**新代码对 DB 里已有的旧行完全兼容**： +现有 7 个 `bucket_name='workspace'` 的 row 直接命中 dict key，无需迁移。 +重启 backend 后就能正常软删。 + +### 4.2 历史异构部署（如果你接手的环境曾跑过下面任一形态） + +#### 4.2.1 `.env` 曾是复数桶名（`S3_WORKSPACE_BUCKET=workspaces` 等） + +如果该部署曾用复数 `.env` 上传过文件，`storage_objects.bucket_name` 列里会有 +复数残留。新代码 dict key 是单数，访问旧行仍会 `KeyError: 'workspaces'`。 +**部署新代码前必须先跑一次数据修复**（先 SELECT 看数量）： + +```sql +SELECT storage_backend, bucket_name, COUNT(*) +FROM storage_objects +WHERE object_status = 'available' +GROUP BY storage_backend, bucket_name +ORDER BY storage_backend, bucket_name; + +UPDATE storage_objects +SET bucket_name = 'workspace' +WHERE bucket_name = 'workspaces' + AND object_status = 'available'; + +-- 同样模式处理 version / run-log / trash 残留 +``` + +#### 4.2.2 `bucket_name` 残留 `Path` 形式（`/data/workspace` 等） + +如果历史版本曾把 `actual_bucket_name("workspace")` 的路径形式（`/data/workspace`、 +`./data/workspace`、`data/workspace`）写进 `bucket_name` 列，需要归一化成裸字符串： + +```sql +UPDATE storage_objects +SET bucket_name = 'workspace' +WHERE bucket_name LIKE '%/workspace' AND storage_backend = 'local' + AND object_status IN ('available', 'deleted'); +``` + +**只改 `available` / `deleted` 行**；正在上传中的 row（`object_status='uploading'`） +不要碰，会丢数据。 + +#### 4.2.3 异构 backend 的旧行 + +如果某个 row 的 `storage_backend` 与当前部署不一致（例如旧 row 是 `s3`、当前部署是 +`local`），物理文件位置在旧 backend 那边，**软删时只翻 DB 状态，物理文件没移动**。 +处理：人工导出后清理，或者重新上传。 + +### 4.3 监控（建议加） + +`storage_objects` 行 `bucket_name` 与 `actual_bucket_name(purpose)` 不一致的比例 +（按 `usage_type` 分组 COUNT）。生产环境应长期趋近于 0。 + +```sql +SELECT usage_type, bucket_name, COUNT(*) +FROM storage_objects +WHERE object_status = 'available' +GROUP BY usage_type, bucket_name +ORDER BY usage_type, bucket_name; +``` + +--- + +## 5. 本地验收清单 + +```bash +# 1. 语法 +uv run python -m compileall common/src backend/src runtime/src schedule/src + +# 2. 测试 +uv run --package backend pytest backend/tests -q +# 期望：30 passed + +# 3. bucket 标识符一致性（s3 风格字符串，不带路径） +grep -rn "bucket_name = settings\." \ + --include='*.py' backend/src common/src 2>/dev/null +# 期望：零结果 — bucket_name 全部走 actual_bucket_name(...) + +# 4. 单复数对齐 +grep -n 'S3_WORKSPACE_BUCKET\|S3_VERSION_BUCKET\|S3_RUN_LOG_BUCKET\|S3_TRASH_BUCKET' \ + .env.example +# 期望：所有桶名与代码 default 一致（参考 .env.example 注释） + +# 5. 命名残留扫 +grep -rn '"workspaces"\|"versions"\|"run-logs"' \ + --include='*.py' --include='*.yml' --include='*.env*' \ + backend/src common/src docker-compose.yml .env .env.example 2>/dev/null +# 期望：除 `__tablename__ = "workspaces"`（DB 表名）和 `auth.py` 的 JSON key 外，零结果 +``` + +--- + +## 6. 回归测试覆盖 + +| 场景 | 触发方式 | 期望 | +|---|---|---| +| 同名 script 不同父目录 | `test_soft_delete_object_sets_is_deleted` 旁边的新测试 | 两条 row 同名共存 | +| 软删 → 物理文件落到 trash 桶的 `source_purpose/{key}` | 手工：上传 → 删 → `mc ls s3/trash/` | 文件在；源桶里已删 | +| 软删后 `storage_objects` 7 字段同步 | 软删后 `SELECT * FROM storage_objects WHERE storage_object_id=...` | 见 §2.1 表格 | +| `restore_object` 把 trash 文件写回源桶 | 软删 → restore → 查源桶 | 源桶恢复；trash 文件保留（orphan） | +| `purge_trash_object` 拒绝误删源数据 | 手工：把 `bucket_name` 改成源桶后调 purge | 409 | +| local 模式软删走通 | local 部署下软删一个 script | 不抛 `KeyError: 'workspace'`；trash 落到 `./data/trash//` | +| `actual_bucket_name` 两种模式返回相同字符串 | 写测试或 trace | s3 模式 = local 模式（与 `.env` 对齐）| + +--- + +## 7. 监控项（建议加） + +1. `storage_objects` 行 `bucket_name` 与 `actual_bucket_name(purpose)` 不一致的比例 + （按 `usage_type` 分组 COUNT）。生产环境应长期趋近于 0。 +2. `object_status='deleted'` 但 `bucket_name != actual_bucket_name('trash')` 的行数 — + 应为 0（软删失败回滚残留）。 +3. trash 桶目录大小 vs `object_status='deleted'` 行数（孤儿文件比例）。 + 没实现 reaper 前这个会单调上升。 + +--- + +## 8. 已知未实现项 + +- **Trash reaper**：`s3_trash_retention_days` 已声明，但代码没有扫表清理。 + `restore_object` 故意保留 trash 文件作 orphan，等 reaper 兜底。 + 第一次实现建议在 `schedule/src/schedule/` 加一个 tick job： + ```sql + SELECT storage_object_id FROM storage_objects + WHERE deleted_at < UTC_TIMESTAMP() - INTERVAL :retention_days DAY + AND object_status = 'deleted' + LIMIT 100; + ``` + 然后调 `/internal/v1/admin/trash/purge`。 +- **跨 backend trash 兼容**：当前只在 `item.storage_backend == settings.storage_backend` + 的行上做物理移动。异构 backend 的旧行（少）软删时只翻 DB 状态，trash 文件实际没移动。 + 远程 8.153.151.51:8890 应该都是同一 backend，目前无影响。 +- **历史 trash_key 兼容读**：新代码用 `item.object_key` 读 trash，**不**回退到 + `item.trash_key`。如果迁移前有按旧 `trash_key = f"{bucket_name}/{object_key}"` + 写入的已删行，`object_key` 还是源 key，restore 会找不到 trash 文件（404）。 + 处理：在 `restore_object` 加 fallback — 先按 `item.object_key` 试，失败再按 `item.trash_key` + 试一次（兼容窗口期）。建议在 reaper 上线后删除 fallback。 +- **DB 测试种子数据**：`storage_objects` 表里有 `workspace_id='00000000000000000000000002'` + 的行（全 0 + 02，不是有效 ULID 格式），对应的 `object_key` 在磁盘 `data/workspace/` 下 + 找不到对应文件（磁盘上是真实 workspace ULID 子目录 `00000000000BM630VT9ARVFZPC`、 + `01KZR60SWWTXXM0KNZ2GJMT36A`）。这些是测试种子。修复 bucket 标识符后，软删会从 KeyError + 变成 StorageNotFoundError —— KeyError 解决后真实地暴露了「字节缺失」。处理：清掉这些 + 测试 row，或补齐磁盘文件。 + +--- + +## 9. 仍未合并到 develop 的本地改动 + +```text +M frontend/vite.config.ts (4+/2-) +?? data/ (untracked — 本地 docker volume 残留，可忽略) +``` + +`frontend/vite.config.ts` 的改动不在本次 trash / bucket 修复范围内，等下次 commit 一起提。 +`data/` 是 `STORAGE_BACKEND=local` 的对象存储本地根，git ignore 不应跟踪。 diff --git a/Makefile b/Makefile index 6ff8fbf..8163249 100644 --- a/Makefile +++ b/Makefile @@ -1,28 +1,20 @@ -# ==================== 环境变量配置 ==================== -# 设置默认值（如果命令行没传，就用这里的默认路径） -WORKSPACES_ROOT ?= ./test/workspaces -PUBLIC_BASE_URL ?= http://192.168.139.3 -RUNTIME_HOST ?= 0.0.0.0 -RUNTIME_PORT ?= 8001 -RUNTIME_BASE_URL ?= http://127.0.0.1:8001 -BACKEND_HOST ?= 0.0.0.0 -BACKEND_PORT ?= 8000 +.PHONY: sync backend runtime schedule migrate up down -# 将变量导出给 Makefile 启动的所有子进程 -export WORKSPACES_ROOT -export PUBLIC_BASE_URL -export RUNTIME_BASE_URL - -.PHONY: runtime backend - -runtime: - uv run --package runtime uvicorn runtime.main:app --host $(RUNTIME_HOST) --port $(RUNTIME_PORT) - -runtime-dev: - uv run --package runtime uvicorn runtime.main:app --host $(RUNTIME_HOST) --port $(RUNTIME_PORT) --reload +sync: + uv sync --all-packages backend: - uv run --package backend uvicorn backend.main:app --host $(BACKEND_HOST) --port $(BACKEND_PORT) + uv run --package backend --env-file .env uvicorn backend.main:app --host 0.0.0.0 --port 8010 --reload -backend-dev: - uv run --package backend uvicorn backend.main:app --host $(BACKEND_HOST) --port $(BACKEND_PORT) --reload \ No newline at end of file +runtime: + uv run --package runtime --env-file .env uvicorn runtime.main:app --host 0.0.0.0 --port 8012 --reload + +schedule: + uv run --package schedule --env-file .env uvicorn schedule.main:app --host 0.0.0.0 --port 8013 --reload + + +up: + docker compose up -d --build + +down: + docker compose down diff --git a/README.md b/README.md new file mode 100644 index 0000000..c09cf66 --- /dev/null +++ b/README.md @@ -0,0 +1,208 @@ +# Model Platform + +自托管的 **Jupyter 模型开发平台**：交互式 workspace、DAG 调度、对象存储工件、 +按 workspace 隔离的运行时 — 全部经一个 Nginx 网关对外。 + +> 技术栈：React Router SPA · FastAPI · APScheduler · MySQL · S3 兼容存储 +> （或本地文件系统，`STORAGE_BACKEND=local`）· 共享 Jupyter · rclone FUSE 挂载（仅 s3 模式） +> 单一入口（Nginx :80）；其他服务只在 Docker 内网互通。 + +## 它做什么 + +| 能力 | 位置 | +|---|---| +| workspace 内 notebook 编辑，行级锁 | `backend/jupyter.py` + `scripts.is_locked` | +| Jupyter 鉴权路由（浏览器永远拿不到 runtime token） | `nginx/default.conf` + `auth_request` + `backend/jupyter.py` | +| notebook / script / version / run_log 的对象存储（s3 / local 二选一） | `common/storage/` + `backend/scripts.py` | +| DAG 调度：节点、边、cron、手动触发、重试、快照 | `backend/schedules.py` + `backend/schedule_runs.py` + `schedule/`（5 个模块） | +| DAG 执行走 MySQL Outbox（无 Redis，无进程内队列） | `schedule/orchestrator.py` + `schedule/worker.py` | +| 每个 workspace 一个 Jupyter 子进程池，配 asyncio 锁 | `runtime/process.py` | +| runtime 内 rclone FUSE 把 workspace 桶挂上来（s3 模式） | `runtime/mount.py` | +| 仅 MySQL 持久化（26 张表，软删除，无外键） | `common/db/models/` | + +## 架构一览 + +``` + ┌────────────────────┐ + │ Browser (SPA) │ + └─────────┬──────────┘ + │ HTTPS / WS + ┌─────────▼──────────┐ + │ Nginx (only :80) │ ← templates/default.conf + │ /api/ /jupyter/ /storage/ + └────┬───────┬──────┘ + │ │ + ┌──────────────┘ └─────────────┐ + ▼ ▼ + ┌──────────────────┐ ┌──────────────────────┐ + │ FastAPI Backend │ │ Runtime (Jupyter) │ + │ + /internal/v1 │ control │ - rclone FUSE mount │ (P0-1) + │ /objects │ token-Auth │ │ + │ (storage) ├──────────────►│ - subprocess pool │ + │ - DAG CRUD │ │ (per workspace) │ + │ - script CRUD │ └──────────┬───────────┘ + │ - auth_request │ │ FUSE / shared vol + │ - /api/v1/... │ ▼ + └────┬──────┬──────┘ ┌──────────────────────┐ + │ │ │ Object storage │ + │ └──────── HTTP ───────►│ (s3: S3 service / │ + ▼ │ local: shared vol) │ + ┌────────────┐ │ 4 buckets per usage │ + │ MySQL │◄───────── poll ─────│ │ + │ - 26 tbls │ └──────────────────────┘ + │ - outbox │ + │ - jobstore │ + └────┬───────┘ + ▲ + │ outbox poll + ┌────┴──────────────────────────┐ + │ Schedule Executor │ + │ - CronScheduler (APScheduler) │ + │ - DispatchOrchestrator │ + │ - NodeExecutor (worker) │ + │ - SchedulerService (facade) │ + └───────────────────────────────┘ +``` + +对象存储通过 `STORAGE_BACKEND`（s3 | local）二选一。s3 模式下 4 个 purpose 命名桶 +（`workspace` / `version` / `run-log` / `trash`）是独立的 S3 bucket；local 模式下 +是 `LOCAL_STORAGE_BASE_DIR` 的子目录，通过 Docker volume `local-storage` 共享。 +详见 `DEVELOP.md` §存储。 + +详细设计见 `ARCHITECTURE.md`。实现的偏离和近期重构记录在 `HANDOVER.md`。 + +## 目录结构 + +```text +frontend/ React Router SPA +backend/ FastAPI：公开 API + 内部存储 API +runtime/ Jupyter 子进程管理 + rclone FUSE +schedule/ DAG 调度器（5 模块：context/scheduler/ + orchestrator/worker/service） +common/ 配置、SQLAlchemy 模型、存储 SDK、 + outbox 事件、jobstore +migrations/ Alembic 基线 + 各特性 migration +nginx/ （仅概念 — 见下方「容器」一节） +scripts/ nginx-entrypoint.sh（模板渲染） +docker-compose.yml 4 服务 — web / backend / runtime / schedule +default.conf Nginx 模板（挂载，启动时渲染） +.env.example common.config.Settings 消费的所有环境变量 +``` + +## 容器 + +| 服务 | 镜像 | 暴露 | 用途 | +|---|---|---|---| +| `web` | `nginx:alpine` | 宿主机 `:8888` → `:80` | SPA、`/api/` 反向代理、`/jupyter/{ws}/` auth_request 代理、`/storage/` S3 直通（仅 s3 模式） | +| `backend` | `Dockerfile` | 仅内网 | DAG CRUD、script CRUD、schedule 触发、`/api/v1/auth/jupyter`、`/internal/v1/objects` 服务间 RPC(共享 `INTERNAL_SERVICE_TOKEN` 鉴权,P0-1)| +| `runtime` | `Dockerfile` | 仅内网 | 每个 workspace 一个 Jupyter 子进程池、rclone FUSE 挂载 `workspace` 桶（s3 模式） | +| `schedule` | `Dockerfile` | 仅内网 | cron tick + DAG 执行（轮询 MySQL Outbox） | + +架构**故意只暴露一个宿主机端口**(网关);其他服务都在 Docker 内网。 +这一点在 `docker-compose.yml` 里强制执行 — backend / runtime / schedule 都没有 `ports:`。在 P0-1 之前,backend 与 runtime 曾短暂地把 `8891` / `8892` 映射到宿主机;此映射已被删除,改用 `INTERNAL_SERVICE_TOKEN` 头对 `/internal/v1/*` 做服务间鉴权,见 `API.md §9`。 + +## 快速启动 + +```bash +cp .env.example .env +# 编辑 .env — 至少改 MYSQL 密码，以及（s3 模式下）S3 凭据。 + +# 静态检查 +uv sync --all-packages +uv run --frozen --package backend python -m compileall -q backend/src common/src +uv run --frozen --package schedule python -m compileall -q schedule/src +uv run --frozen --package runtime python -m compileall -q runtime/src + +# 应用 schema +uv run --frozen --package backend alembic upgrade head + +# 启动整套服务 +docker compose config # 校验 +docker compose up -d --build +docker compose ps +``` + +访问 `http://localhost:8888`。 + +### 日志 + +```bash +docker compose logs -f backend +docker compose logs -f schedule +docker compose logs -f runtime +``` + +### 停服（保留 MySQL + S3 / local-storage 数据卷） + +```bash +docker compose down +``` + +### 抹数据 + +```bash +docker compose down -v +``` + +## 配置 + +所有环境变量在 `common/src/common/config.py` 里用 pydantic-settings 的 `Settings` +类一次性声明，外面套一层 `@lru_cache` 单例。新增环境变量： + +1. 在 `common/src/common/config.py` 的 `Settings` 里加字段（带合理 default，使 dev 启动不需要设） +2. 在 `.env.example` 加一行带注释 +3. 调用点用 `settings.`，永远不要用 `os.environ["..."]` + +完整环境变量列表和含义见 `DEVELOP.md`。 + +## 存储布局 + +4 个 purpose 命名桶。从 `StorageObjects.usage_type` 到桶的映射由 +**单一入口**（`backend/storage_api.py:resolve_bucket`）决定： + +| `usage_type` | 桶（环境变量） | 默认名 | +|---|---|---| +| `working_copy`、`public_script`、`data_resource`、`snapshot` | `S3_WORKSPACE_BUCKET` | `workspace` | +| `version_artifact` | `S3_VERSION_BUCKET` | `version` | +| `run_log`、`run_result` | `S3_RUN_LOG_BUCKET` | `run-log` | +| （软删除目标） | `S3_TRASH_BUCKET` | `trash` | + +`STORAGE_BACKEND=s3` 模式下是 4 个独立 S3 桶。`STORAGE_BACKEND=local` 模式下 +是 `LOCAL_STORAGE_BASE_DIR`（默认 `/data`）下的 4 个子目录： + +``` +/data/ +├── workspace/ # S3_WORKSPACE_BUCKET +├── version/ # S3_VERSION_BUCKET +├── run_log/ # S3_RUN_LOG_BUCKET +└── trash/ # S3_TRASH_BUCKET +``` + +某个 workspace 的 `artifact_bucket` 列（非 NULL 时）覆盖该 workspace 的默认桶， +无视 `usage_type` — 适合把付费客户隔离到专属桶。 + +对象 key 是两层扁平路径 — `workspace_id` 加服务端签发的 `ulid`： + +``` +//{.} +``` + +文件名、扩展名、MIME、逻辑路径都放在 `StorageObjects` 和 `Scripts` 行里，不进 +object key — 重新组织存储不需要重写数据库。 + +Backend 代码从不写容器本地文件系统（`STORAGE_BACKEND=local` 模式除外，那里共享 +`local-storage` volume 就是规范存储）。Schedule Executor 在 `tempfile.TemporaryDirectory()` +里暂存节点工件（自动清理）。只有 `runtime` 容器保留宿主 volume — s3 模式下 rclone FUSE +挂载需要；local 模式下是 no-op 透传。 + +## 文档 + +- `README.md`（本文）— 快速导读 +- `ARCHITECTURE.md` — 设计图 + 简化历史 +- `HANDOVER.md` — 实现偏离、近期重构、待办事项 +- `DEVELOP.md` — 开发指南（环境变量、代码规约、常用操作） +- `CLAUDE.md` — agent 面向的本仓库规约 + +## 许可 + +内部。 \ No newline at end of file diff --git a/REFACTOR_NOTES.md b/REFACTOR_NOTES.md new file mode 100644 index 0000000..9eb69f2 --- /dev/null +++ b/REFACTOR_NOTES.md @@ -0,0 +1,61 @@ +# 本次重构说明 + +## 已完成 + +### 前端 + +- 将原 `frontend/src` 页面迁入 React Router SPA 结构； +- 主入口改为 `app/routes/platform.tsx`； +- 拆分为 `components / features / routes / services / styles`； +- 保留脚本管理、目录管理、Jupyter 编辑、版本发布、调度画布、运行记录和系统管理功能； +- 将原 Hash 页面切换改为 React Router 路径：`/`、`/scripts`、`/schedules`、`/system`； +- Nginx 使用 SPA fallback，刷新子路径不会 404。 + +### 后端 + +- 合并 Platform API 和 Storage API 到同一个 `backend` 进程； +- 删除 Redis 容器、依赖、Streams 消费和锁实现； +- 文件锁改为 MySQL `edit_sessions` 租约； +- Jupyter 短期票据由单 Runtime 进程内存保存； +- Schedule Executor 内置 APScheduler，并使用 MySQL `apscheduler_jobs`； +- Backend 创建运行和 Outbox 后，通过内部 HTTP 尝试立即推送； +- Schedule Executor 轮询 MySQL Outbox 作为失败兜底； +- 增加迁移 `20260730_0004`，兼容旧数据库的 `redis_lock_key -> lock_key`。 + +### Docker + +最终容器： + +```text +mysql +s3 # 外部 S3-兼容服务（MinIO/RustFS/SeaweedFS/...），由运维在 compose 外启动 +jupyter # 注释保留；当前实现未在 compose 启此独立容器 +migrate（一次性） +backend +runtime +schedule +gateway +``` + +> 补充说明：`STORAGE_BACKEND=local` 模式下不需要外部 S3 服务，backend +> 与 runtime 共享 docker 卷 `local-storage`，挂载到 +> `LOCAL_STORAGE_BASE_DIR`（默认 `/data/storage`）；runtime 也跳过 +> rclone FUSE 挂载（见 `runtime/src/runtime/mount.py`）。 + +## 已执行检查 + +- Python 全项目 `compileall` 通过； +- SQLAlchemy 模型导入通过，共加载 26 张表； +- Alembic `upgrade head --sql` 离线生成通过； +- `docker-compose.yml` YAML 解析通过； +- 前端 feature、route 和 root 文件 TypeScript 静态检查通过。 + +## 未在当前沙箱执行 + +当前执行环境没有 Docker 命令，并且无法连接 npm/PyPI，因此没有在沙箱内完成： + +- `docker compose up --build`； +- 正式 `pnpm install && pnpm build`； +- Python 依赖在线安装后的集成测试。 + +请在安装了 Docker Desktop且网络可访问依赖仓库的 Windows 电脑上执行根目录 README 中的启动命令。 diff --git a/alembic.ini b/alembic.ini new file mode 100644 index 0000000..a312124 --- /dev/null +++ b/alembic.ini @@ -0,0 +1,41 @@ +[alembic] +script_location = %(here)s/migrations +prepend_sys_path = . +path_separator = os + +# The real connection URL must be injected via DATABASE_URL; never store credentials here. +sqlalchemy.url = driver://user:pass@localhost/dbname + +[loggers] +keys = root,sqlalchemy,alembic + +[handlers] +keys = console + +[formatters] +keys = generic + +[logger_root] +level = WARN +handlers = console +qualname = + +[logger_sqlalchemy] +level = WARN +handlers = +qualname = sqlalchemy.engine + +[logger_alembic] +level = INFO +handlers = +qualname = alembic + +[handler_console] +class = StreamHandler +args = (sys.stderr,) +level = NOTSET +formatter = generic + +[formatter_generic] +format = %(levelname)-5.5s [%(name)s] %(message)s +datefmt = %H:%M:%S diff --git a/backend/Dockerfile b/backend/Dockerfile index 6e57073..85b5977 100644 --- a/backend/Dockerfile +++ b/backend/Dockerfile @@ -1,22 +1,21 @@ -# backend/Dockerfile -FROM python:3.12-slim +FROM python-base:3.12 -WORKDIR /app +ENV UV_HTTP_TIMEOUT=300 +ENV UV_DEFAULT_INDEX=https://pypi.tuna.tsinghua.edu.cn/simple/ -# 安装 uv -COPY --from=ghcr.io/astral-sh/uv:latest /uv /bin/uv -# 1. 拷贝根目录配置与 lock 文件 +COPY common/pyproject.toml common/pyproject.toml +COPY backend/pyproject.toml backend/pyproject.toml COPY pyproject.toml uv.lock ./ +RUN uv sync --no-dev --package backend --no-install-workspace + -# 2. 拷贝 common 模块与 backend 模块 COPY common ./common COPY backend ./backend +COPY alembic.ini ./ +COPY migrations ./migrations +RUN uv sync --frozen --no-dev --package backend -# 3. 安装 backend 及其所有依赖 (包括本地 common) -RUN UV_DEFAULT_INDEX="https://pypi.tuna.tsinghua.edu.cn/simple/" uv sync --frozen --no-dev --no-editable --package backend EXPOSE 8000 - -# 4. 启动服务 (通过 uv run 指定运行 backend 包) -CMD ["uv", "run", "--package", "backend", "uvicorn", "backend.main:app", "--host", "0.0.0.0", "--port", "8000"] \ No newline at end of file +CMD ["uv", "run", "--frozen", "--no-dev", "--package", "backend", "gunicorn", "--config", "backend/gunicorn.conf.py", "backend.main:app"] diff --git a/backend/README.md b/backend/README.md index e69de29..5e943a5 100644 --- a/backend/README.md +++ b/backend/README.md @@ -0,0 +1,8 @@ +# Backend + +统一 FastAPI 管理服务。包含用户、Workspace、脚本、稳定版本、调度定义、 +立即运行以及 S3 对象接口。原 `platform_api` 与 `storage_api` 已在此 +模块合并，外部 REST 契约保持不变。 + +底层走的是 `common.storage.AsyncStorageBackend` 抽象，按 +`settings.storage_backend` 切换 s3 / local 两种实现。 \ No newline at end of file diff --git a/backend/gunicorn.conf.py b/backend/gunicorn.conf.py new file mode 100644 index 0000000..a326fea --- /dev/null +++ b/backend/gunicorn.conf.py @@ -0,0 +1,14 @@ +"""Gunicorn configuration for the backend service. + +All values can be overridden with GUNICORN_* environment variables. +""" + +import os + +bind = os.getenv("GUNICORN_BIND", "0.0.0.0:8000") +workers = int(os.getenv("GUNICORN_WORKERS", "1")) +worker_class = "uvicorn.workers.UvicornWorker" +timeout = int(os.getenv("GUNICORN_TIMEOUT", "120")) +graceful_timeout = int(os.getenv("GUNICORN_GRACEFUL_TIMEOUT", "30")) +accesslog = "-" +errorlog = "-" diff --git a/backend/pyproject.toml b/backend/pyproject.toml index 88165c2..0161f9a 100644 --- a/backend/pyproject.toml +++ b/backend/pyproject.toml @@ -1,32 +1,44 @@ [project] name = "backend" -version = "0.1.0" -description = "Add your description here" -readme = "README.md" -authors = [ - { name = "tao.chen", email = "93983997+taochen-ct@users.noreply.github.com" } -] +version = "0.2.0" requires-python = ">=3.12" dependencies = [ - "boto3>=1.43.57", - "fastapi>=0.140.0", - "httpx>=0.28.1", - "loguru>=0.7.3", - "pydantic>=2.13.4", - "uvicorn>=0.51.0", + "common", + "fastapi==0.116.1", + "uvicorn[standard]==0.35.0", + "httpx==0.28.1", + "croniter==6.2.4", + "alembic==1.18.5", + "cryptography==49.0.0", + "gunicorn>=26.0.0", + "passlib==1.7.4", + "bcrypt>=4.0,<4.1", + "loguru>=0.7.2", + "aiofiles>=25.1.0", ] -[project.scripts] -backend = "backend:main" +[tool.uv.sources] +common = { workspace = true } [[tool.uv.index]] url = "https://pypi.tuna.tsinghua.edu.cn/simple/" default = true -[tool.uv.sources] -common = { workspace = true } [build-system] requires = ["hatchling"] build-backend = "hatchling.build" +[dependency-groups] +dev = [ + "pytest>=9.1.1", + "pytest-asyncio>=1.4.0", + "respx>=0.23.1", +] + +[tool.hatch.build.targets.wheel] +packages = ["src/backend"] + +[tool.pytest.ini_options] +asyncio_mode = "auto" +testpaths = ["tests"] diff --git a/backend/src/backend/__init__.py b/backend/src/backend/__init__.py index bc831a6..d90a9a5 100644 --- a/backend/src/backend/__init__.py +++ b/backend/src/backend/__init__.py @@ -1,2 +1 @@ -def main() -> None: - print("Hello from backend!") +"""Platform API application.""" diff --git a/backend/src/backend/admin.py b/backend/src/backend/admin.py new file mode 100644 index 0000000..73628ee --- /dev/null +++ b/backend/src/backend/admin.py @@ -0,0 +1,258 @@ +"""旧版工作区级员工管理接口。 + +路由前缀为 ``/api/v1/admin``，依赖当前工作区的管理员权限。新的系统级用户、 +工作区和成员管理接口在 ``platform.py``；本模块主要保留给兼容旧前端调用。 +""" + +from __future__ import annotations + +from typing import Any, Literal + +from common.auth.passwords import hash_password +from common.db.models import Roles, Users, WorkspaceMembers +from common.ids import new_ulid +from fastapi import APIRouter, Depends, HTTPException, status +from pydantic import BaseModel, ConfigDict, Field +from sqlalchemy import delete, func, or_, select +from sqlalchemy.ext.asyncio import AsyncSession + +from backend.dependencies import ( + RequestContext, + database_session, + request_context, +) + +router = APIRouter(prefix="/api/v1/admin", tags=["admin"]) + + +class EmployeeCreate(BaseModel): + model_config = ConfigDict(extra="forbid") + + username: str = Field(min_length=2, max_length=64) + display_name: str = Field(min_length=1, max_length=100) + email: str | None = Field(default=None, max_length=255) + role_code: Literal["admin", "developer"] = "developer" + password: str = Field(min_length=8, max_length=72) + + +class EmployeeUpdate(BaseModel): + model_config = ConfigDict(extra="forbid") + + display_name: str | None = Field(default=None, min_length=1, max_length=100) + email: str | None = Field(default=None, max_length=255) + role_code: Literal["admin", "developer"] | None = None + status: Literal["active", "disabled", "locked"] | None = None + + +def require_admin(context: RequestContext) -> None: + # 统一在路由入口处做角色判断，避免每个 CRUD 分支重复写权限代码。 + if not context.is_admin: + raise HTTPException(status.HTTP_403_FORBIDDEN, "仅管理员可以管理员工") + + +def employee_payload(user: Users, role: Roles) -> dict[str, Any]: + return { + "user_id": user.user_id, + "username": user.username, + "display_name": user.display_name, + "email": user.email, + "status": user.status, + "role_code": role.role_code, + "role_name": role.role_name, + "created_at": user.created_at.isoformat(), + } + + +async def member_row( + user_id: str, + context: RequestContext, + session: AsyncSession, +) -> tuple[Users, WorkspaceMembers, Roles]: + row = ( + await session.execute( + select(Users, WorkspaceMembers, Roles) + .join( + WorkspaceMembers, + WorkspaceMembers.user_id == Users.user_id, + ) + .join(Roles, Roles.role_id == WorkspaceMembers.role_id) + .where( + WorkspaceMembers.workspace_id == context.workspace.workspace_id, + Users.user_id == user_id, + ) + ) + ).first() + if row is None: + raise HTTPException(status.HTTP_404_NOT_FOUND, "员工不存在") + return row + + +# 旧版接口：列出当前工作区内的员工及其角色。 +@router.get("/employees") +async def list_employees( + context: RequestContext = Depends(request_context), + session: AsyncSession = Depends(database_session), +) -> dict[str, Any]: + rows = ( + await session.execute( + select(Users, Roles) + .join( + WorkspaceMembers, + WorkspaceMembers.user_id == Users.user_id, + ) + .join(Roles, Roles.role_id == WorkspaceMembers.role_id) + .where( + WorkspaceMembers.workspace_id == context.workspace.workspace_id, + WorkspaceMembers.member_status == "active", + ) + .order_by(Users.created_at, Users.user_id) + ) + ).all() + return { + "request_id": context.request_id, + "data": [employee_payload(user, role) for user, role in rows], + "meta": {"count": len(rows), "can_manage": context.is_admin}, + } + + +# 旧版接口：在当前工作区中创建员工及成员关系。 +@router.post("/employees", status_code=status.HTTP_201_CREATED) +async def create_employee( + payload: EmployeeCreate, + context: RequestContext = Depends(request_context), + session: AsyncSession = Depends(database_session), +) -> dict[str, Any]: + require_admin(context) + username = payload.username.strip() + display_name = payload.display_name.strip() + duplicate_conditions = [Users.username == username] + if payload.email: + duplicate_conditions.append(Users.email == payload.email.strip()) + duplicate = await session.scalar( + select(Users.user_id).where(or_(*duplicate_conditions)) + ) + if duplicate is not None: + raise HTTPException(status.HTTP_409_CONFLICT, "用户名或邮箱已存在") + role = await session.scalar( + select(Roles).where(Roles.role_code == payload.role_code) + ) + if role is None: + raise HTTPException(status.HTTP_422_UNPROCESSABLE_ENTITY, "角色不存在") + user = Users( + user_id=new_ulid(), + username=username, + display_name=display_name, + email=payload.email.strip() if payload.email else None, + password_hash=hash_password(payload.password), + status="active", + # 不变量: workspace admin == 系统管理员, 因此 role_code == "admin" + # 时必须同步把 platform_role_id 指向 admin role, 否则 + # resolve_is_system_admin 会返回 False, 但前端的 role_code + # 判断仍然命中, 两端出现分歧。 + platform_role_id=role.role_id if role.role_code == "admin" else None, + ) + session.add(user) + session.add( + WorkspaceMembers( + workspace_id=context.workspace.workspace_id, + user_id=user.user_id, + role_id=role.role_id, + member_status="active", + ) + ) + await session.flush() + await session.refresh(user) + return { + "request_id": context.request_id, + "data": employee_payload(user, role), + "meta": {}, + } + + +# 旧版接口：更新员工显示信息、角色或状态。 +@router.patch("/employees/{user_id}") +async def update_employee( + user_id: str, + payload: EmployeeUpdate, + context: RequestContext = Depends(request_context), + session: AsyncSession = Depends(database_session), +) -> dict[str, Any]: + require_admin(context) + user, membership, role = await member_row(user_id, context, session) + if payload.display_name is not None: + user.display_name = payload.display_name.strip() + if payload.email is not None: + user.email = payload.email.strip() or None + if payload.status is not None and payload.status != user.status: + if user.user_id == context.user.user_id and payload.status != "active": + raise HTTPException( + status.HTTP_409_CONFLICT, + "不能停用当前登录账号", + ) + if role.role_code == "admin" and payload.status in ("disabled", "locked"): + raise HTTPException( + status.HTTP_409_CONFLICT, + "不能停用或锁定管理员账号", + ) + user.status = payload.status + if payload.role_code is not None and payload.role_code != role.role_code: + if role.role_code == "admin" and payload.role_code != "admin": + raise HTTPException( + status.HTTP_409_CONFLICT, + "不能降级管理员账号", + ) + if ( + user.user_id == context.user.user_id + and payload.role_code != "admin" + ): + raise HTTPException( + status.HTTP_409_CONFLICT, + "不能降级自身管理员角色", + ) + next_role = await session.scalar( + select(Roles).where(Roles.role_code == payload.role_code) + ) + if next_role is None: + raise HTTPException(status.HTTP_422_UNPROCESSABLE_ENTITY, "角色不存在") + membership.role_id = next_role.role_id + role = next_role + await session.flush() + await session.refresh(user) + return { + "request_id": context.request_id, + "data": employee_payload(user, role), + "meta": {}, + } + + +# 旧版接口：移除当前工作区中的员工成员关系。 +@router.delete("/employees/{user_id}") +async def delete_employee( + user_id: str, + context: RequestContext = Depends(request_context), + session: AsyncSession = Depends(database_session), +) -> dict[str, Any]: + require_admin(context) + if user_id == context.user.user_id: + raise HTTPException(status.HTTP_409_CONFLICT, "不能删除当前登录员工") + user, _, role = await member_row(user_id, context, session) + if role.role_code == "admin": + raise HTTPException(status.HTTP_409_CONFLICT, "管理员账号不能删除") + await session.execute( + delete(WorkspaceMembers).where( + WorkspaceMembers.workspace_id == context.workspace.workspace_id, + WorkspaceMembers.user_id == user_id, + ) + ) + memberships = await session.scalar( + select(func.count()) + .select_from(WorkspaceMembers) + .where(WorkspaceMembers.user_id == user_id) + ) + if int(memberships or 0) == 0: + user.status = "disabled" + return { + "request_id": context.request_id, + "data": {"user_id": user_id, "deleted": True}, + "meta": {}, + } diff --git a/backend/src/backend/auth.py b/backend/src/backend/auth.py new file mode 100644 index 0000000..d23ebab --- /dev/null +++ b/backend/src/backend/auth.py @@ -0,0 +1,291 @@ +"""登录与会话认证接口。 + +中文导读：浏览器调用登录接口后，后端把 JWT 放入 HttpOnly Cookie；之后前端 +通过 ``fetch(..., credentials='same-origin')`` 自动携带 Cookie。需要工作区的 +接口会继续由 ``request_context`` 校验 ``workspace_id`` 和成员权限。 + +Cookie+JWT authentication endpoints. + +The user-facing flow is: + 1. POST /api/v1/auth/login — verify password, set HttpOnly cookie + 2. every other /api/ request reads the cookie via + ``backend.dependencies.request_context`` + 3. POST /api/v1/auth/logout — clear the cookie + 4. GET /api/v1/auth/me — return the current user + +Service-to-service calls do not use these endpoints — they live on the +shared Docker network and have no application-layer auth. See +``docker-compose.yml`` and ``schedule/service.py``. +""" + +from __future__ import annotations + +from typing import Any + +from common.auth.jwt import JwtError, issue_jwt, verify_jwt_token +from common.auth.membership import resolve_is_system_admin +from common.auth.passwords import verify_password +from common.config import settings +from common.db.models import Roles, Users, WorkspaceMembers, Workspaces +from common.ids import new_ulid +from fastapi import APIRouter, Depends, HTTPException, Request, Response, status +from sqlalchemy import select +from sqlalchemy.ext.asyncio import AsyncSession + +from backend.dependencies import database_session, load_user_permissions + +router = APIRouter(tags=["auth"]) + +# Cookie 配置：生产环境走 HTTPS 时应设置 Secure；本地 HTTP 开发环境会根据 +# 实际请求协议决定是否设置，避免浏览器因 Secure Cookie 而丢弃登录状态。 +COOKIE_NAME = "access_token" +COOKIE_TTL_SECONDS = 24 * 60 * 60 +COOKIE_SAMESITE = "lax" + + +def _set_session_cookie(request: Request, response: Response, token: str) -> None: + forwarded_scheme = request.headers.get("x-forwarded-proto", request.url.scheme) + # 反向代理通常用 X-Forwarded-Proto 告诉后端原始协议；若代理未传该头， + # 可通过配置强制启用 Secure，防止 HTTPS 场景下出现不安全 Cookie。 + secure = forwarded_scheme == "https" or settings.cookie_force_secure + response.set_cookie( + key=COOKIE_NAME, + value=token, + max_age=COOKIE_TTL_SECONDS, + path="/", + httponly=True, + secure=secure, + samesite=COOKIE_SAMESITE, + ) + + +def _clear_session_cookie(response: Response) -> None: + response.delete_cookie(key=COOKIE_NAME, path="/") + + +def _user_payload( + user: Users, + role_code: str | None = None, + *, + is_system_admin: bool = False, + permissions: list[str] | None = None, +) -> dict[str, Any]: + return { + "user_id": user.user_id, + "username": user.username, + "display_name": user.display_name, + "email": user.email, + "status": user.status, + "role_code": role_code, + "is_system_admin": is_system_admin, + "permissions": list(permissions) if permissions is not None else [], + } + + +def _workspace_payload( + workspace: Workspaces, + role: Roles, +) -> dict[str, Any]: + return { + "workspace_id": workspace.workspace_id, + "workspace_code": workspace.workspace_code, + "workspace_name": workspace.workspace_name, + "role_code": role.role_code, + "role_name": role.role_name, + } + + +# 校验账号密码，设置登录 Cookie，并返回用户可进入的工作区列表。 +@router.post("/api/v1/auth/login") +async def login( + request: Request, + response: Response, + session: AsyncSession = Depends(database_session), +) -> dict[str, Any]: + """Verify username/password and issue a session cookie. + + Returns the user record and the workspaces they are an active + member of (joined earliest first, which doubles as the default + selection until the user picks a different one in the UI). The + workspace list is informational — the JWT itself does not bind a + workspace; each request specifies its own ``?workspace_id=``. + """ + body = await request.json() + username = (body or {}).get("username", "").strip() + password = (body or {}).get("password", "") + if not username or not password: + raise HTTPException( + status.HTTP_400_BAD_REQUEST, + "username and password are required", + ) + + user = await session.scalar( + select(Users).where( + Users.username == username, + Users.status == "active", + ) + ) + if user is None or not verify_password(password, user.password_hash): + # Unified 401 to prevent username enumeration. + raise HTTPException( + status.HTTP_401_UNAUTHORIZED, + "invalid username or password", + ) + + # 返回用户可进入的工作区列表；当前没有默认工作区字段，因此最早加入的 + # 工作区作为前端的初始选择。 + rows = ( + await session.execute( + select(Workspaces, Roles, WorkspaceMembers.joined_at) + .join( + WorkspaceMembers, + WorkspaceMembers.workspace_id == Workspaces.workspace_id, + ) + .join(Roles, Roles.role_id == WorkspaceMembers.role_id) + .where( + WorkspaceMembers.user_id == user.user_id, + WorkspaceMembers.member_status == "active", + Workspaces.status == "active", + ) + .order_by(WorkspaceMembers.joined_at.asc()) + ) + ).all() + if not rows: + raise HTTPException( + status.HTTP_403_FORBIDDEN, + "user is not a member of any active workspace", + ) + + workspaces = [] + default_workspace_id: str | None = None + for workspace, role, joined_at in rows: + workspaces.append(_workspace_payload(workspace, role)) + if default_workspace_id is None: + default_workspace_id = workspace.workspace_id + + # Workspace role is always inherited from the user's platform role + # (``Users.platform_role_id``); the iteration over rows above was + # a legacy way to find the "highest" workspace role and is no + # longer correct now that admin/developer are platform-only. + user_role_code: str | None = None + if user.platform_role_id is not None: + platform_role_row = await session.scalar( + select(Roles).where(Roles.role_id == user.platform_role_id) + ) + if platform_role_row is not None: + user_role_code = platform_role_row.role_code + + token = issue_jwt(user.user_id, ttl_seconds=COOKIE_TTL_SECONDS) + _set_session_cookie(request, response, token) + + is_system_admin = await resolve_is_system_admin(session, user) + permissions = await load_user_permissions(session, user) + + return { + "request_id": new_ulid(), + "data": { + "user": _user_payload( + user, + user_role_code, + is_system_admin=is_system_admin, + permissions=permissions, + ), + "workspaces": workspaces, + "default_workspace_id": default_workspace_id, + }, + "meta": {}, + } + + +# 清除浏览器 Cookie，使当前会话立即失效。 +@router.post("/api/v1/auth/logout") +async def logout(response: Response) -> dict[str, Any]: + """Clear the session cookie. Idempotent.""" + _clear_session_cookie(response) + return { + "request_id": new_ulid(), + "data": {"logged_out": True}, + "meta": {}, + } + + +# 返回当前登录用户、权限和可访问工作区，用于前端初始化登录态。 +@router.get("/api/v1/auth/me") +async def me( + request: Request, + session: AsyncSession = Depends(database_session), +) -> dict[str, Any]: + """Return the current user record based on the session cookie. + + Does not require a workspace_id — useful for the frontend to + bootstrap identity on app load before any workspace has been + selected. Workspace list is included so the login screen can be + skipped on subsequent visits. + """ + token = request.cookies.get(COOKIE_NAME) + if not token: + raise HTTPException(status.HTTP_401_UNAUTHORIZED, "not authenticated") + try: + payload = verify_jwt_token(token) + except JwtError as exc: + raise HTTPException(status.HTTP_401_UNAUTHORIZED, str(exc)) from exc + + user_id = payload.get("sub") + if not user_id: + raise HTTPException(status.HTTP_401_UNAUTHORIZED, "invalid token") + + user = await session.get(Users, user_id) + if user is None or user.status != "active": + raise HTTPException(status.HTTP_401_UNAUTHORIZED, "user not found") + + rows = ( + await session.execute( + select(Workspaces, Roles, WorkspaceMembers.joined_at) + .join( + WorkspaceMembers, + WorkspaceMembers.workspace_id == Workspaces.workspace_id, + ) + .join(Roles, Roles.role_id == WorkspaceMembers.role_id) + .where( + WorkspaceMembers.user_id == user.user_id, + WorkspaceMembers.member_status == "active", + Workspaces.status == "active", + ) + .order_by(WorkspaceMembers.joined_at.asc()) + ) + ).all() + + workspaces = [_workspace_payload(ws, role) for ws, role, _ in rows] + default_workspace_id = workspaces[0]["workspace_id"] if workspaces else None + user_role_code: str | None = None + if user.platform_role_id is not None: + platform_role_row = await session.scalar( + select(Roles).where(Roles.role_id == user.platform_role_id) + ) + if platform_role_row is not None: + user_role_code = platform_role_row.role_code + + is_system_admin = await resolve_is_system_admin(session, user) + permissions = await load_user_permissions(session, user) + + return { + "request_id": new_ulid(), + "data": { + "user": _user_payload( + user, + user_role_code, + is_system_admin=is_system_admin, + permissions=permissions, + ), + "workspaces": workspaces, + "default_workspace_id": default_workspace_id, + }, + "meta": {}, + } + + +__all__ = [ + "COOKIE_NAME", + "COOKIE_TTL_SECONDS", + "router", +] diff --git a/backend/src/backend/dependencies.py b/backend/src/backend/dependencies.py new file mode 100644 index 0000000..5ea6575 --- /dev/null +++ b/backend/src/backend/dependencies.py @@ -0,0 +1,231 @@ +"""公共 FastAPI 依赖。 + +中文导读： + +* ``database_session``：为一次请求提供数据库事务；成功提交、异常回滚。 +* ``current_user``：只验证登录 Cookie 并得到当前用户，不关心工作区。 +* ``request_context``：普通业务接口最常使用的依赖，同时验证用户、 + ``workspace_id``、工作区成员关系和角色权限。 + +路由函数把这些函数写进 ``Depends(...)`` 后，FastAPI 会先完成校验，再把 +结果作为参数传给路由函数；因此业务代码无需重复解析 Cookie 或查询成员关系。 + +FastAPI dependencies for the public API. + +Two distinct concerns live here: + +* :func:`database_session` — yields a transactional ``AsyncSession`` + bound to the request's app-state factory. The session commits on + success and rolls back on exception via ``common.db.session_scope``. + +* :func:`request_context` — verifies the ``access_token`` cookie + (HS256-signed JWT), then loads the user's active membership for the + ``workspace_id`` query parameter. Returns a :class:`RequestContext` + dataclass that downstream handlers read for identity, role, and the + request id. + +The 56 ``Depends(request_context)`` call sites elsewhere in the +backend expect this exact dataclass shape; the workspace and role are +always present, so handlers do not need to handle the "no workspace +selected" case. + +Routes that need a workspace context must declare the query parameter +explicitly, even when the path also contains a resource id (e.g. +``/schedules/{schedule_id}``). This keeps authorization decisions +local to the request and prevents the "implicit workspace" footgun +where a path-resource lookup quietly overrides what the user asked +for. +""" + +from __future__ import annotations + +from collections.abc import AsyncIterator +from dataclasses import dataclass + +from common.auth.jwt import JwtError, verify_jwt_token +from common.auth.membership import ( + MembershipError, + load_active_membership, + resolve_is_system_admin, +) +from common.db import session_scope +from common.db.models import Permissions, RolePermissions, Roles, Users, Workspaces +from common.ids import new_ulid +from fastapi import Depends, HTTPException, Query, Request, status +from sqlalchemy import select +from sqlalchemy.ext.asyncio import AsyncSession + +ACCESS_TOKEN_COOKIE = "access_token" + + +@dataclass(frozen=True) +class RequestContext: + """已完成认证和工作区授权后的请求上下文。 + + 路由依赖 ``request_context`` 后会得到该对象，用其中的用户、工作区和角色 + 执行业务权限判断，避免在每个接口中重复查询。 + """ + + request_id: str + user: Users + workspace: Workspaces + role: Roles + # True when the requester holds the platform-scoped admin role (via + # Users.platform_role_id). Same flag exposed on ``/api/v1/auth/me`` + # so the frontend can render the platform-admin entry point. System + # admins have full control over every workspace — including disabled + # ones — so ``request_context`` lets them through and this flag is + # the single signal handlers use to gate platform-only operations. + is_system_admin: bool = False + + @property + def is_admin(self) -> bool: + """Workspace admin OR system admin (the latter is strictly stronger).""" + return self.is_system_admin or self.role.role_code == "admin" + + +async def database_session(request: Request) -> AsyncIterator[AsyncSession]: + # 一个请求对应一个事务范围，避免不同请求意外共用同一个 Session。 + async with session_scope(request.app.state.session_factory) as session: + yield session + + +async def current_user( + request: Request, + session: AsyncSession = Depends(database_session), +) -> Users: + """Resolve the authenticated user from the ``access_token`` cookie. + + Returns the active ``Users`` row or raises 401. Does NOT load a + workspace — use :func:`request_context` for handlers that need + a workspace-scoped context, or use ``Depends(current_user)`` for + workspace-agnostic endpoints (e.g. ``/api/v1/auth/me``). + """ + # 登录接口写入 HttpOnly Cookie；后续浏览器请求会自动携带它。 + token = request.cookies.get(ACCESS_TOKEN_COOKIE) + if not token: + raise HTTPException(status.HTTP_401_UNAUTHORIZED, "not authenticated") + try: + payload = verify_jwt_token(token) + except JwtError as exc: + raise HTTPException(status.HTTP_401_UNAUTHORIZED, str(exc)) from exc + user_id = payload.get("sub") + if not user_id: + raise HTTPException(status.HTTP_401_UNAUTHORIZED, "invalid token") + user = await session.get(Users, user_id) + if user is None or user.status != "active": + raise HTTPException(status.HTTP_401_UNAUTHORIZED, "user not found") + return user + + +async def request_context( + request: Request, + workspace_id: str = Query( + ..., + min_length=26, + max_length=26, + description="Workspace context for this request (CHAR(26) ULID).", + ), + session: AsyncSession = Depends(database_session), +) -> RequestContext: + """Verify JWT and load the user's membership for ``workspace_id``. + + The ``request_id`` comes from the ``X-Request-ID`` header if + present, else from a freshly minted ULID. Handlers receive the + same :class:`RequestContext` shape they had under the old + header-based implementation, so the 56 ``Depends(request_context)`` + call sites in this repo stay working without changes — they just + now pass ``?workspace_id=`` instead of the old headers. + + System admins (Users.platform_role_id → admin role) bypass the + active-membership requirement: they can address disabled workspaces + because they own the platform. Non-admin users still need an + active ``WorkspaceMembers`` row in an active ``Workspaces`` row. + """ + # 大多数工作区接口通过这个依赖统一完成“登录 + 工作区 + 角色”三层校验。 + user = await current_user(request, session) + is_system_admin = await resolve_is_system_admin(session, user) + + if is_system_admin: + # System admin: any workspace (active or disabled) is fine. + # Still 404 if the workspace_id is genuinely unknown — the + # ``?workspace_id=`` query param is part of the URL contract. + workspace = await session.get(Workspaces, workspace_id) + if workspace is None: + raise HTTPException( + status.HTTP_404_NOT_FOUND, + "workspace not found", + ) + # Synthesize a role object so the rest of ``RequestContext`` + # (and downstream ``is_admin`` checks) keep working without + # branching on whether a real WorkspaceMembers row exists. + role = await _load_admin_role(session) + if role is None: + # The seed migration creates this row, so this is a + # hard config error if it's missing. + raise HTTPException( + status.HTTP_500_INTERNAL_SERVER_ERROR, + "admin role not configured", + ) + else: + try: + _user, workspace, role = await load_active_membership( + session, user.user_id, workspace_id, + ) + except MembershipError as exc: + raise HTTPException( + status.HTTP_403_FORBIDDEN, + "active workspace membership is required", + ) from exc + + request_id = request.headers.get("X-Request-ID") or new_ulid() + return RequestContext( + request_id=request_id, + user=user, + workspace=workspace, + role=role, + is_system_admin=is_system_admin, + ) + + +async def _load_admin_role(session: AsyncSession) -> Roles | None: + """Return the singleton ``role_code='admin'`` row (None if absent).""" + return await session.scalar( + select(Roles).where(Roles.role_code == "admin") + ) + + +async def load_user_permissions( + session: AsyncSession, user: Users +) -> list[str]: + """Return the user's effective platform permission_codes. + + Resolves ``Users.platform_role_id`` → ``RolePermissions`` → + ``Permissions.permission_code``, filtered by ``is_deleted = 0`` on + both sides. Returns an empty list when the user has no platform + role assigned (e.g. brand-new account before role assignment). + + This is the single source of truth for the menu permissions + consumed by ``GET /api/v1/auth/me``. Endpoint-level authorization + keeps using ``system_admin_context`` (which keys off + ``role_code == 'admin'``); permissions are a frontend-display + concern only. + """ + if user.platform_role_id is None: + return [] + rows = ( + await session.execute( + select(Permissions.permission_code) + .join( + RolePermissions, + RolePermissions.permission_id == Permissions.permission_id, + ) + .where( + RolePermissions.role_id == user.platform_role_id, + RolePermissions.is_deleted == 0, + Permissions.is_deleted == 0, + ) + .order_by(Permissions.permission_code) + ) + ).all() + return [row[0] for row in rows] diff --git a/backend/src/backend/jupyter.py b/backend/src/backend/jupyter.py new file mode 100644 index 0000000..74d01e1 --- /dev/null +++ b/backend/src/backend/jupyter.py @@ -0,0 +1,171 @@ +"""Jupyter 访问的鉴权桥接。 + +浏览器访问 ``/jupyter/{workspace_id}/...`` 时，Nginx 会先向本模块的 +``/api/v1/auth/jupyter`` 发起内部 auth_request。后端验证用户、工作区和文件 +锁，再把实际 Jupyter 地址与内部令牌写进响应头，由 Nginx 转发请求。 + +本模块只负责鉴权和路由选择；真正启动、管理 Jupyter 进程的是 ``runtime`` 服务。 +""" + +import re + +from common.auth.jwt import JwtError, verify_jwt_token +from common.auth.membership import MembershipError, load_active_membership +from common.db.models import Scripts +from fastapi import APIRouter, Depends, HTTPException, Request, Response +from fastapi.security import HTTPAuthorizationCredentials, HTTPBearer +from sqlalchemy import select +from sqlalchemy.ext.asyncio import AsyncSession + +from backend.dependencies import database_session +from backend.runtime_client import RuntimeClientError + +router = APIRouter(tags=["jupyter"]) +security = HTTPBearer(auto_error=False) + + +def extract_notebook_path( + uri: str, + workspace_id: str, +) -> str | None: + """Pull the relative notebook path out of the original request URI. + + Only ``/jupyter/{workspace_id}/notebooks/*.ipynb`` requests are + subject to file-level lock checks; everything else (tree views, + ``/api/contents`` and WebSocket upgrades) bypasses the lock. + """ + pattern = rf"^/jupyter/{re.escape(workspace_id)}/notebooks/(.+\.ipynb)" + match = re.match(pattern, uri) + if match: + return match.group(1) + return None + + +async def check_notebook_is_locked( + session: AsyncSession, + workspace_id: str, + notebook_path: str, + user_id: str, +) -> bool: + """Decide whether the request is allowed to read the notebook. + + Returns ``True`` only when the notebook exists, is owned by a + different user, and is currently locked. The owner is always let + through; a missing row is treated as "not owned yet" and allowed. + """ + # 锁信息保存在数据库的 Scripts 记录中，而非前端内存；因此多浏览器/多用户 + # 访问同一 notebook 时也能得到一致结果。 + statement = select(Scripts.owner_user_id, Scripts.is_locked).where( + Scripts.workspace_id == workspace_id, + Scripts.script_name == notebook_path, + Scripts.script_type == "notebook", + Scripts.is_deleted == 0, + ) + row = (await session.execute(statement)).one_or_none() + if row is None: + return False + owner_user_id, is_locked = row + if owner_user_id == user_id: + return False + return bool(is_locked) + + +async def load_active_membership_or_403( + session: AsyncSession, + user_id: str, + workspace_id: str, +): + """Resolve the user's active membership, raising 403 if missing. + + Thin wrapper around :func:`common.auth.membership.load_active_membership` + that maps the library's ``MembershipError`` to a FastAPI 403. + """ + try: + return await load_active_membership(session, user_id, workspace_id) + except MembershipError as exc: + raise HTTPException( + status_code=403, + detail="active workspace membership is required", + ) from exc + + +# 供 Nginx auth_request 调用：验证访问 Jupyter 的身份、成员关系和文件锁， +# 再返回应转发到的 Jupyter 地址及内部令牌。 +@router.get("/api/v1/auth/jupyter") +async def verify_jupyter_access( + request: Request, + response: Response, + auth: HTTPAuthorizationCredentials | None = Depends(security), + session: AsyncSession = Depends(database_session), +) -> dict: + """Nginx auth_request subrequest handler. + + Returns 200 with two response headers so Nginx can proxy the + request to the user's Jupyter instance: + + * ``x-upstream-addr`` = ``{base_url}:{port}`` + * ``x-jupyter-internal-token`` = the runtime-issued token + """ + workspace_id = request.headers.get("X-Original-Workspace-Id") + original_uri = request.headers.get("X-Original-URI", "") + + if not workspace_id: + raise HTTPException( + status_code=400, + detail="Missing Workspace ID", + ) + + cookie_token = request.cookies.get("access_token") + bearer_token = auth.credentials if auth else None + token = cookie_token or bearer_token + try: + payload = verify_jwt_token(token) + except JwtError as exc: + raise HTTPException( + status_code=401, + detail=str(exc), + ) from exc + user_id = payload.get("sub") + if not user_id: + raise HTTPException( + status_code=401, + detail="Invalid Authentication Token", + ) + + await load_active_membership_or_403(session, user_id, workspace_id) + + notebook_path = extract_notebook_path(original_uri, workspace_id) + if notebook_path and await check_notebook_is_locked( + session, + workspace_id, + notebook_path, + user_id, + ): + raise HTTPException( + status_code=403, + detail=f"Notebook '{notebook_path}' is currently locked", + ) + + runtime_client = request.app.state.runtime_client + ws_info = await runtime_client.get_workspace(workspace_id) + if not ws_info or ws_info.get("status") != "running": + try: + ws_info = await runtime_client.start_workspace(workspace_id) + except RuntimeClientError as exc: + raise HTTPException( + status_code=exc.status_code, + detail=exc.detail, + ) from exc + + target_port = ws_info.get("port") + jupyter_token = ws_info.get("token") + jupyter_base_url = ws_info.get("base_url") + if not target_port: + raise HTTPException( + status_code=500, + detail="Jupyter instance returned no port", + ) + + response.headers["x-upstream-addr"] = f"{jupyter_base_url}:{target_port}" + response.headers["x-jupyter-internal-token"] = jupyter_token or "" + return {"status": "ok"} diff --git a/backend/src/backend/main.py b/backend/src/backend/main.py index 0a474d5..f9df7ae 100644 --- a/backend/src/backend/main.py +++ b/backend/src/backend/main.py @@ -1,147 +1,142 @@ -# coding=utf-8 +"""后端服务总入口。 + +这里负责两件事： +1. 在应用启动时准备数据库、对象存储以及 Runtime 的 HTTP 客户端； +2. 将各业务模块的路由注册到同一个 FastAPI 应用中。 + +浏览器请求先经过 Nginx 的 ``/api/`` 反向代理，随后才会到达本文件创建的 +应用。具体接口实现分别在 ``auth.py``、``scripts.py``、``schedules.py`` 等 +模块中。 """ -@Time :2026/7/27 -@Author :tao.chen -""" -import os -import re + +from __future__ import annotations + +import time +from collections.abc import AsyncIterator +from contextlib import asynccontextmanager +from typing import Any + import httpx -from fastapi import FastAPI, Request, Response, HTTPException, Depends, status -from fastapi.security import HTTPBearer, HTTPAuthorizationCredentials -from typing import Optional +from common.config import settings +from common.db import create_database_engine, create_session_factory +from common.logging import configure_logging +from common.service_app import create_service_app +from common.storage import ( + PURPOSE_BUCKETS, + AsyncStorageBackend, + actual_bucket_name, + build_storage_config, + create_storage, +) +from fastapi import Request +from fastapi.responses import JSONResponse from loguru import logger -app = FastAPI(title="Jupyter Auth & Router Backend") +from backend.admin import router as admin_router +from backend.auth import router as auth_router +from backend.jupyter import router as jupyter_router +from backend.platform import router as platform_router +from backend.rclone_rc_client import RcloneRCClient +from backend.resources import router as resources_router +from backend.runtime_client import RuntimeClient +from backend.schedule_runs import router as schedule_runs_router +from backend.schedules import router as schedules_router +from backend.scripts import router as scripts_router +from backend.storage_api import router as storage_api_router -RUNTIME_BASE_URL = os.getenv("RUNTIME_BASE_URL", "http://127.0.0.1:8001") -security = HTTPBearer(auto_error=False) +configure_logging(settings.log_level) -# ------------------------------------------------------------------ -# 1. Runtime 交互 Client -# ------------------------------------------------------------------ -class RuntimeClient: - """与 Runtime 进程管理器服务交互""" +@asynccontextmanager +async def lifespan(app: Any) -> AsyncIterator[None]: + # 生命周期内创建的对象挂在 app.state 上，路由通过 Depends 或 Request + # 取得它们；这样每个请求不会重复创建数据库连接或 HTTP 客户端。 + engine = create_database_engine(settings.database_url) + app.state.session_factory = create_session_factory(engine) - @staticmethod - async def get_workspace(workspace_id: str) -> Optional[dict]: - """按需查询单个 workspace 进程""" - async with httpx.AsyncClient(base_url=RUNTIME_BASE_URL) as client: - try: - resp = await client.post( - "/api/v1/jupyter", - json={"action": "get", "workspace_id": workspace_id}, - timeout=3.0, - ) - if resp.status_code == 200: - return resp.json() - return None - except httpx.RequestError: - return None - - @staticmethod - async def start_workspace(workspace_id: str) -> dict: - """进程未运行时主动触发启动""" - async with httpx.AsyncClient(base_url=RUNTIME_BASE_URL) as client: - resp = await client.post( - "/api/v1/jupyter", - json={"action": "start", "workspace_id": workspace_id}, - timeout=10.0, - ) - if resp.status_code == 200: - return resp.json() - raise HTTPException( - status_code=500, detail="Failed to start Jupyter instance" - ) - - -# ------------------------------------------------------------------ -# 2. 数据库与权限模拟 (请根据实际 MySQL ORM 修改) -# ------------------------------------------------------------------ -async def check_notebook_is_locked(workspace_id: str, notebook_path: str) -> bool: - """ - 查数据库：判断特定 Notebook 文件是否被锁定 - :param workspace_id: 工作区 ID - :param notebook_path: 相对路径，如 "test.ipynb" 或 "folder/demo.ipynb" - """ - # 模拟锁定数据库：假定 test_locked.ipynb 被锁定 - locked_notebooks = { - ("test1234", "test_locked.ipynb"): True, + # 存储接口与业务路由运行在同一个 backend 进程中，因此直接复用存储对象， + # 不需要再通过 HTTP 调用自己。字典键使用真实桶名，便于上传会话和存储 + # 对象记录直接定位对应的存储后端。 + app.state.object_stores: dict[str, StorageBackend | AsyncStorageBackend] = { # noqa: F821 + actual_bucket_name(purpose): create_storage(build_storage_config(purpose)) + for purpose in PURPOSE_BUCKETS } - return locked_notebooks.get((workspace_id, notebook_path), False) + app.state.default_bucket = settings.s3_workspace_bucket + # P0-1 fix: runtime's /api/v1/jupyter is token-guarded. The token is + # the same ``INTERNAL_SERVICE_TOKEN`` value used by /internal/v1/* — + # reusing one mechanism instead of inventing a second one. + runtime_http_client = httpx.AsyncClient( + base_url=settings.runtime_api_url, + timeout=httpx.Timeout(30.0), + headers={ + "X-Internal-Service-Token": settings.internal_service_token, + }, + ) + app.state.runtime_client = RuntimeClient(runtime_http_client) + # rclone 目录缓存刷新属于尽力而为的后台动作，不能拖慢用户保存文件的请求。 + rclone_http_client = httpx.AsyncClient( + base_url=settings.rclone_rc_url, + timeout=httpx.Timeout(30.0), + ) + app.state.rclone_rc_client = RcloneRCClient(rclone_http_client) + try: + yield + finally: + await rclone_http_client.aclose() + await runtime_http_client.aclose() + await engine.dispose() -def verify_jwt_token(token: str) -> str: - """校验 JWT 令牌""" - if token == "invalid-token": - raise HTTPException(status_code=401, detail="Invalid Authentication Token") - return "user_001" +app = create_service_app( + settings.service_name, + lifespan=lifespan, +) +# 面向浏览器的公开 API：认证、脚本、资源、调度和系统管理。 +app.include_router(auth_router) +app.include_router(jupyter_router) +app.include_router(resources_router) +app.include_router(schedule_runs_router) +app.include_router(schedules_router) +app.include_router(scripts_router) +app.include_router(admin_router) +app.include_router(platform_router) + +# 内部存储接口额外加上 /internal 前缀，供后端服务间调用，不作为普通前端 API。 +app.include_router(storage_api_router, prefix="/internal") -def extract_notebook_path(uri: str, workspace_id: str) -> Optional[str]: - """ - 从原始请求 URI 中提取请求的 .ipynb 文件相对路径 - 例如: /jupyter/test1234/notebooks/folder/test.ipynb -> folder/test.ipynb - """ - pattern = rf"^/jupyter/{re.escape(workspace_id)}/notebooks/(.+\.ipynb)" - match = re.match(pattern, uri) - if match: - return match.group(1) - return None - - -# ------------------------------------------------------------------ -# 3. 核心 Auth 接口 (针对 Nginx auth_request) -# ------------------------------------------------------------------ -@app.get("/api/v1/auth/jupyter") -async def verify_jupyter_access( - request: Request, - response: Response, - auth: Optional[HTTPAuthorizationCredentials] = Depends(security), -): - # 获取 Nginx 传入的元数据 - workspace_id = request.headers.get("X-Original-Workspace-Id") - original_uri = request.headers.get("X-Original-URI", "") - - cookie_token = request.cookies.get("access_token") - bearer_token = auth.credentials if auth else None - token = bearer_token or cookie_token - - # if not token: - # raise HTTPException(status_code=401, detail="Missing Authentication Token") - - if not workspace_id: - raise HTTPException(status_code=400, detail="Missing Workspace ID") - - # 基础身份认证 - # current_user_id = verify_jwt_token(token) - - # 精准锁校验：只有在访问 .ipynb 文件时才检查 is_locked - notebook_path = extract_notebook_path(original_uri, workspace_id) - if notebook_path: - is_locked = await check_notebook_is_locked(workspace_id, notebook_path) - if is_locked: - raise HTTPException( - status_code=403, - detail=f"Notebook '{notebook_path}' is currently locked", - ) - - # 获取或启动 Jupyter 子进程 - ws_info = await RuntimeClient.get_workspace(workspace_id) - - if not ws_info or ws_info.get("status") != "running": - ws_info = await RuntimeClient.start_workspace(workspace_id) - - target_port = ws_info.get("port") - jupyter_token = ws_info.get("token") - jupyter_base_url = ws_info.get("base_url") - - if not target_port: - raise HTTPException( - status_code=500, detail="Jupyter instance returned no port" +@app.middleware("http") +async def access_log(request: Request, call_next): + # 每个 HTTP 请求都记录方法、路径、状态码和耗时；排查页面请求失败时， + # Docker Desktop 中 backend 容器的 Logs 就会显示这里生成的日志。 + start = time.perf_counter() + try: + response = await call_next(request) + except Exception: + elapsed_ms = (time.perf_counter() - start) * 1000 + logger.exception( + "request failed {method} {path} after {ms:.1f}ms", + method=request.method, path=request.url.path, ms=elapsed_ms, ) + raise + elapsed_ms = (time.perf_counter() - start) * 1000 + logger.info( + "{method} {path} -> {status} in {ms:.1f}ms", + method=request.method, path=request.url.path, + status=response.status_code, ms=elapsed_ms, + ) + return response - # 通过 Response Header 返回 Upstream 地址与 Token 给 Nginx - response.headers["x-upstream-addr"] = f"{jupyter_base_url}:{target_port}" - response.headers["x-jupyter-internal-token"] = jupyter_token or "" - return {"status": "ok"} + +@app.exception_handler(Exception) +async def unhandled_exception_handler(request: Request, exc: Exception): + logger.error( + "unhandled exception on {method} {path} from {client}: {exc!r}", + method=request.method, path=request.url.path, + client=request.client.host if request.client else "-", + exc=exc, + ) + return JSONResponse( + status_code=500, + content={"detail": "Internal server error"}, + ) diff --git a/backend/src/backend/platform.py b/backend/src/backend/platform.py new file mode 100644 index 0000000..3af6959 --- /dev/null +++ b/backend/src/backend/platform.py @@ -0,0 +1,1277 @@ +"""系统级管理接口。 + +中文导读：本模块管理全平台的用户、工作区、成员关系和角色权限。它使用 +``system_admin_context`` 进行平台管理员校验，因此不要求请求者先加入某个具体 +工作区；普通工作区内的业务接口则使用 ``request_context``。 + +System-admin (platform-scope) endpoints for workspace & membership management. + +All routes under ``/api/v1/platform/*`` are gated by +:func:`system_admin_context`, which requires the requester to hold a +``Users.platform_role_id`` pointing to a ``Roles`` row whose +``role_code == 'admin'``. Unlike ``backend.dependencies.request_context``, +this dependency does NOT require an active workspace membership — system +admins can manage workspaces before/without being a member of any. + +Endpoints +--------- + +Workspace CRUD:: + + GET /workspaces — list non-deleted workspaces + POST /workspaces — create a new workspace + GET /workspaces/{workspace_id} — single workspace (incl. disabled) + PATCH /workspaces/{workspace_id} — update editable fields + DELETE /workspaces/{workspace_id} — soft delete (cascades memberships) + +Workspace membership CRUD:: + + GET /workspaces/{workspace_id}/members — list active members + POST /workspaces/{workspace_id}/members — add a member + PATCH /workspaces/{workspace_id}/members/{user_id} — update role/status + DELETE /workspaces/{workspace_id}/members/{user_id} — remove a member + +Platform employee roster:: + + GET /employees — list all non-deleted users + POST /employees — create a user without workspace membership + PATCH /employees/{user_id} — update a user's profile, status or platform role + DELETE /employees/{user_id} — soft delete a user (cascades to workspace memberships) + +Role menu-permission management:: + + GET /roles — list platform roles with their permission_codes + GET /roles/{role_code}/permissions — one role's permission_codes + PATCH /roles/{role_code}/permissions — replace a role's permission set (diff-based) + +Invariants +---------- + +* Every workspace must always retain at least one active ``admin`` member. + This is enforced on member PATCH/DELETE AND on + ``PATCH /employees/{user_id}`` demotions, because workspace role is + inherited from ``users.platform_role_id`` and demoting a platform + admin cascades to all of their active memberships. +* A system admin cannot remove their own workspace membership via + ``DELETE .../members/{self}``; the only escape is to delete the entire + workspace, which cascades membership soft-deletion. +* ``DELETE /workspaces/{id}`` is allowed from any non-disabled status and + sets ``status='disabled'`` + ``is_deleted=1`` + ``deleted_at`` on the + workspace and every one of its active memberships. +* The ``admin`` role must always keep ``system.view`` + ``system.manage`` + menu permissions; non-admin roles may never hold ``system.*`` + permissions. Menu permissions gate frontend rendering only — API + authorization always keys off ``role_code == 'admin'``. +""" + +from __future__ import annotations + +import datetime +import re +from dataclasses import dataclass +from typing import Any, Literal + +from common.auth.passwords import hash_password +from common.db.models import ( + Permissions, + RolePermissions, + Roles, + Users, + WorkspaceMembers, + Workspaces, +) +from common.ids import new_ulid +from fastapi import APIRouter, Depends, HTTPException, Request, status +from pydantic import BaseModel, ConfigDict, Field +from sqlalchemy import func, insert, or_, select, update +from sqlalchemy.ext.asyncio import AsyncSession + +from backend.dependencies import current_user, database_session + +router = APIRouter(prefix="/api/v1/platform", tags=["platform"]) + + +# --------------------------------------------------------------------------- +# Constants +# --------------------------------------------------------------------------- + +WORKSPACE_CODE_PATTERN = re.compile(r"^[a-z0-9-]{3,32}$") +LIST_PAGE_SIZE = 100 + +WORKSPACE_EDITABLE_STATUS = ("active", "archived") +MEMBER_ROLE_CODES = ("admin", "developer") +MEMBER_STATUS_VALUES = ("active", "disabled", "locked") + + +# --------------------------------------------------------------------------- +# Schemas +# --------------------------------------------------------------------------- + + +# 创建工作区时前端提交的请求体；禁止未声明字段。 +class WorkspaceCreate(BaseModel): + model_config = ConfigDict(extra="forbid") + + workspace_code: str = Field(min_length=3, max_length=32) + workspace_name: str = Field(min_length=1, max_length=150) + quota_bytes: int = Field(default=0, ge=0) + description: str | None = Field(default=None, max_length=1000) + + +# 编辑工作区时允许修改的字段；禁用操作必须走删除接口而不是直接传状态。 +class WorkspaceUpdate(BaseModel): + model_config = ConfigDict(extra="forbid") + + workspace_name: str | None = Field(default=None, min_length=1, max_length=150) + quota_bytes: int | None = Field(default=None, ge=0) + description: str | None = Field(default=None, max_length=1000) + # 'disabled' is rejected here on purpose — soft delete must go through DELETE. + status: Literal["active", "archived"] | None = None + + +# 将已存在用户加入工作区的请求体；角色继承用户的平台角色。 +class MemberCreate(BaseModel): + """Add a user to a workspace. Role is inherited from the user's + platform role (Users.platform_role_id) — not set here.""" + + model_config = ConfigDict(extra="forbid") + + user_id: str = Field(min_length=26, max_length=26) + + +# 更新成员在该工作区中的可用状态，不直接在这里修改平台角色。 +class MemberUpdate(BaseModel): + """Update a workspace membership's status. Role cannot be changed + via this endpoint — workspace role is always inherited from the + user's platform role. To change a member's role, PATCH + /platform/employees/{user_id} instead.""" + + model_config = ConfigDict(extra="forbid") + + member_status: Literal["active", "disabled", "locked"] | None = None + + +# 新建平台用户的请求体；创建用户不等同于把用户加入某个工作区。 +class PlatformEmployeeCreate(BaseModel): + model_config = ConfigDict(extra="forbid") + + username: str = Field(min_length=2, max_length=64) + display_name: str = Field(min_length=1, max_length=100) + email: str | None = Field(default=None, max_length=255) + password: str = Field(min_length=8, max_length=72) + role_code: Literal["admin", "developer"] | None = None + + +# 修改平台用户资料、状态或平台角色的请求体。 +class PlatformEmployeeUpdate(BaseModel): + model_config = ConfigDict(extra="forbid") + + display_name: str | None = Field(default=None, min_length=1, max_length=100) + email: str | None = Field(default=None, max_length=255) + status: Literal["active", "disabled", "locked"] | None = None + role_code: Literal["admin", "developer"] | None = None + + +# 用完整权限集合替换某个平台角色菜单权限的请求体。 +class RolePermissionsPatch(BaseModel): + """Replace a platform role's permission set wholesale. + + Empty list is allowed (revokes all permissions) for non-`admin` + roles. The PATCH endpoint rejects emptying an `admin` role of its + system.* permissions; see ``patch_role_permissions`` for the + load-bearing guard order. + """ + + model_config = ConfigDict(extra="forbid") + + permission_codes: list[str] = Field(default_factory=list, max_length=64) + + +# --------------------------------------------------------------------------- +# System-admin context dependency +# --------------------------------------------------------------------------- + + +@dataclass(frozen=True) +class SystemAdminContext: + """通过系统管理员校验后的上下文，只包含当前用户和请求追踪 ID。""" + + """Resolved identity for a system-admin request. + + Carries the request id, the authenticated user row, and the resolved + ``Roles`` row the user holds via ``Users.platform_role_id``. By + construction the role's ``role_code`` is ``"admin"``. + """ + + request_id: str + user: Users + platform_role: Roles + + +async def system_admin_context( + request: Request, + session: AsyncSession = Depends(database_session), +) -> SystemAdminContext: + """验证当前用户是否为平台管理员，供 /api/v1/platform 下的路由依赖。""" + """Resolve the requester as a system admin. + + Steps: + 1. Reuse :func:`backend.dependencies.current_user` to validate the JWT + cookie and fetch the active ``Users`` row (raises 401 on failure). + 2. Require ``Users.platform_role_id`` to point to a row whose + ``role_code == 'admin'`` — anything else is 403. + """ + user = await current_user(request, session) + if user.platform_role_id is None: + raise HTTPException( + status.HTTP_403_FORBIDDEN, + "需要系统管理员权限", + ) + platform_role = await session.scalar( + select(Roles).where(Roles.role_id == user.platform_role_id) + ) + if platform_role is None or platform_role.role_code != "admin": + raise HTTPException( + status.HTTP_403_FORBIDDEN, + "需要系统管理员权限", + ) + request_id = request.headers.get("X-Request-ID") or new_ulid() + return SystemAdminContext( + request_id=request_id, + user=user, + platform_role=platform_role, + ) + + +# --------------------------------------------------------------------------- +# Payload helpers +# --------------------------------------------------------------------------- + + +def workspace_payload(workspace: Workspaces) -> dict[str, Any]: + return { + "workspace_id": workspace.workspace_id, + "workspace_code": workspace.workspace_code, + "workspace_name": workspace.workspace_name, + "active_root_uri": workspace.active_root_uri, + "quota_bytes": workspace.quota_bytes, + "status": workspace.status, + "description": workspace.description, + "created_by": workspace.created_by, + "created_at": workspace.created_at.isoformat(), + "updated_at": ( + workspace.updated_at.isoformat() if workspace.updated_at else None + ), + } + + +def member_payload( + user: Users, + role: Roles, + membership: WorkspaceMembers, +) -> dict[str, Any]: + return { + "user_id": user.user_id, + "username": user.username, + "display_name": user.display_name, + "email": user.email, + "user_status": user.status, + "role_code": role.role_code, + "role_name": role.role_name, + "member_status": membership.member_status, + "joined_at": membership.joined_at.isoformat(), + } + + +def platform_employee_payload( + user: Users, + role: Roles | None, +) -> dict[str, Any]: + return { + "user_id": user.user_id, + "username": user.username, + "display_name": user.display_name, + "email": user.email, + "status": user.status, + "role_code": role.role_code if role is not None else None, + "role_name": role.role_name if role is not None else None, + "created_at": user.created_at.isoformat(), + } + + +# --------------------------------------------------------------------------- +# Internal helpers +# --------------------------------------------------------------------------- + + +async def _load_workspace(session: AsyncSession, workspace_id: str) -> Workspaces: + workspace = await session.get(Workspaces, workspace_id) + if workspace is None: + raise HTTPException(status.HTTP_404_NOT_FOUND, "workspace 不存在") + return workspace + + +async def _load_role_by_code(session: AsyncSession, role_code: str) -> Roles: + role = await session.scalar(select(Roles).where(Roles.role_code == role_code)) + if role is None: + raise HTTPException( + status.HTTP_422_UNPROCESSABLE_ENTITY, + f"角色 {role_code} 不存在", + ) + return role + + +async def _count_active_admins( + session: AsyncSession, + workspace_id: str, + exclude_user_id: str | None = None, +) -> int: + """Count active admin members of ``workspace_id``. + + Pass ``exclude_user_id`` when checking "would X be the last admin?" + before mutating X. + """ + admin_role = await _load_role_by_code(session, "admin") + stmt = ( + select(func.count()) + .select_from(WorkspaceMembers) + .where( + WorkspaceMembers.workspace_id == workspace_id, + WorkspaceMembers.role_id == admin_role.role_id, + WorkspaceMembers.member_status == "active", + WorkspaceMembers.is_deleted == 0, + ) + ) + if exclude_user_id is not None: + stmt = stmt.where(WorkspaceMembers.user_id != exclude_user_id) + return int(await session.scalar(stmt) or 0) + + +async def _count_active_system_admins( + session: AsyncSession, + exclude_user_id: str | None = None, +) -> int: + """Count active system admins across the platform. + + Pass ``exclude_user_id`` when checking "would X be the last admin?" + before mutating X. + """ + admin_role = await _load_role_by_code(session, "admin") + stmt = ( + select(func.count()) + .select_from(Users) + .where( + Users.status == "active", + Users.is_deleted == 0, + Users.platform_role_id == admin_role.role_id, + ) + ) + if exclude_user_id is not None: + stmt = stmt.where(Users.user_id != exclude_user_id) + return int(await session.scalar(stmt) or 0) + + +def _envelope(request_id: str, data: Any, meta: dict[str, Any] | None = None) -> dict[str, Any]: + return { + "request_id": request_id, + "data": data, + "meta": meta or {}, + } + + +# --------------------------------------------------------------------------- +# Platform employee roster +# --------------------------------------------------------------------------- + + +# 列出整个平台的非删除用户；不局限于某一个工作区。 +@router.get("/employees") +async def list_platform_employees( + context: SystemAdminContext = Depends(system_admin_context), + session: AsyncSession = Depends(database_session), +) -> dict[str, Any]: + """List every non-soft-deleted platform user.""" + rows = ( + await session.execute( + select(Users, Roles) + .outerjoin(Roles, Roles.role_id == Users.platform_role_id) + .where(Users.is_deleted == 0) + .order_by(Users.created_at, Users.user_id) + ) + ).all() + return _envelope( + context.request_id, + [platform_employee_payload(user, role) for user, role in rows], + {"count": len(rows)}, + ) + + +# 创建平台用户；后续可再通过成员接口把该用户加入工作区。 +@router.post("/employees", status_code=status.HTTP_201_CREATED) +async def create_platform_employee( + payload: PlatformEmployeeCreate, + context: SystemAdminContext = Depends(system_admin_context), + session: AsyncSession = Depends(database_session), +) -> dict[str, Any]: + """Create a platform user without assigning workspace membership.""" + username = payload.username.strip() + display_name = payload.display_name.strip() + duplicate_conditions = [Users.username == username] + if payload.email: + duplicate_conditions.append(Users.email == payload.email.strip()) + duplicate = await session.scalar( + select(Users.user_id).where(or_(*duplicate_conditions)) + ) + if duplicate is not None: + raise HTTPException(status.HTTP_409_CONFLICT, "用户名或邮箱已存在") + + new_role: Roles | None = None + if payload.role_code is not None: + new_role = await _load_role_by_code(session, payload.role_code) + + user = Users( + user_id=new_ulid(), + username=username, + display_name=display_name, + email=payload.email.strip() if payload.email else None, + password_hash=hash_password(payload.password), + status="active", + platform_role_id=new_role.role_id if new_role is not None else None, + ) + session.add(user) + await session.flush() + await session.refresh(user) + return _envelope( + context.request_id, + platform_employee_payload(user, new_role), + ) + + +# 更新平台用户资料、账号状态或平台角色，同时保护最少管理员等约束。 +@router.patch("/employees/{user_id}") +async def update_platform_employee( + user_id: str, + payload: PlatformEmployeeUpdate, + context: SystemAdminContext = Depends(system_admin_context), + session: AsyncSession = Depends(database_session), +) -> dict[str, Any]: + """Update a platform user's profile, status, or platform role. + + Changing ``role_code`` cascades: every active ``workspace_members`` + row of the user is rewritten to the new role (workspace role is + inherited from the platform role). Demoting admin → developer is + rejected with 409 when it would leave any workspace without an + active admin member, or the platform without an active system + admin. Self-demotion is always rejected. + """ + user = await session.get(Users, user_id) + if user is None or user.is_deleted != 0: + raise HTTPException(status.HTTP_404_NOT_FOUND, "用户不存在") + + is_self = user_id == context.user.user_id + if ( + is_self + and payload.status is not None + and payload.status != "active" + ): + raise HTTPException(status.HTTP_409_CONFLICT, "不能停用当前登录账号") + + current_role: Roles | None = None + if user.platform_role_id is not None: + current_role = await session.scalar( + select(Roles).where(Roles.role_id == user.platform_role_id) + ) + is_current_system_admin = ( + user.status == "active" + and current_role is not None + and current_role.role_code == "admin" + ) + + new_role: Roles | None = None + if payload.role_code is not None: + new_role = await _load_role_by_code(session, payload.role_code) + next_status = payload.status if payload.status is not None else user.status + + leaves_admin_pool = ( + is_current_system_admin + and ( + next_status != "active" + or (new_role is not None and new_role.role_code != "admin") + ) + ) + if leaves_admin_pool: + remaining = await _count_active_system_admins( + session, exclude_user_id=user_id, + ) + if remaining == 0: + raise HTTPException( + status.HTTP_409_CONFLICT, + "platform 必须保留至少一个 active 系统管理员", + ) + + # Workspace-level last-admin guard for the demote path. The role_code + # sync below rewrites ``workspace_members.role_id`` for every active + # membership of this user, so demoting admin → developer would + # silently strip workspace admin coverage anywhere this user is the + # sole active admin member. ``update_member`` / ``remove_member`` + # guard the same invariant via ``_count_active_admins``; this + # endpoint must too, now that it can change workspace roles. + demotes_admin = ( + is_current_system_admin + and new_role is not None + and new_role.role_code != "admin" + ) + if demotes_admin: + assert current_role is not None # implied by is_current_system_admin + admin_memberships = ( + await session.execute( + select(WorkspaceMembers.workspace_id) + .where( + WorkspaceMembers.user_id == user_id, + WorkspaceMembers.role_id == current_role.role_id, + WorkspaceMembers.member_status == "active", + WorkspaceMembers.is_deleted == 0, + ) + ) + ).all() + orphaned: list[str] = [] + for (ws_id,) in admin_memberships: + remaining_ws = await _count_active_admins( + session, ws_id, exclude_user_id=user_id, + ) + if remaining_ws == 0: + orphaned.append(ws_id) + if orphaned: + codes = ( + await session.execute( + select(Workspaces.workspace_code).where( + Workspaces.workspace_id.in_(orphaned) + ) + ) + ).all() + names = sorted(row[0] for row in codes) + raise HTTPException( + status.HTTP_409_CONFLICT, + f"以下 workspace 将失去唯一 active admin: {names};" + "请先在这些 workspace 中指定其他 admin,再降级该用户", + ) + + if is_self and new_role is not None and new_role.role_code != "admin": + raise HTTPException(status.HTTP_409_CONFLICT, "不能降级自身管理员角色") + + if payload.display_name is not None: + user.display_name = payload.display_name.strip() + if payload.email is not None: + user.email = payload.email.strip() or None + if payload.status is not None: + user.status = payload.status + if new_role is not None: + user.platform_role_id = new_role.role_id + # Workspace role is always inherited from the platform role + # (§7.5/§7.6 cannot change it). Keep workspace_members.role_id + # in sync so downstream reads — `/me` workspaces[].role_code, + # load_active_membership, §7.7 DELETE last-admin guard — + # see the up-to-date role. Without this sync, a user demoted + # from admin → developer would still appear as admin in every + # workspace they belong to until they leave and re-join. + await session.execute( + update(WorkspaceMembers) + .where( + WorkspaceMembers.user_id == user.user_id, + WorkspaceMembers.is_deleted == 0, + ) + .values(role_id=new_role.role_id) + ) + + await session.flush() + await session.refresh(user) + + response_role: Roles | None = None + if user.platform_role_id is not None: + response_role = await session.scalar( + select(Roles).where(Roles.role_id == user.platform_role_id) + ) + return _envelope( + context.request_id, platform_employee_payload(user, response_role), + ) + + +# 软删除平台用户，并级联标记其工作区成员关系为删除。 +@router.delete("/employees/{user_id}") +async def delete_platform_employee( + user_id: str, + context: SystemAdminContext = Depends(system_admin_context), + session: AsyncSession = Depends(database_session), +) -> dict[str, Any]: + """Soft delete a platform user and cascade-soft-delete workspace memberships.""" + user = await session.get(Users, user_id) + if user is None or user.is_deleted != 0: + raise HTTPException(status.HTTP_404_NOT_FOUND, "用户不存在") + + if user_id == context.user.user_id: + raise HTTPException(status.HTTP_409_CONFLICT, "不能删除当前登录账号") + + if user.status == "active" and user.platform_role_id is not None: + current_admin_role = await session.scalar( + select(Roles).where(Roles.role_id == user.platform_role_id) + ) + if current_admin_role is not None and current_admin_role.role_code == "admin": + remaining = await _count_active_system_admins( + session, exclude_user_id=user_id, + ) + if remaining == 0: + raise HTTPException( + status.HTTP_409_CONFLICT, + "platform 必须保留至少一个 active 系统管理员", + ) + + now = datetime.datetime.utcnow() + user.status = "disabled" + user.is_deleted = 1 + user.deleted_at = now + await session.execute( + update(WorkspaceMembers) + .where( + WorkspaceMembers.user_id == user_id, + WorkspaceMembers.is_deleted == 0, + ) + .values(is_deleted=1, deleted_at=now) + ) + await session.flush() + return _envelope( + context.request_id, + {"user_id": user_id, "deleted": True}, + ) + + +# --------------------------------------------------------------------------- +# Workspace CRUD +# --------------------------------------------------------------------------- + + +# 列出平台中全部未删除工作区。 +@router.get("/workspaces") +async def list_workspaces( + context: SystemAdminContext = Depends(system_admin_context), + session: AsyncSession = Depends(database_session), +) -> dict[str, Any]: + """List active/archived workspaces. Soft-deleted rows are filtered out. + + Silent ``pageSize=100`` cap — YAGNI on real pagination until needed. + """ + rows = ( + await session.execute( + select(Workspaces) + .where( + Workspaces.status != "disabled", + Workspaces.is_deleted == 0, + ) + .order_by(Workspaces.created_at, Workspaces.workspace_id) + .limit(LIST_PAGE_SIZE) + ) + ).scalars().all() + return _envelope( + context.request_id, + [workspace_payload(w) for w in rows], + {"count": len(rows), "page_size": LIST_PAGE_SIZE}, + ) + + +# 创建工作区，并将当前系统管理员初始化为该工作区管理员。 +@router.post("/workspaces", status_code=status.HTTP_201_CREATED) +async def create_workspace( + payload: WorkspaceCreate, + context: SystemAdminContext = Depends(system_admin_context), + session: AsyncSession = Depends(database_session), +) -> dict[str, Any]: + """Create a workspace and auto-join the creator as an admin member.""" + if not WORKSPACE_CODE_PATTERN.fullmatch(payload.workspace_code): + raise HTTPException( + status.HTTP_422_UNPROCESSABLE_ENTITY, + "workspace_code 必须匹配 ^[a-z0-9-]{3,32}$", + ) + duplicate = await session.scalar( + select(Workspaces.workspace_id).where( + Workspaces.workspace_code == payload.workspace_code, + ) + ) + if duplicate is not None: + raise HTTPException(status.HTTP_409_CONFLICT, "workspace_code 已存在") + + admin_role = await _load_role_by_code(session, "admin") + workspace_id = new_ulid() + workspace = Workspaces( + workspace_id=workspace_id, + workspace_code=payload.workspace_code, + workspace_name=payload.workspace_name, + active_root_uri=f"s3://workspaces/{workspace_id}/", + quota_bytes=payload.quota_bytes, + status="active", + created_by=context.user.user_id, + description=payload.description, + ) + session.add(workspace) + session.add( + WorkspaceMembers( + workspace_id=workspace_id, + user_id=context.user.user_id, + role_id=admin_role.role_id, + member_status="active", + ) + ) + await session.flush() + await session.refresh(workspace) + return _envelope(context.request_id, workspace_payload(workspace)) + + +# 读取单个工作区详情，包含已归档或禁用状态。 +@router.get("/workspaces/{workspace_id}") +async def get_workspace( + workspace_id: str, + context: SystemAdminContext = Depends(system_admin_context), + session: AsyncSession = Depends(database_session), +) -> dict[str, Any]: + """Fetch a single workspace — even soft-deleted ones are reachable.""" + workspace = await _load_workspace(session, workspace_id) + return _envelope(context.request_id, workspace_payload(workspace)) + + +# 更新工作区可编辑属性，例如名称、配额、描述和归档状态。 +@router.patch("/workspaces/{workspace_id}") +async def update_workspace( + workspace_id: str, + payload: WorkspaceUpdate, + context: SystemAdminContext = Depends(system_admin_context), + session: AsyncSession = Depends(database_session), +) -> dict[str, Any]: + """Patch editable workspace fields. ``status='disabled'`` is rejected.""" + workspace = await _load_workspace(session, workspace_id) + if workspace.status == "disabled": + raise HTTPException( + status.HTTP_409_CONFLICT, + "workspace 已删除,无法修改", + ) + if payload.workspace_name is not None: + workspace.workspace_name = payload.workspace_name.strip() + if payload.quota_bytes is not None: + workspace.quota_bytes = payload.quota_bytes + if payload.description is not None: + workspace.description = payload.description + if payload.status is not None: + workspace.status = payload.status + await session.flush() + await session.refresh(workspace) + return _envelope(context.request_id, workspace_payload(workspace)) + + +# 软删除/禁用工作区，并级联处理其活动成员关系。 +@router.delete("/workspaces/{workspace_id}") +async def delete_workspace( + workspace_id: str, + context: SystemAdminContext = Depends(system_admin_context), + session: AsyncSession = Depends(database_session), +) -> dict[str, Any]: + """Soft-delete a workspace and cascade-soft-delete its memberships. + + Allowed from any non-disabled status (active or archived). The + membership cascade is what lets system admins leave a workspace — + there is no per-member DELETE escape for self-removal. + """ + workspace = await _load_workspace(session, workspace_id) + if workspace.status == "disabled": + raise HTTPException( + status.HTTP_409_CONFLICT, + "workspace 已被删除", + ) + now = datetime.datetime.utcnow() + workspace.status = "disabled" + workspace.is_deleted = 1 + workspace.deleted_at = now + await session.execute( + update(WorkspaceMembers) + .where( + WorkspaceMembers.workspace_id == workspace_id, + WorkspaceMembers.is_deleted == 0, + ) + .values(is_deleted=1, deleted_at=now) + ) + await session.flush() + await session.refresh(workspace) + return _envelope(context.request_id, workspace_payload(workspace)) + + +# --------------------------------------------------------------------------- +# Workspace membership CRUD +# --------------------------------------------------------------------------- + + +# 列出一个工作区的活动成员与成员状态。 +@router.get("/workspaces/{workspace_id}/members") +async def list_members( + workspace_id: str, + context: SystemAdminContext = Depends(system_admin_context), + session: AsyncSession = Depends(database_session), +) -> dict[str, Any]: + """List active and historical (non-soft-deleted) members of a workspace.""" + await _load_workspace(session, workspace_id) + rows = ( + await session.execute( + select(Users, Roles, WorkspaceMembers) + .join( + WorkspaceMembers, + WorkspaceMembers.user_id == Users.user_id, + ) + .join(Roles, Roles.role_id == WorkspaceMembers.role_id) + .where( + WorkspaceMembers.workspace_id == workspace_id, + WorkspaceMembers.is_deleted == 0, + ) + .order_by(WorkspaceMembers.joined_at, Users.user_id) + .limit(LIST_PAGE_SIZE) + ) + ).all() + return _envelope( + context.request_id, + [member_payload(u, r, m) for u, r, m in rows], + {"count": len(rows), "page_size": LIST_PAGE_SIZE}, + ) + + +# 将已有平台用户加入指定工作区。 +@router.post( + "/workspaces/{workspace_id}/members", + status_code=status.HTTP_201_CREATED, +) +async def add_member( + workspace_id: str, + payload: MemberCreate, + context: SystemAdminContext = Depends(system_admin_context), + session: AsyncSession = Depends(database_session), +) -> dict[str, Any]: + """Add a user to a workspace. The new row starts with member_status='active'. + + The role is inherited from the target user's ``platform_role_id``; + the request body does NOT take a ``role_code``. To change a member's + role, PATCH ``/api/v1/platform/employees/{user_id}`` instead. + """ + await _load_workspace(session, workspace_id) + user = await session.get(Users, payload.user_id) + if user is None or user.is_deleted != 0: + raise HTTPException(status.HTTP_404_NOT_FOUND, "用户不存在") + if user.status != "active": + raise HTTPException( + status.HTTP_409_CONFLICT, + f"用户状态为 {user.status},无法加入 workspace", + ) + if user.platform_role_id is None: + raise HTTPException( + status.HTTP_409_CONFLICT, + "目标用户尚未分配平台角色,无法加入 workspace;" + "请先 PATCH /api/v1/platform/employees/{user_id} 设置 role_code", + ) + role = await session.scalar( + select(Roles).where( + Roles.role_id == user.platform_role_id, + Roles.is_deleted == 0, + ) + ) + if role is None: + raise HTTPException( + status.HTTP_422_UNPROCESSABLE_ENTITY, + "用户的平台角色行不存在或已被删除", + ) + # ``WorkspaceMembers`` 的主键是 ``(workspace_id, user_id)`` 复合 PK, + # 而 ``remove_member`` / ``delete_platform_employee`` 都是软删除 (保留行, + # 仅置 ``is_deleted=1``). 因此这里必须按主键查整行,而不是只看活跃行: + # 否则软删行会被 active-duplicate 检查漏过,然后 INSERT 直接撞 PK. + existing = await session.scalar( + select(WorkspaceMembers).where( + WorkspaceMembers.workspace_id == workspace_id, + WorkspaceMembers.user_id == payload.user_id, + ) + ) + if existing is not None: + if existing.is_deleted == 0: + raise HTTPException( + status.HTTP_409_CONFLICT, + "用户已是该 workspace 成员;workspace 角色继承自平台角色," + "要变更请 PATCH /api/v1/platform/employees/{user_id} 修改 role_code", + ) + # 复活软删除行. 保留 ``joined_at`` 作为历史记录;``role_id`` 重新继承 + # 当前用户的平台角色 (用户在中间可能改过 platform_role);清掉 + # ``deleted_at`` 标记本轮已不在软删状态. + existing.is_deleted = 0 + existing.deleted_at = None + existing.role_id = role.role_id + existing.member_status = "active" + await session.flush() + await session.refresh(existing) + return _envelope( + context.request_id, member_payload(user, role, existing), + ) + membership = WorkspaceMembers( + workspace_id=workspace_id, + user_id=payload.user_id, + role_id=role.role_id, + member_status="active", + ) + session.add(membership) + await session.flush() + await session.refresh(membership) + return _envelope(context.request_id, member_payload(user, role, membership)) + + +# 更新成员状态，例如禁用或锁定；同时保证工作区不会失去最后一个管理员。 +@router.patch("/workspaces/{workspace_id}/members/{user_id}") +async def update_member( + workspace_id: str, + user_id: str, + payload: MemberUpdate, + context: SystemAdminContext = Depends(system_admin_context), + session: AsyncSession = Depends(database_session), +) -> dict[str, Any]: + """Update a workspace membership's status. Role is not editable here. + + Workspace role is always inherited from the user's platform role + (``Users.platform_role_id``). To change role, PATCH + ``/api/v1/platform/employees/{user_id}`` instead. + + Last-admin guard still applies to ``member_status`` changes: setting + the only active admin to ``disabled``/``locked`` would leave the + workspace without admin coverage. + """ + await _load_workspace(session, workspace_id) + row = ( + await session.execute( + select(Users, Roles, WorkspaceMembers) + .join( + WorkspaceMembers, + WorkspaceMembers.user_id == Users.user_id, + ) + .join(Roles, Roles.role_id == WorkspaceMembers.role_id) + .where( + WorkspaceMembers.workspace_id == workspace_id, + WorkspaceMembers.user_id == user_id, + WorkspaceMembers.is_deleted == 0, + ) + ) + ).first() + if row is None: + raise HTTPException(status.HTTP_404_NOT_FOUND, "成员不存在") + user, role, membership = row + + if payload.member_status is not None and payload.member_status != membership.member_status: + if ( + role.role_code == "admin" + and payload.member_status != "active" + ): + remaining = await _count_active_admins( + session, workspace_id, exclude_user_id=user_id, + ) + if remaining == 0: + raise HTTPException( + status.HTTP_409_CONFLICT, + "workspace 必须保留至少一个 admin", + ) + membership.member_status = payload.member_status + + await session.flush() + await session.refresh(membership) + return _envelope(context.request_id, member_payload(user, role, membership)) + + +# 移除某个工作区成员，并保护最后一名管理员及当前操作者的安全约束。 +@router.delete("/workspaces/{workspace_id}/members/{user_id}") +async def remove_member( + workspace_id: str, + user_id: str, + context: SystemAdminContext = Depends(system_admin_context), + session: AsyncSession = Depends(database_session), +) -> dict[str, Any]: + """Soft-delete a workspace membership. + + System admins cannot remove themselves — the only escape is to delete + the entire workspace, which cascades membership soft-deletion. + """ + await _load_workspace(session, workspace_id) + if user_id == context.user.user_id: + raise HTTPException( + status.HTTP_403_FORBIDDEN, + "系统管理员不能把自己从 workspace 移除;如需退出,请删除整个 workspace", + ) + row = ( + await session.execute( + select(Roles, WorkspaceMembers) + .join(Roles, Roles.role_id == WorkspaceMembers.role_id) + .where( + WorkspaceMembers.workspace_id == workspace_id, + WorkspaceMembers.user_id == user_id, + WorkspaceMembers.is_deleted == 0, + ) + ) + ).first() + if row is None: + raise HTTPException(status.HTTP_404_NOT_FOUND, "成员不存在") + role, membership = row + if role.role_code == "admin" and membership.member_status == "active": + remaining = await _count_active_admins( + session, workspace_id, exclude_user_id=user_id, + ) + if remaining == 0: + raise HTTPException( + status.HTTP_409_CONFLICT, + "workspace 必须保留至少一个 admin", + ) + membership.is_deleted = 1 + membership.deleted_at = datetime.datetime.utcnow() + await session.flush() + return _envelope( + context.request_id, + {"workspace_id": workspace_id, "user_id": user_id, "removed": True}, + ) + + +# --------------------------------------------------------------------------- +# Platform role permission management +# --------------------------------------------------------------------------- +# +# Menu permissions for the platform admin UI. The auth gate +# (system_admin_context) still keys off role_code == "admin"; these +# endpoints only control the menu items the frontend renders, not +# which API calls a user may make. See migrations/ +# versions/e5f6a7b8c9d0_seed_role_permissions_and_fix_scope.py for +# the seed values. + + +async def _load_platform_role_by_code( + session: AsyncSession, role_code: str +) -> Roles: + """Load a platform-scoped role by code; 404 if missing or not platform-scope.""" + role = await session.scalar( + select(Roles).where( + Roles.role_code == role_code, Roles.is_deleted == 0, + ) + ) + if role is None or role.role_scope != "platform": + raise HTTPException( + status.HTTP_404_NOT_FOUND, f"platform 角色 {role_code} 不存在", + ) + return role + + +async def _load_role_permission_codes( + session: AsyncSession, role_id: str +) -> list[str]: + """Return the active permission_codes for a role, ordered by code.""" + rows = ( + await session.execute( + select(Permissions.permission_code) + .join( + RolePermissions, + RolePermissions.permission_id == Permissions.permission_id, + ) + .where( + RolePermissions.role_id == role_id, + RolePermissions.is_deleted == 0, + Permissions.is_deleted == 0, + ) + .order_by(Permissions.permission_code) + ) + ).all() + return [row[0] for row in rows] + + +def _role_payload(role: Roles, permission_codes: list[str]) -> dict[str, Any]: + return { + "role_id": role.role_id, + "role_code": role.role_code, + "role_name": role.role_name, + "is_builtin": bool(role.is_builtin), + "permission_codes": permission_codes, + } + + +# 列出平台角色及其拥有的菜单权限代码。 +@router.get("/roles") +async def list_platform_roles( + context: SystemAdminContext = Depends(system_admin_context), + session: AsyncSession = Depends(database_session), +) -> dict[str, Any]: + """List every platform-scoped role with its current permission_codes.""" + roles = ( + ( + await session.scalars( + select(Roles) + .where(Roles.role_scope == "platform", Roles.is_deleted == 0) + .order_by(Roles.role_code) + ) + ).all() + ) + payload = [] + for role in roles: + codes = await _load_role_permission_codes(session, role.role_id) + payload.append(_role_payload(role, codes)) + return _envelope( + context.request_id, payload, {"count": len(payload)}, + ) + + +# 获取一个角色当前配置的权限代码集合。 +@router.get("/roles/{role_code}/permissions") +async def get_role_permissions( + role_code: str, + context: SystemAdminContext = Depends(system_admin_context), + session: AsyncSession = Depends(database_session), +) -> dict[str, Any]: + """Return one platform role's permission_codes.""" + role = await _load_platform_role_by_code(session, role_code) + codes = await _load_role_permission_codes(session, role.role_id) + return _envelope( + context.request_id, _role_payload(role, codes), + ) + + +# 以请求中的完整集合更新角色权限，并保留管理员角色的必要系统权限。 +@router.patch("/roles/{role_code}/permissions") +async def patch_role_permissions( + role_code: str, + payload: RolePermissionsPatch, + context: SystemAdminContext = Depends(system_admin_context), + session: AsyncSession = Depends(database_session), +) -> dict[str, Any]: + """Replace a platform role's permission set wholesale. + + Guard order: + + 1. Load the role. Reject 404 if it is missing or not + platform-scoped. + 2. Admin role: the patched ``permission_codes`` MUST still include + both ``system.view`` and ``system.manage``. Otherwise every + active admin loses the menu entry to this very endpoint and + the platform locks itself out. Reject with 409. (No last-admin + count is needed here — menu permissions never gate API access; + ``system_admin_context`` keys off ``role_code == 'admin'``.) + 3. Non-admin role: ``system.*`` codes are rejected with 422 — + they would render a system-admin menu entry whose API calls + all 403. + 4. Validate every code resolves to a non-deleted ``Permissions`` + row; unknown codes → 422. + 5. Write: diff-based. Only soft-delete codes leaving the set, + only insert codes entering it. The ``(role_id, permission_id)`` + PRIMARY KEY still occupies soft-deleted rows, so a blanket + "delete-all then insert-all" would IntegrityError. + Repeat-with-same-payload is a no-op. + """ + role = await _load_platform_role_by_code(session, role_code) + + new_codes = list(dict.fromkeys(payload.permission_codes)) + + if role.role_code == "admin": + keeps_admin_entry = ( + "system.view" in new_codes and "system.manage" in new_codes + ) + if not keeps_admin_entry: + raise HTTPException( + status.HTTP_409_CONFLICT, + "admin 角色必须保留 system.view 与 system.manage 权限", + ) + else: + # Menu permissions are a frontend-display signal only — backend + # authorization keeps keying off role_code == "admin". Letting a + # non-admin role hold system.* permissions would render the + # system-admin entry in the developer's UI while every + # /api/v1/platform/* call still returns 403. Reject with 422 so + # the failure is unambiguous about *what* the input violated. + leaked_system = [ + code for code in new_codes if code.startswith("system.") + ] + if leaked_system: + raise HTTPException( + status.HTTP_422_UNPROCESSABLE_ENTITY, + f"非 admin 角色不能拥有 system.* 权限: {leaked_system}", + ) + + # 4. Validate every requested permission_code exists and is live. + if new_codes: + rows = ( + await session.execute( + select(Permissions.permission_code).where( + Permissions.permission_code.in_(new_codes), + Permissions.is_deleted == 0, + ) + ) + ).all() + found = {row[0] for row in rows} + missing = [code for code in new_codes if code not in found] + if missing: + raise HTTPException( + status.HTTP_422_UNPROCESSABLE_ENTITY, + f"未知的 permission_code: {missing}", + ) + + # 5. Write: diff-based soft-delete + insert. + # The (role_id, permission_id) PRIMARY KEY still occupies the slot + # of soft-deleted rows, so a "delete-all then insert-all" approach + # would IntegrityError on any code that was already linked. + # Instead: only soft-delete codes NOT in the new set, only INSERT + # codes NOT already active. Repeat-with-same-payload is a no-op. + now = datetime.datetime.utcnow() + current_codes = set( + await _load_role_permission_codes(session, role.role_id) + ) + new_set = set(new_codes) + + codes_to_drop = current_codes - new_set + codes_to_add = new_set - current_codes + + if codes_to_drop: + # Resolve to permission_ids then soft-delete by id pair. + drop_ids = ( + await session.execute( + select(Permissions.permission_id).where( + Permissions.permission_code.in_(codes_to_drop), + Permissions.is_deleted == 0, + ) + ) + ).all() + drop_id_values = [row[0] for row in drop_ids] + await session.execute( + update(RolePermissions) + .where( + RolePermissions.role_id == role.role_id, + RolePermissions.permission_id.in_(drop_id_values), + RolePermissions.is_deleted == 0, + ) + .values(is_deleted=1, deleted_at=now) + ) + + if codes_to_add: + add_ids = ( + await session.execute( + select(Permissions.permission_id).where( + Permissions.permission_code.in_(codes_to_add), + Permissions.is_deleted == 0, + ) + ) + ).all() + if add_ids: + await session.execute( + insert(RolePermissions), + [ + {"role_id": role.role_id, "permission_id": pid} + for pid, in add_ids + ], + ) + + await session.flush() + final_codes = await _load_role_permission_codes(session, role.role_id) + return _envelope( + context.request_id, _role_payload(role, final_codes), + ) + + +__all__ = [ + "SystemAdminContext", + "router", + "system_admin_context", +] diff --git a/backend/src/backend/rclone_rc_client.py b/backend/src/backend/rclone_rc_client.py new file mode 100644 index 0000000..0be8d55 --- /dev/null +++ b/backend/src/backend/rclone_rc_client.py @@ -0,0 +1,92 @@ +"""rclone RC client. + +Used by the backend to invalidate the rclone FUSE directory cache after +writing a new workspace object (notebook / script / upload). The runtime +container already starts rclone with ``--rc --rc-addr 0.0.0.0:5572 +--rc-no-auth`` (see ``runtime/src/runtime/mount.py``), so we only need +an HTTP client here — no extra runtime plumbing. + +Failure is logged and swallowed: the VFS refresh is best-effort and +must never bubble up to the API caller. +""" + +from __future__ import annotations + +from dataclasses import dataclass +from typing import Any + +import httpx +from loguru import logger + + +@dataclass(frozen=True) +class RcloneRCError(Exception): + status_code: int + detail: Any + + +_RCLONE_RC_TRANSPORT_ERROR = RcloneRCError( + 503, + { + "code": "RCLONE_RC_UNAVAILABLE", + "message": "rclone RC 暂时不可用", + "retryable": True, + "details": {}, + }, +) + + +class RcloneRCClient: + def __init__(self, client: httpx.AsyncClient) -> None: + self.client = client + + async def _request( + self, + method: str, + path: str, + payload: dict[str, Any], + ) -> dict[str, Any]: + try: + response = await self.client.request(method, path, json=payload) + except httpx.RequestError as exc: + raise _RCLONE_RC_TRANSPORT_ERROR from exc + if response.is_error: + try: + detail = response.json().get("detail", response.text) + except ValueError: + detail = response.text + raise RcloneRCError(response.status_code, detail) + return response.json() + + async def vfs_refresh( + self, + dir_path: str, + *, + recursive: bool = True, + ) -> None: + """Invalidate the FUSE dir-cache for ``dir_path``. + + Fires ``POST /vfs/refresh`` against the rclone RC server with + ``_async=true`` so the call returns immediately while rclone + performs the directory walk in the background. Errors are + logged and swallowed — a refresh failure must never fail the + API call that triggered it. + """ + try: + await self._request( + "POST", + "/vfs/refresh", + { + "dir": dir_path, + "recursive": recursive, + "_async": True, + }, + ) + logger.info( + f"vfs_refresh dir={dir_path} recursive={recursive} ok" + ) + except Exception as exc: # noqa: BLE001 - best-effort + logger.warning(f"vfs_refresh dir={dir_path} failed: {exc}") + + +__all__ = ["RcloneRCClient", "RcloneRCError"] \ No newline at end of file diff --git a/backend/src/backend/resources.py b/backend/src/backend/resources.py new file mode 100644 index 0000000..bd4baf1 --- /dev/null +++ b/backend/src/backend/resources.py @@ -0,0 +1,523 @@ +"""工作区数据资源 API。 + +资源上传分为三步：创建上传会话 → 写入文件字节 → 绑定为可见的数据资源。 +这种拆分使前端可以分别处理元数据、文件传输和最终展示；资源文件本身由存储层 +保存，数据库只保存资源与存储对象的关联关系。 +""" + +from __future__ import annotations + +import os +from datetime import UTC, datetime +from pathlib import Path, PurePosixPath +from typing import Any + +from common.db.models import DataResources, StorageObjects +from common.ids import new_ulid +from common.storage import workspaces_root +from common.storage.schemas import ( + CreateUploadRequest, + DownloadUrlRequest, +) +from fastapi import APIRouter, Depends, Header, HTTPException, Query, Request, status +from sqlalchemy import func, select +from sqlalchemy.ext.asyncio import AsyncSession + +from backend.dependencies import ( + RequestContext, + database_session, + request_context, +) +from backend.schemas import ( + CompleteResourceUploadRequest, + CreateResourceUploadRequest, + DownloadUrlRequest, + ResourceRelativePathRequest, +) +from backend.services.storage import ( + acquire_named_lock, + create_download_url_payload, + create_upload_record, + release_named_lock, + soft_delete_object, + upload_bytes_to_session, +) + +router = APIRouter(prefix="/api/v1/data-resources", tags=["data-resources"]) + + +def compute_jupyter_relative_path(script_path: str, resource_relative: str) -> str: + """从当前脚本所在目录算到资源文件的 Jupyter 相对路径。 + + script_path / resource_relative 都是相对于 user root_dir 的 POSIX 路径。 + """ + script_dir = PurePosixPath(script_path).parent.as_posix() + if not script_dir or script_dir == ".": + return resource_relative + return os.path.relpath(resource_relative, start=script_dir) + + +def resource_directory( + object_key: str, + workspace_id: str, + owner_user_id: str, +) -> str: + """从 object_key 解析资源所在目录(相对于用户根目录,根目录返回 "")。 + + object_key 形如 ``{ws_id}/{user_id}/{target_path}/{file_name}``; + 不匹配该前缀的键(如无 ws/user 前缀的旧数据)统一视为根目录。 + """ + prefix = f"{workspace_id}/{owner_user_id}/" + if not object_key.startswith(prefix): + return "" + tail = object_key[len(prefix):] + directory, _, _ = tail.rpartition("/") + return directory + + +def resource_payload( + resource: DataResources, + storage_object: StorageObjects, +) -> dict[str, Any]: + # ``object_key`` now follows ``{ws_id}/{user_id}/{target_path}/{file_name}`` + # (target_path may be empty). Legacy objects still live under + # ``.resources/{file_name}`` and must remain readable. Both shapes share + # the same derivation: strip the workspace/user prefix and use the rest + # as the Jupyter-relative path. + workspace_prefix = f"{resource.workspace_id}/" + user_prefix = f"{resource.owner_user_id}/" + jupyter_accessible_path = "" + absolute_path = "" + if storage_object.object_key and storage_object.object_key.startswith( + workspace_prefix + ): + remainder = storage_object.object_key[len(workspace_prefix):] + if remainder.startswith(user_prefix): + tail = remainder[len(user_prefix):] + jupyter_accessible_path = tail + absolute_path = ( + workspaces_root() + / resource.workspace_id + / resource.owner_user_id + / Path(tail) + ).as_posix() + else: + jupyter_accessible_path = remainder + elif storage_object.object_key: + jupyter_accessible_path = storage_object.object_key + return { + "resource_id": resource.resource_id, + "workspace_id": resource.workspace_id, + "storage_object_id": resource.storage_object_id, + "owner_user_id": resource.owner_user_id, + "resource_name": resource.resource_name, + "description": resource.description, + "visibility": resource.visibility, + "status": resource.status, + "created_at": resource.created_at.isoformat(), + "updated_at": resource.updated_at.isoformat(), + "file": { + "file_name": storage_object.file_name, + "file_extension": storage_object.file_extension, + "mime_type": storage_object.mime_type, + "size_bytes": storage_object.size_bytes, + "content_hash": storage_object.content_hash, + "object_status": storage_object.object_status, + }, + "jupyter_accessible_path": jupyter_accessible_path, + "absolute_path": absolute_path, + } + + +def can_view(resource: DataResources, context: RequestContext) -> bool: + # 2026-08-11: 临时取消"用户间目录互相不可见"约束 + # 同一 workspace 内的成员现在可以查看彼此的 private 资源。 + # 还原: 取消下方注释,恢复 owner_user_id 检查。 + return ( + # resource.owner_user_id == context.user.user_id + # or + resource.visibility in {"workspace", "public"} + or context.is_admin + ) + + +# 根据当前脚本位置计算资源的相对路径，便于 Notebook 中用相对路径读取文件。 +@router.post("/{resource_id}/jupyter-relative-path") +async def resource_jupyter_relative_path( + resource_id: str, + payload: ResourceRelativePathRequest, + context: RequestContext = Depends(request_context), + session: AsyncSession = Depends(database_session), +) -> dict[str, Any]: + resource, storage_object = await get_visible_resource( + resource_id, context, session + ) + script_path = payload.script_path.strip() + if ( + script_path.startswith("/") + or "\\" in script_path + or any(seg == ".." for seg in script_path.split("/")) + or any(ord(c) < 0x20 or ord(c) == 0x7F for c in script_path) + ): + raise HTTPException(422, "script_path must be a clean Jupyter-relative POSIX path") + + resource_path = resource_payload(resource, storage_object)["jupyter_accessible_path"] + # Both legacy ``.resources/`` files and new flat/nested paths are valid. + relative = compute_jupyter_relative_path(script_path, resource_path) + return { + "request_id": context.request_id, + "data": {"relative_path": relative}, + "meta": {}, + } + + +# 上传第 1 步：创建上传会话，登记文件名、大小、类型等预期元数据。 +@router.post("/uploads", status_code=status.HTTP_201_CREATED) +async def create_resource_upload( + payload: CreateResourceUploadRequest, + request: Request, + context: RequestContext = Depends(request_context), + session: AsyncSession = Depends(database_session), + idempotency_key: str = Header( + min_length=8, + max_length=128, + alias="Idempotency-Key", + ), +) -> dict[str, Any]: + data = await create_upload_record( + CreateUploadRequest( + workspace_id=context.workspace.workspace_id, + user_id=context.user.user_id, + usage_type="data_resource", + file_name=payload.file_name, + content_type=payload.content_type, + expected_size_bytes=payload.expected_size_bytes, + expected_hash=payload.expected_hash, + idempotency_key=idempotency_key, + target_path=payload.target_path, + ), + session, + request, + ) + return {"request_id": context.request_id, "data": data, "meta": {}} + + +# 上传第 2 步：将浏览器传来的二进制文件写入已创建的上传会话。 +@router.put("/uploads/{upload_id}") +async def upload_resource_bytes( + upload_id: str, + request: Request, + context: RequestContext = Depends(request_context), + session: AsyncSession = Depends(database_session), +) -> dict[str, Any]: + """Server-proxied upload step 2: PUT the raw bytes here. + + Replaces the old 3-step presign-PUT flow. The new flow is: + POST /uploads → {upload_id, upload_path, ...} + PUT /uploads/{upload_id} ← this route + (3) The frontend then calls a separate bind route to attach the + resulting StorageObjects row to a DataResources row. + """ + item = await upload_bytes_to_session(upload_id, session, request) + return { + "request_id": context.request_id, + "data": {"storage_object_id": item.storage_object_id}, + "meta": {}, + } + + +# 上传第 3 步：把已完成的上传会话绑定为工作区可见的数据资源。 +@router.post("/uploads/{upload_id}/bind") +async def bind_resource( + upload_id: str, + payload: CompleteResourceUploadRequest, + request: Request, + context: RequestContext = Depends(request_context), + session: AsyncSession = Depends(database_session), +) -> dict[str, Any]: + """Bind a completed upload to a DataResources row. + + Caller must have already PUT the bytes (see ``PUT /uploads/{id}``). + This route attaches the resource_name / description / visibility to + the StorageObjects row + creates the DataResources row that points + to it. + """ + from common.db.models import UploadSessions + upload = await session.scalar( + select(UploadSessions).where(UploadSessions.upload_id == upload_id) + ) + if upload is None: + raise HTTPException(status.HTTP_404_NOT_FOUND, "upload not found") + if upload.storage_object_id is None: + raise HTTPException( + status.HTTP_409_CONFLICT, + "upload has no completed object; PUT the bytes first", + ) + if upload.workspace_id != context.workspace.workspace_id: + raise HTTPException( + status.HTTP_403_FORBIDDEN, + "upload belongs to another workspace", + ) + if upload.user_id != context.user.user_id: + raise HTTPException( + status.HTTP_403_FORBIDDEN, + "upload belongs to another user", + ) + if upload.usage_type != "data_resource": + raise HTTPException( + status.HTTP_409_CONFLICT, + "upload was not created for a data resource", + ) + item = await session.get(StorageObjects, upload.storage_object_id) + if item is None: + raise HTTPException( + status.HTTP_409_CONFLICT, + "storage object metadata is missing", + ) + + # 幂等重试:同一 storage object 已有 active 的资源行,直接返回, + # 不重复创建、不触发同名查重。已删除的旧绑定行(status != active) + # 不会命中,走下方新建流程。 + existing = await session.scalar( + select(DataResources).where( + DataResources.storage_object_id == item.storage_object_id, + DataResources.status == "active", + ) + ) + if existing is not None: + return { + "request_id": context.request_id, + "data": resource_payload(existing, item), + "meta": {"reused": True}, + } + + # Persist resource_name / description / visibility override. + item.visibility = payload.visibility + # (description lives on DataResources, not on StorageObjects.) + + # 同名查重按「owner + 目录 + 名称」维度:目录从 object_key 解析, + # 不同目录、不同 owner 均允许重名。 + new_directory = resource_directory( + item.object_key, + context.workspace.workspace_id, + context.user.user_id, + ) + # 查重 + 建行没有唯一索引兜底,按 (owner, 目录, 名称) 加命名锁, + # 避免两个并发 bind 同时通过检查产生重复行。 + lock_name = await acquire_named_lock( + session, + f"bind-resource:{context.workspace.workspace_id}:" + f"{context.user.user_id}:{new_directory}:{payload.resource_name}", + ) + try: + same_name_rows = ( + await session.execute( + select(DataResources, StorageObjects) + .join( + StorageObjects, + StorageObjects.storage_object_id + == DataResources.storage_object_id, + ) + .where( + DataResources.workspace_id == context.workspace.workspace_id, + DataResources.owner_user_id == context.user.user_id, + DataResources.resource_name == payload.resource_name, + DataResources.status == "active", + ) + ) + ).all() + for existing_resource, existing_object in same_name_rows: + existing_directory = resource_directory( + existing_object.object_key, + existing_resource.workspace_id, + existing_resource.owner_user_id, + ) + if existing_directory == new_directory: + raise HTTPException( + status.HTTP_409_CONFLICT, + "a data resource with this name already exists in this directory", + ) + + existing = DataResources( + resource_id=new_ulid(), + workspace_id=context.workspace.workspace_id, + storage_object_id=item.storage_object_id, + owner_user_id=context.user.user_id, + resource_name=payload.resource_name, + description=payload.description, + visibility=payload.visibility, + status="active", + ) + session.add(existing) + await session.flush() + await session.refresh(existing) + finally: + await release_named_lock(session, lock_name) + return { + "request_id": context.request_id, + "data": resource_payload(existing, item), + "meta": {"reused": False}, + } + + +# 列出当前工作区可见的数据资源，可按可见性或关键字筛选。 +@router.get("") +async def list_resources( + context: RequestContext = Depends(request_context), + session: AsyncSession = Depends(database_session), + visibility: str | None = Query(default=None), + keyword: str | None = Query(default=None, max_length=100), +) -> dict[str, Any]: + statement = ( + select(DataResources, StorageObjects) + .join( + StorageObjects, + StorageObjects.storage_object_id + == DataResources.storage_object_id, + ) + .where( + DataResources.workspace_id == context.workspace.workspace_id, + DataResources.status == "active", + StorageObjects.object_status == "available", + ) + .order_by(DataResources.updated_at.desc()) + ) + # 2026-08-11: 临时取消"用户间目录互相不可见"约束 + # 列表接口现在返回 workspace 内全部 active 资源(不再按 owner / visibility 过滤)。 + # 还原: 删除下面这段注释,恢复原来的 if not context.is_admin: ... 块。 + # if not context.is_admin: + # statement = statement.where( + # or_( + # DataResources.owner_user_id == context.user.user_id, + # DataResources.visibility.in_(["workspace", "public"]), + # ) + # ) + if visibility: + if visibility not in {"private", "workspace", "public"}: + raise HTTPException( + status.HTTP_422_UNPROCESSABLE_ENTITY, + "invalid visibility", + ) + statement = statement.where(DataResources.visibility == visibility) + if keyword: + statement = statement.where( + DataResources.resource_name.like(f"%{keyword.strip()}%") + ) + rows = (await session.execute(statement)).all() + return { + "request_id": context.request_id, + "data": [ + resource_payload(resource, storage_object) + for resource, storage_object in rows + ], + "meta": {"count": len(rows)}, + } + + +async def get_visible_resource( + resource_id: str, + context: RequestContext, + session: AsyncSession, +) -> tuple[DataResources, StorageObjects]: + row = ( + await session.execute( + select(DataResources, StorageObjects) + .join( + StorageObjects, + StorageObjects.storage_object_id + == DataResources.storage_object_id, + ) + .where( + DataResources.resource_id == resource_id, + DataResources.workspace_id + == context.workspace.workspace_id, + DataResources.status == "active", + ) + ) + ).one_or_none() + if row is None or not can_view(row[0], context): + raise HTTPException(status.HTTP_404_NOT_FOUND, "resource not found") + return row + + +# 查询单个数据资源的元数据与其关联文件信息。 +@router.get("/{resource_id}") +async def get_resource( + resource_id: str, + context: RequestContext = Depends(request_context), + session: AsyncSession = Depends(database_session), +) -> dict[str, Any]: + resource, storage_object = await get_visible_resource( + resource_id, + context, + session, + ) + return { + "request_id": context.request_id, + "data": resource_payload(resource, storage_object), + "meta": {}, + } + + +# 为资源文件生成带时效的下载链接。 +@router.post("/{resource_id}/download-url") +async def resource_download_url( + resource_id: str, + payload: DownloadUrlRequest, + request: Request, + context: RequestContext = Depends(request_context), + session: AsyncSession = Depends(database_session), +) -> dict[str, Any]: + resource, _ = await get_visible_resource( + resource_id, + context, + session, + ) + data = await create_download_url_payload( + await session.get(StorageObjects, resource.storage_object_id), + DownloadUrlRequest(expires_seconds=payload.expires_seconds), + request, + ) + return {"request_id": context.request_id, "data": data["data"], "meta": {}} + + +# 软删除数据资源及其关联对象，遵循存储层的回收站策略。 +@router.delete("/{resource_id}") +async def delete_resource( + resource_id: str, + request: Request, + context: RequestContext = Depends(request_context), + session: AsyncSession = Depends(database_session), +) -> dict[str, Any]: + resource, _ = await get_visible_resource( + resource_id, + context, + session, + ) + if ( + resource.owner_user_id != context.user.user_id + and not context.is_admin + ): + raise HTTPException( + status.HTTP_403_FORBIDDEN, + "resource can only be deleted by its owner or an administrator", + ) + # 底层 storage object 可能被其他 active 资源行共享;仍有引用时 + # 只删 DataResources 行,不动底层对象。 + shared_count = await session.scalar( + select(func.count(DataResources.resource_id)).where( + DataResources.storage_object_id == resource.storage_object_id, + DataResources.resource_id != resource.resource_id, + DataResources.status == "active", + ) + ) + if not shared_count: + await soft_delete_object(resource.storage_object_id, request, session) + resource.status = "deleted" + resource.deleted_at = datetime.now(UTC).replace(tzinfo=None) + resource.is_deleted = 1 + return { + "request_id": context.request_id, + "data": {"resource_id": resource_id, "status": "deleted"}, + "meta": {}, + } diff --git a/backend/src/backend/runtime_client.py b/backend/src/backend/runtime_client.py new file mode 100644 index 0000000..1ca859b --- /dev/null +++ b/backend/src/backend/runtime_client.py @@ -0,0 +1,361 @@ +"""Backend 调用 Runtime 服务的轻量 HTTP 客户端。 + +后端不直接管理 Jupyter 进程；涉及工作区运行时状态、编辑会话或访问票据时， +会通过本客户端请求 Docker 内网中的 ``runtime`` 服务。网络错误会统一包装成 +``RuntimeClientError``，让路由返回可识别的 503 错误。 +""" + +from __future__ import annotations + +from dataclasses import dataclass +from typing import Any + +import httpx +from loguru import logger + + +@dataclass(frozen=True) +class RuntimeClientError(Exception): + status_code: int + detail: Any + + +_RUNTIME_TRANSPORT_ERROR = RuntimeClientError( + 503, + { + "code": "RUNTIME_UNAVAILABLE", + "message": "Runtime Manager 暂时不可用", + "retryable": True, + "details": {}, + }, +) + + +class RuntimeClient: + def __init__(self, client: httpx.AsyncClient) -> None: + self.client = client + + async def _request( + self, + method: str, + path: str, + payload: dict[str, Any], + ) -> dict[str, Any]: + # 将 httpx 的连接/响应异常转换为项目统一的业务异常，调用方无需关心 + # 底层 HTTP 客户端的具体异常类型。 + try: + response = await self.client.request(method, path, json=payload) + except httpx.RequestError as exc: + raise _RUNTIME_TRANSPORT_ERROR from exc + if response.is_error: + try: + detail = response.json().get("detail", response.text) + except ValueError: + detail = response.text + raise RuntimeClientError(response.status_code, detail) + return response.json() + + async def get_workspace( + self, + workspace_id: str, + ) -> dict[str, Any] | None: + """Ask Runtime for a live workspace. + + Returns the workspace descriptor when the Jupyter process is + ``running``; returns ``None`` when the workspace is not yet + started or has been torn down so callers can fall through to + :meth:`start_workspace`. + """ + try: + return await self._request( + "POST", + "/api/v1/jupyter", + {"action": "get", "workspace_id": workspace_id}, + ) + except RuntimeClientError as exc: + if exc.status_code == 404: + return None + raise + + async def start_workspace( + self, + workspace_id: str, + ) -> dict[str, Any]: + return await self._request( + "POST", + "/api/v1/jupyter", + {"action": "start", "workspace_id": workspace_id}, + ) + + async def _ensure_workspace( + self, + workspace_id: str, + ) -> dict[str, Any]: + """Return a running workspace descriptor, starting it if needed. + + Mirrors the lazy-start pattern used by + :func:`backend.jupyter.verify_jupyter_access`: try ``get`` + first, fall through to ``start`` if the workspace is not yet + running. Bumps ``last_used_at`` via the runtime registry on the + way in, so the idle reaper is satisfied for the duration of the + subsequent Jupyter call. + """ + ws = await self.get_workspace(workspace_id) + if not ws or ws.get("status") != "running": + ws = await self.start_workspace(workspace_id) + if not ws.get("port") or not ws.get("token"): + raise RuntimeClientError( + 500, + { + "code": "JUPYTER_DESCRIPTOR_INVALID", + "message": "Runtime returned no port/token for workspace", + "retryable": False, + "details": {"workspace_id": workspace_id}, + }, + ) + return ws + + async def _jupyter_request( + self, + workspace_id: str, + ws: dict[str, Any], + method: str, + contents_path: str, + body: dict[str, Any] | None = None, + ) -> dict[str, Any]: + """Send a request to the workspace Jupyter's contents API. + + ``contents_path`` is the suffix after ``/api/contents/`` — + pass ``""`` for the root, ``"foo.ipynb"`` for a file, or + ``"foo.ipynb/checkpoints"`` for the checkpoint subresource. + The URL is built from the per-workspace ``base_url`` + ``port`` + and authenticated with the runtime-issued token. We reuse the + existing ``httpx.AsyncClient`` (its ``base_url`` is overridden + by the absolute URL we hand it). + """ + suffix = contents_path.lstrip("/") + url = ( + f"{ws['base_url']}:{ws['port']}" + f"/jupyter/{workspace_id}/api/contents/{suffix}" + ) + logger.debug(f"{method} {url}") + logger.debug(body) + headers = {"Authorization": f"token {ws['token']}"} + try: + if body is None: + response = await self.client.request( + method, url, headers=headers + ) + else: + response = await self.client.request( + method, url, json=body, headers=headers + ) + except httpx.RequestError as exc: + raise _RUNTIME_TRANSPORT_ERROR from exc + if response.is_error: + try: + detail = response.json() + except ValueError: + detail = response.text + raise RuntimeClientError(response.status_code, detail) + if response.status_code == 204 or not response.content: + return {} + return response.json() + + async def create_notebook( + self, + workspace_id: str, + *, + name: str, + cells: list[dict[str, Any]] | None = None, + checkpoint: bool = True, + ) -> dict[str, Any]: + """Create a notebook at the given path in the workspace. + + Uses Jupyter's ``PUT /api/contents/{name}`` (the + "save / create at path" verb — ``POST /api/contents/`` only + creates an auto-incremented ``Untitled.ipynb``). The body + carries ``type``, ``format`` and the full notebook ``content``. + + Auto-starts the workspace's Jupyter if it is not yet running. + When ``checkpoint`` is true (default), follows up with a + ``POST /api/contents/{name}/checkpoints`` so the file is + visible in the live notebook tree without an extra refresh — + checkpoint failures are logged but do not fail the create, + because the underlying ``PUT`` already persisted the bytes. + + ``name`` must be a safe basename or relative path; the Jupyter + side will reject anything that escapes the workspace root. + """ + ws = await self._ensure_workspace(workspace_id) + body = { + "type": "notebook", + "format": "json", + "content": { + "cells": cells if cells is not None else [], + "metadata": {}, + "nbformat": 4, + "nbformat_minor": 5, + }, + } + result = await self._jupyter_request( + workspace_id, ws, "PUT", name, body=body + ) + if checkpoint: + try: + await self._jupyter_request( + workspace_id, ws, "POST", f"{name}/checkpoints" + ) + except RuntimeClientError as exc: + logger.warning( + f"checkpoint after create_notebook({name}) failed: " + f"{exc.status_code} {exc.detail}" + ) + return result + + async def upload_file( + self, + workspace_id: str, + *, + name: str, + content: str, + content_type: str = "text/plain", + checkpoint: bool = True, + ) -> dict[str, Any]: + """Upload a text file to the workspace. + + Uses ``PUT /api/contents/{name}`` with ``type="file"`` and + ``format="text"``. ``content_type`` is accepted for API + symmetry with :meth:`create_notebook`; only ``text/*`` and + ``application/json`` are supported here — binary uploads + require the base64 pathway which is not yet wired up and + raise :class:`ValueError`. + + When ``checkpoint`` is true (default), the same follow-up + checkpoint call as :meth:`create_notebook` is made; failures + are swallowed. + """ + if not ( + content_type.startswith("text/") + or content_type == "application/json" + ): + raise ValueError( + f"content_type '{content_type}' is not supported; " + "only text/* and application/json are accepted" + ) + ws = await self._ensure_workspace(workspace_id) + body = { + "type": "file", + "format": "text", + "content": content, + } + result = await self._jupyter_request( + workspace_id, ws, "PUT", name, body=body + ) + if checkpoint: + try: + await self._jupyter_request( + workspace_id, ws, "POST", f"{name}/checkpoints" + ) + except RuntimeClientError as exc: + logger.warning( + f"checkpoint after upload_file({name}) failed: " + f"{exc.status_code} {exc.detail}" + ) + return result + + async def delete_file( + self, + workspace_id: str, + *, + name: str, + ) -> None: + """Delete a file from the workspace Jupyter. + + Uses ``DELETE /api/contents/{name}`` on the workspace Jupyter. + Jupyter returns ``204 No Content`` on success; any 4xx/5xx is + surfaced as :class:`RuntimeClientError`. Non-recursive — Jupyter + rejects directory deletes unless the directory is empty. + """ + ws = await self._ensure_workspace(workspace_id) + await self._jupyter_request(workspace_id, ws, "DELETE", name) + + async def get_file( + self, + workspace_id: str, + *, + name: str, + ) -> dict[str, Any]: + """Read a file's contents descriptor from the workspace Jupyter. + + Returns the Jupyter contents payload (``type``, ``content``, + ``format``, ``mimetype``, ``size``, ...). For ``type="notebook"`` + the ``content`` field is the notebook dict + (``cells``/``metadata``/``nbformat``); for ``type="file"`` it is + the raw text when ``format="text"`` or base64-encoded bytes when + ``format="base64"``. + """ + ws = await self._ensure_workspace(workspace_id) + return await self._jupyter_request(workspace_id, ws, "GET", name) + + async def create_directory( + self, + workspace_id: str, + *, + name: str, + ) -> dict[str, Any]: + """Create a directory in the workspace Jupyter. + + Uses ``PUT /api/contents/{name}`` with body ``{"type": "directory"}``. + The directory's on-disk name in Jupyter is ``name``; callers that + want the display basename preserved separately should pass a ULID + (or other stable identifier) as ``name`` and keep the human-readable + basename in the database. + """ + ws = await self._ensure_workspace(workspace_id) + return await self._jupyter_request( + workspace_id, ws, "PUT", name, body={"type": "directory"} + ) + + async def delete_directory( + self, + workspace_id: str, + *, + name: str, + ) -> None: + """Delete a directory from the workspace Jupyter. + + Uses ``DELETE /api/contents/{name}`` on the workspace Jupyter. + Jupyter returns ``204 No Content`` on success; any 4xx/5xx is + surfaced as :class:`RuntimeClientError`. Non-recursive — Jupyter + returns ``409 Conflict`` when the directory is not empty; the + caller is responsible for emptying the directory first. + """ + ws = await self._ensure_workspace(workspace_id) + await self._jupyter_request(workspace_id, ws, "DELETE", name) + + async def ensure_directory( + self, + workspace_id: str, + name: str, + ) -> None: + """Ensure a directory exists in the workspace Jupyter. + + Lazy-backfill primitive used during create flows: performs a + ``GET /api/contents/{name}`` first, returning immediately if the + directory already exists. On ``404 Not Found`` it falls through + to :meth:`create_directory`. Any other error is propagated as a + :class:`RuntimeClientError`. The GET-first pattern avoids the + ambiguity of calling ``PUT /api/contents/{name}`` on a path that + may already exist. + """ + ws = await self._ensure_workspace(workspace_id) + try: + await self._jupyter_request(workspace_id, ws, "GET", name) + except RuntimeClientError as exc: + if exc.status_code == 404: + await self.create_directory(workspace_id, name=name) + return + raise + +__all__ = ["RuntimeClient", "RuntimeClientError"] diff --git a/backend/src/backend/schedule_client.py b/backend/src/backend/schedule_client.py new file mode 100644 index 0000000..dd1f119 --- /dev/null +++ b/backend/src/backend/schedule_client.py @@ -0,0 +1,16 @@ +"""Schedule executor HTTP-side dispatch is intentionally a no-op. + +Architecture V3.1 §2.3 documents two dispatch paths: + ① Backend HTTP push to Schedule Executor (immediate runs) + ② Schedule Executor polling MySQL Outbox (immediate + cron runs; contract) + +We run path ② only. Path ① is an optimisation layered on top of ② and was +removed alongside the INTERNAL_SERVICE_TOKEN cleanup. Backend writes the +``schedule.run.requested`` Outbox event in the same transaction as the +``ScheduleRuns`` row, then commits; the executor picks it up on its next +poll. No additional auth / token / header is involved — the Outbox is the +single source of truth for run dispatch. + +This module is kept as a docstring-only placeholder so future readers +(LLM and human) can grep for ``schedule_client`` and find the rationale. +""" diff --git a/backend/src/backend/schedule_runs.py b/backend/src/backend/schedule_runs.py new file mode 100644 index 0000000..43ff14f --- /dev/null +++ b/backend/src/backend/schedule_runs.py @@ -0,0 +1,433 @@ +"""调度运行的触发与查询 API。 + +手动运行接口只负责在 MySQL 中创建 ``ScheduleRuns`` 和 Outbox 事件；真正执行 +任务的是 ``schedule`` 容器，它轮询 Outbox 后运行 DAG 节点。本模块还提供运行 +历史、节点状态、日志和结果文件的查询入口。 +""" + +from __future__ import annotations + +from datetime import UTC, datetime +from typing import Any, Literal +from urllib.parse import quote + +from common.config import settings +from common.db.models import ( + ScheduleNodeRuns, + ScheduleRuns, + StorageObjects, +) +from common.scheduler import ( + DagTooLarge, + InvalidDag, + InvalidNodeArguments, + ScheduleNotFound, + TriggerError, + create_scheduled_run, + normalize_idempotency_key, +) +from common.schemas import StrictModel +from fastapi import ( + APIRouter, + Depends, + Header, + HTTPException, + Query, + Request, + Response, + status, +) +from pydantic import Field +from sqlalchemy import select +from sqlalchemy.ext.asyncio import AsyncSession + +from backend.dependencies import ( + RequestContext, + database_session, + request_context, +) + +router = APIRouter(tags=["schedule-runs"]) +RunStatus = Literal[ + "queued", + "running", + "succeeded", + "failed", + "cancelled", + "timed_out", +] + + +# 手动触发调度时允许前端附带的运行原因；严格拒绝未定义字段。 +class RunScheduleRequest(StrictModel): + reason: str = Field(default="manual_run", min_length=1, max_length=255) + + +def _iso(value: datetime | None) -> str | None: + if value is None: + return None + if value.tzinfo is None: + value = value.replace(tzinfo=UTC) + return value.astimezone(UTC).isoformat() + + +def _http_error_from_trigger(exc: TriggerError) -> HTTPException: + # 调度领域异常统一转换为 HTTP 状态码，前端可据此区分“找不到任务”、 + # “DAG 无效”和“请求参数不合法”等情况。 + if isinstance(exc, ScheduleNotFound): + return HTTPException(status.HTTP_404_NOT_FOUND, str(exc)) + if isinstance(exc, InvalidDag): + return HTTPException( + status.HTTP_409_CONFLICT, + detail={ + "code": "SCHEDULE_DAG_INVALID", + "message": str(exc), + "errors": exc.errors, + }, + ) + if isinstance(exc, DagTooLarge): + return HTTPException(status.HTTP_409_CONFLICT, str(exc)) + if isinstance(exc, InvalidNodeArguments): + return HTTPException(status.HTTP_422_UNPROCESSABLE_ENTITY, exc.message) + return HTTPException(status.HTTP_409_CONFLICT, str(exc)) + + +def run_summary(item: ScheduleRuns) -> dict[str, Any]: + return { + "run_id": item.run_id, + "schedule_id": item.schedule_id, + "workspace_id": item.workspace_id, + "workflow_version": item.workflow_version, + "trigger_type": item.trigger_type, + "run_status": item.run_status, + "state_version": item.state_version, + "queued_at": _iso(item.queued_at), + "started_at": _iso(item.started_at), + "finished_at": _iso(item.finished_at), + "duration_ms": item.duration_ms, + "error_code": item.error_code, + "error_message": item.error_message, + "logs_object_id": item.logs_object_id, + "result_object_id": item.result_object_id, + } + + +def node_run_payload(item: ScheduleNodeRuns) -> dict[str, Any]: + return { + "node_run_id": item.node_run_id, + "run_id": item.run_id, + "node_id": item.node_id, + "versions_id": item.versions_id, + "attempt_no": item.attempt_no, + "node_status": item.node_status, + "state_version": item.state_version, + "started_at": _iso(item.started_at), + "finished_at": _iso(item.finished_at), + "duration_ms": item.duration_ms, + "exit_code": item.exit_code, + "message": item.message, + "logs_object_id": item.logs_object_id, + "result_object_id": item.result_object_id, + } + + +async def run_detail( + item: ScheduleRuns, + session: AsyncSession, +) -> dict[str, Any]: + node_runs = list( + ( + await session.scalars( + select(ScheduleNodeRuns) + .where(ScheduleNodeRuns.run_id == item.run_id) + .order_by( + ScheduleNodeRuns.created_at, + ScheduleNodeRuns.attempt_no, + ) + ) + ).all() + ) + return { + **run_summary(item), + "node_runs": [node_run_payload(node_run) for node_run in node_runs], + } + + +async def _visible_run( + run_id: str, + context: RequestContext, + session: AsyncSession, +) -> ScheduleRuns: + item = await session.scalar( + select(ScheduleRuns).where( + ScheduleRuns.run_id == run_id, + ScheduleRuns.workspace_id == context.workspace.workspace_id, + ) + ) + if item is None: + raise HTTPException(status.HTTP_404_NOT_FOUND, "schedule run not found") + return item + + +async def _visible_node_run( + run_id: str, + node_run_id: str, + context: RequestContext, + session: AsyncSession, +) -> ScheduleNodeRuns: + await _visible_run(run_id, context, session) + item = await session.scalar( + select(ScheduleNodeRuns).where( + ScheduleNodeRuns.node_run_id == node_run_id, + ScheduleNodeRuns.run_id == run_id, + ) + ) + if item is None: + raise HTTPException(status.HTTP_404_NOT_FOUND, "schedule node run not found") + return item + + +async def _visible_artifact( + storage_object_id: str | None, + *, + usage_type: Literal["run_log", "run_result"], + context: RequestContext, + session: AsyncSession, +) -> StorageObjects | None: + if storage_object_id is None: + return None + item = await session.scalar( + select(StorageObjects).where( + StorageObjects.storage_object_id == storage_object_id, + StorageObjects.workspace_id == context.workspace.workspace_id, + StorageObjects.usage_type == usage_type, + StorageObjects.object_status == "available", + StorageObjects.is_deleted == 0, + ) + ) + if ( + item is None + or item.storage_backend != settings.storage_backend + or not item.bucket_name + or not item.object_key + ): + return None + + return item + + +def _artifact_payload(item: StorageObjects | None, *, url: str) -> dict[str, Any] | None: + if item is None: + return None + return { + "url": url, + "file_name": item.file_name, + "mime_type": item.mime_type, + "size_bytes": item.size_bytes, + } + + +async def _artifact_bytes( + item: StorageObjects, + request: Request, +) -> bytes: + return await request.app.state.object_stores[item.bucket_name].get( + item.object_key, + ) + + +# 立即触发一次调度：写入运行记录和 Outbox，由 schedule 容器异步接手执行。 +@router.post( + "/api/v1/schedules/{schedule_id}/run", + status_code=status.HTTP_202_ACCEPTED, +) +async def run_schedule_now( + schedule_id: str, + request: Request, + payload: RunScheduleRequest | None = None, + idempotency_key: str = Header(alias="Idempotency-Key"), + context: RequestContext = Depends(request_context), + session: AsyncSession = Depends(database_session), +) -> dict[str, Any]: + reason = payload.reason if payload is not None else "manual_run" + trigger_type: Literal["manual", "cron"] = "cron" if reason == "cron" else "manual" + try: + key = normalize_idempotency_key( + context.workspace.workspace_id, + schedule_id, + idempotency_key, + ) + except TriggerError as exc: + raise HTTPException( + status.HTTP_422_UNPROCESSABLE_ENTITY, str(exc), + ) from exc + + try: + run, is_new = await create_scheduled_run( + session, + schedule_id=schedule_id, + workspace_id=context.workspace.workspace_id, + triggered_by_user_id=context.user.user_id, + trigger_type=trigger_type, + idempotency_key=key, + trace_id=context.request_id, + ) + except TriggerError as exc: + raise _http_error_from_trigger(exc) from exc + + # Commit before responding so the Outbox row is visible to the + # executor's next MySQL poll — the executor's _database_event_loop + # picks it up. We intentionally do NOT HTTP-push. + await session.commit() + await session.refresh(run) + return { + "request_id": context.request_id, + "data": await run_detail(run, session), + "meta": {"reused": not is_new}, + } + + +# 按调度或状态筛选运行历史，供前端运行记录列表展示。 +@router.get("/api/v1/schedule-runs") +async def list_schedule_runs( + schedule_id: str | None = Query(default=None), + run_status: RunStatus | None = Query(default=None, alias="status"), + limit: int = Query(default=50, ge=1, le=200), + context: RequestContext = Depends(request_context), + session: AsyncSession = Depends(database_session), +) -> dict[str, Any]: + statement = select(ScheduleRuns).where( + ScheduleRuns.workspace_id == context.workspace.workspace_id, + ) + if schedule_id: + statement = statement.where(ScheduleRuns.schedule_id == schedule_id) + if run_status: + statement = statement.where(ScheduleRuns.run_status == run_status) + items = list( + ( + await session.scalars( + statement.order_by(ScheduleRuns.queued_at.desc()).limit(limit) + ) + ).all() + ) + return { + "request_id": context.request_id, + "data": [run_summary(item) for item in items], + "meta": {"count": len(items)}, + } + + +# 查询一次运行的详情，包括每个节点的执行状态。 +@router.get("/api/v1/schedule-runs/{run_id}") +async def get_schedule_run( + run_id: str, + context: RequestContext = Depends(request_context), + session: AsyncSession = Depends(database_session), +) -> dict[str, Any]: + item = await _visible_run(run_id, context, session) + return { + "request_id": context.request_id, + "data": await run_detail(item, session), + "meta": {}, + } + + +# 返回某个节点运行关联的日志/结果产物元数据及可访问地址。 +@router.get( + "/api/v1/schedule-runs/{run_id}/node-runs/{node_run_id}/artifacts" +) +async def get_schedule_node_run_artifacts( + run_id: str, + node_run_id: str, + context: RequestContext = Depends(request_context), + session: AsyncSession = Depends(database_session), +) -> dict[str, Any]: + node_run = await _visible_node_run(run_id, node_run_id, context, session) + log_artifact = await _visible_artifact( + node_run.logs_object_id, + usage_type="run_log", + context=context, + session=session, + ) + result_artifact = await _visible_artifact( + node_run.result_object_id, + usage_type="run_result", + context=context, + session=session, + ) + base_path = f"/api/v1/schedule-runs/{run_id}/node-runs/{node_run_id}" + workspace_query = f"workspace_id={context.workspace.workspace_id}" + return { + "request_id": context.request_id, + "data": { + "run_id": run_id, + "node_run_id": node_run_id, + "log": _artifact_payload( + log_artifact, + url=f"{base_path}/logs?{workspace_query}", + ), + "result": _artifact_payload( + result_artifact, + url=f"{base_path}/result?{workspace_query}", + ), + }, + "meta": {}, + } + + +# 读取节点运行日志正文，通常由前端日志面板按需调用。 +@router.get( + "/api/v1/schedule-runs/{run_id}/node-runs/{node_run_id}/logs" +) +async def read_schedule_node_run_logs( + run_id: str, + node_run_id: str, + request: Request, + context: RequestContext = Depends(request_context), + session: AsyncSession = Depends(database_session), +) -> Response: + node_run = await _visible_node_run(run_id, node_run_id, context, session) + item = await _visible_artifact( + node_run.logs_object_id, + usage_type="run_log", + context=context, + session=session, + ) + if item is None: + raise HTTPException(status.HTTP_404_NOT_FOUND, "schedule node log not found") + return Response( + content=await _artifact_bytes(item, request), + media_type=item.mime_type or "text/plain; charset=utf-8", + headers={"Cache-Control": "no-store"}, + ) + + +# 为节点运行结果生成下载响应或重定向地址。 +@router.get( + "/api/v1/schedule-runs/{run_id}/node-runs/{node_run_id}/result" +) +async def download_schedule_node_run_result( + run_id: str, + node_run_id: str, + request: Request, + context: RequestContext = Depends(request_context), + session: AsyncSession = Depends(database_session), +) -> Response: + node_run = await _visible_node_run(run_id, node_run_id, context, session) + item = await _visible_artifact( + node_run.result_object_id, + usage_type="run_result", + context=context, + session=session, + ) + if item is None: + raise HTTPException(status.HTTP_404_NOT_FOUND, "schedule node result not found") + encoded_name = quote(item.file_name, safe="") + return Response( + content=await _artifact_bytes(item, request), + media_type=item.mime_type or "application/octet-stream", + headers={ + "Cache-Control": "no-store", + "Content-Disposition": f"attachment; filename*=UTF-8''{encoded_name}", + }, + ) diff --git a/backend/src/backend/schedule_schemas.py b/backend/src/backend/schedule_schemas.py new file mode 100644 index 0000000..0f9f170 --- /dev/null +++ b/backend/src/backend/schedule_schemas.py @@ -0,0 +1,218 @@ +from __future__ import annotations + +from datetime import datetime +from typing import Any + +from common.db.models.schedules import FailurePolicy, PythonVersion, TriggerType +from common.schemas import StrictModel +from pydantic import Field, field_validator, model_validator + + +def _required_text(value: str) -> str: + normalized = value.strip() + if not normalized: + raise ValueError("value must not be blank") + return normalized + + +class CreateScheduleRequest(StrictModel): + schedule_name: str = Field(min_length=1, max_length=255) + description: str | None = Field(default=None, max_length=1000) + trigger_type: TriggerType = "cron" + cron_expression: str | None = Field(default=None, max_length=128) + timezone: str = Field(default="Asia/Shanghai", min_length=1, max_length=64) + enabled: bool = False + max_concurrency: int = Field(default=1, ge=1, le=64) + failure_policy: FailurePolicy = "stop" + + @field_validator("schedule_name", "timezone") + @classmethod + def validate_required_text(cls, value: str) -> str: + return _required_text(value) + + @field_validator("description", "cron_expression") + @classmethod + def normalize_optional_text(cls, value: str | None) -> str | None: + if value is None: + return None + normalized = value.strip() + return normalized or None + + @model_validator(mode="after") + def validate_trigger(self) -> CreateScheduleRequest: + if self.trigger_type == "cron" and not self.cron_expression: + raise ValueError("cron_expression is required for cron schedules") + if self.trigger_type != "cron" and self.cron_expression: + raise ValueError( + "cron_expression is only allowed for cron schedules" + ) + return self + + +class UpdateScheduleRequest(StrictModel): + workflow_version: int = Field(ge=1) + schedule_name: str | None = Field(default=None, min_length=1, max_length=255) + description: str | None = Field(default=None, max_length=1000) + trigger_type: TriggerType | None = None + cron_expression: str | None = Field(default=None, max_length=128) + timezone: str | None = Field(default=None, min_length=1, max_length=64) + enabled: bool | None = None + max_concurrency: int | None = Field(default=None, ge=1, le=64) + failure_policy: FailurePolicy | None = None + + @field_validator("schedule_name", "timezone") + @classmethod + def validate_optional_required_text( + cls, + value: str | None, + ) -> str | None: + return _required_text(value) if value is not None else None + + @field_validator("description", "cron_expression") + @classmethod + def normalize_optional_text(cls, value: str | None) -> str | None: + if value is None: + return None + normalized = value.strip() + return normalized or None + + @model_validator(mode="after") + def require_change(self) -> UpdateScheduleRequest: + if self.model_fields_set == {"workflow_version"}: + raise ValueError("at least one schedule field must be updated") + for field in ( + "schedule_name", + "trigger_type", + "timezone", + "enabled", + "max_concurrency", + "failure_policy", + ): + if field in self.model_fields_set and getattr(self, field) is None: + raise ValueError(f"{field} cannot be null") + return self + + +class WorkflowVersionRequest(StrictModel): + workflow_version: int = Field(ge=1) + + +class DeleteScheduleNodeRequest(WorkflowVersionRequest): + """删除节点时可显式确认一并清理其已经结束的运行记录。""" + + delete_execution_history: bool = False + + +class CreateScheduleNodeRequest(StrictModel): + workflow_version: int = Field(ge=1) + node_key: str = Field( + min_length=1, + max_length=64, + pattern=r"^[A-Za-z0-9][A-Za-z0-9_-]*$", + ) + node_name: str = Field(min_length=1, max_length=255) + versions_id: str = Field(min_length=26, max_length=26) + timeout_seconds: int = Field(default=600, ge=1, le=86_400) + retry_count: int = Field(default=0, ge=0, le=10) + retry_interval_sec: int = Field(default=5, ge=0, le=3600) + position_x: float = Field(default=0, ge=-100_000, le=100_000) + position_y: float = Field(default=0, ge=-100_000, le=100_000) + arguments_json: dict[str, Any] = Field(default_factory=dict, max_length=100) + env_refs_json: dict[str, str] = Field(default_factory=dict, max_length=100) + python_version: PythonVersion = Field(default="3.12") + + @field_validator("node_key", "node_name") + @classmethod + def validate_required_text(cls, value: str) -> str: + return _required_text(value) + + +class UpdateScheduleNodeRequest(StrictModel): + workflow_version: int = Field(ge=1) + node_name: str | None = Field(default=None, min_length=1, max_length=255) + versions_id: str | None = Field(default=None, min_length=26, max_length=26) + timeout_seconds: int | None = Field(default=None, ge=1, le=86_400) + retry_count: int | None = Field(default=None, ge=0, le=10) + retry_interval_sec: int | None = Field(default=None, ge=0, le=3600) + position_x: float | None = Field( + default=None, + ge=-100_000, + le=100_000, + ) + position_y: float | None = Field( + default=None, + ge=-100_000, + le=100_000, + ) + arguments_json: dict[str, Any] | None = Field( + default=None, + max_length=100, + ) + env_refs_json: dict[str, str] | None = Field( + default=None, + max_length=100, + ) + python_version: PythonVersion | None = Field(default=None) + + @field_validator("node_name") + @classmethod + def validate_optional_required_text( + cls, + value: str | None, + ) -> str | None: + return _required_text(value) if value is not None else None + + @model_validator(mode="after") + def require_change(self) -> UpdateScheduleNodeRequest: + if self.model_fields_set == {"workflow_version"}: + raise ValueError("at least one node field must be updated") + for field in self.model_fields_set - {"workflow_version"}: + if getattr(self, field) is None: + raise ValueError(f"{field} cannot be null") + return self + + +class CreateScheduleEdgeRequest(StrictModel): + workflow_version: int = Field(ge=1) + source_node_id: str = Field(min_length=26, max_length=26) + target_node_id: str = Field(min_length=26, max_length=26) + condition_expr: str | None = Field(default=None, max_length=1000) + + @field_validator("condition_expr") + @classmethod + def normalize_condition(cls, value: str | None) -> str | None: + if value is None: + return None + normalized = value.strip() + return normalized or None + + @model_validator(mode="after") + def reject_self_edge(self) -> CreateScheduleEdgeRequest: + if self.source_node_id == self.target_node_id: + raise ValueError("an edge cannot connect a node to itself") + return self + + +class UpdateScheduleEdgeRequest(StrictModel): + workflow_version: int = Field(ge=1) + condition_expr: str | None = Field(default=None, max_length=1000) + + @field_validator("condition_expr") + @classmethod + def normalize_condition(cls, value: str | None) -> str | None: + if value is None: + return None + normalized = value.strip() + return normalized or None + + +class CronPreviewRequest(StrictModel): + cron_expression: str = Field(min_length=1, max_length=128) + timezone: str = Field(default="Asia/Shanghai", min_length=1, max_length=64) + count: int = Field(default=5, ge=1, le=20) + base_time: datetime | None = None + + @field_validator("cron_expression", "timezone") + @classmethod + def validate_required_text(cls, value: str) -> str: + return _required_text(value) diff --git a/backend/src/backend/schedules.py b/backend/src/backend/schedules.py new file mode 100644 index 0000000..bfe6cc7 --- /dev/null +++ b/backend/src/backend/schedules.py @@ -0,0 +1,1237 @@ +"""DAG 调度定义 API。 + +这里管理调度任务本身：名称、Cron、节点、边和 DAG 校验；实际的定时轮询与节点 +执行由独立的 ``schedule`` 容器完成。``workflow_version`` 用于乐观并发控制： +前端修改画布时必须携带当前版本，避免两个人的编辑互相覆盖。 +""" + +from __future__ import annotations + +import heapq +from datetime import UTC, datetime +from decimal import Decimal +from typing import Any +from zoneinfo import ZoneInfo, ZoneInfoNotFoundError + +from common.db.models import ( + ScheduleEdges, + ScheduleNodeRuns, + ScheduleNodes, + ScheduleRuns, + Schedules, + Scripts, + StorageObjects, + Versions, +) +from common.ids import new_ulid +from croniter import CroniterBadCronError, croniter +from fastapi import APIRouter, Depends, HTTPException, Query, Request, status +from sqlalchemy import delete, func, or_, select +from sqlalchemy.ext.asyncio import AsyncSession + +from backend.dependencies import ( + RequestContext, + database_session, + request_context, +) +from backend.schedule_schemas import ( + CreateScheduleEdgeRequest, + CreateScheduleNodeRequest, + CreateScheduleRequest, + CronPreviewRequest, + DeleteScheduleNodeRequest, + UpdateScheduleEdgeRequest, + UpdateScheduleNodeRequest, + UpdateScheduleRequest, + WorkflowVersionRequest, +) +from backend.services.storage import soft_delete_object + +router = APIRouter(tags=["schedules"]) + +_ACTIVE_RUN_STATUSES = ("queued", "running") + + +def _iso(value: datetime | None) -> str | None: + if value is None: + return None + if value.tzinfo is None: + value = value.replace(tzinfo=UTC) + return value.astimezone(UTC).isoformat() + + +def _mysql_utc(value: datetime) -> datetime: + if value.tzinfo is None: + value = value.replace(tzinfo=UTC) + return value.astimezone(UTC).replace(tzinfo=None) + + +def _execution_artifact_ids( + items: list[ScheduleRuns | ScheduleNodeRuns], +) -> set[str]: + """收集运行日志和结果产物，供删除记录时一并移入回收站。""" + + return { + storage_object_id + for item in items + for storage_object_id in (item.logs_object_id, item.result_object_id) + if storage_object_id + } + + +async def _delete_execution_artifacts( + storage_object_ids: set[str], + request: Request, + session: AsyncSession, +) -> None: + """软删除可删除的运行产物。 + + 不可变对象是运行审计原件，存储层不允许移动或删除它们。调用方随后会 + 删除运行记录本身，因此不可变原件不会再通过该调度节点暴露；保留原件也 + 不应阻塞节点或调度方案的删除。 + """ + + if not storage_object_ids: + return + + mutable_storage_object_ids = set( + ( + await session.scalars( + select(StorageObjects.storage_object_id).where( + StorageObjects.storage_object_id.in_(storage_object_ids), + StorageObjects.is_immutable == 0, + ) + ) + ).all() + ) + for storage_object_id in sorted(mutable_storage_object_ids): + await soft_delete_object(storage_object_id, request, session) + + +def _timezone(value: str) -> ZoneInfo: + try: + return ZoneInfo(value) + except ZoneInfoNotFoundError as exc: + raise HTTPException( + status.HTTP_422_UNPROCESSABLE_ENTITY, + f"unknown timezone: {value}", + ) from exc + + +def cron_preview( + expression: str, + timezone_name: str, + *, + count: int, + base_time: datetime | None = None, +) -> dict[str, Any]: + # 该接口只计算并预览下几次触发时间，不会创建或修改任何调度任务。 + if len(expression.split()) != 5: + raise HTTPException( + status.HTTP_422_UNPROCESSABLE_ENTITY, + "cron_expression must contain exactly five fields: " + "minute hour day month weekday", + ) + timezone = _timezone(timezone_name) + if base_time is None: + local_base = datetime.now(timezone) + elif base_time.tzinfo is None: + local_base = base_time.replace(tzinfo=timezone) + else: + local_base = base_time.astimezone(timezone) + try: + if not croniter.is_valid(expression, strict=True): + raise CroniterBadCronError(expression) + iterator = croniter(expression, local_base) + occurrences = [ + iterator.get_next(datetime) + for _ in range(count) + ] + except (CroniterBadCronError, ValueError, OverflowError) as exc: + raise HTTPException( + status.HTTP_422_UNPROCESSABLE_ENTITY, + "invalid cron_expression", + ) from exc + return { + "cron_expression": expression, + "timezone": timezone_name, + "base_time": local_base.isoformat(), + "occurrences": [ + { + "local_time": occurrence.isoformat(), + "utc_time": occurrence.astimezone(UTC).isoformat(), + } + for occurrence in occurrences + ], + } + + +def schedule_payload( + item: Schedules, + *, + node_count: int = 0, + edge_count: int = 0, +) -> dict[str, Any]: + return { + "schedule_id": item.schedule_id, + "workspace_id": item.workspace_id, + "schedule_name": item.schedule_name, + "description": item.description, + "trigger_type": item.trigger_type, + "cron_expression": item.cron_expression, + "timezone": item.timezone, + "enabled": bool(item.enabled), + "workflow_version": item.workflow_version, + "max_concurrency": item.max_concurrency, + "failure_policy": item.failure_policy, + "last_run_at": _iso(item.last_run_at), + "next_run_at": _iso(item.next_run_at), + "created_by": item.created_by, + "updated_by": item.updated_by, + "created_at": _iso(item.created_at), + "updated_at": _iso(item.updated_at), + "node_count": node_count, + "edge_count": edge_count, + } + + +def node_payload( + item: ScheduleNodes, + version: Versions, + script: Scripts, +) -> dict[str, Any]: + return { + "node_id": item.node_id, + "schedule_id": item.schedule_id, + "node_key": item.node_key, + "node_name": item.node_name, + "versions_id": item.versions_id, + "python_version": item.python_version, + "timeout_seconds": item.timeout_seconds, + "retry_count": item.retry_count, + "retry_interval_sec": item.retry_interval_sec, + "position_x": float(item.position_x), + "position_y": float(item.position_y), + "arguments_json": item.arguments_json or {}, + "env_refs_json": item.env_refs_json or {}, + "created_at": _iso(item.created_at), + "updated_at": _iso(item.updated_at), + "version": { + "versions_id": version.versions_id, + "version_label": version.version_label, + "script_id": version.script_id, + "script_name": script.script_name, + "script_type": script.script_type, + "content_hash": version.content_hash, + "created_at": _iso(version.created_at), + }, + } + + +def edge_payload(item: ScheduleEdges) -> dict[str, Any]: + return { + "edge_id": item.edge_id, + "schedule_id": item.schedule_id, + "source_node_id": item.source_node_id, + "target_node_id": item.target_node_id, + "condition_expr": item.condition_expr, + "created_at": _iso(item.created_at), + } + + +def validate_dag( + nodes: list[ScheduleNodes], + edges: list[ScheduleEdges], +) -> dict[str, Any]: + node_by_id = {item.node_id: item for item in nodes} + indegree = {item.node_id: 0 for item in nodes} + outgoing: dict[str, set[str]] = { + item.node_id: set() + for item in nodes + } + errors: list[dict[str, Any]] = [] + seen_edges: set[tuple[str, str]] = set() + + if not nodes: + errors.append( + { + "code": "DAG_EMPTY", + "message": "schedule must contain at least one node", + } + ) + + for edge in edges: + if ( + edge.source_node_id not in node_by_id + or edge.target_node_id not in node_by_id + ): + errors.append( + { + "code": "DAG_EDGE_NODE_MISSING", + "message": "edge references a node outside the schedule", + "edge_id": edge.edge_id, + } + ) + continue + pair = (edge.source_node_id, edge.target_node_id) + if edge.source_node_id == edge.target_node_id: + errors.append( + { + "code": "DAG_SELF_EDGE", + "message": "a node cannot depend on itself", + "edge_id": edge.edge_id, + } + ) + continue + if pair in seen_edges: + errors.append( + { + "code": "DAG_DUPLICATE_EDGE", + "message": "duplicate directed edge", + "edge_id": edge.edge_id, + } + ) + continue + seen_edges.add(pair) + outgoing[edge.source_node_id].add(edge.target_node_id) + indegree[edge.target_node_id] += 1 + + root_ids = sorted( + (node_id for node_id, degree in indegree.items() if degree == 0), + key=lambda node_id: node_by_id[node_id].node_key, + ) + leaf_ids = sorted( + (node_id for node_id, targets in outgoing.items() if not targets), + key=lambda node_id: node_by_id[node_id].node_key, + ) + queue = [ + (node_by_id[node_id].node_key, node_id) + for node_id in root_ids + ] + heapq.heapify(queue) + remaining_indegree = dict(indegree) + ordered_ids: list[str] = [] + while queue: + _, node_id = heapq.heappop(queue) + ordered_ids.append(node_id) + for target_id in sorted( + outgoing[node_id], + key=lambda value: node_by_id[value].node_key, + ): + remaining_indegree[target_id] -= 1 + if remaining_indegree[target_id] == 0: + heapq.heappush( + queue, + (node_by_id[target_id].node_key, target_id), + ) + + if len(ordered_ids) != len(nodes): + cycle_node_ids = sorted( + ( + node_id + for node_id, degree in remaining_indegree.items() + if degree > 0 + ), + key=lambda node_id: node_by_id[node_id].node_key, + ) + errors.append( + { + "code": "DAG_CYCLE", + "message": "schedule graph contains a directed cycle", + "node_ids": cycle_node_ids, + } + ) + + return { + "valid": not errors, + "node_count": len(nodes), + "edge_count": len(edges), + "root_node_ids": root_ids, + "leaf_node_ids": leaf_ids, + "topological_order": ordered_ids, + "errors": errors, + } + + +async def schedule_row( + schedule_id: str, + context: RequestContext, + session: AsyncSession, + *, + for_update: bool = False, +) -> Schedules: + statement = select(Schedules).where( + Schedules.schedule_id == schedule_id, + Schedules.workspace_id == context.workspace.workspace_id, + Schedules.deleted_at.is_(None), + ) + if for_update: + statement = statement.with_for_update() + item = await session.scalar(statement) + if item is None: + raise HTTPException(status.HTTP_404_NOT_FOUND, "schedule not found") + return item + + +def require_workflow_version(item: Schedules, expected: int) -> None: + if item.workflow_version != expected: + raise HTTPException( + status.HTTP_412_PRECONDITION_FAILED, + detail={ + "code": "WORKFLOW_VERSION_CONFLICT", + "message": "schedule was modified by another request", + "expected": expected, + "current": item.workflow_version, + }, + ) + + +async def graph_rows( + schedule_id: str, + session: AsyncSession, +) -> tuple[ + list[tuple[ScheduleNodes, Versions, Scripts]], + list[ScheduleEdges], +]: + node_rows = ( + await session.execute( + select(ScheduleNodes, Versions, Scripts) + .join( + Versions, + Versions.versions_id == ScheduleNodes.versions_id, + ) + .join(Scripts, Scripts.script_id == Versions.script_id) + .where(ScheduleNodes.schedule_id == schedule_id) + .order_by(ScheduleNodes.created_at, ScheduleNodes.node_key) + ) + ).all() + edges = list( + ( + await session.scalars( + select(ScheduleEdges) + .where(ScheduleEdges.schedule_id == schedule_id) + .order_by(ScheduleEdges.created_at, ScheduleEdges.edge_id) + ) + ).all() + ) + return list(node_rows), edges + + +async def detail_payload( + item: Schedules, + session: AsyncSession, +) -> dict[str, Any]: + node_rows, edges = await graph_rows(item.schedule_id, session) + nodes = [row[0] for row in node_rows] + return { + **schedule_payload( + item, + node_count=len(nodes), + edge_count=len(edges), + ), + "nodes": [ + node_payload(node, version, script) + for node, version, script in node_rows + ], + "edges": [edge_payload(edge) for edge in edges], + "dag_validation": validate_dag(nodes, edges), + } + + +async def accessible_version( + versions_id: str, + context: RequestContext, + session: AsyncSession, +) -> tuple[Versions, Scripts]: + row = ( + await session.execute( + select(Versions, Scripts) + .join(Scripts, Scripts.script_id == Versions.script_id) + .where( + Versions.versions_id == versions_id, + Versions.workspace_id == context.workspace.workspace_id, + ) + ) + ).one_or_none() + if row is None: + raise HTTPException( + status.HTTP_404_NOT_FOUND, + "stable version not found", + ) + version, script = row + if not ( + context.is_admin + or version.created_by == context.user.user_id + or version.visibility in {"workspace", "public"} + ): + raise HTTPException( + status.HTTP_403_FORBIDDEN, + "stable version is not visible to this user", + ) + return version, script + + +def _apply_next_run(item: Schedules) -> None: + _timezone(item.timezone) + if item.trigger_type != "cron": + if item.cron_expression: + raise HTTPException( + status.HTTP_422_UNPROCESSABLE_ENTITY, + "cron_expression is only allowed for cron schedules", + ) + item.cron_expression = None + item.next_run_at = None + return + if not item.cron_expression: + raise HTTPException( + status.HTTP_422_UNPROCESSABLE_ENTITY, + "cron_expression is required for cron schedules", + ) + preview = cron_preview( + item.cron_expression, + item.timezone, + count=1, + ) + item.next_run_at = ( + _mysql_utc( + datetime.fromisoformat(preview["occurrences"][0]["utc_time"]) + ) + if item.enabled + else None + ) + + +async def _require_valid_when_enabled( + item: Schedules, + session: AsyncSession, +) -> None: + if not item.enabled: + return + node_rows, edges = await graph_rows(item.schedule_id, session) + validation = validate_dag([row[0] for row in node_rows], edges) + if not validation["valid"]: + raise HTTPException( + status.HTTP_409_CONFLICT, + detail={ + "code": "DAG_INVALID", + "message": "an enabled schedule must contain a valid DAG", + "validation": validation, + }, + ) + + +# 根据 Cron 表达式预览未来触发时间，不会保存或执行任务。 +@router.post("/api/v1/cron/preview") +async def preview_cron( + payload: CronPreviewRequest, + context: RequestContext = Depends(request_context), +) -> dict[str, Any]: + return { + "request_id": context.request_id, + "data": cron_preview( + payload.cron_expression, + payload.timezone, + count=payload.count, + base_time=payload.base_time, + ), + "meta": {}, + } + + +# 列出调度产生的可展示版本/产物，供前端结果面板使用。 +@router.get("/api/v1/schedule-artifacts") +async def list_schedule_artifacts( + limit: int = Query(default=100, ge=1, le=500), + context: RequestContext = Depends(request_context), + session: AsyncSession = Depends(database_session), +) -> dict[str, Any]: + rows = ( + await session.execute( + select(Versions, Scripts) + .join(Scripts, Scripts.script_id == Versions.script_id) + .where( + Versions.workspace_id == context.workspace.workspace_id, + Versions.schedule_hidden_at.is_(None), + ) + .order_by(Versions.created_at.desc()) + .limit(limit) + ) + ).all() + visible = [ + (version, script) + for version, script in rows + if ( + context.is_admin + or version.created_by == context.user.user_id + or version.visibility in {"workspace", "public"} + ) + ] + data = [ + { + "versions_id": version.versions_id, + "version_label": version.version_label, + "script_id": script.script_id, + "script_name": script.script_name, + "script_type": script.script_type, + "content_hash": version.content_hash, + "file_size_bytes": version.file_size_bytes, + "visibility": version.visibility, + "created_by": version.created_by, + "created_at": _iso(version.created_at), + } + for version, script in visible + ] + return { + "request_id": context.request_id, + "data": data, + "meta": {"count": len(data)}, + } + + +# 列出当前工作区的调度定义及其节点、边数量等摘要信息。 +@router.get("/api/v1/schedules") +async def list_schedules( + context: RequestContext = Depends(request_context), + session: AsyncSession = Depends(database_session), +) -> dict[str, Any]: + items = list( + ( + await session.scalars( + select(Schedules) + .where( + Schedules.workspace_id == context.workspace.workspace_id, + Schedules.deleted_at.is_(None), + ) + .order_by(Schedules.updated_at.desc()) + ) + ).all() + ) + ids = [item.schedule_id for item in items] + node_counts: dict[str, int] = {} + edge_counts: dict[str, int] = {} + if ids: + node_counts = { + schedule_id: int(count) + for schedule_id, count in ( + await session.execute( + select( + ScheduleNodes.schedule_id, + func.count(ScheduleNodes.node_id), + ) + .where(ScheduleNodes.schedule_id.in_(ids)) + .group_by(ScheduleNodes.schedule_id) + ) + ).all() + } + edge_counts = { + schedule_id: int(count) + for schedule_id, count in ( + await session.execute( + select( + ScheduleEdges.schedule_id, + func.count(ScheduleEdges.edge_id), + ) + .where(ScheduleEdges.schedule_id.in_(ids)) + .group_by(ScheduleEdges.schedule_id) + ) + ).all() + } + return { + "request_id": context.request_id, + "data": [ + schedule_payload( + item, + node_count=node_counts.get(item.schedule_id, 0), + edge_count=edge_counts.get(item.schedule_id, 0), + ) + for item in items + ], + "meta": {"count": len(items)}, + } + + +# 创建新的 DAG 调度定义；初始状态不包含节点和边。 +@router.post( + "/api/v1/schedules", + status_code=status.HTTP_201_CREATED, +) +async def create_schedule( + payload: CreateScheduleRequest, + context: RequestContext = Depends(request_context), + session: AsyncSession = Depends(database_session), +) -> dict[str, Any]: + duplicate = await session.scalar( + select(Schedules).where( + Schedules.workspace_id == context.workspace.workspace_id, + Schedules.schedule_name == payload.schedule_name, + Schedules.deleted_at.is_(None), + ) + ) + if duplicate is not None: + raise HTTPException( + status.HTTP_409_CONFLICT, + "an active schedule with the same name already exists", + ) + if payload.enabled: + raise HTTPException( + status.HTTP_409_CONFLICT, + "a new empty schedule cannot be enabled", + ) + item = Schedules( + schedule_id=new_ulid(), + workspace_id=context.workspace.workspace_id, + schedule_name=payload.schedule_name, + description=payload.description, + trigger_type=payload.trigger_type, + cron_expression=payload.cron_expression, + timezone=payload.timezone, + enabled=0, + workflow_version=1, + max_concurrency=payload.max_concurrency, + failure_policy=payload.failure_policy, + created_by=context.user.user_id, + updated_by=context.user.user_id, + ) + _apply_next_run(item) + session.add(item) + await session.flush() + await session.refresh(item) + return { + "request_id": context.request_id, + "data": await detail_payload(item, session), + "meta": {}, + } + + +# 获取一个调度的完整画布数据，包括节点、边和当前工作流版本。 +@router.get("/api/v1/schedules/{schedule_id}") +async def get_schedule( + schedule_id: str, + context: RequestContext = Depends(request_context), + session: AsyncSession = Depends(database_session), +) -> dict[str, Any]: + item = await schedule_row(schedule_id, context, session) + return { + "request_id": context.request_id, + "data": await detail_payload(item, session), + "meta": {}, + } + + +# 更新调度基本属性，如名称、Cron、时区、是否启用和并发策略。 +@router.put("/api/v1/schedules/{schedule_id}") +@router.patch("/api/v1/schedules/{schedule_id}") +async def update_schedule( + schedule_id: str, + payload: UpdateScheduleRequest, + context: RequestContext = Depends(request_context), + session: AsyncSession = Depends(database_session), +) -> dict[str, Any]: + item = await schedule_row( + schedule_id, + context, + session, + for_update=True, + ) + require_workflow_version(item, payload.workflow_version) + mutable_fields = { + "schedule_name", + "description", + "trigger_type", + "cron_expression", + "timezone", + "enabled", + "max_concurrency", + "failure_policy", + } + if ( + "schedule_name" in payload.model_fields_set + and payload.schedule_name != item.schedule_name + ): + duplicate = await session.scalar( + select(Schedules).where( + Schedules.workspace_id == context.workspace.workspace_id, + Schedules.schedule_name == payload.schedule_name, + Schedules.schedule_id != schedule_id, + Schedules.deleted_at.is_(None), + ) + ) + if duplicate is not None: + raise HTTPException( + status.HTTP_409_CONFLICT, + "an active schedule with the same name already exists", + ) + for field in mutable_fields & payload.model_fields_set: + value = getattr(payload, field) + setattr(item, field, int(value) if field == "enabled" else value) + if ( + "trigger_type" in payload.model_fields_set + and item.trigger_type != "cron" + and "cron_expression" not in payload.model_fields_set + ): + item.cron_expression = None + _apply_next_run(item) + await _require_valid_when_enabled(item, session) + item.updated_by = context.user.user_id + item.workflow_version += 1 + await session.flush() + await session.refresh(item) + return { + "request_id": context.request_id, + "data": await detail_payload(item, session), + "meta": {}, + } + + +# 删除调度定义；请求携带 workflow_version 以避免误删他人刚修改的画布。 +@router.delete("/api/v1/schedules/{schedule_id}") +async def delete_schedule( + schedule_id: str, + payload: WorkflowVersionRequest, + request: Request, + context: RequestContext = Depends(request_context), + session: AsyncSession = Depends(database_session), +) -> dict[str, Any]: + item = await schedule_row( + schedule_id, + context, + session, + for_update=True, + ) + require_workflow_version(item, payload.workflow_version) + + # 正在执行的 DAG 依赖运行快照和日志对象。此时删除会让执行器无法安全收尾， + # 因此要求先等待运行结束，避免影响现有运行功能。 + active_run_id = await session.scalar( + select(ScheduleRuns.run_id) + .where( + ScheduleRuns.schedule_id == schedule_id, + ScheduleRuns.run_status.in_(_ACTIVE_RUN_STATUSES), + ) + .limit(1) + ) + if active_run_id is not None: + raise HTTPException( + status.HTTP_409_CONFLICT, + "schedule has active runs; wait for completion before deletion", + ) + + schedule_runs = list( + ( + await session.scalars( + select(ScheduleRuns).where(ScheduleRuns.schedule_id == schedule_id) + ) + ).all() + ) + run_ids = [run.run_id for run in schedule_runs] + node_runs = ( + list( + ( + await session.scalars( + select(ScheduleNodeRuns).where( + ScheduleNodeRuns.run_id.in_(run_ids) + ) + ) + ).all() + ) + if run_ids + else [] + ) + + # 调度删除会清理节点、边和所有运行记录；运行日志/结果文件同时移入回收站。 + await _delete_execution_artifacts( + _execution_artifact_ids([*schedule_runs, *node_runs]), + request, + session, + ) + if run_ids: + await session.execute( + delete(ScheduleNodeRuns).where(ScheduleNodeRuns.run_id.in_(run_ids)) + ) + await session.execute( + delete(ScheduleRuns).where(ScheduleRuns.run_id.in_(run_ids)) + ) + await session.execute( + delete(ScheduleEdges).where(ScheduleEdges.schedule_id == schedule_id) + ) + await session.execute( + delete(ScheduleNodes).where(ScheduleNodes.schedule_id == schedule_id) + ) + item.enabled = 0 + item.next_run_at = None + item.deleted_at = _mysql_utc(datetime.now(UTC)) + item.updated_by = context.user.user_id + item.workflow_version += 1 + await session.flush() + return { + "request_id": context.request_id, + "data": { + "schedule_id": item.schedule_id, + "deleted": True, + "workflow_version": item.workflow_version, + }, + "meta": {}, + } + + +# 向调度画布新增一个执行节点，并关联已发布的脚本版本。 +@router.post( + "/api/v1/schedules/{schedule_id}/nodes", + status_code=status.HTTP_201_CREATED, +) +async def create_schedule_node( + schedule_id: str, + payload: CreateScheduleNodeRequest, + context: RequestContext = Depends(request_context), + session: AsyncSession = Depends(database_session), +) -> dict[str, Any]: + schedule = await schedule_row( + schedule_id, + context, + session, + for_update=True, + ) + require_workflow_version(schedule, payload.workflow_version) + duplicate = await session.scalar( + select(ScheduleNodes).where( + ScheduleNodes.schedule_id == schedule_id, + ScheduleNodes.node_key == payload.node_key, + ) + ) + if duplicate is not None: + raise HTTPException( + status.HTTP_409_CONFLICT, + "node_key already exists in this schedule", + ) + await accessible_version(payload.versions_id, context, session) + node = ScheduleNodes( + node_id=new_ulid(), + schedule_id=schedule_id, + node_key=payload.node_key, + node_name=payload.node_name, + versions_id=payload.versions_id, + python_version=payload.python_version, + timeout_seconds=payload.timeout_seconds, + retry_count=payload.retry_count, + retry_interval_sec=payload.retry_interval_sec, + position_x=Decimal(str(payload.position_x)), + position_y=Decimal(str(payload.position_y)), + arguments_json=payload.arguments_json, + env_refs_json=payload.env_refs_json, + ) + session.add(node) + schedule.updated_by = context.user.user_id + schedule.workflow_version += 1 + await session.flush() + return { + "request_id": context.request_id, + "data": await detail_payload(schedule, session), + "meta": {"created_node_id": node.node_id}, + } + + +# 更新节点名称、执行参数、超时、重试和画布坐标等配置。 +@router.put("/api/v1/schedules/{schedule_id}/nodes/{node_id}") +async def update_schedule_node( + schedule_id: str, + node_id: str, + payload: UpdateScheduleNodeRequest, + context: RequestContext = Depends(request_context), + session: AsyncSession = Depends(database_session), +) -> dict[str, Any]: + schedule = await schedule_row( + schedule_id, + context, + session, + for_update=True, + ) + require_workflow_version(schedule, payload.workflow_version) + node = await session.scalar( + select(ScheduleNodes).where( + ScheduleNodes.node_id == node_id, + ScheduleNodes.schedule_id == schedule_id, + ) + ) + if node is None: + raise HTTPException(status.HTTP_404_NOT_FOUND, "schedule node not found") + if ( + "versions_id" in payload.model_fields_set + and payload.versions_id is not None + ): + await accessible_version(payload.versions_id, context, session) + mutable_fields = { + "node_name", + "versions_id", + "python_version", + "timeout_seconds", + "retry_count", + "retry_interval_sec", + "position_x", + "position_y", + "arguments_json", + "env_refs_json", + } + for field in mutable_fields & payload.model_fields_set: + value = getattr(payload, field) + if field in {"position_x", "position_y"} and value is not None: + value = Decimal(str(value)) + setattr(node, field, value) + schedule.updated_by = context.user.user_id + schedule.workflow_version += 1 + await session.flush() + return { + "request_id": context.request_id, + "data": await detail_payload(schedule, session), + "meta": {"updated_node_id": node.node_id}, + } + + +# 从调度画布删除节点，并同步清理关联边。 +@router.delete("/api/v1/schedules/{schedule_id}/nodes/{node_id}") +async def delete_schedule_node( + schedule_id: str, + node_id: str, + payload: DeleteScheduleNodeRequest, + request: Request, + context: RequestContext = Depends(request_context), + session: AsyncSession = Depends(database_session), +) -> dict[str, Any]: + schedule = await schedule_row( + schedule_id, + context, + session, + for_update=True, + ) + require_workflow_version(schedule, payload.workflow_version) + node = await session.scalar( + select(ScheduleNodes).where( + ScheduleNodes.node_id == node_id, + ScheduleNodes.schedule_id == schedule_id, + ) + ) + if node is None: + raise HTTPException(status.HTTP_404_NOT_FOUND, "schedule node not found") + active_run_id = await session.scalar( + select(ScheduleRuns.run_id) + .join(ScheduleNodeRuns, ScheduleNodeRuns.run_id == ScheduleRuns.run_id) + .where( + ScheduleRuns.schedule_id == schedule_id, + ScheduleRuns.run_status.in_(_ACTIVE_RUN_STATUSES), + ScheduleNodeRuns.node_id == node_id, + ) + .limit(1) + ) + if active_run_id is not None: + raise HTTPException( + status.HTTP_409_CONFLICT, + "node has active execution; wait for the run to finish before deletion", + ) + node_runs = list( + ( + await session.scalars( + select(ScheduleNodeRuns).where(ScheduleNodeRuns.node_id == node_id) + ) + ).all() + ) + if node_runs and not payload.delete_execution_history: + raise HTTPException( + status.HTTP_409_CONFLICT, + detail={ + "code": "node_execution_history_exists", + "message": "node has execution history; confirmation required", + }, + ) + if node_runs: + await _delete_execution_artifacts( + _execution_artifact_ids(node_runs), + request, + session, + ) + await session.execute( + delete(ScheduleNodeRuns).where(ScheduleNodeRuns.node_id == node_id) + ) + await session.execute( + delete(ScheduleEdges).where( + ScheduleEdges.schedule_id == schedule_id, + or_( + ScheduleEdges.source_node_id == node_id, + ScheduleEdges.target_node_id == node_id, + ), + ) + ) + await session.delete(node) + schedule.updated_by = context.user.user_id + schedule.workflow_version += 1 + await session.flush() + return { + "request_id": context.request_id, + "data": await detail_payload(schedule, session), + "meta": {"deleted_node_id": node_id}, + } + + +# 在两个节点之间新增依赖边，表示目标节点必须等待源节点完成。 +@router.post( + "/api/v1/schedules/{schedule_id}/edges", + status_code=status.HTTP_201_CREATED, +) +async def create_schedule_edge( + schedule_id: str, + payload: CreateScheduleEdgeRequest, + context: RequestContext = Depends(request_context), + session: AsyncSession = Depends(database_session), +) -> dict[str, Any]: + schedule = await schedule_row( + schedule_id, + context, + session, + for_update=True, + ) + require_workflow_version(schedule, payload.workflow_version) + node_ids = set( + ( + await session.scalars( + select(ScheduleNodes.node_id).where( + ScheduleNodes.schedule_id == schedule_id, + ScheduleNodes.node_id.in_( + [payload.source_node_id, payload.target_node_id] + ), + ) + ) + ).all() + ) + if node_ids != {payload.source_node_id, payload.target_node_id}: + raise HTTPException( + status.HTTP_422_UNPROCESSABLE_ENTITY, + "both edge nodes must belong to the schedule", + ) + duplicate = await session.scalar( + select(ScheduleEdges).where( + ScheduleEdges.schedule_id == schedule_id, + ScheduleEdges.source_node_id == payload.source_node_id, + ScheduleEdges.target_node_id == payload.target_node_id, + ) + ) + if duplicate is not None: + raise HTTPException( + status.HTTP_409_CONFLICT, + "directed edge already exists", + ) + edge = ScheduleEdges( + edge_id=new_ulid(), + schedule_id=schedule_id, + source_node_id=payload.source_node_id, + target_node_id=payload.target_node_id, + condition_expr=payload.condition_expr, + ) + session.add(edge) + await session.flush() + node_rows, edges = await graph_rows(schedule_id, session) + validation = validate_dag([row[0] for row in node_rows], edges) + if not validation["valid"]: + raise HTTPException( + status.HTTP_409_CONFLICT, + detail={ + "code": "DAG_INVALID", + "message": "edge would make the schedule graph invalid", + "validation": validation, + }, + ) + schedule.updated_by = context.user.user_id + schedule.workflow_version += 1 + await session.flush() + return { + "request_id": context.request_id, + "data": await detail_payload(schedule, session), + "meta": {"created_edge_id": edge.edge_id}, + } + + +# 修改一条依赖边的条件表达式或其他可编辑字段。 +@router.put("/api/v1/schedules/{schedule_id}/edges/{edge_id}") +async def update_schedule_edge( + schedule_id: str, + edge_id: str, + payload: UpdateScheduleEdgeRequest, + context: RequestContext = Depends(request_context), + session: AsyncSession = Depends(database_session), +) -> dict[str, Any]: + schedule = await schedule_row( + schedule_id, + context, + session, + for_update=True, + ) + require_workflow_version(schedule, payload.workflow_version) + edge = await session.scalar( + select(ScheduleEdges).where( + ScheduleEdges.edge_id == edge_id, + ScheduleEdges.schedule_id == schedule_id, + ) + ) + if edge is None: + raise HTTPException(status.HTTP_404_NOT_FOUND, "schedule edge not found") + edge.condition_expr = payload.condition_expr + schedule.updated_by = context.user.user_id + schedule.workflow_version += 1 + await session.flush() + return { + "request_id": context.request_id, + "data": await detail_payload(schedule, session), + "meta": {"updated_edge_id": edge.edge_id}, + } + + +# 删除节点之间的依赖关系，不会删除节点本身。 +@router.delete("/api/v1/schedules/{schedule_id}/edges/{edge_id}") +async def delete_schedule_edge( + schedule_id: str, + edge_id: str, + payload: WorkflowVersionRequest, + context: RequestContext = Depends(request_context), + session: AsyncSession = Depends(database_session), +) -> dict[str, Any]: + schedule = await schedule_row( + schedule_id, + context, + session, + for_update=True, + ) + require_workflow_version(schedule, payload.workflow_version) + edge = await session.scalar( + select(ScheduleEdges).where( + ScheduleEdges.edge_id == edge_id, + ScheduleEdges.schedule_id == schedule_id, + ) + ) + if edge is None: + raise HTTPException(status.HTTP_404_NOT_FOUND, "schedule edge not found") + await session.delete(edge) + schedule.updated_by = context.user.user_id + schedule.workflow_version += 1 + await session.flush() + return { + "request_id": context.request_id, + "data": await detail_payload(schedule, session), + "meta": {"deleted_edge_id": edge_id}, + } + + +# 校验画布是否为可执行 DAG，例如是否存在环、孤立节点或无效版本。 +@router.post("/api/v1/schedules/{schedule_id}/validate") +async def validate_schedule( + schedule_id: str, + context: RequestContext = Depends(request_context), + session: AsyncSession = Depends(database_session), +) -> dict[str, Any]: + item = await schedule_row(schedule_id, context, session) + node_rows, edges = await graph_rows(schedule_id, session) + return { + "request_id": context.request_id, + "data": { + "schedule_id": item.schedule_id, + "workflow_version": item.workflow_version, + **validate_dag([row[0] for row in node_rows], edges), + }, + "meta": {}, + } diff --git a/backend/src/backend/schemas.py b/backend/src/backend/schemas.py new file mode 100644 index 0000000..958588f --- /dev/null +++ b/backend/src/backend/schemas.py @@ -0,0 +1,78 @@ +from typing import Literal + +from common.schemas import StrictModel +from pydantic import Field, field_validator + + +class CreateResourceUploadRequest(StrictModel): + file_name: str = Field(min_length=1, max_length=255) + content_type: str = Field(min_length=1, max_length=255) + expected_size_bytes: int = Field(ge=0, le=100 * 1024 * 1024) + expected_hash: str | None = Field(default=None, min_length=64, max_length=64) + target_path: str = Field(default="", max_length=1024) + + @field_validator("target_path") + @classmethod + def validate_target_path(cls, value: str) -> str: + # POSIX 相对路径，不能含 ..、绝对前缀、控制字符 + if value and (value.startswith("/") or "\\" in value + or any(seg == ".." for seg in value.split("/")) + or any(ord(c) < 0x20 or ord(c) == 0x7F for c in value)): + raise ValueError("target_path must be a clean relative POSIX path") + # 去前导 /；允许尾部 / + return value.strip("/") + + @field_validator("expected_hash") + @classmethod + def validate_hash(cls, value: str | None) -> str | None: + if value is None: + return None + normalized = value.lower() + if any(character not in "0123456789abcdef" for character in normalized): + raise ValueError("expected_hash must be SHA-256 hex") + return normalized + + +class CompleteResourceUploadRequest(StrictModel): + resource_name: str = Field(min_length=1, max_length=255) + description: str | None = Field(default=None, max_length=1000) + visibility: Literal["private", "workspace", "public"] = "private" + + +class CreateScriptRequest(StrictModel): + script_name: str = Field(min_length=1, max_length=255) + script_type: Literal["python", "notebook"] + content: str = Field(max_length=10 * 1024 * 1024) + visibility: Literal["private", "workspace", "public"] = "private" + parent_path: str | None = Field(default=None, max_length=1024) + + +class CreateWorkspaceDirectoryRequest(StrictModel): + directory_name: str = Field(min_length=1, max_length=255) + parent_path: str = Field(default="", max_length=1024) + + +class UpdateScriptRequest(StrictModel): + content: str = Field(max_length=10 * 1024 * 1024) + + +class LockScriptRequest(StrictModel): + is_locked: bool + + +class PublishVersionRequest(StrictModel): + source_object_id: str | None = Field( + default=None, + min_length=26, + max_length=26, + ) + release_note: str | None = Field(default=None, max_length=1000) + visibility: Literal["private", "workspace", "public"] = "workspace" + + +class DownloadUrlRequest(StrictModel): + expires_seconds: int = Field(default=300, ge=30, le=3600) + + +class ResourceRelativePathRequest(StrictModel): + script_path: str = Field(min_length=1, max_length=512) diff --git a/backend/src/backend/scripts.py b/backend/src/backend/scripts.py new file mode 100644 index 0000000..a48f94f --- /dev/null +++ b/backend/src/backend/scripts.py @@ -0,0 +1,1601 @@ +"""工作区脚本与 Notebook 的公开 API。 + +脚本元数据（名称、归属、锁、版本关系）保存在 MySQL；文件正文和版本产物保存到 +对象存储或本地 ``data/``。本模块负责把两者保持一致，并提供目录、上传、编辑锁、 +版本发布和下载地址等接口。 + +前端的 ``frontend/app/services/api.ts`` 通过 ``/api/v1/scripts`` 和 +``/api/v1/workspace-directories`` 调用这里的路由。 +""" + +import base64 +import hashlib +import json +import mimetypes +from datetime import UTC, datetime +from pathlib import PurePosixPath +from typing import Any + +from common.config import settings +from common.db.models import ( + Scripts, + StorageObjects, + Users, + Versions, +) +from common.ids import new_ulid +from common.storage import actual_bucket_name, build_storage_uri +from common.storage.schemas import ServerObjectRequest +from fastapi import ( + APIRouter, + BackgroundTasks, + Depends, + HTTPException, + Query, + Request, + status, +) +from loguru import logger +from sqlalchemy import func, select +from sqlalchemy.ext.asyncio import AsyncSession + +from backend.dependencies import ( + RequestContext, + database_session, + request_context, +) +from backend.runtime_client import RuntimeClientError +from backend.schemas import ( + CreateScriptRequest, + CreateWorkspaceDirectoryRequest, + DownloadUrlRequest, + LockScriptRequest, + PublishVersionRequest, + UpdateScriptRequest, +) +from backend.services.storage import ( + _resolve_unique_object_key, + create_download_url_payload, + create_server_object_payload, + soft_delete_object, +) + +router = APIRouter(tags=["scripts"]) + + +def normalize_user_path(value: str, *, allow_empty: bool = True) -> str: + normalized = value.replace("\\", "/").strip().strip("/") + if not normalized and allow_empty: + return "" + pure_path = PurePosixPath(normalized) + if ( + pure_path.is_absolute() + or not pure_path.parts + or any( + part in {"", ".", ".."} or any(ord(char) < 32 for char in part) + for part in pure_path.parts + ) + ): + raise HTTPException( + status.HTTP_422_UNPROCESSABLE_ENTITY, + "invalid workspace path", + ) + return pure_path.as_posix() + + +def safe_directory_name(value: str) -> str: + name = value.strip() + if ( + not name + or name in {".", ".."} + or name.startswith(".") + or "/" in name + or "\\" in name + or any(ord(char) < 32 for char in name) + ): + raise HTTPException( + status.HTTP_422_UNPROCESSABLE_ENTITY, + "invalid directory_name", + ) + return name + + +# Version/run artifacts are not part of the user's workspace directory tree. +TREE_EXCLUDED_USAGE_TYPES = ( + "version_artifact", + "snapshot", + "run_log", + "run_result", +) + + +def user_relative_path(context: RequestContext, child_path: str = "") -> str: + base = f"workspace/{context.user.user_id}" + normalized = normalize_user_path(child_path) + return f"{base}/{normalized}" if normalized else base + + +def safe_script_name(value: str, script_type: str) -> str: + name = value.replace("\\", "/").rsplit("/", 1)[-1].strip() + if not name or name in {".", ".."} or any(ord(char) < 32 for char in name): + raise HTTPException( + status.HTTP_422_UNPROCESSABLE_ENTITY, + "invalid script_name", + ) + expected_suffix = ".py" if script_type == "python" else ".ipynb" + if not name.lower().endswith(expected_suffix): + name += expected_suffix + return name + + +def _jupyter_path(script_type: str, script_name: str) -> str: + """Return the Jupyter basename for a newly-created script. + + Callers MUST pass the value returned by ``safe_script_name()`` so the + extension is correct and the name is POSIX-safe. The on-disk filename + is the user's name — there is no separate ULID segment. + """ + expected_suffix = ".py" if script_type == "python" else ".ipynb" + if not script_name.lower().endswith(expected_suffix): + script_name += expected_suffix + return script_name + + +def _derive_jupyter_path( + storage_object: StorageObjects | None, + workspace_id: str, + script_type: str, + script_id: str, +) -> str: + """Return the real Jupyter path for an existing script. + + For normal workspace files the path is taken from + ``StorageObjects.object_key`` with the workspace prefix removed. + For jupyter-only scripts that have no StorageObjects row, fall back + to the flat ``_jupyter_path()`` basename so existing behavior is + preserved. + """ + if storage_object is None or not storage_object.object_key: + return _jupyter_path(script_type, script_id) + return storage_object.object_key.removeprefix(f"{workspace_id}/") + + +def validate_script_content(content: str, script_type: str) -> bytes: + encoded = content.encode("utf-8") + if len(encoded) > 10 * 1024 * 1024: + raise HTTPException( + status.HTTP_413_REQUEST_ENTITY_TOO_LARGE, + "script exceeds 10 MiB", + ) + if script_type == "notebook": + try: + notebook = json.loads(content) + except json.JSONDecodeError as exc: + raise HTTPException( + status.HTTP_422_UNPROCESSABLE_ENTITY, + "notebook content must be valid JSON", + ) from exc + if not isinstance(notebook, dict) or not isinstance( + notebook.get("cells"), + list, + ): + raise HTTPException( + status.HTTP_422_UNPROCESSABLE_ENTITY, + "notebook must contain a cells array", + ) + return encoded + + +def script_payload( + script: Scripts, + storage_object: StorageObjects | dict[str, Any] | None, + *, + owner_display_name: str | None = None, +) -> dict[str, Any]: + if storage_object is None: + relative_path = None + object_key = None + content_hash = None + size_bytes = 0 + elif isinstance(storage_object, dict): + relative_path = storage_object.get("relative_path") + object_key = storage_object.get("object_key") + content_hash = storage_object.get("content_hash") + size_bytes = storage_object.get("size_bytes", 0) + else: + relative_path = storage_object.relative_path + object_key = storage_object.object_key + content_hash = storage_object.content_hash + size_bytes = storage_object.size_bytes + workspace_prefix = f"{script.workspace_id}/" + if object_key: + jupyter_path = ( + object_key.removeprefix(workspace_prefix) + ) + else: + jupyter_path = _jupyter_path(script.script_type, script.script_id) + return { + "script_id": script.script_id, + "workspace_id": script.workspace_id, + "current_object_id": script.current_object_id, + "owner_user_id": script.owner_user_id, + "owner_display_name": owner_display_name, + "script_name": script.script_name, + "script_type": script.script_type, + "visibility": script.visibility, + "status": script.status, + "is_locked": bool(script.is_locked), + "relative_path": relative_path, + "jupyter_path": jupyter_path, + "content_hash": content_hash, + "size_bytes": size_bytes, + "created_at": script.created_at.isoformat(), + "updated_at": script.updated_at.isoformat(), + } + + +def version_payload(version: Versions) -> dict[str, Any]: + return { + "versions_id": version.versions_id, + "workspace_id": version.workspace_id, + "script_id": version.script_id, + "source_object_id": version.source_object_id, + "artifact_object_id": version.artifact_object_id, + "version_no": version.version_no, + "version_label": version.version_label, + "source_path": version.source_path, + "artifact_path": version.artifact_path, + "content_hash": version.content_hash, + "file_size_bytes": version.file_size_bytes, + "visibility": version.visibility, + "release_note": version.release_note, + "created_by": version.created_by, + "created_at": version.created_at.isoformat(), + } + + +async def get_script_row( + script_id: str, + context: RequestContext, + session: AsyncSession, + *, + for_update: bool = False, + allow_missing_storage_object: bool = False, +) -> tuple[Scripts, StorageObjects | None]: + if for_update: + script = await session.scalar( + select(Scripts) + .where( + Scripts.script_id == script_id, + Scripts.workspace_id == context.workspace.workspace_id, + Scripts.status == "active", + ) + .with_for_update() + ) + if script is None: + raise HTTPException( + status.HTTP_404_NOT_FOUND, + "script not found", + ) + storage_object = await session.get( + StorageObjects, + script.current_object_id, + ) + if storage_object is None and not allow_missing_storage_object: + raise HTTPException( + status.HTTP_409_CONFLICT, + "script working-copy metadata is missing", + ) + row = (script, storage_object) + else: + if allow_missing_storage_object: + statement = ( + select(Scripts, StorageObjects) + .outerjoin( + StorageObjects, + StorageObjects.storage_object_id == Scripts.current_object_id, + ) + .where( + Scripts.script_id == script_id, + Scripts.workspace_id == context.workspace.workspace_id, + Scripts.status == "active", + ) + ) + else: + statement = ( + select(Scripts, StorageObjects) + .join( + StorageObjects, + StorageObjects.storage_object_id == Scripts.current_object_id, + ) + .where( + Scripts.script_id == script_id, + Scripts.workspace_id == context.workspace.workspace_id, + Scripts.status == "active", + ) + ) + row = (await session.execute(statement)).one_or_none() + if row is None: + raise HTTPException(status.HTTP_404_NOT_FOUND, "script not found") + script, storage_object = row + return script, storage_object + + +def require_script_modify_access( + script: Scripts, + *, + user_id: str, + is_admin: bool, +) -> None: + """Enforce the V3.1 §4 access rules for write operations on a script. + + Rules: + * admin: always allowed + * owner: always allowed + * non-owner: allowed iff ``is_locked`` is False + + Reads (``list`` / ``get``) intentionally do not call this helper — the + design contract is "everyone in the workspace can see the script + list, but only the owner (or admin) can mutate when locked". + """ + if is_admin or script.owner_user_id == user_id: + return + if not script.is_locked: + return + raise HTTPException( + status.HTTP_403_FORBIDDEN, + "script is locked; only the owner (or an administrator) may modify it", + ) + + +async def create_script_record( + *, + name: str, + script_type: str, + content: bytes, + visibility: str, + parent_path: str | None, + request: Request, + context: RequestContext, + session: AsyncSession, +) -> tuple[Scripts, dict[str, Any]]: + parent = normalize_user_path(parent_path) if parent_path else "" + scoped_prefix = user_relative_path(context) + parent_dir_row = None + if parent: + parent_dir_row = await session.scalar( + select(StorageObjects).where( + StorageObjects.workspace_id == context.workspace.workspace_id, + StorageObjects.object_status == "available", + StorageObjects.object_type == "directory", + StorageObjects.relative_path == f"{scoped_prefix}/{parent}", + ) + ) + if parent_dir_row is None: + raise HTTPException( + status.HTTP_404_NOT_FOUND, + "parent directory not found", + ) + + # The Jupyter-side filename is the sanitized script name; extension and + # path separators are enforced by safe_script_name(). The script_id ULID + # remains the Scripts PK only. + user_id = context.user.user_id + script_id = new_ulid() # PK only — kept as ULID + jupyter_basename = _jupyter_path(script_type, name) # name is already passed through safe_script_name + # The Jupyter path uses the parent directory's user-relative path + # (parent_dir_row.relative_path looks like "workspace/{user_id}/foo/bar"), + # NOT the parent's ULID. Strip the "workspace/{user_id}/" prefix to get + # the Jupyter-relative parent segment. + parent_relative = parent_dir_row.relative_path if parent_dir_row is not None else None + workspace_user_prefix = f"workspace/{user_id}" + if parent_relative and parent_relative.startswith(workspace_user_prefix + "/"): + parent_segment = parent_relative[len(workspace_user_prefix) + 1:] + else: + parent_segment = "" + if parent_segment: + jupyter_path = f"{user_id}/{parent_segment}/{jupyter_basename}" + else: + jupyter_path = f"{user_id}/{jupyter_basename}" + + # StorageObjects relative_path mirrors the Jupyter layout under the + # workspace/user prefix. Compute it before the conflict check so we can + # scope duplicates by the full path, not just the display name. + relative_path = user_relative_path( + context, f"{parent}/{jupyter_basename}" if parent else jupyter_basename + ) + logger.debug(relative_path) + + name_clash = await session.scalar( + select(Scripts.script_id) + .join( + StorageObjects, + StorageObjects.storage_object_id == Scripts.current_object_id, + ) + .where( + Scripts.workspace_id == context.workspace.workspace_id, + Scripts.owner_user_id == context.user.user_id, + Scripts.script_name == name, + Scripts.status == "active", + StorageObjects.relative_path == relative_path, + ) + ) + if name_clash is not None: + raise HTTPException( + status.HTTP_409_CONFLICT, + "a script with the same name already exists at this path", + ) + + # Push the file directly to the workspace's live Jupyter instance. + # Auto-starts the workspace if no Jupyter is running yet. Once + # Jupyter has the file in its local mount, the rclone VFS will + # eventually replicate it back to object storage. + runtime_client = request.app.state.runtime_client + workspace_id = context.workspace.workspace_id + content_hash = hashlib.sha256(content).hexdigest() + size_bytes = len(content) + # Jupyter Contents API PUT does not create intermediate directories. + # Make sure the user's own subfolder exists at the Jupyter level before + # we try to write into it; on nested creates, also ensure the parent + # directory under it. ensure_directory is idempotent (GET-first pattern). + try: + await runtime_client.ensure_directory(workspace_id, user_id) + if parent_segment: + await runtime_client.ensure_directory( + workspace_id, f"{user_id}/{parent_segment}" + ) + except RuntimeClientError as exc: + raise HTTPException( + status_code=exc.status_code, + detail=exc.detail, + ) from exc + + try: + if script_type == "notebook": + notebook = json.loads(content.decode("utf-8")) + logger.debug(notebook) + jupyter_resp = await runtime_client.create_notebook( + workspace_id, + name=jupyter_path, + cells=notebook.get("cells"), + ) + else: + jupyter_resp = await runtime_client.upload_file( + workspace_id, + name=jupyter_path, + content=content.decode("utf-8"), + content_type=(mimetypes.guess_type(jupyter_basename)[0] or "text/plain"), + ) + logger.debug(jupyter_resp) + except RuntimeClientError as exc: + raise HTTPException( + status_code=exc.status_code, + detail=exc.detail, + ) from exc + + # Build a real StorageObjects row so the file participates in + # workspace-tree / list / get queries that JOIN this table. The + # bytes live in the Jupyter mount; rclone replicates them to S3 + # asynchronously. We mark the row "available" because the + # file is queryable as a workspace file from the user's POV; the + # storage_uri points at where the replicated bytes will land. + object_id = new_ulid() + object_key = f"{workspace_id}/{jupyter_path}" + # Route through conflict helper to handle cross-entity collisions and + # to keep the "at most one is_deleted=0 per (backend, bucket, key)" + # invariant. For same-entity re-upload after soft-delete, the helper + # returns the original key because deleted rows are excluded from its + # "available" filter; for cross-entity collision it appends a ULID + # suffix so the new row gets a distinct object_key. + object_key = await _resolve_unique_object_key(session, object_key) + object_key_hash = hashlib.sha256(object_key.encode("utf-8")).digest() + bucket_name = actual_bucket_name("workspace") + mime_type = mimetypes.guess_type(jupyter_basename)[0] + storage_object = StorageObjects( + storage_object_id=object_id, + workspace_id=context.workspace.workspace_id, + owner_user_id=context.user.user_id, + object_type="file", + usage_type="working_copy", + storage_backend=settings.storage_backend, + bucket_name=bucket_name, + object_key=object_key, + object_key_hash=object_key_hash, + storage_uri=build_storage_uri(bucket_name, object_key), + file_name=name, + file_extension=PurePosixPath(jupyter_basename).suffix.lower() or None, + mime_type=mime_type, + size_bytes=size_bytes, + content_hash=content_hash, + visibility=visibility, + is_immutable=0, + object_status="available", + created_by=context.user.user_id, + relative_path=relative_path, + path_hash=hashlib.sha256(relative_path.encode("utf-8")).digest(), + ) + script = Scripts( + script_id=script_id, + workspace_id=context.workspace.workspace_id, + current_object_id=object_id, + owner_user_id=context.user.user_id, + script_name=name, + script_type=script_type, + visibility=visibility, + status="active", + ) + try: + session.add(storage_object) + await session.flush() + except Exception: + # Best-effort compensating cleanup: remove the Jupyter file we + # just created so a failed flush does not leave an orphan on disk. + try: + await runtime_client.delete_file(workspace_id, name=jupyter_path) + except RuntimeClientError as cleanup_exc: + if cleanup_exc.status_code == 404: + pass + else: + logger.warning( + f"failed to clean up Jupyter file {jupyter_path} " + f"after DB flush error: {cleanup_exc.status_code} {cleanup_exc.detail}" + ) + raise + await session.refresh(storage_object) + + try: + session.add(script) + await session.flush() + except Exception: + # If the Scripts row fails, the active transaction rolls back the + # StorageObjects insert as well because both share the same session. + # Still attempt to remove the Jupyter file we just pushed. + try: + await runtime_client.delete_file(workspace_id, name=jupyter_path) + except RuntimeClientError as cleanup_exc: + if cleanup_exc.status_code == 404: + pass + else: + logger.warning( + f"failed to clean up Jupyter file {jupyter_path} " + f"after DB flush error: {cleanup_exc.status_code} {cleanup_exc.detail}" + ) + raise + await session.refresh(script) + return script, storage_object + + +# 新建空的 Python 脚本或 Notebook：同时创建数据库元数据和初始文件内容。 +@router.post("/api/v1/scripts", status_code=status.HTTP_201_CREATED) +async def create_script( + payload: CreateScriptRequest, + request: Request, + background_tasks: BackgroundTasks, + context: RequestContext = Depends(request_context), + session: AsyncSession = Depends(database_session), +) -> dict[str, Any]: + name = safe_script_name(payload.script_name, payload.script_type) + content = validate_script_content(payload.content, payload.script_type) + script, storage_data = await create_script_record( + name=name, + script_type=payload.script_type, + content=content, + visibility=payload.visibility, + parent_path=payload.parent_path, + request=request, + context=context, + session=session, + ) + # background_tasks.add_task( + # request.app.state.rclone_rc_client.vfs_refresh, + # dir_path=context.workspace.workspace_id, + # recursive=True, + # ) + return { + "request_id": context.request_id, + "data": script_payload(script, storage_data), + "meta": {}, + } + + +# 上传现有脚本文件：校验文件名/类型后写入存储，并建立 Scripts 记录。 +@router.post( + "/api/v1/scripts/upload", + status_code=status.HTTP_201_CREATED, +) +async def upload_script( + request: Request, + background_tasks: BackgroundTasks, + file_name: str = Query(min_length=1, max_length=255), + parent_path: str = Query(default="", max_length=1024), + visibility: str = Query( + default="workspace", + pattern="^(private|workspace|public)$", + ), + context: RequestContext = Depends(request_context), + session: AsyncSession = Depends(database_session), +) -> dict[str, Any]: + suffix = PurePosixPath(file_name).suffix.lower() + if suffix not in {".py", ".ipynb"}: + raise HTTPException( + status.HTTP_422_UNPROCESSABLE_ENTITY, + "only .py and .ipynb files can be uploaded", + ) + script_type = "notebook" if suffix == ".ipynb" else "python" + name = safe_script_name(file_name, script_type) + body = await request.body() + if len(body) > 10 * 1024 * 1024: + raise HTTPException( + status.HTTP_413_REQUEST_ENTITY_TOO_LARGE, + "script exceeds 10 MiB", + ) + try: + text = body.decode("utf-8") + except UnicodeDecodeError as exc: + raise HTTPException( + status.HTTP_422_UNPROCESSABLE_ENTITY, + "script must use UTF-8 encoding", + ) from exc + content = validate_script_content(text, script_type) + script, storage_data = await create_script_record( + name=name, + script_type=script_type, + content=content, + visibility=visibility, + parent_path=parent_path, + request=request, + context=context, + session=session, + ) + # background_tasks.add_task( + # request.app.state.rclone_rc_client.vfs_refresh, + # dir_path=context.workspace.workspace_id, + # recursive=True, + # ) + return { + "request_id": context.request_id, + "data": script_payload(script, storage_data), + "meta": {}, + } + + +# 返回旧版一次性完整目录树，保留给兼容旧前端；新页面通常按目录懒加载。 +@router.get("/api/v1/workspace-tree") +async def get_workspace_tree( + context: RequestContext = Depends(request_context), + session: AsyncSession = Depends(database_session), +) -> dict[str, Any]: + # Workspace object storage uses implicit directories (object key prefixes) + # plus explicit directory rows, so we derive the tree from + # ``StorageObjects.relative_path`` and ``StorageObjects.object_type`` + # rather than walking a local filesystem. Only paths that start with the + # user's scoped prefix (and that are currently active) contribute. + scoped_prefix = user_relative_path(context) + if scoped_prefix: + like_prefix = f"{scoped_prefix}%" + else: + like_prefix = "%" + + rows = ( + await session.execute( + select(StorageObjects.relative_path, StorageObjects.object_type).where( + StorageObjects.workspace_id == context.workspace.workspace_id, + StorageObjects.object_status == "available", + StorageObjects.is_deleted == 0, + StorageObjects.relative_path.like(like_prefix), + StorageObjects.usage_type.notin_(TREE_EXCLUDED_USAGE_TYPES), + ) + ) + ).all() + + directories: dict[str, dict[str, str]] = {} + for relative, object_type in rows: + if not relative: + continue + # Strip the scoped prefix so the returned paths are workspace-local. + if scoped_prefix and relative.startswith(scoped_prefix + "/"): + trimmed = relative[len(scoped_prefix) + 1 :] + elif relative == scoped_prefix: + continue + else: + trimmed = relative + # Explicit directory rows are included directly, then we still + # materialise every ancestor directory. + if object_type == "directory" and trimmed: + directories.setdefault( + trimmed, + { + "path": trimmed, + "name": trimmed.rsplit("/", 1)[-1], + "parent_path": "" + if "/" not in trimmed + else trimmed.rsplit("/", 1)[0], + }, + ) + parts = trimmed.split("/")[:-1] + for index in range(1, len(parts) + 1): + directory_path = "/".join(parts[:index]) + directories.setdefault( + directory_path, + { + "path": directory_path, + "name": parts[index - 1], + "parent_path": "" if index == 1 else "/".join(parts[: index - 1]), + }, + ) + sorted_dirs = sorted(directories.values(), key=lambda item: item["path"]) + return { + "request_id": context.request_id, + "data": {"directories": sorted_dirs}, + "meta": {"directory_count": len(sorted_dirs)}, + } + + +# 查询某个目录下的直接子目录，供前端按需展开工作区树。 +@router.get("/api/v1/workspace-directories") +async def list_workspace_directories( + parent_path: str = Query(default=""), + context: RequestContext = Depends(request_context), + session: AsyncSession = Depends(database_session), +) -> dict[str, Any]: + """List direct child directories of a workspace path. + + Empty ``parent_path`` returns the directories immediately under the + user's scoped root. Only available, non-deleted StorageObjects are + considered. + """ + scoped_prefix = user_relative_path(context) + parent = normalize_user_path(parent_path) + target_prefix = f"{scoped_prefix}/{parent}" if parent else scoped_prefix + descendant_prefix = f"{target_prefix}/" + + rows = ( + await session.execute( + select(StorageObjects.relative_path).where( + StorageObjects.workspace_id == context.workspace.workspace_id, + StorageObjects.object_status == "available", + StorageObjects.is_deleted == 0, + StorageObjects.relative_path.like(f"{descendant_prefix}%"), + ~StorageObjects.relative_path.like(f"{descendant_prefix}%/%"), + StorageObjects.object_type == "directory", + StorageObjects.usage_type.notin_(TREE_EXCLUDED_USAGE_TYPES), + ) + ) + ).all() + + directories: dict[str, dict[str, Any]] = {} + for (relative,) in rows: + if not relative or not relative.startswith(descendant_prefix): + continue + suffix = relative[len(descendant_prefix) :] + if "/" in suffix: + continue + child_path = f"{parent}/{suffix}" if parent else suffix + directories.setdefault( + child_path, + { + "path": child_path, + "name": suffix, + "parent_path": parent, + "has_children": False, + }, + ) + + for directory in directories.values(): + # directory['path'] is already workspace-relative and includes the parent segment. + child_prefix = f"{scoped_prefix}/{directory['path']}/" + has_children = await session.scalar( + select(StorageObjects.storage_object_id).where( + StorageObjects.workspace_id == context.workspace.workspace_id, + StorageObjects.object_status == "available", + StorageObjects.is_deleted == 0, + StorageObjects.relative_path.like(f"{child_prefix}%"), + ~StorageObjects.relative_path.like(f"{child_prefix}%/%"), + StorageObjects.usage_type.notin_(TREE_EXCLUDED_USAGE_TYPES), + ).limit(1) + ) + directory["has_children"] = has_children is not None + + sorted_dirs = sorted(directories.values(), key=lambda item: item["path"]) + return { + "request_id": context.request_id, + "data": {"directories": sorted_dirs}, + "meta": {"directory_count": len(sorted_dirs)}, + } + + +# 在工作区内创建逻辑目录；目录信息由脚本相对路径推导，不对应容器本地文件夹。 +@router.post( + "/api/v1/workspace-directories", + status_code=status.HTTP_201_CREATED, +) +async def create_workspace_directory( + payload: CreateWorkspaceDirectoryRequest, + request: Request, + context: RequestContext = Depends(request_context), + session: AsyncSession = Depends(database_session), +) -> dict[str, Any]: + name = safe_directory_name(payload.directory_name) + parent = normalize_user_path(payload.parent_path) + child_path = f"{parent}/{name}" if parent else name + relative_path = user_relative_path(context, child_path) + scoped_prefix = user_relative_path(context) + path_hash = hashlib.sha256(relative_path.encode("utf-8")).digest() + parent_dir_row = None + if parent: + parent_relative = f"{scoped_prefix}/{parent}" + # Only an available directory row at the exact parent path counts as + # a parent. Capture it so we can derive the new Jupyter path from its + # relative_path (which uses the user-supplied directory names). + parent_dir_row = await session.scalar( + select(StorageObjects).where( + StorageObjects.workspace_id == context.workspace.workspace_id, + StorageObjects.object_status == "available", + StorageObjects.object_type == "directory", + StorageObjects.relative_path == parent_relative, + ) + ) + if parent_dir_row is None: + raise HTTPException( + status.HTTP_404_NOT_FOUND, + "parent directory not found", + ) + # Conflict check relies on SELECT ... FOR UPDATE over (workspace_id, storage_backend, + # path_hash). The underlying index is no longer unique, so conflict determination is + # fully application-level: a soft-deleted row at the same path can be revived, while + # an available row is a conflict. Concurrent inserts are no longer serialized by a + # DB unique constraint; callers must ensure the FOR UPDATE lock covers the race. + existing = await session.scalar( + select(StorageObjects) + .where( + StorageObjects.workspace_id == context.workspace.workspace_id, + StorageObjects.storage_backend == settings.storage_backend, + StorageObjects.path_hash == path_hash, + ) + .with_for_update() + ) + if existing is not None and existing.object_status == "available": + raise HTTPException( + status.HTTP_409_CONFLICT, + "a file or directory with the same path already exists", + ) + + # The Jupyter-facing directory name is the sanitized user-supplied name. + # dir_id is still generated up front because the DB PK and path_hash + # collision-revival reuse depend on it. Create the directory in Jupyter + # before persisting the DB row; if persistence fails we clean up. + dir_id = new_ulid() # PK only — not in the path anymore + user_id = context.user.user_id + parent_relative = parent_dir_row.relative_path if parent_dir_row is not None else None + workspace_user_prefix = f"workspace/{user_id}" + if parent_relative and parent_relative.startswith(workspace_user_prefix + "/"): + parent_segment = parent_relative[len(workspace_user_prefix) + 1:] + else: + parent_segment = "" + if parent_segment: + jupyter_path = f"{user_id}/{parent_segment}/{name}" + else: + jupyter_path = f"{user_id}/{name}" + runtime_client = request.app.state.runtime_client + workspace_id = context.workspace.workspace_id + + # Jupyter Contents API PUT does not create intermediate directories. + # Make sure the user's own subfolder exists at the Jupyter level before + # we try to write into it; on nested creates, also ensure the parent + # directory under it. ensure_directory is idempotent (GET-first pattern). + try: + await runtime_client.ensure_directory(workspace_id, user_id) + if parent_segment: + await runtime_client.ensure_directory( + workspace_id, f"{user_id}/{parent_segment}" + ) + except RuntimeClientError as exc: + raise HTTPException( + status_code=exc.status_code, + detail=exc.detail, + ) from exc + + try: + await runtime_client.create_directory(workspace_id, name=jupyter_path) + except RuntimeClientError as exc: + raise HTTPException( + status_code=exc.status_code, + detail=exc.detail, + ) from exc + + if existing is None: + directory = StorageObjects( + storage_object_id=dir_id, + workspace_id=context.workspace.workspace_id, + storage_backend=settings.storage_backend, + object_type="directory", + usage_type="working_copy", + storage_uri=f"inline://directory/{relative_path}", + file_name=name, + relative_path=relative_path, + path_hash=path_hash, + object_status="available", + size_bytes=0, + visibility="private", + created_by=context.user.user_id, + ) + session.add(directory) + else: + directory = existing + directory.object_status = "available" + directory.is_deleted = 0 + directory.deleted_at = None + directory.object_type = "directory" + directory.usage_type = "working_copy" + directory.storage_uri = f"inline://directory/{relative_path}" + directory.file_name = name + directory.size_bytes = 0 + directory.visibility = "private" + directory.created_by = context.user.user_id + + try: + await session.flush() + except Exception: + # Best-effort compensating cleanup: remove the Jupyter directory we + # just created so a failed flush does not leave an orphan on disk. + try: + await runtime_client.delete_directory(workspace_id, name=jupyter_path) + except RuntimeClientError as cleanup_exc: + if cleanup_exc.status_code == 404: + pass + else: + logger.warning( + f"failed to clean up Jupyter directory {jupyter_path} " + f"after DB flush error: {cleanup_exc.status_code} {cleanup_exc.detail}" + ) + raise + return { + "request_id": context.request_id, + "data": { + "storage_object_id": directory.storage_object_id, + "path": child_path, + "name": name, + "parent_path": parent, + }, + "meta": {}, + } + + +# 删除逻辑目录及其下属脚本记录；实际文件按存储层的软删除规则处理。 +@router.delete("/api/v1/workspace-directories") +async def delete_workspace_directory( + request: Request, + path: str = Query(min_length=1, max_length=1024), + context: RequestContext = Depends(request_context), + session: AsyncSession = Depends(database_session), +) -> dict[str, Any]: + directory_path = normalize_user_path(path, allow_empty=False) + + scoped_prefix = user_relative_path(context) + target_relative = f"{scoped_prefix}/{directory_path}" + target_dir_row = await session.scalar( + select(StorageObjects).where( + StorageObjects.workspace_id == context.workspace.workspace_id, + StorageObjects.object_type == "directory", + StorageObjects.relative_path == target_relative, + StorageObjects.object_status == "available", + ) + ) + if target_dir_row is None: + raise HTTPException(status.HTTP_404_NOT_FOUND, "directory not found") + target_ulid = target_dir_row.storage_object_id + + child_prefix = f"{target_relative}/" + descendants = ( + ( + await session.execute( + select(StorageObjects).where( + StorageObjects.workspace_id == context.workspace.workspace_id, + StorageObjects.object_status == "available", + StorageObjects.relative_path.like(f"{child_prefix}%"), + ).order_by(func.length(StorageObjects.relative_path).desc()) + ) + ) + .scalars() + .all() + ) + + deleted_scripts = 0 + + for descendant in descendants: + if descendant.object_type == "file": + await soft_delete_object( + descendant.storage_object_id, request, session + ) + script = await session.scalar( + select(Scripts).where( + Scripts.workspace_id == context.workspace.workspace_id, + Scripts.current_object_id == descendant.storage_object_id, + ) + ) + if script is not None: + script.status = "deleted" + script.is_deleted = 1 + script.deleted_at = datetime.now(UTC).replace(tzinfo=None) + deleted_scripts += 1 + descendant.object_status = "deleted" + descendant.is_deleted = 1 + descendant.deleted_at = datetime.now(UTC).replace(tzinfo=None) + else: + descendant.object_status = "deleted" + descendant.is_deleted = 1 + descendant.deleted_at = datetime.now(UTC).replace(tzinfo=None) + + target_dir_row.object_status = "deleted" + target_dir_row.is_deleted = 1 + target_dir_row.deleted_at = datetime.now(UTC).replace(tzinfo=None) + + await session.flush() + + return { + "request_id": context.request_id, + "data": { + "path": directory_path, + "status": "deleted", + "deleted_scripts": deleted_scripts, + "versions_preserved": True, + }, + "meta": {}, + } + + +# 列出当前工作区中用户有权查看的脚本与 Notebook 元数据。 +@router.get("/api/v1/scripts") +async def list_scripts( + context: RequestContext = Depends(request_context), + session: AsyncSession = Depends(database_session), +) -> dict[str, Any]: + statement = ( + select(Scripts, StorageObjects, Users.display_name) + .join( + StorageObjects, + StorageObjects.storage_object_id == Scripts.current_object_id, + ) + .outerjoin(Users, Users.user_id == Scripts.owner_user_id) + .where( + Scripts.workspace_id == context.workspace.workspace_id, + Scripts.status == "active", + ) + .order_by(Scripts.updated_at.desc()) + ) + rows = (await session.execute(statement)).all() + return { + "request_id": context.request_id, + "data": [ + script_payload(script, storage_object, owner_display_name=owner_display_name) + for script, storage_object, owner_display_name in rows + ], + "meta": {"count": len(rows)}, + } + + +# 读取脚本正文或 Notebook JSON；编辑器打开文件时调用此接口。 +@router.get("/api/v1/scripts/{script_id}/content") +async def get_script_content( + script_id: str, + request: Request, + context: RequestContext = Depends(request_context), + session: AsyncSession = Depends(database_session), +) -> dict[str, Any]: + """获取脚本内容（用于只读预览）。 + + 适用于被锁定的脚本，非所有者只能查看内容，不能编辑。 + 该接口不检查锁状态，调用方需自行判断权限。 + """ + script, storage_object = await get_script_row( + script_id, + context, + session, + allow_missing_storage_object=True, + ) + + # 获取 Jupyter 路径 + workspace_id = context.workspace.workspace_id + jupyter_path = _derive_jupyter_path( + storage_object, workspace_id, script.script_type, script.script_id + ) + + # 从 Jupyter 读取内容 + runtime_client = request.app.state.runtime_client + try: + content_data = await runtime_client.get_file( + workspace_id, + name=jupyter_path, + ) + except RuntimeClientError as exc: + raise HTTPException( + status_code=exc.status_code, + detail=exc.detail, + ) from exc + + return { + "request_id": context.request_id, + "data": { + "script_id": script.script_id, + "script_type": script.script_type, + "content": content_data.get("content"), + "format": content_data.get("format"), + }, + "meta": {}, + } + + +# 查询单个脚本的元数据，例如类型、路径、锁状态和拥有者。 +@router.get("/api/v1/scripts/{script_id}") +async def get_script( + script_id: str, + context: RequestContext = Depends(request_context), + session: AsyncSession = Depends(database_session), +) -> dict[str, Any]: + script, storage_object = await get_script_row( + script_id, + context, + session, + ) + return { + "request_id": context.request_id, + "data": script_payload(script, storage_object), + "meta": {}, + } + + +# 保存编辑器提交的新内容；会校验工作区权限和文件编辑锁。 +@router.put("/api/v1/scripts/{script_id}") +async def update_script( + script_id: str, + payload: UpdateScriptRequest, + request: Request, + context: RequestContext = Depends(request_context), + session: AsyncSession = Depends(database_session), +) -> dict[str, Any]: + # Load the working-copy StorageObject as well as the Scripts row. + # Jupyter-only scripts may not have a StorageObjects row; allow that + # case and fall back to the flat _jupyter_path() name. + script, storage_object = await get_script_row( + script_id, + context, + session, + for_update=True, + allow_missing_storage_object=True, + ) + require_script_modify_access( + script, + user_id=context.user.user_id, + is_admin=context.is_admin, + ) + content = validate_script_content(payload.content, script.script_type) + runtime_client = request.app.state.runtime_client + workspace_id = context.workspace.workspace_id + jupyter_path = _derive_jupyter_path( + storage_object, workspace_id, script.script_type, script.script_id + ) + try: + if script.script_type == "notebook": + notebook = json.loads(content.decode("utf-8")) + jupyter_resp = await runtime_client.create_notebook( + workspace_id, + name=jupyter_path, + cells=notebook.get("cells"), + ) + else: + jupyter_resp = await runtime_client.upload_file( + workspace_id, + name=jupyter_path, + content=content.decode("utf-8"), + content_type=(mimetypes.guess_type(jupyter_path)[0] or "text/plain"), + ) + except RuntimeClientError as exc: + raise HTTPException( + status_code=exc.status_code, + detail=exc.detail, + ) from exc + + script.updated_at = datetime.now(UTC).replace(tzinfo=None) + content_hash = hashlib.sha256(content).hexdigest() + size_bytes = len(content) + relative_path = f"workspace/{jupyter_path}" + + # Persist the new content fingerprint so cache/dedup/hash checks + # downstream see the post-edit values. Filename is fixed in + # update_script, so object_key / path_hash are unchanged — only the + # body-derived fields move. relative_path is re-derived to keep the + # response in sync with the workspace-prefixed convention used at + # create time. + if storage_object is not None: + storage_object.content_hash = content_hash + storage_object.size_bytes = size_bytes + storage_object.relative_path = relative_path + await session.flush() + await session.refresh(storage_object) + storage_data: StorageObjects | dict[str, Any] = storage_object + else: + # Jupyter-only script: no StorageObjects row to update, but the + # response still needs the workspace-prefixed path shape so the + # frontend's slice(2) reducer produces the expected basename. + storage_data = { + "storage_object_id": script.current_object_id, + "relative_path": relative_path, + "object_key": f"{workspace_id}/{jupyter_path}", + "content_hash": content_hash, + "size_bytes": size_bytes, + } + return { + "request_id": context.request_id, + "data": script_payload(script, storage_data), + "meta": {}, + } + + +# 修改脚本锁定状态，避免其他用户同时编辑同一份文件。 +@router.patch("/api/v1/scripts/{script_id}/lock") +async def set_script_lock( + script_id: str, + payload: LockScriptRequest, + context: RequestContext = Depends(request_context), + session: AsyncSession = Depends(database_session), +) -> dict[str, Any]: + script = await session.scalar( + select(Scripts) + .where( + Scripts.script_id == script_id, + Scripts.workspace_id == context.workspace.workspace_id, + Scripts.status == "active", + ) + .with_for_update() + ) + if script is None: + raise HTTPException( + status.HTTP_404_NOT_FOUND, + "script not found", + ) + require_script_modify_access( + script, + user_id=context.user.user_id, + is_admin=context.is_admin, + ) + script.is_locked = 1 if payload.is_locked else 0 + await session.commit() + storage_object = await session.scalar( + select(StorageObjects).where( + StorageObjects.storage_object_id == script.current_object_id + ) + ) + return { + "request_id": context.request_id, + "data": script_payload(script, storage_object) + if storage_object + else script_payload(script, {}), + "meta": {}, + } + + +# 软删除脚本；元数据标记删除，历史版本可按规则继续保留。 +@router.delete("/api/v1/scripts/{script_id}") +async def delete_script( + script_id: str, + request: Request, + context: RequestContext = Depends(request_context), + session: AsyncSession = Depends(database_session), +) -> dict[str, Any]: + # Load the working-copy StorageObject if it exists. Jupyter-only + # scripts may not have a StorageObjects row; in that case we only + # flip the Scripts row to deleted. + script, storage_object = await get_script_row( + script_id, + context, + session, + for_update=True, + allow_missing_storage_object=True, + ) + require_script_modify_access( + script, + user_id=context.user.user_id, + is_admin=context.is_admin, + ) + + if storage_object is not None: + await soft_delete_object( + storage_object.storage_object_id, request, session + ) + storage_object.object_status = "deleted" + storage_object.is_deleted = 1 + storage_object.deleted_at = datetime.now(UTC).replace(tzinfo=None) + + script.status = "deleted" + script.deleted_at = datetime.now(UTC).replace(tzinfo=None) + script.is_deleted = 1 + return { + "request_id": context.request_id, + "data": { + "script_id": script.script_id, + "status": script.status, + "versions_preserved": True, + }, + "meta": {}, + } + + +# 将当前脚本内容发布为不可变版本，供调度节点和回溯下载使用。 +@router.post( + "/api/v1/scripts/{script_id}/versions", + status_code=status.HTTP_201_CREATED, +) +async def publish_version( + script_id: str, + payload: PublishVersionRequest, + request: Request, + context: RequestContext = Depends(request_context), + session: AsyncSession = Depends(database_session), +) -> dict[str, Any]: + # Load the working-copy StorageObject if it exists. Jupyter-only + # scripts may not have a StorageObjects row; fall back to the flat + # _jupyter_path() name when reading from Jupyter. + script, storage_object = await get_script_row( + script_id, + context, + session, + for_update=True, + allow_missing_storage_object=True, + ) + require_script_modify_access( + script, + user_id=context.user.user_id, + is_admin=context.is_admin, + ) + if ( + payload.source_object_id + and payload.source_object_id != script.current_object_id + ): + raise HTTPException( + status.HTTP_412_PRECONDITION_FAILED, + "source_object_id is not the current working copy", + ) + + # Read the working-copy content from the workspace's Jupyter + # instance. Notebooks come back as a dict (json.dumps it); text + # files come back as a UTF-8 string. + runtime_client = request.app.state.runtime_client + workspace_id = context.workspace.workspace_id + jupyter_path = _derive_jupyter_path( + storage_object, workspace_id, script.script_type, script.script_id + ) + try: + contents = await runtime_client.get_file(workspace_id, name=jupyter_path) + except RuntimeClientError as exc: + raise HTTPException( + status_code=exc.status_code, + detail=exc.detail, + ) from exc + if contents.get("type") == "notebook": + content = json.dumps(contents.get("content", {}), ensure_ascii=False).encode( + "utf-8" + ) + else: + content = (contents.get("content") or "").encode("utf-8") + + content_hash = hashlib.sha256(content).hexdigest() + existing = await session.scalar( + select(Versions).where( + Versions.script_id == script.script_id, + Versions.content_hash == content_hash, + ) + ) + if existing is not None: + return { + "request_id": context.request_id, + "data": version_payload(existing), + "meta": {"reused": True}, + } + content_type = ( + mimetypes.guess_type(script.script_name)[0] or "application/octet-stream" + ) + artifact = await create_server_object_payload( + ServerObjectRequest( + workspace_id=context.workspace.workspace_id, + user_id=context.user.user_id, + usage_type="version_artifact", + file_name=script.script_name, + content_type=content_type, + content_base64=base64.b64encode(content).decode("ascii"), + visibility=payload.visibility, + is_immutable=True, + idempotency_key=f"version:{script.script_id}:{content_hash}", + ), + request, + session, + ) + current_max = await session.scalar( + select(func.max(Versions.version_no)).where( + Versions.script_id == script.script_id + ) + ) + artifact_data = artifact["data"] + version_no = int(current_max or 0) + 1 + version = Versions( + versions_id=new_ulid(), + workspace_id=context.workspace.workspace_id, + script_id=script.script_id, + source_object_id=script.current_object_id, + artifact_object_id=artifact_data["storage_object_id"], + version_no=version_no, + version_label=f"v{version_no}.0", + source_path=jupyter_path, + artifact_path=artifact_data["storage_uri"], + content_hash=content_hash, + file_size_bytes=len(content), + visibility=payload.visibility, + release_note=payload.release_note, + created_by=context.user.user_id, + ) + session.add(version) + await session.flush() + await session.refresh(version) + return { + "request_id": context.request_id, + "data": version_payload(version), + "meta": {"reused": False}, + } + + +# 列出某脚本已经发布的历史版本。 +@router.get("/api/v1/scripts/{script_id}/versions") +async def list_versions( + script_id: str, + context: RequestContext = Depends(request_context), + session: AsyncSession = Depends(database_session), +) -> dict[str, Any]: + await get_script_row(script_id, context, session) + versions = ( + await session.scalars( + select(Versions) + .where(Versions.script_id == script_id) + .order_by(Versions.version_no.desc()) + ) + ).all() + return { + "request_id": context.request_id, + "data": [version_payload(version) for version in versions], + "meta": {"count": len(versions)}, + } + + +# 读取脚本最近一次发布的版本；未发布时返回空结果。 +@router.get("/api/v1/scripts/{script_id}/latest-version") +async def latest_version( + script_id: str, + context: RequestContext = Depends(request_context), + session: AsyncSession = Depends(database_session), +) -> dict[str, Any]: + """Return just the latest version's ``version_label`` + ``versions_id``. + + Lightweight alternative to :func:`list_versions` for the editor + header that only needs to show "vN · ". Validates the script + exists in the caller's workspace, then queries the single most + recent version row. Returns ``data: null`` when the script has no + published versions yet (so the frontend can render an empty label + without a 404 round-trip). + """ + script = await session.scalar( + select(Scripts.script_id).where( + Scripts.script_id == script_id, + Scripts.workspace_id == context.workspace.workspace_id, + Scripts.status == "active", + ) + ) + if script is None: + raise HTTPException( + status.HTTP_404_NOT_FOUND, + "script not found", + ) + latest = await session.scalar( + select(Versions) + .where( + Versions.script_id == script_id, + ) + .order_by(Versions.version_no.desc()) + .limit(1) + ) + if latest is None: + return { + "request_id": context.request_id, + "data": None, + "meta": {"has_versions": False}, + } + return { + "request_id": context.request_id, + "data": { + "versions_id": latest.versions_id, + "version_label": latest.version_label, + }, + "meta": {"has_versions": True}, + } + + +# 查询单个发布版本的元数据和关联脚本信息。 +@router.get("/api/v1/versions/{versions_id}") +async def get_version( + versions_id: str, + context: RequestContext = Depends(request_context), + session: AsyncSession = Depends(database_session), +) -> dict[str, Any]: + version = await session.get(Versions, versions_id) + if version is None or version.workspace_id != context.workspace.workspace_id: + raise HTTPException(status.HTTP_404_NOT_FOUND, "version not found") + return { + "request_id": context.request_id, + "data": version_payload(version), + "meta": {}, + } + + +# 隐藏/删除一个发布版本；是否保留实际产物由存储删除策略决定。 +@router.delete("/api/v1/versions/{versions_id}") +async def delete_version( + versions_id: str, + context: RequestContext = Depends(request_context), + session: AsyncSession = Depends(database_session), +) -> dict[str, Any]: + # Join to Scripts so the lock + owner check rides on the script record, + # not on whoever happened to publish this specific version. Lock state + # is a property of the script as a whole (architecture V3.1 §4), not of + # any one version of it. + row = ( + await session.execute( + select(Versions, Scripts) + .join( + Scripts, + Scripts.script_id == Versions.script_id, + ) + .where( + Versions.versions_id == versions_id, + Versions.workspace_id == context.workspace.workspace_id, + ) + ) + ).one_or_none() + if row is None: + raise HTTPException(status.HTTP_404_NOT_FOUND, "稳定版本不存在") + version, script = row + require_script_modify_access( + script, + user_id=context.user.user_id, + is_admin=context.is_admin, + ) + + version.schedule_hidden_at = datetime.now(UTC).replace(tzinfo=None) + await session.flush() + return { + "request_id": context.request_id, + "data": { + "versions_id": versions_id, + "deleted": True, + "artifact_preserved": True, + }, + "meta": { + "message": "已从调度稳定版本列表移除；稳定版本和历史记录保持不变", + }, + } + + +# 为某个版本产物生成带时效的下载地址，而非把大文件直接经 API 返回。 +@router.post("/api/v1/versions/{versions_id}/download-url") +async def version_download_url( + versions_id: str, + payload: DownloadUrlRequest, + request: Request, + context: RequestContext = Depends(request_context), + session: AsyncSession = Depends(database_session), +) -> dict[str, Any]: + version = await session.get(Versions, versions_id) + if version is None or version.workspace_id != context.workspace.workspace_id: + raise HTTPException(status.HTTP_404_NOT_FOUND, "version not found") + data = await create_download_url_payload( + await session.get(StorageObjects, version.artifact_object_id), + DownloadUrlRequest(expires_seconds=payload.expires_seconds), + request, + ) + return {"request_id": context.request_id, "data": data["data"], "meta": {}} diff --git a/backend/src/backend/services/__init__.py b/backend/src/backend/services/__init__.py new file mode 100644 index 0000000..e69de29 diff --git a/backend/src/backend/services/storage.py b/backend/src/backend/services/storage.py new file mode 100644 index 0000000..65ed927 --- /dev/null +++ b/backend/src/backend/services/storage.py @@ -0,0 +1,761 @@ +"""In-process storage helpers. + +The HTTP ``/internal/v1/*`` routes in ``backend.storage_api`` are wrappers +around these. Other backend modules (``scripts``, ``resources``) and the +schedule worker call these helpers directly instead of going through an +HTTP client — the storage layer lives in the same process, so the +indirection is pointless. + +Functions: + + create_upload_record — open a new upload session, returning + the upload_path (PUT-bytes) + session row. + upload_bytes_to_session — read raw bytes from request, validate, + call AsyncStorageBackend.put, build + StorageObjects row. + create_server_object_payload — server-side single-call upload (bytes + in JSON via base64). Used for small + artifacts (≤100 KiB). + create_download_url_payload — build a presigned GET URL for one + StorageObjects row. + soft_delete_object — copy-to-trash + delete source + flip row + to "deleted" with deleted_at stamp. + +These helpers raise ``HTTPException`` directly because they share an +HTTP-shaped error contract with the routes; callers can let the +exception propagate. +""" + +from __future__ import annotations + +import base64 +import binascii +import hashlib +from datetime import timedelta +from pathlib import PurePosixPath +from typing import Any + +from common.config import settings +from common.db.models import StorageObjects, UploadSessions +from common.ids import new_ulid +from common.storage import USAGE_TYPE_TO_PURPOSE, actual_bucket_name, build_storage_uri +from common.storage.schemas import ( + CreateUploadRequest, + DownloadUrlRequest, + ServerObjectRequest, +) +from fastapi import HTTPException, Request, status +from sqlalchemy import select, text +from sqlalchemy.exc import IntegrityError +from sqlalchemy.ext.asyncio import AsyncSession + +# ── shared low-level helpers (module-private) ──────────────────────────── + + +def _safe_file_name(value: str) -> str: + """Sanitize a user-supplied filename for use in an S3 object key. + + Rules: + - Strip whitespace + - Reject empty / `.` / `..` (path traversal) + - Reject leading `.` (hidden files) + - Reject `/`, `\\`, and any control character (ASCII < 0x20 or 0x7F) + - Reject leading/trailing whitespace already handled by .strip() + The Jupyter-side editor selection depends on the suffix, so callers + use ``PurePosixPath(safe).suffix`` to recover the extension. + """ + name = value.strip() + if ( + not name + or name in {".", ".."} + or name.startswith(".") + or "/" in name + or "\\" in name + or any(ord(char) < 0x20 or ord(char) == 0x7F for char in name) + ): + raise HTTPException( + status_code=status.HTTP_422_UNPROCESSABLE_ENTITY, + detail="invalid file_name", + ) + return name + + +def _safe_path_segment(segment: str) -> str: + """Single path segment: keep alnum / dash / underscore / dot; collapse the rest.""" + cleaned = "".join( + c if c.isalnum() or c in "-_." else "_" for c in segment + ).strip("._") + return cleaned or "untitled" + + +def _utcnow_naive() -> Any: + from datetime import UTC, datetime + return datetime.now(UTC).replace(tzinfo=None) + + +def _hash_bytes(value: str) -> bytes: + import hashlib as _h + return _h.sha256(value.encode("utf-8")).digest() + + +def _object_key_tail(object_key: str, workspace_id: str, user_id: str) -> str: + """object_key 去掉 ``{ws_id}/{user_id}/`` 前缀后的相对路径部分。""" + prefix = f"{workspace_id}/{user_id}/" + if object_key.startswith(prefix): + return object_key[len(prefix):] + return object_key + + +def _strip_uniqueness_suffix(tail: str) -> str: + """去掉 _resolve_unique_object_key 追加的 ``-`` 后缀,恢复请求时的原始路径。""" + dir_part, sep, name = tail.rpartition("/") + stem, dot, ext = name.rpartition(".") + if not dot: + stem, ext = name, "" + base, dash, suffix = stem.rpartition("-") + if dash and len(suffix) == 26 and suffix.isalnum(): + stem = base + name = f"{stem}.{ext}" if ext else stem + return f"{dir_part}{sep}{name}" + + +async def acquire_named_lock( + session: AsyncSession, + name: str, + *, + timeout_seconds: int = 10, +) -> str: + """获取 MySQL 命名锁(绑定当前会话连接),返回实际锁名。 + + MySQL 锁名上限 64 字符,统一哈希压缩。调用方必须在同一 session 上 + 用 release_named_lock 释放 —— 连接归还连接池时锁不会自动释放。 + """ + digest = hashlib.sha256(name.encode("utf-8")).hexdigest() + lock_name = f"mp:{digest[:61]}" + acquired = await session.scalar( + text("SELECT GET_LOCK(:name, :timeout)").bindparams( + name=lock_name, timeout=timeout_seconds + ) + ) + if acquired != 1: + raise HTTPException( + status.HTTP_503_SERVICE_UNAVAILABLE, + "failed to acquire storage lock; retry later", + ) + return lock_name + + +async def release_named_lock(session: AsyncSession, lock_name: str) -> None: + await session.execute( + text("SELECT RELEASE_LOCK(:name)").bindparams(name=lock_name) + ) + + +async def _mark_upload_failed_and_raise( + request: Request, + upload_id: str, + http_status: int, + detail: str, +) -> None: + """Flip ``upload_status`` to a terminal state and raise, surviving the + surrounding ``session_scope`` rollback AND keeping the named-lock + connection pristine. + + Why a separate session: this helper is invoked from inside the + ``acquire_named_lock`` critical section in + :func:`upload_bytes_to_session` (size / hash / storage-put failure + branches). Committing on the lock-bound session could return its + connection to the pool, after which the enclosing + ``finally: release_named_lock`` would check out a different + connection and leak the ``mp:`` lock for up to ``pool_recycle`` + seconds — re-opening the same-key upload race the lock exists to + close. Opening a fresh session from the factory commits the status + flip independently and leaves the lock connection untouched so + ``release_named_lock`` runs on the same connection that ran + ``GET_LOCK``. + + Why this helper exists at all: the route handler wraps every request + in ``session_scope`` (see ``backend.dependencies.database_session``), + which rolls back on exception. Without this helper, a naive + ``upload.upload_status = "failed"; raise HTTPException(...)`` would + lose the status flip and leave the row stuck in ``created``/``uploading`` + forever. + """ + session_factory = request.app.state.session_factory + async with session_factory() as session: + upload = await session.scalar( + select(UploadSessions).where(UploadSessions.upload_id == upload_id) + ) + if upload is not None: + upload.upload_status = "failed" + await session.commit() + raise HTTPException(http_status, detail) + + +async def _resolve_unique_object_key( + session: AsyncSession, + object_key: str, +) -> str: + """Append a ULID suffix when an available object already occupies ``object_key``. + + Keeps the original file name in metadata; only the on-disk key gets + the disambiguation suffix so repeated uploads of the same name never + overwrite existing bytes. + """ + key = object_key + for _ in range(5): + key_hash = _hash_bytes(key) + existing = await session.scalar( + select(StorageObjects).where( + StorageObjects.object_key_hash == key_hash, + StorageObjects.object_status == "available", + # Defense-in-depth: `object_status` is the canonical + # active/deleted flag, but `is_deleted` mirrors it. Filter + # both so future code that flips one without the other + # cannot bypass the active-row check. + StorageObjects.is_deleted == 0, + ) + ) + if existing is None: + return key + suffix = new_ulid().lower() + parts = key.rsplit("/", 1) + dir_part = parts[0] if len(parts) > 1 else "" + name = parts[-1] + # Preserve extension if present (ignore leading dot / hidden files). + if "." in name[1:]: + stem, ext = name.rsplit(".", 1) + new_name = f"{stem}-{suffix}.{ext}" + else: + new_name = f"{name}-{suffix}" + key = f"{dir_part}/{new_name}" if dir_part else new_name + return key + + +def _build_storage_object( + *, + upload: UploadSessions, + file_name: str, + content_type: str, + size_bytes: int, + content_hash: str | None, + visibility: str, + is_immutable: bool, + usage_type: str, + owner_user_id: str | None = None, +) -> StorageObjects: + """Build the StorageObjects row that pairs with a completed UploadSessions row.""" + safe_name = _safe_file_name(file_name) + return StorageObjects( + storage_object_id=new_ulid(), + workspace_id=upload.workspace_id, + owner_user_id=owner_user_id or upload.user_id, + object_type="file", + usage_type=usage_type, + storage_backend=settings.storage_backend, + bucket_name=upload.bucket_name, + object_key=upload.object_key, + object_key_hash=upload.object_key_hash, + storage_uri=build_storage_uri(upload.bucket_name, upload.object_key), + file_name=safe_name, + file_extension=PurePosixPath(safe_name).suffix.lower() or None, + mime_type=content_type, + size_bytes=size_bytes, + content_hash=content_hash, + object_etag=None, + visibility=visibility, + is_immutable=int(is_immutable), + object_status="available", + created_by=upload.user_id, + ) + + +# ── create_upload_record ──────────────────────────────────────────────── + + +def _resolve_bucket_for_usage( + usage_type: str, + *, + workspace_artifact_bucket: str | None, +) -> str: + """Mirror of storage_api.resolve_bucket, but pure (no DB / Request).""" + from backend.storage_api import BUCKET_FOR_USAGE + if workspace_artifact_bucket: + return workspace_artifact_bucket + return BUCKET_FOR_USAGE.get( + usage_type, actual_bucket_name(USAGE_TYPE_TO_PURPOSE.get(usage_type, "workspace")) + ) + + +async def create_upload_record( + payload: CreateUploadRequest, + session: AsyncSession, + request: Request, +) -> dict[str, Any]: + """Create or reuse an UploadSessions row. + + Returns ``{upload_id, status, upload_path, expires_at}`` for a fresh + session; or ``{upload_id, status: "completed", storage_object: {...}}`` + when the idempotency key hits an already-completed upload. + """ + from backend.storage_api import ( + normalized_idempotency_key, + require_workspace_member, + ) + + workspace = await require_workspace_member( + session, payload.workspace_id, payload.user_id + ) + stored_key = normalized_idempotency_key( + payload.workspace_id, payload.user_id, payload.idempotency_key + ) + safe_name = _safe_file_name(payload.file_name) + # Defense-in-depth: schemas already validate target_path, but the + # helper is called directly from some callers so re-validate here. + target_path = (payload.target_path or "").strip("/") + if target_path and (target_path.startswith("/") or "\\" in target_path + or any(seg == ".." for seg in target_path.split("/")) + or any(ord(c) < 0x20 or ord(c) == 0x7F for c in target_path)): + raise HTTPException( + status.HTTP_422_UNPROCESSABLE_ENTITY, + "target_path must be a clean relative POSIX path", + ) + clean_target = "/".join( + _safe_path_segment(seg) for seg in target_path.split("/") if seg + ) + requested_tail = f"{clean_target}/{safe_name}" if clean_target else safe_name + + existing = await session.scalar( + select(UploadSessions).where(UploadSessions.idempotency_key == stored_key) + ) + if existing is not None: + # 已有会话的 object_key 可能带唯一化后缀,比较前恢复原始路径, + # 保证同一文件的幂等重试通过、不同路径/文件名的复用报 409。 + existing_tail = _strip_uniqueness_suffix( + _object_key_tail( + existing.object_key, payload.workspace_id, payload.user_id + ) + ) + if ( + existing.workspace_id != payload.workspace_id + or existing.user_id != payload.user_id + or existing.expected_size_bytes != payload.expected_size_bytes + or existing.expected_hash != payload.expected_hash + or existing.content_type != payload.content_type + or existing_tail != requested_tail + ): + raise HTTPException( + status.HTTP_409_CONFLICT, + "idempotency key was used with different upload metadata", + ) + upload = existing + else: + from datetime import timedelta + + from backend.storage_api import utcnow + + bucket_name = _resolve_bucket_for_usage( + payload.usage_type, + workspace_artifact_bucket=workspace.artifact_bucket, + ) + object_key = ( + f"{payload.workspace_id}/{payload.user_id}/{requested_tail}" + ) + object_key = await _resolve_unique_object_key(session, object_key) + upload = UploadSessions( + upload_id=new_ulid(), + workspace_id=payload.workspace_id, + user_id=payload.user_id, + idempotency_key=stored_key, + bucket_name=bucket_name, + object_key=object_key, + object_key_hash=_hash_bytes(object_key), + upload_status="created", + expires_at=utcnow() + timedelta(minutes=15), + expected_size_bytes=payload.expected_size_bytes, + expected_hash=payload.expected_hash, + content_type=payload.content_type, + file_name=payload.file_name, + usage_type=payload.usage_type, + visibility=payload.visibility, + is_immutable=int(payload.is_immutable), + ) + session.add(upload) + await session.flush() + + if upload.upload_status == "completed" and upload.storage_object_id: + from backend.storage_api import storage_payload + storage_object = await session.get(StorageObjects, upload.storage_object_id) + if storage_object is None or storage_object.object_status != "available": + upload.storage_object_id = None + upload.upload_status = "created" + else: + return { + "upload_id": upload.upload_id, + "status": upload.upload_status, + "storage_object": storage_payload(storage_object), + } + + if upload.upload_status not in {"created", "uploading"}: + raise HTTPException( + status.HTTP_409_CONFLICT, + f"upload cannot continue from status {upload.upload_status}", + ) + + return { + "upload_id": upload.upload_id, + "status": upload.upload_status, + "upload_path": f"/internal/v1/uploads/{upload.upload_id}", + "expires_at": upload.expires_at.isoformat(), + } + + +# ── upload_bytes_to_session (server-proxied PUT) ──────────────────────── + + +async def upload_bytes_to_session( + upload_id: str, + session: AsyncSession, + request: Request, +) -> StorageObjects: + """Read raw bytes from the request body, validate against the + UploadSessions row, write via AsyncStorageBackend.put, and build the + StorageObjects row. Returns the row (caller may serialize it). + """ + upload = await session.scalar( + select(UploadSessions) + .where(UploadSessions.upload_id == upload_id) + .with_for_update() + ) + if upload is None: + raise HTTPException(status.HTTP_404_NOT_FOUND, "upload not found") + if upload.upload_status == "completed" and upload.storage_object_id: + item = await session.get(StorageObjects, upload.storage_object_id) + if item is None or item.object_status != "available": + upload.storage_object_id = None + upload.upload_status = "created" + else: + return item + if upload.upload_status not in {"created", "uploading"}: + raise HTTPException( + status.HTTP_409_CONFLICT, + f"upload cannot continue from status {upload.upload_status}", + ) + if upload.expires_at < _utcnow_naive(): + # 同样需要事务外 commit,否则 expired 状态会被外层 session_scope + # 回滚,客户端重试永远得到 created。 + upload.upload_status = "expired" + await session.commit() + raise HTTPException(status.HTTP_409_CONFLICT, "upload expired") + + # 按 object_key 串行化「PUT + INSERT」临界区:否则两个同 key 并发上传 + # 会互相覆盖字节,后 INSERT 的一方 409 时字节已被污染。 + lock_name = await acquire_named_lock( + session, f"upload:{upload.bucket_name}:{upload.object_key}" + ) + try: + # 锁内复查:创建会话后该 key 可能已被其他对象占用,必要时换 key。 + collision = await session.scalar( + select(StorageObjects.storage_object_id).where( + StorageObjects.bucket_name == upload.bucket_name, + StorageObjects.object_key_hash == upload.object_key_hash, + StorageObjects.object_status == "available", + StorageObjects.is_deleted == 0, + ) + ) + if collision is not None: + # 新 key 带随机 ULID 后缀,不会与其他并发会话冲突,无需重新加锁。 + new_key = await _resolve_unique_object_key( + session, upload.object_key + ) + upload.object_key = new_key + upload.object_key_hash = _hash_bytes(new_key) + + content = await request.body() + actual_size = len(content) + + if ( + upload.expected_size_bytes is not None + and actual_size != upload.expected_size_bytes + ): + await _mark_upload_failed_and_raise( + request, upload.upload_id, + status.HTTP_409_CONFLICT, + "uploaded bytes size does not match expected_size_bytes", + ) + + actual_hash = hashlib.sha256(content).hexdigest() if content else "" + if upload.expected_hash and actual_hash != upload.expected_hash: + await _mark_upload_failed_and_raise( + request, upload.upload_id, + status.HTTP_409_CONFLICT, + "uploaded bytes hash does not match expected_hash", + ) + + s3_metadata: dict[str, str] = {} + if actual_hash: + s3_metadata["sha256"] = actual_hash + + try: + await request.app.state.object_stores[upload.bucket_name].put( + upload.object_key, + content, + content_type=upload.content_type, + metadata=s3_metadata or None, + ) + except Exception as exc: + await _mark_upload_failed_and_raise( + request, upload.upload_id, + status.HTTP_503_SERVICE_UNAVAILABLE, + f"failed to write object to storage: {exc}", + ) + + item = _build_storage_object( + upload=upload, + file_name=upload.file_name, + content_type=upload.content_type, + size_bytes=actual_size, + content_hash=actual_hash or None, + visibility=upload.visibility, + is_immutable=bool(upload.is_immutable), + usage_type=upload.usage_type, + ) + session.add(item) + try: + await session.flush() + except IntegrityError as exc: + # 锁内已复查,走到这里说明发生了极罕见的跨锁竞争; + # 事务会由上层回滚,这里只需报错。 + raise HTTPException( + status.HTTP_409_CONFLICT, + "a file with this name already exists at this path; rename and retry", + ) from exc + await session.refresh(item) + upload.storage_object_id = item.storage_object_id + upload.upload_status = "completed" + upload.completed_at = _utcnow_naive() + return item + finally: + await release_named_lock(session, lock_name) + + +# ── create_server_object_payload ──────────────────────────────────────── + + +async def create_server_object_payload( + payload: ServerObjectRequest, + request: Request, + session: AsyncSession, +) -> dict[str, Any]: + """Server-side single-call upload (JSON body, base64 content). + + Used by scripts.py when publishing version artifacts and by the + schedule worker for run logs / run results. + """ + from backend.storage_api import storage_payload + + try: + content = base64.b64decode(payload.content_base64, validate=True) + except (binascii.Error, ValueError) as exc: + raise HTTPException( + status.HTTP_422_UNPROCESSABLE_ENTITY, + "content_base64 is invalid", + ) from exc + if len(content) > 100 * 1024 * 1024: + raise HTTPException( + status.HTTP_413_REQUEST_ENTITY_TOO_LARGE, + "object exceeds 100 MiB server-side upload limit", + ) + + content_hash = hashlib.sha256(content).hexdigest() + upload_result = await create_upload_record( + CreateUploadRequest( + workspace_id=payload.workspace_id, + user_id=payload.user_id, + usage_type=payload.usage_type, + file_name=payload.file_name, + content_type=payload.content_type, + expected_size_bytes=len(content), + expected_hash=content_hash, + idempotency_key=payload.idempotency_key, + visibility=payload.visibility, + is_immutable=payload.is_immutable, + ), + session, + request, + ) + if upload_result.get("status") == "completed": + existing_data = upload_result["storage_object"] + if ( + payload.relative_path + and existing_data + and existing_data.get("relative_path") != payload.relative_path + ): + existing_item = await session.get( + StorageObjects, existing_data["storage_object_id"] + ) + if existing_item is not None: + existing_item.relative_path = payload.relative_path + await session.flush() + existing_data = storage_payload(existing_item) + return {"data": existing_data, "meta": {"reused": True}} + + upload = await session.get(UploadSessions, upload_result["upload_id"]) + if upload is None: + raise HTTPException( + status.HTTP_500_INTERNAL_SERVER_ERROR, + "upload record disappeared", + ) + + try: + await request.app.state.object_stores[upload.bucket_name].put( + upload.object_key, + content, + content_type=payload.content_type, + metadata={"sha256": content_hash}, + ) + except Exception as exc: + await _mark_upload_failed_and_raise( + request, upload.upload_id, + status.HTTP_503_SERVICE_UNAVAILABLE, + f"failed to write object to storage: {exc}", + ) + + item = _build_storage_object( + upload=upload, + file_name=payload.file_name, + content_type=payload.content_type, + size_bytes=len(content), + content_hash=content_hash, + visibility=payload.visibility, + is_immutable=payload.is_immutable, + usage_type=payload.usage_type, + ) + session.add(item) + await session.flush() + # Force SQLAlchemy to round-trip server-default columns + # (created_at / updated_at / is_deleted) in the *current* async + # context. ``eager_defaults="auto"`` on the mapper does not + # guarantee a refresh for non-PK server defaults; without this, + # ``storage_payload`` (a sync helper) would later trigger a lazy + # refresh through the async driver and raise MissingGreenlet. + await session.refresh( + item, + attribute_names=["created_at", "updated_at", "is_deleted"], + ) + item.relative_path = payload.relative_path + upload.storage_object_id = item.storage_object_id + upload.upload_status = "completed" + upload.completed_at = _utcnow_naive() + return {"data": storage_payload(item), "meta": {"reused": False}} + + +# ── create_download_url_payload ───────────────────────────────────────── + + +async def create_download_url_payload( + item: StorageObjects, + payload: DownloadUrlRequest, + request: Request, +) -> dict[str, Any]: + """Build a presigned GET URL for one StorageObjects row. + + The caller (route handler in storage_api / scripts.py / resources.py) + loads the StorageObjects row + validates ownership/visibility; this + helper just builds the URL. + """ + if item is None or item.object_status != "available": + raise HTTPException(status.HTTP_404_NOT_FOUND, "object not found") + if ( + item.storage_backend != settings.storage_backend + or not item.bucket_name + or not item.object_key + ): + raise HTTPException( + status.HTTP_409_CONFLICT, + "object does not support a presigned URL", + ) + url = await request.app.state.object_stores[item.bucket_name].get_url( + item.object_key, + expires_in=timedelta(seconds=payload.expires_seconds), + ) + # Public-host rewriting is now nginx's job (location /storage/). In the + # future the boto3 client should be built with the public endpoint so + # generate_presigned_url returns a public URL directly. + return { + "data": { + "storage_object_id": item.storage_object_id, + "presigned_url": url, + "method": "GET", + "expires_in_seconds": payload.expires_seconds, + } + } + + +# ── soft_delete_object ────────────────────────────────────────────────── + + +async def soft_delete_object( + storage_object_id: str, + request: Request, + session: AsyncSession, +) -> dict[str, Any]: + """Soft-delete a storage object: copy to trash bucket, delete source, + flip the row to ``"deleted"``. Immutable objects are rejected. + """ + item = await session.scalar( + select(StorageObjects) + .where(StorageObjects.storage_object_id == storage_object_id) + .with_for_update() + ) + if item is None: + raise HTTPException(status.HTTP_404_NOT_FOUND, "object not found") + if item.is_immutable: + raise HTTPException( + status.HTTP_409_CONFLICT, + "immutable object cannot be deleted", + ) + if item.object_status == "deleted": + return { + "data": { + "storage_object_id": storage_object_id, + "object_status": item.object_status, + "trash_key": item.trash_key, + "trash_bucket": actual_bucket_name("trash"), + } + } + if item.storage_backend == settings.storage_backend and item.bucket_name and item.object_key: + source_purpose = USAGE_TYPE_TO_PURPOSE.get(item.usage_type, "workspace") + # 尾部拼 storage_object_id,防止同名文件多次删除在回收站互相覆盖。 + trash_key = f"{source_purpose}/{item.object_key}-{item.storage_object_id}" + trash_bucket = actual_bucket_name("trash") + try: + object_stores = request.app.state.object_stores + # P0-5 / B2: 流式迁移,避免 get() 全量加载导致 10G 对象 OOM。 + # LOCAL 后端的 put 是 aiofiles 流式写入,get_stream + put 零字节驻留; + # S3 后端目前 put 仍会 ``b"".join(chunks)`` 物化到内存,S3 大对象 + # 的 OOM 修复需要把 put 改成 multipart upload —— 后续工单。 + stream = object_stores[item.bucket_name].get_stream(item.object_key) + await object_stores[trash_bucket].put(trash_key, stream) + await object_stores[item.bucket_name].delete(item.object_key) + except Exception as exc: + raise HTTPException( + status.HTTP_502_BAD_GATEWAY, + f"failed to move object to trash: {exc}", + ) from exc + item.trash_key = trash_key + item.bucket_name = trash_bucket + item.object_key = trash_key + item.object_key_hash = _hash_bytes(trash_key) + item.storage_uri = build_storage_uri(trash_bucket, trash_key) + item.object_status = "deleted" + item.deleted_at = _utcnow_naive() + item.is_deleted = 1 + return { + "data": { + "storage_object_id": storage_object_id, + "object_status": item.object_status, + "trash_key": item.trash_key, + "trash_bucket": actual_bucket_name("trash"), + } + } diff --git a/backend/src/backend/storage_api.py b/backend/src/backend/storage_api.py new file mode 100644 index 0000000..843c5c2 --- /dev/null +++ b/backend/src/backend/storage_api.py @@ -0,0 +1,427 @@ +"""后端内部对象存储接口与通用辅助函数。 + +这些路由由 ``main.py`` 额外挂载到 ``/internal``，用于上传会话、对象记录和 +下载地址等内部协作。业务路由通常直接复用本模块/``services.storage`` 的函数， +而不是让浏览器直接调用这些内部接口。 +""" + +from __future__ import annotations + +import hashlib +import secrets +from collections.abc import AsyncIterator +from datetime import UTC, datetime, timedelta +from pathlib import PurePosixPath +from typing import Any + +from common.config import settings +from common.db import session_scope +from common.db.models import ( + StorageObjects, + UploadSessions, + Users, + WorkspaceMembers, + Workspaces, +) +from common.ids import new_ulid +from common.storage import USAGE_TYPE_TO_PURPOSE, actual_bucket_name, build_storage_uri +from common.storage.schemas import ( + CreateUploadRequest, + ServerObjectRequest, +) +from fastapi import APIRouter, Depends, Header, HTTPException, Request, status +from sqlalchemy import select +from sqlalchemy.ext.asyncio import AsyncSession + +from backend.services.storage import ( + create_server_object_payload, + create_upload_record, + upload_bytes_to_session, +) + + +# Header name for the service-to-service token. Schedule worker is the +# only legitimate caller — every other consumer goes through the +# JWT-protected ``/api/v1/data-resources/*`` routes. The header name +# mirrors the ``X-Internal-*`` convention used elsewhere in the stack. +INTERNAL_SERVICE_TOKEN_HEADER = "x-internal-service-token" + + +def require_internal_service( + x_internal_service_token: str | None = Header(default=None), +) -> None: + """Enforce a shared secret for /internal/v1/* routes. + + Compares the supplied header against ``settings.internal_service_token`` + with a constant-time check. The token is configured identically on the + backend and the schedule container via ``INTERNAL_SERVICE_TOKEN``; the + default in ``Settings`` is a development-only placeholder that callers + must override in any non-dev deployment. + """ + expected = settings.internal_service_token + if not expected: + raise HTTPException( + status.HTTP_503_SERVICE_UNAVAILABLE, + "internal service token not configured", + ) + if not x_internal_service_token or not secrets.compare_digest( + x_internal_service_token, expected + ): + raise HTTPException(status.HTTP_401_UNAUTHORIZED, "internal service token required") + + +def utcnow() -> datetime: + return datetime.now(UTC).replace(tzinfo=None) + + +def hash_bytes(value: str) -> bytes: + return hashlib.sha256(value.encode("utf-8")).digest() + + +def normalized_idempotency_key(workspace_id: str, user_id: str, value: str) -> str: + digest = hashlib.sha256( + f"{workspace_id}:{user_id}:{value}".encode() + ).hexdigest() + return f"v1:{digest}" + + +def safe_file_name(value: str) -> str: + name = value.replace("\\", "/").rsplit("/", 1)[-1].strip() + if not name or name in {".", ".."} or any(ord(char) < 32 for char in name): + raise HTTPException(status.HTTP_422_UNPROCESSABLE_ENTITY, "invalid file_name") + return name + + +# 预先解析的“用途 → 桶名”映射，供只读调用方快速使用。若工作区配置了专属 +# artifact_bucket，则必须调用 resolve_bucket()，让工作区级覆盖规则生效。 +BUCKET_FOR_USAGE: dict[str, str] = { + usage_type: actual_bucket_name(purpose) + for usage_type, purpose in USAGE_TYPE_TO_PURPOSE.items() +} + + +def resolve_bucket( + usage_type: str, + *, + workspace: Workspaces, +) -> str: + """Pick the bucket for ``usage_type``. + + ``workspace.artifact_bucket`` (per-workspace override) wins over the + usage-type default. An unknown ``usage_type`` falls through to the + workspace bucket so we never silently drop an object. + """ + if workspace.artifact_bucket: + return workspace.artifact_bucket + purpose = USAGE_TYPE_TO_PURPOSE.get(usage_type, "workspace") + return actual_bucket_name(purpose) + + +def storage_payload(item: StorageObjects) -> dict[str, Any]: + return { + "storage_object_id": item.storage_object_id, + "workspace_id": item.workspace_id, + "owner_user_id": item.owner_user_id, + "storage_backend": item.storage_backend, + "usage_type": item.usage_type, + "storage_uri": item.storage_uri, + "object_key": item.object_key, + "relative_path": item.relative_path, + "file_name": item.file_name, + "file_extension": item.file_extension, + "mime_type": item.mime_type, + "size_bytes": item.size_bytes, + "content_hash": item.content_hash, + "visibility": item.visibility, + "is_immutable": bool(item.is_immutable), + "object_status": item.object_status, + "created_at": item.created_at.isoformat(), + "updated_at": item.updated_at.isoformat(), + } + + +# 内部路由由 main.py 以 /internal 前缀挂载。数据库引擎、Session 工厂和对象 +# 存储实例均在应用生命周期中创建；本模块只定义路由和供 services.storage 复用的 +# 存储辅助函数（如 storage_payload、resolve_bucket、BUCKET_FOR_USAGE）。 +router = APIRouter(tags=["internal-storage"]) + + +async def database_session(request: Request) -> AsyncIterator[AsyncSession]: + async with session_scope(request.app.state.session_factory) as session: + yield session + + +async def require_workspace_member( + session: AsyncSession, workspace_id: str, user_id: str +) -> Workspaces: + statement = ( + select(Workspaces) + .join( + WorkspaceMembers, WorkspaceMembers.workspace_id == Workspaces.workspace_id + ) + .join(Users, Users.user_id == WorkspaceMembers.user_id) + .where( + Workspaces.workspace_id == workspace_id, + Workspaces.status == "active", + WorkspaceMembers.user_id == user_id, + WorkspaceMembers.member_status == "active", + Users.status == "active", + ) + ) + workspace = await session.scalar(statement) + if workspace is None: + raise HTTPException( + status.HTTP_403_FORBIDDEN, "user is not an active workspace member" + ) + return workspace + + +async def create_upload_record( + payload: CreateUploadRequest, session: AsyncSession, request: Request +) -> dict[str, Any]: + workspace = await require_workspace_member( + session, payload.workspace_id, payload.user_id + ) + stored_key = normalized_idempotency_key( + payload.workspace_id, payload.user_id, payload.idempotency_key + ) + existing = await session.scalar( + select(UploadSessions).where(UploadSessions.idempotency_key == stored_key) + ) + if existing is not None: + if ( + existing.workspace_id != payload.workspace_id + or existing.user_id != payload.user_id + or existing.expected_size_bytes != payload.expected_size_bytes + or existing.expected_hash != payload.expected_hash + or existing.content_type != payload.content_type + ): + raise HTTPException( + status.HTTP_409_CONFLICT, + "idempotency key was used with different upload metadata", + ) + upload = existing + else: + upload_id = new_ulid() + bucket_name = resolve_bucket(payload.usage_type, workspace=workspace) + # Keep the opaque upload id while preserving the original extension. + # Jupyter selects its editor from this suffix, so an extensionless + # object would make notebooks look like generic JSON/text files. + file_extension = PurePosixPath(safe_file_name(payload.file_name)).suffix.lower() + object_key = f"{payload.workspace_id}/{payload.user_id}/{upload_id}{file_extension}" + upload = UploadSessions( + upload_id=upload_id, + workspace_id=payload.workspace_id, + user_id=payload.user_id, + idempotency_key=stored_key, + bucket_name=bucket_name, + object_key=object_key, + object_key_hash=hash_bytes(object_key), + upload_status="created", + expires_at=utcnow() + timedelta(minutes=15), + expected_size_bytes=payload.expected_size_bytes, + expected_hash=payload.expected_hash, + content_type=payload.content_type, + file_name=payload.file_name, + usage_type=payload.usage_type, + visibility=payload.visibility, + is_immutable=int(payload.is_immutable), + ) + session.add(upload) + await session.flush() + + if upload.upload_status == "completed" and upload.storage_object_id: + storage_object = await session.get(StorageObjects, upload.storage_object_id) + if storage_object is None or storage_object.object_status != "available": + # The previously-completed object was deleted (or never + # materialized). Treat the idempotency hit as a tombstone + # and fall through to a fresh upload: clear the pointer so + # complete_upload_record re-validates the bucket + key. + upload.storage_object_id = None + upload.upload_status = "created" + else: + return { + "upload_id": upload.upload_id, + "status": upload.upload_status, + "storage_object": storage_payload(storage_object), + } + if upload.upload_status not in {"created", "uploading"}: + raise HTTPException( + status.HTTP_409_CONFLICT, + f"upload cannot continue from status {upload.upload_status}", + ) + + # Two-step server-proxied upload: the caller PUTs the raw bytes to + # ``upload_path`` after this response, which routes through + # ``backend.resources.upload_bytes_to_session`` (the canonical helper + # in ``services.storage``). + return { + "upload_id": upload.upload_id, + "status": upload.upload_status, + "upload_path": f"/internal/v1/uploads/{upload.upload_id}", + "expires_at": upload.expires_at.isoformat(), + } + + +def _public_base_url(request: Request) -> str: + """Return the public base URL the client should use. + + Falls back to the inbound request's ``Host`` header and the scheme + Nginx forwards via ``X-Forwarded-Proto`` so the resulting + presigned URL always points at the public edge rather than the + in-cluster S3 endpoint. + """ + forwarded_proto = request.headers.get("x-forwarded-proto", "").strip() + scheme = forwarded_proto or request.url.scheme or "http" + host = ( + request.headers.get("x-forwarded-host", "").strip() + or request.headers.get("host", "").strip() + ) + if not host: + raise HTTPException( + status.HTTP_400_BAD_REQUEST, + "cannot determine public host for presigned URL", + ) + return f"{scheme}://{host}" + + +@router.post( + "/v1/objects", + dependencies=[Depends(require_internal_service)], +) +async def create_server_object( + payload: ServerObjectRequest, + request: Request, + session: AsyncSession = Depends(database_session), +) -> dict[str, Any]: + """Server-side single-call object upload used by the schedule worker. + + Token-guarded: the only legitimate caller is the schedule service + that writes ``run_log`` / ``run_result`` artifacts after a notebook + finishes. Frontend users upload through the JWT-protected + ``/api/v1/data-resources/*`` routes instead. + """ + return await create_server_object_payload(payload, request, session) + + +@router.post("/v1/objects/{storage_object_id}/restore") +async def restore_object( + storage_object_id: str, + request: Request, + session: AsyncSession = Depends(database_session), +) -> dict[str, Any]: + """Restore a soft-deleted object from the trash bucket. + + Copies the bytes back to the source bucket + key and flips the + row back to ``object_status='available'``. If the source bucket + is missing the object (e.g. the trash copy was the only one), the + restore still works because we copy *from* trash rather than + renaming in place. The trash copy is left in place — the reaper + will collect it on the next sweep; this is intentional so a + failed restore does not destroy the only copy. + """ + item = await session.scalar( + select(StorageObjects) + .where(StorageObjects.storage_object_id == storage_object_id) + .with_for_update() + ) + if item is None: + raise HTTPException(status.HTTP_404_NOT_FOUND, "object not found") + if item.object_status != "deleted": + raise HTTPException(status.HTTP_409_CONFLICT, "object is not in trash") + if not item.trash_key or not item.bucket_name or not item.object_key: + raise HTTPException( + status.HTTP_409_CONFLICT, "object has no trash pointer; cannot restore" + ) + try: + # P0-5 / B2: 与 soft_delete_object 对称,从回收站恢复也走流式, + # 不再把整个对象加载进内存。S3 put 物化限制同 soft_delete_object。 + object_stores = request.app.state.object_stores + source_purpose, _, trash_tail = item.object_key.partition("/") + if not source_purpose: + source_purpose = "workspace" + # trash key 尾部带 ``-{storage_object_id}`` 后缀(防同名覆盖), + # 恢复时剥掉;旧数据没有该后缀,endswith 判断天然兼容。 + id_suffix = f"-{item.storage_object_id}" + source_key = ( + trash_tail[: -len(id_suffix)] + if trash_tail.endswith(id_suffix) + else trash_tail + ) + target_bucket = actual_bucket_name(source_purpose) + stream = object_stores[actual_bucket_name("trash")].get_stream(item.object_key) + await object_stores[target_bucket].put(source_key, stream) + except Exception as exc: + raise HTTPException( + status.HTTP_502_BAD_GATEWAY, + f"failed to restore from trash: {exc}", + ) from exc + item.object_status = "available" + item.deleted_at = None + item.bucket_name = target_bucket + item.object_key = source_key + item.object_key_hash = hash_bytes(source_key) + item.storage_uri = build_storage_uri(target_bucket, source_key) + # Keep trash_key so the reaper can clean up the duplicate on its + # next pass; we don't try to delete it here because a partial + # failure would leave the user with no data. + return { + "data": storage_payload(item), + } + + +# 管理动作：永久清理超过保留期限或指定的回收站对象。 +@router.post("/v1/admin/trash/purge") +async def purge_trash_object( + payload: dict[str, Any], + request: Request, + session: AsyncSession = Depends(database_session), +) -> dict[str, Any]: + """Physically delete a trashed object. + + Admin / reaper endpoint — given a ``storage_object_id``, deletes + the bytes from the trash bucket and hard-deletes the DB row. + The route is split from ``delete_object`` because the regular + delete path is the one users hit, and reaper runs need a way to + finalize the lifecycle without re-entering the soft-delete branch. + """ + storage_object_id = (payload or {}).get("storage_object_id", "").strip() + if not storage_object_id: + raise HTTPException( + status.HTTP_400_BAD_REQUEST, "storage_object_id is required" + ) + item = await session.scalar( + select(StorageObjects) + .where(StorageObjects.storage_object_id == storage_object_id) + .with_for_update() + ) + if item is None: + return {"data": {"storage_object_id": storage_object_id, "purged": False}} + if item.object_status != "deleted": + raise HTTPException( + status.HTTP_409_CONFLICT, + "object is not in trash; refuse to hard-delete live data", + ) + if item.bucket_name != actual_bucket_name("trash"): + raise HTTPException( + status.HTTP_409_CONFLICT, + "object is not in trash bucket; refuse to hard-delete", + ) + if item.trash_key: + try: + await request.app.state.object_stores[actual_bucket_name("trash")].delete( + item.object_key + ) + except Exception as exc: + raise HTTPException( + status.HTTP_502_BAD_GATEWAY, + f"failed to delete from trash: {exc}", + ) from exc + await session.delete(item) + return {"data": {"storage_object_id": storage_object_id, "purged": True}} + + +# 检查内部存储后端是否可用，供健康检查和排障使用。 +@router.get("/health/storage") +async def internal_health() -> dict[str, str]: + return {"status": "ready", "service": "storage-api"} diff --git a/backend/tests/__init__.py b/backend/tests/__init__.py new file mode 100644 index 0000000..e69de29 diff --git a/backend/tests/test_resources.py b/backend/tests/test_resources.py new file mode 100644 index 0000000..10b0df3 --- /dev/null +++ b/backend/tests/test_resources.py @@ -0,0 +1,407 @@ +"""Unit tests for data-resource path derivation and helpers. + +These tests do not need a database because they exercise pure helpers. +""" + +from __future__ import annotations + +import datetime +from types import SimpleNamespace +from unittest.mock import MagicMock + +import pytest +from backend.resources import ( + compute_jupyter_relative_path, + resource_directory, + resource_payload, +) +from backend.services.storage import ( + _object_key_tail, + _safe_file_name, + _safe_path_segment, + _strip_uniqueness_suffix, +) + + +_BIND_WS = "01WS0000000000000000000A" +_BIND_USER = "01USR0000000000000000000A" + + +class _ExecuteResult: + def __init__(self, rows): + self._rows = rows + + def all(self): + return self._rows + + +class _BindSessionMock: + """Mocked session for bind_resource. + + ``new_object_key`` is the StorageObjects row of the upload being bound; + ``same_name_rows`` are (DataResources, StorageObjects) candidates already + in the workspace with the same resource_name. + """ + + def __init__( + self, + new_object_key: str, + same_name_rows=(), + reused_resource=None, + usage_type: str = "data_resource", + ): + self._new_object_key = new_object_key + self._same_name_rows = list(same_name_rows) + self._reused_resource = reused_resource + self._usage_type = usage_type + self._scalar_calls = 0 + + async def scalar(self, _stmt): + from sqlalchemy import TextClause + + if isinstance(_stmt, TextClause): + return 1 # GET_LOCK 成功 + self._scalar_calls += 1 + # 1st scalar: UploadSessions lookup; 2nd: active 绑定行复用检查。 + if self._scalar_calls == 1: + return SimpleNamespace( + upload_id="01UPL0000000000000000000B", + storage_object_id="01OBJ0000000000000000000B", + workspace_id=_BIND_WS, + user_id=_BIND_USER, + usage_type=self._usage_type, + ) + if self._scalar_calls == 2 and self._reused_resource is not None: + return self._reused_resource + return None + + async def get(self, _model, _pk): + return _make_storage_object(self._new_object_key) + + async def execute(self, _stmt): + from sqlalchemy import TextClause + + if isinstance(_stmt, TextClause): + return _ExecuteResult([]) # RELEASE_LOCK + # 模拟 SQL 的 owner 过滤与 self-exclusion:只返回属于当前用户且 + # 不是当前 upload 已绑定对象的同名候选。 + rows = [ + (resource, storage_object) + for resource, storage_object in self._same_name_rows + if resource.owner_user_id == _BIND_USER + and resource.storage_object_id != "01OBJ0000000000000000000B" + ] + return _ExecuteResult(rows) + + def add(self, _obj): + pass + + async def flush(self): + pass + + async def refresh(self, _obj, attribute_names=None): + _obj.created_at = datetime.datetime.now(datetime.UTC).replace(tzinfo=None) + _obj.updated_at = datetime.datetime.now(datetime.UTC).replace(tzinfo=None) + + +def _bind_context(): + return SimpleNamespace( + request_id="01REQ0000000000000000000A", + user=SimpleNamespace(user_id=_BIND_USER), + workspace=SimpleNamespace(workspace_id=_BIND_WS), + ) + + +def _bind_payload(): + return SimpleNamespace( + resource_name="data.csv", description=None, visibility="private" + ) + + +@pytest.mark.asyncio +async def test_bind_resource_rejects_duplicate_name_in_same_directory() -> None: + """Same resource_name in the same directory raises 409.""" + from backend.resources import bind_resource + + existing_rows = [ + ( + _make_resource(_BIND_WS, _BIND_USER), + _make_storage_object(f"{_BIND_WS}/{_BIND_USER}/data.csv"), + ) + ] + session = _BindSessionMock( + new_object_key=f"{_BIND_WS}/{_BIND_USER}/data.csv", + same_name_rows=existing_rows, + ) + with pytest.raises(Exception) as exc_info: + await bind_resource( + upload_id="01UPL0000000000000000000B", + payload=_bind_payload(), + request=MagicMock(), + context=_bind_context(), + session=session, + ) + assert exc_info.value.status_code == 409 + assert "already exists" in exc_info.value.detail + + +@pytest.mark.asyncio +async def test_bind_resource_allows_same_name_in_different_directory() -> None: + """Same resource_name in a different directory binds successfully.""" + from backend.resources import bind_resource + + existing_rows = [ + ( + _make_resource(_BIND_WS, _BIND_USER), + _make_storage_object(f"{_BIND_WS}/{_BIND_USER}/data.csv"), + ) + ] + session = _BindSessionMock( + new_object_key=f"{_BIND_WS}/{_BIND_USER}/subdir/data.csv", + same_name_rows=existing_rows, + ) + result = await bind_resource( + upload_id="01UPL0000000000000000000B", + payload=_bind_payload(), + request=MagicMock(), + context=_bind_context(), + session=session, + ) + assert result["data"]["resource_name"] == "data.csv" + assert result["data"]["jupyter_accessible_path"] == "subdir/data.csv" + + +@pytest.mark.asyncio +async def test_bind_resource_allows_same_name_when_workspace_empty() -> None: + """No same-name rows at all: bind succeeds (root directory).""" + from backend.resources import bind_resource + + session = _BindSessionMock( + new_object_key=f"{_BIND_WS}/{_BIND_USER}/data.csv", + ) + result = await bind_resource( + upload_id="01UPL0000000000000000000A", + payload=_bind_payload(), + request=MagicMock(), + context=_bind_context(), + session=session, + ) + assert result["data"]["resource_name"] == "data.csv" + + +@pytest.mark.asyncio +async def test_bind_resource_allows_same_name_for_different_owner() -> None: + """其他用户在同目录下的同名资源不阻塞当前用户的绑定。""" + from backend.resources import bind_resource + + other_user = "01USR0000000000000000000B" + existing_rows = [ + ( + _make_resource(_BIND_WS, other_user), + _make_storage_object(f"{_BIND_WS}/{other_user}/data.csv"), + ) + ] + session = _BindSessionMock( + new_object_key=f"{_BIND_WS}/{_BIND_USER}/data.csv", + same_name_rows=existing_rows, + ) + result = await bind_resource( + upload_id="01UPL0000000000000000000C", + payload=_bind_payload(), + request=MagicMock(), + context=_bind_context(), + session=session, + ) + assert result["data"]["resource_name"] == "data.csv" + + +@pytest.mark.asyncio +async def test_bind_resource_allows_rebinding_same_storage_object() -> None: + """重新绑定同一 upload_id 应走 idempotent 复用路径,不触发 409。""" + from backend.resources import bind_resource + + new_object_key = f"{_BIND_WS}/{_BIND_USER}/data.csv" + existing_resource = _make_resource(_BIND_WS, _BIND_USER) + existing_resource.resource_name = "data.csv" + existing_resource.storage_object_id = "01OBJ0000000000000000000B" + existing_object = _make_storage_object(new_object_key) + existing_object.storage_object_id = "01OBJ0000000000000000000B" + same_name_rows = [(existing_resource, existing_object)] + session = _BindSessionMock( + new_object_key=new_object_key, + same_name_rows=same_name_rows, + reused_resource=existing_resource, + ) + result = await bind_resource( + upload_id="01UPL0000000000000000000B", + payload=_bind_payload(), + request=MagicMock(), + context=_bind_context(), + session=session, + ) + assert result["meta"]["reused"] is True + assert result["data"]["resource_name"] == "data.csv" + + +@pytest.mark.asyncio +async def test_bind_resource_rejects_non_data_resource_upload() -> None: + """其他用途(如 working_copy)的 upload session 不能 bind 成数据资源。""" + from backend.resources import bind_resource + + session = _BindSessionMock( + new_object_key=f"{_BIND_WS}/{_BIND_USER}/data.csv", + usage_type="working_copy", + ) + with pytest.raises(Exception) as exc_info: + await bind_resource( + upload_id="01UPL0000000000000000000B", + payload=_bind_payload(), + request=MagicMock(), + context=_bind_context(), + session=session, + ) + assert exc_info.value.status_code == 409 + assert "not created for a data resource" in exc_info.value.detail + + +def test_safe_path_segment_cleans_special_characters(): + assert _safe_path_segment("train") == "train" + assert _safe_path_segment("train v1") == "train_v1" + assert _safe_path_segment("../foo") == "foo" + assert _safe_path_segment("a/b") == "a_b" + assert _safe_path_segment("...") == "untitled" + + +def test_safe_file_name_rejects_traversal_and_hidden(): + assert _safe_file_name("data.csv") == "data.csv" + with pytest.raises(Exception): + _safe_file_name("../data.csv") + with pytest.raises(Exception): + _safe_file_name(".hidden.csv") + + +def _make_resource(workspace_id: str, owner_user_id: str): + return SimpleNamespace( + resource_id="01RES0000000000000000000A", + workspace_id=workspace_id, + storage_object_id="01OBJ0000000000000000000A", + owner_user_id=owner_user_id, + resource_name="sample", + description=None, + visibility="workspace", + status="active", + created_at=datetime.datetime.now(datetime.UTC).replace(tzinfo=None), + updated_at=datetime.datetime.now(datetime.UTC).replace(tzinfo=None), + ) + + +def _make_storage_object(object_key: str): + return SimpleNamespace( + storage_object_id="01OBJ0000000000000000000A", + object_key=object_key, + file_name="data.csv", + file_extension=".csv", + mime_type="text/csv", + size_bytes=42, + content_hash="a" * 64, + object_status="available", + ) + + +def test_resource_payload_legacy_dot_resources(): + ws = "01WS00000000000000000000A" + user = "01USR000000000000000000A" + payload = resource_payload( + _make_resource(ws, user), + _make_storage_object(f"{ws}/{user}/.resources/data.csv"), + ) + assert payload["jupyter_accessible_path"] == ".resources/data.csv" + assert payload["absolute_path"].endswith(f"{ws}/{user}/.resources/data.csv") + + +def test_resource_payload_new_flat_path(): + ws = "01WS00000000000000000000A" + user = "01USR000000000000000000A" + payload = resource_payload( + _make_resource(ws, user), + _make_storage_object(f"{ws}/{user}/data.csv"), + ) + assert payload["jupyter_accessible_path"] == "data.csv" + assert payload["absolute_path"].endswith(f"{ws}/{user}/data.csv") + + +def test_resource_payload_new_nested_path(): + ws = "01WS00000000000000000000A" + user = "01USR000000000000000000A" + payload = resource_payload( + _make_resource(ws, user), + _make_storage_object(f"{ws}/{user}/train/v1/data.csv"), + ) + assert payload["jupyter_accessible_path"] == "train/v1/data.csv" + assert payload["absolute_path"].endswith(f"{ws}/{user}/train/v1/data.csv") + + +def test_compute_jupyter_relative_path_for_legacy_and_new_paths(): + # Legacy .resources path still resolves correctly. + assert compute_jupyter_relative_path("notebooks/exp.ipynb", ".resources/data.csv") == "../.resources/data.csv" + # New nested path resolves relative to the script directory. + assert compute_jupyter_relative_path("notebooks/exp.ipynb", "train/v1/data.csv") == "../train/v1/data.csv" + # Same-directory script. + assert compute_jupyter_relative_path("exp.ipynb", "data.csv") == "data.csv" + + +def test_resource_directory_parses_object_key(): + ws = "01WS00000000000000000000A" + user = "01USR000000000000000000A" + # 根目录文件:目录为 ""。 + assert resource_directory(f"{ws}/{user}/data.csv", ws, user) == "" + # 子目录 / 多级目录。 + assert resource_directory(f"{ws}/{user}/sub/data.csv", ws, user) == "sub" + assert ( + resource_directory(f"{ws}/{user}/train/v1/data.csv", ws, user) + == "train/v1" + ) + # 旧版 .resources 布局:目录为 ".resources"。 + assert ( + resource_directory(f"{ws}/{user}/.resources/data.csv", ws, user) + == ".resources" + ) + # 不匹配 ws/user 前缀的键按根目录处理。 + assert resource_directory("other-bucket-key.csv", ws, user) == "" + + +def test_object_key_tail_and_strip_uniqueness_suffix(): + ws = "01WS00000000000000000000A" + user = "01USR000000000000000000A" + # 前缀剥离。 + assert _object_key_tail(f"{ws}/{user}/sub/data.csv", ws, user) == "sub/data.csv" + assert _object_key_tail("no-prefix.csv", ws, user) == "no-prefix.csv" + # 带唯一化后缀的 key 恢复为原始路径(26 位 ULID 后缀)。 + suffix = "01arz3ndektsv4rrffq69g5fav" # 26 chars + assert ( + _strip_uniqueness_suffix(f"sub/data-{suffix}.csv") == "sub/data.csv" + ) + assert _strip_uniqueness_suffix(f"data-{suffix}.csv") == "data.csv" + assert _strip_uniqueness_suffix(f"data-{suffix}") == "data" + # 普通文件名不受影响(短后缀、无连字符、目录含连字符)。 + assert _strip_uniqueness_suffix("sub/data.csv") == "sub/data.csv" + assert _strip_uniqueness_suffix("data-v1.csv") == "data-v1.csv" + assert _strip_uniqueness_suffix("my-dir/data.csv") == "my-dir/data.csv" + assert _strip_uniqueness_suffix("data") == "data" + + +def test_data_resources_model_allows_duplicate_storage_object_reference() -> None: + """With uk_data_resources_object dropped, no unique index covers + storage_object_id, so multiple DataResources rows may reference the + same storage object. + """ + from common.db.models import DataResources + + index_names = {idx.name for idx in DataResources.__table__.indexes} + assert "uk_data_resources_object" not in index_names + for idx in DataResources.__table__.indexes: + if idx.unique: + cols = {c.name for c in idx.columns} + assert "storage_object_id" not in cols, ( + f"unexpected unique index {idx.name} on storage_object_id" + ) diff --git a/backend/tests/test_runtime_client_directories.py b/backend/tests/test_runtime_client_directories.py new file mode 100644 index 0000000..78627f3 --- /dev/null +++ b/backend/tests/test_runtime_client_directories.py @@ -0,0 +1,274 @@ +"""Unit tests for the three new directory methods on RuntimeClient. + +Covers: +- `create_directory` — PUT with `{"type": "directory"}` body. +- `delete_directory` — DELETE, surfaces Jupyter's 409 on non-empty dirs. +- `ensure_directory` — GET-first, falls back to `create_directory` on 404. + +Uses `respx` to mock httpx transport so we don't need a live Jupyter. +The runtime descriptor (`get_workspace`) is patched to a synchronous return +so `_ensure_workspace` short-circuits without hitting the Runtime service. +""" + +from __future__ import annotations + +import httpx +import pytest +import respx +from backend.runtime_client import RuntimeClient, RuntimeClientError + +WORKSPACE_ID = "01HWS0000000000000000000A" +BASE_URL = "http://runtime" +PORT = 34567 +TOKEN = "test-token" +JUPYTER_URL = f"{BASE_URL}:{PORT}/jupyter/{WORKSPACE_ID}/api/contents" + + +def _running_descriptor() -> dict: + return { + "status": "running", + "workspace_id": WORKSPACE_ID, + "base_url": BASE_URL, + "port": PORT, + "token": TOKEN, + } + + +@pytest.fixture +def client() -> httpx.AsyncClient: + return httpx.AsyncClient(timeout=httpx.Timeout(5.0)) + + +@pytest.fixture +def runtime(client: httpx.AsyncClient) -> RuntimeClient: + rt = RuntimeClient(client) + # Bypass the real `_ensure_workspace` so tests don't have to mock the + # Runtime service. The descriptor is otherwise identical to what the + # production path returns. + rt._ensure_workspace = _ensure_workspace_stub # type: ignore[assignment] + return rt + + +async def _ensure_workspace_stub(workspace_id: str) -> dict: + return _running_descriptor() + + +# --------------------------------------------------------------------------- +# create_directory +# --------------------------------------------------------------------------- + + +async def test_create_directory_happy_path(runtime: RuntimeClient) -> None: + with respx.mock(assert_all_called=True) as router: + route = router.put(f"{JUPYTER_URL}/01DIRAAAAAAAAAAAAAAA").mock( + return_value=httpx.Response( + 201, + json={ + "name": "01DIRAAAAAAAAAAAAAAA", + "type": "directory", + "path": "01DIRAAAAAAAAAAAAAAA", + }, + ) + ) + + async with httpx.AsyncClient() as transport: + runtime.client = transport # type: ignore[assignment] + result = await runtime.create_directory( + WORKSPACE_ID, name="01DIRAAAAAAAAAAAAAAA" + ) + + assert route.called + assert result["type"] == "directory" + # Verify request body shape (PUT contents/ directory). + request = route.calls[0].request + assert request.headers["Authorization"] == f"token {TOKEN}" + assert request.headers["Content-Type"] == "application/json" + assert request.content == b'{"type":"directory"}' + + +async def test_create_directory_nested_path(runtime: RuntimeClient) -> None: + """Nested path `{parent_ulid}/{dir_ulid}` lands on Jupyter correctly.""" + nested = "01DIR_PARENT_ULID/01DIR_CHILD_ULID" + with respx.mock(assert_all_called=True) as router: + route = router.put(f"{JUPYTER_URL}/{nested}").mock( + return_value=httpx.Response( + 201, json={"name": "01DIR_CHILD_ULID", "type": "directory"} + ) + ) + + async with httpx.AsyncClient() as transport: + runtime.client = transport # type: ignore[assignment] + await runtime.create_directory(WORKSPACE_ID, name=nested) + + assert route.called + + +async def test_create_directory_propagates_jupyter_4xx( + runtime: RuntimeClient, +) -> None: + with respx.mock() as router: + put_route = router.put(f"{JUPYTER_URL}/01DIRAAAAAAAAAAAAAAA").mock( + return_value=httpx.Response( + 400, + json={ + "detail": { + "code": "BAD_REQUEST", + "message": "invalid name", + } + }, + ) + ) + + async with httpx.AsyncClient() as transport: + runtime.client = transport # type: ignore[assignment] + with pytest.raises(RuntimeClientError) as exc_info: + await runtime.create_directory( + WORKSPACE_ID, name="01DIRAAAAAAAAAAAAAAA" + ) + + assert put_route.called + assert exc_info.value.status_code == 400 + # `_jupyter_request` surfaces the whole JSON body as `detail` — the + # inner `detail` envelope is preserved verbatim. + assert exc_info.value.detail == { + "detail": { + "code": "BAD_REQUEST", + "message": "invalid name", + } + } + + +# --------------------------------------------------------------------------- +# delete_directory +# --------------------------------------------------------------------------- + + +async def test_delete_directory_happy_path(runtime: RuntimeClient) -> None: + with respx.mock(assert_all_called=True) as router: + route = router.delete(f"{JUPYTER_URL}/01DIRAAAAAAAAAAAAAAA").mock( + return_value=httpx.Response(204) + ) + + async with httpx.AsyncClient() as transport: + runtime.client = transport # type: ignore[assignment] + result = await runtime.delete_directory( + WORKSPACE_ID, name="01DIRAAAAAAAAAAAAAAA" + ) + + assert route.called + assert result is None + + +async def test_delete_directory_non_empty_409(runtime: RuntimeClient) -> None: + """Jupyter rejects non-empty directory deletes with 409; surface as-is.""" + with respx.mock() as router: + router.delete(f"{JUPYTER_URL}/01DIRAAAAAAAAAAAAAAA").mock( + return_value=httpx.Response( + 409, + json={ + "detail": { + "code": "DIRECTORY_NOT_EMPTY", + "message": "Directory is not empty", + } + }, + ) + ) + + async with httpx.AsyncClient() as transport: + runtime.client = transport # type: ignore[assignment] + with pytest.raises(RuntimeClientError) as exc_info: + await runtime.delete_directory( + WORKSPACE_ID, name="01DIRAAAAAAAAAAAAAAA" + ) + + assert exc_info.value.status_code == 409 + + +# --------------------------------------------------------------------------- +# ensure_directory +# --------------------------------------------------------------------------- + + +async def test_ensure_directory_already_exists(runtime: RuntimeClient) -> None: + """GET succeeds → no PUT. The lazy-backfill is a no-op.""" + put_called = False + + def _track_put(request: httpx.Request) -> httpx.Response: + nonlocal put_called + put_called = True + return httpx.Response(201, json={}) + + with respx.mock(assert_all_called=False) as router: + router.get(f"{JUPYTER_URL}/01DIRAAAAAAAAAAAAAAA").mock( + return_value=httpx.Response( + 200, + json={"name": "01DIRAAAAAAAAAAAAAAA", "type": "directory"}, + ) + ) + router.put(f"{JUPYTER_URL}/01DIRAAAAAAAAAAAAAAA").mock( + side_effect=_track_put + ) + + async with httpx.AsyncClient() as transport: + runtime.client = transport # type: ignore[assignment] + await runtime.ensure_directory( + WORKSPACE_ID, "01DIRAAAAAAAAAAAAAAA" + ) + + assert not put_called, "PUT must not be issued when GET already shows the dir exists" + + +async def test_ensure_directory_missing_creates_it( + runtime: RuntimeClient, +) -> None: + """GET 404 → PUT. The lazy-backfill creates the missing directory.""" + with respx.mock(assert_all_called=True) as router: + get_route = router.get(f"{JUPYTER_URL}/01DIRAAAAAAAAAAAAAAA").mock( + return_value=httpx.Response( + 404, json={"detail": {"code": "NOT_FOUND"}} + ) + ) + put_route = router.put(f"{JUPYTER_URL}/01DIRAAAAAAAAAAAAAAA").mock( + return_value=httpx.Response( + 201, json={"type": "directory"} + ) + ) + + async with httpx.AsyncClient() as transport: + runtime.client = transport # type: ignore[assignment] + await runtime.ensure_directory( + WORKSPACE_ID, "01DIRAAAAAAAAAAAAAAA" + ) + + assert get_route.called + assert put_route.called + + +async def test_ensure_directory_propagates_non_404_error( + runtime: RuntimeClient, +) -> None: + """GET 500 → propagate; do NOT fall through to PUT.""" + put_called = False + + def _track_put(request: httpx.Request) -> httpx.Response: + nonlocal put_called + put_called = True + return httpx.Response(201, json={}) + + with respx.mock(assert_all_called=False) as router: + router.get(f"{JUPYTER_URL}/01DIRAAAAAAAAAAAAAAA").mock( + return_value=httpx.Response(500, text="internal error") + ) + router.put(f"{JUPYTER_URL}/01DIRAAAAAAAAAAAAAAA").mock( + side_effect=_track_put + ) + + async with httpx.AsyncClient() as transport: + runtime.client = transport # type: ignore[assignment] + with pytest.raises(RuntimeClientError) as exc_info: + await runtime.ensure_directory( + WORKSPACE_ID, "01DIRAAAAAAAAAAAAAAA" + ) + + assert exc_info.value.status_code == 500 + assert not put_called, "PUT must not be issued after a non-404 GET error" \ No newline at end of file diff --git a/backend/tests/test_scripts.py b/backend/tests/test_scripts.py new file mode 100644 index 0000000..8b6ebeb --- /dev/null +++ b/backend/tests/test_scripts.py @@ -0,0 +1,723 @@ +"""Tests for script storage-layer behavior after unique-index removal. + +These are intentionally unit-level: they mock the async SQLAlchemy session +and the Jupyter runtime client so the suite stays fast and does not need a +live database. The tests verify the code-level guarantees that back the +"delete then re-upload" flow: + + * StorageObjects is flushed before Scripts on creation. + * All soft-delete paths flip ``is_deleted = 1`` alongside ``status='deleted'`` + / ``object_status='deleted'``. + * The ORM models no longer declare the dropped unique indexes. +""" + +from __future__ import annotations + +from types import SimpleNamespace +from unittest.mock import AsyncMock, MagicMock + +import pytest +from common.config import settings +from common.db.models import DataResources, Scripts, StorageObjects + + +def _index_names(table) -> set[str]: + return {idx.name for idx in table.indexes} + + +def test_scripts_model_dropped_unique_indexes() -> None: + """The dropped unique indexes must not be declared on the Scripts model.""" + names = _index_names(Scripts.__table__) + assert "uk_scripts_current_object" not in names + assert "uk_scripts_workspace_name" not in names + assert "uk_scripts_workspace_name_active" not in names + + +def test_data_resources_model_dropped_unique_index() -> None: + """The dropped unique index must not be declared on DataResources.""" + names = _index_names(DataResources.__table__) + assert "uk_data_resources_object" not in names + + +def test_storage_objects_unique_index_is_active_only() -> None: + """After dropping uk_storage_bucket_key and adding + uk_storage_bucket_key_active (conditional UNIQUE via generated column), + the only physical unique index on storage_objects covers only the + `object_status='available'` subset (NULL-permissive slot for soft-deleted). + + The workspace path lookup index is non-unique; path uniqueness for + active objects is enforced by the application-level conflict check + in ``scripts.py`` and ``services.storage._resolve_unique_object_key``. + """ + indexes_by_name = {idx.name: idx for idx in StorageObjects.__table__.indexes} + assert "uk_storage_bucket_key" not in indexes_by_name + assert "uk_storage_bucket_key_active" in indexes_by_name + assert indexes_by_name["uk_storage_bucket_key_active"].unique is True + # The active-column index must include the generated column. + assert any( + col.name == "object_key_hash_active" + for col in indexes_by_name["uk_storage_bucket_key_active"].columns + ) + + +def _make_context() -> SimpleNamespace: + return SimpleNamespace( + request_id="01REQ0000000000000000000A", + user=SimpleNamespace(user_id="01USR0000000000000000000A"), + workspace=SimpleNamespace(workspace_id="01WS0000000000000000000A"), + is_admin=False, + ) + + +def _make_request() -> MagicMock: + request = MagicMock() + request.app.state.runtime_client.delete_file = AsyncMock(return_value=None) + return request + + +class _AsyncSessionMock: + """Minimal AsyncSession stand-in that records flush/add order.""" + + def __init__(self) -> None: + self.added: list[object] = [] + self.flush_order: list[str] = [] + self._refreshed: list[object] = [] + self.scalar_results: list[Any] = [] + + def add(self, obj: object) -> None: + self.added.append(obj) + + async def flush(self) -> None: + # Record the kind of object that triggered this flush. + self.flush_order.append(type(self.added[-1]).__name__) + + async def refresh(self, obj: object, attribute_names: list[str] | None = None) -> None: + self._refreshed.append(obj) + + async def scalar(self, *_args, **_kwargs) -> None: + if self.scalar_results: + return self.scalar_results.pop(0) + return None + + +@pytest.mark.asyncio +async def test_create_script_record_flushes_storage_object_before_script() -> None: + """StorageObjects must flush first so path conflicts surface early.""" + from backend.scripts import create_script_record + + session = _AsyncSessionMock() + request = _make_request() + context = _make_context() + + runtime_client = request.app.state.runtime_client + runtime_client.ensure_directory = AsyncMock(return_value=None) + runtime_client.create_notebook = AsyncMock(return_value={"name": "test.ipynb"}) + + script, storage_object = await create_script_record( + name="test.ipynb", + script_type="notebook", + content=b'{"cells": []}', + visibility="private", + parent_path=None, + request=request, + context=context, + session=session, + ) + + assert isinstance(storage_object, StorageObjects) + assert isinstance(script, Scripts) + assert session.flush_order == ["StorageObjects", "Scripts"] + assert session._refreshed == [storage_object, script] + + +@pytest.mark.asyncio +async def test_create_script_record_storage_object_flush_failure_does_not_add_script() -> None: + """If the StorageObjects flush fails, the Scripts row must never be added.""" + from backend.scripts import create_script_record + + class FailingSession(_AsyncSessionMock): + async def flush(self) -> None: + self.flush_order.append(type(self.added[-1]).__name__) + raise RuntimeError("duplicate path") + + session = FailingSession() + request = _make_request() + context = _make_context() + + runtime_client = request.app.state.runtime_client + runtime_client.ensure_directory = AsyncMock(return_value=None) + runtime_client.create_notebook = AsyncMock(return_value={"name": "test.ipynb"}) + + with pytest.raises(RuntimeError, match="duplicate path"): + await create_script_record( + name="test.ipynb", + script_type="notebook", + content=b'{"cells": []}', + visibility="private", + parent_path=None, + request=request, + context=context, + session=session, + ) + + # Only the StorageObjects row was ever added. + assert len(session.added) == 1 + assert isinstance(session.added[0], StorageObjects) + # The Jupyter cleanup was attempted. + request.app.state.runtime_client.delete_file.assert_awaited_once() + + +@pytest.mark.asyncio +async def test_create_script_record_allows_reupload_after_delete() -> None: + """Without uk_scripts_workspace_name_active, re-uploading a script with + the same name after the previous one was soft-deleted succeeds. + """ + from backend.scripts import create_script_record + + session = _AsyncSessionMock() + request = _make_request() + context = _make_context() + + runtime_client = request.app.state.runtime_client + runtime_client.ensure_directory = AsyncMock(return_value=None) + runtime_client.create_notebook = AsyncMock(return_value={"name": "test.ipynb"}) + + script1, storage_object1 = await create_script_record( + name="test.ipynb", + script_type="notebook", + content=b'{"cells": []}', + visibility="private", + parent_path=None, + request=request, + context=context, + session=session, + ) + + # Simulate the first script and its storage object being soft-deleted. + script1.status = "deleted" + script1.is_deleted = 1 + storage_object1.object_status = "deleted" + storage_object1.is_deleted = 1 + + # Re-upload with the same name: the Scripts layer must not reject it. + script2, storage_object2 = await create_script_record( + name="test.ipynb", + script_type="notebook", + content=b'{"cells": []}', + visibility="private", + parent_path=None, + request=request, + context=context, + session=session, + ) + + assert script2.script_name == script1.script_name + assert script2.script_id != script1.script_id + assert script2.status == "active" + assert storage_object2.storage_object_id != storage_object1.storage_object_id + # Each upload flushes its own StorageObject then Script. + assert session.flush_order == [ + "StorageObjects", + "Scripts", + "StorageObjects", + "Scripts", + ] + + +@pytest.mark.asyncio +async def test_create_script_record_allows_same_name_different_parent() -> None: + """Same filename in different parent directories of the same user + must coexist — they correspond to different Jupyter paths + (/user/foo.ipynb vs /user/test/foo.ipynb). + """ + from backend.scripts import create_script_record + + session = _AsyncSessionMock() + request = _make_request() + context = _make_context() + + runtime_client = request.app.state.runtime_client + runtime_client.ensure_directory = AsyncMock(return_value=None) + runtime_client.create_notebook = AsyncMock(return_value={"name": "x.ipynb"}) + + s1, so1 = await create_script_record( + name="x.ipynb", script_type="notebook", content=b'{"cells":[]}', + visibility="private", parent_path=None, + request=request, context=context, session=session, + ) + + # Prime the mock to return a directory row for the upcoming parent lookup. + user_prefix = f"workspace/{context.user.user_id}" + session.scalar_results.append( + StorageObjects( + storage_object_id="01OBJ0000000000000000AB", + workspace_id=context.workspace.workspace_id, + owner_user_id=context.user.user_id, + object_type="directory", + usage_type="working_copy", + storage_backend=settings.storage_backend, + storage_uri="s3://bucket/key", + file_name="test", + created_by=context.user.user_id, + relative_path=f"{user_prefix}/test", + ) + ) + + # Second upload with the same name but in a subdirectory must NOT 409. + s2, so2 = await create_script_record( + name="x.ipynb", script_type="notebook", content=b'{"cells":[]}', + visibility="private", parent_path="test", + request=request, context=context, session=session, + ) + + assert s1.script_id != s2.script_id + assert so1.storage_object_id != so2.storage_object_id + # Different relative_paths because parent_path differs. + assert so1.relative_path != so2.relative_path + assert so1.relative_path.endswith("x.ipynb") + assert so2.relative_path.endswith("test/x.ipynb") + + +@pytest.mark.asyncio +async def test_create_script_after_soft_delete_does_not_conflict() -> None: + """Resurrection regression: with uk_storage_bucket_key_active being a + conditional UNIQUE (NULL when object_status != 'available'), re-uploading + a script whose previous StorageObjects row is soft-deleted does NOT + raise IntegrityError — the generated column is NULL for the deleted row, + so it does not occupy the UNIQUE slot. + """ + from backend.scripts import create_script_record + + session = _AsyncSessionMock() + request = _make_request() + context = _make_context() + + runtime_client = request.app.state.runtime_client + runtime_client.ensure_directory = AsyncMock(return_value=None) + runtime_client.create_notebook = AsyncMock(return_value={"name": "x.ipynb"}) + + s1, so1 = await create_script_record( + name="x.ipynb", script_type="notebook", content=b'{"cells":[]}', + visibility="private", parent_path=None, + request=request, context=context, session=session, + ) + s1.status = "deleted" + s1.is_deleted = 1 + so1.object_status = "deleted" + so1.is_deleted = 1 + + s2, so2 = await create_script_record( + name="x.ipynb", script_type="notebook", content=b'{"cells":[]}', + visibility="private", parent_path=None, + request=request, context=context, session=session, + ) + assert s2.script_id != s1.script_id + assert so2.storage_object_id != so1.storage_object_id + # Same object_key (no ULID suffix) — soft-deleted row excluded from + # _resolve_unique_object_key's "available" filter. + assert so2.object_key == so1.object_key + + +def test_scripts_model_allows_duplicate_active_name() -> None: + """With uk_scripts_workspace_name_active dropped, no unique index covers + (workspace_id, script_name, script_type), so duplicate active names are + allowed at the ORM level. + """ + for idx in Scripts.__table__.indexes: + if idx.unique: + cols = {c.name for c in idx.columns} + assert not ( + {"workspace_id", "script_name", "script_type"} <= cols + ), f"unexpected unique index {idx.name} on script name" + + +def test_scripts_model_allows_duplicate_current_object_id() -> None: + """With uk_scripts_current_object dropped, no unique index covers + current_object_id, so multiple scripts may point to the same storage object. + """ + names = _index_names(Scripts.__table__) + assert "uk_scripts_current_object" not in names + for idx in Scripts.__table__.indexes: + if idx.unique and len(idx.columns) == 1: + assert "current_object_id" not in {c.name for c in idx.columns} + + +def _script_row() -> Scripts: + from datetime import UTC, datetime + + return Scripts( + script_id="01SCR0000000000000000000A", + workspace_id="01WS0000000000000000000A", + current_object_id="01OBJ0000000000000000000A", + owner_user_id="01USR0000000000000000000A", + script_name="test.ipynb", + script_type="notebook", + visibility="private", + status="active", + created_at=datetime.now(UTC).replace(tzinfo=None), + updated_at=datetime.now(UTC).replace(tzinfo=None), + ) + + +def _storage_object_row() -> StorageObjects: + return StorageObjects( + storage_object_id="01OBJ0000000000000000000A", + workspace_id="01WS0000000000000000000A", + object_type="file", + usage_type="working_copy", + # 与生产构造器一致（common/db/models/storage.py 注释 "s3"） + storage_backend=settings.storage_backend, + storage_uri="s3://bucket/key", + file_name="test.ipynb", + created_by="01USR0000000000000000000A", + ) + + +@pytest.mark.asyncio +async def test_delete_script_route_sets_is_deleted(monkeypatch: pytest.MonkeyPatch) -> None: + """Soft-deleting a script via the route handler flips is_deleted=1.""" + from backend.scripts import delete_script + + script = _script_row() + storage_object = _storage_object_row() + + request = _make_request() + context = _make_context() + session = AsyncMock() + + async def _fake_get_script_row( + _script_id: str, + _context: SimpleNamespace, + _session: AsyncMock, + *, + for_update: bool = False, + allow_missing_storage_object: bool = False, + ) -> tuple[Scripts, StorageObjects]: + return script, storage_object + + monkeypatch.setattr("backend.scripts.get_script_row", _fake_get_script_row) + mock_soft_delete = AsyncMock( + return_value={ + "data": { + "storage_object_id": storage_object.storage_object_id, + "object_status": "deleted", + "trash_key": "trash/bucket/key", + "trash_bucket": "trash", + } + } + ) + monkeypatch.setattr("backend.scripts.soft_delete_object", mock_soft_delete) + + result = await delete_script( + script_id=script.script_id, + request=request, + context=context, + session=session, + ) + + assert script.status == "deleted" + assert script.is_deleted == 1 + assert script.deleted_at is not None + assert storage_object.object_status == "deleted" + assert storage_object.is_deleted == 1 + assert storage_object.deleted_at is not None + assert result["data"]["status"] == "deleted" + mock_soft_delete.assert_awaited_once_with( + storage_object.storage_object_id, request, session + ) + + +@pytest.mark.asyncio +async def test_delete_resource_sets_is_deleted(monkeypatch: pytest.MonkeyPatch) -> None: + """Soft-deleting a data resource must write is_deleted=1 on the row.""" + from backend.resources import delete_resource + + resource = DataResources( + resource_id="01RES0000000000000000000A", + workspace_id="01WS0000000000000000000A", + storage_object_id="01OBJ0000000000000000000A", + owner_user_id="01USR0000000000000000000A", + resource_name="data.csv", + visibility="workspace", # visible so can_view passes + status="active", + ) + storage_object = _storage_object_row() + + request = _make_request() + context = _make_context() + + class _Result: + def one_or_none(self): + return (resource, storage_object) + + session = AsyncMock() + session.execute = AsyncMock(return_value=_Result()) + + with monkeypatch.context() as mp: + mp.setattr( + "backend.resources.soft_delete_object", + AsyncMock(return_value={"data": {}}), + ) + result = await delete_resource( + resource_id=resource.resource_id, + request=request, + context=context, + session=session, + ) + + assert resource.status == "deleted" + assert resource.is_deleted == 1 + assert resource.deleted_at is not None + assert result["data"]["status"] == "deleted" + + +@pytest.mark.asyncio +async def test_soft_delete_object_sets_is_deleted() -> None: + """The shared helper must flip is_deleted=1 on StorageObjects.""" + from backend.services.storage import soft_delete_object + + item = _storage_object_row() + item.storage_backend = "local" # 测的是"非默认后端短路 trash、只翻 DB"路径 + item.object_status = "available" + + request = MagicMock() + request.app.state.object_stores = {} + + session = AsyncMock() + session.scalar = AsyncMock(return_value=item) + + result = await soft_delete_object( + storage_object_id=item.storage_object_id, + request=request, + session=session, + ) + + assert item.object_status == "deleted" + assert item.is_deleted == 1 + assert item.deleted_at is not None + assert result["data"]["object_status"] == "deleted" + + +@pytest.mark.asyncio +async def test_soft_delete_object_streams_via_get_stream() -> None: + """P0-5 / B2: 必须走 ``get_stream()`` 流式迁移,不能调 ``get()`` + 把整个对象加载到内存(10G 对象会 OOM)。""" + from backend.services.storage import soft_delete_object + + item = _storage_object_row() + # 让 storage_backend 与 settings 一致,确保走"移动到 trash"分支 + item.storage_backend = settings.storage_backend + item.object_status = "available" + item.bucket_name = "workspace" + item.object_key = "ws/u/file.bin" + item.usage_type = "working_copy" + + source_store = MagicMock() + # 模拟一个 async generator 作为 get_stream 的返回值 + async def _fake_stream(_key, _chunk_size=65536): + yield b"chunk-1" + yield b"chunk-2" + source_store.get_stream.side_effect = _fake_stream + source_store.put = AsyncMock() + source_store.delete = AsyncMock() + + trash_store = MagicMock() + trash_store.put = AsyncMock() + trash_store.delete = AsyncMock() + + request = MagicMock() + request.app.state.object_stores = { + item.bucket_name: source_store, + "trash": trash_store, + } + + session = AsyncMock() + session.scalar = AsyncMock(return_value=item) + + # 捕获原 key —— helper 在移动后会把 item.object_key 重写成 trash_key, + # 不捕获的话下面断言会拿不到原值。 + original_object_key = item.object_key + + await soft_delete_object( + storage_object_id=item.storage_object_id, + request=request, + session=session, + ) + + # get_stream 必须被调用;get() 不应被调用 —— 否则仍是全量加载路径 + source_store.get_stream.assert_called_once_with(original_object_key) + source_store.get.assert_not_called() + # 源对象删除 —— 同样用原始 key 断言(item.object_key 已被 helper 改写) + source_store.delete.assert_awaited_once_with(original_object_key) + # 目标桶写入走 put(接受 async iter),trash_key 尾部拼 storage_object_id + trash_store.put.assert_awaited_once() + put_args, _ = trash_store.put.call_args + assert put_args[0] == f"workspace/{original_object_key}-{item.storage_object_id}" + + +@pytest.mark.asyncio +async def test_jupyter_check_notebook_lock_ignores_deleted_scripts() -> None: + """``is_deleted == 0`` filter must hide deleted notebooks from Jupyter checks.""" + from backend.jupyter import check_notebook_is_locked + + session = AsyncMock() + session.execute = AsyncMock() + session.execute.return_value.one_or_none = MagicMock(return_value=None) + + result = await check_notebook_is_locked( + workspace_id="01WS0000000000000000000A", + notebook_path="test.ipynb", + user_id="01USR0000000000000000000A", + session=session, + ) + + assert result is False + # Verify the query carries the is_deleted filter. + call = session.execute.await_args + statement = call[0][0] + compiled = str(statement.compile(compile_kwargs={"literal_binds": True})) + assert "is_deleted" in compiled + + +@pytest.mark.asyncio +async def test_update_script_writes_back_storage_object_metadata( + monkeypatch: pytest.MonkeyPatch, +) -> None: + """update_script must persist the new content_hash/size_bytes/relative_path + onto the StorageObjects row (not just into the response dict). Pre-fix the + DB row kept the create-time values forever, so cache/dedup/hash checks + downstream saw stale data. + + Also locks in the workspace-prefixed relative_path convention so the + frontend's slice(2) reducer produces a non-empty path even for root scripts. + """ + import hashlib + + from backend.schemas import UpdateScriptRequest + from backend.scripts import update_script + + script = _script_row() + storage_object = _storage_object_row() + # Seed stale metadata as it would have looked after create_script_record. + user_id = "01USR0000000000000000000A" + storage_object.object_key = f"{script.workspace_id}/{user_id}/test.ipynb" + storage_object.content_hash = hashlib.sha256(b"old").hexdigest() + storage_object.size_bytes = 3 + storage_object.relative_path = f"workspace/{user_id}/test.ipynb" + + request = _make_request() + runtime_client = request.app.state.runtime_client + runtime_client.upload_file = AsyncMock(return_value={"name": "test.ipynb"}) + runtime_client.create_notebook = AsyncMock(return_value={"name": "test.ipynb"}) + context = _make_context() + session = AsyncMock() + session.flush = AsyncMock() + session.refresh = AsyncMock() + + async def _fake_get_script_row( + _script_id: str, + _context: SimpleNamespace, + _session: AsyncMock, + *, + for_update: bool = False, + allow_missing_storage_object: bool = False, + ) -> tuple[Scripts, StorageObjects]: + return script, storage_object + + monkeypatch.setattr("backend.scripts.get_script_row", _fake_get_script_row) + + new_content = '{"cells": [{"cell_type": "code", "source": ["print(1)"]}]}\n' + payload = UpdateScriptRequest(content=new_content) + + result = await update_script( + script_id=script.script_id, + payload=payload, + request=request, + context=context, + session=session, + ) + + expected_hash = hashlib.sha256(new_content.encode("utf-8")).hexdigest() + expected_size = len(new_content.encode("utf-8")) + + # DB row carries the new metadata. + assert storage_object.content_hash == expected_hash + assert storage_object.size_bytes == expected_size + assert storage_object.relative_path == f"workspace/{user_id}/test.ipynb" + + # Response echoes the same values. + data = result["data"] + assert data["content_hash"] == expected_hash + assert data["size_bytes"] == expected_size + assert data["relative_path"] == f"workspace/{user_id}/test.ipynb" + + # session.flush + refresh were called to push the new metadata to DB. + session.flush.assert_awaited_once() + session.refresh.assert_awaited_once_with(storage_object) + # The runtime client received the create_notebook for the Jupyter path + # (script_type='notebook' goes through create_notebook, not upload_file). + runtime_client.create_notebook.assert_awaited_once() + + +@pytest.mark.asyncio +async def test_update_script_jupyter_only_uses_dict_fallback( + monkeypatch: pytest.MonkeyPatch, +) -> None: + """When a script has no StorageObjects row (jupyter-only fallback path), + update_script must still return the workspace-prefixed relative_path + so the frontend's slice(2) reducer yields the basename — not an empty + string. No DB write should be attempted when there is no row to update. + """ + import hashlib + + from backend.schemas import UpdateScriptRequest + from backend.scripts import update_script + + script = _script_row() + user_id = "01USR0000000000000000000A" + + request = _make_request() + runtime_client = request.app.state.runtime_client + runtime_client.upload_file = AsyncMock(return_value={"name": "test.ipynb"}) + runtime_client.create_notebook = AsyncMock(return_value={"name": "test.ipynb"}) + context = _make_context() + session = AsyncMock() + session.flush = AsyncMock() + session.refresh = AsyncMock() + + async def _fake_get_script_row( + _script_id: str, + _context: SimpleNamespace, + _session: AsyncMock, + *, + for_update: bool = False, + allow_missing_storage_object: bool = False, + ) -> tuple[Scripts, None]: + return script, None + + monkeypatch.setattr("backend.scripts.get_script_row", _fake_get_script_row) + + payload = UpdateScriptRequest(content='{"cells": []}\n') + result = await update_script( + script_id=script.script_id, + payload=payload, + request=request, + context=context, + session=session, + ) + + expected_hash = hashlib.sha256(b'{"cells": []}\n').hexdigest() + + data = result["data"] + # Jupyter-only fallback derives jupyter_path from the script_id via + # _jupyter_path(), which appends ".ipynb". The contract we lock in is + # "always workspace-prefixed, never bare user_id/basename" so the + # frontend's slice(2) reducer is safe. + assert data["relative_path"] == f"workspace/{script.script_id}.ipynb" + assert data["relative_path"].startswith("workspace/") + assert data["content_hash"] == expected_hash + assert data["size_bytes"] == len(b'{"cells": []}\n') + + # No DB write should be attempted when there is no StorageObjects row. + session.flush.assert_not_awaited() + session.refresh.assert_not_awaited() diff --git a/backend/tests/test_storage_upload_status.py b/backend/tests/test_storage_upload_status.py new file mode 100644 index 0000000..a5a46f1 --- /dev/null +++ b/backend/tests/test_storage_upload_status.py @@ -0,0 +1,149 @@ +"""Unit tests for ``upload_bytes_to_session`` failure-path status persistence. + +P0-5 / B1: the route handler wraps every request in ``session_scope`` +(``backend.dependencies.database_session``), which rolls back on +exception. A naive ``upload.upload_status = "failed"; raise HTTPException(...)`` +loses the status flip and leaves the row stuck in ``created``/``uploading`` +forever. The fix is ``_mark_upload_failed_and_raise`` which opens a +*separate* session from ``request.app.state.session_factory`` and commits +the status change on it before raising. The outer rollback is then a +no-op on the caller's session, AND the named-lock connection (acquired +inside the helper's caller for the size/hash/storage failure branches) +stays pristine so ``release_named_lock`` runs on the same connection that +ran ``GET_LOCK`` — closing the lock-leak window Codex flagged. + +These tests exercise the helper directly + the expired-status branch of +``upload_bytes_to_session`` with a mocked session. +""" +from __future__ import annotations + +from datetime import datetime, timedelta +from types import SimpleNamespace +from unittest.mock import AsyncMock, MagicMock + +import pytest +from fastapi import HTTPException + +from backend.services.storage import ( + _mark_upload_failed_and_raise, + upload_bytes_to_session, +) + + +def _make_upload(upload_id: str = "01UPL0000000000000000000A") -> SimpleNamespace: + """Minimal UploadSessions row with the columns the helper touches.""" + return SimpleNamespace( + upload_id=upload_id, + upload_status="uploading", + expires_at=None, + expected_size_bytes=None, + expected_hash=None, + bucket_name="workspace", + object_key="ws/user/file.bin", + object_key_hash=b"\x00" * 32, + ) + + +class _AsyncContextManager: + """Async context manager that yields ``session`` on enter. + + Used to mock the result of ``session_factory()`` without pulling in + a real SQLAlchemy engine. + """ + + def __init__(self, session: object) -> None: + self._session = session + + async def __aenter__(self) -> object: + return self._session + + async def __aexit__(self, *_args: object) -> None: + return None + + +async def test_mark_upload_failed_and_raise_persists_status() -> None: + """The helper must open a fresh session, mark 'failed', commit, then + raise — surviving both the surrounding ``session_scope`` rollback and + the named-lock connection-pool leak the original implementation opened + up by committing on the lock-bound session. + """ + upload = _make_upload() + fresh_session = SimpleNamespace( + scalar=AsyncMock(return_value=upload), + commit=AsyncMock(), + ) + session_factory = MagicMock(return_value=_AsyncContextManager(fresh_session)) + request = SimpleNamespace( + app=SimpleNamespace(state=SimpleNamespace(session_factory=session_factory)), + ) + + with pytest.raises(HTTPException) as excinfo: + await _mark_upload_failed_and_raise( + request, upload.upload_id, 409, "size mismatch", + ) + + assert excinfo.value.status_code == 409 + assert excinfo.value.detail == "size mismatch" + assert upload.upload_status == "failed" + # The fresh session — not the caller's — must have committed. + fresh_session.scalar.assert_awaited_once() + fresh_session.commit.assert_awaited_once() + session_factory.assert_called_once() + + +async def test_mark_upload_failed_and_raise_raises_even_if_upload_missing() -> None: + """If the row has been hard-deleted between the caller's lookup and the + helper's separate-session write, the helper still raises HTTPException + with the requested status/detail — it just skips the commit. The caller + still gets the same error contract. + """ + fresh_session = SimpleNamespace( + scalar=AsyncMock(return_value=None), + commit=AsyncMock(), + ) + session_factory = MagicMock(return_value=_AsyncContextManager(fresh_session)) + request = SimpleNamespace( + app=SimpleNamespace(state=SimpleNamespace(session_factory=session_factory)), + ) + + with pytest.raises(HTTPException) as excinfo: + await _mark_upload_failed_and_raise( + request, "01UPL0000000000000000000A", 503, "boom", + ) + + assert excinfo.value.status_code == 503 + assert excinfo.value.detail == "boom" + # No row to update, so commit must NOT have been called. + fresh_session.commit.assert_not_awaited() + + +async def test_upload_bytes_to_session_expired_commits_status() -> None: + """The early ``if upload.expires_at < now`` branch must also + commit the expired status, otherwise a client retrying after the + expiry window would see ``created`` again and re-upload. + + The expired branch is OUTSIDE the named-lock critical section, so it + can still commit on the caller's session safely — no separate-session + detour is needed. + """ + expired_upload = _make_upload() + expired_upload.upload_status = "created" + # expires_at < utcnow_naive() triggers the expired branch. + expired_upload.expires_at = datetime.utcnow() - timedelta(minutes=1) + + session = SimpleNamespace( + scalar=AsyncMock(return_value=expired_upload), + commit=AsyncMock(), + get=AsyncMock(return_value=None), + ) + request = SimpleNamespace( + app=SimpleNamespace(state=SimpleNamespace(object_stores={})), + ) + + with pytest.raises(HTTPException) as excinfo: + await upload_bytes_to_session("01UPL0000000000000000000A", session, request) + + assert excinfo.value.status_code == 409 + assert "expired" in excinfo.value.detail + assert expired_upload.upload_status == "expired" + session.commit.assert_awaited() \ No newline at end of file diff --git a/common/README.md b/common/README.md index ca981d3..261a432 100644 --- a/common/README.md +++ b/common/README.md @@ -1,2 +1,5 @@ -## 初始化 alembic -uv run alembic init src/common/migrations \ No newline at end of file +# Common + +后端公共配置、标识、错误模型、日志和基础工具目录。 + +业务模块不得在本目录外重复定义公共 DTO 或错误码。 diff --git a/common/pyproject.toml b/common/pyproject.toml index 8424d06..8c1e319 100644 --- a/common/pyproject.toml +++ b/common/pyproject.toml @@ -1,27 +1,34 @@ [project] name = "common" -version = "0.1.0" -description = "Add your description here" -readme = "README.md" -authors = [ - { name = "tao.chen", email = "93983997+taochen-ct@users.noreply.github.com" } -] +version = "0.2.0" requires-python = ">=3.12" dependencies = [ - "alembic>=1.18.5", + "SQLAlchemy==2.0.51", + "greenlet>=3.0.0", + "apscheduler>=3.11.3", + "asyncmy==0.2.11", + "boto3>=1.34,<2", + "fastapi==0.116.1", "pydantic-settings>=2.14.2", - "pymysql>=1.2.0", - "sqlalchemy>=2.0.51", + "loguru>=0.7.2", + "passlib==1.7.4", + "bcrypt>=4.0,<4.1", + "aiofiles>=25.1.0", + "aioboto3>=15.5.0", ] [build-system] requires = ["hatchling"] build-backend = "hatchling.build" - [tool.hatch.build.targets.wheel] packages = ["src/common"] [[tool.uv.index]] url = "https://pypi.tuna.tsinghua.edu.cn/simple/" default = true + +[dependency-groups] +dev = [ + "pytest>=9.1.1", +] diff --git a/common/src/common/__init__.py b/common/src/common/__init__.py index a130cb4..ab78a93 100644 --- a/common/src/common/__init__.py +++ b/common/src/common/__init__.py @@ -1,2 +1 @@ -def main() -> None: - print("Hello from common!") +"""Shared backend building blocks.""" diff --git a/common/src/common/auth/__init__.py b/common/src/common/auth/__init__.py new file mode 100644 index 0000000..7de60de --- /dev/null +++ b/common/src/common/auth/__init__.py @@ -0,0 +1,14 @@ +"""Authentication primitives shared across services. + +Houses the HS256 JWT issuer/verifier, bcrypt password helpers, and the +single canonical active-membership loader. The previous implementation +lived inline inside ``backend/jupyter.py`` and ``backend/dependencies.py``; +moving it here lets the schedule service verify the same tokens (when +service-to-service auth is reintroduced) and keeps the dependency +inversion clean. + +Service-to-service HTTP calls in this repository do NOT currently +authenticate at the application layer (see ``docker-compose.yml``: only +the gateway exposes a host port). The JWT and membership helpers are +used exclusively by user-facing endpoints. +""" diff --git a/common/src/common/auth/jwt.py b/common/src/common/auth/jwt.py new file mode 100644 index 0000000..2759e46 --- /dev/null +++ b/common/src/common/auth/jwt.py @@ -0,0 +1,140 @@ +"""HS256 JWT issuance and verification. + +The implementation is intentionally minimal: a hand-rolled HS256 +signer/verifier so the project does not depend on PyJWT. It deliberately +ignores the ``alg`` header on the verify side and always recomputes +HMAC-SHA256, which means a forged ``"alg":"none"`` token still fails +signature validation. + +Token payload contract: + sub - user_id (CHAR(26) ULID) + exp - unix seconds; mandatory + iat - unix seconds; mandatory (used for last_logout_at checks if added later) + +Any other claim is preserved by the verifier but not interpreted here. +""" + +from __future__ import annotations + +import base64 +import hashlib +import hmac +import json +import time +from typing import Any + +from common.config import settings + +JWT_SECRET: str = settings.jwt_secret +JWT_ALGORITHM: str = "HS256" +DEFAULT_TTL_SECONDS: int = 24 * 60 * 60 + + +def _b64encode(raw: bytes) -> str: + return base64.urlsafe_b64encode(raw).rstrip(b"=").decode("ascii") + + +def _b64decode(value: str) -> bytes: + padding = "=" * (-len(value) % 4) + return base64.urlsafe_b64decode(value + padding) + + +def issue_jwt( + user_id: str, + *, + ttl_seconds: int = DEFAULT_TTL_SECONDS, + extra_claims: dict[str, Any] | None = None, + now: int | None = None, +) -> str: + """Sign a JWT for ``user_id`` and return the compact serialization. + + The header is fixed ``{"alg":"HS256","typ":"JWT"}``; the signature + uses HMAC-SHA256 over ``{header_b64}.{payload_b64}`` keyed by + ``settings.jwt_secret``. + + ``ttl_seconds`` defaults to 24h. ``extra_claims`` is merged into the + payload after ``sub``/``iat``/``exp`` are populated and would + overwrite those if callers passed the same keys — kept simple on + purpose so we never accidentally bypass the contract. + """ + if not user_id: + raise ValueError("user_id is required") + + issued_at = int(time.time()) if now is None else int(now) + payload: dict[str, Any] = { + "sub": user_id, + "iat": issued_at, + "exp": issued_at + int(ttl_seconds), + } + if extra_claims: + payload.update(extra_claims) + + header = {"alg": JWT_ALGORITHM, "typ": "JWT"} + header_b64 = _b64encode(json.dumps(header, separators=(",", ":")).encode()) + payload_b64 = _b64encode(json.dumps(payload, separators=(",", ":")).encode()) + signing_input = f"{header_b64}.{payload_b64}".encode() + signature = hmac.new( + JWT_SECRET.encode(), + signing_input, + hashlib.sha256, + ).digest() + signature_b64 = _b64encode(signature) + return f"{header_b64}.{payload_b64}.{signature_b64}" + + +class JwtError(Exception): + """Raised on missing / malformed / expired / wrong-signature tokens.""" + + +def verify_jwt_token(token: str | None) -> dict[str, Any]: + """Verify an HS256-signed JWT and return its payload. + + The function is the inverse of :func:`issue_jwt`. The ``alg`` header + is read for completeness but the signature is always recomputed + under HS256 — a token claiming ``alg":"none"`` is rejected because + its signature segment will not match a recomputed HMAC. + + Raises :class:`JwtError` on every failure mode; callers translate to + 401 in HTTP contexts. + """ + if not token: + raise JwtError("missing authentication token") + + try: + header_b64, payload_b64, signature_b64 = token.split(".", 2) + except ValueError as exc: + raise JwtError("malformed token") from exc + + signing_input = f"{header_b64}.{payload_b64}".encode() + expected = hmac.new( + JWT_SECRET.encode(), + signing_input, + hashlib.sha256, + ).digest() + try: + signature = _b64decode(signature_b64) + except Exception as exc: # pragma: no cover - malformed b64 + raise JwtError("malformed signature") from exc + if not hmac.compare_digest(expected, signature): + raise JwtError("signature mismatch") + + try: + payload = json.loads(_b64decode(payload_b64)) + except Exception as exc: + raise JwtError("malformed payload") from exc + + exp = payload.get("exp") + if not isinstance(exp, (int, float)) or exp < time.time(): + raise JwtError("token expired") + + return payload + + +__all__ = [ + "DEFAULT_TTL_SECONDS", + "JWT_ALGORITHM", + "JWT_SECRET", + "JwtError", + "issue_jwt", + "verify_jwt_token", +] diff --git a/common/src/common/auth/membership.py b/common/src/common/auth/membership.py new file mode 100644 index 0000000..fed91ad --- /dev/null +++ b/common/src/common/auth/membership.py @@ -0,0 +1,98 @@ +"""Active workspace-membership loader. + +Single canonical implementation that verifies a user is currently a +member of the given workspace. Used by the dependency-injection layer +in the backend; ``jupyter.py`` previously carried its own near-duplicate +that is now reduced to a thin caller. + +The query joins Users / WorkspaceMembers / Workspaces / Roles so +callers receive the role they need for ``is_admin``-style checks +without a second round-trip. +""" + +from __future__ import annotations + +from sqlalchemy import select +from sqlalchemy.ext.asyncio import AsyncSession + +from common.db.models import Roles, Users, WorkspaceMembers, Workspaces + + +class MembershipError(Exception): + """Raised when the user is not an active member of the workspace.""" + + +async def resolve_is_system_admin( + session: AsyncSession, + user: Users, +) -> bool: + """Return True iff the user holds a platform-scoped admin role. + + The check is: ``Users.status == 'active'`` AND + ``Users.platform_role_id`` points to a ``Roles`` row whose + ``role_code == 'admin'``. Any other shape (no platform_role_id, + disabled user, wrong role code) returns False — the frontend reads + this to decide whether to show the system-admin entry point. + + System admins own the platform: they can address disabled workspaces, + bypass per-workspace membership checks, etc. Anything that wants to + gate "platform-only" behavior (deleting a workspace, soft-deleting + a user globally, …) should consult this flag — it is the single + source of truth. + """ + from sqlalchemy import select + + if user.status != "active" or user.platform_role_id is None: + return False + platform_role = await session.scalar( + select(Roles).where(Roles.role_id == user.platform_role_id) + ) + return platform_role is not None and platform_role.role_code == "admin" + + +async def load_active_membership( + session: AsyncSession, + user_id: str, + workspace_id: str, +) -> tuple[Users, Workspaces, Roles]: + """Return ``(user, workspace, role)`` for an active membership. + + All four conditions must hold: ``Users.status == 'active'``, + ``WorkspaceMembers.member_status == 'active'``, + ``Workspaces.status == 'active'``, and the row exists at all. + Raises :class:`MembershipError` otherwise. + """ + statement = ( + select(Users, Workspaces, Roles) + .join( + WorkspaceMembers, + WorkspaceMembers.user_id == Users.user_id, + ) + .join( + Workspaces, + Workspaces.workspace_id == WorkspaceMembers.workspace_id, + ) + .join( + Roles, + Roles.role_id == WorkspaceMembers.role_id, + ) + .where( + Users.user_id == user_id, + Users.status == "active", + WorkspaceMembers.workspace_id == workspace_id, + WorkspaceMembers.member_status == "active", + Workspaces.status == "active", + ) + ) + row = (await session.execute(statement)).one_or_none() + if row is None: + raise MembershipError("active workspace membership is required") + user, workspace, role = row + return user, workspace, role + + +__all__ = [ + "MembershipError", + "load_active_membership", + "resolve_is_system_admin", +] diff --git a/common/src/common/auth/passwords.py b/common/src/common/auth/passwords.py new file mode 100644 index 0000000..e3a986d --- /dev/null +++ b/common/src/common/auth/passwords.py @@ -0,0 +1,54 @@ +"""Bcrypt password hashing helpers. + +Uses passlib's :class:`CryptContext` so the algorithm choice stays in +one place — when (not if) we move to argon2 we change the ``schemes`` +list and existing hashes still verify. + +The :data:`make_unusable_password` helper returns a bcrypt hash of a +random 32-byte secret. It is intentionally verifiable (to keep the +``verify_password`` path symmetric) but cannot be matched by any +human-supplied plaintext, so it is safe to assign to service accounts +that should never log in interactively. +""" + +from __future__ import annotations + +import secrets + +from passlib.context import CryptContext + +_crypt_context = CryptContext(schemes=["bcrypt"], deprecated="auto") + + +def hash_password(plain: str) -> str: + """Hash ``plain`` with bcrypt and return the encoded digest.""" + if not plain: + raise ValueError("plain must be a non-empty string") + return _crypt_context.hash(plain) + + +def verify_password(plain: str, hashed: str) -> bool: + """Return True iff ``plain`` matches ``hashed`` under the active scheme.""" + if not plain or not hashed: + return False + try: + return _crypt_context.verify(plain, hashed) + except ValueError: + return False + + +def make_unusable_password() -> str: + """Return a bcrypt hash of a random 32-byte secret. + + Service accounts store this so the ``Users.password_hash`` column is + populated and any stray login attempt is rejected by the password + check (random secret → impossible to brute force offline). + """ + return _crypt_context.hash(secrets.token_urlsafe(32)) + + +__all__ = [ + "hash_password", + "make_unusable_password", + "verify_password", +] diff --git a/common/src/common/config.py b/common/src/common/config.py new file mode 100644 index 0000000..4d78060 --- /dev/null +++ b/common/src/common/config.py @@ -0,0 +1,220 @@ +"""Centralised configuration via pydantic-settings. + +All Python services (backend / schedule / runtime) import ``settings`` from +this module. The instance is a process-wide singleton (lru_cache wrapped), +so each field is parsed from the environment exactly once at first access. + +Rules for adding a new variable: + + 1. Add the field here with a sensible default that lets local dev + boot without the env var set. + 2. Use ``settings.`` at the call site. Never re-introduce + ``os.environ`` / ``os.getenv`` for application config — they + bypass this central registry. + 3. Document the env var name in ``.env.example`` so operators know + it exists. +""" + +from __future__ import annotations + +from functools import lru_cache + +from pydantic import Field +from pydantic_settings import BaseSettings, SettingsConfigDict + + +class Settings(BaseSettings): + # ── database ────────────────────────────────────────────────── + database_url: str = Field( + default=( + "mysql+asyncmy://model_platform:model_platform@mysql:3306/" + "model_platform?charset=utf8mb4" + ), + description="SQLAlchemy async URI for the platform MySQL.", + ) + + # ── JWT ─────────────────────────────────────────────────────── + jwt_secret: str = Field( + default="dev-only-not-for-production", + description="HS256 secret used by backend's jupyter auth_request.", + ) + demo_auth_enabled: bool = Field( + default=False, + description="Enable the self-hosted UI's short-lived demo session cookie.", + ) + cookie_force_secure: bool = Field( + default=False, + description=( + "Force the ``Secure`` flag on the session cookie even when the " + "inbound request scheme is plain HTTP. Enable this when running " + "behind a TLS-terminating reverse proxy that strips or rewrites " + "``X-Forwarded-Proto`` — without it the cookie is written without " + "the Secure flag and modern browsers will refuse to send it back " + "over HTTPS." + ), + ) + + # ── service identity ────────────────────────────────────────── + service_name: str = Field( + default="service", + description="Service label surfaced in lifespan / health checks.", + ) + + # ── logging ─────────────────────────────────────────────────── + log_level: str = Field( + default="DEBUG", + description=( + "loguru stderr sink level. One of DEBUG/INFO/WARNING/ERROR/CRITICAL; " + "anything else falls back to INFO inside configure_logging()." + ), + ) + + # ── runtime container endpoint ─────────────────────────────── + runtime_api_url: str = Field( + default="http://runtime:8000", + description="Backend → Runtime HTTP endpoint.", + ) + rclone_rc_url: str = Field( + default="http://runtime:5572", + description="Backend → rclone RC HTTP endpoint (VFS cache invalidation).", + ) + + # ── object storage backend selection ───────────────────────── + storage_backend: str = Field( + default="s3", + description=( + "Which storage backend the deployment uses. ``s3`` (default) " + "reads the ``s3_*`` settings and connects to an S3-compatible " + "service. ``local`` uses on-disk filesystems under " + "``local_storage_base_dir`` — useful for dev / single-node / " + "air-gapped deployments." + ), + ) + local_storage_base_dir: str = Field( + default="/data", + description=( + "Root directory for the local-filesystem storage backend. The 4 " + "buckets become subdirectories: ``/workspace``, " + "``/version``, ``/run_log``, ``/trash``. " + "Default ``/data``; this directory must be a shared Docker " + "volume between the backend and runtime containers in local mode." + ), + ) + + # ── S3-compatible object storage ───────────────────────────── + s3_endpoint: str = Field( + default="http://s3:9000", + description="S3 endpoint for the object-storage upstream.", + ) + s3_access_key: str = Field( + default="modelplatform", + description="boto3 access key for S3-compatible storage.", + ) + s3_secret_key: str = Field( + default="modelplatformsecret", + description="boto3 secret key for S3-compatible storage.", + ) + s3_workspace_bucket: str = Field( + default="workspace", + description=( + "Bucket for workspace files (notebooks / scripts / working " + "copies). Layout: s3:////..." + ), + ) + s3_version_bucket: str = Field( + default="version", + description="Bucket for immutable script-version artifacts.", + ) + s3_run_log_bucket: str = Field( + default="run-log", + description="Bucket for schedule run logs.", + ) + s3_trash_bucket: str = Field( + default="trash", + description=( + "Bucket for soft-deleted objects. The source bucket key is " + "preserved as a prefix so a restore is a same-key move. " + "Trash is reaped on a schedule out of band." + ), + ) + s3_trash_retention_days: int = Field( + default=30, + description=( + "How long a trashed object is retained before reaping. " + "Tracked in the database (StorageObjects.deleted_at) so the " + "reaper can run as a single SQL sweep." + ), + ) + + # ── schedule → backend API ──────────────────────────────────── + backend_api_url: str = Field( + default="http://backend:8000", + description="Schedule → Backend HTTP base URL (cron post-back).", + ) + + # ── service-to-service auth for /internal/v1/* (P0-1 fix) ───── + # Shared secret between backend and the schedule worker. The schedule + # posts the value in the ``X-Internal-Service-Token`` header when it + # uploads ``run_log`` / ``run_result`` artifacts. Backend's storage + # API rejects requests whose header does not match this value. + # Override via ``INTERNAL_SERVICE_TOKEN``; the placeholder default is + # safe for local dev with the matching schedule config but must be + # replaced in any non-dev deployment. + internal_service_token: str = Field( + default="dev-only-internal-token-not-for-production", + description=( + "Shared secret for service-to-service auth on /internal/v1/*. " + "Set identically on backend and schedule via INTERNAL_SERVICE_TOKEN." + ), + ) + + # ── runtime public base URL ────────────────────────────────── + public_base_url: str = Field( + default="http://runtime", + description="Public base URL for the runtime container.", + ) + + # ── schedule execution tuning ──────────────────────────────── + schedule_event_namespace: str = Field( + default="model-platform-local", + description=( + "Namespace prepended to schedule outbox event types. Each " + "deployment that shares a database must use a unique value." + ), + ) + schedule_execution_concurrency: int = Field( + default=4, + description=( + "Max concurrent notebooks running in the schedule worker. " + "Each notebook is dispatched as an asyncio task bounded by " + "a semaphore; the polling loop is never blocked." + ), + ) + + # ── readiness probes ────────────────────────────────────────── + readiness_targets: str = Field( + default="", + description=( + "Comma-separated host:port list checked by /health/ready. " + "Empty disables the check." + ), + ) + + model_config = SettingsConfigDict( + env_file=".env", + env_file_encoding="utf-8", + case_sensitive=False, + extra="ignore", + ) + + +@lru_cache +def get_settings() -> Settings: + """Construct (and cache) the singleton Settings instance.""" + return Settings() + + +settings: Settings = get_settings() + + +__all__ = ["Settings", "get_settings", "settings"] diff --git a/common/src/common/db/__init__.py b/common/src/common/db/__init__.py index f0ff52a..10b0d03 100644 --- a/common/src/common/db/__init__.py +++ b/common/src/common/db/__init__.py @@ -1,5 +1,17 @@ -# coding=utf-8 -""" -@Time :2026/7/29 -@Author :tao.chen -""" +"""Shared SQLAlchemy database models and infrastructure.""" + +from common.db.models import Base +from common.db.session import ( + AsyncSessionFactory, + create_database_engine, + create_session_factory, + session_scope, +) + +__all__ = [ + "AsyncSessionFactory", + "Base", + "create_database_engine", + "create_session_factory", + "session_scope", +] diff --git a/common/src/common/db/base.py b/common/src/common/db/base.py index 3f203ac..c861dd9 100644 --- a/common/src/common/db/base.py +++ b/common/src/common/db/base.py @@ -1,14 +1,10 @@ -# coding=utf-8 """ @Time :2026/7/29 -@Author :tao.chen +@Author :tao.chen """ + from sqlalchemy.orm import DeclarativeBase + class Base(DeclarativeBase): pass - -# 导入所有实体模型，确保 Base.metadata 能收集到所有表 -# from common.db.models.notebook import NotebookModel -# from common.db.models.workspace import WorkspaceModel -# from common.db.models.job import JobRunModel \ No newline at end of file diff --git a/common/src/common/db/models/__init__.py b/common/src/common/db/models/__init__.py index f0ff52a..c03058d 100644 --- a/common/src/common/db/models/__init__.py +++ b/common/src/common/db/models/__init__.py @@ -1,5 +1,35 @@ -# coding=utf-8 -""" -@Time :2026/7/29 -@Author :tao.chen -""" +from common.db.base import Base +from common.db.models.events import ConsumerInbox, OutboxEvents +from common.db.models.identity import Permissions, RolePermissions, Roles, Users +from common.db.models.schedules import ( + ScheduleEdges, + ScheduleNodeRuns, + ScheduleNodes, + ScheduleRuns, + Schedules, +) +from common.db.models.scripts import Scripts, Versions +from common.db.models.storage import DataResources, StorageObjects, UploadSessions +from common.db.models.workspaces import WorkspaceMembers, Workspaces + +__all__ = [ + "Base", + "ConsumerInbox", + "DataResources", + "OutboxEvents", + "Permissions", + "RolePermissions", + "Roles", + "ScheduleEdges", + "ScheduleNodeRuns", + "ScheduleNodes", + "ScheduleRuns", + "Schedules", + "Scripts", + "StorageObjects", + "UploadSessions", + "Users", + "Versions", + "WorkspaceMembers", + "Workspaces", +] diff --git a/common/src/common/db/models/events.py b/common/src/common/db/models/events.py new file mode 100644 index 0000000..e870eda --- /dev/null +++ b/common/src/common/db/models/events.py @@ -0,0 +1,78 @@ +import datetime + +from sqlalchemy import JSON, Index, String, text +from sqlalchemy.dialects.mysql import CHAR, DATETIME, INTEGER, SMALLINT, TINYINT +from sqlalchemy.orm import Mapped, mapped_column + +from common.db.base import Base + + +class ConsumerInbox(Base): + __tablename__ = "consumer_inbox" + __table_args__ = ( + Index("idx_consumer_inbox_status", "consumer_name", "process_status", "created_at"), + {"comment": "消费者幂等 Inbox，防止 Stream 重投导致重复执行"}, + ) + + consumer_name: Mapped[str] = mapped_column(String(128), primary_key=True) + event_id: Mapped[str] = mapped_column(CHAR(26), primary_key=True) + process_status: Mapped[str] = mapped_column( + String(16), + nullable=False, + server_default=text("'processing'"), + comment="processing/succeeded/failed", + ) + created_at: Mapped[datetime.datetime] = mapped_column( + DATETIME(fsp=3), nullable=False, server_default=text("CURRENT_TIMESTAMP(3)") + ) + message_id: Mapped[str | None] = mapped_column( + String(128), comment="Inbox message ID" + ) + processed_at: Mapped[datetime.datetime | None] = mapped_column(DATETIME(fsp=3)) + error_message: Mapped[str | None] = mapped_column(String(2000)) + is_deleted: Mapped[int] = mapped_column( + TINYINT(1), nullable=False, server_default=text("0") + ) + deleted_at: Mapped[datetime.datetime | None] = mapped_column(DATETIME(fsp=3)) + + +class OutboxEvents(Base): + __tablename__ = "outbox_events" + __table_args__ = ( + Index("idx_outbox_aggregate", "aggregate_type", "aggregate_id", "created_at"), + Index("idx_outbox_idempotency", "idempotency_key"), + Index("idx_outbox_pending", "event_status", "available_at", "created_at"), + {"comment": "事务 Outbox；提交后发布到内部事件总线"}, + ) + + event_id: Mapped[str] = mapped_column(CHAR(26), primary_key=True) + aggregate_type: Mapped[str] = mapped_column(String(64), nullable=False) + aggregate_id: Mapped[str] = mapped_column(String(128), nullable=False) + event_type: Mapped[str] = mapped_column(String(128), nullable=False) + schema_version: Mapped[int] = mapped_column( + SMALLINT, nullable=False, server_default=text("1") + ) + payload_json: Mapped[dict] = mapped_column(JSON, nullable=False) + event_status: Mapped[str] = mapped_column( + String(16), + nullable=False, + server_default=text("'pending'"), + comment="pending/published/failed", + ) + available_at: Mapped[datetime.datetime] = mapped_column( + DATETIME(fsp=3), nullable=False, server_default=text("CURRENT_TIMESTAMP(3)") + ) + retry_count: Mapped[int] = mapped_column( + INTEGER, nullable=False, server_default=text("0") + ) + created_at: Mapped[datetime.datetime] = mapped_column( + DATETIME(fsp=3), nullable=False, server_default=text("CURRENT_TIMESTAMP(3)") + ) + trace_id: Mapped[str | None] = mapped_column(String(64)) + idempotency_key: Mapped[str | None] = mapped_column(String(128)) + published_at: Mapped[datetime.datetime | None] = mapped_column(DATETIME(fsp=3)) + last_error: Mapped[str | None] = mapped_column(String(2000)) + is_deleted: Mapped[int] = mapped_column( + TINYINT(1), nullable=False, server_default=text("0") + ) + deleted_at: Mapped[datetime.datetime | None] = mapped_column(DATETIME(fsp=3)) diff --git a/common/src/common/db/models/identity.py b/common/src/common/db/models/identity.py new file mode 100644 index 0000000..58c4364 --- /dev/null +++ b/common/src/common/db/models/identity.py @@ -0,0 +1,116 @@ +import datetime + +from sqlalchemy import Index, String, text +from sqlalchemy.dialects.mysql import CHAR, DATETIME, TINYINT +from sqlalchemy.orm import Mapped, mapped_column + +from common.db.base import Base + + +class Permissions(Base): + __tablename__ = "permissions" + __table_args__ = ( + Index("idx_permissions_module", "module_code"), + Index("uk_permissions_code", "permission_code", unique=True), + {"comment": "权限点"}, + ) + + permission_id: Mapped[str] = mapped_column(CHAR(26), primary_key=True) + permission_code: Mapped[str] = mapped_column(String(128), nullable=False) + permission_name: Mapped[str] = mapped_column(String(100), nullable=False) + module_code: Mapped[str] = mapped_column(String(64), nullable=False) + created_at: Mapped[datetime.datetime] = mapped_column( + DATETIME(fsp=3), nullable=False, server_default=text("CURRENT_TIMESTAMP(3)") + ) + description: Mapped[str | None] = mapped_column(String(500)) + is_deleted: Mapped[int] = mapped_column( + TINYINT(1), nullable=False, server_default=text("0") + ) + deleted_at: Mapped[datetime.datetime | None] = mapped_column(DATETIME(fsp=3)) + + +class Roles(Base): + __tablename__ = "roles" + __table_args__ = ( + Index("uk_roles_code", "role_code", unique=True), + {"comment": "角色"}, + ) + + role_id: Mapped[str] = mapped_column(CHAR(26), primary_key=True) + role_code: Mapped[str] = mapped_column(String(64), nullable=False) + role_name: Mapped[str] = mapped_column(String(100), nullable=False) + role_scope: Mapped[str] = mapped_column( + String(16), nullable=False, comment="platform/workspace" + ) + is_builtin: Mapped[int] = mapped_column( + TINYINT(1), nullable=False, server_default=text("0") + ) + created_at: Mapped[datetime.datetime] = mapped_column( + DATETIME(fsp=3), nullable=False, server_default=text("CURRENT_TIMESTAMP(3)") + ) + updated_at: Mapped[datetime.datetime] = mapped_column( + DATETIME(fsp=3), + nullable=False, + server_default=text("CURRENT_TIMESTAMP(3) ON UPDATE CURRENT_TIMESTAMP(3)"), + ) + description: Mapped[str | None] = mapped_column(String(500)) + is_deleted: Mapped[int] = mapped_column( + TINYINT(1), nullable=False, server_default=text("0") + ) + deleted_at: Mapped[datetime.datetime | None] = mapped_column(DATETIME(fsp=3)) + + +class RolePermissions(Base): + __tablename__ = "role_permissions" + __table_args__ = ( + Index("fk_role_permissions_permission", "permission_id"), + {"comment": "角色权限"}, + ) + + role_id: Mapped[str] = mapped_column(CHAR(26), primary_key=True) + permission_id: Mapped[str] = mapped_column(CHAR(26), primary_key=True) + created_at: Mapped[datetime.datetime] = mapped_column( + DATETIME(fsp=3), nullable=False, server_default=text("CURRENT_TIMESTAMP(3)") + ) + is_deleted: Mapped[int] = mapped_column( + TINYINT(1), nullable=False, server_default=text("0") + ) + deleted_at: Mapped[datetime.datetime | None] = mapped_column(DATETIME(fsp=3)) + + +class Users(Base): + __tablename__ = "users" + __table_args__ = ( + Index("fk_users_platform_role", "platform_role_id"), + Index("idx_users_status", "status"), + Index("uk_users_email", "email", unique=True), + Index("uk_users_username", "username", unique=True), + {"comment": "平台用户"}, + ) + + user_id: Mapped[str] = mapped_column(CHAR(26), primary_key=True) + username: Mapped[str] = mapped_column(String(64), nullable=False) + display_name: Mapped[str] = mapped_column(String(100), nullable=False) + password_hash: Mapped[str] = mapped_column(String(255), nullable=False) + status: Mapped[str] = mapped_column( + String(16), + nullable=False, + server_default=text("'active'"), + comment="active/disabled/locked", + ) + created_at: Mapped[datetime.datetime] = mapped_column( + DATETIME(fsp=3), nullable=False, server_default=text("CURRENT_TIMESTAMP(3)") + ) + updated_at: Mapped[datetime.datetime] = mapped_column( + DATETIME(fsp=3), + nullable=False, + server_default=text("CURRENT_TIMESTAMP(3) ON UPDATE CURRENT_TIMESTAMP(3)"), + ) + email: Mapped[str | None] = mapped_column(String(255)) + platform_role_id: Mapped[str | None] = mapped_column(CHAR(26)) + avatar_uri: Mapped[str | None] = mapped_column(String(1000)) + last_login_at: Mapped[datetime.datetime | None] = mapped_column(DATETIME(fsp=3)) + is_deleted: Mapped[int] = mapped_column( + TINYINT(1), nullable=False, server_default=text("0") + ) + deleted_at: Mapped[datetime.datetime | None] = mapped_column(DATETIME(fsp=3)) diff --git a/common/src/common/db/models/schedules.py b/common/src/common/db/models/schedules.py new file mode 100644 index 0000000..512ffb8 --- /dev/null +++ b/common/src/common/db/models/schedules.py @@ -0,0 +1,253 @@ +import datetime +import decimal +from typing import Literal + +from sqlalchemy import DECIMAL, JSON, Index, Integer, String, Text, text +from sqlalchemy.dialects.mysql import BIGINT, CHAR, DATETIME, INTEGER, TINYINT +from sqlalchemy.orm import Mapped, mapped_column + +from common.db.base import Base + +TriggerType = Literal["manual", "cron", "api"] +FailurePolicy = Literal["stop", "continue"] +PythonVersion = Literal["3.8", "3.10", "3.12"] + + +class Schedules(Base): + __tablename__ = "schedules" + __table_args__ = ( + Index("fk_schedules_created_by", "created_by"), + Index("fk_schedules_updated_by", "updated_by"), + Index("idx_schedules_due", "enabled", "next_run_at"), + Index("idx_schedules_workspace", "workspace_id", "enabled", "updated_at"), + {"comment": "调度方案"}, + ) + + schedule_id: Mapped[str] = mapped_column(CHAR(26), primary_key=True) + workspace_id: Mapped[str] = mapped_column(CHAR(26), nullable=False) + schedule_name: Mapped[str] = mapped_column(String(255), nullable=False) + trigger_type: Mapped[str] = mapped_column( + String(16), + nullable=False, + server_default=text("'cron'"), + comment="manual/cron/api", + ) + timezone: Mapped[str] = mapped_column( + String(64), nullable=False, server_default=text("'Asia/Shanghai'") + ) + enabled: Mapped[int] = mapped_column( + TINYINT(1), nullable=False, server_default=text("0") + ) + workflow_version: Mapped[int] = mapped_column( + INTEGER, nullable=False, server_default=text("1") + ) + max_concurrency: Mapped[int] = mapped_column( + INTEGER, nullable=False, server_default=text("1") + ) + failure_policy: Mapped[str] = mapped_column( + String(24), nullable=False, server_default=text("'stop'"), comment="stop/continue" + ) + created_by: Mapped[str] = mapped_column(CHAR(26), nullable=False) + updated_by: Mapped[str] = mapped_column(CHAR(26), nullable=False) + created_at: Mapped[datetime.datetime] = mapped_column( + DATETIME(fsp=3), nullable=False, server_default=text("CURRENT_TIMESTAMP(3)") + ) + updated_at: Mapped[datetime.datetime] = mapped_column( + DATETIME(fsp=3), + nullable=False, + server_default=text("CURRENT_TIMESTAMP(3) ON UPDATE CURRENT_TIMESTAMP(3)"), + ) + description: Mapped[str | None] = mapped_column(String(1000)) + cron_expression: Mapped[str | None] = mapped_column(String(128)) + last_run_at: Mapped[datetime.datetime | None] = mapped_column(DATETIME(fsp=3)) + next_run_at: Mapped[datetime.datetime | None] = mapped_column(DATETIME(fsp=3)) + is_deleted: Mapped[int] = mapped_column( + TINYINT(1), nullable=False, server_default=text("0") + ) + deleted_at: Mapped[datetime.datetime | None] = mapped_column(DATETIME(fsp=3)) + + +class ScheduleRuns(Base): + __tablename__ = "schedule_runs" + __table_args__ = ( + Index("fk_schedule_runs_logs", "logs_object_id"), + Index("fk_schedule_runs_result", "result_object_id"), + Index("fk_schedule_runs_user", "triggered_by"), + Index("idx_schedule_runs_schedule", "schedule_id", "created_at"), + Index("idx_schedule_runs_status", "run_status", "queued_at"), + Index( + "idx_schedule_runs_workspace_status", + "workspace_id", + "run_status", + "queued_at", + ), + Index("uk_schedule_runs_idempotency", "idempotency_key", unique=True), + {"comment": "调度运行"}, + ) + + run_id: Mapped[str] = mapped_column(CHAR(26), primary_key=True) + schedule_id: Mapped[str] = mapped_column(CHAR(26), nullable=False) + workspace_id: Mapped[str] = mapped_column(CHAR(26), nullable=False) + workflow_version: Mapped[int] = mapped_column(INTEGER, nullable=False) + trigger_type: Mapped[str] = mapped_column( + String(16), nullable=False, comment="manual/cron/api/retry" + ) + idempotency_key: Mapped[str] = mapped_column(String(128), nullable=False) + run_status: Mapped[str] = mapped_column( + String(24), + nullable=False, + server_default=text("'queued'"), + comment="queued/running/succeeded/failed/cancelled/timed_out", + ) + state_version: Mapped[int] = mapped_column( + INTEGER, nullable=False, server_default=text("0"), comment="乐观锁版本" + ) + schedule_snapshot: Mapped[dict] = mapped_column( + JSON, nullable=False, comment="执行时 DAG 快照" + ) + queued_at: Mapped[datetime.datetime] = mapped_column( + DATETIME(fsp=3), nullable=False, server_default=text("CURRENT_TIMESTAMP(3)") + ) + created_at: Mapped[datetime.datetime] = mapped_column( + DATETIME(fsp=3), nullable=False, server_default=text("CURRENT_TIMESTAMP(3)") + ) + triggered_by: Mapped[str | None] = mapped_column(CHAR(26)) + started_at: Mapped[datetime.datetime | None] = mapped_column(DATETIME(fsp=3)) + finished_at: Mapped[datetime.datetime | None] = mapped_column(DATETIME(fsp=3)) + duration_ms: Mapped[int | None] = mapped_column(BIGINT) + error_code: Mapped[str | None] = mapped_column(String(64)) + error_message: Mapped[str | None] = mapped_column(Text) + logs_object_id: Mapped[str | None] = mapped_column(CHAR(26)) + result_object_id: Mapped[str | None] = mapped_column(CHAR(26)) + is_deleted: Mapped[int] = mapped_column( + TINYINT(1), nullable=False, server_default=text("0") + ) + deleted_at: Mapped[datetime.datetime | None] = mapped_column(DATETIME(fsp=3)) + + +class ScheduleNodes(Base): + __tablename__ = "schedule_nodes" + __table_args__ = ( + Index("idx_schedule_nodes_version", "versions_id"), + Index("uk_schedule_nodes_key", "schedule_id", "node_key", unique=True), + {"comment": "DAG 节点，必须引用稳定版本"}, + ) + + node_id: Mapped[str] = mapped_column(CHAR(26), primary_key=True) + schedule_id: Mapped[str] = mapped_column(CHAR(26), nullable=False) + node_key: Mapped[str] = mapped_column( + String(64), nullable=False, comment="画布内稳定标识" + ) + node_name: Mapped[str] = mapped_column(String(255), nullable=False) + versions_id: Mapped[str] = mapped_column(CHAR(26), nullable=False) + python_version: Mapped[str] = mapped_column( + String(8), nullable=False, server_default=text("'3.12'"), + comment="节点执行 Python 版本（3.8/3.10/3.12）", + ) + timeout_seconds: Mapped[int] = mapped_column( + INTEGER, nullable=False, server_default=text("600") + ) + retry_count: Mapped[int] = mapped_column( + INTEGER, nullable=False, server_default=text("0") + ) + retry_interval_sec: Mapped[int] = mapped_column( + INTEGER, nullable=False, server_default=text("5") + ) + position_x: Mapped[decimal.Decimal] = mapped_column( + DECIMAL(10, 2), nullable=False, server_default=text("0.00") + ) + position_y: Mapped[decimal.Decimal] = mapped_column( + DECIMAL(10, 2), nullable=False, server_default=text("0.00") + ) + created_at: Mapped[datetime.datetime] = mapped_column( + DATETIME(fsp=3), nullable=False, server_default=text("CURRENT_TIMESTAMP(3)") + ) + updated_at: Mapped[datetime.datetime] = mapped_column( + DATETIME(fsp=3), + nullable=False, + server_default=text("CURRENT_TIMESTAMP(3) ON UPDATE CURRENT_TIMESTAMP(3)"), + ) + arguments_json: Mapped[dict | None] = mapped_column(JSON) + env_refs_json: Mapped[dict | None] = mapped_column( + JSON, comment="只存密钥引用，不存明文密钥" + ) + is_deleted: Mapped[int] = mapped_column( + TINYINT(1), nullable=False, server_default=text("0") + ) + deleted_at: Mapped[datetime.datetime | None] = mapped_column(DATETIME(fsp=3)) + + +class ScheduleEdges(Base): + __tablename__ = "schedule_edges" + __table_args__ = ( + Index("fk_schedule_edges_source", "source_node_id"), + Index("idx_schedule_edges_target", "target_node_id"), + Index( + "uk_schedule_edges_pair", + "schedule_id", + "source_node_id", + "target_node_id", + unique=True, + ), + {"comment": "DAG 有向边"}, + ) + + edge_id: Mapped[str] = mapped_column(CHAR(26), primary_key=True) + schedule_id: Mapped[str] = mapped_column(CHAR(26), nullable=False) + source_node_id: Mapped[str] = mapped_column(CHAR(26), nullable=False) + target_node_id: Mapped[str] = mapped_column(CHAR(26), nullable=False) + created_at: Mapped[datetime.datetime] = mapped_column( + DATETIME(fsp=3), nullable=False, server_default=text("CURRENT_TIMESTAMP(3)") + ) + condition_expr: Mapped[str | None] = mapped_column(String(1000)) + is_deleted: Mapped[int] = mapped_column( + TINYINT(1), nullable=False, server_default=text("0") + ) + deleted_at: Mapped[datetime.datetime | None] = mapped_column(DATETIME(fsp=3)) + + +class ScheduleNodeRuns(Base): + __tablename__ = "schedule_node_runs" + __table_args__ = ( + Index("fk_node_runs_logs", "logs_object_id"), + Index("fk_node_runs_node", "node_id"), + Index("fk_node_runs_result", "result_object_id"), + Index("idx_node_runs_status", "run_id", "node_status"), + Index("idx_node_runs_version", "versions_id"), + Index( + "uk_node_runs_attempt", "run_id", "node_id", "attempt_no", unique=True + ), + {"comment": "调度节点运行与重试"}, + ) + + node_run_id: Mapped[str] = mapped_column(CHAR(26), primary_key=True) + run_id: Mapped[str] = mapped_column(CHAR(26), nullable=False) + node_id: Mapped[str] = mapped_column(CHAR(26), nullable=False) + versions_id: Mapped[str] = mapped_column(CHAR(26), nullable=False) + attempt_no: Mapped[int] = mapped_column( + INTEGER, nullable=False, server_default=text("1") + ) + node_status: Mapped[str] = mapped_column( + String(24), + nullable=False, + server_default=text("'queued'"), + comment="queued/running/succeeded/failed/skipped/cancelled/timed_out", + ) + state_version: Mapped[int] = mapped_column( + INTEGER, nullable=False, server_default=text("0"), comment="乐观锁版本" + ) + created_at: Mapped[datetime.datetime] = mapped_column( + DATETIME(fsp=3), nullable=False, server_default=text("CURRENT_TIMESTAMP(3)") + ) + started_at: Mapped[datetime.datetime | None] = mapped_column(DATETIME(fsp=3)) + finished_at: Mapped[datetime.datetime | None] = mapped_column(DATETIME(fsp=3)) + duration_ms: Mapped[int | None] = mapped_column(BIGINT) + exit_code: Mapped[int | None] = mapped_column(Integer) + message: Mapped[str | None] = mapped_column(String(2000)) + metrics_json: Mapped[dict | None] = mapped_column(JSON) + logs_object_id: Mapped[str | None] = mapped_column(CHAR(26)) + result_object_id: Mapped[str | None] = mapped_column(CHAR(26)) + is_deleted: Mapped[int] = mapped_column( + TINYINT(1), nullable=False, server_default=text("0") + ) + deleted_at: Mapped[datetime.datetime | None] = mapped_column(DATETIME(fsp=3)) diff --git a/common/src/common/db/models/scripts.py b/common/src/common/db/models/scripts.py new file mode 100644 index 0000000..06a7d31 --- /dev/null +++ b/common/src/common/db/models/scripts.py @@ -0,0 +1,107 @@ +import datetime + +from sqlalchemy import Index, String, text +from sqlalchemy.dialects.mysql import BIGINT, CHAR, DATETIME, INTEGER, TINYINT +from sqlalchemy.orm import Mapped, mapped_column + +from common.db.base import Base + + +class Scripts(Base): + __tablename__ = "scripts" + __table_args__ = ( + Index("idx_scripts_owner", "owner_user_id", "status"), + Index( + "idx_scripts_workspace", + "workspace_id", + "script_type", + "visibility", + "status", + ), + {"comment": "可执行 Python/Notebook 脚本"}, + ) + + script_id: Mapped[str] = mapped_column(CHAR(26), primary_key=True) + workspace_id: Mapped[str] = mapped_column(CHAR(26), nullable=False) + current_object_id: Mapped[str] = mapped_column( + CHAR(26), nullable=False, comment="当前工作副本" + ) + owner_user_id: Mapped[str] = mapped_column(CHAR(26), nullable=False) + script_name: Mapped[str] = mapped_column(String(255), nullable=False) + script_type: Mapped[str] = mapped_column( + String(16), nullable=False, comment="python/notebook" + ) + visibility: Mapped[str] = mapped_column( + String(16), nullable=False, server_default=text("'private'") + ) + status: Mapped[str] = mapped_column( + String(16), nullable=False, server_default=text("'active'") + ) + created_at: Mapped[datetime.datetime] = mapped_column( + DATETIME(fsp=3), nullable=False, server_default=text("CURRENT_TIMESTAMP(3)") + ) + updated_at: Mapped[datetime.datetime] = mapped_column( + DATETIME(fsp=3), + nullable=False, + server_default=text("CURRENT_TIMESTAMP(3) ON UPDATE CURRENT_TIMESTAMP(3)"), + ) + is_deleted: Mapped[int] = mapped_column( + TINYINT(1), nullable=False, server_default=text("0") + ) + is_locked: Mapped[int] = mapped_column( + TINYINT(1), nullable=False, server_default=text("1") + ) + deleted_at: Mapped[datetime.datetime | None] = mapped_column(DATETIME(fsp=3)) + + +class Versions(Base): + __tablename__ = "versions" + __table_args__ = ( + Index("fk_versions_source_object", "source_object_id"), + Index("idx_versions_creator", "created_by", "created_at"), + Index("idx_versions_workspace_created", "workspace_id", "created_at"), + Index("uk_versions_artifact", "artifact_object_id", unique=True), + Index("uk_versions_script_hash", "script_id", "content_hash", unique=True), + Index("uk_versions_script_no", "script_id", "version_no", unique=True), + {"comment": "不可变稳定版本；调度节点必须引用 versions_id"}, + ) + + versions_id: Mapped[str] = mapped_column( + CHAR(26), primary_key=True, comment="稳定版本唯一 ID" + ) + workspace_id: Mapped[str] = mapped_column(CHAR(26), nullable=False) + script_id: Mapped[str] = mapped_column(CHAR(26), nullable=False) + source_object_id: Mapped[str] = mapped_column( + CHAR(26), nullable=False, comment="发布时的源对象" + ) + artifact_object_id: Mapped[str] = mapped_column( + CHAR(26), nullable=False, comment="S3 不可变版本制品" + ) + version_no: Mapped[int] = mapped_column(INTEGER, nullable=False) + version_label: Mapped[str] = mapped_column( + String(32), nullable=False, comment="例如 v1.0" + ) + source_path: Mapped[str] = mapped_column( + String(1024), nullable=False, comment="发布时路径快照" + ) + artifact_path: Mapped[str] = mapped_column(String(1500), nullable=False) + content_hash: Mapped[str] = mapped_column(CHAR(64), nullable=False) + file_size_bytes: Mapped[int] = mapped_column( + BIGINT, nullable=False, server_default=text("0") + ) + visibility: Mapped[str] = mapped_column( + String(16), nullable=False, server_default=text("'private'") + ) + created_by: Mapped[str] = mapped_column(CHAR(26), nullable=False) + created_at: Mapped[datetime.datetime] = mapped_column( + DATETIME(fsp=3), nullable=False, server_default=text("CURRENT_TIMESTAMP(3)") + ) + release_note: Mapped[str | None] = mapped_column(String(1000)) + schedule_hidden_at: Mapped[datetime.datetime | None] = mapped_column( + DATETIME(fsp=3), + comment="从调度稳定版本列表移除的时间；不影响版本和运行历史", + ) + is_deleted: Mapped[int] = mapped_column( + TINYINT(1), nullable=False, server_default=text("0") + ) + deleted_at: Mapped[datetime.datetime | None] = mapped_column(DATETIME(fsp=3)) diff --git a/common/src/common/db/models/storage.py b/common/src/common/db/models/storage.py new file mode 100644 index 0000000..05bbc1c --- /dev/null +++ b/common/src/common/db/models/storage.py @@ -0,0 +1,238 @@ +import datetime + +from sqlalchemy import BINARY, JSON, Computed, Index, String, text +from sqlalchemy.dialects.mysql import BIGINT, CHAR, DATETIME, TINYINT +from sqlalchemy.orm import Mapped, mapped_column + +from common.db.base import Base + + +class StorageObjects(Base): + __tablename__ = "storage_objects" + # Force SQLAlchemy to round-trip server-default columns (created_at, + # updated_at, ...) via a follow-up SELECT after INSERT. Without this, + # attributes populated by `server_default` stay unloaded on the ORM + # object and the next sync read (e.g. storage_payload in storage_api.py) + # triggers a lazy refresh through the async driver, which raises + # MissingGreenlet because the read happens outside an awaitable. + __mapper_args__ = {"eager_defaults": "auto"} + __table_args__ = ( + Index("fk_storage_created_by", "created_by"), + Index("idx_storage_content_hash", "content_hash"), + Index("idx_storage_owner", "owner_user_id", "object_status"), + Index("idx_storage_parent", "parent_object_id"), + Index( + "idx_storage_workspace_path", + "workspace_id", + "storage_backend", + "path_hash", + ), + Index( + "idx_storage_workspace_usage", + "workspace_id", + "usage_type", + "object_status", + ), + Index( + "uk_storage_bucket_key_active", + "storage_backend", + "bucket_name", + "object_key_hash_active", + unique=True, + ), + {"comment": "Workspace 文件和 RustFS 对象的统一元数据"}, + ) + + storage_object_id: Mapped[str] = mapped_column(CHAR(26), primary_key=True) + workspace_id: Mapped[str] = mapped_column(CHAR(26), nullable=False) + object_type: Mapped[str] = mapped_column( + String(16), nullable=False, comment="file/directory" + ) + usage_type: Mapped[str] = mapped_column( + String(32), + nullable=False, + comment="working_copy/public_script/data_resource/version_artifact/snapshot/run_log/run_result", + ) + storage_backend: Mapped[str] = mapped_column( + String(16), nullable=False, comment="s3" + ) + storage_uri: Mapped[str] = mapped_column(String(1500), nullable=False) + file_name: Mapped[str] = mapped_column(String(255), nullable=False) + size_bytes: Mapped[int] = mapped_column( + BIGINT, nullable=False, server_default=text("0") + ) + visibility: Mapped[str] = mapped_column( + String(16), + nullable=False, + server_default=text("'private'"), + comment="private/workspace/public", + ) + is_immutable: Mapped[int] = mapped_column( + TINYINT(1), nullable=False, server_default=text("0") + ) + object_status: Mapped[str] = mapped_column( + String(24), + nullable=False, + server_default=text("'available'"), + comment="uploading/available/deleting/deleted/failed", + ) + created_by: Mapped[str] = mapped_column(CHAR(26), nullable=False) + created_at: Mapped[datetime.datetime] = mapped_column( + DATETIME(fsp=3), nullable=False, server_default=text("CURRENT_TIMESTAMP(3)") + ) + updated_at: Mapped[datetime.datetime] = mapped_column( + DATETIME(fsp=3), + nullable=False, + server_default=text("CURRENT_TIMESTAMP(3) ON UPDATE CURRENT_TIMESTAMP(3)"), + ) + owner_user_id: Mapped[str | None] = mapped_column(CHAR(26)) + parent_object_id: Mapped[str | None] = mapped_column(CHAR(26)) + relative_path: Mapped[str | None] = mapped_column( + String(1024), comment="Workspace 相对路径" + ) + path_hash: Mapped[bytes | None] = mapped_column( + BINARY(32), comment="SHA-256(relative_path)，由应用写入" + ) + bucket_name: Mapped[str | None] = mapped_column(String(128)) + object_key: Mapped[str | None] = mapped_column(String(1024)) + object_key_hash: Mapped[bytes | None] = mapped_column( + BINARY(32), comment="SHA-256(object_key)，由应用写入" + ) + object_key_hash_active: Mapped[bytes | None] = mapped_column( + BINARY(32), + Computed( + "CASE WHEN object_status = 'available' THEN object_key_hash ELSE NULL END", + persisted=False, + ), + comment="VIRTUAL generated column used by uk_storage_bucket_key_active", + ) + file_extension: Mapped[str | None] = mapped_column(String(32)) + mime_type: Mapped[str | None] = mapped_column(String(255)) + content_hash: Mapped[str | None] = mapped_column( + CHAR(64), comment="SHA-256 hex" + ) + object_etag: Mapped[str | None] = mapped_column(String(255)) + is_deleted: Mapped[int] = mapped_column( + TINYINT(1), nullable=False, server_default=text("0") + ) + deleted_at: Mapped[datetime.datetime | None] = mapped_column(DATETIME(fsp=3)) + trash_key: Mapped[str | None] = mapped_column( + String(1100), + comment=( + "Path inside the trash bucket where the soft-deleted bytes " + "live. Format: '{source_purpose}/{object_key}-{storage_object_id}' " + "(id 后缀防止同名文件多次删除互相覆盖); 恢复时去掉该 id 后缀 " + "拷回原 object_key。NULL while the row is still available." + ), + ) + + +class DataResources(Base): + __tablename__ = "data_resources" + __table_args__ = ( + Index("idx_data_resources_owner", "owner_user_id", "status"), + Index( + "idx_data_resources_workspace", + "workspace_id", + "visibility", + "status", + ), + {"comment": "数据资源"}, + ) + + resource_id: Mapped[str] = mapped_column(CHAR(26), primary_key=True) + workspace_id: Mapped[str] = mapped_column(CHAR(26), nullable=False) + storage_object_id: Mapped[str] = mapped_column(CHAR(26), nullable=False) + owner_user_id: Mapped[str] = mapped_column(CHAR(26), nullable=False) + resource_name: Mapped[str] = mapped_column(String(255), nullable=False) + visibility: Mapped[str] = mapped_column( + String(16), nullable=False, server_default=text("'private'") + ) + status: Mapped[str] = mapped_column( + String(16), nullable=False, server_default=text("'active'") + ) + created_at: Mapped[datetime.datetime] = mapped_column( + DATETIME(fsp=3), nullable=False, server_default=text("CURRENT_TIMESTAMP(3)") + ) + updated_at: Mapped[datetime.datetime] = mapped_column( + DATETIME(fsp=3), + nullable=False, + server_default=text("CURRENT_TIMESTAMP(3) ON UPDATE CURRENT_TIMESTAMP(3)"), + ) + description: Mapped[str | None] = mapped_column(String(1000)) + schema_json: Mapped[dict | None] = mapped_column( + JSON, comment="字段结构、行数等可选元数据" + ) + is_deleted: Mapped[int] = mapped_column( + TINYINT(1), nullable=False, server_default=text("0") + ) + deleted_at: Mapped[datetime.datetime | None] = mapped_column(DATETIME(fsp=3)) + + +class UploadSessions(Base): + __tablename__ = "upload_sessions" + __table_args__ = ( + Index("fk_upload_sessions_storage_object", "storage_object_id"), + Index("fk_upload_sessions_user", "user_id"), + Index("idx_upload_sessions_expiry", "upload_status", "expires_at"), + Index( + "idx_upload_sessions_object_key", "bucket_name", "object_key_hash" + ), + Index( + "idx_upload_sessions_workspace", "workspace_id", "user_id", "created_at" + ), + Index("uk_upload_sessions_idempotency", "idempotency_key", unique=True), + {"comment": "RustFS 预签名上传会话；URL 本身不持久化"}, + ) + + upload_id: Mapped[str] = mapped_column(CHAR(26), primary_key=True) + workspace_id: Mapped[str] = mapped_column(CHAR(26), nullable=False) + user_id: Mapped[str] = mapped_column(CHAR(26), nullable=False) + idempotency_key: Mapped[str] = mapped_column(String(128), nullable=False) + bucket_name: Mapped[str] = mapped_column(String(128), nullable=False) + object_key: Mapped[str] = mapped_column(String(1024), nullable=False) + object_key_hash: Mapped[bytes] = mapped_column(BINARY(32), nullable=False) + upload_status: Mapped[str] = mapped_column( + String(24), + nullable=False, + server_default=text("'created'"), + comment="created/uploading/completed/expired/aborted/failed", + ) + expires_at: Mapped[datetime.datetime] = mapped_column(DATETIME(fsp=3), nullable=False) + created_at: Mapped[datetime.datetime] = mapped_column( + DATETIME(fsp=3), nullable=False, server_default=text("CURRENT_TIMESTAMP(3)") + ) + updated_at: Mapped[datetime.datetime] = mapped_column( + DATETIME(fsp=3), + nullable=False, + server_default=text("CURRENT_TIMESTAMP(3) ON UPDATE CURRENT_TIMESTAMP(3)"), + ) + multipart_upload_id: Mapped[str | None] = mapped_column(String(255)) + expected_size_bytes: Mapped[int | None] = mapped_column(BIGINT) + expected_hash: Mapped[str | None] = mapped_column(CHAR(64)) + content_type: Mapped[str | None] = mapped_column(String(255)) + storage_object_id: Mapped[str | None] = mapped_column(CHAR(26)) + completed_at: Mapped[datetime.datetime | None] = mapped_column(DATETIME(fsp=3)) + # Metadata persisted at session creation so step 2 (PUT bytes) can build + # the StorageObjects row without re-sending them. Replaces the + # CompleteUploadRequest payload that lived between presign-PUT and head(). + file_name: Mapped[str] = mapped_column(String(255), nullable=False, server_default="") + usage_type: Mapped[str] = mapped_column( + String(32), + nullable=False, + server_default=text("'working_copy'"), + comment="data_resource/version_artifact/snapshot/run_log/run_result/working_copy/public_script", + ) + visibility: Mapped[str] = mapped_column( + String(16), + nullable=False, + server_default=text("'private'"), + comment="private/workspace/public", + ) + is_immutable: Mapped[int] = mapped_column( + TINYINT(1), nullable=False, server_default=text("0") + ) + is_deleted: Mapped[int] = mapped_column( + TINYINT(1), nullable=False, server_default=text("0") + ) + deleted_at: Mapped[datetime.datetime | None] = mapped_column(DATETIME(fsp=3)) diff --git a/common/src/common/db/models/workspaces.py b/common/src/common/db/models/workspaces.py new file mode 100644 index 0000000..f029f5d --- /dev/null +++ b/common/src/common/db/models/workspaces.py @@ -0,0 +1,84 @@ +import datetime + +from sqlalchemy import Index, String, text +from sqlalchemy.dialects.mysql import BIGINT, CHAR, DATETIME, TINYINT +from sqlalchemy.orm import Mapped, mapped_column + +from common.db.base import Base + + +class Workspaces(Base): + __tablename__ = "workspaces" + __table_args__ = ( + Index("fk_workspaces_created_by", "created_by"), + Index("idx_workspaces_status", "status"), + Index("uk_workspaces_code", "workspace_code", unique=True), + {"comment": "Workspace"}, + ) + + workspace_id: Mapped[str] = mapped_column(CHAR(26), primary_key=True) + workspace_code: Mapped[str] = mapped_column(String(64), nullable=False) + workspace_name: Mapped[str] = mapped_column(String(150), nullable=False) + active_root_uri: Mapped[str] = mapped_column( + String(1500), nullable=False, comment="活动工作区，建议 NFS/PVC/file URI" + ) + quota_bytes: Mapped[int] = mapped_column( + BIGINT, nullable=False, server_default=text("0"), comment="0 表示不限额" + ) + used_bytes: Mapped[int] = mapped_column( + BIGINT, nullable=False, server_default=text("0") + ) + status: Mapped[str] = mapped_column( + String(24), + nullable=False, + server_default=text("'active'"), + comment="creating/active/suspended/deleting/deleted", + ) + created_by: Mapped[str] = mapped_column(CHAR(26), nullable=False) + created_at: Mapped[datetime.datetime] = mapped_column( + DATETIME(fsp=3), nullable=False, server_default=text("CURRENT_TIMESTAMP(3)") + ) + updated_at: Mapped[datetime.datetime] = mapped_column( + DATETIME(fsp=3), + nullable=False, + server_default=text("CURRENT_TIMESTAMP(3) ON UPDATE CURRENT_TIMESTAMP(3)"), + ) + description: Mapped[str | None] = mapped_column(String(1000)) + artifact_bucket: Mapped[str | None] = mapped_column( + String(128), comment="S3 bucket" + ) + artifact_prefix: Mapped[str | None] = mapped_column( + String(512), comment="S3 object key prefix" + ) + is_deleted: Mapped[int] = mapped_column( + TINYINT(1), nullable=False, server_default=text("0") + ) + deleted_at: Mapped[datetime.datetime | None] = mapped_column(DATETIME(fsp=3)) + + +class WorkspaceMembers(Base): + __tablename__ = "workspace_members" + __table_args__ = ( + Index("idx_workspace_members_role", "role_id"), + Index("idx_workspace_members_user", "user_id", "member_status"), + {"comment": "Workspace 成员与角色"}, + ) + + workspace_id: Mapped[str] = mapped_column(CHAR(26), primary_key=True) + user_id: Mapped[str] = mapped_column(CHAR(26), primary_key=True) + role_id: Mapped[str] = mapped_column(CHAR(26), nullable=False) + member_status: Mapped[str] = mapped_column( + String(16), nullable=False, server_default=text("'active'") + ) + joined_at: Mapped[datetime.datetime] = mapped_column( + DATETIME(fsp=3), nullable=False, server_default=text("CURRENT_TIMESTAMP(3)") + ) + updated_at: Mapped[datetime.datetime] = mapped_column( + DATETIME(fsp=3), + nullable=False, + server_default=text("CURRENT_TIMESTAMP(3) ON UPDATE CURRENT_TIMESTAMP(3)"), + ) + is_deleted: Mapped[int] = mapped_column( + TINYINT(1), nullable=False, server_default=text("0") + ) + deleted_at: Mapped[datetime.datetime | None] = mapped_column(DATETIME(fsp=3)) diff --git a/common/src/common/db/session.py b/common/src/common/db/session.py index f0ff52a..0b9836d 100644 --- a/common/src/common/db/session.py +++ b/common/src/common/db/session.py @@ -1,5 +1,57 @@ -# coding=utf-8 -""" -@Time :2026/7/29 -@Author :tao.chen -""" +from __future__ import annotations + +from collections.abc import AsyncIterator +from contextlib import asynccontextmanager + +from sqlalchemy.ext.asyncio import ( + AsyncEngine, + AsyncSession, + async_sessionmaker, + create_async_engine, +) + +AsyncSessionFactory = async_sessionmaker[AsyncSession] + + +def create_database_engine( + database_url: str, + *, + echo: bool = False, + pool_pre_ping: bool = True, + pool_size: int = 10, + max_overflow: int = 20, + pool_recycle: int = 1800, +) -> AsyncEngine: + """Create an async SQLAlchemy engine without storing global connection state.""" + return create_async_engine( + database_url, + echo=echo, + pool_pre_ping=pool_pre_ping, + pool_size=pool_size, + max_overflow=max_overflow, + pool_recycle=pool_recycle, + ) + + +def create_session_factory(engine: AsyncEngine) -> AsyncSessionFactory: + """Create the shared async session factory used by FastAPI dependencies.""" + return async_sessionmaker( + bind=engine, + class_=AsyncSession, + expire_on_commit=False, + autoflush=False, + ) + + +@asynccontextmanager +async def session_scope( + factory: AsyncSessionFactory, +) -> AsyncIterator[AsyncSession]: + """Commit a unit of work or roll it back when an exception is raised.""" + async with factory() as session: + try: + yield session + await session.commit() + except Exception: + await session.rollback() + raise diff --git a/common/src/common/eventing.py b/common/src/common/eventing.py new file mode 100644 index 0000000..3003ec1 --- /dev/null +++ b/common/src/common/eventing.py @@ -0,0 +1,73 @@ +from __future__ import annotations + +from datetime import UTC, datetime +from typing import Any + +from sqlalchemy.ext.asyncio import AsyncSession + +from common.config import settings +from common.db.models import OutboxEvents +from common.ids import new_ulid + + +def utcnow() -> datetime: + return datetime.now(UTC).replace(tzinfo=None) + + +def event_time(value: datetime | None = None) -> str: + item = value or utcnow() + if item.tzinfo is None: + item = item.replace(tzinfo=UTC) + return item.astimezone(UTC).isoformat().replace("+00:00", "Z") + + +def schedule_event_type(event_type: str) -> str: + """Return the deployment-scoped schedule event type. + + Multiple development deployments currently share one MySQL database. + Scoping the event type prevents a scheduler from another deployment from + claiming and acknowledging work that only this deployment can execute. + """ + namespace = settings.schedule_event_namespace.strip().strip(".") + if not namespace: + raise ValueError("SCHEDULE_EVENT_NAMESPACE must not be empty") + return f"{namespace}.{event_type}" + + +async def add_outbox_event( + session: AsyncSession, + *, + event_type: str, + producer: str, + trace_id: str, + aggregate_type: str, + aggregate_id: str, + idempotency_key: str, + payload: dict[str, Any], + available_at: datetime | None = None, +) -> OutboxEvents: + event_id = new_ulid() + item = OutboxEvents( + event_id=event_id, + aggregate_type=aggregate_type, + aggregate_id=aggregate_id, + event_type=event_type, + schema_version=1, + payload_json=payload, + event_status="pending", + available_at=available_at or utcnow(), + retry_count=0, + trace_id=trace_id, + idempotency_key=idempotency_key, + ) + session.add(item) + return item + + +__all__ = [ + "add_outbox_event", + "event_time", + "schedule_event_type", + "utcnow", +] + diff --git a/common/src/common/ids.py b/common/src/common/ids.py new file mode 100644 index 0000000..f738c8a --- /dev/null +++ b/common/src/common/ids.py @@ -0,0 +1,17 @@ +from __future__ import annotations + +import secrets +import time + +_CROCKFORD32 = "0123456789ABCDEFGHJKMNPQRSTVWXYZ" + + +def new_ulid() -> str: + """Return a lexicographically sortable 26-character ULID.""" + timestamp_ms = int(time.time_ns() // 1_000_000) + value = (timestamp_ms << 80) | secrets.randbits(80) + encoded = ["0"] * 26 + for index in range(25, -1, -1): + encoded[index] = _CROCKFORD32[value & 31] + value >>= 5 + return "".join(encoded) diff --git a/common/src/common/logging.py b/common/src/common/logging.py new file mode 100644 index 0000000..2553a66 --- /dev/null +++ b/common/src/common/logging.py @@ -0,0 +1,39 @@ +"""Centralised loguru configuration. Call :func:`configure_logging` once per process, as early as possible. Idempotent.""" + +from __future__ import annotations + +import sys + +from loguru import logger + +_DEFAULT_FORMAT = ( + "{time:YYYY-MM-DD HH:mm:ss.SSS} " + "{level: <8} | " + "{name}:{function}:{line} - " + "{message}" +) +_ALLOWED_LEVELS = {"DEBUG", "INFO", "WARNING", "ERROR", "CRITICAL"} +_CONFIGURED: bool = False + + +def configure_logging(level: str = "INFO") -> str: + """Configure loguru's default stderr sink and return the normalised level.""" + global _CONFIGURED + if _CONFIGURED: + return level.upper() + + normalised = level.upper() if level.upper() in _ALLOWED_LEVELS else "INFO" + + logger.remove() + logger.add( + sys.stderr, + level=normalised, + format=_DEFAULT_FORMAT, + backtrace=True, + diagnose=False, + enqueue=False, + catch=True, + ) + + _CONFIGURED = True + return normalised diff --git a/common/src/common/migrations/README b/common/src/common/migrations/README deleted file mode 100644 index 98e4f9c..0000000 --- a/common/src/common/migrations/README +++ /dev/null @@ -1 +0,0 @@ -Generic single-database configuration. \ No newline at end of file diff --git a/common/src/common/migrations/env.py b/common/src/common/migrations/env.py deleted file mode 100644 index 3f8cba8..0000000 --- a/common/src/common/migrations/env.py +++ /dev/null @@ -1,78 +0,0 @@ -from logging.config import fileConfig - -from sqlalchemy import engine_from_config -from sqlalchemy import pool - -from alembic import context -from common.db.base import Base -# this is the Alembic Config object, which provides -# access to the values within the .ini file in use. -config = context.config - -# Interpret the config file for Python logging. -# This line sets up loggers basically. -if config.config_file_name is not None: - fileConfig(config.config_file_name) - -# add your model's MetaData object here -# for 'autogenerate' support -# from myapp import mymodel -# target_metadata = mymodel.Base.metadata -target_metadata = Base.metadata - -# other values from the config, defined by the needs of env.py, -# can be acquired: -# my_important_option = config.get_main_option("my_important_option") -# ... etc. - - -def run_migrations_offline() -> None: - """Run migrations in 'offline' mode. - - This configures the context with just a URL - and not an Engine, though an Engine is acceptable - here as well. By skipping the Engine creation - we don't even need a DBAPI to be available. - - Calls to context.execute() here emit the given string to the - script output. - - """ - url = config.get_main_option("sqlalchemy.url") - context.configure( - url=url, - target_metadata=target_metadata, - literal_binds=True, - dialect_opts={"paramstyle": "named"}, - ) - - with context.begin_transaction(): - context.run_migrations() - - -def run_migrations_online() -> None: - """Run migrations in 'online' mode. - - In this scenario we need to create an Engine - and associate a connection with the context. - - """ - connectable = engine_from_config( - config.get_section(config.config_ini_section, {}), - prefix="sqlalchemy.", - poolclass=pool.NullPool, - ) - - with connectable.connect() as connection: - context.configure( - connection=connection, target_metadata=target_metadata - ) - - with context.begin_transaction(): - context.run_migrations() - - -if context.is_offline_mode(): - run_migrations_offline() -else: - run_migrations_online() diff --git a/common/src/common/scheduler/__init__.py b/common/src/common/scheduler/__init__.py new file mode 100644 index 0000000..b6e4ad9 --- /dev/null +++ b/common/src/common/scheduler/__init__.py @@ -0,0 +1,80 @@ +"""Shared APScheduler jobstore helpers. + +The same ``apscheduler_jobs`` table is consumed by the FastAPI backend +(to enqueue cron jobs through ``add_job``) and by the schedule executor +(to run them). Both sides agree on the table name and the URL form +that :class:`SQLAlchemyJobStore` expects, so this module is the single +source of truth for that contract. + +``apscheduler`` is declared as an optional peer dependency: only the +schedule service imports it at runtime. This module therefore exposes +the table name and URL helpers but defers actual jobstore construction +to the caller. +""" + +from __future__ import annotations + +from typing import TYPE_CHECKING + +from common.scheduler.trigger import ( + SYSTEM_CRON_USER_ID, + DagTooLarge, + InvalidDag, + InvalidNodeArguments, + ScheduleNotFound, + TriggerError, + create_scheduled_run, + normalize_idempotency_key, + parse_node_arguments, +) + +if TYPE_CHECKING: # pragma: no cover - typing only + from apscheduler.jobstores.sqlalchemy import SQLAlchemyJobStore + + +JOBSTORE_TABLE = "apscheduler_jobs" + + +def to_sync_database_url(database_url: str) -> str: + """Rewrite ``mysql+asyncmy://`` to ``mysql+pymysql://``. + + APScheduler's ``SQLAlchemyJobStore`` uses a synchronous engine; the + project's default async URL needs to be downgraded before it can + drive the jobstore. + """ + return database_url.replace("mysql+asyncmy://", "mysql+pymysql://", 1) + + +def build_sqlalchemy_jobstore( + database_url: str, + *, + tablename: str = JOBSTORE_TABLE, +) -> SQLAlchemyJobStore: + """Instantiate a :class:`SQLAlchemyJobStore` for the canonical table. + + The caller is responsible for ensuring APScheduler and its sync + driver (``pymysql``) are installed; ``common`` does not depend on + either. + """ + from apscheduler.jobstores.sqlalchemy import SQLAlchemyJobStore + + return SQLAlchemyJobStore( + url=to_sync_database_url(database_url), + tablename=tablename, + ) + + +__all__ = [ + "JOBSTORE_TABLE", + "SYSTEM_CRON_USER_ID", + "DagTooLarge", + "InvalidDag", + "InvalidNodeArguments", + "ScheduleNotFound", + "TriggerError", + "build_sqlalchemy_jobstore", + "create_scheduled_run", + "normalize_idempotency_key", + "parse_node_arguments", + "to_sync_database_url", +] diff --git a/common/src/common/scheduler/trigger.py b/common/src/common/scheduler/trigger.py new file mode 100644 index 0000000..ce14826 --- /dev/null +++ b/common/src/common/scheduler/trigger.py @@ -0,0 +1,369 @@ +"""Shared schedule-trigger logic. + +Both the user-facing manual run endpoint (``backend.schedule_runs``) +and the schedule service's cron tick handler call into this module to +materialize a ``ScheduleRuns`` row plus the corresponding +``schedule.run.requested`` outbox event. The outbox is the single +source of truth for run dispatch — the schedule executor polls MySQL +and picks the row up. + +The schedule executor does NOT take any application-layer auth from +this codebase. Service-to-service calls on the shared Docker network +are intentionally unauthenticated; the ``triggered_by`` field stores +the user_id that originated the run (a human for manual runs, the +fixed ``_system_cron`` user for cron ticks) and the executor +re-verifies that user against ``Users.status='active'`` before doing +work. +""" + +from __future__ import annotations + +import hashlib +from typing import Any, Literal + +from sqlalchemy import select +from sqlalchemy.ext.asyncio import AsyncSession + +from common.db.models import ( + ScheduleEdges, + ScheduleNodes, + ScheduleRuns, + Schedules, + Scripts, + Versions, +) +from common.eventing import add_outbox_event, schedule_event_type, utcnow +from common.ids import new_ulid + +# Cron 运行需要一个可审计的 ``triggered_by``。这里使用基线迁移已经创建的 +# 管理员用户；该值必须是 26 个字符，才能写入 ``ScheduleRuns.triggered_by`` +# 的 ``CHAR(26)`` 字段。 +SYSTEM_CRON_USER_ID = "00000000000000000000000001" + + +TriggerType = Literal["manual", "cron", "api"] + + +class TriggerError(Exception): + """Raised when a run cannot be created. Subclasses carry the + appropriate HTTP status when surfaced from the backend router.""" + + +class ScheduleNotFound(TriggerError): + pass + + +class InvalidDag(TriggerError): + def __init__(self, errors: list[dict[str, Any]]) -> None: + super().__init__("schedule must contain a valid non-empty DAG") + self.errors = errors + + +class DagTooLarge(TriggerError): + pass + + +class InvalidNodeArguments(TriggerError): + def __init__(self, message: str) -> None: + super().__init__(message) + self.message = message + + +def normalize_idempotency_key( + workspace_id: str, + schedule_id: str, + value: str, + *, + min_length: int = 8, +) -> str: + """SHA-256 the (workspace, schedule, header) triple and prefix a + version tag. Centralized so the schedule service and the backend + router produce the same key. + """ + normalized = value.strip() + if len(normalized) < min_length: + raise TriggerError( + f"Idempotency-Key must contain at least {min_length} characters" + ) + digest = hashlib.sha256( + f"{workspace_id}:{schedule_id}:{normalized}".encode() + ).hexdigest() + return f"run:v1:{digest}" + + +def parse_node_arguments(value: dict[str, Any] | None) -> list[str]: + """Turn a node's ``arguments_json`` dict into a list of CLI args. + + Mirrors the backend's old ``_arguments`` helper but raises + :class:`InvalidNodeArguments` instead of an HTTP exception, so + the schedule service can use it without importing FastAPI. + """ + payload = value or {} + raw = payload.get("_args") + result: list[str] = [str(item) for item in raw] if isinstance(raw, list) else [] + for key, item in payload.items(): + if key == "_args": + continue + option = f"--{key.replace('_', '-')}" + if item is True: + result.append(option) + elif item is False or item is None: + continue + elif isinstance(item, list): + for list_item in item: + result.extend((option, str(list_item))) + elif isinstance(item, (str, int, float)): + result.extend((option, str(item))) + else: + raise InvalidNodeArguments( + f"node argument {key!r} must be a scalar or list" + ) + return result + + +async def _load_schedule( + session: AsyncSession, + schedule_id: str, + workspace_id: str, + *, + for_update: bool = False, +) -> Schedules: + statement = select(Schedules).where( + Schedules.schedule_id == schedule_id, + Schedules.workspace_id == workspace_id, + Schedules.deleted_at.is_(None), + ) + if for_update: + statement = statement.with_for_update() + item = await session.scalar(statement) + if item is None: + raise ScheduleNotFound("schedule not found") + return item + + +async def _load_graph( + session: AsyncSession, + schedule_id: str, +) -> tuple[ + list[tuple[ScheduleNodes, Versions, Scripts]], + list[ScheduleEdges], +]: + node_rows = ( + await session.execute( + select(ScheduleNodes, Versions, Scripts) + .join(Versions, Versions.versions_id == ScheduleNodes.versions_id) + .join(Scripts, Scripts.script_id == Versions.script_id) + .where(ScheduleNodes.schedule_id == schedule_id) + .order_by(ScheduleNodes.created_at, ScheduleNodes.node_key) + ) + ).all() + edges = list( + ( + await session.scalars( + select(ScheduleEdges) + .where(ScheduleEdges.schedule_id == schedule_id) + .order_by(ScheduleEdges.created_at, ScheduleEdges.edge_id) + ) + ).all() + ) + return list(node_rows), edges + + +def _validate_dag( + nodes: list[ScheduleNodes], + edges: list[ScheduleEdges], + *, + max_nodes: int = 100, + max_edges: int = 500, +) -> list[dict[str, Any]]: + errors: list[dict[str, Any]] = [] + if not nodes: + errors.append( + { + "code": "DAG_EMPTY", + "message": "schedule must contain at least one node", + } + ) + return errors + if len(nodes) > max_nodes or len(edges) > max_edges: + raise DagTooLarge( + f"schedule exceeds the v1 execution size limit " + f"({len(nodes)} nodes / {len(edges)} edges > {max_nodes}/{max_edges})" + ) + + # Cycle detection via Kahn's algorithm. + in_degree: dict[str, int] = {n.node_id: 0 for n in nodes} + adjacency: dict[str, list[str]] = {n.node_id: [] for n in nodes} + for edge in edges: + if edge.source_node_id not in in_degree or edge.target_node_id not in in_degree: + errors.append( + { + "code": "DAG_EDGE_REFERENCES_MISSING_NODE", + "message": f"edge {edge.edge_id} references unknown node", + "edge_id": edge.edge_id, + } + ) + continue + adjacency[edge.source_node_id].append(edge.target_node_id) + in_degree[edge.target_node_id] += 1 + queue = [nid for nid, d in in_degree.items() if d == 0] + ordered: list[str] = [] + while queue: + queue.sort() + current = queue.pop(0) + ordered.append(current) + for neighbor in adjacency[current]: + in_degree[neighbor] -= 1 + if in_degree[neighbor] == 0: + queue.append(neighbor) + if len(ordered) != len(nodes): + cycle_nodes = [nid for nid, d in in_degree.items() if d > 0] + errors.append( + { + "code": "DAG_CYCLE", + "message": "schedule graph contains a directed cycle", + "node_ids": cycle_nodes, + } + ) + return errors + + +def _build_snapshot( + schedule: Schedules, + node_rows: list[tuple[ScheduleNodes, Versions, Scripts]], + edges: list[ScheduleEdges], +) -> dict[str, Any]: + return { + "schedule_name": schedule.schedule_name, + "workflow_version": schedule.workflow_version, + "max_concurrency": schedule.max_concurrency, + "failure_policy": schedule.failure_policy, + "nodes": [ + { + "node_id": node.node_id, + "node_key": node.node_key, + "versions_id": version.versions_id, + "script_type": script.script_type, + "artifact_object_id": version.artifact_object_id, + "artifact_path": version.artifact_path, + "timeout_seconds": node.timeout_seconds, + "retry_count": node.retry_count, + "retry_interval_sec": node.retry_interval_sec, + "arguments": parse_node_arguments(node.arguments_json), + } + for node, version, script in node_rows + ], + "edges": [ + { + "source_node_id": edge.source_node_id, + "target_node_id": edge.target_node_id, + } + for edge in edges + ], + } + + +async def create_scheduled_run( + session: AsyncSession, + *, + schedule_id: str, + workspace_id: str, + triggered_by_user_id: str, + trigger_type: TriggerType, + idempotency_key: str, + trace_id: str | None = None, +) -> tuple[ScheduleRuns, bool]: + """Create a ``ScheduleRuns`` row + outbox event in this session. + + Returns ``(run, is_new)``. ``is_new=False`` means the + idempotency_key was already used and the existing run is returned + unchanged. ``triggered_by_user_id`` is stored as-is — for cron + triggers pass :data:`SYSTEM_CRON_USER_ID`. + + The caller owns the transaction: ``create_scheduled_run`` flushes + the new row to surface the unique-constraint violation on + ``idempotency_key`` deterministically, then leaves the commit to + the caller's session lifecycle. Both the backend's + ``request_context`` (which uses ``session_scope``) and the + schedule service's own session scope can wrap this call. + """ + # 1) Existing run short-circuit (re-using a known idempotency key). + existing = await session.scalar( + select(ScheduleRuns).where(ScheduleRuns.idempotency_key == idempotency_key) + ) + if existing is not None: + if ( + existing.workspace_id != workspace_id + or existing.schedule_id != schedule_id + ): + raise TriggerError("Idempotency-Key belongs to another schedule run") + return existing, False + + # 2) Lock + load schedule for the duration of this transaction. + schedule = await _load_schedule( + session, schedule_id, workspace_id, for_update=True, + ) + + # 3) Build snapshot (validates node arguments eagerly). + node_rows, edges = await _load_graph(session, schedule_id) + snapshot = _build_snapshot(schedule, node_rows, edges) + + # 4) Validate DAG after snapshot so parse_node_arguments errors + # surface first. + errors = _validate_dag( + [n for n, _v, _s in node_rows], edges, + ) + if errors: + raise InvalidDag(errors) + + # 5) Persist run + outbox. + now = utcnow() + run = ScheduleRuns( + run_id=new_ulid(), + schedule_id=schedule.schedule_id, + workspace_id=schedule.workspace_id, + workflow_version=schedule.workflow_version, + trigger_type=trigger_type, + idempotency_key=idempotency_key, + run_status="queued", + state_version=0, + schedule_snapshot=snapshot, + queued_at=now, + triggered_by=triggered_by_user_id, + ) + session.add(run) + schedule.last_run_at = now + + await add_outbox_event( + session, + event_type=schedule_event_type("schedule.run.requested"), + producer="platform-api", + trace_id=trace_id, + aggregate_type="schedule_run", + aggregate_id=run.run_id, + idempotency_key=idempotency_key, + payload={ + "workspace_id": run.workspace_id, + "schedule_id": run.schedule_id, + "run_id": run.run_id, + "workflow_version": run.workflow_version, + "trigger_type": run.trigger_type, + "triggered_by": run.triggered_by, + "schedule_snapshot": snapshot, + }, + ) + await session.flush() + return run, True + + +__all__ = [ + "SYSTEM_CRON_USER_ID", + "DagTooLarge", + "InvalidDag", + "InvalidNodeArguments", + "ScheduleNotFound", + "TriggerError", + "create_scheduled_run", + "normalize_idempotency_key", + "parse_node_arguments", +] diff --git a/common/src/common/schemas.py b/common/src/common/schemas.py new file mode 100644 index 0000000..b8c8553 --- /dev/null +++ b/common/src/common/schemas.py @@ -0,0 +1,14 @@ +"""Shared Pydantic base models for service-level request validation.""" + +from __future__ import annotations + +from pydantic import BaseModel, ConfigDict + + +class StrictModel(BaseModel): + """Base model that rejects unknown fields.""" + + model_config = ConfigDict(extra="forbid") + + +__all__ = ["StrictModel"] diff --git a/common/src/common/service_app.py b/common/src/common/service_app.py new file mode 100644 index 0000000..2317dcb --- /dev/null +++ b/common/src/common/service_app.py @@ -0,0 +1,81 @@ +from __future__ import annotations + +import asyncio +from collections.abc import Callable +from datetime import UTC, datetime +from typing import Any + +from fastapi import FastAPI, Response, status + +from common.config import settings + + +def _target_list() -> list[str]: + value = settings.readiness_targets + return [item.strip() for item in value.split(",") if item.strip()] + + +async def _check_tcp_target(target: str) -> dict[str, Any]: + host, port_text = target.rsplit(":", 1) + try: + reader, writer = await asyncio.wait_for( + asyncio.open_connection(host, int(port_text)), + timeout=1.5, + ) + writer.close() + await writer.wait_closed() + return {"target": target, "status": "ok"} + except (OSError, TimeoutError, ValueError) as exc: + return { + "target": target, + "status": "error", + "detail": type(exc).__name__, + } + + +def create_service_app( + service_name: str, + *, + lifespan: Callable[..., Any] | None = None, +) -> FastAPI: + app = FastAPI( + title=f"{service_name} service", + version="0.1.0", + docs_url="/docs", + redoc_url=None, + lifespan=lifespan, + ) + + def base_payload(state: str) -> dict[str, str]: + return { + "status": state, + "service": service_name, + "timestamp": datetime.now(UTC).isoformat(), + } + + @app.get("/") + async def root() -> dict[str, str]: + return base_payload("running") + + @app.get("/health/live") + async def live() -> dict[str, str]: + return base_payload("ok") + + @app.get("/api/v1/health") + async def public_health() -> dict[str, str]: + return base_payload("ok") + + @app.get("/health/ready") + async def ready(response: Response) -> dict[str, Any]: + checks = await asyncio.gather( + *(_check_tcp_target(target) for target in _target_list()) + ) + ready_state = all(check["status"] == "ok" for check in checks) + if not ready_state: + response.status_code = status.HTTP_503_SERVICE_UNAVAILABLE + return { + **base_payload("ready" if ready_state else "not_ready"), + "checks": checks, + } + + return app diff --git a/common/src/common/storage/__init__.py b/common/src/common/storage/__init__.py index f0ff52a..4a4906d 100644 --- a/common/src/common/storage/__init__.py +++ b/common/src/common/storage/__init__.py @@ -1,5 +1,50 @@ -# coding=utf-8 -""" -@Time :2026/7/29 -@Author :tao.chen +"""统一存储层，同时支持同步和异步，通过 config["mode"] 切换。 + +对上层暴露的公开 API： + + from storage import create_storage, StorageBackend, AsyncStorageBackend, ObjectMeta + from storage.exceptions import StorageError, StorageNotFoundError, ... + +用法： + # 同步（默认 mode="sync"） + storage = create_storage({"type": "local", "base_dir": "./data"}) + storage.put("a/b.txt", b"hello") + + # 异步：加一个 mode 字段 + storage = create_storage({"type": "local", "mode": "async", "base_dir": "./data"}) + await storage.put("a/b.txt", b"hello") + +切换本地/S3，或切换同步/异步，业务代码都不用改，只改配置： + storage = create_storage({"type": "s3", "mode": "async", "bucket": "my-bucket"}) """ + +from .base import AsyncStorageBackend, ObjectMeta, StorageBackend +from .factory import ( + PURPOSE_BUCKETS, + RCLONE_REMOTE_NAME, + USAGE_TYPE_TO_PURPOSE, + actual_bucket_name, + build_storage_config, + build_storage_uri, + create_storage, + rclone_remote_spec, + workspaces_root, +) +from .registry import register_backend, registered_backends + +__all__ = [ + "PURPOSE_BUCKETS", + "RCLONE_REMOTE_NAME", + "USAGE_TYPE_TO_PURPOSE", + "AsyncStorageBackend", + "ObjectMeta", + "StorageBackend", + "actual_bucket_name", + "build_storage_config", + "build_storage_uri", + "create_storage", + "rclone_remote_spec", + "register_backend", + "registered_backends", + "workspaces_root", +] diff --git a/common/src/common/storage/backends/__init__.py b/common/src/common/storage/backends/__init__.py new file mode 100644 index 0000000..97afac8 --- /dev/null +++ b/common/src/common/storage/backends/__init__.py @@ -0,0 +1,11 @@ +"""导入本模块即可触发所有内置后端的 @register_backend 注册。 + +新增内置后端时，在这里加一行 import 即可； +如果是第三方/业务自己的后端，不需要改这个文件， +只要在使用前 import 一次那个模块（让装饰器执行）就够了。 +""" + +from . import ( + local, # noqa: F401 + s3, # noqa: F401 +) diff --git a/common/src/common/storage/backends/local.py b/common/src/common/storage/backends/local.py new file mode 100644 index 0000000..9c2785e --- /dev/null +++ b/common/src/common/storage/backends/local.py @@ -0,0 +1,246 @@ +"""本地文件系统存储后端。 + +- 同步实现 `LocalStorageBackend`：标准库文件 I/O +- 异步实现 `LocalAsyncStorageBackend`：aiofiles 做实际读写， + stat/exists/delete/mkdir/目录遍历这类轻量元数据操作用 + asyncio.to_thread 包一层，避免阻塞事件循环 + （只有创建异步实例时才需要装 aiofiles，同步实现零依赖） +""" + +import asyncio +import os +import shutil +from collections.abc import AsyncIterator, Iterable +from datetime import timedelta +from pathlib import Path +from typing import BinaryIO + +from ..base import AsyncData, AsyncStorageBackend, ObjectMeta, StorageBackend, SyncData +from ..exceptions import StorageAlreadyExistsError, StorageNotFoundError +from ..registry import register_backend + + +def _resolve(base_dir: Path, key: str) -> Path: + key = key.strip("/") + path = (base_dir / key).resolve() + if base_dir not in path.parents and path != base_dir: + raise ValueError(f"非法 key，路径穿越到 base_dir 之外: {key!r}") + return path + + +def _meta(key: str, path: Path) -> ObjectMeta: + st = path.stat() + return ObjectMeta(key=key, size=st.st_size, last_modified=st.st_mtime) + + +# ==================== 同步实现 ==================== + + +@register_backend("local", mode="sync") +class LocalStorageBackend(StorageBackend): + """配置示例: {"type": "local", "mode": "sync", "base_dir": "/data/storage"}""" + + def __init__(self, base_dir: str, **_ignored): + self.base_dir = Path(base_dir).resolve() + self.base_dir.mkdir(parents=True, exist_ok=True) + + def _resolve(self, key: str) -> Path: + return _resolve(self.base_dir, key) + + def put( + self, + key: str, + data: SyncData, + *, + overwrite: bool = True, + content_type: str | None = None, + metadata: dict | None = None, + ) -> ObjectMeta: + path = self._resolve(key) + if path.exists() and not overwrite: + raise StorageAlreadyExistsError(f"key 已存在: {key}") + path.parent.mkdir(parents=True, exist_ok=True) + + if isinstance(data, bytes): + path.write_bytes(data) + else: + with open(path, "wb") as f: + shutil.copyfileobj(data, f) + # local FS 没有对象级 metadata；content_type / metadata 暂存忽略。 + return _meta(key, path) + + def get(self, key: str) -> bytes: + path = self._resolve(key) + if not path.is_file(): + raise StorageNotFoundError(f"key 不存在: {key}") + return path.read_bytes() + + def get_stream(self, key: str) -> BinaryIO: + path = self._resolve(key) + if not path.is_file(): + raise StorageNotFoundError(f"key 不存在: {key}") + return open(path, "rb") + + def delete(self, key: str) -> None: + try: + self._resolve(key).unlink() + except FileNotFoundError: + pass + + def exists(self, key: str) -> bool: + return self._resolve(key).is_file() + + def stat(self, key: str) -> ObjectMeta: + path = self._resolve(key) + if not path.is_file(): + raise StorageNotFoundError(f"key 不存在: {key}") + return _meta(key, path) + + def list(self, prefix: str = "") -> Iterable[ObjectMeta]: + search_root = self._resolve(prefix) if prefix else self.base_dir + if search_root.is_dir(): + candidates = search_root.rglob("*") + else: + candidates = search_root.parent.glob(f"{search_root.name}*") + + for path in candidates: + if path.is_file(): + key = str(path.relative_to(self.base_dir)).replace(os.sep, "/") + yield _meta(key, path) + + def get_url(self, key: str, *, expires_in: timedelta | None = None) -> str: + path = self._resolve(key) + if not path.is_file(): + raise StorageNotFoundError(f"key 不存在: {key}") + return path.as_uri() + + def copy(self, src_key: str, dst_key: str) -> ObjectMeta: + src_path = self._resolve(src_key) + if not src_path.is_file(): + raise StorageNotFoundError(f"key 不存在: {src_key}") + dst_path = self._resolve(dst_key) + dst_path.parent.mkdir(parents=True, exist_ok=True) + shutil.copy2(src_path, dst_path) + return _meta(dst_key, dst_path) + + +# ==================== 异步实现 ==================== + + +@register_backend("local", mode="async") +class LocalAsyncStorageBackend(AsyncStorageBackend): + """配置示例: {"type": "local", "mode": "async", "base_dir": "/data/storage"} + + 需要: pip install aiofiles + """ + + def __init__(self, base_dir: str, **_ignored): + self.base_dir = Path(base_dir).resolve() + self.base_dir.mkdir(parents=True, exist_ok=True) + + def _resolve(self, key: str) -> Path: + return _resolve(self.base_dir, key) + + async def put( + self, + key: str, + data: AsyncData, + *, + overwrite: bool = True, + content_type: str | None = None, + metadata: dict | None = None, + ) -> ObjectMeta: + import aiofiles + + path = self._resolve(key) + if not overwrite and await asyncio.to_thread(path.exists): + raise StorageAlreadyExistsError(f"key 已存在: {key}") + await asyncio.to_thread(path.parent.mkdir, parents=True, exist_ok=True) + + async with aiofiles.open(path, "wb") as f: + if isinstance(data, (bytes, bytearray)): + await f.write(data) + else: + async for chunk in data: + await f.write(chunk) + + # local FS 没有对象级 metadata；content_type / metadata 暂存忽略。 + return await asyncio.to_thread(_meta, key, path) + + async def get(self, key: str) -> bytes: + import aiofiles + + path = self._resolve(key) + if not await asyncio.to_thread(path.is_file): + raise StorageNotFoundError(f"key 不存在: {key}") + async with aiofiles.open(path, "rb") as f: + return await f.read() + + def get_stream(self, key: str, chunk_size: int = 64 * 1024) -> AsyncIterator[bytes]: + async def _iter(): + import aiofiles + + path = self._resolve(key) + if not await asyncio.to_thread(path.is_file): + raise StorageNotFoundError(f"key 不存在: {key}") + async with aiofiles.open(path, "rb") as f: + while True: + chunk = await f.read(chunk_size) + if not chunk: + break + yield chunk + + return _iter() + + async def delete(self, key: str) -> None: + path = self._resolve(key) + + def _unlink(): + try: + path.unlink() + except FileNotFoundError: + pass + + await asyncio.to_thread(_unlink) + + async def exists(self, key: str) -> bool: + return await asyncio.to_thread(self._resolve(key).is_file) + + async def stat(self, key: str) -> ObjectMeta: + path = self._resolve(key) + if not await asyncio.to_thread(path.is_file): + raise StorageNotFoundError(f"key 不存在: {key}") + return await asyncio.to_thread(_meta, key, path) + + def list(self, prefix: str = "") -> AsyncIterator[ObjectMeta]: + async def _iter(): + search_root = self._resolve(prefix) if prefix else self.base_dir + + def _collect(): + if search_root.is_dir(): + candidates = list(search_root.rglob("*")) + else: + candidates = list(search_root.parent.glob(f"{search_root.name}*")) + return [p for p in candidates if p.is_file()] + + files = await asyncio.to_thread(_collect) + for path in files: + key = str(path.relative_to(self.base_dir)).replace(os.sep, "/") + yield await asyncio.to_thread(_meta, key, path) + + return _iter() + + async def get_url(self, key: str, *, expires_in: timedelta | None = None) -> str: + path = self._resolve(key) + if not await asyncio.to_thread(path.is_file): + raise StorageNotFoundError(f"key 不存在: {key}") + return path.as_uri() + + async def copy(self, src_key: str, dst_key: str) -> ObjectMeta: + src_path = self._resolve(src_key) + if not await asyncio.to_thread(src_path.is_file): + raise StorageNotFoundError(f"key 不存在: {src_key}") + dst_path = self._resolve(dst_key) + await asyncio.to_thread(dst_path.parent.mkdir, parents=True, exist_ok=True) + await asyncio.to_thread(shutil.copy2, src_path, dst_path) + return await asyncio.to_thread(_meta, dst_key, dst_path) diff --git a/common/src/common/storage/backends/s3.py b/common/src/common/storage/backends/s3.py new file mode 100644 index 0000000..6f067a2 --- /dev/null +++ b/common/src/common/storage/backends/s3.py @@ -0,0 +1,421 @@ +"""S3（及兼容协议）存储后端。 + +- 同步实现 `S3StorageBackend`：boto3 +- 异步实现 `S3AsyncStorageBackend`：aioboto3 + +两者只在各自 __init__ 里做 lazy import，互不强制依赖： +只用同步模式不需要装 aioboto3，只用异步模式不需要额外装 boto3 +（aioboto3 本身依赖 botocore，异常类型从它里面拿）。 +""" + +from collections.abc import AsyncIterator, Iterable +from datetime import timedelta +from typing import BinaryIO + +from ..base import AsyncData, AsyncStorageBackend, ObjectMeta, StorageBackend, SyncData +from ..exceptions import ( + StorageAlreadyExistsError, + StorageConnectionError, + StorageNotFoundError, +) +from ..registry import register_backend + + +def _meta_from_head(key: str, head: dict) -> ObjectMeta: + return ObjectMeta( + key=key, + size=head.get("ContentLength", 0), + last_modified=head["LastModified"].timestamp() if head.get("LastModified") else None, + etag=head.get("ETag"), + ) + + +# ==================== 同步实现 ==================== + + +@register_backend("s3", mode="sync") +class S3StorageBackend(StorageBackend): + """配置示例: + { + "type": "s3", "mode": "sync", + "bucket": "my-bucket", "prefix": "app1/", + "region_name": "cn-north-1", "endpoint_url": "https://s3.example.com", + "aws_access_key_id": "...", "aws_secret_access_key": "...", + } + + 需要: pip install boto3 + """ + + def __init__( + self, + bucket: str, + prefix: str = "", + region_name: str | None = None, + endpoint_url: str | None = None, + aws_access_key_id: str | None = None, + aws_secret_access_key: str | None = None, + **_ignored, + ): + try: + import boto3 + from botocore.exceptions import BotoCoreError, ClientError + except ImportError as e: + raise ImportError("使用同步 S3 存储后端需要先安装 boto3: pip install boto3") from e + + self._ClientError = ClientError + self._BotoCoreError = BotoCoreError + self.bucket = bucket + self.prefix = prefix.strip("/") + "/" if prefix.strip("/") else "" + + try: + self.client = boto3.client( + "s3", + region_name=region_name, + endpoint_url=endpoint_url, + aws_access_key_id=aws_access_key_id, + aws_secret_access_key=aws_secret_access_key, + ) + except (BotoCoreError, ClientError) as e: + raise StorageConnectionError(f"初始化 S3 client 失败: {e}") from e + + def _full_key(self, key: str) -> str: + return f"{self.prefix}{key.lstrip('/')}" + + def put(self, key: str, data: SyncData, *, overwrite: bool = True) -> ObjectMeta: + full_key = self._full_key(key) + if not overwrite and self.exists(key): + raise StorageAlreadyExistsError(f"key 已存在: {key}") + body = data if isinstance(data, bytes) else data.read() + try: + self.client.put_object(Bucket=self.bucket, Key=full_key, Body=body) + except (self._ClientError, self._BotoCoreError) as e: + raise StorageConnectionError(f"上传失败 key={key}: {e}") from e + return self.stat(key) + + def get(self, key: str) -> bytes: + full_key = self._full_key(key) + try: + resp = self.client.get_object(Bucket=self.bucket, Key=full_key) + return resp["Body"].read() + except self._ClientError as e: + if e.response.get("Error", {}).get("Code") in ("NoSuchKey", "404"): + raise StorageNotFoundError(f"key 不存在: {key}") from e + raise StorageConnectionError(f"读取失败 key={key}: {e}") from e + + def get_stream(self, key: str) -> BinaryIO: + full_key = self._full_key(key) + try: + resp = self.client.get_object(Bucket=self.bucket, Key=full_key) + return resp["Body"] + except self._ClientError as e: + if e.response.get("Error", {}).get("Code") in ("NoSuchKey", "404"): + raise StorageNotFoundError(f"key 不存在: {key}") from e + raise StorageConnectionError(f"读取失败 key={key}: {e}") from e + + def delete(self, key: str) -> None: + try: + self.client.delete_object(Bucket=self.bucket, Key=self._full_key(key)) + except (self._ClientError, self._BotoCoreError) as e: + raise StorageConnectionError(f"删除失败 key={key}: {e}") from e + + def exists(self, key: str) -> bool: + try: + self.client.head_object(Bucket=self.bucket, Key=self._full_key(key)) + return True + except self._ClientError as e: + if e.response.get("Error", {}).get("Code") in ("404", "NoSuchKey"): + return False + raise StorageConnectionError(f"检查 exists 失败 key={key}: {e}") from e + + def stat(self, key: str) -> ObjectMeta: + try: + head = self.client.head_object(Bucket=self.bucket, Key=self._full_key(key)) + except self._ClientError as e: + if e.response.get("Error", {}).get("Code") in ("404", "NoSuchKey"): + raise StorageNotFoundError(f"key 不存在: {key}") from e + raise StorageConnectionError(f"获取元信息失败 key={key}: {e}") from e + return _meta_from_head(key, head) + + def list(self, prefix: str = "") -> Iterable[ObjectMeta]: + full_prefix = self._full_key(prefix) + paginator = self.client.get_paginator("list_objects_v2") + try: + for page in paginator.paginate(Bucket=self.bucket, Prefix=full_prefix): + for obj in page.get("Contents", []): + key = obj["Key"][len(self.prefix):] if self.prefix else obj["Key"] + yield ObjectMeta( + key=key, + size=obj["Size"], + last_modified=obj["LastModified"].timestamp(), + etag=obj.get("ETag"), + ) + except (self._ClientError, self._BotoCoreError) as e: + raise StorageConnectionError(f"列举对象失败 prefix={prefix}: {e}") from e + + def get_url(self, key: str, *, expires_in: timedelta | None = None) -> str: + expires_seconds = int(expires_in.total_seconds()) if expires_in else 3600 + try: + return self.client.generate_presigned_url( + "get_object", + Params={"Bucket": self.bucket, "Key": self._full_key(key)}, + ExpiresIn=expires_seconds, + ) + except (self._ClientError, self._BotoCoreError) as e: + raise StorageConnectionError(f"生成预签名 URL 失败 key={key}: {e}") from e + + def copy(self, src_key: str, dst_key: str) -> ObjectMeta: + try: + self.client.copy_object( + Bucket=self.bucket, + Key=self._full_key(dst_key), + CopySource={"Bucket": self.bucket, "Key": self._full_key(src_key)}, + ) + except self._ClientError as e: + if e.response.get("Error", {}).get("Code") in ("404", "NoSuchKey"): + raise StorageNotFoundError(f"key 不存在: {src_key}") from e + raise StorageConnectionError(f"复制失败 {src_key} -> {dst_key}: {e}") from e + return self.stat(dst_key) + + +# ==================== 异步实现 ==================== + + +@register_backend("s3", mode="async") +class S3AsyncStorageBackend(AsyncStorageBackend): + """配置示例同上，把 "mode" 改成 "async" 即可。 + + 需要: pip install aioboto3 + + 每次操作默认通过 `async with session.client(...)` 拿一个短生命周期 + client；用 `async with create_storage(...) as storage:` 可以复用同一个 + client（见 __aenter__/__aexit__）。 + """ + + def __init__( + self, + bucket: str, + prefix: str = "", + region_name: str | None = None, + endpoint_url: str | None = None, + aws_access_key_id: str | None = None, + aws_secret_access_key: str | None = None, + **_ignored, + ): + try: + import aioboto3 + from botocore.exceptions import BotoCoreError, ClientError + except ImportError as e: + raise ImportError( + "使用异步 S3 存储后端需要先安装 aioboto3: pip install aioboto3" + ) from e + + self._ClientError = ClientError + self._BotoCoreError = BotoCoreError + self.bucket = bucket + self.prefix = prefix.strip("/") + "/" if prefix.strip("/") else "" + self._client_kwargs = dict( + region_name=region_name, + endpoint_url=endpoint_url, + aws_access_key_id=aws_access_key_id, + aws_secret_access_key=aws_secret_access_key, + ) + self._session = aioboto3.Session() + self._persistent_client = None + self._persistent_cm = None + + def _client_cm(self): + return self._session.client("s3", **self._client_kwargs) + + async def __aenter__(self) -> "S3AsyncStorageBackend": + self._persistent_cm = self._client_cm() + self._persistent_client = await self._persistent_cm.__aenter__() + return self + + async def __aexit__(self, exc_type, exc, tb) -> None: + if self._persistent_cm is not None: + await self._persistent_cm.__aexit__(exc_type, exc, tb) + self._persistent_cm = None + self._persistent_client = None + + async def aclose(self) -> None: + await self.__aexit__(None, None, None) + + def _full_key(self, key: str) -> str: + return f"{self.prefix}{key.lstrip('/')}" + + async def _run(self, coro_fn): + if self._persistent_client is not None: + return await coro_fn(self._persistent_client) + async with self._client_cm() as client: + return await coro_fn(client) + + async def put( + self, + key: str, + data: AsyncData, + *, + overwrite: bool = True, + content_type: str | None = None, + metadata: dict | None = None, + ) -> ObjectMeta: + full_key = self._full_key(key) + if not overwrite and await self.exists(key): + raise StorageAlreadyExistsError(f"key 已存在: {key}") + + if isinstance(data, (bytes, bytearray)): + body = bytes(data) + else: + chunks = [] + async for chunk in data: + chunks.append(chunk) + body = b"".join(chunks) + + # 过滤掉空 dict / None，避免 boto3 报 "parameter must be a non-empty + # non-null dictionary of strings" 这种空请求参数错误。 + meta = {k: str(v) for k, v in (metadata or {}).items() if v is not None} or None + + async def _op(client): + kwargs = {"Bucket": self.bucket, "Key": full_key, "Body": body} + if content_type: + kwargs["ContentType"] = content_type + if meta: + kwargs["Metadata"] = meta + await client.put_object(**kwargs) + + try: + await self._run(_op) + except (self._ClientError, self._BotoCoreError) as e: + raise StorageConnectionError(f"上传失败 key={key}: {e}") from e + return await self.stat(key) + + async def get(self, key: str) -> bytes: + full_key = self._full_key(key) + + async def _op(client): + resp = await client.get_object(Bucket=self.bucket, Key=full_key) + async with resp["Body"] as stream: + return await stream.read() + + try: + return await self._run(_op) + except self._ClientError as e: + if e.response.get("Error", {}).get("Code") in ("NoSuchKey", "404"): + raise StorageNotFoundError(f"key 不存在: {key}") from e + raise StorageConnectionError(f"读取失败 key={key}: {e}") from e + + def get_stream(self, key: str, chunk_size: int = 64 * 1024) -> AsyncIterator[bytes]: + async def _iter(): + full_key = self._full_key(key) + + async def _op(client): + return await client.get_object(Bucket=self.bucket, Key=full_key) + + try: + resp = await self._run(_op) + except self._ClientError as e: + if e.response.get("Error", {}).get("Code") in ("NoSuchKey", "404"): + raise StorageNotFoundError(f"key 不存在: {key}") from e + raise StorageConnectionError(f"读取失败 key={key}: {e}") from e + + async with resp["Body"] as stream: + while True: + chunk = await stream.read(chunk_size) + if not chunk: + break + yield chunk + + return _iter() + + async def delete(self, key: str) -> None: + async def _op(client): + await client.delete_object(Bucket=self.bucket, Key=self._full_key(key)) + + try: + await self._run(_op) + except (self._ClientError, self._BotoCoreError) as e: + raise StorageConnectionError(f"删除失败 key={key}: {e}") from e + + async def exists(self, key: str) -> bool: + async def _op(client): + await client.head_object(Bucket=self.bucket, Key=self._full_key(key)) + + try: + await self._run(_op) + return True + except self._ClientError as e: + if e.response.get("Error", {}).get("Code") in ("404", "NoSuchKey"): + return False + raise StorageConnectionError(f"检查 exists 失败 key={key}: {e}") from e + + async def stat(self, key: str) -> ObjectMeta: + async def _op(client): + return await client.head_object(Bucket=self.bucket, Key=self._full_key(key)) + + try: + head = await self._run(_op) + except self._ClientError as e: + if e.response.get("Error", {}).get("Code") in ("404", "NoSuchKey"): + raise StorageNotFoundError(f"key 不存在: {key}") from e + raise StorageConnectionError(f"获取元信息失败 key={key}: {e}") from e + return _meta_from_head(key, head) + + def list(self, prefix: str = "") -> AsyncIterator[ObjectMeta]: + async def _iter(): + full_prefix = self._full_key(prefix) + + async def _paginate(client): + paginator = client.get_paginator("list_objects_v2") + results = [] + async for page in paginator.paginate(Bucket=self.bucket, Prefix=full_prefix): + for obj in page.get("Contents", []): + key = obj["Key"][len(self.prefix):] if self.prefix else obj["Key"] + results.append( + ObjectMeta( + key=key, + size=obj["Size"], + last_modified=obj["LastModified"].timestamp(), + etag=obj.get("ETag"), + ) + ) + return results + + try: + metas = await self._run(_paginate) + except (self._ClientError, self._BotoCoreError) as e: + raise StorageConnectionError(f"列举对象失败 prefix={prefix}: {e}") from e + + for meta in metas: + yield meta + + return _iter() + + async def get_url(self, key: str, *, expires_in: timedelta | None = None) -> str: + expires_seconds = int(expires_in.total_seconds()) if expires_in else 3600 + + async def _op(client): + return await client.generate_presigned_url( + "get_object", + Params={"Bucket": self.bucket, "Key": self._full_key(key)}, + ExpiresIn=expires_seconds, + ) + + try: + return await self._run(_op) + except (self._ClientError, self._BotoCoreError) as e: + raise StorageConnectionError(f"生成预签名 URL 失败 key={key}: {e}") from e + + async def copy(self, src_key: str, dst_key: str) -> ObjectMeta: + async def _op(client): + await client.copy_object( + Bucket=self.bucket, + Key=self._full_key(dst_key), + CopySource={"Bucket": self.bucket, "Key": self._full_key(src_key)}, + ) + + try: + await self._run(_op) + except self._ClientError as e: + if e.response.get("Error", {}).get("Code") in ("404", "NoSuchKey"): + raise StorageNotFoundError(f"key 不存在: {src_key}") from e + raise StorageConnectionError(f"复制失败 {src_key} -> {dst_key}: {e}") from e + return await self.stat(dst_key) diff --git a/common/src/common/storage/base.py b/common/src/common/storage/base.py new file mode 100644 index 0000000..30b40d3 --- /dev/null +++ b/common/src/common/storage/base.py @@ -0,0 +1,156 @@ +"""同步 / 异步存储后端统一抽象接口。 + +`StorageBackend` 是同步接口，`AsyncStorageBackend` 是异步接口， +两者共用同一个 `ObjectMeta` 数据结构，方法签名尽量保持对称 +（异步版本每个方法多一个 await，get_stream/list 变成异步生成器）， +这样业务代码从同步切到异步时心智负担最小。 + +上层通过 `storage.create_storage(config)` 统一创建实例， +用 `config["mode"]` 决定拿到的是同步实现还是异步实现。 +""" + +from abc import ABC, abstractmethod +from collections.abc import AsyncIterator, Iterable +from dataclasses import dataclass, field +from datetime import timedelta +from typing import BinaryIO, Union + +SyncData = Union[bytes, BinaryIO] +AsyncData = Union[bytes, "AsyncIterator[bytes]"] + + +@dataclass +class ObjectMeta: + """list/stat 等操作返回的对象元信息，做了跨后端的字段归一化。""" + + key: str + size: int + last_modified: float | None = None # unix timestamp + etag: str | None = None + extra: dict = field(default_factory=dict) # 后端特有的额外信息 + + +class StorageBackend(ABC): + """同步存储后端统一抽象基类。""" + + @abstractmethod + def put( + self, + key: str, + data: SyncData, + *, + overwrite: bool = True, + content_type: str | None = None, + metadata: dict | None = None, + ) -> ObjectMeta: + """写入对象。overwrite=False 时 key 已存在应抛出 StorageAlreadyExistsError。 + + ``content_type`` 和 ``metadata`` 是可选的（与异步 put 语义一致）。 + """ + + @abstractmethod + def get(self, key: str) -> bytes: + """读取对象内容，不存在时抛出 StorageNotFoundError。""" + + @abstractmethod + def get_stream(self, key: str) -> BinaryIO: + """以流方式读取对象，适合大文件。""" + + @abstractmethod + def delete(self, key: str) -> None: + """删除对象。删除不存在的 key 不应报错（幂等）。""" + + @abstractmethod + def exists(self, key: str) -> bool: + ... + + @abstractmethod + def stat(self, key: str) -> ObjectMeta: + """不存在时抛出 StorageNotFoundError。""" + + @abstractmethod + def list(self, prefix: str = "") -> Iterable[ObjectMeta]: + """按前缀列出对象。""" + + @abstractmethod + def get_url(self, key: str, *, expires_in: timedelta | None = None) -> str: + """获取可访问 URL；本地存储返回 file://，S3 返回预签名 URL。""" + + def copy(self, src_key: str, dst_key: str) -> ObjectMeta: + """默认实现：读出来再写进去。后端可覆盖为更高效的原生实现。""" + data = self.get(src_key) + return self.put(dst_key, data) + + def close(self) -> None: + """释放后端持有的资源（连接池等）。不需要的后端可以不覆盖。""" + return + + def __enter__(self) -> "StorageBackend": + return self + + def __exit__(self, exc_type, exc, tb) -> None: + self.close() + + +class AsyncStorageBackend(ABC): + """异步存储后端统一抽象基类。""" + + @abstractmethod + async def put( + self, + key: str, + data: AsyncData, + *, + overwrite: bool = True, + content_type: str | None = None, + metadata: dict | None = None, + ) -> ObjectMeta: + """data 可以是 bytes，也可以是异步字节流（async generator）。 + + ``content_type`` 和 ``metadata`` 是可选的：S3 后端会把它们分别透传 + 成 ``ContentType`` 请求头和 ``Metadata`` dict；local 后端目前忽略 + 这两个参数（本地 FS 没有对象级 metadata）。 + """ + + @abstractmethod + async def get(self, key: str) -> bytes: + """不存在时抛出 StorageNotFoundError。""" + + @abstractmethod + def get_stream(self, key: str, chunk_size: int = 64 * 1024) -> AsyncIterator[bytes]: + """异步分块读取，用法: `async for chunk in backend.get_stream(key):`。 + 普通方法（非 async def），返回值本身就是异步生成器。 + """ + + @abstractmethod + async def delete(self, key: str) -> None: + """幂等：删除不存在的 key 不应报错。""" + + @abstractmethod + async def exists(self, key: str) -> bool: + ... + + @abstractmethod + async def stat(self, key: str) -> ObjectMeta: + """不存在时抛出 StorageNotFoundError。""" + + @abstractmethod + def list(self, prefix: str = "") -> AsyncIterator[ObjectMeta]: + """用法: `async for meta in backend.list(prefix):`。""" + + @abstractmethod + async def get_url(self, key: str, *, expires_in: timedelta | None = None) -> str: + ... + + async def copy(self, src_key: str, dst_key: str) -> ObjectMeta: + data = await self.get(src_key) + return await self.put(dst_key, data) + + async def aclose(self) -> None: + return None + + async def __aenter__(self) -> "AsyncStorageBackend": + return self + + async def __aexit__(self, exc_type, exc, tb) -> None: + await self.aclose() diff --git a/common/src/common/storage/example_usage.py b/common/src/common/storage/example_usage.py new file mode 100644 index 0000000..4c2d06a --- /dev/null +++ b/common/src/common/storage/example_usage.py @@ -0,0 +1,115 @@ +"""使用示例：同一套 create_storage()，靠 config["mode"] 切换同步/异步。""" + +import asyncio + +from common.storage import create_storage +from common.storage.exceptions import StorageNotFoundError + + +def sync_demo(): + # mode 默认就是 "sync"，可以不写 + storage = create_storage({"type": "local", "base_dir": "./data_sync"}) + + storage.put("docs/hello.txt", b"hello world") + print(storage.get("docs/hello.txt")) + print(storage.exists("docs/hello.txt")) + print(list(storage.list("docs/"))) + print(storage.get_url("docs/hello.txt")) + + try: + storage.get("docs/not_exist.txt") + except StorageNotFoundError: + print("按预期抛出 StorageNotFoundError") + + # 换成同步 S3，只改配置： + # storage = create_storage({"type": "s3", "bucket": "my-bucket"}) + + +async def async_demo(): + # 只加一个 "mode": "async"，其余配置和参数不变 + storage = create_storage({"type": "local", "mode": "async", "base_dir": "./data_async"}) + + await storage.put("docs/hello.txt", b"hello world") + print(await storage.get("docs/hello.txt")) + print(await storage.exists("docs/hello.txt")) + + async for meta in storage.list("docs/"): + print(meta) + + chunks = [] + async for chunk in storage.get_stream("docs/hello.txt"): + chunks.append(chunk) + print(b"".join(chunks)) + + try: + await storage.get("docs/not_exist.txt") + except StorageNotFoundError: + print("按预期抛出 StorageNotFoundError") + + # 并发写入，异步模式的典型优势场景 + tasks = [storage.put(f"batch/{i}.txt", f"content-{i}".encode()) for i in range(10)] + await asyncio.gather(*tasks) + print("并发写入 10 个对象完成") + + # 换成异步 S3，只改配置： + # storage = create_storage({"type": "s3", "mode": "async", "bucket": "my-bucket"}) + # 高吞吐场景复用连接： + # async with create_storage({"type": "s3", "mode": "async", "bucket": "my-bucket"}) as s3: + # await s3.put("a.txt", b"1") + + +def extend_with_new_backend_demo(): + """演示独立扩展一种新的存储方式（同步+异步各一个），不用改现有代码。""" + import io + import time + + from common.storage.base import ObjectMeta, StorageBackend + from common.storage.exceptions import StorageNotFoundError + from common.storage.registry import register_backend + + @register_backend("memory", mode="sync") + class MemoryStorageBackend(StorageBackend): + def __init__(self, **_ignored): + self._store = {} + + def put(self, key, data, *, overwrite=True): + body = data if isinstance(data, bytes) else data.read() + self._store[key] = body + return ObjectMeta(key=key, size=len(body), last_modified=time.time()) + + def get(self, key): + if key not in self._store: + raise StorageNotFoundError(key) + return self._store[key] + + def get_stream(self, key): + return io.BytesIO(self.get(key)) + + def delete(self, key): + self._store.pop(key, None) + + def exists(self, key): + return key in self._store + + def stat(self, key): + if key not in self._store: + raise StorageNotFoundError(key) + return ObjectMeta(key=key, size=len(self._store[key])) + + def list(self, prefix=""): + for key, body in self._store.items(): + if key.startswith(prefix): + yield ObjectMeta(key=key, size=len(body)) + + def get_url(self, key, *, expires_in=None): + return f"memory://{key}" + + mem_storage = create_storage({"type": "memory", "mode": "sync"}) + mem_storage.put("a.txt", b"in-memory content") + print(mem_storage.get("a.txt")) + + +if __name__ == "__main__": + sync_demo() + asyncio.run(async_demo()) + extend_with_new_backend_demo() diff --git a/common/src/common/storage/exceptions.py b/common/src/common/storage/exceptions.py new file mode 100644 index 0000000..26d9e8b --- /dev/null +++ b/common/src/common/storage/exceptions.py @@ -0,0 +1,21 @@ +"""存储层统一异常。同步/异步后端共用同一套异常类型。""" + + +class StorageError(Exception): + """所有存储相关异常的基类。""" + + +class StorageNotFoundError(StorageError): + """指定的 key 不存在。""" + + +class StorageAlreadyExistsError(StorageError): + """在要求不覆盖的场景下，key 已存在。""" + + +class StorageConnectionError(StorageError): + """连接/网络层面的错误（如 S3 网络超时、权限问题等）。""" + + +class StorageConfigError(StorageError): + """配置错误，例如缺少必需参数、backend 类型未注册等。""" diff --git a/common/src/common/storage/factory.py b/common/src/common/storage/factory.py new file mode 100644 index 0000000..1b152c1 --- /dev/null +++ b/common/src/common/storage/factory.py @@ -0,0 +1,200 @@ +"""统一入口：根据配置字典创建具体的存储后端实例。 + +配置里的 "mode" 字段决定拿到同步还是异步实现，默认 "sync"（向后兼容）。 + + # 同步（默认） + storage = create_storage({"type": "local", "base_dir": "./data"}) + storage.put("a.txt", b"hello") + + # 异步：只需加一个 mode 字段，其余配置不变 + storage = create_storage({"type": "local", "mode": "async", "base_dir": "./data"}) + await storage.put("a.txt", b"hello") + + # S3 同理 + storage = create_storage({"type": "s3", "mode": "async", "bucket": "my-bucket"}) + +上层业务代码应该只从这里拿实例，不要直接 import 具体的 XxxStorageBackend / +XxxAsyncStorageBackend 类。 +""" + +from pathlib import Path +from typing import Any, Union + +from .backends import local, s3 # noqa: F401 # 触发内置后端注册 +from .base import AsyncStorageBackend, StorageBackend +from .exceptions import StorageConfigError +from .registry import get_backend_class + +AnyStorageBackend = Union[StorageBackend, AsyncStorageBackend] + + +def create_storage(config: dict[str, Any]) -> AnyStorageBackend: + """根据配置创建存储后端。 + + Args: + config: 必须包含 "type" 字段（如 "local" / "s3"）； + 可选 "mode" 字段（"sync" 默认 / "async"）； + 其余字段作为 kwargs 传给对应后端的构造函数。 + + Returns: + mode="sync" 时返回 StorageBackend 实例（同步方法）； + mode="async" 时返回 AsyncStorageBackend 实例（方法需要 await）。 + """ + config = dict(config) # 不修改调用方传入的原字典 + backend_type = config.pop("type", None) + mode = config.pop("mode", "sync") + + if not backend_type: + raise StorageConfigError("配置缺少 'type' 字段，例如 'local' 或 's3'") + + backend_cls = get_backend_class(backend_type, mode) + try: + return backend_cls(**config) + except TypeError as e: + raise StorageConfigError( + f"创建后端 (mode={mode}, type={backend_type}) 失败，参数不匹配: {e}" + ) from e + + +# 4 个目的化桶的名字（key for app.state.object_stores）。 +# 这些字符串作为 ``app.state.object_stores`` 的 dict key，也存入 +# ``StorageObjects.bucket_name`` / ``UploadSessions.bucket_name``。 +# 在 s3 模式下它们同时是真正的 S3 bucket 名；在 local 模式下它们只是 +# 与 s3 bucket 对应的标识符，真正的本地子目录由 ``build_storage_config`` +# 用 ``local_storage_base_dir/`` 拼装。 +PURPOSE_BUCKETS: tuple[str, ...] = ("workspace", "version", "run_log", "trash") + + +# Map an upload's usage_type to its purpose (which then resolves to the +# actual bucket / directory via ``actual_bucket_name``). Keeping the +# purpose as the intermediate value means s3 mode and local mode share +# the same routing logic — only the final ``actual_bucket_name`` differs. +USAGE_TYPE_TO_PURPOSE: dict[str, str] = { + "working_copy": "workspace", + "public_script": "workspace", + "data_resource": "workspace", + "snapshot": "workspace", + "version_artifact": "version", + "run_log": "run_log", + "run_result": "run_log", +} + + +def actual_bucket_name(purpose: str) -> str: + """把 purpose 名称解析成实际桶标识符（runtime 数据会存在这个字符串里）。 + + local / s3 两种模式都返回 ``settings.s3__bucket`` 里配置的 + 字符串（如 ``"workspace"`` / ``"run-logs"``），因此两种模式下 + ``app.state.object_stores`` 的 key、``StorageObjects.bucket_name``、 + ``UploadSessions.bucket_name`` 都保持一致。 + + local 模式下真正的本地文件系统布局由 ``build_storage_config`` 负责： + 它会把 ``local_storage_base_dir`` 与这里的 **purpose**（不是返回值） + 拼接，得到 ``create_storage`` 的 ``base_dir``。本函数只关心返回统一 + 的 bucket 标识符，不参与路径拼接。 + """ + from common.config import settings # 延迟 import 避免循环 + return getattr(settings, f"s3_{purpose}_bucket") + + +def build_storage_uri(bucket_name: str, object_key: str) -> str: + """根据 ``settings.storage_backend`` 构造对象的 storage_uri。 + + - s3 模式:``s3://{bucket_name}/{object_key}`` + - local 模式:``file://{local_storage_base_dir}/{bucket_name}/{object_key}`` + + ``bucket_name`` 是统一的 s3 风格 bucket 标识符（见 ``actual_bucket_name``）， + 不是本地文件系统路径。local 模式下需要把它映射到 ``local_storage_base_dir`` + 下的子目录，再转成 ``file://`` URI，避免生成 ``s3:///data/...`` 这种非法 URI。 + """ + from common.config import settings # 延迟 import 避免循环 + + if settings.storage_backend == "local": + absolute_path = (Path(settings.local_storage_base_dir) / bucket_name).resolve().as_posix() + return f"file://{absolute_path}/{object_key}" + return f"s3://{bucket_name}/{object_key}" + + +def build_storage_config(bucket_name: str) -> dict[str, Any]: + """根据 ``settings.storage_backend`` 构造 ``create_storage()`` 的入参。 + + 上层（lifespan 等）只用 ``PURPOSE_BUCKETS`` 循环调用一次， + 业务代码完全不感知本地 / S3 的差别。 + + Args: + bucket_name: 桶名，必须是 ``PURPOSE_BUCKETS`` 之一。 + + Returns: + 直接喂给 ``create_storage(...)`` 的 dict。 + """ + # 延迟 import：避免 storage -> config -> storage 的循环依赖 + from common.config import settings + + if bucket_name not in PURPOSE_BUCKETS: + raise StorageConfigError( + f"未知 bucket 名称 {bucket_name!r}，可选值: {PURPOSE_BUCKETS}" + ) + + if settings.storage_backend == "local": + return { + "type": "local", + "mode": "async", + "base_dir": str(Path(settings.local_storage_base_dir) / bucket_name), + } + + if settings.storage_backend == "s3": + return { + "type": "s3", + "mode": "async", + "bucket": getattr(settings, f"s3_{bucket_name}_bucket"), + "endpoint_url": settings.s3_endpoint, + "aws_access_key_id": settings.s3_access_key, + "aws_secret_access_key": settings.s3_secret_key, + } + + raise StorageConfigError( + f"settings.storage_backend={settings.storage_backend!r} 不支持，" + f"可选值: 's3', 'local'" + ) + + +def workspaces_root() -> Path: + """返回 runtime 视角下 workspace bucket 的本地路径。 + + 唯一权威入口。``settings.local_storage_base_dir`` 是与存储相关的 + 唯一路径设置(其它路径都从这里推导): + + - s3 模式(默认):``${local_storage_base_dir}/workspaces``(rclone 把 + S3 workspace bucket 挂到这里)。 + - local 模式:``${local_storage_base_dir}/workspace``(直接读写 + 本地目录,无 FUSE 层)。 + + ``runtime.mount.WORKSPACES_ROOT`` 等于本函数返回值,业务代码不要自己 + 拼路径。 + """ + from common.config import settings # 延迟 import 避免循环 + base = Path(settings.local_storage_base_dir) + return base / "workspace" + + +# rclone remote 名字。跟 docker-compose 里的 ``RCLONE_CONFIG__*`` 命名空间 +# 对应——rclone 通过 env var 名前缀来定位 remote 配置块,所以这里的常量名 +# 必须跟 ``RCLONE_CONFIG_S3_*`` 的 ``S3`` 部分一致。 +RCLONE_REMOTE_NAME: str = "s3" + + +def rclone_remote_spec() -> str: + """rclone mount 用的 remote spec (s3 模式才合法)。 + + 格式 ``:``——``runtime.mount.start_rclone_mount`` + 直接喂给 ``rclone mount ``。 + + 唯一权威入口:local 模式下没有 rclone,抛 ``StorageConfigError``。 + """ + from common.config import settings + if settings.storage_backend != "s3": + raise StorageConfigError( + "rclone_remote_spec() 仅在 STORAGE_BACKEND=s3 时合法;" + f"当前 settings.storage_backend={settings.storage_backend!r}" + ) + return f"{RCLONE_REMOTE_NAME}:{settings.s3_workspace_bucket}" diff --git a/common/src/common/storage/registry.py b/common/src/common/storage/registry.py new file mode 100644 index 0000000..2d061cb --- /dev/null +++ b/common/src/common/storage/registry.py @@ -0,0 +1,61 @@ +"""后端注册表，用 (mode, name) 作为 key 同时管理同步和异步实现。 + +新增一种存储方式的同步或异步实现时，不需要改 factory.py： + @register_backend("local", mode="sync") + class LocalStorageBackend(StorageBackend): ... + + @register_backend("local", mode="async") + class LocalAsyncStorageBackend(AsyncStorageBackend): ... + +只要保证模块被 import 一次即可（backends/__init__.py 里统一 import）。 +""" + +from typing import Union + +from .base import AsyncStorageBackend, StorageBackend +from .exceptions import StorageConfigError + +BackendClass = Union[type[StorageBackend], type[AsyncStorageBackend]] + +_REGISTRY: dict[tuple[str, str], BackendClass] = {} + +VALID_MODES = ("sync", "async") + + +def _check_mode(mode: str) -> None: + if mode not in VALID_MODES: + raise StorageConfigError(f"不支持的 mode: {mode!r}，可选值: {VALID_MODES}") + + +def register_backend(name: str, mode: str = "sync"): + """类装饰器：把一个后端类注册为 (mode, name) 对应的实现。""" + _check_mode(mode) + + def _decorator(cls: BackendClass) -> BackendClass: + key = (mode, name) + if key in _REGISTRY and _REGISTRY[key] is not cls: + raise StorageConfigError( + f"存储后端 (mode={mode}, type={name}) 已被注册为 {_REGISTRY[key]!r}" + ) + _REGISTRY[key] = cls + return cls + + return _decorator + + +def get_backend_class(name: str, mode: str = "sync") -> BackendClass: + _check_mode(mode) + key = (mode, name) + try: + return _REGISTRY[key] + except KeyError: + available = ", ".join( + f"{m}:{n}" for (m, n) in sorted(_REGISTRY) + ) or "(无)" + raise StorageConfigError( + f"未知的存储后端 (mode={mode}, type={name})，当前已注册: {available}" + ) + + +def registered_backends() -> dict[tuple[str, str], BackendClass]: + return dict(_REGISTRY) diff --git a/common/src/common/storage/rustfs.py b/common/src/common/storage/rustfs.py deleted file mode 100644 index f0ff52a..0000000 --- a/common/src/common/storage/rustfs.py +++ /dev/null @@ -1,5 +0,0 @@ -# coding=utf-8 -""" -@Time :2026/7/29 -@Author :tao.chen -""" diff --git a/common/src/common/storage/schemas.py b/common/src/common/storage/schemas.py new file mode 100644 index 0000000..b3d7b35 --- /dev/null +++ b/common/src/common/storage/schemas.py @@ -0,0 +1,84 @@ +"""Pydantic request models for the internal ``/internal/v1/...`` storage surface.""" + +from __future__ import annotations + +from typing import Literal + +from pydantic import Field, field_validator + +from common.schemas import StrictModel + +__all__ = [ + "CreateUploadRequest", + "DownloadUrlRequest", + "ServerObjectRequest", +] + + +class CreateUploadRequest(StrictModel): + workspace_id: str = Field(min_length=26, max_length=26) + user_id: str = Field(min_length=26, max_length=26) + usage_type: Literal[ + "data_resource", + "version_artifact", + "snapshot", + "run_log", + "run_result", + "working_copy", + "public_script", + ] + file_name: str = Field(min_length=1, max_length=255) + content_type: str = Field(min_length=1, max_length=255) + expected_size_bytes: int = Field(ge=0, le=100 * 1024 * 1024) + expected_hash: str | None = Field(default=None, min_length=64, max_length=64) + idempotency_key: str = Field(min_length=8, max_length=128) + visibility: Literal["private", "workspace", "public"] = "private" + is_immutable: bool = False + + target_path: str = Field(default="", max_length=1024) + + @field_validator("target_path") + @classmethod + def validate_target_path(cls, value: str) -> str: + # POSIX 相对路径，不能含 ..、绝对前缀、控制字符 + if value and (value.startswith("/") or "\\" in value + or any(seg == ".." for seg in value.split("/")) + or any(ord(c) < 0x20 or ord(c) == 0x7F for c in value)): + raise ValueError("target_path must be a clean relative POSIX path") + # 去前导 /；允许尾部 / + return value.strip("/") + + @field_validator("expected_hash") + @classmethod + def validate_hash(cls, value: str | None) -> str | None: + if value is None: + return None + normalized = value.lower() + if any(character not in "0123456789abcdef" for character in normalized): + raise ValueError("expected_hash must be lowercase SHA-256 hex") + return normalized + + +class ServerObjectRequest(StrictModel): + workspace_id: str = Field(min_length=26, max_length=26) + user_id: str = Field(min_length=26, max_length=26) + usage_type: Literal[ + "data_resource", + "version_artifact", + "snapshot", + "run_log", + "run_result", + "working_copy", + "public_script", + ] + file_name: str = Field(min_length=1, max_length=255) + content_type: str = Field(min_length=1, max_length=255) + content_base64: str = Field(min_length=1) + visibility: Literal["private", "workspace", "public"] = "private" + is_immutable: bool = False + idempotency_key: str = Field(min_length=8, max_length=128) + relative_path: str | None = Field(default=None, max_length=1024) + + +class DownloadUrlRequest(StrictModel): + expires_seconds: int = Field(default=300, ge=30, le=3600) diff --git a/common/src/common/utils.py b/common/src/common/utils.py index 3c4e086..40725ec 100644 --- a/common/src/common/utils.py +++ b/common/src/common/utils.py @@ -1,7 +1,67 @@ -# coding=utf-8 +"""Shared low-level utilities used across services. + +Currently home to two general-purpose helpers that have no runtime- +specific concerns: + +- :func:`get_free_port` asks the kernel for a currently-unused TCP port + by binding to ``:0`` and reading back the assigned port number. +- :func:`start_process` launches a subprocess with stdout/stderr merged + into a per-pid log file under ``log_dir`` and returns the final log + path so callers can log it themselves with whatever logger they use. """ -@Time :2026/7/27 -@Author :tao.chen -""" -def hello_world(): - return 'Hello World!' \ No newline at end of file + +from __future__ import annotations + +import os +import socket +import subprocess +import time +from pathlib import Path + + +def get_free_port() -> int: + with socket.socket(socket.AF_INET, socket.SOCK_STREAM) as s: + s.bind(("", 0)) + s.listen(1) + port = s.getsockname()[1] + return port + + +def start_process( + cmd: list[str], + workspace_path: Path, + log_dir: str | Path = "/tmp/process_logs", + env: dict[str, str] | None = None, +) -> tuple[subprocess.Popen, Path]: + """Launch ``cmd`` as a subprocess and return ``(process, log_file)``. + + stderr is merged into a per-pid log file under ``log_dir``; the + temp ``process_start_*.log`` is renamed to ``process__*.log`` + once the real pid is known. The caller logs "I started this" with + its own context — this function does not log on its own. + """ + log_dir_path = Path(log_dir) + log_dir_path.mkdir(parents=True, exist_ok=True) + + start_time = time.strftime("%Y%m%d_%H%M%S") + temp_log = log_dir_path / f"process_start_{start_time}.log" + + # 构建合并后的环境变量 + full_env = os.environ.copy() + if env: + full_env.update(env) + + with open(temp_log, "a", buffering=1) as log_file: + process = subprocess.Popen( + cmd, + cwd=workspace_path, + stdout=log_file, + stderr=subprocess.STDOUT, + start_new_session=True, + env=full_env, + ) + + final_log = log_dir_path / f"process_{process.pid}_{start_time}.log" + temp_log.replace(final_log) + + return process, final_log \ No newline at end of file diff --git a/default.conf b/default.conf index 7572404..5ac3a4e 100644 --- a/default.conf +++ b/default.conf @@ -1,13 +1,15 @@ +# ---------------------------------------------------------------------------- +# NOTE: this file is mounted into the nginx container as a TEMPLATE. +# scripts/nginx-entrypoint.sh (mounted as /docker-entrypoint.sh) substitutes +# the single ${S3_ENDPOINT} placeholder at container start. The rendered +# output is written to /etc/nginx/conf.d/default.conf and execs nginx. +# ---------------------------------------------------------------------------- + map $http_upgrade $connection_upgrade { default upgrade; '' close; } -upstream rustfs_backend { - server 8.153.151.51:9000; - keepalive 64; -} - server { listen 80; server_name localhost; @@ -16,6 +18,9 @@ server { # 指定 Docker 内置 DNS 解析器，并设置 30 秒缓存 resolver 127.0.0.11 valid=30s ipv6=off; + # S3 upstream — full URL passed through to proxy_pass below. + set $s3_backend "${S3_ENDPOINT}"; + location / { root /usr/share/nginx/html; # 前端静态文件存放在容器中的路径 index index.html index.htm; @@ -24,7 +29,7 @@ server { try_files $uri $uri/ /index.html; } - # （可选）针对静态文件资源加长期缓存优化 + #（可选）针对静态文件资源加长期缓存优化 # location ~* \.(?:css|js|jpg|jpeg|gif|png|ico|cur|gz|svg|svgz|mp4|ogg|ogv|webm|htc|woff|woff2)$ { # root /usr/share/nginx/html; # expires 2h; # 前端静态缓存2小时 @@ -34,23 +39,36 @@ server { location /api/ { proxy_pass http://backend:8000/api/; proxy_set_header Host $host; + proxy_set_header X-Forwarded-Host $http_host; + proxy_set_header X-Forwarded-Proto $scheme; proxy_set_header X-Real-IP $remote_addr; proxy_set_header X-Forwarded-For $proxy_add_x_forwarded_for; + # Explicitly forward the session cookie set by + # POST /api/v1/auth/login. nginx forwards it by default, but + # spelling it out keeps the auth contract visible. + proxy_set_header Cookie $http_cookie; + # Defense in depth: blank out the legacy identity headers so a + # malicious client cannot bypass the cookie-based auth flow + # by stuffing X-User-ID / X-Workspace-ID into the request. + # The backend's RequestContext no longer reads them (it + # derives identity from the access_token cookie), so this is + # belt-and-suspenders against a future regression. + proxy_set_header X-User-ID ""; + proxy_set_header X-Workspace-ID ""; } # ========================================================================= - # 1. RustFS 对象存储服务转发 (/storage/) + # 1. S3 对象存储服务转发 (/storage/) # ========================================================================= location /storage/ { - # 核心：透传 Host，确保 RustFS 生成的 Presigned URL 包含公网地址 + # 核心：透传 Host，确保 S3 生成的 Presigned URL 包含公网地址 proxy_set_header Host $http_host; proxy_set_header X-Real-IP $remote_addr; proxy_set_header X-Forwarded-For $proxy_add_x_forwarded_for; proxy_set_header X-Forwarded-Proto $scheme; - # 转发至 RustFS - # 注意：如果 RustFS 内部接口也是 /storage/...，请把末尾的 / 去掉 - proxy_pass http://rustfs_backend/; + # 转发至 S3（$s3_backend 来自 set 指令；尾斜杠保留 location /storage/ 前缀剥离语义） + proxy_pass $s3_backend/; # HTTP/1.1 长连接支持 proxy_http_version 1.1; @@ -98,6 +116,15 @@ server { proxy_set_header Host $host; proxy_set_header X-Real-IP $remote_addr; proxy_set_header X-Forwarded-For $proxy_add_x_forwarded_for; + # Defense in depth: do not let the browser-supplied identity + # headers leak past the auth subrequest. The auth subrequest + # only forwards the Cookie + Authorization it cares about; + # the actual Jupyter upstream is fully trusted (the address + # comes from the backend's runtime registry), so a leaked + # X-User-ID here would not matter for the proxy target but + # could pollute audit logs. + proxy_set_header X-User-ID ""; + proxy_set_header X-Workspace-ID ""; } # 2. 内部 Auth 子请求 location @@ -117,10 +144,17 @@ server { proxy_set_header Cookie $http_cookie; proxy_set_header Authorization $http_authorization; + # Defense in depth: the auth subrequest reads the session + # cookie / Authorization header, not the legacy identity + # headers. Blank them out so a poisoned client header cannot + # be confused for an authenticated identity if the backend + # code is ever refactored to read them again. + proxy_set_header X-User-ID ""; + proxy_set_header X-Workspace-ID ""; } # 拒绝其余非法路径 location /jupyter/ { return 403 "Access Denied"; } -} \ No newline at end of file +} diff --git a/docker-compose.yml b/docker-compose.yml index e1666c8..3a46f51 100644 --- a/docker-compose.yml +++ b/docker-compose.yml @@ -1,61 +1,229 @@ -version: '3.8' - services: - web: - image: nginx:alpine - ports: - - "8888:80" - restart: unless-stopped - volumes: - - ./default.conf:/etc/nginx/conf.d/default.conf:ro - - ./frontend/build/client:/usr/share/nginx/html:ro - depends_on: + migrate: + build: + context: . + dockerfile: backend/Dockerfile + image: model-development-backend:latest + logging: + driver: json-file + options: + max-size: "200m" + max-file: "10" + restart: "no" + command: + - uv + - run + - --frozen + - --no-dev + - --package - backend + - alembic + - upgrade + - head + environment: + DATABASE_URL: ${DATABASE_URL:?DATABASE_URL is required} + INITIAL_ADMIN_PASSWORD: ${INITIAL_ADMIN_PASSWORD:-admin12345} + UV_OFFLINE: "1" + UV_NO_SYNC: "1" + + web: + build: + context: . + dockerfile: frontend/Dockerfile + image: model-development-web:latest + logging: + driver: json-file + options: + max-size: "200m" + max-file: "10" + restart: unless-stopped + # Architecture §2.2: this is the only service exposed to the host. The + # default.conf file is mounted as a template; scripts/nginx-entrypoint.sh + # parses ${S3_ENDPOINT} and writes the rendered config to + # /etc/nginx/conf.d/default.conf before exec'ing nginx. + ports: + - "${GATEWAY_PORT:-8888}:80" + environment: + S3_ENDPOINT: ${S3_ENDPOINT:?S3_ENDPOINT is required} + depends_on: + backend: + condition: service_healthy + runtime: + condition: service_healthy + volumes: + - ${PWD}:/app + - ./default.conf:/etc/nginx/conf.d/default.conf.template:ro + healthcheck: + test: [ "CMD-SHELL", "wget -qO- http://127.0.0.1/ >/dev/null" ] + interval: 10s + timeout: 3s + retries: 12 + start_period: 10s backend: build: context: . dockerfile: backend/Dockerfile - ports: - - "8000:8000" + image: model-development-backend:latest + logging: + driver: json-file + options: + max-size: "200m" + max-file: "10" + restart: unless-stopped + # No host port: architecture §2.2 — only Nginx is externally reachable. + # The previous ``8891:8000`` mapping (P0-1) was removed: the + # ``/internal/v1/*`` storage control plane is now guarded by a + # shared ``INTERNAL_SERVICE_TOKEN`` instead of network isolation. + # No local-FS volume: backend stores everything in S3 (S3_*). environment: - - RUNTIME_BASE_URL=http://runtime:8001 - volumes: - - ./backend:/app/backend:ro - - ./common:/app/common:ro + DATABASE_URL: ${DATABASE_URL:?DATABASE_URL is required} + SERVICE_NAME: model-platform-backend + SCHEDULE_EVENT_NAMESPACE: ${SCHEDULE_EVENT_NAMESPACE:-model-platform-local} + JWT_SECRET: ${JWT_SECRET:?JWT_SECRET is required} + DEMO_AUTH_ENABLED: ${DEMO_AUTH_ENABLED:-false} + INITIAL_ADMIN_PASSWORD: ${INITIAL_ADMIN_PASSWORD:-admin12345} + RUNTIME_API_URL: http://runtime:8000 + # P0-1 fix: shared secret required by /internal/v1/* routes. + INTERNAL_SERVICE_TOKEN: ${INTERNAL_SERVICE_TOKEN:?INTERNAL_SERVICE_TOKEN is required} + STORAGE_BACKEND: ${STORAGE_BACKEND:-s3} + LOCAL_STORAGE_BASE_DIR: ${LOCAL_STORAGE_BASE_DIR:-/data} + # S3_* only matter when STORAGE_BACKEND=s3. Defaults are kept so local + # mode boots without them; override in .env when switching to s3. + S3_ENDPOINT: ${S3_ENDPOINT:-http://s3:9000} + S3_ACCESS_KEY: ${S3_ACCESS_KEY:-} + S3_SECRET_KEY: ${S3_SECRET_KEY:-} + S3_WORKSPACE_BUCKET: ${S3_WORKSPACE_BUCKET:-workspace} + S3_VERSION_BUCKET: ${S3_VERSION_BUCKET:-version} + S3_RUN_LOG_BUCKET: ${S3_RUN_LOG_BUCKET:-run-log} + S3_TRASH_BUCKET: ${S3_TRASH_BUCKET:-trash} + S3_TRASH_RETENTION_DAYS: ${S3_TRASH_RETENTION_DAYS:-30} + READINESS_TARGETS: ${MYSQL_HOST:?MYSQL_HOST is required}:${MYSQL_PORT:-3306},runtime:8000 + UV_OFFLINE: "1" + UV_NO_SYNC: "1" depends_on: - - runtime + migrate: + condition: service_completed_successfully + runtime: + condition: service_healthy + volumes: + - ${PWD}:/app + - ./data:/data + - /app/.venv + healthcheck: + test: [ "CMD-SHELL", "curl -fsS http://127.0.0.1:8000/health/ready >/dev/null" ] + interval: 10s + timeout: 5s + retries: 18 + start_period: 20s runtime: build: context: . dockerfile: runtime/Dockerfile + image: model-development-runtime:latest + logging: + driver: json-file + options: + max-size: "200m" + max-file: "10" + restart: unless-stopped + # No host port: architecture §2.2 — only Nginx is externally reachable. + # The previous ``8892:8000`` mapping (P0-1) was removed: the runtime + # container is reachable only from the Docker internal network and + # Nginx-authenticated Jupyter paths. cap_add: - SYS_ADMIN devices: - /dev/fuse:/dev/fuse security_opt: - apparmor:unconfined - - ports: - - "8001:8001" + # No host port: architecture §2.2 — only Nginx is externally reachable. environment: - - PUBLIC_BASE_URL=http://runtime - # --- Rclone 动态环境变量配置 (对应名称 rustfs) --- - - RCLONE_CONFIG_RUSTFS_TYPE=s3 - - RCLONE_CONFIG_RUSTFS_PROVIDER=Other - - RCLONE_CONFIG_RUSTFS_ACCESS_KEY_ID=BdsXeamEnvSDQnk8tRxh - - RCLONE_CONFIG_RUSTFS_SECRET_ACCESS_KEY=mmBVc3RqzbT2VX3ysKGnirYH5kYD3ww3wFtMVvrb - # 替换为你的 RustFS 服务地址（如果是同 docker-compose 网络下的服务，可以直接填服务名:端口） - - RCLONE_CONFIG_RUSTFS_ENDPOINT=http://8.153.151.51:9000 - # 自建 S3 建议强制开启 Path-style 访问 (http://endpoint/bucket) - - RCLONE_CONFIG_RUSTFS_ENV_AUTH=false - - RCLONE_CONFIG_RUSTFS_FORCE_PATH_STYLE=true - - RCLONE_CONFIG_RUSTFS_REGION=other - - # --- Runtime 逻辑环境变量 --- - - REMOTE_BUCKET=rustfs:workspaces - - WORKSPACES_ROOT=/app/workspaces + DATABASE_URL: ${DATABASE_URL:?DATABASE_URL is required} + SERVICE_NAME: runtime-manager + # P0-1 fix: runtime's /api/v1/jupyter is token-guarded. The same + # ``INTERNAL_SERVICE_TOKEN`` value backend uses for /internal/v1/* + # auth — see ``require_internal_service`` in runtime/main.py. + INTERNAL_SERVICE_TOKEN: ${INTERNAL_SERVICE_TOKEN:?INTERNAL_SERVICE_TOKEN is required} + STORAGE_BACKEND: ${STORAGE_BACKEND:-s3} + LOCAL_STORAGE_BASE_DIR: ${LOCAL_STORAGE_BASE_DIR:-/data} + # WORKSPACES_ROOT defaults to /data/workspace (settings.workspaces_root); + # in local mode runtime skips the rclone mount and reads directly from + # ${LOCAL_STORAGE_BASE_DIR}/workspace instead. + PUBLIC_BASE_URL: http://runtime + # rclone config only used when STORAGE_BACKEND=s3 (mount skipped in local mode). + # The remote spec ("s3:") is derived in + # common.storage.rclone_remote_spec(); no REMOTE_BUCKET env needed. + RCLONE_CONFIG_S3_TYPE: s3 + RCLONE_CONFIG_S3_PROVIDER: Other + RCLONE_CONFIG_S3_ACCESS_KEY_ID: ${S3_ACCESS_KEY:-} + RCLONE_CONFIG_S3_SECRET_ACCESS_KEY: ${S3_SECRET_KEY:-} + RCLONE_CONFIG_S3_ENDPOINT: ${S3_ENDPOINT:-http://s3:9000} + RCLONE_CONFIG_S3_ENV_AUTH: "false" + RCLONE_CONFIG_S3_FORCE_PATH_STYLE: "true" + RCLONE_CONFIG_S3_REGION: other + UV_OFFLINE: "1" + UV_NO_SYNC: "1" volumes: - - ./runtime:/app/runtime:ro - - ./common:/app/common:ro \ No newline at end of file + - ${PWD}:/app + - ./data:/data + - /app/.venv + healthcheck: + # Two-mode: s3 needs rclone FUSE mount; local skips rclone and uses a + # bind mount instead. In s3 mode the grep succeeds (mount must be up or + # startup would have crashed). In local mode the grep fails (no fuse.rclone) + # so the || branch runs: just verify /data/workspace is a directory. + test: [ "CMD-SHELL", "curl -fsS http://127.0.0.1:8000/api/v1/health >/dev/null" ] + interval: 10s + timeout: 5s + retries: 18 + start_period: 30s + + schedule: + build: + context: . + dockerfile: schedule/Dockerfile + image: model-development-schedule:latest + logging: + driver: json-file + options: + max-size: "200m" + max-file: "10" + restart: unless-stopped + # No host port: architecture §2.2 — only Nginx is externally reachable. + # No local-FS volume: schedule executes nodes via tempfile.TemporaryDirectory + # under Python's default temp dir (cleaned per-run); artifacts live in S3. + environment: + DATABASE_URL: ${DATABASE_URL:?DATABASE_URL is required} + SERVICE_NAME: schedule-executor + SCHEDULE_EVENT_NAMESPACE: ${SCHEDULE_EVENT_NAMESPACE:-model-platform-local} + BACKEND_API_URL: http://backend:8000 + # P0-1 fix: must match the backend's INTERNAL_SERVICE_TOKEN exactly. + INTERNAL_SERVICE_TOKEN: ${INTERNAL_SERVICE_TOKEN:?INTERNAL_SERVICE_TOKEN is required} + STORAGE_BACKEND: ${STORAGE_BACKEND:-s3} + LOCAL_STORAGE_BASE_DIR: ${LOCAL_STORAGE_BASE_DIR:-/data} + S3_ENDPOINT: ${S3_ENDPOINT:-http://s3:9000} + S3_ACCESS_KEY: ${S3_ACCESS_KEY:-} + S3_SECRET_KEY: ${S3_SECRET_KEY:-} + S3_WORKSPACE_BUCKET: ${S3_WORKSPACE_BUCKET:-workspace} + S3_VERSION_BUCKET: ${S3_VERSION_BUCKET:-version} + S3_RUN_LOG_BUCKET: ${S3_RUN_LOG_BUCKET:-run-log} + # Backend's health endpoint already verifies MySQL; the scheduler only + # needs MySQL and Backend to be ready in either local or S3 mode. + READINESS_TARGETS: ${MYSQL_HOST:?MYSQL_HOST is required}:${MYSQL_PORT:-3306},backend:8000 + UV_OFFLINE: "1" + UV_NO_SYNC: "1" + depends_on: + backend: + condition: service_healthy + volumes: + - ${PWD}:/app + - ./data:/data + - /app/.venv + healthcheck: + test: [ "CMD-SHELL", "curl -fsS http://127.0.0.1:8000/health/ready >/dev/null" ] + interval: 10s + timeout: 5s + retries: 18 + start_period: 20s diff --git a/extras/requirements-py310 b/extras/requirements-py310 new file mode 100644 index 0000000..a63537d --- /dev/null +++ b/extras/requirements-py310 @@ -0,0 +1,13 @@ +# this is not system dependences +nbclient +nbformat +ipykernel +httpx +apscheduler +pymysql +loguru +# custom +pandas +numpy +scikit-learn +pyspark \ No newline at end of file diff --git a/extras/requirements-py312 b/extras/requirements-py312 new file mode 100644 index 0000000..a63537d --- /dev/null +++ b/extras/requirements-py312 @@ -0,0 +1,13 @@ +# this is not system dependences +nbclient +nbformat +ipykernel +httpx +apscheduler +pymysql +loguru +# custom +pandas +numpy +scikit-learn +pyspark \ No newline at end of file diff --git a/extras/requirements-py38 b/extras/requirements-py38 new file mode 100644 index 0000000..a63537d --- /dev/null +++ b/extras/requirements-py38 @@ -0,0 +1,13 @@ +# this is not system dependences +nbclient +nbformat +ipykernel +httpx +apscheduler +pymysql +loguru +# custom +pandas +numpy +scikit-learn +pyspark \ No newline at end of file diff --git a/frontend/.agents/skills/react-router/SKILL.md b/frontend/.agents/skills/react-router/SKILL.md new file mode 100644 index 0000000..949e3ae --- /dev/null +++ b/frontend/.agents/skills/react-router/SKILL.md @@ -0,0 +1,122 @@ +--- +name: react-router +description: Build applications with React Router in Framework, Data, Declarative, and unstable RSC modes. Use when configuring routes, route modules, loaders, actions, forms, fetchers, navigation, pending UI, SSR/SPA/pre-rendering, middleware, URL params/search params, or React Router upgrades. +license: MIT +--- + +# React Router + +React Router is mode-specific. Before changing an app, identify the mode, load the matching reference, then read the installed docs for the installed package version. + +## Identify the Mode + +Do not apply Framework/Data patterns to a Declarative app unless you are intentionally migrating modes. + +### Framework Mode + +Use Framework Mode guidance when you see: + +- `@react-router/dev` in dependencies +- `react-router.config.ts` +- `app/routes.ts` +- `app/entry.server.tsx` and/or `app/entry.client.tsx` files +- route modules under `app/routes/` +- route exports like `loader`, `action`, `clientLoader`, `clientAction`, `ErrorBoundary`, `meta`, `links`, or `headers` +- imports from `./+types/...` +- the React Router Vite plugin from `@react-router/dev/vite` + +Framework examples usually use the default `app/` directory, but check `react-router.config.ts` for a custom `appDirectory` before assuming exact paths. + +Then read `references/framework-mode.md`. + +### Data Mode + +Use Data Mode guidance when you see: + +- `createBrowserRouter`, `createHashRouter`, `createMemoryRouter`, or `createStaticRouter` +- `` +- route objects with properties like `path`, `children`, `loader`, `action`, `Component`, `ErrorBoundary`, or `lazy` +- data APIs without the Framework Vite plugin + +Then read `references/data-mode.md`. + +### Declarative Mode + +Use Declarative Mode guidance when you see: + +- ``, ``, or `` +- `` and `` JSX route configuration +- route components passed with `element={}` +- no data router, no route module convention, and no loaders/actions + +Then read `references/declarative-mode.md`. + +### RSC Framework and RSC Data Modes + +React Server Components support is unstable and exists in both Framework and Data variants. Use RSC guidance when you see: + +- `unstable_reactRouterRSC` +- `@vitejs/plugin-rsc` +- `unstable_RSCRouteConfig` +- RSC entry files such as `entry.rsc` +- `ServerComponent`, `ServerErrorBoundary`, `ServerLayout`, or `ServerHydrateFallback` +- React directives or boundary packages such as `"use client"`, `"server-only"`, or `"client-only"` + +For RSC Framework, read both `references/framework-mode.md` and `references/rsc.md`. +For RSC Data, read both `references/data-mode.md` and `references/rsc.md`. + +## Use Installed Docs as Source of Truth + +React Router ships markdown docs in the package so guidance can match the installed version: + +```txt +node_modules/react-router/docs/ +``` + +Key docs paths: + +```txt +node_modules/react-router/docs/index.md +node_modules/react-router/docs/start/ +node_modules/react-router/docs/how-to/ +node_modules/react-router/docs/explanation/ +node_modules/react-router/docs/upgrading/ +``` + +When this skill references `react-router/docs/...`, read the matching file under `node_modules/react-router/docs/`. If the installed version does not include local docs, use the repo `docs/` directory when working inside the React Router repository; in a consuming app, fall back to version-matched website docs. + +Most docs include a mode marker near the top: + +```txt +[MODES: framework, data, declarative] +``` + +Only apply a doc when its mode marker matches the app mode. If a task spans modes, prefer the section or file that matches the current app. + +RSC is documented primarily in: + +```txt +node_modules/react-router/docs/how-to/react-server-components.md +``` + +## Skill References + +Load the relevant reference after identifying the mode: + +| Reference | Use When | +| -------------------------------- | --------------------------------------------- | +| `references/framework-mode.md` | Framework Mode or RSC Framework base behavior | +| `references/data-mode.md` | Data Mode or RSC Data base behavior | +| `references/declarative-mode.md` | Declarative Mode | +| `references/rsc.md` | Any unstable RSC app | + +## Mode Migration Doc Index + +If the user explicitly asks to switch modes, read the target mode reference plus the migration-relevant docs: + +| Migration | Docs to read | +| ------------------------------------ | ------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- | +| Declarative → Data | `react-router/docs/start/modes.md`, `react-router/docs/start/data/routing.md`, `react-router/docs/start/data/data-loading.md`, `react-router/docs/start/data/actions.md` | +| Declarative/Data → Framework | `react-router/docs/start/modes.md`, `react-router/docs/start/framework/routing.md`, `react-router/docs/start/framework/route-module.md`, `react-router/docs/how-to/route-module-type-safety.md` | +| Framework SPA/SSR/pre-render changes | `react-router/docs/start/framework/rendering.md`, `react-router/docs/how-to/spa.md`, `react-router/docs/how-to/pre-rendering.md`, `react-router/docs/start/framework/data-loading.md`, `react-router/docs/start/framework/actions.md` | +| Future flags/upgrades | `react-router/docs/upgrading/future.md` and relevant files under `react-router/docs/upgrading/` | diff --git a/frontend/.agents/skills/react-router/references/data-mode.md b/frontend/.agents/skills/react-router/references/data-mode.md new file mode 100644 index 0000000..ea3480c --- /dev/null +++ b/frontend/.agents/skills/react-router/references/data-mode.md @@ -0,0 +1,165 @@ +# Data Mode + +Data Mode uses data routers such as `createBrowserRouter` and renders with ``. It gives an app route objects, loaders, actions, pending UI, fetchers, and SSR primitives without adopting the Framework Vite plugin or route-module file conventions. + +Use this reference after the main skill identifies a Data Mode app. + +## Read the Local Docs by Mode + +Start with: + +```txt +react-router/docs/start/modes.md +react-router/docs/start/data/index.md +``` + +Then use the Data docs under: + +```txt +react-router/docs/start/data/ +``` + +Those files cover installation, route objects, routing, data loading, actions, navigation, pending UI, and testing. For task-specific details, read relevant files in: + +```txt +react-router/docs/how-to/ +react-router/docs/explanation/ +``` + +Always check the `[MODES: data, ...]` marker in a doc before applying it. + +## Data Router Shape + +Typical setup: + +```tsx +import { createBrowserRouter, RouterProvider } from "react-router"; + +const router = createBrowserRouter([ + { + path: "/", + Component: Root, + loader: rootLoader, + children: [ + { index: true, Component: Home }, + { + path: "projects/:projectId", + Component: Project, + loader: projectLoader, + }, + ], + }, +]); + +root.render(); +``` + +Look for route object arrays and APIs such as: + +- `createBrowserRouter` +- `createHashRouter` +- `createMemoryRouter` +- `RouterProvider` +- `loader` +- `action` +- `Component` +- `ErrorBoundary` +- `lazy` +- `children` + +## Route Objects and Routing + +Before editing route configuration, read: + +```txt +react-router/docs/start/data/routing.md +react-router/docs/start/data/route-object.md +``` + +Rules: + +- Keep route objects outside render when possible. +- Use nested routes for shared layouts and data boundaries. +- Use index routes for default child content. +- Use dynamic segments and splats according to route-object docs. +- Prefer `Component`/`ErrorBoundary` route object properties in Data Mode examples unless the existing app uses `element` consistently. + +## Data and Mutations + +Before working on data loading or mutations, read: + +```txt +react-router/docs/start/data/data-loading.md +react-router/docs/start/data/actions.md +``` + +Rules: + +- Load route data with route `loader` functions. +- Mutate route data with route `action` functions. +- Prefer loaders/actions over route-level `useEffect` fetching. +- Use `request`, `params`, and returned/throwable Responses as described in the docs. +- Let React Router revalidate after actions unless there is a documented reason to customize revalidation. + +Common patterns: + +- Validation failure from an action: return `data({ errors, values }, { status: 400 })`, then render errors with `useActionData()` or `fetcher.data`. +- Missing record in a loader: throw `data("Not Found", { status: 404 })` and render the route `ErrorBoundary`. +- Search/filter data: parse `new URL(request.url).searchParams` in the loader so the URL is shareable and bookmarkable. + +## Forms, Fetchers, and Pending UI + +For forms and pending UI, read: + +```txt +react-router/docs/start/data/actions.md +react-router/docs/start/data/pending-ui.md +react-router/docs/how-to/fetchers.md +react-router/docs/explanation/form-vs-fetcher.md +``` + +Rules of thumb: + +- Search/filter form that updates the URL: `
`. +- Mutation that should change URL/history or redirect after completion: ``. +- Mutation that should keep the user on the same page: `useFetcher` / ``. +- Optimistic UI: derive from `fetcher.formData` or `navigation.formData`. + +## Navigation and URL State + +Before changing navigation or search params, read: + +```txt +react-router/docs/start/data/navigating.md +react-router/docs/how-to/search-params.md +react-router/docs/explanation/location.md +``` + +Rules: + +- Use ``/`` for user-initiated internal navigation. +- Use `redirect` in loaders/actions when navigation follows data loading or mutations. +- Use `useNavigate` for imperative client-side event navigation. +- Treat URL params as strings and validate/parse them. +- Preserve unrelated search params unless intentionally resetting them. + +## SSR in Data Mode + +Data Mode SSR is manual and lower-level than Framework Mode. Before implementing or changing SSR, read the Data Mode custom/SSR docs and match existing server abstractions. + +Start with: + +```txt +react-router/docs/start/data/custom.md +``` + +Look for APIs like `createStaticHandler`, `createStaticRouter`, `StaticRouterProvider`, and hydration data handling in the current app before changing anything. + +## RSC Data + +If this Data Mode app uses `unstable_RSCRouteConfig`, RSC route config, or low-level RSC server APIs, also read: + +```txt +references/rsc.md +react-router/docs/how-to/react-server-components.md +``` diff --git a/frontend/.agents/skills/react-router/references/declarative-mode.md b/frontend/.agents/skills/react-router/references/declarative-mode.md new file mode 100644 index 0000000..982a3c4 --- /dev/null +++ b/frontend/.agents/skills/react-router/references/declarative-mode.md @@ -0,0 +1,123 @@ +# Declarative Mode + +Declarative Mode is React Router's simplest mode. It uses router components like `` and JSX routes with ``/``. It does not provide loaders, actions, fetchers, or data-router pending UI. + +Use this reference after the main skill identifies a Declarative Mode app. + +## Read the Local Docs by Mode + +Start with: + +```txt +react-router/docs/start/modes.md +react-router/docs/start/declarative/index.md +``` + +Then use the Declarative docs under: + +```txt +react-router/docs/start/declarative/ +``` + +Those files cover installation, routing, navigation, and URL values. For conceptual details, read relevant files in: + +```txt +react-router/docs/explanation/ +``` + +Always check the `[MODES: declarative, ...]` marker in a doc before applying it. + +## Declarative Router Shape + +Typical setup: + +```tsx +import { BrowserRouter, Routes, Route } from "react-router"; + +function App() { + return ( + + + } /> + } /> + }> + } /> + } /> + + + + ); +} +``` + +Look for APIs such as: + +- `` +- `` +- `` +- `` +- `` +- `element={}` +- `useRoutes` + +## Routing + +Before editing routes, read: + +```txt +react-router/docs/start/declarative/routing.md +``` + +Rules: + +- Use `` and `` for route configuration. +- Use nested routes with `` for shared layout. +- Use index routes for default child UI. +- Use route params and splats according to the declarative routing docs. +- Do not add route object loaders/actions to a Declarative router. + +## Navigation + +Before changing navigation, read: + +```txt +react-router/docs/start/declarative/navigating.md +``` + +Rules: + +- Use `` or `` for user-initiated internal navigation. +- Use `NavLink` when active styling matters. +- Use `useNavigate` for imperative navigation from event handlers or effects. +- Do not use plain `` for internal navigation unless intentionally forcing a full document navigation. + +## URL Values + +Before changing params, search params, or location state, read: + +```txt +react-router/docs/start/declarative/url-values.md +react-router/docs/explanation/location.md +``` + +Rules: + +- Use `useParams` for dynamic route params. +- Use `useSearchParams` for query string state. +- Use `useLocation` for the current location object and navigation state. +- Validate and parse URL params; they are strings and can be absent. +- Preserve unrelated search params unless intentionally resetting them. + +## Mode Boundary + +Declarative Mode does not have Data/Framework APIs such as: + +- `loader` +- `action` +- `` +- `useFetcher` +- `useNavigation` +- route module exports +- generated `./+types` route types + +If the user asks for route data loading, DB/API-backed data, CRUD, form mutations, validation returned from submissions, revalidation, pending UI, optimistic UI, or fetchers, recommend Data Mode or Framework Mode depending on how much structure they want. Ask before migrating unless they already requested it. diff --git a/frontend/.agents/skills/react-router/references/framework-mode.md b/frontend/.agents/skills/react-router/references/framework-mode.md new file mode 100644 index 0000000..5c3e096 --- /dev/null +++ b/frontend/.agents/skills/react-router/references/framework-mode.md @@ -0,0 +1,213 @@ +# Framework Mode + +Framework Mode is React Router's full-stack mode. It uses the React Router Vite plugin, route config in `app/routes.ts`, route modules, generated route types, and rendering strategies such as SSR, SPA mode, and pre-rendering. + +Use this reference after the main skill identifies a Framework Mode app. + +## Read the Local Docs by Mode + +Start with: + +```txt +react-router/docs/start/modes.md +react-router/docs/start/framework/index.md +``` + +Then use the Framework docs under: + +```txt +react-router/docs/start/framework/ +``` + +Those files cover installation, routing, route modules, data loading, actions, navigation, pending UI, rendering, deploying, and testing. For task-specific details, read relevant files in: + +```txt +react-router/docs/how-to/ +react-router/docs/explanation/ +``` + +Always check the `[MODES: framework, ...]` marker in a doc before applying it. + +## Framework Shape + +Examples usually assume the default `appDirectory` of `app`. Check `react-router.config.ts` before assuming exact paths. + +Look for these files and conventions: + +```txt +react-router.config.ts +app/root.tsx +app/routes.ts +app/routes/**/*.tsx +route modules importing from ./+types/... +``` + +Typical route module: + +```tsx +import type { Route } from "./+types/product"; + +export async function loader({ params }: Route.LoaderArgs) { + return { product: await getProduct(params.productId) }; +} + +export default function Product({ loaderData }: Route.ComponentProps) { + return

{loaderData.product.name}

; +} +``` + +## Route Configuration + +Framework apps use `app/routes.ts`. Many apps use file-system routing via `flatRoutes()`, but manual route config is also supported. + +Before editing routes, read: + +```txt +react-router/docs/start/framework/routing.md +``` + +If the app uses file-route conventions, read: + +```txt +react-router/docs/how-to/file-route-conventions.md +``` + +## Route Modules + +Route modules are the main unit of Framework Mode. Before adding or changing route exports, read: + +```txt +react-router/docs/start/framework/route-module.md +``` + +Common exports include: + +| Export | Use | +| --------------------------------- | ------------------------------------------------------------------- | +| `default` | Route component rendered for the match | +| `loader` | Server data loading for SSR/pre-rendering/server data requests | +| `clientLoader` | Browser-only data loading or supplementing server loader data | +| `action` | Server mutation called by ``, `useSubmit`, or fetchers | +| `clientAction` | Browser-only mutation or client-side wrapper around a server action | +| `ErrorBoundary` | UI for errors thrown by this route's loaders/actions/component | +| `HydrateFallback` | Initial fallback while client loader hydration runs | +| `links` / `meta` | Route document links and metadata | +| `handle` | Arbitrary route metadata consumed via `useMatches` | +| `shouldRevalidate` | Overrides default loader revalidation behavior | +| `middleware` / `clientMiddleware` | Server/client request pipeline hooks when enabled | + +Use generated `Route.*` types from `./+types/` for route module args and props. + +## Layout and Root Route Rules + +- `app/root.tsx` is the root route and should contain global document/app shell concerns. +- Put global providers, app-wide nav, app-wide footer, scripts/meta/links, and document structure in `root.tsx` when appropriate. +- Use nested routes/layout routes for section-specific layouts. +- Do not flatten routes that should share UI or data boundaries. + +Useful docs: + +```txt +react-router/docs/explanation/special-files.md +react-router/docs/start/framework/routing.md +``` + +## Data and Mutations + +Before working on route data: + +```txt +react-router/docs/start/framework/data-loading.md +react-router/docs/start/framework/actions.md +``` + +Framework rules: + +- Load route data with `loader` or `clientLoader`. +- Mutate route data with `action` or `clientAction`. +- Prefer route loaders/actions over ad hoc `useEffect` fetching for route data. +- Use `data()`/Responses and redirects according to the docs. +- Let React Router revalidate after actions unless the docs point you to `shouldRevalidate`. +- In SSR/server data routes, keep Node-only/database code in server-only modules and call it from `loader`/`action`, not from browser-rendered component code. + +Common patterns: + +- Validation failure from an action: return `data({ errors, values }, { status: 400 })`, then render errors from `Route.ComponentProps["actionData"]` or `fetcher.data`. +- Missing record in a loader: throw `data("Not Found", { status: 404 })` and render the route `ErrorBoundary`. +- Search/filter data: parse the route request URL/search params in the loader so the URL is shareable and bookmarkable. + +## Forms, Fetchers, and Pending UI + +For forms and pending UI, read: + +```txt +react-router/docs/start/framework/actions.md +react-router/docs/start/framework/pending-ui.md +react-router/docs/how-to/fetchers.md +react-router/docs/explanation/form-vs-fetcher.md +``` + +Rules of thumb: + +- Search/filter form that updates the URL: ``. +- Mutation that should change URL/history or redirect after completion: ``. +- Mutation that should keep the user on the same page: `useFetcher` / ``. +- Optimistic UI: derive from `fetcher.formData` or `navigation.formData`. + +## Type Safety + +Before changing generated route types or typed URL behavior, read: + +```txt +react-router/docs/how-to/route-module-type-safety.md +react-router/docs/explanation/type-safety.md +``` + +Rules: + +- Import types from `./+types/`. +- Use `Route.LoaderArgs`, `Route.ActionArgs`, `Route.ComponentProps`, etc. +- Use type-only imports where appropriate. +- Do not edit generated `.react-router/types` files. + +## Metadata + +Before changing `meta`, read: + +```txt +react-router/docs/how-to/meta.md +react-router/docs/start/framework/route-module.md +``` + +Important: `meta` receives `loaderData`; do not use deprecated `data` args. + +## Rendering Strategy + +Framework Mode can be SSR, SPA, pre-rendered, or mixed depending on config and route behavior. Before changing rendering behavior, read: + +```txt +react-router/docs/start/framework/rendering.md +react-router/docs/how-to/spa.md +react-router/docs/how-to/pre-rendering.md +react-router/docs/explanation/hydration.md +``` + +## Middleware, Sessions, and Auth + +Before implementing middleware or auth/session flows, read: + +```txt +react-router/docs/how-to/middleware.md +react-router/docs/explanation/sessions-and-cookies.md +``` + +Middleware and context APIs are version/config sensitive. Check the installed React Router version and the app's `react-router.config.ts` before implementing. + +## RSC Framework + +If this Framework app uses `unstable_reactRouterRSC` or `@vitejs/plugin-rsc`, also read: + +```txt +references/rsc.md +react-router/docs/how-to/react-server-components.md +``` diff --git a/frontend/.agents/skills/react-router/references/rsc.md b/frontend/.agents/skills/react-router/references/rsc.md new file mode 100644 index 0000000..b940821 --- /dev/null +++ b/frontend/.agents/skills/react-router/references/rsc.md @@ -0,0 +1,90 @@ +# React Server Components (RSC) + +React Router's RSC support is unstable and exists in two variants: + +- **RSC Framework Mode**: Framework Mode with the unstable RSC Vite plugin. +- **RSC Data Mode**: lower-level RSC runtime APIs and manual bundler/server integration. + +Use this reference in addition to `framework-mode.md` or `data-mode.md` after the main skill identifies an RSC app. + +## Read the Local RSC Docs + +Start with: + +```txt +react-router/docs/how-to/react-server-components.md +``` + +Then read the relevant base mode docs: + +```txt +react-router/docs/start/framework/ +react-router/docs/start/data/ +``` + +RSC docs may describe differences from non-RSC mode rather than repeating every Framework/Data concept, so keep both layers in mind. + +## Detect RSC Framework Mode + +Look for: + +- `unstable_reactRouterRSC` imported from `@react-router/dev/vite` +- `@vitejs/plugin-rsc` +- `vite.config.ts` with `plugins: [reactRouterRSC(), rsc()]` +- Framework route modules plus RSC route exports +- RSC entry files such as `entry.rsc` + +RSC Framework Mode uses a different Vite plugin from non-RSC Framework Mode. Do not swap it for the regular `reactRouter()` plugin. + +## Detect RSC Data Mode + +Look for: + +- `unstable_RSCRouteConfig` +- route config passed to lower-level RSC APIs +- APIs such as `unstable_matchRSCServerRequest`, `unstable_routeRSCServerRequest`, `unstable_RSCHydratedRouter`, or `unstable_RSCStaticRouter` +- custom bundler/server setup around RSC + +RSC Data Mode is more manual than RSC Framework Mode. Match the app's bundler and server abstractions before changing routes or entries. + +## RSC Route Module Differences + +In RSC Framework Mode, many normal Framework Mode concepts still apply, but routes can use server component exports. + +Important route-module concepts from the RSC docs include: + +- `ServerComponent` instead of the usual client `default` component +- `ServerErrorBoundary` paired with `ErrorBoundary` +- `ServerLayout` paired with `Layout` +- `ServerHydrateFallback` paired with `HydrateFallback` +- server-rendered React elements returned from loaders/actions + +A route module cannot export both the normal client component and its server component counterpart for the same role. Read the RSC docs before adding these exports. + +## Client/Server Boundaries + +RSC code must respect React's client/server split: + +- Use `"use client"` for components that need hooks, browser APIs, or event handlers. +- Use server-only modules for server data access and secrets. +- In RSC Framework Mode, prefer the `server-only` and `client-only` boundary imports described in the docs. +- Do not assume `.server`/`.client` file naming works the same way in RSC Framework Mode; read the RSC docs before relying on those conventions. + +## Data Loading in RSC + +RSC changes where data can be loaded: + +- Server Components can fetch data directly on the server. +- Loaders/actions may still exist and can have RSC-specific behavior. +- Client components still need client-safe data and cannot directly access server-only modules. + +When choosing between a server component fetch, a loader, and a client loader/action, follow the RSC docs and match existing app patterns. + +## Stability + +RSC APIs are explicitly unstable. Before implementing or refactoring RSC code: + +- Check the installed React Router version. +- Check the installed `@vitejs/plugin-rsc` version. +- Read the app's existing RSC entry/config files. +- Prefer minimal changes that match current patterns. diff --git a/frontend/.gitignore b/frontend/.gitignore index 039ee62..271afdb 100644 --- a/frontend/.gitignore +++ b/frontend/.gitignore @@ -5,3 +5,7 @@ # React Router /.react-router/ /build/ + +# monaco-editor 预构建 min/vs(由 scripts/copy-monaco.mjs 在 postinstall 时 +# 从 node_modules/monaco-editor/min/vs 复制生成,不要提交) +/public/monaco/vs/ diff --git a/frontend/.nvmrc b/frontend/.nvmrc new file mode 100644 index 0000000..8dfc5cb --- /dev/null +++ b/frontend/.nvmrc @@ -0,0 +1 @@ +24.18.1 diff --git a/frontend/Dockerfile b/frontend/Dockerfile index 5ec026f..77b70e6 100644 --- a/frontend/Dockerfile +++ b/frontend/Dockerfile @@ -1,22 +1,17 @@ -FROM node:24-alpine AS development-dependencies-env -COPY . /app +FROM node-pnpm-base:24.18.1 AS frontend-build WORKDIR /app -RUN npm ci +COPY frontend/package.json frontend/pnpm-lock.yaml ./ +RUN pnpm config set registry https://registry.npmmirror.com && \ + pnpm install --frozen-lockfile +COPY frontend/ ./ +RUN pnpm typecheck && pnpm build -FROM node:24-alpine AS production-dependencies-env -COPY ./package.json package-lock.json /app/ -WORKDIR /app -RUN npm ci --omit=dev - -FROM node:24-alpine AS build-env -COPY . /app/ -COPY --from=development-dependencies-env /app/node_modules /app/node_modules -WORKDIR /app -RUN npm run build - -FROM node:24-alpine -COPY ./package.json package-lock.json /app/ -COPY --from=production-dependencies-env /app/node_modules /app/node_modules -COPY --from=build-env /app/build /app/build -WORKDIR /app -CMD ["npm", "run", "start"] +FROM nginx-base:alpine +COPY ./default.conf /etc/nginx/conf.d/default.conf.template +COPY ./scripts/nginx-entrypoint.sh /usr/local/bin/model-platform-entrypoint.sh +RUN sed -i 's/\r$//' /usr/local/bin/model-platform-entrypoint.sh \ + && chmod +x /usr/local/bin/model-platform-entrypoint.sh +COPY --from=frontend-build /app/build/client /usr/share/nginx/html +EXPOSE 80 +ENTRYPOINT ["/usr/local/bin/model-platform-entrypoint.sh"] +CMD ["nginx", "-g", "daemon off;"] diff --git a/frontend/README.md b/frontend/README.md index 5c4780a..a3844c1 100644 --- a/frontend/README.md +++ b/frontend/README.md @@ -1,87 +1,22 @@ -# Welcome to React Router! +# Frontend -A modern, production-ready template for building full-stack React applications using React Router. +React Router SPA，已将原前端按功能拆分到 `app/`： -[![Open in StackBlitz](https://developer.stackblitz.com/img/open_in_stackblitz.svg)](https://stackblitz.com/github/remix-run/react-router-templates/tree/main/default) +```text +app/components/ 通用图标组件 +app/features/platform/ 主框架、脚本工作区和 Jupyter 编辑 +app/features/schedules/ DAG 调度页面 +app/features/admin/ 工作台与系统管理 +app/routes/ React Router 路由入口 +app/services/ API 请求、DTO 和演示上下文 +app/styles/ 分功能样式 +``` -## Features - -- 🚀 Server-side rendering -- ⚡️ Hot Module Replacement (HMR) -- 📦 Asset bundling and optimization -- 🔄 Data loading and mutations -- 🔒 TypeScript by default -- 🎉 TailwindCSS for styling -- 📖 [React Router docs](https://reactrouter.com/) - -## Getting Started - -### Installation - -Install the dependencies: +本地开发： ```bash -npm install +pnpm install +pnpm dev ``` -### Development - -Start the development server with HMR: - -```bash -npm run dev -``` - -Your application will be available at `http://localhost:5173`. - -## Building for Production - -Create a production build: - -```bash -npm run build -``` - -## Deployment - -### Docker Deployment - -To build and run using Docker: - -```bash -docker build -t my-app . - -# Run the container -docker run -p 3000:3000 my-app -``` - -The containerized application can be deployed to any platform that supports Docker, including: - -- AWS ECS -- Google Cloud Run -- Azure Container Apps -- Digital Ocean App Platform -- Fly.io -- Railway - -### DIY Deployment - -If you're familiar with deploying Node applications, the built-in app server is production-ready. - -Make sure to deploy the output of `npm run build` - -``` -├── package.json -├── package-lock.json (or pnpm-lock.yaml, or bun.lockb) -├── build/ -│ ├── client/ # Static assets -│ └── server/ # Server-side code -``` - -## Styling - -This template comes with [Tailwind CSS](https://tailwindcss.com/) already configured for a simple default starting experience. You can use whatever CSS framework you prefer. - ---- - -Built with ❤️ using React Router. +生产构建由根目录 `nginx/Dockerfile` 完成，构建结果复制到 Nginx 静态目录。 diff --git a/frontend/app/app.css b/frontend/app/app.css index 99345d8..c5fff16 100644 --- a/frontend/app/app.css +++ b/frontend/app/app.css @@ -1,15 +1,110 @@ -@import "tailwindcss"; - -@theme { - --font-sans: "Inter", ui-sans-serif, system-ui, sans-serif, - "Apple Color Emoji", "Segoe UI Emoji", "Segoe UI Symbol", "Noto Color Emoji"; +@font-face { + font-family: "Inter"; + src: url("/fonts/Inter-Variable.ttf") format("truetype"); + font-style: normal; + font-weight: 100 900; + font-display: swap; } -html, -body { - @apply bg-white dark:bg-gray-950; - - @media (prefers-color-scheme: dark) { - color-scheme: dark; - } +@font-face { + font-family: "Inter"; + src: url("/fonts/Inter-Variable-Italic.ttf") format("truetype"); + font-style: italic; + font-weight: 100 900; + font-display: swap; } + +:root { + font-family: Inter, "PingFang SC", "Microsoft YaHei", system-ui, sans-serif; + color: #1f354b; + background: #f3f6f9; + font-synthesis: none; + text-rendering: optimizeLegibility; +} + +* { box-sizing: border-box; } +html, body { margin: 0; min-width: 1024px; min-height: 100%; } +button, input, select, textarea { font: inherit; } + +.auth-loading, +.login-page { + min-height: 100vh; + display: flex; + align-items: center; + justify-content: center; +} + +.auth-loading { + gap: 12px; + color: #61758a; +} + +.auth-loading > span { + width: 20px; + height: 20px; + border: 2px solid #bfd5e9; + border-top-color: #1677ff; + border-radius: 50%; + animation: auth-spin 0.8s linear infinite; +} + +@keyframes auth-spin { to { transform: rotate(360deg); } } + +.login-page { + padding: 48px; + background: + radial-gradient(circle at 20% 15%, rgba(22, 119, 255, 0.12), transparent 34%), + linear-gradient(145deg, #eef5fb, #f8fafc 55%, #edf3f8); +} + +.login-card { + width: 420px; + padding: 42px; + border: 1px solid #dbe6ef; + border-radius: 18px; + background: rgba(255, 255, 255, 0.96); + box-shadow: 0 22px 60px rgba(37, 63, 88, 0.14); +} + +.login-brand { + width: 46px; + height: 46px; + display: grid; + place-items: center; + border-radius: 13px; + color: white; + background: linear-gradient(145deg, #1177e8, #25a1f2); +} + +.login-kicker { + margin: 26px 0 8px; + color: #1677ff; + font-size: 11px; + font-weight: 750; + letter-spacing: 0.14em; +} + +.login-card h1 { margin: 0; font-size: 27px; } +.login-description { margin: 10px 0 28px; color: #728398; } +.login-card form { display: grid; gap: 18px; } +.login-card label { display: grid; gap: 8px; color: #465b70; font-size: 13px; } +.login-card input { + width: 100%; + padding: 12px 14px; + border: 1px solid #cad8e5; + border-radius: 9px; + outline: none; + color: #18334f; + background: white; +} +.login-card input:focus { border-color: #1677ff; box-shadow: 0 0 0 3px rgba(22, 119, 255, 0.12); } +.login-card button { + min-height: 44px; + border: 0; + border-radius: 9px; + color: white; + background: #1677ff; + cursor: pointer; +} +.login-card button:disabled { opacity: 0.65; cursor: wait; } +.login-error { margin: -4px 0 0; color: #d4380d; font-size: 13px; } diff --git a/frontend/app/components/admin/DashboardPage.tsx b/frontend/app/components/admin/DashboardPage.tsx new file mode 100644 index 0000000..3c0a960 --- /dev/null +++ b/frontend/app/components/admin/DashboardPage.tsx @@ -0,0 +1,90 @@ +import Icon from "../../components/common/Icon"; +import { useAuth } from "../../context/AuthContext"; + +import "../../styles/admin.css"; +import "../../styles/dashboard.css"; + +export function DashboardPage({ + scriptCount, + online, + onNavigate, +}: { + scriptCount: number; + online: boolean; + onNavigate: (page: "scripts" | "schedules" | "system") => void; +}) { + const { user, currentWorkspace } = useAuth(); + const isSystemAdmin = user?.is_system_admin ?? false; + + return ( +
+
+
+ MODEL DEVELOPMENT PLATFORM +

下午好，{user?.display_name ?? "用户"}

+

+ 当前位于 {currentWorkspace?.workspace_name ?? "（未选择 Workspace）"} + ，可以继续构建脚本或配置调度。 +

+
+ {online ? "服务正常" : "服务连接中"} +
+
+
{scriptCount}工作副本
+
2Workspace
+
4平台用户
+
{online ? "正常" : "检查中"}平台状态
+
+
+ + + {isSystemAdmin && ( + + )} +
+
+
+
运行趋势

近 7 天调度执行

成功率 92.6%
+
+ {[38, 55, 44, 73, 61, 86, 78].map((value, index) => ( +
+ {Math.round(value / 7)} + + {["周一", "周二", "周三", "周四", "周五", "周六", "今天"][index]} +
+ ))} +
+
成功 75失败 6
+
+
+
脚本资产

类型分布

+
+
{scriptCount}全部脚本
+
+ Notebook{Math.max(1, Math.round(scriptCount * .67))} 个 · 67% + Python{Math.max(0, scriptCount - Math.round(scriptCount * .67))} 个 · 33% + 稳定版本3 个已发布 +
+
+
+
+
ACTIVITY

最近平台活动

+
+
操作内容执行人状态时间
+ {[ + ["数据探索.ipynb 发布稳定版本 v3.0", "张三", "成功", "16:42"], + ["每日模型训练流程完成调度运行", "Scheduler", "成功", "15:25"], + ["批量预测.py 更新工作副本", "王五", "已同步", "14:18"], + ["风险验证流程完成 DAG 校验", "李四", "成功", "11:06"], + ].map((row) => ( +
+ {row[0]} + {row[1]}{row[2]}{row[3]} +
+ ))} +
+
+
+
+ ); +} diff --git a/frontend/app/components/admin/ProjectManagementPage.tsx b/frontend/app/components/admin/ProjectManagementPage.tsx new file mode 100644 index 0000000..5f3471e --- /dev/null +++ b/frontend/app/components/admin/ProjectManagementPage.tsx @@ -0,0 +1,553 @@ +import { useEffect, useState } from "react"; + +import { ApiRequestError, type Employee, type Workspace, type WorkspaceMember } from "../../services/api"; +import { useApi, useAuth } from "../../context/AuthContext"; +import Icon from "../common/Icon"; +import { UserMultiSelect } from "./UserMultiSelect"; + +const EMPTY_PROJECT_FORM = { + workspace_code: "", + workspace_name: "", + quota_bytes: 0, + description: "", +}; + +export function ProjectManagementPage({ + onNotify, + onConnectionChange, +}: { + onNotify: (notice: { tone: "success" | "error" | "info"; message: string }) => void; + onConnectionChange: (online: boolean) => void; +}) { + const api = useApi(); + const { user, refreshWorkspaces } = useAuth(); + const [projects, setProjects] = useState([]); + const [projectLoading, setProjectLoading] = useState(true); + const [projectDialogOpen, setProjectDialogOpen] = useState(false); + const [projectForm, setProjectForm] = useState(EMPTY_PROJECT_FORM); + const [editingProject, setEditingProject] = useState(null); + const [importMemberDialogOpen, setImportMemberDialogOpen] = useState(false); + const [selectedProject, setSelectedProject] = useState(null); + const [availableUsers, setAvailableUsers] = useState([]); + const [selectedUserIds, setSelectedUserIds] = useState([]); + const [projectSearchTerm, setProjectSearchTerm] = useState(""); + const [saving, setSaving] = useState(false); + const [membersDrawerOpen, setMembersDrawerOpen] = useState(false); + const [currentProjectMembers, setCurrentProjectMembers] = useState([]); + const [membersLoading, setMembersLoading] = useState(false); + + const canManage = user?.role_code === "admin"; + + const loadProjects = async (): Promise => { + setProjectLoading(true); + try { + const workspaceList = await api.listWorkspaces(); + setProjects(workspaceList); + onConnectionChange(true); + } catch (error) { + onConnectionChange(false); + onNotify({ + tone: "error", + message: error instanceof Error ? error.message : "项目列表加载失败", + }); + } finally { + setProjectLoading(false); + } + }; + + useEffect(() => { + void loadProjects(); + }, []); + + const openCreateProject = (): void => { + setEditingProject(null); + setProjectForm(EMPTY_PROJECT_FORM); + setProjectDialogOpen(true); + }; + + const openEditProject = (project: Workspace): void => { + setEditingProject(project); + setProjectForm({ + workspace_code: project.workspace_code, + workspace_name: project.workspace_name, + quota_bytes: project.quota_bytes, + description: project.description ?? "", + }); + setProjectDialogOpen(true); + }; + + const submitProject = async (event: React.FormEvent): Promise => { + event.preventDefault(); + if (!projectForm.workspace_name.trim()) { + onNotify({ tone: "error", message: "请输入项目名称" }); + return; + } + if (!projectForm.workspace_code.trim() && !editingProject) { + onNotify({ tone: "error", message: "请输入项目编码" }); + return; + } + setSaving(true); + try { + if (editingProject) { + const updated = await api.updateWorkspace(editingProject.workspace_id, { + workspace_name: projectForm.workspace_name.trim(), + quota_bytes: projectForm.quota_bytes, + description: projectForm.description.trim() || undefined, + }); + setProjects((current) => + current.map((p) => (p.workspace_id === updated.workspace_id ? updated : p)) + ); + onNotify({ tone: "success", message: "项目信息已更新" }); + } else { + const generatedCode = projectForm.workspace_code.trim() || projectForm.workspace_name.trim().toLowerCase().replace(/[^a-z0-9-]/g, "-").slice(0, 32); + const created = await api.createWorkspace({ + workspace_code: generatedCode, + workspace_name: projectForm.workspace_name.trim(), + quota_bytes: projectForm.quota_bytes, + description: projectForm.description.trim() || undefined, + }); + setProjects((current) => [...current, created]); + void refreshWorkspaces(); + onNotify({ tone: "success", message: "项目已创建" }); + } + setProjectDialogOpen(false); + } catch (error) { + onNotify({ + tone: "error", + message: error instanceof ApiRequestError ? error.message : (editingProject ? "更新项目失败" : "创建项目失败"), + }); + } finally { + setSaving(false); + } + }; + + const deleteProject = async (project: Workspace): Promise => { + if (!window.confirm(`确定要删除项目"${project.workspace_name}"吗？此操作将级联软删所有成员。`)) return; + try { + await api.deleteWorkspace(project.workspace_id); + setProjects((current) => current.filter((p) => p.workspace_id !== project.workspace_id)); + onNotify({ tone: "success", message: "项目已删除" }); + } catch (error) { + onNotify({ + tone: "error", + message: error instanceof Error ? error.message : "删除项目失败", + }); + } + }; + + const openImportMemberDialog = async (project: Workspace): Promise => { + setSelectedProject(project); + setSelectedUserIds([]); + setImportMemberDialogOpen(true); + + // 加载可用用户和项目成员 + try { + const [allUsers, currentMembers] = await Promise.all([ + api.listPlatformEmployees(), + api.listWorkspaceMembers(project.workspace_id), + ]); + setAvailableUsers(allUsers); + setCurrentProjectMembers(currentMembers); + } catch (error) { + // 如果加载失败，仍显示所有用户 + const allUsers = await api.listPlatformEmployees(); + setAvailableUsers(allUsers); + setCurrentProjectMembers([]); + } + }; + + const openMembersDrawer = (project: Workspace): void => { + setSelectedProject(project); + setMembersDrawerOpen(true); + void loadProjectMembers(project.workspace_id); + }; + + const loadProjectMembers = async (workspaceId: string): Promise => { + setMembersLoading(true); + try { + const members = await api.listWorkspaceMembers(workspaceId); + setCurrentProjectMembers(members); + } catch (error) { + onNotify({ + tone: "error", + message: error instanceof Error ? error.message : "成员列表加载失败", + }); + } finally { + setMembersLoading(false); + } + }; + + const removeMember = async (userId: string): Promise => { + if (!selectedProject) return; + // 管理员不能被移除 + const targetMember = currentProjectMembers.find((m) => m.user_id === userId); + if (targetMember?.role_code === "admin") { + onNotify({ tone: "error", message: "管理员不能被移除" }); + return; + } + try { + await api.deleteWorkspaceMember(selectedProject.workspace_id, userId); + setCurrentProjectMembers((current) => current.filter((m) => m.user_id !== userId)); + onNotify({ tone: "success", message: "成员已移除" }); + } catch (error) { + onNotify({ + tone: "error", + message: error instanceof ApiRequestError ? error.message : "移除成员失败", + }); + } + }; + + const updateMemberRole = async (userId: string, roleCode: "admin" | "developer"): Promise => { + if (!selectedProject) return; + try { + const updated = await api.updateWorkspaceMember(selectedProject.workspace_id, userId, { + role_code: roleCode, + }); + setCurrentProjectMembers((current) => + current.map((m) => (m.user_id === userId ? { ...m, role_code: roleCode, role_name: roleCode === "admin" ? "管理员" : "开发人员" } : m)) + ); + onNotify({ tone: "success", message: "成员角色已更新" }); + } catch (error) { + onNotify({ + tone: "error", + message: error instanceof ApiRequestError ? error.message : "更新成员角色失败", + }); + } + }; + + const importMember = async (): Promise => { + if (!selectedProject || selectedUserIds.length === 0) { + onNotify({ tone: "error", message: "请选择要添加的用户" }); + return; + } + setSaving(true); + try { + await Promise.all( + selectedUserIds.map((userId) => + api.addWorkspaceMember(selectedProject.workspace_id, { + user_id: userId, + }) + ) + ); + setImportMemberDialogOpen(false); + onNotify({ tone: "success", message: `已添加 ${selectedUserIds.length} 名成员` }); + } catch (error) { + onNotify({ + tone: "error", + message: error instanceof ApiRequestError ? error.message : "添加成员失败", + }); + } finally { + setSaving(false); + } + }; + + return ( +
+
+
+ + setProjectSearchTerm(event.target.value)} + /> +
+ +
+ + {!canManage &&
当前为开发人员，只能查看项目列表。
} +
+
+ 项目名称 + 项目编码 + 状态 + 配额 + 操作 +
+ {projectLoading ? ( +

正在加载项目…

+ ) : projects.length === 0 ? ( +

暂无项目

+ ) : ( + projects + .filter((project) => { + const term = projectSearchTerm.toLowerCase().trim(); + if (!term) return true; + return ( + project.workspace_name.toLowerCase().includes(term) || + project.workspace_code.toLowerCase().includes(term) || + (project.description && project.description.toLowerCase().includes(term)) + ); + }) + .map((project) => ( +
+ + {project.workspace_name.slice(0, 1)} + + {project.workspace_name} + {project.description ?? "无描述"} + + + {project.workspace_code} + + + {project.status === "active" ? "正常" : project.status === "archived" ? "已归档" : "已删除"} + + + {project.quota_bytes > 0 ? `${(project.quota_bytes / 1024 / 1024 / 1024).toFixed(1)} GB` : "无限制"} + + + + {/* */} + + +
+ )) + )} +
+ + {projectDialogOpen && ( +
+
+
+
+ PROJECT +

{editingProject ? "编辑项目" : "新建项目"}

+
+ +
+ void submitProject(event)}> + {!editingProject && ( + + )} + + +