diff --git a/.env.example b/.env.example index 1d1a188..1d587aa 100644 --- a/.env.example +++ b/.env.example @@ -24,27 +24,42 @@ JWT_SECRET=change-this-development-secret # ============================================================================ INITIAL_ADMIN_PASSWORD=admin12345 -# Object storage (S3-compatible, RustFS). -# RUSTFS_ENDPOINT is the single upstream URL consumed by all 4 services: +# Object storage. Two modes are supported: +# STORAGE_BACKEND=s3 — connects to an S3-compatible service (MinIO, +# RustFS, SeaweedFS, AWS S3, …). Requires the +# S3_* block below. +# STORAGE_BACKEND=local — stores objects on the local filesystem under +# LOCAL_STORAGE_BASE_DIR. Backend and runtime +# share this directory via a Docker volume +# (docker-compose.yml mounts `local-storage`). +# Useful for dev, single-node, air-gapped. +STORAGE_BACKEND=s3 +LOCAL_STORAGE_BASE_DIR=/data + +# Object storage (S3-compatible). Only used when STORAGE_BACKEND=s3. +# S3_ENDPOINT is the single upstream URL consumed by all 4 services: # - nginx (via scripts/nginx-entrypoint.sh, which parses host + port) # - backend / runtime / schedule (passed through to boto3 / rclone) -# RUSTFS_ACCESS_KEY / RUSTFS_SECRET_KEY are read by Python code in +# S3_ACCESS_KEY / S3_SECRET_KEY are read by Python code in # backend/ and schedule/ (boto3 credentials). # -# RustFS buckets are purpose-named. Currently we have: -# RUSTFS_WORKSPACE_BUCKET — workspace files (notebooks, scripts, working +# S3 buckets are purpose-named: +# S3_WORKSPACE_BUCKET — workspace files (notebooks, scripts, working # copies); layout is ``s3:////...``. -# Future: RUSTFS_VERSION_BUCKET, RUSTFS_RUN_LOG_BUCKET, ... -RUSTFS_HOST=127.0.0.1 -RUSTFS_PORT=9000 -RUSTFS_ENDPOINT=http://127.0.0.1:9000 -RUSTFS_ACCESS_KEY=change-me -RUSTFS_SECRET_KEY=change-me -RUSTFS_WORKSPACE_BUCKET=workspaces -RUSTFS_VERSION_BUCKET=versions -RUSTFS_RUN_LOG_BUCKET=run-logs -RUSTFS_TRASH_BUCKET=trash -RUSTFS_TRASH_RETENTION_DAYS=30 +# S3_VERSION_BUCKET — immutable script-version artifacts. +# S3_RUN_LOG_BUCKET — schedule run logs and execution results. +# S3_TRASH_BUCKET — soft-deleted objects; source bucket key is preserved +# as a prefix so restore is a same-key move. +S3_HOST=127.0.0.1 +S3_PORT=9000 +S3_ENDPOINT=http://127.0.0.1:9000 +S3_ACCESS_KEY=change-me +S3_SECRET_KEY=change-me +S3_WORKSPACE_BUCKET=workspaces +S3_VERSION_BUCKET=versions +S3_RUN_LOG_BUCKET=run-logs +S3_TRASH_BUCKET=trash +S3_TRASH_RETENTION_DAYS=30 # rclone RC (HTTP control API). The runtime container starts rclone with # `--rc --rc-addr 0.0.0.0:5572 --rc-no-auth` (see runtime/src/runtime/mount.py), diff --git a/API.md b/API.md index 2e20559..e4daa7f 100644 --- a/API.md +++ b/API.md @@ -25,9 +25,10 @@ 4. [调度 (`/api/v1/schedules/...` + `/api/v1/schedule-runs/...`)](#四调度) 5. [数据资源 (`/api/v1/data-resources/...`)](#五数据资源) 6. [管理后台 (`/api/v1/admin/...`)](#六管理后台) -7. [Jupyter 路由 (Nginx `auth_request`)](#七jupyter-路由) -8. [对象存储控制面 (`/internal/v1/...`,同进程 RPC)](#八对象存储控制面) -9. [健康检查](#九健康检查) +7. [系统管理 (`/api/v1/platform/...`)](#七系统管理-apiv1platform) +8. [Jupyter 路由 (Nginx `auth_request`)](#八jupyter-路由) +9. [对象存储控制面 (`/internal/v1/...`,同进程 RPC)](#九对象存储控制面) +10. [健康检查](#十健康检查) --- @@ -44,6 +45,8 @@ - 缺失或过期 → HTTP `401`。 - 有效但用户不在 workspace → HTTP `403`(由 Nginx `auth_request` 透传给客户端)。 +> `/api/v1/auth/me` 与 `/api/v1/auth/login` 响应中的 `data.user` 对象额外携带 `is_system_admin: bool` 字段,派生自 `users.platform_role_id` 指向的角色 `role_code == 'admin'` 且用户状态为 `active`。前端据此决定是否渲染"系统管理"入口。详见 §七。 + --- ## 二、统一约定 @@ -89,7 +92,7 @@ ### 3.2 `POST /api/v1/workspace-directories` -创建一个**逻辑目录**(RustFS 上是隐式前缀,无需落对象)。 +创建一个**逻辑目录**(对象存储上是隐式前缀,无需落对象)。 - **请求体**: ```json @@ -160,7 +163,7 @@ ### 3.7 `POST /api/v1/scripts/upload?file_name=...&parent_path=...&visibility=...` -multipart/binary 形式上传大文件(走 presigned PUT)。 +multipart/binary 形式上传大文件(走 server-proxied PUT,详见 §九)。 - **查询参数**: `file_name`(必填)、`parent_path`、`visibility` - **请求体**: 原始文件字节(`Content-Type` 必须与脚本类型匹配) @@ -200,7 +203,7 @@ multipart/binary 形式上传大文件(走 presigned PUT)。 ### 3.11 `POST /api/v1/scripts/{script_id}/versions` -发布一个**稳定版本**(immutable,绑定到 `RUSTFS_VERSION_BUCKET`)。门禁同 §3.8。 +发布一个**稳定版本**(immutable,绑定到 `S3_VERSION_BUCKET`)。门禁同 §3.8。 - **请求体**: ```json @@ -240,11 +243,11 @@ multipart/binary 形式上传大文件(走 presigned PUT)。 ### 3.14 `DELETE /api/v1/versions/{versions_id}` -从调度候选中**隐藏**此版本(不删除 RustFS 对象)。门禁:**owner 校验基于所属 `Scripts` 的 owner**——即"按整本 script 判定",而非"按版本发布者判定"。 +从调度候选中**隐藏**此版本(不删除对象存储里的对象)。门禁:**owner 校验基于所属 `Scripts` 的 owner**——即"按整本 script 判定",而非"按版本发布者判定"。 ### 3.15 `POST /api/v1/versions/{versions_id}/download-url` -生成 RustFS 的 presigned download URL。 +生成对象存储的 presigned download URL(走 S3 兼容协议,local 模式下该 endpoint 在 s3 模式才生效)。 - **请求体**: ```json @@ -396,8 +399,8 @@ queued ──→ running ──┬─→ succeeded | 方法 | 路径 | 说明 | |---|---|---| -| `POST` | `/api/v1/data-resources/uploads` | 创建上传会话,返回 presigned PUT URL + `upload_id` | -| `POST` | `/api/v1/data-resources/uploads/{upload_id}/complete` | 完成上传(写 `StorageObjects` 行) | +| `POST` | `/api/v1/data-resources/uploads` | 创建上传会话,返回 `upload_id` + `upload_path` | +| `PUT` | `/api/v1/data-resources/uploads/{upload_id}` | 上传字节(请求体即文件内容) | | `GET` | `/api/v1/data-resources` | 列表(workspace 范围) | | `GET` | `/api/v1/data-resources/{id}` | 详情 | | `POST` | `/api/v1/data-resources/{id}/download-url` | 生成 presigned GET URL | @@ -417,15 +420,18 @@ queued ──→ running ──┬─→ succeeded **完整上传流程(前端应实现的模式)**: ``` -1. POST /uploads → {upload_id, presigned_url} -2. PUT presigned_url with file bytes -3. POST /uploads/{upload_id}/complete -4. 服务器完成 → 200 {data: StorageObjectPayload} +1. POST /uploads → {upload_id, upload_path, expires_at} +2. PUT upload_path with raw file bytes (Content-Type: application/octet-stream) +3. 服务器端走 backend.put() → 200 {data: StorageObjectPayload} ``` +字节经过 backend 进程(server-proxied upload),最大 100 MiB,由 backend +直接调 `AsyncStorageBackend.put()` 写入存储(不再走 presigned PUT 直传)。 +前端无需关心 S3 协议或签名。 + **小对象(<100 KiB)捷径**:直接调 `create_server_object` 把字节 base64 放进 -`content_base64` 字段(走 `StorageClient.create_server_object`),前端无需 -走 presigned PUT 三步。 +`content_base64` 字段(JSON 体里走),内部走同一条 `AsyncStorageBackend.put` +路径,前端无需分两步。 --- @@ -484,11 +490,139 @@ Base 前缀 `/api/v1/admin`。 > `PATCH` / `DELETE` 员工接口**不**涉及密码字段,也不返回密码相关信息。 -## 七、Jupyter 路由 +## 七、系统管理 (`/api/v1/platform/...`) + +平台级(跨 workspace)管理接口,用于管理 workspace 实体与 workspace 成员。 +所有端点要求调用者是**系统管理员**——其 `users.platform_role_id` 指向 +`role_code='admin'` 的角色行,且 `users.status == 'active'`。系统管理员判定 +通过 `GET /api/v1/auth/me` 响应中的 `data.user.is_system_admin` 字段(详见 §一)。 + +| 方法 | 路径 | 说明 | +|---|---|---| +| `GET` | `/api/v1/platform/workspaces` | 列 workspace(`active`/`archived`);已软删的过滤掉 | +| `POST` | `/api/v1/platform/workspaces` | 创建 workspace(返回 201);创建者自动成为 admin 成员 | +| `GET` | `/api/v1/platform/workspaces/{workspace_id}` | 单个 workspace(含已 disabled 的,用于恢复) | +| `PATCH` | `/api/v1/platform/workspaces/{workspace_id}` | 改 workspace 字段;`status` 仅允许 `active`/`archived` | +| `DELETE` | `/api/v1/platform/workspaces/{workspace_id}` | 软删 workspace;级联软删其成员 | +| `GET` | `/api/v1/platform/workspaces/{workspace_id}/members` | 列成员 | +| `POST` | `/api/v1/platform/workspaces/{workspace_id}/members` | 添加成员(返回 201) | +| `PATCH` | `/api/v1/platform/workspaces/{workspace_id}/members/{user_id}` | 改成员角色/状态 | +| `DELETE` | `/api/v1/platform/workspaces/{workspace_id}/members/{user_id}` | 软删成员 | + +> **不变量**: +> - 每个 workspace 必须始终保留至少一个 `admin` 角色的活跃成员;对最后 admin 做降级 / 停用 / 删除 → 409。 +> - 系统管理员不能通过 `DELETE .../members/{self}` 把自己移除(403)。唯一退出方式是 `DELETE /workspaces/{id}` 软删整个 workspace,后者会级联软删所有成员。 +> - 列表类接口静默 `pageSize=100` 上限,无客户端分页参数(YAGNI)。 +> - 跨 workspace 操作**不**需要 `?workspace_id=` query 参数,与 `/api/v1/admin/...`(workspace 内成员管理)不要混淆。 + +### 7.1 `POST /api/v1/platform/workspaces` + +创建 workspace;创建者(当前系统管理员)自动成为该 workspace 的 `admin` 成员。 + +- **请求体字段**: + +| 字段 | 类型 | 必填 | 限制 | 说明 | +|---|---|---|---|---| +| `workspace_code` | string | 是 | regex `^[a-z0-9-]{3,32}$`(类似 git repo 名) | 创建后冻结,不可改 | +| `workspace_name` | string | 是 | 1~150 字符 | 显示名称 | +| `quota_bytes` | int | 否 | ≥0,默认 `0` | 配额字节数,`0` 表示无配额 | +| `description` | string | 否 | ≤1000 字符 | | + +- **服务端自动生成字段**(不接收): + - `workspace_id`(ULID) + - `active_root_uri`(`s3://workspaces/{workspace_id}/`) + - `status`(`"active"`) + - `created_by`(当前管理员 `user_id`) + - `created_at` / `updated_at`(DB 自动) + +- **响应 201**:见下 §7.2 `WorkspacePayload`。 + +### 7.2 `GET /api/v1/platform/workspaces/{workspace_id}` / `WorkspacePayload` + +- **响应 200**: + ```json + { + "request_id": "...", + "data": { + "workspace_id": "01HXY...", + "workspace_code": "model-development", + "workspace_name": "模型开发 Workspace", + "active_root_uri": "s3://workspaces/01HXY.../", + "quota_bytes": 0, + "status": "active", + "description": null, + "created_by": "01HXY...", + "created_at": "2026-08-04T12:00:00.000", + "updated_at": null + }, + "meta": {"count": ..., "page_size": 100} + } + ``` + +### 7.3 `PATCH /api/v1/platform/workspaces/{workspace_id}` + +部分更新。**不可改**:`workspace_id`、`workspace_code`、`active_root_uri`、`created_by`、时间戳、软删标记。 + +- **请求体字段**(全部可选): + +| 字段 | 类型 | 限制 | 说明 | +|---|---|---|---| +| `workspace_name` | string | 1~150 | | +| `quota_bytes` | int | ≥0 | | +| `description` | string | ≤1000 | | +| `status` | string | `active` \| `archived` | **不允许 `disabled`**——软删须走 DELETE | + +- 错误:`status="disabled"` → 422;已 disabled 的 workspace → 409。 + +### 7.4 `DELETE /api/v1/platform/workspaces/{workspace_id}` + +软删除。允许从 `active` 或 `archived` 状态调用。 + +- **副作用**: + - 该 workspace 行:`status='disabled'`、`is_deleted=1`、`deleted_at=NOW()` + - **级联**:所有未删除的 `workspace_members` 行同步 `is_deleted=1`、`deleted_at=NOW()` +- 已 disabled 的 workspace 再删 → 409。 + +### 7.5 `POST /api/v1/platform/workspaces/{workspace_id}/members` + +添加成员。 + +- **请求体字段**: + +| 字段 | 类型 | 必填 | 限制 | 说明 | +|---|---|---|---|---| +| `user_id` | string | 是 | 26 字符 ULID | | +| `role_code` | string | 是 | `admin` \| `developer` | **不可填 `system_admin`**(那是用户级身份,不是 workspace 角色) | + +- 服务端默认 `member_status='active'`。 +- 用户不存在 → 404;用户已是该 workspace 成员 → 409。 + +### 7.6 `PATCH /api/v1/platform/workspaces/{workspace_id}/members/{user_id}` + +修改成员的角色或状态。 + +- **请求体字段**(全部可选): + +| 字段 | 类型 | 限制 | 说明 | +|---|---|---|---| +| `role_code` | string | `admin` \| `developer` | 降级最后 admin → 409 | +| `member_status` | string | `active` \| `disabled` \| `locked` | 停用 / 锁定最后 admin → 409 | + +### 7.7 `DELETE /api/v1/platform/workspaces/{workspace_id}/members/{user_id}` + +软删除成员。 + +- **自我移除保护**:`user_id == 当前管理员 user_id` → 403 "系统管理员不能把自己从 workspace 移除;如需退出,请删除整个 workspace" +- **末位 admin 保护**:若删除的是最后一个 `admin` 角色活跃成员 → 409 +- 不存在的成员 → 404 + +--- + +## 八、Jupyter 路由 > **本节是 Nginx 行为,不是直接 HTTP 端点**。前端**不要**直接调用。 -### 7.1 浏览器 → 用户打开 notebook +### 8.1 浏览器 → 用户打开 notebook 用户在前端点击某个 notebook,前端拼出 URL: ``` @@ -498,7 +632,7 @@ GET /jupyter/{workspace_id}/api/contents/{相对路径}.ipynb WS /jupyter/{workspace_id}/api/kernels/... ``` -### 7.2 Nginx `auth_request` 鉴权 +### 8.2 Nginx `auth_request` 鉴权 Nginx 收到上述请求后,**先**发一个内部子请求: ``` @@ -525,7 +659,7 @@ Nginx: auth_request_set 捕获这两个变量,proxy_pass 到子进程并注入 浏览器收到响应,**自始至终未接触 Jupyter Token** ``` -### 7.3 鉴权失败码 +### 8.3 鉴权失败码 | 状态 | 触发条件 | |---|---| @@ -538,14 +672,18 @@ Nginx 把这些状态原样透传给浏览器,前端可在 `onerror` 里判断 --- -## 八、对象存储控制面 +## 九、对象存储控制面 > 路径前缀 `/internal/v1/...`,**前端不要直接调用**。这是 backend 内部 -> 异步消息处理(Schedule worker)用的 RPC 端点,经 `StorageClient` HTTP -> 客户端访问。Backend 通过 ASGI `auth_request_set` 路由转发,外部无法 -> 访问。 +> 异步消息处理(Schedule worker)用的 RPC 端点,经 in-process ASGI 直接 +> 转发(`storage_app` 路由被 `app.include_router` 进同一个 backend 进程), +> 外部无法访问。 -### 8.1 `POST /internal/v1/uploads` +底层抽象:`common.storage.AsyncStorageBackend`(`put/get/delete/exists/stat/ +list/get_url/copy`)。按 `settings.storage_backend` 选实现:`"s3"` 走 +S3-兼容服务,`"local"` 走 `LOCAL_STORAGE_BASE_DIR` 子目录。 + +### 9.1 `POST /internal/v1/uploads` 创建上传会话。`Idempotency-Key` 必填,同 key + 同元数据 → 复用;同 key + 不同元数据 → 409。 @@ -558,23 +696,36 @@ Nginx 把这些状态原样透传给浏览器,前端可在 `onerror` 里判断 "content_type": "text/x-python", "expected_size_bytes": 1024, "expected_hash": "", - "idempotency_key": "..." + "idempotency_key": "...", + "visibility": "private", + "is_immutable": false } ``` -返回 `{upload_id, bucket_name, object_key, presigned_url, expires_in_seconds}`。 +返回 `{upload_id, status, upload_path, expires_at}`。`upload_path` 是 +第 9.2 步要 PUT 的端点(本进程内 `/internal/v1/uploads/{upload_id}`)。 -### 8.2 `POST /internal/v1/uploads/{upload_id}/complete` +### 9.2 `PUT /internal/v1/uploads/{upload_id}` -完成上传。从 RustFS 读 HEAD → 校验 hash → 写 `StorageObjects` 行。 +完成上传(server-proxied PUT)。**请求体即原始字节**,`Content-Type: +application/octet-stream`。后端 `await request.body()` 读字节 → 校验 +size + sha256 → 调 `await backend.put(key, bytes, content_type=..., +metadata={"sha256": ...})` → 写 `StorageObjects` 行 → 标 session 为 +completed。最大 100 MiB。 -### 8.3 `POST /internal/v1/uploads/{upload_id}/abort` +> 历史:旧版本这一步是 `POST /uploads/{id}/complete`,靠 +> presigned-PUT + head() 验证。已被 server-proxied PUT 取代。 -主动放弃。释放 `UploadSessions` 行,对象不入库。 +### 9.3 `POST /internal/v1/uploads/{upload_id}/abort` -### 8.4 `POST /internal/v1/objects` +主动放弃。删除可能已经写了一半的对象字节,释放 `UploadSessions` 行。 -**单步创建**(不走 presigned PUT,字节随请求体直传)。适用 < 100 KiB 对象。 +### 9.4 `POST /internal/v1/objects` + +**单步创建**(不走两步上传,字节 base64 进 JSON 体)。适用 < 100 KiB +对象(避免 multipart/大请求体的前端复杂度)。内部直接调 +`AsyncStorageBackend.put(key, content, content_type=..., +metadata={"sha256": ...})`。 ```json { @@ -586,31 +737,39 @@ Nginx 把这些状态原样透传给浏览器,前端可在 `onerror` 里判断 "content_base64": "PHN0ZXAtY29udGVudD4=", "visibility": "private", "is_immutable": false, - "idempotency_key": "..." + "idempotency_key": "...", + "relative_path": null } ``` -### 8.5 `POST /internal/v1/objects/{storage_object_id}/download-url` +### 9.5 `POST /internal/v1/objects/{storage_object_id}/download-url` -生成 presigned GET URL。 +生成 presigned GET URL(s3 模式:`AsyncStorageBackend.get_url()`; +local 模式:目前抛 `NotImplementedError`,需要 native FS serving 配合 +nginx 静态 location)。 -### 8.6 `DELETE /internal/v1/objects/{storage_object_id}` +### 9.6 `DELETE /internal/v1/objects/{storage_object_id}` -软删。`is_immutable == 1` 的对象拒绝删除。 +软删。`is_immutable == 1` 的对象拒绝删除。`move_to_trash` 走跨后端 +`copy + delete`(同一进程内的两个 backend 实例)。 -### 8.7 usage_type → 桶路由(自动) +### 9.7 usage_type → 桶路由(自动) | usage_type | 实际桶(env var) | 默认桶名 | |---|---|---| -| `working_copy`, `public_script`, `data_resource`, `snapshot` | `RUSTFS_WORKSPACE_BUCKET` | `workspaces` | -| `version_artifact` | `RUSTFS_VERSION_BUCKET` | `versions` | -| `run_log`, `run_result` | `RUSTFS_RUN_LOG_BUCKET` | `run-logs` | +| `working_copy`, `public_script`, `data_resource`, `snapshot` | `S3_WORKSPACE_BUCKET` | `workspaces` | +| `version_artifact` | `S3_VERSION_BUCKET` | `versions` | +| `run_log`, `run_result` | `S3_RUN_LOG_BUCKET` | `run-logs` | +| (soft-delete target) | `S3_TRASH_BUCKET` | `trash` | 若 `Workspaces.artifact_bucket` 非空,优先用 per-workspace 桶(覆盖 usage_type 路由)。 +桶在 `STORAGE_BACKEND=s3` 时是 4 个独立 S3 bucket,在 +`STORAGE_BACKEND=local` 时是 `LOCAL_STORAGE_BASE_DIR` 下的 4 个子目录。 + --- -## 九、健康检查 +## 十、健康检查 | 方法 | 路径 | 用途 | |---|---|---| @@ -620,7 +779,8 @@ Nginx 把这些状态原样透传给浏览器,前端可在 `onerror` 里判断 | `GET` | `/api/v1/health` | 公开健康检查(前端可访问) | `/health/ready` 支持 `READINESS_TARGETS` 环境变量,逗号分隔的 `host:port` -列表,例如 `mysql:3306,rustfs:9000`,全部 TCP 通则返回 200,否则 503。 +列表,例如 `mysql:3306,s3:9000`,全部 TCP 通则返回 200,否则 503。 +`STORAGE_BACKEND=local` 模式下不需要 S3 host,列表里删掉即可。 --- @@ -630,18 +790,20 @@ Nginx 把这些状态原样透传给浏览器,前端可在 `onerror` 里判断 |---|---|---| | 400 | 参数错误 | Pydantic 校验失败 | | 401 | 未鉴权 | JWT 缺失/无效 | -| 403 | 鉴权失败 | 非 workspace 成员 / `is_locked` 阻写 | +| 403 | 鉴权失败 | 非 workspace 成员 / `is_locked` 阻写 / **非系统管理员访问 `/api/v1/platform/*`** / 系统管理员自我移除 workspace 成员 | | 404 | 不存在 | resource_id / script_id / schedule_id 找不到 | -| 409 | 冲突 | DAG 无效 / 同 idempotency_key 不同元数据 / 目标已存在 / `is_immutable` 阻删 | +| 409 | 冲突 | DAG 无效 / 同 idempotency_key 不同元数据 / 目标已存在 / `is_immutable` 阻删 / **workspace 末位 admin 保护** | | 412 | 条件失败 | `source_object_id` 与当前工作副本不一致 | | 413 | 太大 | 内容超过 100 MiB / 10 MiB | -| 422 | 语义错误 | 文件名非法 / cron 表达式非法 / 路径逃逸 | +| 422 | 语义错误 | 文件名非法 / cron 表达式非法 / 路径逃逸 / **`workspace_code` 不匹配 `^[a-z0-9-]{3,32}$` / `status="disabled"` 走 PATCH** | | 500 | 内部错误 | DB / 存储不可达 | ## 附录 B — 状态枚举 | 类型 | 取值 | |---|---| +| `Workspaces.status` | `active` / `archived` / `disabled`(`disabled` 由 DELETE 设置,PATCH 不允许设) | +| `WorkspaceMembers.member_status` | `active` / `disabled` / `locked` | | `StorageObjects.usage_type` | `data_resource` / `version_artifact` / `snapshot` / `run_log` / `run_result` / `working_copy` / `public_script` | | `StorageObjects.object_status` | `available` / `deleted` | | `StorageObjects.visibility` | `private` / `workspace` / `public` | diff --git a/ARCHITECTURE.md b/ARCHITECTURE.md index 633687e..141ceec 100644 --- a/ARCHITECTURE.md +++ b/ARCHITECTURE.md @@ -24,7 +24,7 @@ Schedule Executor（APScheduler） |-- MySQL APSchedulerJobStore |-- MySQL Outbox 轮询兜底 |-- DAG 节点执行与重试 - |-- RustFS 日志/结果 + |-- S3 日志/结果 +-- Backend 内部 Storage API ``` diff --git a/CLAUDE.md b/CLAUDE.md index 32cb110..e65c5ca 100644 --- a/CLAUDE.md +++ b/CLAUDE.md @@ -3,7 +3,7 @@ ## Current architecture - `frontend`: React Router SPA. Production files are built in `nginx/Dockerfile`. -- `backend`: public FastAPI API and internal RustFS storage API in one process. +- `backend`: public FastAPI API and internal S3 storage API in one process. - `runtime`: shared Jupyter lifecycle, MySQL edit leases and short-lived in-memory access tickets. - `schedule`: APScheduler, MySQL JobStore, MySQL Outbox polling and DAG execution. - `common`: SQLAlchemy models, database/session helpers, IDs and object-store helpers. @@ -51,7 +51,7 @@ pnpm build - Cron jobs are persisted by APScheduler in MySQL table `apscheduler_jobs`. - Runtime must stay single-replica while file leases and Jupyter tickets use the simplified implementation. - Never expose the internal Jupyter token to the browser. -- Never delete Docker volumes when preserving MySQL or RustFS data is required. +- Never delete Docker volumes when preserving MySQL or storage data is required. ## Main entrypoints diff --git a/DEVELOP.md b/DEVELOP.md index 32e1d95..c06f437 100644 --- a/DEVELOP.md +++ b/DEVELOP.md @@ -12,7 +12,7 @@ common/ Pure-Python shared library db/ SQLAlchemy 2.0 async engine, session_scope, Base db/models/ 26 tables in 9 domain files (zero FK, zero relationship) scheduler/ build_sqlalchemy_jobstore (delayed import) - storage/ RustFSObjectStore + StorageClient + Pydantic schemas + storage/ AsyncStorageBackend abstraction (s3 + local impls) + Pydantic schemas eventing.py add_outbox_event / utcnow / event_time service_app.py /health/ready TCP probe, /api/v1/health schemas.py StrictModel base @@ -21,14 +21,14 @@ common/ Pure-Python shared library backend/ Public FastAPI service + internal /internal/v1/* sub-app main.py lifespan + route registration jupyter.py /api/v1/auth/jupyter — the ONLY auth entry - scripts.py CRUD for scripts/notebooks (workspace_fs=rustfs) + scripts.py CRUD for scripts/notebooks (object storage via AsyncStorageBackend) schedules.py DAG CRUD: schedules, nodes, edges schedule_runs.py Trigger / list / get runs schedule_schemas.py Pydantic request/response models admin.py Admin endpoints resources.py Misc data resources storage_api.py /internal/v1/* (sub-app merged into main) - storage_client.py HTTP client for the storage sub-app + storage_client.py Stub (HTTP client removed post-migration; rewrite pending) schedule_client.py Placeholder module (was the HTTP-push executor client) runtime_client.py Self-contained httpx wrapper for the runtime jupyter.py auth_request handler @@ -41,7 +41,7 @@ schedule/ Schedule Executor (DAG worker) worker.py NodeExecutor (notebook / python execution) service.py SchedulerService facade (composes the three) main.py Lifespan + FastAPI app - storage_client.py SchedulerStorageClient (subclass of common StorageClient) + storage_client.py Stub (SchedulerStorageClient rewrite pending — use AsyncStorageBackend directly) execution.py execute_artifact (notebook + python paths) notebook_runner.py Subprocess entry point (nbclient) @@ -68,16 +68,17 @@ All env vars go through one place: `common/src/common/config.py`. from common.config import settings settings.database_url # str -settings.rustfs_endpoint # str (full URL, e.g. "http://rustfs:9000") -settings.rustfs_access_key # str -settings.rustfs_secret_key # str -settings.rustfs_workspace_bucket -settings.rustfs_version_bucket -settings.rustfs_run_log_bucket -settings.jwt_secret # HS256 secret for the auth_request handler -settings.workspace_root # schedule subprocess cwd; backend ignores -settings.workspaces_root # runtime rclone FUSE mount point -settings.remote_bucket # rclone remote spec (e.g. "rustfs:workspaces") +settings.storage_backend # str: "s3" (default) or "local" +settings.local_storage_base_dir # str: root dir for storage data (default "/data"); see "Storage" below for per-mode derivation +settings.s3_endpoint # str (full URL, e.g. "http://s3:9000"; s3 mode only) +settings.s3_access_key # str (s3 mode only) +settings.s3_secret_key # str (s3 mode only) +settings.s3_workspace_bucket # str (s3 mode only) +settings.s3_version_bucket # str (s3 mode only) +settings.s3_run_log_bucket # str (s3 mode only) +settings.s3_trash_bucket # str (s3 mode only) +settings.s3_trash_retention_days # int (s3 mode only) +settings.jwt_secret # HS256 secret for the auth_request handler settings.backend_api_url # schedule → backend HTTP base settings.runtime_api_url # backend → runtime HTTP base settings.public_base_url # runtime public base URL @@ -125,12 +126,31 @@ grep -rnE 'os\.(environ\[?["\x27][A-Z_]+|getenv\(["\x27][A-Z_]+)' --include="*.p ### Storage -- All object bytes go to **RustFS** via boto3. -- Use `StorageClient` (HTTP) or `RustFSObjectStore` (direct) — never - the local filesystem. -- `bucket_name` is one of: `RUSTFS_WORKSPACE_BUCKET` (default - `workspaces`), `RUSTFS_VERSION_BUCKET` (default `versions`, - reserved), `RUSTFS_RUN_LOG_BUCKET` (default `run-logs`, reserved). +- All object bytes go through `common.storage.AsyncStorageBackend`, + created by `create_storage(config)` from `common.storage.factory`. +- Two backends are registered: `local` (filesystem, local mode) and + `s3` (S3-compatible service, s3 mode). Selection is per-deployment + via `settings.storage_backend` (`"s3"` default, `"local"` for + dev / single-node / air-gapped). +- The factory helper `build_storage_config(bucket_name)` returns the + right `create_storage` kwargs for each of the 4 purpose buckets + (`workspace`, `version`, `run_log`, `trash`). Use it in lifespan code; + route handlers don't see the difference. +- Bucket resolution from `usage_type` is in **one place** + (`backend/storage_api.py:resolve_bucket`); route handlers only know + about `app.state.object_stores[bucket_name]`. +- The runtime's view of the workspace bucket on disk is exposed by + `common.storage.workspaces_root()`: + - `s3` mode: `${settings.local_storage_base_dir}/workspaces` + (default `/data/workspaces`, the rclone FUSE mount target). + - `local` mode: `${settings.local_storage_base_dir}/workspace` + (default `/data/workspace`, a subdir of the shared local-storage + volume). + `settings.local_storage_base_dir` is the **only** path setting; the + helper handles the per-mode suffix. Don't read `settings.workspaces_root` + or any other path setting directly in runtime code — use this helper. +- The pre-2026 abstraction (`RustFSObjectStore` / `common.storage.client` + / `StorageClient` HTTP wrapper) is gone. Don't reintroduce it. ### Auth @@ -202,9 +222,13 @@ cd frontend && pnpm install && cd .. ```bash # Backend (terminal 1) export DATABASE_URL="mysql+asyncmy://model_platform:model_platform@127.0.0.1:3306/model_platform?charset=utf8mb4" -export RUSTFS_ACCESS_KEY=modelplatform -export RUSTFS_SECRET_KEY=modelplatformsecret -export RUSTFS_ENDPOINT=http://127.0.0.1:9000 +export STORAGE_BACKEND=s3 +export S3_ACCESS_KEY=modelplatform +export S3_SECRET_KEY=modelplatformsecret +export S3_ENDPOINT=http://127.0.0.1:9000 +# Or for local mode: +# export STORAGE_BACKEND=local +# export LOCAL_STORAGE_BASE_DIR=/data uv run --frozen --package backend uvicorn backend.main:app --host 0.0.0.0 --port 8000 --reload # Schedule Executor (terminal 2) @@ -246,7 +270,7 @@ PYTHONPATH="backend/src:common/src" uv run --frozen --package backend python -c from backend.main import app from common.config import settings print('backend:', len(app.routes), 'routes') -print('settings ok:', settings.rustfs_endpoint) +print('settings ok:', settings.s3_endpoint) " ``` @@ -284,36 +308,42 @@ See "Adding a new env var" above. uv run --frozen --package backend alembic upgrade head ``` -### Wire a new RustFS bucket +### Wire a new storage bucket -The current three buckets are wired in `backend/storage_api.py:resolve_bucket`: +The current 4 buckets are wired in `backend/storage_api.py:resolve_bucket`: ```python BUCKET_FOR_USAGE: dict[str, str] = { - "working_copy": settings.rustfs_workspace_bucket, - "public_script": settings.rustfs_workspace_bucket, - "data_resource": settings.rustfs_workspace_bucket, - "snapshot": settings.rustfs_workspace_bucket, - "version_artifact": settings.rustfs_version_bucket, - "run_log": settings.rustfs_run_log_bucket, - "run_result": settings.rustfs_run_log_bucket, + "working_copy": settings.s3_workspace_bucket, + "public_script": settings.s3_workspace_bucket, + "data_resource": settings.s3_workspace_bucket, + "snapshot": settings.s3_workspace_bucket, + "version_artifact": settings.s3_version_bucket, + "run_log": settings.s3_run_log_bucket, + "run_result": settings.s3_run_log_bucket, } ``` -To add a fourth bucket: +The constant `PURPOSE_BUCKETS = ("workspace", "version", "run_log", "trash")` +in `common.storage.factory` enumerates the four backends built in the +backend lifespan. To add a fifth bucket: -1. Add the env var to `Settings`: +1. Add the env var to `Settings` (s3 mode only): ```python - rustfs__bucket: str = Field(default="", description="...") + s3__bucket: str = Field(default="", description="...") ``` 2. Add to `.env.example` with a one-line comment. -3. Extend the `Literal` in `common/storage/schemas.py` (in - `ServerObjectRequest.usage_type`, `CreateUploadRequest.usage_type`, - `CompleteUploadRequest.usage_type`) to include the new value. -4. Add an entry in `BUCKET_FOR_USAGE` mapping the new `usage_type` to +3. Append `""` to the `PURPOSE_BUCKETS` tuple in + `common/storage/factory.py`. `build_storage_config("")` + will then automatically read `settings.s3__bucket` (s3 + mode) or use `/` (local mode). +4. Extend the `Literal` in `common/storage/schemas.py` (in + `CreateUploadRequest.usage_type`, `ServerObjectRequest.usage_type`) + to include the new value. +5. Add an entry in `BUCKET_FOR_USAGE` mapping the new `usage_type` to the new bucket env var. -5. Add the bucket to the `ensure_bucket` loop in - `backend/main.py` lifespan. +6. Pre-create the bucket (s3 mode) or subdirectory (local mode) in the + deployment. The backend no longer auto-creates buckets. A workspace's `artifact_bucket` column (when non-null) overrides the default for that workspace, regardless of `usage_type`. @@ -354,7 +384,7 @@ contexts. Add `greenlet>=3.0.0` to `common/pyproject.toml` and Either MySQL isn't running, or the network namespace doesn't allow `mysql:3306` resolution. Inside the Docker network, services reach each other by service name (`mysql`, `backend`, `runtime`, -`schedule`, `rustfs`). +`schedule`, `s3`). ### Jupyter routing 401s diff --git a/README.md b/README.md index b1c0fc8..f955d7f 100644 --- a/README.md +++ b/README.md @@ -5,8 +5,9 @@ an interactive workspace, a DAG scheduler, an object-storage-backed artifact store, and per-workspace runtime isolation — all behind a single Nginx gateway. -> Stack: React Router SPA · FastAPI · APScheduler · MySQL · RustFS (S3) -> · shared Jupyter · FUSE mount via rclone +> Stack: React Router SPA · FastAPI · APScheduler · MySQL · S3-compatible +> storage (or local filesystem via `STORAGE_BACKEND=local`) · shared +> Jupyter · FUSE mount via rclone (s3 mode only) > Single ingress (Nginx :80); all other services are Docker-internal. ## What it does @@ -15,11 +16,11 @@ gateway. |---|---| | Workspace-scoped notebook editing with row-level lock | `backend/jupyter.py` + `scripts.is_locked` | | Authenticated Jupyter routing (browser never sees the runtime token) | `nginx/default.conf` + `auth_request` + `backend/jupyter.py` | -| Object storage for notebooks / scripts / versions / run logs (RustFS, S3 API) | `common/storage/` + `backend/scripts.py` | +| Object storage for notebooks / scripts / versions / run logs (s3 / local toggle) | `common/storage/` + `backend/scripts.py` | | DAG-style scheduling: nodes, edges, cron, manual trigger, retries, snapshots | `backend/schedules.py` + `backend/schedule_runs.py` + `schedule/` (5 modules) | | DAG execution via MySQL Outbox (no Redis, no in-process queues) | `schedule/orchestrator.py` + `schedule/worker.py` | | Per-workspace Jupyter sub-process pool with asyncio locks | `runtime/process.py` | -| rclone FUSE mount of the workspace bucket into the runtime | `runtime/mount.py` | +| rclone FUSE mount of the workspace bucket into the runtime (s3 mode) | `runtime/mount.py` | | MySQL-only persistence (26 tables, soft-delete, no foreign keys) | `common/db/models/` | ## Architecture at a glance @@ -42,13 +43,13 @@ gateway. │ (storage) ├──────────────►│ - subprocess pool │ │ - DAG CRUD │ │ (per workspace) │ │ - script CRUD │ └──────────┬───────────┘ - │ - auth_request │ │ FUSE + │ - auth_request │ │ FUSE / shared vol │ - /api/v1/... │ ▼ └────┬──────┬──────┘ ┌──────────────────────┐ - │ │ │ RustFS (S3) │ - │ └──────── HTTP ───────►│ bucket: workspaces │ - ▼ │ bucket: versions │ - ┌────────────┐ │ bucket: run-logs │ + │ │ │ Object storage │ + │ └──────── HTTP ───────►│ (s3: S3 service / │ + ▼ │ local: shared vol) │ + ┌────────────┐ │ 4 buckets per usage │ │ MySQL │◄───────── poll ─────│ │ │ - 26 tbls │ └──────────────────────┘ │ - outbox │ @@ -65,6 +66,11 @@ gateway. └───────────────────────────────┘ ``` +Object storage is selectable via `STORAGE_BACKEND` (s3 | local). In s3 mode +the 4 purpose-named buckets (`workspaces` / `versions` / `run-logs` / `trash`) +are S3 buckets; in local mode they're subdirectories of `LOCAL_STORAGE_BASE_DIR`, +shared via the `local-storage` Docker volume. See `DEVELOP.md` §Storage. + Detailed design lives in `ARCHITECTURE.md`. Implementation deviations and recent refactors are recorded in `HANDOVER.md`. @@ -90,9 +96,9 @@ default.conf Nginx template (mounted, rendered at start) | Service | Image | Exposed | Purpose | |---|---|---|---| -| `web` | `nginx:alpine` | host `:8888` → `:80` | SPA, `/api/` reverse-proxy, `/jupyter/{ws}/` auth_request proxy, `/storage/` RustFS passthrough | +| `web` | `nginx:alpine` | host `:8888` → `:80` | SPA, `/api/` reverse-proxy, `/jupyter/{ws}/` auth_request proxy, `/storage/` S3 passthrough (s3 mode only) | | `backend` | `Dockerfile` | internal only | DAG CRUD, script CRUD, schedule triggers, `/api/v1/auth/jupyter`, `/internal/v1/*` storage control plane | -| `runtime` | `Dockerfile` | internal only | Per-workspace Jupyter sub-process pool, rclone FUSE mount of `workspaces` bucket | +| `runtime` | `Dockerfile` | internal only | Per-workspace Jupyter sub-process pool, rclone FUSE mount of `workspaces` bucket (s3 mode) | | `schedule` | `Dockerfile` | internal only | Cron tick + DAG execution via MySQL Outbox polling | The architecture **deliberately has only one host port** (the gateway); @@ -103,7 +109,7 @@ all other services are on the Docker internal network. This is enforced in ```bash cp .env.example .env -# Edit .env — at minimum change MYSQL password and RUSTFS credentials. +# Edit .env — at minimum change MYSQL password and (in s3 mode) S3 credentials. # Static check uv sync --all-packages @@ -130,7 +136,7 @@ docker compose logs -f schedule docker compose logs -f runtime ``` -### Tear down (keeps MySQL + RustFS volumes) +### Tear down (keeps MySQL + S3 / local-storage volumes) ```bash docker compose down @@ -157,14 +163,28 @@ See `DEVELOP.md` for the full list of variables and their meanings. ## Storage layout -Three purpose-named RustFS buckets. The mapping from `StorageObjects.usage_type` -to bucket is decided in **one place** (`storage_api.py:resolve_bucket`): +Four purpose-named buckets. The mapping from `StorageObjects.usage_type` +to bucket is decided in **one place** (`backend/storage_api.py:resolve_bucket`): | `usage_type` | Bucket (env var) | Default name | |---|---|---| -| `working_copy`, `public_script`, `data_resource`, `snapshot` | `RUSTFS_WORKSPACE_BUCKET` | `workspaces` | -| `version_artifact` | `RUSTFS_VERSION_BUCKET` | `versions` | -| `run_log`, `run_result` | `RUSTFS_RUN_LOG_BUCKET` | `run-logs` | +| `working_copy`, `public_script`, `data_resource`, `snapshot` | `S3_WORKSPACE_BUCKET` | `workspaces` | +| `version_artifact` | `S3_VERSION_BUCKET` | `versions` | +| `run_log`, `run_result` | `S3_RUN_LOG_BUCKET` | `run-logs` | +| (soft-delete target) | `S3_TRASH_BUCKET` | `trash` | + +In `STORAGE_BACKEND=s3` mode these are 4 separate S3 buckets. In +`STORAGE_BACKEND=local` mode they are 4 subdirectories under +`LOCAL_STORAGE_BASE_DIR` (default `/data`), so the layout above +becomes: + +``` +/data/ +├── workspace/ # S3_WORKSPACE_BUCKET +├── version/ # S3_VERSION_BUCKET +├── run_log/ # S3_RUN_LOG_BUCKET +└── trash/ # S3_TRASH_BUCKET +``` A workspace's `artifact_bucket` column (when non-null) overrides the default for that workspace, regardless of `usage_type` — useful for @@ -174,24 +194,19 @@ The object key is a flat two-level path — `workspace_id` and a server- issued `ulid` for the object: ``` -s3://workspaces/ -└── / - ├── # working_copy / data_resource / snapshot / ... - ├── - └── ... - -s3://versions// # immutable script versions -s3://run-logs// # node run logs and results +//{.} ``` The file name, extension, content type, and logical path live in the -`StorageObjects` and `Scripts` rows, not in the S3 key, so the bucket -can be re-organised without a database rewrite. +`StorageObjects` and `Scripts` rows, not in the object key, so the +storage can be re-organised without a database rewrite. -Backend code never writes to the container's local filesystem. Schedule -Executor stages node artifacts in `tempfile.TemporaryDirectory()` (auto- -cleaned). Only the `runtime` container keeps a host volume — it is required -by the rclone FUSE mount. +Backend code never writes to the container's local filesystem (except +in `STORAGE_BACKEND=local` mode, where the shared `local-storage` volume +is the canonical store). Schedule Executor stages node artifacts in +`tempfile.TemporaryDirectory()` (auto-cleaned). Only the `runtime` +container keeps a host volume — required by the rclone FUSE mount in +s3 mode, and a no-op pass-through in local mode. ## Documentation diff --git a/REFACTOR_NOTES.md b/REFACTOR_NOTES.md index 4b17b93..9eb69f2 100644 --- a/REFACTOR_NOTES.md +++ b/REFACTOR_NOTES.md @@ -28,8 +28,8 @@ ```text mysql -rustfs -jupyter +s3 # 外部 S3-兼容服务（MinIO/RustFS/SeaweedFS/...），由运维在 compose 外启动 +jupyter # 注释保留；当前实现未在 compose 启此独立容器 migrate（一次性） backend runtime @@ -37,6 +37,11 @@ schedule gateway ``` +> 补充说明：`STORAGE_BACKEND=local` 模式下不需要外部 S3 服务，backend +> 与 runtime 共享 docker 卷 `local-storage`，挂载到 +> `LOCAL_STORAGE_BASE_DIR`（默认 `/data/storage`）；runtime 也跳过 +> rclone FUSE 挂载（见 `runtime/src/runtime/mount.py`）。 + ## 已执行检查 - Python 全项目 `compileall` 通过； diff --git a/alembic.ini b/alembic.ini index c1c4732..a312124 100644 --- a/alembic.ini +++ b/alembic.ini @@ -3,7 +3,7 @@ script_location = %(here)s/migrations prepend_sys_path = . path_separator = os -# 真实连接串必须通过 DATABASE_URL 注入，禁止在仓库内保存数据库密码。 +# The real connection URL must be injected via DATABASE_URL; never store credentials here. sqlalchemy.url = driver://user:pass@localhost/dbname [loggers] diff --git a/backend/Dockerfile b/backend/Dockerfile index c55b11d..ec9ccb0 100644 --- a/backend/Dockerfile +++ b/backend/Dockerfile @@ -1,30 +1,4 @@ -FROM python:3.12-slim-bookworm - -ENV PYTHONDONTWRITEBYTECODE=1 \ - PYTHONUNBUFFERED=1 \ - PYTHONPATH=/app \ - PATH="/app/.venv/bin:${PATH}" \ - TZ=Asia/Shanghai -WORKDIR /app -COPY --from=ghcr.io/astral-sh/uv:latest /uv /bin/uv - -RUN ( \ - sed -i 's/deb.debian.org/mirrors.tuna.tsinghua.edu.cn/g' /etc/apt/sources.list.d/debian.sources 2>/dev/null || \ - sed -i 's/deb.debian.org/mirrors.tuna.tsinghua.edu.cn/g' /etc/apt/sources.list 2>/dev/null \ - ) || true - -RUN apt-get -o Acquire::Retries=5 update && apt-get install -y --no-install-recommends \ - curl \ - ca-certificates \ - gcc \ - g++ \ - python3-dev \ - build-essential \ - tzdata \ - && ln -snf /usr/share/zoneinfo/$TZ /etc/localtime \ - && echo $TZ > /etc/timezone \ - && dpkg-reconfigure --frontend noninteractive tzdata \ - && rm -rf /var/lib/apt/lists/* +FROM python-base:3.12 COPY common ./common COPY backend ./backend diff --git a/backend/README.md b/backend/README.md index eea5476..5e943a5 100644 --- a/backend/README.md +++ b/backend/README.md @@ -1,5 +1,8 @@ # Backend 统一 FastAPI 管理服务。包含用户、Workspace、脚本、稳定版本、调度定义、 -立即运行以及 RustFS 对象接口。原 `platform_api` 与 `storage_api` 已在此 +立即运行以及 S3 对象接口。原 `platform_api` 与 `storage_api` 已在此 模块合并，外部 REST 契约保持不变。 + +底层走的是 `common.storage.AsyncStorageBackend` 抽象，按 +`settings.storage_backend` 切换 s3 / local 两种实现。 \ No newline at end of file diff --git a/backend/pyproject.toml b/backend/pyproject.toml index f51f2ed..2750a53 100644 --- a/backend/pyproject.toml +++ b/backend/pyproject.toml @@ -14,6 +14,7 @@ dependencies = [ "passlib==1.7.4", "bcrypt>=4.0,<4.1", "loguru>=0.7.2", + "aiofiles>=25.1.0", ] [tool.uv.sources] diff --git a/backend/src/backend/auth.py b/backend/src/backend/auth.py index f302db3..c7bc069 100644 --- a/backend/src/backend/auth.py +++ b/backend/src/backend/auth.py @@ -22,6 +22,7 @@ from sqlalchemy.ext.asyncio import AsyncSession from backend.dependencies import database_session from common.auth.jwt import JwtError, issue_jwt, verify_jwt_token +from common.auth.membership import resolve_is_system_admin from common.auth.passwords import verify_password from common.db.models import Roles, Users, WorkspaceMembers, Workspaces from common.ids import new_ulid @@ -55,7 +56,12 @@ def _clear_session_cookie(response: Response) -> None: response.delete_cookie(key=COOKIE_NAME, path="/") -def _user_payload(user: Users, role_code: str | None = None) -> dict[str, Any]: +def _user_payload( + user: Users, + role_code: str | None = None, + *, + is_system_admin: bool = False, +) -> dict[str, Any]: return { "user_id": user.user_id, "username": user.username, @@ -63,6 +69,7 @@ def _user_payload(user: Users, role_code: str | None = None) -> dict[str, Any]: "email": user.email, "status": user.status, "role_code": role_code, + "is_system_admin": is_system_admin, } @@ -161,10 +168,12 @@ async def login( token = issue_jwt(user.user_id, ttl_seconds=COOKIE_TTL_SECONDS) _set_session_cookie(request, response, token) + is_system_admin = await resolve_is_system_admin(session, user) + return { "request_id": new_ulid(), "data": { - "user": _user_payload(user, user_role_code), + "user": _user_payload(user, user_role_code, is_system_admin=is_system_admin), "workspaces": workspaces, "default_workspace_id": default_workspace_id, }, @@ -238,10 +247,12 @@ async def me( if user_role_code is None and rows: user_role_code = rows[0][1].role_code + is_system_admin = await resolve_is_system_admin(session, user) + return { "request_id": new_ulid(), "data": { - "user": _user_payload(user, user_role_code), + "user": _user_payload(user, user_role_code, is_system_admin=is_system_admin), "workspaces": workspaces, "default_workspace_id": default_workspace_id, }, diff --git a/backend/src/backend/dependencies.py b/backend/src/backend/dependencies.py index ec2806e..ea79f91 100644 --- a/backend/src/backend/dependencies.py +++ b/backend/src/backend/dependencies.py @@ -31,10 +31,15 @@ from dataclasses import dataclass from typing import AsyncIterator from fastapi import Depends, HTTPException, Query, Request, status +from sqlalchemy import select from sqlalchemy.ext.asyncio import AsyncSession from common.auth.jwt import JwtError, verify_jwt_token -from common.auth.membership import MembershipError, load_active_membership +from common.auth.membership import ( + MembershipError, + load_active_membership, + resolve_is_system_admin, +) from common.db import session_scope from common.db.models import Roles, Users, Workspaces from common.ids import new_ulid @@ -49,10 +54,18 @@ class RequestContext: user: Users workspace: Workspaces role: Roles + # True when the requester holds the platform-scoped admin role (via + # Users.platform_role_id). Same flag exposed on ``/api/v1/auth/me`` + # so the frontend can render the platform-admin entry point. System + # admins have full control over every workspace — including disabled + # ones — so ``request_context`` lets them through and this flag is + # the single signal handlers use to gate platform-only operations. + is_system_admin: bool = False @property def is_admin(self) -> bool: - return self.role.role_code == "admin" + """Workspace admin OR system admin (the latter is strictly stronger).""" + return self.is_system_admin or self.role.role_code == "admin" async def database_session(request: Request) -> AsyncIterator[AsyncSession]: @@ -97,7 +110,7 @@ async def request_context( ), session: AsyncSession = Depends(database_session), ) -> RequestContext: - """Verify JWT and load the user's active membership for ``workspace_id``. + """Verify JWT and load the user's membership for ``workspace_id``. The ``request_id`` comes from the ``X-Request-ID`` header if present, else from a freshly minted ULID. Handlers receive the @@ -105,21 +118,59 @@ async def request_context( header-based implementation, so the 56 ``Depends(request_context)`` call sites in this repo stay working without changes — they just now pass ``?workspace_id=`` instead of the old headers. + + System admins (Users.platform_role_id → admin role) bypass the + active-membership requirement: they can address disabled workspaces + because they own the platform. Non-admin users still need an + active ``WorkspaceMembers`` row in an active ``Workspaces`` row. """ user = await current_user(request, session) - try: - _user, workspace, role = await load_active_membership( - session, user.user_id, workspace_id, - ) - except MembershipError as exc: - raise HTTPException( - status.HTTP_403_FORBIDDEN, - "active workspace membership is required", - ) from exc + is_system_admin = await resolve_is_system_admin(session, user) + + if is_system_admin: + # System admin: any workspace (active or disabled) is fine. + # Still 404 if the workspace_id is genuinely unknown — the + # ``?workspace_id=`` query param is part of the URL contract. + workspace = await session.get(Workspaces, workspace_id) + if workspace is None: + raise HTTPException( + status.HTTP_404_NOT_FOUND, + "workspace not found", + ) + # Synthesize a role object so the rest of ``RequestContext`` + # (and downstream ``is_admin`` checks) keep working without + # branching on whether a real WorkspaceMembers row exists. + role = await _load_admin_role(session) + if role is None: + # The seed migration creates this row, so this is a + # hard config error if it's missing. + raise HTTPException( + status.HTTP_500_INTERNAL_SERVER_ERROR, + "admin role not configured", + ) + else: + try: + _user, workspace, role = await load_active_membership( + session, user.user_id, workspace_id, + ) + except MembershipError as exc: + raise HTTPException( + status.HTTP_403_FORBIDDEN, + "active workspace membership is required", + ) from exc + request_id = request.headers.get("X-Request-ID") or new_ulid() return RequestContext( request_id=request_id, user=user, workspace=workspace, role=role, + is_system_admin=is_system_admin, + ) + + +async def _load_admin_role(session: AsyncSession) -> Roles | None: + """Return the singleton ``role_code='admin'`` row (None if absent).""" + return await session.scalar( + select(Roles).where(Roles.role_code == "admin") ) diff --git a/backend/src/backend/main.py b/backend/src/backend/main.py index 6752caf..78a8a8d 100644 --- a/backend/src/backend/main.py +++ b/backend/src/backend/main.py @@ -10,9 +10,16 @@ from fastapi.routing import APIRoute from common.config import settings from common.db import create_database_engine, create_session_factory from common.service_app import create_service_app -from common.storage import RustFSObjectStore +from common.storage import ( + AsyncStorageBackend, + PURPOSE_BUCKETS, + actual_bucket_name, + build_storage_config, + create_storage, +) from backend.admin import router as admin_router from backend.auth import router as auth_router +from backend.platform import router as platform_router from backend.jupyter import router as jupyter_router from backend.resources import router as resources_router from backend.runtime_client import RuntimeClient @@ -21,7 +28,6 @@ from backend.schedule_runs import router as schedule_runs_router from backend.schedules import router as schedules_router from backend.scripts import router as scripts_router from backend.storage_api import app as storage_app -from backend.storage_client import StorageClient @asynccontextmanager @@ -29,32 +35,17 @@ async def lifespan(app: Any) -> AsyncIterator[None]: engine = create_database_engine(settings.database_url) app.state.session_factory = create_session_factory(engine) - # Storage API is now part of the backend process. Platform routers keep - # their existing client contract, but calls are dispatched in-process. - app.state.object_store = RustFSObjectStore( - internal_endpoint=settings.rustfs_endpoint, - access_key=settings.rustfs_access_key, - secret_key=settings.rustfs_secret_key, - ) - # Ensure all four purpose-named buckets exist; the storage edge picks - # the right one per upload (see resolve_bucket in storage_api.py). - for bucket in ( - settings.rustfs_workspace_bucket, - settings.rustfs_version_bucket, - settings.rustfs_run_log_bucket, - settings.rustfs_trash_bucket, - ): - await asyncio.to_thread( - app.state.object_store.ensure_bucket, - bucket, - ) - app.state.default_bucket = settings.rustfs_workspace_bucket - storage_http_client = httpx.AsyncClient( - transport=httpx.ASGITransport(app=app), - base_url="http://backend.internal", - timeout=httpx.Timeout(30.0), - ) - app.state.storage_client = StorageClient(storage_http_client) + # Storage API is part of the backend process. Platform routers call + # the helpers in ``backend.services.storage`` directly (in-process), + # so no HTTP client is needed. The dict is keyed by the actual + # bucket name (e.g. "versions"), matching ``UploadSessions.bucket_name`` + # and ``StorageObjects.bucket_name`` so call sites can do + # ``object_stores[upload.bucket_name].put(...)`` directly. + app.state.object_stores: dict[str, AsyncStorageBackend] = { + actual_bucket_name(purpose): create_storage(build_storage_config(purpose)) + for purpose in PURPOSE_BUCKETS + } + app.state.default_bucket = settings.s3_workspace_bucket runtime_http_client = httpx.AsyncClient( base_url=settings.runtime_api_url, timeout=httpx.Timeout(30.0), @@ -71,7 +62,6 @@ async def lifespan(app: Any) -> AsyncIterator[None]: finally: await rclone_http_client.aclose() await runtime_http_client.aclose() - await storage_http_client.aclose() await engine.dispose() @@ -86,6 +76,7 @@ app.include_router(schedule_runs_router) app.include_router(schedules_router) app.include_router(scripts_router) app.include_router(admin_router) +app.include_router(platform_router) # Reuse the proven storage endpoints without running another FastAPI service. for route in storage_app.routes: diff --git a/backend/src/backend/platform.py b/backend/src/backend/platform.py new file mode 100644 index 0000000..5bc48cb --- /dev/null +++ b/backend/src/backend/platform.py @@ -0,0 +1,607 @@ +"""System-admin (platform-scope) endpoints for workspace & membership management. + +All routes under ``/api/v1/platform/*`` are gated by +:func:`system_admin_context`, which requires the requester to hold a +``Users.platform_role_id`` pointing to a ``Roles`` row whose +``role_code == 'admin'``. Unlike ``backend.dependencies.request_context``, +this dependency does NOT require an active workspace membership — system +admins can manage workspaces before/without being a member of any. + +Endpoints +--------- + +Workspace CRUD:: + + GET /workspaces — list non-deleted workspaces + POST /workspaces — create a new workspace + GET /workspaces/{workspace_id} — single workspace (incl. disabled) + PATCH /workspaces/{workspace_id} — update editable fields + DELETE /workspaces/{workspace_id} — soft delete (cascades memberships) + +Workspace membership CRUD:: + + GET /workspaces/{workspace_id}/members — list active members + POST /workspaces/{workspace_id}/members — add a member + PATCH /workspaces/{workspace_id}/members/{user_id} — update role/status + DELETE /workspaces/{workspace_id}/members/{user_id} — remove a member + +Invariants +---------- + +* Every workspace must always retain at least one active ``admin`` member. +* A system admin cannot remove their own workspace membership via + ``DELETE .../members/{self}``; the only escape is to delete the entire + workspace, which cascades membership soft-deletion. +* ``DELETE /workspaces/{id}`` is allowed from any non-disabled status and + sets ``status='disabled'`` + ``is_deleted=1`` + ``deleted_at`` on the + workspace and every one of its active memberships. +""" + +from __future__ import annotations + +import datetime +import re +from dataclasses import dataclass +from typing import Any, Literal + +from fastapi import APIRouter, Depends, HTTPException, Request, status +from pydantic import BaseModel, ConfigDict, Field +from sqlalchemy import func, select, update +from sqlalchemy.ext.asyncio import AsyncSession + +from backend.dependencies import current_user, database_session +from common.db.models import Roles, Users, WorkspaceMembers, Workspaces +from common.ids import new_ulid + + +router = APIRouter(prefix="/api/v1/platform", tags=["platform"]) + + +# --------------------------------------------------------------------------- +# Constants +# --------------------------------------------------------------------------- + +WORKSPACE_CODE_PATTERN = re.compile(r"^[a-z0-9-]{3,32}$") +LIST_PAGE_SIZE = 100 + +WORKSPACE_EDITABLE_STATUS = ("active", "archived") +MEMBER_ROLE_CODES = ("admin", "developer") +MEMBER_STATUS_VALUES = ("active", "disabled", "locked") + + +# --------------------------------------------------------------------------- +# Schemas +# --------------------------------------------------------------------------- + + +class WorkspaceCreate(BaseModel): + model_config = ConfigDict(extra="forbid") + + workspace_code: str = Field(min_length=3, max_length=32) + workspace_name: str = Field(min_length=1, max_length=150) + quota_bytes: int = Field(default=0, ge=0) + description: str | None = Field(default=None, max_length=1000) + + +class WorkspaceUpdate(BaseModel): + model_config = ConfigDict(extra="forbid") + + workspace_name: str | None = Field(default=None, min_length=1, max_length=150) + quota_bytes: int | None = Field(default=None, ge=0) + description: str | None = Field(default=None, max_length=1000) + # 'disabled' is rejected here on purpose — soft delete must go through DELETE. + status: Literal["active", "archived"] | None = None + + +class MemberCreate(BaseModel): + model_config = ConfigDict(extra="forbid") + + user_id: str = Field(min_length=26, max_length=26) + role_code: Literal["admin", "developer"] + + +class MemberUpdate(BaseModel): + model_config = ConfigDict(extra="forbid") + + role_code: Literal["admin", "developer"] | None = None + member_status: Literal["active", "disabled", "locked"] | None = None + + +# --------------------------------------------------------------------------- +# System-admin context dependency +# --------------------------------------------------------------------------- + + +@dataclass(frozen=True) +class SystemAdminContext: + """Resolved identity for a system-admin request. + + Carries the request id, the authenticated user row, and the resolved + ``Roles`` row the user holds via ``Users.platform_role_id``. By + construction the role's ``role_code`` is ``"admin"``. + """ + + request_id: str + user: Users + platform_role: Roles + + +async def system_admin_context( + request: Request, + session: AsyncSession = Depends(database_session), +) -> SystemAdminContext: + """Resolve the requester as a system admin. + + Steps: + 1. Reuse :func:`backend.dependencies.current_user` to validate the JWT + cookie and fetch the active ``Users`` row (raises 401 on failure). + 2. Require ``Users.platform_role_id`` to point to a row whose + ``role_code == 'admin'`` — anything else is 403. + """ + user = await current_user(request, session) + if user.platform_role_id is None: + raise HTTPException( + status.HTTP_403_FORBIDDEN, + "需要系统管理员权限", + ) + platform_role = await session.scalar( + select(Roles).where(Roles.role_id == user.platform_role_id) + ) + if platform_role is None or platform_role.role_code != "admin": + raise HTTPException( + status.HTTP_403_FORBIDDEN, + "需要系统管理员权限", + ) + request_id = request.headers.get("X-Request-ID") or new_ulid() + return SystemAdminContext( + request_id=request_id, + user=user, + platform_role=platform_role, + ) + + +# --------------------------------------------------------------------------- +# Payload helpers +# --------------------------------------------------------------------------- + + +def workspace_payload(workspace: Workspaces) -> dict[str, Any]: + return { + "workspace_id": workspace.workspace_id, + "workspace_code": workspace.workspace_code, + "workspace_name": workspace.workspace_name, + "active_root_uri": workspace.active_root_uri, + "quota_bytes": workspace.quota_bytes, + "status": workspace.status, + "description": workspace.description, + "created_by": workspace.created_by, + "created_at": workspace.created_at.isoformat(), + "updated_at": ( + workspace.updated_at.isoformat() if workspace.updated_at else None + ), + } + + +def member_payload( + user: Users, + role: Roles, + membership: WorkspaceMembers, +) -> dict[str, Any]: + return { + "user_id": user.user_id, + "username": user.username, + "display_name": user.display_name, + "email": user.email, + "user_status": user.status, + "role_code": role.role_code, + "role_name": role.role_name, + "member_status": membership.member_status, + "joined_at": membership.joined_at.isoformat(), + } + + +# --------------------------------------------------------------------------- +# Internal helpers +# --------------------------------------------------------------------------- + + +async def _load_workspace(session: AsyncSession, workspace_id: str) -> Workspaces: + workspace = await session.get(Workspaces, workspace_id) + if workspace is None: + raise HTTPException(status.HTTP_404_NOT_FOUND, "workspace 不存在") + return workspace + + +async def _load_role_by_code(session: AsyncSession, role_code: str) -> Roles: + role = await session.scalar(select(Roles).where(Roles.role_code == role_code)) + if role is None: + raise HTTPException( + status.HTTP_422_UNPROCESSABLE_ENTITY, + f"角色 {role_code} 不存在", + ) + return role + + +async def _count_active_admins( + session: AsyncSession, + workspace_id: str, + exclude_user_id: str | None = None, +) -> int: + """Count active admin members of ``workspace_id``. + + Pass ``exclude_user_id`` when checking "would X be the last admin?" + before mutating X. + """ + admin_role = await _load_role_by_code(session, "admin") + stmt = ( + select(func.count()) + .select_from(WorkspaceMembers) + .where( + WorkspaceMembers.workspace_id == workspace_id, + WorkspaceMembers.role_id == admin_role.role_id, + WorkspaceMembers.member_status == "active", + WorkspaceMembers.is_deleted == 0, + ) + ) + if exclude_user_id is not None: + stmt = stmt.where(WorkspaceMembers.user_id != exclude_user_id) + return int(await session.scalar(stmt) or 0) + + +def _envelope(request_id: str, data: Any, meta: dict[str, Any] | None = None) -> dict[str, Any]: + return { + "request_id": request_id, + "data": data, + "meta": meta or {}, + } + + +# --------------------------------------------------------------------------- +# Workspace CRUD +# --------------------------------------------------------------------------- + + +@router.get("/workspaces") +async def list_workspaces( + context: SystemAdminContext = Depends(system_admin_context), + session: AsyncSession = Depends(database_session), +) -> dict[str, Any]: + """List active/archived workspaces. Soft-deleted rows are filtered out. + + Silent ``pageSize=100`` cap — YAGNI on real pagination until needed. + """ + rows = ( + await session.execute( + select(Workspaces) + .where( + Workspaces.status != "disabled", + Workspaces.is_deleted == 0, + ) + .order_by(Workspaces.created_at, Workspaces.workspace_id) + .limit(LIST_PAGE_SIZE) + ) + ).scalars().all() + return _envelope( + context.request_id, + [workspace_payload(w) for w in rows], + {"count": len(rows), "page_size": LIST_PAGE_SIZE}, + ) + + +@router.post("/workspaces", status_code=status.HTTP_201_CREATED) +async def create_workspace( + payload: WorkspaceCreate, + context: SystemAdminContext = Depends(system_admin_context), + session: AsyncSession = Depends(database_session), +) -> dict[str, Any]: + """Create a workspace and auto-join the creator as an admin member.""" + if not WORKSPACE_CODE_PATTERN.fullmatch(payload.workspace_code): + raise HTTPException( + status.HTTP_422_UNPROCESSABLE_ENTITY, + "workspace_code 必须匹配 ^[a-z0-9-]{3,32}$", + ) + duplicate = await session.scalar( + select(Workspaces.workspace_id).where( + Workspaces.workspace_code == payload.workspace_code, + ) + ) + if duplicate is not None: + raise HTTPException(status.HTTP_409_CONFLICT, "workspace_code 已存在") + + admin_role = await _load_role_by_code(session, "admin") + workspace_id = new_ulid() + workspace = Workspaces( + workspace_id=workspace_id, + workspace_code=payload.workspace_code, + workspace_name=payload.workspace_name, + active_root_uri=f"s3://workspaces/{workspace_id}/", + quota_bytes=payload.quota_bytes, + status="active", + created_by=context.user.user_id, + description=payload.description, + ) + session.add(workspace) + session.add( + WorkspaceMembers( + workspace_id=workspace_id, + user_id=context.user.user_id, + role_id=admin_role.role_id, + member_status="active", + ) + ) + await session.flush() + await session.refresh(workspace) + return _envelope(context.request_id, workspace_payload(workspace)) + + +@router.get("/workspaces/{workspace_id}") +async def get_workspace( + workspace_id: str, + context: SystemAdminContext = Depends(system_admin_context), + session: AsyncSession = Depends(database_session), +) -> dict[str, Any]: + """Fetch a single workspace — even soft-deleted ones are reachable.""" + workspace = await _load_workspace(session, workspace_id) + return _envelope(context.request_id, workspace_payload(workspace)) + + +@router.patch("/workspaces/{workspace_id}") +async def update_workspace( + workspace_id: str, + payload: WorkspaceUpdate, + context: SystemAdminContext = Depends(system_admin_context), + session: AsyncSession = Depends(database_session), +) -> dict[str, Any]: + """Patch editable workspace fields. ``status='disabled'`` is rejected.""" + workspace = await _load_workspace(session, workspace_id) + if workspace.status == "disabled": + raise HTTPException( + status.HTTP_409_CONFLICT, + "workspace 已删除,无法修改", + ) + if payload.workspace_name is not None: + workspace.workspace_name = payload.workspace_name.strip() + if payload.quota_bytes is not None: + workspace.quota_bytes = payload.quota_bytes + if payload.description is not None: + workspace.description = payload.description + if payload.status is not None: + workspace.status = payload.status + await session.flush() + await session.refresh(workspace) + return _envelope(context.request_id, workspace_payload(workspace)) + + +@router.delete("/workspaces/{workspace_id}") +async def delete_workspace( + workspace_id: str, + context: SystemAdminContext = Depends(system_admin_context), + session: AsyncSession = Depends(database_session), +) -> dict[str, Any]: + """Soft-delete a workspace and cascade-soft-delete its memberships. + + Allowed from any non-disabled status (active or archived). The + membership cascade is what lets system admins leave a workspace — + there is no per-member DELETE escape for self-removal. + """ + workspace = await _load_workspace(session, workspace_id) + if workspace.status == "disabled": + raise HTTPException( + status.HTTP_409_CONFLICT, + "workspace 已被删除", + ) + now = datetime.datetime.utcnow() + workspace.status = "disabled" + workspace.is_deleted = 1 + workspace.deleted_at = now + await session.execute( + update(WorkspaceMembers) + .where( + WorkspaceMembers.workspace_id == workspace_id, + WorkspaceMembers.is_deleted == 0, + ) + .values(is_deleted=1, deleted_at=now) + ) + await session.flush() + await session.refresh(workspace) + return _envelope(context.request_id, workspace_payload(workspace)) + + +# --------------------------------------------------------------------------- +# Workspace membership CRUD +# --------------------------------------------------------------------------- + + +@router.get("/workspaces/{workspace_id}/members") +async def list_members( + workspace_id: str, + context: SystemAdminContext = Depends(system_admin_context), + session: AsyncSession = Depends(database_session), +) -> dict[str, Any]: + """List active and historical (non-soft-deleted) members of a workspace.""" + await _load_workspace(session, workspace_id) + rows = ( + await session.execute( + select(Users, Roles, WorkspaceMembers) + .join( + WorkspaceMembers, + WorkspaceMembers.user_id == Users.user_id, + ) + .join(Roles, Roles.role_id == WorkspaceMembers.role_id) + .where( + WorkspaceMembers.workspace_id == workspace_id, + WorkspaceMembers.is_deleted == 0, + ) + .order_by(WorkspaceMembers.joined_at, Users.user_id) + .limit(LIST_PAGE_SIZE) + ) + ).all() + return _envelope( + context.request_id, + [member_payload(u, r, m) for u, r, m in rows], + {"count": len(rows), "page_size": LIST_PAGE_SIZE}, + ) + + +@router.post( + "/workspaces/{workspace_id}/members", + status_code=status.HTTP_201_CREATED, +) +async def add_member( + workspace_id: str, + payload: MemberCreate, + context: SystemAdminContext = Depends(system_admin_context), + session: AsyncSession = Depends(database_session), +) -> dict[str, Any]: + """Add a user to a workspace. The new row starts with member_status='active'.""" + await _load_workspace(session, workspace_id) + user = await session.get(Users, payload.user_id) + if user is None: + raise HTTPException(status.HTTP_404_NOT_FOUND, "用户不存在") + role = await _load_role_by_code(session, payload.role_code) + duplicate = await session.scalar( + select(WorkspaceMembers.user_id).where( + WorkspaceMembers.workspace_id == workspace_id, + WorkspaceMembers.user_id == payload.user_id, + WorkspaceMembers.is_deleted == 0, + ) + ) + if duplicate is not None: + raise HTTPException( + status.HTTP_409_CONFLICT, + "用户已是该 workspace 成员", + ) + membership = WorkspaceMembers( + workspace_id=workspace_id, + user_id=payload.user_id, + role_id=role.role_id, + member_status="active", + ) + session.add(membership) + await session.flush() + await session.refresh(membership) + return _envelope(context.request_id, member_payload(user, role, membership)) + + +@router.patch("/workspaces/{workspace_id}/members/{user_id}") +async def update_member( + workspace_id: str, + user_id: str, + payload: MemberUpdate, + context: SystemAdminContext = Depends(system_admin_context), + session: AsyncSession = Depends(database_session), +) -> dict[str, Any]: + """Update a member's role and/or status. Last-admin guard applies.""" + await _load_workspace(session, workspace_id) + row = ( + await session.execute( + select(Users, Roles, WorkspaceMembers) + .join( + WorkspaceMembers, + WorkspaceMembers.user_id == Users.user_id, + ) + .join(Roles, Roles.role_id == WorkspaceMembers.role_id) + .where( + WorkspaceMembers.workspace_id == workspace_id, + WorkspaceMembers.user_id == user_id, + WorkspaceMembers.is_deleted == 0, + ) + ) + ).first() + if row is None: + raise HTTPException(status.HTTP_404_NOT_FOUND, "成员不存在") + user, role, membership = row + + next_role = role + if payload.role_code is not None and payload.role_code != role.role_code: + if ( + role.role_code == "admin" + and payload.role_code != "admin" + and membership.member_status == "active" + ): + remaining = await _count_active_admins( + session, workspace_id, exclude_user_id=user_id, + ) + if remaining == 0: + raise HTTPException( + status.HTTP_409_CONFLICT, + "workspace 必须保留至少一个 admin", + ) + next_role = await _load_role_by_code(session, payload.role_code) + membership.role_id = next_role.role_id + + if payload.member_status is not None and payload.member_status != membership.member_status: + if ( + role.role_code == "admin" + and payload.member_status != "active" + ): + remaining = await _count_active_admins( + session, workspace_id, exclude_user_id=user_id, + ) + if remaining == 0: + raise HTTPException( + status.HTTP_409_CONFLICT, + "workspace 必须保留至少一个 admin", + ) + membership.member_status = payload.member_status + + await session.flush() + await session.refresh(membership) + return _envelope(context.request_id, member_payload(user, next_role, membership)) + + +@router.delete("/workspaces/{workspace_id}/members/{user_id}") +async def remove_member( + workspace_id: str, + user_id: str, + context: SystemAdminContext = Depends(system_admin_context), + session: AsyncSession = Depends(database_session), +) -> dict[str, Any]: + """Soft-delete a workspace membership. + + System admins cannot remove themselves — the only escape is to delete + the entire workspace, which cascades membership soft-deletion. + """ + await _load_workspace(session, workspace_id) + if user_id == context.user.user_id: + raise HTTPException( + status.HTTP_403_FORBIDDEN, + "系统管理员不能把自己从 workspace 移除;如需退出,请删除整个 workspace", + ) + row = ( + await session.execute( + select(Roles, WorkspaceMembers) + .join(Roles, Roles.role_id == WorkspaceMembers.role_id) + .where( + WorkspaceMembers.workspace_id == workspace_id, + WorkspaceMembers.user_id == user_id, + WorkspaceMembers.is_deleted == 0, + ) + ) + ).first() + if row is None: + raise HTTPException(status.HTTP_404_NOT_FOUND, "成员不存在") + role, membership = row + if role.role_code == "admin" and membership.member_status == "active": + remaining = await _count_active_admins( + session, workspace_id, exclude_user_id=user_id, + ) + if remaining == 0: + raise HTTPException( + status.HTTP_409_CONFLICT, + "workspace 必须保留至少一个 admin", + ) + membership.is_deleted = 1 + membership.deleted_at = datetime.datetime.utcnow() + await session.flush() + return _envelope( + context.request_id, + {"workspace_id": workspace_id, "user_id": user_id, "removed": True}, + ) + + +__all__ = [ + "router", + "SystemAdminContext", + "system_admin_context", +] \ No newline at end of file diff --git a/backend/src/backend/resources.py b/backend/src/backend/resources.py index 0c9dab1..e57b61e 100644 --- a/backend/src/backend/resources.py +++ b/backend/src/backend/resources.py @@ -1,5 +1,6 @@ from __future__ import annotations +import base64 from datetime import UTC, datetime from typing import Any @@ -9,6 +10,11 @@ from sqlalchemy.ext.asyncio import AsyncSession from common.db.models import DataResources, StorageObjects from common.ids import new_ulid +from common.storage.schemas import ( + CreateUploadRequest, + DownloadUrlRequest, + ServerObjectRequest, +) from backend.dependencies import ( RequestContext, database_session, @@ -19,6 +25,13 @@ from backend.schemas import ( CreateResourceUploadRequest, DownloadUrlRequest, ) +from backend.services.storage import ( + create_download_url_payload, + create_server_object_payload, + create_upload_record, + soft_delete_object, + upload_bytes_to_session, +) router = APIRouter(prefix="/api/v1/data-resources", tags=["data-resources"]) @@ -68,52 +81,95 @@ async def create_resource_upload( alias="Idempotency-Key", ), ) -> dict[str, Any]: - data = await request.app.state.storage_client.create_upload( - { - "workspace_id": context.workspace.workspace_id, - "user_id": context.user.user_id, - "usage_type": "data_resource", - "file_name": payload.file_name, - "content_type": payload.content_type, - "expected_size_bytes": payload.expected_size_bytes, - "expected_hash": payload.expected_hash, - "idempotency_key": idempotency_key, - } + data = await create_upload_record( + CreateUploadRequest( + workspace_id=context.workspace.workspace_id, + user_id=context.user.user_id, + usage_type="data_resource", + file_name=payload.file_name, + content_type=payload.content_type, + expected_size_bytes=payload.expected_size_bytes, + expected_hash=payload.expected_hash, + idempotency_key=idempotency_key, + ), + session, + request, ) return {"request_id": context.request_id, "data": data, "meta": {}} -@router.post("/uploads/{upload_id}/complete") -async def complete_resource_upload( +@router.put("/uploads/{upload_id}") +async def upload_resource_bytes( + upload_id: str, + request: Request, + context: RequestContext = Depends(request_context), + session: AsyncSession = Depends(database_session), +) -> dict[str, Any]: + """Server-proxied upload step 2: PUT the raw bytes here. + + Replaces the old 3-step presign-PUT flow. The new flow is: + POST /uploads → {upload_id, upload_path, ...} + PUT /uploads/{upload_id} ← this route + (3) The frontend then calls a separate bind route to attach the + resulting StorageObjects row to a DataResources row. + """ + item = await upload_bytes_to_session(upload_id, session, request) + return { + "request_id": context.request_id, + "data": {"storage_object_id": item.storage_object_id}, + "meta": {}, + } + + +@router.post("/uploads/{upload_id}/bind") +async def bind_resource( upload_id: str, payload: CompleteResourceUploadRequest, request: Request, context: RequestContext = Depends(request_context), session: AsyncSession = Depends(database_session), ) -> dict[str, Any]: - storage_data = await request.app.state.storage_client.complete_upload( - upload_id, - { - "usage_type": "data_resource", - "file_name": payload.resource_name, - "visibility": payload.visibility, - "is_immutable": False, - }, + """Bind a completed upload to a DataResources row. + + Caller must have already PUT the bytes (see ``PUT /uploads/{id}``). + This route attaches the resource_name / description / visibility to + the StorageObjects row + creates the DataResources row that points + to it. + """ + from common.db.models import UploadSessions + upload = await session.scalar( + select(UploadSessions).where(UploadSessions.upload_id == upload_id) ) - if storage_data["workspace_id"] != context.workspace.workspace_id: + if upload is None: + raise HTTPException(status.HTTP_404_NOT_FOUND, "upload not found") + if upload.storage_object_id is None: + raise HTTPException( + status.HTTP_409_CONFLICT, + "upload has no completed object; PUT the bytes first", + ) + if upload.workspace_id != context.workspace.workspace_id: raise HTTPException( status.HTTP_403_FORBIDDEN, "upload belongs to another workspace", ) - if storage_data["owner_user_id"] != context.user.user_id: + if upload.user_id != context.user.user_id: raise HTTPException( status.HTTP_403_FORBIDDEN, "upload belongs to another user", ) + item = await session.get(StorageObjects, upload.storage_object_id) + if item is None: + raise HTTPException( + status.HTTP_409_CONFLICT, + "storage object metadata is missing", + ) + # Persist resource_name / description / visibility override. + item.visibility = payload.visibility + # (description lives on DataResources, not on StorageObjects.) + existing = await session.scalar( select(DataResources).where( - DataResources.storage_object_id - == storage_data["storage_object_id"] + DataResources.storage_object_id == item.storage_object_id ) ) reused = existing is not None @@ -121,7 +177,7 @@ async def complete_resource_upload( existing = DataResources( resource_id=new_ulid(), workspace_id=context.workspace.workspace_id, - storage_object_id=storage_data["storage_object_id"], + storage_object_id=item.storage_object_id, owner_user_id=context.user.user_id, resource_name=payload.resource_name, description=payload.description, @@ -131,18 +187,9 @@ async def complete_resource_upload( session.add(existing) await session.flush() await session.refresh(existing) - storage_object = await session.get( - StorageObjects, - existing.storage_object_id, - ) - if storage_object is None: - raise HTTPException( - status.HTTP_409_CONFLICT, - "storage object metadata is missing", - ) return { "request_id": context.request_id, - "data": resource_payload(existing, storage_object), + "data": resource_payload(existing, item), "meta": {"reused": reused}, } @@ -254,11 +301,12 @@ async def resource_download_url( context, session, ) - data = await request.app.state.storage_client.create_download_url( - resource.storage_object_id, - payload.expires_seconds, + data = await create_download_url_payload( + await session.get(StorageObjects, resource.storage_object_id), + DownloadUrlRequest(expires_seconds=payload.expires_seconds), + request, ) - return {"request_id": context.request_id, "data": data, "meta": {}} + return {"request_id": context.request_id, "data": data["data"], "meta": {}} @router.delete("/{resource_id}") @@ -281,9 +329,7 @@ async def delete_resource( status.HTTP_403_FORBIDDEN, "resource can only be deleted by its owner or an administrator", ) - await request.app.state.storage_client.delete_object( - resource.storage_object_id - ) + await soft_delete_object(resource.storage_object_id, request, session) resource.status = "deleted" resource.deleted_at = datetime.now(UTC).replace(tzinfo=None) return { diff --git a/backend/src/backend/scripts.py b/backend/src/backend/scripts.py index 84a36b6..dbe8940 100644 --- a/backend/src/backend/scripts.py +++ b/backend/src/backend/scripts.py @@ -1,6 +1,7 @@ from __future__ import annotations import asyncio +import base64 import hashlib import json import mimetypes @@ -34,6 +35,11 @@ from backend.dependencies import ( request_context, ) from backend.runtime_client import RuntimeClientError +from backend.services.storage import ( + create_download_url_payload, + create_server_object_payload, +) +from common.storage.schemas import ServerObjectRequest from backend.schemas import ( CreateScriptRequest, CreateWorkspaceDirectoryRequest, @@ -359,13 +365,13 @@ async def create_script_record( # Build a real StorageObjects row so the file participates in # workspace-tree / list / get queries that JOIN this table. The - # bytes live in the Jupyter mount; rclone replicates them to - # RustFS asynchronously. We mark the row "available" because the + # bytes live in the Jupyter mount; rclone replicates them to S3 + # asynchronously. We mark the row "available" because the # file is queryable as a workspace file from the user's POV; the # storage_uri points at where the replicated bytes will land. object_id = new_ulid() object_key = f"{workspace_id}/{jupyter_name}" - bucket_name = settings.rustfs_workspace_bucket + bucket_name = settings.s3_workspace_bucket relative_path = user_relative_path(context, jupyter_name) mime_type = mimetypes.guess_type(jupyter_name)[0] storage_object = StorageObjects( @@ -374,7 +380,7 @@ async def create_script_record( owner_user_id=context.user.user_id, object_type="file", usage_type="working_copy", - storage_backend="rustfs", + storage_backend="s3", bucket_name=bucket_name, object_key=object_key, object_key_hash=hashlib.sha256(object_key.encode("utf-8")).digest(), @@ -587,7 +593,7 @@ async def create_workspace_directory( status.HTTP_404_NOT_FOUND, "parent directory not found", ) - # RustFS has no real directory objects — the prefix is implicitly + # S3 has no real directory objects — the prefix is implicitly # created when a file is uploaded. Conflict detection is best-effort. existing = await session.scalar( select(StorageObjects.storage_object_id).where( @@ -931,33 +937,38 @@ async def publish_version( mimetypes.guess_type(script.script_name)[0] or "application/octet-stream" ) - artifact = await request.app.state.storage_client.create_server_object( - workspace_id=context.workspace.workspace_id, - user_id=context.user.user_id, - usage_type="version_artifact", - file_name=script.script_name, - content_type=content_type, - content=content, - visibility=payload.visibility, - is_immutable=True, - idempotency_key=f"version:{script.script_id}:{content_hash}", + artifact = await create_server_object_payload( + ServerObjectRequest( + workspace_id=context.workspace.workspace_id, + user_id=context.user.user_id, + usage_type="version_artifact", + file_name=script.script_name, + content_type=content_type, + content_base64=base64.b64encode(content).decode("ascii"), + visibility=payload.visibility, + is_immutable=True, + idempotency_key=f"version:{script.script_id}:{content_hash}", + ), + request, + session, ) current_max = await session.scalar( select(func.max(Versions.version_no)).where( Versions.script_id == script.script_id ) ) + artifact_data = artifact["data"] version_no = int(current_max or 0) + 1 version = Versions( versions_id=new_ulid(), workspace_id=context.workspace.workspace_id, script_id=script.script_id, source_object_id=script.current_object_id, - artifact_object_id=artifact["storage_object_id"], + artifact_object_id=artifact_data["storage_object_id"], version_no=version_no, version_label=f"v{version_no}.0", source_path=jupyter_name, - artifact_path=artifact["storage_uri"], + artifact_path=artifact_data["storage_uri"], content_hash=content_hash, file_size_bytes=len(content), visibility=payload.visibility, @@ -1123,8 +1134,9 @@ async def version_download_url( or version.workspace_id != context.workspace.workspace_id ): raise HTTPException(status.HTTP_404_NOT_FOUND, "version not found") - data = await request.app.state.storage_client.create_download_url( - version.artifact_object_id, - payload.expires_seconds, + data = await create_download_url_payload( + await session.get(StorageObjects, version.artifact_object_id), + DownloadUrlRequest(expires_seconds=payload.expires_seconds), + request, ) - return {"request_id": context.request_id, "data": data, "meta": {}} + return {"request_id": context.request_id, "data": data["data"], "meta": {}} diff --git a/backend/src/backend/services/__init__.py b/backend/src/backend/services/__init__.py new file mode 100644 index 0000000..e69de29 diff --git a/backend/src/backend/services/storage.py b/backend/src/backend/services/storage.py new file mode 100644 index 0000000..0789b43 --- /dev/null +++ b/backend/src/backend/services/storage.py @@ -0,0 +1,512 @@ +"""In-process storage helpers. + +The HTTP ``/internal/v1/*`` routes in ``backend.storage_api`` are wrappers +around these. Other backend modules (``scripts``, ``resources``) and the +schedule worker call these helpers directly instead of going through an +HTTP client — the storage layer lives in the same process, so the +indirection is pointless. + +Functions: + + create_upload_record — open a new upload session, returning + the upload_path (PUT-bytes) + session row. + upload_bytes_to_session — read raw bytes from request, validate, + call AsyncStorageBackend.put, build + StorageObjects row. + create_server_object_payload — server-side single-call upload (bytes + in JSON via base64). Used for small + artifacts (≤100 KiB). + create_download_url_payload — build a presigned GET URL for one + StorageObjects row. + soft_delete_object — copy-to-trash + delete source + flip row + to "deleted" with deleted_at stamp. + +These helpers raise ``HTTPException`` directly because they share an +HTTP-shaped error contract with the routes; callers can let the +exception propagate. +""" + +from __future__ import annotations + +import base64 +import binascii +import hashlib +from datetime import timedelta +from pathlib import PurePosixPath +from typing import Any + +from fastapi import HTTPException, Request, status +from sqlalchemy import select +from sqlalchemy.ext.asyncio import AsyncSession + +from common.config import settings +from common.db.models import StorageObjects, UploadSessions +from common.ids import new_ulid +from common.storage.schemas import ( + CreateUploadRequest, + DownloadUrlRequest, + ServerObjectRequest, +) + + +# ── shared low-level helpers (module-private) ──────────────────────────── + + +def _safe_file_name(value: str) -> str: + return value.strip() or "upload.bin" + + +def _utcnow_naive() -> Any: + from datetime import datetime, UTC + return datetime.now(UTC).replace(tzinfo=None) + + +def _hash_bytes(value: str) -> bytes: + import hashlib as _h + return _h.sha256(value.encode("utf-8")).digest() + + +def _build_storage_object( + *, + upload: UploadSessions, + file_name: str, + content_type: str, + size_bytes: int, + content_hash: str | None, + visibility: str, + is_immutable: bool, + usage_type: str, + owner_user_id: str | None = None, +) -> StorageObjects: + """Build the StorageObjects row that pairs with a completed UploadSessions row.""" + safe_name = _safe_file_name(file_name) + return StorageObjects( + storage_object_id=new_ulid(), + workspace_id=upload.workspace_id, + owner_user_id=owner_user_id or upload.user_id, + object_type="file", + usage_type=usage_type, + storage_backend="s3", + bucket_name=upload.bucket_name, + object_key=upload.object_key, + object_key_hash=upload.object_key_hash, + storage_uri=f"s3://{upload.bucket_name}/{upload.object_key}", + file_name=safe_name, + file_extension=PurePosixPath(safe_name).suffix.lower() or None, + mime_type=content_type, + size_bytes=size_bytes, + content_hash=content_hash, + object_etag=None, + visibility=visibility, + is_immutable=int(is_immutable), + object_status="available", + created_by=upload.user_id, + ) + + +# ── create_upload_record ──────────────────────────────────────────────── + + +def _resolve_bucket_for_usage( + usage_type: str, + *, + workspace_artifact_bucket: str | None, +) -> str: + """Mirror of storage_api.resolve_bucket, but pure (no DB / Request).""" + from backend.storage_api import BUCKET_FOR_USAGE + if workspace_artifact_bucket: + return workspace_artifact_bucket + return BUCKET_FOR_USAGE.get(usage_type, settings.s3_workspace_bucket) + + +async def create_upload_record( + payload: CreateUploadRequest, + session: AsyncSession, + request: Request, +) -> dict[str, Any]: + """Create or reuse an UploadSessions row. + + Returns ``{upload_id, status, upload_path, expires_at}`` for a fresh + session; or ``{upload_id, status: "completed", storage_object: {...}}`` + when the idempotency key hits an already-completed upload. + """ + from backend.storage_api import ( + require_workspace_member, + normalized_idempotency_key, + BUCKET_FOR_USAGE, + ) + + workspace = await require_workspace_member( + session, payload.workspace_id, payload.user_id + ) + stored_key = normalized_idempotency_key( + payload.workspace_id, payload.user_id, payload.idempotency_key + ) + existing = await session.scalar( + select(UploadSessions).where(UploadSessions.idempotency_key == stored_key) + ) + if existing is not None: + if ( + existing.workspace_id != payload.workspace_id + or existing.user_id != payload.user_id + or existing.expected_size_bytes != payload.expected_size_bytes + or existing.expected_hash != payload.expected_hash + or existing.content_type != payload.content_type + ): + raise HTTPException( + status.HTTP_409_CONFLICT, + "idempotency key was used with different upload metadata", + ) + upload = existing + else: + from datetime import timedelta + from backend.storage_api import utcnow + + bucket_name = _resolve_bucket_for_usage( + payload.usage_type, + workspace_artifact_bucket=workspace.artifact_bucket, + ) + # Keep the opaque upload id while preserving the original extension. + # Jupyter selects its editor from this suffix, so an extensionless + # object would make notebooks look like generic JSON/text files. + file_extension = PurePosixPath(_safe_file_name(payload.file_name)).suffix.lower() + object_key = f"{payload.workspace_id}/{new_ulid()}{file_extension}" + upload = UploadSessions( + upload_id=new_ulid(), + workspace_id=payload.workspace_id, + user_id=payload.user_id, + idempotency_key=stored_key, + bucket_name=bucket_name, + object_key=object_key, + object_key_hash=_hash_bytes(object_key), + upload_status="created", + expires_at=utcnow() + timedelta(minutes=15), + expected_size_bytes=payload.expected_size_bytes, + expected_hash=payload.expected_hash, + content_type=payload.content_type, + file_name=payload.file_name, + usage_type=payload.usage_type, + visibility=payload.visibility, + is_immutable=int(payload.is_immutable), + ) + session.add(upload) + await session.flush() + + if upload.upload_status == "completed" and upload.storage_object_id: + from backend.storage_api import storage_payload + storage_object = await session.get(StorageObjects, upload.storage_object_id) + if storage_object is None or storage_object.object_status != "available": + upload.storage_object_id = None + upload.upload_status = "created" + else: + return { + "upload_id": upload.upload_id, + "status": upload.upload_status, + "storage_object": storage_payload(storage_object), + } + + if upload.upload_status not in {"created", "uploading"}: + raise HTTPException( + status.HTTP_409_CONFLICT, + f"upload cannot continue from status {upload.upload_status}", + ) + + return { + "upload_id": upload.upload_id, + "status": upload.upload_status, + "upload_path": f"/internal/v1/uploads/{upload.upload_id}", + "expires_at": upload.expires_at.isoformat(), + } + + +# ── upload_bytes_to_session (server-proxied PUT) ──────────────────────── + + +async def upload_bytes_to_session( + upload_id: str, + session: AsyncSession, + request: Request, +) -> StorageObjects: + """Read raw bytes from the request body, validate against the + UploadSessions row, write via AsyncStorageBackend.put, and build the + StorageObjects row. Returns the row (caller may serialize it). + """ + upload = await session.scalar( + select(UploadSessions) + .where(UploadSessions.upload_id == upload_id) + .with_for_update() + ) + if upload is None: + raise HTTPException(status.HTTP_404_NOT_FOUND, "upload not found") + if upload.upload_status == "completed" and upload.storage_object_id: + item = await session.get(StorageObjects, upload.storage_object_id) + if item is None or item.object_status != "available": + upload.storage_object_id = None + upload.upload_status = "created" + else: + return item + if upload.upload_status not in {"created", "uploading"}: + raise HTTPException( + status.HTTP_409_CONFLICT, + f"upload cannot continue from status {upload.upload_status}", + ) + if upload.expires_at < _utcnow_naive(): + upload.upload_status = "expired" + raise HTTPException(status.HTTP_409_CONFLICT, "upload expired") + + content = await request.body() + actual_size = len(content) + + if ( + upload.expected_size_bytes is not None + and actual_size != upload.expected_size_bytes + ): + upload.upload_status = "failed" + raise HTTPException( + status.HTTP_409_CONFLICT, + "uploaded bytes size does not match expected_size_bytes", + ) + + actual_hash = hashlib.sha256(content).hexdigest() if content else "" + if upload.expected_hash and actual_hash != upload.expected_hash: + upload.upload_status = "failed" + raise HTTPException( + status.HTTP_409_CONFLICT, + "uploaded bytes hash does not match expected_hash", + ) + + s3_metadata: dict[str, str] = {} + if actual_hash: + s3_metadata["sha256"] = actual_hash + + try: + await request.app.state.object_stores[upload.bucket_name].put( + upload.object_key, + content, + content_type=upload.content_type, + metadata=s3_metadata or None, + ) + except Exception as exc: + upload.upload_status = "failed" + raise HTTPException( + status.HTTP_503_SERVICE_UNAVAILABLE, + f"failed to write object to storage: {exc}", + ) from exc + + item = _build_storage_object( + upload=upload, + file_name=upload.file_name, + content_type=upload.content_type, + size_bytes=actual_size, + content_hash=actual_hash or None, + visibility=upload.visibility, + is_immutable=bool(upload.is_immutable), + usage_type=upload.usage_type, + ) + session.add(item) + await session.flush() + await session.refresh(item) + upload.storage_object_id = item.storage_object_id + upload.upload_status = "completed" + upload.completed_at = _utcnow_naive() + return item + + +# ── create_server_object_payload ──────────────────────────────────────── + + +async def create_server_object_payload( + payload: ServerObjectRequest, + request: Request, + session: AsyncSession, +) -> dict[str, Any]: + """Server-side single-call upload (JSON body, base64 content). + + Used by scripts.py when publishing version artifacts and by the + schedule worker for run logs / run results. + """ + from backend.storage_api import storage_payload + + try: + content = base64.b64decode(payload.content_base64, validate=True) + except (binascii.Error, ValueError) as exc: + raise HTTPException( + status.HTTP_422_UNPROCESSABLE_ENTITY, + "content_base64 is invalid", + ) from exc + if len(content) > 100 * 1024 * 1024: + raise HTTPException( + status.HTTP_413_REQUEST_ENTITY_TOO_LARGE, + "object exceeds 100 MiB server-side upload limit", + ) + + content_hash = hashlib.sha256(content).hexdigest() + upload_result = await create_upload_record( + CreateUploadRequest( + workspace_id=payload.workspace_id, + user_id=payload.user_id, + usage_type=payload.usage_type, + file_name=payload.file_name, + content_type=payload.content_type, + expected_size_bytes=len(content), + expected_hash=content_hash, + idempotency_key=payload.idempotency_key, + visibility=payload.visibility, + is_immutable=payload.is_immutable, + ), + session, + request, + ) + if upload_result.get("status") == "completed": + existing_data = upload_result["storage_object"] + if ( + payload.relative_path + and existing_data + and existing_data.get("relative_path") != payload.relative_path + ): + existing_item = await session.get( + StorageObjects, existing_data["storage_object_id"] + ) + if existing_item is not None: + existing_item.relative_path = payload.relative_path + await session.flush() + existing_data = storage_payload(existing_item) + return {"data": existing_data, "meta": {"reused": True}} + + upload = await session.get(UploadSessions, upload_result["upload_id"]) + if upload is None: + raise HTTPException( + status.HTTP_500_INTERNAL_SERVER_ERROR, + "upload record disappeared", + ) + + try: + await request.app.state.object_stores[upload.bucket_name].put( + upload.object_key, + content, + content_type=payload.content_type, + metadata={"sha256": content_hash}, + ) + except Exception as exc: + upload.upload_status = "failed" + raise HTTPException( + status.HTTP_503_SERVICE_UNAVAILABLE, + f"failed to write object to storage: {exc}", + ) from exc + + item = _build_storage_object( + upload=upload, + file_name=payload.file_name, + content_type=payload.content_type, + size_bytes=len(content), + content_hash=content_hash, + visibility=payload.visibility, + is_immutable=payload.is_immutable, + usage_type=payload.usage_type, + ) + session.add(item) + await session.flush() + item.relative_path = payload.relative_path + upload.storage_object_id = item.storage_object_id + upload.upload_status = "completed" + upload.completed_at = _utcnow_naive() + return {"data": storage_payload(item), "meta": {"reused": False}} + + +# ── create_download_url_payload ───────────────────────────────────────── + + +async def create_download_url_payload( + item: StorageObjects, + payload: DownloadUrlRequest, + request: Request, +) -> dict[str, Any]: + """Build a presigned GET URL for one StorageObjects row. + + The caller (route handler in storage_api / scripts.py / resources.py) + loads the StorageObjects row + validates ownership/visibility; this + helper just builds the URL. + """ + if item is None or item.object_status != "available": + raise HTTPException(status.HTTP_404_NOT_FOUND, "object not found") + if ( + item.storage_backend != "s3" + or not item.bucket_name + or not item.object_key + ): + raise HTTPException( + status.HTTP_409_CONFLICT, + "object does not support a presigned URL", + ) + url = await request.app.state.object_stores[item.bucket_name].get_url( + item.object_key, + expires_in=timedelta(seconds=payload.expires_seconds), + ) + # Public-host rewriting is now nginx's job (location /storage/). In the + # future the boto3 client should be built with the public endpoint so + # generate_presigned_url returns a public URL directly. + return { + "data": { + "storage_object_id": item.storage_object_id, + "presigned_url": url, + "method": "GET", + "expires_in_seconds": payload.expires_seconds, + } + } + + +# ── soft_delete_object ────────────────────────────────────────────────── + + +async def soft_delete_object( + storage_object_id: str, + request: Request, + session: AsyncSession, +) -> dict[str, Any]: + """Soft-delete a storage object: copy to trash bucket, delete source, + flip the row to ``"deleted"``. Immutable objects are rejected. + """ + item = await session.scalar( + select(StorageObjects) + .where(StorageObjects.storage_object_id == storage_object_id) + .with_for_update() + ) + if item is None: + raise HTTPException(status.HTTP_404_NOT_FOUND, "object not found") + if item.is_immutable: + raise HTTPException( + status.HTTP_409_CONFLICT, + "immutable object cannot be deleted", + ) + if item.object_status == "deleted": + return { + "data": { + "storage_object_id": storage_object_id, + "object_status": item.object_status, + "trash_key": item.trash_key, + "trash_bucket": settings.s3_trash_bucket, + } + } + if item.storage_backend == "s3" and item.bucket_name and item.object_key: + trash_key = f"{item.bucket_name}/{item.object_key}" + try: + object_stores = request.app.state.object_stores + data = await object_stores[item.bucket_name].get(item.object_key) + await object_stores[settings.s3_trash_bucket].put(trash_key, data) + await object_stores[item.bucket_name].delete(item.object_key) + except Exception as exc: + raise HTTPException( + status.HTTP_502_BAD_GATEWAY, + f"failed to move object to trash: {exc}", + ) from exc + item.trash_key = trash_key + item.object_status = "deleted" + item.deleted_at = _utcnow_naive() + return { + "data": { + "storage_object_id": storage_object_id, + "object_status": item.object_status, + "trash_key": item.trash_key, + "trash_bucket": settings.s3_trash_bucket, + } + } \ No newline at end of file diff --git a/backend/src/backend/storage_api.py b/backend/src/backend/storage_api.py index 248aba7..59c1ab0 100644 --- a/backend/src/backend/storage_api.py +++ b/backend/src/backend/storage_api.py @@ -1,14 +1,8 @@ from __future__ import annotations -import asyncio -import base64 -import binascii import hashlib -import mimetypes -import secrets from contextlib import asynccontextmanager from datetime import UTC, datetime, timedelta -from pathlib import PurePosixPath from typing import Any, AsyncIterator from fastapi import Depends, HTTPException, Request, status @@ -22,15 +16,29 @@ from common.db.models import ( UploadSessions, Users, WorkspaceMembers, - Workspaces) + Workspaces, +) from common.ids import new_ulid from common.service_app import create_service_app -from common.storage import RustFSObjectStore +from common.storage import ( + AsyncStorageBackend, + PURPOSE_BUCKETS, + actual_bucket_name, + build_storage_config, + create_storage, +) from common.storage.schemas import ( - CompleteUploadRequest, CreateUploadRequest, DownloadUrlRequest, - ServerObjectRequest) + ServerObjectRequest, +) +from backend.services.storage import ( + create_download_url_payload, + create_server_object_payload, + create_upload_record, + soft_delete_object, + upload_bytes_to_session, +) def utcnow() -> datetime: @@ -41,10 +49,7 @@ def hash_bytes(value: str) -> bytes: return hashlib.sha256(value.encode("utf-8")).digest() -def normalized_idempotency_key( - workspace_id: str, - user_id: str, - value: str) -> str: +def normalized_idempotency_key(workspace_id: str, user_id: str, value: str) -> str: digest = hashlib.sha256( f"{workspace_id}:{user_id}:{value}".encode("utf-8") ).hexdigest() @@ -58,20 +63,26 @@ def safe_file_name(value: str) -> str: return name -# Map an upload's usage_type to the RustFS bucket that should hold the -# resulting object. ``usage_type`` is the only signal available at the -# storage edge (the request comes from either the public API or the -# internal schedule worker), so we make the routing decision in one place -# here and let every other layer — server-object create, multipart upload, -# direct put — inherit the mapping. +# Map an upload's usage_type to its purpose (which then resolves to the +# actual bucket / directory via ``actual_bucket_name``). Keeping the +# purpose as the intermediate value means s3 mode and local mode share +# the same routing logic — only the final ``actual_bucket_name`` differs. +USAGE_TYPE_TO_PURPOSE: dict[str, str] = { + "working_copy": "workspace", + "public_script": "workspace", + "data_resource": "workspace", + "snapshot": "workspace", + "version_artifact": "version", + "run_log": "run_log", + "run_result": "run_log", +} + +# Pre-resolved bucket map (for read-only callers like services/storage.py). +# Re-resolved at module load; re-resolve via resolve_bucket() if the +# workspace.artifact_bucket override matters. BUCKET_FOR_USAGE: dict[str, str] = { - "working_copy": settings.rustfs_workspace_bucket, - "public_script": settings.rustfs_workspace_bucket, - "data_resource": settings.rustfs_workspace_bucket, - "snapshot": settings.rustfs_workspace_bucket, - "version_artifact": settings.rustfs_version_bucket, - "run_log": settings.rustfs_run_log_bucket, - "run_result": settings.rustfs_run_log_bucket, + usage_type: actual_bucket_name(purpose) + for usage_type, purpose in USAGE_TYPE_TO_PURPOSE.items() } @@ -88,7 +99,8 @@ def resolve_bucket( """ if workspace.artifact_bucket: return workspace.artifact_bucket - return BUCKET_FOR_USAGE.get(usage_type, settings.rustfs_workspace_bucket) + purpose = USAGE_TYPE_TO_PURPOSE.get(usage_type, "workspace") + return actual_bucket_name(purpose) def storage_payload(item: StorageObjects) -> dict[str, Any]: @@ -118,35 +130,23 @@ def storage_payload(item: StorageObjects) -> dict[str, Any]: async def lifespan(app: Any) -> AsyncIterator[None]: engine = create_database_engine(settings.database_url) app.state.session_factory = create_session_factory(engine) - app.state.object_store = RustFSObjectStore( - internal_endpoint=settings.rustfs_endpoint, - access_key=settings.rustfs_access_key, - secret_key=settings.rustfs_secret_key, - ) - app.state.default_bucket = settings.rustfs_workspace_bucket - # Ensure every purpose-named bucket exists up front, including the - # trash bucket. The trash bucket is shared across all workspaces - # and usage_types; the source key is preserved as a prefix so a - # restore is a same-key move back to the source bucket. - for bucket in ( - settings.rustfs_workspace_bucket, - settings.rustfs_version_bucket, - settings.rustfs_run_log_bucket, - settings.rustfs_trash_bucket, - ): - await asyncio.to_thread( - app.state.object_store.ensure_bucket, - bucket, - ) + # Buckets are pre-provisioned by the deployment; the storage layer no + # longer auto-creates them. ``build_storage_config`` picks s3 vs local + # based on ``settings.storage_backend``. The dict is keyed by the + # actual bucket name so ``object_stores[upload.bucket_name]`` + # works without a reverse mapping. + app.state.object_stores: dict[str, AsyncStorageBackend] = { + actual_bucket_name(purpose): create_storage(build_storage_config(purpose)) + for purpose in PURPOSE_BUCKETS + } + app.state.default_bucket = settings.s3_workspace_bucket try: yield finally: await engine.dispose() -app = create_service_app( - settings.service_name, - lifespan=lifespan) +app = create_service_app(settings.service_name, lifespan=lifespan) async def database_session(request: Request) -> AsyncIterator[AsyncSession]: @@ -155,46 +155,41 @@ async def database_session(request: Request) -> AsyncIterator[AsyncSession]: async def require_workspace_member( - session: AsyncSession, - workspace_id: str, - user_id: str) -> Workspaces: + session: AsyncSession, workspace_id: str, user_id: str +) -> Workspaces: statement = ( select(Workspaces) .join( - WorkspaceMembers, - WorkspaceMembers.workspace_id == Workspaces.workspace_id) + WorkspaceMembers, WorkspaceMembers.workspace_id == Workspaces.workspace_id + ) .join(Users, Users.user_id == WorkspaceMembers.user_id) .where( Workspaces.workspace_id == workspace_id, Workspaces.status == "active", WorkspaceMembers.user_id == user_id, WorkspaceMembers.member_status == "active", - Users.status == "active") + Users.status == "active", + ) ) workspace = await session.scalar(statement) if workspace is None: raise HTTPException( - status.HTTP_403_FORBIDDEN, - "user is not an active workspace member") + status.HTTP_403_FORBIDDEN, "user is not an active workspace member" + ) return workspace async def create_upload_record( - payload: CreateUploadRequest, - session: AsyncSession, - request: Request) -> dict[str, Any]: + payload: CreateUploadRequest, session: AsyncSession, request: Request +) -> dict[str, Any]: workspace = await require_workspace_member( - session, - payload.workspace_id, - payload.user_id) + session, payload.workspace_id, payload.user_id + ) stored_key = normalized_idempotency_key( - payload.workspace_id, - payload.user_id, - payload.idempotency_key) + payload.workspace_id, payload.user_id, payload.idempotency_key + ) existing = await session.scalar( - select(UploadSessions).where( - UploadSessions.idempotency_key == stored_key - ) + select(UploadSessions).where(UploadSessions.idempotency_key == stored_key) ) if existing is not None: if ( @@ -206,7 +201,8 @@ async def create_upload_record( ): raise HTTPException( status.HTTP_409_CONFLICT, - "idempotency key was used with different upload metadata") + "idempotency key was used with different upload metadata", + ) upload = existing else: upload_id = new_ulid() @@ -214,9 +210,7 @@ async def create_upload_record( # Keep the opaque upload id while preserving the original extension. # Jupyter selects its editor from this suffix, so an extensionless # object would make notebooks look like generic JSON/text files. - file_extension = PurePosixPath( - safe_file_name(payload.file_name) - ).suffix.lower() + file_extension = PurePosixPath(safe_file_name(payload.file_name)).suffix.lower() object_key = f"{payload.workspace_id}/{upload_id}{file_extension}" upload = UploadSessions( upload_id=upload_id, @@ -230,14 +224,17 @@ async def create_upload_record( expires_at=utcnow() + timedelta(minutes=15), expected_size_bytes=payload.expected_size_bytes, expected_hash=payload.expected_hash, - content_type=payload.content_type) + content_type=payload.content_type, + file_name=payload.file_name, + usage_type=payload.usage_type, + visibility=payload.visibility, + is_immutable=int(payload.is_immutable), + ) session.add(upload) await session.flush() if upload.upload_status == "completed" and upload.storage_object_id: - storage_object = await session.get( - StorageObjects, - upload.storage_object_id) + storage_object = await session.get(StorageObjects, upload.storage_object_id) if storage_object is None or storage_object.object_status != "available": # The previously-completed object was deleted (or never # materialized). Treat the idempotency hit as a tombstone @@ -254,24 +251,16 @@ async def create_upload_record( if upload.upload_status not in {"created", "uploading"}: raise HTTPException( status.HTTP_409_CONFLICT, - f"upload cannot continue from status {upload.upload_status}") + f"upload cannot continue from status {upload.upload_status}", + ) - url, headers = request.app.state.object_store.presign_put( - bucket_name=upload.bucket_name, - object_key=upload.object_key, - content_type=upload.content_type or "application/octet-stream", - expected_hash=upload.expected_hash, - expires_seconds=900) - presigned_url = request.app.state.object_store.rewrite_to_public_path( - url, - public_base_url=_public_base_url(request), - ) + # Two-step server-proxied upload: the caller PUTs the raw bytes to + # ``upload_path`` after this response, which routes through + # ``upload_bytes_to_session`` below. return { "upload_id": upload.upload_id, "status": upload.upload_status, - "method": "PUT", - "presigned_url": presigned_url, - "required_headers": headers, + "upload_path": f"/internal/v1/uploads/{upload.upload_id}", "expires_at": upload.expires_at.isoformat(), } @@ -282,7 +271,7 @@ def _public_base_url(request: Request) -> str: Falls back to the inbound request's ``Host`` header and the scheme Nginx forwards via ``X-Forwarded-Proto`` so the resulting presigned URL always points at the public edge rather than the - in-cluster RustFS endpoint. + in-cluster S3 endpoint. """ forwarded_proto = request.headers.get("x-forwarded-proto", "").strip() scheme = forwarded_proto or request.url.scheme or "http" @@ -298,11 +287,15 @@ def _public_base_url(request: Request) -> str: return f"{scheme}://{host}" -async def complete_upload_record( - upload_id: str, - payload: CompleteUploadRequest, - session: AsyncSession, - request: Request) -> StorageObjects: +async def upload_bytes_to_session( + upload_id: str, session: AsyncSession, request: Request +) -> StorageObjects: + """Server-proxied upload: read raw bytes from the request body, validate + against the ``UploadSessions`` expectations, call ``backend.put``, and + create the ``StorageObjects`` row. + + Replaces the old presign-PUT + head-validate flow. + """ upload = await session.scalar( select(UploadSessions) .where(UploadSessions.upload_id == upload_id) @@ -313,9 +306,7 @@ async def complete_upload_record( if upload.upload_status == "completed" and upload.storage_object_id: item = await session.get(StorageObjects, upload.storage_object_id) if item is None or item.object_status != "available": - # The linked storage object was deleted. Reset the upload so - # the caller can re-upload the same bytes and create a - # fresh, available object. + # Linked object was deleted; allow re-upload with the same id. upload.storage_object_id = None upload.upload_status = "created" else: @@ -323,22 +314,15 @@ async def complete_upload_record( if upload.upload_status not in {"created", "uploading"}: raise HTTPException( status.HTTP_409_CONFLICT, - f"upload cannot be completed from status {upload.upload_status}") + f"upload cannot continue from status {upload.upload_status}", + ) if upload.expires_at < utcnow(): upload.upload_status = "expired" raise HTTPException(status.HTTP_409_CONFLICT, "upload expired") - try: - head = await asyncio.to_thread( - request.app.state.object_store.head, - bucket_name=upload.bucket_name, - object_key=upload.object_key) - except Exception as exc: - raise HTTPException( - status.HTTP_409_CONFLICT, - "uploaded object is not available") from exc + content = await request.body() + actual_size = len(content) - actual_size = int(head.get("ContentLength", 0)) if ( upload.expected_size_bytes is not None and actual_size != upload.expected_size_bytes @@ -346,57 +330,59 @@ async def complete_upload_record( upload.upload_status = "failed" raise HTTPException( status.HTTP_409_CONFLICT, - "uploaded object size does not match expected_size_bytes") - actual_content_type = str( - head.get("ContentType") or "application/octet-stream" - ) - if upload.content_type and actual_content_type != upload.content_type: - upload.upload_status = "failed" - raise HTTPException( - status.HTTP_409_CONFLICT, - "uploaded object content type does not match") - metadata = { - str(key).lower(): str(value).lower() - for key, value in dict(head.get("Metadata") or {}).items() - } - actual_hash = metadata.get("sha256") - if not actual_hash: - actual_hash = await asyncio.to_thread( - request.app.state.object_store.sha256, - bucket_name=upload.bucket_name, - object_key=upload.object_key) + "uploaded bytes size does not match expected_size_bytes", + ) + + actual_hash = hashlib.sha256(content).hexdigest() if content else "" if upload.expected_hash and actual_hash != upload.expected_hash: upload.upload_status = "failed" raise HTTPException( - status.HTTP_409_CONFLICT, - "uploaded object hash does not match expected_hash") + status.HTTP_409_CONFLICT, "uploaded bytes hash does not match expected_hash" + ) - # The object key is just ``{workspace_id}/{ulid}`` — it does not encode - # the file name. Use the original file name from the upload session - # (carried via payload.file_name) so the StorageObjects row still - # records the user-visible name + extension. - file_name = safe_file_name(payload.file_name) + # Round-trip content_type + sha256 metadata through the storage backend + # so the next head() (or our own put signature) can recover them. + s3_metadata: dict[str, str] = {} + if actual_hash: + s3_metadata["sha256"] = actual_hash + + try: + await request.app.state.object_stores[upload.bucket_name].put( + upload.object_key, + content, + content_type=upload.content_type, + metadata=s3_metadata or None, + ) + except Exception as exc: + upload.upload_status = "failed" + raise HTTPException( + status.HTTP_503_SERVICE_UNAVAILABLE, + f"failed to write object to storage: {exc}", + ) from exc + + file_name = safe_file_name(upload.file_name_hint or "upload.bin") item = StorageObjects( storage_object_id=new_ulid(), workspace_id=upload.workspace_id, owner_user_id=upload.user_id, object_type="file", - usage_type=payload.usage_type, - storage_backend="rustfs", + usage_type=upload.usage_type, + storage_backend="s3", bucket_name=upload.bucket_name, object_key=upload.object_key, object_key_hash=upload.object_key_hash, storage_uri=f"s3://{upload.bucket_name}/{upload.object_key}", file_name=file_name, file_extension=PurePosixPath(file_name).suffix.lower() or None, - mime_type=actual_content_type, + mime_type=upload.content_type, size_bytes=actual_size, - content_hash=actual_hash, - object_etag=str(head.get("ETag", "")).strip('"') or None, - visibility=payload.visibility, - is_immutable=int(payload.is_immutable), + content_hash=actual_hash or None, + object_etag=None, + visibility=upload.visibility, + is_immutable=int(upload.is_immutable), object_status="available", - created_by=upload.user_id) + created_by=upload.user_id, + ) session.add(item) await session.flush() await session.refresh(item) @@ -410,33 +396,29 @@ async def complete_upload_record( async def create_upload( payload: CreateUploadRequest, request: Request, - session: AsyncSession = Depends(database_session)) -> dict[str, Any]: + session: AsyncSession = Depends(database_session), +) -> dict[str, Any]: return { "data": await create_upload_record(payload, session, request), } -@app.post( - "/internal/v1/uploads/{upload_id}/complete") -async def complete_upload( - upload_id: str, - payload: CompleteUploadRequest, - request: Request, - session: AsyncSession = Depends(database_session)) -> dict[str, Any]: - item = await complete_upload_record( - upload_id, - payload, - session, - request) +@app.put("/internal/v1/uploads/{upload_id}") +async def upload_bytes( + upload_id: str, request: Request, session: AsyncSession = Depends(database_session) +) -> dict[str, Any]: + """Server-proxied upload: PUT raw bytes in the request body. Replaces the + old ``POST /uploads/{id}/complete`` flow that paired presigned-PUT with + a head()-validate step. + """ + item = await upload_bytes_to_session(upload_id, session, request) return {"data": storage_payload(item)} -@app.post( - "/internal/v1/uploads/{upload_id}/abort") +@app.post("/internal/v1/uploads/{upload_id}/abort") async def abort_upload( - upload_id: str, - request: Request, - session: AsyncSession = Depends(database_session)) -> dict[str, Any]: + upload_id: str, request: Request, session: AsyncSession = Depends(database_session) +) -> dict[str, Any]: upload = await session.scalar( select(UploadSessions) .where(UploadSessions.upload_id == upload_id) @@ -446,202 +428,52 @@ async def abort_upload( raise HTTPException(status.HTTP_404_NOT_FOUND, "upload not found") if upload.upload_status == "completed": raise HTTPException( - status.HTTP_409_CONFLICT, - "completed upload cannot be aborted") + status.HTTP_409_CONFLICT, "completed upload cannot be aborted" + ) if upload.upload_status != "aborted": - await asyncio.to_thread( - request.app.state.object_store.delete, - bucket_name=upload.bucket_name, - object_key=upload.object_key) + await request.app.state.object_stores[upload.bucket_name].delete( + upload.object_key + ) upload.upload_status = "aborted" return {"data": {"upload_id": upload_id, "status": "aborted"}} -@app.post( - "/internal/v1/objects") +@app.post("/internal/v1/objects") async def create_server_object( payload: ServerObjectRequest, request: Request, - session: AsyncSession = Depends(database_session)) -> dict[str, Any]: - try: - content = base64.b64decode(payload.content_base64, validate=True) - except (binascii.Error, ValueError) as exc: - raise HTTPException( - status.HTTP_422_UNPROCESSABLE_ENTITY, - "content_base64 is invalid") from exc - if len(content) > 100 * 1024 * 1024: - raise HTTPException( - status.HTTP_413_REQUEST_ENTITY_TOO_LARGE, - "object exceeds 100 MiB server-side upload limit") - content_hash = hashlib.sha256(content).hexdigest() - upload_result = await create_upload_record( - CreateUploadRequest( - workspace_id=payload.workspace_id, - user_id=payload.user_id, - usage_type=payload.usage_type, - file_name=payload.file_name, - content_type=payload.content_type, - expected_size_bytes=len(content), - expected_hash=content_hash, - idempotency_key=payload.idempotency_key), - session, - request) - if upload_result.get("status") == "completed": - existing_data = upload_result["storage_object"] - if ( - payload.relative_path - and existing_data - and existing_data.get("relative_path") != payload.relative_path - ): - existing_item = await session.get( - StorageObjects, - existing_data["storage_object_id"], - ) - if existing_item is not None: - existing_item.relative_path = payload.relative_path - await session.flush() - existing_data = storage_payload(existing_item) - return {"data": existing_data, "meta": {"reused": True}} - - upload = await session.get(UploadSessions, upload_result["upload_id"]) - if upload is None: - raise HTTPException( - status.HTTP_500_INTERNAL_SERVER_ERROR, - "upload record disappeared") - await asyncio.to_thread( - request.app.state.object_store.put_bytes, - bucket_name=upload.bucket_name, - object_key=upload.object_key, - content=content, - content_type=payload.content_type, - content_hash=content_hash) - item = await complete_upload_record( - upload.upload_id, - CompleteUploadRequest( - usage_type=payload.usage_type, - file_name=payload.file_name, - visibility=payload.visibility, - is_immutable=payload.is_immutable), - session, - request) - item.relative_path = payload.relative_path - await session.flush() - return {"data": storage_payload(item), "meta": {"reused": False}} + session: AsyncSession = Depends(database_session), +) -> dict[str, Any]: + return await create_server_object_payload(payload, request, session) -@app.post( - "/internal/v1/objects/{storage_object_id}/download-url") +@app.post("/internal/v1/objects/{storage_object_id}/download-url") async def create_download_url( storage_object_id: str, payload: DownloadUrlRequest, request: Request, - session: AsyncSession = Depends(database_session)) -> dict[str, Any]: + session: AsyncSession = Depends(database_session), +) -> dict[str, Any]: item = await session.get(StorageObjects, storage_object_id) - if item is None or item.object_status != "available": - raise HTTPException(status.HTTP_404_NOT_FOUND, "object not found") - if ( - item.storage_backend != "rustfs" - or not item.bucket_name - or not item.object_key - ): - raise HTTPException( - status.HTTP_409_CONFLICT, - "object does not support a presigned URL") - url = request.app.state.object_store.presign_get( - bucket_name=item.bucket_name, - object_key=item.object_key, - file_name=item.file_name, - expires_seconds=payload.expires_seconds) - presigned_url = request.app.state.object_store.rewrite_to_public_path( - url, - public_base_url=_public_base_url(request), - ) - return { - "data": { - "storage_object_id": item.storage_object_id, - "presigned_url": presigned_url, - "method": "GET", - "expires_in_seconds": payload.expires_seconds, - } - } + return await create_download_url_payload(item, payload, request) -@app.delete( - "/internal/v1/objects/{storage_object_id}") +@app.delete("/internal/v1/objects/{storage_object_id}") async def delete_object( storage_object_id: str, request: Request, - session: AsyncSession = Depends(database_session)) -> dict[str, Any]: - """Soft-delete a storage object. - - The bytes are copied to ``rustfs_trash_bucket`` under the same key - (preserved as ``{source_bucket}/{object_key}`` so a restore is a - same-name move), the source key is then deleted from its origin - bucket, and the row's ``object_status`` flips to ``"deleted"`` with - ``deleted_at`` stamped for the reaper. - - Immutable artifacts (version snapshots, run logs) are not - trashed — the policy is enforced by ``is_immutable`` and an - explicit 409. The reaper will physically delete trashed objects - older than ``rustfs_trash_retention_days`` (out of scope for this - endpoint; the field is the contract). - """ - item = await session.scalar( - select(StorageObjects) - .where(StorageObjects.storage_object_id == storage_object_id) - .with_for_update() - ) - if item is None: - raise HTTPException(status.HTTP_404_NOT_FOUND, "object not found") - if item.is_immutable: - raise HTTPException( - status.HTTP_409_CONFLICT, - "immutable object cannot be deleted") - if item.object_status == "deleted": - return { - "data": { - "storage_object_id": storage_object_id, - "object_status": item.object_status, - "trash_key": item.trash_key, - } - } - if item.storage_backend == "rustfs" and item.bucket_name and item.object_key: - trash_key = f"{item.bucket_name}/{item.object_key}" - try: - await asyncio.to_thread( - request.app.state.object_store.move_to_trash, - source_bucket=item.bucket_name, - source_key=item.object_key, - trash_bucket=settings.rustfs_trash_bucket, - trash_key=trash_key, - ) - except Exception as exc: - # If the move fails, leave the source intact and surface the - # error. We do NOT mark the row as deleted in that case — - # otherwise we'd have a row pointing to non-existent bytes. - raise HTTPException( - status.HTTP_502_BAD_GATEWAY, - f"failed to move object to trash: {exc}", - ) from exc - item.trash_key = trash_key - item.object_status = "deleted" - item.deleted_at = utcnow() - return { - "data": { - "storage_object_id": storage_object_id, - "object_status": item.object_status, - "trash_key": item.trash_key, - "trash_bucket": settings.rustfs_trash_bucket, - } - } + session: AsyncSession = Depends(database_session), +) -> dict[str, Any]: + """Soft-delete a storage object. See ``backend.services.storage.soft_delete_object``.""" + return await soft_delete_object(storage_object_id, request, session) -@app.post( - "/internal/v1/objects/{storage_object_id}/restore") +@app.post("/internal/v1/objects/{storage_object_id}/restore") async def restore_object( storage_object_id: str, request: Request, - session: AsyncSession = Depends(database_session)) -> dict[str, Any]: + session: AsyncSession = Depends(database_session), +) -> dict[str, Any]: """Restore a soft-deleted object from the trash bucket. Copies the bytes back to the source bucket + key and flips the @@ -660,21 +492,16 @@ async def restore_object( if item is None: raise HTTPException(status.HTTP_404_NOT_FOUND, "object not found") if item.object_status != "deleted": - raise HTTPException( - status.HTTP_409_CONFLICT, - "object is not in trash") + raise HTTPException(status.HTTP_409_CONFLICT, "object is not in trash") if not item.trash_key or not item.bucket_name or not item.object_key: raise HTTPException( - status.HTTP_409_CONFLICT, - "object has no trash pointer; cannot restore") - try: - await asyncio.to_thread( - request.app.state.object_store.copy, - source_bucket=settings.rustfs_trash_bucket, - source_key=item.trash_key, - dest_bucket=item.bucket_name, - dest_key=item.object_key, + status.HTTP_409_CONFLICT, "object has no trash pointer; cannot restore" ) + try: + # Cross-backend copy: get from trash, put back to source bucket. + object_stores = request.app.state.object_stores + data = await object_stores[settings.s3_trash_bucket].get(item.trash_key) + await object_stores[item.bucket_name].put(item.object_key, data) except Exception as exc: raise HTTPException( status.HTTP_502_BAD_GATEWAY, @@ -690,12 +517,12 @@ async def restore_object( } -@app.post( - "/internal/v1/admin/trash/purge") +@app.post("/internal/v1/admin/trash/purge") async def purge_trash_object( payload: dict[str, Any], request: Request, - session: AsyncSession = Depends(database_session)) -> dict[str, Any]: + session: AsyncSession = Depends(database_session), +) -> dict[str, Any]: """Physically delete a trashed object. Admin / reaper endpoint — given a ``storage_object_id``, deletes @@ -707,8 +534,8 @@ async def purge_trash_object( storage_object_id = (payload or {}).get("storage_object_id", "").strip() if not storage_object_id: raise HTTPException( - status.HTTP_400_BAD_REQUEST, - "storage_object_id is required") + status.HTTP_400_BAD_REQUEST, "storage_object_id is required" + ) item = await session.scalar( select(StorageObjects) .where(StorageObjects.storage_object_id == storage_object_id) @@ -719,13 +546,12 @@ async def purge_trash_object( if item.object_status != "deleted": raise HTTPException( status.HTTP_409_CONFLICT, - "object is not in trash; refuse to hard-delete live data") + "object is not in trash; refuse to hard-delete live data", + ) if item.trash_key: try: - await asyncio.to_thread( - request.app.state.object_store.delete, - bucket_name=settings.rustfs_trash_bucket, - object_key=item.trash_key, + await request.app.state.object_stores[settings.s3_trash_bucket].delete( + item.trash_key ) except Exception as exc: raise HTTPException( diff --git a/backend/src/backend/storage_client.py b/backend/src/backend/storage_client.py deleted file mode 100644 index bf924d3..0000000 --- a/backend/src/backend/storage_client.py +++ /dev/null @@ -1,66 +0,0 @@ -"""Backend-bound storage client. - -Re-exports :class:`StorageClient` under the same name used by callers in -``backend/``. The default client raises :class:`StorageClientError` from -``common.storage.client`` so it stays usable from non-FastAPI contexts. -Inside FastAPI route handlers we want HTTP-shaped errors, so this module -also exposes :class:`BackendStorageClient`, a thin wrapper that translates -the framework-agnostic errors into ``HTTPException``. -""" - -from __future__ import annotations - -from typing import Any - -from fastapi import HTTPException, status - -from common.storage.client import ( - StorageClient, - StorageClientError, - StorageRequestFailed, - StorageUnavailable, -) - - -__all__ = ["BackendStorageClient", "StorageClient", "StorageClientError"] - - -def _to_http_exception(exc: StorageClientError) -> HTTPException: - if isinstance(exc, StorageUnavailable): - return HTTPException( - status.HTTP_503_SERVICE_UNAVAILABLE, - { - "code": "STORAGE_UNAVAILABLE", - "message": "Storage service temporarily unavailable", - "retryable": True, - "details": {}, - }, - ) - if isinstance(exc, StorageRequestFailed): - return HTTPException(exc.status_code, exc.detail) - return HTTPException( - status.HTTP_500_INTERNAL_SERVER_ERROR, - "Storage client error", - ) - - -class BackendStorageClient(StorageClient): - """Storage client that raises ``HTTPException`` for web callers.""" - - async def _request( - self, - method: str, - path: str, - *, - payload: dict[str, Any] | None = None, - ) -> dict[str, Any]: - try: - return await super()._request(method, path, payload=payload) - except StorageClientError as exc: - raise _to_http_exception(exc) from exc - - -# Re-bind the imported symbol so existing backend call sites that import -# ``StorageClient`` from this module transparently get the FastAPI-bound -# variant without changing every import statement. -StorageClient = BackendStorageClient diff --git a/common/pyproject.toml b/common/pyproject.toml index 3de5494..d39a628 100644 --- a/common/pyproject.toml +++ b/common/pyproject.toml @@ -12,6 +12,8 @@ dependencies = [ "pydantic-settings>=2.14.2", "passlib==1.7.4", "bcrypt>=4.0,<4.1", + "aiofiles>=25.1.0", + "aioboto3>=15.5.0", ] [build-system] diff --git a/common/src/common/auth/membership.py b/common/src/common/auth/membership.py index b6e528d..fed91ad 100644 --- a/common/src/common/auth/membership.py +++ b/common/src/common/auth/membership.py @@ -22,6 +22,34 @@ class MembershipError(Exception): """Raised when the user is not an active member of the workspace.""" +async def resolve_is_system_admin( + session: AsyncSession, + user: Users, +) -> bool: + """Return True iff the user holds a platform-scoped admin role. + + The check is: ``Users.status == 'active'`` AND + ``Users.platform_role_id`` points to a ``Roles`` row whose + ``role_code == 'admin'``. Any other shape (no platform_role_id, + disabled user, wrong role code) returns False — the frontend reads + this to decide whether to show the system-admin entry point. + + System admins own the platform: they can address disabled workspaces, + bypass per-workspace membership checks, etc. Anything that wants to + gate "platform-only" behavior (deleting a workspace, soft-deleting + a user globally, …) should consult this flag — it is the single + source of truth. + """ + from sqlalchemy import select + + if user.status != "active" or user.platform_role_id is None: + return False + platform_role = await session.scalar( + select(Roles).where(Roles.role_id == user.platform_role_id) + ) + return platform_role is not None and platform_role.role_code == "admin" + + async def load_active_membership( session: AsyncSession, user_id: str, @@ -66,4 +94,5 @@ async def load_active_membership( __all__ = [ "MembershipError", "load_active_membership", + "resolve_is_system_admin", ] diff --git a/common/src/common/config.py b/common/src/common/config.py index d0a9a55..4bb9995 100644 --- a/common/src/common/config.py +++ b/common/src/common/config.py @@ -59,35 +59,57 @@ class Settings(BaseSettings): description="Backend → rclone RC HTTP endpoint (VFS cache invalidation).", ) - # ── RustFS object storage ──────────────────────────────────── - rustfs_endpoint: str = Field( - default="http://rustfs:9000", - description="S3 endpoint for the RustFS upstream.", + # ── object storage backend selection ───────────────────────── + storage_backend: str = Field( + default="s3", + description=( + "Which storage backend the deployment uses. ``s3`` (default) " + "reads the ``s3_*`` settings and connects to an S3-compatible " + "service. ``local`` uses on-disk filesystems under " + "``local_storage_base_dir`` — useful for dev / single-node / " + "air-gapped deployments." + ), ) - rustfs_access_key: str = Field( + local_storage_base_dir: str = Field( + default="/data", + description=( + "Root directory for the local-filesystem storage backend. The 4 " + "buckets become subdirectories: ``/workspace``, " + "``/version``, ``/run_log``, ``/trash``. " + "Default ``/data``; this directory must be a shared Docker " + "volume between the backend and runtime containers in local mode." + ), + ) + + # ── S3-compatible object storage ───────────────────────────── + s3_endpoint: str = Field( + default="http://s3:9000", + description="S3 endpoint for the object-storage upstream.", + ) + s3_access_key: str = Field( default="modelplatform", - description="boto3 access key for RustFS.", + description="boto3 access key for S3-compatible storage.", ) - rustfs_secret_key: str = Field( + s3_secret_key: str = Field( default="modelplatformsecret", - description="boto3 secret key for RustFS.", + description="boto3 secret key for S3-compatible storage.", ) - rustfs_workspace_bucket: str = Field( + s3_workspace_bucket: str = Field( default="workspaces", description=( "Bucket for workspace files (notebooks / scripts / working " "copies). Layout: s3:////..." ), ) - rustfs_version_bucket: str = Field( + s3_version_bucket: str = Field( default="versions", description="Bucket for immutable script-version artifacts.", ) - rustfs_run_log_bucket: str = Field( + s3_run_log_bucket: str = Field( default="run-logs", description="Bucket for schedule run logs.", ) - rustfs_trash_bucket: str = Field( + s3_trash_bucket: str = Field( default="trash", description=( "Bucket for soft-deleted objects. The source bucket key is " @@ -95,7 +117,7 @@ class Settings(BaseSettings): "Trash is reaped on a schedule out of band." ), ) - rustfs_trash_retention_days: int = Field( + s3_trash_retention_days: int = Field( default=30, description=( "How long a trashed object is retained before reaping. " @@ -104,26 +126,6 @@ class Settings(BaseSettings): ), ) - # ── local FS roots ──────────────────────────────────────────── - workspace_root: str = Field( - default="/app/workspaces", - description=( - "Schedule subprocess cwd; staging area for notebook_runner. " - "Backend no longer writes here — workspace files live in " - "RustFS via rustfs_workspace_bucket." - ), - ) - workspaces_root: str = Field( - default="/app/workspaces", - description="Runtime rclone FUSE mount point for the workspace bucket.", - ) - - # ── rclone remote spec ──────────────────────────────────────── - remote_bucket: str = Field( - default="rustfs:workspaces", - description="rclone remote spec for the workspace bucket.", - ) - # ── schedule → backend API ──────────────────────────────────── backend_api_url: str = Field( default="http://backend:8000", diff --git a/common/src/common/db/models/__init__.py b/common/src/common/db/models/__init__.py index 15ad04a..c03058d 100644 --- a/common/src/common/db/models/__init__.py +++ b/common/src/common/db/models/__init__.py @@ -1,8 +1,6 @@ from common.db.base import Base -from common.db.models.audit import AuditLogs from common.db.models.events import ConsumerInbox, OutboxEvents from common.db.models.identity import Permissions, RolePermissions, Roles, Users -from common.db.models.runtime import WorkspaceOperations from common.db.models.schedules import ( ScheduleEdges, ScheduleNodeRuns, @@ -16,7 +14,6 @@ from common.db.models.workspaces import WorkspaceMembers, Workspaces __all__ = [ "Base", - "AuditLogs", "ConsumerInbox", "DataResources", "OutboxEvents", @@ -34,6 +31,5 @@ __all__ = [ "Users", "Versions", "WorkspaceMembers", - "WorkspaceOperations", "Workspaces", ] diff --git a/common/src/common/db/models/audit.py b/common/src/common/db/models/audit.py deleted file mode 100644 index ed9edba..0000000 --- a/common/src/common/db/models/audit.py +++ /dev/null @@ -1,38 +0,0 @@ -import datetime -from typing import Optional - -from sqlalchemy import Index, JSON, String, text -from sqlalchemy.dialects.mysql import BIGINT, CHAR, DATETIME, TINYINT -from sqlalchemy.orm import Mapped, mapped_column - -from common.db.base import Base - - -class AuditLogs(Base): - __tablename__ = "audit_logs" - __table_args__ = ( - Index("idx_audit_action_time", "action_code", "created_at"), - Index("idx_audit_actor_time", "actor_user_id", "created_at"), - Index("idx_audit_workspace_time", "workspace_id", "created_at"), - {"comment": "操作审计日志"}, - ) - - audit_id: Mapped[int] = mapped_column(BIGINT, primary_key=True) - action_code: Mapped[str] = mapped_column(String(128), nullable=False) - target_type: Mapped[str] = mapped_column(String(64), nullable=False) - operation_status: Mapped[str] = mapped_column( - String(16), nullable=False, server_default=text("'success'") - ) - created_at: Mapped[datetime.datetime] = mapped_column( - DATETIME(fsp=3), nullable=False, server_default=text("CURRENT_TIMESTAMP(3)") - ) - workspace_id: Mapped[Optional[str]] = mapped_column(CHAR(26)) - actor_user_id: Mapped[Optional[str]] = mapped_column(CHAR(26)) - target_id: Mapped[Optional[str]] = mapped_column(String(128)) - client_ip: Mapped[Optional[str]] = mapped_column(String(45)) - user_agent: Mapped[Optional[str]] = mapped_column(String(1000)) - detail_json: Mapped[Optional[dict]] = mapped_column(JSON) - is_deleted: Mapped[int] = mapped_column( - TINYINT(1), nullable=False, server_default=text("0") - ) - deleted_at: Mapped[Optional[datetime.datetime]] = mapped_column(DATETIME(fsp=3)) diff --git a/common/src/common/db/models/runtime.py b/common/src/common/db/models/runtime.py deleted file mode 100644 index 1299077..0000000 --- a/common/src/common/db/models/runtime.py +++ /dev/null @@ -1,55 +0,0 @@ -import datetime -from typing import Optional - -from sqlalchemy import Index, String, Text, text -from sqlalchemy.dialects.mysql import CHAR, DATETIME, INTEGER, TINYINT -from sqlalchemy.orm import Mapped, mapped_column - -from common.db.base import Base - - -class WorkspaceOperations(Base): - __tablename__ = "workspace_operations" - __table_args__ = ( - Index("fk_workspace_operations_user", "requested_by"), - Index("idx_workspace_operations_runtime", "runtime_id", "created_at"), - Index( - "idx_workspace_operations_workspace", - "workspace_id", - "operation_status", - "created_at", - ), - Index("uk_workspace_operations_request", "request_id", unique=True), - {"comment": "无状态 Backend 的 Workspace/Jupyter 异步操作记录"}, - ) - - operation_id: Mapped[str] = mapped_column(CHAR(26), primary_key=True) - workspace_id: Mapped[str] = mapped_column(CHAR(26), nullable=False) - operation_type: Mapped[str] = mapped_column( - String(24), - nullable=False, - comment="open/close/mount/unmount/start/stop/restart/recycle", - ) - operation_status: Mapped[str] = mapped_column( - String(24), - nullable=False, - server_default=text("'pending'"), - comment="pending/running/succeeded/failed/cancelled", - ) - state_version: Mapped[int] = mapped_column( - INTEGER, nullable=False, server_default=text("0"), comment="乐观锁版本" - ) - requested_by: Mapped[str] = mapped_column(CHAR(26), nullable=False) - created_at: Mapped[datetime.datetime] = mapped_column( - DATETIME(fsp=3), nullable=False, server_default=text("CURRENT_TIMESTAMP(3)") - ) - runtime_id: Mapped[Optional[str]] = mapped_column(CHAR(26)) - request_id: Mapped[Optional[str]] = mapped_column(String(128)) - started_at: Mapped[Optional[datetime.datetime]] = mapped_column(DATETIME(fsp=3)) - finished_at: Mapped[Optional[datetime.datetime]] = mapped_column(DATETIME(fsp=3)) - error_code: Mapped[Optional[str]] = mapped_column(String(64)) - error_message: Mapped[Optional[str]] = mapped_column(Text) - is_deleted: Mapped[int] = mapped_column( - TINYINT(1), nullable=False, server_default=text("0") - ) - deleted_at: Mapped[Optional[datetime.datetime]] = mapped_column(DATETIME(fsp=3)) diff --git a/common/src/common/db/models/scripts.py b/common/src/common/db/models/scripts.py index 153453b..535b0de 100644 --- a/common/src/common/db/models/scripts.py +++ b/common/src/common/db/models/scripts.py @@ -84,7 +84,7 @@ class Versions(Base): CHAR(26), nullable=False, comment="发布时的源对象" ) artifact_object_id: Mapped[str] = mapped_column( - CHAR(26), nullable=False, comment="RustFS 不可变版本制品" + CHAR(26), nullable=False, comment="S3 不可变版本制品" ) version_no: Mapped[int] = mapped_column(INTEGER, nullable=False) version_label: Mapped[str] = mapped_column( diff --git a/common/src/common/db/models/storage.py b/common/src/common/db/models/storage.py index 521b7aa..85edf86 100644 --- a/common/src/common/db/models/storage.py +++ b/common/src/common/db/models/storage.py @@ -201,6 +201,25 @@ class UploadSessions(Base): content_type: Mapped[Optional[str]] = mapped_column(String(255)) storage_object_id: Mapped[Optional[str]] = mapped_column(CHAR(26)) completed_at: Mapped[Optional[datetime.datetime]] = mapped_column(DATETIME(fsp=3)) + # Metadata persisted at session creation so step 2 (PUT bytes) can build + # the StorageObjects row without re-sending them. Replaces the + # CompleteUploadRequest payload that lived between presign-PUT and head(). + file_name: Mapped[str] = mapped_column(String(255), nullable=False, server_default="") + usage_type: Mapped[str] = mapped_column( + String(32), + nullable=False, + server_default=text("'working_copy'"), + comment="data_resource/version_artifact/snapshot/run_log/run_result/working_copy/public_script", + ) + visibility: Mapped[str] = mapped_column( + String(16), + nullable=False, + server_default=text("'private'"), + comment="private/workspace/public", + ) + is_immutable: Mapped[int] = mapped_column( + TINYINT(1), nullable=False, server_default=text("0") + ) is_deleted: Mapped[int] = mapped_column( TINYINT(1), nullable=False, server_default=text("0") ) diff --git a/common/src/common/db/models/workspaces.py b/common/src/common/db/models/workspaces.py index 23df009..f482d3e 100644 --- a/common/src/common/db/models/workspaces.py +++ b/common/src/common/db/models/workspaces.py @@ -46,10 +46,10 @@ class Workspaces(Base): ) description: Mapped[Optional[str]] = mapped_column(String(1000)) artifact_bucket: Mapped[Optional[str]] = mapped_column( - String(128), comment="RustFS bucket" + String(128), comment="S3 bucket" ) artifact_prefix: Mapped[Optional[str]] = mapped_column( - String(512), comment="RustFS object key prefix" + String(512), comment="S3 object key prefix" ) is_deleted: Mapped[int] = mapped_column( TINYINT(1), nullable=False, server_default=text("0") diff --git a/common/src/common/migrations/README b/common/src/common/migrations/README deleted file mode 100644 index 98e4f9c..0000000 --- a/common/src/common/migrations/README +++ /dev/null @@ -1 +0,0 @@ -Generic single-database configuration. \ No newline at end of file diff --git a/common/src/common/migrations/env.py b/common/src/common/migrations/env.py deleted file mode 100644 index b83bc31..0000000 --- a/common/src/common/migrations/env.py +++ /dev/null @@ -1,78 +0,0 @@ -from logging.config import fileConfig - -from sqlalchemy import engine_from_config -from sqlalchemy import pool - -from alembic import context -from common.db.models import Base -# this is the Alembic Config object, which provides -# access to the values within the .ini file in use. -config = context.config - -# Interpret the config file for Python logging. -# This line sets up loggers basically. -if config.config_file_name is not None: - fileConfig(config.config_file_name) - -# add your model's MetaData object here -# for 'autogenerate' support -# from myapp import mymodel -# target_metadata = mymodel.Base.metadata -target_metadata = Base.metadata - -# other values from the config, defined by the needs of env.py, -# can be acquired: -# my_important_option = config.get_main_option("my_important_option") -# ... etc. - - -def run_migrations_offline() -> None: - """Run migrations in 'offline' mode. - - This configures the context with just a URL - and not an Engine, though an Engine is acceptable - here as well. By skipping the Engine creation - we don't even need a DBAPI to be available. - - Calls to context.execute() here emit the given string to the - script output. - - """ - url = config.get_main_option("sqlalchemy.url") - context.configure( - url=url, - target_metadata=target_metadata, - literal_binds=True, - dialect_opts={"paramstyle": "named"}, - ) - - with context.begin_transaction(): - context.run_migrations() - - -def run_migrations_online() -> None: - """Run migrations in 'online' mode. - - In this scenario we need to create an Engine - and associate a connection with the context. - - """ - connectable = engine_from_config( - config.get_section(config.config_ini_section, {}), - prefix="sqlalchemy.", - poolclass=pool.NullPool, - ) - - with connectable.connect() as connection: - context.configure( - connection=connection, target_metadata=target_metadata - ) - - with context.begin_transaction(): - context.run_migrations() - - -if context.is_offline_mode(): - run_migrations_offline() -else: - run_migrations_online() diff --git a/common/src/common/migrations/script.py.mako b/common/src/common/migrations/script.py.mako deleted file mode 100644 index 1101630..0000000 --- a/common/src/common/migrations/script.py.mako +++ /dev/null @@ -1,28 +0,0 @@ -"""${message} - -Revision ID: ${up_revision} -Revises: ${down_revision | comma,n} -Create Date: ${create_date} - -""" -from typing import Sequence, Union - -from alembic import op -import sqlalchemy as sa -${imports if imports else ""} - -# revision identifiers, used by Alembic. -revision: str = ${repr(up_revision)} -down_revision: Union[str, Sequence[str], None] = ${repr(down_revision)} -branch_labels: Union[str, Sequence[str], None] = ${repr(branch_labels)} -depends_on: Union[str, Sequence[str], None] = ${repr(depends_on)} - - -def upgrade() -> None: - """Upgrade schema.""" - ${upgrades if upgrades else "pass"} - - -def downgrade() -> None: - """Downgrade schema.""" - ${downgrades if downgrades else "pass"} diff --git a/common/src/common/storage/__init__.py b/common/src/common/storage/__init__.py index 61ffe4b..bc82ba0 100644 --- a/common/src/common/storage/__init__.py +++ b/common/src/common/storage/__init__.py @@ -1,6 +1,46 @@ -"""Storage building blocks shared by backend and schedule services.""" +"""统一存储层，同时支持同步和异步，通过 config["mode"] 切换。 -from common.storage.client import StorageClient -from common.storage.rustfs import RustFSObjectStore +对上层暴露的公开 API： -__all__ = ["RustFSObjectStore", "StorageClient"] + from storage import create_storage, StorageBackend, AsyncStorageBackend, ObjectMeta + from storage.exceptions import StorageError, StorageNotFoundError, ... + +用法： + # 同步（默认 mode="sync"） + storage = create_storage({"type": "local", "base_dir": "./data"}) + storage.put("a/b.txt", b"hello") + + # 异步：加一个 mode 字段 + storage = create_storage({"type": "local", "mode": "async", "base_dir": "./data"}) + await storage.put("a/b.txt", b"hello") + +切换本地/S3，或切换同步/异步，业务代码都不用改，只改配置： + storage = create_storage({"type": "s3", "mode": "async", "bucket": "my-bucket"}) +""" + +from .base import AsyncStorageBackend, ObjectMeta, StorageBackend +from .factory import ( + PURPOSE_BUCKETS, + RCLONE_REMOTE_NAME, + actual_bucket_name, + build_storage_config, + create_storage, + rclone_remote_spec, + workspaces_root, +) +from .registry import register_backend, registered_backends + +__all__ = [ + "create_storage", + "build_storage_config", + "actual_bucket_name", + "workspaces_root", + "rclone_remote_spec", + "RCLONE_REMOTE_NAME", + "PURPOSE_BUCKETS", + "StorageBackend", + "AsyncStorageBackend", + "ObjectMeta", + "register_backend", + "registered_backends", +] diff --git a/common/src/common/storage/backends/__init__.py b/common/src/common/storage/backends/__init__.py new file mode 100644 index 0000000..82440f2 --- /dev/null +++ b/common/src/common/storage/backends/__init__.py @@ -0,0 +1,9 @@ +"""导入本模块即可触发所有内置后端的 @register_backend 注册。 + +新增内置后端时，在这里加一行 import 即可； +如果是第三方/业务自己的后端，不需要改这个文件， +只要在使用前 import 一次那个模块（让装饰器执行）就够了。 +""" + +from . import local # noqa: F401 +from . import s3 # noqa: F401 diff --git a/common/src/common/storage/backends/local.py b/common/src/common/storage/backends/local.py new file mode 100644 index 0000000..f3d86e3 --- /dev/null +++ b/common/src/common/storage/backends/local.py @@ -0,0 +1,245 @@ +"""本地文件系统存储后端。 + +- 同步实现 `LocalStorageBackend`：标准库文件 I/O +- 异步实现 `LocalAsyncStorageBackend`：aiofiles 做实际读写， + stat/exists/delete/mkdir/目录遍历这类轻量元数据操作用 + asyncio.to_thread 包一层，避免阻塞事件循环 + （只有创建异步实例时才需要装 aiofiles，同步实现零依赖） +""" + +import asyncio +import os +import shutil +from datetime import timedelta +from pathlib import Path +from typing import AsyncIterator, BinaryIO, Iterable, Optional + +from ..base import AsyncData, AsyncStorageBackend, ObjectMeta, StorageBackend, SyncData +from ..exceptions import StorageAlreadyExistsError, StorageNotFoundError +from ..registry import register_backend + + +def _resolve(base_dir: Path, key: str) -> Path: + key = key.strip("/") + path = (base_dir / key).resolve() + if base_dir not in path.parents and path != base_dir: + raise ValueError(f"非法 key，路径穿越到 base_dir 之外: {key!r}") + return path + + +def _meta(key: str, path: Path) -> ObjectMeta: + st = path.stat() + return ObjectMeta(key=key, size=st.st_size, last_modified=st.st_mtime) + + +# ==================== 同步实现 ==================== + + +@register_backend("local", mode="sync") +class LocalStorageBackend(StorageBackend): + """配置示例: {"type": "local", "mode": "sync", "base_dir": "/data/storage"}""" + + def __init__(self, base_dir: str, **_ignored): + self.base_dir = Path(base_dir).resolve() + self.base_dir.mkdir(parents=True, exist_ok=True) + + def _resolve(self, key: str) -> Path: + return _resolve(self.base_dir, key) + + def put( + self, + key: str, + data: SyncData, + *, + overwrite: bool = True, + content_type: Optional[str] = None, + metadata: Optional[dict] = None, + ) -> ObjectMeta: + path = self._resolve(key) + if path.exists() and not overwrite: + raise StorageAlreadyExistsError(f"key 已存在: {key}") + path.parent.mkdir(parents=True, exist_ok=True) + + if isinstance(data, bytes): + path.write_bytes(data) + else: + with open(path, "wb") as f: + shutil.copyfileobj(data, f) + # local FS 没有对象级 metadata；content_type / metadata 暂存忽略。 + return _meta(key, path) + + def get(self, key: str) -> bytes: + path = self._resolve(key) + if not path.is_file(): + raise StorageNotFoundError(f"key 不存在: {key}") + return path.read_bytes() + + def get_stream(self, key: str) -> BinaryIO: + path = self._resolve(key) + if not path.is_file(): + raise StorageNotFoundError(f"key 不存在: {key}") + return open(path, "rb") + + def delete(self, key: str) -> None: + try: + self._resolve(key).unlink() + except FileNotFoundError: + pass + + def exists(self, key: str) -> bool: + return self._resolve(key).is_file() + + def stat(self, key: str) -> ObjectMeta: + path = self._resolve(key) + if not path.is_file(): + raise StorageNotFoundError(f"key 不存在: {key}") + return _meta(key, path) + + def list(self, prefix: str = "") -> Iterable[ObjectMeta]: + search_root = self._resolve(prefix) if prefix else self.base_dir + if search_root.is_dir(): + candidates = search_root.rglob("*") + else: + candidates = search_root.parent.glob(f"{search_root.name}*") + + for path in candidates: + if path.is_file(): + key = str(path.relative_to(self.base_dir)).replace(os.sep, "/") + yield _meta(key, path) + + def get_url(self, key: str, *, expires_in: Optional[timedelta] = None) -> str: + path = self._resolve(key) + if not path.is_file(): + raise StorageNotFoundError(f"key 不存在: {key}") + return path.as_uri() + + def copy(self, src_key: str, dst_key: str) -> ObjectMeta: + src_path = self._resolve(src_key) + if not src_path.is_file(): + raise StorageNotFoundError(f"key 不存在: {src_key}") + dst_path = self._resolve(dst_key) + dst_path.parent.mkdir(parents=True, exist_ok=True) + shutil.copy2(src_path, dst_path) + return _meta(dst_key, dst_path) + + +# ==================== 异步实现 ==================== + + +@register_backend("local", mode="async") +class LocalAsyncStorageBackend(AsyncStorageBackend): + """配置示例: {"type": "local", "mode": "async", "base_dir": "/data/storage"} + + 需要: pip install aiofiles + """ + + def __init__(self, base_dir: str, **_ignored): + self.base_dir = Path(base_dir).resolve() + self.base_dir.mkdir(parents=True, exist_ok=True) + + def _resolve(self, key: str) -> Path: + return _resolve(self.base_dir, key) + + async def put( + self, + key: str, + data: AsyncData, + *, + overwrite: bool = True, + content_type: Optional[str] = None, + metadata: Optional[dict] = None, + ) -> ObjectMeta: + import aiofiles + + path = self._resolve(key) + if not overwrite and await asyncio.to_thread(path.exists): + raise StorageAlreadyExistsError(f"key 已存在: {key}") + await asyncio.to_thread(path.parent.mkdir, parents=True, exist_ok=True) + + async with aiofiles.open(path, "wb") as f: + if isinstance(data, (bytes, bytearray)): + await f.write(data) + else: + async for chunk in data: + await f.write(chunk) + + # local FS 没有对象级 metadata；content_type / metadata 暂存忽略。 + return await asyncio.to_thread(_meta, key, path) + + async def get(self, key: str) -> bytes: + import aiofiles + + path = self._resolve(key) + if not await asyncio.to_thread(path.is_file): + raise StorageNotFoundError(f"key 不存在: {key}") + async with aiofiles.open(path, "rb") as f: + return await f.read() + + def get_stream(self, key: str, chunk_size: int = 64 * 1024) -> AsyncIterator[bytes]: + async def _iter(): + import aiofiles + + path = self._resolve(key) + if not await asyncio.to_thread(path.is_file): + raise StorageNotFoundError(f"key 不存在: {key}") + async with aiofiles.open(path, "rb") as f: + while True: + chunk = await f.read(chunk_size) + if not chunk: + break + yield chunk + + return _iter() + + async def delete(self, key: str) -> None: + path = self._resolve(key) + + def _unlink(): + try: + path.unlink() + except FileNotFoundError: + pass + + await asyncio.to_thread(_unlink) + + async def exists(self, key: str) -> bool: + return await asyncio.to_thread(self._resolve(key).is_file) + + async def stat(self, key: str) -> ObjectMeta: + path = self._resolve(key) + if not await asyncio.to_thread(path.is_file): + raise StorageNotFoundError(f"key 不存在: {key}") + return await asyncio.to_thread(_meta, key, path) + + def list(self, prefix: str = "") -> AsyncIterator[ObjectMeta]: + async def _iter(): + search_root = self._resolve(prefix) if prefix else self.base_dir + + def _collect(): + if search_root.is_dir(): + candidates = list(search_root.rglob("*")) + else: + candidates = list(search_root.parent.glob(f"{search_root.name}*")) + return [p for p in candidates if p.is_file()] + + files = await asyncio.to_thread(_collect) + for path in files: + key = str(path.relative_to(self.base_dir)).replace(os.sep, "/") + yield await asyncio.to_thread(_meta, key, path) + + return _iter() + + async def get_url(self, key: str, *, expires_in: Optional[timedelta] = None) -> str: + path = self._resolve(key) + if not await asyncio.to_thread(path.is_file): + raise StorageNotFoundError(f"key 不存在: {key}") + return path.as_uri() + + async def copy(self, src_key: str, dst_key: str) -> ObjectMeta: + src_path = self._resolve(src_key) + if not await asyncio.to_thread(src_path.is_file): + raise StorageNotFoundError(f"key 不存在: {src_key}") + dst_path = self._resolve(dst_key) + await asyncio.to_thread(dst_path.parent.mkdir, parents=True, exist_ok=True) + await asyncio.to_thread(shutil.copy2, src_path, dst_path) + return await asyncio.to_thread(_meta, dst_key, dst_path) diff --git a/common/src/common/storage/backends/s3.py b/common/src/common/storage/backends/s3.py new file mode 100644 index 0000000..ed82e4b --- /dev/null +++ b/common/src/common/storage/backends/s3.py @@ -0,0 +1,420 @@ +"""S3（及兼容协议）存储后端。 + +- 同步实现 `S3StorageBackend`：boto3 +- 异步实现 `S3AsyncStorageBackend`：aioboto3 + +两者只在各自 __init__ 里做 lazy import，互不强制依赖： +只用同步模式不需要装 aioboto3，只用异步模式不需要额外装 boto3 +（aioboto3 本身依赖 botocore，异常类型从它里面拿）。 +""" + +from datetime import timedelta +from typing import AsyncIterator, BinaryIO, Iterable, Optional + +from ..base import AsyncData, AsyncStorageBackend, ObjectMeta, StorageBackend, SyncData +from ..exceptions import ( + StorageAlreadyExistsError, + StorageConnectionError, + StorageNotFoundError, +) +from ..registry import register_backend + + +def _meta_from_head(key: str, head: dict) -> ObjectMeta: + return ObjectMeta( + key=key, + size=head.get("ContentLength", 0), + last_modified=head["LastModified"].timestamp() if head.get("LastModified") else None, + etag=head.get("ETag"), + ) + + +# ==================== 同步实现 ==================== + + +@register_backend("s3", mode="sync") +class S3StorageBackend(StorageBackend): + """配置示例: + { + "type": "s3", "mode": "sync", + "bucket": "my-bucket", "prefix": "app1/", + "region_name": "cn-north-1", "endpoint_url": "https://s3.example.com", + "aws_access_key_id": "...", "aws_secret_access_key": "...", + } + + 需要: pip install boto3 + """ + + def __init__( + self, + bucket: str, + prefix: str = "", + region_name: Optional[str] = None, + endpoint_url: Optional[str] = None, + aws_access_key_id: Optional[str] = None, + aws_secret_access_key: Optional[str] = None, + **_ignored, + ): + try: + import boto3 + from botocore.exceptions import BotoCoreError, ClientError + except ImportError as e: + raise ImportError("使用同步 S3 存储后端需要先安装 boto3: pip install boto3") from e + + self._ClientError = ClientError + self._BotoCoreError = BotoCoreError + self.bucket = bucket + self.prefix = prefix.strip("/") + "/" if prefix.strip("/") else "" + + try: + self.client = boto3.client( + "s3", + region_name=region_name, + endpoint_url=endpoint_url, + aws_access_key_id=aws_access_key_id, + aws_secret_access_key=aws_secret_access_key, + ) + except (BotoCoreError, ClientError) as e: + raise StorageConnectionError(f"初始化 S3 client 失败: {e}") from e + + def _full_key(self, key: str) -> str: + return f"{self.prefix}{key.lstrip('/')}" + + def put(self, key: str, data: SyncData, *, overwrite: bool = True) -> ObjectMeta: + full_key = self._full_key(key) + if not overwrite and self.exists(key): + raise StorageAlreadyExistsError(f"key 已存在: {key}") + body = data if isinstance(data, bytes) else data.read() + try: + self.client.put_object(Bucket=self.bucket, Key=full_key, Body=body) + except (self._ClientError, self._BotoCoreError) as e: + raise StorageConnectionError(f"上传失败 key={key}: {e}") from e + return self.stat(key) + + def get(self, key: str) -> bytes: + full_key = self._full_key(key) + try: + resp = self.client.get_object(Bucket=self.bucket, Key=full_key) + return resp["Body"].read() + except self._ClientError as e: + if e.response.get("Error", {}).get("Code") in ("NoSuchKey", "404"): + raise StorageNotFoundError(f"key 不存在: {key}") from e + raise StorageConnectionError(f"读取失败 key={key}: {e}") from e + + def get_stream(self, key: str) -> BinaryIO: + full_key = self._full_key(key) + try: + resp = self.client.get_object(Bucket=self.bucket, Key=full_key) + return resp["Body"] + except self._ClientError as e: + if e.response.get("Error", {}).get("Code") in ("NoSuchKey", "404"): + raise StorageNotFoundError(f"key 不存在: {key}") from e + raise StorageConnectionError(f"读取失败 key={key}: {e}") from e + + def delete(self, key: str) -> None: + try: + self.client.delete_object(Bucket=self.bucket, Key=self._full_key(key)) + except (self._ClientError, self._BotoCoreError) as e: + raise StorageConnectionError(f"删除失败 key={key}: {e}") from e + + def exists(self, key: str) -> bool: + try: + self.client.head_object(Bucket=self.bucket, Key=self._full_key(key)) + return True + except self._ClientError as e: + if e.response.get("Error", {}).get("Code") in ("404", "NoSuchKey"): + return False + raise StorageConnectionError(f"检查 exists 失败 key={key}: {e}") from e + + def stat(self, key: str) -> ObjectMeta: + try: + head = self.client.head_object(Bucket=self.bucket, Key=self._full_key(key)) + except self._ClientError as e: + if e.response.get("Error", {}).get("Code") in ("404", "NoSuchKey"): + raise StorageNotFoundError(f"key 不存在: {key}") from e + raise StorageConnectionError(f"获取元信息失败 key={key}: {e}") from e + return _meta_from_head(key, head) + + def list(self, prefix: str = "") -> Iterable[ObjectMeta]: + full_prefix = self._full_key(prefix) + paginator = self.client.get_paginator("list_objects_v2") + try: + for page in paginator.paginate(Bucket=self.bucket, Prefix=full_prefix): + for obj in page.get("Contents", []): + key = obj["Key"][len(self.prefix):] if self.prefix else obj["Key"] + yield ObjectMeta( + key=key, + size=obj["Size"], + last_modified=obj["LastModified"].timestamp(), + etag=obj.get("ETag"), + ) + except (self._ClientError, self._BotoCoreError) as e: + raise StorageConnectionError(f"列举对象失败 prefix={prefix}: {e}") from e + + def get_url(self, key: str, *, expires_in: Optional[timedelta] = None) -> str: + expires_seconds = int(expires_in.total_seconds()) if expires_in else 3600 + try: + return self.client.generate_presigned_url( + "get_object", + Params={"Bucket": self.bucket, "Key": self._full_key(key)}, + ExpiresIn=expires_seconds, + ) + except (self._ClientError, self._BotoCoreError) as e: + raise StorageConnectionError(f"生成预签名 URL 失败 key={key}: {e}") from e + + def copy(self, src_key: str, dst_key: str) -> ObjectMeta: + try: + self.client.copy_object( + Bucket=self.bucket, + Key=self._full_key(dst_key), + CopySource={"Bucket": self.bucket, "Key": self._full_key(src_key)}, + ) + except self._ClientError as e: + if e.response.get("Error", {}).get("Code") in ("404", "NoSuchKey"): + raise StorageNotFoundError(f"key 不存在: {src_key}") from e + raise StorageConnectionError(f"复制失败 {src_key} -> {dst_key}: {e}") from e + return self.stat(dst_key) + + +# ==================== 异步实现 ==================== + + +@register_backend("s3", mode="async") +class S3AsyncStorageBackend(AsyncStorageBackend): + """配置示例同上，把 "mode" 改成 "async" 即可。 + + 需要: pip install aioboto3 + + 每次操作默认通过 `async with session.client(...)` 拿一个短生命周期 + client；用 `async with create_storage(...) as storage:` 可以复用同一个 + client（见 __aenter__/__aexit__）。 + """ + + def __init__( + self, + bucket: str, + prefix: str = "", + region_name: Optional[str] = None, + endpoint_url: Optional[str] = None, + aws_access_key_id: Optional[str] = None, + aws_secret_access_key: Optional[str] = None, + **_ignored, + ): + try: + import aioboto3 + from botocore.exceptions import BotoCoreError, ClientError + except ImportError as e: + raise ImportError( + "使用异步 S3 存储后端需要先安装 aioboto3: pip install aioboto3" + ) from e + + self._ClientError = ClientError + self._BotoCoreError = BotoCoreError + self.bucket = bucket + self.prefix = prefix.strip("/") + "/" if prefix.strip("/") else "" + self._client_kwargs = dict( + region_name=region_name, + endpoint_url=endpoint_url, + aws_access_key_id=aws_access_key_id, + aws_secret_access_key=aws_secret_access_key, + ) + self._session = aioboto3.Session() + self._persistent_client = None + self._persistent_cm = None + + def _client_cm(self): + return self._session.client("s3", **self._client_kwargs) + + async def __aenter__(self) -> "S3AsyncStorageBackend": + self._persistent_cm = self._client_cm() + self._persistent_client = await self._persistent_cm.__aenter__() + return self + + async def __aexit__(self, exc_type, exc, tb) -> None: + if self._persistent_cm is not None: + await self._persistent_cm.__aexit__(exc_type, exc, tb) + self._persistent_cm = None + self._persistent_client = None + + async def aclose(self) -> None: + await self.__aexit__(None, None, None) + + def _full_key(self, key: str) -> str: + return f"{self.prefix}{key.lstrip('/')}" + + async def _run(self, coro_fn): + if self._persistent_client is not None: + return await coro_fn(self._persistent_client) + async with self._client_cm() as client: + return await coro_fn(client) + + async def put( + self, + key: str, + data: AsyncData, + *, + overwrite: bool = True, + content_type: Optional[str] = None, + metadata: Optional[dict] = None, + ) -> ObjectMeta: + full_key = self._full_key(key) + if not overwrite and await self.exists(key): + raise StorageAlreadyExistsError(f"key 已存在: {key}") + + if isinstance(data, (bytes, bytearray)): + body = bytes(data) + else: + chunks = [] + async for chunk in data: + chunks.append(chunk) + body = b"".join(chunks) + + # 过滤掉空 dict / None，避免 boto3 报 "parameter must be a non-empty + # non-null dictionary of strings" 这种空请求参数错误。 + meta = {k: str(v) for k, v in (metadata or {}).items() if v is not None} or None + + async def _op(client): + kwargs = {"Bucket": self.bucket, "Key": full_key, "Body": body} + if content_type: + kwargs["ContentType"] = content_type + if meta: + kwargs["Metadata"] = meta + await client.put_object(**kwargs) + + try: + await self._run(_op) + except (self._ClientError, self._BotoCoreError) as e: + raise StorageConnectionError(f"上传失败 key={key}: {e}") from e + return await self.stat(key) + + async def get(self, key: str) -> bytes: + full_key = self._full_key(key) + + async def _op(client): + resp = await client.get_object(Bucket=self.bucket, Key=full_key) + async with resp["Body"] as stream: + return await stream.read() + + try: + return await self._run(_op) + except self._ClientError as e: + if e.response.get("Error", {}).get("Code") in ("NoSuchKey", "404"): + raise StorageNotFoundError(f"key 不存在: {key}") from e + raise StorageConnectionError(f"读取失败 key={key}: {e}") from e + + def get_stream(self, key: str, chunk_size: int = 64 * 1024) -> AsyncIterator[bytes]: + async def _iter(): + full_key = self._full_key(key) + + async def _op(client): + return await client.get_object(Bucket=self.bucket, Key=full_key) + + try: + resp = await self._run(_op) + except self._ClientError as e: + if e.response.get("Error", {}).get("Code") in ("NoSuchKey", "404"): + raise StorageNotFoundError(f"key 不存在: {key}") from e + raise StorageConnectionError(f"读取失败 key={key}: {e}") from e + + async with resp["Body"] as stream: + while True: + chunk = await stream.read(chunk_size) + if not chunk: + break + yield chunk + + return _iter() + + async def delete(self, key: str) -> None: + async def _op(client): + await client.delete_object(Bucket=self.bucket, Key=self._full_key(key)) + + try: + await self._run(_op) + except (self._ClientError, self._BotoCoreError) as e: + raise StorageConnectionError(f"删除失败 key={key}: {e}") from e + + async def exists(self, key: str) -> bool: + async def _op(client): + await client.head_object(Bucket=self.bucket, Key=self._full_key(key)) + + try: + await self._run(_op) + return True + except self._ClientError as e: + if e.response.get("Error", {}).get("Code") in ("404", "NoSuchKey"): + return False + raise StorageConnectionError(f"检查 exists 失败 key={key}: {e}") from e + + async def stat(self, key: str) -> ObjectMeta: + async def _op(client): + return await client.head_object(Bucket=self.bucket, Key=self._full_key(key)) + + try: + head = await self._run(_op) + except self._ClientError as e: + if e.response.get("Error", {}).get("Code") in ("404", "NoSuchKey"): + raise StorageNotFoundError(f"key 不存在: {key}") from e + raise StorageConnectionError(f"获取元信息失败 key={key}: {e}") from e + return _meta_from_head(key, head) + + def list(self, prefix: str = "") -> AsyncIterator[ObjectMeta]: + async def _iter(): + full_prefix = self._full_key(prefix) + + async def _paginate(client): + paginator = client.get_paginator("list_objects_v2") + results = [] + async for page in paginator.paginate(Bucket=self.bucket, Prefix=full_prefix): + for obj in page.get("Contents", []): + key = obj["Key"][len(self.prefix):] if self.prefix else obj["Key"] + results.append( + ObjectMeta( + key=key, + size=obj["Size"], + last_modified=obj["LastModified"].timestamp(), + etag=obj.get("ETag"), + ) + ) + return results + + try: + metas = await self._run(_paginate) + except (self._ClientError, self._BotoCoreError) as e: + raise StorageConnectionError(f"列举对象失败 prefix={prefix}: {e}") from e + + for meta in metas: + yield meta + + return _iter() + + async def get_url(self, key: str, *, expires_in: Optional[timedelta] = None) -> str: + expires_seconds = int(expires_in.total_seconds()) if expires_in else 3600 + + async def _op(client): + return await client.generate_presigned_url( + "get_object", + Params={"Bucket": self.bucket, "Key": self._full_key(key)}, + ExpiresIn=expires_seconds, + ) + + try: + return await self._run(_op) + except (self._ClientError, self._BotoCoreError) as e: + raise StorageConnectionError(f"生成预签名 URL 失败 key={key}: {e}") from e + + async def copy(self, src_key: str, dst_key: str) -> ObjectMeta: + async def _op(client): + await client.copy_object( + Bucket=self.bucket, + Key=self._full_key(dst_key), + CopySource={"Bucket": self.bucket, "Key": self._full_key(src_key)}, + ) + + try: + await self._run(_op) + except self._ClientError as e: + if e.response.get("Error", {}).get("Code") in ("404", "NoSuchKey"): + raise StorageNotFoundError(f"key 不存在: {src_key}") from e + raise StorageConnectionError(f"复制失败 {src_key} -> {dst_key}: {e}") from e + return await self.stat(dst_key) diff --git a/common/src/common/storage/base.py b/common/src/common/storage/base.py new file mode 100644 index 0000000..2dae299 --- /dev/null +++ b/common/src/common/storage/base.py @@ -0,0 +1,155 @@ +"""同步 / 异步存储后端统一抽象接口。 + +`StorageBackend` 是同步接口，`AsyncStorageBackend` 是异步接口， +两者共用同一个 `ObjectMeta` 数据结构，方法签名尽量保持对称 +（异步版本每个方法多一个 await，get_stream/list 变成异步生成器）， +这样业务代码从同步切到异步时心智负担最小。 + +上层通过 `storage.create_storage(config)` 统一创建实例， +用 `config["mode"]` 决定拿到的是同步实现还是异步实现。 +""" + +from abc import ABC, abstractmethod +from dataclasses import dataclass, field +from datetime import timedelta +from typing import AsyncIterator, BinaryIO, Iterable, Optional, Union + +SyncData = Union[bytes, BinaryIO] +AsyncData = Union[bytes, "AsyncIterator[bytes]"] + + +@dataclass +class ObjectMeta: + """list/stat 等操作返回的对象元信息，做了跨后端的字段归一化。""" + + key: str + size: int + last_modified: Optional[float] = None # unix timestamp + etag: Optional[str] = None + extra: dict = field(default_factory=dict) # 后端特有的额外信息 + + +class StorageBackend(ABC): + """同步存储后端统一抽象基类。""" + + @abstractmethod + def put( + self, + key: str, + data: SyncData, + *, + overwrite: bool = True, + content_type: Optional[str] = None, + metadata: Optional[dict] = None, + ) -> ObjectMeta: + """写入对象。overwrite=False 时 key 已存在应抛出 StorageAlreadyExistsError。 + + ``content_type`` 和 ``metadata`` 是可选的（与异步 put 语义一致）。 + """ + + @abstractmethod + def get(self, key: str) -> bytes: + """读取对象内容，不存在时抛出 StorageNotFoundError。""" + + @abstractmethod + def get_stream(self, key: str) -> BinaryIO: + """以流方式读取对象，适合大文件。""" + + @abstractmethod + def delete(self, key: str) -> None: + """删除对象。删除不存在的 key 不应报错（幂等）。""" + + @abstractmethod + def exists(self, key: str) -> bool: + ... + + @abstractmethod + def stat(self, key: str) -> ObjectMeta: + """不存在时抛出 StorageNotFoundError。""" + + @abstractmethod + def list(self, prefix: str = "") -> Iterable[ObjectMeta]: + """按前缀列出对象。""" + + @abstractmethod + def get_url(self, key: str, *, expires_in: Optional[timedelta] = None) -> str: + """获取可访问 URL；本地存储返回 file://，S3 返回预签名 URL。""" + + def copy(self, src_key: str, dst_key: str) -> ObjectMeta: + """默认实现：读出来再写进去。后端可覆盖为更高效的原生实现。""" + data = self.get(src_key) + return self.put(dst_key, data) + + def close(self) -> None: + """释放后端持有的资源（连接池等）。不需要的后端可以不覆盖。""" + return None + + def __enter__(self) -> "StorageBackend": + return self + + def __exit__(self, exc_type, exc, tb) -> None: + self.close() + + +class AsyncStorageBackend(ABC): + """异步存储后端统一抽象基类。""" + + @abstractmethod + async def put( + self, + key: str, + data: AsyncData, + *, + overwrite: bool = True, + content_type: Optional[str] = None, + metadata: Optional[dict] = None, + ) -> ObjectMeta: + """data 可以是 bytes，也可以是异步字节流（async generator）。 + + ``content_type`` 和 ``metadata`` 是可选的：S3 后端会把它们分别透传 + 成 ``ContentType`` 请求头和 ``Metadata`` dict；local 后端目前忽略 + 这两个参数（本地 FS 没有对象级 metadata）。 + """ + + @abstractmethod + async def get(self, key: str) -> bytes: + """不存在时抛出 StorageNotFoundError。""" + + @abstractmethod + def get_stream(self, key: str, chunk_size: int = 64 * 1024) -> AsyncIterator[bytes]: + """异步分块读取，用法: `async for chunk in backend.get_stream(key):`。 + 普通方法（非 async def），返回值本身就是异步生成器。 + """ + + @abstractmethod + async def delete(self, key: str) -> None: + """幂等：删除不存在的 key 不应报错。""" + + @abstractmethod + async def exists(self, key: str) -> bool: + ... + + @abstractmethod + async def stat(self, key: str) -> ObjectMeta: + """不存在时抛出 StorageNotFoundError。""" + + @abstractmethod + def list(self, prefix: str = "") -> AsyncIterator[ObjectMeta]: + """用法: `async for meta in backend.list(prefix):`。""" + + @abstractmethod + async def get_url(self, key: str, *, expires_in: Optional[timedelta] = None) -> str: + ... + + async def copy(self, src_key: str, dst_key: str) -> ObjectMeta: + data = await self.get(src_key) + return await self.put(dst_key, data) + + async def aclose(self) -> None: + return None + + async def __aenter__(self) -> "AsyncStorageBackend": + return self + + async def __aexit__(self, exc_type, exc, tb) -> None: + await self.aclose() diff --git a/common/src/common/storage/client.py b/common/src/common/storage/client.py deleted file mode 100644 index 5fdf1ea..0000000 --- a/common/src/common/storage/client.py +++ /dev/null @@ -1,146 +0,0 @@ -"""HTTP client for the internal ``/internal/v1/...`` storage surface. - -The class accepts an injected ``httpx.AsyncClient`` so callers can wire -it up with either a real network transport or an ``ASGITransport`` for -in-process dispatch. ``SchedulerStorageClient`` extends this class to -add a single ``create_object`` helper used by the schedule service. - -All public methods raise :class:`StorageClientError` (or a subclass) on -failure. Translation to a web-framework exception (e.g. FastAPI's -``HTTPException``) is the caller's responsibility so this client stays -usable from non-FastAPI contexts such as the schedule worker. -""" - -from __future__ import annotations - -import base64 -from typing import Any - -import httpx - - -class StorageClientError(Exception): - """Base class for storage client failures.""" - - -class StorageUnavailable(StorageClientError): - """Transport-level failure; safe to retry.""" - - def __init__(self, message: str) -> None: - super().__init__(message) - self.retryable = True - - -class StorageRequestFailed(StorageClientError): - """Storage endpoint returned a non-2xx response.""" - - def __init__(self, status_code: int, detail: Any) -> None: - super().__init__(f"storage request failed with status {status_code}") - self.status_code = status_code - self.detail = detail - - -class StorageClient: - def __init__(self, client: httpx.AsyncClient) -> None: - self.client = client - - async def _request( - self, - method: str, - path: str, - *, - payload: dict[str, Any] | None = None, - ) -> dict[str, Any]: - try: - response = await self.client.request( - method, - path, - json=payload, - ) - except httpx.RequestError as exc: - raise StorageUnavailable("Storage service temporarily unavailable") from exc - if response.is_error: - try: - detail = response.json().get("detail", response.text) - except ValueError: - detail = response.text - raise StorageRequestFailed(response.status_code, detail) - return response.json() - - async def create_upload( - self, - payload: dict[str, Any], - ) -> dict[str, Any]: - return (await self._request( - "POST", - "/internal/v1/uploads", - payload=payload, - ))["data"] - - async def complete_upload( - self, - upload_id: str, - payload: dict[str, Any], - ) -> dict[str, Any]: - return (await self._request( - "POST", - f"/internal/v1/uploads/{upload_id}/complete", - payload=payload, - ))["data"] - - async def create_server_object( - self, - *, - workspace_id: str, - user_id: str, - usage_type: str, - file_name: str, - content_type: str, - content: bytes, - visibility: str, - is_immutable: bool, - idempotency_key: str, - relative_path: str | None = None, - ) -> dict[str, Any]: - result = await self._request( - "POST", - "/internal/v1/objects", - payload={ - "workspace_id": workspace_id, - "user_id": user_id, - "usage_type": usage_type, - "file_name": file_name, - "content_type": content_type, - "content_base64": base64.b64encode(content).decode("ascii"), - "visibility": visibility, - "is_immutable": is_immutable, - "idempotency_key": idempotency_key, - "relative_path": relative_path, - }, - ) - return result["data"] - - async def create_download_url( - self, - storage_object_id: str, - expires_seconds: int, - ) -> dict[str, Any]: - return (await self._request( - "POST", - f"/internal/v1/objects/{storage_object_id}/download-url", - payload={"expires_seconds": expires_seconds}, - ))["data"] - - async def delete_object(self, storage_object_id: str) -> dict[str, Any]: - return (await self._request( - "DELETE", - f"/internal/v1/objects/{storage_object_id}", - ))["data"] - - -__all__ = [ - "StorageClient", - "StorageClientError", - "StorageRequestFailed", - "StorageUnavailable", -] diff --git a/common/src/common/storage/example_usage.py b/common/src/common/storage/example_usage.py new file mode 100644 index 0000000..eb2607a --- /dev/null +++ b/common/src/common/storage/example_usage.py @@ -0,0 +1,116 @@ +"""使用示例：同一套 create_storage()，靠 config["mode"] 切换同步/异步。""" + +import asyncio + +from common.storage import create_storage +from common.storage.exceptions import StorageNotFoundError + + +def sync_demo(): + # mode 默认就是 "sync"，可以不写 + storage = create_storage({"type": "local", "base_dir": "./data_sync"}) + + storage.put("docs/hello.txt", b"hello world") + print(storage.get("docs/hello.txt")) + print(storage.exists("docs/hello.txt")) + print(list(storage.list("docs/"))) + print(storage.get_url("docs/hello.txt")) + + try: + storage.get("docs/not_exist.txt") + except StorageNotFoundError: + print("按预期抛出 StorageNotFoundError") + + # 换成同步 S3，只改配置： + # storage = create_storage({"type": "s3", "bucket": "my-bucket"}) + + +async def async_demo(): + # 只加一个 "mode": "async"，其余配置和参数不变 + storage = create_storage({"type": "local", "mode": "async", "base_dir": "./data_async"}) + + await storage.put("docs/hello.txt", b"hello world") + print(await storage.get("docs/hello.txt")) + print(await storage.exists("docs/hello.txt")) + + async for meta in storage.list("docs/"): + print(meta) + + chunks = [] + async for chunk in storage.get_stream("docs/hello.txt"): + chunks.append(chunk) + print(b"".join(chunks)) + + try: + await storage.get("docs/not_exist.txt") + except StorageNotFoundError: + print("按预期抛出 StorageNotFoundError") + + # 并发写入，异步模式的典型优势场景 + tasks = [storage.put(f"batch/{i}.txt", f"content-{i}".encode()) for i in range(10)] + await asyncio.gather(*tasks) + print("并发写入 10 个对象完成") + + # 换成异步 S3，只改配置： + # storage = create_storage({"type": "s3", "mode": "async", "bucket": "my-bucket"}) + # 高吞吐场景复用连接： + # async with create_storage({"type": "s3", "mode": "async", "bucket": "my-bucket"}) as s3: + # await s3.put("a.txt", b"1") + + +def extend_with_new_backend_demo(): + """演示独立扩展一种新的存储方式（同步+异步各一个），不用改现有代码。""" + import io + import time + from typing import AsyncIterator, BinaryIO, Iterable, Optional + + from common.storage.base import AsyncStorageBackend, ObjectMeta, StorageBackend + from common.storage.exceptions import StorageNotFoundError + from common.storage.registry import register_backend + + @register_backend("memory", mode="sync") + class MemoryStorageBackend(StorageBackend): + def __init__(self, **_ignored): + self._store = {} + + def put(self, key, data, *, overwrite=True): + body = data if isinstance(data, bytes) else data.read() + self._store[key] = body + return ObjectMeta(key=key, size=len(body), last_modified=time.time()) + + def get(self, key): + if key not in self._store: + raise StorageNotFoundError(key) + return self._store[key] + + def get_stream(self, key): + return io.BytesIO(self.get(key)) + + def delete(self, key): + self._store.pop(key, None) + + def exists(self, key): + return key in self._store + + def stat(self, key): + if key not in self._store: + raise StorageNotFoundError(key) + return ObjectMeta(key=key, size=len(self._store[key])) + + def list(self, prefix=""): + for key, body in self._store.items(): + if key.startswith(prefix): + yield ObjectMeta(key=key, size=len(body)) + + def get_url(self, key, *, expires_in=None): + return f"memory://{key}" + + mem_storage = create_storage({"type": "memory", "mode": "sync"}) + mem_storage.put("a.txt", b"in-memory content") + print(mem_storage.get("a.txt")) + + +if __name__ == "__main__": + sync_demo() + asyncio.run(async_demo()) + extend_with_new_backend_demo() diff --git a/common/src/common/storage/exceptions.py b/common/src/common/storage/exceptions.py new file mode 100644 index 0000000..26d9e8b --- /dev/null +++ b/common/src/common/storage/exceptions.py @@ -0,0 +1,21 @@ +"""存储层统一异常。同步/异步后端共用同一套异常类型。""" + + +class StorageError(Exception): + """所有存储相关异常的基类。""" + + +class StorageNotFoundError(StorageError): + """指定的 key 不存在。""" + + +class StorageAlreadyExistsError(StorageError): + """在要求不覆盖的场景下，key 已存在。""" + + +class StorageConnectionError(StorageError): + """连接/网络层面的错误（如 S3 网络超时、权限问题等）。""" + + +class StorageConfigError(StorageError): + """配置错误，例如缺少必需参数、backend 类型未注册等。""" diff --git a/common/src/common/storage/factory.py b/common/src/common/storage/factory.py new file mode 100644 index 0000000..46ca7cb --- /dev/null +++ b/common/src/common/storage/factory.py @@ -0,0 +1,165 @@ +"""统一入口：根据配置字典创建具体的存储后端实例。 + +配置里的 "mode" 字段决定拿到同步还是异步实现，默认 "sync"（向后兼容）。 + + # 同步（默认） + storage = create_storage({"type": "local", "base_dir": "./data"}) + storage.put("a.txt", b"hello") + + # 异步：只需加一个 mode 字段，其余配置不变 + storage = create_storage({"type": "local", "mode": "async", "base_dir": "./data"}) + await storage.put("a.txt", b"hello") + + # S3 同理 + storage = create_storage({"type": "s3", "mode": "async", "bucket": "my-bucket"}) + +上层业务代码应该只从这里拿实例，不要直接 import 具体的 XxxStorageBackend / +XxxAsyncStorageBackend 类。 +""" + +from pathlib import Path +from typing import Any, Dict, Union + +from .base import AsyncStorageBackend, StorageBackend +from .exceptions import StorageConfigError +from .registry import get_backend_class +from .backends import local, s3 # noqa: F401 # 触发内置后端注册 + +AnyStorageBackend = Union[StorageBackend, AsyncStorageBackend] + + +def create_storage(config: Dict[str, Any]) -> AnyStorageBackend: + """根据配置创建存储后端。 + + Args: + config: 必须包含 "type" 字段（如 "local" / "s3"）； + 可选 "mode" 字段（"sync" 默认 / "async"）； + 其余字段作为 kwargs 传给对应后端的构造函数。 + + Returns: + mode="sync" 时返回 StorageBackend 实例（同步方法）； + mode="async" 时返回 AsyncStorageBackend 实例（方法需要 await）。 + """ + config = dict(config) # 不修改调用方传入的原字典 + backend_type = config.pop("type", None) + mode = config.pop("mode", "sync") + + if not backend_type: + raise StorageConfigError("配置缺少 'type' 字段，例如 'local' 或 's3'") + + backend_cls = get_backend_class(backend_type, mode) + try: + return backend_cls(**config) + except TypeError as e: + raise StorageConfigError( + f"创建后端 (mode={mode}, type={backend_type}) 失败，参数不匹配: {e}" + ) from e + + +# 4 个目的化桶的名字（key for app.state.object_stores）。 +# 在 local 模式下对应 ``local_storage_base_dir/`` 子目录； +# 在 s3 模式下对应 ``settings.s3__bucket``。 +PURPOSE_BUCKETS: tuple[str, ...] = ("workspace", "version", "run_log", "trash") + + +def actual_bucket_name(purpose: str) -> str: + """把 purpose 名称解析成实际桶路径 / 名（runtime 数据会存在这个字符串里）。 + + - s3 模式:`settings.s3__bucket`(e.g. ``"versions"``) + - local 模式:`${local_storage_base_dir}/`(e.g. ``"/data/version"``) + + 这是 ``app.state.object_stores`` 的 dict key——``UploadSessions.bucket_name`` + 和 ``StorageObjects.bucket_name`` 都存这个值,所以 dict 必须用这个串 + 做 key 才能在 ``object_stores[upload.bucket_name]`` 那里直接命中。 + """ + from common.config import settings # 延迟 import 避免循环 + if settings.storage_backend == "local": + return str(Path(settings.local_storage_base_dir) / purpose) + return getattr(settings, f"s3_{purpose}_bucket") + + +def build_storage_config(bucket_name: str) -> Dict[str, Any]: + """根据 ``settings.storage_backend`` 构造 ``create_storage()`` 的入参。 + + 上层（lifespan 等）只用 ``PURPOSE_BUCKETS`` 循环调用一次， + 业务代码完全不感知本地 / S3 的差别。 + + Args: + bucket_name: 桶名，必须是 ``PURPOSE_BUCKETS`` 之一。 + + Returns: + 直接喂给 ``create_storage(...)`` 的 dict。 + """ + # 延迟 import：避免 storage -> config -> storage 的循环依赖 + from common.config import settings + + if bucket_name not in PURPOSE_BUCKETS: + raise StorageConfigError( + f"未知 bucket 名称 {bucket_name!r}，可选值: {PURPOSE_BUCKETS}" + ) + + if settings.storage_backend == "local": + return { + "type": "local", + "mode": "async", + "base_dir": str(Path(settings.local_storage_base_dir) / bucket_name), + } + + if settings.storage_backend == "s3": + return { + "type": "s3", + "mode": "async", + "bucket": getattr(settings, f"s3_{bucket_name}_bucket"), + "endpoint_url": settings.s3_endpoint, + "aws_access_key_id": settings.s3_access_key, + "aws_secret_access_key": settings.s3_secret_key, + } + + raise StorageConfigError( + f"settings.storage_backend={settings.storage_backend!r} 不支持，" + f"可选值: 's3', 'local'" + ) + + +def workspaces_root() -> Path: + """返回 runtime 视角下 workspace bucket 的本地路径。 + + 唯一权威入口。``settings.local_storage_base_dir`` 是与存储相关的 + 唯一路径设置(其它路径都从这里推导): + + - s3 模式(默认):``${local_storage_base_dir}/workspaces``(rclone 把 + S3 workspace bucket 挂到这里)。 + - local 模式:``${local_storage_base_dir}/workspace``(直接读写 + 本地目录,无 FUSE 层)。 + + ``runtime.mount.WORKSPACES_ROOT`` 等于本函数返回值,业务代码不要自己 + 拼路径。 + """ + from common.config import settings # 延迟 import 避免循环 + base = Path(settings.local_storage_base_dir) + if settings.storage_backend == "local": + return base / "workspace" + return base / "workspaces" + + +# rclone remote 名字。跟 docker-compose 里的 ``RCLONE_CONFIG__*`` 命名空间 +# 对应——rclone 通过 env var 名前缀来定位 remote 配置块,所以这里的常量名 +# 必须跟 ``RCLONE_CONFIG_S3_*`` 的 ``S3`` 部分一致。 +RCLONE_REMOTE_NAME: str = "s3" + + +def rclone_remote_spec() -> str: + """rclone mount 用的 remote spec (s3 模式才合法)。 + + 格式 ``:``——``runtime.mount.start_rclone_mount`` + 直接喂给 ``rclone mount ``。 + + 唯一权威入口:local 模式下没有 rclone,抛 ``StorageConfigError``。 + """ + from common.config import settings + if settings.storage_backend != "s3": + raise StorageConfigError( + "rclone_remote_spec() 仅在 STORAGE_BACKEND=s3 时合法;" + f"当前 settings.storage_backend={settings.storage_backend!r}" + ) + return f"{RCLONE_REMOTE_NAME}:{settings.s3_workspace_bucket}" diff --git a/common/src/common/storage/registry.py b/common/src/common/storage/registry.py new file mode 100644 index 0000000..56d0b49 --- /dev/null +++ b/common/src/common/storage/registry.py @@ -0,0 +1,61 @@ +"""后端注册表，用 (mode, name) 作为 key 同时管理同步和异步实现。 + +新增一种存储方式的同步或异步实现时，不需要改 factory.py： + @register_backend("local", mode="sync") + class LocalStorageBackend(StorageBackend): ... + + @register_backend("local", mode="async") + class LocalAsyncStorageBackend(AsyncStorageBackend): ... + +只要保证模块被 import 一次即可（backends/__init__.py 里统一 import）。 +""" + +from typing import Dict, Tuple, Type, Union + +from .base import AsyncStorageBackend, StorageBackend +from .exceptions import StorageConfigError + +BackendClass = Union[Type[StorageBackend], Type[AsyncStorageBackend]] + +_REGISTRY: Dict[Tuple[str, str], BackendClass] = {} + +VALID_MODES = ("sync", "async") + + +def _check_mode(mode: str) -> None: + if mode not in VALID_MODES: + raise StorageConfigError(f"不支持的 mode: {mode!r}，可选值: {VALID_MODES}") + + +def register_backend(name: str, mode: str = "sync"): + """类装饰器：把一个后端类注册为 (mode, name) 对应的实现。""" + _check_mode(mode) + + def _decorator(cls: BackendClass) -> BackendClass: + key = (mode, name) + if key in _REGISTRY and _REGISTRY[key] is not cls: + raise StorageConfigError( + f"存储后端 (mode={mode}, type={name}) 已被注册为 {_REGISTRY[key]!r}" + ) + _REGISTRY[key] = cls + return cls + + return _decorator + + +def get_backend_class(name: str, mode: str = "sync") -> BackendClass: + _check_mode(mode) + key = (mode, name) + try: + return _REGISTRY[key] + except KeyError: + available = ", ".join( + f"{m}:{n}" for (m, n) in sorted(_REGISTRY) + ) or "(无)" + raise StorageConfigError( + f"未知的存储后端 (mode={mode}, type={name})，当前已注册: {available}" + ) + + +def registered_backends() -> Dict[Tuple[str, str], BackendClass]: + return dict(_REGISTRY) diff --git a/common/src/common/storage/rustfs.py b/common/src/common/storage/rustfs.py deleted file mode 100644 index 30c535e..0000000 --- a/common/src/common/storage/rustfs.py +++ /dev/null @@ -1,205 +0,0 @@ -from __future__ import annotations - -import hashlib -from typing import Any, BinaryIO -from urllib.parse import urlsplit, urlunsplit - -import boto3 -from botocore.client import Config -from botocore.exceptions import ClientError - - -class RustFSObjectStore: - def __init__( - self, - *, - internal_endpoint: str, - access_key: str, - secret_key: str, - ) -> None: - common = { - "aws_access_key_id": access_key, - "aws_secret_access_key": secret_key, - "region_name": "us-east-1", - "config": Config( - signature_version="s3v4", - s3={"addressing_style": "path"}, - ), - } - self.internal = boto3.client( - "s3", - endpoint_url=internal_endpoint.rstrip("/"), - **common, - ) - self._internal_endpoint = internal_endpoint.rstrip("/") - - def ensure_bucket(self, bucket_name: str) -> None: - try: - self.internal.head_bucket(Bucket=bucket_name) - except ClientError as exc: - code = str(exc.response.get("Error", {}).get("Code", "")) - if code not in {"404", "NoSuchBucket", "NotFound"}: - raise - self.internal.create_bucket(Bucket=bucket_name) - - def presign_put( - self, - *, - bucket_name: str, - object_key: str, - content_type: str, - expected_hash: str | None, - expires_seconds: int, - ) -> tuple[str, dict[str, str]]: - params: dict[str, Any] = { - "Bucket": bucket_name, - "Key": object_key, - "ContentType": content_type, - } - headers = {"Content-Type": content_type} - if expected_hash: - params["Metadata"] = {"sha256": expected_hash} - headers["x-amz-meta-sha256"] = expected_hash - url = self.internal.generate_presigned_url( - "put_object", - Params=params, - ExpiresIn=expires_seconds, - ) - return url, headers - - def presign_get( - self, - *, - bucket_name: str, - object_key: str, - file_name: str, - expires_seconds: int, - ) -> str: - return self.internal.generate_presigned_url( - "get_object", - Params={ - "Bucket": bucket_name, - "Key": object_key, - "ResponseContentDisposition": ( - f'attachment; filename="{file_name.encode("ascii", "ignore").decode() or "download"}"' - ), - }, - ExpiresIn=expires_seconds, - ) - - def rewrite_to_public_path( - self, - url: str, - *, - public_base_url: str, - ) -> str: - """Rewrite the host of a presigned URL to the public edge. - - Replaces the scheme + host (and strips any trailing slash) with - ``public_base_url``; the bucket prefix is moved under the - ``/storage/`` path so Nginx can forward the call to RustFS - without exposing its port. - """ - parsed = urlsplit(url) - public = urlsplit(public_base_url.rstrip("/")) - netloc = public.netloc - prefix = public.path.rstrip("/") - # Presigned URLs are generated against the internal S3 endpoint - # and always start with ``/{bucket}/...``; relocate the bucket - # segment under ``/storage/...`` so the public edge can route - # the request to the right bucket. - rewritten_path = f"{prefix}/storage{parsed.path}" - return urlunsplit(( - public.scheme, - netloc, - rewritten_path, - parsed.query, - parsed.fragment, - )) - - def put_bytes( - self, - *, - bucket_name: str, - object_key: str, - content: bytes, - content_type: str, - content_hash: str, - ) -> None: - self.internal.put_object( - Bucket=bucket_name, - Key=object_key, - Body=content, - ContentType=content_type, - Metadata={"sha256": content_hash}, - ) - - def head(self, *, bucket_name: str, object_key: str) -> dict[str, Any]: - return self.internal.head_object(Bucket=bucket_name, Key=object_key) - - def get_bytes(self, *, bucket_name: str, object_key: str) -> bytes: - response = self.internal.get_object(Bucket=bucket_name, Key=object_key) - body: BinaryIO = response["Body"] - try: - return body.read() - finally: - body.close() - - def sha256(self, *, bucket_name: str, object_key: str) -> str: - response = self.internal.get_object(Bucket=bucket_name, Key=object_key) - body: BinaryIO = response["Body"] - digest = hashlib.sha256() - while chunk := body.read(1024 * 1024): - digest.update(chunk) - body.close() - return digest.hexdigest() - - def delete(self, *, bucket_name: str, object_key: str) -> None: - self.internal.delete_object(Bucket=bucket_name, Key=object_key) - - def copy( - self, - *, - source_bucket: str, - source_key: str, - dest_bucket: str, - dest_key: str, - ) -> None: - """Server-side copy ``source_bucket/source_key`` → ``dest_bucket/dest_key``. - - ``CopySource`` is a single header string of the form - ``/{bucket}/{key}`` — must NOT be URL-encoded or quoted. - """ - self.internal.copy_object( - Bucket=dest_bucket, - Key=dest_key, - CopySource={"Bucket": source_bucket, "Key": source_key}, - ) - - def move_to_trash( - self, - *, - source_bucket: str, - source_key: str, - trash_bucket: str, - trash_key: str, - ) -> None: - """Copy an object into the trash bucket and delete the source. - - The copy is a server-side operation in RustFS (no data flows - through the client). The source delete is best-effort: if it - fails after the copy succeeds the trash holds the only copy of - the bytes, which is exactly the point — the caller can retry. - """ - self.copy( - source_bucket=source_bucket, - source_key=source_key, - dest_bucket=trash_bucket, - dest_key=trash_key, - ) - try: - self.delete(bucket_name=source_bucket, object_key=source_key) - except ClientError: - # Source was already gone, or transient delete failure — - # the trash copy is what matters; caller logs and moves on. - pass diff --git a/common/src/common/storage/schemas.py b/common/src/common/storage/schemas.py index 8c63bd2..2538558 100644 --- a/common/src/common/storage/schemas.py +++ b/common/src/common/storage/schemas.py @@ -10,7 +10,6 @@ from common.schemas import StrictModel __all__ = [ - "CompleteUploadRequest", "CreateUploadRequest", "DownloadUrlRequest", "ServerObjectRequest", @@ -34,6 +33,8 @@ class CreateUploadRequest(StrictModel): expected_size_bytes: int = Field(ge=0, le=100 * 1024 * 1024) expected_hash: str | None = Field(default=None, min_length=64, max_length=64) idempotency_key: str = Field(min_length=8, max_length=128) + visibility: Literal["private", "workspace", "public"] = "private" + is_immutable: bool = False @field_validator("expected_hash") @classmethod @@ -46,21 +47,6 @@ class CreateUploadRequest(StrictModel): return normalized -class CompleteUploadRequest(StrictModel): - usage_type: Literal[ - "data_resource", - "version_artifact", - "snapshot", - "run_log", - "run_result", - "working_copy", - "public_script", - ] - file_name: str = Field(min_length=1, max_length=255) - visibility: Literal["private", "workspace", "public"] = "private" - is_immutable: bool = False - - class ServerObjectRequest(StrictModel): workspace_id: str = Field(min_length=26, max_length=26) user_id: str = Field(min_length=26, max_length=26) diff --git a/default.conf b/default.conf index 1f357ce..5ac3a4e 100644 --- a/default.conf +++ b/default.conf @@ -1,7 +1,7 @@ # ---------------------------------------------------------------------------- # NOTE: this file is mounted into the nginx container as a TEMPLATE. # scripts/nginx-entrypoint.sh (mounted as /docker-entrypoint.sh) substitutes -# the single ${RUSTFS_ENDPOINT} placeholder at container start. The rendered +# the single ${S3_ENDPOINT} placeholder at container start. The rendered # output is written to /etc/nginx/conf.d/default.conf and execs nginx. # ---------------------------------------------------------------------------- @@ -18,8 +18,8 @@ server { # 指定 Docker 内置 DNS 解析器，并设置 30 秒缓存 resolver 127.0.0.11 valid=30s ipv6=off; - # RustFS upstream — full URL passed through to proxy_pass below. - set $rustfs_backend "${RUSTFS_ENDPOINT}"; + # S3 upstream — full URL passed through to proxy_pass below. + set $s3_backend "${S3_ENDPOINT}"; location / { root /usr/share/nginx/html; # 前端静态文件存放在容器中的路径 @@ -58,17 +58,17 @@ server { } # ========================================================================= - # 1. RustFS 对象存储服务转发 (/storage/) + # 1. S3 对象存储服务转发 (/storage/) # ========================================================================= location /storage/ { - # 核心：透传 Host，确保 RustFS 生成的 Presigned URL 包含公网地址 + # 核心：透传 Host，确保 S3 生成的 Presigned URL 包含公网地址 proxy_set_header Host $http_host; proxy_set_header X-Real-IP $remote_addr; proxy_set_header X-Forwarded-For $proxy_add_x_forwarded_for; proxy_set_header X-Forwarded-Proto $scheme; - # 转发至 RustFS（$rustfs_endpoint 来自 set 指令；尾斜杠保留 location /storage/ 前缀剥离语义） - proxy_pass $rustfs_backend/; + # 转发至 S3（$s3_backend 来自 set 指令；尾斜杠保留 location /storage/ 前缀剥离语义） + proxy_pass $s3_backend/; # HTTP/1.1 长连接支持 proxy_http_version 1.1; diff --git a/docker-compose.yml b/docker-compose.yml index 3601256..e4e1264 100644 --- a/docker-compose.yml +++ b/docker-compose.yml @@ -22,14 +22,14 @@ services: context: . dockerfile: frontend/Dockerfile restart: unless-stopped - # Architecture §2.2: this is the only service exposed to the host. The + # Architecture §2.2: this is the only service exposed to the host. The # default.conf file is mounted as a template; scripts/nginx-entrypoint.sh - # parses ${RUSTFS_ENDPOINT} and writes the rendered config to + # parses ${S3_ENDPOINT} and writes the rendered config to # /etc/nginx/conf.d/default.conf before exec'ing nginx. ports: - "${GATEWAY_PORT:-8888}:80" environment: - RUSTFS_ENDPOINT: ${RUSTFS_ENDPOINT:?RUSTFS_ENDPOINT is required} + S3_ENDPOINT: ${S3_ENDPOINT:?S3_ENDPOINT is required} depends_on: backend: condition: service_healthy @@ -49,8 +49,8 @@ services: context: . dockerfile: backend/Dockerfile restart: unless-stopped - # No host port: architecture §2.2 — only Nginx is externally reachable. - # No local-FS volume: backend stores everything in RustFS (RUSTFS_*). + # No host port: architecture §2.2 — only Nginx is externally reachable. + # No local-FS volume: backend stores everything in S3 (S3_*). environment: DATABASE_URL: ${DATABASE_URL:?DATABASE_URL is required} SERVICE_NAME: model-platform-backend @@ -59,15 +59,19 @@ services: DEMO_AUTH_ENABLED: ${DEMO_AUTH_ENABLED:-false} INITIAL_ADMIN_PASSWORD: ${INITIAL_ADMIN_PASSWORD:-admin12345} RUNTIME_API_URL: http://runtime:8000 - RUSTFS_ENDPOINT: ${RUSTFS_ENDPOINT:?RUSTFS_ENDPOINT is required} - RUSTFS_ACCESS_KEY: ${RUSTFS_ACCESS_KEY:?RUSTFS_ACCESS_KEY is required} - RUSTFS_SECRET_KEY: ${RUSTFS_SECRET_KEY:?RUSTFS_SECRET_KEY is required} - RUSTFS_WORKSPACE_BUCKET: ${RUSTFS_WORKSPACE_BUCKET:-workspaces} - RUSTFS_VERSION_BUCKET: ${RUSTFS_VERSION_BUCKET:-versions} - RUSTFS_RUN_LOG_BUCKET: ${RUSTFS_RUN_LOG_BUCKET:-run-logs} - RUSTFS_TRASH_BUCKET: ${RUSTFS_TRASH_BUCKET:-trash} - RUSTFS_TRASH_RETENTION_DAYS: ${RUSTFS_TRASH_RETENTION_DAYS:-30} - READINESS_TARGETS: ${MYSQL_HOST:?MYSQL_HOST is required}:${MYSQL_PORT:-3306},${RUSTFS_HOST:?RUSTFS_HOST is required}:${RUSTFS_PORT:-9000},runtime:8000 + STORAGE_BACKEND: ${STORAGE_BACKEND:-s3} + LOCAL_STORAGE_BASE_DIR: ${LOCAL_STORAGE_BASE_DIR:-/data} + # S3_* only matter when STORAGE_BACKEND=s3. Defaults are kept so local + # mode boots without them; override in .env when switching to s3. + S3_ENDPOINT: ${S3_ENDPOINT:-http://s3:9000} + S3_ACCESS_KEY: ${S3_ACCESS_KEY:-} + S3_SECRET_KEY: ${S3_SECRET_KEY:-} + S3_WORKSPACE_BUCKET: ${S3_WORKSPACE_BUCKET:-workspaces} + S3_VERSION_BUCKET: ${S3_VERSION_BUCKET:-versions} + S3_RUN_LOG_BUCKET: ${S3_RUN_LOG_BUCKET:-run-logs} + S3_TRASH_BUCKET: ${S3_TRASH_BUCKET:-trash} + S3_TRASH_RETENTION_DAYS: ${S3_TRASH_RETENTION_DAYS:-30} + READINESS_TARGETS: ${MYSQL_HOST:?MYSQL_HOST is required}:${MYSQL_PORT:-3306},${S3_HOST:-s3}:${S3_PORT:-9000},runtime:8000 depends_on: migrate: condition: service_completed_successfully @@ -76,6 +80,7 @@ services: volumes: - ./backend:/app/backend - ./common:/app/common + - ./data:/data healthcheck: test: ["CMD-SHELL", "curl -fsS http://127.0.0.1:8000/health/ready >/dev/null"] interval: 10s @@ -98,24 +103,31 @@ services: environment: DATABASE_URL: ${DATABASE_URL:?DATABASE_URL is required} SERVICE_NAME: runtime-manager - WORKSPACES_ROOT: /app/workspaces + STORAGE_BACKEND: ${STORAGE_BACKEND:-s3} + LOCAL_STORAGE_BASE_DIR: ${LOCAL_STORAGE_BASE_DIR:-/data} + # WORKSPACES_ROOT defaults to /data/workspaces (settings.workspaces_root); + # in local mode runtime skips the rclone mount and reads directly from + # ${LOCAL_STORAGE_BASE_DIR}/workspace instead. PUBLIC_BASE_URL: http://runtime - REMOTE_BUCKET: rustfs:${RUSTFS_WORKSPACE_BUCKET:-workspaces} - RCLONE_CONFIG_RUSTFS_TYPE: s3 - RCLONE_CONFIG_RUSTFS_PROVIDER: Other - RCLONE_CONFIG_RUSTFS_ACCESS_KEY_ID: ${RUSTFS_ACCESS_KEY:?RUSTFS_ACCESS_KEY is required} - RCLONE_CONFIG_RUSTFS_SECRET_ACCESS_KEY: ${RUSTFS_SECRET_KEY:?RUSTFS_SECRET_KEY is required} - RCLONE_CONFIG_RUSTFS_ENDPOINT: ${RUSTFS_ENDPOINT:?RUSTFS_ENDPOINT is required} - RCLONE_CONFIG_RUSTFS_ENV_AUTH: "false" - RCLONE_CONFIG_RUSTFS_FORCE_PATH_STYLE: "true" - RCLONE_CONFIG_RUSTFS_REGION: other + # rclone config only used when STORAGE_BACKEND=s3 (mount skipped in local mode). + # The remote spec ("s3:") is derived in + # common.storage.rclone_remote_spec(); no REMOTE_BUCKET env needed. + RCLONE_CONFIG_S3_TYPE: s3 + RCLONE_CONFIG_S3_PROVIDER: Other + RCLONE_CONFIG_S3_ACCESS_KEY_ID: ${S3_ACCESS_KEY:-} + RCLONE_CONFIG_S3_SECRET_ACCESS_KEY: ${S3_SECRET_KEY:-} + RCLONE_CONFIG_S3_ENDPOINT: ${S3_ENDPOINT:-http://s3:9000} + RCLONE_CONFIG_S3_ENV_AUTH: "false" + RCLONE_CONFIG_S3_FORCE_PATH_STYLE: "true" + RCLONE_CONFIG_S3_REGION: other depends_on: migrate: condition: service_completed_successfully volumes: - ./runtime:/app/runtime + - ./data:/data healthcheck: - test: ["CMD-SHELL", "grep -q ' /app/workspaces .* - fuse.rclone ' /proc/self/mountinfo && curl -fsS http://127.0.0.1:8000/api/v1/health >/dev/null"] + test: ["CMD-SHELL", "grep -q ' /data/workspaces .* - fuse.rclone ' /proc/self/mountinfo && curl -fsS http://127.0.0.1:8000/api/v1/health >/dev/null"] interval: 10s timeout: 5s retries: 18 @@ -126,21 +138,23 @@ services: context: . dockerfile: schedule/Dockerfile restart: unless-stopped - # No host port: architecture §2.2 — only Nginx is externally reachable. - # No local-FS volume: schedule executes nodes via tempfile.TemporaryDirectory - # under Python's default temp dir (cleaned per-run); artifacts live in RustFS. + # No host port: architecture §2.2 — only Nginx is externally reachable. + # No local-FS volume: schedule executes nodes via tempfile.TemporaryDirectory + # under Python's default temp dir (cleaned per-run); artifacts live in S3. environment: DATABASE_URL: ${DATABASE_URL:?DATABASE_URL is required} SERVICE_NAME: schedule-executor SCHEDULE_EVENT_NAMESPACE: ${SCHEDULE_EVENT_NAMESPACE:-model-platform-local} BACKEND_API_URL: http://backend:8000 - RUSTFS_ENDPOINT: ${RUSTFS_ENDPOINT:?RUSTFS_ENDPOINT is required} - RUSTFS_ACCESS_KEY: ${RUSTFS_ACCESS_KEY:?RUSTFS_ACCESS_KEY is required} - RUSTFS_SECRET_KEY: ${RUSTFS_SECRET_KEY:?RUSTFS_SECRET_KEY is required} - RUSTFS_WORKSPACE_BUCKET: ${RUSTFS_WORKSPACE_BUCKET:-workspaces} - RUSTFS_VERSION_BUCKET: ${RUSTFS_VERSION_BUCKET:-versions} - RUSTFS_RUN_LOG_BUCKET: ${RUSTFS_RUN_LOG_BUCKET:-run-logs} - READINESS_TARGETS: ${MYSQL_HOST:?MYSQL_HOST is required}:${MYSQL_PORT:-3306},${RUSTFS_HOST:?RUSTFS_HOST is required}:${RUSTFS_PORT:-9000},backend:8000 + STORAGE_BACKEND: ${STORAGE_BACKEND:-s3} + LOCAL_STORAGE_BASE_DIR: ${LOCAL_STORAGE_BASE_DIR:-/data} + S3_ENDPOINT: ${S3_ENDPOINT:-http://s3:9000} + S3_ACCESS_KEY: ${S3_ACCESS_KEY:-} + S3_SECRET_KEY: ${S3_SECRET_KEY:-} + S3_WORKSPACE_BUCKET: ${S3_WORKSPACE_BUCKET:-workspaces} + S3_VERSION_BUCKET: ${S3_VERSION_BUCKET:-versions} + S3_RUN_LOG_BUCKET: ${S3_RUN_LOG_BUCKET:-run-logs} + READINESS_TARGETS: ${MYSQL_HOST:?MYSQL_HOST is required}:${MYSQL_PORT:-3306},${S3_HOST:-s3}:${S3_PORT:-9000},backend:8000 depends_on: backend: condition: service_healthy diff --git a/frontend/Dockerfile b/frontend/Dockerfile index 36b9919..13cb9b4 100644 --- a/frontend/Dockerfile +++ b/frontend/Dockerfile @@ -1,16 +1,11 @@ -FROM node:24.18.1-slim AS frontend-build +FROM node-pnpm-base:24.18.1 AS frontend-build WORKDIR /app -RUN corepack enable && corepack prepare pnpm@10.15.1 --activate COPY frontend/package.json frontend/pnpm-lock.yaml ./ RUN pnpm install --frozen-lockfile COPY frontend/ ./ RUN pnpm typecheck && pnpm build -FROM nginx:alpine -ENV TZ=Asia/Shanghai -RUN apk add --no-cache tzdata \ - && ln -snf /usr/share/zoneinfo/$TZ /etc/localtime \ - && echo $TZ > /etc/timezone +FROM nginx-base:alpine COPY ./default.conf /etc/nginx/conf.d/default.conf.template COPY ./scripts/nginx-entrypoint.sh /usr/local/bin/model-platform-entrypoint.sh RUN sed -i 's/\r$//' /usr/local/bin/model-platform-entrypoint.sh \ diff --git a/frontend/app/components/admin/DashboardPage.tsx b/frontend/app/components/admin/DashboardPage.tsx new file mode 100644 index 0000000..bcbd9c0 --- /dev/null +++ b/frontend/app/components/admin/DashboardPage.tsx @@ -0,0 +1,83 @@ +import Icon from "../../components/common/Icon"; +import { useAuth } from "../../context/AuthContext"; + +export function DashboardPage({ + scriptCount, + online, + onNavigate, +}: { + scriptCount: number; + online: boolean; + onNavigate: (page: "scripts" | "schedules" | "system") => void; +}) { + const { user, currentWorkspace } = useAuth(); + return ( +
+
+
+ MODEL DEVELOPMENT PLATFORM +

下午好，{user?.display_name ?? "用户"}

+

+ 当前位于 {currentWorkspace?.workspace_name ?? "（未选择 Workspace）"} + ，可以继续构建脚本或配置调度。 +

+
+ {online ? "服务正常" : "服务连接中"} +
+
+
{scriptCount}工作副本
+
2Workspace
+
4平台用户
+
{online ? "正常" : "检查中"}平台状态
+
+
+ + + +
+
+
+
运行趋势

近 7 天调度执行

成功率 92.6%
+
+ {[38, 55, 44, 73, 61, 86, 78].map((value, index) => ( +
+ {Math.round(value / 7)} + + {["周一", "周二", "周三", "周四", "周五", "周六", "今天"][index]} +
+ ))} +
+
成功 75失败 6
+
+
+
脚本资产

类型分布

+
+
{scriptCount}全部脚本
+
+ Notebook{Math.max(1, Math.round(scriptCount * .67))} 个 · 67% + Python{Math.max(0, scriptCount - Math.round(scriptCount * .67))} 个 · 33% + 稳定版本3 个已发布 +
+
+
+
+
ACTIVITY

最近平台活动

+
+
操作内容执行人状态时间
+ {[ + ["数据探索.ipynb 发布稳定版本 v3.0", "张三", "成功", "16:42"], + ["每日模型训练流程完成调度运行", "Scheduler", "成功", "15:25"], + ["批量预测.py 更新工作副本", "王五", "已同步", "14:18"], + ["风险验证流程完成 DAG 校验", "李四", "成功", "11:06"], + ].map((row) => ( +
+ {row[0]} + {row[1]}{row[2]}{row[3]} +
+ ))} +
+
+
+
+ ); +} diff --git a/frontend/app/components/admin/ProjectManagementPage.tsx b/frontend/app/components/admin/ProjectManagementPage.tsx new file mode 100644 index 0000000..1b3ec65 --- /dev/null +++ b/frontend/app/components/admin/ProjectManagementPage.tsx @@ -0,0 +1,364 @@ +import { useEffect, useState } from "react"; + +import { ApiRequestError, type Employee, type Workspace } from "../../services/api"; +import { useApi, useAuth } from "../../context/AuthContext"; +import Icon from "../common/Icon"; +import { UserMultiSelect } from "./UserMultiSelect"; + +const EMPTY_PROJECT_FORM = { + workspace_code: "", + workspace_name: "", + quota_bytes: 0, + description: "", +}; + +export function ProjectManagementPage({ + onNotify, + onConnectionChange, +}: { + onNotify: (notice: { tone: "success" | "error" | "info"; message: string }) => void; + onConnectionChange: (online: boolean) => void; +}) { + const api = useApi(); + const { user } = useAuth(); + const [projects, setProjects] = useState([]); + const [projectLoading, setProjectLoading] = useState(true); + const [projectDialogOpen, setProjectDialogOpen] = useState(false); + const [projectForm, setProjectForm] = useState(EMPTY_PROJECT_FORM); + const [editingProject, setEditingProject] = useState(null); + const [importMemberDialogOpen, setImportMemberDialogOpen] = useState(false); + const [selectedProject, setSelectedProject] = useState(null); + const [availableUsers, setAvailableUsers] = useState([]); + const [selectedUserIds, setSelectedUserIds] = useState([]); + const [selectedRoleCode, setSelectedRoleCode] = useState<"admin" | "developer">("developer"); + const [projectSearchTerm, setProjectSearchTerm] = useState(""); + const [saving, setSaving] = useState(false); + + const canManage = user?.role_code === "admin"; + + const loadProjects = async (): Promise => { + setProjectLoading(true); + try { + const workspaceList = await api.listWorkspaces(); + setProjects(workspaceList); + onConnectionChange(true); + } catch (error) { + onConnectionChange(false); + onNotify({ + tone: "error", + message: error instanceof Error ? error.message : "项目列表加载失败", + }); + } finally { + setProjectLoading(false); + } + }; + + const loadAvailableUsers = async (): Promise => { + try { + const list = await api.listEmployees(); + setAvailableUsers(list); + } catch (error) { + onNotify({ + tone: "error", + message: error instanceof Error ? error.message : "用户列表加载失败", + }); + } + }; + + useEffect(() => { + void loadProjects(); + }, []); + + const openCreateProject = (): void => { + setEditingProject(null); + setProjectForm(EMPTY_PROJECT_FORM); + setProjectDialogOpen(true); + }; + + const openEditProject = (project: Workspace): void => { + setEditingProject(project); + setProjectForm({ + workspace_code: project.workspace_code, + workspace_name: project.workspace_name, + quota_bytes: project.quota_bytes, + description: project.description ?? "", + }); + setProjectDialogOpen(true); + }; + + const submitProject = async (event: React.FormEvent): Promise => { + event.preventDefault(); + if (!projectForm.workspace_name.trim()) { + onNotify({ tone: "error", message: "请输入项目名称" }); + return; + } + if (!projectForm.workspace_code.trim() && !editingProject) { + onNotify({ tone: "error", message: "请输入项目编码" }); + return; + } + setSaving(true); + try { + if (editingProject) { + const updated = await api.updateWorkspace(editingProject.workspace_id, { + workspace_name: projectForm.workspace_name.trim(), + quota_bytes: projectForm.quota_bytes, + description: projectForm.description.trim() || undefined, + }); + setProjects((current) => + current.map((p) => (p.workspace_id === updated.workspace_id ? updated : p)) + ); + onNotify({ tone: "success", message: "项目信息已更新" }); + } else { + const generatedCode = projectForm.workspace_code.trim() || projectForm.workspace_name.trim().toLowerCase().replace(/[^a-z0-9-]/g, "-").slice(0, 32); + const created = await api.createWorkspace({ + workspace_code: generatedCode, + workspace_name: projectForm.workspace_name.trim(), + quota_bytes: projectForm.quota_bytes, + description: projectForm.description.trim() || undefined, + }); + setProjects((current) => [...current, created]); + onNotify({ tone: "success", message: "项目已创建" }); + } + setProjectDialogOpen(false); + } catch (error) { + onNotify({ + tone: "error", + message: error instanceof ApiRequestError ? error.message : (editingProject ? "更新项目失败" : "创建项目失败"), + }); + } finally { + setSaving(false); + } + }; + + const deleteProject = async (project: Workspace): Promise => { + if (!window.confirm(`确定要删除项目"${project.workspace_name}"吗？此操作将级联软删所有成员。`)) return; + try { + await api.deleteWorkspace(project.workspace_id); + setProjects((current) => current.filter((p) => p.workspace_id !== project.workspace_id)); + onNotify({ tone: "success", message: "项目已删除" }); + } catch (error) { + onNotify({ + tone: "error", + message: error instanceof Error ? error.message : "删除项目失败", + }); + } + }; + + const openImportMemberDialog = (project: Workspace): void => { + setSelectedProject(project); + setSelectedUserIds([]); + setSelectedRoleCode("developer"); + void loadAvailableUsers(); + setImportMemberDialogOpen(true); + }; + + const importMember = async (): Promise => { + if (!selectedProject || selectedUserIds.length === 0) { + onNotify({ tone: "error", message: "请选择要添加的用户" }); + return; + } + setSaving(true); + try { + await Promise.all( + selectedUserIds.map((userId) => + api.addWorkspaceMember(selectedProject.workspace_id, { + user_id: userId, + role_code: selectedRoleCode, + }) + ) + ); + setImportMemberDialogOpen(false); + onNotify({ tone: "success", message: `已添加 ${selectedUserIds.length} 名成员` }); + } catch (error) { + onNotify({ + tone: "error", + message: error instanceof ApiRequestError ? error.message : "添加成员失败", + }); + } finally { + setSaving(false); + } + }; + + return ( +
+
+
+ + setProjectSearchTerm(event.target.value)} + /> +
+ +
+ + {!canManage &&
当前为开发人员，只能查看项目列表。
} +
+
+ 项目名称 + 项目编码 + 状态 + 配额 + 操作 +
+ {projectLoading ? ( +

正在加载项目…

+ ) : projects.length === 0 ? ( +

暂无项目

+ ) : ( + projects + .filter((project) => { + const term = projectSearchTerm.toLowerCase().trim(); + if (!term) return true; + return ( + project.workspace_name.toLowerCase().includes(term) || + project.workspace_code.toLowerCase().includes(term) || + (project.description && project.description.toLowerCase().includes(term)) + ); + }) + .map((project) => ( +
+ + {project.workspace_name.slice(0, 1)} + + {project.workspace_name} + {project.description ?? "无描述"} + + + {project.workspace_code} + + + {project.status === "active" ? "正常" : project.status === "archived" ? "已归档" : "已删除"} + + + {project.quota_bytes > 0 ? `${(project.quota_bytes / 1024 / 1024 / 1024).toFixed(1)} GB` : "无限制"} + + + + + +
+ )) + )} +
+ + {projectDialogOpen && ( +
+
+
+
+ PROJECT +

{editingProject ? "编辑项目" : "新建项目"}

+
+ +
+
void submitProject(event)}> + {!editingProject && ( + + )} + + +