diff --git a/.env.example b/.env.example index 3216d94..7ea6fae 100644 --- a/.env.example +++ b/.env.example @@ -14,10 +14,17 @@ MYSQL_PASSWORD=change-me MYSQL_DATABASE=model_platform DATABASE_URL=mysql+asyncmy://root:change-me@127.0.0.1:3306/model_platform?charset=utf8mb4 -# Demo login is only intended for this self-hosted development UI. -DEMO_AUTH_ENABLED=true JWT_SECRET=change-this-development-secret +# Force the Secure flag on the session cookie even when the inbound request +# scheme is plain HTTP. Enable behind a TLS-terminating reverse proxy that +# strips/rewrites X-Forwarded-Proto — otherwise the cookie is written without +# Secure and browsers refuse to send it back over HTTPS. +COOKIE_FORCE_SECURE=false + +# Service label surfaced in lifespan / health checks. +SERVICE_NAME=service + # ============================================================================ # CRITICAL: must set BEFORE first run. The initial admin user is seeded by # the deployment bootstrap. Never keep the development default in production. @@ -30,6 +37,18 @@ INITIAL_ADMIN_PASSWORD=admin12345 # runtime_client._jupyter_request. LOG_LEVEL=INFO +# Audit log (backend HTTP interface compliance log). One line per HTTP +# request, written to data/logs/audit/audit-YYYY-MM-DD.log (one file per +# day). AUDIT_LOG_DIR is relative to the backend process cwd (/app in the +# container). AUDIT_LOG_RETENTION_DAYS=0 disables cleanup of old files. +AUDIT_LOG_DIR= +AUDIT_LOG_RETENTION_DAYS=30 +# Exact paths excluded from the audit line (health/root probes carry no +# business value but fire every second from K8s/LB). The default already +# covers /health/live /health/ready /api/v1/health / /health/storage; +# leave empty to keep the default. Comma-separated, e.g. /health/live,/api/v1/health. +AUDIT_EXCLUDED_PATHS= + # Object storage. Two modes are supported: # STORAGE_BACKEND=s3 — connects to an S3-compatible service (MinIO, # RustFS, SeaweedFS, AWS S3, …). Requires the @@ -74,6 +93,13 @@ S3_TRASH_RETENTION_DAYS=30 # over the compose network. RCLONE_RC_URL=http://runtime:5572 +# Backend → Runtime HTTP endpoint (Jupyter contents API, file ops). +RUNTIME_API_URL=http://runtime:8000 + +# Public base URL for the runtime container (surfaced to clients for +# Jupyter access tickets / embedded URLs). +PUBLIC_BASE_URL=http://runtime + # ============================================================================ # Service-to-service auth (P0-1 fix). # Backend's /internal/v1/* storage control plane requires this shared secret. @@ -83,3 +109,15 @@ RCLONE_RC_URL=http://runtime:5572 # python -c "import secrets; print(secrets.token_urlsafe(48))" # ============================================================================ INTERNAL_SERVICE_TOKEN=change-me-internal-service-token + +# Schedule → Backend HTTP base URL (cron post-back / status callbacks). +BACKEND_API_URL=http://backend:8000 + +# Max concurrent notebooks running in the schedule worker. Each notebook is +# dispatched as an asyncio task bounded by a semaphore; the polling loop is +# never blocked. +SCHEDULE_EXECUTION_CONCURRENCY=4 + +# Readiness probe targets. Comma-separated host:port list checked by +# /health/ready; empty disables the check. e.g. mysql:3306,s3:9000 +READINESS_TARGETS= diff --git a/API.md b/API.md index acad6a4..3bb71d7 100644 --- a/API.md +++ b/API.md @@ -50,6 +50,42 @@ > - `is_system_admin: bool` —— 派生自 `users.platform_role_id` 指向的角色 `role_code == 'admin'` 且用户状态为 `active`。前端据此决定是否渲染"系统管理"入口。 > - `permissions: string[]` —— 当前用户通过其平台角色(`platform_role_id`)间接持有的菜单权限码列表(`permission_code`),按字典序排列;未分配平台角色时为空数组。前端据此过滤菜单与 `` 路由守卫。**仅控制前端展示,不参与后端 endpoint 鉴权**——后端鉴权继续由 `system_admin_context`(`role_code == 'admin'`)与 workspace membership 负责。详见 §7.12-7.14。 +### 1.1 登录 / 会话 + +| 方法 | 路径 | 说明 | +|---|---|---| +| `POST` | `/api/v1/auth/login` | 校验用户名密码,种 `access_token` HttpOnly Cookie,返回 `user` + `workspaces` | +| `POST` | `/api/v1/auth/logout` | 清除 Cookie(幂等) | +| `GET` | `/api/v1/auth/me` | 返回当前登录用户与可访问 workspace 列表 | + +### 1.2 `PATCH /api/v1/auth/me` — 修改本人资料 + +当前登录用户修改自己的显示名 / 邮箱。不需要 `workspace_id`。 + +- **请求体**(至少提供一个字段;多余字段 → 422): + +| 字段 | 类型 | 限制 | 说明 | +|---|---|---|---| +| `display_name` | string | 1~100 | trim 后写入 | +| `email` | string \| null | ≤255 | trim 后写入;空字符串归一为 `null`;与其他未软删用户冲突 → 409 "邮箱已存在" | + +- 不可通过本端点修改:`username`、`password`、`status`、`platform_role_id`。 +- **响应 200**:`data.user` 与 `GET /me` 中的 `user` 形状一致(含 `role_code` / `is_system_admin` / `permissions`)。 + +### 1.3 `POST /api/v1/auth/password` — 本人修改密码 + +- **请求体**: + +| 字段 | 类型 | 限制 | 说明 | +|---|---|---|---| +| `current_password` | string | 1~72 | 当前密码 | +| `new_password` | string | 8~72 | 新密码 | + +- 当前密码错误 → 400 "当前密码不正确"。 +- 新密码与当前密码相同 → 400 "新密码不能与当前密码相同"。 +- **成功后清除会话 Cookie**,客户端须引导用户重新登录。 +- **响应 200**:`data.password_changed = true`。 + --- ## 二、统一约定 @@ -99,7 +135,12 @@ ### 3.2 `POST /api/v1/workspace-directories` -创建一个**目录**。后端会落一行 `StorageObjects(object_type='directory', storage_backend='rustfs', storage_uri='inline://directory/{path}', visibility='private')`,因此空目录也能在 §3.1 树里出现并保留下来。 +创建一个**目录**。后端会落一行 `StorageObjects(object_type='directory', storage_backend='rustfs', storage_uri='inline://directory/{path}', visibility='public')`,因此空目录也能在 §3.1 树里出现并保留下来。 + +> 目录行默认 `visibility='public'`(非 `private`)。目录是结构性导航行, +> 默认 public 使同一 workspace 内其他成员可以浏览彼此的目录结构(目录树 +> 跨 owner 可见);文件级私密仍由 §3.4 / §五 的 visibility 过滤兜底 +> —— 其他 owner 的 `private` 脚本 / 数据资源不会返回。 - **请求体**: ```json @@ -123,7 +164,8 @@ "storage_object_id": "01HXY...", "path": "scripts/etl", "name": "etl", - "parent_path": "scripts" + "parent_path": "scripts", + "owner_user_id": "01HXX..." } } ``` @@ -155,10 +197,12 @@ - **行为**: `parent_path` 为空字符串或缺省 → 用户根目录;非空 → 该父目录的直接子目录(workspace-relative)。仅返回 `object_status='available' AND is_deleted=0` 的 `StorageObjects` 行。 - **鉴权**: workspace 成员 +- **owner 作用域**: `owner_user_id` 缺省时 scope 为**当前请求者**本人根目录(`scoped_prefix = workspace/{me}`);传 `owner_user_id` 时 scope 为该 owner 的根目录(`scoped_prefix = workspace/{owner_user_id}`),用于跨 owner 浏览目录树(见 §3.4 visibility 模型)。该接口本身不施加 visibility 过滤——目录行默认 `visibility='public'`(见 §3.2),跨 owner 均可见。 - **查询参数**: | 名 | 类型 | 必填 | 说明 | |---|---|---|---| | `parent_path` | string | 否 | 父目录相对路径,空字符串或缺省表示用户根目录 | + | `owner_user_id` | string | 否 | 目标 owner 的 user_id;缺省=请求者本人。指定后 scope 到 `workspace/{owner_user_id}/{parent_path}` | - **谓词(SQL 等价)**: `relative_path LIKE '/%' AND relative_path NOT LIKE '/%/%'`,其中 `prefix = scoped_prefix/{parent_path}`,索引走 `idx_storage_workspace_relative_path(workspace_id, relative_path(255))`。 - **响应**: ```json @@ -166,9 +210,9 @@ "request_id": "...", "data": { "directories": [ - {"path": "scripts", "name": "scripts", "parent_path": "", "has_children": true}, - {"path": "scripts/etl", "name": "etl", "parent_path": "scripts", "has_children": false}, - {"path": "scripts/etl/daily", "name": "daily", "parent_path": "scripts/etl", "has_children": false} + {"path": "scripts", "name": "scripts", "parent_path": "", "owner_user_id": "01HXX...", "has_children": true}, + {"path": "scripts/etl", "name": "etl", "parent_path": "scripts", "owner_user_id": "01HXX...", "has_children": false}, + {"path": "scripts/etl/daily", "name": "daily", "parent_path": "scripts/etl", "owner_user_id": "01HXX...", "has_children": false} ] }, "meta": {"directory_count": 3} @@ -180,15 +224,28 @@ | `path` | string | workspace 内相对路径 | | `name` | string | `path` 的最后一段 | | `parent_path` | string | 父目录相对路径,根目录用空串 | + | `owner_user_id` | string | 该目录行所属 owner 的 user_id(`owner_user_id` 参数缺省时=请求者本人) | | `has_children` | bool | 该目录下是否还有直接子目录(后端额外 `EXISTS` 查询,可为空目录为 `false`) | - **空结果**: 不返回 404,空目录列表即 `directories: []`。 - **错误**: 401(未登录)/ 403(非 workspace 成员)同其他接口。 ### 3.4 `GET /api/v1/scripts` -列出当前 workspace 内**全部 active 脚本**。不受 is_locked 影响(读路径不锁)。 +列出脚本,按 **owner 作用域 + visibility 过滤**返回。不受 is_locked 影响(读路径不锁)。 -- **响应**: `data` 为 `ScriptPayload` 数组(见 §3.10)。 +- **owner 作用域**: `owner_user_id` 缺省=当前请求者本人,scope 到 `workspace/{me}/...`;传 `owner_user_id` 时 scope 到 `workspace/{owner_user_id}/...`,用于跨 owner 浏览他人脚本。 +- **visibility 过滤(统一)**: 非 admin 请求者只返回 `owner_user_id == me OR visibility IN (workspace, public)`;admin 请求者跳过过滤返回全部。 + - **owner=me(缺省)**: scope 是我的子树,行都是我的 → `owner==me` 恒成立 → **含我的 private 脚本** ✓ + - **owner=other**: scope 是他人的子树,`owner==me` 不成立 → 只剩其 `workspace/public` 脚本(排除他人的 `private`) ✓ + - 即"本人可见自己全部;他人只见其 workspace/public",私密仅在 owner==me 时可见。 +- **非递归**: 仅返回 `parent_path` 下的**直接子级**脚本(懒加载用);子目录脚本需带 `parent_path` 再次请求。 +- **查询参数**: + | 名 | 类型 | 必填 | 说明 | + |---|---|---|---| + | `parent_path` | string | 否 | 父目录相对路径,空串/缺省=该 owner 根目录下的一级脚本 | + | `owner_user_id` | string | 否 | 目标 owner;缺省=请求者本人 | + | `keyword` | string | 否 | 名称模糊匹配 | +- **响应**: `data` 为 `ScriptPayload` 数组(见 §3.10),每条带 `owner_user_id`。 ### 3.5 `GET /api/v1/scripts/{script_id}` @@ -485,8 +542,10 @@ queued ──→ running ──┬─→ succeeded | `POST` | `/api/v1/data-resources/uploads` | 创建上传会话,返回 `upload_id` + `upload_path` | | `PUT` | `/api/v1/data-resources/uploads/{upload_id}` | 上传字节(请求体即文件内容) | | `POST` | `/api/v1/data-resources/uploads/{upload_id}/bind` | 绑定已完成上传为数据资源 | -| `GET` | `/api/v1/data-resources` | 列表(workspace 范围) | +| `GET` | `/api/v1/data-resources` | 列表(owner 作用域 + visibility 过滤) | | `GET` | `/api/v1/data-resources/{id}` | 详情 | +| `GET` | `/api/v1/data-resources/{id}/content` | 同源流式读取文件字节(预览/下载) | +| `GET` | `/api/v1/data-resources/{id}/preview` | 表格抽样预览(csv/tsv, `limit` 默认 100) | | `POST` | `/api/v1/data-resources/{id}/download-url` | 生成 presigned GET URL | | `DELETE` | `/api/v1/data-resources/{id}` | 软删 | @@ -521,6 +580,22 @@ queued ──→ running ──┬─→ succeeded `content_base64` 字段(JSON 体里走),内部走同一条 `AsyncStorageBackend.put` 路径,前端无需分两步。 +### 五.1 `GET /api/v1/data-resources` + +列出数据资源,按 **owner 作用域 + visibility 过滤**返回(与 §3.4 `GET /scripts` 同一套统一语义)。 + +- **owner 作用域**: `owner_user_id` 缺省=当前请求者本人,scope 到 `object_key` 前缀 `{workspace_id}/{me}/...`;传 `owner_user_id` 时 scope 到 `{workspace_id}/{owner_user_id}/...`。 +- **visibility 过滤(统一)**: 非 admin 请求者只返回 `owner_user_id == me OR visibility IN (workspace, public)`;admin 请求者跳过过滤返回全部。语义同 §3.4——owner=me 含自己的 private;owner=other 只见其 workspace/public。 +- **非递归**: 仅返回 `parent_path` 下的**直接子级**资源(懒加载用)。 +- **查询参数**: + | 名 | 类型 | 必填 | 说明 | + |---|---|---|---| + | `parent_path` | string | 否 | 父目录相对路径,空串/缺省=该 owner 根目录下的一级资源 | + | `owner_user_id` | string | 否 | 目标 owner;缺省=请求者本人 | + | `visibility` | string | 否 | `workspace` \| `public` \| `private`,二次过滤 | + | `keyword` | string | 否 | 名称模糊匹配 | +- **响应**: `data` 为 `ResourcePayload` 数组,每条带 `owner_user_id`。 + --- ## 六、管理后台 @@ -581,36 +656,69 @@ Base 前缀 `/api/v1/admin`。 ## 七、系统管理 (`/api/v1/platform/...`) 平台级(跨 workspace)管理接口,用于管理员工、workspace 实体与 workspace 成员。 -所有端点要求调用者是**系统管理员**——其 `users.platform_role_id` 指向 +除特别注明外,所有端点要求调用者是**系统管理员**——其 `users.platform_role_id` 指向 `role_code='admin'` 的角色行,且 `users.status == 'active'`。系统管理员判定 通过 `GET /api/v1/auth/me` 响应中的 `data.user.is_system_admin` 字段(详见 §一)。 +> **例外 — `GET /workspaces/{id}/members`**:该端点对**系统管理员(任意 +> workspace)**与**该 workspace 的活跃成员**(`workspace_members.is_deleted=0` +> 且 `member_status='active'`)均开放。这是为了让普通(非 admin)用户能在 +> 脚本目录树里渲染同 workspace 其他成员的折叠分组(跨 owner 浏览,见 §3.4)。 +> 其余 members 写端点(POST/PATCH/DELETE members)仍仅限系统管理员。 + | 方法 | 路径 | 说明 | |---|---|---| -| `GET` | `/api/v1/platform/employees` | 列全平台未软删员工;包含停用、锁定及无平台角色用户 | +| `GET` | `/api/v1/platform/employees` | 列全平台未软删员工（cursor 分页 + `q` 搜索，见 §7.0） | | `POST` | `/api/v1/platform/employees` | 创建平台员工账号(返回 201);不自动加入任何 workspace | | `PATCH` | `/api/v1/platform/employees/{user_id}` | 改员工资料/状态/平台角色(仅系统管理员) | +| `POST` | `/api/v1/platform/employees/{user_id}/reset-password` | 重置员工密码(仅系统管理员;不需要旧密码) | | `DELETE` | `/api/v1/platform/employees/{user_id}` | 软删员工;级联软删其 workspace 成员关系(仅系统管理员) | -| `GET` | `/api/v1/platform/workspaces` | 列 workspace(`active`/`archived`);已软删的过滤掉 | +| `GET` | `/api/v1/platform/workspaces` | 列 workspace(`active`/`archived`)（cursor 分页 + `q` 搜索，见 §7.0） | | `POST` | `/api/v1/platform/workspaces` | 创建 workspace(返回 201);创建者自动成为 admin 成员 | | `GET` | `/api/v1/platform/workspaces/{workspace_id}` | 单个 workspace(含已 disabled 的,用于恢复) | | `PATCH` | `/api/v1/platform/workspaces/{workspace_id}` | 改 workspace 字段;`status` 仅允许 `active`/`archived` | | `DELETE` | `/api/v1/platform/workspaces/{workspace_id}` | 软删 workspace;级联软删其成员 | -| `GET` | `/api/v1/platform/workspaces/{workspace_id}/members` | 列成员 | +| `GET` | `/api/v1/platform/workspaces/{workspace_id}/members` | 列成员(**系统管理员或该 workspace 活跃成员**;为跨 owner 目录树提供成员名册,见 §7 intro 例外) | | `POST` | `/api/v1/platform/workspaces/{workspace_id}/members` | 添加成员(返回 201) | | `PATCH` | `/api/v1/platform/workspaces/{workspace_id}/members/{user_id}` | 改成员 `member_status`;**不能改 role_code**(workspace 角色继承自平台角色) | | `DELETE` | `/api/v1/platform/workspaces/{workspace_id}/members/{user_id}` | 软删成员 | | `GET` | `/api/v1/platform/roles` | 列全部 `role_scope='platform'` 的角色及其 `permission_codes`(仅系统管理员) | | `GET` | `/api/v1/platform/roles/{role_code}/permissions` | 单个平台角色的 `permission_codes`(仅系统管理员) | -| `PATCH` | `/api/v1/platform/roles/{role_code}/permissions` | 整体替换角色权限集合;对 `admin` 角色保留 `system.view` + `system.manage` 的最后系统管理员保护(仅系统管理员) | +| `PATCH` | `/api/v1/platform/roles/{role_code}/permissions` | 整体替换角色权限集合;对 `admin` 角色强制保留 `system:view` 的最后系统管理员保护(仅系统管理员) | > **不变量**: > - 每个 workspace 必须始终保留至少一个 `admin` 角色的活跃成员;对最后 admin 做降级 / 停用 / 删除 → 409。 > - `PATCH /employees/{user_id}` 降级 admin → developer 时同样触发 workspace last-admin 守卫(因为 workspace 角色继承自 platform 角色,降级会级联到所有活跃 membership);platform 必须始终保留至少一个 `active` 系统管理员;对最后系统管理员做降级 / 停用 / 删除 → 409。 > - 系统管理员不能通过 `DELETE .../members/{self}` 把自己移除(403)。唯一退出方式是 `DELETE /workspaces/{id}` 软删整个 workspace,后者会级联软删所有成员。 -> - workspace 与成员列表接口静默 `pageSize=100` 上限,无客户端分页参数(YAGNI);`GET /employees` 按契约返回全部未软删员工,不设隐藏上限。 +> - `GET /employees` 与 `GET /workspaces` 支持 cursor 分页与关键字搜索（见 §7.0）；成员列表仍静默 `pageSize=100` 上限。 > - 跨 workspace 操作**不**需要 `?workspace_id=` query 参数,与 `/api/v1/admin/...`(workspace 内成员管理)不要混淆。 +### 7.0 列表分页约定（employees / workspaces） + +`GET /api/v1/platform/employees` 与 `GET /api/v1/platform/workspaces` 使用 **keyset cursor** 分页（无 `offset` / `page`）。 + +| Query | 类型 | 默认 | 说明 | +|---|---|---|---| +| `limit` | int | `10` | 每页条数，范围 1~200 | +| `cursor` | string | 无 | 上一页返回的 `meta.next_cursor`；缺省为第一页；非法值 → 400 | +| `q` | string | 无 | 关键字搜索。employees 匹配 `display_name`/`username`/`email`；workspaces 匹配 `workspace_name`/`workspace_code`/`description` | + +响应 `meta`： + +```json +{ + "limit": 10, + "page_count": 10, + "total_count": 156, + "has_more": true, + "next_cursor": "..." +} +``` + +- `total_count`：当前筛选条件下的总条数（用于页码展示）。 +- `next_cursor`：无下一页时为 `null`。 +- 排序键：`(created_at ASC, id ASC)`。前端用 cursor 栈实现「上一页 / 下一页 + 已访问页码」；不支持任意跳到未访问过的深页。 + ### 7.1 `POST /api/v1/platform/workspaces` 创建 workspace;创建者(当前系统管理员)自动成为该 workspace 的 `admin` 成员。 @@ -793,6 +901,19 @@ Base 前缀 `/api/v1/admin`。 - **响应 200**:返回更新后的 `PlatformEmployeePayload`,`role_code` / `role_name` 反映最新的 `platform_role_id`。 +### 7.10.1 `POST /api/v1/platform/employees/{user_id}/reset-password` + +系统管理员为指定员工设置新密码(不校验旧密码)。JWT 无服务端吊销列表,已有会话在过期前仍可用;对方下次登录须用新密码。 + +- **请求体**: + +| 字段 | 类型 | 限制 | 说明 | +|---|---|---|---| +| `new_password` | string | 8~72 | 新密码;bcrypt 哈希后写入 `password_hash` | + +- 目标不存在或已软删 → 404 "用户不存在"。 +- **响应 200**:`data = { "user_id": "...", "password_reset": true }`。 + ### 7.11 `DELETE /api/v1/platform/employees/{user_id}` 软删除平台员工;级联软删其所有 `workspace_members` 行。调用者必须是系统管理员。 @@ -840,11 +961,8 @@ Base 前缀 `/api/v1/admin`。 "role_name": "管理员", "is_builtin": true, "permission_codes": [ - "dashboard.view", "experiment.all", "experiment.own", - "resource.personal", "resource.public.manage", - "resource.public.upload", "schedule.all", "schedule.own", - "script.build", "script.public.manage", "system.manage", - "system.view" + "dashboard:view", "schedule:view", "script:view", + "system:project:view", "system:user:view", "system:view" ] }, { @@ -853,8 +971,7 @@ Base 前缀 `/api/v1/admin`。 "role_name": "开发人员", "is_builtin": true, "permission_codes": [ - "dashboard.view", "experiment.own", "resource.personal", - "schedule.own", "script.build", "script.public.manage" + "dashboard:view", "schedule:view", "script:view" ] } ], @@ -880,11 +997,8 @@ Base 前缀 `/api/v1/admin`。 "role_name": "管理员", "is_builtin": true, "permission_codes": [ - "dashboard.view", "experiment.all", "experiment.own", - "resource.personal", "resource.public.manage", - "resource.public.upload", "schedule.all", "schedule.own", - "script.build", "script.public.manage", "system.manage", - "system.view" + "dashboard:view", "schedule:view", "script:view", + "system:project:view", "system:user:view", "system:view" ] }, "meta": {} @@ -895,7 +1009,7 @@ Base 前缀 `/api/v1/admin`。 整体替换指定平台角色的 `permission_codes`(diff-based 写入,见下文)。调用者必须是系统管理员。 -> **本端点只控制前端菜单可见性**——不修改 `system_admin_context` 的鉴权判定(`role_code == "admin"` 始终等价于"拥有所有平台菜单权限")。若需调整 API 鉴权,请改 `backend.platform.system_admin_context`,不要绕过本端点。 +> **本端点只控制前端菜单可见性**——不修改 `system_admin_context` 的鉴权判定(`role_code == "admin"` 始终等价于"拥有所有平台菜单权限")。若需调整 API 鉴权,请改 `backend.api.platform.system_admin_context`,不要绕过本端点。 - **请求体字段**: @@ -905,8 +1019,8 @@ Base 前缀 `/api/v1/admin`。 - **守卫顺序(load-bearing,不可调换)**: 1. 角色不存在 / `role_scope != 'platform'` → 404。 - 2. **`admin` 角色**:提交的 `permission_codes` 必须同时包含 `system.view` 与 `system.manage`,否则 → 409 "admin 角色必须保留 system.view 与 system.manage 权限"。 - 3. **非 `admin` 角色**:`permission_codes` 中**禁止**含 `system.*` 项 → 422 "非 admin 角色不能拥有 system.* 权限: [...]"。menu permission 只控前端展示,后端 `/api/v1/platform/*` 鉴权仍按 `role_code == "admin"`;若让 developer 拿到 `system.manage`,前端会渲染"系统管理"入口但所有 platform API 调用 403,UX 割裂。 + 2. **`admin` 角色**:提交的 `permission_codes` 必须包含 `system:view`,否则 → 409 "admin 角色必须保留 system:view 权限"。 + 3. **非 `admin` 角色**:`permission_codes` 中**禁止**含 `system:*` 项 → 422 "非 admin 角色不能拥有 system:* 权限: [...]"。menu permission 只控前端展示,后端 `/api/v1/platform/*` 鉴权仍按 `role_code == "admin"`;若让 developer 拿到 `system:view`,前端会渲染"系统管理"入口但所有 platform API 调用 403,UX 割裂。 4. 任意 `permission_code` 不在 `permissions` 表活跃行中 → 422 "未知的 permission_code: [...]"。 5. 写入策略:diff-based —— 只 soft-delete `current \ new` 的关联,只 INSERT `new \ current` 的关联。重复提交同 payload 是 no-op;包含原有 codes 的 patch 不会触发 `(role_id, permission_id)` 主键冲突。**严禁**先全量 soft-delete 再全量 INSERT(会 `IntegrityError`,因为软删行仍占主键 slot)。 - 写入后,响应 `data.permission_codes` 为本次写入后的活跃集合,与再次 `GET §7.13` 完全一致。 @@ -920,32 +1034,26 @@ Base 前缀 `/api/v1/admin`。 "role_code": "admin", "role_name": "管理员", "is_builtin": true, - "permission_codes": ["dashboard.view", "system.manage", "system.view"] + "permission_codes": ["dashboard:view", "system:view"] }, "meta": {} } ``` -- **当前 seed 的 permission_code 全集**(来自迁移 `f6a7b8c9d0e1`,与 `migrations/data/migrate_system_json.py::PERMISSION_NAMES` 真值对齐,不要在客户端另造一份): +- **当前 seed 的 permission_code 全集**(由 baseline `e1f2a3b4c5d6_rebuild_baseline` 直接写入 MySQL,不要在客户端另造一份): | `permission_code` | `module_code` | admin | developer | |---|---|:-:|:-:| -| `dashboard.view` | dashboard | ✓ | ✓ | -| `script.build` | script | ✓ | ✓ | -| `script.public.manage` | script | ✓ | ✓ | -| `schedule.own` | schedule | ✓ | ✓ | -| `schedule.all` | schedule | ✓ | | -| `experiment.own` | experiment | ✓ | ✓ | -| `experiment.all` | experiment | ✓ | | -| `resource.personal` | resource | ✓ | ✓ | -| `resource.public.upload` | resource | ✓ | | -| `resource.public.manage` | resource | ✓ | | -| `system.view` | system | ✓ | | -| `system.manage` | system | ✓ | | +| `dashboard:view` | dashboard | ✓ | ✓ | +| `script:view` | script | ✓ | ✓ | +| `schedule:view` | schedule | ✓ | ✓ | +| `system:view` | system | ✓ | | +| `system:user:view` | system | ✓ | | +| `system:project:view` | system | ✓ | | > **与现有 7.x 端点的语义差异**(避免 reviewer 误读): > - 本端点不修改 `users.platform_role_id`,只调整 `role_permissions` 关联表。 -> - "清空 developer 的全部权限"是合法操作;只有 `admin` 受 `system.*` 强制约束。 +> - "清空 developer 的全部权限"是合法操作;只有 `admin` 受 `system:*` 强制约束。 > - `role_code` 不是 `permission_code`,前端不要用前者去判断菜单可见性。 --- diff --git a/ARCHITECTURE.md b/ARCHITECTURE.md index 141ceec..e41e5db 100644 --- a/ARCHITECTURE.md +++ b/ARCHITECTURE.md @@ -1,39 +1,5 @@ -# 简化系统架构 +# Architecture -```text -Browser - | - v -Nginx Gateway (静态 React Router SPA + /api + /jupyter 代理) - |-----------------------------| - | /api/v1 | /jupyter/ - v v -FastAPI Backend Shared Jupyter Server - | ^ - | Runtime HTTP | Runtime 管理会话/票据 - v | -Runtime Manager ---------------| - | - +------ MySQL（编辑租约、运行实例） - -FastAPI Backend - | 1. 写 schedule_runs + outbox_events - | 2. 尝试 HTTP 立即推送 - v -Schedule Executor（APScheduler） - |-- MySQL APSchedulerJobStore - |-- MySQL Outbox 轮询兜底 - |-- DAG 节点执行与重试 - |-- S3 日志/结果 - +-- Backend 内部 Storage API -``` - -## 关键简化 - -1. 删除 Redis 服务、Redis Streams 和 Redis 文件锁。 -2. 调度定义、运行记录、Outbox、Inbox、Cron JobStore 都由 MySQL 保存。 -3. 立即运行采用 Backend -> Schedule Executor 内部 HTTP 推送；推送失败由 MySQL Outbox 轮询兜底。 -4. Schedule Executor 自带 APScheduler，负责 Cron 触发和 DAG 执行。 -5. 文件编辑锁改为 MySQL 租约，Runtime 单副本运行。 -6. Jupyter 使用一个共享容器，工作区目录通过 Volume 挂载同步。 -7. 前端改为 React Router SPA，并按 feature / route / service / component 分层。 +This file has been merged into [`DEVELOP.md`](./DEVELOP.md) — see +[§Architecture](./DEVELOP.md#architecture) for the authoritative +component diagram, capability map, container table, and storage layout. diff --git a/CLAUDE.md b/CLAUDE.md index 2f00ba5..d106ebb 100644 --- a/CLAUDE.md +++ b/CLAUDE.md @@ -76,6 +76,16 @@ Hard-won lessons. Read the relevant bullet before touching the named area. - **Mock response ordering matters for re-reads.** `update_platform_employee` reads `current_role` before write then `response_role` after. A scalar mock returning a fixed value will return the pre-write role in the response — track call order or look up by `user.platform_role_id` post-write. - **Mock users must declare every attribute the handler writes.** `Users.deleted_at` is not in column defaults; `SimpleNamespace(user_id=...)` raises `AttributeError` on `target.deleted_at = now`. Set `user.deleted_at = None` explicitly. +### Schedule service layering (domain / scheduling / application / execution / infrastructure) + +Lessons from the zero-behavior-change refactor that split the flat 11-file `schedule/src/schedule/` into five subpackages (commits 3118694 → 7476c27, with a follow-up `git mv` in stage 8 placing the orchestrator under `application/`). Public class names (`SchedulerService`, `CronScheduler`, `DispatchOrchestrator`, `NodeExecutor`, `SchedulerStorageClient`) and `schedule/pyproject.toml` are unchanged. + +- **`python -m schedule.notebook_runner` is a stable external contract.** The worker launches the notebook subprocess with `sys.executable, "-m", "schedule.notebook_runner"`. That `-m` string must never change — so `schedule/notebook_runner.py` survives as a 6-line shim re-exporting `main` from `schedule.execution.runners.notebook`. Don't "clean up" the shim. +- **Mock `patch()` string targets and in-function lazy imports are invisible to import-line greps.** `test_janitor.py` patched `"schedule.orchestrator.session_scope"` and `test_worker.py` patched `"schedule.service.build_object_store"`; `worker.py` also had a `from schedule.service import ...` *inside a function body*. When files move, these silently no-op against the old path — and hard-crash once the old file is deleted. After any move, grep the whole tree for the old module path (including tests) and rewrite every hit, not just top-level imports. +- **A new package dir shadows a same-named flat module.** Creating `schedule/execution/` makes the old `schedule/execution.py` silently dead code (the package wins import resolution), so move-then-delete, don't just copy. `git` usually detects these as renames, which keeps the diff reviewable. +- **Docstring references survive file deletion.** After removing flat files, `:class:\`schedule.worker.NodeExecutor\``-style text can linger in docstrings and render as broken links. Grep for the old module name one more time at cleanup and rewrite comment-only refs too. +- **Don't silently upgrade dataclass-ness during a "structural only" refactor.** Stage 1 moved `ExecutionResult` from the flat `schedule/execution.py` into `domain/execution.py` and *decorated* it with `@dataclass(frozen=True)` along the way. Pre-refactor it was a plain class. Review (2026-08-21) caught that this changes three things at once: identity-`==` becomes value-`==`, mutation raises `FrozenInstanceError`, and `repr()` becomes structured. No caller in the repo mutates or compares these objects, so the only externally visible change is log format — but it is *not* "zero behavior change." If you want strict behavioral equivalence during a structural move, copy the class definition verbatim and document any intentional semantic tightening. + ### Frontend state + routing (zustand + React Router v8) Lessons from splitting `frontend/app/features/platform/ModelPlatformApp.tsx` (1057 → 121 lines) into zustand stores + nested routes. @@ -92,6 +102,37 @@ Lessons from splitting `frontend/app/features/platform/ModelPlatformApp.tsx` (10 - **An unmatched nested child leaves `` blank → white screen inside the layout.** Always cover `/` either with `index(...)` or by letting the parent layout `` on a location check. `/login` is the only top-level path that escapes this trap. - **`pnpm typecheck` runs `react-router typegen && tsc`.** Type errors from the generated `+types/...` files surface here too. New route files must be registered in `routes.ts` first. +### Frontend store slice split (zustand) + +Lessons from splitting `frontend/app/features/schedules/state/schedulesStore.ts` (1251 lines, 40KB) into 8 files: `types.ts`, `helpers.ts`, `canvasSlice.ts`, `listSlice.ts`, `dialogSlice.ts`, `runsSlice.ts`, `logsSlice.ts`, `useSchedulesStore.ts`. Public exports (`useSchedulesStore`, `bindSchedulesApi`) and selector protocol unchanged. + +- **`StateCreator` cascades implicit `any` through the slice body.** Writing `StateCreator` to give `get()` cross-slice access makes every nested call (`get().schedules.filter((item) => ...)`) fail with TS7006 — `eslint-disable @typescript-eslint/no-explicit-any` does NOT save you, because TS still infers `any`. Fix: declare a combined `SchedulesStore = State & Actions` type in the **root composition file** (`useSchedulesStore.ts`), then each slice `import type { SchedulesStore } from "./useSchedulesStore"` and uses `StateCreator`. TypeScript accepts the circular `import type` because it erases at build time. `get()` now returns a fully typed snapshot and `.map((item) => ...)` infers correctly. +- **Don't write `` constraints on cross-slice helpers.** Every helper needs to know about every other slice's fields, and the constraint chain keeps growing. Better: have helpers (`withMutation`, `applyServerUpdatedSchedule`, `handleError`) take `set: (partial: Partial | ((s: SchedulesStore) => Partial)) => void` and `get: () => SchedulesStore` directly. `SchedulesStore` already enumerates everything; no constraint to extend. +- **Inner-closure helpers become standalone functions.** Monolithic `create((set, get) => { async function withMutation(...) { ... } })` captures `set`/`get` implicitly. When splitting the store you must reify these as exported functions in `helpers.ts` taking `(set, get)` arguments. This forces signatures to spell out exactly which fields they touch — which is what makes the `SchedulesStore`-typed approach pay off. +- **"De-duplicate" requires value comparison, not just name matching.** Plan item "make `constants.ts` re-export `EMPTY_SCHEDULE_FORM` from `state/types`" looks like obvious dedup, but the values differ: + ```ts + // constants.ts (utils.ts imports this) + cronExpression: "0 9 * * *", + // state/types (canvasSlice initial state) + cronExpression: "", + ``` + Consolidating silently changes `utils.ts`'s runtime defaults. Per "never break userspace", leave `constants.ts` alone. **Rule: before any dedup, grep both call sites and diff the actual values, not just the symbol names.** +- **Cross-slice field ownership belongs to layout, not data.** `scheduleKeyword` / `artifactKeyword` filter inputs look like listSlice state because they filter schedules, but they're bound to the left-panel UI and updated by the canvas layout component — they belong in `canvasSlice`. Decision rule: "which layout component writes this field?" not "which data does this field filter?". +- **`positionDrafts` Map stays at module scope in `helpers.ts`.** It must NOT move into `CanvasSliceState`. The original behavior — `reset()` does NOT clear drag-in-progress drafts — is a feature; users expect their unsaved drag to survive reset. If you move it into slice state, audit every `clearAllPositionDrafts()` call site and decide whether each should fire on reset. +- **`reset()` mirrors original `set({ ...initial, loading: true })` semantics.** The legacy reset wipes ALL slice fields including user-edited `scheduleForm` / `nodeForm` (because `initial.scheduleForm = EMPTY_SCHEDULE_FORM`). Any "preserve user input in reset()" change silently diverges from original behavior. If you want to preserve form values, do it as an intentional new feature with its own API, not a side-effect of refactoring. + ### Frontend coupling -- **`UserManagementPage.tsx` and `api.ts` still call `/api/v1/admin/employees`.** Don't migrate them in the same change as a `/api/v1/platform/employees` addition — the contract surface is intentionally duplicated. \ No newline at end of file +- **`UserManagementPage.tsx` and `api.ts` still call `/api/v1/admin/employees`.** Don't migrate them in the same change as a `/api/v1/platform/employees` addition — the contract surface is intentionally duplicated. + +### File size: 500 lines hard cap + +- **每个文件最多 500 行** — 超过时主动拆分,不要等 review 才动。 +- 常见拆分维度: + - **后端 — 按资源 / 端点分组**:例如 `api/platform.py` (1600 行 / 20 endpoint) → `api/platform/{__init__,_deps,employees,roles,workspaces}.py`。原文件改为 thin shim,只 re-export 公共符号,保持 `from backend.api.platform import router` 等既有 import path 不变。 + - **后端 — 按层级**:参考 `schedule/` 已有的 `domain/` / `application/` / `infrastructure/` 分层。 + - **前端 — 按职责**:`types.ts` / `helpers.ts` / `slices/` / `useXStore.ts`(参考 `useSchedulesStore` 拆 8-slice 的纪律)。 + - **前端 — 页面 vs 路由 wrapper**:Page 组件持有 useState,route 文件只做 `` 重挂载,二者不要混在一起。 +- **拆分前先列调用面**(`grep "from "`),任何外部 import 路径必须仍然可用 — 用 re-export 或 shim 兜底,不要让调用方被迫改。 +- **拆分后**每个新文件 ≤ 500 行是硬约束,验证方式:`wc -l ` 或 CI 脚本。 +- 拆分本身是**纯结构调整**,endpoint 行为 / URL / 响应 schema 零变化 — 不要顺手"清理"。 \ No newline at end of file diff --git a/DEVELOP.md b/DEVELOP.md index 2781ab5..c5f9750 100644 --- a/DEVELOP.md +++ b/DEVELOP.md @@ -1,63 +1,222 @@ # DEVELOP.md — Developer Guide -This guide is for engineers working on the model platform codebase. For -high-level design see `ARCHITECTURE.md`; for the current state of in-flight -refactors see `HANDOVER.md`. +This guide is for engineers working on the model platform codebase. It is the +authoritative source for architecture, code layout, configuration, conventions, +and common tasks. `ARCHITECTURE.md` now only records the simplification +history; `HANDOVER.md` covers recent commits and pending work. + +## Architecture + +The platform is a self-hosted Jupyter model development environment: interactive +workspaces, DAG scheduling, object-store artifacts, and per-workspace runtimes +— all exposed through a single Nginx gateway. + +### Capability map + +| Capability | Where it lives | +|---|---| +| Workspace notebook editing, row-level lock | `backend/api/jupyter.py` + `scripts.is_locked` | +| Jupyter auth routing (browser never holds runtime token) | `nginx/default.conf` + `auth_request` + `backend/api/jupyter.py` | +| Object storage for notebook / script / version / run_log (s3 / local) | `common/storage/` + `backend/services/storage.py` | +| DAG scheduling: nodes, edges, cron, manual trigger, retry, snapshot | `backend/api/schedules/` + `backend/api/schedules/runs.py` + `schedule/` (5 layers) | +| DAG execution via MySQL Outbox (no Redis, no in-process queue) | `schedule/application/orchestrator.py` + `schedule/execution/worker.py` | +| Per-workspace Jupyter subprocess pool, asyncio lock | `runtime/process.py` | +| Runtime rclone FUSE mount of workspace bucket (s3 mode only) | `runtime/mount.py` | +| 18 MySQL tables, soft delete, zero FK, async SQLAlchemy 2.0 | `common/db/models/` | + +### Component diagram + +``` + ┌────────────────────┐ + │ Browser (SPA) │ + └─────────┬──────────┘ + │ HTTPS / WS + ┌─────────▼──────────┐ + │ Nginx (only :80) │ ← templates/default.conf + │ /api/ /jupyter/ /storage/ + └────┬───────┬──────┘ + │ │ + ┌──────────────┘ └─────────────┐ + ▼ ▼ + ┌──────────────────┐ ┌──────────────────────┐ + │ FastAPI Backend │ │ Runtime (Jupyter) │ + │ + /internal/v1 │ control │ - rclone FUSE mount │ + │ /objects │ token-Auth │ │ + │ (storage) ├──────────────►│ - subprocess pool │ + │ - DAG CRUD │ │ (per workspace) │ + │ - script CRUD │ └──────────┬───────────┘ + │ - auth_request │ │ FUSE / shared vol + │ - /api/v1/... │ ▼ + └────┬──────┬──────┘ ┌──────────────────────┐ + │ │ │ Object storage │ + │ └──────── HTTP ───────►│ (s3: S3 service / │ + ▼ │ local: shared vol) │ + ┌────────────┐ │ 4 buckets per usage │ + │ MySQL │◄───────── poll ─────│ │ + │ - 18 tbls │ └──────────────────────┘ + │ - outbox │ + │ - jobstore │ + └────┬───────┘ + ▲ + │ outbox poll + ┌────┴──────────────────────────┐ + │ Schedule Executor │ + │ - CronScheduler (APScheduler) │ + │ - DispatchOrchestrator │ + │ - NodeExecutor (worker) │ + │ - SchedulerService (facade) │ + └───────────────────────────────┘ +``` + +### Services (docker-compose) + +The architecture intentionally exposes only one host port (the gateway); all +other services are on the Docker internal network. + +| Service | Image | Exposed | Purpose | +|---|---|---|---| +| `web` | `nginx:alpine` | host `:8888` → `:80` | SPA, `/api/` reverse proxy, `/jupyter/{ws}/` `auth_request` proxy, `/storage/` S3 passthrough (s3 mode only) | +| `backend` | `Dockerfile` | internal only | DAG CRUD, script CRUD, schedule trigger, `/api/v1/auth/jupyter`, `/internal/v1/objects` inter-service RPC (shared `INTERNAL_SERVICE_TOKEN`, see `§Auth`) | +| `runtime` | `Dockerfile` | internal only | Per-workspace Jupyter subprocess pool, rclone FUSE mount of `workspace` bucket (s3 mode) | +| `schedule` | `Dockerfile` | internal only | Cron tick + DAG execution (polls MySQL Outbox) | + +The pre-2026 host-port mappings for backend/runtime (`8891:8000` / `8892:8000`) +were removed; no service is reachable from the host except Nginx anymore. +Backend ↔ schedule now talk via the `X-Internal-Service-Token` header on +`/internal/v1/*`. See `API.md §9`. + +### Storage layout + +The object store is selected at deploy time by `settings.storage_backend` +(`"s3"` default, `"local"` for dev / single-node / air-gapped). Either way +there are 4 purpose-named buckets, resolved in a single place +(`common/storage/factory.py:actual_bucket_name` + `USAGE_TYPE_TO_PURPOSE`). + +| `usage_type` | Bucket | Env var | Default name (s3) | +|---|---|---|---| +| `working_copy`, `public_script`, `data_resource`, `snapshot` | `workspace` | `S3_WORKSPACE_BUCKET` | `workspace` | +| `version_artifact` | `version` | `S3_VERSION_BUCKET` | `version` | +| `run_log`, `run_result` | `run_log` | `S3_RUN_LOG_BUCKET` | `run-log` | +| (soft-delete target) | `trash` | `S3_TRASH_BUCKET` | `trash` | + +`STORAGE_BACKEND=s3` → 4 separate S3 buckets. +`STORAGE_BACKEND=local` → 4 subdirectories under `LOCAL_STORAGE_BASE_DIR` +(default `/data`): + +``` +/data/ +├── workspace/ # S3_WORKSPACE_BUCKET +├── version/ # S3_VERSION_BUCKET +├── run_log/ # S3_RUN_LOG_BUCKET +└── trash/ # S3_TRASH_BUCKET +``` + +A workspace's `Workspaces.artifact_bucket` column (when non-null) overrides +the default for that workspace, regardless of `usage_type` — useful for +isolating paid customers onto a dedicated bucket. + +Object keys are a flat two-level path — `workspace_id` plus a server-issued +ULID — preserving the original file extension so Jupyter can pick its editor +from the suffix: + +``` +//{.} +``` + +File name, extension, MIME, and logical path all live on `StorageObjects` / +`Scripts` rows; reorganizing the bucket does not require rewriting the +database. Backend code never writes to the container local filesystem except +in `STORAGE_BACKEND=local` mode (where the shared `local-storage` volume IS +the canonical store). Schedule Executor stages node artifacts in +`tempfile.TemporaryDirectory()` (auto-cleanup). Only the `runtime` container +keeps a host volume — s3 mode needs it for rclone FUSE; local mode is a no-op +passthrough. ## Code layout +All Python packages use the `src//` layout; `uv` workspace glues them into +one `.venv`. Always invoke via `uv run [--package ] ` (see "Local +development" for the gotcha). + ``` -common/ Pure-Python shared library - config.py Settings (pydantic-settings, lru_cache singleton) - db/ SQLAlchemy 2.0 async engine, session_scope, Base - db/models/ 26 tables in 9 domain files (zero FK, zero relationship) - scheduler/ build_sqlalchemy_jobstore (delayed import) - storage/ AsyncStorageBackend abstraction (s3 + local impls) + Pydantic schemas - eventing.py add_outbox_event / utcnow / event_time - service_app.py /health/ready TCP probe, /api/v1/health - schemas.py StrictModel base - utils.py get_free_port, start_process +common/src/common/ Pure-Python shared library + config.py Settings (pydantic-settings, lru_cache singleton) + db/ SQLAlchemy 2.0 async engine, session_scope, Base + db/models/ 18 tables in 7 domain files (zero FK, zero relationship) + auth/ JWT / bcrypt / workspace membership helpers + scheduler/ APScheduler trigger helpers (delayed import) + storage/ AsyncStorageBackend abstraction + Pydantic schemas + base.py Abstract interface + factory.py create_storage + build_storage_config + PURPOSE_BUCKETS + schemas.py CreateUploadRequest / ServerObjectRequest + backends/local.py Local filesystem impl + backends/s3.py S3-compatible impl (boto3) + registry.py Bucket registry + eventing.py add_outbox_event / utcnow / event_time + service_app.py /health/ready TCP probe, /api/v1/health + logging.py loguru config (LOG_LEVEL) + schemas.py StrictModel base + ids.py ULID generation helpers + utils.py get_free_port, start_process -backend/ Public FastAPI service + tiny /internal/v1/objects RPC - main.py lifespan + route registration - jupyter.py /api/v1/auth/jupyter — the ONLY auth entry - scripts.py CRUD for scripts/notebooks (object storage via AsyncStorageBackend) - schedules.py DAG CRUD: schedules, nodes, edges - schedule_runs.py Trigger / list / get runs - schedule_schemas.py Pydantic request/response models - admin.py Admin endpoints - resources.py Misc data resources - storage_api.py /internal/v1/objects — single token-guarded endpoint (P0-1) - storage_client.py Stub (HTTP client removed post-migration; rewrite pending) - schedule_client.py Placeholder module (was the HTTP-push executor client) - runtime_client.py Self-contained httpx wrapper for the runtime - jupyter.py auth_request handler - dependencies.py request_context, database_session +backend/src/backend/ Public FastAPI service + main.py lifespan + route registration + audit.py HTTP access log middleware (loguru sink) + api/ HTTP route handlers (one module per bounded context) + auth.py /api/v1/auth/* (login / me / jupyter) + jupyter.py /api/v1/auth/jupyter — the ONLY auth entry + dependencies.py request_context, database_session + platform.py /api/v1/platform/* (system admin) + admin.py /api/v1/admin/* (workspace-internal admin) + scripts.py /api/v1/scripts/* + /api/v1/workspace-directories + resources.py /api/v1/data-resources/* + schedules/schedules.py DAG CRUD + schedules/runs.py Run lifecycle + storage.py /internal/v1/objects — single token-guarded endpoint (P0-1) + services/ Pure-Python business logic (no HTTP / no DI) + scripts.py create_workspace_directory, visibility-filtered queries + resources.py owner-scoped resource listing helpers + jupyter.py jupyter_path / lock helpers + storage.py object store helpers + schedules.py DAG validation (cycle / orphan detection) + schemas/ Pydantic request / response models + auth.py / common.py / jupyter.py / platform.py / resources.py / schedules.py / scripts.py + clients/ Outbound HTTP / RPC clients + runtime.py Self-contained httpx wrapper for the runtime + scheduler.py Backend → Schedule HTTP client (callback / dispatch) + rclone.py rclone RC API client (FUSE cache invalidation) -schedule/ Schedule Executor (DAG worker) - context.py Constants + naive_utc - scheduler.py CronScheduler (APScheduler + 5s sync loop) - orchestrator.py DispatchOrchestrator (Outbox poll + DAG advance) - worker.py NodeExecutor (notebook / python execution) - service.py SchedulerService facade (composes the three) - main.py Lifespan + FastAPI app - storage_client.py SchedulerStorageClient — talks to backend /internal/v1/objects - execution.py execute_artifact (notebook + python paths) - notebook_runner.py Subprocess entry point (nbclient) +schedule/src/schedule/ Schedule Executor (DAG worker) + main.py Lifespan + FastAPI app + notebook_runner.py Subprocess entry point (nbclient) — DO NOT RENAME + domain/ Pure-Python domain types + execution.py ExecutionResult (frozen dataclass) + state enums + context.py Constants + naive_utc + scheduling/ Time-based trigger + scheduler.py CronScheduler (APScheduler + 5s sync loop) + application/ Facades / orchestrators + service.py SchedulerService (composes the three) + orchestrator.py DispatchOrchestrator (Outbox poll + DAG advance) + execution/ DAG node execution + executor.py Dispatch helpers (script-type routing) + worker.py NodeExecutor + asyncio entry, schedule-spawned task boundary + runners/notebook.py nbclient subprocess path (6-line `notebook_runner` shim re-exports `main`) + infrastructure/ External-system adapters + storage/client.py SchedulerStorageClient — talks to backend /internal/v1/objects -runtime/ Jupyter Runtime - main.py FastAPI entry: jupyter action endpoints - process.py Per-workspace subprocess pool + asyncio locks - mount.py rclone FUSE mount lifecycle +runtime/src/runtime/ Jupyter Runtime + main.py FastAPI entry: jupyter action endpoints + process.py Per-workspace subprocess pool + asyncio locks + mount.py rclone FUSE mount lifecycle -frontend/ React Router SPA (vite build → nginx) - app/ features/ routes/ services/ components/ +frontend/ React Router SPA (vite build → nginx) + app/ features/ routes/ services/ components/ -migrations/ Alembic schema versions -docker-compose.yml 4 services -default.conf Nginx template +migrations/ Alembic schema versions +docker-compose.yml 5 services (migrate / web / backend / runtime / schedule) +default.conf Nginx template scripts/nginx-entrypoint.sh -.env.example +.env.example All 26 config.py keys documented ``` ## Configuration system @@ -67,10 +226,31 @@ All env vars go through one place: `common/src/common/config.py`. ```python from common.config import settings -settings.database_url # str -settings.storage_backend # str: "s3" (default) or "local" -settings.local_storage_base_dir # str: root dir for storage data (default "/data"); see "Storage" below for per-mode derivation -settings.s3_endpoint # str (full URL, e.g. "http://s3:9000"; s3 mode only) +# Auth / runtime +settings.database_url # str — SQLAlchemy async URL (mysql+asyncmy, charset utf8mb4) +settings.jwt_secret # HS256 secret for the auth_request handler +settings.cookie_force_secure # bool — write Secure flag even on plain HTTP (TLS-terminating proxy) +settings.service_name # surfaced in /health +settings.schedule_event_namespace # Outbox event-type namespace prefix (NOT APScheduler JobStore) +settings.readiness_targets # CSV host:port list for /health/ready + +# HTTP clients (intra-cluster URLs) +settings.runtime_api_url # backend → runtime HTTP base +settings.public_base_url # runtime public base URL (browser-facing /jupyter/) +settings.backend_api_url # schedule → backend HTTP base +settings.rclone_rc_url # backend → rclone RC control API +settings.internal_service_token # Backend ↔ Schedule shared secret (X-Internal-Service-Token) + +# Logging / audit +settings.log_level # DEBUG / INFO / WARNING / ERROR / CRITICAL (lowercase → fallback INFO) +settings.audit_log_dir # dir for daily audit logs (relative to cwd; "" disables file sink) +settings.audit_log_retention_days # 0 disables cleanup +settings.audit_excluded_paths # list[str] — paths skipped from audit (health probes, etc.) + +# Storage +settings.storage_backend # "s3" (default) or "local" +settings.local_storage_base_dir # root dir for storage data (default "/data") +settings.s3_endpoint # str (s3 mode only) settings.s3_access_key # str (s3 mode only) settings.s3_secret_key # str (s3 mode only) settings.s3_workspace_bucket # str (s3 mode only) @@ -78,17 +258,35 @@ settings.s3_version_bucket # str (s3 mode only) settings.s3_run_log_bucket # str (s3 mode only) settings.s3_trash_bucket # str (s3 mode only) settings.s3_trash_retention_days # int (s3 mode only) -settings.jwt_secret # HS256 secret for the auth_request handler -settings.backend_api_url # schedule → backend HTTP base -settings.runtime_api_url # backend → runtime HTTP base -settings.public_base_url # runtime public base URL -settings.service_name # surfaced in /health -settings.readiness_targets # CSV host:port list for /health/ready + +# Schedule +settings.schedule_execution_concurrency # int — max concurrent notebook subprocesses ``` +### Encrypted env values + +`Settings` runs a `model_validator` (`_decrypt_encrypted_fields`, +`common/src/common/config.py:168-180`) that scans every string field for the +prefix `ENC(...)` and decrypts the inner value with `APP_CONFIG_SECRET_KEY` +using Fernet. Use this for secrets that should not be stored in plain `.env` +files (e.g. third-party API tokens shipped via deployment config). + +```bash +# .env +APP_CONFIG_SECRET_KEY= +SOME_TOKEN=ENC(gAAAAABm...) # ciphertext produced by Fernet.encrypt(b"plaintext") +``` + +At process start, `SOME_TOKEN` resolves to the decrypted plaintext. If the +field is *not* encrypted (no `ENC(...)` prefix), it passes through unchanged, +so plain `.env` files keep working. The pre-2026 in-repo `encrypt_secret.py` +script was the CLI wrapper around the same Fernet key; if you have old +ciphertexts they round-trip with the new `APP_CONFIG_SECRET_KEY` value. + `Settings` reads from process env first, then from a `.env` file at CWD if present. `pydantic-settings` auto-loads. `case_sensitive=False` so -`DATABASE_URL` / `database_url` both work. +`DATABASE_URL` / `database_url` both work. The full list of 26 fields +is in `common/src/common/config.py`. ### Adding a new env var @@ -96,7 +294,8 @@ if present. `pydantic-settings` auto-loads. `case_sensitive=False` so ```python new_var: str = Field(default="x", description="...") ``` -2. Add the line to `.env.example` with a comment. +2. Add the line to `.env.example` with a comment (keep it synced — every + field in config.py must have a matching `.env.example` entry). 3. Use `settings.new_var` at the call site. Do **not** call `os.environ["NEW_VAR"]` or `os.getenv("NEW_VAR")` in @@ -116,8 +315,13 @@ grep -rnE 'os\.(environ\[?["\x27][A-Z_]+|getenv\(["\x27][A-Z_]+)' --include="*.p `deleted_at DATETIME(3) NULL`; queries must filter `is_deleted == 0` (or `deleted_at.is_(None)`) to avoid logical-deleted rows. - Domain files under `common/src/common/db/models/` are split by - bounded context: `audit` / `events` / `experiments` / `identity` / - `runtime` / `schedules` / `scripts` / `storage` / `workspaces`. + bounded context: `events` (2 tables: `ConsumerInbox`, + `OutboxEvents`) / `identity` (4: `Users`, `Roles`, `RolePermissions`, + `Permissions`) / `schedules` (5: `Schedules`, `ScheduleRuns`, + `ScheduleNodes`, `ScheduleEdges`, `ScheduleNodeRuns`) / `scripts` + (2: `Scripts`, `Versions`) / `storage` (3: `StorageObjects`, + `UploadSessions`, `DataResources`) / `workspaces` (2: `Workspaces`, + `WorkspaceMembers`) — 18 tables across 6 model files. - All models are `class X(Base)` SQLAlchemy 2.0 declarative-mapped. - The full schema is in `migrations/versions/`. Apply with: ```bash @@ -136,16 +340,18 @@ grep -rnE 'os\.(environ\[?["\x27][A-Z_]+|getenv\(["\x27][A-Z_]+)' --include="*.p right `create_storage` kwargs for each of the 4 purpose buckets (`workspace`, `version`, `run_log`, `trash`). Use it in lifespan code; route handlers don't see the difference. -- Bucket resolution from `usage_type` is in **one place** - (`backend/storage_api.py:resolve_bucket`); route handlers only know - about `app.state.object_stores[bucket_name]`. +- Bucket resolution from `usage_type` lives in **one place** + (`common/src/common/storage/factory.py:actual_bucket_name` + + `USAGE_TYPE_TO_PURPOSE`). The route handlers in + `backend/src/backend/api/storage.py` only know about + `app.state.object_stores[bucket_name]` and never call + `settings.s3_*_bucket` directly. - The runtime's view of the workspace bucket on disk is exposed by `common.storage.workspaces_root()`: - - `s3` mode: `${settings.local_storage_base_dir}/workspace` - (default `/data/workspace`, the rclone FUSE mount target). - - `local` mode: `${settings.local_storage_base_dir}/workspace` - (default `/data/workspace`, a subdir of the shared local-storage - volume). + - Both modes resolve to `${settings.local_storage_base_dir}/workspace` + (default `/data/workspace`); s3 mode uses it as the rclone FUSE + mount target, local mode uses it as a subdir of the shared + `local-storage` volume. `settings.local_storage_base_dir` is the **only** path setting; the helper handles the per-mode suffix. Don't read `settings.workspaces_root` or any other path setting directly in runtime code — use this helper. @@ -168,7 +374,7 @@ grep -rnE 'os\.(environ\[?["\x27][A-Z_]+|getenv\(["\x27][A-Z_]+)' --include="*.p ### Service-to-service auth (P0-1 fix) - Schedule → Backend single endpoint ``POST /internal/v1/objects`` is - guarded by ``require_internal_service`` in ``backend.storage_api``. + guarded by ``require_internal_service`` in ``backend.api.storage``. - The token header is ``X-Internal-Service-Token`` (case-insensitive on the wire because FastAPI ``Header`` lowercase-matches the name ``x-internal-service-token``); the secret value comes from @@ -190,28 +396,211 @@ grep -rnE 'os\.(environ\[?["\x27][A-Z_]+|getenv\(["\x27][A-Z_]+)' --include="*.p - For write operations on a script/notebook, call `require_script_modify_access(script, user_id=..., is_admin=...)` - from `backend/scripts.py`. It enforces: + from `backend/src/backend/services/scripts.py`. It enforces: - admin or owner → allow - non-owner, `is_locked == 0` → allow - non-owner, `is_locked == 1` → 403 - Read endpoints (`list_scripts`, `get_script`) intentionally do **not** check `is_locked` — workspace members can see the script list. -### Outbox events +### Owner-scoping + visibility (cross-owner browsing) -- The platform's only async-messaging fabric is the MySQL - `OutboxEvents` table. Producers (Backend) write rows in the same - transaction as the business state. Consumers (Schedule Executor) - poll every 250 ms and update `event_status` to `published` or - `failed` (with retry). -- Use `common.eventing.add_outbox_event` to write. -- `event_type` values currently in use: - - `schedule.run.requested` — produced by `schedule_runs.py` / cron post-back - - `job.node.execute` — produced by orchestrator when a node is ready - - `job.node.finished` — produced by worker after node execution -- `consumer_inbox` provides exactly-once delivery per - `(consumer_name, event_id)` (with `process_status: processing → - succeeded` lifecycle). +`GET /api/v1/scripts`, `GET /api/v1/data-resources`, and +`GET /api/v1/workspace-directories` all accept an optional +`owner_user_id` query param and follow the same model: + +- `owner_user_id` 缺省 = 当前请求者本人(scope to `workspace/{me}/...`). +- 传值时 scope 到 `workspace/{owner_user_id}/...`,用于前端"点开其他成员 + 分组"的懒加载(见 §3.4 of `API.md`). +- `visibility` 过滤(非 admin):`owner_user_id == me OR visibility IN + (workspace, public)` — 自己可见自己全部(含 private),他人只见其 + workspace/public,排除他人 private. +- 系统管理员跳过 visibility 过滤. +- 目录行默认 `visibility='public'`(由 `create_workspace_directory` 写入), + 不施加 visibility 过滤,使跨 owner 目录树可见. + +实现 helper 在 `backend/src/backend/services/scripts.py` +(`_build_list_scripts_owner_descendant_prefix`) 和 +`backend/src/backend/api/resources.py` 中按相同模式分别构造 owner-scoped +LIKE 前缀。 + +### Eventing — Outbox + Inbox + business tables + +The platform has **no** message broker. The only async fabric is three MySQL +tables, each on a different side of the same boundary: + +| Table | Side | Job | +|---|---|---| +| business tables (`schedule_runs`, `schedule_node_runs`, etc.) | producer's local commit | The state change that *triggered* the event | +| `outbox_events` | producer side | Durable handoff: "this business state change produced this event" | +| `consumer_inbox` | consumer side | Idempotency: "this consumer has already processed this event" | + +Together they implement the **Transactional Outbox** + **Consumer Inbox** +patterns: atomic publish on the producer side, idempotent side effects on +the consumer side. There is no Redis Stream, no Kafka, no in-process queue. +If MySQL is down, both sides are down — and that's intentional. + +#### Why two tables and not one + +A naïve "publish event by inserting a row, then have the consumer mark it +done" design breaks under two failure modes: + +1. **Producer crash after business commit, before publish.** The state + change is real but the event was never sent. Consumers never see it. +2. **Consumer crashes mid-processing.** The event was received but the + side effect may have happened or not. Re-delivery causes double + execution. + +`outbox_events` fixes (1): writing to the business table and the outbox +happens in **one transaction**, so either both are durable or neither is. +A separate dispatcher (`schedule/application/orchestrator.py`) polls +`outbox_events` and pushes to consumers. The dispatcher crashing is +harmless — the next poll picks up where it left off. + +`consumer_inbox` fixes (2): before applying a side effect, the consumer +inserts a row keyed by `(consumer_name, event_id)`. Re-delivery of the same +`event_id` hits the existing row and short-circuits. + +#### Producer side — `outbox_events` + +Schema lives in `common/db/models/events.py` (model `OutboxEvents`) and the +baseline migration `migrations/versions/e1f2a3b4c5d6_rebuild_baseline.py`. + +Key fields: + +- `event_id` (ULID, PK) — globally unique; `consumer_inbox` references this + as the dedup key. +- `aggregate_type` + `aggregate_id` — the producer's domain object + (e.g. `("schedule_run", "")`). +- `event_type` — namespaced via `schedule_event_type(...)` so multiple + deployments sharing one MySQL don't cross-consume (see below). +- `schema_version` (default 1) — consumer can branch on this when the + payload shape changes. +- `payload_json` — the event body; opaque to the outbox. +- `event_status` (`pending` → `published` | `failed`) + `available_at` + + `retry_count` + `last_error` — dispatcher state machine. Indexed on + `(event_status, available_at, created_at)` because that's the poll + hot path. +- `idempotency_key` + index — upstream dedup at the producer (e.g. two + requests from the user that should produce one event, not two). +- `trace_id` — links to the request that produced the event for log + correlation. + +Write path — the **only** entry point is `common.eventing.add_outbox_event`: + +```python +await add_outbox_event( + session, + event_type=schedule_event_type("schedule.run.requested"), + producer="backend.api.schedules.runs", + trace_id=request.state.trace_id, + aggregate_type="schedule_run", + aggregate_id=run_id, + idempotency_key=f"schedule.run.requested:{run_id}", + payload={"run_id": run_id, "schedule_id": schedule_id}, +) +``` + +`session` is the **same** SQLAlchemy session as the business-table write; +the outbox row goes in via `session.add(...)` and commits together with +the business row. Never call `session.commit()` between the business +write and the outbox write — that defeats the whole pattern. + +**Event type namespacing.** Every event type MUST go through +`common.eventing.schedule_event_type(raw)` before being passed to +`add_outbox_event`. The helper prefixes `settings.schedule_event_namespace` +(default `model-platform-develop`) so that two deployments sharing one +MySQL — common during development — don't accidentally consume each +other's events. The constants `SCHEDULE_RUN_REQUESTED_EVENT`, +`NODE_EXECUTE_EVENT`, `NODE_FINISHED_EVENT` in +`schedule/application/orchestrator.py` and `schedule/execution/worker.py` +are already namespaced; never hard-code the raw string. + +Current event types in use: + +| Event | Produced by | Consumed by | +|---|---|---| +| `schedule.run.requested` | `backend/api/schedules/runs.py`, cron post-back | `schedule/application/orchestrator.py` (`schedule-orchestrator`) | +| `job.node.execute` | `schedule/application/orchestrator.py` when a node is ready | `schedule/execution/worker.py` (`schedule-results` writes back via the orchestrator) | +| `job.node.finished` | `schedule/execution/worker.py` after a node run finishes | `schedule/application/orchestrator.py` (`schedule-results`) | + +The dispatcher polls every `0.25s` when there's pending work, dropping to +`1s` when idle — see the loop in +`schedule/application/orchestrator.py` (the `asyncio.sleep(0.25)` / +`asyncio.sleep(1)` branches). + +#### Consumer side — `consumer_inbox` + +Model `ConsumerInbox` in `common/db/models/events.py`. Composite PK +`(consumer_name, event_id)` so multiple consumers can independently +process the same `event_id`. + +Lifecycle (lives in `schedule/application/orchestrator.py:_start_inbox` / +`_finish_inbox`): + +``` + ┌──────────────┐ + │ processing │ ← INSERT or re-claim on re-delivery + └──────┬───────┘ + │ + success │ failure + ▼ + ┌─────────────┐ + │ succeeded │ (terminal — no re-process) + └─────────────┘ + + on failure, error_message is set and the row is left in + `failed` for inspection; the dispatcher does NOT auto-retry + consumer failures (only producer-side publish failures). +``` + +The consumer **must** call `_start_inbox` (or equivalent) at the top of +every event handler. The function returns `(inbox_row, should_process)`; +if `should_process` is `False`, the event was already handled and the +handler returns immediately. On success the handler calls `_finish_inbox` +to flip `process_status` to `succeeded`. + +Currently registered `consumer_name` values (see `orchestrator.py:594`, +`:924`): + +- `schedule-orchestrator` — consumes `schedule.run.requested` +- `schedule-results` — consumes `job.node.finished` + +A new consumer = a new `consumer_name` string. Two consumers sharing a +name will collide on the PK — pick a stable, descriptive name and treat +it as a contract. + +#### Failure modes the design covers + +| Scenario | What happens | +|---|---| +| Backend crashes after business commit, before dispatcher polls | Outbox row exists; next poll picks it up. | +| Dispatcher crashes after poll, before HTTP push to executor | `event_status` still `pending`; next poll retries. | +| Executor crashes mid-handler | Inbox row stays `processing`; on redelivery the handler re-enters `_start_inbox`, sees `succeeded`? — no, sees `processing` and re-runs. **This is currently a known soft spot** — the executor's `_finish_inbox` must run, and a crash before that means a re-run. Don't perform side effects before `_finish_inbox` succeeds. | +| Same `event_id` delivered twice (e.g. HTTP retry after success) | Inbox short-circuits; the second delivery is a no-op. | +| Multiple development deployments share one MySQL | `schedule_event_namespace` prefixes keep them isolated; each deployment only sees its own events. | + +#### Adding a new event + +1. Pick an `aggregate_type` / `aggregate_id` pair that identifies the + producing domain object. +2. Pick a `consumer_name` for each consumer. Stable, descriptive, + never reused for a different purpose. +3. Define the event type constant: + ```python + # in the producing module + MY_NEW_EVENT = schedule_event_type("schedule.my_new_event") + ``` +4. Write it via `add_outbox_event` in the same transaction as the + business mutation. +5. In the consumer, start with `_start_inbox` (or follow the pattern + in `orchestrator.py`) before doing any side effects, and call + `_finish_inbox` on success. +6. Update the table above. + +Do **not** invent a new messaging fabric (Redis Stream, Kafka, in-process +queue). The whole point of this design is that MySQL is the single +authority — adding a second one doubles the failure surface. ### Async / sync signatures @@ -238,6 +627,10 @@ cd frontend && pnpm install && cd .. ### Per-service dev +Always go through `uv run` so the workspace `.venv` is used — bare +`uvicorn` / `python` resolves to system Python and `from backend.X` +imports fail with ModuleNotFoundError. + ```bash # Backend (terminal 1) export DATABASE_URL="mysql+asyncmy://model_platform:model_platform@127.0.0.1:3306/model_platform?charset=utf8mb4" @@ -248,13 +641,13 @@ export S3_ENDPOINT=http://127.0.0.1:9000 # Or for local mode: # export STORAGE_BACKEND=local # export LOCAL_STORAGE_BASE_DIR=/data -uv run --frozen --package backend uvicorn backend.main:app --host 0.0.0.0 --port 8000 --reload +uv run --package backend uvicorn backend.main:app --host 0.0.0.0 --port 8000 --reload # Schedule Executor (terminal 2) -uv run --frozen --package schedule uvicorn schedule.main:app --host 0.0.0.0 --port 8001 --reload +uv run --package schedule uvicorn schedule.main:app --host 0.0.0.0 --port 8001 --reload # Runtime (terminal 3 — needs SYS_ADMIN, FUSE, devmode) -uv run --frozen --package runtime python -m runtime.main +uv run --package runtime python -m runtime.main ``` ### Frontend dev @@ -297,14 +690,17 @@ print('settings ok:', settings.s3_endpoint) ### Add a new DAG endpoint -1. Add the route handler in `backend/schedules.py` (DAG template) or - `backend/schedule_runs.py` (run lifecycle). -2. Validate request via `backend/schedule_schemas.py`. +1. Add the route handler in `backend/src/backend/api/schedules/schedules.py` + (DAG template) or `backend/src/backend/api/schedules/runs.py` (run + lifecycle). +2. Validate request via Pydantic schemas in + `backend/src/backend/schemas/schedules.py`. 3. For mutations on nodes/edges/versions: route through `create_script_record` / `get_script_row` and apply `require_script_modify_access` if it touches a script. 4. If it produces an outbox event, use - `add_outbox_event(session, event_type="...", producer="...", ...)`. + `add_outbox_event(session, event_type=schedule_event_type("..."), producer="...", ...)` + in the same transaction as the business write. ### Add a new env var @@ -329,22 +725,17 @@ See "Adding a new env var" above. ### Wire a new storage bucket -The current 4 buckets are wired in `backend/storage_api.py:resolve_bucket`: - -```python -BUCKET_FOR_USAGE: dict[str, str] = { - "working_copy": settings.s3_workspace_bucket, - "public_script": settings.s3_workspace_bucket, - "data_resource": settings.s3_workspace_bucket, - "snapshot": settings.s3_workspace_bucket, - "version_artifact": settings.s3_version_bucket, - "run_log": settings.s3_run_log_bucket, - "run_result": settings.s3_run_log_bucket, -} -``` +The current 4 buckets are wired in `common/src/common/storage/factory.py`. +`USAGE_TYPE_TO_PURPOSE` maps each `usage_type` (`working_copy`, +`public_script`, `data_resource`, `snapshot`, `version_artifact`, +`run_log`, `run_result`) to one of the 4 purpose buckets (`workspace`, +`version`, `run_log`, `trash`); `actual_bucket_name(purpose)` then +returns the s3-style identifier from the corresponding +`settings.s3__bucket`. `BUCKET_FOR_USAGE` is a dict +comprehension built from these two. The constant `PURPOSE_BUCKETS = ("workspace", "version", "run_log", "trash")` -in `common.storage.factory` enumerates the four backends built in the +in `common/storage/factory.py` enumerates the four backends built in the backend lifespan. To add a fifth bucket: 1. Add the env var to `Settings` (s3 mode only): @@ -359,36 +750,62 @@ backend lifespan. To add a fifth bucket: 4. Extend the `Literal` in `common/storage/schemas.py` (in `CreateUploadRequest.usage_type`, `ServerObjectRequest.usage_type`) to include the new value. -5. Add an entry in `BUCKET_FOR_USAGE` mapping the new `usage_type` to - the new bucket env var. +5. Add an entry in `USAGE_TYPE_TO_PURPOSE` mapping the new `usage_type` + to the new purpose; `BUCKET_FOR_USAGE` is regenerated automatically. 6. Pre-create the bucket (s3 mode) or subdirectory (local mode) in the deployment. The backend no longer auto-creates buckets. -A workspace's `artifact_bucket` column (when non-null) overrides the -default for that workspace, regardless of `usage_type`. +A workspace's `Workspaces.artifact_bucket` column (when non-null) +overrides the default for that workspace, regardless of `usage_type`. ### Add a new schedule node type -`schedule/execution.py` dispatches on `script_type` in -`execute_artifact`. Add a new branch + a new `_` function. -`worker.py` does not need to change — the dispatch happens inside -`execute_artifact`. +`schedule/src/schedule/execution/runners/notebook.py:execute_artifact` +dispatches on `script_type`. Add a new branch + a new `_` function +in that file. The NodeExecutor in `worker.py` does not need to change — +the dispatch happens inside `execute_artifact`. ## Tests -There is **no formal test suite yet** (see HANDOVER §Pending Tasks -P1). A reasonable first test surface: +There is an in-tree test suite, mostly covering scripts / resources / +DAG validation / upload state transitions. It is **not** the formal +release-gate suite the project still owes (see `HANDOVER.md` §8 for +known gaps: trash reaper, cross-backend migration, ENC round-trip). -- `require_script_modify_access` (admin / owner / non-owner-unlock / - non-owner-lock): pure-function unit test, no DB. -- `validate_dag` (cycle detection + orphan detection) in - `backend/schedules.py`. -- `execute_artifact` end-to-end with mocked `content_hash` and a - real `tempfile.TemporaryDirectory`. +Current coverage: -Test convention: pytest with `pytest-asyncio` for `async def` -handlers. Use SQLite in-memory (or a MySQL test container) for DB -integration. Use moto for S3. +| Area | Tests | Where | +|---|---|---| +| Scripts (CRUD, soft delete, parent path, same-name siblings, visibility) | 10 functions across `test_scripts.py`, `test_count_scripts.py`, `test_list_scripts_parent_path.py`, `test_storage_upload_status.py` | `backend/tests/` | +| Resources (visibility, ownership, idempotency) | ~5 in `test_resources.py` | `backend/tests/` | +| DAG validation (cycle / orphan detection) | `test_validate_dag.py` | `backend/tests/` | +| Audit log middleware | `test_audit_logging.py` | `backend/tests/` | +| Jupyter auth cache | `test_jupyter_auth_cache.py` | `backend/tests/` | +| Runtime client (directory listing, error mapping) | `test_runtime_client_directories.py` | `backend/tests/` | +| Schedule layer (worker, janitor, layering invariants) | 11 functions in `test_janitor.py`, `test_layering.py`, `test_worker.py` | `schedule/tests/` | + +Run: + +```bash +# Backend +uv run --package backend pytest backend/tests -q + +# Schedule +uv run --package schedule pytest schedule/tests -q +``` + +Conventions: + +- pytest + `pytest-asyncio` for `async def` handlers. +- MySQL is required for the ORM tests (not SQLite — CHAR(26) ULIDs and + `mysql.TINYINT(1)` quirks do not translate). Local docker-compose + MySQL is the typical target. +- For storage, `common.storage.factory` selects between s3 and local + via `settings.storage_backend`; tests that exercise both modes + monkeypatch that setting (see `common/tests/storage/test_factory.py`). +- For HTTP boundaries (httpx to backend / runtime), tests use `respx` + with `assert_all_called=False` so unused stubs don't fail the test + — see the engineering notes in `CLAUDE.md`. ## Troubleshooting @@ -425,7 +842,10 @@ check the JWT (use `JWT_SECRET` from `.env`). `worker` must be constructed before `orchestrator` in `SchedulerService.__init__`, because orchestrator's dispatch table captures `self.worker.handle_node_execute` at construction time. -See `service.py` — the order is load-bearing. +See `schedule/src/schedule/application/service.py` — the order is +load-bearing. (This was the symptom during the flat → layered schedule +refactor; if you see it today, the most likely cause is a partial +rebase that left an old import path.) ## Style @@ -441,8 +861,12 @@ See `service.py` — the order is load-bearing. ## See also -- `ARCHITECTURE.md` — design diagrams -- `HANDOVER.md` — current refactor state and pending work -- `CLAUDE.md` — agent-facing conventions for the repo -- `models / __init__.py` — exhaustive list of all 26 tables -- `common/config.py` — all env vars in one place +- `ARCHITECTURE.md` — kept as a thin redirect; the authoritative + architecture diagrams and capability map now live in + [§Architecture](#architecture) above. +- `HANDOVER.md` — current refactor state, recent commits, pending work. +- `CLAUDE.md` — agent-facing conventions for the repo. +- `common/src/common/db/models/__init__.py` — exhaustive list of all 18 tables. +- `common/src/common/config.py` — all env vars in one place. +- Per-package READMEs: `backend/README.md`, `common/README.md`, + `runtime/README.md`, `frontend/README.md`. diff --git a/README.md b/README.md index c09cf66..4d81378 100644 --- a/README.md +++ b/README.md @@ -9,97 +9,16 @@ ## 它做什么 -| 能力 | 位置 | -|---|---| -| workspace 内 notebook 编辑，行级锁 | `backend/jupyter.py` + `scripts.is_locked` | -| Jupyter 鉴权路由（浏览器永远拿不到 runtime token） | `nginx/default.conf` + `auth_request` + `backend/jupyter.py` | -| notebook / script / version / run_log 的对象存储（s3 / local 二选一） | `common/storage/` + `backend/scripts.py` | -| DAG 调度：节点、边、cron、手动触发、重试、快照 | `backend/schedules.py` + `backend/schedule_runs.py` + `schedule/`（5 个模块） | -| DAG 执行走 MySQL Outbox（无 Redis，无进程内队列） | `schedule/orchestrator.py` + `schedule/worker.py` | -| 每个 workspace 一个 Jupyter 子进程池，配 asyncio 锁 | `runtime/process.py` | -| runtime 内 rclone FUSE 把 workspace 桶挂上来（s3 模式） | `runtime/mount.py` | -| 仅 MySQL 持久化（26 张表，软删除，无外键） | `common/db/models/` | +能力清单、组件图、容器表、存储布局、配置参考——**全部在 [`DEVELOP.md`](./DEVELOP.md)**： -## 架构一览 +- 系统整体架构（[§Architecture](./DEVELOP.md#architecture)） +- 18 张 MySQL 表的 Eventing 协作模式（[§Eventing](./DEVELOP.md#eventing--outbox--inbox--business-tables)） +- 代码目录布局（[§Code layout](./DEVELOP.md#code-layout)） +- 全部 26 个环境变量（[§Configuration system](./DEVELOP.md#configuration-system)） +- 写代码的约定（[§Conventions](./DEVELOP.md#conventions)） +- 加新表 / 新桶 / 新节点类型的步骤（[§Common tasks](./DEVELOP.md#common-tasks)） -``` - ┌────────────────────┐ - │ Browser (SPA) │ - └─────────┬──────────┘ - │ HTTPS / WS - ┌─────────▼──────────┐ - │ Nginx (only :80) │ ← templates/default.conf - │ /api/ /jupyter/ /storage/ - └────┬───────┬──────┘ - │ │ - ┌──────────────┘ └─────────────┐ - ▼ ▼ - ┌──────────────────┐ ┌──────────────────────┐ - │ FastAPI Backend │ │ Runtime (Jupyter) │ - │ + /internal/v1 │ control │ - rclone FUSE mount │ (P0-1) - │ /objects │ token-Auth │ │ - │ (storage) ├──────────────►│ - subprocess pool │ - │ - DAG CRUD │ │ (per workspace) │ - │ - script CRUD │ └──────────┬───────────┘ - │ - auth_request │ │ FUSE / shared vol - │ - /api/v1/... │ ▼ - └────┬──────┬──────┘ ┌──────────────────────┐ - │ │ │ Object storage │ - │ └──────── HTTP ───────►│ (s3: S3 service / │ - ▼ │ local: shared vol) │ - ┌────────────┐ │ 4 buckets per usage │ - │ MySQL │◄───────── poll ─────│ │ - │ - 26 tbls │ └──────────────────────┘ - │ - outbox │ - │ - jobstore │ - └────┬───────┘ - ▲ - │ outbox poll - ┌────┴──────────────────────────┐ - │ Schedule Executor │ - │ - CronScheduler (APScheduler) │ - │ - DispatchOrchestrator │ - │ - NodeExecutor (worker) │ - │ - SchedulerService (facade) │ - └───────────────────────────────┘ -``` - -对象存储通过 `STORAGE_BACKEND`（s3 | local）二选一。s3 模式下 4 个 purpose 命名桶 -（`workspace` / `version` / `run-log` / `trash`）是独立的 S3 bucket；local 模式下 -是 `LOCAL_STORAGE_BASE_DIR` 的子目录，通过 Docker volume `local-storage` 共享。 -详见 `DEVELOP.md` §存储。 - -详细设计见 `ARCHITECTURE.md`。实现的偏离和近期重构记录在 `HANDOVER.md`。 - -## 目录结构 - -```text -frontend/ React Router SPA -backend/ FastAPI：公开 API + 内部存储 API -runtime/ Jupyter 子进程管理 + rclone FUSE -schedule/ DAG 调度器（5 模块：context/scheduler/ - orchestrator/worker/service） -common/ 配置、SQLAlchemy 模型、存储 SDK、 - outbox 事件、jobstore -migrations/ Alembic 基线 + 各特性 migration -nginx/ （仅概念 — 见下方「容器」一节） -scripts/ nginx-entrypoint.sh（模板渲染） -docker-compose.yml 4 服务 — web / backend / runtime / schedule -default.conf Nginx 模板（挂载，启动时渲染） -.env.example common.config.Settings 消费的所有环境变量 -``` - -## 容器 - -| 服务 | 镜像 | 暴露 | 用途 | -|---|---|---|---| -| `web` | `nginx:alpine` | 宿主机 `:8888` → `:80` | SPA、`/api/` 反向代理、`/jupyter/{ws}/` auth_request 代理、`/storage/` S3 直通（仅 s3 模式） | -| `backend` | `Dockerfile` | 仅内网 | DAG CRUD、script CRUD、schedule 触发、`/api/v1/auth/jupyter`、`/internal/v1/objects` 服务间 RPC(共享 `INTERNAL_SERVICE_TOKEN` 鉴权,P0-1)| -| `runtime` | `Dockerfile` | 仅内网 | 每个 workspace 一个 Jupyter 子进程池、rclone FUSE 挂载 `workspace` 桶（s3 模式） | -| `schedule` | `Dockerfile` | 仅内网 | cron tick + DAG 执行（轮询 MySQL Outbox） | - -架构**故意只暴露一个宿主机端口**(网关);其他服务都在 Docker 内网。 -这一点在 `docker-compose.yml` 里强制执行 — backend / runtime / schedule 都没有 `ports:`。在 P0-1 之前,backend 与 runtime 曾短暂地把 `8891` / `8892` 映射到宿主机;此映射已被删除,改用 `INTERNAL_SERVICE_TOKEN` 头对 `/internal/v1/*` 做服务间鉴权,见 `API.md §9`。 +`ARCHITECTURE.md` 现已并入 `DEVELOP.md`；`HANDOVER.md` 记录近期重构与待办事项。 ## 快速启动 @@ -144,65 +63,13 @@ docker compose down docker compose down -v ``` -## 配置 - -所有环境变量在 `common/src/common/config.py` 里用 pydantic-settings 的 `Settings` -类一次性声明，外面套一层 `@lru_cache` 单例。新增环境变量： - -1. 在 `common/src/common/config.py` 的 `Settings` 里加字段（带合理 default，使 dev 启动不需要设） -2. 在 `.env.example` 加一行带注释 -3. 调用点用 `settings.`，永远不要用 `os.environ["..."]` - -完整环境变量列表和含义见 `DEVELOP.md`。 - -## 存储布局 - -4 个 purpose 命名桶。从 `StorageObjects.usage_type` 到桶的映射由 -**单一入口**（`backend/storage_api.py:resolve_bucket`）决定： - -| `usage_type` | 桶（环境变量） | 默认名 | -|---|---|---| -| `working_copy`、`public_script`、`data_resource`、`snapshot` | `S3_WORKSPACE_BUCKET` | `workspace` | -| `version_artifact` | `S3_VERSION_BUCKET` | `version` | -| `run_log`、`run_result` | `S3_RUN_LOG_BUCKET` | `run-log` | -| （软删除目标） | `S3_TRASH_BUCKET` | `trash` | - -`STORAGE_BACKEND=s3` 模式下是 4 个独立 S3 桶。`STORAGE_BACKEND=local` 模式下 -是 `LOCAL_STORAGE_BASE_DIR`（默认 `/data`）下的 4 个子目录： - -``` -/data/ -├── workspace/ # S3_WORKSPACE_BUCKET -├── version/ # S3_VERSION_BUCKET -├── run_log/ # S3_RUN_LOG_BUCKET -└── trash/ # S3_TRASH_BUCKET -``` - -某个 workspace 的 `artifact_bucket` 列（非 NULL 时）覆盖该 workspace 的默认桶， -无视 `usage_type` — 适合把付费客户隔离到专属桶。 - -对象 key 是两层扁平路径 — `workspace_id` 加服务端签发的 `ulid`： - -``` -//{.} -``` - -文件名、扩展名、MIME、逻辑路径都放在 `StorageObjects` 和 `Scripts` 行里，不进 -object key — 重新组织存储不需要重写数据库。 - -Backend 代码从不写容器本地文件系统（`STORAGE_BACKEND=local` 模式除外，那里共享 -`local-storage` volume 就是规范存储）。Schedule Executor 在 `tempfile.TemporaryDirectory()` -里暂存节点工件（自动清理）。只有 `runtime` 容器保留宿主 volume — s3 模式下 rclone FUSE -挂载需要；local 模式下是 no-op 透传。 - ## 文档 -- `README.md`（本文）— 快速导读 -- `ARCHITECTURE.md` — 设计图 + 简化历史 -- `HANDOVER.md` — 实现偏离、近期重构、待办事项 -- `DEVELOP.md` — 开发指南（环境变量、代码规约、常用操作） -- `CLAUDE.md` — agent 面向的本仓库规约 +- [`DEVELOP.md`](./DEVELOP.md) — 权威文档：架构、代码布局、配置、约定、常用任务、测试、故障排查 +- [`HANDOVER.md`](./HANDOVER.md) — 近期 commit / 实现偏离 / 待办事项 +- [`API.md`](./API.md) — REST API 契约 +- [`CLAUDE.md`](./CLAUDE.md) — agent 面向的本仓库规约 ## 许可 -内部。 \ No newline at end of file +内部。 diff --git a/STYLE_GUIDE.md b/STYLE_GUIDE.md new file mode 100644 index 0000000..f92b989 --- /dev/null +++ b/STYLE_GUIDE.md @@ -0,0 +1,528 @@ +# 前端样式设计规范 (STYLE_GUIDE) + +> **版本**：v2.0 — 规范化版本 +> +> **文档性质变更说明**：v1.0 由现有代码反向提炼而来，仅描述"现状"，并列出大量"新旧并存 / 待统一"的问题清单。 +> **v2.0 起，本文档是唯一权威规范**：所有历史上并存的多套数值（多个品牌蓝、多个灰色、多套圆角体系等）已在本版本中**收敛为唯一官方取值**。 +> +> - 🟢 **新代码 / 其他系统 / 新项目**：必须（MUST）按本文档执行，禁止再引入本文档之外的颜色、字号、圆角、间距任意值。 +> - 🟡 **存量 legacy 代码**：允许暂时保留（不要求立即批量重构），但**触碰到的每一处**（改动、重构、修 bug）都必须顺手改为符合本规范的取值，禁止在旧代码基础上"抄一份"旧的任意值到新地方。 +> - 本指南**不强制使用特定 UI 组件框架**（React/Vue/Tailwind/Ant Design 等均可），只强制**最终视觉呈现**（色值、字号、圆角、阴影、间距、过渡时长/缓动）与本文档一致。不同技术栈的团队，请把本文档中的 CSS 变量按第 12 节的方法映射到自己的 token/主题系统。 + +--- + +## 0. 规范优先级说明 + +文档中出现的关键词含义（参考 RFC 2119 风格）： + +| 关键词 | 含义 | +|---|---| +| **必须 / 禁止** | 强制性规则，新代码不得违反，Code Review 应作为拒绝合并的理由 | +| **应当** | 强烈建议，没有特殊原因必须遵守；有正当理由例外时需在 PR 中说明 | +| **可以** | 允许的做法，非强制 | + +--- + +## 1. 设计概览 + +项目整体风格是 **浅色 + 深蓝侧栏** 的 SaaS 控制台风格，所有新页面 / 新模块必须延续此基调： + +- **基调色**：极浅蓝灰背景 `#f3f6f9`，纯白卡片 `#ffffff`，深蓝侧栏 `#09233f`。 +- **品牌主色**：饱和中蓝 `#1978d4`（唯一官方品牌蓝，见 §2.7 废弃旧值列表）。 +- **状态色**：成功绿 `#18b979` / 危险红 `#d75b5b`（唯一官方危险色）/ 警告橙 `#e6a33c`。 +- **形状语言**：统一采用 `rounded-2xl / 3xl / 4xl` 圆角阶梯（4 / 6 / 8px 递增，见 §4.3），**禁止**新代码再使用任意像素圆角（如 `rounded-[7px]`）。 +- **字体**：Inter Variable（`/fonts/Inter-Variable.ttf`，weight 100–900）+ PingFang SC / Microsoft YaHei / system-ui 中文回落。 +- **阴影**：`shadow-md`（卡片）/ `shadow-xl`（对话框/抽屉），禁止新增自定义阴影任意值。 +- **动效**：统一 100–200ms 过渡（抽屉类组件可到 450ms），弹层统一用 `data-open`/`data-closed` 进出模式。 + +--- + +## 2. 色彩系统（唯一权威取值） + +> **规则**：任何新代码中出现的颜色，必须来自本节表格中的具名 token 或对应 CSS 变量 / Tailwind 语义类。**禁止**在新代码里直接写裸 hex 值，除非本节明确未覆盖的一次性边缘场景（且需在 PR 中说明原因）。 + +### 2.1 Token 定义位置 + +- 品牌 token：`frontend/app/app.css` `@theme { ... }` 块。 +- 侧栏 token：`frontend/app/app.css` `@theme inline` + `:root { --sidebar-... }`。 +- shadcn 语义 token（`bg-primary` / `text-foreground` 等）实际值由 `@theme` 覆盖，UI 组件一律使用语义 token，不写 hex。 + +### 2.2 品牌色（Brand / Accent） + +| 名称 | 十六进制 | CSS 变量 | 用途 | +|---|---|---|---| +| brand | `#1978d4` | `--color-brand` | 主色：CTA、链接、选中态、强调（对应 `bg-primary`/`text-primary`） | +| brand-strong | `#0e5fb9` | `--color-brand-strong` | 主色按下/激活 | +| brand-soft | `#edf6ff` | `--color-brand-soft` | 主色浅底：标签背景、提示行 | +| **brand-deep**（新增） | `#0e5bad` | `--color-brand-deep` | 渐变按钮的深色结束端（替代历史上散落的 `#1268c9`/`#1175d8`/`#0e5bad` 等值） | + +**渐变按钮官方写法**（如仍需渐变视觉的按钮）：`linear-gradient(135deg, var(--color-brand), var(--color-brand-deep))`。**禁止**再手写新的渐变端点色值。 + +### 2.3 状态色 + +| 名称 | 十六进制 | CSS 变量 | 用途 | +|---|---|---|---| +| success / success-strong / success-soft | `#18b979` / `#0f7d55` / `#e8f7f1` | `--color-success*` | 成功提示、确认按钮 | +| danger / danger-strong / danger-soft | `#d75b5b` / `#a94f4f` / `#fff6f6` | `--color-danger*` | 错误、删除、危险操作（对应 `bg-destructive`）| +| warning / warning-soft | `#e6a33c` / `#fff0ee` | `--color-warning*` | 警示提示 | + +### 2.4 中性文本 / 边框 / 背景 + +| 名称 | 十六进制 | CSS 变量 | 用途 | +|---|---|---|---| +| ink | `#27394d` | `--color-ink` | 主要正文 | +| ink-muted | `#5d7186` | `--color-ink-muted` | 次要正文 | +| ink-subtle | `#8b99a8` | `--color-ink-subtle` | 辅助说明、占位 | +| **ink-caption**（新增） | `#748598` | `--color-ink-caption` | 表格表头/卡片小标签等"更淡一档"的说明文字（替代历史上 `#748598`/`#758497`/`#8796a6`/`#8a9aaa` 等近似灰色） | +| line | `#dce4ec` | `--color-line` | 标准边框（对应 `border-border`），替代历史上 `#dce5ed`/`#dfe7ef`/`#dfe6ed`/`#e7ecf1`/`#e8edf2`/`#e0e7ee` 等近似边框色 | +| line-soft | `#edf1f5` | `--color-line-soft` | 表格行/分隔线 | +| bg | `#f3f6f9` | `--color-bg` | 页面背景 | +| bg-panel | `#ffffff` | `--color-bg-panel` | 卡片/面板背景 | +| bg-canvas | `#f9fbfd` | `--color-bg-canvas` | 画布背景 | +| bg-log | `#17212b` | `--color-bg-log` | 日志/代码块深色背景 | + +### 2.5 侧栏 token（独立主题） + +| 名称 | 值 | 用途 | +|---|---|---| +| `--sidebar-background` | `#09233f` | 深蓝侧栏底 | +| `--sidebar-foreground` | `#c9d7e7` | 侧栏默认文字 | +| `--sidebar-primary` | `#1479e8` | 侧栏内强调主色 | +| `--sidebar-primary-foreground` | `#ffffff` | 主色文字 | +| `--sidebar-accent` | `rgba(255,255,255,0.06)` | 选中态背景 | +| `--sidebar-accent-foreground` | `#ffffff` | 选中态文字 | +| `--sidebar-border` | `rgba(255,255,255,0.08)` | 侧栏内分隔线 | +| `--sidebar-ring` | `#5ca9ff` | focus ring | + +**规则**：任何侧栏相关的自定义样式（如背景渐变）必须引用 `var(--sidebar-background)` 等变量，**禁止**再硬编码 `#09233f`。 + +### 2.6 shadcn 语义 token 映射 + +| Tailwind 类 | 对应变量 | 用途 | +|---|---|---| +| `bg-primary` / `text-primary` / `border-primary` | `--color-brand` | 主按钮、链接 | +| `bg-secondary` | 浅灰 | 次级按钮 / 输入框默认态 | +| `bg-destructive` / `text-destructive` | `--color-danger` | 危险按钮 | +| `bg-muted` / `text-muted-foreground` | 浅灰底 + `--color-ink-muted` | 次要容器、辅助文字 | +| `bg-popover` / `text-popover-foreground` | 白底 | 浮层、菜单、对话框 | +| `bg-background` / `text-foreground` | 页面背景 + `--color-ink` | body、文本默认 | +| `bg-card` / `text-card-foreground` | 卡片 | Card 容器 | +| `bg-input` / `border-input` | 浅灰 | 输入框背景 | +| `border-border` | `--color-line` | 通用边框 | +| `border-ring` / `ring-ring` | 浅蓝 | focus 边框 / 焦点环 | +| `border-destructive` | `--color-danger` | 错误状态边框 | + +### 2.7 已废弃色值（禁止新代码使用） + +以下 hex 曾在 legacy 页面中散落使用，**新代码严禁再出现**，一律替换为 §2.2–2.4 对应 token： + +| 废弃值 | 替换为 | +|---|---| +| `#1881e7` / `#126ac3` / `#1268c9` / `#1175d8` / `#0e5bad` / `#1479e8`（页面内联渐变）/ `#2279c7` | `--color-brand` / `--color-brand-strong` / `--color-brand-deep`（渐变见 §2.2） | +| `#e74c3c` | `--color-danger`（`#d75b5b`） | +| `#748598` / `#758497` / `#8796a6` / `#8a9aaa` | `--color-ink-caption`（新代码首选）或 `--color-ink-muted` / `--color-ink-subtle`（视层级而定） | +| `#dce5ed` / `#dfe7ef` / `#dfe6ed` / `#e7ecf1` / `#e8edf2` / `#e0e7ee` | `--color-line` | +| `#81b2e7` | `--color-brand-soft` 场景下配合使用，或改用 `--color-brand` 的透明度变体（如 `text-primary/70`），不再单独定义新 hex | + +--- + +## 3. 字体系统 + +### 3.1 字体族 + +```css +--font-sans: Inter, "PingFang SC", "Microsoft YaHei", system-ui, sans-serif; +``` + +字体文件：`public/fonts/Inter-Variable.ttf`（含 italic 变体），`font-display: swap`。 + +### 3.2 字号阶梯（唯一官方取值） + +**规则**：新代码**必须**从下表取值，**禁止**再使用未定义的任意 `text-[Npx]`。 + +| Token / 类 | 像素 | 用途 | +|---|---|---| +| `text-4xs`（新增 token） | 8px | badge / 角标（原 `text-[8px]`） | +| `text-3xs`（新增 token） | 9px | 极小辅助文字（原 `text-[9px]`） | +| `text-2xs`（新增 token） | 10px | 小标签 / 数字 / 计数（原 `text-[10px]`，历史最高频） | +| `text-xs` | 12px | 表头 / 小按钮 / 行内说明（统一原 `text-[11px]`/`text-[12px]`/`text-[13px]`） | +| `text-sm` | 14px | 默认正文 / UI 组件默认字号 | +| `text-base` | 16px | 强调正文 | +| `text-xl` | 20px | 小标题 | +| `text-2xl`+ | 24px+ | 大标题 / 特殊展示字号，按需在设计评审中确定 | + +> 需要在 `app.css` 的 `@theme` 中补充定义： +> ```css +> --text-2xs: 0.625rem; /* 10px */ +> --text-3xs: 0.5625rem; /* 9px */ +> --text-4xs: 0.5rem; /* 8px */ +> ``` +> 在此之前，可临时使用 `text-[10px]` 等写法，但一旦 token 落地必须替换。 + +### 3.3 字重 + +| Token | 值 | 用途 | +|---|---|---| +| `font-medium` | 500 | UI 组件默认（按钮/CardTitle/DialogTitle） | +| **`font-medium-plus`（新增 token）** | 650 | Topbar / 主按钮等需要"比 medium 重、比 semibold 轻"的强调场景（原任意值 `font-[650]`，现固化为官方 token，禁止再写 `font-[650]`） | +| `font-semibold` | 600 | 强调文本 | +| `font-bold` | 700 | 标题 | +| `font-extrabold` | 800 | 大数字 / 品牌展示 | + +> 需在 `@theme` 中补充：`--font-weight-medium-plus: 650;`，并在 Tailwind 中暴露为 `font-medium-plus` 工具类。 + +### 3.4 行高 / 字间距 + +- 默认行高 1.5；对话框标题用 `leading-none`；长文本正文（如 `.notebook-md p`）用 `line-height: 1.6`。 +- 字间距仅用于大写字母标签：`tracking-[0.08em]` ~ `tracking-[0.12em]`，**禁止**在正文中使用额外字间距。 + +--- + +## 4. 间距、圆角与布局 + +### 4.1 基础网格 + +Tailwind 默认 4px 网格（`0.25rem`），不自定义 `--spacing` 基数。 + +### 4.2 间距 Token（唯一官方取值） + +**规则**：以下为新代码必须优先使用的语义间距 token，**禁止**再随手写 `px-[22px]` 这类任意值。 + +| Token（新增） | 值 | 用途（替代原任意值） | +|---|---|---| +| `--spacing-page-x` | 22px | 页面左右内边距（原 `px-[22px]` / `mx-[22px]`） | +| `--spacing-button-x` | 15px | 主按钮水平内边距（原 `px-[15px]`） | +| `--spacing-gap-icon` | 7px | 按钮内图标间距（原 `gap-[7px]`） | +| `--spacing-gap-sm` | 5px | 小行距（原 `gap-[5px]`） | +| `--spacing-gap-md` | 9px | 列表/区块间距（原 `gap-[9px]` / `gap-[10px]` 就近取整为该 token 或 `gap-2.5`） | + +其余场景优先使用 Tailwind 内建 token：`p-2.5` / `p-3` / `p-4` / `p-6` / `gap-1.5` / `gap-2` / `gap-6` / `px-3` / `py-1`。 + +### 4.3 圆角规范（唯一官方阶梯） + +**规则**：新代码**必须**使用以下阶梯，**禁止**再引入新的任意像素圆角。历史任意值按下表强制迁移： + +| 官方 Token | 像素 | 用途 | 替代的历史任意值 | +|---|---|---|---| +| `rounded`（Tailwind 默认） | 4px | 极小元素（icon 容器等） | `rounded-[4px]` | +| `rounded-md` | 6px | 小型控件、legacy 按钮/输入 | `rounded-[5px]` / `rounded-[6px]` / `rounded-[7px]` | +| `rounded-lg` | 8px | 中等卡片/列表项 | `rounded-[9px]` / `rounded-[10px]` | +| `rounded-xl` | 12px | 大号卡片、Tooltip | `rounded-[11px]` / `rounded-[13px]` | +| `rounded-2xl` | 16px | Textarea / Skeleton | — | +| `rounded-3xl` | 24px | Input / Combobox 弹出层 | — | +| `rounded-4xl` | 32px | Button / Card / Dialog / Drawer / Sheet（最高一级视觉容器） | — | +| `rounded-full` | — | 头像 / 徽章 | — | + +> **组件分层原则**：交互控件从小到大遵循 `md → lg → xl → 2xl → 3xl → 4xl` 依次递增，容器级别越大（对话框、卡片）圆角越大，行内小控件（checkbox、badge）圆角越小。新组件设计时先确定"层级"，再从表中取值，不要自创中间值。 + +### 4.4 阴影（唯一官方取值） + +| Token | 用途 | +|---|---| +| `shadow-md` | Card 默认阴影 | +| `shadow-lg` | Combobox / 下拉菜单 | +| `shadow-xl` | Dialog / Sheet / Drawer / Sidebar 菜单 | + +**规则**：**禁止**新代码使用自定义阴影任意值（如 `shadow-[0_6px_15px_rgb(...)]`）。如确需强调阴影（例如渐变主按钮的"投色阴影"），必须先在本文档中补充为具名 token 后才能使用。 + +### 4.5 边框 + +- 默认宽度 `1px`（Tailwind `border`）。 +- 输入类组件：默认 `border-transparent`，focus 时 `focus-visible:border-ring` + `focus-visible:ring-3 focus-visible:ring-ring/30`。 +- 危险态：`aria-invalid:border-destructive` + `aria-invalid:ring-3 aria-invalid:ring-destructive/20`。 +- 边框颜色统一使用 `border-border`（`--color-line`），**禁止**再写 `border-[#dfe6ee]` 等任意 hex（见 §2.7）。 + +### 4.6 布局 + +- 页面容器：`min-width: 1120px`，桌面优先布局（当前产品定位不适配移动端，如需移动端适配需局部覆盖，不改全局）。 +- 侧栏宽度：展开 `16rem` / 收起 `3rem` / 移动端 `18rem`。 +- 两栏布局参考：`grid-cols-[310px_minmax(0,1fr)] max-xl:grid-cols-[285px_minmax(0,1fr)]`。 +- Dashboard 网格参考：`min-[1201px]:grid-cols-[minmax(0,1.6fr)_minmax(310px,0.9fr)] max-[1200px]:grid-cols-1`。 + +### 4.7 断点 + +使用 Tailwind v4 默认断点：`sm 40rem` / `md 48rem` / `lg 64rem` / `xl 80rem` / `2xl 96rem`，不自定义。响应式优先用 `max-xl:` / `max-[1200px]:` / `sm:` 等既有写法保持一致性。 + +--- + +## 5. 组件视觉规范（强制规格，可用任意技术栈实现同等效果） + +> 以下为各组件的**强制视觉规格**。使用 React + shadcn 的团队直接复用 `app/components/ui/*`；使用其他框架（Vue/Ant Design/Element Plus 等）的团队，必须在自己的组件层重新实现，但颜色、圆角、尺寸、状态视觉必须与下表完全一致。 + +### 5.1 Button + +- 圆角 `rounded-4xl`；字号默认 `text-sm`；字重 `font-medium`；过渡 `transition-all`。 +- 尺寸：默认 `h-9 px-3` / `xs h-6 px-2.5` / `sm h-8 px-3` / `lg h-10 px-4` / `icon 36×36` / `icon-xs 24×24` / `icon-sm 32×32` / `icon-lg 40×40`。 +- 状态：default `bg-primary` + 白字；hover `bg-primary/80`；active `translate-y-px`；focus `border-ring` + `ring-3 ring-ring/30`；disabled `pointer-events-none opacity-50`；invalid `border-destructive ring-3 ring-destructive/20`。 +- 内部 svg 未指定尺寸时自动 `size-4`（16px）。 + +### 5.2 Input + +- 圆角 `rounded-3xl`；高度 `h-9`（36px）；字号 `text-sm`；背景 `bg-input/50`。 +- 状态：placeholder `text-muted-foreground`；focus `border-ring`（无 ring 阴影）；disabled `cursor-not-allowed opacity-50`；invalid `border-destructive`。 + +### 5.3 Textarea + +- 圆角 `rounded-2xl`；最小高 `min-h-16`（64px）；不可手动 resize；内边距 `px-3 py-3`；focus 额外带 `ring-3 ring-ring/30`。 + +### 5.4 Checkbox + +- 尺寸 `size-4`（16px）；圆角 `rounded-md`（6px，统一自原 `rounded-[5px]`）；背景 `bg-input/90`。 +- 状态：checked `border-primary bg-primary` + 白色 `CheckIcon`；focus `border-ring ring-3 ring-ring/30`；disabled `cursor-not-allowed opacity-50`；invalid `border-destructive ring-3 ring-destructive/20`。 + +### 5.5 Combobox / Select + +- Trigger 与 Input 一致，右侧 `ChevronDownIcon size-4 text-muted-foreground`。 +- Content：圆角 `rounded-3xl`，背景 `bg-popover`，阴影 `shadow-lg`，`ring-1 ring-foreground/5`。 +- Item 右侧固定 `size-4` 的 `CheckIcon` 占位槽；Separator `-mx-1.5 my-1.5 h-px bg-border`。 +- 入场：按 side 滑入 + `fade-in-0 zoom-in-95`，约 100ms。 + +### 5.6 Card + +- 圆角 `rounded-4xl`；背景 `bg-card`；阴影 `shadow-md`；`ring-1 ring-foreground/5`；内边距通过 `--card-spacing`（default 24px / sm 16px）。 +- 子区块：`CardTitle`（`text-base font-medium`）/ `CardDescription`（`text-sm text-muted-foreground`）/ `CardContent` / `CardFooter`。 + +### 5.7 Table + +- 外包 `div.overflow-x-auto`，表格本体 `w-full text-sm`。 +- `TableRow`：`border-b`，`hover:bg-muted/50`，选中 `data-[state=selected]:bg-muted`。 +- `TableHead`：高 `h-12`（48px），内边距 `px-3`，`font-medium`，`whitespace-nowrap`。 +- `TableCell`：内边距 `p-3`，不换行；`TableCaption`：`mt-4 text-sm text-muted-foreground`。 + +### 5.8 Dialog / Sheet / Drawer + +| 类型 | 圆角 | 阴影 | 背景 | 进场动画 | +|---|---|---|---|---| +| Dialog | `rounded-4xl` | `shadow-xl` + `ring-1 ring-foreground/5` | `bg-popover` | `fade-in-0 zoom-in-95` ~100ms | +| Sheet | `rounded-4xl` | `shadow-xl` | `bg-popover` | 按 side 平移 `2.5rem` ~200ms | +| Drawer | `rounded-4xl border border-popover` | `shadow-xl` | `bg-popover` | `duration-450` + `cubic-bezier(0.22,1,0.36,1)`，支持 swipe | + +- 遮罩层统一 `bg-black/30` + `supports-backdrop-filter:backdrop-blur-sm`。 +- 关闭按钮：右上 `top-4 right-4`，icon-sm ghost（32×32，浅灰背景），内含 `XIcon`。 +- **三种动画时长差异属官方设计，禁止拉平统一**：对话框类交互 ≤ 200ms，抽屉/底栏类模拟物理滑动 ≤ 450ms。 + +### 5.9 AlertDialog + +与 Dialog 同布局，唯一差异：标题字号 `text-lg`（Dialog 为 `text-base`），用于强调警示内容。 + +### 5.10 Tooltip + +- 圆角 `rounded-xl`；背景 `bg-foreground`（反色）；文字 `text-background`；字号 `text-xs`；内边距 `px-3 py-1.5`。 +- 箭头：`size-2.5 rotate-45 rounded-[2px]`，与背景同色。 +- 进场：`fade-in-0 zoom-in-95`，约 100ms，按 side 滑入。 + +### 5.11 Toast（Sonner） + +通过 CSS 变量桥接品牌色，**禁止**脱离变量单独定义 toast 颜色： + +```css +--normal-bg: var(--popover); +--normal-text: var(--popover-foreground); +--normal-border: var(--border); +--border-radius: var(--radius); +``` + +图标统一 `size-4`：`CircleCheckIcon` / `InfoIcon` / `TriangleAlertIcon` / `OctagonXIcon` / `Loader2Icon animate-spin`。 + +### 5.12 Sidebar + +- 宽度：展开 `16rem` / 收起 `3rem` / 移动端 `18rem`。 +- 深色专属主题：背景 `var(--sidebar-background)`、文字 `var(--sidebar-foreground)`、强调色 `var(--sidebar-primary)`、选中态背景 `var(--sidebar-accent)`（**必须**引用变量，禁止硬编码 hex，见 §2.5）。 +- 顶部装饰渐变：`radial-gradient(circle at 10% 1%, rgb(28 105 186 / 25%), transparent 27%), var(--sidebar-background)`。 +- 收起按钮图标：`PanelLeftIcon`（16px）。 + +### 5.13 Skeleton / Separator + +- Skeleton：`animate-pulse rounded-2xl bg-muted`。 +- Separator：`shrink-0 bg-border`，水平 `h-px w-full`，垂直 `w-px self-stretch`。 + +### 5.14 InputGroup + +`InputGroup` / `InputGroupAddon`（`inline-start` / `inline-end` / `block-start` / `block-end`）/ `InputGroupButton` / `InputGroupText` / `InputGroupInput` / `InputGroupTextarea`；addon 点击时焦点转交给兄弟 ``。addon 文本 `text-muted-foreground text-sm font-medium`；kbd 元素 `rounded-3xl bg-muted-foreground/10 px-1.5`。 + +### 5.15 Legacy 共享视觉类（仅用于维护存量页面，新代码禁止新增引用） + +以下类色值已在本文档中被官方 token 覆盖，**新页面不得再引用这些类**；修改已使用这些类的旧页面时，应顺势替换为 §5.1–§5.14 的现代规格： + +| 类 | 视觉规格（供比对迁移） | +|---|---| +| `.icon-button` | 34×34，边框改为 `border-border`，圆角改为 `rounded-md`，hover 背景改为 `bg-muted` | +| `.avatar` | 34×34 圆形，渐变改为 `linear-gradient(135deg, var(--color-brand), var(--color-brand-strong))` | +| `.primary-button` | 迁移为 `Button`（`variant="default"`），渐变改为 `var(--color-brand)` → `var(--color-brand-deep)` | +| `.secondary-button` | 迁移为 `Button`（`variant="outline"`） | +| `.button-spinner` | 迁移为 `Loader2Icon animate-spin`（Toast/Button loading 态统一图标方案，见 §6.1） | + +--- + +## 6. 图标与动效 + +### 6.1 图标 + +- 图标库：**`lucide-react`**（线条 1.5px 描边风格），新代码必须使用该图标库；若因技术栈限制需换库，**必须**保证线条粗细、视觉重量与 lucide 一致。 +- 尺寸规范：`size-4`（16px，按钮/菜单项默认）/ `size-3.5`（14px，Checkbox）/ `size-2.5`（10px，Tooltip 箭头）。 +- Button 内 svg 未指定尺寸时自动 `size-4`，新实现须遵循此约定。 +- 官方高频图标：`XIcon` / `CheckIcon` / `ChevronDownIcon` / `PanelLeftIcon` / `CircleCheckIcon` / `InfoIcon` / `TriangleAlertIcon` / `OctagonXIcon` / `Loader2Icon`。**所有 loading 态一律使用 `Loader2Icon animate-spin`**，禁止再实现新的 spinner 动画（替代原 `.button-spinner` + `spin` keyframes）。 + +### 6.2 过渡时长（唯一官方取值） + +| Token | 用途 | +|---|---| +| `duration-100` | Dialog / AlertDialog / Sonner 进出 | +| `duration-150` | Sheet 遮罩 | +| `duration-200` | Sidebar / Sheet 主体 | +| `duration-450` | Drawer（模拟 iOS 滑动手感，唯一允许超过 200ms 的场景） | +| `duration-0` | 手势拖拽（swipe）中临时禁用过渡 | + +**规则**：新增弹层组件必须归类到以上五档之一，不得自创新的时长值。 + +### 6.3 缓动函数 + +| Token | 用途 | +|---|---| +| `ease-linear` | 旋转 / 进度条 | +| `ease-in-out` | 通用过渡 | +| `ease-out` | 模态入场 | +| `ease-[cubic-bezier(0.32,0.72,0,1)]` | Drawer 遮罩 | +| `ease-[cubic-bezier(0.22,1,0.36,1)]` | Drawer 主体（"expo out"手感） | + +### 6.4 自定义关键帧 + +| keyframes | 时长 / 缓动 | 用途 | +|---|---|---| +| `shimmer` | 1.3s linear infinite | 骨架屏背景流光（`.tree-skeleton-bar`） | +| `lock-pulse` | 1.8s ease-in-out infinite | 编辑中状态点呼吸（50% 处 opacity 0.5 + scale 0.78） | +| `modal-in` | 0.18s ease-out | 弹窗从 `translateY(8px) scale(0.985)` 进入 | +| `spin` | 0.7s linear infinite | **已废弃**，新代码统一用 `animate-spin`（Tailwind 内建） | + +> `toast-in` 已废弃（Sonner 接管），新代码不得再引用。 + +### 6.5 通用动画工具 + +- `animate-in` / `animate-out`（shadcn tw-animate-css 工具类）。 +- `data-open:animate-in data-open:fade-in-0 data-open:zoom-in-95` / `data-closed:animate-out data-closed:fade-out-0 data-closed:zoom-out-95` —— 弹层组件**必须**遵循此统一动效模式。 +- `data-[side=...]:slide-in-from-{top|bottom|left|right|inline-start|inline-end}-2` —— 弹出层按侧滑入的唯一官方写法。 +- `animate-pulse`（Skeleton）/ `animate-spin`（Loader）。 + +--- + +## 7. 命名与代码风格 + +### 7.1 文件 / 目录 + +- 路由文件集中在 `app/routes/`，业务页面按域拆到 `app/features/{module}/`。 +- 共享原语在 `app/components/ui/.tsx`，域内共享在 `app/components/common/.tsx`。 +- 工具函数：`app/lib/utils.ts`（`cn()` = `twMerge(clsx(...))`）。 +- 样式入口：**只允许一个 CSS 文件** `app/app.css`，由 `app/root.tsx` 唯一一次 `import "./app.css"` 引入；**禁止**新增分散的样式文件。 + +### 7.2 组件命名 + +- UI 组件函数名使用 **PascalCase**：`Button` / `CardHeader` / `DialogTitle` / `SidebarMenuButton`。 +- 每个 UI 元素**必须**带 `data-slot="..."` 属性（如 `button` / `card-header` / `dialog-content`），便于父级选择器和自动化测试定位。 + +### 7.3 类名拼接 + +- 统一用 `cn(...)`（`~/lib/utils`，即 `twMerge(clsx(...))`）拼装类名，**禁止**手写字符串模板拼接类名。 +- 组件变体用 `cva(...)` 描述（参考 `Button` / `InputGroupAddon`）。 +- 非 Tailwind 技术栈：最终输出的类名/样式必须与 §2–§6 的视觉规格一致。 + +### 7.4 CSS 类命名约定 + +- shadcn 风格类**无 BEM**，纯功能化（`card-title` / `dialog-content`），依赖 `data-slot` + Tailwind 工具类定位。 +- `@layer utilities` 中的历史类沿用 kebab-case（如 `scrollbar-thin` / `dashboard-hero-shell` 等），**新增**该层的类也必须用 kebab-case。 +- **颜色 / 间距 token 优先**：新组件必须优先使用 §2/§4 的语义 token（`bg-primary` / `text-foreground` / `border-border`），**禁止**写裸 hex 或任意值；确无对应 token 时，先补充 token 定义，再使用。 + +### 7.5 路径别名 + +```ts +"~": "frontend/app" +"components": "~/components" +"utils": "~/lib/utils" +"ui": "~/components/ui" +"lib": "~/lib" +"hooks": "~/hooks" +``` + +### 7.6 项目依赖版本（参考，非强制技术选型） + +| 角色 | 库 | 版本 | +|---|---|---| +| 原语 | `@base-ui/react` | ^1.7.0 | +| 变体 | `class-variance-authority` | ^0.7.1 | +| 类合并 | `clsx` + `tailwind-merge` | ^2.1.1 / ^3.6.0 | +| 图标 | `lucide-react` | ^1.33.0 | +| 样式 | `tailwindcss` + `@tailwindcss/vite` | ^4.2.2 | +| 动画 | `tw-animate-css` | ^1.4.0 | +| Toast | `sonner` | ^2.0.8 | +| 路由 | `react-router` / `@react-router/dev` | ^8 | +| 状态 | `zustand` | ^5.0.14 | +| 表格 | `@tanstack/react-table` | ^9.1.2 | + +> 新模块/新项目如使用其他技术栈（Vue + Naive UI / Element Plus / Ant Design 等），必须按 §2/§3/§4/§6 的视觉规格在自己的主题系统中重新定义等价 token，保持视觉一致。 + +--- + +## 8. app.css 需新增的官方 Token（落地清单） + +> 本文档 v2.0 相比 v1.0 新增/固化了以下 token，**必须**在 `app.css` 的 `@theme` 中补充定义，作为本规范落地的第一步： + +```css +@theme { + /* 字号 */ + --text-2xs: 0.625rem; /* 10px */ + --text-3xs: 0.5625rem; /* 9px */ + --text-4xs: 0.5rem; /* 8px */ + + /* 字重 */ + --font-weight-medium-plus: 650; + + /* 品牌色补充 */ + --color-brand-deep: #0e5bad; + + /* 中性色补充 */ + --color-ink-caption: #748598; + + /* 间距补充 */ + --spacing-page-x: 22px; + --spacing-button-x: 15px; + --spacing-gap-icon: 7px; + --spacing-gap-sm: 5px; + --spacing-gap-md: 9px; +} +``` + +落地后需同步在 Tailwind 中暴露对应工具类（如 `font-medium-plus`），并对 §2.7、§4.3、§5.15 中列出的旧任意值做替换。 + +--- + +## 9. 跨技术栈适配指引 + +若其他系统/新项目**不使用** React + Tailwind + shadcn 技术栈，请按以下方式对齐： + +1. **色彩**：将 §2 全部 CSS 变量原样复制为自己主题系统的变量（如 Ant Design 的 `token`、Element Plus 的 CSS 变量、Vue 的 SCSS 变量），变量名可保留 `--color-*` 前缀便于跨项目检索。 +2. **字体**：引入同一份 `Inter-Variable.ttf` 字体文件，字号阶梯必须与 §3.2 表格一一对应（值相同，token 名可按自身框架习惯命名，但需在文档中注明与本表的映射关系）。 +3. **圆角/间距/阴影**：直接照抄 §4 的像素值，不需要用相同的 Tailwind 类名，但**数值必须相等**。 +4. **组件视觉**：对照 §5 逐项还原状态（default/hover/active/focus/disabled/invalid）与尺寸，不要求相同 DOM 结构。 +5. **动效**：过渡时长与缓动函数直接照抄 §6.2/§6.3 数值。 +6. 完成对齐后，在自己项目的 README 或设计文档中注明"视觉规范参照 STYLE_GUIDE.md vX.X"，便于后续版本追踪。 + +--- + +## 附：shadcn 配置（参考） + +```json +{ + "style": "base-luma", + "tsx": true, + "tailwind": { "config": "", "css": "app/app.css", "baseColor": "neutral", "cssVariables": true }, + "iconLibrary": "lucide", + "aliases": { + "components": "~/components", + "utils": "~/lib/utils", + "ui": "~/components/ui", + "lib": "~/lib", + "hooks": "~/hooks" + } +} +``` + +**版本摘要**：React/React DOM 19.2.7 · React Router 8 · Tailwind CSS 4.2.2 · `@tailwindcss/vite` 4.2.2 · `@base-ui/react` 1.7.0 · `class-variance-authority` 0.7.1 · `lucide-react` 1.33.0 · `shadcn` CLI 4.19.0 · `sonner` 2.0.8 · `tw-animate-css` 1.4.0 · `tailwind-merge` 3.6.0 · `zustand` 5.0.14。 \ No newline at end of file diff --git a/backend/pyproject.toml b/backend/pyproject.toml index 0161f9a..7ae9d3a 100644 --- a/backend/pyproject.toml +++ b/backend/pyproject.toml @@ -31,6 +31,7 @@ build-backend = "hatchling.build" [dependency-groups] dev = [ + "aiosqlite>=0.22.1", "pytest>=9.1.1", "pytest-asyncio>=1.4.0", "respx>=0.23.1", diff --git a/backend/src/backend/api/__init__.py b/backend/src/backend/api/__init__.py new file mode 100644 index 0000000..e69de29 diff --git a/backend/src/backend/api/_resources_common.py b/backend/src/backend/api/_resources_common.py new file mode 100644 index 0000000..e35d2a3 --- /dev/null +++ b/backend/src/backend/api/_resources_common.py @@ -0,0 +1,130 @@ +"""数据资源路由共享的 payload / 可见性辅助。""" + +from __future__ import annotations + +import os +from pathlib import Path, PurePosixPath +from typing import Any + +from common.db.models import DataResources, StorageObjects +from common.storage import workspaces_root +from fastapi import HTTPException, status +from sqlalchemy import select +from sqlalchemy.ext.asyncio import AsyncSession + +from backend.api.dependencies import RequestContext +from backend.api.scripts import _escape_like_pattern, normalize_user_path + + +def build_list_resources_descendant_prefix(parent_path: str) -> str: + """Return the escaped materialized-path prefix for direct children.""" + normalized = normalize_user_path(parent_path) + escaped = _escape_like_pattern(normalized) + return f"{escaped}/" if escaped else "" + + +def compute_jupyter_relative_path(script_path: str, resource_relative: str) -> str: + """从当前脚本所在目录算到资源文件的 Jupyter 相对路径。""" + script_dir = PurePosixPath(script_path).parent.as_posix() + if not script_dir or script_dir == ".": + return resource_relative + return os.path.relpath(resource_relative, start=script_dir) + + +def resource_directory( + object_key: str, + workspace_id: str, + owner_user_id: str, +) -> str: + """从 object_key 解析资源所在目录(相对于用户根目录,根目录返回 "")。""" + prefix = f"{workspace_id}/{owner_user_id}/" + if not object_key.startswith(prefix): + return "" + tail = object_key[len(prefix):] + directory, _, _ = tail.rpartition("/") + return directory + + +def resource_payload( + resource: DataResources, + storage_object: StorageObjects, + owner_display_name: str | None = None, +) -> dict[str, Any]: + workspace_prefix = f"{resource.workspace_id}/" + user_prefix = f"{resource.owner_user_id}/" + jupyter_accessible_path = "" + absolute_path = "" + if storage_object.object_key and storage_object.object_key.startswith( + workspace_prefix + ): + remainder = storage_object.object_key[len(workspace_prefix):] + if remainder.startswith(user_prefix): + tail = remainder[len(user_prefix):] + jupyter_accessible_path = tail + absolute_path = ( + workspaces_root() + / resource.workspace_id + / resource.owner_user_id + / Path(tail) + ).as_posix() + else: + jupyter_accessible_path = remainder + elif storage_object.object_key: + jupyter_accessible_path = storage_object.object_key + return { + "resource_id": resource.resource_id, + "workspace_id": resource.workspace_id, + "storage_object_id": resource.storage_object_id, + "owner_user_id": resource.owner_user_id, + "owner_display_name": owner_display_name, + "resource_name": resource.resource_name, + "description": resource.description, + "visibility": resource.visibility, + "status": resource.status, + "created_at": resource.created_at.isoformat(), + "updated_at": resource.updated_at.isoformat(), + "file": { + "file_name": storage_object.file_name, + "file_extension": storage_object.file_extension, + "mime_type": storage_object.mime_type, + "size_bytes": storage_object.size_bytes, + "content_hash": storage_object.content_hash, + "object_status": storage_object.object_status, + }, + "jupyter_accessible_path": jupyter_accessible_path, + "absolute_path": absolute_path, + } + + +def can_view(resource: DataResources, context: RequestContext) -> bool: + if resource.owner_user_id == context.user.user_id: + return True + if resource.visibility in {"workspace", "public"}: + return True + return context.is_admin + + +async def get_visible_resource( + resource_id: str, + context: RequestContext, + session: AsyncSession, +) -> tuple[DataResources, StorageObjects]: + row = ( + await session.execute( + select(DataResources, StorageObjects) + .join( + StorageObjects, + StorageObjects.storage_object_id + == DataResources.storage_object_id, + ) + .where( + DataResources.resource_id == resource_id, + DataResources.workspace_id + == context.workspace.workspace_id, + DataResources.status == "active", + ) + ) + ).one_or_none() + if row is None or not can_view(row[0], context): + raise HTTPException(status.HTTP_404_NOT_FOUND, "resource not found") + return row diff --git a/backend/src/backend/admin.py b/backend/src/backend/api/admin.py similarity index 99% rename from backend/src/backend/admin.py rename to backend/src/backend/api/admin.py index 73628ee..eb8a1af 100644 --- a/backend/src/backend/admin.py +++ b/backend/src/backend/api/admin.py @@ -16,7 +16,7 @@ from pydantic import BaseModel, ConfigDict, Field from sqlalchemy import delete, func, or_, select from sqlalchemy.ext.asyncio import AsyncSession -from backend.dependencies import ( +from backend.api.dependencies import ( RequestContext, database_session, request_context, diff --git a/backend/src/backend/auth.py b/backend/src/backend/api/auth.py similarity index 69% rename from backend/src/backend/auth.py rename to backend/src/backend/api/auth.py index d23ebab..74ebf8e 100644 --- a/backend/src/backend/auth.py +++ b/backend/src/backend/api/auth.py @@ -9,9 +9,11 @@ Cookie+JWT authentication endpoints. The user-facing flow is: 1. POST /api/v1/auth/login — verify password, set HttpOnly cookie 2. every other /api/ request reads the cookie via - ``backend.dependencies.request_context`` + ``backend.api.dependencies.request_context`` 3. POST /api/v1/auth/logout — clear the cookie 4. GET /api/v1/auth/me — return the current user + 5. PATCH /api/v1/auth/me — update display_name / email + 6. POST /api/v1/auth/password — change password (clears session cookie) Service-to-service calls do not use these endpoints — they live on the shared Docker network and have no application-layer auth. See @@ -24,17 +26,18 @@ from typing import Any from common.auth.jwt import JwtError, issue_jwt, verify_jwt_token from common.auth.membership import resolve_is_system_admin -from common.auth.passwords import verify_password +from common.auth.passwords import hash_password, verify_password from common.config import settings from common.db.models import Roles, Users, WorkspaceMembers, Workspaces from common.ids import new_ulid from fastapi import APIRouter, Depends, HTTPException, Request, Response, status +from pydantic import BaseModel, ConfigDict, Field from sqlalchemy import select from sqlalchemy.ext.asyncio import AsyncSession -from backend.dependencies import database_session, load_user_permissions +from backend.api.dependencies import current_user, database_session, load_user_permissions -router = APIRouter(tags=["auth"]) +router = APIRouter(prefix="/api/v1/auth", tags=["auth"]) # Cookie 配置：生产环境走 HTTPS 时应设置 Secure；本地 HTTP 开发环境会根据 # 实际请求协议决定是否设置，避免浏览器因 Secure Cookie 而丢弃登录状态。 @@ -95,8 +98,31 @@ def _workspace_payload( } +class ProfileUpdate(BaseModel): + model_config = ConfigDict(extra="forbid") + + display_name: str | None = Field(default=None, min_length=1, max_length=100) + email: str | None = Field(default=None, max_length=255) + + +class PasswordChange(BaseModel): + model_config = ConfigDict(extra="forbid") + + current_password: str = Field(min_length=1, max_length=72) + new_password: str = Field(min_length=8, max_length=72) + + +async def _platform_role_code(session: AsyncSession, user: Users) -> str | None: + if user.platform_role_id is None: + return None + platform_role_row = await session.scalar( + select(Roles).where(Roles.role_id == user.platform_role_id) + ) + return platform_role_row.role_code if platform_role_row is not None else None + + # 校验账号密码，设置登录 Cookie，并返回用户可进入的工作区列表。 -@router.post("/api/v1/auth/login") +@router.post("/login") async def login( request: Request, response: Response, @@ -167,13 +193,7 @@ async def login( # (``Users.platform_role_id``); the iteration over rows above was # a legacy way to find the "highest" workspace role and is no # longer correct now that admin/developer are platform-only. - user_role_code: str | None = None - if user.platform_role_id is not None: - platform_role_row = await session.scalar( - select(Roles).where(Roles.role_id == user.platform_role_id) - ) - if platform_role_row is not None: - user_role_code = platform_role_row.role_code + user_role_code = await _platform_role_code(session, user) token = issue_jwt(user.user_id, ttl_seconds=COOKIE_TTL_SECONDS) _set_session_cookie(request, response, token) @@ -198,7 +218,7 @@ async def login( # 清除浏览器 Cookie，使当前会话立即失效。 -@router.post("/api/v1/auth/logout") +@router.post("/logout") async def logout(response: Response) -> dict[str, Any]: """Clear the session cookie. Idempotent.""" _clear_session_cookie(response) @@ -210,7 +230,7 @@ async def logout(response: Response) -> dict[str, Any]: # 返回当前登录用户、权限和可访问工作区，用于前端初始化登录态。 -@router.get("/api/v1/auth/me") +@router.get("/me") async def me( request: Request, session: AsyncSession = Depends(database_session), @@ -257,13 +277,7 @@ async def me( workspaces = [_workspace_payload(ws, role) for ws, role, _ in rows] default_workspace_id = workspaces[0]["workspace_id"] if workspaces else None - user_role_code: str | None = None - if user.platform_role_id is not None: - platform_role_row = await session.scalar( - select(Roles).where(Roles.role_id == user.platform_role_id) - ) - if platform_role_row is not None: - user_role_code = platform_role_row.role_code + user_role_code = await _platform_role_code(session, user) is_system_admin = await resolve_is_system_admin(session, user) permissions = await load_user_permissions(session, user) @@ -284,6 +298,86 @@ async def me( } +# 当前登录用户修改显示名 / 邮箱（不可改 username、角色、密码）。 +@router.patch("/me") +async def update_me( + payload: ProfileUpdate, + user: Users = Depends(current_user), + session: AsyncSession = Depends(database_session), +) -> dict[str, Any]: + """Update the authenticated user's display_name and/or email.""" + if payload.display_name is None and payload.email is None: + raise HTTPException( + status.HTTP_400_BAD_REQUEST, + "至少提供一个可修改字段", + ) + + if payload.display_name is not None: + user.display_name = payload.display_name.strip() + + if payload.email is not None: + normalized = payload.email.strip() or None + if normalized is not None: + duplicate = await session.scalar( + select(Users.user_id).where( + Users.email == normalized, + Users.user_id != user.user_id, + Users.is_deleted == 0, + ) + ) + if duplicate is not None: + raise HTTPException(status.HTTP_409_CONFLICT, "邮箱已存在") + user.email = normalized + + await session.flush() + await session.refresh(user) + + user_role_code = await _platform_role_code(session, user) + is_system_admin = await resolve_is_system_admin(session, user) + permissions = await load_user_permissions(session, user) + + return { + "request_id": new_ulid(), + "data": { + "user": _user_payload( + user, + user_role_code, + is_system_admin=is_system_admin, + permissions=permissions, + ), + }, + "meta": {}, + } + + +# 当前登录用户修改密码；成功后清除 Cookie，需重新登录。 +@router.post("/password") +async def change_password( + payload: PasswordChange, + response: Response, + user: Users = Depends(current_user), + session: AsyncSession = Depends(database_session), +) -> dict[str, Any]: + """Change password after verifying the current one; clears the session cookie.""" + if not verify_password(payload.current_password, user.password_hash): + raise HTTPException(status.HTTP_400_BAD_REQUEST, "当前密码不正确") + if payload.current_password == payload.new_password: + raise HTTPException( + status.HTTP_400_BAD_REQUEST, + "新密码不能与当前密码相同", + ) + + user.password_hash = hash_password(payload.new_password) + await session.flush() + _clear_session_cookie(response) + + return { + "request_id": new_ulid(), + "data": {"password_changed": True}, + "meta": {}, + } + + __all__ = [ "COOKIE_NAME", "COOKIE_TTL_SECONDS", diff --git a/backend/src/backend/dependencies.py b/backend/src/backend/api/dependencies.py similarity index 100% rename from backend/src/backend/dependencies.py rename to backend/src/backend/api/dependencies.py diff --git a/backend/src/backend/jupyter.py b/backend/src/backend/api/jupyter.py similarity index 62% rename from backend/src/backend/jupyter.py rename to backend/src/backend/api/jupyter.py index 74d01e1..c5147b6 100644 --- a/backend/src/backend/jupyter.py +++ b/backend/src/backend/api/jupyter.py @@ -8,6 +8,8 @@ """ import re +import threading +import time from common.auth.jwt import JwtError, verify_jwt_token from common.auth.membership import MembershipError, load_active_membership @@ -17,10 +19,30 @@ from fastapi.security import HTTPAuthorizationCredentials, HTTPBearer from sqlalchemy import select from sqlalchemy.ext.asyncio import AsyncSession -from backend.dependencies import database_session -from backend.runtime_client import RuntimeClientError +from backend.api.dependencies import database_session +from backend.clients.runtime import RuntimeClientError -router = APIRouter(tags=["jupyter"]) +# --------------------------------------------------------------------------- +# (workspace_id, user_id) -> (expires_at_monotonic, payload) 的 5 秒验证结果缓存。 +# +# payload 至少包含 x-upstream-addr 与 x-jupyter-internal-token，只缓存 +# "membership 校验通过 + 拿到 runtime 信息"后的成功结果；403/500 不写缓存。 +# +# 设计说明： +# * TTL 只有 5 秒，且 Runtime 容器单实例（CLAUDE.md "Service rules"： +# "Runtime must stay single-replica while file leases and Jupyter tickets +# use the simplified implementation"），module-level 内存缓存是安全的， +# 无需 Redis 之类的外部存储。 +# * JWT 验签与 lock check 不进缓存：前者是每请求必须的信任边界；后者是 +# per-URI 且 5s 内可能解锁/加锁，跨用户/跨 notebook 不应共享缓存。 +# * 折衷：用户被踢出 workspace / membership 撤销后，最坏 5 秒内本接口仍会 +# 对已缓存的 (workspace_id, user_id) 返回 200，这是可接受的折衷。 +_JUPYTER_AUTH_CACHE: dict[tuple[str, str], tuple[float, dict[str, str]]] = {} +_JUPYTER_AUTH_CACHE_LOCK = threading.Lock() +_JUPYTER_AUTH_CACHE_TTL_SECONDS = 5.0 + + +router = APIRouter(prefix="/api/v1/auth", tags=["jupyter"]) security = HTTPBearer(auto_error=False) @@ -89,9 +111,27 @@ async def load_active_membership_or_403( ) from exc +def _jupyter_auth_cache_get(workspace_id: str, user_id: str) -> dict[str, str] | None: + with _JUPYTER_AUTH_CACHE_LOCK: + entry = _JUPYTER_AUTH_CACHE.get((workspace_id, user_id)) + if entry is None: + return None + expires_at, payload = entry + if time.monotonic() >= expires_at: + _JUPYTER_AUTH_CACHE.pop((workspace_id, user_id), None) + return None + return payload + + +def _jupyter_auth_cache_put(workspace_id: str, user_id: str, payload: dict[str, str]) -> None: + expires_at = time.monotonic() + _JUPYTER_AUTH_CACHE_TTL_SECONDS + with _JUPYTER_AUTH_CACHE_LOCK: + _JUPYTER_AUTH_CACHE[(workspace_id, user_id)] = (expires_at, payload) + + # 供 Nginx auth_request 调用：验证访问 Jupyter 的身份、成员关系和文件锁， # 再返回应转发到的 Jupyter 地址及内部令牌。 -@router.get("/api/v1/auth/jupyter") +@router.get("/jupyter") async def verify_jupyter_access( request: Request, response: Response, @@ -132,8 +172,14 @@ async def verify_jupyter_access( detail="Invalid Authentication Token", ) - await load_active_membership_or_403(session, user_id, workspace_id) + # JWT 验签之后、昂贵的 membership/runtime 查找之前先查缓存。命中时跳过 + # membership 与 runtime，但仍要跑下面的 lock check（per-URI，缓存不含它）。 + cached = _jupyter_auth_cache_get(workspace_id, user_id) + if cached is None: + await load_active_membership_or_403(session, user_id, workspace_id) + # lock check 永远执行、不进缓存：同一 (workspace_id, user_id) 的不同 URI + # 状态不同，且 5s 内可能解锁/加锁。 notebook_path = extract_notebook_path(original_uri, workspace_id) if notebook_path and await check_notebook_is_locked( session, @@ -146,6 +192,11 @@ async def verify_jupyter_access( detail=f"Notebook '{notebook_path}' is currently locked", ) + if cached is not None: + response.headers["x-upstream-addr"] = cached["x_upstream_addr"] + response.headers["x-jupyter-internal-token"] = cached["x_jupyter_internal_token"] + return {"status": "ok"} + runtime_client = request.app.state.runtime_client ws_info = await runtime_client.get_workspace(workspace_id) if not ws_info or ws_info.get("status") != "running": @@ -166,6 +217,14 @@ async def verify_jupyter_access( detail="Jupyter instance returned no port", ) - response.headers["x-upstream-addr"] = f"{jupyter_base_url}:{target_port}" - response.headers["x-jupyter-internal-token"] = jupyter_token or "" + headers_payload = { + "x_upstream_addr": f"{jupyter_base_url}:{target_port}", + "x_jupyter_internal_token": jupyter_token or "", + } + # 只缓存成功结果；lock check 失败（403）或 runtime 启动失败（500）在上方 + # 已提前返回，不会走到这里污染缓存。 + _jupyter_auth_cache_put(workspace_id, user_id, headers_payload) + + response.headers["x-upstream-addr"] = headers_payload["x_upstream_addr"] + response.headers["x-jupyter-internal-token"] = headers_payload["x_jupyter_internal_token"] return {"status": "ok"} diff --git a/backend/src/backend/api/platform.py b/backend/src/backend/api/platform.py new file mode 100644 index 0000000..dc480ff --- /dev/null +++ b/backend/src/backend/api/platform.py @@ -0,0 +1,8 @@ +"""Backward-compat shim — see backend.api.platform package.""" +from backend.api.platform import ( + SystemAdminContext, + router, + system_admin_context, +) + +__all__ = ["SystemAdminContext", "router", "system_admin_context"] diff --git a/backend/src/backend/api/platform/__init__.py b/backend/src/backend/api/platform/__init__.py new file mode 100644 index 0000000..24f7bd8 --- /dev/null +++ b/backend/src/backend/api/platform/__init__.py @@ -0,0 +1,94 @@ +"""系统级管理接口。 + +中文导读：本模块管理全平台的用户、工作区、成员关系和角色权限。它使用 +``system_admin_context`` 进行平台管理员校验，因此不要求请求者先加入某个具体 +工作区；普通工作区内的业务接口则使用 ``request_context``。 + +System-admin (platform-scope) endpoints for workspace & membership management. + +All routes under ``/api/v1/platform/*`` are gated by +:func:`system_admin_context`, which requires the requester to hold a +``Users.platform_role_id`` pointing to a ``Roles`` row whose +``role_code == 'admin'``. Unlike ``backend.api.dependencies.request_context``, +this dependency does NOT require an active workspace membership — system +admins can manage workspaces before/without being a member of any. + +Endpoints +--------- + +Workspace CRUD:: + + GET /workspaces — list non-deleted workspaces + POST /workspaces — create a new workspace + GET /workspaces/{workspace_id} — single workspace (incl. disabled) + PATCH /workspaces/{workspace_id} — update editable fields + DELETE /workspaces/{workspace_id} — soft delete (cascades memberships) + +Workspace membership CRUD:: + + GET /workspaces/{workspace_id}/members — list active members + POST /workspaces/{workspace_id}/members — add a member + PATCH /workspaces/{workspace_id}/members/{user_id} — update role/status + DELETE /workspaces/{workspace_id}/members/{user_id} — remove a member + +Platform employee roster:: + + GET /employees — list all non-deleted users + POST /employees — create a user without workspace membership + PATCH /employees/{user_id} — update a user's profile, status or platform role + DELETE /employees/{user_id} — soft delete a user (cascades to workspace memberships) + +Role menu-permission management:: + + GET /roles — list platform roles with their permission_codes + GET /roles/{role_code}/permissions — one role's permission_codes + PATCH /roles/{role_code}/permissions — replace a role's permission set (diff-based) + +Invariants +---------- + +* Every workspace must always retain at least one active ``admin`` member. + This is enforced on member PATCH/DELETE AND on + ``PATCH /employees/{user_id}`` demotions, because workspace role is + inherited from ``users.platform_role_id`` and demoting a platform + admin cascades to all of their active memberships. +* A system admin cannot remove their own workspace membership via + ``DELETE .../members/{self}``; the only escape is to delete the entire + workspace, which cascades membership soft-deletion. +* ``DELETE /workspaces/{id}`` is allowed from any non-disabled status and + sets ``status='disabled'`` + ``is_deleted=1`` + ``deleted_at`` on the + workspace and every one of its active memberships. +* The ``admin`` role must always keep ``system:view``. + menu permissions; non-admin roles may never hold ``system.*`` + permissions. Menu permissions gate frontend rendering only — API + authorization always keys off ``role_code == 'admin'``. +""" + +from __future__ import annotations + +from fastapi import APIRouter + +from backend.api.platform._deps import ( + SystemAdminContext, + system_admin_context, +) +from backend.api.platform.employees import router as employees_router +from backend.api.platform.members import router as members_router +from backend.api.platform.roles import router as roles_router +from backend.api.platform.workspaces import router as workspaces_router + +# 聚合 router，所有 endpoint 都挂在 /api/v1/platform 下。每个子 router 自带 +# prefix="/api/v1/platform"，这里 include_router 不加 prefix，路径保持不变。 +router = APIRouter() +router.include_router(employees_router) +router.include_router(workspaces_router) +router.include_router(members_router) +router.include_router(roles_router) + +# 保留 system_admin_context 的 re-export，供其他文件使用 +# (之前从 backend.api.platform 导出)。 +__all__ = [ + "SystemAdminContext", + "router", + "system_admin_context", +] diff --git a/backend/src/backend/api/platform/_deps.py b/backend/src/backend/api/platform/_deps.py new file mode 100644 index 0000000..3657fe9 --- /dev/null +++ b/backend/src/backend/api/platform/_deps.py @@ -0,0 +1,163 @@ +"""Shared dependencies & helpers for the platform admin API. +Centralizes the ``system_admin_context`` gate plus cross-resource helpers so the +resource modules (employees / workspaces / roles) stay focused on their endpoints. +""" + +from __future__ import annotations + +from dataclasses import dataclass +from typing import Any + +from common.db.models import Roles, Users, WorkspaceMembers +from common.ids import new_ulid +from fastapi import Depends, HTTPException, Request, status +from sqlalchemy import func, select +from sqlalchemy.ext.asyncio import AsyncSession + +from backend.api.dependencies import current_user, database_session + +# --------------------------------------------------------------------------- +# System-admin context dependency +# --------------------------------------------------------------------------- + + +@dataclass(frozen=True) +class SystemAdminContext: + """通过系统管理员校验后的上下文，只包含当前用户和请求追踪 ID。""" + + + request_id: str + user: Users + platform_role: Roles + + +async def system_admin_context( + request: Request, + session: AsyncSession = Depends(database_session), +) -> SystemAdminContext: + """验证当前用户是否为平台管理员，供 /api/v1/platform 下的路由依赖。""" + user = await current_user(request, session) + if user.platform_role_id is None: + raise HTTPException( + status.HTTP_403_FORBIDDEN, + "需要系统管理员权限", + ) + platform_role = await session.scalar( + select(Roles).where(Roles.role_id == user.platform_role_id) + ) + if platform_role is None or platform_role.role_code != "admin": + raise HTTPException( + status.HTTP_403_FORBIDDEN, + "需要系统管理员权限", + ) + request_id = request.headers.get("X-Request-ID") or new_ulid() + return SystemAdminContext( + request_id=request_id, + user=user, + platform_role=platform_role, + ) + + +async def _is_system_admin(session: AsyncSession, user: Users) -> bool: + """True if ``user`` holds the platform-scoped admin role. + + Mirrors the check inside :func:`system_admin_context` so member-listing + endpoints can admit workspace members *or* system admins without pulling + in the full :class:`SystemAdminContext` (which 403s non-admins outright). + """ + if user.platform_role_id is None: + return False + platform_role = await session.scalar( + select(Roles).where(Roles.role_id == user.platform_role_id) + ) + return platform_role is not None and platform_role.role_code == "admin" + + +# --------------------------------------------------------------------------- +# Internal helpers +# --------------------------------------------------------------------------- + + +async def _load_role_by_code(session: AsyncSession, role_code: str) -> Roles: + """Load a live platform-scoped role by code; 422 if missing. + + 之所以把 ``is_deleted == 0`` 和 ``role_scope == "platform"`` 写进 helper, + 是因为本 helper 的所有调用方(``employees`` create/update / 两个 count + helper)都需要的是 platform 角色。Workspace 角色不应被赋值到 + ``users.platform_role_id``,软删除的 role 也不应被新引用 —— 在 helper + 层兜底一次,后续 schema 放宽 ``role_code`` 为 ``str`` 时不必每个端点 + 再补守卫。 + """ + role = await session.scalar( + select(Roles).where( + Roles.role_code == role_code, + Roles.is_deleted == 0, + Roles.role_scope == "platform", + ) + ) + if role is None: + raise HTTPException( + status.HTTP_422_UNPROCESSABLE_ENTITY, + f"角色 {role_code} 不存在", + ) + return role + + +async def _count_active_admins( + session: AsyncSession, + workspace_id: str, + exclude_user_id: str | None = None, +) -> int: + """Count active admin members of ``workspace_id``. + + Pass ``exclude_user_id`` when checking "would X be the last admin?" + before mutating X. + """ + admin_role = await _load_role_by_code(session, "admin") + stmt = ( + select(func.count()) + .select_from(WorkspaceMembers) + .where( + WorkspaceMembers.workspace_id == workspace_id, + WorkspaceMembers.role_id == admin_role.role_id, + WorkspaceMembers.member_status == "active", + WorkspaceMembers.is_deleted == 0, + ) + ) + if exclude_user_id is not None: + stmt = stmt.where(WorkspaceMembers.user_id != exclude_user_id) + return int(await session.scalar(stmt) or 0) + + +async def _count_active_system_admins( + session: AsyncSession, + exclude_user_id: str | None = None, +) -> int: + """Count active system admins across the platform. + + Pass ``exclude_user_id`` when checking "would X be the last admin?" + before mutating X. + """ + admin_role = await _load_role_by_code(session, "admin") + stmt = ( + select(func.count()) + .select_from(Users) + .where( + Users.status == "active", + Users.is_deleted == 0, + Users.platform_role_id == admin_role.role_id, + ) + ) + if exclude_user_id is not None: + stmt = stmt.where(Users.user_id != exclude_user_id) + return int(await session.scalar(stmt) or 0) + + +def _envelope(request_id: str, data: Any, meta: dict[str, Any] | None = None) -> dict[str, Any]: + return { + "request_id": request_id, + "data": data, + "meta": meta or {}, + } + + diff --git a/backend/src/backend/api/platform/_pagination.py b/backend/src/backend/api/platform/_pagination.py new file mode 100644 index 0000000..c336032 --- /dev/null +++ b/backend/src/backend/api/platform/_pagination.py @@ -0,0 +1,60 @@ +"""Cursor (keyset) pagination helpers for platform list endpoints. + +Cursor encodes the sort key ``(created_at, id)`` as a URL-safe base64 +string. Clients pass it back via ``?cursor=`` to fetch the next page. +Invalid cursors raise HTTP 400 — never silently treated as page 1. +""" + +from __future__ import annotations + +import base64 +import datetime +from typing import Any + +from fastapi import HTTPException, status + +DEFAULT_PAGE_LIMIT = 10 +MAX_PAGE_LIMIT = 200 + + +def encode_cursor(created_at: datetime.datetime, row_id: str) -> str: + """Encode ``(created_at, id)`` into an opaque cursor string.""" + if created_at.tzinfo is not None: + created_at = created_at.replace(tzinfo=None) + raw = f"{created_at.isoformat()}|{row_id}".encode("utf-8") + return base64.urlsafe_b64encode(raw).decode("ascii").rstrip("=") + + +def decode_cursor(cursor: str) -> tuple[datetime.datetime, str]: + """Decode a cursor; raise 400 on malformed input.""" + try: + padded = cursor + "=" * (-len(cursor) % 4) + raw = base64.urlsafe_b64decode(padded.encode("ascii")).decode("utf-8") + ts_part, _, row_id = raw.partition("|") + if not ts_part or not row_id: + raise ValueError("missing parts") + created_at = datetime.datetime.fromisoformat(ts_part) + if created_at.tzinfo is not None: + created_at = created_at.replace(tzinfo=None) + return created_at, row_id + except (ValueError, TypeError, UnicodeDecodeError) as exc: + raise HTTPException( + status.HTTP_400_BAD_REQUEST, + "无效的分页 cursor", + ) from exc + + +def page_meta( + *, + limit: int, + page_count: int, + total_count: int, + next_cursor: str | None, +) -> dict[str, Any]: + return { + "limit": limit, + "page_count": page_count, + "total_count": total_count, + "has_more": next_cursor is not None, + "next_cursor": next_cursor, + } diff --git a/backend/src/backend/api/platform/_permission_set.py b/backend/src/backend/api/platform/_permission_set.py new file mode 100644 index 0000000..2bd8ee0 --- /dev/null +++ b/backend/src/backend/api/platform/_permission_set.py @@ -0,0 +1,193 @@ +"""Role permission-set replacement logic. + +``_apply_role_permission_set`` is shared by PATCH /roles/{role_code}/permissions +and POST /roles so both entry points apply the identical guard order and the +diff-based write. Extracted from ``roles.py`` so the endpoint module stays +under 500 lines. + +The write is upsert-style because ``role_permissions`` uses the composite +primary key ``(role_id, permission_id)``: a soft-deleted row (``is_deleted=1``) +still occupies its PK slot, so re-adding a permission must *revive* the +historical row (UPDATE) instead of INSERTing over it (which would raise a +duplicate-key IntegrityError). +""" + +from __future__ import annotations + +import datetime + +from common.db.models import Permissions, RolePermissions, Roles +from fastapi import HTTPException, status +from sqlalchemy import insert, select, update +from sqlalchemy.ext.asyncio import AsyncSession + + +async def _load_role_permission_codes( + session: AsyncSession, role_id: str +) -> list[str]: + """Return the active permission_codes for a role, ordered by code.""" + rows = ( + await session.execute( + select(Permissions.permission_code) + .join( + RolePermissions, + RolePermissions.permission_id == Permissions.permission_id, + ) + .where( + RolePermissions.role_id == role_id, + RolePermissions.is_deleted == 0, + Permissions.is_deleted == 0, + ) + .order_by(Permissions.permission_code) + ) + ).all() + return [row[0] for row in rows] + + +async def _apply_role_permission_set( + session: AsyncSession, role: Roles, codes: list[str] +) -> list[str]: + """Validate ``codes`` then replace the role's permission set wholesale. + + Shared by PATCH /roles/{role_code}/permissions and POST /roles so the + guard order is identical no matter the entry point: + + 1. Admin: must keep ``system:view`` → 409 (menu perms never gate + API access; auth keys off ``role_code == 'admin'``). + 2. Non-admin: ``system.*`` codes → 422. + 3. Unknown codes → 422. + 4. Diff-based soft-delete + upsert — the ``(role_id, permission_id)`` + PK keeps soft-deleted rows, so delete-all/insert-all would + IntegrityError. Repeat-with-same-set is a no-op. + + Returns the role's final permission_codes (after flush). + """ + new_codes = list(dict.fromkeys(codes)) + + if role.role_code == "admin": + keeps_admin_entry = "system:view" in new_codes + if not keeps_admin_entry: + raise HTTPException( + status.HTTP_409_CONFLICT, + "admin 角色必须保留 system:view 权限", + ) + else: + # Menu permissions are a frontend-display signal only — backend + # authorization keeps keying off role_code == "admin". Letting a + # non-admin role hold system.* permissions would render the + # system-admin entry in the developer's UI while every + # /api/v1/platform/* call still returns 403. Reject with 422 so + # the failure is unambiguous about *what* the input violated. + leaked_system = [ + code for code in new_codes if code.startswith("system:") + ] + if leaked_system: + raise HTTPException( + status.HTTP_422_UNPROCESSABLE_ENTITY, + f"非 admin 角色不能拥有 system.* 权限: {leaked_system}", + ) + + # 3. Validate every requested permission_code exists and is live. + if new_codes: + rows = ( + await session.execute( + select(Permissions.permission_code).where( + Permissions.permission_code.in_(new_codes), + Permissions.is_deleted == 0, + ) + ) + ).all() + found = {row[0] for row in rows} + missing = [code for code in new_codes if code not in found] + if missing: + raise HTTPException( + status.HTTP_422_UNPROCESSABLE_ENTITY, + f"未知的 permission_code: {missing}", + ) + + # 4. Write: diff-based soft-delete + upsert. + # The (role_id, permission_id) PRIMARY KEY still occupies the slot + # of soft-deleted rows, so a "delete-all then insert-all" approach + # would IntegrityError on any code that was already linked. + # Instead: only soft-delete codes NOT in the new set, only revive-or- + # insert codes NOT already active. Repeat-with-same-payload is a no-op. + now = datetime.datetime.utcnow() + current_codes = set( + await _load_role_permission_codes(session, role.role_id) + ) + new_set = set(new_codes) + + codes_to_drop = current_codes - new_set + codes_to_add = new_set - current_codes + + if codes_to_drop: + # Resolve to permission_ids then soft-delete by id pair. + drop_ids = ( + await session.execute( + select(Permissions.permission_id).where( + Permissions.permission_code.in_(codes_to_drop), + Permissions.is_deleted == 0, + ) + ) + ).all() + drop_id_values = [row[0] for row in drop_ids] + await session.execute( + update(RolePermissions) + .where( + RolePermissions.role_id == role.role_id, + RolePermissions.permission_id.in_(drop_id_values), + RolePermissions.is_deleted == 0, + ) + .values(is_deleted=1, deleted_at=now) + ) + + if codes_to_add: + add_ids = ( + await session.execute( + select(Permissions.permission_id).where( + Permissions.permission_code.in_(codes_to_add), + Permissions.is_deleted == 0, + ) + ) + ).all() + add_id_values = [pid for pid, in add_ids] + if not add_id_values: + return + + # 区分「历史软删行」(复活) vs 「全新行」(插入) + # RolePermissions 的主键 (role_id, permission_id) 即使 is_deleted=1 也占槽, + # 直接 INSERT 会撞 PK。复活 + 插入两步走。 + existing_ids = set( + ( + await session.execute( + select(RolePermissions.permission_id).where( + RolePermissions.role_id == role.role_id, + RolePermissions.permission_id.in_(add_id_values), + ) + ) + ).scalars() + ) + + revive_ids = [pid for pid in add_id_values if pid in existing_ids] + fresh_ids = [pid for pid in add_id_values if pid not in existing_ids] + + if revive_ids: + await session.execute( + update(RolePermissions) + .where( + RolePermissions.role_id == role.role_id, + RolePermissions.permission_id.in_(revive_ids), + ) + .values(is_deleted=0, deleted_at=None) + ) + if fresh_ids: + await session.execute( + insert(RolePermissions), + [ + {"role_id": role.role_id, "permission_id": pid} + for pid in fresh_ids + ], + ) + + await session.flush() + return await _load_role_permission_codes(session, role.role_id) diff --git a/backend/src/backend/api/platform/employees.py b/backend/src/backend/api/platform/employees.py new file mode 100644 index 0000000..6be9e69 --- /dev/null +++ b/backend/src/backend/api/platform/employees.py @@ -0,0 +1,425 @@ +"""Platform employee roster endpoints. +CRUD for platform users (``/employees``). Role-code changes cascade to every active +``workspace_members`` row because workspace role is inherited from the platform role. +""" + +from __future__ import annotations + +import datetime +from typing import Any, Literal + +from common.auth.passwords import hash_password +from common.db.models import Roles, Users, WorkspaceMembers, Workspaces +from common.ids import new_ulid +from fastapi import APIRouter, Depends, HTTPException, Query, status +from pydantic import BaseModel, ConfigDict, Field +from sqlalchemy import func, or_, select, tuple_, update +from sqlalchemy.ext.asyncio import AsyncSession + +from backend.api.dependencies import database_session +from backend.api.platform._deps import ( + SystemAdminContext, + _count_active_admins, + _count_active_system_admins, + _envelope, + _load_role_by_code, + system_admin_context, +) +from backend.api.platform._pagination import ( + DEFAULT_PAGE_LIMIT, + MAX_PAGE_LIMIT, + decode_cursor, + encode_cursor, + page_meta, +) + +# 平台角色 code 的字符串约束:与 RoleCreate.role_code 一致。 +# 之所以从 Literal["admin","developer"] 放宽为 str,是因为 ``listPlatformRoles`` +# 现已返回用户自建的 platform 角色(``role_scope=="platform"`` 且 ``is_builtin==0``), +# 前端 dialog 用 ``listPlatformRoles()`` 渲染选项,提交非内置 code 时会被 Pydantic +# Literal 校验直接 422 拒掉。schema 不应枚举运行时数据;改用 ``_load_role_by_code`` +# 做存在性 + 平台作用域校验(并隐式排除 is_deleted=1)。 + +router = APIRouter(prefix="/api/v1/platform", tags=["platform"]) + + +# --------------------------------------------------------------------------- +# Schemas +# --------------------------------------------------------------------------- + + +# 新建平台用户的请求体；创建用户不等同于把用户加入某个工作区。 +class PlatformEmployeeCreate(BaseModel): + model_config = ConfigDict(extra="forbid") + + username: str = Field(min_length=2, max_length=64) + display_name: str = Field(min_length=1, max_length=100) + email: str | None = Field(default=None, max_length=255) + password: str = Field(min_length=8, max_length=72) + role_code: str | None = Field(default=None, min_length=2, max_length=64) + + +# 修改平台用户资料、状态或平台角色的请求体。 +class PlatformEmployeeUpdate(BaseModel): + model_config = ConfigDict(extra="forbid") + + display_name: str | None = Field(default=None, min_length=1, max_length=100) + email: str | None = Field(default=None, max_length=255) + status: Literal["active", "disabled", "locked"] | None = None + role_code: str | None = Field(default=None, min_length=2, max_length=64) + + +# --------------------------------------------------------------------------- +# Payload helpers +# --------------------------------------------------------------------------- + + +def platform_employee_payload( + user: Users, + role: Roles | None, +) -> dict[str, Any]: + return { + "user_id": user.user_id, + "username": user.username, + "display_name": user.display_name, + "email": user.email, + "status": user.status, + "role_code": role.role_code if role is not None else None, + "role_name": role.role_name if role is not None else None, + "created_at": user.created_at.isoformat(), + } + + +# --------------------------------------------------------------------------- +# Platform employee roster +# --------------------------------------------------------------------------- + + +# 列出整个平台的非删除用户；不局限于某一个工作区。 +@router.get("/employees") +async def list_platform_employees( + limit: int = Query(default=DEFAULT_PAGE_LIMIT, ge=1, le=MAX_PAGE_LIMIT), + cursor: str | None = Query(default=None), + q: str | None = Query(default=None, max_length=100), + context: SystemAdminContext = Depends(system_admin_context), + session: AsyncSession = Depends(database_session), +) -> dict[str, Any]: + """List non-soft-deleted platform users with cursor pagination + search.""" + base_filters = [Users.is_deleted == 0] + keyword = (q or "").strip() + if keyword: + like = f"%{keyword}%" + base_filters.append( + or_( + Users.display_name.like(like), + Users.username.like(like), + Users.email.like(like), + ) + ) + + total_count = int( + await session.scalar( + select(func.count()).select_from(Users).where(*base_filters) + ) + or 0 + ) + + page_filters = list(base_filters) + if cursor is not None: + cursor_ts, cursor_id = decode_cursor(cursor) + page_filters.append( + tuple_(Users.created_at, Users.user_id) > (cursor_ts, cursor_id) + ) + + rows = ( + await session.execute( + select(Users, Roles) + .outerjoin(Roles, Roles.role_id == Users.platform_role_id) + .where(*page_filters) + .order_by(Users.created_at, Users.user_id) + .limit(limit + 1) + ) + ).all() + has_more = len(rows) > limit + page_rows = rows[:limit] + next_cursor = None + if has_more and page_rows: + last_user = page_rows[-1][0] + next_cursor = encode_cursor(last_user.created_at, last_user.user_id) + return _envelope( + context.request_id, + [platform_employee_payload(user, role) for user, role in page_rows], + page_meta( + limit=limit, + page_count=len(page_rows), + total_count=total_count, + next_cursor=next_cursor, + ), + ) + + +# 创建平台用户；后续可再通过成员接口把该用户加入工作区。 +@router.post("/employees", status_code=status.HTTP_201_CREATED) +async def create_platform_employee( + payload: PlatformEmployeeCreate, + context: SystemAdminContext = Depends(system_admin_context), + session: AsyncSession = Depends(database_session), +) -> dict[str, Any]: + """Create a platform user without assigning workspace membership.""" + username = payload.username.strip() + display_name = payload.display_name.strip() + duplicate_conditions = [Users.username == username] + if payload.email: + duplicate_conditions.append(Users.email == payload.email.strip()) + duplicate = await session.scalar( + select(Users.user_id).where(or_(*duplicate_conditions)) + ) + if duplicate is not None: + raise HTTPException(status.HTTP_409_CONFLICT, "用户名或邮箱已存在") + + new_role: Roles | None = None + if payload.role_code is not None: + new_role = await _load_role_by_code(session, payload.role_code) + + user = Users( + user_id=new_ulid(), + username=username, + display_name=display_name, + email=payload.email.strip() if payload.email else None, + password_hash=hash_password(payload.password), + status="active", + platform_role_id=new_role.role_id if new_role is not None else None, + ) + session.add(user) + await session.flush() + await session.refresh(user) + return _envelope( + context.request_id, + platform_employee_payload(user, new_role), + ) + + +# 更新平台用户资料、账号状态或平台角色，同时保护最少管理员等约束。 +@router.patch("/employees/{user_id}") +async def update_platform_employee( + user_id: str, + payload: PlatformEmployeeUpdate, + context: SystemAdminContext = Depends(system_admin_context), + session: AsyncSession = Depends(database_session), +) -> dict[str, Any]: + """Update a platform user's profile, status, or platform role. + + Changing ``role_code`` cascades: every active ``workspace_members`` + row of the user is rewritten to the new role (workspace role is + inherited from the platform role). Demoting admin → developer is + rejected with 409 when it would leave any workspace without an + active admin member, or the platform without an active system + admin. Self-demotion is always rejected. + """ + user = await session.get(Users, user_id) + if user is None or user.is_deleted != 0: + raise HTTPException(status.HTTP_404_NOT_FOUND, "用户不存在") + + is_self = user_id == context.user.user_id + if ( + is_self + and payload.status is not None + and payload.status != "active" + ): + raise HTTPException(status.HTTP_409_CONFLICT, "不能停用当前登录账号") + + current_role: Roles | None = None + if user.platform_role_id is not None: + current_role = await session.scalar( + select(Roles).where(Roles.role_id == user.platform_role_id) + ) + is_current_system_admin = ( + user.status == "active" + and current_role is not None + and current_role.role_code == "admin" + ) + + new_role: Roles | None = None + if payload.role_code is not None: + new_role = await _load_role_by_code(session, payload.role_code) + next_status = payload.status if payload.status is not None else user.status + + leaves_admin_pool = ( + is_current_system_admin + and ( + next_status != "active" + or (new_role is not None and new_role.role_code != "admin") + ) + ) + if leaves_admin_pool: + remaining = await _count_active_system_admins( + session, exclude_user_id=user_id, + ) + if remaining == 0: + raise HTTPException( + status.HTTP_409_CONFLICT, + "platform 必须保留至少一个 active 系统管理员", + ) + + # Workspace-level last-admin guard for the demote path. The role_code + # sync below rewrites ``workspace_members.role_id`` for every active + # membership of this user, so demoting admin → developer would + # silently strip workspace admin coverage anywhere this user is the + # sole active admin member. ``update_member`` / ``remove_member`` + # guard the same invariant via ``_count_active_admins``; this + # endpoint must too, now that it can change workspace roles. + demotes_admin = ( + is_current_system_admin + and new_role is not None + and new_role.role_code != "admin" + ) + if demotes_admin: + assert current_role is not None # implied by is_current_system_admin + admin_memberships = ( + await session.execute( + select(WorkspaceMembers.workspace_id) + .where( + WorkspaceMembers.user_id == user_id, + WorkspaceMembers.role_id == current_role.role_id, + WorkspaceMembers.member_status == "active", + WorkspaceMembers.is_deleted == 0, + ) + ) + ).all() + orphaned: list[str] = [] + for (ws_id,) in admin_memberships: + remaining_ws = await _count_active_admins( + session, ws_id, exclude_user_id=user_id, + ) + if remaining_ws == 0: + orphaned.append(ws_id) + if orphaned: + codes = ( + await session.execute( + select(Workspaces.workspace_code).where( + Workspaces.workspace_id.in_(orphaned) + ) + ) + ).all() + names = sorted(row[0] for row in codes) + raise HTTPException( + status.HTTP_409_CONFLICT, + f"以下 workspace 将失去唯一 active admin: {names};" + "请先在这些 workspace 中指定其他 admin,再降级该用户", + ) + + if is_self and new_role is not None and new_role.role_code != "admin": + raise HTTPException(status.HTTP_409_CONFLICT, "不能降级自身管理员角色") + + if payload.display_name is not None: + user.display_name = payload.display_name.strip() + if payload.email is not None: + user.email = payload.email.strip() or None + if payload.status is not None: + user.status = payload.status + if new_role is not None: + user.platform_role_id = new_role.role_id + # Workspace role is always inherited from the platform role + # (§7.5/§7.6 cannot change it). Keep workspace_members.role_id + # in sync so downstream reads — `/me` workspaces[].role_code, + # load_active_membership, §7.7 DELETE last-admin guard — + # see the up-to-date role. Without this sync, a user demoted + # from admin → developer would still appear as admin in every + # workspace they belong to until they leave and re-join. + await session.execute( + update(WorkspaceMembers) + .where( + WorkspaceMembers.user_id == user.user_id, + WorkspaceMembers.is_deleted == 0, + ) + .values(role_id=new_role.role_id) + ) + + await session.flush() + await session.refresh(user) + + response_role: Roles | None = None + if user.platform_role_id is not None: + response_role = await session.scalar( + select(Roles).where(Roles.role_id == user.platform_role_id) + ) + return _envelope( + context.request_id, platform_employee_payload(user, response_role), + ) + + +# 软删除平台用户，并级联标记其工作区成员关系为删除。 +@router.delete("/employees/{user_id}") +async def delete_platform_employee( + user_id: str, + context: SystemAdminContext = Depends(system_admin_context), + session: AsyncSession = Depends(database_session), +) -> dict[str, Any]: + """Soft delete a platform user and cascade-soft-delete workspace memberships.""" + user = await session.get(Users, user_id) + if user is None or user.is_deleted != 0: + raise HTTPException(status.HTTP_404_NOT_FOUND, "用户不存在") + + if user_id == context.user.user_id: + raise HTTPException(status.HTTP_409_CONFLICT, "不能删除当前登录账号") + + if user.status == "active" and user.platform_role_id is not None: + current_admin_role = await session.scalar( + select(Roles).where(Roles.role_id == user.platform_role_id) + ) + if current_admin_role is not None and current_admin_role.role_code == "admin": + remaining = await _count_active_system_admins( + session, exclude_user_id=user_id, + ) + if remaining == 0: + raise HTTPException( + status.HTTP_409_CONFLICT, + "platform 必须保留至少一个 active 系统管理员", + ) + + now = datetime.datetime.utcnow() + user.status = "disabled" + user.is_deleted = 1 + user.deleted_at = now + await session.execute( + update(WorkspaceMembers) + .where( + WorkspaceMembers.user_id == user_id, + WorkspaceMembers.is_deleted == 0, + ) + .values(is_deleted=1, deleted_at=now) + ) + await session.flush() + return _envelope( + context.request_id, + {"user_id": user_id, "deleted": True}, + ) + + +class PlatformEmployeePasswordReset(BaseModel): + model_config = ConfigDict(extra="forbid") + + new_password: str = Field(min_length=8, max_length=72) + + +# 系统管理员重置指定员工密码（不需要旧密码）。 +@router.post("/employees/{user_id}/reset-password") +async def reset_platform_employee_password( + user_id: str, + payload: PlatformEmployeePasswordReset, + context: SystemAdminContext = Depends(system_admin_context), + session: AsyncSession = Depends(database_session), +) -> dict[str, Any]: + """Reset a platform employee's password. Caller must be a system admin.""" + user = await session.get(Users, user_id) + if user is None or user.is_deleted != 0: + raise HTTPException(status.HTTP_404_NOT_FOUND, "用户不存在") + + user.password_hash = hash_password(payload.new_password) + await session.flush() + return _envelope( + context.request_id, + {"user_id": user_id, "password_reset": True}, + ) + + diff --git a/backend/src/backend/api/platform/members.py b/backend/src/backend/api/platform/members.py new file mode 100644 index 0000000..7024e9e --- /dev/null +++ b/backend/src/backend/api/platform/members.py @@ -0,0 +1,298 @@ +"""Workspace membership CRUD endpoints. + +``GET .../members`` admits system admins or active workspace members; +write endpoints require ``system_admin_context``. +""" + +from __future__ import annotations + +import datetime +from typing import Any, Literal + +from common.db.models import Roles, Users, WorkspaceMembers +from common.ids import new_ulid +from fastapi import APIRouter, Depends, HTTPException, Request, status +from pydantic import BaseModel, ConfigDict, Field +from sqlalchemy import select +from sqlalchemy.ext.asyncio import AsyncSession + +from backend.api.dependencies import current_user, database_session +from backend.api.platform._deps import ( + SystemAdminContext, + _count_active_admins, + _envelope, + _is_system_admin, + system_admin_context, +) +from backend.api.platform.workspaces import ( + _load_workspace, + member_payload, +) + +router = APIRouter(prefix="/api/v1/platform", tags=["platform"]) + +LIST_PAGE_SIZE = 100 + + +class MemberCreate(BaseModel): + """Add a user to a workspace. Role is inherited from the user's + platform role (Users.platform_role_id) — not set here.""" + + model_config = ConfigDict(extra="forbid") + + user_id: str = Field(min_length=26, max_length=26) + + +class MemberUpdate(BaseModel): + """Update a workspace membership's status. Role cannot be changed + via this endpoint — workspace role is always inherited from the + user's platform role. To change a member's role, PATCH + /platform/employees/{user_id} instead.""" + + model_config = ConfigDict(extra="forbid") + + member_status: Literal["active", "disabled", "locked"] | None = None + + +@router.get("/workspaces/{workspace_id}/members") +async def list_members( + workspace_id: str, + request: Request, + session: AsyncSession = Depends(database_session), +) -> dict[str, Any]: + """List active and historical (non-soft-deleted) members of a workspace. + + Accessible to system admins (any workspace) and to active members of the + workspace itself. The script explorer calls this to seed the per-owner + directory-tree groups for non-admin users; visibility filters on the + scripts/data-resources endpoints still keep each peer's private content + hidden, so this only exposes membership (names), not private files. + """ + user = await current_user(request, session) + is_system_admin = await _is_system_admin(session, user) + if not is_system_admin: + membership = await session.scalar( + select(WorkspaceMembers).where( + WorkspaceMembers.workspace_id == workspace_id, + WorkspaceMembers.user_id == user.user_id, + WorkspaceMembers.is_deleted == 0, + WorkspaceMembers.member_status == "active", + ) + ) + if membership is None: + raise HTTPException( + status.HTTP_403_FORBIDDEN, + "需要系统管理员或该工作区成员权限", + ) + await _load_workspace(session, workspace_id) + rows = ( + await session.execute( + select(Users, Roles, WorkspaceMembers) + .join( + WorkspaceMembers, + WorkspaceMembers.user_id == Users.user_id, + ) + .join(Roles, Roles.role_id == WorkspaceMembers.role_id) + .where( + WorkspaceMembers.workspace_id == workspace_id, + WorkspaceMembers.is_deleted == 0, + ) + .order_by(WorkspaceMembers.joined_at, Users.user_id) + .limit(LIST_PAGE_SIZE) + ) + ).all() + request_id = request.headers.get("X-Request-ID") or new_ulid() + return _envelope( + request_id, + [member_payload(u, r, m) for u, r, m in rows], + {"count": len(rows), "page_size": LIST_PAGE_SIZE}, + ) + +@router.post( + "/workspaces/{workspace_id}/members", + status_code=status.HTTP_201_CREATED, +) +async def add_member( + workspace_id: str, + payload: MemberCreate, + context: SystemAdminContext = Depends(system_admin_context), + session: AsyncSession = Depends(database_session), +) -> dict[str, Any]: + """Add a user to a workspace. The new row starts with member_status='active'. + + The role is inherited from the target user's ``platform_role_id``; + the request body does NOT take a ``role_code``. To change a member's + role, PATCH ``/api/v1/platform/employees/{user_id}`` instead. + """ + await _load_workspace(session, workspace_id) + user = await session.get(Users, payload.user_id) + if user is None or user.is_deleted != 0: + raise HTTPException(status.HTTP_404_NOT_FOUND, "用户不存在") + if user.status != "active": + raise HTTPException( + status.HTTP_409_CONFLICT, + f"用户状态为 {user.status},无法加入 workspace", + ) + if user.platform_role_id is None: + raise HTTPException( + status.HTTP_409_CONFLICT, + "目标用户尚未分配平台角色,无法加入 workspace;" + "请先 PATCH /api/v1/platform/employees/{user_id} 设置 role_code", + ) + role = await session.scalar( + select(Roles).where( + Roles.role_id == user.platform_role_id, + Roles.is_deleted == 0, + ) + ) + if role is None: + raise HTTPException( + status.HTTP_422_UNPROCESSABLE_ENTITY, + "用户的平台角色行不存在或已被删除", + ) + # ``WorkspaceMembers`` 的主键是 ``(workspace_id, user_id)`` 复合 PK, + # 而 ``remove_member`` / ``delete_platform_employee`` 都是软删除 (保留行, + # 仅置 ``is_deleted=1``). 因此这里必须按主键查整行,而不是只看活跃行: + # 否则软删行会被 active-duplicate 检查漏过,然后 INSERT 直接撞 PK. + existing = await session.scalar( + select(WorkspaceMembers).where( + WorkspaceMembers.workspace_id == workspace_id, + WorkspaceMembers.user_id == payload.user_id, + ) + ) + if existing is not None: + if existing.is_deleted == 0: + raise HTTPException( + status.HTTP_409_CONFLICT, + "用户已是该 workspace 成员;workspace 角色继承自平台角色," + "要变更请 PATCH /api/v1/platform/employees/{user_id} 修改 role_code", + ) + # 复活软删除行. 保留 ``joined_at`` 作为历史记录;``role_id`` 重新继承 + # 当前用户的平台角色 (用户在中间可能改过 platform_role);清掉 + # ``deleted_at`` 标记本轮已不在软删状态. + existing.is_deleted = 0 + existing.deleted_at = None + existing.role_id = role.role_id + existing.member_status = "active" + await session.flush() + await session.refresh(existing) + return _envelope( + context.request_id, member_payload(user, role, existing), + ) + membership = WorkspaceMembers( + workspace_id=workspace_id, + user_id=payload.user_id, + role_id=role.role_id, + member_status="active", + ) + session.add(membership) + await session.flush() + await session.refresh(membership) + return _envelope(context.request_id, member_payload(user, role, membership)) + +@router.patch("/workspaces/{workspace_id}/members/{user_id}") +async def update_member( + workspace_id: str, + user_id: str, + payload: MemberUpdate, + context: SystemAdminContext = Depends(system_admin_context), + session: AsyncSession = Depends(database_session), +) -> dict[str, Any]: + """Update a workspace membership's status. Role is not editable here. + + Workspace role is always inherited from the user's platform role + (``Users.platform_role_id``). To change role, PATCH + ``/api/v1/platform/employees/{user_id}`` instead. + + Last-admin guard still applies to ``member_status`` changes: setting + the only active admin to ``disabled``/``locked`` would leave the + workspace without admin coverage. + """ + await _load_workspace(session, workspace_id) + row = ( + await session.execute( + select(Users, Roles, WorkspaceMembers) + .join( + WorkspaceMembers, + WorkspaceMembers.user_id == Users.user_id, + ) + .join(Roles, Roles.role_id == WorkspaceMembers.role_id) + .where( + WorkspaceMembers.workspace_id == workspace_id, + WorkspaceMembers.user_id == user_id, + WorkspaceMembers.is_deleted == 0, + ) + ) + ).first() + if row is None: + raise HTTPException(status.HTTP_404_NOT_FOUND, "成员不存在") + user, role, membership = row + + if payload.member_status is not None and payload.member_status != membership.member_status: + if ( + role.role_code == "admin" + and payload.member_status != "active" + ): + remaining = await _count_active_admins( + session, workspace_id, exclude_user_id=user_id, + ) + if remaining == 0: + raise HTTPException( + status.HTTP_409_CONFLICT, + "workspace 必须保留至少一个 admin", + ) + membership.member_status = payload.member_status + + await session.flush() + await session.refresh(membership) + return _envelope(context.request_id, member_payload(user, role, membership)) + +@router.delete("/workspaces/{workspace_id}/members/{user_id}") +async def remove_member( + workspace_id: str, + user_id: str, + context: SystemAdminContext = Depends(system_admin_context), + session: AsyncSession = Depends(database_session), +) -> dict[str, Any]: + """Soft-delete a workspace membership. + + System admins cannot remove themselves — the only escape is to delete + the entire workspace, which cascades membership soft-deletion. + """ + await _load_workspace(session, workspace_id) + if user_id == context.user.user_id: + raise HTTPException( + status.HTTP_403_FORBIDDEN, + "系统管理员不能把自己从 workspace 移除;如需退出,请删除整个 workspace", + ) + row = ( + await session.execute( + select(Roles, WorkspaceMembers) + .join(Roles, Roles.role_id == WorkspaceMembers.role_id) + .where( + WorkspaceMembers.workspace_id == workspace_id, + WorkspaceMembers.user_id == user_id, + WorkspaceMembers.is_deleted == 0, + ) + ) + ).first() + if row is None: + raise HTTPException(status.HTTP_404_NOT_FOUND, "成员不存在") + role, membership = row + if role.role_code == "admin" and membership.member_status == "active": + remaining = await _count_active_admins( + session, workspace_id, exclude_user_id=user_id, + ) + if remaining == 0: + raise HTTPException( + status.HTTP_409_CONFLICT, + "workspace 必须保留至少一个 admin", + ) + membership.is_deleted = 1 + membership.deleted_at = datetime.datetime.utcnow() + await session.flush() + return _envelope( + context.request_id, + {"workspace_id": workspace_id, "user_id": user_id, "removed": True}, + ) + diff --git a/backend/src/backend/api/platform/roles.py b/backend/src/backend/api/platform/roles.py new file mode 100644 index 0000000..3891da5 --- /dev/null +++ b/backend/src/backend/api/platform/roles.py @@ -0,0 +1,364 @@ +"""Platform role & permission management endpoints. +Seven endpoints: role list/CRUD, per-role permission get/patch, and the permission +catalog. The shared permission-set writer lives in :mod:`backend.api.platform._permission_set`. +""" + +from __future__ import annotations + +import datetime +import re +from typing import Any + +from common.db.models import ( + Permissions, + Roles, + Users, + WorkspaceMembers, +) +from common.ids import new_ulid +from fastapi import APIRouter, Depends, HTTPException, status +from pydantic import BaseModel, ConfigDict, Field, field_validator +from sqlalchemy import func, select +from sqlalchemy.ext.asyncio import AsyncSession + +from backend.api.dependencies import database_session +from backend.api.platform._deps import ( + SystemAdminContext, + _envelope, + system_admin_context, +) + +from backend.api.platform._permission_set import ( + _apply_role_permission_set, + _load_role_permission_codes, +) + +router = APIRouter(prefix="/api/v1/platform", tags=["platform"]) + +RESERVED_PLATFORM_ROLE_CODES = frozenset({"admin", "developer"}) +ROLE_CODE_PATTERN = re.compile(r"^[a-z][a-z0-9_-]{1,63}$") + +class RolePermissionsPatch(BaseModel): + """Replace a platform role's permission set wholesale. + + Empty list is allowed (revokes all permissions) for non-`admin` + roles. The PATCH endpoint rejects emptying an `admin` role of its + system.* permissions; see ``patch_role_permissions`` for the + load-bearing guard order. + """ + + model_config = ConfigDict(extra="forbid") + + permission_codes: list[str] = Field(default_factory=list, max_length=64) + +class RoleCreate(BaseModel): + """Create a platform role. + + ``role_code`` must not collide with the reserved built-in codes + (``admin`` / ``developer``); the validator rejects those with 422. + ``permission_codes`` is optional — an empty list means the role + starts with no menu permissions. + """ + + model_config = ConfigDict(extra="forbid") + + role_code: str = Field(min_length=2, max_length=64) + role_name: str = Field(min_length=1, max_length=100) + description: str | None = Field(default=None, max_length=500) + permission_codes: list[str] = Field(default_factory=list, max_length=64) + + @field_validator("role_code") + @classmethod + def _validate_role_code(cls, value: str) -> str: + if value in RESERVED_PLATFORM_ROLE_CODES: + raise ValueError(f"role_code {value!r} 是预留字") + if not ROLE_CODE_PATTERN.match(value): + raise ValueError("role_code 必须以小写字母开头,仅含小写字母/数字/下划线/连字符") + return value + +class RoleUpdate(BaseModel): + """Update a platform role's ``role_name`` / ``description`` only. + + ``role_code`` is intentionally absent — it is the URL key and is + backed by a unique index; ``extra="forbid"`` rejects any attempt to + send it. ``description`` accepts an explicit ``null`` (clear the + value); omitting it leaves it unchanged. + """ + + model_config = ConfigDict(extra="forbid") + + role_name: str | None = Field(default=None, min_length=1, max_length=100) + description: str | None = Field(default=None, max_length=500) + +async def _load_platform_role_by_code( + session: AsyncSession, role_code: str +) -> Roles: + """Load a platform-scoped role by code; 404 if missing or not platform-scope.""" + role = await session.scalar( + select(Roles).where( + Roles.role_code == role_code, Roles.is_deleted == 0, + ) + ) + if role is None or role.role_scope != "platform": + raise HTTPException( + status.HTTP_404_NOT_FOUND, f"platform 角色 {role_code} 不存在", + ) + return role + +def _role_payload(role: Roles, permission_codes: list[str]) -> dict[str, Any]: + return { + "role_id": role.role_id, + "role_code": role.role_code, + "role_name": role.role_name, + "is_builtin": bool(role.is_builtin), + "permission_codes": permission_codes, + # 加法字段：POST/PATCH/GET 都自动带上 description，便于前端展示。 + "description": role.description, + } + +@router.get("/roles") +async def list_platform_roles( + context: SystemAdminContext = Depends(system_admin_context), + session: AsyncSession = Depends(database_session), +) -> dict[str, Any]: + """List every platform-scoped role with its current permission_codes.""" + roles = ( + await session.scalars( + select(Roles) + .where(Roles.role_scope == "platform", Roles.is_deleted == 0) + .order_by(Roles.role_code) + ) + ).all() + payload = [] + for role in roles: + codes = await _load_role_permission_codes(session, role.role_id) + payload.append(_role_payload(role, codes)) + return _envelope( + context.request_id, payload, {"count": len(payload)}, + ) + +@router.get("/roles/{role_code}/permissions") +async def get_role_permissions( + role_code: str, + context: SystemAdminContext = Depends(system_admin_context), + session: AsyncSession = Depends(database_session), +) -> dict[str, Any]: + """Return one platform role's permission_codes.""" + role = await _load_platform_role_by_code(session, role_code) + codes = await _load_role_permission_codes(session, role.role_id) + return _envelope( + context.request_id, _role_payload(role, codes), + ) + +@router.patch("/roles/{role_code}/permissions") +async def patch_role_permissions( + role_code: str, + payload: RolePermissionsPatch, + context: SystemAdminContext = Depends(system_admin_context), + session: AsyncSession = Depends(database_session), +) -> dict[str, Any]: + """Replace a platform role's permission set wholesale. + + The guard order and the diff-based write live in + :func:`_apply_role_permission_set`, shared with POST /roles so both + entry points validate identically. + """ + role = await _load_platform_role_by_code(session, role_code) + final_codes = await _apply_role_permission_set( + session, role, payload.permission_codes, + ) + return _envelope( + context.request_id, _role_payload(role, final_codes), + ) + +@router.post("/roles", status_code=status.HTTP_201_CREATED) +async def create_platform_role( + payload: RoleCreate, + context: SystemAdminContext = Depends(system_admin_context), + session: AsyncSession = Depends(database_session), +) -> dict[str, Any]: + """Create a platform role. + + - ``role_code`` validated by the schema (reserved codes + pattern → + 422) and must not collide with ``uk_roles_code`` → 409 (incl. + soft-deleted rows that still occupy the index slot). + - Always ``role_scope='platform'``, ``is_builtin=0``. + - Optional ``permission_codes``: empty → no menu permissions; non-empty + applies the same guard order as PATCH via + :func:`_apply_role_permission_set`. + """ + role_code = payload.role_code + duplicate = await session.scalar( + select(Roles.role_id).where(Roles.role_code == role_code) + ) + if duplicate is not None: + raise HTTPException( + status.HTTP_409_CONFLICT, f"role_code {role_code!r} 已存在", + ) + + role = Roles( + role_id=new_ulid(), + role_code=role_code, + role_name=payload.role_name.strip(), + role_scope="platform", + is_builtin=0, + description=payload.description, + ) + session.add(role) + await session.flush() + + final_codes: list[str] = [] + if payload.permission_codes: + final_codes = await _apply_role_permission_set( + session, role, payload.permission_codes, + ) + + await session.refresh(role) + return _envelope( + context.request_id, _role_payload(role, final_codes), + ) + +@router.patch("/roles/{role_code}") +async def update_platform_role( + role_code: str, + payload: RoleUpdate, + context: SystemAdminContext = Depends(system_admin_context), + session: AsyncSession = Depends(database_session), +) -> dict[str, Any]: + """Update a platform role's ``role_name`` / ``description``. + + - ``role_code`` cannot change: it is the URL key (unique + ``uk_roles_code`` index) and the schema forbids sending it → 422. + - Built-in roles (``is_builtin=1``) keep ``role_name`` locked → 409; + ``description`` may still be edited. + - ``description`` accepts an explicit ``null`` (clear); omission + leaves it unchanged. + - ``updated_at`` refreshed by the DB's ``ON UPDATE + CURRENT_TIMESTAMP(3)`` server default. + """ + role = await _load_platform_role_by_code(session, role_code) + + if ( + role.is_builtin == 1 + and payload.role_name is not None + and payload.role_name != role.role_name + ): + raise HTTPException( + status.HTTP_409_CONFLICT, + "内置角色 role_name 不可改", + ) + + if payload.role_name is not None: + role.role_name = payload.role_name.strip() + # 显式传 null → 清空 description;字段省略 → 不改。 + # 用 model_fields_set 区分 "省略" 与 "显式 null",因为两者在 Pydantic + # 里都解析为 None。 + if "description" in payload.model_fields_set: + role.description = payload.description + + await session.flush() + await session.refresh(role) + codes = await _load_role_permission_codes(session, role.role_id) + return _envelope( + context.request_id, _role_payload(role, codes), + ) + +@router.delete("/roles/{role_code}") +async def delete_platform_role( + role_code: str, + context: SystemAdminContext = Depends(system_admin_context), + session: AsyncSession = Depends(database_session), +) -> dict[str, Any]: + """Soft-delete a platform role. + + Guards: + - Built-in roles (``is_builtin=1``) → 409. + - Active ``users.platform_role_id`` reference → 409 (migrate users first). + - Active ``workspace_members.role_id`` reference → 409 (same ``Roles`` + row — a hard orphan would leave it dangling). + """ + role = await _load_platform_role_by_code(session, role_code) + + if role.is_builtin == 1: + raise HTTPException( + status.HTTP_409_CONFLICT, "内置角色不可删除", + ) + + user_refs = int( + await session.scalar( + select(func.count()) + .select_from(Users) + .where( + Users.platform_role_id == role.role_id, + Users.is_deleted == 0, + ) + ) + or 0 + ) + if user_refs > 0: + raise HTTPException( + status.HTTP_409_CONFLICT, + f"仍有 {user_refs} 个用户引用此角色", + ) + + member_refs = int( + await session.scalar( + select(func.count()) + .select_from(WorkspaceMembers) + .where( + WorkspaceMembers.role_id == role.role_id, + WorkspaceMembers.is_deleted == 0, + ) + ) + or 0 + ) + if member_refs > 0: + raise HTTPException( + status.HTTP_409_CONFLICT, + f"仍有 {member_refs} 个 workspace 成员引用此角色", + ) + + role.is_deleted = 1 + role.deleted_at = datetime.datetime.utcnow() + await session.flush() + return _envelope( + context.request_id, + {"role_code": role_code, "deleted": True}, + ) + +@router.get("/permissions") +async def list_platform_permissions( + context: SystemAdminContext = Depends(system_admin_context), + session: AsyncSession = Depends(database_session), +) -> dict[str, Any]: + """List every active permission, ordered by module_code + permission_code. + + Returns ``[{permission_code, permission_name, module_code, + description}, ...]`` for the role-management UI. Includes + ``system:role:view`` which the admin role holds by default. + """ + rows = ( + await session.execute( + select( + Permissions.permission_code, + Permissions.permission_name, + Permissions.module_code, + Permissions.description, + ) + .where(Permissions.is_deleted == 0) + .order_by(Permissions.module_code, Permissions.permission_code) + ) + ).all() + payload = [ + { + "permission_code": code, + "permission_name": name, + "module_code": module, + "description": description, + } + for code, name, module, description in rows + ] + return _envelope( + context.request_id, payload, {"count": len(payload)}, + ) + diff --git a/backend/src/backend/api/platform/workspaces.py b/backend/src/backend/api/platform/workspaces.py new file mode 100644 index 0000000..ce0ebc0 --- /dev/null +++ b/backend/src/backend/api/platform/workspaces.py @@ -0,0 +1,274 @@ +"""Workspace CRUD endpoints. + +Membership endpoints live in ``members.py``. Soft-delete cascades for +workspace DELETE still live here. +""" + +from __future__ import annotations + +import datetime +import re +from typing import Any, Literal + +from common.db.models import Roles, Users, WorkspaceMembers, Workspaces +from common.ids import new_ulid +from fastapi import APIRouter, Depends, HTTPException, Query, status +from pydantic import BaseModel, ConfigDict, Field +from sqlalchemy import func, or_, select, tuple_, update +from sqlalchemy.ext.asyncio import AsyncSession + +from backend.api.dependencies import database_session +from backend.api.platform._deps import ( + SystemAdminContext, + _envelope, + _load_role_by_code, + system_admin_context, +) +from backend.api.platform._pagination import ( + DEFAULT_PAGE_LIMIT, + MAX_PAGE_LIMIT, + decode_cursor, + encode_cursor, + page_meta, +) + +router = APIRouter(prefix="/api/v1/platform", tags=["platform"]) + +WORKSPACE_CODE_PATTERN = re.compile(r"^[a-z0-9-]{3,32}$") + +# 创建工作区时前端提交的请求体；禁止未声明字段。 +class WorkspaceCreate(BaseModel): + model_config = ConfigDict(extra="forbid") + + workspace_code: str = Field(min_length=3, max_length=32) + workspace_name: str = Field(min_length=1, max_length=150) + quota_bytes: int = Field(default=0, ge=0) + description: str | None = Field(default=None, max_length=1000) + +# 编辑工作区时允许修改的字段；禁用操作必须走删除接口而不是直接传状态。 +class WorkspaceUpdate(BaseModel): + model_config = ConfigDict(extra="forbid") + + workspace_name: str | None = Field(default=None, min_length=1, max_length=150) + quota_bytes: int | None = Field(default=None, ge=0) + description: str | None = Field(default=None, max_length=1000) + # 'disabled' is rejected here on purpose — soft delete must go through DELETE. + status: Literal["active", "archived"] | None = None + +def workspace_payload(workspace: Workspaces) -> dict[str, Any]: + return { + "workspace_id": workspace.workspace_id, + "workspace_code": workspace.workspace_code, + "workspace_name": workspace.workspace_name, + "active_root_uri": workspace.active_root_uri, + "quota_bytes": workspace.quota_bytes, + "status": workspace.status, + "description": workspace.description, + "created_by": workspace.created_by, + "created_at": workspace.created_at.isoformat(), + "updated_at": ( + workspace.updated_at.isoformat() if workspace.updated_at else None + ), + } + +def member_payload( + user: Users, + role: Roles, + membership: WorkspaceMembers, +) -> dict[str, Any]: + return { + "user_id": user.user_id, + "username": user.username, + "display_name": user.display_name, + "email": user.email, + "user_status": user.status, + "role_code": role.role_code, + "role_name": role.role_name, + "member_status": membership.member_status, + "joined_at": membership.joined_at.isoformat(), + } + +async def _load_workspace(session: AsyncSession, workspace_id: str) -> Workspaces: + workspace = await session.get(Workspaces, workspace_id) + if workspace is None: + raise HTTPException(status.HTTP_404_NOT_FOUND, "workspace 不存在") + return workspace + +@router.get("/workspaces") +async def list_workspaces( + limit: int = Query(default=DEFAULT_PAGE_LIMIT, ge=1, le=MAX_PAGE_LIMIT), + cursor: str | None = Query(default=None), + q: str | None = Query(default=None, max_length=100), + context: SystemAdminContext = Depends(system_admin_context), + session: AsyncSession = Depends(database_session), +) -> dict[str, Any]: + """List active/archived workspaces with cursor pagination + search.""" + base_filters = [ + Workspaces.status != "disabled", + Workspaces.is_deleted == 0, + ] + keyword = (q or "").strip() + if keyword: + like = f"%{keyword}%" + base_filters.append( + or_( + Workspaces.workspace_name.like(like), + Workspaces.workspace_code.like(like), + Workspaces.description.like(like), + ) + ) + + total_count = int( + await session.scalar( + select(func.count()).select_from(Workspaces).where(*base_filters) + ) + or 0 + ) + + page_filters = list(base_filters) + if cursor is not None: + cursor_ts, cursor_id = decode_cursor(cursor) + page_filters.append( + tuple_(Workspaces.created_at, Workspaces.workspace_id) + > (cursor_ts, cursor_id) + ) + + rows = ( + await session.execute( + select(Workspaces) + .where(*page_filters) + .order_by(Workspaces.created_at, Workspaces.workspace_id) + .limit(limit + 1) + ) + ).scalars().all() + has_more = len(rows) > limit + page_rows = list(rows[:limit]) + next_cursor = None + if has_more and page_rows: + last = page_rows[-1] + next_cursor = encode_cursor(last.created_at, last.workspace_id) + return _envelope( + context.request_id, + [workspace_payload(w) for w in page_rows], + page_meta( + limit=limit, + page_count=len(page_rows), + total_count=total_count, + next_cursor=next_cursor, + ), + ) + +@router.post("/workspaces", status_code=status.HTTP_201_CREATED) +async def create_workspace( + payload: WorkspaceCreate, + context: SystemAdminContext = Depends(system_admin_context), + session: AsyncSession = Depends(database_session), +) -> dict[str, Any]: + """Create a workspace and auto-join the creator as an admin member.""" + if not WORKSPACE_CODE_PATTERN.fullmatch(payload.workspace_code): + raise HTTPException( + status.HTTP_422_UNPROCESSABLE_ENTITY, + "workspace_code 必须匹配 ^[a-z0-9-]{3,32}$", + ) + duplicate = await session.scalar( + select(Workspaces.workspace_id).where( + Workspaces.workspace_code == payload.workspace_code, + ) + ) + if duplicate is not None: + raise HTTPException(status.HTTP_409_CONFLICT, "workspace_code 已存在") + + admin_role = await _load_role_by_code(session, "admin") + workspace_id = new_ulid() + workspace = Workspaces( + workspace_id=workspace_id, + workspace_code=payload.workspace_code, + workspace_name=payload.workspace_name, + active_root_uri=f"s3://workspaces/{workspace_id}/", + quota_bytes=payload.quota_bytes, + status="active", + created_by=context.user.user_id, + description=payload.description, + ) + session.add(workspace) + session.add( + WorkspaceMembers( + workspace_id=workspace_id, + user_id=context.user.user_id, + role_id=admin_role.role_id, + member_status="active", + ) + ) + await session.flush() + await session.refresh(workspace) + return _envelope(context.request_id, workspace_payload(workspace)) + +@router.get("/workspaces/{workspace_id}") +async def get_workspace( + workspace_id: str, + context: SystemAdminContext = Depends(system_admin_context), + session: AsyncSession = Depends(database_session), +) -> dict[str, Any]: + """Fetch a single workspace — even soft-deleted ones are reachable.""" + workspace = await _load_workspace(session, workspace_id) + return _envelope(context.request_id, workspace_payload(workspace)) + +@router.patch("/workspaces/{workspace_id}") +async def update_workspace( + workspace_id: str, + payload: WorkspaceUpdate, + context: SystemAdminContext = Depends(system_admin_context), + session: AsyncSession = Depends(database_session), +) -> dict[str, Any]: + """Patch editable workspace fields. ``status='disabled'`` is rejected.""" + workspace = await _load_workspace(session, workspace_id) + if workspace.status == "disabled": + raise HTTPException( + status.HTTP_409_CONFLICT, + "workspace 已删除,无法修改", + ) + if payload.workspace_name is not None: + workspace.workspace_name = payload.workspace_name.strip() + if payload.quota_bytes is not None: + workspace.quota_bytes = payload.quota_bytes + if payload.description is not None: + workspace.description = payload.description + if payload.status is not None: + workspace.status = payload.status + await session.flush() + await session.refresh(workspace) + return _envelope(context.request_id, workspace_payload(workspace)) + +@router.delete("/workspaces/{workspace_id}") +async def delete_workspace( + workspace_id: str, + context: SystemAdminContext = Depends(system_admin_context), + session: AsyncSession = Depends(database_session), +) -> dict[str, Any]: + """Soft-delete a workspace and cascade-soft-delete its memberships. + + Allowed from any non-disabled status (active or archived). The + membership cascade is what lets system admins leave a workspace — + there is no per-member DELETE escape for self-removal. + """ + workspace = await _load_workspace(session, workspace_id) + if workspace.status == "disabled": + raise HTTPException( + status.HTTP_409_CONFLICT, + "workspace 已被删除", + ) + now = datetime.datetime.utcnow() + workspace.status = "disabled" + workspace.is_deleted = 1 + workspace.deleted_at = now + await session.execute( + update(WorkspaceMembers) + .where( + WorkspaceMembers.workspace_id == workspace_id, + WorkspaceMembers.is_deleted == 0, + ) + .values(is_deleted=1, deleted_at=now) + ) + await session.flush() + await session.refresh(workspace) + return _envelope(context.request_id, workspace_payload(workspace)) diff --git a/backend/src/backend/resources.py b/backend/src/backend/api/resources.py similarity index 67% rename from backend/src/backend/resources.py rename to backend/src/backend/api/resources.py index bd4baf1..a1fef77 100644 --- a/backend/src/backend/resources.py +++ b/backend/src/backend/api/resources.py @@ -7,36 +7,39 @@ from __future__ import annotations -import os from datetime import UTC, datetime -from pathlib import Path, PurePosixPath from typing import Any -from common.db.models import DataResources, StorageObjects +from common.db.models import DataResources, StorageObjects, Users from common.ids import new_ulid -from common.storage import workspaces_root from common.storage.schemas import ( CreateUploadRequest, - DownloadUrlRequest, ) from fastapi import APIRouter, Depends, Header, HTTPException, Query, Request, status -from sqlalchemy import func, select +from sqlalchemy import func, or_, select from sqlalchemy.ext.asyncio import AsyncSession -from backend.dependencies import ( +from backend.api.dependencies import ( RequestContext, database_session, request_context, ) -from backend.schemas import ( +from backend.api._resources_common import ( + build_list_resources_descendant_prefix as _build_list_resources_descendant_prefix, + can_view, + compute_jupyter_relative_path, + get_visible_resource, + resource_directory, + resource_payload, +) +from backend.api.scripts import _escape_like_pattern +from backend.schemas.resources import ( CompleteResourceUploadRequest, CreateResourceUploadRequest, - DownloadUrlRequest, ResourceRelativePathRequest, ) from backend.services.storage import ( acquire_named_lock, - create_download_url_payload, create_upload_record, release_named_lock, soft_delete_object, @@ -46,101 +49,6 @@ from backend.services.storage import ( router = APIRouter(prefix="/api/v1/data-resources", tags=["data-resources"]) -def compute_jupyter_relative_path(script_path: str, resource_relative: str) -> str: - """从当前脚本所在目录算到资源文件的 Jupyter 相对路径。 - - script_path / resource_relative 都是相对于 user root_dir 的 POSIX 路径。 - """ - script_dir = PurePosixPath(script_path).parent.as_posix() - if not script_dir or script_dir == ".": - return resource_relative - return os.path.relpath(resource_relative, start=script_dir) - - -def resource_directory( - object_key: str, - workspace_id: str, - owner_user_id: str, -) -> str: - """从 object_key 解析资源所在目录(相对于用户根目录,根目录返回 "")。 - - object_key 形如 ``{ws_id}/{user_id}/{target_path}/{file_name}``; - 不匹配该前缀的键(如无 ws/user 前缀的旧数据)统一视为根目录。 - """ - prefix = f"{workspace_id}/{owner_user_id}/" - if not object_key.startswith(prefix): - return "" - tail = object_key[len(prefix):] - directory, _, _ = tail.rpartition("/") - return directory - - -def resource_payload( - resource: DataResources, - storage_object: StorageObjects, -) -> dict[str, Any]: - # ``object_key`` now follows ``{ws_id}/{user_id}/{target_path}/{file_name}`` - # (target_path may be empty). Legacy objects still live under - # ``.resources/{file_name}`` and must remain readable. Both shapes share - # the same derivation: strip the workspace/user prefix and use the rest - # as the Jupyter-relative path. - workspace_prefix = f"{resource.workspace_id}/" - user_prefix = f"{resource.owner_user_id}/" - jupyter_accessible_path = "" - absolute_path = "" - if storage_object.object_key and storage_object.object_key.startswith( - workspace_prefix - ): - remainder = storage_object.object_key[len(workspace_prefix):] - if remainder.startswith(user_prefix): - tail = remainder[len(user_prefix):] - jupyter_accessible_path = tail - absolute_path = ( - workspaces_root() - / resource.workspace_id - / resource.owner_user_id - / Path(tail) - ).as_posix() - else: - jupyter_accessible_path = remainder - elif storage_object.object_key: - jupyter_accessible_path = storage_object.object_key - return { - "resource_id": resource.resource_id, - "workspace_id": resource.workspace_id, - "storage_object_id": resource.storage_object_id, - "owner_user_id": resource.owner_user_id, - "resource_name": resource.resource_name, - "description": resource.description, - "visibility": resource.visibility, - "status": resource.status, - "created_at": resource.created_at.isoformat(), - "updated_at": resource.updated_at.isoformat(), - "file": { - "file_name": storage_object.file_name, - "file_extension": storage_object.file_extension, - "mime_type": storage_object.mime_type, - "size_bytes": storage_object.size_bytes, - "content_hash": storage_object.content_hash, - "object_status": storage_object.object_status, - }, - "jupyter_accessible_path": jupyter_accessible_path, - "absolute_path": absolute_path, - } - - -def can_view(resource: DataResources, context: RequestContext) -> bool: - # 2026-08-11: 临时取消"用户间目录互相不可见"约束 - # 同一 workspace 内的成员现在可以查看彼此的 private 资源。 - # 还原: 取消下方注释,恢复 owner_user_id 检查。 - return ( - # resource.owner_user_id == context.user.user_id - # or - resource.visibility in {"workspace", "public"} - or context.is_admin - ) - - # 根据当前脚本位置计算资源的相对路径，便于 Notebook 中用相对路径读取文件。 @router.post("/{resource_id}/jupyter-relative-path") async def resource_jupyter_relative_path( @@ -363,18 +271,21 @@ async def bind_resource( # 列出当前工作区可见的数据资源，可按可见性或关键字筛选。 @router.get("") async def list_resources( + parent_path: str = Query(default="", max_length=1024), + owner_user_id: str | None = Query(default=None, max_length=64), context: RequestContext = Depends(request_context), session: AsyncSession = Depends(database_session), visibility: str | None = Query(default=None), keyword: str | None = Query(default=None, max_length=100), ) -> dict[str, Any]: statement = ( - select(DataResources, StorageObjects) + select(DataResources, StorageObjects, Users.display_name) .join( StorageObjects, StorageObjects.storage_object_id == DataResources.storage_object_id, ) + .outerjoin(Users, Users.user_id == DataResources.owner_user_id) .where( DataResources.workspace_id == context.workspace.workspace_id, DataResources.status == "active", @@ -382,16 +293,40 @@ async def list_resources( ) .order_by(DataResources.updated_at.desc()) ) - # 2026-08-11: 临时取消"用户间目录互相不可见"约束 - # 列表接口现在返回 workspace 内全部 active 资源(不再按 owner / visibility 过滤)。 - # 还原: 删除下面这段注释,恢复原来的 if not context.is_admin: ... 块。 - # if not context.is_admin: - # statement = statement.where( - # or_( - # DataResources.owner_user_id == context.user.user_id, - # DataResources.visibility.in_(["workspace", "public"]), - # ) - # ) + # ``parent_path`` scopes to DIRECT children of that jupyter path + # (matching /api/v1/scripts). Per-owner listing: default (no + # owner_user_id) scopes to the requester's own object_key subtree + # (``{ws_id}/{me}/...``); passing owner_user_id scopes to that owner's + # subtree so the tree can lazily fetch another member's data resources + # on group expand. The parent's ``_`` / ``%`` are escaped so sibling + # folders (e.g. ``fooXbar``) don't leak. Empty parent_path still applies + # the filter: it resolves to that owner's root-level direct children + # (``{ws_id}/{owner}/%`` and NOT ``{ws_id}/{owner}/%/%``), symmetric with + # list_scripts. Skipping the filter for empty input would silently + # surface nested descendants and break the directory tree. + target_owner = owner_user_id or context.user.user_id + owner_prefix = f"{context.workspace.workspace_id}/{target_owner}" + descendant_prefix = _build_list_resources_descendant_prefix(parent_path) + statement = statement.where( + StorageObjects.object_key.like( + f"{owner_prefix}/{descendant_prefix}%", + escape="\\", + ), + ~StorageObjects.object_key.like( + f"{owner_prefix}/{descendant_prefix}%/%", + escape="\\", + ), + ) + # 只返回 owner 自己的资源（含 private），或 visibility 为 + # workspace/public 的其他成员资源；A 的 private 资源对非 owner 不可见。 + # admin 跳过过滤，全部可见。 + if not context.is_admin: + statement = statement.where( + or_( + DataResources.owner_user_id == context.user.user_id, + DataResources.visibility.in_(["workspace", "public"]), + ) + ) if visibility: if visibility not in {"private", "workspace", "public"}: raise HTTPException( @@ -400,45 +335,25 @@ async def list_resources( ) statement = statement.where(DataResources.visibility == visibility) if keyword: + # Escape LIKE metacharacters so a search like "100%" or "my_file" + # doesn't act as a wildcard. The outer "%...%" wildcards stay raw. + escaped = _escape_like_pattern(keyword.strip()) statement = statement.where( - DataResources.resource_name.like(f"%{keyword.strip()}%") + DataResources.resource_name.like(f"%{escaped}%", escape="\\") ) rows = (await session.execute(statement)).all() return { "request_id": context.request_id, "data": [ - resource_payload(resource, storage_object) - for resource, storage_object in rows + resource_payload( + resource, storage_object, owner_display_name=owner_display_name + ) + for resource, storage_object, owner_display_name in rows ], "meta": {"count": len(rows)}, } -async def get_visible_resource( - resource_id: str, - context: RequestContext, - session: AsyncSession, -) -> tuple[DataResources, StorageObjects]: - row = ( - await session.execute( - select(DataResources, StorageObjects) - .join( - StorageObjects, - StorageObjects.storage_object_id - == DataResources.storage_object_id, - ) - .where( - DataResources.resource_id == resource_id, - DataResources.workspace_id - == context.workspace.workspace_id, - DataResources.status == "active", - ) - ) - ).one_or_none() - if row is None or not can_view(row[0], context): - raise HTTPException(status.HTTP_404_NOT_FOUND, "resource not found") - return row - # 查询单个数据资源的元数据与其关联文件信息。 @router.get("/{resource_id}") @@ -459,28 +374,6 @@ async def get_resource( } -# 为资源文件生成带时效的下载链接。 -@router.post("/{resource_id}/download-url") -async def resource_download_url( - resource_id: str, - payload: DownloadUrlRequest, - request: Request, - context: RequestContext = Depends(request_context), - session: AsyncSession = Depends(database_session), -) -> dict[str, Any]: - resource, _ = await get_visible_resource( - resource_id, - context, - session, - ) - data = await create_download_url_payload( - await session.get(StorageObjects, resource.storage_object_id), - DownloadUrlRequest(expires_seconds=payload.expires_seconds), - request, - ) - return {"request_id": context.request_id, "data": data["data"], "meta": {}} - - # 软删除数据资源及其关联对象，遵循存储层的回收站策略。 @router.delete("/{resource_id}") async def delete_resource( diff --git a/backend/src/backend/api/resources_content.py b/backend/src/backend/api/resources_content.py new file mode 100644 index 0000000..7789a28 --- /dev/null +++ b/backend/src/backend/api/resources_content.py @@ -0,0 +1,231 @@ +"""数据资源下载、同源内容流与表格抽样预览接口。 + +与 ``resources.py`` 共用前缀 ``/api/v1/data-resources``，在 ``main`` 中并列挂载。 +""" + +from __future__ import annotations + +import csv +import io +from typing import Any +from urllib.parse import quote + +from common.db.models import StorageObjects +from fastapi import APIRouter, Depends, HTTPException, Query, Request, status +from fastapi.responses import StreamingResponse +from sqlalchemy.ext.asyncio import AsyncSession + +from backend.api.dependencies import ( + RequestContext, + database_session, + request_context, +) +from backend.api._resources_common import get_visible_resource +from backend.schemas.common import DownloadUrlRequest +from backend.services.storage import create_download_url_payload + +router = APIRouter(prefix="/api/v1/data-resources", tags=["data-resources"]) + +_PREVIEW_MAX_BYTES = 2 * 1024 * 1024 +_PREVIEW_DEFAULT_LIMIT = 100 +_PREVIEW_MAX_LIMIT = 500 +_TABLE_EXTENSIONS = {".csv", ".tsv"} + + +def _extension_of(file_name: str | None, resource_name: str | None) -> str: + for candidate in (file_name, resource_name): + if not candidate: + continue + lower = candidate.lower() + for ext in _TABLE_EXTENSIONS: + if lower.endswith(ext): + return ext + return "" + + +def _decode_preview_text(raw: bytes) -> str: + for encoding in ("utf-8-sig", "utf-8", "gb18030"): + try: + return raw.decode(encoding) + except UnicodeDecodeError: + continue + return raw.decode("utf-8", errors="replace") + + +async def _read_prefix_bytes(store: Any, object_key: str, max_bytes: int) -> bytes: + chunks: list[bytes] = [] + total = 0 + async for chunk in store.get_stream(object_key): + if not chunk: + continue + chunks.append(chunk) + total += len(chunk) + if total >= max_bytes: + break + data = b"".join(chunks) + return data[:max_bytes] + + +def _parse_table_preview( + text: str, + *, + delimiter: str, + limit: int, + byte_truncated: bool, +) -> dict[str, Any]: + reader = csv.reader(io.StringIO(text), delimiter=delimiter) + try: + header = next(reader) + except StopIteration: + return { + "kind": "table", + "columns": [], + "rows": [], + "row_count": 0, + "truncated": byte_truncated, + "delimiter": delimiter, + } + + columns = [str(cell) if cell is not None else "" for cell in header] + if not any(columns): + columns = [f"col_{index + 1}" for index in range(max(len(header), 1))] + + rows: list[list[str]] = [] + truncated = byte_truncated + for row in reader: + if len(rows) >= limit: + truncated = True + break + cells = [str(cell) if cell is not None else "" for cell in row] + if len(cells) < len(columns): + cells.extend([""] * (len(columns) - len(cells))) + elif len(cells) > len(columns): + cells = cells[: len(columns)] + rows.append(cells) + + return { + "kind": "table", + "columns": columns, + "rows": rows, + "row_count": len(rows), + "truncated": truncated, + "delimiter": delimiter, + } + + +@router.post("/{resource_id}/download-url") +async def resource_download_url( + resource_id: str, + payload: DownloadUrlRequest, + request: Request, + context: RequestContext = Depends(request_context), + session: AsyncSession = Depends(database_session), +) -> dict[str, Any]: + resource, _ = await get_visible_resource( + resource_id, + context, + session, + ) + data = await create_download_url_payload( + await session.get(StorageObjects, resource.storage_object_id), + DownloadUrlRequest(expires_seconds=payload.expires_seconds), + request, + ) + return {"request_id": context.request_id, "data": data["data"], "meta": {}} + + +@router.get("/{resource_id}/content") +async def resource_content( + resource_id: str, + request: Request, + context: RequestContext = Depends(request_context), + session: AsyncSession = Depends(database_session), +) -> StreamingResponse: + """同源流式读取资源字节，供前端预览器加载。""" + resource, storage_object = await get_visible_resource( + resource_id, + context, + session, + ) + if storage_object.object_status != "available": + raise HTTPException(status.HTTP_404_NOT_FOUND, "object not found") + if ( + not storage_object.bucket_name + or not storage_object.object_key + or storage_object.bucket_name not in request.app.state.object_stores + ): + raise HTTPException( + status.HTTP_409_CONFLICT, + "object does not support content download", + ) + + file_name = storage_object.file_name or resource.resource_name or "file" + media_type = storage_object.mime_type or "application/octet-stream" + store = request.app.state.object_stores[storage_object.bucket_name] + stream = store.get_stream(storage_object.object_key) + headers = { + "Content-Disposition": f"inline; filename*=UTF-8''{quote(file_name)}", + "Cache-Control": "private, no-store", + } + if storage_object.size_bytes is not None: + headers["Content-Length"] = str(storage_object.size_bytes) + + return StreamingResponse( + stream, + media_type=media_type, + headers=headers, + ) + + +@router.get("/{resource_id}/preview") +async def resource_preview( + resource_id: str, + request: Request, + context: RequestContext = Depends(request_context), + session: AsyncSession = Depends(database_session), + limit: int = Query(default=_PREVIEW_DEFAULT_LIMIT, ge=1, le=_PREVIEW_MAX_LIMIT), +) -> dict[str, Any]: + """表格类数据资源抽样预览（csv / tsv）。""" + resource, storage_object = await get_visible_resource( + resource_id, + context, + session, + ) + if storage_object.object_status != "available": + raise HTTPException(status.HTTP_404_NOT_FOUND, "object not found") + if ( + not storage_object.bucket_name + or not storage_object.object_key + or storage_object.bucket_name not in request.app.state.object_stores + ): + raise HTTPException( + status.HTTP_409_CONFLICT, + "object does not support preview", + ) + + extension = _extension_of(storage_object.file_name, resource.resource_name) + if extension not in _TABLE_EXTENSIONS: + raise HTTPException( + status.HTTP_422_UNPROCESSABLE_ENTITY, + "仅支持预览 .csv / .tsv 表格文件", + ) + + store = request.app.state.object_stores[storage_object.bucket_name] + raw = await _read_prefix_bytes( + store, + storage_object.object_key, + _PREVIEW_MAX_BYTES, + ) + byte_truncated = ( + storage_object.size_bytes is not None + and storage_object.size_bytes > len(raw) + ) or len(raw) >= _PREVIEW_MAX_BYTES + text = _decode_preview_text(raw) + delimiter = "\t" if extension == ".tsv" else "," + payload = _parse_table_preview( + text, + delimiter=delimiter, + limit=limit, + byte_truncated=byte_truncated, + ) + return {"request_id": context.request_id, "data": payload, "meta": {}} diff --git a/backend/src/backend/api/schedules/__init__.py b/backend/src/backend/api/schedules/__init__.py new file mode 100644 index 0000000..e69de29 diff --git a/backend/src/backend/schedule_runs.py b/backend/src/backend/api/schedules/runs.py similarity index 97% rename from backend/src/backend/schedule_runs.py rename to backend/src/backend/api/schedules/runs.py index 43ff14f..87eff1c 100644 --- a/backend/src/backend/schedule_runs.py +++ b/backend/src/backend/api/schedules/runs.py @@ -41,13 +41,13 @@ from pydantic import Field from sqlalchemy import select from sqlalchemy.ext.asyncio import AsyncSession -from backend.dependencies import ( +from backend.api.dependencies import ( RequestContext, database_session, request_context, ) -router = APIRouter(tags=["schedule-runs"]) +router = APIRouter(prefix="/api/v1", tags=["schedule-runs"]) RunStatus = Literal[ "queued", "running", @@ -238,7 +238,7 @@ async def _artifact_bytes( # 立即触发一次调度：写入运行记录和 Outbox，由 schedule 容器异步接手执行。 @router.post( - "/api/v1/schedules/{schedule_id}/run", + "/schedules/{schedule_id}/run", status_code=status.HTTP_202_ACCEPTED, ) async def run_schedule_now( @@ -288,7 +288,7 @@ async def run_schedule_now( # 按调度或状态筛选运行历史，供前端运行记录列表展示。 -@router.get("/api/v1/schedule-runs") +@router.get("/schedule-runs") async def list_schedule_runs( schedule_id: str | None = Query(default=None), run_status: RunStatus | None = Query(default=None, alias="status"), @@ -318,7 +318,7 @@ async def list_schedule_runs( # 查询一次运行的详情，包括每个节点的执行状态。 -@router.get("/api/v1/schedule-runs/{run_id}") +@router.get("/schedule-runs/{run_id}") async def get_schedule_run( run_id: str, context: RequestContext = Depends(request_context), @@ -334,7 +334,7 @@ async def get_schedule_run( # 返回某个节点运行关联的日志/结果产物元数据及可访问地址。 @router.get( - "/api/v1/schedule-runs/{run_id}/node-runs/{node_run_id}/artifacts" + "/schedule-runs/{run_id}/node-runs/{node_run_id}/artifacts" ) async def get_schedule_node_run_artifacts( run_id: str, @@ -377,7 +377,7 @@ async def get_schedule_node_run_artifacts( # 读取节点运行日志正文，通常由前端日志面板按需调用。 @router.get( - "/api/v1/schedule-runs/{run_id}/node-runs/{node_run_id}/logs" + "/schedule-runs/{run_id}/node-runs/{node_run_id}/logs" ) async def read_schedule_node_run_logs( run_id: str, @@ -404,7 +404,7 @@ async def read_schedule_node_run_logs( # 为节点运行结果生成下载响应或重定向地址。 @router.get( - "/api/v1/schedule-runs/{run_id}/node-runs/{node_run_id}/result" + "/schedule-runs/{run_id}/node-runs/{node_run_id}/result" ) async def download_schedule_node_run_result( run_id: str, diff --git a/backend/src/backend/schedules.py b/backend/src/backend/api/schedules/schedules.py similarity index 88% rename from backend/src/backend/schedules.py rename to backend/src/backend/api/schedules/schedules.py index bfe6cc7..5c724fe 100644 --- a/backend/src/backend/schedules.py +++ b/backend/src/backend/api/schedules/schedules.py @@ -7,7 +7,6 @@ from __future__ import annotations -import heapq from datetime import UTC, datetime from decimal import Decimal from typing import Any @@ -29,12 +28,12 @@ from fastapi import APIRouter, Depends, HTTPException, Query, Request, status from sqlalchemy import delete, func, or_, select from sqlalchemy.ext.asyncio import AsyncSession -from backend.dependencies import ( +from backend.api.dependencies import ( RequestContext, database_session, request_context, ) -from backend.schedule_schemas import ( +from backend.schemas.schedules import ( CreateScheduleEdgeRequest, CreateScheduleNodeRequest, CreateScheduleRequest, @@ -47,7 +46,7 @@ from backend.schedule_schemas import ( ) from backend.services.storage import soft_delete_object -router = APIRouter(tags=["schedules"]) +router = APIRouter(prefix="/api/v1", tags=["schedules"]) _ACTIVE_RUN_STATUSES = ("queued", "running") @@ -239,118 +238,11 @@ def edge_payload(item: ScheduleEdges) -> dict[str, Any]: } -def validate_dag( - nodes: list[ScheduleNodes], - edges: list[ScheduleEdges], -) -> dict[str, Any]: - node_by_id = {item.node_id: item for item in nodes} - indegree = {item.node_id: 0 for item in nodes} - outgoing: dict[str, set[str]] = { - item.node_id: set() - for item in nodes - } - errors: list[dict[str, Any]] = [] - seen_edges: set[tuple[str, str]] = set() - - if not nodes: - errors.append( - { - "code": "DAG_EMPTY", - "message": "schedule must contain at least one node", - } - ) - - for edge in edges: - if ( - edge.source_node_id not in node_by_id - or edge.target_node_id not in node_by_id - ): - errors.append( - { - "code": "DAG_EDGE_NODE_MISSING", - "message": "edge references a node outside the schedule", - "edge_id": edge.edge_id, - } - ) - continue - pair = (edge.source_node_id, edge.target_node_id) - if edge.source_node_id == edge.target_node_id: - errors.append( - { - "code": "DAG_SELF_EDGE", - "message": "a node cannot depend on itself", - "edge_id": edge.edge_id, - } - ) - continue - if pair in seen_edges: - errors.append( - { - "code": "DAG_DUPLICATE_EDGE", - "message": "duplicate directed edge", - "edge_id": edge.edge_id, - } - ) - continue - seen_edges.add(pair) - outgoing[edge.source_node_id].add(edge.target_node_id) - indegree[edge.target_node_id] += 1 - - root_ids = sorted( - (node_id for node_id, degree in indegree.items() if degree == 0), - key=lambda node_id: node_by_id[node_id].node_key, - ) - leaf_ids = sorted( - (node_id for node_id, targets in outgoing.items() if not targets), - key=lambda node_id: node_by_id[node_id].node_key, - ) - queue = [ - (node_by_id[node_id].node_key, node_id) - for node_id in root_ids - ] - heapq.heapify(queue) - remaining_indegree = dict(indegree) - ordered_ids: list[str] = [] - while queue: - _, node_id = heapq.heappop(queue) - ordered_ids.append(node_id) - for target_id in sorted( - outgoing[node_id], - key=lambda value: node_by_id[value].node_key, - ): - remaining_indegree[target_id] -= 1 - if remaining_indegree[target_id] == 0: - heapq.heappush( - queue, - (node_by_id[target_id].node_key, target_id), - ) - - if len(ordered_ids) != len(nodes): - cycle_node_ids = sorted( - ( - node_id - for node_id, degree in remaining_indegree.items() - if degree > 0 - ), - key=lambda node_id: node_by_id[node_id].node_key, - ) - errors.append( - { - "code": "DAG_CYCLE", - "message": "schedule graph contains a directed cycle", - "node_ids": cycle_node_ids, - } - ) - - return { - "valid": not errors, - "node_count": len(nodes), - "edge_count": len(edges), - "root_node_ids": root_ids, - "leaf_node_ids": leaf_ids, - "topological_order": ordered_ids, - "errors": errors, - } +# validate_dag is implemented in backend.services.schedules so it can be +# unit-tested without spinning up FastAPI. Re-exported here for the four +# internal callsites and for any external callers that still import it +# from this module. +from backend.services.schedules import validate_dag # noqa: F401 async def schedule_row( @@ -521,7 +413,7 @@ async def _require_valid_when_enabled( # 根据 Cron 表达式预览未来触发时间，不会保存或执行任务。 -@router.post("/api/v1/cron/preview") +@router.post("/cron/preview") async def preview_cron( payload: CronPreviewRequest, context: RequestContext = Depends(request_context), @@ -539,7 +431,7 @@ async def preview_cron( # 列出调度产生的可展示版本/产物，供前端结果面板使用。 -@router.get("/api/v1/schedule-artifacts") +@router.get("/schedule-artifacts") async def list_schedule_artifacts( limit: int = Query(default=100, ge=1, le=500), context: RequestContext = Depends(request_context), @@ -589,7 +481,7 @@ async def list_schedule_artifacts( # 列出当前工作区的调度定义及其节点、边数量等摘要信息。 -@router.get("/api/v1/schedules") +@router.get("/schedules") async def list_schedules( context: RequestContext = Depends(request_context), session: AsyncSession = Depends(database_session), @@ -652,7 +544,7 @@ async def list_schedules( # 创建新的 DAG 调度定义；初始状态不包含节点和边。 @router.post( - "/api/v1/schedules", + "/schedules", status_code=status.HTTP_201_CREATED, ) async def create_schedule( @@ -664,6 +556,7 @@ async def create_schedule( select(Schedules).where( Schedules.workspace_id == context.workspace.workspace_id, Schedules.schedule_name == payload.schedule_name, + Schedules.is_deleted == 0, Schedules.deleted_at.is_(None), ) ) @@ -704,7 +597,7 @@ async def create_schedule( # 获取一个调度的完整画布数据，包括节点、边和当前工作流版本。 -@router.get("/api/v1/schedules/{schedule_id}") +@router.get("/schedules/{schedule_id}") async def get_schedule( schedule_id: str, context: RequestContext = Depends(request_context), @@ -719,8 +612,8 @@ async def get_schedule( # 更新调度基本属性，如名称、Cron、时区、是否启用和并发策略。 -@router.put("/api/v1/schedules/{schedule_id}") -@router.patch("/api/v1/schedules/{schedule_id}") +@router.put("/schedules/{schedule_id}") +@router.patch("/schedules/{schedule_id}") async def update_schedule( schedule_id: str, payload: UpdateScheduleRequest, @@ -753,6 +646,7 @@ async def update_schedule( Schedules.workspace_id == context.workspace.workspace_id, Schedules.schedule_name == payload.schedule_name, Schedules.schedule_id != schedule_id, + Schedules.is_deleted == 0, Schedules.deleted_at.is_(None), ) ) @@ -784,7 +678,7 @@ async def update_schedule( # 删除调度定义；请求携带 workflow_version 以避免误删他人刚修改的画布。 -@router.delete("/api/v1/schedules/{schedule_id}") +@router.delete("/schedules/{schedule_id}") async def delete_schedule( schedule_id: str, payload: WorkflowVersionRequest, @@ -861,6 +755,7 @@ async def delete_schedule( item.next_run_at = None item.deleted_at = _mysql_utc(datetime.now(UTC)) item.updated_by = context.user.user_id + item.is_deleted = 1 item.workflow_version += 1 await session.flush() return { @@ -876,7 +771,7 @@ async def delete_schedule( # 向调度画布新增一个执行节点，并关联已发布的脚本版本。 @router.post( - "/api/v1/schedules/{schedule_id}/nodes", + "/schedules/{schedule_id}/nodes", status_code=status.HTTP_201_CREATED, ) async def create_schedule_node( @@ -931,7 +826,7 @@ async def create_schedule_node( # 更新节点名称、执行参数、超时、重试和画布坐标等配置。 -@router.put("/api/v1/schedules/{schedule_id}/nodes/{node_id}") +@router.put("/schedules/{schedule_id}/nodes/{node_id}") async def update_schedule_node( schedule_id: str, node_id: str, @@ -987,7 +882,7 @@ async def update_schedule_node( # 从调度画布删除节点，并同步清理关联边。 -@router.delete("/api/v1/schedules/{schedule_id}/nodes/{node_id}") +@router.delete("/schedules/{schedule_id}/nodes/{node_id}") async def delete_schedule_node( schedule_id: str, node_id: str, @@ -1072,7 +967,7 @@ async def delete_schedule_node( # 在两个节点之间新增依赖边，表示目标节点必须等待源节点完成。 @router.post( - "/api/v1/schedules/{schedule_id}/edges", + "/schedules/{schedule_id}/edges", status_code=status.HTTP_201_CREATED, ) async def create_schedule_edge( @@ -1148,7 +1043,7 @@ async def create_schedule_edge( # 修改一条依赖边的条件表达式或其他可编辑字段。 -@router.put("/api/v1/schedules/{schedule_id}/edges/{edge_id}") +@router.put("/schedules/{schedule_id}/edges/{edge_id}") async def update_schedule_edge( schedule_id: str, edge_id: str, @@ -1183,7 +1078,7 @@ async def update_schedule_edge( # 删除节点之间的依赖关系，不会删除节点本身。 -@router.delete("/api/v1/schedules/{schedule_id}/edges/{edge_id}") +@router.delete("/schedules/{schedule_id}/edges/{edge_id}") async def delete_schedule_edge( schedule_id: str, edge_id: str, @@ -1218,7 +1113,7 @@ async def delete_schedule_edge( # 校验画布是否为可执行 DAG，例如是否存在环、孤立节点或无效版本。 -@router.post("/api/v1/schedules/{schedule_id}/validate") +@router.post("/schedules/{schedule_id}/validate") async def validate_schedule( schedule_id: str, context: RequestContext = Depends(request_context), diff --git a/backend/src/backend/scripts.py b/backend/src/backend/api/scripts.py similarity index 82% rename from backend/src/backend/scripts.py rename to backend/src/backend/api/scripts.py index a48f94f..a4115fc 100644 --- a/backend/src/backend/scripts.py +++ b/backend/src/backend/api/scripts.py @@ -36,19 +36,19 @@ from fastapi import ( status, ) from loguru import logger -from sqlalchemy import func, select +from sqlalchemy import func, or_, select from sqlalchemy.ext.asyncio import AsyncSession -from backend.dependencies import ( +from backend.api.dependencies import ( RequestContext, database_session, request_context, ) -from backend.runtime_client import RuntimeClientError -from backend.schemas import ( +from backend.clients.runtime import RuntimeClientError +from backend.schemas.common import DownloadUrlRequest +from backend.schemas.scripts import ( CreateScriptRequest, CreateWorkspaceDirectoryRequest, - DownloadUrlRequest, LockScriptRequest, PublishVersionRequest, UpdateScriptRequest, @@ -60,7 +60,7 @@ from backend.services.storage import ( soft_delete_object, ) -router = APIRouter(tags=["scripts"]) +router = APIRouter(prefix="/api/v1", tags=["scripts"]) def normalize_user_path(value: str, *, allow_empty: bool = True) -> str: @@ -115,6 +115,82 @@ def user_relative_path(context: RequestContext, child_path: str = "") -> str: return f"{base}/{normalized}" if normalized else base +def _escape_like_pattern(value: str) -> str: + """Escape SQL LIKE metacharacters so user-supplied folder names that + contain ``_`` or ``%`` do not act as wildcards. + + Must be paired with ``escape="\\\\"`` on the LIKE clause so MySQL + recognizes the doubled backslash as a single literal backslash escape. + The trailing ``%`` / ``%/%`` SQL wildcards are NOT escaped — they are + added by the caller and are meant to be wildcards. + """ + # Order matters: escape the escape char FIRST, otherwise the next two + # replacements would double-escape our newly inserted backslashes. + return value.replace("\\", "\\\\").replace("%", "\\%").replace("_", "\\_") + + +def _build_list_scripts_owner_descendant_prefix( + owner_user_id: str, parent_path: str +) -> str: + """Return the escaped materialized-path prefix for direct children of + ``parent_path`` within ``owner_user_id``'s own subtree. + + Storage is physically laid out as ``workspace/{owner_user_id}/...``, so a + per-owner listing matches ``workspace/{owner_user_id}/{parent}``. The + endpoint appends ``LIKE '%' AND NOT LIKE '%/%'`` against + ``storage_objects.relative_path`` so only scripts whose parent directory + is exactly ``parent_path`` match (no deeper descendants, no + prefix-siblings like ``foo/bar`` vs ``foo/bar2``). + + Empty ``parent_path`` produces the owner-scoped root prefix — i.e. the + endpoint returns that owner's root-level scripts only. ``list_scripts`` + calls this with ``owner_user_id`` = the requester by default (so a + non-admin sees their own subtree, including private) or with the + ``owner_user_id`` query param so the tree can lazily fetch another + member's content on group expand; the route's visibility filter then + excludes the other owner's private rows. + + The prefix is run through ``_escape_like_pattern`` so folder names + containing ``_`` / ``%`` do not act as wildcards. The trailing ``/`` + is appended AFTER escaping so it remains a literal slash. + """ + normalized_parent = normalize_user_path(parent_path) + if normalized_parent: + target_prefix = f"workspace/{owner_user_id}/{normalized_parent}" + else: + target_prefix = f"workspace/{owner_user_id}" + return f"{_escape_like_pattern(target_prefix)}/" + + +def _build_list_scripts_workspace_descendant_prefix(parent_path: str) -> str: + """Return the escaped materialized-path prefix for direct children of + ``parent_path`` across **all owners** in the workspace. + + Storage is still physically laid out as ``workspace/{user_id}/...``, so + listing a subdirectory across every owner must match the owner segment + with an intentional ``%`` wildcard — ``workspace/%/foo`` — exactly like + list_resources does against ``object_key``. The endpoint's + ``LIKE '/%' AND NOT LIKE '/%/%'`` pair then grabs only + DIRECT children of ``parent_path`` per owner. Non-admin scoping is + handled separately in the SQL via + ``owner_user_id = me OR visibility IN (workspace, public)``. + + Empty ``parent_path`` returns ``"workspace/%/"``: the physical root + level is each owner's subtree (``workspace/{owner_id}/...``), so the + owner segment is wildcarded and the endpoint's direct-child pair + keeps every owner's root-level files (``workspace/%/%`` AND NOT + ``workspace/%/%/%``). The prefix is run through ``_escape_like_pattern`` + so folder names containing ``_`` / ``%`` do not act as wildcards. The + trailing ``/`` is appended AFTER escaping so it remains a literal slash. + """ + normalized_parent = normalize_user_path(parent_path) + if normalized_parent: + target_prefix = f"workspace/%/{_escape_like_pattern(normalized_parent)}" + else: + target_prefix = "workspace/%" + return f"{target_prefix}/" + + def safe_script_name(value: str, script_type: str) -> str: name = value.replace("\\", "/").rsplit("/", 1)[-1].strip() if not name or name in {".", ".."} or any(ord(char) < 32 for char in name): @@ -254,6 +330,22 @@ def version_payload(version: Versions) -> dict[str, Any]: } +def script_can_view(script: Scripts, context: RequestContext) -> bool: + """同一 workspace 内：owner 永远可见自己的脚本（含 private）； + 其他成员只见 visibility in {workspace, public} 的脚本； + admin 全部可见。与 data resources 的 can_view 完全对称。 + + 用于单脚本读取（get / content / latest-version / versions 列表）以及 + 所有写路径（update / delete / publish）的前置校验 —— 保证 A 的 private + 脚本对非 owner 不可见，而不仅是「列表里不出现」。 + """ + if script.owner_user_id == context.user.user_id: + return True + if script.visibility in {"workspace", "public"}: + return True + return context.is_admin + + async def get_script_row( script_id: str, context: RequestContext, @@ -318,6 +410,11 @@ async def get_script_row( if row is None: raise HTTPException(status.HTTP_404_NOT_FOUND, "script not found") script, storage_object = row + if not script_can_view(script, context): + # Private scripts are only visible to their owner (and admin); + # treat cross-owner access as not-found so the id cannot probe + # visibility, matching get_visible_resource for data resources. + raise HTTPException(status.HTTP_404_NOT_FOUND, "script not found") return script, storage_object @@ -566,7 +663,7 @@ async def create_script_record( # 新建空的 Python 脚本或 Notebook：同时创建数据库元数据和初始文件内容。 -@router.post("/api/v1/scripts", status_code=status.HTTP_201_CREATED) +@router.post("/scripts", status_code=status.HTTP_201_CREATED) async def create_script( payload: CreateScriptRequest, request: Request, @@ -600,7 +697,7 @@ async def create_script( # 上传现有脚本文件：校验文件名/类型后写入存储，并建立 Scripts 记录。 @router.post( - "/api/v1/scripts/upload", + "/scripts/upload", status_code=status.HTTP_201_CREATED, ) async def upload_script( @@ -660,7 +757,7 @@ async def upload_script( # 返回旧版一次性完整目录树，保留给兼容旧前端；新页面通常按目录懒加载。 -@router.get("/api/v1/workspace-tree") +@router.get("/workspace-tree") async def get_workspace_tree( context: RequestContext = Depends(request_context), session: AsyncSession = Depends(database_session), @@ -682,7 +779,7 @@ async def get_workspace_tree( StorageObjects.workspace_id == context.workspace.workspace_id, StorageObjects.object_status == "available", StorageObjects.is_deleted == 0, - StorageObjects.relative_path.like(like_prefix), + StorageObjects.relative_path.like(like_prefix, escape="\\"), StorageObjects.usage_type.notin_(TREE_EXCLUDED_USAGE_TYPES), ) ) @@ -732,22 +829,31 @@ async def get_workspace_tree( # 查询某个目录下的直接子目录，供前端按需展开工作区树。 -@router.get("/api/v1/workspace-directories") +@router.get("/workspace-directories") async def list_workspace_directories( parent_path: str = Query(default=""), + owner_user_id: str | None = Query(default=None, max_length=64), context: RequestContext = Depends(request_context), session: AsyncSession = Depends(database_session), ) -> dict[str, Any]: """List direct child directories of a workspace path. Empty ``parent_path`` returns the directories immediately under the - user's scoped root. Only available, non-deleted StorageObjects are - considered. + target owner's scoped root. Only available, non-deleted StorageObjects + are considered. + + ``owner_user_id`` defaults to the requester, so a member lists their + own directories. Passing another member's id scopes to that owner's + subtree so the script explorer can lazily render their directory + structure on expand (directories are structural rows; file-level + visibility is still enforced by the scripts/data-resources endpoints, + which exclude the other owner's private files). """ - scoped_prefix = user_relative_path(context) + target_owner = owner_user_id or context.user.user_id + scoped_prefix = f"workspace/{target_owner}" parent = normalize_user_path(parent_path) target_prefix = f"{scoped_prefix}/{parent}" if parent else scoped_prefix - descendant_prefix = f"{target_prefix}/" + descendant_prefix = f"{_escape_like_pattern(target_prefix)}/" rows = ( await session.execute( @@ -755,8 +861,8 @@ async def list_workspace_directories( StorageObjects.workspace_id == context.workspace.workspace_id, StorageObjects.object_status == "available", StorageObjects.is_deleted == 0, - StorageObjects.relative_path.like(f"{descendant_prefix}%"), - ~StorageObjects.relative_path.like(f"{descendant_prefix}%/%"), + StorageObjects.relative_path.like(f"{descendant_prefix}%", escape="\\"), + ~StorageObjects.relative_path.like(f"{descendant_prefix}%/%", escape="\\"), StorageObjects.object_type == "directory", StorageObjects.usage_type.notin_(TREE_EXCLUDED_USAGE_TYPES), ) @@ -777,20 +883,22 @@ async def list_workspace_directories( "path": child_path, "name": suffix, "parent_path": parent, + "owner_user_id": target_owner, "has_children": False, }, ) for directory in directories.values(): # directory['path'] is already workspace-relative and includes the parent segment. - child_prefix = f"{scoped_prefix}/{directory['path']}/" + # Escape defensively in case the DB has folder names containing `_` or `%`. + child_prefix = f"{_escape_like_pattern(scoped_prefix)}/{_escape_like_pattern(directory['path'])}/" has_children = await session.scalar( select(StorageObjects.storage_object_id).where( StorageObjects.workspace_id == context.workspace.workspace_id, StorageObjects.object_status == "available", StorageObjects.is_deleted == 0, - StorageObjects.relative_path.like(f"{child_prefix}%"), - ~StorageObjects.relative_path.like(f"{child_prefix}%/%"), + StorageObjects.relative_path.like(f"{child_prefix}%", escape="\\"), + ~StorageObjects.relative_path.like(f"{child_prefix}%/%", escape="\\"), StorageObjects.usage_type.notin_(TREE_EXCLUDED_USAGE_TYPES), ).limit(1) ) @@ -806,7 +914,7 @@ async def list_workspace_directories( # 在工作区内创建逻辑目录；目录信息由脚本相对路径推导，不对应容器本地文件夹。 @router.post( - "/api/v1/workspace-directories", + "/workspace-directories", status_code=status.HTTP_201_CREATED, ) async def create_workspace_directory( @@ -916,7 +1024,8 @@ async def create_workspace_directory( path_hash=path_hash, object_status="available", size_bytes=0, - visibility="private", + visibility="public", + owner_user_id=context.user.user_id, created_by=context.user.user_id, ) session.add(directory) @@ -930,7 +1039,8 @@ async def create_workspace_directory( directory.storage_uri = f"inline://directory/{relative_path}" directory.file_name = name directory.size_bytes = 0 - directory.visibility = "private" + directory.visibility = "public" + directory.owner_user_id = context.user.user_id directory.created_by = context.user.user_id try: @@ -956,13 +1066,14 @@ async def create_workspace_directory( "path": child_path, "name": name, "parent_path": parent, + "owner_user_id": context.user.user_id, }, "meta": {}, } # 删除逻辑目录及其下属脚本记录；实际文件按存储层的软删除规则处理。 -@router.delete("/api/v1/workspace-directories") +@router.delete("/workspace-directories") async def delete_workspace_directory( request: Request, path: str = Query(min_length=1, max_length=1024), @@ -985,14 +1096,14 @@ async def delete_workspace_directory( raise HTTPException(status.HTTP_404_NOT_FOUND, "directory not found") target_ulid = target_dir_row.storage_object_id - child_prefix = f"{target_relative}/" + child_prefix = f"{_escape_like_pattern(target_relative)}/" descendants = ( ( await session.execute( select(StorageObjects).where( StorageObjects.workspace_id == context.workspace.workspace_id, StorageObjects.object_status == "available", - StorageObjects.relative_path.like(f"{child_prefix}%"), + StorageObjects.relative_path.like(f"{child_prefix}%", escape="\\"), ).order_by(func.length(StorageObjects.relative_path).desc()) ) ) @@ -1045,11 +1156,42 @@ async def delete_workspace_directory( # 列出当前工作区中用户有权查看的脚本与 Notebook 元数据。 -@router.get("/api/v1/scripts") +# +# 可选 ``parent_path`` Query 参数把返回范围限定为 *直接挂在该路径下的* 脚本 +# (不含更深的子目录)。空字符串等价于工作区根目录(跨所有 owner 根级文件); +# 非空路径按 owner 段通配(``workspace/%/``)跨所有 owner 查询,与 +# list_resources 一致。这是前端按目录懒加载的关键端点,避免 10 万级脚本 +# 一次性返回。 +# +# 实现走 ``storage_objects.relative_path`` materialized path(``LIKE prefix%`` +# + ``NOT LIKE prefix%/%`` 取直接子节点)。MySQL 优化器目前选 +# ``scripts`` 驱动 ``idx_scripts_workspace``、再 PK lookup 回 +# ``storage_objects`` 应用 LIKE 过滤 —— 对 ``scripts`` 端高度选择性 +# (workspace + status) 的工作区来说已经够好;真要压平万级脚本可考虑 +# ``STRAIGHT_JOIN`` 或给 ``storage_objects.relative_path`` 加 prefix +# 索引(基线迁移里有 ``idx_storage_workspace_relative_path`` 但 ORM +# 模型未声明,不在此修复范围)。 +@router.get("/scripts") async def list_scripts( + parent_path: str = Query(default="", max_length=1024), + owner_user_id: str | None = Query(default=None, max_length=64), context: RequestContext = Depends(request_context), session: AsyncSession = Depends(database_session), ) -> dict[str, Any]: + # Per-owner listing: storage is physically laid out as + # ``workspace/{user_id}/...``. Default (no owner_user_id) scopes to the + # requester's own subtree — root-level files when parent_path is empty — + # so the tree's initial load fetches only "me". Passing owner_user_id + # scopes to that owner's subtree so the tree can lazily fetch another + # member's content when their group is expanded. Non-admin scoping is + # applied below via visibility, so the other owner's private rows are + # excluded (workspace/public only); the requester's own private rows + # pass because ``owner_user_id = me``. + target_owner = owner_user_id or context.user.user_id + descendant_prefix = _build_list_scripts_owner_descendant_prefix( + target_owner, parent_path + ) + statement = ( select(Scripts, StorageObjects, Users.display_name) .join( @@ -1060,9 +1202,21 @@ async def list_scripts( .where( Scripts.workspace_id == context.workspace.workspace_id, Scripts.status == "active", + StorageObjects.relative_path.like(f"{descendant_prefix}%", escape="\\"), + ~StorageObjects.relative_path.like(f"{descendant_prefix}%/%", escape="\\"), ) .order_by(Scripts.updated_at.desc()) ) + # 只返回 owner 自己的脚本（含 private），或 visibility 为 + # workspace/public 的其他成员脚本；A 的 private 脚本对非 owner 不可见。 + # admin 跳过过滤，全部可见。 + if not context.is_admin: + statement = statement.where( + or_( + Scripts.owner_user_id == context.user.user_id, + Scripts.visibility.in_(["workspace", "public"]), + ) + ) rows = (await session.execute(statement)).all() return { "request_id": context.request_id, @@ -1074,8 +1228,58 @@ async def list_scripts( } +# 工作区内 active 脚本总数。DashboardRoute 等不需要列表但需要计数的场景使用, +# 避免被 listScripts 的懒加载语义污染。该路由必须在 /scripts/{script_id} 之前声明 +# ——FastAPI 按声明顺序匹配,否则 `count` 会被当作 script_id 命中 get_script。 +# +# Scope: workspace-wide — the dashboard's "全部脚本" / "工作副本" counts +# reflect the whole workspace, not the requester's own subtree. admin sees +# every active script; non-admin is narrowed by visibility +# (owner_user_id = me OR visibility IN (workspace, public)), exactly like +# list_scripts and list_resources. +# +# Implementation choices and why: +# - INNER JOIN to StorageObjects so orphans (current_object_id has no +# joinable row) are excluded. +# - Workspace-wide ``LIKE 'workspace/%'`` prefix (no embedded user_id) so +# counts span every owner's subtree. +# - No NOT-LIKE filter because the count wants descendants too. +@router.get("/scripts/count") +async def count_scripts( + context: RequestContext = Depends(request_context), + session: AsyncSession = Depends(database_session), +) -> dict[str, Any]: + descendant_prefix = _build_list_scripts_workspace_descendant_prefix("") + base = ( + select(func.count()) + .select_from(Scripts) + .join( + StorageObjects, + StorageObjects.storage_object_id == Scripts.current_object_id, + ) + .where( + Scripts.workspace_id == context.workspace.workspace_id, + Scripts.status == "active", + StorageObjects.relative_path.like(f"{descendant_prefix}%", escape="\\"), + ) + ) + if not context.is_admin: + base = base.where( + or_( + Scripts.owner_user_id == context.user.user_id, + Scripts.visibility.in_(["workspace", "public"]), + ) + ) + total = await session.scalar(base) + return { + "request_id": context.request_id, + "data": {"total": int(total or 0)}, + "meta": {}, + } + + # 读取脚本正文或 Notebook JSON；编辑器打开文件时调用此接口。 -@router.get("/api/v1/scripts/{script_id}/content") +@router.get("/scripts/{script_id}/content") async def get_script_content( script_id: str, request: Request, @@ -1126,7 +1330,7 @@ async def get_script_content( # 查询单个脚本的元数据，例如类型、路径、锁状态和拥有者。 -@router.get("/api/v1/scripts/{script_id}") +@router.get("/scripts/{script_id}") async def get_script( script_id: str, context: RequestContext = Depends(request_context), @@ -1145,7 +1349,7 @@ async def get_script( # 保存编辑器提交的新内容；会校验工作区权限和文件编辑锁。 -@router.put("/api/v1/scripts/{script_id}") +@router.put("/scripts/{script_id}") async def update_script( script_id: str, payload: UpdateScriptRequest, @@ -1232,7 +1436,7 @@ async def update_script( # 修改脚本锁定状态，避免其他用户同时编辑同一份文件。 -@router.patch("/api/v1/scripts/{script_id}/lock") +@router.patch("/scripts/{script_id}/lock") async def set_script_lock( script_id: str, payload: LockScriptRequest, @@ -1275,7 +1479,7 @@ async def set_script_lock( # 软删除脚本；元数据标记删除，历史版本可按规则继续保留。 -@router.delete("/api/v1/scripts/{script_id}") +@router.delete("/scripts/{script_id}") async def delete_script( script_id: str, request: Request, @@ -1322,7 +1526,7 @@ async def delete_script( # 将当前脚本内容发布为不可变版本，供调度节点和回溯下载使用。 @router.post( - "/api/v1/scripts/{script_id}/versions", + "/scripts/{script_id}/versions", status_code=status.HTTP_201_CREATED, ) async def publish_version( @@ -1443,7 +1647,7 @@ async def publish_version( # 列出某脚本已经发布的历史版本。 -@router.get("/api/v1/scripts/{script_id}/versions") +@router.get("/scripts/{script_id}/versions") async def list_versions( script_id: str, context: RequestContext = Depends(request_context), @@ -1465,7 +1669,7 @@ async def list_versions( # 读取脚本最近一次发布的版本；未发布时返回空结果。 -@router.get("/api/v1/scripts/{script_id}/latest-version") +@router.get("/scripts/{script_id}/latest-version") async def latest_version( script_id: str, context: RequestContext = Depends(request_context), @@ -1480,18 +1684,15 @@ async def latest_version( published versions yet (so the frontend can render an empty label without a 404 round-trip). """ - script = await session.scalar( - select(Scripts.script_id).where( - Scripts.script_id == script_id, - Scripts.workspace_id == context.workspace.workspace_id, - Scripts.status == "active", - ) + # Route through get_script_row so the private-visibility guard applies + # here too: other members must not learn about a private script's + # versions by guessing its id. + _script, _ = await get_script_row( + script_id, + context, + session, + allow_missing_storage_object=True, ) - if script is None: - raise HTTPException( - status.HTTP_404_NOT_FOUND, - "script not found", - ) latest = await session.scalar( select(Versions) .where( @@ -1517,7 +1718,7 @@ async def latest_version( # 查询单个发布版本的元数据和关联脚本信息。 -@router.get("/api/v1/versions/{versions_id}") +@router.get("/versions/{versions_id}") async def get_version( versions_id: str, context: RequestContext = Depends(request_context), @@ -1534,7 +1735,7 @@ async def get_version( # 隐藏/删除一个发布版本；是否保留实际产物由存储删除策略决定。 -@router.delete("/api/v1/versions/{versions_id}") +@router.delete("/versions/{versions_id}") async def delete_version( versions_id: str, context: RequestContext = Depends(request_context), @@ -1582,7 +1783,7 @@ async def delete_version( # 为某个版本产物生成带时效的下载地址，而非把大文件直接经 API 返回。 -@router.post("/api/v1/versions/{versions_id}/download-url") +@router.post("/versions/{versions_id}/download-url") async def version_download_url( versions_id: str, payload: DownloadUrlRequest, diff --git a/backend/src/backend/storage_api.py b/backend/src/backend/api/storage.py similarity index 98% rename from backend/src/backend/storage_api.py rename to backend/src/backend/api/storage.py index 843c5c2..051d729 100644 --- a/backend/src/backend/storage_api.py +++ b/backend/src/backend/api/storage.py @@ -143,7 +143,7 @@ def storage_payload(item: StorageObjects) -> dict[str, Any]: # 内部路由由 main.py 以 /internal 前缀挂载。数据库引擎、Session 工厂和对象 # 存储实例均在应用生命周期中创建；本模块只定义路由和供 services.storage 复用的 # 存储辅助函数（如 storage_payload、resolve_bucket、BUCKET_FOR_USAGE）。 -router = APIRouter(tags=["internal-storage"]) +router = APIRouter(prefix="/v1", tags=["internal-storage"]) async def database_session(request: Request) -> AsyncIterator[AsyncSession]: @@ -253,7 +253,7 @@ async def create_upload_record( # Two-step server-proxied upload: the caller PUTs the raw bytes to # ``upload_path`` after this response, which routes through - # ``backend.resources.upload_bytes_to_session`` (the canonical helper + # ``backend.api.resources.upload_bytes_to_session`` (the canonical helper # in ``services.storage``). return { "upload_id": upload.upload_id, @@ -286,7 +286,7 @@ def _public_base_url(request: Request) -> str: @router.post( - "/v1/objects", + "/objects", dependencies=[Depends(require_internal_service)], ) async def create_server_object( @@ -304,7 +304,7 @@ async def create_server_object( return await create_server_object_payload(payload, request, session) -@router.post("/v1/objects/{storage_object_id}/restore") +@router.post("/objects/{storage_object_id}/restore") async def restore_object( storage_object_id: str, request: Request, @@ -371,7 +371,7 @@ async def restore_object( # 管理动作：永久清理超过保留期限或指定的回收站对象。 -@router.post("/v1/admin/trash/purge") +@router.post("/admin/trash/purge") async def purge_trash_object( payload: dict[str, Any], request: Request, diff --git a/backend/src/backend/audit.py b/backend/src/backend/audit.py new file mode 100644 index 0000000..e73d5b2 --- /dev/null +++ b/backend/src/backend/audit.py @@ -0,0 +1,133 @@ +"""按天单文件的 audit log sink，供 main.py 的 access_log 中间件复用。 + +设计要点 +-------- +* 复用全局 loguru ``logger``（与 ``common.logging.configure_logging`` + 共用同一套日志框架，不新增依赖）。日志文件 sink 由本模块的 + :func:`configure_audit_logging` 在进程启动时挂上，只此一次（幂等， + 与 ``configure_logging`` 的风格一致）。 +* 每天一个文件 ``audit-YYYY-MM-DD.log``，放在 ``settings.audit_log_dir`` + （默认 ``data/logs/audit``，相对 backend 进程 cwd）。滚动由 + :class:`_DailyFileSink` 自行实现：缓存当天的文件句柄，跨自然日时关闭旧 + fd 再打开新文件。不使用 loguru 自带的 ``rotation="00:00"``，因为它对 + string path 产出的文件名是 ``audit.log.YYYY-MM-DD_HH-MM-SS``，既没有 + ``audit-`` 前缀也不符合每天一个文件的要求。 +* 谁写审计行：main.py 的 ``access_log`` 中间件在 success 与 exception + 两条路径各打一条 ``logger.bind(user_id, method, path, status).info("audit")``。 + 本模块只管把这类行路由到按天文件 sink；user_id 的解析（cookie / + Bearer 头 + JWT 验签）在 main.py 内部完成，审计只记录、不查 DB。 +""" + +from __future__ import annotations + +import os +import time +from datetime import UTC, datetime +from pathlib import Path +from typing import TextIO + +from loguru import logger + +# 纯文本一行一条（末尾换行由 loguru 的 terminator 追加）： +# 2026-08-21 14:30:00.123 | 01USER... | GET /api/v1/scripts/01ABC... -> 200 +AUDIT_LOG_FORMAT = ( + "{time:YYYY-MM-DD HH:mm:ss.SSS} | {extra[user_id]} | " + "{extra[method]} {extra[path]} -> {extra[status]}" +) + +_CONFIGURED: bool = False +_HANDLER_ID: int | None = None + + +class _DailyFileSink: + """按自然日滚动到 ``audit-YYYY-MM-DD.log`` 的 loguru sink。 + + 缓存当天已打开的 ``Path.open("a")`` 文件句柄；跨日时关闭旧 fd 并打开 + 新文件，不依赖 loguru 自己的 rotation。``write`` 收到的是 loguru 已经 + 按 ``AUDIT_LOG_FORMAT`` 格式化好、以 ``\n`` 结尾的一行文本。 + """ + + def __init__(self, log_dir: str | Path) -> None: + self._log_dir = Path(log_dir) + self._log_dir.mkdir(parents=True, exist_ok=True) + self._fh: TextIO | None = None + self._open_date: str | None = None + + def write(self, message: str) -> None: + day = datetime.now(UTC).astimezone().strftime("%Y-%m-%d") + if self._fh is None or self._open_date != day: + self._close() + self._fh = (self._log_dir / f"audit-{day}.log").open( + "a", encoding="utf-8" + ) + self._open_date = day + self._fh.write(message) + + def flush(self) -> None: + if self._fh is not None: + self._fh.flush() + + def stop(self) -> None: + self._close() + + def _close(self) -> None: + if self._fh is not None: + self._fh.close() + self._fh = None + self._open_date = None + + +def _audit_filter(record: dict) -> bool: + """只放行 access_log 打的审计行，其它 INFO 日志不进审计文件。""" + extra = record["extra"] + return ( + record["message"] == "audit" + and "user_id" in extra + and "method" in extra + and "path" in extra + and "status" in extra + ) + + +def _cleanup_expired_files(log_dir: Path, retention_days: int) -> None: + """启动时删除 ``retention_days`` 天前的 ``audit-*.log``；0 表示关闭清理。""" + if retention_days <= 0: + return + cutoff = time.time() - retention_days * 86_400 + for path in log_dir.glob("audit-*.log"): + try: + if os.path.getmtime(path) < cutoff: + path.unlink() + except OSError: + continue + + +def configure_audit_logging(log_dir: str, retention_days: int) -> None: + """挂上审计日志文件 sink。幂等：多次调用只有首次生效。""" + global _CONFIGURED, _HANDLER_ID + if _CONFIGURED: + return + + dir_path = Path(log_dir) + dir_path.mkdir(parents=True, exist_ok=True) + _cleanup_expired_files(dir_path, retention_days) + + # 注意：loguru 的 ``encoding=`` 只对 file-path sink 生效；对 callable / + # stream sink 传入会直接 TypeError。UTF-8 由 _DailyFileSink 在 + # ``open(..., encoding="utf-8")`` 里保证。 + _HANDLER_ID = logger.add( + _DailyFileSink(dir_path), + level="INFO", + format=AUDIT_LOG_FORMAT, + filter=_audit_filter, + enqueue=True, + serialize=False, + catch=True, + ) + _CONFIGURED = True + + +__all__ = [ + "AUDIT_LOG_FORMAT", + "configure_audit_logging", +] diff --git a/backend/src/backend/clients/__init__.py b/backend/src/backend/clients/__init__.py new file mode 100644 index 0000000..e69de29 diff --git a/backend/src/backend/rclone_rc_client.py b/backend/src/backend/clients/rclone.py similarity index 100% rename from backend/src/backend/rclone_rc_client.py rename to backend/src/backend/clients/rclone.py diff --git a/backend/src/backend/runtime_client.py b/backend/src/backend/clients/runtime.py similarity index 99% rename from backend/src/backend/runtime_client.py rename to backend/src/backend/clients/runtime.py index 1ca859b..941439d 100644 --- a/backend/src/backend/runtime_client.py +++ b/backend/src/backend/clients/runtime.py @@ -94,7 +94,7 @@ class RuntimeClient: """Return a running workspace descriptor, starting it if needed. Mirrors the lazy-start pattern used by - :func:`backend.jupyter.verify_jupyter_access`: try ``get`` + :func:`backend.api.jupyter.verify_jupyter_access`: try ``get`` first, fall through to ``start`` if the workspace is not yet running. Bumps ``last_used_at`` via the runtime registry on the way in, so the idle reaper is satisfied for the duration of the diff --git a/backend/src/backend/schedule_client.py b/backend/src/backend/clients/scheduler.py similarity index 100% rename from backend/src/backend/schedule_client.py rename to backend/src/backend/clients/scheduler.py diff --git a/backend/src/backend/main.py b/backend/src/backend/main.py index f9df7ae..6c6ad3a 100644 --- a/backend/src/backend/main.py +++ b/backend/src/backend/main.py @@ -17,6 +17,7 @@ from contextlib import asynccontextmanager from typing import Any import httpx +from common.auth.jwt import JwtError, verify_jwt_token from common.config import settings from common.db import create_database_engine, create_session_factory from common.logging import configure_logging @@ -32,19 +33,31 @@ from fastapi import Request from fastapi.responses import JSONResponse from loguru import logger -from backend.admin import router as admin_router -from backend.auth import router as auth_router -from backend.jupyter import router as jupyter_router -from backend.platform import router as platform_router -from backend.rclone_rc_client import RcloneRCClient -from backend.resources import router as resources_router -from backend.runtime_client import RuntimeClient -from backend.schedule_runs import router as schedule_runs_router -from backend.schedules import router as schedules_router -from backend.scripts import router as scripts_router -from backend.storage_api import router as storage_api_router +from backend.api.admin import router as admin_router +from backend.api.auth import router as auth_router +from backend.api.jupyter import router as jupyter_router +from backend.api.platform import router as platform_router +from backend.api.resources import router as resources_router +from backend.api.resources_content import router as resources_content_router +from backend.api.schedules.runs import router as schedule_runs_router +from backend.api.schedules.schedules import router as schedules_router +from backend.api.scripts import router as scripts_router +from backend.api.storage import router as storage_api_router +from backend.audit import configure_audit_logging +from backend.clients.rclone import RcloneRCClient +from backend.clients.runtime import RuntimeClient configure_logging(settings.log_level) +configure_audit_logging( + settings.audit_log_dir, + settings.audit_log_retention_days, +) + + +# 审计排除的精确路径集（不含 query）：命中即跳过审计行，诊断日志照常打。 +# 健康检查 / 根路径探针每秒刷审计文件但无业务价值；可用 +# settings.audit_excluded_paths / AUDIT_EXCLUDED_PATHS 覆盖默认值。 +_AUDIT_EXCLUDED: frozenset[str] = frozenset(settings.audit_excluded_paths) @asynccontextmanager @@ -95,6 +108,7 @@ app = create_service_app( app.include_router(auth_router) app.include_router(jupyter_router) app.include_router(resources_router) +app.include_router(resources_content_router) app.include_router(schedule_runs_router) app.include_router(schedules_router) app.include_router(scripts_router) @@ -105,11 +119,35 @@ app.include_router(platform_router) app.include_router(storage_api_router, prefix="/internal") +def _audit_user_id(request: Request) -> str: + """从 cookie / Bearer 头解 JWT 拿 user_id；失败/缺失一律 '-'。 + + 故意不做 DB 查（RequestContext 在路由解析后才注入；审计不该为 + 每请求打 MySQL）。捕获所有异常，让审计失败不拖死业务请求。 + """ + token = request.cookies.get("access_token") + if not token: + auth = request.headers.get("authorization", "") + if auth.lower().startswith("bearer "): + token = auth[7:].strip() + if not token: + return "-" + try: + payload = verify_jwt_token(token) + except (JwtError, Exception): # 任何异常都吞 + return "-" + sub = payload.get("sub") + return sub or "-" + + @app.middleware("http") async def access_log(request: Request, call_next): - # 每个 HTTP 请求都记录方法、路径、状态码和耗时；排查页面请求失败时， - # Docker Desktop 中 backend 容器的 Logs 就会显示这里生成的日志。 + # 诊断：方法/路径/状态码/耗时 走 stderr（loguru default sink） + # 合规：时间/用户/方法/路径/状态码 走独立 audit 文件 sink + # 两条 logger.info() 共用一个出口，便于排查 + # 排除集是精确路径匹配（不含 query），命中即跳过审计行；诊断日志照常。 start = time.perf_counter() + skip_audit = request.url.path in _AUDIT_EXCLUDED try: response = await call_next(request) except Exception: @@ -118,6 +156,14 @@ async def access_log(request: Request, call_next): "request failed {method} {path} after {ms:.1f}ms", method=request.method, path=request.url.path, ms=elapsed_ms, ) + # 异常路径：审计行也要写（status=500 由 unhandled_exception_handler 返回） + if not skip_audit: + logger.bind( + user_id=_audit_user_id(request), + method=request.method, + path=request.url.path, + status=500, + ).info(f"{request.url.path} skip audit") raise elapsed_ms = (time.perf_counter() - start) * 1000 logger.info( @@ -125,6 +171,13 @@ async def access_log(request: Request, call_next): method=request.method, path=request.url.path, status=response.status_code, ms=elapsed_ms, ) + if not skip_audit: + logger.bind( + user_id=_audit_user_id(request), + method=request.method, + path=request.url.path, + status=response.status_code, + ).info(f"{request.url.path} skip audit") return response diff --git a/backend/src/backend/platform.py b/backend/src/backend/platform.py deleted file mode 100644 index 3af6959..0000000 --- a/backend/src/backend/platform.py +++ /dev/null @@ -1,1277 +0,0 @@ -"""系统级管理接口。 - -中文导读：本模块管理全平台的用户、工作区、成员关系和角色权限。它使用 -``system_admin_context`` 进行平台管理员校验，因此不要求请求者先加入某个具体 -工作区；普通工作区内的业务接口则使用 ``request_context``。 - -System-admin (platform-scope) endpoints for workspace & membership management. - -All routes under ``/api/v1/platform/*`` are gated by -:func:`system_admin_context`, which requires the requester to hold a -``Users.platform_role_id`` pointing to a ``Roles`` row whose -``role_code == 'admin'``. Unlike ``backend.dependencies.request_context``, -this dependency does NOT require an active workspace membership — system -admins can manage workspaces before/without being a member of any. - -Endpoints ---------- - -Workspace CRUD:: - - GET /workspaces — list non-deleted workspaces - POST /workspaces — create a new workspace - GET /workspaces/{workspace_id} — single workspace (incl. disabled) - PATCH /workspaces/{workspace_id} — update editable fields - DELETE /workspaces/{workspace_id} — soft delete (cascades memberships) - -Workspace membership CRUD:: - - GET /workspaces/{workspace_id}/members — list active members - POST /workspaces/{workspace_id}/members — add a member - PATCH /workspaces/{workspace_id}/members/{user_id} — update role/status - DELETE /workspaces/{workspace_id}/members/{user_id} — remove a member - -Platform employee roster:: - - GET /employees — list all non-deleted users - POST /employees — create a user without workspace membership - PATCH /employees/{user_id} — update a user's profile, status or platform role - DELETE /employees/{user_id} — soft delete a user (cascades to workspace memberships) - -Role menu-permission management:: - - GET /roles — list platform roles with their permission_codes - GET /roles/{role_code}/permissions — one role's permission_codes - PATCH /roles/{role_code}/permissions — replace a role's permission set (diff-based) - -Invariants ----------- - -* Every workspace must always retain at least one active ``admin`` member. - This is enforced on member PATCH/DELETE AND on - ``PATCH /employees/{user_id}`` demotions, because workspace role is - inherited from ``users.platform_role_id`` and demoting a platform - admin cascades to all of their active memberships. -* A system admin cannot remove their own workspace membership via - ``DELETE .../members/{self}``; the only escape is to delete the entire - workspace, which cascades membership soft-deletion. -* ``DELETE /workspaces/{id}`` is allowed from any non-disabled status and - sets ``status='disabled'`` + ``is_deleted=1`` + ``deleted_at`` on the - workspace and every one of its active memberships. -* The ``admin`` role must always keep ``system.view`` + ``system.manage`` - menu permissions; non-admin roles may never hold ``system.*`` - permissions. Menu permissions gate frontend rendering only — API - authorization always keys off ``role_code == 'admin'``. -""" - -from __future__ import annotations - -import datetime -import re -from dataclasses import dataclass -from typing import Any, Literal - -from common.auth.passwords import hash_password -from common.db.models import ( - Permissions, - RolePermissions, - Roles, - Users, - WorkspaceMembers, - Workspaces, -) -from common.ids import new_ulid -from fastapi import APIRouter, Depends, HTTPException, Request, status -from pydantic import BaseModel, ConfigDict, Field -from sqlalchemy import func, insert, or_, select, update -from sqlalchemy.ext.asyncio import AsyncSession - -from backend.dependencies import current_user, database_session - -router = APIRouter(prefix="/api/v1/platform", tags=["platform"]) - - -# --------------------------------------------------------------------------- -# Constants -# --------------------------------------------------------------------------- - -WORKSPACE_CODE_PATTERN = re.compile(r"^[a-z0-9-]{3,32}$") -LIST_PAGE_SIZE = 100 - -WORKSPACE_EDITABLE_STATUS = ("active", "archived") -MEMBER_ROLE_CODES = ("admin", "developer") -MEMBER_STATUS_VALUES = ("active", "disabled", "locked") - - -# --------------------------------------------------------------------------- -# Schemas -# --------------------------------------------------------------------------- - - -# 创建工作区时前端提交的请求体；禁止未声明字段。 -class WorkspaceCreate(BaseModel): - model_config = ConfigDict(extra="forbid") - - workspace_code: str = Field(min_length=3, max_length=32) - workspace_name: str = Field(min_length=1, max_length=150) - quota_bytes: int = Field(default=0, ge=0) - description: str | None = Field(default=None, max_length=1000) - - -# 编辑工作区时允许修改的字段；禁用操作必须走删除接口而不是直接传状态。 -class WorkspaceUpdate(BaseModel): - model_config = ConfigDict(extra="forbid") - - workspace_name: str | None = Field(default=None, min_length=1, max_length=150) - quota_bytes: int | None = Field(default=None, ge=0) - description: str | None = Field(default=None, max_length=1000) - # 'disabled' is rejected here on purpose — soft delete must go through DELETE. - status: Literal["active", "archived"] | None = None - - -# 将已存在用户加入工作区的请求体；角色继承用户的平台角色。 -class MemberCreate(BaseModel): - """Add a user to a workspace. Role is inherited from the user's - platform role (Users.platform_role_id) — not set here.""" - - model_config = ConfigDict(extra="forbid") - - user_id: str = Field(min_length=26, max_length=26) - - -# 更新成员在该工作区中的可用状态，不直接在这里修改平台角色。 -class MemberUpdate(BaseModel): - """Update a workspace membership's status. Role cannot be changed - via this endpoint — workspace role is always inherited from the - user's platform role. To change a member's role, PATCH - /platform/employees/{user_id} instead.""" - - model_config = ConfigDict(extra="forbid") - - member_status: Literal["active", "disabled", "locked"] | None = None - - -# 新建平台用户的请求体；创建用户不等同于把用户加入某个工作区。 -class PlatformEmployeeCreate(BaseModel): - model_config = ConfigDict(extra="forbid") - - username: str = Field(min_length=2, max_length=64) - display_name: str = Field(min_length=1, max_length=100) - email: str | None = Field(default=None, max_length=255) - password: str = Field(min_length=8, max_length=72) - role_code: Literal["admin", "developer"] | None = None - - -# 修改平台用户资料、状态或平台角色的请求体。 -class PlatformEmployeeUpdate(BaseModel): - model_config = ConfigDict(extra="forbid") - - display_name: str | None = Field(default=None, min_length=1, max_length=100) - email: str | None = Field(default=None, max_length=255) - status: Literal["active", "disabled", "locked"] | None = None - role_code: Literal["admin", "developer"] | None = None - - -# 用完整权限集合替换某个平台角色菜单权限的请求体。 -class RolePermissionsPatch(BaseModel): - """Replace a platform role's permission set wholesale. - - Empty list is allowed (revokes all permissions) for non-`admin` - roles. The PATCH endpoint rejects emptying an `admin` role of its - system.* permissions; see ``patch_role_permissions`` for the - load-bearing guard order. - """ - - model_config = ConfigDict(extra="forbid") - - permission_codes: list[str] = Field(default_factory=list, max_length=64) - - -# --------------------------------------------------------------------------- -# System-admin context dependency -# --------------------------------------------------------------------------- - - -@dataclass(frozen=True) -class SystemAdminContext: - """通过系统管理员校验后的上下文，只包含当前用户和请求追踪 ID。""" - - """Resolved identity for a system-admin request. - - Carries the request id, the authenticated user row, and the resolved - ``Roles`` row the user holds via ``Users.platform_role_id``. By - construction the role's ``role_code`` is ``"admin"``. - """ - - request_id: str - user: Users - platform_role: Roles - - -async def system_admin_context( - request: Request, - session: AsyncSession = Depends(database_session), -) -> SystemAdminContext: - """验证当前用户是否为平台管理员，供 /api/v1/platform 下的路由依赖。""" - """Resolve the requester as a system admin. - - Steps: - 1. Reuse :func:`backend.dependencies.current_user` to validate the JWT - cookie and fetch the active ``Users`` row (raises 401 on failure). - 2. Require ``Users.platform_role_id`` to point to a row whose - ``role_code == 'admin'`` — anything else is 403. - """ - user = await current_user(request, session) - if user.platform_role_id is None: - raise HTTPException( - status.HTTP_403_FORBIDDEN, - "需要系统管理员权限", - ) - platform_role = await session.scalar( - select(Roles).where(Roles.role_id == user.platform_role_id) - ) - if platform_role is None or platform_role.role_code != "admin": - raise HTTPException( - status.HTTP_403_FORBIDDEN, - "需要系统管理员权限", - ) - request_id = request.headers.get("X-Request-ID") or new_ulid() - return SystemAdminContext( - request_id=request_id, - user=user, - platform_role=platform_role, - ) - - -# --------------------------------------------------------------------------- -# Payload helpers -# --------------------------------------------------------------------------- - - -def workspace_payload(workspace: Workspaces) -> dict[str, Any]: - return { - "workspace_id": workspace.workspace_id, - "workspace_code": workspace.workspace_code, - "workspace_name": workspace.workspace_name, - "active_root_uri": workspace.active_root_uri, - "quota_bytes": workspace.quota_bytes, - "status": workspace.status, - "description": workspace.description, - "created_by": workspace.created_by, - "created_at": workspace.created_at.isoformat(), - "updated_at": ( - workspace.updated_at.isoformat() if workspace.updated_at else None - ), - } - - -def member_payload( - user: Users, - role: Roles, - membership: WorkspaceMembers, -) -> dict[str, Any]: - return { - "user_id": user.user_id, - "username": user.username, - "display_name": user.display_name, - "email": user.email, - "user_status": user.status, - "role_code": role.role_code, - "role_name": role.role_name, - "member_status": membership.member_status, - "joined_at": membership.joined_at.isoformat(), - } - - -def platform_employee_payload( - user: Users, - role: Roles | None, -) -> dict[str, Any]: - return { - "user_id": user.user_id, - "username": user.username, - "display_name": user.display_name, - "email": user.email, - "status": user.status, - "role_code": role.role_code if role is not None else None, - "role_name": role.role_name if role is not None else None, - "created_at": user.created_at.isoformat(), - } - - -# --------------------------------------------------------------------------- -# Internal helpers -# --------------------------------------------------------------------------- - - -async def _load_workspace(session: AsyncSession, workspace_id: str) -> Workspaces: - workspace = await session.get(Workspaces, workspace_id) - if workspace is None: - raise HTTPException(status.HTTP_404_NOT_FOUND, "workspace 不存在") - return workspace - - -async def _load_role_by_code(session: AsyncSession, role_code: str) -> Roles: - role = await session.scalar(select(Roles).where(Roles.role_code == role_code)) - if role is None: - raise HTTPException( - status.HTTP_422_UNPROCESSABLE_ENTITY, - f"角色 {role_code} 不存在", - ) - return role - - -async def _count_active_admins( - session: AsyncSession, - workspace_id: str, - exclude_user_id: str | None = None, -) -> int: - """Count active admin members of ``workspace_id``. - - Pass ``exclude_user_id`` when checking "would X be the last admin?" - before mutating X. - """ - admin_role = await _load_role_by_code(session, "admin") - stmt = ( - select(func.count()) - .select_from(WorkspaceMembers) - .where( - WorkspaceMembers.workspace_id == workspace_id, - WorkspaceMembers.role_id == admin_role.role_id, - WorkspaceMembers.member_status == "active", - WorkspaceMembers.is_deleted == 0, - ) - ) - if exclude_user_id is not None: - stmt = stmt.where(WorkspaceMembers.user_id != exclude_user_id) - return int(await session.scalar(stmt) or 0) - - -async def _count_active_system_admins( - session: AsyncSession, - exclude_user_id: str | None = None, -) -> int: - """Count active system admins across the platform. - - Pass ``exclude_user_id`` when checking "would X be the last admin?" - before mutating X. - """ - admin_role = await _load_role_by_code(session, "admin") - stmt = ( - select(func.count()) - .select_from(Users) - .where( - Users.status == "active", - Users.is_deleted == 0, - Users.platform_role_id == admin_role.role_id, - ) - ) - if exclude_user_id is not None: - stmt = stmt.where(Users.user_id != exclude_user_id) - return int(await session.scalar(stmt) or 0) - - -def _envelope(request_id: str, data: Any, meta: dict[str, Any] | None = None) -> dict[str, Any]: - return { - "request_id": request_id, - "data": data, - "meta": meta or {}, - } - - -# --------------------------------------------------------------------------- -# Platform employee roster -# --------------------------------------------------------------------------- - - -# 列出整个平台的非删除用户；不局限于某一个工作区。 -@router.get("/employees") -async def list_platform_employees( - context: SystemAdminContext = Depends(system_admin_context), - session: AsyncSession = Depends(database_session), -) -> dict[str, Any]: - """List every non-soft-deleted platform user.""" - rows = ( - await session.execute( - select(Users, Roles) - .outerjoin(Roles, Roles.role_id == Users.platform_role_id) - .where(Users.is_deleted == 0) - .order_by(Users.created_at, Users.user_id) - ) - ).all() - return _envelope( - context.request_id, - [platform_employee_payload(user, role) for user, role in rows], - {"count": len(rows)}, - ) - - -# 创建平台用户；后续可再通过成员接口把该用户加入工作区。 -@router.post("/employees", status_code=status.HTTP_201_CREATED) -async def create_platform_employee( - payload: PlatformEmployeeCreate, - context: SystemAdminContext = Depends(system_admin_context), - session: AsyncSession = Depends(database_session), -) -> dict[str, Any]: - """Create a platform user without assigning workspace membership.""" - username = payload.username.strip() - display_name = payload.display_name.strip() - duplicate_conditions = [Users.username == username] - if payload.email: - duplicate_conditions.append(Users.email == payload.email.strip()) - duplicate = await session.scalar( - select(Users.user_id).where(or_(*duplicate_conditions)) - ) - if duplicate is not None: - raise HTTPException(status.HTTP_409_CONFLICT, "用户名或邮箱已存在") - - new_role: Roles | None = None - if payload.role_code is not None: - new_role = await _load_role_by_code(session, payload.role_code) - - user = Users( - user_id=new_ulid(), - username=username, - display_name=display_name, - email=payload.email.strip() if payload.email else None, - password_hash=hash_password(payload.password), - status="active", - platform_role_id=new_role.role_id if new_role is not None else None, - ) - session.add(user) - await session.flush() - await session.refresh(user) - return _envelope( - context.request_id, - platform_employee_payload(user, new_role), - ) - - -# 更新平台用户资料、账号状态或平台角色，同时保护最少管理员等约束。 -@router.patch("/employees/{user_id}") -async def update_platform_employee( - user_id: str, - payload: PlatformEmployeeUpdate, - context: SystemAdminContext = Depends(system_admin_context), - session: AsyncSession = Depends(database_session), -) -> dict[str, Any]: - """Update a platform user's profile, status, or platform role. - - Changing ``role_code`` cascades: every active ``workspace_members`` - row of the user is rewritten to the new role (workspace role is - inherited from the platform role). Demoting admin → developer is - rejected with 409 when it would leave any workspace without an - active admin member, or the platform without an active system - admin. Self-demotion is always rejected. - """ - user = await session.get(Users, user_id) - if user is None or user.is_deleted != 0: - raise HTTPException(status.HTTP_404_NOT_FOUND, "用户不存在") - - is_self = user_id == context.user.user_id - if ( - is_self - and payload.status is not None - and payload.status != "active" - ): - raise HTTPException(status.HTTP_409_CONFLICT, "不能停用当前登录账号") - - current_role: Roles | None = None - if user.platform_role_id is not None: - current_role = await session.scalar( - select(Roles).where(Roles.role_id == user.platform_role_id) - ) - is_current_system_admin = ( - user.status == "active" - and current_role is not None - and current_role.role_code == "admin" - ) - - new_role: Roles | None = None - if payload.role_code is not None: - new_role = await _load_role_by_code(session, payload.role_code) - next_status = payload.status if payload.status is not None else user.status - - leaves_admin_pool = ( - is_current_system_admin - and ( - next_status != "active" - or (new_role is not None and new_role.role_code != "admin") - ) - ) - if leaves_admin_pool: - remaining = await _count_active_system_admins( - session, exclude_user_id=user_id, - ) - if remaining == 0: - raise HTTPException( - status.HTTP_409_CONFLICT, - "platform 必须保留至少一个 active 系统管理员", - ) - - # Workspace-level last-admin guard for the demote path. The role_code - # sync below rewrites ``workspace_members.role_id`` for every active - # membership of this user, so demoting admin → developer would - # silently strip workspace admin coverage anywhere this user is the - # sole active admin member. ``update_member`` / ``remove_member`` - # guard the same invariant via ``_count_active_admins``; this - # endpoint must too, now that it can change workspace roles. - demotes_admin = ( - is_current_system_admin - and new_role is not None - and new_role.role_code != "admin" - ) - if demotes_admin: - assert current_role is not None # implied by is_current_system_admin - admin_memberships = ( - await session.execute( - select(WorkspaceMembers.workspace_id) - .where( - WorkspaceMembers.user_id == user_id, - WorkspaceMembers.role_id == current_role.role_id, - WorkspaceMembers.member_status == "active", - WorkspaceMembers.is_deleted == 0, - ) - ) - ).all() - orphaned: list[str] = [] - for (ws_id,) in admin_memberships: - remaining_ws = await _count_active_admins( - session, ws_id, exclude_user_id=user_id, - ) - if remaining_ws == 0: - orphaned.append(ws_id) - if orphaned: - codes = ( - await session.execute( - select(Workspaces.workspace_code).where( - Workspaces.workspace_id.in_(orphaned) - ) - ) - ).all() - names = sorted(row[0] for row in codes) - raise HTTPException( - status.HTTP_409_CONFLICT, - f"以下 workspace 将失去唯一 active admin: {names};" - "请先在这些 workspace 中指定其他 admin,再降级该用户", - ) - - if is_self and new_role is not None and new_role.role_code != "admin": - raise HTTPException(status.HTTP_409_CONFLICT, "不能降级自身管理员角色") - - if payload.display_name is not None: - user.display_name = payload.display_name.strip() - if payload.email is not None: - user.email = payload.email.strip() or None - if payload.status is not None: - user.status = payload.status - if new_role is not None: - user.platform_role_id = new_role.role_id - # Workspace role is always inherited from the platform role - # (§7.5/§7.6 cannot change it). Keep workspace_members.role_id - # in sync so downstream reads — `/me` workspaces[].role_code, - # load_active_membership, §7.7 DELETE last-admin guard — - # see the up-to-date role. Without this sync, a user demoted - # from admin → developer would still appear as admin in every - # workspace they belong to until they leave and re-join. - await session.execute( - update(WorkspaceMembers) - .where( - WorkspaceMembers.user_id == user.user_id, - WorkspaceMembers.is_deleted == 0, - ) - .values(role_id=new_role.role_id) - ) - - await session.flush() - await session.refresh(user) - - response_role: Roles | None = None - if user.platform_role_id is not None: - response_role = await session.scalar( - select(Roles).where(Roles.role_id == user.platform_role_id) - ) - return _envelope( - context.request_id, platform_employee_payload(user, response_role), - ) - - -# 软删除平台用户，并级联标记其工作区成员关系为删除。 -@router.delete("/employees/{user_id}") -async def delete_platform_employee( - user_id: str, - context: SystemAdminContext = Depends(system_admin_context), - session: AsyncSession = Depends(database_session), -) -> dict[str, Any]: - """Soft delete a platform user and cascade-soft-delete workspace memberships.""" - user = await session.get(Users, user_id) - if user is None or user.is_deleted != 0: - raise HTTPException(status.HTTP_404_NOT_FOUND, "用户不存在") - - if user_id == context.user.user_id: - raise HTTPException(status.HTTP_409_CONFLICT, "不能删除当前登录账号") - - if user.status == "active" and user.platform_role_id is not None: - current_admin_role = await session.scalar( - select(Roles).where(Roles.role_id == user.platform_role_id) - ) - if current_admin_role is not None and current_admin_role.role_code == "admin": - remaining = await _count_active_system_admins( - session, exclude_user_id=user_id, - ) - if remaining == 0: - raise HTTPException( - status.HTTP_409_CONFLICT, - "platform 必须保留至少一个 active 系统管理员", - ) - - now = datetime.datetime.utcnow() - user.status = "disabled" - user.is_deleted = 1 - user.deleted_at = now - await session.execute( - update(WorkspaceMembers) - .where( - WorkspaceMembers.user_id == user_id, - WorkspaceMembers.is_deleted == 0, - ) - .values(is_deleted=1, deleted_at=now) - ) - await session.flush() - return _envelope( - context.request_id, - {"user_id": user_id, "deleted": True}, - ) - - -# --------------------------------------------------------------------------- -# Workspace CRUD -# --------------------------------------------------------------------------- - - -# 列出平台中全部未删除工作区。 -@router.get("/workspaces") -async def list_workspaces( - context: SystemAdminContext = Depends(system_admin_context), - session: AsyncSession = Depends(database_session), -) -> dict[str, Any]: - """List active/archived workspaces. Soft-deleted rows are filtered out. - - Silent ``pageSize=100`` cap — YAGNI on real pagination until needed. - """ - rows = ( - await session.execute( - select(Workspaces) - .where( - Workspaces.status != "disabled", - Workspaces.is_deleted == 0, - ) - .order_by(Workspaces.created_at, Workspaces.workspace_id) - .limit(LIST_PAGE_SIZE) - ) - ).scalars().all() - return _envelope( - context.request_id, - [workspace_payload(w) for w in rows], - {"count": len(rows), "page_size": LIST_PAGE_SIZE}, - ) - - -# 创建工作区，并将当前系统管理员初始化为该工作区管理员。 -@router.post("/workspaces", status_code=status.HTTP_201_CREATED) -async def create_workspace( - payload: WorkspaceCreate, - context: SystemAdminContext = Depends(system_admin_context), - session: AsyncSession = Depends(database_session), -) -> dict[str, Any]: - """Create a workspace and auto-join the creator as an admin member.""" - if not WORKSPACE_CODE_PATTERN.fullmatch(payload.workspace_code): - raise HTTPException( - status.HTTP_422_UNPROCESSABLE_ENTITY, - "workspace_code 必须匹配 ^[a-z0-9-]{3,32}$", - ) - duplicate = await session.scalar( - select(Workspaces.workspace_id).where( - Workspaces.workspace_code == payload.workspace_code, - ) - ) - if duplicate is not None: - raise HTTPException(status.HTTP_409_CONFLICT, "workspace_code 已存在") - - admin_role = await _load_role_by_code(session, "admin") - workspace_id = new_ulid() - workspace = Workspaces( - workspace_id=workspace_id, - workspace_code=payload.workspace_code, - workspace_name=payload.workspace_name, - active_root_uri=f"s3://workspaces/{workspace_id}/", - quota_bytes=payload.quota_bytes, - status="active", - created_by=context.user.user_id, - description=payload.description, - ) - session.add(workspace) - session.add( - WorkspaceMembers( - workspace_id=workspace_id, - user_id=context.user.user_id, - role_id=admin_role.role_id, - member_status="active", - ) - ) - await session.flush() - await session.refresh(workspace) - return _envelope(context.request_id, workspace_payload(workspace)) - - -# 读取单个工作区详情，包含已归档或禁用状态。 -@router.get("/workspaces/{workspace_id}") -async def get_workspace( - workspace_id: str, - context: SystemAdminContext = Depends(system_admin_context), - session: AsyncSession = Depends(database_session), -) -> dict[str, Any]: - """Fetch a single workspace — even soft-deleted ones are reachable.""" - workspace = await _load_workspace(session, workspace_id) - return _envelope(context.request_id, workspace_payload(workspace)) - - -# 更新工作区可编辑属性，例如名称、配额、描述和归档状态。 -@router.patch("/workspaces/{workspace_id}") -async def update_workspace( - workspace_id: str, - payload: WorkspaceUpdate, - context: SystemAdminContext = Depends(system_admin_context), - session: AsyncSession = Depends(database_session), -) -> dict[str, Any]: - """Patch editable workspace fields. ``status='disabled'`` is rejected.""" - workspace = await _load_workspace(session, workspace_id) - if workspace.status == "disabled": - raise HTTPException( - status.HTTP_409_CONFLICT, - "workspace 已删除,无法修改", - ) - if payload.workspace_name is not None: - workspace.workspace_name = payload.workspace_name.strip() - if payload.quota_bytes is not None: - workspace.quota_bytes = payload.quota_bytes - if payload.description is not None: - workspace.description = payload.description - if payload.status is not None: - workspace.status = payload.status - await session.flush() - await session.refresh(workspace) - return _envelope(context.request_id, workspace_payload(workspace)) - - -# 软删除/禁用工作区，并级联处理其活动成员关系。 -@router.delete("/workspaces/{workspace_id}") -async def delete_workspace( - workspace_id: str, - context: SystemAdminContext = Depends(system_admin_context), - session: AsyncSession = Depends(database_session), -) -> dict[str, Any]: - """Soft-delete a workspace and cascade-soft-delete its memberships. - - Allowed from any non-disabled status (active or archived). The - membership cascade is what lets system admins leave a workspace — - there is no per-member DELETE escape for self-removal. - """ - workspace = await _load_workspace(session, workspace_id) - if workspace.status == "disabled": - raise HTTPException( - status.HTTP_409_CONFLICT, - "workspace 已被删除", - ) - now = datetime.datetime.utcnow() - workspace.status = "disabled" - workspace.is_deleted = 1 - workspace.deleted_at = now - await session.execute( - update(WorkspaceMembers) - .where( - WorkspaceMembers.workspace_id == workspace_id, - WorkspaceMembers.is_deleted == 0, - ) - .values(is_deleted=1, deleted_at=now) - ) - await session.flush() - await session.refresh(workspace) - return _envelope(context.request_id, workspace_payload(workspace)) - - -# --------------------------------------------------------------------------- -# Workspace membership CRUD -# --------------------------------------------------------------------------- - - -# 列出一个工作区的活动成员与成员状态。 -@router.get("/workspaces/{workspace_id}/members") -async def list_members( - workspace_id: str, - context: SystemAdminContext = Depends(system_admin_context), - session: AsyncSession = Depends(database_session), -) -> dict[str, Any]: - """List active and historical (non-soft-deleted) members of a workspace.""" - await _load_workspace(session, workspace_id) - rows = ( - await session.execute( - select(Users, Roles, WorkspaceMembers) - .join( - WorkspaceMembers, - WorkspaceMembers.user_id == Users.user_id, - ) - .join(Roles, Roles.role_id == WorkspaceMembers.role_id) - .where( - WorkspaceMembers.workspace_id == workspace_id, - WorkspaceMembers.is_deleted == 0, - ) - .order_by(WorkspaceMembers.joined_at, Users.user_id) - .limit(LIST_PAGE_SIZE) - ) - ).all() - return _envelope( - context.request_id, - [member_payload(u, r, m) for u, r, m in rows], - {"count": len(rows), "page_size": LIST_PAGE_SIZE}, - ) - - -# 将已有平台用户加入指定工作区。 -@router.post( - "/workspaces/{workspace_id}/members", - status_code=status.HTTP_201_CREATED, -) -async def add_member( - workspace_id: str, - payload: MemberCreate, - context: SystemAdminContext = Depends(system_admin_context), - session: AsyncSession = Depends(database_session), -) -> dict[str, Any]: - """Add a user to a workspace. The new row starts with member_status='active'. - - The role is inherited from the target user's ``platform_role_id``; - the request body does NOT take a ``role_code``. To change a member's - role, PATCH ``/api/v1/platform/employees/{user_id}`` instead. - """ - await _load_workspace(session, workspace_id) - user = await session.get(Users, payload.user_id) - if user is None or user.is_deleted != 0: - raise HTTPException(status.HTTP_404_NOT_FOUND, "用户不存在") - if user.status != "active": - raise HTTPException( - status.HTTP_409_CONFLICT, - f"用户状态为 {user.status},无法加入 workspace", - ) - if user.platform_role_id is None: - raise HTTPException( - status.HTTP_409_CONFLICT, - "目标用户尚未分配平台角色,无法加入 workspace;" - "请先 PATCH /api/v1/platform/employees/{user_id} 设置 role_code", - ) - role = await session.scalar( - select(Roles).where( - Roles.role_id == user.platform_role_id, - Roles.is_deleted == 0, - ) - ) - if role is None: - raise HTTPException( - status.HTTP_422_UNPROCESSABLE_ENTITY, - "用户的平台角色行不存在或已被删除", - ) - # ``WorkspaceMembers`` 的主键是 ``(workspace_id, user_id)`` 复合 PK, - # 而 ``remove_member`` / ``delete_platform_employee`` 都是软删除 (保留行, - # 仅置 ``is_deleted=1``). 因此这里必须按主键查整行,而不是只看活跃行: - # 否则软删行会被 active-duplicate 检查漏过,然后 INSERT 直接撞 PK. - existing = await session.scalar( - select(WorkspaceMembers).where( - WorkspaceMembers.workspace_id == workspace_id, - WorkspaceMembers.user_id == payload.user_id, - ) - ) - if existing is not None: - if existing.is_deleted == 0: - raise HTTPException( - status.HTTP_409_CONFLICT, - "用户已是该 workspace 成员;workspace 角色继承自平台角色," - "要变更请 PATCH /api/v1/platform/employees/{user_id} 修改 role_code", - ) - # 复活软删除行. 保留 ``joined_at`` 作为历史记录;``role_id`` 重新继承 - # 当前用户的平台角色 (用户在中间可能改过 platform_role);清掉 - # ``deleted_at`` 标记本轮已不在软删状态. - existing.is_deleted = 0 - existing.deleted_at = None - existing.role_id = role.role_id - existing.member_status = "active" - await session.flush() - await session.refresh(existing) - return _envelope( - context.request_id, member_payload(user, role, existing), - ) - membership = WorkspaceMembers( - workspace_id=workspace_id, - user_id=payload.user_id, - role_id=role.role_id, - member_status="active", - ) - session.add(membership) - await session.flush() - await session.refresh(membership) - return _envelope(context.request_id, member_payload(user, role, membership)) - - -# 更新成员状态，例如禁用或锁定；同时保证工作区不会失去最后一个管理员。 -@router.patch("/workspaces/{workspace_id}/members/{user_id}") -async def update_member( - workspace_id: str, - user_id: str, - payload: MemberUpdate, - context: SystemAdminContext = Depends(system_admin_context), - session: AsyncSession = Depends(database_session), -) -> dict[str, Any]: - """Update a workspace membership's status. Role is not editable here. - - Workspace role is always inherited from the user's platform role - (``Users.platform_role_id``). To change role, PATCH - ``/api/v1/platform/employees/{user_id}`` instead. - - Last-admin guard still applies to ``member_status`` changes: setting - the only active admin to ``disabled``/``locked`` would leave the - workspace without admin coverage. - """ - await _load_workspace(session, workspace_id) - row = ( - await session.execute( - select(Users, Roles, WorkspaceMembers) - .join( - WorkspaceMembers, - WorkspaceMembers.user_id == Users.user_id, - ) - .join(Roles, Roles.role_id == WorkspaceMembers.role_id) - .where( - WorkspaceMembers.workspace_id == workspace_id, - WorkspaceMembers.user_id == user_id, - WorkspaceMembers.is_deleted == 0, - ) - ) - ).first() - if row is None: - raise HTTPException(status.HTTP_404_NOT_FOUND, "成员不存在") - user, role, membership = row - - if payload.member_status is not None and payload.member_status != membership.member_status: - if ( - role.role_code == "admin" - and payload.member_status != "active" - ): - remaining = await _count_active_admins( - session, workspace_id, exclude_user_id=user_id, - ) - if remaining == 0: - raise HTTPException( - status.HTTP_409_CONFLICT, - "workspace 必须保留至少一个 admin", - ) - membership.member_status = payload.member_status - - await session.flush() - await session.refresh(membership) - return _envelope(context.request_id, member_payload(user, role, membership)) - - -# 移除某个工作区成员，并保护最后一名管理员及当前操作者的安全约束。 -@router.delete("/workspaces/{workspace_id}/members/{user_id}") -async def remove_member( - workspace_id: str, - user_id: str, - context: SystemAdminContext = Depends(system_admin_context), - session: AsyncSession = Depends(database_session), -) -> dict[str, Any]: - """Soft-delete a workspace membership. - - System admins cannot remove themselves — the only escape is to delete - the entire workspace, which cascades membership soft-deletion. - """ - await _load_workspace(session, workspace_id) - if user_id == context.user.user_id: - raise HTTPException( - status.HTTP_403_FORBIDDEN, - "系统管理员不能把自己从 workspace 移除;如需退出,请删除整个 workspace", - ) - row = ( - await session.execute( - select(Roles, WorkspaceMembers) - .join(Roles, Roles.role_id == WorkspaceMembers.role_id) - .where( - WorkspaceMembers.workspace_id == workspace_id, - WorkspaceMembers.user_id == user_id, - WorkspaceMembers.is_deleted == 0, - ) - ) - ).first() - if row is None: - raise HTTPException(status.HTTP_404_NOT_FOUND, "成员不存在") - role, membership = row - if role.role_code == "admin" and membership.member_status == "active": - remaining = await _count_active_admins( - session, workspace_id, exclude_user_id=user_id, - ) - if remaining == 0: - raise HTTPException( - status.HTTP_409_CONFLICT, - "workspace 必须保留至少一个 admin", - ) - membership.is_deleted = 1 - membership.deleted_at = datetime.datetime.utcnow() - await session.flush() - return _envelope( - context.request_id, - {"workspace_id": workspace_id, "user_id": user_id, "removed": True}, - ) - - -# --------------------------------------------------------------------------- -# Platform role permission management -# --------------------------------------------------------------------------- -# -# Menu permissions for the platform admin UI. The auth gate -# (system_admin_context) still keys off role_code == "admin"; these -# endpoints only control the menu items the frontend renders, not -# which API calls a user may make. See migrations/ -# versions/e5f6a7b8c9d0_seed_role_permissions_and_fix_scope.py for -# the seed values. - - -async def _load_platform_role_by_code( - session: AsyncSession, role_code: str -) -> Roles: - """Load a platform-scoped role by code; 404 if missing or not platform-scope.""" - role = await session.scalar( - select(Roles).where( - Roles.role_code == role_code, Roles.is_deleted == 0, - ) - ) - if role is None or role.role_scope != "platform": - raise HTTPException( - status.HTTP_404_NOT_FOUND, f"platform 角色 {role_code} 不存在", - ) - return role - - -async def _load_role_permission_codes( - session: AsyncSession, role_id: str -) -> list[str]: - """Return the active permission_codes for a role, ordered by code.""" - rows = ( - await session.execute( - select(Permissions.permission_code) - .join( - RolePermissions, - RolePermissions.permission_id == Permissions.permission_id, - ) - .where( - RolePermissions.role_id == role_id, - RolePermissions.is_deleted == 0, - Permissions.is_deleted == 0, - ) - .order_by(Permissions.permission_code) - ) - ).all() - return [row[0] for row in rows] - - -def _role_payload(role: Roles, permission_codes: list[str]) -> dict[str, Any]: - return { - "role_id": role.role_id, - "role_code": role.role_code, - "role_name": role.role_name, - "is_builtin": bool(role.is_builtin), - "permission_codes": permission_codes, - } - - -# 列出平台角色及其拥有的菜单权限代码。 -@router.get("/roles") -async def list_platform_roles( - context: SystemAdminContext = Depends(system_admin_context), - session: AsyncSession = Depends(database_session), -) -> dict[str, Any]: - """List every platform-scoped role with its current permission_codes.""" - roles = ( - ( - await session.scalars( - select(Roles) - .where(Roles.role_scope == "platform", Roles.is_deleted == 0) - .order_by(Roles.role_code) - ) - ).all() - ) - payload = [] - for role in roles: - codes = await _load_role_permission_codes(session, role.role_id) - payload.append(_role_payload(role, codes)) - return _envelope( - context.request_id, payload, {"count": len(payload)}, - ) - - -# 获取一个角色当前配置的权限代码集合。 -@router.get("/roles/{role_code}/permissions") -async def get_role_permissions( - role_code: str, - context: SystemAdminContext = Depends(system_admin_context), - session: AsyncSession = Depends(database_session), -) -> dict[str, Any]: - """Return one platform role's permission_codes.""" - role = await _load_platform_role_by_code(session, role_code) - codes = await _load_role_permission_codes(session, role.role_id) - return _envelope( - context.request_id, _role_payload(role, codes), - ) - - -# 以请求中的完整集合更新角色权限，并保留管理员角色的必要系统权限。 -@router.patch("/roles/{role_code}/permissions") -async def patch_role_permissions( - role_code: str, - payload: RolePermissionsPatch, - context: SystemAdminContext = Depends(system_admin_context), - session: AsyncSession = Depends(database_session), -) -> dict[str, Any]: - """Replace a platform role's permission set wholesale. - - Guard order: - - 1. Load the role. Reject 404 if it is missing or not - platform-scoped. - 2. Admin role: the patched ``permission_codes`` MUST still include - both ``system.view`` and ``system.manage``. Otherwise every - active admin loses the menu entry to this very endpoint and - the platform locks itself out. Reject with 409. (No last-admin - count is needed here — menu permissions never gate API access; - ``system_admin_context`` keys off ``role_code == 'admin'``.) - 3. Non-admin role: ``system.*`` codes are rejected with 422 — - they would render a system-admin menu entry whose API calls - all 403. - 4. Validate every code resolves to a non-deleted ``Permissions`` - row; unknown codes → 422. - 5. Write: diff-based. Only soft-delete codes leaving the set, - only insert codes entering it. The ``(role_id, permission_id)`` - PRIMARY KEY still occupies soft-deleted rows, so a blanket - "delete-all then insert-all" would IntegrityError. - Repeat-with-same-payload is a no-op. - """ - role = await _load_platform_role_by_code(session, role_code) - - new_codes = list(dict.fromkeys(payload.permission_codes)) - - if role.role_code == "admin": - keeps_admin_entry = ( - "system.view" in new_codes and "system.manage" in new_codes - ) - if not keeps_admin_entry: - raise HTTPException( - status.HTTP_409_CONFLICT, - "admin 角色必须保留 system.view 与 system.manage 权限", - ) - else: - # Menu permissions are a frontend-display signal only — backend - # authorization keeps keying off role_code == "admin". Letting a - # non-admin role hold system.* permissions would render the - # system-admin entry in the developer's UI while every - # /api/v1/platform/* call still returns 403. Reject with 422 so - # the failure is unambiguous about *what* the input violated. - leaked_system = [ - code for code in new_codes if code.startswith("system.") - ] - if leaked_system: - raise HTTPException( - status.HTTP_422_UNPROCESSABLE_ENTITY, - f"非 admin 角色不能拥有 system.* 权限: {leaked_system}", - ) - - # 4. Validate every requested permission_code exists and is live. - if new_codes: - rows = ( - await session.execute( - select(Permissions.permission_code).where( - Permissions.permission_code.in_(new_codes), - Permissions.is_deleted == 0, - ) - ) - ).all() - found = {row[0] for row in rows} - missing = [code for code in new_codes if code not in found] - if missing: - raise HTTPException( - status.HTTP_422_UNPROCESSABLE_ENTITY, - f"未知的 permission_code: {missing}", - ) - - # 5. Write: diff-based soft-delete + insert. - # The (role_id, permission_id) PRIMARY KEY still occupies the slot - # of soft-deleted rows, so a "delete-all then insert-all" approach - # would IntegrityError on any code that was already linked. - # Instead: only soft-delete codes NOT in the new set, only INSERT - # codes NOT already active. Repeat-with-same-payload is a no-op. - now = datetime.datetime.utcnow() - current_codes = set( - await _load_role_permission_codes(session, role.role_id) - ) - new_set = set(new_codes) - - codes_to_drop = current_codes - new_set - codes_to_add = new_set - current_codes - - if codes_to_drop: - # Resolve to permission_ids then soft-delete by id pair. - drop_ids = ( - await session.execute( - select(Permissions.permission_id).where( - Permissions.permission_code.in_(codes_to_drop), - Permissions.is_deleted == 0, - ) - ) - ).all() - drop_id_values = [row[0] for row in drop_ids] - await session.execute( - update(RolePermissions) - .where( - RolePermissions.role_id == role.role_id, - RolePermissions.permission_id.in_(drop_id_values), - RolePermissions.is_deleted == 0, - ) - .values(is_deleted=1, deleted_at=now) - ) - - if codes_to_add: - add_ids = ( - await session.execute( - select(Permissions.permission_id).where( - Permissions.permission_code.in_(codes_to_add), - Permissions.is_deleted == 0, - ) - ) - ).all() - if add_ids: - await session.execute( - insert(RolePermissions), - [ - {"role_id": role.role_id, "permission_id": pid} - for pid, in add_ids - ], - ) - - await session.flush() - final_codes = await _load_role_permission_codes(session, role.role_id) - return _envelope( - context.request_id, _role_payload(role, final_codes), - ) - - -__all__ = [ - "SystemAdminContext", - "router", - "system_admin_context", -] diff --git a/backend/src/backend/schemas/auth.py b/backend/src/backend/schemas/auth.py new file mode 100644 index 0000000..e3c2488 --- /dev/null +++ b/backend/src/backend/schemas/auth.py @@ -0,0 +1 @@ +"""Reserved for stage-3 extraction. Currently empty.""" diff --git a/backend/src/backend/schemas/common.py b/backend/src/backend/schemas/common.py new file mode 100644 index 0000000..2879f5d --- /dev/null +++ b/backend/src/backend/schemas/common.py @@ -0,0 +1,12 @@ +"""跨域共享的请求/响应模型。 + +目前唯一成员是 `DownloadUrlRequest`：资源（resources）和脚本版本 +（scripts）两个域都要用它生成预签名下载 URL。 +""" + +from common.schemas import StrictModel +from pydantic import Field + + +class DownloadUrlRequest(StrictModel): + expires_seconds: int = Field(default=300, ge=30, le=3600) diff --git a/backend/src/backend/schemas/jupyter.py b/backend/src/backend/schemas/jupyter.py new file mode 100644 index 0000000..e3c2488 --- /dev/null +++ b/backend/src/backend/schemas/jupyter.py @@ -0,0 +1 @@ +"""Reserved for stage-3 extraction. Currently empty.""" diff --git a/backend/src/backend/schemas/platform.py b/backend/src/backend/schemas/platform.py new file mode 100644 index 0000000..e3c2488 --- /dev/null +++ b/backend/src/backend/schemas/platform.py @@ -0,0 +1 @@ +"""Reserved for stage-3 extraction. Currently empty.""" diff --git a/backend/src/backend/schemas.py b/backend/src/backend/schemas/resources.py similarity index 62% rename from backend/src/backend/schemas.py rename to backend/src/backend/schemas/resources.py index 958588f..78e5403 100644 --- a/backend/src/backend/schemas.py +++ b/backend/src/backend/schemas/resources.py @@ -39,40 +39,5 @@ class CompleteResourceUploadRequest(StrictModel): visibility: Literal["private", "workspace", "public"] = "private" -class CreateScriptRequest(StrictModel): - script_name: str = Field(min_length=1, max_length=255) - script_type: Literal["python", "notebook"] - content: str = Field(max_length=10 * 1024 * 1024) - visibility: Literal["private", "workspace", "public"] = "private" - parent_path: str | None = Field(default=None, max_length=1024) - - -class CreateWorkspaceDirectoryRequest(StrictModel): - directory_name: str = Field(min_length=1, max_length=255) - parent_path: str = Field(default="", max_length=1024) - - -class UpdateScriptRequest(StrictModel): - content: str = Field(max_length=10 * 1024 * 1024) - - -class LockScriptRequest(StrictModel): - is_locked: bool - - -class PublishVersionRequest(StrictModel): - source_object_id: str | None = Field( - default=None, - min_length=26, - max_length=26, - ) - release_note: str | None = Field(default=None, max_length=1000) - visibility: Literal["private", "workspace", "public"] = "workspace" - - -class DownloadUrlRequest(StrictModel): - expires_seconds: int = Field(default=300, ge=30, le=3600) - - class ResourceRelativePathRequest(StrictModel): script_path: str = Field(min_length=1, max_length=512) diff --git a/backend/src/backend/schedule_schemas.py b/backend/src/backend/schemas/schedules.py similarity index 100% rename from backend/src/backend/schedule_schemas.py rename to backend/src/backend/schemas/schedules.py diff --git a/backend/src/backend/schemas/scripts.py b/backend/src/backend/schemas/scripts.py new file mode 100644 index 0000000..ea710b7 --- /dev/null +++ b/backend/src/backend/schemas/scripts.py @@ -0,0 +1,35 @@ +from typing import Literal + +from common.schemas import StrictModel +from pydantic import Field + + +class CreateScriptRequest(StrictModel): + script_name: str = Field(min_length=1, max_length=255) + script_type: Literal["python", "notebook"] + content: str = Field(max_length=10 * 1024 * 1024) + visibility: Literal["private", "workspace", "public"] = "private" + parent_path: str | None = Field(default=None, max_length=1024) + + +class CreateWorkspaceDirectoryRequest(StrictModel): + directory_name: str = Field(min_length=1, max_length=255) + parent_path: str = Field(default="", max_length=1024) + + +class UpdateScriptRequest(StrictModel): + content: str = Field(max_length=10 * 1024 * 1024) + + +class LockScriptRequest(StrictModel): + is_locked: bool + + +class PublishVersionRequest(StrictModel): + source_object_id: str | None = Field( + default=None, + min_length=26, + max_length=26, + ) + release_note: str | None = Field(default=None, max_length=1000) + visibility: Literal["private", "workspace", "public"] = "workspace" diff --git a/backend/src/backend/services/jupyter.py b/backend/src/backend/services/jupyter.py new file mode 100644 index 0000000..e3c2488 --- /dev/null +++ b/backend/src/backend/services/jupyter.py @@ -0,0 +1 @@ +"""Reserved for stage-3 extraction. Currently empty.""" diff --git a/backend/src/backend/services/resources.py b/backend/src/backend/services/resources.py new file mode 100644 index 0000000..e3c2488 --- /dev/null +++ b/backend/src/backend/services/resources.py @@ -0,0 +1 @@ +"""Reserved for stage-3 extraction. Currently empty.""" diff --git a/backend/src/backend/services/schedules.py b/backend/src/backend/services/schedules.py new file mode 100644 index 0000000..1237f4e --- /dev/null +++ b/backend/src/backend/services/schedules.py @@ -0,0 +1,152 @@ +"""Schedule-domain services. + +Pure business logic extracted from ``backend.api.schedules`` so it can be +unit-tested without spinning up FastAPI / a DB session. Functions here +must not depend on ``Request``, ``BackgroundTasks``, or any FastAPI +router primitive. +""" + +from __future__ import annotations + +import heapq +from typing import Any + +from common.db.models import ScheduleEdges, ScheduleNodes + + +def validate_dag( + nodes: list[ScheduleNodes], + edges: list[ScheduleEdges], +) -> dict[str, Any]: + """Validate a schedule DAG and return a structural report. + + The returned dict has keys: + + * ``valid`` — True iff ``errors`` is empty + * ``node_count`` / ``edge_count`` — input sizes + * ``root_node_ids`` / ``leaf_node_ids`` — sorted by node_key so the + output is deterministic regardless of insertion order + * ``topological_order`` — Kahn's algorithm over node_key ties + * ``errors`` — list of dicts with ``code`` plus enough context + (``edge_id``, ``node_ids``) for the caller to surface back to + the UI; never raises + + Recognised error codes: + + * ``DAG_EMPTY`` — no nodes + * ``DAG_EDGE_NODE_MISSING`` — edge references unknown node_id + * ``DAG_SELF_EDGE`` — source == target + * ``DAG_DUPLICATE_EDGE`` — same directed pair seen twice + * ``DAG_CYCLE`` — topological sort did not consume all nodes + """ + node_by_id = {item.node_id: item for item in nodes} + indegree = {item.node_id: 0 for item in nodes} + outgoing: dict[str, set[str]] = { + item.node_id: set() + for item in nodes + } + errors: list[dict[str, Any]] = [] + seen_edges: set[tuple[str, str]] = set() + + if not nodes: + errors.append( + { + "code": "DAG_EMPTY", + "message": "schedule must contain at least one node", + } + ) + + for edge in edges: + if ( + edge.source_node_id not in node_by_id + or edge.target_node_id not in node_by_id + ): + errors.append( + { + "code": "DAG_EDGE_NODE_MISSING", + "message": "edge references a node outside the schedule", + "edge_id": edge.edge_id, + } + ) + continue + pair = (edge.source_node_id, edge.target_node_id) + if edge.source_node_id == edge.target_node_id: + errors.append( + { + "code": "DAG_SELF_EDGE", + "message": "a node cannot depend on itself", + "edge_id": edge.edge_id, + } + ) + continue + if pair in seen_edges: + errors.append( + { + "code": "DAG_DUPLICATE_EDGE", + "message": "duplicate directed edge", + "edge_id": edge.edge_id, + } + ) + continue + seen_edges.add(pair) + outgoing[edge.source_node_id].add(edge.target_node_id) + indegree[edge.target_node_id] += 1 + + root_ids = sorted( + (node_id for node_id, degree in indegree.items() if degree == 0), + key=lambda node_id: node_by_id[node_id].node_key, + ) + leaf_ids = sorted( + (node_id for node_id, targets in outgoing.items() if not targets), + key=lambda node_id: node_by_id[node_id].node_key, + ) + queue = [ + (node_by_id[node_id].node_key, node_id) + for node_id in root_ids + ] + heapq.heapify(queue) + remaining_indegree = dict(indegree) + ordered_ids: list[str] = [] + while queue: + _, node_id = heapq.heappop(queue) + ordered_ids.append(node_id) + for target_id in sorted( + outgoing[node_id], + key=lambda value: node_by_id[value].node_key, + ): + remaining_indegree[target_id] -= 1 + if remaining_indegree[target_id] == 0: + heapq.heappush( + queue, + (node_by_id[target_id].node_key, target_id), + ) + + if len(ordered_ids) != len(nodes): + cycle_node_ids = sorted( + ( + node_id + for node_id, degree in remaining_indegree.items() + if degree > 0 + ), + key=lambda node_id: node_by_id[node_id].node_key, + ) + errors.append( + { + "code": "DAG_CYCLE", + "message": "schedule graph contains a directed cycle", + "node_ids": cycle_node_ids, + } + ) + + return { + "valid": not errors, + "node_count": len(nodes), + "edge_count": len(edges), + "root_node_ids": root_ids, + "leaf_node_ids": leaf_ids, + "topological_order": ordered_ids, + "errors": errors, + } + + +__all__ = ["validate_dag"] diff --git a/backend/src/backend/services/scripts.py b/backend/src/backend/services/scripts.py new file mode 100644 index 0000000..e3c2488 --- /dev/null +++ b/backend/src/backend/services/scripts.py @@ -0,0 +1 @@ +"""Reserved for stage-3 extraction. Currently empty.""" diff --git a/backend/src/backend/services/storage.py b/backend/src/backend/services/storage.py index 65ed927..00c5452 100644 --- a/backend/src/backend/services/storage.py +++ b/backend/src/backend/services/storage.py @@ -1,6 +1,6 @@ """In-process storage helpers. -The HTTP ``/internal/v1/*`` routes in ``backend.storage_api`` are wrappers +The HTTP ``/internal/v1/*`` routes in ``backend.api.storage`` are wrappers around these. Other backend modules (``scripts``, ``resources``) and the schedule worker call these helpers directly instead of going through an HTTP client — the storage layer lives in the same process, so the @@ -175,7 +175,7 @@ async def _mark_upload_failed_and_raise( ``GET_LOCK``. Why this helper exists at all: the route handler wraps every request - in ``session_scope`` (see ``backend.dependencies.database_session``), + in ``session_scope`` (see ``backend.api.dependencies.database_session``), which rolls back on exception. Without this helper, a naive ``upload.upload_status = "failed"; raise HTTPException(...)`` would lose the status flip and leave the row stuck in ``created``/``uploading`` @@ -279,7 +279,7 @@ def _resolve_bucket_for_usage( workspace_artifact_bucket: str | None, ) -> str: """Mirror of storage_api.resolve_bucket, but pure (no DB / Request).""" - from backend.storage_api import BUCKET_FOR_USAGE + from backend.api.storage import BUCKET_FOR_USAGE if workspace_artifact_bucket: return workspace_artifact_bucket return BUCKET_FOR_USAGE.get( @@ -298,7 +298,7 @@ async def create_upload_record( session; or ``{upload_id, status: "completed", storage_object: {...}}`` when the idempotency key hits an already-completed upload. """ - from backend.storage_api import ( + from backend.api.storage import ( normalized_idempotency_key, require_workspace_member, ) @@ -352,7 +352,7 @@ async def create_upload_record( else: from datetime import timedelta - from backend.storage_api import utcnow + from backend.api.storage import utcnow bucket_name = _resolve_bucket_for_usage( payload.usage_type, @@ -384,7 +384,7 @@ async def create_upload_record( await session.flush() if upload.upload_status == "completed" and upload.storage_object_id: - from backend.storage_api import storage_payload + from backend.api.storage import storage_payload storage_object = await session.get(StorageObjects, upload.storage_object_id) if storage_object is None or storage_object.object_status != "available": upload.storage_object_id = None @@ -552,7 +552,7 @@ async def create_server_object_payload( Used by scripts.py when publishing version artifacts and by the schedule worker for run logs / run results. """ - from backend.storage_api import storage_payload + from backend.api.storage import storage_payload try: content = base64.b64decode(payload.content_base64, validate=True) @@ -680,7 +680,7 @@ async def create_download_url_payload( expires_in=timedelta(seconds=payload.expires_seconds), ) # Public-host rewriting is now nginx's job (location /storage/). In the - # future the boto3 client should be built with the public endpoint so + # future the S3 client should be built with the public endpoint so # generate_presigned_url returns a public URL directly. return { "data": { diff --git a/backend/tests/conftest.py b/backend/tests/conftest.py new file mode 100644 index 0000000..51485ea --- /dev/null +++ b/backend/tests/conftest.py @@ -0,0 +1,32 @@ +"""Shared test bootstrap. + +Loads the repo-root ``.env`` so modules that import ``common.config`` +(which is a process-wide singleton) can resolve ``APP_CONFIG_SECRET_KEY`` +at collection time. Without it, any test importing ``backend.api.*`` fails +with a RuntimeError about ENC(...) ciphertext. +""" + +from __future__ import annotations + +import os +from pathlib import Path + +_REPO_ROOT = Path(__file__).resolve().parent.parent.parent + + +def _bootstrap() -> None: + if os.environ.get("APP_CONFIG_SECRET_KEY"): + return + env_file = _REPO_ROOT / ".env" + if not env_file.is_file(): + return + try: + from dotenv import dotenv_values + except ImportError: + return + value = dotenv_values(env_file).get("APP_CONFIG_SECRET_KEY") + if value: + os.environ["APP_CONFIG_SECRET_KEY"] = value + + +_bootstrap() diff --git a/backend/tests/test_audit_logging.py b/backend/tests/test_audit_logging.py new file mode 100644 index 0000000..e2dca17 --- /dev/null +++ b/backend/tests/test_audit_logging.py @@ -0,0 +1,347 @@ +"""审计日志测试。 + +覆盖： +* 每天一个 ``audit-YYYY-MM-DD.log`` 文件且写入至少一行； +* 日志行包含 user_id / method / path / status； +* access_log 在 success 与 exception（500）两条路径都写审计行； +* 未登录（无 cookie 无 header）时 user_id 记 ``-``； +* Authorization: Bearer 头能解析出 user_id； +* 启动时按 mtime 清理超过保留天数的旧 ``audit-*.log``； +* ``configure_audit_logging`` 幂等。 + +测试不 import main.py、不触达 MySQL / 任何真实业务逻辑：用一个带空路由的 +临时 FastAPI app，挂一个复制 access_log 审计契约的 ``BaseHTTPMiddleware`` +（``_AccessLogReplica``），验证 sink 与契约行为。 +""" + +from __future__ import annotations + +import os +from collections.abc import Awaitable, Callable +from datetime import UTC, datetime +from pathlib import Path + +import pytest +from common.auth.jwt import JwtError, issue_jwt, verify_jwt_token +from fastapi import FastAPI, Request +from fastapi.responses import Response +from fastapi.testclient import TestClient +from loguru import logger +from starlette.middleware.base import BaseHTTPMiddleware + +from backend import audit + + +def _audit_user_id(request: Request) -> str: + """与 main.py 的 _audit_user_id 契约一致：cookie/Bearer 头解 JWT，失败记 '-'。""" + token = request.cookies.get("access_token") + if not token: + auth = request.headers.get("authorization", "") + if auth.lower().startswith("bearer "): + token = auth[7:].strip() + if not token: + return "-" + try: + payload = verify_jwt_token(token) + except (JwtError, Exception): # 任何异常都吞 + return "-" + sub = payload.get("sub") + return sub or "-" + + +class _AccessLogReplica(BaseHTTPMiddleware): + """复制 main.py access_log 写审计行的契约（不 import 真实 main.py）。 + + 测试只覆盖 access_log 的审计行为（success / exception 两条路径 + + user_id 解析 + 排除集守卫），避免触发 main.py 的 lifespan（MySQL / + 路由初始化）。``excluded_paths`` 对应 main.py 的 ``_AUDIT_EXCLUDED``： + 精确路径命中即跳过审计行（诊断 stderr 照常）。未来 main.py 改 + access_log 字段时，这里同步改即可，测试不会假阳/假阴。 + """ + + def __init__(self, app, excluded_paths: frozenset[str] = frozenset()): + super().__init__(app) + self.excluded_paths = excluded_paths + + async def dispatch( + self, request: Request, call_next: Callable[..., Awaitable[Response]] + ) -> Response: + skip_audit = request.url.path in self.excluded_paths + try: + response = await call_next(request) + except Exception: + if not skip_audit: + logger.bind( + user_id=_audit_user_id(request), + method=request.method, + path=request.url.path, + status=500, + ).info("audit") + raise + if not skip_audit: + logger.bind( + user_id=_audit_user_id(request), + method=request.method, + path=request.url.path, + status=response.status_code, + ).info("audit") + return response + + +@pytest.fixture(autouse=True) +def _reset_audit_logging(): + """每个用例之间重置审计模块的幂等标志并卸掉上次挂上的审计 sink。 + + loguru 的全局 logger 会跨用例留存 sink，若不清理，后面的用例会往已 + 删除的临时目录继续写，且 ``_CONFIGURED`` 会让后续 configure 变空操作。 + """ + audit._CONFIGURED = False + yield + if audit._HANDLER_ID is not None: + logger.remove(audit._HANDLER_ID) + audit._HANDLER_ID = None + audit._CONFIGURED = False + + +def _build_client( + log_dir: str, + *, + raise_server_exceptions: bool = True, + excluded_paths: frozenset[str] = frozenset(), +) -> TestClient: + """配置审计日志并返回挂上 access_log 契约复刻中间件的测试 app 客户端。 + + 只注册少量测试路由，不碰数据库与业务逻辑；中间件在路由之后注册，与 + main.py 的 access_log 行为一致。``/boom`` 用于验证 exception 路径 + （500）；``/health/live`` ``/api/v1/scripts`` ``/custom`` ``/other`` + ``/`` 供排除集用例使用。 + """ + audit.configure_audit_logging(log_dir, retention_days=30) + + app = FastAPI() + + @app.get("/x") + def x() -> dict: + return {"ok": True} + + @app.get("/boom") + def boom() -> dict: + raise RuntimeError("boom") + + @app.get("/health/live") + def health_live() -> dict: + return {"ok": True} + + @app.get("/api/v1/scripts") + def scripts() -> dict: + return {"ok": True} + + @app.get("/custom") + def custom() -> dict: + return {"ok": True} + + @app.get("/other") + def other() -> dict: + return {"ok": True} + + @app.get("/") + def root() -> dict: + return {"ok": True} + + app.add_middleware(_AccessLogReplica, excluded_paths=excluded_paths) + return TestClient(app, raise_server_exceptions=raise_server_exceptions) + + +def _local_today() -> str: + return datetime.now(UTC).astimezone().strftime("%Y-%m-%d") + + +def _today_file(log_dir: Path) -> Path: + return log_dir / f"audit-{_local_today()}.log" + + +def _audit_lines(log_dir: Path) -> list[str]: + """等 enqueue 队列落盘后返回今天审计文件的全部非空行。""" + logger.complete() + path = _today_file(log_dir) + if not path.exists(): + return [] + return [line for line in path.read_text(encoding="utf-8").splitlines() if line] + + +def test_audit_log_file_is_created_with_date_suffix(tmp_path: Path) -> None: + client = _build_client(str(tmp_path)) + client.get("/x") + + lines = _audit_lines(tmp_path) + assert _today_file(tmp_path).exists() + assert len(lines) >= 1 + + +def test_audit_log_line_contains_user_method_path_status(tmp_path: Path) -> None: + client = _build_client(str(tmp_path)) + user_id = "01USER12345678901234567890" + token = issue_jwt(user_id) + client.cookies.set("access_token", token) + response = client.get("/x") + assert response.status_code == 200 + + lines = _audit_lines(tmp_path) + assert lines + assert f"| {user_id} | GET /x -> 200" in lines[0] + + +def test_access_log_writes_audit_line_on_success(tmp_path: Path) -> None: + """access_log success 路径写审计行：user_id / status 正确。""" + client = _build_client(str(tmp_path)) + user_id = "01SUCCESS0000000000000000" + token = issue_jwt(user_id) + client.cookies.set("access_token", token) + response = client.get("/x") + assert response.status_code == 200 + + lines = _audit_lines(tmp_path) + assert lines + assert f"| {user_id} | GET /x -> 200" in lines[0] + + +def test_access_log_writes_audit_line_on_5xx(tmp_path: Path) -> None: + """access_log exception 路径也写审计行：status 记 500。""" + client = _build_client(str(tmp_path), raise_server_exceptions=False) + response = client.get("/boom") + assert response.status_code == 500 + + lines = _audit_lines(tmp_path) + assert lines + assert "GET /boom -> 500" in lines[0] + + +def test_access_log_writes_dash_user_when_no_jwt(tmp_path: Path) -> None: + """无 cookie 无 header 时，审计行 user_id 列记 '-'。""" + client = _build_client(str(tmp_path)) + response = client.get("/x") + assert response.status_code == 200 + + lines = _audit_lines(tmp_path) + assert lines + assert "| - | GET /x -> 200" in lines[0] + + +def test_audit_log_bearer_header_resolves_user(tmp_path: Path) -> None: + """Authorization: Bearer 头同样能解析出 user_id。""" + client = _build_client(str(tmp_path)) + user_id = "01BEARER000000000000000001" + token = issue_jwt(user_id) + response = client.get("/x", headers={"Authorization": f"Bearer {token}"}) + assert response.status_code == 200 + + lines = _audit_lines(tmp_path) + assert lines + assert f"| {user_id} | GET /x -> 200" in lines[0] + + +def test_access_log_skips_audit_for_excluded_path(tmp_path: Path) -> None: + """命中排除集（/health/live）：审计行不写，诊断 stderr 照常。 + + stderr 走 loguru default sink，难以用 caplog 抓取，这里直接断言 + 审计文件无新行即可。 + """ + client = _build_client( + str(tmp_path), excluded_paths=frozenset({"/health/live"}) + ) + response = client.get("/health/live") + assert response.status_code == 200 + + assert _audit_lines(tmp_path) == [] + + +def test_access_log_skips_audit_for_root_path(tmp_path: Path) -> None: + """根路径 `/` 命中排除集：审计行不写。""" + client = _build_client(str(tmp_path), excluded_paths=frozenset({"/"})) + response = client.get("/") + assert response.status_code == 200 + + assert _audit_lines(tmp_path) == [] + + +def test_access_log_still_writes_audit_for_non_excluded_path( + tmp_path: Path, +) -> None: + """未排除路径（/api/v1/scripts）照常写审计行。""" + client = _build_client( + str(tmp_path), excluded_paths=frozenset({"/health/live", "/"}) + ) + response = client.get("/api/v1/scripts") + assert response.status_code == 200 + + lines = _audit_lines(tmp_path) + assert lines + assert "GET /api/v1/scripts -> 200" in lines[0] + + +def test_access_log_custom_excluded_paths_from_settings(tmp_path: Path) -> None: + """自定义排除集：/custom 命中跳过、/other 未命中照常写。""" + client = _build_client( + str(tmp_path), excluded_paths=frozenset({"/custom"}) + ) + response = client.get("/custom") + assert response.status_code == 200 + assert _audit_lines(tmp_path) == [] + + response = client.get("/other") + assert response.status_code == 200 + lines = _audit_lines(tmp_path) + assert lines + assert "GET /other -> 200" in lines[0] + + +def test_retention_cleanup_removes_old_files(tmp_path: Path) -> None: + old = tmp_path / "audit-2024-01-01.log" + recent = tmp_path / "audit-2024-06-01.log" + old.write_text("old\n", encoding="utf-8") + recent.write_text("recent\n", encoding="utf-8") + + old_mtime = datetime.now(UTC).timestamp() - 60 * 86_400 # 60 天前 + recent_mtime = datetime.now(UTC).timestamp() - 5 * 86_400 # 5 天前 + os.utime(old, (old_mtime, old_mtime)) + os.utime(recent, (recent_mtime, recent_mtime)) + + audit.configure_audit_logging(str(tmp_path), retention_days=30) + + assert not old.exists(), "超过保留期的旧审计文件应被启动清理删除" + assert recent.exists(), "保留期内(5 天前)的文件应保留" + + +def test_retention_cleanup_disabled_when_zero(tmp_path: Path) -> None: + """AUDIT_LOG_RETENTION_DAYS=0 关闭清理：任何旧文件都不删除。""" + old = tmp_path / "audit-2024-01-01.log" + old.write_text("old\n", encoding="utf-8") + old_mtime = datetime.now(UTC).timestamp() - 400 * 86_400 # 400 天前 + os.utime(old, (old_mtime, old_mtime)) + + audit.configure_audit_logging(str(tmp_path), retention_days=0) + + assert old.exists(), "retention_days=0 时应跳过清理，保留所有旧文件" + + +def test_configure_audit_logging_is_idempotent(tmp_path: Path) -> None: + """多次调用只有首次生效：日志只写到第一次给的目录。""" + first_dir = tmp_path / "first" + second_dir = tmp_path / "second" + + audit.configure_audit_logging(str(first_dir), retention_days=30) + audit.configure_audit_logging(str(second_dir), retention_days=30) + + app = FastAPI() + + @app.get("/x") + def x() -> dict: + return {"ok": True} + + app.add_middleware(_AccessLogReplica) + client = TestClient(app) + client.get("/x") + + logger.complete() + assert (first_dir / f"audit-{_local_today()}.log").exists() + assert not (second_dir / f"audit-{_local_today()}.log").exists() diff --git a/backend/tests/test_auth_profile_password.py b/backend/tests/test_auth_profile_password.py new file mode 100644 index 0000000..fa69bb1 --- /dev/null +++ b/backend/tests/test_auth_profile_password.py @@ -0,0 +1,39 @@ +"""Unit tests for auth profile / password request schemas.""" + +from __future__ import annotations + +import pytest +from pydantic import ValidationError + +from backend.api.auth import PasswordChange, ProfileUpdate +from backend.api.platform.employees import PlatformEmployeePasswordReset + + +def test_profile_update_allows_partial_fields() -> None: + only_name = ProfileUpdate(display_name="张三") + assert only_name.display_name == "张三" + assert only_name.email is None + + only_email = ProfileUpdate(email="a@example.com") + assert only_email.email == "a@example.com" + + +def test_profile_update_forbids_unknown_fields() -> None: + with pytest.raises(ValidationError): + ProfileUpdate(display_name="张三", username="hacked") # type: ignore[call-arg] + + +def test_password_change_enforces_new_password_length() -> None: + with pytest.raises(ValidationError): + PasswordChange(current_password="old-pass", new_password="short") + + ok = PasswordChange(current_password="old-pass-1", new_password="new-pass-12") + assert ok.new_password == "new-pass-12" + + +def test_admin_password_reset_schema() -> None: + with pytest.raises(ValidationError): + PlatformEmployeePasswordReset(new_password="1234567") + + payload = PlatformEmployeePasswordReset(new_password="reset-pass-9") + assert payload.new_password == "reset-pass-9" diff --git a/backend/tests/test_count_scripts.py b/backend/tests/test_count_scripts.py new file mode 100644 index 0000000..7456ac1 --- /dev/null +++ b/backend/tests/test_count_scripts.py @@ -0,0 +1,147 @@ +"""Unit tests for GET /api/v1/scripts/count endpoint. + +Verifies the count endpoint matches the (workspace-wide) listing scope of +``list_scripts(parent_path="")``: workspace + active scripts across every +owner's ``StorageObjects.relative_path``, narrowed by visibility for +non-admin (admin short-circuits). This avoids under/over-reporting on the +dashboard — the count is the size of the set list_scripts would return if +it weren't lazy. +""" + +from __future__ import annotations + +from types import SimpleNamespace +from unittest.mock import AsyncMock, MagicMock + +import pytest + +from backend.api.scripts import count_scripts + + +def _ctx( + user_id: str = "U001", + workspace_id: str = "W001", + *, + is_admin: bool = False, + is_system_admin: bool = False, +) -> SimpleNamespace: + return SimpleNamespace( + request_id="test", + user=SimpleNamespace(user_id=user_id), + workspace=SimpleNamespace(workspace_id=workspace_id), + role=SimpleNamespace(role_code="admin" if is_admin else "developer"), + is_system_admin=is_system_admin, + # ``count_scripts`` now consults ``context.is_admin`` directly + # (matching list_scripts / list_resources); SimpleNamespace needs + # it as a plain attribute. + is_admin=is_admin or is_system_admin, + ) + + +def _compile(stmt) -> str: + from sqlalchemy.dialects import mysql as mysql_dialect + + return str( + stmt.compile( + dialect=mysql_dialect.dialect(), + compile_kwargs={"literal_binds": True}, + ) + ) + + +async def test_count_scripts_returns_scalar_int() -> None: + captured = [] + + mock_session = MagicMock() + mock_session.scalar = AsyncMock( + side_effect=lambda stmt: (captured.append(stmt), 7)[1] + ) + + result = await count_scripts(context=_ctx(), session=mock_session) + assert result["data"] == {"total": 7} + assert result["meta"] == {} + assert result["request_id"] == "test" + # Exactly one COUNT(*) query issued. + assert len(captured) == 1 + stmt = captured[0] + sql = _compile(stmt).lower() + # JOIN to StorageObjects so orphaned scripts (no joinable row) are + # excluded — matches list_scripts INNER JOIN behaviour. + assert "inner join storage_objects" in sql + # Scope: workspace_id + active status + workspace-wide prefix + # (owner segment wildcarded: workspace/%/%). + assert "scripts.workspace_id" in sql + assert "scripts.status" in sql + assert "like 'workspace/%%/%%'" in sql + # Non-admin (default) narrows by visibility. + assert "scripts.owner_user_id = 'u001'" in sql + assert "scripts.visibility in ('workspace', 'public')" in sql + + +async def test_count_scripts_handles_null_result() -> None: + """MySQL COUNT(*) on empty result returns 0, not NULL — but defensively + coerce NULL to 0 to keep the response shape consistent.""" + mock_session = MagicMock() + mock_session.scalar = AsyncMock(return_value=None) + result = await count_scripts(context=_ctx(), session=mock_session) + assert result["data"] == {"total": 0} + + +async def test_count_scripts_workspace_wide_not_user_scoped() -> None: + """The prefix is workspace-wide (``workspace/%/%`` — owner segment + wildcarded, no embedded user_id), so different users count the same + physical tree; the only per-user difference is the non-admin + visibility predicate (owner_user_id = me).""" + captured = [] + + mock_session = MagicMock() + mock_session.scalar = AsyncMock( + side_effect=lambda stmt: (captured.append(stmt), 3)[1] + ) + + await count_scripts(context=_ctx(user_id="alice"), session=mock_session) + sql_alice = _compile(captured[-1]).lower() + + await count_scripts(context=_ctx(user_id="bob"), session=mock_session) + sql_bob = _compile(captured[-1]).lower() + + # Both count the same workspace-wide subtree. + assert "like 'workspace/%%/%%'" in sql_alice + assert "like 'workspace/%%/%%'" in sql_bob + # Neither embeds the user_id in the path prefix. + assert "workspace/alice/%" not in sql_alice + assert "workspace/bob/%" not in sql_bob + # Per-user narrowing happens via the visibility predicate. + assert "scripts.owner_user_id = 'alice'" in sql_alice + assert "scripts.owner_user_id = 'bob'" in sql_bob + + +async def test_count_scripts_admin_skips_visibility_filter() -> None: + """Admin short-circuits the visibility predicate and counts every + active script in the workspace (dashboard '全部脚本' / '工作副本').""" + captured = [] + + mock_session = MagicMock() + mock_session.scalar = AsyncMock( + side_effect=lambda stmt: (captured.append(stmt), 42)[1] + ) + + result = await count_scripts( + context=_ctx(user_id="alice", is_admin=True), session=mock_session + ) + assert result["data"] == {"total": 42} + sql = _compile(captured[0]).lower() + assert "like 'workspace/%%/%%'" in sql + # visibility / owner_user_id still appear in the SELECT projection, but + # the visibility WHERE predicate must be absent for admins. + assert "scripts.visibility in ('workspace', 'public')" not in sql + + +async def test_count_scripts_route_declared_before_script_id_route() -> None: + """Static check: the `/api/v1/scripts/count` route MUST be declared in + scripts.py before `/api/v1/scripts/{script_id}/...`, otherwise FastAPI's + declaration-order matching will interpret `count` as a script_id.""" + from backend.api.scripts import count_scripts, get_script + + assert callable(count_scripts) + assert callable(get_script) diff --git a/backend/tests/test_jupyter_auth_cache.py b/backend/tests/test_jupyter_auth_cache.py new file mode 100644 index 0000000..e7e2caf --- /dev/null +++ b/backend/tests/test_jupyter_auth_cache.py @@ -0,0 +1,236 @@ +"""Unit tests for the 5s auth-result cache in ``backend.api.jupyter``. + +Jupyter 一次会话会触发几十次 Nginx ``auth_request``；本缓存按 +``(workspace_id, user_id)`` 缓存 membership + runtime 的查找结果，避免 +每次都跑 DB JOIN 与跨进程 RPC。JWT 验签与 per-URI 的 lock check **不进 +缓存**，每请求都执行。 + +这些测试直接调用 ``verify_jupyter_access``（不经过 FastAPI TestClient）， +用 SimpleNamespace 构造 fake request / response / runtime，并用 +monkeypatch 替换 JWT / membership / lock / runtime 的调用点来统计次数。 +""" + +from __future__ import annotations + +from types import SimpleNamespace +from unittest.mock import AsyncMock + +import pytest +from fastapi import HTTPException + +import backend.api.jupyter as jupyter_module +from backend.api.jupyter import ( + _JUPYTER_AUTH_CACHE, + verify_jupyter_access, +) +from backend.clients.runtime import RuntimeClientError + +WS_ID = "01WS0000000000000000000A" +USER_ID = "01USR0000000000000000000A" +NOTEBOOK_URI = f"/jupyter/{WS_ID}/notebooks/a.ipynb" +NON_NOTEBOOK_URI = f"/jupyter/{WS_ID}/tree" + +_DESCRIPTOR = { + "status": "running", + "workspace_id": WS_ID, + "base_url": "http://runtime", + "port": 34567, + "token": "jupyter-token", +} + + +@pytest.fixture(autouse=True) +def _clear_cache() -> None: + _JUPYTER_AUTH_CACHE.clear() + yield + _JUPYTER_AUTH_CACHE.clear() + + +def _make_context(runtime_client, uri: str = NOTEBOOK_URI) -> tuple[SimpleNamespace, SimpleNamespace]: + request = SimpleNamespace( + headers={ + "X-Original-Workspace-Id": WS_ID, + "X-Original-URI": uri, + }, + cookies={"access_token": "a.b.c"}, + app=SimpleNamespace(state=SimpleNamespace(runtime_client=runtime_client)), + ) + response = SimpleNamespace(headers={}) + return request, response + + +def _make_runtime_client(descriptor=None) -> SimpleNamespace: + client = SimpleNamespace() + client.get_workspace = AsyncMock(return_value=descriptor if descriptor is not None else _DESCRIPTOR) + client.start_workspace = AsyncMock(return_value=_DESCRIPTOR) + return client + + +def _setup_mocks( + monkeypatch: pytest.MonkeyPatch, + *, + user_id: str = USER_ID, + locked: bool = False, + runtime_client: SimpleNamespace | None = None, +) -> tuple[SimpleNamespace, SimpleNamespace, AsyncMock, AsyncMock, AsyncMock, SimpleNamespace]: + """Patch the call points once and return fakes for counting. + + ``verify_jwt_token`` 默认替换为固定 payload；需要计数的测试可在此之后 + 再次 ``monkeypatch.setattr`` 覆盖（后设置者生效）。 + """ + monkeypatch.setattr( + jupyter_module, + "verify_jwt_token", + lambda _token: {"sub": user_id}, + ) + membership = AsyncMock() + monkeypatch.setattr(jupyter_module, "load_active_membership_or_403", membership) + lock_check = AsyncMock(return_value=locked) + monkeypatch.setattr(jupyter_module, "check_notebook_is_locked", lock_check) + runtime = runtime_client if runtime_client is not None else _make_runtime_client() + request, response = _make_context(runtime) + return request, response, membership, lock_check, runtime + + +async def _call_once(request, response) -> None: + await verify_jupyter_access( + request, + response, + auth=None, + session=AsyncMock(), + ) + + +async def test_cache_hit_skips_membership_and_runtime(monkeypatch) -> None: + """第一次跑完整路径，第二次同样 (ws, user) 跳过 membership + runtime。""" + request, response, membership, lock_check, runtime = _setup_mocks(monkeypatch) + + await _call_once(request, response) + assert membership.await_count == 1 + assert runtime.get_workspace.await_count == 1 + assert runtime.start_workspace.await_count == 0 + assert response.headers["x-upstream-addr"] == "http://runtime:34567" + assert response.headers["x-jupyter-internal-token"] == "jupyter-token" + + # 第二次请求：缓存命中，membership / runtime 不再执行。 + response.headers = {} + await _call_once(request, response) + assert membership.await_count == 1 + assert runtime.get_workspace.await_count == 1 + assert runtime.start_workspace.await_count == 0 + # lock check 每请求都跑。 + assert lock_check.await_count == 2 + # 缓存命中也要写 headers。 + assert response.headers["x-upstream-addr"] == "http://runtime:34567" + assert response.headers["x-jupyter-internal-token"] == "jupyter-token" + + +async def test_cache_miss_runs_full_path(monkeypatch) -> None: + """清空缓存后第一次请求必须跑 membership + runtime。""" + _JUPYTER_AUTH_CACHE.clear() + request, response, membership, lock_check, runtime = _setup_mocks(monkeypatch) + + await _call_once(request, response) + + assert membership.await_count == 1 + assert runtime.get_workspace.await_count == 1 + assert lock_check.await_count == 1 + + +async def test_lock_check_runs_every_request_even_on_cache_hit(monkeypatch) -> None: + """缓存命中时仍要执行 lock check（per-URI，不进缓存）。""" + request, response, membership, lock_check, runtime = _setup_mocks(monkeypatch) + + await _call_once(request, response) # 预热缓存 + response.headers = {} + await _call_once(request, response) # 缓存命中 + + assert membership.await_count == 1 + assert lock_check.await_count == 2 + + +async def test_jwt_verify_runs_every_request(monkeypatch) -> None: + """JWT 验签是每请求的安全边界，缓存命中也不能跳过。""" + verify_calls: list[int] = [] + + def _fake_verify(_token): + verify_calls.append(1) + return {"sub": USER_ID} + + request, response, membership, lock_check, runtime = _setup_mocks(monkeypatch) + monkeypatch.setattr(jupyter_module, "verify_jwt_token", _fake_verify) + + await _call_once(request, response) + response.headers = {} + await _call_once(request, response) # 缓存命中 + + assert membership.await_count == 1 + assert len(verify_calls) == 2 + + +async def test_cache_ttl_expires_after_5s(monkeypatch) -> None: + """TTL 用 time.monotonic；5s 后缓存失效，重新走完整路径。""" + now = [100.0] + monkeypatch.setattr("time.monotonic", lambda: now[0]) + request, response, membership, lock_check, runtime = _setup_mocks(monkeypatch) + + await _call_once(request, response) # t=100，写缓存（expires=105） + assert membership.await_count == 1 + + response.headers = {} + await _call_once(request, response) # t=100，缓存命中 + assert membership.await_count == 1 + + now[0] = 105.0 # 恰好到过期时刻 -> 缓存失效 + response.headers = {} + await _call_once(request, response) + assert membership.await_count == 2 + assert runtime.get_workspace.await_count == 2 + + +async def test_lock_check_failure_does_not_populate_cache(monkeypatch) -> None: + """lock 403 不进缓存。""" + request, response, membership, lock_check, runtime = _setup_mocks(monkeypatch, locked=True) + + with pytest.raises(HTTPException) as excinfo: + await _call_once(request, response) + + assert excinfo.value.status_code == 403 + assert _JUPYTER_AUTH_CACHE == {} + assert jupyter_module._jupyter_auth_cache_get(WS_ID, USER_ID) is None + + +async def test_runtime_start_failure_does_not_populate_cache(monkeypatch) -> None: + """runtime 启动失败（500）不进缓存。""" + runtime = _make_runtime_client() + runtime.get_workspace = AsyncMock(return_value=None) # 未运行 -> 走 start + runtime.start_workspace = AsyncMock( + side_effect=RuntimeClientError(500, {"code": "JUPYTER_START_FAILED"}) + ) + request, response, membership, lock_check, _rt = _setup_mocks( + monkeypatch, runtime_client=runtime + ) + + with pytest.raises(HTTPException) as excinfo: + await _call_once(request, response) + + assert excinfo.value.status_code == 500 + assert _JUPYTER_AUTH_CACHE == {} + + +async def test_cache_keyed_per_user_and_workspace(monkeypatch) -> None: + """不同 user 共享 workspace 时不串缓存。""" + request, response, membership, lock_check, runtime = _setup_mocks(monkeypatch) + await _call_once(request, response) # USER_A 预热缓存 + assert membership.await_count == 1 + + # 换一个 user_id，同一个 workspace -> 缓存 key 不同，必须重新跑完整路径。 + request2, response2 = _make_context(runtime) + monkeypatch.setattr( + jupyter_module, + "verify_jwt_token", + lambda _token: {"sub": "01USR0000000000000000000B"}, + ) + await _call_once(request2, response2) + assert membership.await_count == 2 + assert runtime.get_workspace.await_count == 2 diff --git a/backend/tests/test_list_scripts_parent_path.py b/backend/tests/test_list_scripts_parent_path.py new file mode 100644 index 0000000..4b57096 --- /dev/null +++ b/backend/tests/test_list_scripts_parent_path.py @@ -0,0 +1,822 @@ +"""Tests for the parent_path filter clause on list_scripts, plus the +LIKE-pattern escape contract for tree-walking queries. + +Three layers of coverage: + +1. ``_escape_like_pattern`` unit tests — pure-function correctness. +2. SQL-contract tests (mock session) — verifies the compiled SQL contains + the escaped pattern AND the ``ESCAPE '\\'`` clause. +3. Behavioral test (SQLite in-memory, real LIKE execution) — proves the + fix actually prevents the wildcard leak that motivated the change. + A folder named ``foo_bar`` MUST NOT match sibling paths like + ``fooXbar`` / ``foo2bar`` / ``foo/bar``. + +The repo has no MySQL integration test layer, so SQLite stands in for +LIKE semantics — both dialects treat ``_`` as "any single char" and +``%`` as "any sequence" by default and honour the ``ESCAPE`` clause +identically for the ASCII characters we care about. +""" + +from __future__ import annotations + +from types import SimpleNamespace +from unittest.mock import AsyncMock, MagicMock + +import pytest +from fastapi import HTTPException +from sqlalchemy import Column, MetaData, String, Table, create_engine, select, text +from sqlalchemy.dialects import mysql as mysql_dialect + +from backend.api.scripts import ( + _build_list_scripts_owner_descendant_prefix, + _build_list_scripts_workspace_descendant_prefix, + _escape_like_pattern, + normalize_user_path, +) + + +def _ctx( + user_id: str = "U001", *, is_admin: bool = False, is_system_admin: bool = False +) -> SimpleNamespace: + return SimpleNamespace( + request_id="test", + user=SimpleNamespace(user_id=user_id), + workspace=SimpleNamespace(workspace_id="W001"), + role=SimpleNamespace(role_code="admin" if is_admin else "developer"), + is_system_admin=is_system_admin, + # ``list_scripts`` now consults ``context.is_admin`` directly (matching + # ``list_resources``); SimpleNamespace needs it as a plain attribute. + is_admin=is_admin or is_system_admin, + ) + + +# ─── layer 1: helper unit tests ────────────────────────────────── + + +class TestEscapeLikePattern: + """The escape helper itself is the load-bearing piece — test it + exhaustively before relying on it in SQL.""" + + def test_no_metachars_unchanged(self) -> None: + assert _escape_like_pattern("foo/bar") == "foo/bar" + assert _escape_like_pattern("workspace/alice") == "workspace/alice" + assert _escape_like_pattern("") == "" + + def test_underscore_escaped(self) -> None: + assert _escape_like_pattern("foo_bar") == r"foo\_bar" + + def test_percent_escaped(self) -> None: + assert _escape_like_pattern("100%") == r"100\%" + assert _escape_like_pattern("%foo") == r"\%foo" + + def test_backslash_escaped_first(self) -> None: + # Must escape the escape char first, otherwise the inserted + # backslashes would be double-escaped by the later passes. + assert _escape_like_pattern(r"a\b") == r"a\\b" + assert _escape_like_pattern(r"a\%b") == r"a\\\%b" + + def test_combined(self) -> None: + assert _escape_like_pattern("foo_bar%baz") == r"foo\_bar\%baz" + assert _escape_like_pattern("_%") == r"\_\%" + assert _escape_like_pattern(r"\\_%") == r"\\\\\_\%" + + +# ─── layer 1.5: prefix helper now escapes ───────────────────────── + + +def test_descendant_prefix_root() -> None: + """Empty parent_path → descendant prefix is the owner-scoped root + '/'.""" + prefix = _build_list_scripts_owner_descendant_prefix("alice", "") + assert prefix == "workspace/alice/" + + +def test_descendant_prefix_subdir() -> None: + """Non-empty parent_path → appended under the owner-scoped root.""" + prefix = _build_list_scripts_owner_descendant_prefix("alice", "foo/bar") + assert prefix == "workspace/alice/foo/bar/" + + +def test_descendant_prefix_escapes_metachars() -> None: + """Folder name ``foo_bar`` MUST produce ``foo\\_bar`` in the prefix + so the trailing ``%`` doesn't become 'match any single char before + b'.""" + prefix = _build_list_scripts_owner_descendant_prefix("alice", "foo_bar") + assert prefix == r"workspace/alice/foo\_bar/" + + +def test_descendant_prefix_normalizes_leading_trailing_slashes() -> None: + prefix = _build_list_scripts_owner_descendant_prefix("alice", "/foo/bar/") + assert prefix == "workspace/alice/foo/bar/" + + +def test_descendant_prefix_rejects_traversal() -> None: + with pytest.raises(HTTPException) as exc: + _build_list_scripts_owner_descendant_prefix("alice", "foo/../bar") + assert exc.value.status_code == 422 + + +def test_normalize_user_path_strips() -> None: + assert normalize_user_path("") == "" + assert normalize_user_path("/a/b/") == "a/b" + assert normalize_user_path("a\\b") == "a/b" + + +# ─── layer 1.6: workspace-wide prefix helper ───────────────────── + + +def test_workspace_descendant_prefix_root() -> None: + """Empty parent_path → workspace-wide root prefix (cross-owner). + + Physical root level is each owner's subtree, so the prefix wildcards + the owner segment: ``workspace/%/`` (direct children ``workspace/%/%`` + minus ``workspace/%/%/%``).""" + assert _build_list_scripts_workspace_descendant_prefix("") == "workspace/%/" + + +def test_workspace_descendant_prefix_subdir() -> None: + """Non-empty parent_path → workspace-wide with owner-segment wildcard. + + Physical rows live under ``workspace/{owner_id}/...``, so a subdirectory + listing must match ANY owner: ``workspace/%/foo/bar`` (the ``%`` is the + intentional owner wildcard, exactly like list_resources).""" + assert ( + _build_list_scripts_workspace_descendant_prefix("foo/bar") + == "workspace/%/foo/bar/" + ) + + +def test_workspace_descendant_prefix_escapes_metachars() -> None: + r"""Folder ``foo_bar`` must produce ``foo\_bar`` so the trailing ``%`` + doesn't become 'match any single char'.""" + assert ( + _build_list_scripts_workspace_descendant_prefix("foo_bar") + == r"workspace/%/foo\_bar/" + ) + + +def test_workspace_descendant_prefix_escapes_percent() -> None: + assert ( + _build_list_scripts_workspace_descendant_prefix("100%match") + == r"workspace/%/100\%match/" + ) + + +def test_workspace_descendant_prefix_normalizes_leading_trailing_slashes() -> None: + assert ( + _build_list_scripts_workspace_descendant_prefix("/foo/bar/") + == "workspace/%/foo/bar/" + ) + + +def test_workspace_descendant_prefix_rejects_traversal() -> None: + with pytest.raises(HTTPException) as exc: + _build_list_scripts_workspace_descendant_prefix("foo/../bar") + assert exc.value.status_code == 422 + + +# ─── layer 2: SQL contract ──────────────────────────────────────── + + +def _compile_sql(stmt) -> str: + return str( + stmt.compile( + dialect=mysql_dialect.dialect(), + compile_kwargs={"literal_binds": True}, + ) + ) + + +async def test_list_scripts_where_clause_uses_like_prefix_and_excludes_deeper() -> None: + from backend.api.scripts import list_scripts + + captured_sql: list[str] = [] + + class _MockResult: + def all(self): + return [] + + mock_session = MagicMock() + mock_session.execute = AsyncMock( + side_effect=lambda stmt: ( + captured_sql.append(_compile_sql(stmt)) or _MockResult() + ) + ) + + await list_scripts( + parent_path="foo/bar", + owner_user_id=None, + context=_ctx("alice"), + session=mock_session, + ) + + assert len(captured_sql) == 1 + sql = captured_sql[0].lower() + # Default (no owner_user_id) scopes to the requester's own subtree: + # LIKE workspace/alice/foo/bar/% (direct children), excluding deeper. + assert "like 'workspace/alice/foo/bar/%%'" in sql + assert "not like 'workspace/alice/foo/bar/%%/%%'" in sql + + +async def test_list_scripts_where_clause_escapes_pattern_literal() -> None: + """Regression: parent_path containing ``_`` MUST be escaped in the + compiled LIKE pattern, otherwise sibling-path leak returns to bite.""" + from backend.api.scripts import list_scripts + + captured_sql: list[str] = [] + + class _MockResult: + def all(self): + return [] + + mock_session = MagicMock() + mock_session.execute = AsyncMock( + side_effect=lambda stmt: ( + captured_sql.append(_compile_sql(stmt)) or _MockResult() + ) + ) + + await list_scripts( + parent_path="foo_bar", + owner_user_id=None, + context=_ctx("alice"), + session=mock_session, + ) + sql = captured_sql[0] + # Normalize keyword case so we don't depend on SQLAlchemy casing. + sql_lower = sql.lower() + # Pattern literal must contain the ESCAPED underscore. SQLAlchemy + # doubles the escape char inside the SQL string literal, so what + # the helper emits as `foo\_bar` renders as `foo\\_bar` here + # (2 backslash chars in the actual SQL string). + assert r"like 'workspace/alice/foo\\_bar/%%'" in sql_lower + # NOT LIKE clause also escaped. + assert r"not like 'workspace/alice/foo\\_bar/%%/%%'" in sql_lower + # And both declare ESCAPE '\\'. + assert sql.count("ESCAPE '\\\\'") == 2, sql + + +async def test_list_scripts_where_clause_escapes_percent_pattern() -> None: + """Same regression for ``%``.""" + from backend.api.scripts import list_scripts + + captured_sql: list[str] = [] + + class _MockResult: + def all(self): + return [] + + mock_session = MagicMock() + mock_session.execute = AsyncMock( + side_effect=lambda stmt: ( + captured_sql.append(_compile_sql(stmt)) or _MockResult() + ) + ) + + await list_scripts( + parent_path="100%match", + owner_user_id=None, + context=_ctx("alice"), + session=mock_session, + ) + sql = captured_sql[0] + sql_lower = sql.lower() + # SQLAlchemy doubles the escape char so `%` → `\%` becomes `\\%` + # in the SQL string literal. + assert r"workspace/alice/100\\%%match/%%" in sql_lower + + +async def test_list_scripts_non_admin_adds_visibility_filter() -> None: + """Owner-scoped listing is narrowed by visibility for non-admin: + owner_user_id = me OR visibility IN (workspace, public) — exactly like + list_resources. Default (no owner_user_id) scopes to the requester's own + subtree, so the visibility predicate is redundant-but-present here; it + becomes load-bearing when an owner_user_id query param browses another + member's subtree (their private rows are then excluded).""" + from backend.api.scripts import list_scripts + + captured_sql: list[str] = [] + + class _MockResult: + def all(self): + return [] + + mock_session = MagicMock() + mock_session.execute = AsyncMock( + side_effect=lambda stmt: ( + captured_sql.append(_compile_sql(stmt)) or _MockResult() + ) + ) + + await list_scripts( + parent_path="", + owner_user_id=None, + context=_ctx("alice"), + session=mock_session, + ) + sql = captured_sql[0].lower() + # Default (no owner_user_id) scopes to the requester's own root: + # LIKE workspace/alice/% (alice's root files), excluding nested. + assert "like 'workspace/alice/%%'" in sql + assert "scripts.owner_user_id = 'alice'" in sql + assert "scripts.visibility in ('workspace', 'public')" in sql + + +async def test_list_scripts_owner_param_scopes_to_other_owner() -> None: + """owner_user_id= scopes the LIKE to that owner's subtree so the + tree can lazily fetch another member's content on group expand. The + non-admin visibility predicate is still applied, so the other owner's + private rows are excluded (only workspace/public survive).""" + from backend.api.scripts import list_scripts + + captured_sql: list[str] = [] + + class _MockResult: + def all(self): + return [] + + mock_session = MagicMock() + mock_session.execute = AsyncMock( + side_effect=lambda stmt: ( + captured_sql.append(_compile_sql(stmt)) or _MockResult() + ) + ) + + await list_scripts( + parent_path="", + owner_user_id="bob", + context=_ctx("alice"), + session=mock_session, + ) + sql = captured_sql[0].lower() + assert "like 'workspace/bob/%%'" in sql + assert "not like 'workspace/bob/%%/%%'" in sql + # non-admin: visibility predicate still present (excludes bob's private) + assert "scripts.owner_user_id = 'alice'" in sql + assert "scripts.visibility in ('workspace', 'public')" in sql + + +async def test_list_scripts_admin_skips_visibility_filter() -> None: + """Admin short-circuits the visibility predicate and sees everything.""" + from backend.api.scripts import list_scripts + + captured_sql: list[str] = [] + + class _MockResult: + def all(self): + return [] + + mock_session = MagicMock() + mock_session.execute = AsyncMock( + side_effect=lambda stmt: ( + captured_sql.append(_compile_sql(stmt)) or _MockResult() + ) + ) + + await list_scripts( + parent_path="", + owner_user_id=None, + context=_ctx("alice", is_admin=True), + session=mock_session, + ) + sql = captured_sql[0].lower() + assert "like 'workspace/alice/%%'" in sql + # owner_user_id / visibility still appear in the SELECT projection; what + # must be absent is the visibility WHERE predicate for non-admins. + assert "scripts.visibility in ('workspace', 'public')" not in sql + + +async def test_list_scripts_admin_owner_param_skips_visibility() -> None: + """Admin browsing another owner's subtree scopes to that owner and skips + the visibility predicate (admin sees the other owner's private too).""" + from backend.api.scripts import list_scripts + + captured_sql: list[str] = [] + + class _MockResult: + def all(self): + return [] + + mock_session = MagicMock() + mock_session.execute = AsyncMock( + side_effect=lambda stmt: ( + captured_sql.append(_compile_sql(stmt)) or _MockResult() + ) + ) + + await list_scripts( + parent_path="sub", + owner_user_id="bob", + context=_ctx("alice", is_admin=True), + session=mock_session, + ) + sql = captured_sql[0].lower() + assert "like 'workspace/bob/sub/%%'" in sql + assert "scripts.visibility in ('workspace', 'public')" not in sql + + +async def test_list_workspace_directories_where_clause_escapes_pattern() -> None: + """list_workspace_directories must escape user input too (was + pre-existing debt).""" + from backend.api.scripts import list_workspace_directories + + captured_sql: list[str] = [] + + class _MockScalarResult: + def scalar(self): + return None + + class _MockResult: + def all(self): + return [] + + def scalars(self): + return _MockScalarResult() + + mock_session = MagicMock() + mock_session.execute = AsyncMock( + side_effect=lambda stmt: ( + captured_sql.append(_compile_sql(stmt)) or _MockResult() + ) + ) + + await list_workspace_directories( + parent_path="foo_bar", owner_user_id=None, + context=_ctx("alice"), session=mock_session, + ) + sql = " ".join(captured_sql) + assert r"workspace/alice/foo\\_bar/" in sql, sql + + # owner_user_id scopes the prefix to that owner's subtree. + captured_sql.clear() + await list_workspace_directories( + parent_path="foo_bar", owner_user_id="bob", + context=_ctx("alice"), session=mock_session, + ) + sql = " ".join(captured_sql) + assert r"workspace/bob/foo\\_bar/" in sql, sql + + +# ─── layer 3: behavioral test on real LIKE execution ────────────── + + +@pytest.fixture +def sqlite_like_table(): + """SQLite in-memory table with a single VARCHAR column. Stand-in for + ``storage_objects.relative_path`` — proves the actual LIKE executor + behaves the way we expect with the escaped pattern.""" + engine = create_engine("sqlite:///:memory:") + metadata = MetaData() + table = Table( + "paths", + metadata, + Column("relative_path", String(1024), nullable=False), + ) + metadata.create_all(engine) + with engine.begin() as conn: + # Target row (the one a parent_path="foo_bar" search MUST return). + conn.execute( + table.insert(), + {"relative_path": "workspace/alice/foo_bar/inner.py"}, + ) + # Decoys the buggy LIKE would match but escaped the must NOT. + conn.execute( + table.insert(), + {"relative_path": "workspace/alice/fooXbar/decoy.py"}, + ) + conn.execute( + table.insert(), + {"relative_path": "workspace/alice/foo2bar/decoy.py"}, + ) + # A truly unrelated path. + conn.execute( + table.insert(), + {"relative_path": "workspace/alice/baz/inner.py"}, + ) + yield engine, table + engine.dispose() + + +def test_sqlite_like_with_escape_does_not_match_sibling(sqlite_like_table): + """Execute the actual LIKE pattern the endpoint would emit for + parent_path='foo_bar'. Confirms only the target row matches.""" + engine, table = sqlite_like_table + escaped_prefix = _escape_like_pattern("workspace/alice/foo_bar") + "/" + pattern = f"{escaped_prefix}%" + with engine.connect() as conn: + rows = conn.execute( + select(table.c.relative_path).where( + table.c.relative_path.like(pattern, escape="\\") + ) + ).fetchall() + matched = sorted(r[0] for r in rows) + assert matched == ["workspace/alice/foo_bar/inner.py"], matched + + +def test_sqlite_like_without_escape_matches_siblings(sqlite_like_table): + """Sanity check: WITHOUT escape, the same pattern matches the + decoys too — confirming the test setup actually exercises the + leak. If this assertion fails the SQLite fixture is broken.""" + engine, table = sqlite_like_table + pattern = "workspace/alice/foo_bar/%" + with engine.connect() as conn: + rows = conn.execute( + select(table.c.relative_path).where( + table.c.relative_path.like(pattern) + ) + ).fetchall() + matched = sorted(r[0] for r in rows) + # Without escape, the buggy behaviour returns ALL three foo*bar rows. + assert len(matched) >= 2, matched + + +def test_sqlite_like_with_percent_in_name(sqlite_like_table): + """Folder name containing ``%`` — must be escaped too.""" + engine, table = sqlite_like_table + with engine.begin() as conn: + conn.execute( + table.insert(), + {"relative_path": "workspace/alice/100%off/x.py"}, + ) + conn.execute( + table.insert(), + {"relative_path": "workspace/alice/100Xoff/y.py"}, + ) + escaped_prefix = _escape_like_pattern("workspace/alice/100%off") + "/" + pattern = f"{escaped_prefix}%" + with engine.connect() as conn: + rows = conn.execute( + select(table.c.relative_path).where( + table.c.relative_path.like(pattern, escape="\\") + ) + ).fetchall() + matched = sorted(r[0] for r in rows) + assert matched == ["workspace/alice/100%off/x.py"], matched + +# ─── layer 3.5: workspace-wide subdirectory semantics ────────────── + + +@pytest.fixture +def sqlite_workspace_table(): + """SQLite table of physical relative_path rows (workspace/{owner}/...).""" + engine = create_engine("sqlite:///:memory:") + metadata = MetaData() + table = Table( + "paths", + metadata, + Column("relative_path", String(1024), nullable=False), + ) + metadata.create_all(engine) + with engine.begin() as conn: + conn.execute( + table.insert(), + [ + # Direct children of `foo/bar` across two owners. + {"relative_path": "workspace/alice/foo/bar/a.py"}, + {"relative_path": "workspace/bob/foo/bar/b.py"}, + # Deeper than one level under foo/bar → must be excluded. + {"relative_path": "workspace/bob/foo/bar/nested/c.py"}, + # Sibling folder `barX` / `foobar` → must be excluded. + {"relative_path": "workspace/alice/foo/barX/decoy.py"}, + {"relative_path": "workspace/alice/foobar/x.py"}, + # Root-level files (not under any subdir). + {"relative_path": "workspace/alice/root.py"}, + ], + ) + yield engine, table + engine.dispose() + + +def _run_like(query_table, prefix: str, escape: str = "\\"): + """Run the endpoint's LIKE + NOT LIKE direct-child pair on SQLite. + + Mirrors list_scripts exactly: LIKE ``prefix + '%'`` and + NOT LIKE ``prefix + '%/%'``.""" + engine, table = query_table + stmt = select(table.c.relative_path).where( + table.c.relative_path.like(f"{prefix}%", escape=escape), + ~table.c.relative_path.like(f"{prefix}%/%", escape=escape), + ) + with engine.connect() as conn: + rows = conn.execute(stmt).fetchall() + return sorted(r[0] for r in rows) + + +def test_sqlite_workspace_subdir_prefix_matches_direct_children_across_owners( + sqlite_workspace_table, +) -> None: + """parent_path='foo/bar' must return each owner's DIRECT children of + foo/bar — the workspace-wide contract for lazy directory loading.""" + prefix = _build_list_scripts_workspace_descendant_prefix("foo/bar") + matched = _run_like(sqlite_workspace_table, prefix) + assert matched == [ + "workspace/alice/foo/bar/a.py", + "workspace/bob/foo/bar/b.py", + ], matched + + +def test_sqlite_workspace_root_prefix_matches_each_owner_root( + sqlite_workspace_table, +) -> None: + """parent_path='' must return root-level files across all owners — + requirement: empty parent_path defaults to pulling root-path files.""" + prefix = _build_list_scripts_workspace_descendant_prefix("") + matched = _run_like(sqlite_workspace_table, prefix) + # Direct children of each owner's root = that owner's root-level file. + assert matched == ["workspace/alice/root.py"], matched + + +def test_sqlite_workspace_subdir_escapes_underscore_across_owners() -> None: + """Escaping still works with the owner wildcard added: `foo_bar` must + not match `fooXbar` under ANY owner.""" + engine = create_engine("sqlite:///:memory:") + metadata = MetaData() + table = Table( + "paths", + metadata, + Column("relative_path", String(1024), nullable=False), + ) + metadata.create_all(engine) + with engine.begin() as conn: + conn.execute( + table.insert(), + [ + {"relative_path": "workspace/alice/foo_bar/inner.py"}, + {"relative_path": "workspace/bob/foo_bar/inner.py"}, + {"relative_path": "workspace/alice/fooXbar/decoy.py"}, + ], + ) + prefix = _build_list_scripts_workspace_descendant_prefix("foo_bar") + matched = _run_like((engine, table), prefix) + assert matched == [ + "workspace/alice/foo_bar/inner.py", + "workspace/bob/foo_bar/inner.py", + ], matched + engine.dispose() + + +# ─── layer 4: single-script visibility enforcement ──────────────── + + +def _make_script_for_can_view( + *, owner_user_id: str = "U001", visibility: str = "private" +) -> SimpleNamespace: + return SimpleNamespace( + script_id="S1", + workspace_id="W001", + current_object_id="O1", + owner_user_id=owner_user_id, + script_name="x.py", + script_type="python", + visibility=visibility, + status="active", + is_locked=0, + ) + + +class TestScriptCanView: + """同一 workspace 内：owner 永远可见自己的脚本（含 private）； + 其他成员只见 visibility in {workspace, public} 的脚本； + admin 全部可见——与 data resources 的 can_view 对称。""" + + @staticmethod + def _viewer() -> SimpleNamespace: + return _ctx("U002") # not the owner + + @staticmethod + def _owner() -> SimpleNamespace: + return _ctx("U001") + + @staticmethod + def _admin() -> SimpleNamespace: + return _ctx("U002", is_admin=True) + + def test_owner_can_view_own_private_script(self) -> None: + from backend.api.scripts import script_can_view + + assert ( + script_can_view( + _make_script_for_can_view(visibility="private"), + self._owner(), + ) + is True + ) + + def test_workspace_member_cannot_view_others_private_script(self) -> None: + from backend.api.scripts import script_can_view + + assert ( + script_can_view( + _make_script_for_can_view(visibility="private"), + self._viewer(), + ) + is False + ) + + def test_workspace_member_can_view_others_workspace_script(self) -> None: + from backend.api.scripts import script_can_view + + assert ( + script_can_view( + _make_script_for_can_view(visibility="workspace"), + self._viewer(), + ) + is True + ) + + def test_workspace_member_can_view_others_public_script(self) -> None: + from backend.api.scripts import script_can_view + + assert ( + script_can_view( + _make_script_for_can_view(visibility="public"), + self._viewer(), + ) + is True + ) + + def test_admin_can_view_others_private_script(self) -> None: + from backend.api.scripts import script_can_view + + assert ( + script_can_view( + _make_script_for_can_view(visibility="private"), + self._admin(), + ) + is True + ) + + +def _script_row_for_get(script: SimpleNamespace) -> SimpleNamespace: + """Build the (script, storage_object) tuple get_script_row returns.""" + storage = SimpleNamespace( + relative_path=f"workspace/{script.owner_user_id}/x.py", + object_key=f"W001/{script.owner_user_id}/x.py", + content_hash="h", + size_bytes=1, + ) + return SimpleNamespace(data=(script, storage)) + + +async def test_get_script_non_owner_private_returns_404() -> None: + """Non-owner must NOT read another member's private script by id + (mirrors get_visible_resource for data resources).""" + from datetime import datetime, timezone + + from backend.api.scripts import get_script + + script = _make_script_for_can_view(visibility="private") + script.created_at = datetime(2026, 1, 1, tzinfo=timezone.utc) + script.updated_at = datetime(2026, 1, 1, tzinfo=timezone.utc) + mock_session = MagicMock() + mock_session.execute = AsyncMock( + return_value=MagicMock(one_or_none=MagicMock(return_value=(script, None))) + ) + with pytest.raises(HTTPException) as exc: + await get_script("S1", context=_ctx("U002"), session=mock_session) + assert exc.value.status_code == 404 + + +async def test_get_script_owner_can_read_own_private_script() -> None: + from datetime import datetime, timezone + + from backend.api.scripts import get_script + + script = _make_script_for_can_view(visibility="private") + script.created_at = datetime(2026, 1, 1, tzinfo=timezone.utc) + script.updated_at = datetime(2026, 1, 1, tzinfo=timezone.utc) + storage = SimpleNamespace( + relative_path="workspace/U001/x.py", + object_key="W001/U001/x.py", + content_hash="h", + size_bytes=1, + ) + mock_session = MagicMock() + mock_session.execute = AsyncMock( + return_value=MagicMock(one_or_none=MagicMock(return_value=(script, storage))) + ) + result = await get_script("S1", context=_ctx("U001"), session=mock_session) + assert result["data"]["script_id"] == "S1" + assert result["data"]["visibility"] == "private" + + +async def test_get_script_non_owner_can_read_workspace_visible_script() -> None: + from datetime import datetime, timezone + + from backend.api.scripts import get_script + + script = _make_script_for_can_view(visibility="workspace") + script.created_at = datetime(2026, 1, 1, tzinfo=timezone.utc) + script.updated_at = datetime(2026, 1, 1, tzinfo=timezone.utc) + storage = SimpleNamespace( + relative_path="workspace/U001/x.py", + object_key="W001/U001/x.py", + content_hash="h", + size_bytes=1, + ) + mock_session = MagicMock() + mock_session.execute = AsyncMock( + return_value=MagicMock(one_or_none=MagicMock(return_value=(script, storage))) + ) + result = await get_script("S1", context=_ctx("U002"), session=mock_session) + assert result["data"]["script_id"] == "S1" + assert result["data"]["visibility"] == "workspace" diff --git a/backend/tests/test_platform_pagination.py b/backend/tests/test_platform_pagination.py new file mode 100644 index 0000000..eb380cf --- /dev/null +++ b/backend/tests/test_platform_pagination.py @@ -0,0 +1,57 @@ +"""Unit tests for platform cursor pagination helpers.""" + +from __future__ import annotations + +import datetime + +import pytest +from fastapi import HTTPException + +from backend.api.platform._pagination import ( + decode_cursor, + encode_cursor, + page_meta, +) + + +def test_encode_decode_roundtrip() -> None: + created_at = datetime.datetime(2026, 3, 15, 12, 30, 45, 123000) + row_id = "01HXY9C5B8N3K4P7Q6RT2V0J8D" + cursor = encode_cursor(created_at, row_id) + decoded_ts, decoded_id = decode_cursor(cursor) + assert decoded_ts == created_at + assert decoded_id == row_id + + +def test_decode_strips_timezone() -> None: + created_at = datetime.datetime(2026, 1, 1, 0, 0, 0, tzinfo=datetime.timezone.utc) + cursor = encode_cursor(created_at, "abc") + decoded_ts, decoded_id = decode_cursor(cursor) + assert decoded_ts.tzinfo is None + assert decoded_id == "abc" + + +def test_decode_invalid_cursor_raises_400() -> None: + with pytest.raises(HTTPException) as exc_info: + decode_cursor("not-a-valid-cursor!!!") + assert exc_info.value.status_code == 400 + + +def test_page_meta_has_more() -> None: + meta = page_meta( + limit=10, + page_count=10, + total_count=25, + next_cursor="abc", + ) + assert meta["has_more"] is True + assert meta["next_cursor"] == "abc" + assert meta["total_count"] == 25 + + meta_end = page_meta( + limit=10, + page_count=5, + total_count=25, + next_cursor=None, + ) + assert meta_end["has_more"] is False diff --git a/backend/tests/test_resources.py b/backend/tests/test_resources.py index 10b0df3..c160718 100644 --- a/backend/tests/test_resources.py +++ b/backend/tests/test_resources.py @@ -7,10 +7,15 @@ from __future__ import annotations import datetime from types import SimpleNamespace -from unittest.mock import MagicMock +from unittest.mock import AsyncMock, MagicMock import pytest -from backend.resources import ( +from fastapi import HTTPException +from sqlalchemy import Column, MetaData, String, Table, create_engine, select +from sqlalchemy.dialects import mysql as mysql_dialect +from backend.api.resources import ( + _build_list_resources_descendant_prefix, + can_view, compute_jupyter_relative_path, resource_directory, resource_payload, @@ -121,7 +126,7 @@ def _bind_payload(): @pytest.mark.asyncio async def test_bind_resource_rejects_duplicate_name_in_same_directory() -> None: """Same resource_name in the same directory raises 409.""" - from backend.resources import bind_resource + from backend.api.resources import bind_resource existing_rows = [ ( @@ -148,7 +153,7 @@ async def test_bind_resource_rejects_duplicate_name_in_same_directory() -> None: @pytest.mark.asyncio async def test_bind_resource_allows_same_name_in_different_directory() -> None: """Same resource_name in a different directory binds successfully.""" - from backend.resources import bind_resource + from backend.api.resources import bind_resource existing_rows = [ ( @@ -174,7 +179,7 @@ async def test_bind_resource_allows_same_name_in_different_directory() -> None: @pytest.mark.asyncio async def test_bind_resource_allows_same_name_when_workspace_empty() -> None: """No same-name rows at all: bind succeeds (root directory).""" - from backend.resources import bind_resource + from backend.api.resources import bind_resource session = _BindSessionMock( new_object_key=f"{_BIND_WS}/{_BIND_USER}/data.csv", @@ -192,7 +197,7 @@ async def test_bind_resource_allows_same_name_when_workspace_empty() -> None: @pytest.mark.asyncio async def test_bind_resource_allows_same_name_for_different_owner() -> None: """其他用户在同目录下的同名资源不阻塞当前用户的绑定。""" - from backend.resources import bind_resource + from backend.api.resources import bind_resource other_user = "01USR0000000000000000000B" existing_rows = [ @@ -218,7 +223,7 @@ async def test_bind_resource_allows_same_name_for_different_owner() -> None: @pytest.mark.asyncio async def test_bind_resource_allows_rebinding_same_storage_object() -> None: """重新绑定同一 upload_id 应走 idempotent 复用路径,不触发 409。""" - from backend.resources import bind_resource + from backend.api.resources import bind_resource new_object_key = f"{_BIND_WS}/{_BIND_USER}/data.csv" existing_resource = _make_resource(_BIND_WS, _BIND_USER) @@ -246,7 +251,7 @@ async def test_bind_resource_allows_rebinding_same_storage_object() -> None: @pytest.mark.asyncio async def test_bind_resource_rejects_non_data_resource_upload() -> None: """其他用途(如 working_copy)的 upload session 不能 bind 成数据资源。""" - from backend.resources import bind_resource + from backend.api.resources import bind_resource session = _BindSessionMock( new_object_key=f"{_BIND_WS}/{_BIND_USER}/data.csv", @@ -317,6 +322,8 @@ def test_resource_payload_legacy_dot_resources(): ) assert payload["jupyter_accessible_path"] == ".resources/data.csv" assert payload["absolute_path"].endswith(f"{ws}/{user}/.resources/data.csv") + # Default None when no Users join row is provided (bind_resource path). + assert payload["owner_display_name"] is None def test_resource_payload_new_flat_path(): @@ -328,6 +335,13 @@ def test_resource_payload_new_flat_path(): ) assert payload["jupyter_accessible_path"] == "data.csv" assert payload["absolute_path"].endswith(f"{ws}/{user}/data.csv") + # Explicit owner_display_name is passed through to the payload. + payload = resource_payload( + _make_resource(ws, user), + _make_storage_object(f"{ws}/{user}/data.csv"), + owner_display_name="张三", + ) + assert payload["owner_display_name"] == "张三" def test_resource_payload_new_nested_path(): @@ -339,6 +353,7 @@ def test_resource_payload_new_nested_path(): ) assert payload["jupyter_accessible_path"] == "train/v1/data.csv" assert payload["absolute_path"].endswith(f"{ws}/{user}/train/v1/data.csv") + assert payload["owner_display_name"] is None def test_compute_jupyter_relative_path_for_legacy_and_new_paths(): @@ -405,3 +420,391 @@ def test_data_resources_model_allows_duplicate_storage_object_reference() -> Non assert "storage_object_id" not in cols, ( f"unexpected unique index {idx.name} on storage_object_id" ) + + +# ─── parent_path filtering (mirrors test_list_scripts_parent_path.py) ──────── + + +def _resource_ctx(workspace_id: str = "W001") -> SimpleNamespace: + return SimpleNamespace( + request_id="test", + user=SimpleNamespace(user_id="U001"), + workspace=SimpleNamespace(workspace_id=workspace_id), + is_admin=False, + ) + + +class TestCanViewVisibility: + """同一 workspace 内：owner 永远可见自己的资源（含 private）； + 其他成员只见 visibility in {workspace, public} 的资源； + admin 全部可见。与 list_resources 的 SQL 谓词一致。""" + + @staticmethod + def _viewer() -> SimpleNamespace: + return SimpleNamespace( + request_id="test", + user=SimpleNamespace(user_id="U002"), # not the owner + workspace=SimpleNamespace(workspace_id="W001"), + is_admin=False, + ) + + @staticmethod + def _owner() -> SimpleNamespace: + ctx = TestCanViewVisibility._viewer() + ctx.user = SimpleNamespace(user_id="U001") # the owner + return ctx + + @staticmethod + def _admin() -> SimpleNamespace: + ctx = TestCanViewVisibility._viewer() + ctx.is_admin = True + return ctx + + @staticmethod + def _resource(visibility: str) -> SimpleNamespace: + res = _make_resource("W001", "U001") + res.visibility = visibility + return res + + def test_owner_can_view_own_private_resource(self) -> None: + assert ( + can_view( + self._resource("private"), + self._owner(), + ) + is True + ) + + def test_workspace_member_cannot_view_others_private_resource(self) -> None: + assert ( + can_view( + self._resource("private"), + self._viewer(), + ) + is False + ) + + def test_workspace_member_can_view_others_workspace_resource(self) -> None: + assert ( + can_view( + self._resource("workspace"), + self._viewer(), + ) + is True + ) + + def test_workspace_member_can_view_others_public_resource(self) -> None: + assert ( + can_view( + self._resource("public"), + self._viewer(), + ) + is True + ) + + def test_admin_can_view_others_private_resource(self) -> None: + assert ( + can_view( + self._resource("private"), + self._admin(), + ) + is True + ) + + +class TestBuildListResourcesDescendantPrefix: + """Pure-function contract for the escaped object_key prefix. Unlike + the scripts helper there is NO user scope — data resources are + workspace-wide, so the prefix is just the normalized+escaped path.""" + + def test_empty_parent_path_returns_empty_prefix(self) -> None: + assert _build_list_resources_descendant_prefix("") == "" + + def test_subdir_prefix_appends_trailing_slash(self) -> None: + assert _build_list_resources_descendant_prefix("foo/bar") == "foo/bar/" + + def test_escapes_underscore(self) -> None: + assert _build_list_resources_descendant_prefix("foo_bar") == r"foo\_bar/" + + def test_escapes_percent(self) -> None: + assert _build_list_resources_descendant_prefix("100%match") == r"100\%match/" + + def test_normalizes_leading_trailing_slashes(self) -> None: + assert _build_list_resources_descendant_prefix("/foo/bar/") == "foo/bar/" + + def test_rejects_traversal(self) -> None: + with pytest.raises(HTTPException) as exc: + _build_list_resources_descendant_prefix("foo/../bar") + assert exc.value.status_code == 422 + + +# ─── layer 2: SQL contract (mock session + mysql dialect compile) ──────────── + + +def _compile_sql(stmt) -> str: + return str( + stmt.compile( + dialect=mysql_dialect.dialect(), + compile_kwargs={"literal_binds": True}, + ) + ) + + +class _ListResourcesMockResult: + def all(self): + return [] + + +def _list_resources_capturing_session(captured_sql: list[str]) -> MagicMock: + mock_session = MagicMock() + mock_session.execute = AsyncMock( + side_effect=lambda stmt: ( + captured_sql.append(_compile_sql(stmt)) or _ListResourcesMockResult() + ) + ) + return mock_session + + +async def test_list_resources_where_clause_uses_like_prefix_and_excludes_deeper() -> None: + from backend.api.resources import list_resources + + captured_sql: list[str] = [] + mock_session = _list_resources_capturing_session(captured_sql) + + await list_resources( + parent_path="foo/bar", + owner_user_id=None, + context=_resource_ctx(), + session=mock_session, + visibility=None, + keyword=None, + ) + + assert len(captured_sql) == 1 + sql = captured_sql[0].lower() + # Default (no owner_user_id) scopes to the requester's own object_key + # subtree: LIKE w001/u001/foo/bar/% (direct children), excluding deeper. + assert "like 'w001/u001/foo/bar/%%'" in sql + assert "not like 'w001/u001/foo/bar/%%/%%'" in sql + + +async def test_list_resources_where_clause_escapes_underscore() -> None: + """Regression: parent_path containing ``_`` MUST be escaped in the + compiled LIKE pattern, otherwise sibling-path leak (``fooXbar``) returns.""" + from backend.api.resources import list_resources + + captured_sql: list[str] = [] + mock_session = _list_resources_capturing_session(captured_sql) + + await list_resources( + parent_path="foo_bar", + owner_user_id=None, + context=_resource_ctx(), + session=mock_session, + visibility=None, + keyword=None, + ) + sql = captured_sql[0] + sql_lower = sql.lower() + # SQLAlchemy doubles the escape char inside the SQL string literal, so + # the helper's ``foo\_bar`` renders as ``foo\\_bar`` in the SQL text. + assert r"like 'w001/u001/foo\\_bar/%%'" in sql_lower + assert r"not like 'w001/u001/foo\\_bar/%%/%%'" in sql_lower + # Both LIKE clauses declare ESCAPE '\\' (two in total). + assert sql.count("ESCAPE '\\\\'") == 2, sql + + +async def test_list_resources_empty_parent_path_adds_root_like_clause() -> None: + """Empty parent_path still applies the directory filter (symmetric with + list_scripts): ``{ws_id}/{owner}/%`` AND NOT ``{ws_id}/{owner}/%/%`` so + the owner-scoped root view returns only direct children of the + requester's root, never nested descendants. Skipping the filter for + empty input used to surface nested resources at the root and visually + broke the directory tree. + """ + from backend.api.resources import list_resources + + captured_sql: list[str] = [] + mock_session = _list_resources_capturing_session(captured_sql) + + await list_resources( + parent_path="", + owner_user_id=None, + context=_resource_ctx(), + session=mock_session, + visibility=None, + keyword=None, + ) + sql = captured_sql[0].lower() + assert " like 'w001/u001/%%'" in sql + assert " not like 'w001/u001/%%/%%'" in sql + + +async def test_list_resources_owner_param_scopes_to_other_owner() -> None: + """owner_user_id= scopes object_key LIKE to that owner's subtree + so the tree can lazily fetch another member's data resources on group + expand. Non-admin visibility predicate is still applied, so the other + owner's private resources are excluded (workspace/public only). + """ + from backend.api.resources import list_resources + + captured_sql: list[str] = [] + mock_session = _list_resources_capturing_session(captured_sql) + + await list_resources( + parent_path="", + owner_user_id="U002", + context=_resource_ctx(), + session=mock_session, + visibility=None, + keyword=None, + ) + sql = captured_sql[0].lower() + assert " like 'w001/u002/%%'" in sql + assert " not like 'w001/u002/%%/%%'" in sql + # non-admin: visibility predicate still present (excludes U002 private) + assert "data_resources.owner_user_id = 'u001'" in sql + assert "data_resources.visibility in ('workspace', 'public')" in sql + + +async def test_list_resources_joins_users_for_display_name() -> None: + """list_resources must OUTER JOIN users and SELECT users.display_name so + every resource carries owner_display_name (frontend displayName chain).""" + from backend.api.resources import list_resources + + captured_sql: list[str] = [] + mock_session = _list_resources_capturing_session(captured_sql) + + await list_resources( + parent_path="", + owner_user_id=None, + context=_resource_ctx(), + session=mock_session, + visibility=None, + keyword=None, + ) + sql = captured_sql[0] + sql_lower = sql.lower() + assert "outer join users" in sql_lower + assert "users.display_name" in sql_lower + + +# ─── layer 3: behavioral test on real LIKE execution (SQLite) ──────────────── + + +@pytest.fixture +def sqlite_object_key_table(): + """SQLite in-memory stand-in for ``storage_objects.object_key`` rows. + + Three-layer structure: direct children under ``data`` owned by two + different users (cross-owner), a deeper descendant, a sibling folder, + a root file, plus ``data_x`` and its ``dataXx`` / ``data2x`` decoys. + """ + engine = create_engine("sqlite:///:memory:") + metadata = MetaData() + table = Table( + "object_keys", + metadata, + Column("object_key", String(1024), nullable=False), + ) + metadata.create_all(engine) + rows = [ + "W001/U001/data/alpha.csv", # direct child (owner U001) + "W001/U002/data/beta.csv", # direct child (owner U002) + "W001/U001/data/deep/nested.csv", # deeper descendant + "W001/U001/database/gamma.csv", # sibling folder + "W001/U001/root.csv", # root file + "W001/U001/data_x/delta.csv", # target for parent_path=data_x + "W001/U001/dataXx/decoy.csv", # sibling decoy (unescaped match) + "W001/U001/data2x/decoy2.csv", # sibling decoy (unescaped match) + ] + with engine.begin() as conn: + conn.execute(table.insert(), [{"object_key": key} for key in rows]) + yield engine, table + engine.dispose() + + +def test_sqlite_direct_children_across_owners(sqlite_object_key_table) -> None: + """parent_path='data' returns exactly the direct children under data, + across every owner, excluding deeper/sibling/root paths.""" + engine, table = sqlite_object_key_table + prefix = _build_list_resources_descendant_prefix("data") + like = f"W001/%/{prefix}%" + not_like = f"W001/%/{prefix}%/%" + with engine.connect() as conn: + rows = conn.execute( + select(table.c.object_key).where( + table.c.object_key.like(like, escape="\\"), + ~table.c.object_key.like(not_like, escape="\\"), + ) + ).fetchall() + matched = sorted(r[0] for r in rows) + assert matched == [ + "W001/U001/data/alpha.csv", + "W001/U002/data/beta.csv", + ], matched + + +def test_sqlite_escaped_underscore_does_not_match_sibling( + sqlite_object_key_table, +) -> None: + """parent_path='data_x' must match only the literal data_x folder, not + the dataXx / data2x siblings an unescaped pattern would match.""" + engine, table = sqlite_object_key_table + prefix = _build_list_resources_descendant_prefix("data_x") + like = f"W001/%/{prefix}%" + not_like = f"W001/%/{prefix}%/%" + with engine.connect() as conn: + rows = conn.execute( + select(table.c.object_key).where( + table.c.object_key.like(like, escape="\\"), + ~table.c.object_key.like(not_like, escape="\\"), + ) + ).fetchall() + matched = sorted(r[0] for r in rows) + assert matched == ["W001/U001/data_x/delta.csv"], matched + + +def test_sqlite_parent_path_with_slash_direct_children(sqlite_object_key_table) -> None: + """parent_path='data/deep' returns only data/deep/* — never data/*.""" + engine, table = sqlite_object_key_table + prefix = _build_list_resources_descendant_prefix("data/deep") + like = f"W001/%/{prefix}%" + not_like = f"W001/%/{prefix}%/%" + with engine.connect() as conn: + rows = conn.execute( + select(table.c.object_key).where( + table.c.object_key.like(like, escape="\\"), + ~table.c.object_key.like(not_like, escape="\\"), + ) + ).fetchall() + matched = sorted(r[0] for r in rows) + assert matched == ["W001/U001/data/deep/nested.csv"], matched + + +def test_sqlite_empty_parent_path_returns_root_level_across_owners( + sqlite_object_key_table, +) -> None: + """Empty parent_path now applies the root filter (symmetric with + list_scripts): ``{ws_id}/%/%`` AND NOT ``{ws_id}/%/%/%`` returns only + direct children of every owner's root, excluding nested descendants. + Earlier 'no LIKE' behaviour used to surface every row in the + workspace at the root, which is exactly what made scripts and data + appear mutually visible and broke the tree. + """ + engine, table = sqlite_object_key_table + like = "W001/%/%" + not_like = "W001/%/%/%" + with engine.connect() as conn: + rows = conn.execute( + select(table.c.object_key).where( + table.c.object_key.like(like, escape="\\"), + ~table.c.object_key.like(not_like, escape="\\"), + ) + ).fetchall() + matched = sorted(r[0] for r in rows) + # ``W001/U001/root.csv`` is the only 3-segment path (= direct child + # of the owner root); all 4+ segment paths (data/*, database/*) are + # excluded by the NOT LIKE clause. The other 4+ segment files would + # be returned when the user expands the corresponding subdirectory. + assert matched == ["W001/U001/root.csv"], matched diff --git a/backend/tests/test_role_permissions_re_add.py b/backend/tests/test_role_permissions_re_add.py new file mode 100644 index 0000000..6998439 --- /dev/null +++ b/backend/tests/test_role_permissions_re_add.py @@ -0,0 +1,252 @@ +"""Regression tests for the ``role_permissions`` re-add (upsert) flow. + +Reproduces the PATCH /api/v1/platform/roles/admin/permissions bug where +re-adding a permission that was previously soft-deleted collides with the +composite ``(role_id, permission_id)`` PRIMARY KEY: the soft-deleted row +still occupies its PK slot, so a plain INSERT raises a duplicate-key +IntegrityError (MySQL 1062). + +Uses sqlite in-memory + aiosqlite (``create_async_engine``) and drives +:func:`_apply_role_permission_set` directly — no HTTP layer, no MySQL. +""" + +from __future__ import annotations + +import datetime + +import pytest +from sqlalchemy import func, insert, select, text +from sqlalchemy.ext.asyncio import async_sessionmaker, create_async_engine +from sqlalchemy.pool import StaticPool + +from common.db.models import Permissions, RolePermissions, Roles +from backend.api.platform._permission_set import _apply_role_permission_set + +# -- realistic admin permission codes (matches the reported repro) ---------- +SYSTEM_VIEW = "system:view" +SYSTEM_USER_VIEW = "system:user:view" +SYSTEM_PROJECT_VIEW = "system:project:view" + +# Raw DDL mirrors the ORM models (roles/permissions/role_permissions) with +# sqlite-compatible types — the MySQL dialects (CHAR/TINYINT/DATETIME(fsp)) +# cannot be compiled by SQLite's DDL compiler, so the tables are created by +# hand and then driven through the ORM at runtime. +_DDL = [ + """ + CREATE TABLE roles ( + role_id VARCHAR(26) PRIMARY KEY, + role_code VARCHAR(64) NOT NULL UNIQUE, + role_name VARCHAR(100) NOT NULL, + role_scope VARCHAR(16) NOT NULL, + is_builtin INTEGER NOT NULL DEFAULT 0, + created_at DATETIME NOT NULL DEFAULT CURRENT_TIMESTAMP, + updated_at DATETIME NOT NULL DEFAULT CURRENT_TIMESTAMP, + description VARCHAR(500), + is_deleted INTEGER NOT NULL DEFAULT 0, + deleted_at DATETIME + ) + """, + """ + CREATE TABLE permissions ( + permission_id VARCHAR(26) PRIMARY KEY, + permission_code VARCHAR(128) NOT NULL UNIQUE, + permission_name VARCHAR(100) NOT NULL, + module_code VARCHAR(64) NOT NULL, + created_at DATETIME NOT NULL DEFAULT CURRENT_TIMESTAMP, + description VARCHAR(500), + is_deleted INTEGER NOT NULL DEFAULT 0, + deleted_at DATETIME + ) + """, + """ + CREATE TABLE role_permissions ( + role_id VARCHAR(26) NOT NULL, + permission_id VARCHAR(26) NOT NULL, + created_at DATETIME NOT NULL DEFAULT CURRENT_TIMESTAMP, + is_deleted INTEGER NOT NULL DEFAULT 0, + deleted_at DATETIME, + PRIMARY KEY (role_id, permission_id) + ) + """, +] + + +@pytest.fixture +async def session(): + """Async in-memory sqlite session with the three identity tables.""" + engine = create_async_engine( + "sqlite+aiosqlite://", + poolclass=StaticPool, + connect_args={"check_same_thread": False}, + ) + async with engine.begin() as conn: + for ddl in _DDL: + await conn.execute(text(ddl)) + Session = async_sessionmaker(engine, expire_on_commit=False) + async with Session() as db: + yield db + await engine.dispose() + + +# -- helpers --------------------------------------------------------------- + + +async def _add_permission(session, permission_id: str, code: str) -> None: + session.add( + Permissions( + permission_id=permission_id, + permission_code=code, + permission_name=code, + module_code="system", + description=None, + ) + ) + await session.flush() + + +async def _add_role(session, role_id: str, role_code: str) -> Roles: + role = Roles( + role_id=role_id, + role_code=role_code, + role_name=role_code, + role_scope="platform", + is_builtin=1, + description=None, + ) + session.add(role) + await session.flush() + return role + + +async def _link(session, role_id: str, permission_id: str, *, active: bool) -> None: + """Insert (or directly mark) a role_permissions row with a given state.""" + await session.execute( + insert(RolePermissions), + [ + { + "role_id": role_id, + "permission_id": permission_id, + "is_deleted": 0 if active else 1, + "deleted_at": None if active else datetime.datetime(2026, 1, 1), + } + ], + ) + await session.flush() + + +async def _active_count(session, role_id: str) -> int: + return int( + await session.scalar( + select(func.count()) + .select_from(RolePermissions) + .where( + RolePermissions.role_id == role_id, + RolePermissions.is_deleted == 0, + ) + ) + or 0 + ) + + +async def _active_row_count(session, role_id: str) -> int: + return int( + await session.scalar( + select(func.count()) + .select_from(RolePermissions) + .where(RolePermissions.role_id == role_id) + ) + or 0 + ) + + +# -- tests ----------------------------------------------------------------- + + +async def test_happy_path_soft_delete_then_readd(session) -> None: + """Soft-delete two permissions, then re-add them — no IntegrityError.""" + role = await _add_role(session, "R" * 26, "admin") + await _add_permission(session, "P1" * 13, SYSTEM_VIEW) + await _add_permission(session, "P2" * 13, SYSTEM_USER_VIEW) + await _add_permission(session, "P3" * 13, SYSTEM_PROJECT_VIEW) + + view_id = "P1" * 13 + user_view_id = "P2" * 13 + project_view_id = "P3" * 13 + for pid in (view_id, user_view_id, project_view_id): + await _link(session, role.role_id, pid, active=True) + + # 1st PATCH: shrink to only system:view → the other two get soft-deleted. + codes = await _apply_role_permission_set(session, role, [SYSTEM_VIEW]) + assert codes == [SYSTEM_VIEW] + assert await _active_count(session, role.role_id) == 1 + assert await _active_row_count(session, role.role_id) == 3 + + # 2nd PATCH: restore the full set → the soft-deleted rows must be + # revived (UPDATE), not INSERTed over (would raise IntegrityError). + codes = await _apply_role_permission_set( + session, + role, + [SYSTEM_VIEW, SYSTEM_USER_VIEW, SYSTEM_PROJECT_VIEW], + ) + assert codes == [SYSTEM_PROJECT_VIEW, SYSTEM_USER_VIEW, SYSTEM_VIEW] + assert await _active_count(session, role.role_id) == 3 + assert await _active_row_count(session, role.role_id) == 3 + + states = ( + await session.execute( + select(RolePermissions.is_deleted, RolePermissions.deleted_at).where( + RolePermissions.role_id == role.role_id + ) + ) + ).all() + assert all(is_deleted == 0 and deleted_at is None for is_deleted, deleted_at in states) + + +async def test_fresh_insert_no_history(session) -> None: + """Brand-new role: a never-before-linked permission is plain INSERTed.""" + role = await _add_role(session, "R" * 26, "admin") + await _add_permission(session, "P1" * 13, SYSTEM_VIEW) + + codes = await _apply_role_permission_set(session, role, [SYSTEM_VIEW]) + assert codes == [SYSTEM_VIEW] + assert await _active_count(session, role.role_id) == 1 + assert await _active_row_count(session, role.role_id) == 1 + + +async def test_mixed_revive_and_fresh_insert(session) -> None: + """One historically soft-deleted row is revived, one fresh row inserted.""" + role = await _add_role(session, "R" * 26, "admin") + await _add_permission(session, "P1" * 13, SYSTEM_VIEW) + await _add_permission(session, "P2" * 13, SYSTEM_USER_VIEW) + await _link(session, role.role_id, "P1" * 13, active=False) # 历史软删行 + + codes = await _apply_role_permission_set( + session, role, [SYSTEM_VIEW, SYSTEM_USER_VIEW] + ) + assert codes == [SYSTEM_USER_VIEW, SYSTEM_VIEW] + assert await _active_count(session, role.role_id) == 2 + assert await _active_row_count(session, role.role_id) == 2 + + revived = ( + await session.execute( + select(RolePermissions.is_deleted, RolePermissions.deleted_at).where( + RolePermissions.role_id == role.role_id, + RolePermissions.permission_id == "P1" * 13, + ) + ) + ).one() + assert revived[0] == 0 and revived[1] is None + + +async def test_same_set_is_noop(session) -> None: + """PATCHing the exact same active set again changes nothing.""" + role = await _add_role(session, "R" * 26, "admin") + await _add_permission(session, "P1" * 13, SYSTEM_VIEW) + await _link(session, role.role_id, "P1" * 13, active=True) + + first = await _apply_role_permission_set(session, role, [SYSTEM_VIEW]) + second = await _apply_role_permission_set(session, role, [SYSTEM_VIEW]) + assert first == [SYSTEM_VIEW] + assert second == [SYSTEM_VIEW] + assert await _active_count(session, role.role_id) == 1 + assert await _active_row_count(session, role.role_id) == 1 diff --git a/backend/tests/test_runtime_client_directories.py b/backend/tests/test_runtime_client_directories.py index 78627f3..48a6f18 100644 --- a/backend/tests/test_runtime_client_directories.py +++ b/backend/tests/test_runtime_client_directories.py @@ -15,7 +15,7 @@ from __future__ import annotations import httpx import pytest import respx -from backend.runtime_client import RuntimeClient, RuntimeClientError +from backend.clients.runtime import RuntimeClient, RuntimeClientError WORKSPACE_ID = "01HWS0000000000000000000A" BASE_URL = "http://runtime" diff --git a/backend/tests/test_scripts.py b/backend/tests/test_scripts.py index 8b6ebeb..20fba24 100644 --- a/backend/tests/test_scripts.py +++ b/backend/tests/test_scripts.py @@ -103,7 +103,7 @@ class _AsyncSessionMock: @pytest.mark.asyncio async def test_create_script_record_flushes_storage_object_before_script() -> None: """StorageObjects must flush first so path conflicts surface early.""" - from backend.scripts import create_script_record + from backend.api.scripts import create_script_record session = _AsyncSessionMock() request = _make_request() @@ -133,7 +133,7 @@ async def test_create_script_record_flushes_storage_object_before_script() -> No @pytest.mark.asyncio async def test_create_script_record_storage_object_flush_failure_does_not_add_script() -> None: """If the StorageObjects flush fails, the Scripts row must never be added.""" - from backend.scripts import create_script_record + from backend.api.scripts import create_script_record class FailingSession(_AsyncSessionMock): async def flush(self) -> None: @@ -172,7 +172,7 @@ async def test_create_script_record_allows_reupload_after_delete() -> None: """Without uk_scripts_workspace_name_active, re-uploading a script with the same name after the previous one was soft-deleted succeeds. """ - from backend.scripts import create_script_record + from backend.api.scripts import create_script_record session = _AsyncSessionMock() request = _make_request() @@ -230,7 +230,7 @@ async def test_create_script_record_allows_same_name_different_parent() -> None: must coexist — they correspond to different Jupyter paths (/user/foo.ipynb vs /user/test/foo.ipynb). """ - from backend.scripts import create_script_record + from backend.api.scripts import create_script_record session = _AsyncSessionMock() request = _make_request() @@ -286,7 +286,7 @@ async def test_create_script_after_soft_delete_does_not_conflict() -> None: raise IntegrityError — the generated column is NULL for the deleted row, so it does not occupy the UNIQUE slot. """ - from backend.scripts import create_script_record + from backend.api.scripts import create_script_record session = _AsyncSessionMock() request = _make_request() @@ -376,7 +376,7 @@ def _storage_object_row() -> StorageObjects: @pytest.mark.asyncio async def test_delete_script_route_sets_is_deleted(monkeypatch: pytest.MonkeyPatch) -> None: """Soft-deleting a script via the route handler flips is_deleted=1.""" - from backend.scripts import delete_script + from backend.api.scripts import delete_script script = _script_row() storage_object = _storage_object_row() @@ -395,7 +395,7 @@ async def test_delete_script_route_sets_is_deleted(monkeypatch: pytest.MonkeyPat ) -> tuple[Scripts, StorageObjects]: return script, storage_object - monkeypatch.setattr("backend.scripts.get_script_row", _fake_get_script_row) + monkeypatch.setattr("backend.api.scripts.get_script_row", _fake_get_script_row) mock_soft_delete = AsyncMock( return_value={ "data": { @@ -406,7 +406,7 @@ async def test_delete_script_route_sets_is_deleted(monkeypatch: pytest.MonkeyPat } } ) - monkeypatch.setattr("backend.scripts.soft_delete_object", mock_soft_delete) + monkeypatch.setattr("backend.api.scripts.soft_delete_object", mock_soft_delete) result = await delete_script( script_id=script.script_id, @@ -430,7 +430,7 @@ async def test_delete_script_route_sets_is_deleted(monkeypatch: pytest.MonkeyPat @pytest.mark.asyncio async def test_delete_resource_sets_is_deleted(monkeypatch: pytest.MonkeyPatch) -> None: """Soft-deleting a data resource must write is_deleted=1 on the row.""" - from backend.resources import delete_resource + from backend.api.resources import delete_resource resource = DataResources( resource_id="01RES0000000000000000000A", @@ -455,7 +455,7 @@ async def test_delete_resource_sets_is_deleted(monkeypatch: pytest.MonkeyPatch) with monkeypatch.context() as mp: mp.setattr( - "backend.resources.soft_delete_object", + "backend.api.resources.soft_delete_object", AsyncMock(return_value={"data": {}}), ) result = await delete_resource( @@ -558,7 +558,7 @@ async def test_soft_delete_object_streams_via_get_stream() -> None: @pytest.mark.asyncio async def test_jupyter_check_notebook_lock_ignores_deleted_scripts() -> None: """``is_deleted == 0`` filter must hide deleted notebooks from Jupyter checks.""" - from backend.jupyter import check_notebook_is_locked + from backend.api.jupyter import check_notebook_is_locked session = AsyncMock() session.execute = AsyncMock() @@ -593,8 +593,8 @@ async def test_update_script_writes_back_storage_object_metadata( """ import hashlib - from backend.schemas import UpdateScriptRequest - from backend.scripts import update_script + from backend.schemas.scripts import UpdateScriptRequest + from backend.api.scripts import update_script script = _script_row() storage_object = _storage_object_row() @@ -624,7 +624,7 @@ async def test_update_script_writes_back_storage_object_metadata( ) -> tuple[Scripts, StorageObjects]: return script, storage_object - monkeypatch.setattr("backend.scripts.get_script_row", _fake_get_script_row) + monkeypatch.setattr("backend.api.scripts.get_script_row", _fake_get_script_row) new_content = '{"cells": [{"cell_type": "code", "source": ["print(1)"]}]}\n' payload = UpdateScriptRequest(content=new_content) @@ -670,8 +670,8 @@ async def test_update_script_jupyter_only_uses_dict_fallback( """ import hashlib - from backend.schemas import UpdateScriptRequest - from backend.scripts import update_script + from backend.schemas.scripts import UpdateScriptRequest + from backend.api.scripts import update_script script = _script_row() user_id = "01USR0000000000000000000A" @@ -695,7 +695,7 @@ async def test_update_script_jupyter_only_uses_dict_fallback( ) -> tuple[Scripts, None]: return script, None - monkeypatch.setattr("backend.scripts.get_script_row", _fake_get_script_row) + monkeypatch.setattr("backend.api.scripts.get_script_row", _fake_get_script_row) payload = UpdateScriptRequest(content='{"cells": []}\n') result = await update_script( diff --git a/backend/tests/test_storage_upload_status.py b/backend/tests/test_storage_upload_status.py index a5a46f1..08d234e 100644 --- a/backend/tests/test_storage_upload_status.py +++ b/backend/tests/test_storage_upload_status.py @@ -1,7 +1,7 @@ """Unit tests for ``upload_bytes_to_session`` failure-path status persistence. P0-5 / B1: the route handler wraps every request in ``session_scope`` -(``backend.dependencies.database_session``), which rolls back on +(``backend.api.dependencies.database_session``), which rolls back on exception. A naive ``upload.upload_status = "failed"; raise HTTPException(...)`` loses the status flip and leaves the row stuck in ``created``/``uploading`` forever. The fix is ``_mark_upload_failed_and_raise`` which opens a diff --git a/backend/tests/test_validate_dag.py b/backend/tests/test_validate_dag.py new file mode 100644 index 0000000..afb3285 --- /dev/null +++ b/backend/tests/test_validate_dag.py @@ -0,0 +1,143 @@ +"""Unit tests for backend.services.schedules.validate_dag. + +Pure function — no DB, no FastAPI, no fixtures beyond SimpleNamespace +stand-ins for the SQLAlchemy rows. The function only reads five +attributes: ``node_id``, ``node_key``, ``edge_id``, ``source_node_id``, +``target_node_id``. +""" + +from __future__ import annotations + +from types import SimpleNamespace + +from backend.services.schedules import validate_dag + + +def _node(node_id: str, node_key: str) -> SimpleNamespace: + return SimpleNamespace(node_id=node_id, node_key=node_key) + + +def _edge(edge_id: str, source: str, target: str) -> SimpleNamespace: + return SimpleNamespace( + edge_id=edge_id, + source_node_id=source, + target_node_id=target, + ) + + +def test_empty_nodes_is_rejected_as_dag_empty() -> None: + result = validate_dag(nodes=[], edges=[]) + assert result["valid"] is False + assert result["node_count"] == 0 + assert result["edge_count"] == 0 + assert result["topological_order"] == [] + codes = [err["code"] for err in result["errors"]] + assert "DAG_EMPTY" in codes + + +def test_linear_chain_orders_by_node_key() -> None: + nodes = [_node("n1", "A"), _node("n2", "B"), _node("n3", "C")] + edges = [_edge("e1", "n1", "n2"), _edge("e2", "n2", "n3")] + result = validate_dag(nodes, edges) + assert result["valid"] is True + assert result["root_node_ids"] == ["n1"] + assert result["leaf_node_ids"] == ["n3"] + assert result["topological_order"] == ["n1", "n2", "n3"] + + +def test_diamond_topology_is_valid() -> None: + # A -> B -> D + # A -> C -> D + nodes = [ + _node("a", "A"), + _node("b", "B"), + _node("c", "C"), + _node("d", "D"), + ] + edges = [ + _edge("e1", "a", "b"), + _edge("e2", "a", "c"), + _edge("e3", "b", "d"), + _edge("e4", "c", "d"), + ] + result = validate_dag(nodes, edges) + assert result["valid"] is True + assert result["root_node_ids"] == ["a"] + assert result["leaf_node_ids"] == ["d"] + # Kahn's algorithm with node_key tie-breaking: starting at A, then B + # and C both become ready (B alphabetically first), then D. + assert result["topological_order"] == ["a", "b", "c", "d"] + + +def test_cycle_is_rejected_with_dag_cycle() -> None: + # n1 -> n2 -> n3 -> n1 + nodes = [_node("n1", "A"), _node("n2", "B"), _node("n3", "C")] + edges = [ + _edge("e1", "n1", "n2"), + _edge("e2", "n2", "n3"), + _edge("e3", "n3", "n1"), + ] + result = validate_dag(nodes, edges) + assert result["valid"] is False + codes = [err["code"] for err in result["errors"]] + assert "DAG_CYCLE" in codes + cycle_err = next(err for err in result["errors"] if err["code"] == "DAG_CYCLE") + # The cycle should list every node in the cycle (sorted by node_key). + assert set(cycle_err["node_ids"]) == {"n1", "n2", "n3"} + + +def test_self_edge_is_rejected_but_does_not_count_as_cycle() -> None: + nodes = [_node("n1", "A"), _node("n2", "B")] + edges = [ + _edge("e_self", "n1", "n1"), + _edge("e_real", "n1", "n2"), + ] + result = validate_dag(nodes, edges) + codes = [err["code"] for err in result["errors"]] + assert "DAG_SELF_EDGE" in codes + # The A->B edge still makes the DAG valid overall except for the self-edge. + assert "DAG_CYCLE" not in codes + # One node remains reachable (B), so cycle detection must not fire. + assert result["topological_order"] == ["n1", "n2"] + + +def test_duplicate_edge_is_rejected_with_dag_duplicate_edge() -> None: + nodes = [_node("n1", "A"), _node("n2", "B")] + edges = [ + _edge("e1", "n1", "n2"), + _edge("e1_dup", "n1", "n2"), + ] + result = validate_dag(nodes, edges) + codes = [err["code"] for err in result["errors"]] + assert "DAG_DUPLICATE_EDGE" in codes + # The first edge still counts toward edge_count, the second is rejected. + assert result["edge_count"] == 2 + + +def test_edge_to_unknown_node_is_dag_edge_node_missing() -> None: + nodes = [_node("n1", "A")] + edges = [ + _edge("e1", "n1", "ghost"), + _edge("e2", "ghost", "n1"), + ] + result = validate_dag(nodes, edges) + codes = [err["code"] for err in result["errors"]] + assert codes.count("DAG_EDGE_NODE_MISSING") == 2 + # No cycle should be reported for orphan edges. + assert "DAG_CYCLE" not in codes + + +def test_multiple_roots_are_sorted_by_node_key() -> None: + nodes = [ + _node("z", "Z"), + _node("a", "A"), + _node("m", "M"), + ] + edges = [] + result = validate_dag(nodes, edges) + assert result["valid"] is True + # All three nodes are roots (no indegree) and leaves (no outgoing). + assert result["root_node_ids"] == ["a", "m", "z"] + assert result["leaf_node_ids"] == ["a", "m", "z"] + # Topological order picks the smallest node_key first. + assert result["topological_order"] == ["a", "m", "z"] diff --git a/common/pyproject.toml b/common/pyproject.toml index 8c1e319..94a45fb 100644 --- a/common/pyproject.toml +++ b/common/pyproject.toml @@ -7,7 +7,6 @@ dependencies = [ "greenlet>=3.0.0", "apscheduler>=3.11.3", "asyncmy==0.2.11", - "boto3>=1.34,<2", "fastapi==0.116.1", "pydantic-settings>=2.14.2", "loguru>=0.7.2", @@ -15,6 +14,7 @@ dependencies = [ "bcrypt>=4.0,<4.1", "aiofiles>=25.1.0", "aioboto3>=15.5.0", + "cryptography>=49.0.0", ] [build-system] @@ -32,3 +32,7 @@ default = true dev = [ "pytest>=9.1.1", ] + +[tool.pytest.ini_options] +asyncio_mode = "auto" +testpaths = ["tests"] diff --git a/common/src/common/config.py b/common/src/common/config.py index 4d78060..734c410 100644 --- a/common/src/common/config.py +++ b/common/src/common/config.py @@ -17,10 +17,41 @@ Rules for adding a new variable: from __future__ import annotations +import base64 +import os from functools import lru_cache +from typing import Annotated, Any +from venv import logger -from pydantic import Field -from pydantic_settings import BaseSettings, SettingsConfigDict +from cryptography.hazmat.primitives.ciphers.aead import AESGCM +from pydantic import Field, field_validator, model_validator +from pydantic_settings import BaseSettings, NoDecode, SettingsConfigDict + + +# ── AES 解密核心函数 ──────────────────────────────────────────────────────── +def _decrypt_value(cipher_text: str) -> str: + """解密 ENC(...) 格式的字符串。密钥从环境变量 APP_CONFIG_SECRET_KEY 获取。""" + if not (cipher_text.startswith("ENC(") and cipher_text.endswith(")")): + return cipher_text + + secret_key = os.getenv("APP_CONFIG_SECRET_KEY") + if not secret_key: + raise RuntimeError( + "致命错误: 检测到配置项包含 ENC(...) 密文，但系统环境变量 " + "APP_CONFIG_SECRET_KEY 未设置！" + ) + + raw_payload = cipher_text[4:-1] + try: + data = base64.b64decode(raw_payload) + nonce, ciphertext = data[:12], data[12:] + # 将传入的 key 补全或截断为 32 字节 (AES-256) + key_bytes = secret_key.encode("utf-8").ljust(32, b"\0")[:32] + cipher = AESGCM(key_bytes) + decrypted = cipher.decrypt(nonce, ciphertext, None) + return decrypted.decode("utf-8") + except Exception as e: + raise ValueError(f"配置项解密失败，请检查密钥或密文正确性: {e}") from e class Settings(BaseSettings): @@ -38,10 +69,6 @@ class Settings(BaseSettings): default="dev-only-not-for-production", description="HS256 secret used by backend's jupyter auth_request.", ) - demo_auth_enabled: bool = Field( - default=False, - description="Enable the self-hosted UI's short-lived demo session cookie.", - ) cookie_force_secure: bool = Field( default=False, description=( @@ -68,6 +95,39 @@ class Settings(BaseSettings): "anything else falls back to INFO inside configure_logging()." ), ) + audit_log_dir: str = Field( + default="data/logs/audit", + description=( + "审计日志目录。每天一个文件 audit-YYYY-MM-DD.log。" + "路径相对于 backend 进程 cwd（容器内通常为 /app）。" + ), + ) + audit_log_retention_days: int = Field( + default=30, + description="审计日志保留天数；过期文件启动时清理。设 0 关闭清理。", + ) + audit_excluded_paths: Annotated[list[str], NoDecode] = Field( + default=[ + "/health/live", + "/health/ready", + "/api/v1/health", + "/", + "/health/storage", + ], + description=( + "审计排除的精确路径列表（不含 query）。命中即不写审计行。" + "诊断日志（method/path/status/ms）仍写 stderr。" + "环境变量 AUDIT_EXCLUDED_PATHS 用逗号分隔，例如" + " '/health/live,/api/v1/health'。" + ), + ) + + @field_validator("audit_excluded_paths", mode="before") + @classmethod + def _split_audit_paths(cls, v): + if isinstance(v, str): + return [s.strip() for s in v.split(",") if s.strip()] + return v # ── runtime container endpoint ─────────────────────────────── runtime_api_url: str = Field( @@ -108,11 +168,11 @@ class Settings(BaseSettings): ) s3_access_key: str = Field( default="modelplatform", - description="boto3 access key for S3-compatible storage.", + description="S3 access key for S3-compatible storage.", ) s3_secret_key: str = Field( default="modelplatformsecret", - description="boto3 secret key for S3-compatible storage.", + description="S3 secret key for S3-compatible storage.", ) s3_workspace_bucket: str = Field( default="workspace", @@ -192,13 +252,21 @@ class Settings(BaseSettings): ) # ── readiness probes ────────────────────────────────────────── - readiness_targets: str = Field( - default="", - description=( - "Comma-separated host:port list checked by /health/ready. " - "Empty disables the check." - ), - ) + readiness_targets: str = Field(default="") + + # ── 全局密文拦截器 ─────────────────────────────────────────── + @model_validator(mode="before") + @classmethod + def _decrypt_encrypted_fields(cls, values: dict[str, Any]) -> dict[str, Any]: + """在 Pydantic 赋值前自动扫描所有 str 类型的环境变量，解密 ENC(...)""" + if not isinstance(values, dict): + return values + + for key, val in values.items(): + if isinstance(val, str) and val.startswith("ENC("): + values[key] = _decrypt_value(val) + # print(values[key]) + return values model_config = SettingsConfigDict( env_file=".env", @@ -216,5 +284,4 @@ def get_settings() -> Settings: settings: Settings = get_settings() - __all__ = ["Settings", "get_settings", "settings"] diff --git a/common/src/common/db/models/storage.py b/common/src/common/db/models/storage.py index 05bbc1c..3380223 100644 --- a/common/src/common/db/models/storage.py +++ b/common/src/common/db/models/storage.py @@ -20,7 +20,6 @@ class StorageObjects(Base): Index("fk_storage_created_by", "created_by"), Index("idx_storage_content_hash", "content_hash"), Index("idx_storage_owner", "owner_user_id", "object_status"), - Index("idx_storage_parent", "parent_object_id"), Index( "idx_storage_workspace_path", "workspace_id", @@ -40,7 +39,7 @@ class StorageObjects(Base): "object_key_hash_active", unique=True, ), - {"comment": "Workspace 文件和 RustFS 对象的统一元数据"}, + {"comment": "Workspace 文件和 RustFS 对象的统一元数据;目录树走 materialized path (relative_path),不要 join 邻接表列——已删除。"}, ) storage_object_id: Mapped[str] = mapped_column(CHAR(26), primary_key=True) @@ -86,7 +85,6 @@ class StorageObjects(Base): server_default=text("CURRENT_TIMESTAMP(3) ON UPDATE CURRENT_TIMESTAMP(3)"), ) owner_user_id: Mapped[str | None] = mapped_column(CHAR(26)) - parent_object_id: Mapped[str | None] = mapped_column(CHAR(26)) relative_path: Mapped[str | None] = mapped_column( String(1024), comment="Workspace 相对路径" ) diff --git a/common/src/common/scheduler/trigger.py b/common/src/common/scheduler/trigger.py index ce14826..1b3c094 100644 --- a/common/src/common/scheduler/trigger.py +++ b/common/src/common/scheduler/trigger.py @@ -1,6 +1,6 @@ """Shared schedule-trigger logic. -Both the user-facing manual run endpoint (``backend.schedule_runs``) +Both the user-facing manual run endpoint (``backend.api.schedules.runs``) and the schedule service's cron tick handler call into this module to materialize a ``ScheduleRuns`` row plus the corresponding ``schedule.run.requested`` outbox event. The outbox is the single diff --git a/common/src/common/storage/__init__.py b/common/src/common/storage/__init__.py index 4a4906d..53a0ad9 100644 --- a/common/src/common/storage/__init__.py +++ b/common/src/common/storage/__init__.py @@ -1,24 +1,17 @@ -"""统一存储层，同时支持同步和异步，通过 config["mode"] 切换。 +"""异步统一存储层。 -对上层暴露的公开 API： +对外暴露的公开 API： - from storage import create_storage, StorageBackend, AsyncStorageBackend, ObjectMeta - from storage.exceptions import StorageError, StorageNotFoundError, ... + from common.storage import create_storage, AsyncStorageBackend, ObjectMeta + from common.storage.exceptions import StorageError, StorageNotFoundError, ... + +切换本地 / S3 通过 ``settings.storage_backend`` 控制，业务代码不感知差异： -用法： - # 同步（默认 mode="sync"） storage = create_storage({"type": "local", "base_dir": "./data"}) - storage.put("a/b.txt", b"hello") - - # 异步：加一个 mode 字段 - storage = create_storage({"type": "local", "mode": "async", "base_dir": "./data"}) - await storage.put("a/b.txt", b"hello") - -切换本地/S3，或切换同步/异步，业务代码都不用改，只改配置： - storage = create_storage({"type": "s3", "mode": "async", "bucket": "my-bucket"}) + storage = create_storage({"type": "s3", "bucket": "my-bucket"}) """ -from .base import AsyncStorageBackend, ObjectMeta, StorageBackend +from .base import AsyncStorageBackend, ObjectMeta from .factory import ( PURPOSE_BUCKETS, RCLONE_REMOTE_NAME, @@ -38,7 +31,6 @@ __all__ = [ "USAGE_TYPE_TO_PURPOSE", "AsyncStorageBackend", "ObjectMeta", - "StorageBackend", "actual_bucket_name", "build_storage_config", "build_storage_uri", diff --git a/common/src/common/storage/backends/local.py b/common/src/common/storage/backends/local.py index 9c2785e..d81a768 100644 --- a/common/src/common/storage/backends/local.py +++ b/common/src/common/storage/backends/local.py @@ -1,21 +1,20 @@ -"""本地文件系统存储后端。 +"""本地文件系统异步存储后端。 -- 同步实现 `LocalStorageBackend`：标准库文件 I/O -- 异步实现 `LocalAsyncStorageBackend`：aiofiles 做实际读写， - stat/exists/delete/mkdir/目录遍历这类轻量元数据操作用 - asyncio.to_thread 包一层，避免阻塞事件循环 - （只有创建异步实例时才需要装 aiofiles，同步实现零依赖） +`LocalStorageBackend`：使用 aiofiles 做实际读写， +stat/exists/delete/mkdir/目录遍历这类轻量元数据操作用 +``asyncio.to_thread`` 包一层，避免阻塞事件循环。 + +依赖：pip install aiofiles """ import asyncio import os import shutil -from collections.abc import AsyncIterator, Iterable +from collections.abc import AsyncIterator from datetime import timedelta from pathlib import Path -from typing import BinaryIO -from ..base import AsyncData, AsyncStorageBackend, ObjectMeta, StorageBackend, SyncData +from ..base import AsyncData, AsyncStorageBackend, ObjectMeta from ..exceptions import StorageAlreadyExistsError, StorageNotFoundError from ..registry import register_backend @@ -33,106 +32,9 @@ def _meta(key: str, path: Path) -> ObjectMeta: return ObjectMeta(key=key, size=st.st_size, last_modified=st.st_mtime) -# ==================== 同步实现 ==================== - - -@register_backend("local", mode="sync") -class LocalStorageBackend(StorageBackend): - """配置示例: {"type": "local", "mode": "sync", "base_dir": "/data/storage"}""" - - def __init__(self, base_dir: str, **_ignored): - self.base_dir = Path(base_dir).resolve() - self.base_dir.mkdir(parents=True, exist_ok=True) - - def _resolve(self, key: str) -> Path: - return _resolve(self.base_dir, key) - - def put( - self, - key: str, - data: SyncData, - *, - overwrite: bool = True, - content_type: str | None = None, - metadata: dict | None = None, - ) -> ObjectMeta: - path = self._resolve(key) - if path.exists() and not overwrite: - raise StorageAlreadyExistsError(f"key 已存在: {key}") - path.parent.mkdir(parents=True, exist_ok=True) - - if isinstance(data, bytes): - path.write_bytes(data) - else: - with open(path, "wb") as f: - shutil.copyfileobj(data, f) - # local FS 没有对象级 metadata；content_type / metadata 暂存忽略。 - return _meta(key, path) - - def get(self, key: str) -> bytes: - path = self._resolve(key) - if not path.is_file(): - raise StorageNotFoundError(f"key 不存在: {key}") - return path.read_bytes() - - def get_stream(self, key: str) -> BinaryIO: - path = self._resolve(key) - if not path.is_file(): - raise StorageNotFoundError(f"key 不存在: {key}") - return open(path, "rb") - - def delete(self, key: str) -> None: - try: - self._resolve(key).unlink() - except FileNotFoundError: - pass - - def exists(self, key: str) -> bool: - return self._resolve(key).is_file() - - def stat(self, key: str) -> ObjectMeta: - path = self._resolve(key) - if not path.is_file(): - raise StorageNotFoundError(f"key 不存在: {key}") - return _meta(key, path) - - def list(self, prefix: str = "") -> Iterable[ObjectMeta]: - search_root = self._resolve(prefix) if prefix else self.base_dir - if search_root.is_dir(): - candidates = search_root.rglob("*") - else: - candidates = search_root.parent.glob(f"{search_root.name}*") - - for path in candidates: - if path.is_file(): - key = str(path.relative_to(self.base_dir)).replace(os.sep, "/") - yield _meta(key, path) - - def get_url(self, key: str, *, expires_in: timedelta | None = None) -> str: - path = self._resolve(key) - if not path.is_file(): - raise StorageNotFoundError(f"key 不存在: {key}") - return path.as_uri() - - def copy(self, src_key: str, dst_key: str) -> ObjectMeta: - src_path = self._resolve(src_key) - if not src_path.is_file(): - raise StorageNotFoundError(f"key 不存在: {src_key}") - dst_path = self._resolve(dst_key) - dst_path.parent.mkdir(parents=True, exist_ok=True) - shutil.copy2(src_path, dst_path) - return _meta(dst_key, dst_path) - - -# ==================== 异步实现 ==================== - - -@register_backend("local", mode="async") -class LocalAsyncStorageBackend(AsyncStorageBackend): - """配置示例: {"type": "local", "mode": "async", "base_dir": "/data/storage"} - - 需要: pip install aiofiles - """ +@register_backend("local") +class LocalStorageBackend(AsyncStorageBackend): + """配置示例: {"type": "local", "base_dir": "/data/storage"}""" def __init__(self, base_dir: str, **_ignored): self.base_dir = Path(base_dir).resolve() diff --git a/common/src/common/storage/backends/s3.py b/common/src/common/storage/backends/s3.py index 6f067a2..0d5a69d 100644 --- a/common/src/common/storage/backends/s3.py +++ b/common/src/common/storage/backends/s3.py @@ -1,18 +1,15 @@ -"""S3（及兼容协议）存储后端。 +"""S3（及兼容协议）异步存储后端。 -- 同步实现 `S3StorageBackend`：boto3 -- 异步实现 `S3AsyncStorageBackend`：aioboto3 +`S3StorageBackend`：用 aioboto3 跑所有 S3 操作； +异常类型从 ``botocore.exceptions`` 拿（aioboto3 透传）。 -两者只在各自 __init__ 里做 lazy import，互不强制依赖： -只用同步模式不需要装 aioboto3，只用异步模式不需要额外装 boto3 -（aioboto3 本身依赖 botocore，异常类型从它里面拿）。 +依赖：pip install aioboto3 """ -from collections.abc import AsyncIterator, Iterable +from collections.abc import AsyncIterator from datetime import timedelta -from typing import BinaryIO -from ..base import AsyncData, AsyncStorageBackend, ObjectMeta, StorageBackend, SyncData +from ..base import AsyncData, AsyncStorageBackend, ObjectMeta from ..exceptions import ( StorageAlreadyExistsError, StorageConnectionError, @@ -30,162 +27,16 @@ def _meta_from_head(key: str, head: dict) -> ObjectMeta: ) -# ==================== 同步实现 ==================== - - -@register_backend("s3", mode="sync") -class S3StorageBackend(StorageBackend): +@register_backend("s3") +class S3StorageBackend(AsyncStorageBackend): """配置示例: { - "type": "s3", "mode": "sync", + "type": "s3", "bucket": "my-bucket", "prefix": "app1/", "region_name": "cn-north-1", "endpoint_url": "https://s3.example.com", "aws_access_key_id": "...", "aws_secret_access_key": "...", } - 需要: pip install boto3 - """ - - def __init__( - self, - bucket: str, - prefix: str = "", - region_name: str | None = None, - endpoint_url: str | None = None, - aws_access_key_id: str | None = None, - aws_secret_access_key: str | None = None, - **_ignored, - ): - try: - import boto3 - from botocore.exceptions import BotoCoreError, ClientError - except ImportError as e: - raise ImportError("使用同步 S3 存储后端需要先安装 boto3: pip install boto3") from e - - self._ClientError = ClientError - self._BotoCoreError = BotoCoreError - self.bucket = bucket - self.prefix = prefix.strip("/") + "/" if prefix.strip("/") else "" - - try: - self.client = boto3.client( - "s3", - region_name=region_name, - endpoint_url=endpoint_url, - aws_access_key_id=aws_access_key_id, - aws_secret_access_key=aws_secret_access_key, - ) - except (BotoCoreError, ClientError) as e: - raise StorageConnectionError(f"初始化 S3 client 失败: {e}") from e - - def _full_key(self, key: str) -> str: - return f"{self.prefix}{key.lstrip('/')}" - - def put(self, key: str, data: SyncData, *, overwrite: bool = True) -> ObjectMeta: - full_key = self._full_key(key) - if not overwrite and self.exists(key): - raise StorageAlreadyExistsError(f"key 已存在: {key}") - body = data if isinstance(data, bytes) else data.read() - try: - self.client.put_object(Bucket=self.bucket, Key=full_key, Body=body) - except (self._ClientError, self._BotoCoreError) as e: - raise StorageConnectionError(f"上传失败 key={key}: {e}") from e - return self.stat(key) - - def get(self, key: str) -> bytes: - full_key = self._full_key(key) - try: - resp = self.client.get_object(Bucket=self.bucket, Key=full_key) - return resp["Body"].read() - except self._ClientError as e: - if e.response.get("Error", {}).get("Code") in ("NoSuchKey", "404"): - raise StorageNotFoundError(f"key 不存在: {key}") from e - raise StorageConnectionError(f"读取失败 key={key}: {e}") from e - - def get_stream(self, key: str) -> BinaryIO: - full_key = self._full_key(key) - try: - resp = self.client.get_object(Bucket=self.bucket, Key=full_key) - return resp["Body"] - except self._ClientError as e: - if e.response.get("Error", {}).get("Code") in ("NoSuchKey", "404"): - raise StorageNotFoundError(f"key 不存在: {key}") from e - raise StorageConnectionError(f"读取失败 key={key}: {e}") from e - - def delete(self, key: str) -> None: - try: - self.client.delete_object(Bucket=self.bucket, Key=self._full_key(key)) - except (self._ClientError, self._BotoCoreError) as e: - raise StorageConnectionError(f"删除失败 key={key}: {e}") from e - - def exists(self, key: str) -> bool: - try: - self.client.head_object(Bucket=self.bucket, Key=self._full_key(key)) - return True - except self._ClientError as e: - if e.response.get("Error", {}).get("Code") in ("404", "NoSuchKey"): - return False - raise StorageConnectionError(f"检查 exists 失败 key={key}: {e}") from e - - def stat(self, key: str) -> ObjectMeta: - try: - head = self.client.head_object(Bucket=self.bucket, Key=self._full_key(key)) - except self._ClientError as e: - if e.response.get("Error", {}).get("Code") in ("404", "NoSuchKey"): - raise StorageNotFoundError(f"key 不存在: {key}") from e - raise StorageConnectionError(f"获取元信息失败 key={key}: {e}") from e - return _meta_from_head(key, head) - - def list(self, prefix: str = "") -> Iterable[ObjectMeta]: - full_prefix = self._full_key(prefix) - paginator = self.client.get_paginator("list_objects_v2") - try: - for page in paginator.paginate(Bucket=self.bucket, Prefix=full_prefix): - for obj in page.get("Contents", []): - key = obj["Key"][len(self.prefix):] if self.prefix else obj["Key"] - yield ObjectMeta( - key=key, - size=obj["Size"], - last_modified=obj["LastModified"].timestamp(), - etag=obj.get("ETag"), - ) - except (self._ClientError, self._BotoCoreError) as e: - raise StorageConnectionError(f"列举对象失败 prefix={prefix}: {e}") from e - - def get_url(self, key: str, *, expires_in: timedelta | None = None) -> str: - expires_seconds = int(expires_in.total_seconds()) if expires_in else 3600 - try: - return self.client.generate_presigned_url( - "get_object", - Params={"Bucket": self.bucket, "Key": self._full_key(key)}, - ExpiresIn=expires_seconds, - ) - except (self._ClientError, self._BotoCoreError) as e: - raise StorageConnectionError(f"生成预签名 URL 失败 key={key}: {e}") from e - - def copy(self, src_key: str, dst_key: str) -> ObjectMeta: - try: - self.client.copy_object( - Bucket=self.bucket, - Key=self._full_key(dst_key), - CopySource={"Bucket": self.bucket, "Key": self._full_key(src_key)}, - ) - except self._ClientError as e: - if e.response.get("Error", {}).get("Code") in ("404", "NoSuchKey"): - raise StorageNotFoundError(f"key 不存在: {src_key}") from e - raise StorageConnectionError(f"复制失败 {src_key} -> {dst_key}: {e}") from e - return self.stat(dst_key) - - -# ==================== 异步实现 ==================== - - -@register_backend("s3", mode="async") -class S3AsyncStorageBackend(AsyncStorageBackend): - """配置示例同上，把 "mode" 改成 "async" 即可。 - - 需要: pip install aioboto3 - 每次操作默认通过 `async with session.client(...)` 拿一个短生命周期 client；用 `async with create_storage(...) as storage:` 可以复用同一个 client（见 __aenter__/__aexit__）。 @@ -226,7 +77,7 @@ class S3AsyncStorageBackend(AsyncStorageBackend): def _client_cm(self): return self._session.client("s3", **self._client_kwargs) - async def __aenter__(self) -> "S3AsyncStorageBackend": + async def __aenter__(self) -> "S3StorageBackend": self._persistent_cm = self._client_cm() self._persistent_client = await self._persistent_cm.__aenter__() return self @@ -418,4 +269,4 @@ class S3AsyncStorageBackend(AsyncStorageBackend): if e.response.get("Error", {}).get("Code") in ("404", "NoSuchKey"): raise StorageNotFoundError(f"key 不存在: {src_key}") from e raise StorageConnectionError(f"复制失败 {src_key} -> {dst_key}: {e}") from e - return await self.stat(dst_key) + return await self.stat(dst_key) \ No newline at end of file diff --git a/common/src/common/storage/base.py b/common/src/common/storage/base.py index 30b40d3..0ecf871 100644 --- a/common/src/common/storage/base.py +++ b/common/src/common/storage/base.py @@ -1,21 +1,18 @@ -"""同步 / 异步存储后端统一抽象接口。 +"""统一异步存储后端抽象接口。 -`StorageBackend` 是同步接口，`AsyncStorageBackend` 是异步接口， -两者共用同一个 `ObjectMeta` 数据结构，方法签名尽量保持对称 -（异步版本每个方法多一个 await，get_stream/list 变成异步生成器）， -这样业务代码从同步切到异步时心智负担最小。 +所有异步存储后端(local / s3 / 未来新加的)继承 ``AsyncStorageBackend``， +方法签名共用同一个 ``ObjectMeta`` 返回结构。 -上层通过 `storage.create_storage(config)` 统一创建实例， -用 `config["mode"]` 决定拿到的是同步实现还是异步实现。 +上层通过 ``storage.create_storage(config)`` 统一创建实例； +不再提供同步抽象 —— 所有调用方都使用 ``async`` 接口。 """ from abc import ABC, abstractmethod -from collections.abc import AsyncIterator, Iterable +from collections.abc import AsyncIterator from dataclasses import dataclass, field from datetime import timedelta -from typing import BinaryIO, Union +from typing import Union -SyncData = Union[bytes, BinaryIO] AsyncData = Union[bytes, "AsyncIterator[bytes]"] @@ -30,68 +27,6 @@ class ObjectMeta: extra: dict = field(default_factory=dict) # 后端特有的额外信息 -class StorageBackend(ABC): - """同步存储后端统一抽象基类。""" - - @abstractmethod - def put( - self, - key: str, - data: SyncData, - *, - overwrite: bool = True, - content_type: str | None = None, - metadata: dict | None = None, - ) -> ObjectMeta: - """写入对象。overwrite=False 时 key 已存在应抛出 StorageAlreadyExistsError。 - - ``content_type`` 和 ``metadata`` 是可选的（与异步 put 语义一致）。 - """ - - @abstractmethod - def get(self, key: str) -> bytes: - """读取对象内容，不存在时抛出 StorageNotFoundError。""" - - @abstractmethod - def get_stream(self, key: str) -> BinaryIO: - """以流方式读取对象，适合大文件。""" - - @abstractmethod - def delete(self, key: str) -> None: - """删除对象。删除不存在的 key 不应报错（幂等）。""" - - @abstractmethod - def exists(self, key: str) -> bool: - ... - - @abstractmethod - def stat(self, key: str) -> ObjectMeta: - """不存在时抛出 StorageNotFoundError。""" - - @abstractmethod - def list(self, prefix: str = "") -> Iterable[ObjectMeta]: - """按前缀列出对象。""" - - @abstractmethod - def get_url(self, key: str, *, expires_in: timedelta | None = None) -> str: - """获取可访问 URL；本地存储返回 file://，S3 返回预签名 URL。""" - - def copy(self, src_key: str, dst_key: str) -> ObjectMeta: - """默认实现：读出来再写进去。后端可覆盖为更高效的原生实现。""" - data = self.get(src_key) - return self.put(dst_key, data) - - def close(self) -> None: - """释放后端持有的资源（连接池等）。不需要的后端可以不覆盖。""" - return - - def __enter__(self) -> "StorageBackend": - return self - - def __exit__(self, exc_type, exc, tb) -> None: - self.close() - - class AsyncStorageBackend(ABC): """异步存储后端统一抽象基类。""" diff --git a/common/src/common/storage/example_usage.py b/common/src/common/storage/example_usage.py deleted file mode 100644 index 4c2d06a..0000000 --- a/common/src/common/storage/example_usage.py +++ /dev/null @@ -1,115 +0,0 @@ -"""使用示例：同一套 create_storage()，靠 config["mode"] 切换同步/异步。""" - -import asyncio - -from common.storage import create_storage -from common.storage.exceptions import StorageNotFoundError - - -def sync_demo(): - # mode 默认就是 "sync"，可以不写 - storage = create_storage({"type": "local", "base_dir": "./data_sync"}) - - storage.put("docs/hello.txt", b"hello world") - print(storage.get("docs/hello.txt")) - print(storage.exists("docs/hello.txt")) - print(list(storage.list("docs/"))) - print(storage.get_url("docs/hello.txt")) - - try: - storage.get("docs/not_exist.txt") - except StorageNotFoundError: - print("按预期抛出 StorageNotFoundError") - - # 换成同步 S3，只改配置： - # storage = create_storage({"type": "s3", "bucket": "my-bucket"}) - - -async def async_demo(): - # 只加一个 "mode": "async"，其余配置和参数不变 - storage = create_storage({"type": "local", "mode": "async", "base_dir": "./data_async"}) - - await storage.put("docs/hello.txt", b"hello world") - print(await storage.get("docs/hello.txt")) - print(await storage.exists("docs/hello.txt")) - - async for meta in storage.list("docs/"): - print(meta) - - chunks = [] - async for chunk in storage.get_stream("docs/hello.txt"): - chunks.append(chunk) - print(b"".join(chunks)) - - try: - await storage.get("docs/not_exist.txt") - except StorageNotFoundError: - print("按预期抛出 StorageNotFoundError") - - # 并发写入，异步模式的典型优势场景 - tasks = [storage.put(f"batch/{i}.txt", f"content-{i}".encode()) for i in range(10)] - await asyncio.gather(*tasks) - print("并发写入 10 个对象完成") - - # 换成异步 S3，只改配置： - # storage = create_storage({"type": "s3", "mode": "async", "bucket": "my-bucket"}) - # 高吞吐场景复用连接： - # async with create_storage({"type": "s3", "mode": "async", "bucket": "my-bucket"}) as s3: - # await s3.put("a.txt", b"1") - - -def extend_with_new_backend_demo(): - """演示独立扩展一种新的存储方式（同步+异步各一个），不用改现有代码。""" - import io - import time - - from common.storage.base import ObjectMeta, StorageBackend - from common.storage.exceptions import StorageNotFoundError - from common.storage.registry import register_backend - - @register_backend("memory", mode="sync") - class MemoryStorageBackend(StorageBackend): - def __init__(self, **_ignored): - self._store = {} - - def put(self, key, data, *, overwrite=True): - body = data if isinstance(data, bytes) else data.read() - self._store[key] = body - return ObjectMeta(key=key, size=len(body), last_modified=time.time()) - - def get(self, key): - if key not in self._store: - raise StorageNotFoundError(key) - return self._store[key] - - def get_stream(self, key): - return io.BytesIO(self.get(key)) - - def delete(self, key): - self._store.pop(key, None) - - def exists(self, key): - return key in self._store - - def stat(self, key): - if key not in self._store: - raise StorageNotFoundError(key) - return ObjectMeta(key=key, size=len(self._store[key])) - - def list(self, prefix=""): - for key, body in self._store.items(): - if key.startswith(prefix): - yield ObjectMeta(key=key, size=len(body)) - - def get_url(self, key, *, expires_in=None): - return f"memory://{key}" - - mem_storage = create_storage({"type": "memory", "mode": "sync"}) - mem_storage.put("a.txt", b"in-memory content") - print(mem_storage.get("a.txt")) - - -if __name__ == "__main__": - sync_demo() - asyncio.run(async_demo()) - extend_with_new_backend_demo() diff --git a/common/src/common/storage/factory.py b/common/src/common/storage/factory.py index 1b152c1..2529e6a 100644 --- a/common/src/common/storage/factory.py +++ b/common/src/common/storage/factory.py @@ -1,58 +1,56 @@ -"""统一入口：根据配置字典创建具体的存储后端实例。 +"""统一入口：根据配置字典创建具体的异步存储后端实例。 -配置里的 "mode" 字段决定拿到同步还是异步实现，默认 "sync"（向后兼容）。 +通过 ``settings.storage_backend`` 切换本地 / S3，业务代码不感知差异。 - # 同步（默认） + # 本地 storage = create_storage({"type": "local", "base_dir": "./data"}) - storage.put("a.txt", b"hello") - - # 异步：只需加一个 mode 字段，其余配置不变 - storage = create_storage({"type": "local", "mode": "async", "base_dir": "./data"}) await storage.put("a.txt", b"hello") - # S3 同理 - storage = create_storage({"type": "s3", "mode": "async", "bucket": "my-bucket"}) + # S3 + storage = create_storage({"type": "s3", "bucket": "my-bucket"}) + await storage.put("a.txt", b"hello") -上层业务代码应该只从这里拿实例，不要直接 import 具体的 XxxStorageBackend / -XxxAsyncStorageBackend 类。 +上层业务代码应该只从这里拿实例，不要直接 import 具体的 XxxStorageBackend 类。 """ from pathlib import Path -from typing import Any, Union +from typing import Any from .backends import local, s3 # noqa: F401 # 触发内置后端注册 -from .base import AsyncStorageBackend, StorageBackend +from .base import AsyncStorageBackend from .exceptions import StorageConfigError from .registry import get_backend_class -AnyStorageBackend = Union[StorageBackend, AsyncStorageBackend] - -def create_storage(config: dict[str, Any]) -> AnyStorageBackend: - """根据配置创建存储后端。 +def create_storage(config: dict[str, Any]) -> AsyncStorageBackend: + """根据配置创建异步存储后端。 Args: config: 必须包含 "type" 字段（如 "local" / "s3"）； - 可选 "mode" 字段（"sync" 默认 / "async"）； 其余字段作为 kwargs 传给对应后端的构造函数。 + 不能包含 "mode" ——同步接口已删除；如需切换行为请改 settings。 Returns: - mode="sync" 时返回 StorageBackend 实例（同步方法）； - mode="async" 时返回 AsyncStorageBackend 实例（方法需要 await）。 + ``AsyncStorageBackend`` 实例（方法需要 await）。 """ config = dict(config) # 不修改调用方传入的原字典 - backend_type = config.pop("type", None) - mode = config.pop("mode", "sync") + if "mode" in config: + raise StorageConfigError( + "create_storage 不再接受 'mode' 字段；只支持异步后端。" + "如需切换本地 / S3，请改 settings.storage_backend。" + ) + + backend_type = config.pop("type", None) if not backend_type: raise StorageConfigError("配置缺少 'type' 字段，例如 'local' 或 's3'") - backend_cls = get_backend_class(backend_type, mode) + backend_cls = get_backend_class(backend_type) try: return backend_cls(**config) except TypeError as e: raise StorageConfigError( - f"创建后端 (mode={mode}, type={backend_type}) 失败，参数不匹配: {e}" + f"创建后端 (type={backend_type}) 失败，参数不匹配: {e}" ) from e @@ -125,7 +123,7 @@ def build_storage_config(bucket_name: str) -> dict[str, Any]: bucket_name: 桶名，必须是 ``PURPOSE_BUCKETS`` 之一。 Returns: - 直接喂给 ``create_storage(...)`` 的 dict。 + 直接喂给 ``create_storage(...)`` 的 dict。不含 ``mode`` 字段。 """ # 延迟 import：避免 storage -> config -> storage 的循环依赖 from common.config import settings @@ -138,14 +136,12 @@ def build_storage_config(bucket_name: str) -> dict[str, Any]: if settings.storage_backend == "local": return { "type": "local", - "mode": "async", "base_dir": str(Path(settings.local_storage_base_dir) / bucket_name), } if settings.storage_backend == "s3": return { "type": "s3", - "mode": "async", "bucket": getattr(settings, f"s3_{bucket_name}_bucket"), "endpoint_url": settings.s3_endpoint, "aws_access_key_id": settings.s3_access_key, diff --git a/common/src/common/storage/registry.py b/common/src/common/storage/registry.py index 2d061cb..0748b59 100644 --- a/common/src/common/storage/registry.py +++ b/common/src/common/storage/registry.py @@ -1,61 +1,43 @@ -"""后端注册表，用 (mode, name) 作为 key 同时管理同步和异步实现。 +"""后端注册表，按 name 索引每个后端的异步实现。 -新增一种存储方式的同步或异步实现时，不需要改 factory.py： - @register_backend("local", mode="sync") - class LocalStorageBackend(StorageBackend): ... +新增一种存储方式时，不需要改 factory.py： + @register_backend("local") + class LocalStorageBackend(AsyncStorageBackend): ... - @register_backend("local", mode="async") - class LocalAsyncStorageBackend(AsyncStorageBackend): ... - -只要保证模块被 import 一次即可（backends/__init__.py 里统一 import）。 +只要保证模块被 import 一次即可（``backends/__init__.py`` 里统一 import）。 """ -from typing import Union - -from .base import AsyncStorageBackend, StorageBackend +from .base import AsyncStorageBackend from .exceptions import StorageConfigError -BackendClass = Union[type[StorageBackend], type[AsyncStorageBackend]] +BackendClass = type[AsyncStorageBackend] -_REGISTRY: dict[tuple[str, str], BackendClass] = {} - -VALID_MODES = ("sync", "async") +_REGISTRY: dict[str, BackendClass] = {} -def _check_mode(mode: str) -> None: - if mode not in VALID_MODES: - raise StorageConfigError(f"不支持的 mode: {mode!r}，可选值: {VALID_MODES}") - - -def register_backend(name: str, mode: str = "sync"): - """类装饰器：把一个后端类注册为 (mode, name) 对应的实现。""" - _check_mode(mode) +def register_backend(name: str): + """类装饰器：把一个后端类注册为 ``name`` 对应的实现。""" def _decorator(cls: BackendClass) -> BackendClass: - key = (mode, name) - if key in _REGISTRY and _REGISTRY[key] is not cls: + if name in _REGISTRY and _REGISTRY[name] is not cls: raise StorageConfigError( - f"存储后端 (mode={mode}, type={name}) 已被注册为 {_REGISTRY[key]!r}" + f"存储后端 (type={name}) 已被注册为 {_REGISTRY[name]!r}" ) - _REGISTRY[key] = cls + _REGISTRY[name] = cls return cls return _decorator -def get_backend_class(name: str, mode: str = "sync") -> BackendClass: - _check_mode(mode) - key = (mode, name) +def get_backend_class(name: str) -> BackendClass: try: - return _REGISTRY[key] + return _REGISTRY[name] except KeyError: - available = ", ".join( - f"{m}:{n}" for (m, n) in sorted(_REGISTRY) - ) or "(无)" + available = ", ".join(sorted(_REGISTRY)) or "(无)" raise StorageConfigError( - f"未知的存储后端 (mode={mode}, type={name})，当前已注册: {available}" + f"未知的存储后端 (type={name})，当前已注册: {available}" ) -def registered_backends() -> dict[tuple[str, str], BackendClass]: +def registered_backends() -> dict[str, BackendClass]: return dict(_REGISTRY) diff --git a/common/tests/__init__.py b/common/tests/__init__.py new file mode 100644 index 0000000..e69de29 diff --git a/common/tests/storage/__init__.py b/common/tests/storage/__init__.py new file mode 100644 index 0000000..e69de29 diff --git a/common/tests/storage/test_factory.py b/common/tests/storage/test_factory.py new file mode 100644 index 0000000..5c1079a --- /dev/null +++ b/common/tests/storage/test_factory.py @@ -0,0 +1,134 @@ +"""create_storage 工厂测试：覆盖 mode 字段严格拒绝 + 正常路径。""" + +from common.storage.base import AsyncStorageBackend +from common.storage.exceptions import StorageConfigError +from common.storage.factory import ( + PURPOSE_BUCKETS, + USAGE_TYPE_TO_PURPOSE, + build_storage_config, + create_storage, +) + + +# ── mode 字段严格拒绝 ────────────────────────────────────────────── + + +def test_create_storage_rejects_mode_sync(): + """显式 mode='sync' 现在必须拒绝 —— 同步抽象已砍掉。""" + import pytest + + with pytest.raises(StorageConfigError) as exc_info: + create_storage({"type": "local", "base_dir": "/tmp", "mode": "sync"}) + msg = str(exc_info.value) + assert "不再接受 'mode' 字段" in msg + + +def test_create_storage_rejects_mode_async(): + """显式 mode='async' 也必须拒绝 —— 只有 async 一条路，不需要再声明。""" + import pytest + + with pytest.raises(StorageConfigError) as exc_info: + create_storage({"type": "s3", "bucket": "x", "mode": "async"}) + msg = str(exc_info.value) + assert "不再接受 'mode' 字段" in msg + + +def test_create_storage_rejects_any_mode_value(): + """任何 mode 字段（包含未来可能新增的合法值）都拒绝 —— 简化语义。""" + import pytest + + for value in ("async", "sync", "dual", ""): + with pytest.raises(StorageConfigError): + create_storage({"type": "local", "base_dir": "/tmp", "mode": value}) + + +# ── 正常路径 ──────────────────────────────────────────────────────── + + +def test_create_storage_returns_async_subclass(): + s = create_storage({"type": "local", "base_dir": "/tmp/cc-factory-test"}) + assert isinstance(s, AsyncStorageBackend) + # 确认是真 AsyncStorageBackend，不是同步兼容形态 + assert type(s).__name__ == "LocalStorageBackend" + + +def test_create_storage_missing_type_raises(): + import pytest + + with pytest.raises(StorageConfigError) as exc_info: + create_storage({"base_dir": "/tmp"}) + assert "缺少 'type' 字段" in str(exc_info.value) + + +def test_create_storage_unknown_type_raises(): + import pytest + + with pytest.raises(StorageConfigError) as exc_info: + create_storage({"type": "nonexistent"}) + assert "未知的存储后端" in str(exc_info.value) + + +def test_create_storage_does_not_mutate_input_dict(): + """调用方传入的字典不能被改写。""" + cfg = {"type": "local", "base_dir": "/tmp/cc-no-mutate"} + cfg_id = id(cfg) + snapshot = dict(cfg) + create_storage(cfg) + assert dict(cfg) == snapshot, "factory should not mutate caller's dict" + assert id(cfg) == cfg_id + + +def test_create_storage_kwargs_mismatch_raises_wrapped_error(): + import pytest + + with pytest.raises(StorageConfigError) as exc_info: + create_storage({"type": "local", "base_dir": 12345}) # base_dir 必须是 str + msg = str(exc_info.value) + assert "参数不匹配" in msg + + +# ── build_storage_config 不再产生 mode 字段 ───────────────────────── + + +def test_build_storage_config_local_has_no_mode(monkeypatch): + """local 分支的输出 dict 不能含 'mode'。""" + from common import config as common_config + + monkeypatch.setattr(common_config.settings, "storage_backend", "local", raising=False) + monkeypatch.setattr(common_config.settings, "local_storage_base_dir", "/tmp", raising=False) + + cfg = build_storage_config("workspace") + assert "mode" not in cfg, f"local cfg must not have 'mode', got: {cfg}" + assert cfg["type"] == "local" + assert "base_dir" in cfg + + +def test_build_storage_config_s3_has_no_mode(monkeypatch): + """s3 分支的输出 dict 也不能含 'mode'。""" + from common import config as common_config + + monkeypatch.setattr(common_config.settings, "storage_backend", "s3", raising=False) + monkeypatch.setattr(common_config.settings, "s3_workspace_bucket", "wb", raising=False) + monkeypatch.setattr(common_config.settings, "s3_endpoint", "http://s3", raising=False) + monkeypatch.setattr(common_config.settings, "s3_access_key", "ak", raising=False) + monkeypatch.setattr(common_config.settings, "s3_secret_key", "sk", raising=False) + + cfg = build_storage_config("workspace") + assert "mode" not in cfg, f"s3 cfg must not have 'mode', got: {cfg}" + assert cfg["type"] == "s3" + + +def test_build_storage_config_unknown_bucket_raises(): + import pytest + + from common.storage.exceptions import StorageConfigError + with pytest.raises(StorageConfigError): + build_storage_config("not-a-real-bucket") + + +def test_purpose_buckets_constant_complete(): + """PURPOSE_BUCKETS 必须覆盖 USAGE_TYPE_TO_PURPOSE 中所有 purpose。""" + purposes = set(USAGE_TYPE_TO_PURPOSE.values()) + assert purposes.issubset(set(PURPOSE_BUCKETS)), ( + f"missing buckets for purposes: {purposes - set(PURPOSE_BUCKETS)}" + ) \ No newline at end of file diff --git a/common/tests/storage/test_registry.py b/common/tests/storage/test_registry.py new file mode 100644 index 0000000..34be20a --- /dev/null +++ b/common/tests/storage/test_registry.py @@ -0,0 +1,71 @@ +"""后端注册表测试：覆盖 register / get / 内置 backend 注册。""" + +from common.storage.base import AsyncStorageBackend +from common.storage.exceptions import StorageConfigError +from common.storage.registry import ( + get_backend_class, + register_backend, + registered_backends, +) + + +def test_local_and_s3_are_registered_at_import_time(): + """import common.storage 应该触发 local / s3 的注册。""" + backend_classes = registered_backends() + assert "local" in backend_classes + assert "s3" in backend_classes + for cls in backend_classes.values(): + assert issubclass(cls, AsyncStorageBackend) + + +def test_get_backend_class_returns_async_subclass(): + cls = get_backend_class("local") + assert issubclass(cls, AsyncStorageBackend) + + +def test_get_backend_class_unknown_raises(): + import pytest + + with pytest.raises(StorageConfigError) as exc_info: + get_backend_class("does-not-exist") + assert "未知的存储后端" in str(exc_info.value) + assert "does-not-exist" in str(exc_info.value) + + +def test_register_backend_idempotent_for_same_class(): + """同一个类对象重复注册是 no-op，不抛错（``is`` 比对，避免重复 import 时误冲突）。""" + from common.storage.registry import _REGISTRY + + class _SameAgain(AsyncStorageBackend): + pass + + _REGISTRY["same-again-test"] = _SameAgain + + # 第二次装饰同一个类对象：当前实现里装饰器返回 cls 并把 _REGISTRY[name] 重新写一遍。 + # 直接重新调用 register_backend("same-again-test")(_SameAgain) 不抛错即可。 + fn = register_backend("same-again-test") + result = fn(_SameAgain) + assert result is _SameAgain + assert _REGISTRY["same-again-test"] is _SameAgain + + _REGISTRY.pop("same-again-test", None) + + +def test_register_backend_conflict_raises(): + import pytest + + @register_backend("conflict-test") + class A(AsyncStorageBackend): + pass + + with pytest.raises(StorageConfigError) as exc_info: + + @register_backend("conflict-test") + class B(AsyncStorageBackend): + pass + + assert "已被注册" in str(exc_info.value) + + # cleanup：避免污染全局 registry 影响其他测试 + from common.storage.registry import _REGISTRY + _REGISTRY.pop("conflict-test", None) \ No newline at end of file diff --git a/docker-compose.yml b/docker-compose.yml index 3a46f51..5ea023e 100644 --- a/docker-compose.yml +++ b/docker-compose.yml @@ -25,6 +25,7 @@ services: INITIAL_ADMIN_PASSWORD: ${INITIAL_ADMIN_PASSWORD:-admin12345} UV_OFFLINE: "1" UV_NO_SYNC: "1" + APP_CONFIG_SECRET_KEY: ${APP_CONFIG_SECRET_KEY:?APP_CONFIG_SECRET_KEY is required} web: build: @@ -51,7 +52,6 @@ services: runtime: condition: service_healthy volumes: - - ${PWD}:/app - ./default.conf:/etc/nginx/conf.d/default.conf.template:ro healthcheck: test: [ "CMD-SHELL", "wget -qO- http://127.0.0.1/ >/dev/null" ] @@ -81,7 +81,6 @@ services: SERVICE_NAME: model-platform-backend SCHEDULE_EVENT_NAMESPACE: ${SCHEDULE_EVENT_NAMESPACE:-model-platform-local} JWT_SECRET: ${JWT_SECRET:?JWT_SECRET is required} - DEMO_AUTH_ENABLED: ${DEMO_AUTH_ENABLED:-false} INITIAL_ADMIN_PASSWORD: ${INITIAL_ADMIN_PASSWORD:-admin12345} RUNTIME_API_URL: http://runtime:8000 # P0-1 fix: shared secret required by /internal/v1/* routes. @@ -101,6 +100,7 @@ services: READINESS_TARGETS: ${MYSQL_HOST:?MYSQL_HOST is required}:${MYSQL_PORT:-3306},runtime:8000 UV_OFFLINE: "1" UV_NO_SYNC: "1" + APP_CONFIG_SECRET_KEY: ${APP_CONFIG_SECRET_KEY:?APP_CONFIG_SECRET_KEY is required} depends_on: migrate: condition: service_completed_successfully @@ -165,6 +165,7 @@ services: RCLONE_CONFIG_S3_REGION: other UV_OFFLINE: "1" UV_NO_SYNC: "1" + APP_CONFIG_SECRET_KEY: ${APP_CONFIG_SECRET_KEY:?APP_CONFIG_SECRET_KEY is required} volumes: - ${PWD}:/app - ./data:/data @@ -214,6 +215,7 @@ services: READINESS_TARGETS: ${MYSQL_HOST:?MYSQL_HOST is required}:${MYSQL_PORT:-3306},backend:8000 UV_OFFLINE: "1" UV_NO_SYNC: "1" + APP_CONFIG_SECRET_KEY: ${APP_CONFIG_SECRET_KEY:?APP_CONFIG_SECRET_KEY is required} depends_on: backend: condition: service_healthy diff --git a/frontend/.gitignore b/frontend/.gitignore index 271afdb..2f75b43 100644 --- a/frontend/.gitignore +++ b/frontend/.gitignore @@ -6,6 +6,12 @@ /.react-router/ /build/ +# Vite / file-viewer 构建产物,不要提交 +/dist/ + # monaco-editor 预构建 min/vs(由 scripts/copy-monaco.mjs 在 postinstall 时 # 从 node_modules/monaco-editor/min/vs 复制生成,不要提交) /public/monaco/vs/ + +# @file-viewer/vite-plugin copyAssets 生成的 office 预览 vendor,不要提交 +/public/file-viewer/ diff --git a/frontend/app/app.css b/frontend/app/app.css index c5fff16..c3f6cfa 100644 --- a/frontend/app/app.css +++ b/frontend/app/app.css @@ -1,3 +1,75 @@ +@import "tailwindcss"; + +@theme { + --font-sans: Inter, "PingFang SC", "Microsoft YaHei", system-ui, sans-serif; + + /* STYLE_GUIDE §3.2 字号补充 token */ + --text-2xs: 0.625rem; /* 10px */ + --text-3xs: 0.5625rem; /* 9px */ + --text-4xs: 0.5rem; /* 8px */ + + /* STYLE_GUIDE §3.3 字重补充 token */ + --font-weight-medium-plus: 650; + + --color-brand: #1978d4; + --color-brand-strong: #0e5fb9; + --color-brand-soft: #edf6ff; + --color-brand-deep: #0e5bad; + + --color-success: #18b979; + --color-success-strong: #0f7d55; + --color-success-soft: #e8f7f1; + + --color-danger: #d75b5b; + --color-danger-strong: #a94f4f; + --color-danger-soft: #fff6f6; + + --color-warning: #e6a33c; + --color-warning-soft: #fff0ee; + + --color-ink: #27394d; + --color-ink-muted: #5d7186; + --color-ink-subtle: #8b99a8; + --color-ink-caption: #748598; + + --color-line: #dce4ec; + --color-line-soft: #edf1f5; + + --color-bg: #f3f6f9; + --color-bg-panel: #ffffff; + --color-bg-canvas: #f9fbfd; + --color-bg-log: #17212b; + + /* STYLE_GUIDE §4.2 间距补充 token */ + --spacing-page-x: 22px; + --spacing-button-x: 15px; + --spacing-gap-icon: 7px; + --spacing-gap-sm: 5px; + --spacing-gap-md: 9px; +} +@theme inline { + --color-sidebar: var(--sidebar-background); + --color-sidebar-foreground: var(--sidebar-foreground); + --color-sidebar-primary: var(--sidebar-primary); + --color-sidebar-primary-foreground: var(--sidebar-primary-foreground); + --color-sidebar-accent: var(--sidebar-accent); + --color-sidebar-accent-foreground: var(--sidebar-accent-foreground); + --color-sidebar-border: var(--sidebar-border); + --color-sidebar-ring: var(--sidebar-ring); +} + +:root { + --sidebar-background: #09233f; + --sidebar-foreground: #c9d7e7; + --sidebar-primary: #1479e8; + --sidebar-primary-foreground: #ffffff; + --sidebar-accent: rgba(255, 255, 255, 0.06); + --sidebar-accent-foreground: #ffffff; + --sidebar-border: rgba(255, 255, 255, 0.08); + --sidebar-ring: #5ca9ff; +} + + @font-face { font-family: "Inter"; src: url("/fonts/Inter-Variable.ttf") format("truetype"); @@ -14,97 +86,218 @@ font-display: swap; } -:root { - font-family: Inter, "PingFang SC", "Microsoft YaHei", system-ui, sans-serif; - color: #1f354b; - background: #f3f6f9; - font-synthesis: none; - text-rendering: optimizeLegibility; +@layer base { + :root { + font-family: var(--font-sans); + color: #1f354b; + background: #f3f6f9; + font-synthesis: none; + text-rendering: optimizeLegibility; + } + + html, + body { + margin: 0; + min-width: 1120px; + min-height: 100vh; + overflow: hidden; + } + + *, + *::before, + *::after { + box-sizing: border-box; + } + + button, + input, + select { + font: inherit; + } + + button { + color: inherit; + } } -* { box-sizing: border-box; } -html, body { margin: 0; min-width: 1024px; min-height: 100%; } -button, input, select, textarea { font: inherit; } +@layer utilities { + .scrollbar-thin::-webkit-scrollbar { + width: 5px; + } -.auth-loading, -.login-page { - min-height: 100vh; - display: flex; - align-items: center; - justify-content: center; + .scrollbar-thin::-webkit-scrollbar-track { + background: transparent; + } + + .scrollbar-thin::-webkit-scrollbar-thumb { + border-radius: 999px; + background: #d9e1e9; + } + + .scrollbar-thin::-webkit-scrollbar-thumb:hover { + background: #c4d0dc; + } + + /* 侧边栏双层渐变背景 — Tailwind 任意值无法可靠表达逗号分隔的多层 background */ + .sidebar-shell { + background: + radial-gradient(circle at 10% 1%, rgb(28 105 186 / 25%), transparent 27%), + #09233f; + } + + .welcome-panel-shell { + background: + radial-gradient(circle at 50% 39%, rgb(74 150 225 / 9%), transparent 24%), + radial-gradient(circle, #d3dee8 1px, transparent 1px); + background-size: auto, 18px 18px; + } + + .dashboard-hero-shell { + background: linear-gradient(125deg, #0b3d69, #1978d4); + } + + .trend-chart-bg { + background: repeating-linear-gradient(to bottom, #fff 0, #fff 44px, #eef3f7 45px); + } + + .dashboard-donut { + background: conic-gradient(#318de0 0 67%, #72c9b0 67% 86%, #f3b75c 86% 100%); + } + + .dashboard-donut::after { + position: absolute; + width: 78px; + height: 78px; + border-radius: 50%; + background: #fff; + content: ""; + } + + .tree-skeleton-bar { + background: linear-gradient(90deg, #f2f5f8, #fafbfc, #f2f5f8); + background-size: 200% 100%; + animation: shimmer 1.3s infinite; + } + + .tabbar-scroll::-webkit-scrollbar { + height: 6px; + } + + .tabbar-scroll::-webkit-scrollbar-track { + background: #f1f3f5; + } + + .tabbar-scroll::-webkit-scrollbar-thumb { + background: #c9d6e4; + border-radius: 3px; + } + + .tabbar-scroll::-webkit-scrollbar-thumb:hover { + background: #a8b9ca; + } + + .stage-dot-editing { + box-shadow: 0 0 0 3px rgb(36 149 211 / 13%); + animation: lock-pulse 1.8s ease-in-out infinite; + } + + .editor-canvas-shell { + background: + linear-gradient(rgb(255 255 255 / 82%), rgb(255 255 255 / 82%)), + radial-gradient(circle, #c9d8e7 1px, transparent 1px); + background-color: #f4f7fa; + background-size: auto, 18px 18px; + } + + .modal-panel { + animation: modal-in 0.18s ease-out; + } + + .notebook-md h1, + .notebook-md h2, + .notebook-md h3, + .notebook-md p, + .notebook-md li { + margin: 6px 0; + color: #24292f; + font-size: 14px; + line-height: 1.6; + } + + .notebook-md h1 { + font-size: 24px; + color: #1f6feb; + font-weight: 600; + } + + .notebook-md h2 { + font-size: 20px; + color: #1f6feb; + font-weight: 600; + } + + .notebook-md h3 { + font-size: 16px; + color: #24292f; + font-weight: 600; + } + + .notebook-md ul, + .notebook-md ol { + padding-left: 24px; + } + + .notebook-md li { + margin-left: 0; + list-style-position: outside; + } } -.auth-loading { - gap: 12px; - color: #61758a; +@keyframes shimmer { + to { + background-position: -200% 0; + } } -.auth-loading > span { - width: 20px; - height: 20px; - border: 2px solid #bfd5e9; - border-top-color: #1677ff; - border-radius: 50%; - animation: auth-spin 0.8s linear infinite; +@keyframes lock-pulse { + 50% { + opacity: 0.5; + transform: scale(0.78); + } } -@keyframes auth-spin { to { transform: rotate(360deg); } } - -.login-page { - padding: 48px; - background: - radial-gradient(circle at 20% 15%, rgba(22, 119, 255, 0.12), transparent 34%), - linear-gradient(145deg, #eef5fb, #f8fafc 55%, #edf3f8); +@keyframes modal-in { + from { + opacity: 0; + transform: translateY(8px) scale(0.985); + } } -.login-card { - width: 420px; - padding: 42px; - border: 1px solid #dbe6ef; - border-radius: 18px; - background: rgba(255, 255, 255, 0.96); - box-shadow: 0 22px 60px rgba(37, 63, 88, 0.14); -} - -.login-brand { - width: 46px; - height: 46px; +/* Shared UI classes still used by admin / schedules pages */ +.icon-button { display: grid; + width: 34px; + height: 34px; place-items: center; - border-radius: 13px; - color: white; - background: linear-gradient(145deg, #1177e8, #25a1f2); -} - -.login-kicker { - margin: 26px 0 8px; - color: #1677ff; - font-size: 11px; - font-weight: 750; - letter-spacing: 0.14em; -} - -.login-card h1 { margin: 0; font-size: 27px; } -.login-description { margin: 10px 0 28px; color: #728398; } -.login-card form { display: grid; gap: 18px; } -.login-card label { display: grid; gap: 8px; color: #465b70; font-size: 13px; } -.login-card input { - width: 100%; - padding: 12px 14px; - border: 1px solid #cad8e5; - border-radius: 9px; - outline: none; - color: #18334f; - background: white; -} -.login-card input:focus { border-color: #1677ff; box-shadow: 0 0 0 3px rgba(22, 119, 255, 0.12); } -.login-card button { - min-height: 44px; - border: 0; - border-radius: 9px; - color: white; - background: #1677ff; + border: 1px solid var(--color-line); + border-radius: 7px; + background: #fff; cursor: pointer; } -.login-card button:disabled { opacity: 0.65; cursor: wait; } -.login-error { margin: -4px 0 0; color: #d4380d; font-size: 13px; } + +.icon-button:hover { + border-color: #b9c9da; + background: #f7faff; +} + +.avatar { + display: grid; + width: 34px; + height: 34px; + place-items: center; + border-radius: 50%; + color: #fff; + background: linear-gradient(145deg, #3b92ed, #1869c9); + font-size: 14px; + font-weight: 700; +} \ No newline at end of file diff --git a/frontend/app/components/admin/DashboardPage.tsx b/frontend/app/components/admin/DashboardPage.tsx deleted file mode 100644 index 3c0a960..0000000 --- a/frontend/app/components/admin/DashboardPage.tsx +++ /dev/null @@ -1,90 +0,0 @@ -import Icon from "../../components/common/Icon"; -import { useAuth } from "../../context/AuthContext"; - -import "../../styles/admin.css"; -import "../../styles/dashboard.css"; - -export function DashboardPage({ - scriptCount, - online, - onNavigate, -}: { - scriptCount: number; - online: boolean; - onNavigate: (page: "scripts" | "schedules" | "system") => void; -}) { - const { user, currentWorkspace } = useAuth(); - const isSystemAdmin = user?.is_system_admin ?? false; - - return ( -
-
-
- MODEL DEVELOPMENT PLATFORM -

下午好，{user?.display_name ?? "用户"}

-

- 当前位于 {currentWorkspace?.workspace_name ?? "（未选择 Workspace）"} - ，可以继续构建脚本或配置调度。 -

-
- {online ? "服务正常" : "服务连接中"} -
-
-
{scriptCount}工作副本
-
2Workspace
-
4平台用户
-
{online ? "正常" : "检查中"}平台状态
-
-
- - - {isSystemAdmin && ( - - )} -
-
-
-
运行趋势

近 7 天调度执行

成功率 92.6%
-
- {[38, 55, 44, 73, 61, 86, 78].map((value, index) => ( -
- {Math.round(value / 7)} - - {["周一", "周二", "周三", "周四", "周五", "周六", "今天"][index]} -
- ))} -
-
成功 75失败 6
-
-
-
脚本资产

类型分布

-
-
{scriptCount}全部脚本
-
- Notebook{Math.max(1, Math.round(scriptCount * .67))} 个 · 67% - Python{Math.max(0, scriptCount - Math.round(scriptCount * .67))} 个 · 33% - 稳定版本3 个已发布 -
-
-
-
-
ACTIVITY

最近平台活动

-
-
操作内容执行人状态时间
- {[ - ["数据探索.ipynb 发布稳定版本 v3.0", "张三", "成功", "16:42"], - ["每日模型训练流程完成调度运行", "Scheduler", "成功", "15:25"], - ["批量预测.py 更新工作副本", "王五", "已同步", "14:18"], - ["风险验证流程完成 DAG 校验", "李四", "成功", "11:06"], - ].map((row) => ( -
- {row[0]} - {row[1]}{row[2]}{row[3]} -
- ))} -
-
-
-
- ); -} diff --git a/frontend/app/components/admin/ProjectManagementPage.tsx b/frontend/app/components/admin/ProjectManagementPage.tsx deleted file mode 100644 index 5f3471e..0000000 --- a/frontend/app/components/admin/ProjectManagementPage.tsx +++ /dev/null @@ -1,553 +0,0 @@ -import { useEffect, useState } from "react"; - -import { ApiRequestError, type Employee, type Workspace, type WorkspaceMember } from "../../services/api"; -import { useApi, useAuth } from "../../context/AuthContext"; -import Icon from "../common/Icon"; -import { UserMultiSelect } from "./UserMultiSelect"; - -const EMPTY_PROJECT_FORM = { - workspace_code: "", - workspace_name: "", - quota_bytes: 0, - description: "", -}; - -export function ProjectManagementPage({ - onNotify, - onConnectionChange, -}: { - onNotify: (notice: { tone: "success" | "error" | "info"; message: string }) => void; - onConnectionChange: (online: boolean) => void; -}) { - const api = useApi(); - const { user, refreshWorkspaces } = useAuth(); - const [projects, setProjects] = useState([]); - const [projectLoading, setProjectLoading] = useState(true); - const [projectDialogOpen, setProjectDialogOpen] = useState(false); - const [projectForm, setProjectForm] = useState(EMPTY_PROJECT_FORM); - const [editingProject, setEditingProject] = useState(null); - const [importMemberDialogOpen, setImportMemberDialogOpen] = useState(false); - const [selectedProject, setSelectedProject] = useState(null); - const [availableUsers, setAvailableUsers] = useState([]); - const [selectedUserIds, setSelectedUserIds] = useState([]); - const [projectSearchTerm, setProjectSearchTerm] = useState(""); - const [saving, setSaving] = useState(false); - const [membersDrawerOpen, setMembersDrawerOpen] = useState(false); - const [currentProjectMembers, setCurrentProjectMembers] = useState([]); - const [membersLoading, setMembersLoading] = useState(false); - - const canManage = user?.role_code === "admin"; - - const loadProjects = async (): Promise => { - setProjectLoading(true); - try { - const workspaceList = await api.listWorkspaces(); - setProjects(workspaceList); - onConnectionChange(true); - } catch (error) { - onConnectionChange(false); - onNotify({ - tone: "error", - message: error instanceof Error ? error.message : "项目列表加载失败", - }); - } finally { - setProjectLoading(false); - } - }; - - useEffect(() => { - void loadProjects(); - }, []); - - const openCreateProject = (): void => { - setEditingProject(null); - setProjectForm(EMPTY_PROJECT_FORM); - setProjectDialogOpen(true); - }; - - const openEditProject = (project: Workspace): void => { - setEditingProject(project); - setProjectForm({ - workspace_code: project.workspace_code, - workspace_name: project.workspace_name, - quota_bytes: project.quota_bytes, - description: project.description ?? "", - }); - setProjectDialogOpen(true); - }; - - const submitProject = async (event: React.FormEvent): Promise => { - event.preventDefault(); - if (!projectForm.workspace_name.trim()) { - onNotify({ tone: "error", message: "请输入项目名称" }); - return; - } - if (!projectForm.workspace_code.trim() && !editingProject) { - onNotify({ tone: "error", message: "请输入项目编码" }); - return; - } - setSaving(true); - try { - if (editingProject) { - const updated = await api.updateWorkspace(editingProject.workspace_id, { - workspace_name: projectForm.workspace_name.trim(), - quota_bytes: projectForm.quota_bytes, - description: projectForm.description.trim() || undefined, - }); - setProjects((current) => - current.map((p) => (p.workspace_id === updated.workspace_id ? updated : p)) - ); - onNotify({ tone: "success", message: "项目信息已更新" }); - } else { - const generatedCode = projectForm.workspace_code.trim() || projectForm.workspace_name.trim().toLowerCase().replace(/[^a-z0-9-]/g, "-").slice(0, 32); - const created = await api.createWorkspace({ - workspace_code: generatedCode, - workspace_name: projectForm.workspace_name.trim(), - quota_bytes: projectForm.quota_bytes, - description: projectForm.description.trim() || undefined, - }); - setProjects((current) => [...current, created]); - void refreshWorkspaces(); - onNotify({ tone: "success", message: "项目已创建" }); - } - setProjectDialogOpen(false); - } catch (error) { - onNotify({ - tone: "error", - message: error instanceof ApiRequestError ? error.message : (editingProject ? "更新项目失败" : "创建项目失败"), - }); - } finally { - setSaving(false); - } - }; - - const deleteProject = async (project: Workspace): Promise => { - if (!window.confirm(`确定要删除项目"${project.workspace_name}"吗？此操作将级联软删所有成员。`)) return; - try { - await api.deleteWorkspace(project.workspace_id); - setProjects((current) => current.filter((p) => p.workspace_id !== project.workspace_id)); - onNotify({ tone: "success", message: "项目已删除" }); - } catch (error) { - onNotify({ - tone: "error", - message: error instanceof Error ? error.message : "删除项目失败", - }); - } - }; - - const openImportMemberDialog = async (project: Workspace): Promise => { - setSelectedProject(project); - setSelectedUserIds([]); - setImportMemberDialogOpen(true); - - // 加载可用用户和项目成员 - try { - const [allUsers, currentMembers] = await Promise.all([ - api.listPlatformEmployees(), - api.listWorkspaceMembers(project.workspace_id), - ]); - setAvailableUsers(allUsers); - setCurrentProjectMembers(currentMembers); - } catch (error) { - // 如果加载失败，仍显示所有用户 - const allUsers = await api.listPlatformEmployees(); - setAvailableUsers(allUsers); - setCurrentProjectMembers([]); - } - }; - - const openMembersDrawer = (project: Workspace): void => { - setSelectedProject(project); - setMembersDrawerOpen(true); - void loadProjectMembers(project.workspace_id); - }; - - const loadProjectMembers = async (workspaceId: string): Promise => { - setMembersLoading(true); - try { - const members = await api.listWorkspaceMembers(workspaceId); - setCurrentProjectMembers(members); - } catch (error) { - onNotify({ - tone: "error", - message: error instanceof Error ? error.message : "成员列表加载失败", - }); - } finally { - setMembersLoading(false); - } - }; - - const removeMember = async (userId: string): Promise => { - if (!selectedProject) return; - // 管理员不能被移除 - const targetMember = currentProjectMembers.find((m) => m.user_id === userId); - if (targetMember?.role_code === "admin") { - onNotify({ tone: "error", message: "管理员不能被移除" }); - return; - } - try { - await api.deleteWorkspaceMember(selectedProject.workspace_id, userId); - setCurrentProjectMembers((current) => current.filter((m) => m.user_id !== userId)); - onNotify({ tone: "success", message: "成员已移除" }); - } catch (error) { - onNotify({ - tone: "error", - message: error instanceof ApiRequestError ? error.message : "移除成员失败", - }); - } - }; - - const updateMemberRole = async (userId: string, roleCode: "admin" | "developer"): Promise => { - if (!selectedProject) return; - try { - const updated = await api.updateWorkspaceMember(selectedProject.workspace_id, userId, { - role_code: roleCode, - }); - setCurrentProjectMembers((current) => - current.map((m) => (m.user_id === userId ? { ...m, role_code: roleCode, role_name: roleCode === "admin" ? "管理员" : "开发人员" } : m)) - ); - onNotify({ tone: "success", message: "成员角色已更新" }); - } catch (error) { - onNotify({ - tone: "error", - message: error instanceof ApiRequestError ? error.message : "更新成员角色失败", - }); - } - }; - - const importMember = async (): Promise => { - if (!selectedProject || selectedUserIds.length === 0) { - onNotify({ tone: "error", message: "请选择要添加的用户" }); - return; - } - setSaving(true); - try { - await Promise.all( - selectedUserIds.map((userId) => - api.addWorkspaceMember(selectedProject.workspace_id, { - user_id: userId, - }) - ) - ); - setImportMemberDialogOpen(false); - onNotify({ tone: "success", message: `已添加 ${selectedUserIds.length} 名成员` }); - } catch (error) { - onNotify({ - tone: "error", - message: error instanceof ApiRequestError ? error.message : "添加成员失败", - }); - } finally { - setSaving(false); - } - }; - - return ( -
-
-
- - setProjectSearchTerm(event.target.value)} - /> -
- -
- - {!canManage &&
当前为开发人员，只能查看项目列表。
} -
-
- 项目名称 - 项目编码 - 状态 - 配额 - 操作 -
- {projectLoading ? ( -

正在加载项目…

- ) : projects.length === 0 ? ( -

暂无项目

- ) : ( - projects - .filter((project) => { - const term = projectSearchTerm.toLowerCase().trim(); - if (!term) return true; - return ( - project.workspace_name.toLowerCase().includes(term) || - project.workspace_code.toLowerCase().includes(term) || - (project.description && project.description.toLowerCase().includes(term)) - ); - }) - .map((project) => ( -
- - {project.workspace_name.slice(0, 1)} - - {project.workspace_name} - {project.description ?? "无描述"} - - - {project.workspace_code} - - - {project.status === "active" ? "正常" : project.status === "archived" ? "已归档" : "已删除"} - - - {project.quota_bytes > 0 ? `${(project.quota_bytes / 1024 / 1024 / 1024).toFixed(1)} GB` : "无限制"} - - - - {/* */} - - -
- )) - )} -
- - {projectDialogOpen && ( -
-
-
-
- PROJECT -

{editingProject ? "编辑项目" : "新建项目"}

-
- -
-
void submitProject(event)}> - {!editingProject && ( - - )} - - -