From cb0205fcd2981f70e64bb82808fa4dc208d0f705 Mon Sep 17 00:00:00 2001 From: "tao.chen" Date: Fri, 21 Aug 2026 19:26:53 +0800 Subject: [PATCH] =?UTF-8?q?feat(scripts):=20=E8=B7=A8=20owner=20=E6=87=92?= =?UTF-8?q?=E5=8A=A0=E8=BD=BD=E7=9B=AE=E5=BD=95=E6=A0=91=20+=20=E8=B7=A8?= =?UTF-8?q?=E7=94=A8=E6=88=B7=E5=8F=AF=E8=A7=81=20workspace/public?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit 修两个后端接口问题: 1) /api/v1/workspace-directories 返回为空,目录树结构消失 2) 同 workspace 内脚本/数据互相可见但默认排除 private 后端改动 -------- * list_scripts / list_resources / list_workspace_directories 新增 owner_user_id 可选 query 参数;缺省 = 当前请求者本人(scope 到 workspace/{me}/...),传值时 scope 到该 owner 的子树。前端根加载 默认只见自己一级,其他成员以折叠分组呈现。 * visibility 过滤统一:非 admin 请求者只返回 owner==me 或 visibility ∈ {workspace, public};admin 跳过。owner=me 含自己 的 private,owner=other 只剩其 workspace/public,排除他人 private。 * create_workspace_directory 两个分支 visibility 默认 'public' (非 private),使跨 owner 目录树可见;响应新增 owner_user_id 字段。 * platform.list_members 鉴权从 system_admin_context 放宽为 系统管理员或该 workspace 活跃成员(让普通用户也能渲染同 workspace 成员名册,用于跨 owner 分组)。 * main.py 注册 platform 模块(随 list_members 改动补齐导入)。 * .env.example 同步 common/config.py 26 个字段。 前端改动 -------- * ScriptExplorer.memberScriptGroups 改由 members 列表播种分组, display_name 取 members.display_name;inferredDirectories 现在按 owner_user_id 标记,统一跨 owner 目录渲染。删除脚本目录页头与 树分组标题的工作副本数量角标。 * WorkspaceTree 新增 ownerUserId 透传到 store.toggleExpanded; 仅"我"的分组 mount 时 auto-expand,他人分组默认折叠,展开才 调 loadOwnerGroup / owner-scoped loadScripts / loadChildren。 * scriptWorkspaceStore 引入 namespaced cache key (ownerCacheKey = `${ownerUserId ?? me}:${path}`),loadedScriptPaths / loadedChildPaths / loadedOwnerGroups 全部按 owner 隔离; toggleExpanded 用 loadPath === undefined 区分 group 头与真实 目录,修"他人子目录点击不触发接口"的 loadPath 前缀误判 bug。 * api.ts / AuthContext 透传 ownerUserId 给 listScripts / listResources / listWorkspaceDirectories。 文档 ---- * API.md: §3.2 创建目录 visibility 默认 public + 响应加 owner_user_id; §3.3.1 GET directories 加 owner_user_id 参数 + 响应字段; §3.4 GET scripts 改写为 owner 作用域 + visibility 过滤语义; §五.1 GET data-resources 新增,同一套统一语义; §7 intro 例外 — GET members 对系统管理员或 workspace 活跃成员开放。 * DEVELOP.md: Code layout 重写以反映 backend api/services/clients/ schemas 拆分 + schedule domain/scheduling/application/execution/ infrastructure 拆分 + common 子包(auth/storage/backends); Configuration 系统补全 26 个 settings 字段;新增 "Owner-scoping + visibility (cross-owner browsing)" 小节; Per-service dev 注释用 uv run 的源布局要求;Add a new DAG endpoint / storage bucket 路径改为 backend/src/backend/api/* 与 services/*。 测试 ---- * test_list_scripts_parent_path.py / test_resources.py 补充 owner_user_id 参数化直接调用 + LIKE 前缀断言(workspace/{owner}/... 前缀)。 Co-Authored-By: Claude --- .env.example | 28 ++ API.md | 63 ++- DEVELOP.md | 198 +++++++--- backend/src/backend/api/platform.py | 45 ++- backend/src/backend/api/resources.py | 56 +-- backend/src/backend/api/scripts.py | 85 ++-- backend/src/backend/main.py | 8 +- .../tests/test_list_scripts_parent_path.py | 153 ++++++-- backend/tests/test_resources.py | 85 +++- .../components/platform/ScriptExplorer.tsx | 54 ++- frontend/app/context/AuthContext.tsx | 7 +- .../app/features/platform/WorkspaceTree.tsx | 28 +- .../platform/state/scriptWorkspaceStore.ts | 363 ++++++++++++++---- frontend/app/routes/platform.tsx | 9 + frontend/app/services/api.ts | 50 ++- 15 files changed, 929 insertions(+), 303 deletions(-) diff --git a/.env.example b/.env.example index ea864ab..7ea6fae 100644 --- a/.env.example +++ b/.env.example @@ -16,6 +16,15 @@ DATABASE_URL=mysql+asyncmy://root:change-me@127.0.0.1:3306/model_platform?charse JWT_SECRET=change-this-development-secret +# Force the Secure flag on the session cookie even when the inbound request +# scheme is plain HTTP. Enable behind a TLS-terminating reverse proxy that +# strips/rewrites X-Forwarded-Proto — otherwise the cookie is written without +# Secure and browsers refuse to send it back over HTTPS. +COOKIE_FORCE_SECURE=false + +# Service label surfaced in lifespan / health checks. +SERVICE_NAME=service + # ============================================================================ # CRITICAL: must set BEFORE first run. The initial admin user is seeded by # the deployment bootstrap. Never keep the development default in production. @@ -84,6 +93,13 @@ S3_TRASH_RETENTION_DAYS=30 # over the compose network. RCLONE_RC_URL=http://runtime:5572 +# Backend → Runtime HTTP endpoint (Jupyter contents API, file ops). +RUNTIME_API_URL=http://runtime:8000 + +# Public base URL for the runtime container (surfaced to clients for +# Jupyter access tickets / embedded URLs). +PUBLIC_BASE_URL=http://runtime + # ============================================================================ # Service-to-service auth (P0-1 fix). # Backend's /internal/v1/* storage control plane requires this shared secret. @@ -93,3 +109,15 @@ RCLONE_RC_URL=http://runtime:5572 # python -c "import secrets; print(secrets.token_urlsafe(48))" # ============================================================================ INTERNAL_SERVICE_TOKEN=change-me-internal-service-token + +# Schedule → Backend HTTP base URL (cron post-back / status callbacks). +BACKEND_API_URL=http://backend:8000 + +# Max concurrent notebooks running in the schedule worker. Each notebook is +# dispatched as an asyncio task bounded by a semaphore; the polling loop is +# never blocked. +SCHEDULE_EXECUTION_CONCURRENCY=4 + +# Readiness probe targets. Comma-separated host:port list checked by +# /health/ready; empty disables the check. e.g. mysql:3306,s3:9000 +READINESS_TARGETS= diff --git a/API.md b/API.md index c6ec642..9afa626 100644 --- a/API.md +++ b/API.md @@ -99,7 +99,12 @@ ### 3.2 `POST /api/v1/workspace-directories` -创建一个**目录**。后端会落一行 `StorageObjects(object_type='directory', storage_backend='rustfs', storage_uri='inline://directory/{path}', visibility='private')`,因此空目录也能在 §3.1 树里出现并保留下来。 +创建一个**目录**。后端会落一行 `StorageObjects(object_type='directory', storage_backend='rustfs', storage_uri='inline://directory/{path}', visibility='public')`,因此空目录也能在 §3.1 树里出现并保留下来。 + +> 目录行默认 `visibility='public'`(非 `private`)。目录是结构性导航行, +> 默认 public 使同一 workspace 内其他成员可以浏览彼此的目录结构(目录树 +> 跨 owner 可见);文件级私密仍由 §3.4 / §五 的 visibility 过滤兜底 +> —— 其他 owner 的 `private` 脚本 / 数据资源不会返回。 - **请求体**: ```json @@ -123,7 +128,8 @@ "storage_object_id": "01HXY...", "path": "scripts/etl", "name": "etl", - "parent_path": "scripts" + "parent_path": "scripts", + "owner_user_id": "01HXX..." } } ``` @@ -155,10 +161,12 @@ - **行为**: `parent_path` 为空字符串或缺省 → 用户根目录;非空 → 该父目录的直接子目录(workspace-relative)。仅返回 `object_status='available' AND is_deleted=0` 的 `StorageObjects` 行。 - **鉴权**: workspace 成员 +- **owner 作用域**: `owner_user_id` 缺省时 scope 为**当前请求者**本人根目录(`scoped_prefix = workspace/{me}`);传 `owner_user_id` 时 scope 为该 owner 的根目录(`scoped_prefix = workspace/{owner_user_id}`),用于跨 owner 浏览目录树(见 §3.4 visibility 模型)。该接口本身不施加 visibility 过滤——目录行默认 `visibility='public'`(见 §3.2),跨 owner 均可见。 - **查询参数**: | 名 | 类型 | 必填 | 说明 | |---|---|---|---| | `parent_path` | string | 否 | 父目录相对路径,空字符串或缺省表示用户根目录 | + | `owner_user_id` | string | 否 | 目标 owner 的 user_id;缺省=请求者本人。指定后 scope 到 `workspace/{owner_user_id}/{parent_path}` | - **谓词(SQL 等价)**: `relative_path LIKE '/%' AND relative_path NOT LIKE '/%/%'`,其中 `prefix = scoped_prefix/{parent_path}`,索引走 `idx_storage_workspace_relative_path(workspace_id, relative_path(255))`。 - **响应**: ```json @@ -166,9 +174,9 @@ "request_id": "...", "data": { "directories": [ - {"path": "scripts", "name": "scripts", "parent_path": "", "has_children": true}, - {"path": "scripts/etl", "name": "etl", "parent_path": "scripts", "has_children": false}, - {"path": "scripts/etl/daily", "name": "daily", "parent_path": "scripts/etl", "has_children": false} + {"path": "scripts", "name": "scripts", "parent_path": "", "owner_user_id": "01HXX...", "has_children": true}, + {"path": "scripts/etl", "name": "etl", "parent_path": "scripts", "owner_user_id": "01HXX...", "has_children": false}, + {"path": "scripts/etl/daily", "name": "daily", "parent_path": "scripts/etl", "owner_user_id": "01HXX...", "has_children": false} ] }, "meta": {"directory_count": 3} @@ -180,15 +188,28 @@ | `path` | string | workspace 内相对路径 | | `name` | string | `path` 的最后一段 | | `parent_path` | string | 父目录相对路径,根目录用空串 | + | `owner_user_id` | string | 该目录行所属 owner 的 user_id(`owner_user_id` 参数缺省时=请求者本人) | | `has_children` | bool | 该目录下是否还有直接子目录(后端额外 `EXISTS` 查询,可为空目录为 `false`) | - **空结果**: 不返回 404,空目录列表即 `directories: []`。 - **错误**: 401(未登录)/ 403(非 workspace 成员)同其他接口。 ### 3.4 `GET /api/v1/scripts` -列出当前 workspace 内**全部 active 脚本**。不受 is_locked 影响(读路径不锁)。 +列出脚本,按 **owner 作用域 + visibility 过滤**返回。不受 is_locked 影响(读路径不锁)。 -- **响应**: `data` 为 `ScriptPayload` 数组(见 §3.10)。 +- **owner 作用域**: `owner_user_id` 缺省=当前请求者本人,scope 到 `workspace/{me}/...`;传 `owner_user_id` 时 scope 到 `workspace/{owner_user_id}/...`,用于跨 owner 浏览他人脚本。 +- **visibility 过滤(统一)**: 非 admin 请求者只返回 `owner_user_id == me OR visibility IN (workspace, public)`;admin 请求者跳过过滤返回全部。 + - **owner=me(缺省)**: scope 是我的子树,行都是我的 → `owner==me` 恒成立 → **含我的 private 脚本** ✓ + - **owner=other**: scope 是他人的子树,`owner==me` 不成立 → 只剩其 `workspace/public` 脚本(排除他人的 `private`) ✓ + - 即"本人可见自己全部;他人只见其 workspace/public",私密仅在 owner==me 时可见。 +- **非递归**: 仅返回 `parent_path` 下的**直接子级**脚本(懒加载用);子目录脚本需带 `parent_path` 再次请求。 +- **查询参数**: + | 名 | 类型 | 必填 | 说明 | + |---|---|---|---| + | `parent_path` | string | 否 | 父目录相对路径,空串/缺省=该 owner 根目录下的一级脚本 | + | `owner_user_id` | string | 否 | 目标 owner;缺省=请求者本人 | + | `keyword` | string | 否 | 名称模糊匹配 | +- **响应**: `data` 为 `ScriptPayload` 数组(见 §3.10),每条带 `owner_user_id`。 ### 3.5 `GET /api/v1/scripts/{script_id}` @@ -485,7 +506,7 @@ queued ──→ running ──┬─→ succeeded | `POST` | `/api/v1/data-resources/uploads` | 创建上传会话,返回 `upload_id` + `upload_path` | | `PUT` | `/api/v1/data-resources/uploads/{upload_id}` | 上传字节(请求体即文件内容) | | `POST` | `/api/v1/data-resources/uploads/{upload_id}/bind` | 绑定已完成上传为数据资源 | -| `GET` | `/api/v1/data-resources` | 列表(workspace 范围) | +| `GET` | `/api/v1/data-resources` | 列表(owner 作用域 + visibility 过滤) | | `GET` | `/api/v1/data-resources/{id}` | 详情 | | `POST` | `/api/v1/data-resources/{id}/download-url` | 生成 presigned GET URL | | `DELETE` | `/api/v1/data-resources/{id}` | 软删 | @@ -521,6 +542,22 @@ queued ──→ running ──┬─→ succeeded `content_base64` 字段(JSON 体里走),内部走同一条 `AsyncStorageBackend.put` 路径,前端无需分两步。 +### 五.1 `GET /api/v1/data-resources` + +列出数据资源,按 **owner 作用域 + visibility 过滤**返回(与 §3.4 `GET /scripts` 同一套统一语义)。 + +- **owner 作用域**: `owner_user_id` 缺省=当前请求者本人,scope 到 `object_key` 前缀 `{workspace_id}/{me}/...`;传 `owner_user_id` 时 scope 到 `{workspace_id}/{owner_user_id}/...`。 +- **visibility 过滤(统一)**: 非 admin 请求者只返回 `owner_user_id == me OR visibility IN (workspace, public)`;admin 请求者跳过过滤返回全部。语义同 §3.4——owner=me 含自己的 private;owner=other 只见其 workspace/public。 +- **非递归**: 仅返回 `parent_path` 下的**直接子级**资源(懒加载用)。 +- **查询参数**: + | 名 | 类型 | 必填 | 说明 | + |---|---|---|---| + | `parent_path` | string | 否 | 父目录相对路径,空串/缺省=该 owner 根目录下的一级资源 | + | `owner_user_id` | string | 否 | 目标 owner;缺省=请求者本人 | + | `visibility` | string | 否 | `workspace` \| `public` \| `private`,二次过滤 | + | `keyword` | string | 否 | 名称模糊匹配 | +- **响应**: `data` 为 `ResourcePayload` 数组,每条带 `owner_user_id`。 + --- ## 六、管理后台 @@ -581,10 +618,16 @@ Base 前缀 `/api/v1/admin`。 ## 七、系统管理 (`/api/v1/platform/...`) 平台级(跨 workspace)管理接口,用于管理员工、workspace 实体与 workspace 成员。 -所有端点要求调用者是**系统管理员**——其 `users.platform_role_id` 指向 +除特别注明外,所有端点要求调用者是**系统管理员**——其 `users.platform_role_id` 指向 `role_code='admin'` 的角色行,且 `users.status == 'active'`。系统管理员判定 通过 `GET /api/v1/auth/me` 响应中的 `data.user.is_system_admin` 字段(详见 §一)。 +> **例外 — `GET /workspaces/{id}/members`**:该端点对**系统管理员(任意 +> workspace)**与**该 workspace 的活跃成员**(`workspace_members.is_deleted=0` +> 且 `member_status='active'`)均开放。这是为了让普通(非 admin)用户能在 +> 脚本目录树里渲染同 workspace 其他成员的折叠分组(跨 owner 浏览,见 §3.4)。 +> 其余 members 写端点(POST/PATCH/DELETE members)仍仅限系统管理员。 + | 方法 | 路径 | 说明 | |---|---|---| | `GET` | `/api/v1/platform/employees` | 列全平台未软删员工;包含停用、锁定及无平台角色用户 | @@ -596,7 +639,7 @@ Base 前缀 `/api/v1/admin`。 | `GET` | `/api/v1/platform/workspaces/{workspace_id}` | 单个 workspace(含已 disabled 的,用于恢复) | | `PATCH` | `/api/v1/platform/workspaces/{workspace_id}` | 改 workspace 字段;`status` 仅允许 `active`/`archived` | | `DELETE` | `/api/v1/platform/workspaces/{workspace_id}` | 软删 workspace;级联软删其成员 | -| `GET` | `/api/v1/platform/workspaces/{workspace_id}/members` | 列成员 | +| `GET` | `/api/v1/platform/workspaces/{workspace_id}/members` | 列成员(**系统管理员或该 workspace 活跃成员**;为跨 owner 目录树提供成员名册,见 §7 intro 例外) | | `POST` | `/api/v1/platform/workspaces/{workspace_id}/members` | 添加成员(返回 201) | | `PATCH` | `/api/v1/platform/workspaces/{workspace_id}/members/{user_id}` | 改成员 `member_status`;**不能改 role_code**(workspace 角色继承自平台角色) | | `DELETE` | `/api/v1/platform/workspaces/{workspace_id}/members/{user_id}` | 软删成员 | diff --git a/DEVELOP.md b/DEVELOP.md index ce0e26d..bb79217 100644 --- a/DEVELOP.md +++ b/DEVELOP.md @@ -6,58 +6,89 @@ refactors see `HANDOVER.md`. ## Code layout +All Python packages use the `src//` layout; `uv` workspace glues them into +one `.venv`. Always invoke via `uv run [--package ] ` (see "Local +development" for the gotcha). + ``` -common/ Pure-Python shared library - config.py Settings (pydantic-settings, lru_cache singleton) - db/ SQLAlchemy 2.0 async engine, session_scope, Base - db/models/ 26 tables in 9 domain files (zero FK, zero relationship) - scheduler/ build_sqlalchemy_jobstore (delayed import) - storage/ AsyncStorageBackend abstraction (s3 + local impls) + Pydantic schemas - eventing.py add_outbox_event / utcnow / event_time - service_app.py /health/ready TCP probe, /api/v1/health - schemas.py StrictModel base - utils.py get_free_port, start_process +common/src/common/ Pure-Python shared library + config.py Settings (pydantic-settings, lru_cache singleton) + db/ SQLAlchemy 2.0 async engine, session_scope, Base + db/models/ 26 tables in 9 domain files (zero FK, zero relationship) + auth/ JWT / bcrypt / workspace membership helpers + scheduler/ APScheduler trigger helpers (delayed import) + storage/ AsyncStorageBackend abstraction + Pydantic schemas + base.py Abstract interface + factory.py create_storage + build_storage_config + PURPOSE_BUCKETS + schemas.py CreateUploadRequest / ServerObjectRequest + backends/local.py Local filesystem impl + backends/s3.py S3-compatible impl (boto3) + registry.py Bucket registry + eventing.py add_outbox_event / utcnow / event_time + service_app.py /health/ready TCP probe, /api/v1/health + logging.py loguru config (LOG_LEVEL) + schemas.py StrictModel base + ids.py ULID generation helpers + utils.py get_free_port, start_process -backend/ Public FastAPI service + tiny /internal/v1/objects RPC - main.py lifespan + route registration - jupyter.py /api/v1/auth/jupyter — the ONLY auth entry - scripts.py CRUD for scripts/notebooks (object storage via AsyncStorageBackend) - schedules.py DAG CRUD: schedules, nodes, edges - schedule_runs.py Trigger / list / get runs - schedule_schemas.py Pydantic request/response models - admin.py Admin endpoints - resources.py Misc data resources - storage_api.py /internal/v1/objects — single token-guarded endpoint (P0-1) - storage_client.py Stub (HTTP client removed post-migration; rewrite pending) - schedule_client.py Placeholder module (was the HTTP-push executor client) - runtime_client.py Self-contained httpx wrapper for the runtime - jupyter.py auth_request handler - dependencies.py request_context, database_session +backend/src/backend/ Public FastAPI service + main.py lifespan + route registration + audit.py HTTP access log middleware (loguru sink) + api/ HTTP route handlers (one module per bounded context) + auth.py /api/v1/auth/* (login / me / jupyter) + jupyter.py /api/v1/auth/jupyter — the ONLY auth entry + dependencies.py request_context, database_session + platform.py /api/v1/platform/* (system admin) + admin.py /api/v1/admin/* (workspace-internal admin) + scripts.py /api/v1/scripts/* + /api/v1/workspace-directories + resources.py /api/v1/data-resources/* + schedules/schedules.py DAG CRUD + schedules/runs.py Run lifecycle + storage.py /internal/v1/objects — single token-guarded endpoint (P0-1) + services/ Pure-Python business logic (no HTTP / no DI) + scripts.py create_workspace_directory, visibility-filtered queries + resources.py owner-scoped resource listing helpers + jupyter.py jupyter_path / lock helpers + storage.py object store helpers + schedules.py DAG validation (cycle / orphan detection) + schemas/ Pydantic request / response models + auth.py / common.py / jupyter.py / platform.py / resources.py / schedules.py / scripts.py + clients/ Outbound HTTP / RPC clients + runtime.py Self-contained httpx wrapper for the runtime + scheduler.py Backend → Schedule HTTP client (callback / dispatch) + rclone.py rclone RC API client (FUSE cache invalidation) -schedule/ Schedule Executor (DAG worker) - context.py Constants + naive_utc - scheduler.py CronScheduler (APScheduler + 5s sync loop) - orchestrator.py DispatchOrchestrator (Outbox poll + DAG advance) - worker.py NodeExecutor (notebook / python execution) - service.py SchedulerService facade (composes the three) - main.py Lifespan + FastAPI app - storage_client.py SchedulerStorageClient — talks to backend /internal/v1/objects - execution.py execute_artifact (notebook + python paths) - notebook_runner.py Subprocess entry point (nbclient) +schedule/src/schedule/ Schedule Executor (DAG worker) + main.py Lifespan + FastAPI app + notebook_runner.py Subprocess entry point (nbclient) — DO NOT RENAME + domain/ Pure-Python domain types + execution.py ExecutionResult (frozen dataclass) + state enums + context.py Constants + naive_utc + scheduling/ Time-based trigger + scheduler.py CronScheduler (APScheduler + 5s sync loop) + application/ Facades / orchestrators + service.py SchedulerService (composes the three) + orchestrator.py DispatchOrchestrator (Outbox poll + DAG advance) + execution/ DAG node execution + executor.py NodeExecutor (notebook / python dispatch) + worker.py asyncio entry, schedule-spawned task boundary + runners/notebook.py nbclient subprocess path (6-line `notebook_runner` shim re-exports `main`) + infrastructure/ External-system adapters + storage/client.py SchedulerStorageClient — talks to backend /internal/v1/objects -runtime/ Jupyter Runtime - main.py FastAPI entry: jupyter action endpoints - process.py Per-workspace subprocess pool + asyncio locks - mount.py rclone FUSE mount lifecycle +runtime/src/runtime/ Jupyter Runtime + main.py FastAPI entry: jupyter action endpoints + process.py Per-workspace subprocess pool + asyncio locks + mount.py rclone FUSE mount lifecycle -frontend/ React Router SPA (vite build → nginx) - app/ features/ routes/ services/ components/ +frontend/ React Router SPA (vite build → nginx) + app/ features/ routes/ services/ components/ -migrations/ Alembic schema versions -docker-compose.yml 4 services -default.conf Nginx template +migrations/ Alembic schema versions +docker-compose.yml 4 services (gateway / backend / schedule / runtime) +default.conf Nginx template scripts/nginx-entrypoint.sh -.env.example +.env.example All 26 config.py keys documented ``` ## Configuration system @@ -67,10 +98,31 @@ All env vars go through one place: `common/src/common/config.py`. ```python from common.config import settings -settings.database_url # str -settings.storage_backend # str: "s3" (default) or "local" -settings.local_storage_base_dir # str: root dir for storage data (default "/data"); see "Storage" below for per-mode derivation -settings.s3_endpoint # str (full URL, e.g. "http://s3:9000"; s3 mode only) +# Auth / runtime +settings.database_url # str — SQLAlchemy async URL (mysql+asyncmy, charset utf8mb4) +settings.jwt_secret # HS256 secret for the auth_request handler +settings.cookie_force_secure # bool — write Secure flag even on plain HTTP (TLS-terminating proxy) +settings.service_name # surfaced in /health +settings.schedule_event_namespace # APScheduler JobStore namespace + Outbox scope prefix +settings.readiness_targets # CSV host:port list for /health/ready + +# HTTP clients (intra-cluster URLs) +settings.runtime_api_url # backend → runtime HTTP base +settings.public_base_url # runtime public base URL (browser-facing /jupyter/) +settings.backend_api_url # schedule → backend HTTP base +settings.rclone_rc_url # backend → rclone RC control API +settings.internal_service_token # Backend ↔ Schedule shared secret (X-Internal-Service-Token) + +# Logging / audit +settings.log_level # DEBUG / INFO / WARNING / ERROR / CRITICAL (lowercase → fallback INFO) +settings.audit_log_dir # dir for daily audit logs (relative to cwd; "" disables file sink) +settings.audit_log_retention_days # 0 disables cleanup +settings.audit_excluded_paths # list[str] — paths skipped from audit (health probes, etc.) + +# Storage +settings.storage_backend # "s3" (default) or "local" +settings.local_storage_base_dir # root dir for storage data (default "/data") +settings.s3_endpoint # str (s3 mode only) settings.s3_access_key # str (s3 mode only) settings.s3_secret_key # str (s3 mode only) settings.s3_workspace_bucket # str (s3 mode only) @@ -78,17 +130,15 @@ settings.s3_version_bucket # str (s3 mode only) settings.s3_run_log_bucket # str (s3 mode only) settings.s3_trash_bucket # str (s3 mode only) settings.s3_trash_retention_days # int (s3 mode only) -settings.jwt_secret # HS256 secret for the auth_request handler -settings.backend_api_url # schedule → backend HTTP base -settings.runtime_api_url # backend → runtime HTTP base -settings.public_base_url # runtime public base URL -settings.service_name # surfaced in /health -settings.readiness_targets # CSV host:port list for /health/ready + +# Schedule +settings.schedule_execution_concurrency # int — max concurrent notebook subprocesses ``` `Settings` reads from process env first, then from a `.env` file at CWD if present. `pydantic-settings` auto-loads. `case_sensitive=False` so -`DATABASE_URL` / `database_url` both work. +`DATABASE_URL` / `database_url` both work. The full list of 26 fields +is in `common/src/common/config.py`. ### Adding a new env var @@ -96,7 +146,8 @@ if present. `pydantic-settings` auto-loads. `case_sensitive=False` so ```python new_var: str = Field(default="x", description="...") ``` -2. Add the line to `.env.example` with a comment. +2. Add the line to `.env.example` with a comment (keep it synced — every + field in config.py must have a matching `.env.example` entry). 3. Use `settings.new_var` at the call site. Do **not** call `os.environ["NEW_VAR"]` or `os.getenv("NEW_VAR")` in @@ -190,13 +241,34 @@ grep -rnE 'os\.(environ\[?["\x27][A-Z_]+|getenv\(["\x27][A-Z_]+)' --include="*.p - For write operations on a script/notebook, call `require_script_modify_access(script, user_id=..., is_admin=...)` - from `backend/scripts.py`. It enforces: + from `backend/src/backend/services/scripts.py`. It enforces: - admin or owner → allow - non-owner, `is_locked == 0` → allow - non-owner, `is_locked == 1` → 403 - Read endpoints (`list_scripts`, `get_script`) intentionally do **not** check `is_locked` — workspace members can see the script list. +### Owner-scoping + visibility (cross-owner browsing) + +`GET /api/v1/scripts`, `GET /api/v1/data-resources`, and +`GET /api/v1/workspace-directories` all accept an optional +`owner_user_id` query param and follow the same model: + +- `owner_user_id` 缺省 = 当前请求者本人(scope to `workspace/{me}/...`). +- 传值时 scope 到 `workspace/{owner_user_id}/...`,用于前端"点开其他成员 + 分组"的懒加载(见 §3.4 of `API.md`). +- `visibility` 过滤(非 admin):`owner_user_id == me OR visibility IN + (workspace, public)` — 自己可见自己全部(含 private),他人只见其 + workspace/public,排除他人 private. +- 系统管理员跳过 visibility 过滤. +- 目录行默认 `visibility='public'`(由 `create_workspace_directory` 写入), + 不施加 visibility 过滤,使跨 owner 目录树可见. + +实现 helper 在 `backend/src/backend/services/scripts.py` +(`_build_list_scripts_owner_descendant_prefix`) 和 +`backend/src/backend/api/resources.py` 中按相同模式分别构造 owner-scoped +LIKE 前缀。 + ### Outbox events - The platform's only async-messaging fabric is the MySQL @@ -238,6 +310,10 @@ cd frontend && pnpm install && cd .. ### Per-service dev +Always go through `uv run` so the workspace `.venv` is used — bare +`uvicorn` / `python` resolves to system Python and `from backend.X` +imports fail with ModuleNotFoundError. + ```bash # Backend (terminal 1) export DATABASE_URL="mysql+asyncmy://model_platform:model_platform@127.0.0.1:3306/model_platform?charset=utf8mb4" @@ -248,13 +324,13 @@ export S3_ENDPOINT=http://127.0.0.1:9000 # Or for local mode: # export STORAGE_BACKEND=local # export LOCAL_STORAGE_BASE_DIR=/data -uv run --frozen --package backend uvicorn backend.main:app --host 0.0.0.0 --port 8000 --reload +uv run --package backend uvicorn backend.main:app --host 0.0.0.0 --port 8000 --reload # Schedule Executor (terminal 2) -uv run --frozen --package schedule uvicorn schedule.main:app --host 0.0.0.0 --port 8001 --reload +uv run --package schedule uvicorn schedule.main:app --host 0.0.0.0 --port 8001 --reload # Runtime (terminal 3 — needs SYS_ADMIN, FUSE, devmode) -uv run --frozen --package runtime python -m runtime.main +uv run --package runtime python -m runtime.main ``` ### Frontend dev diff --git a/backend/src/backend/api/platform.py b/backend/src/backend/api/platform.py index f368592..7b865ec 100644 --- a/backend/src/backend/api/platform.py +++ b/backend/src/backend/api/platform.py @@ -243,6 +243,21 @@ async def system_admin_context( ) +async def _is_system_admin(session: AsyncSession, user: Users) -> bool: + """True if ``user`` holds the platform-scoped admin role. + + Mirrors the check inside :func:`system_admin_context` so member-listing + endpoints can admit workspace members *or* system admins without pulling + in the full :class:`SystemAdminContext` (which 403s non-admins outright). + """ + if user.platform_role_id is None: + return False + platform_role = await session.scalar( + select(Roles).where(Roles.role_id == user.platform_role_id) + ) + return platform_role is not None and platform_role.role_code == "admin" + + # --------------------------------------------------------------------------- # Payload helpers # --------------------------------------------------------------------------- @@ -809,10 +824,33 @@ async def delete_workspace( @router.get("/workspaces/{workspace_id}/members") async def list_members( workspace_id: str, - context: SystemAdminContext = Depends(system_admin_context), + request: Request, session: AsyncSession = Depends(database_session), ) -> dict[str, Any]: - """List active and historical (non-soft-deleted) members of a workspace.""" + """List active and historical (non-soft-deleted) members of a workspace. + + Accessible to system admins (any workspace) and to active members of the + workspace itself. The script explorer calls this to seed the per-owner + directory-tree groups for non-admin users; visibility filters on the + scripts/data-resources endpoints still keep each peer's private content + hidden, so this only exposes membership (names), not private files. + """ + user = await current_user(request, session) + is_system_admin = await _is_system_admin(session, user) + if not is_system_admin: + membership = await session.scalar( + select(WorkspaceMembers).where( + WorkspaceMembers.workspace_id == workspace_id, + WorkspaceMembers.user_id == user.user_id, + WorkspaceMembers.is_deleted == 0, + WorkspaceMembers.member_status == "active", + ) + ) + if membership is None: + raise HTTPException( + status.HTTP_403_FORBIDDEN, + "需要系统管理员或该工作区成员权限", + ) await _load_workspace(session, workspace_id) rows = ( await session.execute( @@ -830,8 +868,9 @@ async def list_members( .limit(LIST_PAGE_SIZE) ) ).all() + request_id = request.headers.get("X-Request-ID") or new_ulid() return _envelope( - context.request_id, + request_id, [member_payload(u, r, m) for u, r, m in rows], {"count": len(rows), "page_size": LIST_PAGE_SIZE}, ) diff --git a/backend/src/backend/api/resources.py b/backend/src/backend/api/resources.py index b9b855e..8a16e6e 100644 --- a/backend/src/backend/api/resources.py +++ b/backend/src/backend/api/resources.py @@ -51,13 +51,13 @@ def _build_list_resources_descendant_prefix(parent_path: str) -> str: """Return the escaped materialized-path prefix for direct children of ``parent_path`` against ``StorageObjects.object_key``. - Data resources are workspace-wide (no per-user scoping at the API - level). The full object_key is ``{ws_id}/{user_id}/{jupyter_path}``; - we filter on object_key with the pattern ``{ws_id}/%/{parent_path}`` - so any owner whose jupyter_accessible_path starts with parent_path - matches. LIKE wildcards in parent_path are escaped; the ``%`` between - ``{ws_id}/`` and the escaped parent is an intentional SQL wildcard - matching the ``owner_user_id`` segment across all owners. + The full object_key is ``{ws_id}/{owner_user_id}/{jupyter_path}``. + ``list_resources`` prepends ``{ws_id}/{owner_user_id}`` (the requester + by default, or the ``owner_user_id`` query param) to this prefix and + applies ``LIKE '{ws_id}/{owner}/{prefix}%' AND NOT LIKE '...%/%'`` so + only that owner's direct children under ``parent_path`` match. LIKE + wildcards in parent_path are escaped so folder names containing ``_`` + or ``%`` do not act as wildcards. """ normalized = normalize_user_path(parent_path) escaped = _escape_like_pattern(normalized) @@ -383,6 +383,7 @@ async def bind_resource( @router.get("") async def list_resources( parent_path: str = Query(default="", max_length=1024), + owner_user_id: str | None = Query(default=None, max_length=64), context: RequestContext = Depends(request_context), session: AsyncSession = Depends(database_session), visibility: str | None = Query(default=None), @@ -403,23 +404,30 @@ async def list_resources( ) .order_by(DataResources.updated_at.desc()) ) - if parent_path: - # ``parent_path`` scopes to DIRECT children of that jupyter path - # (matching /api/v1/scripts). Data resources are workspace-wide, so - # the middle ``%`` is an intentional wildcard that matches the - # ``owner_user_id`` segment across all owners. The parent's ``_`` / - # ``%`` are escaped so sibling folders (e.g. ``fooXbar``) don't leak. - descendant_prefix = _build_list_resources_descendant_prefix(parent_path) - statement = statement.where( - StorageObjects.object_key.like( - f"{context.workspace.workspace_id}/%/{descendant_prefix}%", - escape="\\", - ), - ~StorageObjects.object_key.like( - f"{context.workspace.workspace_id}/%/{descendant_prefix}%/%", - escape="\\", - ), - ) + # ``parent_path`` scopes to DIRECT children of that jupyter path + # (matching /api/v1/scripts). Per-owner listing: default (no + # owner_user_id) scopes to the requester's own object_key subtree + # (``{ws_id}/{me}/...``); passing owner_user_id scopes to that owner's + # subtree so the tree can lazily fetch another member's data resources + # on group expand. The parent's ``_`` / ``%`` are escaped so sibling + # folders (e.g. ``fooXbar``) don't leak. Empty parent_path still applies + # the filter: it resolves to that owner's root-level direct children + # (``{ws_id}/{owner}/%`` and NOT ``{ws_id}/{owner}/%/%``), symmetric with + # list_scripts. Skipping the filter for empty input would silently + # surface nested descendants and break the directory tree. + target_owner = owner_user_id or context.user.user_id + owner_prefix = f"{context.workspace.workspace_id}/{target_owner}" + descendant_prefix = _build_list_resources_descendant_prefix(parent_path) + statement = statement.where( + StorageObjects.object_key.like( + f"{owner_prefix}/{descendant_prefix}%", + escape="\\", + ), + ~StorageObjects.object_key.like( + f"{owner_prefix}/{descendant_prefix}%/%", + escape="\\", + ), + ) # 只返回 owner 自己的资源（含 private），或 visibility 为 # workspace/public 的其他成员资源；A 的 private 资源对非 owner 不可见。 # admin 跳过过滤，全部可见。 diff --git a/backend/src/backend/api/scripts.py b/backend/src/backend/api/scripts.py index a4c31b4..a4115fc 100644 --- a/backend/src/backend/api/scripts.py +++ b/backend/src/backend/api/scripts.py @@ -129,40 +129,36 @@ def _escape_like_pattern(value: str) -> str: return value.replace("\\", "\\\\").replace("%", "\\%").replace("_", "\\_") -# @deprecated — legacy user-scoped prefix helper. list_scripts/count_scripts -# are now workspace-wide via _build_list_scripts_workspace_descendant_prefix; -# this helper is kept (with its original user-scoped semantics) so existing -# callers/tests that still reference it do not break. -def _build_list_scripts_descendant_prefix( - context: RequestContext, parent_path: str +def _build_list_scripts_owner_descendant_prefix( + owner_user_id: str, parent_path: str ) -> str: """Return the escaped materialized-path prefix for direct children of - ``parent_path`` within the **requester's own subtree**. + ``parent_path`` within ``owner_user_id``'s own subtree. - .. note:: - Legacy user-scoped helper. list_scripts / count_scripts are now - workspace-wide — use - :func:`_build_list_scripts_workspace_descendant_prefix` instead - (visibility filtering handles non-admin scoping in the SQL). + Storage is physically laid out as ``workspace/{owner_user_id}/...``, so a + per-owner listing matches ``workspace/{owner_user_id}/{parent}``. The + endpoint appends ``LIKE '%' AND NOT LIKE '%/%'`` against + ``storage_objects.relative_path`` so only scripts whose parent directory + is exactly ``parent_path`` match (no deeper descendants, no + prefix-siblings like ``foo/bar`` vs ``foo/bar2``). - The endpoint appends ``LIKE '/%' AND NOT LIKE '/%/%'`` - against ``storage_objects.relative_path`` so only scripts whose parent - directory is exactly ``parent_path`` (no deeper descendants, no - prefix-siblings like ``foo/bar`` vs ``foo/bar2``) match. - - Empty ``parent_path`` produces the user-scoped root prefix — i.e. the - endpoint returns root-level scripts only, not the full workspace. + Empty ``parent_path`` produces the owner-scoped root prefix — i.e. the + endpoint returns that owner's root-level scripts only. ``list_scripts`` + calls this with ``owner_user_id`` = the requester by default (so a + non-admin sees their own subtree, including private) or with the + ``owner_user_id`` query param so the tree can lazily fetch another + member's content on group expand; the route's visibility filter then + excludes the other owner's private rows. The prefix is run through ``_escape_like_pattern`` so folder names containing ``_`` / ``%`` do not act as wildcards. The trailing ``/`` is appended AFTER escaping so it remains a literal slash. """ normalized_parent = normalize_user_path(parent_path) - scoped_prefix = user_relative_path(context) if normalized_parent: - target_prefix = f"{scoped_prefix}/{normalized_parent}" + target_prefix = f"workspace/{owner_user_id}/{normalized_parent}" else: - target_prefix = scoped_prefix + target_prefix = f"workspace/{owner_user_id}" return f"{_escape_like_pattern(target_prefix)}/" @@ -836,16 +832,25 @@ async def get_workspace_tree( @router.get("/workspace-directories") async def list_workspace_directories( parent_path: str = Query(default=""), + owner_user_id: str | None = Query(default=None, max_length=64), context: RequestContext = Depends(request_context), session: AsyncSession = Depends(database_session), ) -> dict[str, Any]: """List direct child directories of a workspace path. Empty ``parent_path`` returns the directories immediately under the - user's scoped root. Only available, non-deleted StorageObjects are - considered. + target owner's scoped root. Only available, non-deleted StorageObjects + are considered. + + ``owner_user_id`` defaults to the requester, so a member lists their + own directories. Passing another member's id scopes to that owner's + subtree so the script explorer can lazily render their directory + structure on expand (directories are structural rows; file-level + visibility is still enforced by the scripts/data-resources endpoints, + which exclude the other owner's private files). """ - scoped_prefix = user_relative_path(context) + target_owner = owner_user_id or context.user.user_id + scoped_prefix = f"workspace/{target_owner}" parent = normalize_user_path(parent_path) target_prefix = f"{scoped_prefix}/{parent}" if parent else scoped_prefix descendant_prefix = f"{_escape_like_pattern(target_prefix)}/" @@ -878,6 +883,7 @@ async def list_workspace_directories( "path": child_path, "name": suffix, "parent_path": parent, + "owner_user_id": target_owner, "has_children": False, }, ) @@ -1018,7 +1024,8 @@ async def create_workspace_directory( path_hash=path_hash, object_status="available", size_bytes=0, - visibility="private", + visibility="public", + owner_user_id=context.user.user_id, created_by=context.user.user_id, ) session.add(directory) @@ -1032,7 +1039,8 @@ async def create_workspace_directory( directory.storage_uri = f"inline://directory/{relative_path}" directory.file_name = name directory.size_bytes = 0 - directory.visibility = "private" + directory.visibility = "public" + directory.owner_user_id = context.user.user_id directory.created_by = context.user.user_id try: @@ -1058,6 +1066,7 @@ async def create_workspace_directory( "path": child_path, "name": name, "parent_path": parent, + "owner_user_id": context.user.user_id, }, "meta": {}, } @@ -1165,17 +1174,23 @@ async def delete_workspace_directory( @router.get("/scripts") async def list_scripts( parent_path: str = Query(default="", max_length=1024), + owner_user_id: str | None = Query(default=None, max_length=64), context: RequestContext = Depends(request_context), session: AsyncSession = Depends(database_session), ) -> dict[str, Any]: - # Workspace-wide listing: storage is physically laid out as - # ``workspace/{user_id}/...``. Empty parent_path wildcards the owner - # segment (``workspace/%/``) so each owner's root files are returned; - # non-empty parent_path embeds the same owner wildcard - # (``workspace/%/foo``) so every owner's ``foo`` subtree matches, - # mirroring list_resources. Non-admin scoping is applied below via - # visibility, matching list_resources (69a9a48). - descendant_prefix = _build_list_scripts_workspace_descendant_prefix(parent_path) + # Per-owner listing: storage is physically laid out as + # ``workspace/{user_id}/...``. Default (no owner_user_id) scopes to the + # requester's own subtree — root-level files when parent_path is empty — + # so the tree's initial load fetches only "me". Passing owner_user_id + # scopes to that owner's subtree so the tree can lazily fetch another + # member's content when their group is expanded. Non-admin scoping is + # applied below via visibility, so the other owner's private rows are + # excluded (workspace/public only); the requester's own private rows + # pass because ``owner_user_id = me``. + target_owner = owner_user_id or context.user.user_id + descendant_prefix = _build_list_scripts_owner_descendant_prefix( + target_owner, parent_path + ) statement = ( select(Scripts, StorageObjects, Users.display_name) diff --git a/backend/src/backend/main.py b/backend/src/backend/main.py index e6a938c..cd8eab5 100644 --- a/backend/src/backend/main.py +++ b/backend/src/backend/main.py @@ -33,16 +33,16 @@ from fastapi import Request from fastapi.responses import JSONResponse from loguru import logger -from backend.audit import configure_audit_logging from backend.api.admin import router as admin_router from backend.api.auth import router as auth_router from backend.api.jupyter import router as jupyter_router from backend.api.platform import router as platform_router from backend.api.resources import router as resources_router -from backend.api.scripts import router as scripts_router from backend.api.schedules.runs import router as schedule_runs_router from backend.api.schedules.schedules import router as schedules_router +from backend.api.scripts import router as scripts_router from backend.api.storage import router as storage_api_router +from backend.audit import configure_audit_logging from backend.clients.rclone import RcloneRCClient from backend.clients.runtime import RuntimeClient @@ -161,7 +161,7 @@ async def access_log(request: Request, call_next): method=request.method, path=request.url.path, status=500, - ).info("audit") + ).info(f"{request.url.path} skip audit") raise elapsed_ms = (time.perf_counter() - start) * 1000 logger.info( @@ -175,7 +175,7 @@ async def access_log(request: Request, call_next): method=request.method, path=request.url.path, status=response.status_code, - ).info("audit") + ).info(f"{request.url.path} skip audit") return response diff --git a/backend/tests/test_list_scripts_parent_path.py b/backend/tests/test_list_scripts_parent_path.py index feb0868..4b57096 100644 --- a/backend/tests/test_list_scripts_parent_path.py +++ b/backend/tests/test_list_scripts_parent_path.py @@ -28,7 +28,7 @@ from sqlalchemy import Column, MetaData, String, Table, create_engine, select, t from sqlalchemy.dialects import mysql as mysql_dialect from backend.api.scripts import ( - _build_list_scripts_descendant_prefix, + _build_list_scripts_owner_descendant_prefix, _build_list_scripts_workspace_descendant_prefix, _escape_like_pattern, normalize_user_path, @@ -85,14 +85,14 @@ class TestEscapeLikePattern: def test_descendant_prefix_root() -> None: - """Empty parent_path → descendant prefix is the scoped root + '/'.""" - prefix = _build_list_scripts_descendant_prefix(_ctx("alice"), "") + """Empty parent_path → descendant prefix is the owner-scoped root + '/'.""" + prefix = _build_list_scripts_owner_descendant_prefix("alice", "") assert prefix == "workspace/alice/" def test_descendant_prefix_subdir() -> None: - """Non-empty parent_path → appended under the scoped root.""" - prefix = _build_list_scripts_descendant_prefix(_ctx("alice"), "foo/bar") + """Non-empty parent_path → appended under the owner-scoped root.""" + prefix = _build_list_scripts_owner_descendant_prefix("alice", "foo/bar") assert prefix == "workspace/alice/foo/bar/" @@ -100,18 +100,18 @@ def test_descendant_prefix_escapes_metachars() -> None: """Folder name ``foo_bar`` MUST produce ``foo\\_bar`` in the prefix so the trailing ``%`` doesn't become 'match any single char before b'.""" - prefix = _build_list_scripts_descendant_prefix(_ctx("alice"), "foo_bar") + prefix = _build_list_scripts_owner_descendant_prefix("alice", "foo_bar") assert prefix == r"workspace/alice/foo\_bar/" def test_descendant_prefix_normalizes_leading_trailing_slashes() -> None: - prefix = _build_list_scripts_descendant_prefix(_ctx("alice"), "/foo/bar/") + prefix = _build_list_scripts_owner_descendant_prefix("alice", "/foo/bar/") assert prefix == "workspace/alice/foo/bar/" def test_descendant_prefix_rejects_traversal() -> None: with pytest.raises(HTTPException) as exc: - _build_list_scripts_descendant_prefix(_ctx("alice"), "foo/../bar") + _build_list_scripts_owner_descendant_prefix("alice", "foo/../bar") assert exc.value.status_code == 422 @@ -202,14 +202,19 @@ async def test_list_scripts_where_clause_uses_like_prefix_and_excludes_deeper() ) ) - await list_scripts(parent_path="foo/bar", context=_ctx("alice"), session=mock_session) + await list_scripts( + parent_path="foo/bar", + owner_user_id=None, + context=_ctx("alice"), + session=mock_session, + ) assert len(captured_sql) == 1 sql = captured_sql[0].lower() - # 中间 ``%`` 是 owner 段通配符(跨所有 owner),父路径本身按字面匹配, - # 与 list_resources 对 object_key 的过滤一致。 - assert "like 'workspace/%%/foo/bar/%%'" in sql - assert "not like 'workspace/%%/foo/bar/%%/%%'" in sql + # Default (no owner_user_id) scopes to the requester's own subtree: + # LIKE workspace/alice/foo/bar/% (direct children), excluding deeper. + assert "like 'workspace/alice/foo/bar/%%'" in sql + assert "not like 'workspace/alice/foo/bar/%%/%%'" in sql async def test_list_scripts_where_clause_escapes_pattern_literal() -> None: @@ -230,7 +235,12 @@ async def test_list_scripts_where_clause_escapes_pattern_literal() -> None: ) ) - await list_scripts(parent_path="foo_bar", context=_ctx("alice"), session=mock_session) + await list_scripts( + parent_path="foo_bar", + owner_user_id=None, + context=_ctx("alice"), + session=mock_session, + ) sql = captured_sql[0] # Normalize keyword case so we don't depend on SQLAlchemy casing. sql_lower = sql.lower() @@ -238,9 +248,9 @@ async def test_list_scripts_where_clause_escapes_pattern_literal() -> None: # doubles the escape char inside the SQL string literal, so what # the helper emits as `foo\_bar` renders as `foo\\_bar` here # (2 backslash chars in the actual SQL string). - assert r"like 'workspace/%%/foo\\_bar/%%'" in sql_lower + assert r"like 'workspace/alice/foo\\_bar/%%'" in sql_lower # NOT LIKE clause also escaped. - assert r"not like 'workspace/%%/foo\\_bar/%%/%%'" in sql_lower + assert r"not like 'workspace/alice/foo\\_bar/%%/%%'" in sql_lower # And both declare ESCAPE '\\'. assert sql.count("ESCAPE '\\\\'") == 2, sql @@ -262,18 +272,26 @@ async def test_list_scripts_where_clause_escapes_percent_pattern() -> None: ) ) - await list_scripts(parent_path="100%match", context=_ctx("alice"), session=mock_session) + await list_scripts( + parent_path="100%match", + owner_user_id=None, + context=_ctx("alice"), + session=mock_session, + ) sql = captured_sql[0] sql_lower = sql.lower() # SQLAlchemy doubles the escape char so `%` → `\%` becomes `\\%` # in the SQL string literal. - assert r"workspace/%%/100\\%%match/%%" in sql_lower + assert r"workspace/alice/100\\%%match/%%" in sql_lower async def test_list_scripts_non_admin_adds_visibility_filter() -> None: - """Workspace-wide listing is narrowed by visibility for non-admin: + """Owner-scoped listing is narrowed by visibility for non-admin: owner_user_id = me OR visibility IN (workspace, public) — exactly like - list_resources. The workspace prefix contains NO user_id (cross-owner).""" + list_resources. Default (no owner_user_id) scopes to the requester's own + subtree, so the visibility predicate is redundant-but-present here; it + becomes load-bearing when an owner_user_id query param browses another + member's subtree (their private rows are then excluded).""" from backend.api.scripts import list_scripts captured_sql: list[str] = [] @@ -289,11 +307,50 @@ async def test_list_scripts_non_admin_adds_visibility_filter() -> None: ) ) - await list_scripts(parent_path="", context=_ctx("alice"), session=mock_session) + await list_scripts( + parent_path="", + owner_user_id=None, + context=_ctx("alice"), + session=mock_session, + ) sql = captured_sql[0].lower() - # Root listing wildcards the owner segment: LIKE workspace/%/% - # (each owner's root files), excluding 3+ segment descendants. - assert "like 'workspace/%%/%%'" in sql + # Default (no owner_user_id) scopes to the requester's own root: + # LIKE workspace/alice/% (alice's root files), excluding nested. + assert "like 'workspace/alice/%%'" in sql + assert "scripts.owner_user_id = 'alice'" in sql + assert "scripts.visibility in ('workspace', 'public')" in sql + + +async def test_list_scripts_owner_param_scopes_to_other_owner() -> None: + """owner_user_id= scopes the LIKE to that owner's subtree so the + tree can lazily fetch another member's content on group expand. The + non-admin visibility predicate is still applied, so the other owner's + private rows are excluded (only workspace/public survive).""" + from backend.api.scripts import list_scripts + + captured_sql: list[str] = [] + + class _MockResult: + def all(self): + return [] + + mock_session = MagicMock() + mock_session.execute = AsyncMock( + side_effect=lambda stmt: ( + captured_sql.append(_compile_sql(stmt)) or _MockResult() + ) + ) + + await list_scripts( + parent_path="", + owner_user_id="bob", + context=_ctx("alice"), + session=mock_session, + ) + sql = captured_sql[0].lower() + assert "like 'workspace/bob/%%'" in sql + assert "not like 'workspace/bob/%%/%%'" in sql + # non-admin: visibility predicate still present (excludes bob's private) assert "scripts.owner_user_id = 'alice'" in sql assert "scripts.visibility in ('workspace', 'public')" in sql @@ -316,15 +373,47 @@ async def test_list_scripts_admin_skips_visibility_filter() -> None: ) await list_scripts( - parent_path="", context=_ctx("alice", is_admin=True), session=mock_session + parent_path="", + owner_user_id=None, + context=_ctx("alice", is_admin=True), + session=mock_session, ) sql = captured_sql[0].lower() - assert "like 'workspace/%%/%%'" in sql + assert "like 'workspace/alice/%%'" in sql # owner_user_id / visibility still appear in the SELECT projection; what # must be absent is the visibility WHERE predicate for non-admins. assert "scripts.visibility in ('workspace', 'public')" not in sql +async def test_list_scripts_admin_owner_param_skips_visibility() -> None: + """Admin browsing another owner's subtree scopes to that owner and skips + the visibility predicate (admin sees the other owner's private too).""" + from backend.api.scripts import list_scripts + + captured_sql: list[str] = [] + + class _MockResult: + def all(self): + return [] + + mock_session = MagicMock() + mock_session.execute = AsyncMock( + side_effect=lambda stmt: ( + captured_sql.append(_compile_sql(stmt)) or _MockResult() + ) + ) + + await list_scripts( + parent_path="sub", + owner_user_id="bob", + context=_ctx("alice", is_admin=True), + session=mock_session, + ) + sql = captured_sql[0].lower() + assert "like 'workspace/bob/sub/%%'" in sql + assert "scripts.visibility in ('workspace', 'public')" not in sql + + async def test_list_workspace_directories_where_clause_escapes_pattern() -> None: """list_workspace_directories must escape user input too (was pre-existing debt).""" @@ -351,11 +440,21 @@ async def test_list_workspace_directories_where_clause_escapes_pattern() -> None ) await list_workspace_directories( - parent_path="foo_bar", context=_ctx("alice"), session=mock_session + parent_path="foo_bar", owner_user_id=None, + context=_ctx("alice"), session=mock_session, ) sql = " ".join(captured_sql) assert r"workspace/alice/foo\\_bar/" in sql, sql + # owner_user_id scopes the prefix to that owner's subtree. + captured_sql.clear() + await list_workspace_directories( + parent_path="foo_bar", owner_user_id="bob", + context=_ctx("alice"), session=mock_session, + ) + sql = " ".join(captured_sql) + assert r"workspace/bob/foo\\_bar/" in sql, sql + # ─── layer 3: behavioral test on real LIKE execution ────────────── diff --git a/backend/tests/test_resources.py b/backend/tests/test_resources.py index 7d75c63..c160718 100644 --- a/backend/tests/test_resources.py +++ b/backend/tests/test_resources.py @@ -573,6 +573,7 @@ async def test_list_resources_where_clause_uses_like_prefix_and_excludes_deeper( await list_resources( parent_path="foo/bar", + owner_user_id=None, context=_resource_ctx(), session=mock_session, visibility=None, @@ -581,9 +582,10 @@ async def test_list_resources_where_clause_uses_like_prefix_and_excludes_deeper( assert len(captured_sql) == 1 sql = captured_sql[0].lower() - # 中间 ``%`` 是 owner 段通配符(跨所有 owner),父路径本身按字面匹配。 - assert "like 'w001/%%/foo/bar/%%'" in sql - assert "not like 'w001/%%/foo/bar/%%/%%'" in sql + # Default (no owner_user_id) scopes to the requester's own object_key + # subtree: LIKE w001/u001/foo/bar/% (direct children), excluding deeper. + assert "like 'w001/u001/foo/bar/%%'" in sql + assert "not like 'w001/u001/foo/bar/%%/%%'" in sql async def test_list_resources_where_clause_escapes_underscore() -> None: @@ -596,6 +598,7 @@ async def test_list_resources_where_clause_escapes_underscore() -> None: await list_resources( parent_path="foo_bar", + owner_user_id=None, context=_resource_ctx(), session=mock_session, visibility=None, @@ -605,15 +608,20 @@ async def test_list_resources_where_clause_escapes_underscore() -> None: sql_lower = sql.lower() # SQLAlchemy doubles the escape char inside the SQL string literal, so # the helper's ``foo\_bar`` renders as ``foo\\_bar`` in the SQL text. - assert r"like 'w001/%%/foo\\_bar/%%'" in sql_lower - assert r"not like 'w001/%%/foo\\_bar/%%/%%'" in sql_lower + assert r"like 'w001/u001/foo\\_bar/%%'" in sql_lower + assert r"not like 'w001/u001/foo\\_bar/%%/%%'" in sql_lower # Both LIKE clauses declare ESCAPE '\\' (two in total). assert sql.count("ESCAPE '\\\\'") == 2, sql -async def test_list_resources_without_parent_path_adds_no_like_clause() -> None: - """Empty parent_path keeps the legacy workspace-wide behaviour — no - object_key LIKE filter at all.""" +async def test_list_resources_empty_parent_path_adds_root_like_clause() -> None: + """Empty parent_path still applies the directory filter (symmetric with + list_scripts): ``{ws_id}/{owner}/%`` AND NOT ``{ws_id}/{owner}/%/%`` so + the owner-scoped root view returns only direct children of the + requester's root, never nested descendants. Skipping the filter for + empty input used to surface nested resources at the root and visually + broke the directory tree. + """ from backend.api.resources import list_resources captured_sql: list[str] = [] @@ -621,14 +629,42 @@ async def test_list_resources_without_parent_path_adds_no_like_clause() -> None: await list_resources( parent_path="", + owner_user_id=None, context=_resource_ctx(), session=mock_session, visibility=None, keyword=None, ) sql = captured_sql[0].lower() - assert " like " not in sql - assert " not like " not in sql + assert " like 'w001/u001/%%'" in sql + assert " not like 'w001/u001/%%/%%'" in sql + + +async def test_list_resources_owner_param_scopes_to_other_owner() -> None: + """owner_user_id= scopes object_key LIKE to that owner's subtree + so the tree can lazily fetch another member's data resources on group + expand. Non-admin visibility predicate is still applied, so the other + owner's private resources are excluded (workspace/public only). + """ + from backend.api.resources import list_resources + + captured_sql: list[str] = [] + mock_session = _list_resources_capturing_session(captured_sql) + + await list_resources( + parent_path="", + owner_user_id="U002", + context=_resource_ctx(), + session=mock_session, + visibility=None, + keyword=None, + ) + sql = captured_sql[0].lower() + assert " like 'w001/u002/%%'" in sql + assert " not like 'w001/u002/%%/%%'" in sql + # non-admin: visibility predicate still present (excludes U002 private) + assert "data_resources.owner_user_id = 'u001'" in sql + assert "data_resources.visibility in ('workspace', 'public')" in sql async def test_list_resources_joins_users_for_display_name() -> None: @@ -641,6 +677,7 @@ async def test_list_resources_joins_users_for_display_name() -> None: await list_resources( parent_path="", + owner_user_id=None, context=_resource_ctx(), session=mock_session, visibility=None, @@ -745,11 +782,29 @@ def test_sqlite_parent_path_with_slash_direct_children(sqlite_object_key_table) assert matched == ["W001/U001/data/deep/nested.csv"], matched -def test_sqlite_empty_parent_path_has_no_like_filter(sqlite_object_key_table) -> None: - """Empty parent_path → no LIKE filter → the endpoint's base WHERE only - (workspace-wide active resources). Stand-in: every row is returned.""" +def test_sqlite_empty_parent_path_returns_root_level_across_owners( + sqlite_object_key_table, +) -> None: + """Empty parent_path now applies the root filter (symmetric with + list_scripts): ``{ws_id}/%/%`` AND NOT ``{ws_id}/%/%/%`` returns only + direct children of every owner's root, excluding nested descendants. + Earlier 'no LIKE' behaviour used to surface every row in the + workspace at the root, which is exactly what made scripts and data + appear mutually visible and broke the tree. + """ engine, table = sqlite_object_key_table + like = "W001/%/%" + not_like = "W001/%/%/%" with engine.connect() as conn: - rows = conn.execute(select(table.c.object_key)).fetchall() + rows = conn.execute( + select(table.c.object_key).where( + table.c.object_key.like(like, escape="\\"), + ~table.c.object_key.like(not_like, escape="\\"), + ) + ).fetchall() matched = sorted(r[0] for r in rows) - assert len(matched) == 8, matched + # ``W001/U001/root.csv`` is the only 3-segment path (= direct child + # of the owner root); all 4+ segment paths (data/*, database/*) are + # excluded by the NOT LIKE clause. The other 4+ segment files would + # be returned when the user expands the corresponding subdirectory. + assert matched == ["W001/U001/root.csv"], matched diff --git a/frontend/app/components/platform/ScriptExplorer.tsx b/frontend/app/components/platform/ScriptExplorer.tsx index ad9e5a4..3436cb1 100644 --- a/frontend/app/components/platform/ScriptExplorer.tsx +++ b/frontend/app/components/platform/ScriptExplorer.tsx @@ -60,6 +60,7 @@ export function ScriptExplorer({ const loadingChildrenPaths = useScriptWorkspaceStore( (s) => s.loadingChildrenPaths, ); + const members = useScriptWorkspaceStore((s) => s.members); const onToggle = useScriptWorkspaceStore((s) => s.toggleExpanded); const dataByOwner = useMemo(() => { @@ -83,9 +84,15 @@ export function ScriptExplorer({ item.visibility === "public", ); + // 用工作区成员列表播种分组 —— 顶层"我 / user1 / user2 / …"折叠分组 + // 的来源。即使某成员尚未加载任何脚本/数据（默认折叠、点击才拉取）， + // 也作为空分组出现，保证目录树结构稳定可见（修"目录树结构消失"）。 const byOwner = new Map(); - // 当前用户的目录树即使没有脚本也要渲染，所以预置空组。 - if (user?.user_id) { + for (const m of members) { + if (!byOwner.has(m.user_id)) byOwner.set(m.user_id, []); + } + // 当前用户兜底（members 未就绪时仍渲染"我"的分组）。 + if (user?.user_id && !byOwner.has(user.user_id)) { byOwner.set(user.user_id, []); } for (const item of visibleScripts) { @@ -93,15 +100,17 @@ export function ScriptExplorer({ list.push(item); byOwner.set(item.owner_user_id, list); } - - // data-only owner(只有数据资源、没有 scripts 的用户)也要出现在分组里, - // 因为 data resources 与 scripts 共享同一棵目录树。 + // data-only owner(只有数据资源、没有 scripts 的用户,且不在 members 列表 + // 里,如已移除成员遗留的资源)也要出现在分组里。 for (const ownerUserId of dataByOwner.keys()) { if (!byOwner.has(ownerUserId)) { byOwner.set(ownerUserId, []); } } + // 成员 id → display_name 优先取 members 列表（最准）。 + const memberName = new Map(members.map((m) => [m.user_id, m.display_name])); + const groups: { user: AuthUser | null; scripts: ScriptItem[]; @@ -110,9 +119,8 @@ export function ScriptExplorer({ }[] = []; for (const [ownerUserId, groupScripts] of byOwner.entries()) { const groupDataResources = dataByOwner.get(ownerUserId) ?? []; - // data-only owner（没有 scripts 的用户）回退到 data resources 的 - // owner_display_name，否则 data-only owner 前端只能显示 userId 末 6 位。 const displayName = + memberName.get(ownerUserId) ?? groupScripts[0]?.owner_display_name ?? groupDataResources[0]?.owner_display_name ?? (ownerUserId === user?.user_id ? user?.display_name : null) ?? @@ -129,14 +137,16 @@ export function ScriptExplorer({ role_code: null, is_system_admin: false, } as AuthUser); - const inferred = inferredDirectories(groupScripts); + const inferred = inferredDirectories(groupScripts, ownerUserId); + // directories flat 数组现在按 owner_user_id 标记，按 owner 切分后与 + // inferred 合并（inferred 补全 fetched 目录行未覆盖的祖先路径）。 + const ownerDirs = directories.filter( + (d) => d.owner_user_id === ownerUserId, + ); groups.push({ user: groupUser, scripts: groupScripts, - directories: - ownerUserId === user?.user_id - ? mergeDirectories(directories, inferred) - : inferred, + directories: mergeDirectories(ownerDirs, inferred), dataResources: groupDataResources, }); } @@ -144,18 +154,19 @@ export function ScriptExplorer({ groups.sort((a, b) => { if (a.user?.user_id === user?.user_id) return -1; if (b.user?.user_id === user?.user_id) return 1; - return (a.user?.user_id ?? "").localeCompare(b.user?.user_id ?? ""); + return (a.user?.display_name ?? a.user?.user_id ?? "").localeCompare( + b.user?.display_name ?? b.user?.user_id ?? "", + ); }); return groups; - }, [filteredScripts, directories, user, dataByOwner]); + }, [filteredScripts, directories, user, dataByOwner, members]); return (