feat(scripts/data-resources): merge tree + add parent_path filter

- Backend: GET /api/v1/data-resources accepts parent_path; LIKE
  '{ws_id}/%/{escaped}/%' AND NOT LIKE '{ws_id}/%/{escaped}/%/%' on
  StorageObjects.object_key (workspace-wide, escapes _ and %, mirrors
  list_scripts parent_path semantics). 13 new tests in
  test_resources.py (helper unit / SQL compile / SQLite behavioral).
- Frontend: listResources gains parentPath arg, propagated through
  WorkspaceBoundApi + AuthContext binding. WorkspaceTreeGroup title
  count and ScriptExplorer header count now include dataResources.
  memberScriptGroups backfills data-only owners so users with only
  data resources still render a group. loadDataResources accepts an
  optional parentPath, default empty preserves prior behavior.
This commit is contained in:
tao.chen
2026-09-02 10:10:41 +08:00
committed by tao.chen
parent 0ac034d7d3
commit a337804700
7 changed files with 296 additions and 9 deletions
+36 -1
View File
@@ -28,7 +28,7 @@ from backend.dependencies import (
database_session,
request_context,
)
from backend.scripts import _escape_like_pattern
from backend.scripts import _escape_like_pattern, normalize_user_path
from backend.schemas import (
CompleteResourceUploadRequest,
CreateResourceUploadRequest,
@@ -47,6 +47,23 @@ from backend.services.storage import (
router = APIRouter(prefix="/api/v1/data-resources", tags=["data-resources"])
def _build_list_resources_descendant_prefix(parent_path: str) -> str:
"""Return the escaped materialized-path prefix for direct children
of ``parent_path`` against ``StorageObjects.object_key``.
Data resources are workspace-wide (no per-user scoping at the API
level). The full object_key is ``{ws_id}/{user_id}/{jupyter_path}``;
we filter on object_key with the pattern ``{ws_id}/%/{parent_path}``
so any owner whose jupyter_accessible_path starts with parent_path
matches. LIKE wildcards in parent_path are escaped; the ``%`` between
``{ws_id}/`` and the escaped parent is an intentional SQL wildcard
matching the ``owner_user_id`` segment across all owners.
"""
normalized = normalize_user_path(parent_path)
escaped = _escape_like_pattern(normalized)
return f"{escaped}/" if escaped else ""
def compute_jupyter_relative_path(script_path: str, resource_relative: str) -> str:
"""从当前脚本所在目录算到资源文件的 Jupyter 相对路径。
@@ -364,6 +381,7 @@ async def bind_resource(
# 列出当前工作区可见的数据资源,可按可见性或关键字筛选。
@router.get("")
async def list_resources(
parent_path: str = Query(default="", max_length=1024),
context: RequestContext = Depends(request_context),
session: AsyncSession = Depends(database_session),
visibility: str | None = Query(default=None),
@@ -383,6 +401,23 @@ async def list_resources(
)
.order_by(DataResources.updated_at.desc())
)
if parent_path:
# ``parent_path`` scopes to DIRECT children of that jupyter path
# (matching /api/v1/scripts). Data resources are workspace-wide, so
# the middle ``%`` is an intentional wildcard that matches the
# ``owner_user_id`` segment across all owners. The parent's ``_`` /
# ``%`` are escaped so sibling folders (e.g. ``fooXbar``) don't leak.
descendant_prefix = _build_list_resources_descendant_prefix(parent_path)
statement = statement.where(
StorageObjects.object_key.like(
f"{context.workspace.workspace_id}/%/{descendant_prefix}%",
escape="\\",
),
~StorageObjects.object_key.like(
f"{context.workspace.workspace_id}/%/{descendant_prefix}%/%",
escape="\\",
),
)
# 2026-08-11: 临时取消"用户间目录互相不可见"约束
# 列表接口现在返回 workspace 内全部 active 资源(不再按 owner / visibility 过滤)。
# 还原: 删除下面这段注释,恢复原来的 if not context.is_admin: ... 块。
+238 -1
View File
@@ -7,10 +7,14 @@ from __future__ import annotations
import datetime
from types import SimpleNamespace
from unittest.mock import MagicMock
from unittest.mock import AsyncMock, MagicMock
import pytest
from fastapi import HTTPException
from sqlalchemy import Column, MetaData, String, Table, create_engine, select
from sqlalchemy.dialects import mysql as mysql_dialect
from backend.resources import (
_build_list_resources_descendant_prefix,
compute_jupyter_relative_path,
resource_directory,
resource_payload,
@@ -405,3 +409,236 @@ def test_data_resources_model_allows_duplicate_storage_object_reference() -> Non
assert "storage_object_id" not in cols, (
f"unexpected unique index {idx.name} on storage_object_id"
)
# ─── parent_path filtering (mirrors test_list_scripts_parent_path.py) ────────
def _resource_ctx(workspace_id: str = "W001") -> SimpleNamespace:
return SimpleNamespace(
request_id="test",
user=SimpleNamespace(user_id="U001"),
workspace=SimpleNamespace(workspace_id=workspace_id),
)
class TestBuildListResourcesDescendantPrefix:
"""Pure-function contract for the escaped object_key prefix. Unlike
the scripts helper there is NO user scope — data resources are
workspace-wide, so the prefix is just the normalized+escaped path."""
def test_empty_parent_path_returns_empty_prefix(self) -> None:
assert _build_list_resources_descendant_prefix("") == ""
def test_subdir_prefix_appends_trailing_slash(self) -> None:
assert _build_list_resources_descendant_prefix("foo/bar") == "foo/bar/"
def test_escapes_underscore(self) -> None:
assert _build_list_resources_descendant_prefix("foo_bar") == r"foo\_bar/"
def test_escapes_percent(self) -> None:
assert _build_list_resources_descendant_prefix("100%match") == r"100\%match/"
def test_normalizes_leading_trailing_slashes(self) -> None:
assert _build_list_resources_descendant_prefix("/foo/bar/") == "foo/bar/"
def test_rejects_traversal(self) -> None:
with pytest.raises(HTTPException) as exc:
_build_list_resources_descendant_prefix("foo/../bar")
assert exc.value.status_code == 422
# ─── layer 2: SQL contract (mock session + mysql dialect compile) ────────────
def _compile_sql(stmt) -> str:
return str(
stmt.compile(
dialect=mysql_dialect.dialect(),
compile_kwargs={"literal_binds": True},
)
)
class _ListResourcesMockResult:
def all(self):
return []
def _list_resources_capturing_session(captured_sql: list[str]) -> MagicMock:
mock_session = MagicMock()
mock_session.execute = AsyncMock(
side_effect=lambda stmt: (
captured_sql.append(_compile_sql(stmt)) or _ListResourcesMockResult()
)
)
return mock_session
async def test_list_resources_where_clause_uses_like_prefix_and_excludes_deeper() -> None:
from backend.resources import list_resources
captured_sql: list[str] = []
mock_session = _list_resources_capturing_session(captured_sql)
await list_resources(
parent_path="foo/bar",
context=_resource_ctx(),
session=mock_session,
visibility=None,
keyword=None,
)
assert len(captured_sql) == 1
sql = captured_sql[0].lower()
# 中间 ``%`` 是 owner 段通配符(跨所有 owner),父路径本身按字面匹配。
assert "like 'w001/%%/foo/bar/%%'" in sql
assert "not like 'w001/%%/foo/bar/%%/%%'" in sql
async def test_list_resources_where_clause_escapes_underscore() -> None:
"""Regression: parent_path containing ``_`` MUST be escaped in the
compiled LIKE pattern, otherwise sibling-path leak (``fooXbar``) returns."""
from backend.resources import list_resources
captured_sql: list[str] = []
mock_session = _list_resources_capturing_session(captured_sql)
await list_resources(
parent_path="foo_bar",
context=_resource_ctx(),
session=mock_session,
visibility=None,
keyword=None,
)
sql = captured_sql[0]
sql_lower = sql.lower()
# SQLAlchemy doubles the escape char inside the SQL string literal, so
# the helper's ``foo\_bar`` renders as ``foo\\_bar`` in the SQL text.
assert r"like 'w001/%%/foo\\_bar/%%'" in sql_lower
assert r"not like 'w001/%%/foo\\_bar/%%/%%'" in sql_lower
# Both LIKE clauses declare ESCAPE '\\' (two in total).
assert sql.count("ESCAPE '\\\\'") == 2, sql
async def test_list_resources_without_parent_path_adds_no_like_clause() -> None:
"""Empty parent_path keeps the legacy workspace-wide behaviour — no
object_key LIKE filter at all."""
from backend.resources import list_resources
captured_sql: list[str] = []
mock_session = _list_resources_capturing_session(captured_sql)
await list_resources(
parent_path="",
context=_resource_ctx(),
session=mock_session,
visibility=None,
keyword=None,
)
sql = captured_sql[0].lower()
assert " like " not in sql
assert " not like " not in sql
# ─── layer 3: behavioral test on real LIKE execution (SQLite) ────────────────
@pytest.fixture
def sqlite_object_key_table():
"""SQLite in-memory stand-in for ``storage_objects.object_key`` rows.
Three-layer structure: direct children under ``data`` owned by two
different users (cross-owner), a deeper descendant, a sibling folder,
a root file, plus ``data_x`` and its ``dataXx`` / ``data2x`` decoys.
"""
engine = create_engine("sqlite:///:memory:")
metadata = MetaData()
table = Table(
"object_keys",
metadata,
Column("object_key", String(1024), nullable=False),
)
metadata.create_all(engine)
rows = [
"W001/U001/data/alpha.csv", # direct child (owner U001)
"W001/U002/data/beta.csv", # direct child (owner U002)
"W001/U001/data/deep/nested.csv", # deeper descendant
"W001/U001/database/gamma.csv", # sibling folder
"W001/U001/root.csv", # root file
"W001/U001/data_x/delta.csv", # target for parent_path=data_x
"W001/U001/dataXx/decoy.csv", # sibling decoy (unescaped match)
"W001/U001/data2x/decoy2.csv", # sibling decoy (unescaped match)
]
with engine.begin() as conn:
conn.execute(table.insert(), [{"object_key": key} for key in rows])
yield engine, table
engine.dispose()
def test_sqlite_direct_children_across_owners(sqlite_object_key_table) -> None:
"""parent_path='data' returns exactly the direct children under data,
across every owner, excluding deeper/sibling/root paths."""
engine, table = sqlite_object_key_table
prefix = _build_list_resources_descendant_prefix("data")
like = f"W001/%/{prefix}%"
not_like = f"W001/%/{prefix}%/%"
with engine.connect() as conn:
rows = conn.execute(
select(table.c.object_key).where(
table.c.object_key.like(like, escape="\\"),
~table.c.object_key.like(not_like, escape="\\"),
)
).fetchall()
matched = sorted(r[0] for r in rows)
assert matched == [
"W001/U001/data/alpha.csv",
"W001/U002/data/beta.csv",
], matched
def test_sqlite_escaped_underscore_does_not_match_sibling(
sqlite_object_key_table,
) -> None:
"""parent_path='data_x' must match only the literal data_x folder, not
the dataXx / data2x siblings an unescaped pattern would match."""
engine, table = sqlite_object_key_table
prefix = _build_list_resources_descendant_prefix("data_x")
like = f"W001/%/{prefix}%"
not_like = f"W001/%/{prefix}%/%"
with engine.connect() as conn:
rows = conn.execute(
select(table.c.object_key).where(
table.c.object_key.like(like, escape="\\"),
~table.c.object_key.like(not_like, escape="\\"),
)
).fetchall()
matched = sorted(r[0] for r in rows)
assert matched == ["W001/U001/data_x/delta.csv"], matched
def test_sqlite_parent_path_with_slash_direct_children(sqlite_object_key_table) -> None:
"""parent_path='data/deep' returns only data/deep/* — never data/*."""
engine, table = sqlite_object_key_table
prefix = _build_list_resources_descendant_prefix("data/deep")
like = f"W001/%/{prefix}%"
not_like = f"W001/%/{prefix}%/%"
with engine.connect() as conn:
rows = conn.execute(
select(table.c.object_key).where(
table.c.object_key.like(like, escape="\\"),
~table.c.object_key.like(not_like, escape="\\"),
)
).fetchall()
matched = sorted(r[0] for r in rows)
assert matched == ["W001/U001/data/deep/nested.csv"], matched
def test_sqlite_empty_parent_path_has_no_like_filter(sqlite_object_key_table) -> None:
"""Empty parent_path → no LIKE filter → the endpoint's base WHERE only
(workspace-wide active resources). Stand-in: every row is returned."""
engine, table = sqlite_object_key_table
with engine.connect() as conn:
rows = conn.execute(select(table.c.object_key)).fetchall()
matched = sorted(r[0] for r in rows)
assert len(matched) == 8, matched
@@ -94,6 +94,14 @@ export function ScriptExplorer({
byOwner.set(item.owner_user_id, list);
}
// data-only owner(只有数据资源、没有 scripts 的用户)也要出现在分组里,
// 因为 data resources 与 scripts 共享同一棵目录树。
for (const ownerUserId of dataByOwner.keys()) {
if (!byOwner.has(ownerUserId)) {
byOwner.set(ownerUserId, []);
}
}
const groups: {
user: AuthUser | null;
scripts: ScriptItem[];
@@ -143,7 +151,7 @@ export function ScriptExplorer({
<div className="explorer__header">
<div>
<h2></h2>
<span>{scripts.length} </span>
<span>{scripts.length + dataResources.length} </span>
</div>
<div className="explorer__actions">
<button
+2 -1
View File
@@ -223,7 +223,8 @@ export function useApi(): WorkspaceBoundApi {
return useMemo<WorkspaceBoundApi>(() => ({
listScripts: (parentPath) => rawApi.listScripts(workspaceId, parentPath),
countScripts: () => rawApi.countScripts(workspaceId),
listResources: (opts) => rawApi.listResources(workspaceId, opts),
listResources: (parentPath, opts) =>
rawApi.listResources(workspaceId, parentPath, opts),
createScript: (input) => rawApi.createScript(workspaceId, input),
uploadScript: (file, parentPath, visibility) =>
rawApi.uploadScript(workspaceId, file, parentPath, visibility),
@@ -150,7 +150,7 @@ export function WorkspaceTreeGroup({
<Icon name="chevron" size={14} />
<Icon name="folder" size={17} />
<span>{title}</span>
<em>{scripts.length}</em>
<em>{scripts.length + (dataResources ?? []).length}</em>
</button>
{open && (
<div className="tree-group__items">
@@ -106,7 +106,7 @@ type State = {
setKeyword: (keyword: string) => void;
reset: () => void;
load: (silent?: boolean) => Promise<void>;
loadDataResources: () => Promise<void>;
loadDataResources: (parentPath?: string) => Promise<void>;
selectScript: (id: string | null) => void;
openTab: (id: string) => void;
closeTab: (id: string, event?: { stopPropagation: () => void }) => Promise<void>;
@@ -383,11 +383,11 @@ export const useScriptWorkspaceStore = create<State>((set, get) => {
}
},
loadDataResources: async () => {
loadDataResources: async (parentPath = "") => {
const api = requireApi();
set({ dataResourcesLoading: true });
try {
const list = await api.listResources();
const list = await api.listResources(parentPath);
set({ dataResources: Array.isArray(list) ? list : [] });
} catch {
set({ dataResources: [] });
+7 -1
View File
@@ -437,9 +437,14 @@ export type ResourceItem = {
export async function listResources(
workspaceId: string,
parentPath: string = "",
opts?: { visibility?: string; keyword?: string },
): Promise<ResourceItem[]> {
// Empty parentPath omits the query string entirely so the backend's
// workspace-wide (root-level) filter is applied symmetrically with
// non-empty paths, matching listScripts.
const parameters = new URLSearchParams();
if (parentPath) parameters.set("parent_path", parentPath);
if (opts?.visibility) parameters.set("visibility", opts.visibility);
if (opts?.keyword) parameters.set("keyword", opts.keyword);
const query = parameters.toString();
@@ -1484,7 +1489,8 @@ export type WorkspaceBoundApi = {
) => Promise<ScriptItem[]>;
countScripts: () => Promise<number>;
listResources: (
opts?: Parameters<typeof listResources>[1],
parentPath?: Parameters<typeof listResources>[1],
opts?: Parameters<typeof listResources>[2],
) => Promise<ResourceItem[]>;
createScript: (
input: Parameters<typeof createScript>[1],