From 5d49ff5e3431d4481cc1e862f13c5b07d881eaf3 Mon Sep 17 00:00:00 2001 From: "tao.chen" <93983997+taochen-ct@users.noreply.github.com> Date: Fri, 14 Aug 2026 18:22:46 +0800 Subject: [PATCH] fix: file upload error --- API.md | 3 + backend/src/backend/resources.py | 13 + backend/src/backend/scripts.py | 95 +--- backend/src/backend/services/storage.py | 9 +- backend/tests/test_resources.py | 191 ++++++++ backend/tests/test_scripts.py | 462 ++++++++++++++++++ common/src/common/db/models/storage.py | 29 +- ...9f36_workspace_tree_relative_path_index.py | 38 -- ...scripts_unique_index_include_deleted_at.py | 54 -- migrations/versions/README.md | 14 + ...p_scripts_data_resources_unique_indexes.py | 63 --- ...ne.py => e1f2a3b4c5d6_rebuild_baseline.py} | 180 +++++-- .../e5f6a7b8c9d0_ensure_demo_login.py | 140 ------ ...0e1_seed_role_permissions_and_fix_scope.py | 185 ------- 14 files changed, 872 insertions(+), 604 deletions(-) create mode 100644 backend/tests/test_resources.py create mode 100644 backend/tests/test_scripts.py delete mode 100644 migrations/versions/3ba4d8489f36_workspace_tree_relative_path_index.py delete mode 100644 migrations/versions/47a76cd261fd_scripts_unique_index_include_deleted_at.py delete mode 100644 migrations/versions/a1b2c3d4e5f6_drop_scripts_data_resources_unique_indexes.py rename migrations/versions/{d4e5f6a7b8c9_squashed_baseline.py => e1f2a3b4c5d6_rebuild_baseline.py} (85%) delete mode 100644 migrations/versions/e5f6a7b8c9d0_ensure_demo_login.py delete mode 100644 migrations/versions/f6a7b8c9d0e1_seed_role_permissions_and_fix_scope.py diff --git a/API.md b/API.md index 37f09f9..2e14b9d 100644 --- a/API.md +++ b/API.md @@ -131,6 +131,7 @@ ### 3.3 `DELETE /api/v1/workspace-directories?path=...` 删除一个目录(以及目录下当前用户拥有的所有 `Scripts`,**会触发 is_locked 校验**)。 +`StorageObjects` 行移到 trash bucket(`settings.s3_trash_bucket`),`object_status` 置为 `deleted`。 - **查询参数**: | 名 | 类型 | 必填 | 说明 | @@ -237,6 +238,7 @@ multipart/binary 形式上传大文件(走 server-proxied PUT,详见 §九)。 ### 3.9 `DELETE /api/v1/scripts/{script_id}` 软删脚本。**版本**(`Versions`)会被保留以供审计。门禁同 §3.8。 +`StorageObjects` 行移到 trash bucket(`settings.s3_trash_bucket`),`object_status` 置为 `deleted`。 ### 3.10 ScriptPayload 字段 @@ -486,6 +488,7 @@ queued ──→ running ──┬─→ succeeded | `GET` | `/api/v1/data-resources/{id}` | 详情 | | `POST` | `/api/v1/data-resources/{id}/download-url` | 生成 presigned GET URL | | `DELETE` | `/api/v1/data-resources/{id}` | 软删 | +字节归档到 trash bucket(`settings.s3_trash_bucket`)。 请求示例(上传):`POST /api/v1/data-resources/uploads` diff --git a/backend/src/backend/resources.py b/backend/src/backend/resources.py index d163d4f..5863c1e 100644 --- a/backend/src/backend/resources.py +++ b/backend/src/backend/resources.py @@ -245,6 +245,19 @@ async def bind_resource( item.visibility = payload.visibility # (description lives on DataResources, not on StorageObjects.) + existing_active = await session.scalar( + select(DataResources).where( + DataResources.workspace_id == context.workspace.workspace_id, + DataResources.resource_name == payload.resource_name, + DataResources.status == "active", + ) + ) + if existing_active is not None: + raise HTTPException( + status.HTTP_409_CONFLICT, + "a data resource with this name already exists in this workspace", + ) + existing = await session.scalar( select(DataResources).where( DataResources.storage_object_id == item.storage_object_id diff --git a/backend/src/backend/scripts.py b/backend/src/backend/scripts.py index e499c58..512804b 100644 --- a/backend/src/backend/scripts.py +++ b/backend/src/backend/scripts.py @@ -48,6 +48,8 @@ from backend.schemas import ( from backend.services.storage import ( create_download_url_payload, create_server_object_payload, + soft_delete_object, + _resolve_unique_object_key, ) router = APIRouter(tags=["scripts"]) @@ -464,6 +466,14 @@ async def create_script_record( # storage_uri points at where the replicated bytes will land. object_id = new_ulid() object_key = f"{workspace_id}/{jupyter_path}" + # Route through conflict helper to handle cross-entity collisions and + # to keep the "at most one is_deleted=0 per (backend, bucket, key)" + # invariant. For same-entity re-upload after soft-delete, the helper + # returns the original key because deleted rows are excluded from its + # "available" filter; for cross-entity collision it appends a ULID + # suffix so the new row gets a distinct object_key. + object_key = await _resolve_unique_object_key(session, object_key) + object_key_hash = hashlib.sha256(object_key.encode("utf-8")).digest() bucket_name = settings.s3_workspace_bucket relative_path = user_relative_path( context, f"{parent}/{jupyter_basename}" if parent else jupyter_basename @@ -478,7 +488,7 @@ async def create_script_record( storage_backend=settings.storage_backend, bucket_name=bucket_name, object_key=object_key, - object_key_hash=hashlib.sha256(object_key.encode("utf-8")).digest(), + object_key_hash=object_key_hash, storage_uri=build_storage_uri(bucket_name, object_key), file_name=name, file_extension=PurePosixPath(jupyter_basename).suffix.lower() or None, @@ -813,13 +823,16 @@ async def create_workspace_directory( status.HTTP_404_NOT_FOUND, "parent directory not found", ) - # Conflict check uses the unique index on (workspace_id, storage_backend, path_hash). - # A soft-deleted row at the same path can be revived; an available row is a conflict. + # Conflict check relies on SELECT ... FOR UPDATE over (workspace_id, storage_backend, + # path_hash). The underlying index is no longer unique, so conflict determination is + # fully application-level: a soft-deleted row at the same path can be revived, while + # an available row is a conflict. Concurrent inserts are no longer serialized by a + # DB unique constraint; callers must ensure the FOR UPDATE lock covers the race. existing = await session.scalar( select(StorageObjects) .where( StorageObjects.workspace_id == context.workspace.workspace_id, - StorageObjects.storage_backend == "rustfs", + StorageObjects.storage_backend == settings.storage_backend, StorageObjects.path_hash == path_hash, ) .with_for_update() @@ -969,22 +982,13 @@ async def delete_workspace_directory( .all() ) - runtime_client = request.app.state.runtime_client - workspace_id = context.workspace.workspace_id deleted_scripts = 0 for descendant in descendants: if descendant.object_type == "file": - jupyter_path = descendant.object_key.removeprefix(f"{workspace_id}/") - try: - await runtime_client.delete_file(workspace_id, name=jupyter_path) - except RuntimeClientError as exc: - if exc.status_code != 404: - logger.warning( - f"delete_workspace_directory: jupyter delete failed " - f"for file {descendant.storage_object_id}: " - f"{exc.status_code} {exc.detail}" - ) + await soft_delete_object( + descendant.storage_object_id, request, session + ) script = await session.scalar( select(Scripts).where( Scripts.workspace_id == context.workspace.workspace_id, @@ -1000,47 +1004,10 @@ async def delete_workspace_directory( descendant.is_deleted = 1 descendant.deleted_at = datetime.now(UTC).replace(tzinfo=None) else: - # Sub-directory jupyter path: derive from relative_path (new rows carry - # the user-supplied name; legacy ULID-pathed rows fall back to the - # storage_object_id for backward compatibility). - descendant_relative = descendant.relative_path or "" - descendant_user_prefix = f"workspace/{descendant.owner_user_id}" - if descendant_relative.startswith(descendant_user_prefix + "/"): - desc_segment = descendant_relative[len(descendant_user_prefix) + 1:] - else: - desc_segment = descendant.storage_object_id - try: - await runtime_client.delete_directory( - workspace_id, name=f"{descendant.owner_user_id}/{desc_segment}" - ) - except RuntimeClientError as exc: - if exc.status_code != 404: - logger.warning( - f"delete_workspace_directory: jupyter delete failed " - f"for directory {descendant.storage_object_id}: " - f"{exc.status_code} {exc.detail}" - ) descendant.object_status = "deleted" descendant.is_deleted = 1 descendant.deleted_at = datetime.now(UTC).replace(tzinfo=None) - # Target directory jupyter path: derive from the user-relative target - # path; fall back to the storage_object_id for legacy ULID-pathed rows. - target_user_prefix = f"workspace/{context.user.user_id}" - if target_relative.startswith(target_user_prefix + "/"): - target_segment = target_relative[len(target_user_prefix) + 1:] - else: - target_segment = target_ulid - try: - await runtime_client.delete_directory(workspace_id, name=f"{context.user.user_id}/{target_segment}") - except RuntimeClientError as exc: - if exc.status_code != 404: - logger.warning( - f"delete_workspace_directory: jupyter delete failed " - f"for target directory {target_ulid}: " - f"{exc.status_code} {exc.detail}" - ) - target_dir_row.object_status = "deleted" target_dir_row.is_deleted = 1 target_dir_row.deleted_at = datetime.now(UTC).replace(tzinfo=None) @@ -1271,8 +1238,8 @@ async def delete_script( session: AsyncSession = Depends(database_session), ) -> dict[str, Any]: # Load the working-copy StorageObject if it exists. Jupyter-only - # scripts may not have a StorageObjects row; fall back to the flat - # _jupyter_path() name in that case. + # scripts may not have a StorageObjects row; in that case we only + # flip the Scripts row to deleted. script, storage_object = await get_script_row( script_id, context, @@ -1286,20 +1253,10 @@ async def delete_script( is_admin=context.is_admin, ) - runtime_client = request.app.state.runtime_client - workspace_id = context.workspace.workspace_id - jupyter_path = _derive_jupyter_path( - storage_object, workspace_id, script.script_type, script.script_id - ) - try: - await runtime_client.delete_file(workspace_id, name=jupyter_path) - except RuntimeClientError as exc: - raise HTTPException( - status_code=exc.status_code, - detail=exc.detail, - ) from exc - - if storage_object: + if storage_object is not None: + await soft_delete_object( + storage_object.storage_object_id, request, session + ) storage_object.object_status = "deleted" storage_object.is_deleted = 1 storage_object.deleted_at = datetime.now(UTC).replace(tzinfo=None) diff --git a/backend/src/backend/services/storage.py b/backend/src/backend/services/storage.py index 33890cc..7575a91 100644 --- a/backend/src/backend/services/storage.py +++ b/backend/src/backend/services/storage.py @@ -117,6 +117,11 @@ async def _resolve_unique_object_key( select(StorageObjects).where( StorageObjects.object_key_hash == key_hash, StorageObjects.object_status == "available", + # Defense-in-depth: `object_status` is the canonical + # active/deleted flag, but `is_deleted` mirrors it. Filter + # both so future code that flips one without the other + # cannot bypass the active-row check. + StorageObjects.is_deleted == 0, ) ) if existing is None: @@ -531,7 +536,7 @@ async def create_download_url_payload( if item is None or item.object_status != "available": raise HTTPException(status.HTTP_404_NOT_FOUND, "object not found") if ( - item.storage_backend != "s3" + item.storage_backend != settings.storage_backend or not item.bucket_name or not item.object_key ): @@ -588,7 +593,7 @@ async def soft_delete_object( "trash_bucket": settings.s3_trash_bucket, } } - if item.storage_backend == "s3" and item.bucket_name and item.object_key: + if item.storage_backend == settings.storage_backend and item.bucket_name and item.object_key: trash_key = f"{item.bucket_name}/{item.object_key}" try: object_stores = request.app.state.object_stores diff --git a/backend/tests/test_resources.py b/backend/tests/test_resources.py new file mode 100644 index 0000000..2e534a7 --- /dev/null +++ b/backend/tests/test_resources.py @@ -0,0 +1,191 @@ +"""Unit tests for data-resource path derivation and helpers. + +These tests do not need a database because they exercise pure helpers. +""" + +from __future__ import annotations + +import datetime +from types import SimpleNamespace +from unittest.mock import AsyncMock, MagicMock + +import pytest + +from types import SimpleNamespace + +from backend.resources import compute_jupyter_relative_path, resource_payload +from backend.services.storage import _safe_file_name, _safe_path_segment + + +@pytest.mark.asyncio +async def test_bind_resource_rejects_duplicate_active_name() -> None: + """Two binds with the same resource_name but different upload_ids raise 409.""" + from backend.resources import bind_resource + + class _BindSessionMock: + def __init__(self, existing_active=None): + self._existing_active = existing_active + self._calls = 0 + + async def scalar(self, _stmt): + self._calls += 1 + # 1st scalar: UploadSessions lookup; 2nd: active name clash check. + if self._calls == 1: + return SimpleNamespace( + upload_id="01UPL0000000000000000000B", + storage_object_id="01OBJ0000000000000000000B", + workspace_id="01WS0000000000000000000A", + user_id="01USR0000000000000000000A", + ) + return self._existing_active + + async def get(self, _model, _pk): + return _make_storage_object("01WS0000000000000000000A/01USR0000000000000000000A/data.csv") + + def add(self, _obj): + pass + + async def flush(self): + pass + + async def refresh(self, _obj, attribute_names=None): + _obj.created_at = datetime.datetime.now(datetime.UTC).replace(tzinfo=None) + _obj.updated_at = datetime.datetime.now(datetime.UTC).replace(tzinfo=None) + + context = SimpleNamespace( + request_id="01REQ0000000000000000000A", + user=SimpleNamespace(user_id="01USR0000000000000000000A"), + workspace=SimpleNamespace(workspace_id="01WS0000000000000000000A"), + ) + payload = SimpleNamespace( + resource_name="data.csv", description=None, visibility="private" + ) + request = MagicMock() + + # First bind succeeds: no active resource with this name yet. + session1 = _BindSessionMock(existing_active=None) + await bind_resource( + upload_id="01UPL0000000000000000000A", + payload=payload, + request=request, + context=context, + session=session1, + ) + + # Second bind with same resource_name fails because an active row exists. + session2 = _BindSessionMock( + existing_active=SimpleNamespace(resource_id="01RES0000000000000000000A") + ) + with pytest.raises(Exception) as exc_info: + await bind_resource( + upload_id="01UPL0000000000000000000B", + payload=payload, + request=request, + context=context, + session=session2, + ) + assert exc_info.value.status_code == 409 + assert "already exists" in exc_info.value.detail + + +def test_safe_path_segment_cleans_special_characters(): + assert _safe_path_segment("train") == "train" + assert _safe_path_segment("train v1") == "train_v1" + assert _safe_path_segment("../foo") == "foo" + assert _safe_path_segment("a/b") == "a_b" + assert _safe_path_segment("...") == "untitled" + + +def test_safe_file_name_rejects_traversal_and_hidden(): + assert _safe_file_name("data.csv") == "data.csv" + with pytest.raises(Exception): + _safe_file_name("../data.csv") + with pytest.raises(Exception): + _safe_file_name(".hidden.csv") + + +def _make_resource(workspace_id: str, owner_user_id: str): + return SimpleNamespace( + resource_id="01RES0000000000000000000A", + workspace_id=workspace_id, + storage_object_id="01OBJ0000000000000000000A", + owner_user_id=owner_user_id, + resource_name="sample", + description=None, + visibility="workspace", + status="active", + created_at=datetime.datetime.now(datetime.UTC).replace(tzinfo=None), + updated_at=datetime.datetime.now(datetime.UTC).replace(tzinfo=None), + ) + + +def _make_storage_object(object_key: str): + return SimpleNamespace( + storage_object_id="01OBJ0000000000000000000A", + object_key=object_key, + file_name="data.csv", + file_extension=".csv", + mime_type="text/csv", + size_bytes=42, + content_hash="a" * 64, + object_status="available", + ) + + +def test_resource_payload_legacy_dot_resources(): + ws = "01WS00000000000000000000A" + user = "01USR000000000000000000A" + payload = resource_payload( + _make_resource(ws, user), + _make_storage_object(f"{ws}/{user}/.resources/data.csv"), + ) + assert payload["jupyter_accessible_path"] == ".resources/data.csv" + assert payload["absolute_path"].endswith(f"{ws}/{user}/.resources/data.csv") + + +def test_resource_payload_new_flat_path(): + ws = "01WS00000000000000000000A" + user = "01USR000000000000000000A" + payload = resource_payload( + _make_resource(ws, user), + _make_storage_object(f"{ws}/{user}/data.csv"), + ) + assert payload["jupyter_accessible_path"] == "data.csv" + assert payload["absolute_path"].endswith(f"{ws}/{user}/data.csv") + + +def test_resource_payload_new_nested_path(): + ws = "01WS00000000000000000000A" + user = "01USR000000000000000000A" + payload = resource_payload( + _make_resource(ws, user), + _make_storage_object(f"{ws}/{user}/train/v1/data.csv"), + ) + assert payload["jupyter_accessible_path"] == "train/v1/data.csv" + assert payload["absolute_path"].endswith(f"{ws}/{user}/train/v1/data.csv") + + +def test_compute_jupyter_relative_path_for_legacy_and_new_paths(): + # Legacy .resources path still resolves correctly. + assert compute_jupyter_relative_path("notebooks/exp.ipynb", ".resources/data.csv") == "../.resources/data.csv" + # New nested path resolves relative to the script directory. + assert compute_jupyter_relative_path("notebooks/exp.ipynb", "train/v1/data.csv") == "../train/v1/data.csv" + # Same-directory script. + assert compute_jupyter_relative_path("exp.ipynb", "data.csv") == "data.csv" + + +def test_data_resources_model_allows_duplicate_storage_object_reference() -> None: + """With uk_data_resources_object dropped, no unique index covers + storage_object_id, so multiple DataResources rows may reference the + same storage object. + """ + from common.db.models import DataResources + + index_names = {idx.name for idx in DataResources.__table__.indexes} + assert "uk_data_resources_object" not in index_names + for idx in DataResources.__table__.indexes: + if idx.unique: + cols = {c.name for c in idx.columns} + assert "storage_object_id" not in cols, ( + f"unexpected unique index {idx.name} on storage_object_id" + ) diff --git a/backend/tests/test_scripts.py b/backend/tests/test_scripts.py new file mode 100644 index 0000000..cf29288 --- /dev/null +++ b/backend/tests/test_scripts.py @@ -0,0 +1,462 @@ +"""Tests for script storage-layer behavior after unique-index removal. + +These are intentionally unit-level: they mock the async SQLAlchemy session +and the Jupyter runtime client so the suite stays fast and does not need a +live database. The tests verify the code-level guarantees that back the +"delete then re-upload" flow: + + * StorageObjects is flushed before Scripts on creation. + * All soft-delete paths flip ``is_deleted = 1`` alongside ``status='deleted'`` + / ``object_status='deleted'``. + * The ORM models no longer declare the dropped unique indexes. +""" + +from __future__ import annotations + +from types import SimpleNamespace +from unittest.mock import AsyncMock, MagicMock + +import pytest + +from common.config import settings +from common.db.models import DataResources, Scripts, StorageObjects + + +def _index_names(table) -> set[str]: + return {idx.name for idx in table.indexes} + + +def test_scripts_model_dropped_unique_indexes() -> None: + """The dropped unique indexes must not be declared on the Scripts model.""" + names = _index_names(Scripts.__table__) + assert "uk_scripts_current_object" not in names + assert "uk_scripts_workspace_name" not in names + assert "uk_scripts_workspace_name_active" not in names + + +def test_data_resources_model_dropped_unique_index() -> None: + """The dropped unique index must not be declared on DataResources.""" + names = _index_names(DataResources.__table__) + assert "uk_data_resources_object" not in names + + +def test_storage_objects_unique_index_is_active_only() -> None: + """After dropping uk_storage_bucket_key and adding + uk_storage_bucket_key_active (conditional UNIQUE via generated column), + the only physical unique index on storage_objects covers only the + `object_status='available'` subset (NULL-permissive slot for soft-deleted). + + The workspace path lookup index is non-unique; path uniqueness for + active objects is enforced by the application-level conflict check + in ``scripts.py`` and ``services.storage._resolve_unique_object_key``. + """ + indexes_by_name = {idx.name: idx for idx in StorageObjects.__table__.indexes} + assert "uk_storage_bucket_key" not in indexes_by_name + assert "uk_storage_bucket_key_active" in indexes_by_name + assert indexes_by_name["uk_storage_bucket_key_active"].unique is True + # The active-column index must include the generated column. + assert any( + col.name == "object_key_hash_active" + for col in indexes_by_name["uk_storage_bucket_key_active"].columns + ) + + +def _make_context() -> SimpleNamespace: + return SimpleNamespace( + request_id="01REQ0000000000000000000A", + user=SimpleNamespace(user_id="01USR0000000000000000000A"), + workspace=SimpleNamespace(workspace_id="01WS0000000000000000000A"), + is_admin=False, + ) + + +def _make_request() -> MagicMock: + request = MagicMock() + request.app.state.runtime_client.delete_file = AsyncMock(return_value=None) + return request + + +class _AsyncSessionMock: + """Minimal AsyncSession stand-in that records flush/add order.""" + + def __init__(self) -> None: + self.added: list[object] = [] + self.flush_order: list[str] = [] + self._refreshed: list[object] = [] + + def add(self, obj: object) -> None: + self.added.append(obj) + + async def flush(self) -> None: + # Record the kind of object that triggered this flush. + self.flush_order.append(type(self.added[-1]).__name__) + + async def refresh(self, obj: object, attribute_names: list[str] | None = None) -> None: + self._refreshed.append(obj) + + async def scalar(self, *_args, **_kwargs) -> None: + return None + + +@pytest.mark.asyncio +async def test_create_script_record_flushes_storage_object_before_script() -> None: + """StorageObjects must flush first so path conflicts surface early.""" + from backend.scripts import create_script_record + + session = _AsyncSessionMock() + request = _make_request() + context = _make_context() + + runtime_client = request.app.state.runtime_client + runtime_client.ensure_directory = AsyncMock(return_value=None) + runtime_client.create_notebook = AsyncMock(return_value={"name": "test.ipynb"}) + + script, storage_object = await create_script_record( + name="test.ipynb", + script_type="notebook", + content=b'{"cells": []}', + visibility="private", + parent_path=None, + request=request, + context=context, + session=session, + ) + + assert isinstance(storage_object, StorageObjects) + assert isinstance(script, Scripts) + assert session.flush_order == ["StorageObjects", "Scripts"] + assert session._refreshed == [storage_object, script] + + +@pytest.mark.asyncio +async def test_create_script_record_storage_object_flush_failure_does_not_add_script() -> None: + """If the StorageObjects flush fails, the Scripts row must never be added.""" + from backend.scripts import create_script_record + + class FailingSession(_AsyncSessionMock): + async def flush(self) -> None: + self.flush_order.append(type(self.added[-1]).__name__) + raise RuntimeError("duplicate path") + + session = FailingSession() + request = _make_request() + context = _make_context() + + runtime_client = request.app.state.runtime_client + runtime_client.ensure_directory = AsyncMock(return_value=None) + runtime_client.create_notebook = AsyncMock(return_value={"name": "test.ipynb"}) + + with pytest.raises(RuntimeError, match="duplicate path"): + await create_script_record( + name="test.ipynb", + script_type="notebook", + content=b'{"cells": []}', + visibility="private", + parent_path=None, + request=request, + context=context, + session=session, + ) + + # Only the StorageObjects row was ever added. + assert len(session.added) == 1 + assert isinstance(session.added[0], StorageObjects) + # The Jupyter cleanup was attempted. + request.app.state.runtime_client.delete_file.assert_awaited_once() + + +@pytest.mark.asyncio +async def test_create_script_record_allows_reupload_after_delete() -> None: + """Without uk_scripts_workspace_name_active, re-uploading a script with + the same name after the previous one was soft-deleted succeeds. + """ + from backend.scripts import create_script_record + + session = _AsyncSessionMock() + request = _make_request() + context = _make_context() + + runtime_client = request.app.state.runtime_client + runtime_client.ensure_directory = AsyncMock(return_value=None) + runtime_client.create_notebook = AsyncMock(return_value={"name": "test.ipynb"}) + + script1, storage_object1 = await create_script_record( + name="test.ipynb", + script_type="notebook", + content=b'{"cells": []}', + visibility="private", + parent_path=None, + request=request, + context=context, + session=session, + ) + + # Simulate the first script and its storage object being soft-deleted. + script1.status = "deleted" + script1.is_deleted = 1 + storage_object1.object_status = "deleted" + storage_object1.is_deleted = 1 + + # Re-upload with the same name: the Scripts layer must not reject it. + script2, storage_object2 = await create_script_record( + name="test.ipynb", + script_type="notebook", + content=b'{"cells": []}', + visibility="private", + parent_path=None, + request=request, + context=context, + session=session, + ) + + assert script2.script_name == script1.script_name + assert script2.script_id != script1.script_id + assert script2.status == "active" + assert storage_object2.storage_object_id != storage_object1.storage_object_id + # Each upload flushes its own StorageObject then Script. + assert session.flush_order == [ + "StorageObjects", + "Scripts", + "StorageObjects", + "Scripts", + ] + + +@pytest.mark.asyncio +async def test_create_script_after_soft_delete_does_not_conflict() -> None: + """Resurrection regression: with uk_storage_bucket_key_active being a + conditional UNIQUE (NULL when object_status != 'available'), re-uploading + a script whose previous StorageObjects row is soft-deleted does NOT + raise IntegrityError — the generated column is NULL for the deleted row, + so it does not occupy the UNIQUE slot. + """ + from backend.scripts import create_script_record + + session = _AsyncSessionMock() + request = _make_request() + context = _make_context() + + runtime_client = request.app.state.runtime_client + runtime_client.ensure_directory = AsyncMock(return_value=None) + runtime_client.create_notebook = AsyncMock(return_value={"name": "x.ipynb"}) + + s1, so1 = await create_script_record( + name="x.ipynb", script_type="notebook", content=b'{"cells":[]}', + visibility="private", parent_path=None, + request=request, context=context, session=session, + ) + s1.status = "deleted" + s1.is_deleted = 1 + so1.object_status = "deleted" + so1.is_deleted = 1 + + s2, so2 = await create_script_record( + name="x.ipynb", script_type="notebook", content=b'{"cells":[]}', + visibility="private", parent_path=None, + request=request, context=context, session=session, + ) + assert s2.script_id != s1.script_id + assert so2.storage_object_id != so1.storage_object_id + # Same object_key (no ULID suffix) — soft-deleted row excluded from + # _resolve_unique_object_key's "available" filter. + assert so2.object_key == so1.object_key + + +def test_scripts_model_allows_duplicate_active_name() -> None: + """With uk_scripts_workspace_name_active dropped, no unique index covers + (workspace_id, script_name, script_type), so duplicate active names are + allowed at the ORM level. + """ + for idx in Scripts.__table__.indexes: + if idx.unique: + cols = {c.name for c in idx.columns} + assert not ( + {"workspace_id", "script_name", "script_type"} <= cols + ), f"unexpected unique index {idx.name} on script name" + + +def test_scripts_model_allows_duplicate_current_object_id() -> None: + """With uk_scripts_current_object dropped, no unique index covers + current_object_id, so multiple scripts may point to the same storage object. + """ + names = _index_names(Scripts.__table__) + assert "uk_scripts_current_object" not in names + for idx in Scripts.__table__.indexes: + if idx.unique and len(idx.columns) == 1: + assert "current_object_id" not in {c.name for c in idx.columns} + + +def _script_row() -> Scripts: + return Scripts( + script_id="01SCR0000000000000000000A", + workspace_id="01WS0000000000000000000A", + current_object_id="01OBJ0000000000000000000A", + owner_user_id="01USR0000000000000000000A", + script_name="test.ipynb", + script_type="notebook", + visibility="private", + status="active", + ) + + +def _storage_object_row() -> StorageObjects: + return StorageObjects( + storage_object_id="01OBJ0000000000000000000A", + workspace_id="01WS0000000000000000000A", + object_type="file", + usage_type="working_copy", + # 与生产构造器一致（common/db/models/storage.py 注释 "s3"） + storage_backend=settings.storage_backend, + storage_uri="s3://bucket/key", + file_name="test.ipynb", + created_by="01USR0000000000000000000A", + ) + + +@pytest.mark.asyncio +async def test_delete_script_route_sets_is_deleted(monkeypatch: pytest.MonkeyPatch) -> None: + """Soft-deleting a script via the route handler flips is_deleted=1.""" + from backend.scripts import delete_script + + script = _script_row() + storage_object = _storage_object_row() + + request = _make_request() + context = _make_context() + session = AsyncMock() + + async def _fake_get_script_row( + _script_id: str, + _context: SimpleNamespace, + _session: AsyncMock, + *, + for_update: bool = False, + allow_missing_storage_object: bool = False, + ) -> tuple[Scripts, StorageObjects]: + return script, storage_object + + monkeypatch.setattr("backend.scripts.get_script_row", _fake_get_script_row) + mock_soft_delete = AsyncMock( + return_value={ + "data": { + "storage_object_id": storage_object.storage_object_id, + "object_status": "deleted", + "trash_key": "trash/bucket/key", + "trash_bucket": "trash", + } + } + ) + monkeypatch.setattr("backend.scripts.soft_delete_object", mock_soft_delete) + + result = await delete_script( + script_id=script.script_id, + request=request, + context=context, + session=session, + ) + + assert script.status == "deleted" + assert script.is_deleted == 1 + assert script.deleted_at is not None + assert storage_object.object_status == "deleted" + assert storage_object.is_deleted == 1 + assert storage_object.deleted_at is not None + assert result["data"]["status"] == "deleted" + mock_soft_delete.assert_awaited_once_with( + storage_object.storage_object_id, request, session + ) + + +@pytest.mark.asyncio +async def test_delete_resource_sets_is_deleted(monkeypatch: pytest.MonkeyPatch) -> None: + """Soft-deleting a data resource must write is_deleted=1 on the row.""" + from backend.resources import delete_resource + + resource = DataResources( + resource_id="01RES0000000000000000000A", + workspace_id="01WS0000000000000000000A", + storage_object_id="01OBJ0000000000000000000A", + owner_user_id="01USR0000000000000000000A", + resource_name="data.csv", + visibility="workspace", # visible so can_view passes + status="active", + ) + storage_object = _storage_object_row() + + request = _make_request() + context = _make_context() + + class _Result: + def one_or_none(self): + return (resource, storage_object) + + session = AsyncMock() + session.execute = AsyncMock(return_value=_Result()) + + with monkeypatch.context() as mp: + mp.setattr( + "backend.resources.soft_delete_object", + AsyncMock(return_value={"data": {}}), + ) + result = await delete_resource( + resource_id=resource.resource_id, + request=request, + context=context, + session=session, + ) + + assert resource.status == "deleted" + assert resource.is_deleted == 1 + assert resource.deleted_at is not None + assert result["data"]["status"] == "deleted" + + +@pytest.mark.asyncio +async def test_soft_delete_object_sets_is_deleted() -> None: + """The shared helper must flip is_deleted=1 on StorageObjects.""" + from backend.services.storage import soft_delete_object + + item = _storage_object_row() + item.storage_backend = "local" # 测的是"非默认后端短路 trash、只翻 DB"路径 + item.object_status = "available" + + request = MagicMock() + request.app.state.object_stores = {} + + session = AsyncMock() + session.scalar = AsyncMock(return_value=item) + + result = await soft_delete_object( + storage_object_id=item.storage_object_id, + request=request, + session=session, + ) + + assert item.object_status == "deleted" + assert item.is_deleted == 1 + assert item.deleted_at is not None + assert result["data"]["object_status"] == "deleted" + + +@pytest.mark.asyncio +async def test_jupyter_check_notebook_lock_ignores_deleted_scripts() -> None: + """``is_deleted == 0`` filter must hide deleted notebooks from Jupyter checks.""" + from backend.jupyter import check_notebook_is_locked + + session = AsyncMock() + session.execute = AsyncMock() + session.execute.return_value.one_or_none = MagicMock(return_value=None) + + result = await check_notebook_is_locked( + workspace_id="01WS0000000000000000000A", + notebook_path="test.ipynb", + user_id="01USR0000000000000000000A", + session=session, + ) + + assert result is False + # Verify the query carries the is_deleted filter. + call = session.execute.await_args + statement = call[0][0] + compiled = str(statement.compile(compile_kwargs={"literal_binds": True})) + assert "is_deleted" in compiled diff --git a/common/src/common/db/models/storage.py b/common/src/common/db/models/storage.py index 2a04b55..07ca099 100644 --- a/common/src/common/db/models/storage.py +++ b/common/src/common/db/models/storage.py @@ -1,7 +1,7 @@ import datetime from typing import Optional -from sqlalchemy import BINARY, Index, JSON, String, text +from sqlalchemy import BINARY, Computed, Index, JSON, String, text from sqlalchemy.dialects.mysql import BIGINT, CHAR, DATETIME, TINYINT from sqlalchemy.orm import Mapped, mapped_column @@ -22,6 +22,12 @@ class StorageObjects(Base): Index("idx_storage_content_hash", "content_hash"), Index("idx_storage_owner", "owner_user_id", "object_status"), Index("idx_storage_parent", "parent_object_id"), + Index( + "idx_storage_workspace_path", + "workspace_id", + "storage_backend", + "path_hash", + ), Index( "idx_storage_workspace_usage", "workspace_id", @@ -29,17 +35,10 @@ class StorageObjects(Base): "object_status", ), Index( - "uk_storage_bucket_key", + "uk_storage_bucket_key_active", "storage_backend", "bucket_name", - "object_key_hash", - unique=True, - ), - Index( - "uk_storage_workspace_path", - "workspace_id", - "storage_backend", - "path_hash", + "object_key_hash_active", unique=True, ), {"comment": "Workspace 文件和 RustFS 对象的统一元数据"}, @@ -56,7 +55,7 @@ class StorageObjects(Base): comment="working_copy/public_script/data_resource/version_artifact/snapshot/run_log/run_result", ) storage_backend: Mapped[str] = mapped_column( - String(16), nullable=False, comment="rustfs" + String(16), nullable=False, comment="s3" ) storage_uri: Mapped[str] = mapped_column(String(1500), nullable=False) file_name: Mapped[str] = mapped_column(String(255), nullable=False) @@ -100,6 +99,14 @@ class StorageObjects(Base): object_key_hash: Mapped[Optional[bytes]] = mapped_column( BINARY(32), comment="SHA-256(object_key)，由应用写入" ) + object_key_hash_active: Mapped[Optional[bytes]] = mapped_column( + BINARY(32), + Computed( + "CASE WHEN object_status = 'available' THEN object_key_hash ELSE NULL END", + persisted=False, + ), + comment="VIRTUAL generated column used by uk_storage_bucket_key_active", + ) file_extension: Mapped[Optional[str]] = mapped_column(String(32)) mime_type: Mapped[Optional[str]] = mapped_column(String(255)) content_hash: Mapped[Optional[str]] = mapped_column( diff --git a/migrations/versions/3ba4d8489f36_workspace_tree_relative_path_index.py b/migrations/versions/3ba4d8489f36_workspace_tree_relative_path_index.py deleted file mode 100644 index c947d84..0000000 --- a/migrations/versions/3ba4d8489f36_workspace_tree_relative_path_index.py +++ /dev/null @@ -1,38 +0,0 @@ -"""Add workspace tree relative_path index. - -The ``list_workspace_directories`` endpoint filters by -``relative_path`` prefixes inside a workspace. A composite index on -``(workspace_id, relative_path(255))`` avoids scanning all rows for a -workspace when listing a subdirectory. - -Revision ID: 3ba4d8489f36 -Revises: f6a7b8c9d0e1 -Create Date: 2026-08-12 -""" - -from collections.abc import Sequence - -from alembic import op -import sqlalchemy as sa - -# revision identifiers, used by Alembic. -revision: str = "3ba4d8489f36" -down_revision: str | Sequence[str] | None = "f6a7b8c9d0e1" -branch_labels: str | Sequence[str] | None = None -depends_on: str | Sequence[str] | None = None - - -def upgrade() -> None: - op.create_index( - "idx_storage_workspace_relative_path", - "storage_objects", - [sa.text("`workspace_id`"), sa.text("`relative_path`(255)")], - unique=False, - ) - - -def downgrade() -> None: - op.drop_index( - "idx_storage_workspace_relative_path", - table_name="storage_objects", - ) diff --git a/migrations/versions/47a76cd261fd_scripts_unique_index_include_deleted_at.py b/migrations/versions/47a76cd261fd_scripts_unique_index_include_deleted_at.py deleted file mode 100644 index eca75b6..0000000 --- a/migrations/versions/47a76cd261fd_scripts_unique_index_include_deleted_at.py +++ /dev/null @@ -1,54 +0,0 @@ -"""Make the scripts workspace/name/type unique index soft-delete aware. - -The old unique index ``uk_scripts_workspace_name`` on -``(workspace_id, script_name, script_type)`` blocked re-uploading a -script after it had been soft-deleted, because the deleted row was still -part of the index. - -Replace it with ``uk_scripts_workspace_name_active`` on -``(workspace_id, script_name, script_type, deleted_at)``. In MySQL a -unique index treats ``NULL`` values as distinct, so a new active row -(``deleted_at IS NULL``) no longer conflicts with a previously deleted -row (``deleted_at IS NOT NULL``), while two active rows with the same -name still conflict as expected. - -Revision ID: 47a76cd261fd -Revises: 3ba4d8489f36 -Create Date: 2026-08-14 -""" - -from collections.abc import Sequence - -from alembic import op - -# revision identifiers, used by Alembic. -revision: str = "47a76cd261fd" -down_revision: str | Sequence[str] | None = "3ba4d8489f36" -branch_labels: str | Sequence[str] | None = None -depends_on: str | Sequence[str] | None = None - - -def upgrade() -> None: - op.drop_index( - "uk_scripts_workspace_name", - table_name="scripts", - ) - op.create_index( - "uk_scripts_workspace_name_active", - "scripts", - ["workspace_id", "script_name", "script_type", "deleted_at"], - unique=True, - ) - - -def downgrade() -> None: - op.drop_index( - "uk_scripts_workspace_name_active", - table_name="scripts", - ) - op.create_index( - "uk_scripts_workspace_name", - "scripts", - ["workspace_id", "script_name", "script_type"], - unique=True, - ) diff --git a/migrations/versions/README.md b/migrations/versions/README.md index 28f2e12..062dee7 100644 --- a/migrations/versions/README.md +++ b/migrations/versions/README.md @@ -5,3 +5,17 @@ - 已发布迁移不得直接修改。 - 新迁移必须同时提供可执行的 `upgrade()` 和 `downgrade()`。 - 自动生成后必须检查数据类型、约束、索引和执行顺序。 +# Migration Revisions + +该目录只保存经过评审和验证的 Alembic 迁移版本。 + +## Baseline + +- `e1f2a3b4c5d6_rebuild_baseline.py` — 2026-08-14 重建的基线迁移，包含完整 schema + seed。 +- 之前的 8 个迁移文件已合并并删除；新环境从此基线开始。 + +## Rules + +- 已发布迁移不得直接修改。 +- 新迁移必须同时提供可执行的 `upgrade()` 和 `downgrade()`。 +- 自动生成后必须检查数据类型、约束、索引和执行顺序。 diff --git a/migrations/versions/a1b2c3d4e5f6_drop_scripts_data_resources_unique_indexes.py b/migrations/versions/a1b2c3d4e5f6_drop_scripts_data_resources_unique_indexes.py deleted file mode 100644 index 5f882a1..0000000 --- a/migrations/versions/a1b2c3d4e5f6_drop_scripts_data_resources_unique_indexes.py +++ /dev/null @@ -1,63 +0,0 @@ -"""Drop unique indexes on Scripts and DataResources. - -StorageObjects now carries the physical uniqueness guarantees: - - ``uk_storage_bucket_key`` - - ``uk_storage_workspace_path`` - -Scripts and DataResources therefore no longer need their own unique -indexes on ``current_object_id`` / ``storage_object_id``, and the -soft-delete-aware ``uk_scripts_workspace_name_active`` index is also -removed. Re-uploading a previously soft-deleted script or resource is -allowed because the StorageObjects layer enforces path uniqueness only -for active objects. - -Revision ID: a1b2c3d4e5f6 -Revises: 47a76cd261fd -Create Date: 2026-08-14 -""" - -from collections.abc import Sequence - -from alembic import op - -# revision identifiers, used by Alembic. -revision: str = "a1b2c3d4e5f6" -down_revision: str | Sequence[str] | None = "47a76cd261fd" -branch_labels: str | Sequence[str] | None = None -depends_on: str | Sequence[str] | None = None - - -def upgrade() -> None: - op.drop_index( - "uk_scripts_workspace_name_active", - table_name="scripts", - ) - op.drop_index( - "uk_scripts_current_object", - table_name="scripts", - ) - op.drop_index( - "uk_data_resources_object", - table_name="data_resources", - ) - - -def downgrade() -> None: - op.create_index( - "uk_scripts_workspace_name_active", - "scripts", - ["workspace_id", "script_name", "script_type", "deleted_at"], - unique=True, - ) - op.create_index( - "uk_scripts_current_object", - "scripts", - ["current_object_id"], - unique=True, - ) - op.create_index( - "uk_data_resources_object", - "data_resources", - ["storage_object_id"], - unique=True, - ) diff --git a/migrations/versions/d4e5f6a7b8c9_squashed_baseline.py b/migrations/versions/e1f2a3b4c5d6_rebuild_baseline.py similarity index 85% rename from migrations/versions/d4e5f6a7b8c9_squashed_baseline.py rename to migrations/versions/e1f2a3b4c5d6_rebuild_baseline.py index ff02650..bc7caaa 100644 --- a/migrations/versions/d4e5f6a7b8c9_squashed_baseline.py +++ b/migrations/versions/e1f2a3b4c5d6_rebuild_baseline.py @@ -1,22 +1,12 @@ -"""squashed baseline — full schema + seed data in one migration +"""rebuild baseline — full schema + seed data in one migration -Single baseline migration combining the previous 5-step chain: - - 8d86e2f82860 initial baseline (20 tables) - b71c4f2a9d10 seed demo users / workspaces / roles / members - 9a1b2c3d4e5f enable password login for seeded users - a2b3c4d5e6f7 add storage_objects.trash_key - c3d4e5f6a7b8 add upload_sessions object-metadata columns - -The column additions from the later migrations are folded directly into -the CREATE TABLE statements, so this file is a from-scratch schema. - -Revision ID: d4e5f6a7b8c9 +Revision ID: e1f2a3b4c5d6 Revises: (none) -Create Date: 2026-08-05 +Create Date: 2026-08-14 """ from collections.abc import Sequence +import hashlib import os from alembic import op @@ -26,14 +16,70 @@ from sqlalchemy.dialects import mysql from common.auth.passwords import hash_password # revision identifiers, used by Alembic. -revision: str = "d4e5f6a7b8c9" +revision: str = "e1f2a3b4c5d6" down_revision: str | Sequence[str] | None = None branch_labels: str | Sequence[str] | None = None depends_on: str | Sequence[str] | None = None +# ── seed constants ─────────────────────────────────────────────── +DISABLED_PASSWORD = "demo-login-disabled" +SEEDED_USERS = ( + ( + "00000000000000000000000001", + "admin", + "Admin", + "0000000000000000000000000A", + ), +) + +ADMIN_ROLE_ID = "0000000000000000000000000A" +DEVELOPER_ROLE_ID = "0000000000000000000000000B" + +PERMISSIONS: list[tuple[str, str, str]] = [ + # (permission_code, permission_name, module_code) + ("dashboard.view", "查看工作台", "dashboard"), + ("script.build", "构建脚本", "script"), + ("script.public.manage", "管理公共脚本", "script"), + ("schedule.own", "管理本人调度", "schedule"), + ("schedule.all", "管理全部调度", "schedule"), + ("experiment.own", "管理本人实验", "experiment"), + ("experiment.all", "管理全部实验", "experiment"), + ("resource.personal", "管理个人资源", "resource"), + ("resource.public.upload", "上传公共资源", "resource"), + ("resource.public.manage", "管理公共资源", "resource"), + ("system.view", "查看系统管理", "system"), + ("system.manage", "管理系统配置", "system"), +] + +# developer gets *.own + personal resource only; no system.*, no *.all. +DEVELOPER_PERMISSION_CODES: list[str] = [ + "dashboard.view", + "script.build", + "script.public.manage", + "schedule.own", + "experiment.own", + "resource.personal", +] + +ADMIN_PERMISSION_CODES: list[str] = [code for code, _, _ in PERMISSIONS] + +CROCKFORD_BASE32 = "0123456789ABCDEFGHJKMNPQRSTVWXYZ" + +def _deterministic_permission_id(code: str) -> str: + """Stable 26-char ULID-shaped id derived from permission_code.""" + digest = hashlib.sha256( + f"model-platform-permission-v1:{code}".encode("utf-8") + ).digest() + value = int.from_bytes(b"\x00" * 6 + digest[:10], byteorder="big") + encoded = ["0"] * 26 + for index in range(25, -1, -1): + encoded[index] = CROCKFORD_BASE32[value & 31] + value >>= 5 + return "".join(encoded) def upgrade() -> None: - """Full schema from scratch (all 20 tables, current model state).""" + """Upgrade schema.""" + # ### commands auto generated by Alembic - please adjust! ### op.create_table('consumer_inbox', sa.Column('consumer_name', sa.String(length=128), nullable=False), sa.Column('event_id', mysql.CHAR(length=26), nullable=False), @@ -67,7 +113,6 @@ def upgrade() -> None: ) op.create_index('idx_data_resources_owner', 'data_resources', ['owner_user_id', 'status'], unique=False) op.create_index('idx_data_resources_workspace', 'data_resources', ['workspace_id', 'visibility', 'status'], unique=False) - op.create_index('uk_data_resources_object', 'data_resources', ['storage_object_id'], unique=True) op.create_table('outbox_events', sa.Column('event_id', mysql.CHAR(length=26), nullable=False), sa.Column('aggregate_type', sa.String(length=64), nullable=False), @@ -179,7 +224,7 @@ def upgrade() -> None: sa.Column('node_key', sa.String(length=64), nullable=False, comment='画布内稳定标识'), sa.Column('node_name', sa.String(length=255), nullable=False), sa.Column('versions_id', mysql.CHAR(length=26), nullable=False), - sa.Column('python_version', sa.String(length=8), nullable=False, server_default=sa.text("'3.12'"), comment='节点执行 Python 版本（3.8/3.10/3.12）'), + sa.Column('python_version', sa.String(length=8), server_default=sa.text("'3.12'"), nullable=False, comment='节点执行 Python 版本（3.8/3.10/3.12）'), sa.Column('timeout_seconds', mysql.INTEGER(), server_default=sa.text('600'), nullable=False), sa.Column('retry_count', mysql.INTEGER(), server_default=sa.text('0'), nullable=False), sa.Column('retry_interval_sec', mysql.INTEGER(), server_default=sa.text('5'), nullable=False), @@ -274,8 +319,6 @@ def upgrade() -> None: ) op.create_index('idx_scripts_owner', 'scripts', ['owner_user_id', 'status'], unique=False) op.create_index('idx_scripts_workspace', 'scripts', ['workspace_id', 'script_type', 'visibility', 'status'], unique=False) - op.create_index('uk_scripts_current_object', 'scripts', ['current_object_id'], unique=True) - op.create_index('uk_scripts_workspace_name', 'scripts', ['workspace_id', 'script_name', 'script_type'], unique=True) op.create_table('storage_objects', sa.Column('storage_object_id', mysql.CHAR(length=26), nullable=False), sa.Column('workspace_id', mysql.CHAR(length=26), nullable=False), @@ -298,23 +341,25 @@ def upgrade() -> None: sa.Column('bucket_name', sa.String(length=128), nullable=True), sa.Column('object_key', sa.String(length=1024), nullable=True), sa.Column('object_key_hash', sa.BINARY(length=32), nullable=True, comment='SHA-256(object_key)，由应用写入'), + sa.Column('object_key_hash_active', sa.BINARY(length=32), sa.Computed("CASE WHEN object_status = 'available' THEN object_key_hash ELSE NULL END", persisted=False), nullable=True, comment='VIRTUAL generated column used by uk_storage_bucket_key_active'), sa.Column('file_extension', sa.String(length=32), nullable=True), sa.Column('mime_type', sa.String(length=255), nullable=True), sa.Column('content_hash', mysql.CHAR(length=64), nullable=True, comment='SHA-256 hex'), sa.Column('object_etag', sa.String(length=255), nullable=True), - sa.Column('trash_key', sa.String(length=1100), nullable=True, comment='Path inside the trash bucket where soft-deleted bytes are stored'), sa.Column('is_deleted', mysql.TINYINT(display_width=1), server_default=sa.text('0'), nullable=False), sa.Column('deleted_at', mysql.DATETIME(fsp=3), nullable=True), + sa.Column('trash_key', sa.String(length=1100), nullable=True, comment="Path inside the trash bucket where the soft-deleted bytes live. Format: '{source_bucket}/{object_key}' so a restore is a same-key copy back to the source bucket. NULL while the row is still available."), sa.PrimaryKeyConstraint('storage_object_id'), - comment='Workspace 文件和 S3 对象的统一元数据' + comment='Workspace 文件和 RustFS 对象的统一元数据' ) op.create_index('fk_storage_created_by', 'storage_objects', ['created_by'], unique=False) op.create_index('idx_storage_content_hash', 'storage_objects', ['content_hash'], unique=False) op.create_index('idx_storage_owner', 'storage_objects', ['owner_user_id', 'object_status'], unique=False) op.create_index('idx_storage_parent', 'storage_objects', ['parent_object_id'], unique=False) + op.create_index('idx_storage_workspace_path', 'storage_objects', ['workspace_id', 'storage_backend', 'path_hash'], unique=False) op.create_index('idx_storage_workspace_usage', 'storage_objects', ['workspace_id', 'usage_type', 'object_status'], unique=False) - op.create_index('uk_storage_bucket_key', 'storage_objects', ['storage_backend', 'bucket_name', 'object_key_hash'], unique=True) - op.create_index('uk_storage_workspace_path', 'storage_objects', ['workspace_id', 'storage_backend', 'path_hash'], unique=True) + op.create_index('uk_storage_bucket_key_active', 'storage_objects', ['storage_backend', 'bucket_name', 'object_key_hash_active'], unique=True) + op.create_index('idx_storage_workspace_relative_path', 'storage_objects', [sa.text('`workspace_id`'), sa.text('`relative_path`(255)')], unique=False) op.create_table('upload_sessions', sa.Column('upload_id', mysql.CHAR(length=26), nullable=False), sa.Column('workspace_id', mysql.CHAR(length=26), nullable=False), @@ -333,15 +378,14 @@ def upgrade() -> None: sa.Column('content_type', sa.String(length=255), nullable=True), sa.Column('storage_object_id', mysql.CHAR(length=26), nullable=True), sa.Column('completed_at', mysql.DATETIME(fsp=3), nullable=True), - # Object-metadata columns added by c3d4e5f6a7b8 (server-proxied upload). - sa.Column('file_name', sa.String(length=255), nullable=False, server_default=''), - sa.Column('usage_type', sa.String(length=32), nullable=False, server_default='working_copy', comment='data_resource/version_artifact/snapshot/run_log/run_result/working_copy/public_script'), - sa.Column('visibility', sa.String(length=16), nullable=False, server_default='private', comment='private/workspace/public'), - sa.Column('is_immutable', mysql.TINYINT(display_width=1), nullable=False, server_default='0'), + sa.Column('file_name', sa.String(length=255), server_default='', nullable=False), + sa.Column('usage_type', sa.String(length=32), server_default=sa.text("'working_copy'"), nullable=False, comment='data_resource/version_artifact/snapshot/run_log/run_result/working_copy/public_script'), + sa.Column('visibility', sa.String(length=16), server_default=sa.text("'private'"), nullable=False, comment='private/workspace/public'), + sa.Column('is_immutable', mysql.TINYINT(display_width=1), server_default=sa.text('0'), nullable=False), sa.Column('is_deleted', mysql.TINYINT(display_width=1), server_default=sa.text('0'), nullable=False), sa.Column('deleted_at', mysql.DATETIME(fsp=3), nullable=True), sa.PrimaryKeyConstraint('upload_id'), - comment='S3 上传会话；URL 本身不持久化' + comment='RustFS 预签名上传会话；URL 本身不持久化' ) op.create_index('fk_upload_sessions_storage_object', 'upload_sessions', ['storage_object_id'], unique=False) op.create_index('fk_upload_sessions_user', 'upload_sessions', ['user_id'], unique=False) @@ -434,10 +478,9 @@ def upgrade() -> None: op.create_index('fk_workspaces_created_by', 'workspaces', ['created_by'], unique=False) op.create_index('idx_workspaces_status', 'workspaces', ['status'], unique=False) op.create_index('uk_workspaces_code', 'workspaces', ['workspace_code'], unique=True) + # ### end Alembic commands ### - # ── seed data (from b71c4f2a9d10) ────────────────────────────── - ADMIN_ROLE_ID = "0000000000000000000000000A" - DEVELOPER_ROLE_ID = "0000000000000000000000000B" + # ── seed data ──────────────────────────────────────────────────── ADMIN_USER_ID = "00000000000000000000000001" DEFAULT_WORKSPACE_ID = "00000000000000000000000002" USERS = ( @@ -491,7 +534,7 @@ def upgrade() -> None: "role_id": ADMIN_ROLE_ID, "role_code": "admin", "role_name": "管理员", - "role_scope": "workspace", + "role_scope": "platform", "is_builtin": 1, "description": "Self-hosted workspace administrator", }, @@ -499,7 +542,7 @@ def upgrade() -> None: "role_id": DEVELOPER_ROLE_ID, "role_code": "developer", "role_name": "开发人员", - "role_scope": "workspace", + "role_scope": "platform", "is_builtin": 1, "description": "Self-hosted workspace developer", }, @@ -549,7 +592,7 @@ def upgrade() -> None: ], ) - # ── enable demo password login (from 9a1b2c3d4e5f) ───────────── + # ── enable demo password login (from e5f6a7b8c9d0) ────────────── password = os.environ.get("INITIAL_ADMIN_PASSWORD", "admin12345") seeded_user_ids = (ADMIN_USER_ID,) users_update = sa.table( @@ -564,9 +607,63 @@ def upgrade() -> None: .values(password_hash=hash_password(password)) ) + # ── seed platform permissions (from f6a7b8c9d0e1) ─────────────── + op.execute( + "UPDATE roles SET role_scope = 'platform' " + "WHERE role_code IN ('admin', 'developer') AND is_deleted = 0" + ) + + permissions_table = sa.table( + "permissions", + sa.column("permission_id", sa.CHAR(26)), + sa.column("permission_code", sa.String(128)), + sa.column("permission_name", sa.String(100)), + sa.column("module_code", sa.String(64)), + sa.column("description", sa.String(500)), + ) + perm_id_by_code: dict[str, str] = {} + rows: list[dict[str, str]] = [] + for code, name, module in PERMISSIONS: + pid = _deterministic_permission_id(code) + perm_id_by_code[code] = pid + rows.append( + { + "permission_id": pid, + "permission_code": code, + "permission_name": name, + "module_code": module, + "description": f"platform 菜单权限：{name}", + } + ) + op.bulk_insert(permissions_table, rows) + + role_permissions_table = sa.table( + "role_permissions", + sa.column("role_id", sa.CHAR(26)), + sa.column("permission_id", sa.CHAR(26)), + ) + rp_rows: list[dict[str, str]] = [] + for code in ADMIN_PERMISSION_CODES: + rp_rows.append( + { + "role_id": ADMIN_ROLE_ID, + "permission_id": perm_id_by_code[code], + } + ) + for code in DEVELOPER_PERMISSION_CODES: + rp_rows.append( + { + "role_id": DEVELOPER_ROLE_ID, + "permission_id": perm_id_by_code[code], + } + ) + op.bulk_insert(role_permissions_table, rp_rows) + + # ### end Alembic commands ### def downgrade() -> None: - """Drop everything (reverse of upgrade).""" + """Downgrade schema.""" + # ### commands auto generated by Alembic - please adjust! ### op.drop_index('uk_workspaces_code', table_name='workspaces') op.drop_index('idx_workspaces_status', table_name='workspaces') op.drop_index('fk_workspaces_created_by', table_name='workspaces') @@ -593,16 +690,15 @@ def downgrade() -> None: op.drop_index('fk_upload_sessions_user', table_name='upload_sessions') op.drop_index('fk_upload_sessions_storage_object', table_name='upload_sessions') op.drop_table('upload_sessions') - op.drop_index('uk_storage_workspace_path', table_name='storage_objects') - op.drop_index('uk_storage_bucket_key', table_name='storage_objects') + op.drop_index('uk_storage_bucket_key_active', table_name='storage_objects') op.drop_index('idx_storage_workspace_usage', table_name='storage_objects') + op.drop_index('idx_storage_workspace_relative_path', table_name='storage_objects') + op.drop_index('idx_storage_workspace_path', table_name='storage_objects') op.drop_index('idx_storage_parent', table_name='storage_objects') op.drop_index('idx_storage_owner', table_name='storage_objects') op.drop_index('idx_storage_content_hash', table_name='storage_objects') op.drop_index('fk_storage_created_by', table_name='storage_objects') op.drop_table('storage_objects') - op.drop_index('uk_scripts_workspace_name', table_name='scripts') - op.drop_index('uk_scripts_current_object', table_name='scripts') op.drop_index('idx_scripts_workspace', table_name='scripts') op.drop_index('idx_scripts_owner', table_name='scripts') op.drop_table('scripts') @@ -644,9 +740,9 @@ def downgrade() -> None: op.drop_index('idx_outbox_idempotency', table_name='outbox_events') op.drop_index('idx_outbox_aggregate', table_name='outbox_events') op.drop_table('outbox_events') - op.drop_index('uk_data_resources_object', table_name='data_resources') op.drop_index('idx_data_resources_workspace', table_name='data_resources') op.drop_index('idx_data_resources_owner', table_name='data_resources') op.drop_table('data_resources') op.drop_index('idx_consumer_inbox_status', table_name='consumer_inbox') op.drop_table('consumer_inbox') + # ### end Alembic commands ### diff --git a/migrations/versions/e5f6a7b8c9d0_ensure_demo_login.py b/migrations/versions/e5f6a7b8c9d0_ensure_demo_login.py deleted file mode 100644 index 5ee6453..0000000 --- a/migrations/versions/e5f6a7b8c9d0_ensure_demo_login.py +++ /dev/null @@ -1,140 +0,0 @@ -"""ensure the self-hosted demo login remains available - -Revision ID: e5f6a7b8c9d0 -Revises: d4e5f6a7b8c9 -Create Date: 2026-08-05 15:31:00 -""" - -from collections.abc import Sequence -import os - -from alembic import op -import sqlalchemy as sa -from sqlalchemy.dialects import mysql - -from common.auth.passwords import hash_password - - -revision: str = "e5f6a7b8c9d0" -down_revision: str | Sequence[str] | None = "d4e5f6a7b8c9" -branch_labels: str | Sequence[str] | None = None -depends_on: str | Sequence[str] | None = None - - -DISABLED_PASSWORD = "demo-login-disabled" -SEEDED_USERS = ( - ( - "00000000000000000000000001", - "admin", - "Admin", - "0000000000000000000000000A", - ), -) - - -def _create_users_table() -> None: - op.create_table( - "users", - sa.Column("user_id", mysql.CHAR(length=26), nullable=False), - sa.Column("username", sa.String(length=64), nullable=False), - sa.Column("display_name", sa.String(length=100), nullable=False), - sa.Column("password_hash", sa.String(length=255), nullable=False), - sa.Column( - "status", - sa.String(length=16), - server_default=sa.text("'active'"), - nullable=False, - comment="active/disabled/locked", - ), - sa.Column( - "created_at", - mysql.DATETIME(fsp=3), - server_default=sa.text("CURRENT_TIMESTAMP(3)"), - nullable=False, - ), - sa.Column( - "updated_at", - mysql.DATETIME(fsp=3), - server_default=sa.text( - "CURRENT_TIMESTAMP(3) ON UPDATE CURRENT_TIMESTAMP(3)" - ), - nullable=False, - ), - sa.Column("email", sa.String(length=255), nullable=True), - sa.Column("platform_role_id", mysql.CHAR(length=26), nullable=True), - sa.Column("avatar_uri", sa.String(length=1000), nullable=True), - sa.Column("last_login_at", mysql.DATETIME(fsp=3), nullable=True), - sa.Column( - "is_deleted", - mysql.TINYINT(display_width=1), - server_default=sa.text("0"), - nullable=False, - ), - sa.Column("deleted_at", mysql.DATETIME(fsp=3), nullable=True), - sa.PrimaryKeyConstraint("user_id"), - comment="平台用户", - ) - op.create_index("fk_users_platform_role", "users", ["platform_role_id"]) - op.create_index("idx_users_status", "users", ["status"]) - op.create_index("uk_users_email", "users", ["email"], unique=True) - op.create_index("uk_users_username", "users", ["username"], unique=True) - - -def upgrade() -> None: - connection = op.get_bind() - if not sa.inspect(connection).has_table("users"): - _create_users_table() - - users = sa.table( - "users", - sa.column("user_id", sa.String), - sa.column("username", sa.String), - sa.column("display_name", sa.String), - sa.column("password_hash", sa.String), - sa.column("status", sa.String), - sa.column("email", sa.String), - sa.column("platform_role_id", sa.String), - ) - existing = { - row.username: row.password_hash - for row in connection.execute( - sa.select(users.c.username, users.c.password_hash).where( - users.c.username.in_([user[1] for user in SEEDED_USERS]) - ) - ) - } - password_hash = hash_password( - os.environ.get("INITIAL_ADMIN_PASSWORD", "admin12345") - ) - - for user_id, username, display_name, role_id in SEEDED_USERS: - if username not in existing: - connection.execute( - users.insert().values( - user_id=user_id, - username=username, - display_name=display_name, - password_hash=password_hash, - status="active", - email=f"{username}@model-platform.local", - platform_role_id=role_id, - ) - ) - continue - - if existing[username] in {None, "", DISABLED_PASSWORD}: - connection.execute( - users.update() - .where(users.c.username == username) - .values(password_hash=password_hash) - ) - - connection.execute( - users.update() - .where(users.c.username == "admin") - .values(status="active") - ) - - -def downgrade() -> None: - """Do not remove or disable accounts that may contain user data.""" diff --git a/migrations/versions/f6a7b8c9d0e1_seed_role_permissions_and_fix_scope.py b/migrations/versions/f6a7b8c9d0e1_seed_role_permissions_and_fix_scope.py deleted file mode 100644 index bffed63..0000000 --- a/migrations/versions/f6a7b8c9d0e1_seed_role_permissions_and_fix_scope.py +++ /dev/null @@ -1,185 +0,0 @@ -"""Seed platform permissions + role_permissions, fix admin/developer role_scope. - -The squashed baseline (d4e5f6a7b8c9) ships the Permissions and -RolePermissions tables empty, and seeds admin/developer with -role_scope='workspace' (an early mistake; the codebase elsewhere treats -both as platform-scoped — see backend/platform.py::system_admin_context -and common/auth/membership.py::resolve_is_system_admin). This migration: - - 1. UPDATE roles SET role_scope='platform' for admin/developer rows. - 2. INSERT 12 permission rows covering the menu groups the frontend - consumes (dashboard / script / schedule / experiment / resource / - system). - 3. INSERT role_permissions join rows: admin gets all 12, developer - gets the 6 `*.own` / personal-resource codes. - -Caveats (read before re-running): - -* ``permission_id`` is derived from a sha256 of the code with the salt - prefix ``model-platform-permission-v1:``. The legacy - ``migrations/data/migrate_system_json.py`` script uses a different - salt (``model-platform-v1:permission:``), so the same - ``permission_code`` maps to a DIFFERENT ``permission_id`` between the - two paths. The legacy script's ``existing.get(permission_code)`` check - keeps the row count correct (it reuses the live row by code), so this - is not a crash; the IDs only matter if a downstream system ever - cross-references by deterministic ID, which nothing does today. - -* ``downgrade()`` is a SOFT delete (``is_deleted=1``). Running - ``alembic downgrade`` followed by ``alembic upgrade`` will collide on - the ``permission_id`` PRIMARY KEY — downgrade is a one-way trip on - any environment that has run this migration. The role_scope fix is - not reverted on downgrade (app code already keys off `platform`). - -* If the legacy one-off ``migrations/data/migrate_system_json.py`` is - ever run AFTER this migration on the same database, its - ``existing.get(permission_code)`` check will keep counts correct but - reuses our rows; running it BEFORE this migration would cause - ``uk_permissions_code`` collisions on upgrade. Run this migration - first on a fresh database. - -Revision ID: f6a7b8c9d0e1 -Revises: e5f6a7b8c9d0 (ensure_demo_login) -Create Date: 2026-08-07 -""" - -from __future__ import annotations - -import hashlib -from collections.abc import Sequence - -from alembic import op -import sqlalchemy as sa - - -CROCKFORD_BASE32 = "0123456789ABCDEFGHJKMNPQRSTVWXYZ" - -# Hardcoded to match d4e5f6a7b8c9_squashed_baseline.py seed values, so the -# role_permissions join rows below resolve against the right role rows. -ADMIN_ROLE_ID = "0000000000000000000000000A" -DEVELOPER_ROLE_ID = "0000000000000000000000000B" - -PERMISSIONS: list[tuple[str, str, str]] = [ - # (permission_code, permission_name, module_code) - ("dashboard.view", "查看工作台", "dashboard"), - ("script.build", "构建脚本", "script"), - ("script.public.manage", "管理公共脚本", "script"), - ("schedule.own", "管理本人调度", "schedule"), - ("schedule.all", "管理全部调度", "schedule"), - ("experiment.own", "管理本人实验", "experiment"), - ("experiment.all", "管理全部实验", "experiment"), - ("resource.personal", "管理个人资源", "resource"), - ("resource.public.upload", "上传公共资源", "resource"), - ("resource.public.manage", "管理公共资源", "resource"), - ("system.view", "查看系统管理", "system"), - ("system.manage", "管理系统配置", "system"), -] - -# developer gets *.own + personal resource only; no system.*, no *.all. -DEVELOPER_PERMISSION_CODES: list[str] = [ - "dashboard.view", - "script.build", - "script.public.manage", - "schedule.own", - "experiment.own", - "resource.personal", -] - -ADMIN_PERMISSION_CODES: list[str] = [code for code, _, _ in PERMISSIONS] - - -def _deterministic_permission_id(code: str) -> str: - """Stable 26-char ULID-shaped id derived from permission_code. - - Mirrors ``migrations/data/migrate_system_json.py::deterministic_legacy_ulid`` - so re-running this migration (or running it after the legacy data - migrator) keeps identical IDs for the same code. - """ - digest = hashlib.sha256( - f"model-platform-permission-v1:{code}".encode("utf-8") - ).digest() - value = int.from_bytes(b"\x00" * 6 + digest[:10], byteorder="big") - encoded = ["0"] * 26 - for index in range(25, -1, -1): - encoded[index] = CROCKFORD_BASE32[value & 31] - value >>= 5 - return "".join(encoded) - - -# revision identifiers, used by Alembic. -revision: str = "f6a7b8c9d0e1" -down_revision: str | Sequence[str] | None = "e5f6a7b8c9d0" - - -def upgrade() -> None: - # (1) role_scope fix: admin/developer were seeded as 'workspace' but - # platform.py::system_admin_context treats them as platform-scoped. - op.execute( - "UPDATE roles SET role_scope = 'platform' " - "WHERE role_code IN ('admin', 'developer') AND is_deleted = 0" - ) - - # (2) Permissions rows. - permissions_table = sa.table( - "permissions", - sa.column("permission_id", sa.CHAR(26)), - sa.column("permission_code", sa.String(128)), - sa.column("permission_name", sa.String(100)), - sa.column("module_code", sa.String(64)), - sa.column("description", sa.String(500)), - ) - perm_id_by_code: dict[str, str] = {} - rows: list[dict[str, str]] = [] - for code, name, module in PERMISSIONS: - pid = _deterministic_permission_id(code) - perm_id_by_code[code] = pid - rows.append( - { - "permission_id": pid, - "permission_code": code, - "permission_name": name, - "module_code": module, - "description": f"platform 菜单权限：{name}", - } - ) - op.bulk_insert(permissions_table, rows) - - # (3) role_permissions join rows. - role_permissions_table = sa.table( - "role_permissions", - sa.column("role_id", sa.CHAR(26)), - sa.column("permission_id", sa.CHAR(26)), - ) - rp_rows: list[dict[str, str]] = [] - for code in ADMIN_PERMISSION_CODES: - rp_rows.append( - { - "role_id": ADMIN_ROLE_ID, - "permission_id": perm_id_by_code[code], - } - ) - for code in DEVELOPER_PERMISSION_CODES: - rp_rows.append( - { - "role_id": DEVELOPER_ROLE_ID, - "permission_id": perm_id_by_code[code], - } - ) - op.bulk_insert(role_permissions_table, rp_rows) - - -def downgrade() -> None: - # Soft-delete what we inserted. The role_scope fix is intentionally - # NOT reverted — the only safe direction is platform, since - # application code already keys off it. - code_list_sql = "(" + ",".join(f"'{c}'" for c in ADMIN_PERMISSION_CODES) + ")" - op.execute( - "UPDATE role_permissions " - "SET is_deleted = 1, deleted_at = CURRENT_TIMESTAMP(3) " - f"WHERE permission_id IN (SELECT permission_id FROM permissions " - f"WHERE is_deleted = 0 AND permission_code IN {code_list_sql})" - ) - op.execute( - "UPDATE permissions SET is_deleted = 1, deleted_at = CURRENT_TIMESTAMP(3) " - f"WHERE is_deleted = 0 AND permission_code IN {code_list_sql}" - ) \ No newline at end of file