tao.chen/model-platform
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

fix: file upload error

Browse Source
This commit is contained in:
tao.chen
2026-08-14 18:22:46 +08:00
parent c65d6dc684
commit 5d49ff5e34
14 changed files with 872 additions and 604 deletions
Download Patch File Download Diff File Expand all files Collapse all files
API.md
+3
View File
@@ -131,6 +131,7 @@
### 3.3 `DELETE /api/v1/workspace-directories?path=...`
删除一个目录(以及目录下当前用户拥有的所有 `Scripts`,**会触发 is_locked 校验**)。
`StorageObjects` 行移到 trash bucket(`settings.s3_trash_bucket`),`object_status` 置为 `deleted`。
- **查询参数**:
| 名 | 类型 | 必填 | 说明 |
@@ -237,6 +238,7 @@ multipart/binary 形式上传大文件(走 server-proxied PUT,详见 §九)。
### 3.9 `DELETE /api/v1/scripts/{script_id}`
软删脚本。**版本**(`Versions`)会被保留以供审计。门禁同 §3.8。
`StorageObjects` 行移到 trash bucket(`settings.s3_trash_bucket`),`object_status` 置为 `deleted`。
### 3.10 ScriptPayload 字段
@@ -486,6 +488,7 @@ queued ──→ running ──┬─→ succeeded
| `GET` | `/api/v1/data-resources/{id}` | 详情 |
| `POST` | `/api/v1/data-resources/{id}/download-url` | 生成 presigned GET URL |
| `DELETE` | `/api/v1/data-resources/{id}` | 软删 |
字节归档到 trash bucket(`settings.s3_trash_bucket`)。
请求示例(上传):`POST /api/v1/data-resources/uploads`
backend/src/backend/resources.py
+13
View File
@@ -245,6 +245,19 @@ async def bind_resource(
item.visibility = payload.visibility
# (description lives on DataResources, not on StorageObjects.)
existing_active = await session.scalar(
select(DataResources).where(
DataResources.workspace_id == context.workspace.workspace_id,
DataResources.resource_name == payload.resource_name,
DataResources.status == "active",
)
)
if existing_active is not None:
raise HTTPException(
status.HTTP_409_CONFLICT,
"a data resource with this name already exists in this workspace",
)
existing = await session.scalar(
select(DataResources).where(
DataResources.storage_object_id == item.storage_object_id
backend/src/backend/scripts.py
+26 -69
View File
@@ -48,6 +48,8 @@ from backend.schemas import (
from backend.services.storage import (
create_download_url_payload,
create_server_object_payload,
soft_delete_object,
_resolve_unique_object_key,
)
router = APIRouter(tags=["scripts"])
@@ -464,6 +466,14 @@ async def create_script_record(
# storage_uri points at where the replicated bytes will land.
object_id = new_ulid()
object_key = f"{workspace_id}/{jupyter_path}"
# Route through conflict helper to handle cross-entity collisions and
# to keep the "at most one is_deleted=0 per (backend, bucket, key)"
# invariant. For same-entity re-upload after soft-delete, the helper
# returns the original key because deleted rows are excluded from its
# "available" filter; for cross-entity collision it appends a ULID
# suffix so the new row gets a distinct object_key.
object_key = await _resolve_unique_object_key(session, object_key)
object_key_hash = hashlib.sha256(object_key.encode("utf-8")).digest()
bucket_name = settings.s3_workspace_bucket
relative_path = user_relative_path(
context, f"{parent}/{jupyter_basename}" if parent else jupyter_basename
@@ -478,7 +488,7 @@ async def create_script_record(
storage_backend=settings.storage_backend,
bucket_name=bucket_name,
object_key=object_key,
object_key_hash=hashlib.sha256(object_key.encode("utf-8")).digest(),
object_key_hash=object_key_hash,
storage_uri=build_storage_uri(bucket_name, object_key),
file_name=name,
file_extension=PurePosixPath(jupyter_basename).suffix.lower() or None,
@@ -813,13 +823,16 @@ async def create_workspace_directory(
status.HTTP_404_NOT_FOUND,
"parent directory not found",
)
# Conflict check uses the unique index on (workspace_id, storage_backend, path_hash).
# A soft-deleted row at the same path can be revived; an available row is a conflict.
# Conflict check relies on SELECT ... FOR UPDATE over (workspace_id, storage_backend,
# path_hash). The underlying index is no longer unique, so conflict determination is
# fully application-level: a soft-deleted row at the same path can be revived, while
# an available row is a conflict. Concurrent inserts are no longer serialized by a
# DB unique constraint; callers must ensure the FOR UPDATE lock covers the race.
existing = await session.scalar(
select(StorageObjects)
.where(
StorageObjects.workspace_id == context.workspace.workspace_id,
StorageObjects.storage_backend == "rustfs",
StorageObjects.storage_backend == settings.storage_backend,
StorageObjects.path_hash == path_hash,
)
.with_for_update()
@@ -969,22 +982,13 @@ async def delete_workspace_directory(
.all()
)
runtime_client = request.app.state.runtime_client
workspace_id = context.workspace.workspace_id
deleted_scripts = 0
for descendant in descendants:
if descendant.object_type == "file":
jupyter_path = descendant.object_key.removeprefix(f"{workspace_id}/")
try:
await runtime_client.delete_file(workspace_id, name=jupyter_path)
except RuntimeClientError as exc:
if exc.status_code != 404:
logger.warning(
f"delete_workspace_directory: jupyter delete failed "
f"for file {descendant.storage_object_id}: "
f"{exc.status_code} {exc.detail}"
)
await soft_delete_object(
descendant.storage_object_id, request, session
)
script = await session.scalar(
select(Scripts).where(
Scripts.workspace_id == context.workspace.workspace_id,
@@ -1000,47 +1004,10 @@ async def delete_workspace_directory(
descendant.is_deleted = 1
descendant.deleted_at = datetime.now(UTC).replace(tzinfo=None)
else:
# Sub-directory jupyter path: derive from relative_path (new rows carry
# the user-supplied name; legacy ULID-pathed rows fall back to the
# storage_object_id for backward compatibility).
descendant_relative = descendant.relative_path or ""
descendant_user_prefix = f"workspace/{descendant.owner_user_id}"
if descendant_relative.startswith(descendant_user_prefix + "/"):
desc_segment = descendant_relative[len(descendant_user_prefix) + 1:]
else:
desc_segment = descendant.storage_object_id
try:
await runtime_client.delete_directory(
workspace_id, name=f"{descendant.owner_user_id}/{desc_segment}"
)
except RuntimeClientError as exc:
if exc.status_code != 404:
logger.warning(
f"delete_workspace_directory: jupyter delete failed "
f"for directory {descendant.storage_object_id}: "
f"{exc.status_code} {exc.detail}"
)
descendant.object_status = "deleted"
descendant.is_deleted = 1
descendant.deleted_at = datetime.now(UTC).replace(tzinfo=None)
# Target directory jupyter path: derive from the user-relative target
# path; fall back to the storage_object_id for legacy ULID-pathed rows.
target_user_prefix = f"workspace/{context.user.user_id}"
if target_relative.startswith(target_user_prefix + "/"):
target_segment = target_relative[len(target_user_prefix) + 1:]
else:
target_segment = target_ulid
try:
await runtime_client.delete_directory(workspace_id, name=f"{context.user.user_id}/{target_segment}")
except RuntimeClientError as exc:
if exc.status_code != 404:
logger.warning(
f"delete_workspace_directory: jupyter delete failed "
f"for target directory {target_ulid}: "
f"{exc.status_code} {exc.detail}"
)
target_dir_row.object_status = "deleted"
target_dir_row.is_deleted = 1
target_dir_row.deleted_at = datetime.now(UTC).replace(tzinfo=None)
@@ -1271,8 +1238,8 @@ async def delete_script(
session: AsyncSession = Depends(database_session),
) -> dict[str, Any]:
# Load the working-copy StorageObject if it exists. Jupyter-only
# scripts may not have a StorageObjects row; fall back to the flat
# _jupyter_path() name in that case.
# scripts may not have a StorageObjects row; in that case we only
# flip the Scripts row to deleted.
script, storage_object = await get_script_row(
script_id,
context,
@@ -1286,20 +1253,10 @@ async def delete_script(
is_admin=context.is_admin,
)
runtime_client = request.app.state.runtime_client
workspace_id = context.workspace.workspace_id
jupyter_path = _derive_jupyter_path(
storage_object, workspace_id, script.script_type, script.script_id
)
try:
await runtime_client.delete_file(workspace_id, name=jupyter_path)
except RuntimeClientError as exc:
raise HTTPException(
status_code=exc.status_code,
detail=exc.detail,
) from exc
if storage_object:
if storage_object is not None:
await soft_delete_object(
storage_object.storage_object_id, request, session
)
storage_object.object_status = "deleted"
storage_object.is_deleted = 1
storage_object.deleted_at = datetime.now(UTC).replace(tzinfo=None)
backend/src/backend/services/storage.py
+7 -2
View File
@@ -117,6 +117,11 @@ async def _resolve_unique_object_key(
select(StorageObjects).where(
StorageObjects.object_key_hash == key_hash,
StorageObjects.object_status == "available",
# Defense-in-depth: `object_status` is the canonical
# active/deleted flag, but `is_deleted` mirrors it. Filter
# both so future code that flips one without the other
# cannot bypass the active-row check.
StorageObjects.is_deleted == 0,
)
)
if existing is None:
@@ -531,7 +536,7 @@ async def create_download_url_payload(
if item is None or item.object_status != "available":
raise HTTPException(status.HTTP_404_NOT_FOUND, "object not found")
if (
item.storage_backend != "s3"
item.storage_backend != settings.storage_backend
or not item.bucket_name
or not item.object_key
):
@@ -588,7 +593,7 @@ async def soft_delete_object(
"trash_bucket": settings.s3_trash_bucket,
}
}
if item.storage_backend == "s3" and item.bucket_name and item.object_key:
if item.storage_backend == settings.storage_backend and item.bucket_name and item.object_key:
trash_key = f"{item.bucket_name}/{item.object_key}"
try:
object_stores = request.app.state.object_stores
backend/tests/test_resources.py
+191
View File
@@ -0,0 +1,191 @@
"""Unit tests for data-resource path derivation and helpers.
These tests do not need a database because they exercise pure helpers.
"""
from __future__ import annotations
import datetime
from types import SimpleNamespace
from unittest.mock import AsyncMock, MagicMock
import pytest
from types import SimpleNamespace
from backend.resources import compute_jupyter_relative_path, resource_payload
from backend.services.storage import _safe_file_name, _safe_path_segment
@pytest.mark.asyncio
async def test_bind_resource_rejects_duplicate_active_name() -> None:
"""Two binds with the same resource_name but different upload_ids raise 409."""
from backend.resources import bind_resource
class _BindSessionMock:
def __init__(self, existing_active=None):
self._existing_active = existing_active
self._calls = 0
async def scalar(self, _stmt):
self._calls += 1
# 1st scalar: UploadSessions lookup; 2nd: active name clash check.
if self._calls == 1:
return SimpleNamespace(
upload_id="01UPL0000000000000000000B",
storage_object_id="01OBJ0000000000000000000B",
workspace_id="01WS0000000000000000000A",
user_id="01USR0000000000000000000A",
)
return self._existing_active
async def get(self, _model, _pk):
return _make_storage_object("01WS0000000000000000000A/01USR0000000000000000000A/data.csv")
def add(self, _obj):
pass
async def flush(self):
pass
async def refresh(self, _obj, attribute_names=None):
_obj.created_at = datetime.datetime.now(datetime.UTC).replace(tzinfo=None)
_obj.updated_at = datetime.datetime.now(datetime.UTC).replace(tzinfo=None)
context = SimpleNamespace(
request_id="01REQ0000000000000000000A",
user=SimpleNamespace(user_id="01USR0000000000000000000A"),
workspace=SimpleNamespace(workspace_id="01WS0000000000000000000A"),
)
payload = SimpleNamespace(
resource_name="data.csv", description=None, visibility="private"
)
request = MagicMock()
# First bind succeeds: no active resource with this name yet.
session1 = _BindSessionMock(existing_active=None)
await bind_resource(
upload_id="01UPL0000000000000000000A",
payload=payload,
request=request,
context=context,
session=session1,
)
# Second bind with same resource_name fails because an active row exists.
session2 = _BindSessionMock(
existing_active=SimpleNamespace(resource_id="01RES0000000000000000000A")
)
with pytest.raises(Exception) as exc_info:
await bind_resource(
upload_id="01UPL0000000000000000000B",
payload=payload,
request=request,
context=context,
session=session2,
)
assert exc_info.value.status_code == 409
assert "already exists" in exc_info.value.detail
def test_safe_path_segment_cleans_special_characters():
assert _safe_path_segment("train") == "train"
assert _safe_path_segment("train v1") == "train_v1"
assert _safe_path_segment("../foo") == "foo"
assert _safe_path_segment("a/b") == "a_b"
assert _safe_path_segment("...") == "untitled"
def test_safe_file_name_rejects_traversal_and_hidden():
assert _safe_file_name("data.csv") == "data.csv"
with pytest.raises(Exception):
_safe_file_name("../data.csv")
with pytest.raises(Exception):
_safe_file_name(".hidden.csv")
def _make_resource(workspace_id: str, owner_user_id: str):
return SimpleNamespace(
resource_id="01RES0000000000000000000A",
workspace_id=workspace_id,
storage_object_id="01OBJ0000000000000000000A",
owner_user_id=owner_user_id,
resource_name="sample",
description=None,
visibility="workspace",
status="active",
created_at=datetime.datetime.now(datetime.UTC).replace(tzinfo=None),
updated_at=datetime.datetime.now(datetime.UTC).replace(tzinfo=None),
)
def _make_storage_object(object_key: str):
return SimpleNamespace(
storage_object_id="01OBJ0000000000000000000A",
object_key=object_key,
file_name="data.csv",
file_extension=".csv",
mime_type="text/csv",
size_bytes=42,
content_hash="a" * 64,
object_status="available",
)
def test_resource_payload_legacy_dot_resources():
ws = "01WS00000000000000000000A"
user = "01USR000000000000000000A"
payload = resource_payload(
_make_resource(ws, user),
_make_storage_object(f"{ws}/{user}/.resources/data.csv"),
)
assert payload["jupyter_accessible_path"] == ".resources/data.csv"
assert payload["absolute_path"].endswith(f"{ws}/{user}/.resources/data.csv")
def test_resource_payload_new_flat_path():
ws = "01WS00000000000000000000A"
user = "01USR000000000000000000A"
payload = resource_payload(
_make_resource(ws, user),
_make_storage_object(f"{ws}/{user}/data.csv"),
)
assert payload["jupyter_accessible_path"] == "data.csv"
assert payload["absolute_path"].endswith(f"{ws}/{user}/data.csv")
def test_resource_payload_new_nested_path():
ws = "01WS00000000000000000000A"
user = "01USR000000000000000000A"
payload = resource_payload(
_make_resource(ws, user),
_make_storage_object(f"{ws}/{user}/train/v1/data.csv"),
)
assert payload["jupyter_accessible_path"] == "train/v1/data.csv"
assert payload["absolute_path"].endswith(f"{ws}/{user}/train/v1/data.csv")
def test_compute_jupyter_relative_path_for_legacy_and_new_paths():
# Legacy .resources path still resolves correctly.
assert compute_jupyter_relative_path("notebooks/exp.ipynb", ".resources/data.csv") == "../.resources/data.csv"
# New nested path resolves relative to the script directory.
assert compute_jupyter_relative_path("notebooks/exp.ipynb", "train/v1/data.csv") == "../train/v1/data.csv"
# Same-directory script.
assert compute_jupyter_relative_path("exp.ipynb", "data.csv") == "data.csv"
def test_data_resources_model_allows_duplicate_storage_object_reference() -> None:
"""With uk_data_resources_object dropped, no unique index covers
storage_object_id, so multiple DataResources rows may reference the
same storage object.
"""
from common.db.models import DataResources
index_names = {idx.name for idx in DataResources.__table__.indexes}
assert "uk_data_resources_object" not in index_names
for idx in DataResources.__table__.indexes:
if idx.unique:
cols = {c.name for c in idx.columns}
assert "storage_object_id" not in cols, (
f"unexpected unique index {idx.name} on storage_object_id"
)
backend/tests/test_scripts.py
+462
View File
@@ -0,0 +1,462 @@
"""Tests for script storage-layer behavior after unique-index removal.
These are intentionally unit-level: they mock the async SQLAlchemy session
and the Jupyter runtime client so the suite stays fast and does not need a
live database. The tests verify the code-level guarantees that back the
"delete then re-upload" flow:
* StorageObjects is flushed before Scripts on creation.
* All soft-delete paths flip ``is_deleted = 1`` alongside ``status='deleted'``
/ ``object_status='deleted'``.
* The ORM models no longer declare the dropped unique indexes.
"""
from __future__ import annotations
from types import SimpleNamespace
from unittest.mock import AsyncMock, MagicMock
import pytest
from common.config import settings
from common.db.models import DataResources, Scripts, StorageObjects
def _index_names(table) -> set[str]:
return {idx.name for idx in table.indexes}
def test_scripts_model_dropped_unique_indexes() -> None:
"""The dropped unique indexes must not be declared on the Scripts model."""
names = _index_names(Scripts.__table__)
assert "uk_scripts_current_object" not in names
assert "uk_scripts_workspace_name" not in names
assert "uk_scripts_workspace_name_active" not in names
def test_data_resources_model_dropped_unique_index() -> None:
"""The dropped unique index must not be declared on DataResources."""
names = _index_names(DataResources.__table__)
assert "uk_data_resources_object" not in names
def test_storage_objects_unique_index_is_active_only() -> None:
"""After dropping uk_storage_bucket_key and adding
uk_storage_bucket_key_active (conditional UNIQUE via generated column),
the only physical unique index on storage_objects covers only the
`object_status='available'` subset (NULL-permissive slot for soft-deleted).
The workspace path lookup index is non-unique; path uniqueness for
active objects is enforced by the application-level conflict check
in ``scripts.py`` and ``services.storage._resolve_unique_object_key``.
"""
indexes_by_name = {idx.name: idx for idx in StorageObjects.__table__.indexes}
assert "uk_storage_bucket_key" not in indexes_by_name
assert "uk_storage_bucket_key_active" in indexes_by_name
assert indexes_by_name["uk_storage_bucket_key_active"].unique is True
# The active-column index must include the generated column.
assert any(
col.name == "object_key_hash_active"
for col in indexes_by_name["uk_storage_bucket_key_active"].columns
)
def _make_context() -> SimpleNamespace:
return SimpleNamespace(
request_id="01REQ0000000000000000000A",
user=SimpleNamespace(user_id="01USR0000000000000000000A"),
workspace=SimpleNamespace(workspace_id="01WS0000000000000000000A"),
is_admin=False,
)
def _make_request() -> MagicMock:
request = MagicMock()
request.app.state.runtime_client.delete_file = AsyncMock(return_value=None)
return request
class _AsyncSessionMock:
"""Minimal AsyncSession stand-in that records flush/add order."""
def __init__(self) -> None:
self.added: list[object] = []
self.flush_order: list[str] = []
self._refreshed: list[object] = []
def add(self, obj: object) -> None:
self.added.append(obj)
async def flush(self) -> None:
# Record the kind of object that triggered this flush.
self.flush_order.append(type(self.added[-1]).__name__)
async def refresh(self, obj: object, attribute_names: list[str] | None = None) -> None:
self._refreshed.append(obj)
async def scalar(self, *_args, **_kwargs) -> None:
return None
@pytest.mark.asyncio
async def test_create_script_record_flushes_storage_object_before_script() -> None:
"""StorageObjects must flush first so path conflicts surface early."""
from backend.scripts import create_script_record
session = _AsyncSessionMock()
request = _make_request()
context = _make_context()
runtime_client = request.app.state.runtime_client
runtime_client.ensure_directory = AsyncMock(return_value=None)
runtime_client.create_notebook = AsyncMock(return_value={"name": "test.ipynb"})
script, storage_object = await create_script_record(
name="test.ipynb",
script_type="notebook",
content=b'{"cells": []}',
visibility="private",
parent_path=None,
request=request,
context=context,
session=session,
)
assert isinstance(storage_object, StorageObjects)
assert isinstance(script, Scripts)
assert session.flush_order == ["StorageObjects", "Scripts"]
assert session._refreshed == [storage_object, script]
@pytest.mark.asyncio
async def test_create_script_record_storage_object_flush_failure_does_not_add_script() -> None:
"""If the StorageObjects flush fails, the Scripts row must never be added."""
from backend.scripts import create_script_record
class FailingSession(_AsyncSessionMock):
async def flush(self) -> None:
self.flush_order.append(type(self.added[-1]).__name__)
raise RuntimeError("duplicate path")
session = FailingSession()
request = _make_request()
context = _make_context()
runtime_client = request.app.state.runtime_client
runtime_client.ensure_directory = AsyncMock(return_value=None)
runtime_client.create_notebook = AsyncMock(return_value={"name": "test.ipynb"})
with pytest.raises(RuntimeError, match="duplicate path"):
await create_script_record(
name="test.ipynb",
script_type="notebook",
content=b'{"cells": []}',
visibility="private",
parent_path=None,
request=request,
context=context,
session=session,
)
# Only the StorageObjects row was ever added.
assert len(session.added) == 1
assert isinstance(session.added[0], StorageObjects)
# The Jupyter cleanup was attempted.
request.app.state.runtime_client.delete_file.assert_awaited_once()
@pytest.mark.asyncio
async def test_create_script_record_allows_reupload_after_delete() -> None:
"""Without uk_scripts_workspace_name_active, re-uploading a script with
the same name after the previous one was soft-deleted succeeds.
"""
from backend.scripts import create_script_record
session = _AsyncSessionMock()
request = _make_request()
context = _make_context()
runtime_client = request.app.state.runtime_client
runtime_client.ensure_directory = AsyncMock(return_value=None)
runtime_client.create_notebook = AsyncMock(return_value={"name": "test.ipynb"})
script1, storage_object1 = await create_script_record(
name="test.ipynb",
script_type="notebook",
content=b'{"cells": []}',
visibility="private",
parent_path=None,
request=request,
context=context,
session=session,
)
# Simulate the first script and its storage object being soft-deleted.
script1.status = "deleted"
script1.is_deleted = 1
storage_object1.object_status = "deleted"
storage_object1.is_deleted = 1
# Re-upload with the same name: the Scripts layer must not reject it.
script2, storage_object2 = await create_script_record(
name="test.ipynb",
script_type="notebook",
content=b'{"cells": []}',
visibility="private",
parent_path=None,
request=request,
context=context,
session=session,
)
assert script2.script_name == script1.script_name
assert script2.script_id != script1.script_id
assert script2.status == "active"
assert storage_object2.storage_object_id != storage_object1.storage_object_id
# Each upload flushes its own StorageObject then Script.
assert session.flush_order == [
"StorageObjects",
"Scripts",
"StorageObjects",
"Scripts",
]
@pytest.mark.asyncio
async def test_create_script_after_soft_delete_does_not_conflict() -> None:
"""Resurrection regression: with uk_storage_bucket_key_active being a
conditional UNIQUE (NULL when object_status != 'available'), re-uploading
a script whose previous StorageObjects row is soft-deleted does NOT
raise IntegrityError — the generated column is NULL for the deleted row,
so it does not occupy the UNIQUE slot.
"""
from backend.scripts import create_script_record
session = _AsyncSessionMock()
request = _make_request()
context = _make_context()
runtime_client = request.app.state.runtime_client
runtime_client.ensure_directory = AsyncMock(return_value=None)
runtime_client.create_notebook = AsyncMock(return_value={"name": "x.ipynb"})
s1, so1 = await create_script_record(
name="x.ipynb", script_type="notebook", content=b'{"cells":[]}',
visibility="private", parent_path=None,
request=request, context=context, session=session,
)
s1.status = "deleted"
s1.is_deleted = 1
so1.object_status = "deleted"
so1.is_deleted = 1
s2, so2 = await create_script_record(
name="x.ipynb", script_type="notebook", content=b'{"cells":[]}',
visibility="private", parent_path=None,
request=request, context=context, session=session,
)
assert s2.script_id != s1.script_id
assert so2.storage_object_id != so1.storage_object_id
# Same object_key (no ULID suffix) — soft-deleted row excluded from
# _resolve_unique_object_key's "available" filter.
assert so2.object_key == so1.object_key
def test_scripts_model_allows_duplicate_active_name() -> None:
"""With uk_scripts_workspace_name_active dropped, no unique index covers
(workspace_id, script_name, script_type), so duplicate active names are
allowed at the ORM level.
"""
for idx in Scripts.__table__.indexes:
if idx.unique:
cols = {c.name for c in idx.columns}
assert not (
{"workspace_id", "script_name", "script_type"} <= cols
), f"unexpected unique index {idx.name} on script name"
def test_scripts_model_allows_duplicate_current_object_id() -> None:
"""With uk_scripts_current_object dropped, no unique index covers
current_object_id, so multiple scripts may point to the same storage object.
"""
names = _index_names(Scripts.__table__)
assert "uk_scripts_current_object" not in names
for idx in Scripts.__table__.indexes:
if idx.unique and len(idx.columns) == 1:
assert "current_object_id" not in {c.name for c in idx.columns}
def _script_row() -> Scripts:
return Scripts(
script_id="01SCR0000000000000000000A",
workspace_id="01WS0000000000000000000A",
current_object_id="01OBJ0000000000000000000A",
owner_user_id="01USR0000000000000000000A",
script_name="test.ipynb",
script_type="notebook",
visibility="private",
status="active",
)
def _storage_object_row() -> StorageObjects:
return StorageObjects(
storage_object_id="01OBJ0000000000000000000A",
workspace_id="01WS0000000000000000000A",
object_type="file",
usage_type="working_copy",
# 与生产构造器一致（common/db/models/storage.py 注释 "s3"）
storage_backend=settings.storage_backend,
storage_uri="s3://bucket/key",
file_name="test.ipynb",
created_by="01USR0000000000000000000A",
)
@pytest.mark.asyncio
async def test_delete_script_route_sets_is_deleted(monkeypatch: pytest.MonkeyPatch) -> None:
"""Soft-deleting a script via the route handler flips is_deleted=1."""
from backend.scripts import delete_script
script = _script_row()
storage_object = _storage_object_row()
request = _make_request()
context = _make_context()
session = AsyncMock()
async def _fake_get_script_row(
_script_id: str,
_context: SimpleNamespace,
_session: AsyncMock,
*,
for_update: bool = False,
allow_missing_storage_object: bool = False,
) -> tuple[Scripts, StorageObjects]:
return script, storage_object
monkeypatch.setattr("backend.scripts.get_script_row", _fake_get_script_row)
mock_soft_delete = AsyncMock(
return_value={
"data": {
"storage_object_id": storage_object.storage_object_id,
"object_status": "deleted",
"trash_key": "trash/bucket/key",
"trash_bucket": "trash",
}
}
)
monkeypatch.setattr("backend.scripts.soft_delete_object", mock_soft_delete)
result = await delete_script(
script_id=script.script_id,
request=request,
context=context,
session=session,
)
assert script.status == "deleted"
assert script.is_deleted == 1
assert script.deleted_at is not None
assert storage_object.object_status == "deleted"
assert storage_object.is_deleted == 1
assert storage_object.deleted_at is not None
assert result["data"]["status"] == "deleted"
mock_soft_delete.assert_awaited_once_with(
storage_object.storage_object_id, request, session
)
@pytest.mark.asyncio
async def test_delete_resource_sets_is_deleted(monkeypatch: pytest.MonkeyPatch) -> None:
"""Soft-deleting a data resource must write is_deleted=1 on the row."""
from backend.resources import delete_resource
resource = DataResources(
resource_id="01RES0000000000000000000A",
workspace_id="01WS0000000000000000000A",
storage_object_id="01OBJ0000000000000000000A",
owner_user_id="01USR0000000000000000000A",
resource_name="data.csv",
visibility="workspace", # visible so can_view passes
status="active",
)
storage_object = _storage_object_row()
request = _make_request()
context = _make_context()
class _Result:
def one_or_none(self):
return (resource, storage_object)
session = AsyncMock()
session.execute = AsyncMock(return_value=_Result())
with monkeypatch.context() as mp:
mp.setattr(
"backend.resources.soft_delete_object",
AsyncMock(return_value={"data": {}}),
)
result = await delete_resource(
resource_id=resource.resource_id,
request=request,
context=context,
session=session,
)
assert resource.status == "deleted"
assert resource.is_deleted == 1
assert resource.deleted_at is not None
assert result["data"]["status"] == "deleted"
@pytest.mark.asyncio
async def test_soft_delete_object_sets_is_deleted() -> None:
"""The shared helper must flip is_deleted=1 on StorageObjects."""
from backend.services.storage import soft_delete_object
item = _storage_object_row()
item.storage_backend = "local" # 测的是"非默认后端短路 trash、只翻 DB"路径
item.object_status = "available"
request = MagicMock()
request.app.state.object_stores = {}
session = AsyncMock()
session.scalar = AsyncMock(return_value=item)
result = await soft_delete_object(
storage_object_id=item.storage_object_id,
request=request,
session=session,
)
assert item.object_status == "deleted"
assert item.is_deleted == 1
assert item.deleted_at is not None
assert result["data"]["object_status"] == "deleted"
@pytest.mark.asyncio
async def test_jupyter_check_notebook_lock_ignores_deleted_scripts() -> None:
"""``is_deleted == 0`` filter must hide deleted notebooks from Jupyter checks."""
from backend.jupyter import check_notebook_is_locked
session = AsyncMock()
session.execute = AsyncMock()
session.execute.return_value.one_or_none = MagicMock(return_value=None)
result = await check_notebook_is_locked(
workspace_id="01WS0000000000000000000A",
notebook_path="test.ipynb",
user_id="01USR0000000000000000000A",
session=session,
)
assert result is False
# Verify the query carries the is_deleted filter.
call = session.execute.await_args
statement = call[0][0]
compiled = str(statement.compile(compile_kwargs={"literal_binds": True}))
assert "is_deleted" in compiled
common/src/common/db/models/storage.py
+18 -11
View File
@@ -1,7 +1,7 @@
import datetime
from typing import Optional
from sqlalchemy import BINARY, Index, JSON, String, text
from sqlalchemy import BINARY, Computed, Index, JSON, String, text
from sqlalchemy.dialects.mysql import BIGINT, CHAR, DATETIME, TINYINT
from sqlalchemy.orm import Mapped, mapped_column
@@ -22,6 +22,12 @@ class StorageObjects(Base):
Index("idx_storage_content_hash", "content_hash"),
Index("idx_storage_owner", "owner_user_id", "object_status"),
Index("idx_storage_parent", "parent_object_id"),
Index(
"idx_storage_workspace_path",
"workspace_id",
"storage_backend",
"path_hash",
),
Index(
"idx_storage_workspace_usage",
"workspace_id",
@@ -29,17 +35,10 @@ class StorageObjects(Base):
"object_status",
),
Index(
"uk_storage_bucket_key",
"uk_storage_bucket_key_active",
"storage_backend",
"bucket_name",
"object_key_hash",
unique=True,
),
Index(
"uk_storage_workspace_path",
"workspace_id",
"storage_backend",
"path_hash",
"object_key_hash_active",
unique=True,
),
{"comment": "Workspace 文件和 RustFS 对象的统一元数据"},
@@ -56,7 +55,7 @@ class StorageObjects(Base):
comment="working_copy/public_script/data_resource/version_artifact/snapshot/run_log/run_result",
)
storage_backend: Mapped[str] = mapped_column(
String(16), nullable=False, comment="rustfs"
String(16), nullable=False, comment="s3"
)
storage_uri: Mapped[str] = mapped_column(String(1500), nullable=False)
file_name: Mapped[str] = mapped_column(String(255), nullable=False)
@@ -100,6 +99,14 @@ class StorageObjects(Base):
object_key_hash: Mapped[Optional[bytes]] = mapped_column(
BINARY(32), comment="SHA-256(object_key)，由应用写入"
)
object_key_hash_active: Mapped[Optional[bytes]] = mapped_column(
BINARY(32),
Computed(
"CASE WHEN object_status = 'available' THEN object_key_hash ELSE NULL END",
persisted=False,
),
comment="VIRTUAL generated column used by uk_storage_bucket_key_active",
)
file_extension: Mapped[Optional[str]] = mapped_column(String(32))
mime_type: Mapped[Optional[str]] = mapped_column(String(255))
content_hash: Mapped[Optional[str]] = mapped_column(
migrations/versions/3ba4d8489f36_workspace_tree_relative_path_index.py
-38
View File
@@ -1,38 +0,0 @@
"""Add workspace tree relative_path index.
The ``list_workspace_directories`` endpoint filters by
``relative_path`` prefixes inside a workspace. A composite index on
``(workspace_id, relative_path(255))`` avoids scanning all rows for a
workspace when listing a subdirectory.
Revision ID: 3ba4d8489f36
Revises: f6a7b8c9d0e1
Create Date: 2026-08-12
"""
from collections.abc import Sequence
from alembic import op
import sqlalchemy as sa
# revision identifiers, used by Alembic.
revision: str = "3ba4d8489f36"
down_revision: str | Sequence[str] | None = "f6a7b8c9d0e1"
branch_labels: str | Sequence[str] | None = None
depends_on: str | Sequence[str] | None = None
def upgrade() -> None:
op.create_index(
"idx_storage_workspace_relative_path",
"storage_objects",
[sa.text("`workspace_id`"), sa.text("`relative_path`(255)")],
unique=False,
)
def downgrade() -> None:
op.drop_index(
"idx_storage_workspace_relative_path",
table_name="storage_objects",
)
migrations/versions/47a76cd261fd_scripts_unique_index_include_deleted_at.py
-54
View File
@@ -1,54 +0,0 @@
"""Make the scripts workspace/name/type unique index soft-delete aware.
The old unique index ``uk_scripts_workspace_name`` on
``(workspace_id, script_name, script_type)`` blocked re-uploading a
script after it had been soft-deleted, because the deleted row was still
part of the index.
Replace it with ``uk_scripts_workspace_name_active`` on
``(workspace_id, script_name, script_type, deleted_at)``. In MySQL a
unique index treats ``NULL`` values as distinct, so a new active row
(``deleted_at IS NULL``) no longer conflicts with a previously deleted
row (``deleted_at IS NOT NULL``), while two active rows with the same
name still conflict as expected.
Revision ID: 47a76cd261fd
Revises: 3ba4d8489f36
Create Date: 2026-08-14
"""
from collections.abc import Sequence
from alembic import op
# revision identifiers, used by Alembic.
revision: str = "47a76cd261fd"
down_revision: str | Sequence[str] | None = "3ba4d8489f36"
branch_labels: str | Sequence[str] | None = None
depends_on: str | Sequence[str] | None = None
def upgrade() -> None:
op.drop_index(
"uk_scripts_workspace_name",
table_name="scripts",
)
op.create_index(
"uk_scripts_workspace_name_active",
"scripts",
["workspace_id", "script_name", "script_type", "deleted_at"],
unique=True,
)
def downgrade() -> None:
op.drop_index(
"uk_scripts_workspace_name_active",
table_name="scripts",
)
op.create_index(
"uk_scripts_workspace_name",
"scripts",
["workspace_id", "script_name", "script_type"],
unique=True,
)
migrations/versions/README.md
+14
View File
@@ -5,3 +5,17 @@
- 已发布迁移不得直接修改。
- 新迁移必须同时提供可执行的 `upgrade()` 和 `downgrade()`。
- 自动生成后必须检查数据类型、约束、索引和执行顺序。
# Migration Revisions
该目录只保存经过评审和验证的 Alembic 迁移版本。
## Baseline
- `e1f2a3b4c5d6_rebuild_baseline.py` — 2026-08-14 重建的基线迁移，包含完整 schema + seed。
- 之前的 8 个迁移文件已合并并删除；新环境从此基线开始。
## Rules
- 已发布迁移不得直接修改。
- 新迁移必须同时提供可执行的 `upgrade()` 和 `downgrade()`。
- 自动生成后必须检查数据类型、约束、索引和执行顺序。
migrations/versions/a1b2c3d4e5f6_drop_scripts_data_resources_unique_indexes.py
-63
View File
@@ -1,63 +0,0 @@
"""Drop unique indexes on Scripts and DataResources.
StorageObjects now carries the physical uniqueness guarantees:
- ``uk_storage_bucket_key``
- ``uk_storage_workspace_path``
Scripts and DataResources therefore no longer need their own unique
indexes on ``current_object_id`` / ``storage_object_id``, and the
soft-delete-aware ``uk_scripts_workspace_name_active`` index is also
removed. Re-uploading a previously soft-deleted script or resource is
allowed because the StorageObjects layer enforces path uniqueness only
for active objects.
Revision ID: a1b2c3d4e5f6
Revises: 47a76cd261fd
Create Date: 2026-08-14
"""
from collections.abc import Sequence
from alembic import op
# revision identifiers, used by Alembic.
revision: str = "a1b2c3d4e5f6"
down_revision: str | Sequence[str] | None = "47a76cd261fd"
branch_labels: str | Sequence[str] | None = None
depends_on: str | Sequence[str] | None = None
def upgrade() -> None:
op.drop_index(
"uk_scripts_workspace_name_active",
table_name="scripts",
)
op.drop_index(
"uk_scripts_current_object",
table_name="scripts",
)
op.drop_index(
"uk_data_resources_object",
table_name="data_resources",
)
def downgrade() -> None:
op.create_index(
"uk_scripts_workspace_name_active",
"scripts",
["workspace_id", "script_name", "script_type", "deleted_at"],
unique=True,
)
op.create_index(
"uk_scripts_current_object",
"scripts",
["current_object_id"],
unique=True,
)
op.create_index(
"uk_data_resources_object",
"data_resources",
["storage_object_id"],
unique=True,
)
migrations/versions/d4e5f6a7b8c9_squashed_baseline.py → migrations/versions/e1f2a3b4c5d6_rebuild_baseline.py
+138 -42
View File
@@ -1,22 +1,12 @@
"""squashed baseline — full schema + seed data in one migration
"""rebuild baseline — full schema + seed data in one migration
Single baseline migration combining the previous 5-step chain:
8d86e2f82860 initial baseline (20 tables)
b71c4f2a9d10 seed demo users / workspaces / roles / members
9a1b2c3d4e5f enable password login for seeded users
a2b3c4d5e6f7 add storage_objects.trash_key
c3d4e5f6a7b8 add upload_sessions object-metadata columns
The column additions from the later migrations are folded directly into
the CREATE TABLE statements, so this file is a from-scratch schema.
Revision ID: d4e5f6a7b8c9
Revision ID: e1f2a3b4c5d6
Revises: (none)
Create Date: 2026-08-05
Create Date: 2026-08-14
"""
from collections.abc import Sequence
import hashlib
import os
from alembic import op
@@ -26,14 +16,70 @@ from sqlalchemy.dialects import mysql
from common.auth.passwords import hash_password
# revision identifiers, used by Alembic.
revision: str = "d4e5f6a7b8c9"
revision: str = "e1f2a3b4c5d6"
down_revision: str | Sequence[str] | None = None
branch_labels: str | Sequence[str] | None = None
depends_on: str | Sequence[str] | None = None
# ── seed constants ───────────────────────────────────────────────
DISABLED_PASSWORD = "demo-login-disabled"
SEEDED_USERS = (
(
"00000000000000000000000001",
"admin",
"Admin",
"0000000000000000000000000A",
),
)
ADMIN_ROLE_ID = "0000000000000000000000000A"
DEVELOPER_ROLE_ID = "0000000000000000000000000B"
PERMISSIONS: list[tuple[str, str, str]] = [
# (permission_code, permission_name, module_code)
("dashboard.view", "查看工作台", "dashboard"),
("script.build", "构建脚本", "script"),
("script.public.manage", "管理公共脚本", "script"),
("schedule.own", "管理本人调度", "schedule"),
("schedule.all", "管理全部调度", "schedule"),
("experiment.own", "管理本人实验", "experiment"),
("experiment.all", "管理全部实验", "experiment"),
("resource.personal", "管理个人资源", "resource"),
("resource.public.upload", "上传公共资源", "resource"),
("resource.public.manage", "管理公共资源", "resource"),
("system.view", "查看系统管理", "system"),
("system.manage", "管理系统配置", "system"),
]
# developer gets *.own + personal resource only; no system.*, no *.all.
DEVELOPER_PERMISSION_CODES: list[str] = [
"dashboard.view",
"script.build",
"script.public.manage",
"schedule.own",
"experiment.own",
"resource.personal",
]
ADMIN_PERMISSION_CODES: list[str] = [code for code, _, _ in PERMISSIONS]
CROCKFORD_BASE32 = "0123456789ABCDEFGHJKMNPQRSTVWXYZ"
def _deterministic_permission_id(code: str) -> str:
"""Stable 26-char ULID-shaped id derived from permission_code."""
digest = hashlib.sha256(
f"model-platform-permission-v1:{code}".encode("utf-8")
).digest()
value = int.from_bytes(b"\x00" * 6 + digest[:10], byteorder="big")
encoded = ["0"] * 26
for index in range(25, -1, -1):
encoded[index] = CROCKFORD_BASE32[value & 31]
value >>= 5
return "".join(encoded)
def upgrade() -> None:
"""Full schema from scratch (all 20 tables, current model state)."""
"""Upgrade schema."""
# ### commands auto generated by Alembic - please adjust! ###
op.create_table('consumer_inbox',
sa.Column('consumer_name', sa.String(length=128), nullable=False),
sa.Column('event_id', mysql.CHAR(length=26), nullable=False),
@@ -67,7 +113,6 @@ def upgrade() -> None:
)
op.create_index('idx_data_resources_owner', 'data_resources', ['owner_user_id', 'status'], unique=False)
op.create_index('idx_data_resources_workspace', 'data_resources', ['workspace_id', 'visibility', 'status'], unique=False)
op.create_index('uk_data_resources_object', 'data_resources', ['storage_object_id'], unique=True)
op.create_table('outbox_events',
sa.Column('event_id', mysql.CHAR(length=26), nullable=False),
sa.Column('aggregate_type', sa.String(length=64), nullable=False),
@@ -179,7 +224,7 @@ def upgrade() -> None:
sa.Column('node_key', sa.String(length=64), nullable=False, comment='画布内稳定标识'),
sa.Column('node_name', sa.String(length=255), nullable=False),
sa.Column('versions_id', mysql.CHAR(length=26), nullable=False),
sa.Column('python_version', sa.String(length=8), nullable=False, server_default=sa.text("'3.12'"), comment='节点执行 Python 版本（3.8/3.10/3.12）'),
sa.Column('python_version', sa.String(length=8), server_default=sa.text("'3.12'"), nullable=False, comment='节点执行 Python 版本（3.8/3.10/3.12）'),
sa.Column('timeout_seconds', mysql.INTEGER(), server_default=sa.text('600'), nullable=False),
sa.Column('retry_count', mysql.INTEGER(), server_default=sa.text('0'), nullable=False),
sa.Column('retry_interval_sec', mysql.INTEGER(), server_default=sa.text('5'), nullable=False),
@@ -274,8 +319,6 @@ def upgrade() -> None:
)
op.create_index('idx_scripts_owner', 'scripts', ['owner_user_id', 'status'], unique=False)
op.create_index('idx_scripts_workspace', 'scripts', ['workspace_id', 'script_type', 'visibility', 'status'], unique=False)
op.create_index('uk_scripts_current_object', 'scripts', ['current_object_id'], unique=True)
op.create_index('uk_scripts_workspace_name', 'scripts', ['workspace_id', 'script_name', 'script_type'], unique=True)
op.create_table('storage_objects',
sa.Column('storage_object_id', mysql.CHAR(length=26), nullable=False),
sa.Column('workspace_id', mysql.CHAR(length=26), nullable=False),
@@ -298,23 +341,25 @@ def upgrade() -> None:
sa.Column('bucket_name', sa.String(length=128), nullable=True),
sa.Column('object_key', sa.String(length=1024), nullable=True),
sa.Column('object_key_hash', sa.BINARY(length=32), nullable=True, comment='SHA-256(object_key)，由应用写入'),
sa.Column('object_key_hash_active', sa.BINARY(length=32), sa.Computed("CASE WHEN object_status = 'available' THEN object_key_hash ELSE NULL END", persisted=False), nullable=True, comment='VIRTUAL generated column used by uk_storage_bucket_key_active'),
sa.Column('file_extension', sa.String(length=32), nullable=True),
sa.Column('mime_type', sa.String(length=255), nullable=True),
sa.Column('content_hash', mysql.CHAR(length=64), nullable=True, comment='SHA-256 hex'),
sa.Column('object_etag', sa.String(length=255), nullable=True),
sa.Column('trash_key', sa.String(length=1100), nullable=True, comment='Path inside the trash bucket where soft-deleted bytes are stored'),
sa.Column('is_deleted', mysql.TINYINT(display_width=1), server_default=sa.text('0'), nullable=False),
sa.Column('deleted_at', mysql.DATETIME(fsp=3), nullable=True),
sa.Column('trash_key', sa.String(length=1100), nullable=True, comment="Path inside the trash bucket where the soft-deleted bytes live. Format: '{source_bucket}/{object_key}' so a restore is a same-key copy back to the source bucket. NULL while the row is still available."),
sa.PrimaryKeyConstraint('storage_object_id'),
comment='Workspace 文件和 S3 对象的统一元数据'
comment='Workspace 文件和 RustFS 对象的统一元数据'
)
op.create_index('fk_storage_created_by', 'storage_objects', ['created_by'], unique=False)
op.create_index('idx_storage_content_hash', 'storage_objects', ['content_hash'], unique=False)
op.create_index('idx_storage_owner', 'storage_objects', ['owner_user_id', 'object_status'], unique=False)
op.create_index('idx_storage_parent', 'storage_objects', ['parent_object_id'], unique=False)
op.create_index('idx_storage_workspace_path', 'storage_objects', ['workspace_id', 'storage_backend', 'path_hash'], unique=False)
op.create_index('idx_storage_workspace_usage', 'storage_objects', ['workspace_id', 'usage_type', 'object_status'], unique=False)
op.create_index('uk_storage_bucket_key', 'storage_objects', ['storage_backend', 'bucket_name', 'object_key_hash'], unique=True)
op.create_index('uk_storage_workspace_path', 'storage_objects', ['workspace_id', 'storage_backend', 'path_hash'], unique=True)
op.create_index('uk_storage_bucket_key_active', 'storage_objects', ['storage_backend', 'bucket_name', 'object_key_hash_active'], unique=True)
op.create_index('idx_storage_workspace_relative_path', 'storage_objects', [sa.text('`workspace_id`'), sa.text('`relative_path`(255)')], unique=False)
op.create_table('upload_sessions',
sa.Column('upload_id', mysql.CHAR(length=26), nullable=False),
sa.Column('workspace_id', mysql.CHAR(length=26), nullable=False),
@@ -333,15 +378,14 @@ def upgrade() -> None:
sa.Column('content_type', sa.String(length=255), nullable=True),
sa.Column('storage_object_id', mysql.CHAR(length=26), nullable=True),
sa.Column('completed_at', mysql.DATETIME(fsp=3), nullable=True),
# Object-metadata columns added by c3d4e5f6a7b8 (server-proxied upload).
sa.Column('file_name', sa.String(length=255), nullable=False, server_default=''),
sa.Column('usage_type', sa.String(length=32), nullable=False, server_default='working_copy', comment='data_resource/version_artifact/snapshot/run_log/run_result/working_copy/public_script'),
sa.Column('visibility', sa.String(length=16), nullable=False, server_default='private', comment='private/workspace/public'),
sa.Column('is_immutable', mysql.TINYINT(display_width=1), nullable=False, server_default='0'),
sa.Column('file_name', sa.String(length=255), server_default='', nullable=False),
sa.Column('usage_type', sa.String(length=32), server_default=sa.text("'working_copy'"), nullable=False, comment='data_resource/version_artifact/snapshot/run_log/run_result/working_copy/public_script'),
sa.Column('visibility', sa.String(length=16), server_default=sa.text("'private'"), nullable=False, comment='private/workspace/public'),
sa.Column('is_immutable', mysql.TINYINT(display_width=1), server_default=sa.text('0'), nullable=False),
sa.Column('is_deleted', mysql.TINYINT(display_width=1), server_default=sa.text('0'), nullable=False),
sa.Column('deleted_at', mysql.DATETIME(fsp=3), nullable=True),
sa.PrimaryKeyConstraint('upload_id'),
comment='S3 上传会话；URL 本身不持久化'
comment='RustFS 预签名上传会话；URL 本身不持久化'
)
op.create_index('fk_upload_sessions_storage_object', 'upload_sessions', ['storage_object_id'], unique=False)
op.create_index('fk_upload_sessions_user', 'upload_sessions', ['user_id'], unique=False)
@@ -434,10 +478,9 @@ def upgrade() -> None:
op.create_index('fk_workspaces_created_by', 'workspaces', ['created_by'], unique=False)
op.create_index('idx_workspaces_status', 'workspaces', ['status'], unique=False)
op.create_index('uk_workspaces_code', 'workspaces', ['workspace_code'], unique=True)
# ### end Alembic commands ###
# ── seed data (from b71c4f2a9d10) ──────────────────────────────
ADMIN_ROLE_ID = "0000000000000000000000000A"
DEVELOPER_ROLE_ID = "0000000000000000000000000B"
# ── seed data ────────────────────────────────────────────────────
ADMIN_USER_ID = "00000000000000000000000001"
DEFAULT_WORKSPACE_ID = "00000000000000000000000002"
USERS = (
@@ -491,7 +534,7 @@ def upgrade() -> None:
"role_id": ADMIN_ROLE_ID,
"role_code": "admin",
"role_name": "管理员",
"role_scope": "workspace",
"role_scope": "platform",
"is_builtin": 1,
"description": "Self-hosted workspace administrator",
},
@@ -499,7 +542,7 @@ def upgrade() -> None:
"role_id": DEVELOPER_ROLE_ID,
"role_code": "developer",
"role_name": "开发人员",
"role_scope": "workspace",
"role_scope": "platform",
"is_builtin": 1,
"description": "Self-hosted workspace developer",
},
@@ -549,7 +592,7 @@ def upgrade() -> None:
],
)
# ── enable demo password login (from 9a1b2c3d4e5f) ─────────────
# ── enable demo password login (from e5f6a7b8c9d0) ──────────────
password = os.environ.get("INITIAL_ADMIN_PASSWORD", "admin12345")
seeded_user_ids = (ADMIN_USER_ID,)
users_update = sa.table(
@@ -564,9 +607,63 @@ def upgrade() -> None:
.values(password_hash=hash_password(password))
)
# ── seed platform permissions (from f6a7b8c9d0e1) ───────────────
op.execute(
"UPDATE roles SET role_scope = 'platform' "
"WHERE role_code IN ('admin', 'developer') AND is_deleted = 0"
)
permissions_table = sa.table(
"permissions",
sa.column("permission_id", sa.CHAR(26)),
sa.column("permission_code", sa.String(128)),
sa.column("permission_name", sa.String(100)),
sa.column("module_code", sa.String(64)),
sa.column("description", sa.String(500)),
)
perm_id_by_code: dict[str, str] = {}
rows: list[dict[str, str]] = []
for code, name, module in PERMISSIONS:
pid = _deterministic_permission_id(code)
perm_id_by_code[code] = pid
rows.append(
{
"permission_id": pid,
"permission_code": code,
"permission_name": name,
"module_code": module,
"description": f"platform 菜单权限：{name}",
}
)
op.bulk_insert(permissions_table, rows)
role_permissions_table = sa.table(
"role_permissions",
sa.column("role_id", sa.CHAR(26)),
sa.column("permission_id", sa.CHAR(26)),
)
rp_rows: list[dict[str, str]] = []
for code in ADMIN_PERMISSION_CODES:
rp_rows.append(
{
"role_id": ADMIN_ROLE_ID,
"permission_id": perm_id_by_code[code],
}
)
for code in DEVELOPER_PERMISSION_CODES:
rp_rows.append(
{
"role_id": DEVELOPER_ROLE_ID,
"permission_id": perm_id_by_code[code],
}
)
op.bulk_insert(role_permissions_table, rp_rows)
# ### end Alembic commands ###
def downgrade() -> None:
"""Drop everything (reverse of upgrade)."""
"""Downgrade schema."""
# ### commands auto generated by Alembic - please adjust! ###
op.drop_index('uk_workspaces_code', table_name='workspaces')
op.drop_index('idx_workspaces_status', table_name='workspaces')
op.drop_index('fk_workspaces_created_by', table_name='workspaces')
@@ -593,16 +690,15 @@ def downgrade() -> None:
op.drop_index('fk_upload_sessions_user', table_name='upload_sessions')
op.drop_index('fk_upload_sessions_storage_object', table_name='upload_sessions')
op.drop_table('upload_sessions')
op.drop_index('uk_storage_workspace_path', table_name='storage_objects')
op.drop_index('uk_storage_bucket_key', table_name='storage_objects')
op.drop_index('uk_storage_bucket_key_active', table_name='storage_objects')
op.drop_index('idx_storage_workspace_usage', table_name='storage_objects')
op.drop_index('idx_storage_workspace_relative_path', table_name='storage_objects')
op.drop_index('idx_storage_workspace_path', table_name='storage_objects')
op.drop_index('idx_storage_parent', table_name='storage_objects')
op.drop_index('idx_storage_owner', table_name='storage_objects')
op.drop_index('idx_storage_content_hash', table_name='storage_objects')
op.drop_index('fk_storage_created_by', table_name='storage_objects')
op.drop_table('storage_objects')
op.drop_index('uk_scripts_workspace_name', table_name='scripts')
op.drop_index('uk_scripts_current_object', table_name='scripts')
op.drop_index('idx_scripts_workspace', table_name='scripts')
op.drop_index('idx_scripts_owner', table_name='scripts')
op.drop_table('scripts')
@@ -644,9 +740,9 @@ def downgrade() -> None:
op.drop_index('idx_outbox_idempotency', table_name='outbox_events')
op.drop_index('idx_outbox_aggregate', table_name='outbox_events')
op.drop_table('outbox_events')
op.drop_index('uk_data_resources_object', table_name='data_resources')
op.drop_index('idx_data_resources_workspace', table_name='data_resources')
op.drop_index('idx_data_resources_owner', table_name='data_resources')
op.drop_table('data_resources')
op.drop_index('idx_consumer_inbox_status', table_name='consumer_inbox')
op.drop_table('consumer_inbox')
# ### end Alembic commands ###
migrations/versions/e5f6a7b8c9d0_ensure_demo_login.py
-140
View File
@@ -1,140 +0,0 @@
"""ensure the self-hosted demo login remains available
Revision ID: e5f6a7b8c9d0
Revises: d4e5f6a7b8c9
Create Date: 2026-08-05 15:31:00
"""
from collections.abc import Sequence
import os
from alembic import op
import sqlalchemy as sa
from sqlalchemy.dialects import mysql
from common.auth.passwords import hash_password
revision: str = "e5f6a7b8c9d0"
down_revision: str | Sequence[str] | None = "d4e5f6a7b8c9"
branch_labels: str | Sequence[str] | None = None
depends_on: str | Sequence[str] | None = None
DISABLED_PASSWORD = "demo-login-disabled"
SEEDED_USERS = (
(
"00000000000000000000000001",
"admin",
"Admin",
"0000000000000000000000000A",
),
)
def _create_users_table() -> None:
op.create_table(
"users",
sa.Column("user_id", mysql.CHAR(length=26), nullable=False),
sa.Column("username", sa.String(length=64), nullable=False),
sa.Column("display_name", sa.String(length=100), nullable=False),
sa.Column("password_hash", sa.String(length=255), nullable=False),
sa.Column(
"status",
sa.String(length=16),
server_default=sa.text("'active'"),
nullable=False,
comment="active/disabled/locked",
),
sa.Column(
"created_at",
mysql.DATETIME(fsp=3),
server_default=sa.text("CURRENT_TIMESTAMP(3)"),
nullable=False,
),
sa.Column(
"updated_at",
mysql.DATETIME(fsp=3),
server_default=sa.text(
"CURRENT_TIMESTAMP(3) ON UPDATE CURRENT_TIMESTAMP(3)"
),
nullable=False,
),
sa.Column("email", sa.String(length=255), nullable=True),
sa.Column("platform_role_id", mysql.CHAR(length=26), nullable=True),
sa.Column("avatar_uri", sa.String(length=1000), nullable=True),
sa.Column("last_login_at", mysql.DATETIME(fsp=3), nullable=True),
sa.Column(
"is_deleted",
mysql.TINYINT(display_width=1),
server_default=sa.text("0"),
nullable=False,
),
sa.Column("deleted_at", mysql.DATETIME(fsp=3), nullable=True),
sa.PrimaryKeyConstraint("user_id"),
comment="平台用户",
)
op.create_index("fk_users_platform_role", "users", ["platform_role_id"])
op.create_index("idx_users_status", "users", ["status"])
op.create_index("uk_users_email", "users", ["email"], unique=True)
op.create_index("uk_users_username", "users", ["username"], unique=True)
def upgrade() -> None:
connection = op.get_bind()
if not sa.inspect(connection).has_table("users"):
_create_users_table()
users = sa.table(
"users",
sa.column("user_id", sa.String),
sa.column("username", sa.String),
sa.column("display_name", sa.String),
sa.column("password_hash", sa.String),
sa.column("status", sa.String),
sa.column("email", sa.String),
sa.column("platform_role_id", sa.String),
)
existing = {
row.username: row.password_hash
for row in connection.execute(
sa.select(users.c.username, users.c.password_hash).where(
users.c.username.in_([user[1] for user in SEEDED_USERS])
)
)
}
password_hash = hash_password(
os.environ.get("INITIAL_ADMIN_PASSWORD", "admin12345")
)
for user_id, username, display_name, role_id in SEEDED_USERS:
if username not in existing:
connection.execute(
users.insert().values(
user_id=user_id,
username=username,
display_name=display_name,
password_hash=password_hash,
status="active",
email=f"{username}@model-platform.local",
platform_role_id=role_id,
)
)
continue
if existing[username] in {None, "", DISABLED_PASSWORD}:
connection.execute(
users.update()
.where(users.c.username == username)
.values(password_hash=password_hash)
)
connection.execute(
users.update()
.where(users.c.username == "admin")
.values(status="active")
)
def downgrade() -> None:
"""Do not remove or disable accounts that may contain user data."""
migrations/versions/f6a7b8c9d0e1_seed_role_permissions_and_fix_scope.py
-185
View File
@@ -1,185 +0,0 @@
"""Seed platform permissions + role_permissions, fix admin/developer role_scope.
The squashed baseline (d4e5f6a7b8c9) ships the Permissions and
RolePermissions tables empty, and seeds admin/developer with
role_scope='workspace' (an early mistake; the codebase elsewhere treats
both as platform-scoped — see backend/platform.py::system_admin_context
and common/auth/membership.py::resolve_is_system_admin). This migration:
1. UPDATE roles SET role_scope='platform' for admin/developer rows.
2. INSERT 12 permission rows covering the menu groups the frontend
consumes (dashboard / script / schedule / experiment / resource /
system).
3. INSERT role_permissions join rows: admin gets all 12, developer
gets the 6 `*.own` / personal-resource codes.
Caveats (read before re-running):
* ``permission_id`` is derived from a sha256 of the code with the salt
prefix ``model-platform-permission-v1:``. The legacy
``migrations/data/migrate_system_json.py`` script uses a different
salt (``model-platform-v1:permission:``), so the same
``permission_code`` maps to a DIFFERENT ``permission_id`` between the
two paths. The legacy script's ``existing.get(permission_code)`` check
keeps the row count correct (it reuses the live row by code), so this
is not a crash; the IDs only matter if a downstream system ever
cross-references by deterministic ID, which nothing does today.
* ``downgrade()`` is a SOFT delete (``is_deleted=1``). Running
``alembic downgrade`` followed by ``alembic upgrade`` will collide on
the ``permission_id`` PRIMARY KEY — downgrade is a one-way trip on
any environment that has run this migration. The role_scope fix is
not reverted on downgrade (app code already keys off `platform`).
* If the legacy one-off ``migrations/data/migrate_system_json.py`` is
ever run AFTER this migration on the same database, its
``existing.get(permission_code)`` check will keep counts correct but
reuses our rows; running it BEFORE this migration would cause
``uk_permissions_code`` collisions on upgrade. Run this migration
first on a fresh database.
Revision ID: f6a7b8c9d0e1
Revises: e5f6a7b8c9d0 (ensure_demo_login)
Create Date: 2026-08-07
"""
from __future__ import annotations
import hashlib
from collections.abc import Sequence
from alembic import op
import sqlalchemy as sa
CROCKFORD_BASE32 = "0123456789ABCDEFGHJKMNPQRSTVWXYZ"
# Hardcoded to match d4e5f6a7b8c9_squashed_baseline.py seed values, so the
# role_permissions join rows below resolve against the right role rows.
ADMIN_ROLE_ID = "0000000000000000000000000A"
DEVELOPER_ROLE_ID = "0000000000000000000000000B"
PERMISSIONS: list[tuple[str, str, str]] = [
# (permission_code, permission_name, module_code)
("dashboard.view", "查看工作台", "dashboard"),
("script.build", "构建脚本", "script"),
("script.public.manage", "管理公共脚本", "script"),
("schedule.own", "管理本人调度", "schedule"),
("schedule.all", "管理全部调度", "schedule"),
("experiment.own", "管理本人实验", "experiment"),
("experiment.all", "管理全部实验", "experiment"),
("resource.personal", "管理个人资源", "resource"),
("resource.public.upload", "上传公共资源", "resource"),
("resource.public.manage", "管理公共资源", "resource"),
("system.view", "查看系统管理", "system"),
("system.manage", "管理系统配置", "system"),
]
# developer gets *.own + personal resource only; no system.*, no *.all.
DEVELOPER_PERMISSION_CODES: list[str] = [
"dashboard.view",
"script.build",
"script.public.manage",
"schedule.own",
"experiment.own",
"resource.personal",
]
ADMIN_PERMISSION_CODES: list[str] = [code for code, _, _ in PERMISSIONS]
def _deterministic_permission_id(code: str) -> str:
"""Stable 26-char ULID-shaped id derived from permission_code.
Mirrors ``migrations/data/migrate_system_json.py::deterministic_legacy_ulid``
so re-running this migration (or running it after the legacy data
migrator) keeps identical IDs for the same code.
"""
digest = hashlib.sha256(
f"model-platform-permission-v1:{code}".encode("utf-8")
).digest()
value = int.from_bytes(b"\x00" * 6 + digest[:10], byteorder="big")
encoded = ["0"] * 26
for index in range(25, -1, -1):
encoded[index] = CROCKFORD_BASE32[value & 31]
value >>= 5
return "".join(encoded)
# revision identifiers, used by Alembic.
revision: str = "f6a7b8c9d0e1"
down_revision: str | Sequence[str] | None = "e5f6a7b8c9d0"
def upgrade() -> None:
# (1) role_scope fix: admin/developer were seeded as 'workspace' but
# platform.py::system_admin_context treats them as platform-scoped.
op.execute(
"UPDATE roles SET role_scope = 'platform' "
"WHERE role_code IN ('admin', 'developer') AND is_deleted = 0"
)
# (2) Permissions rows.
permissions_table = sa.table(
"permissions",
sa.column("permission_id", sa.CHAR(26)),
sa.column("permission_code", sa.String(128)),
sa.column("permission_name", sa.String(100)),
sa.column("module_code", sa.String(64)),
sa.column("description", sa.String(500)),
)
perm_id_by_code: dict[str, str] = {}
rows: list[dict[str, str]] = []
for code, name, module in PERMISSIONS:
pid = _deterministic_permission_id(code)
perm_id_by_code[code] = pid
rows.append(
{
"permission_id": pid,
"permission_code": code,
"permission_name": name,
"module_code": module,
"description": f"platform 菜单权限：{name}",
}
)
op.bulk_insert(permissions_table, rows)
# (3) role_permissions join rows.
role_permissions_table = sa.table(
"role_permissions",
sa.column("role_id", sa.CHAR(26)),
sa.column("permission_id", sa.CHAR(26)),
)
rp_rows: list[dict[str, str]] = []
for code in ADMIN_PERMISSION_CODES:
rp_rows.append(
{
"role_id": ADMIN_ROLE_ID,
"permission_id": perm_id_by_code[code],
}
)
for code in DEVELOPER_PERMISSION_CODES:
rp_rows.append(
{
"role_id": DEVELOPER_ROLE_ID,
"permission_id": perm_id_by_code[code],
}
)
op.bulk_insert(role_permissions_table, rp_rows)
def downgrade() -> None:
# Soft-delete what we inserted. The role_scope fix is intentionally
# NOT reverted — the only safe direction is platform, since
# application code already keys off it.
code_list_sql = "(" + ",".join(f"'{c}'" for c in ADMIN_PERMISSION_CODES) + ")"
op.execute(
"UPDATE role_permissions "
"SET is_deleted = 1, deleted_at = CURRENT_TIMESTAMP(3) "
f"WHERE permission_id IN (SELECT permission_id FROM permissions "
f"WHERE is_deleted = 0 AND permission_code IN {code_list_sql})"
)
op.execute(
"UPDATE permissions SET is_deleted = 1, deleted_at = CURRENT_TIMESTAMP(3) "
f"WHERE is_deleted = 0 AND permission_code IN {code_list_sql}"
)