fix(backend): enhance storage/resource concurrency, idempotency, and deletion safety

Summary of changes:

- Resources Bind Idempotency (High 1+2):
  - Check existing active storage object bindings before duplicate check.
  - Return existing binding (`reused: true`) on retry with same upload_id.
  - Filter by `status == "active"` to bypass dead/deleted rows during reuse check.

- Storage Concurrent Overwrite (High 3):
  - Add `acquire_named_lock` and `release_named_lock` helpers using MySQL `GET_LOCK`/`RELEASE_LOCK` hashed to <= 64 chars.
  - Wrap `upload_bytes_to_session` PUT+INSERT critical section with named lock on `object_key`.
  - Re-check key collision inside lock; append ULID suffix on collision.

- Shared Reference Deletion Protection (Medium 4):
  - Check active references before deleting storage objects in `delete_resource`.
  - Delete only `DataResources` record if storage object is still referenced elsewhere.

- Robust Usage Type Fallback (Medium 5):
  - Replace direct dict lookup for `USAGE_TYPE_TO_PURPOSE[item.usage_type]` with `.get(..., "workspace")` default.

- Idempotency Key Path Matching (Medium 6):
  - Move `file_name`/`target_path` validation forward and include path dimension in comparison.
  - Strip uniqueness suffix via `_strip_uniqueness_suffix` before key comparison to avoid false 409s on valid retries.

- Usage Type & Bind Concurrency Control (Low 7 & 8):
  - Reject bind requests with 409 if upload session purpose is not `data_resource`.
  - Wrap resource duplicate check and creation in named lock using `(owner, directory, name)`.

- Trash Key Uniqueness & Restore Compatibility (Low 9):
  - Update `trash_key` format to `{purpose}/{object_key}-{storage_object_id}` to prevent collisions.
  - Update `object_key_hash` on trash move.
  - Update restore logic in `storage_api.py` to strip suffix while maintaining backward compatibility with legacy keys.

- Dead Code Removal (Low 10):
  - Remove unreachable `upload_status = "failed"` and redundant `session.rollback()` in `IntegrityError` block.

- Tests & Mocks:
  - Add/update 5 test cases covering non-data_resource bind rejection, suffix stripping, and path recovery.
  - Add named lock statement mocks for DB testing.
This commit is contained in:
tao.chen
2026-08-14 20:53:07 +08:00
parent f72dfd10e8
commit b6eb069849
5 changed files with 323 additions and 127 deletions
+65 -3
View File
@@ -15,7 +15,12 @@ from backend.resources import (
resource_directory,
resource_payload,
)
from backend.services.storage import _safe_file_name, _safe_path_segment
from backend.services.storage import (
_object_key_tail,
_safe_file_name,
_safe_path_segment,
_strip_uniqueness_suffix,
)
_BIND_WS = "01WS0000000000000000000A"
@@ -38,21 +43,33 @@ class _BindSessionMock:
in the workspace with the same resource_name.
"""
def __init__(self, new_object_key: str, same_name_rows=(), reused_resource=None):
def __init__(
self,
new_object_key: str,
same_name_rows=(),
reused_resource=None,
usage_type: str = "data_resource",
):
self._new_object_key = new_object_key
self._same_name_rows = list(same_name_rows)
self._reused_resource = reused_resource
self._usage_type = usage_type
self._scalar_calls = 0
async def scalar(self, _stmt):
from sqlalchemy import TextClause
if isinstance(_stmt, TextClause):
return 1 # GET_LOCK 成功
self._scalar_calls += 1
# 1st scalar: UploadSessions lookup; 2nd: storage_object_id reuse check.
# 1st scalar: UploadSessions lookup; 2nd: active 绑定行复用检查。
if self._scalar_calls == 1:
return SimpleNamespace(
upload_id="01UPL0000000000000000000B",
storage_object_id="01OBJ0000000000000000000B",
workspace_id=_BIND_WS,
user_id=_BIND_USER,
usage_type=self._usage_type,
)
if self._scalar_calls == 2 and self._reused_resource is not None:
return self._reused_resource
@@ -62,6 +79,10 @@ class _BindSessionMock:
return _make_storage_object(self._new_object_key)
async def execute(self, _stmt):
from sqlalchemy import TextClause
if isinstance(_stmt, TextClause):
return _ExecuteResult([]) # RELEASE_LOCK
# 模拟 SQL 的 owner 过滤与 self-exclusion:只返回属于当前用户且
# 不是当前 upload 已绑定对象的同名候选。
rows = [
@@ -222,6 +243,27 @@ async def test_bind_resource_allows_rebinding_same_storage_object() -> None:
assert result["data"]["resource_name"] == "data.csv"
@pytest.mark.asyncio
async def test_bind_resource_rejects_non_data_resource_upload() -> None:
"""其他用途(如 working_copy)的 upload session 不能 bind 成数据资源。"""
from backend.resources import bind_resource
session = _BindSessionMock(
new_object_key=f"{_BIND_WS}/{_BIND_USER}/data.csv",
usage_type="working_copy",
)
with pytest.raises(Exception) as exc_info:
await bind_resource(
upload_id="01UPL0000000000000000000B",
payload=_bind_payload(),
request=MagicMock(),
context=_bind_context(),
session=session,
)
assert exc_info.value.status_code == 409
assert "not created for a data resource" in exc_info.value.detail
def test_safe_path_segment_cleans_special_characters():
assert _safe_path_segment("train") == "train"
assert _safe_path_segment("train v1") == "train_v1"
@@ -328,6 +370,26 @@ def test_resource_directory_parses_object_key():
assert resource_directory("other-bucket-key.csv", ws, user) == ""
def test_object_key_tail_and_strip_uniqueness_suffix():
ws = "01WS00000000000000000000A"
user = "01USR000000000000000000A"
# 前缀剥离。
assert _object_key_tail(f"{ws}/{user}/sub/data.csv", ws, user) == "sub/data.csv"
assert _object_key_tail("no-prefix.csv", ws, user) == "no-prefix.csv"
# 带唯一化后缀的 key 恢复为原始路径(26 位 ULID 后缀)。
suffix = "01arz3ndektsv4rrffq69g5fav" # 26 chars
assert (
_strip_uniqueness_suffix(f"sub/data-{suffix}.csv") == "sub/data.csv"
)
assert _strip_uniqueness_suffix(f"data-{suffix}.csv") == "data.csv"
assert _strip_uniqueness_suffix(f"data-{suffix}") == "data"
# 普通文件名不受影响(短后缀、无连字符、目录含连字符)。
assert _strip_uniqueness_suffix("sub/data.csv") == "sub/data.csv"
assert _strip_uniqueness_suffix("data-v1.csv") == "data-v1.csv"
assert _strip_uniqueness_suffix("my-dir/data.csv") == "my-dir/data.csv"
assert _strip_uniqueness_suffix("data") == "data"
def test_data_resources_model_allows_duplicate_storage_object_reference() -> None:
"""With uk_data_resources_object dropped, no unique index covers
storage_object_id, so multiple DataResources rows may reference the