fix(backend): enhance storage/resource concurrency, idempotency, and deletion safety
Summary of changes:
- Resources Bind Idempotency (High 1+2):
- Check existing active storage object bindings before duplicate check.
- Return existing binding (`reused: true`) on retry with same upload_id.
- Filter by `status == "active"` to bypass dead/deleted rows during reuse check.
- Storage Concurrent Overwrite (High 3):
- Add `acquire_named_lock` and `release_named_lock` helpers using MySQL `GET_LOCK`/`RELEASE_LOCK` hashed to <= 64 chars.
- Wrap `upload_bytes_to_session` PUT+INSERT critical section with named lock on `object_key`.
- Re-check key collision inside lock; append ULID suffix on collision.
- Shared Reference Deletion Protection (Medium 4):
- Check active references before deleting storage objects in `delete_resource`.
- Delete only `DataResources` record if storage object is still referenced elsewhere.
- Robust Usage Type Fallback (Medium 5):
- Replace direct dict lookup for `USAGE_TYPE_TO_PURPOSE[item.usage_type]` with `.get(..., "workspace")` default.
- Idempotency Key Path Matching (Medium 6):
- Move `file_name`/`target_path` validation forward and include path dimension in comparison.
- Strip uniqueness suffix via `_strip_uniqueness_suffix` before key comparison to avoid false 409s on valid retries.
- Usage Type & Bind Concurrency Control (Low 7 & 8):
- Reject bind requests with 409 if upload session purpose is not `data_resource`.
- Wrap resource duplicate check and creation in named lock using `(owner, directory, name)`.
- Trash Key Uniqueness & Restore Compatibility (Low 9):
- Update `trash_key` format to `{purpose}/{object_key}-{storage_object_id}` to prevent collisions.
- Update `object_key_hash` on trash move.
- Update restore logic in `storage_api.py` to strip suffix while maintaining backward compatibility with legacy keys.
- Dead Code Removal (Low 10):
- Remove unreachable `upload_status = "failed"` and redundant `session.rollback()` in `IntegrityError` block.
- Tests & Mocks:
- Add/update 5 test cases covering non-data_resource bind rejection, suffix stripping, and path recovery.
- Add named lock statement mocks for DB testing.
This commit is contained in:
@@ -15,7 +15,12 @@ from backend.resources import (
|
||||
resource_directory,
|
||||
resource_payload,
|
||||
)
|
||||
from backend.services.storage import _safe_file_name, _safe_path_segment
|
||||
from backend.services.storage import (
|
||||
_object_key_tail,
|
||||
_safe_file_name,
|
||||
_safe_path_segment,
|
||||
_strip_uniqueness_suffix,
|
||||
)
|
||||
|
||||
|
||||
_BIND_WS = "01WS0000000000000000000A"
|
||||
@@ -38,21 +43,33 @@ class _BindSessionMock:
|
||||
in the workspace with the same resource_name.
|
||||
"""
|
||||
|
||||
def __init__(self, new_object_key: str, same_name_rows=(), reused_resource=None):
|
||||
def __init__(
|
||||
self,
|
||||
new_object_key: str,
|
||||
same_name_rows=(),
|
||||
reused_resource=None,
|
||||
usage_type: str = "data_resource",
|
||||
):
|
||||
self._new_object_key = new_object_key
|
||||
self._same_name_rows = list(same_name_rows)
|
||||
self._reused_resource = reused_resource
|
||||
self._usage_type = usage_type
|
||||
self._scalar_calls = 0
|
||||
|
||||
async def scalar(self, _stmt):
|
||||
from sqlalchemy import TextClause
|
||||
|
||||
if isinstance(_stmt, TextClause):
|
||||
return 1 # GET_LOCK 成功
|
||||
self._scalar_calls += 1
|
||||
# 1st scalar: UploadSessions lookup; 2nd: storage_object_id reuse check.
|
||||
# 1st scalar: UploadSessions lookup; 2nd: active 绑定行复用检查。
|
||||
if self._scalar_calls == 1:
|
||||
return SimpleNamespace(
|
||||
upload_id="01UPL0000000000000000000B",
|
||||
storage_object_id="01OBJ0000000000000000000B",
|
||||
workspace_id=_BIND_WS,
|
||||
user_id=_BIND_USER,
|
||||
usage_type=self._usage_type,
|
||||
)
|
||||
if self._scalar_calls == 2 and self._reused_resource is not None:
|
||||
return self._reused_resource
|
||||
@@ -62,6 +79,10 @@ class _BindSessionMock:
|
||||
return _make_storage_object(self._new_object_key)
|
||||
|
||||
async def execute(self, _stmt):
|
||||
from sqlalchemy import TextClause
|
||||
|
||||
if isinstance(_stmt, TextClause):
|
||||
return _ExecuteResult([]) # RELEASE_LOCK
|
||||
# 模拟 SQL 的 owner 过滤与 self-exclusion:只返回属于当前用户且
|
||||
# 不是当前 upload 已绑定对象的同名候选。
|
||||
rows = [
|
||||
@@ -222,6 +243,27 @@ async def test_bind_resource_allows_rebinding_same_storage_object() -> None:
|
||||
assert result["data"]["resource_name"] == "data.csv"
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_bind_resource_rejects_non_data_resource_upload() -> None:
|
||||
"""其他用途(如 working_copy)的 upload session 不能 bind 成数据资源。"""
|
||||
from backend.resources import bind_resource
|
||||
|
||||
session = _BindSessionMock(
|
||||
new_object_key=f"{_BIND_WS}/{_BIND_USER}/data.csv",
|
||||
usage_type="working_copy",
|
||||
)
|
||||
with pytest.raises(Exception) as exc_info:
|
||||
await bind_resource(
|
||||
upload_id="01UPL0000000000000000000B",
|
||||
payload=_bind_payload(),
|
||||
request=MagicMock(),
|
||||
context=_bind_context(),
|
||||
session=session,
|
||||
)
|
||||
assert exc_info.value.status_code == 409
|
||||
assert "not created for a data resource" in exc_info.value.detail
|
||||
|
||||
|
||||
def test_safe_path_segment_cleans_special_characters():
|
||||
assert _safe_path_segment("train") == "train"
|
||||
assert _safe_path_segment("train v1") == "train_v1"
|
||||
@@ -328,6 +370,26 @@ def test_resource_directory_parses_object_key():
|
||||
assert resource_directory("other-bucket-key.csv", ws, user) == ""
|
||||
|
||||
|
||||
def test_object_key_tail_and_strip_uniqueness_suffix():
|
||||
ws = "01WS00000000000000000000A"
|
||||
user = "01USR000000000000000000A"
|
||||
# 前缀剥离。
|
||||
assert _object_key_tail(f"{ws}/{user}/sub/data.csv", ws, user) == "sub/data.csv"
|
||||
assert _object_key_tail("no-prefix.csv", ws, user) == "no-prefix.csv"
|
||||
# 带唯一化后缀的 key 恢复为原始路径(26 位 ULID 后缀)。
|
||||
suffix = "01arz3ndektsv4rrffq69g5fav" # 26 chars
|
||||
assert (
|
||||
_strip_uniqueness_suffix(f"sub/data-{suffix}.csv") == "sub/data.csv"
|
||||
)
|
||||
assert _strip_uniqueness_suffix(f"data-{suffix}.csv") == "data.csv"
|
||||
assert _strip_uniqueness_suffix(f"data-{suffix}") == "data"
|
||||
# 普通文件名不受影响(短后缀、无连字符、目录含连字符)。
|
||||
assert _strip_uniqueness_suffix("sub/data.csv") == "sub/data.csv"
|
||||
assert _strip_uniqueness_suffix("data-v1.csv") == "data-v1.csv"
|
||||
assert _strip_uniqueness_suffix("my-dir/data.csv") == "my-dir/data.csv"
|
||||
assert _strip_uniqueness_suffix("data") == "data"
|
||||
|
||||
|
||||
def test_data_resources_model_allows_duplicate_storage_object_reference() -> None:
|
||||
"""With uk_data_resources_object dropped, no unique index covers
|
||||
storage_object_id, so multiple DataResources rows may reference the
|
||||
|
||||
Reference in New Issue
Block a user