fix: P0-5 — upload-status rollback, streaming copy (LOCAL only), user re-verify, honest lock
B1: `_mark_upload_failed_and_raise` now commits on a separate session
- Helper takes `request + upload_id`, opens a fresh session from
`request.app.state.session_factory` and commits there before raising.
- Closes the named-lock connection-pool leak Codex flagged: the old
"commit-on-the-same-session" implementation could return the
GET_LOCK connection to the pool before the enclosing
`finally: release_named_lock` ran, leaking `mp:<hash>` for up to
`pool_recycle` and re-opening the same-key upload race.
- Same helper now used by `create_server_object_payload`'s put-failure
branch — two failure paths have identical semantics.
B2: streaming copy for soft-delete + restore (`get_stream() + put()`)
- LOCAL backend: zero-copy (aiofiles stream write). OOM fixed.
- S3 backend: still OOMs on multi-GB objects — `put()` materializes
the async iter via `b"".join(chunks)`. Multipart `put` is a
follow-up; do NOT claim "OOM fixed on production" since production
defaults to S3.
C1: worker re-verifies `Users.status='active' AND is_deleted=0`
- `_assert_user_active` called from `_execution_context` after
resolving `triggered_by`; skips `SYSTEM_CRON_USER_ID`.
- `USER_DISABLED` error_code goes into the `NODE_FINISHED_EVENT`
outbox payload — `schedule_node_runs` has no `error_code` column,
the row only carries the `message` text. Docstrings corrected to
say so explicitly (previous docstring falsely promised row-level
observability).
F1: honest browser-local file lock
- `api.ts` `acquireFileLock/heartbeatFileLock/releaseFileLock/
releaseFileLockOnUnload` are now no-ops with comments stating they
never call the network.
- `scriptWorkspaceStore` dropped `tickHeartbeats`; `tickCleanup`
simplified to just clear cache.
- `useEditSessionLifecycle` dropped its 15s heartbeat `setInterval`.
- `ScriptWorkspace.tsx` renders `.local-lock-banner` info bar when
`isEditing`. Two tabs may still silently last-write — banner is the
only guard (acceptable disclosure-only tradeoff).
Dead code: deleted the duplicate `upload_bytes_to_session` in
`backend/src/backend/storage_api.py`. The `services.storage` import
is now the only source of the function; `create_upload_record`'s
docstring updated to point at `backend.resources`.
This commit is contained in:
@@ -498,6 +498,63 @@ async def test_soft_delete_object_sets_is_deleted() -> None:
|
||||
assert result["data"]["object_status"] == "deleted"
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_soft_delete_object_streams_via_get_stream() -> None:
|
||||
"""P0-5 / B2: 必须走 ``get_stream()`` 流式迁移,不能调 ``get()``
|
||||
把整个对象加载到内存(10G 对象会 OOM)。"""
|
||||
from backend.services.storage import soft_delete_object
|
||||
|
||||
item = _storage_object_row()
|
||||
# 让 storage_backend 与 settings 一致,确保走"移动到 trash"分支
|
||||
item.storage_backend = settings.storage_backend
|
||||
item.object_status = "available"
|
||||
item.bucket_name = "workspace"
|
||||
item.object_key = "ws/u/file.bin"
|
||||
item.usage_type = "working_copy"
|
||||
|
||||
source_store = MagicMock()
|
||||
# 模拟一个 async generator 作为 get_stream 的返回值
|
||||
async def _fake_stream(_key, _chunk_size=65536):
|
||||
yield b"chunk-1"
|
||||
yield b"chunk-2"
|
||||
source_store.get_stream.side_effect = _fake_stream
|
||||
source_store.put = AsyncMock()
|
||||
source_store.delete = AsyncMock()
|
||||
|
||||
trash_store = MagicMock()
|
||||
trash_store.put = AsyncMock()
|
||||
trash_store.delete = AsyncMock()
|
||||
|
||||
request = MagicMock()
|
||||
request.app.state.object_stores = {
|
||||
item.bucket_name: source_store,
|
||||
"trash": trash_store,
|
||||
}
|
||||
|
||||
session = AsyncMock()
|
||||
session.scalar = AsyncMock(return_value=item)
|
||||
|
||||
# 捕获原 key —— helper 在移动后会把 item.object_key 重写成 trash_key,
|
||||
# 不捕获的话下面断言会拿不到原值。
|
||||
original_object_key = item.object_key
|
||||
|
||||
await soft_delete_object(
|
||||
storage_object_id=item.storage_object_id,
|
||||
request=request,
|
||||
session=session,
|
||||
)
|
||||
|
||||
# get_stream 必须被调用;get() 不应被调用 —— 否则仍是全量加载路径
|
||||
source_store.get_stream.assert_called_once_with(original_object_key)
|
||||
source_store.get.assert_not_called()
|
||||
# 源对象删除 —— 同样用原始 key 断言(item.object_key 已被 helper 改写)
|
||||
source_store.delete.assert_awaited_once_with(original_object_key)
|
||||
# 目标桶写入走 put(接受 async iter),trash_key 尾部拼 storage_object_id
|
||||
trash_store.put.assert_awaited_once()
|
||||
put_args, _ = trash_store.put.call_args
|
||||
assert put_args[0] == f"workspace/{original_object_key}-{item.storage_object_id}"
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_jupyter_check_notebook_lock_ignores_deleted_scripts() -> None:
|
||||
"""``is_deleted == 0`` filter must hide deleted notebooks from Jupyter checks."""
|
||||
|
||||
@@ -0,0 +1,149 @@
|
||||
"""Unit tests for ``upload_bytes_to_session`` failure-path status persistence.
|
||||
|
||||
P0-5 / B1: the route handler wraps every request in ``session_scope``
|
||||
(``backend.dependencies.database_session``), which rolls back on
|
||||
exception. A naive ``upload.upload_status = "failed"; raise HTTPException(...)``
|
||||
loses the status flip and leaves the row stuck in ``created``/``uploading``
|
||||
forever. The fix is ``_mark_upload_failed_and_raise`` which opens a
|
||||
*separate* session from ``request.app.state.session_factory`` and commits
|
||||
the status change on it before raising. The outer rollback is then a
|
||||
no-op on the caller's session, AND the named-lock connection (acquired
|
||||
inside the helper's caller for the size/hash/storage failure branches)
|
||||
stays pristine so ``release_named_lock`` runs on the same connection that
|
||||
ran ``GET_LOCK`` — closing the lock-leak window Codex flagged.
|
||||
|
||||
These tests exercise the helper directly + the expired-status branch of
|
||||
``upload_bytes_to_session`` with a mocked session.
|
||||
"""
|
||||
from __future__ import annotations
|
||||
|
||||
from datetime import datetime, timedelta
|
||||
from types import SimpleNamespace
|
||||
from unittest.mock import AsyncMock, MagicMock
|
||||
|
||||
import pytest
|
||||
from fastapi import HTTPException
|
||||
|
||||
from backend.services.storage import (
|
||||
_mark_upload_failed_and_raise,
|
||||
upload_bytes_to_session,
|
||||
)
|
||||
|
||||
|
||||
def _make_upload(upload_id: str = "01UPL0000000000000000000A") -> SimpleNamespace:
|
||||
"""Minimal UploadSessions row with the columns the helper touches."""
|
||||
return SimpleNamespace(
|
||||
upload_id=upload_id,
|
||||
upload_status="uploading",
|
||||
expires_at=None,
|
||||
expected_size_bytes=None,
|
||||
expected_hash=None,
|
||||
bucket_name="workspace",
|
||||
object_key="ws/user/file.bin",
|
||||
object_key_hash=b"\x00" * 32,
|
||||
)
|
||||
|
||||
|
||||
class _AsyncContextManager:
|
||||
"""Async context manager that yields ``session`` on enter.
|
||||
|
||||
Used to mock the result of ``session_factory()`` without pulling in
|
||||
a real SQLAlchemy engine.
|
||||
"""
|
||||
|
||||
def __init__(self, session: object) -> None:
|
||||
self._session = session
|
||||
|
||||
async def __aenter__(self) -> object:
|
||||
return self._session
|
||||
|
||||
async def __aexit__(self, *_args: object) -> None:
|
||||
return None
|
||||
|
||||
|
||||
async def test_mark_upload_failed_and_raise_persists_status() -> None:
|
||||
"""The helper must open a fresh session, mark 'failed', commit, then
|
||||
raise — surviving both the surrounding ``session_scope`` rollback and
|
||||
the named-lock connection-pool leak the original implementation opened
|
||||
up by committing on the lock-bound session.
|
||||
"""
|
||||
upload = _make_upload()
|
||||
fresh_session = SimpleNamespace(
|
||||
scalar=AsyncMock(return_value=upload),
|
||||
commit=AsyncMock(),
|
||||
)
|
||||
session_factory = MagicMock(return_value=_AsyncContextManager(fresh_session))
|
||||
request = SimpleNamespace(
|
||||
app=SimpleNamespace(state=SimpleNamespace(session_factory=session_factory)),
|
||||
)
|
||||
|
||||
with pytest.raises(HTTPException) as excinfo:
|
||||
await _mark_upload_failed_and_raise(
|
||||
request, upload.upload_id, 409, "size mismatch",
|
||||
)
|
||||
|
||||
assert excinfo.value.status_code == 409
|
||||
assert excinfo.value.detail == "size mismatch"
|
||||
assert upload.upload_status == "failed"
|
||||
# The fresh session — not the caller's — must have committed.
|
||||
fresh_session.scalar.assert_awaited_once()
|
||||
fresh_session.commit.assert_awaited_once()
|
||||
session_factory.assert_called_once()
|
||||
|
||||
|
||||
async def test_mark_upload_failed_and_raise_raises_even_if_upload_missing() -> None:
|
||||
"""If the row has been hard-deleted between the caller's lookup and the
|
||||
helper's separate-session write, the helper still raises HTTPException
|
||||
with the requested status/detail — it just skips the commit. The caller
|
||||
still gets the same error contract.
|
||||
"""
|
||||
fresh_session = SimpleNamespace(
|
||||
scalar=AsyncMock(return_value=None),
|
||||
commit=AsyncMock(),
|
||||
)
|
||||
session_factory = MagicMock(return_value=_AsyncContextManager(fresh_session))
|
||||
request = SimpleNamespace(
|
||||
app=SimpleNamespace(state=SimpleNamespace(session_factory=session_factory)),
|
||||
)
|
||||
|
||||
with pytest.raises(HTTPException) as excinfo:
|
||||
await _mark_upload_failed_and_raise(
|
||||
request, "01UPL0000000000000000000A", 503, "boom",
|
||||
)
|
||||
|
||||
assert excinfo.value.status_code == 503
|
||||
assert excinfo.value.detail == "boom"
|
||||
# No row to update, so commit must NOT have been called.
|
||||
fresh_session.commit.assert_not_awaited()
|
||||
|
||||
|
||||
async def test_upload_bytes_to_session_expired_commits_status() -> None:
|
||||
"""The early ``if upload.expires_at < now`` branch must also
|
||||
commit the expired status, otherwise a client retrying after the
|
||||
expiry window would see ``created`` again and re-upload.
|
||||
|
||||
The expired branch is OUTSIDE the named-lock critical section, so it
|
||||
can still commit on the caller's session safely — no separate-session
|
||||
detour is needed.
|
||||
"""
|
||||
expired_upload = _make_upload()
|
||||
expired_upload.upload_status = "created"
|
||||
# expires_at < utcnow_naive() triggers the expired branch.
|
||||
expired_upload.expires_at = datetime.utcnow() - timedelta(minutes=1)
|
||||
|
||||
session = SimpleNamespace(
|
||||
scalar=AsyncMock(return_value=expired_upload),
|
||||
commit=AsyncMock(),
|
||||
get=AsyncMock(return_value=None),
|
||||
)
|
||||
request = SimpleNamespace(
|
||||
app=SimpleNamespace(state=SimpleNamespace(object_stores={})),
|
||||
)
|
||||
|
||||
with pytest.raises(HTTPException) as excinfo:
|
||||
await upload_bytes_to_session("01UPL0000000000000000000A", session, request)
|
||||
|
||||
assert excinfo.value.status_code == 409
|
||||
assert "expired" in excinfo.value.detail
|
||||
assert expired_upload.upload_status == "expired"
|
||||
session.commit.assert_awaited()
|
||||
Reference in New Issue
Block a user