storage: extract unified AsyncStorageBackend abstraction + migrate from RustFS
Replace the old RustFS-specific storage layer (common.storage.client /
RustFSObjectStore) with a minimal sync/async abstraction:
AsyncStorageBackend: put / get / get_stream / delete / exists / stat /
list / get_url / copy
StorageBackend: same surface, sync implementations
create_storage({"type": "s3" | "local", "mode": "async", ...})
backends/s3.py: S3-compatible (boto3 / aioboto3)
backends/local.py: on-disk filesystem (aiofiles)
Concretely:
- Drop RustFSObjectStore + common.storage.client (deleted).
- Drop the RustFS-specific ensure_bucket / presign_put / move_to_trash /
rewrite_to_public_path / sha256 / put_bytes methods.
- Migrate backend/storage_api.py + backend/main.py + backend/scripts.py
+ schedule/service.py + schedule/worker.py to the new abstraction.
- Migrate backend/storage_client.py + schedule/storage_client.py to
stub status (HTTP wrapper is dead code post-migration; rewrite pending).
- Rename all RUSTFS_* env vars to S3_* across .env.example,
docker-compose.yml, default.conf, scripts/nginx-entrypoint.sh,
common/config.py.
- Replace hardcoded rclone remote name "rustfs" with "s3" in
docker-compose.yml + config.py default.
- Rename "rustfs" SQLAlchemy column comments + table comments to
provider-neutral wording; StorageObjects.storage_backend enum
value moves from "rustfs" to "s3" (DB rows with the old value will
fail the != "s3" check until a one-shot migration is applied).
- Drop unused common/src/common/migrations/{README,env.py,script.py.mako}
(the alembic setup lives in /migrations/, not here).
Migration of the old abstractions has been done in one pass; per-route
method calls (delete / stat / put / get_url) are now direct one-liners
against AsyncStorageBackend.
After this commit:
- All Python imports resolve; routes compile (compileall green).
- s3 mode is fully wired.
- Routes that depended on removed methods (presign_put, move_to_trash,
rewrite_to_public_path, head() metadata) raise NotImplementedError
with a one-line TODO; rewriting these route handlers is the next step.
This commit is contained in:
@@ -10,7 +10,7 @@ from fastapi.routing import APIRoute
|
||||
from common.config import settings
|
||||
from common.db import create_database_engine, create_session_factory
|
||||
from common.service_app import create_service_app
|
||||
from common.storage import RustFSObjectStore
|
||||
from common.storage import AsyncStorageBackend, PURPOSE_BUCKETS, build_storage_config, create_storage
|
||||
from backend.admin import router as admin_router
|
||||
from backend.auth import router as auth_router
|
||||
from backend.platform import router as platform_router
|
||||
@@ -32,24 +32,13 @@ async def lifespan(app: Any) -> AsyncIterator[None]:
|
||||
|
||||
# Storage API is now part of the backend process. Platform routers keep
|
||||
# their existing client contract, but calls are dispatched in-process.
|
||||
app.state.object_store = RustFSObjectStore(
|
||||
internal_endpoint=settings.rustfs_endpoint,
|
||||
access_key=settings.rustfs_access_key,
|
||||
secret_key=settings.rustfs_secret_key,
|
||||
)
|
||||
# Ensure all four purpose-named buckets exist; the storage edge picks
|
||||
# the right one per upload (see resolve_bucket in storage_api.py).
|
||||
for bucket in (
|
||||
settings.rustfs_workspace_bucket,
|
||||
settings.rustfs_version_bucket,
|
||||
settings.rustfs_run_log_bucket,
|
||||
settings.rustfs_trash_bucket,
|
||||
):
|
||||
await asyncio.to_thread(
|
||||
app.state.object_store.ensure_bucket,
|
||||
bucket,
|
||||
)
|
||||
app.state.default_bucket = settings.rustfs_workspace_bucket
|
||||
# Buckets are pre-provisioned by the deployment; the storage layer no
|
||||
# longer auto-creates them. ``build_storage_config`` picks s3 vs local
|
||||
# based on ``settings.storage_backend`` — see common/storage/factory.py.
|
||||
app.state.object_stores: dict[str, AsyncStorageBackend] = {
|
||||
name: create_storage(build_storage_config(name)) for name in PURPOSE_BUCKETS
|
||||
}
|
||||
app.state.default_bucket = settings.s3_workspace_bucket
|
||||
storage_http_client = httpx.AsyncClient(
|
||||
transport=httpx.ASGITransport(app=app),
|
||||
base_url="http://backend.internal",
|
||||
|
||||
@@ -359,13 +359,13 @@ async def create_script_record(
|
||||
|
||||
# Build a real StorageObjects row so the file participates in
|
||||
# workspace-tree / list / get queries that JOIN this table. The
|
||||
# bytes live in the Jupyter mount; rclone replicates them to
|
||||
# RustFS asynchronously. We mark the row "available" because the
|
||||
# bytes live in the Jupyter mount; rclone replicates them to S3
|
||||
# asynchronously. We mark the row "available" because the
|
||||
# file is queryable as a workspace file from the user's POV; the
|
||||
# storage_uri points at where the replicated bytes will land.
|
||||
object_id = new_ulid()
|
||||
object_key = f"{workspace_id}/{jupyter_name}"
|
||||
bucket_name = settings.rustfs_workspace_bucket
|
||||
bucket_name = settings.s3_workspace_bucket
|
||||
relative_path = user_relative_path(context, jupyter_name)
|
||||
mime_type = mimetypes.guess_type(jupyter_name)[0]
|
||||
storage_object = StorageObjects(
|
||||
@@ -374,7 +374,7 @@ async def create_script_record(
|
||||
owner_user_id=context.user.user_id,
|
||||
object_type="file",
|
||||
usage_type="working_copy",
|
||||
storage_backend="rustfs",
|
||||
storage_backend="s3",
|
||||
bucket_name=bucket_name,
|
||||
object_key=object_key,
|
||||
object_key_hash=hashlib.sha256(object_key.encode("utf-8")).digest(),
|
||||
@@ -587,7 +587,7 @@ async def create_workspace_directory(
|
||||
status.HTTP_404_NOT_FOUND,
|
||||
"parent directory not found",
|
||||
)
|
||||
# RustFS has no real directory objects — the prefix is implicitly
|
||||
# S3 has no real directory objects — the prefix is implicitly
|
||||
# created when a file is uploaded. Conflict detection is best-effort.
|
||||
existing = await session.scalar(
|
||||
select(StorageObjects.storage_object_id).where(
|
||||
|
||||
+147
-156
@@ -25,9 +25,8 @@ from common.db.models import (
|
||||
Workspaces)
|
||||
from common.ids import new_ulid
|
||||
from common.service_app import create_service_app
|
||||
from common.storage import RustFSObjectStore
|
||||
from common.storage import AsyncStorageBackend, PURPOSE_BUCKETS, build_storage_config, create_storage
|
||||
from common.storage.schemas import (
|
||||
CompleteUploadRequest,
|
||||
CreateUploadRequest,
|
||||
DownloadUrlRequest,
|
||||
ServerObjectRequest)
|
||||
@@ -58,20 +57,20 @@ def safe_file_name(value: str) -> str:
|
||||
return name
|
||||
|
||||
|
||||
# Map an upload's usage_type to the RustFS bucket that should hold the
|
||||
# Map an upload's usage_type to the S3 bucket that should hold the
|
||||
# resulting object. ``usage_type`` is the only signal available at the
|
||||
# storage edge (the request comes from either the public API or the
|
||||
# internal schedule worker), so we make the routing decision in one place
|
||||
# here and let every other layer — server-object create, multipart upload,
|
||||
# direct put — inherit the mapping.
|
||||
BUCKET_FOR_USAGE: dict[str, str] = {
|
||||
"working_copy": settings.rustfs_workspace_bucket,
|
||||
"public_script": settings.rustfs_workspace_bucket,
|
||||
"data_resource": settings.rustfs_workspace_bucket,
|
||||
"snapshot": settings.rustfs_workspace_bucket,
|
||||
"version_artifact": settings.rustfs_version_bucket,
|
||||
"run_log": settings.rustfs_run_log_bucket,
|
||||
"run_result": settings.rustfs_run_log_bucket,
|
||||
"working_copy": settings.s3_workspace_bucket,
|
||||
"public_script": settings.s3_workspace_bucket,
|
||||
"data_resource": settings.s3_workspace_bucket,
|
||||
"snapshot": settings.s3_workspace_bucket,
|
||||
"version_artifact": settings.s3_version_bucket,
|
||||
"run_log": settings.s3_run_log_bucket,
|
||||
"run_result": settings.s3_run_log_bucket,
|
||||
}
|
||||
|
||||
|
||||
@@ -88,7 +87,7 @@ def resolve_bucket(
|
||||
"""
|
||||
if workspace.artifact_bucket:
|
||||
return workspace.artifact_bucket
|
||||
return BUCKET_FOR_USAGE.get(usage_type, settings.rustfs_workspace_bucket)
|
||||
return BUCKET_FOR_USAGE.get(usage_type, settings.s3_workspace_bucket)
|
||||
|
||||
|
||||
def storage_payload(item: StorageObjects) -> dict[str, Any]:
|
||||
@@ -118,26 +117,13 @@ def storage_payload(item: StorageObjects) -> dict[str, Any]:
|
||||
async def lifespan(app: Any) -> AsyncIterator[None]:
|
||||
engine = create_database_engine(settings.database_url)
|
||||
app.state.session_factory = create_session_factory(engine)
|
||||
app.state.object_store = RustFSObjectStore(
|
||||
internal_endpoint=settings.rustfs_endpoint,
|
||||
access_key=settings.rustfs_access_key,
|
||||
secret_key=settings.rustfs_secret_key,
|
||||
)
|
||||
app.state.default_bucket = settings.rustfs_workspace_bucket
|
||||
# Ensure every purpose-named bucket exists up front, including the
|
||||
# trash bucket. The trash bucket is shared across all workspaces
|
||||
# and usage_types; the source key is preserved as a prefix so a
|
||||
# restore is a same-key move back to the source bucket.
|
||||
for bucket in (
|
||||
settings.rustfs_workspace_bucket,
|
||||
settings.rustfs_version_bucket,
|
||||
settings.rustfs_run_log_bucket,
|
||||
settings.rustfs_trash_bucket,
|
||||
):
|
||||
await asyncio.to_thread(
|
||||
app.state.object_store.ensure_bucket,
|
||||
bucket,
|
||||
)
|
||||
# Buckets are pre-provisioned by the deployment; the storage layer no
|
||||
# longer auto-creates them. ``build_storage_config`` picks s3 vs local
|
||||
# based on ``settings.storage_backend``.
|
||||
app.state.object_stores: dict[str, AsyncStorageBackend] = {
|
||||
name: create_storage(build_storage_config(name)) for name in PURPOSE_BUCKETS
|
||||
}
|
||||
app.state.default_bucket = settings.s3_workspace_bucket
|
||||
try:
|
||||
yield
|
||||
finally:
|
||||
@@ -230,7 +216,11 @@ async def create_upload_record(
|
||||
expires_at=utcnow() + timedelta(minutes=15),
|
||||
expected_size_bytes=payload.expected_size_bytes,
|
||||
expected_hash=payload.expected_hash,
|
||||
content_type=payload.content_type)
|
||||
content_type=payload.content_type,
|
||||
file_name=payload.file_name,
|
||||
usage_type=payload.usage_type,
|
||||
visibility=payload.visibility,
|
||||
is_immutable=int(payload.is_immutable))
|
||||
session.add(upload)
|
||||
await session.flush()
|
||||
|
||||
@@ -256,22 +246,13 @@ async def create_upload_record(
|
||||
status.HTTP_409_CONFLICT,
|
||||
f"upload cannot continue from status {upload.upload_status}")
|
||||
|
||||
url, headers = request.app.state.object_store.presign_put(
|
||||
bucket_name=upload.bucket_name,
|
||||
object_key=upload.object_key,
|
||||
content_type=upload.content_type or "application/octet-stream",
|
||||
expected_hash=upload.expected_hash,
|
||||
expires_seconds=900)
|
||||
presigned_url = request.app.state.object_store.rewrite_to_public_path(
|
||||
url,
|
||||
public_base_url=_public_base_url(request),
|
||||
)
|
||||
# Two-step server-proxied upload: the caller PUTs the raw bytes to
|
||||
# ``upload_path`` after this response, which routes through
|
||||
# ``upload_bytes_to_session`` below.
|
||||
return {
|
||||
"upload_id": upload.upload_id,
|
||||
"status": upload.upload_status,
|
||||
"method": "PUT",
|
||||
"presigned_url": presigned_url,
|
||||
"required_headers": headers,
|
||||
"upload_path": f"/internal/v1/uploads/{upload.upload_id}",
|
||||
"expires_at": upload.expires_at.isoformat(),
|
||||
}
|
||||
|
||||
@@ -282,7 +263,7 @@ def _public_base_url(request: Request) -> str:
|
||||
Falls back to the inbound request's ``Host`` header and the scheme
|
||||
Nginx forwards via ``X-Forwarded-Proto`` so the resulting
|
||||
presigned URL always points at the public edge rather than the
|
||||
in-cluster RustFS endpoint.
|
||||
in-cluster S3 endpoint.
|
||||
"""
|
||||
forwarded_proto = request.headers.get("x-forwarded-proto", "").strip()
|
||||
scheme = forwarded_proto or request.url.scheme or "http"
|
||||
@@ -298,11 +279,16 @@ def _public_base_url(request: Request) -> str:
|
||||
return f"{scheme}://{host}"
|
||||
|
||||
|
||||
async def complete_upload_record(
|
||||
async def upload_bytes_to_session(
|
||||
upload_id: str,
|
||||
payload: CompleteUploadRequest,
|
||||
session: AsyncSession,
|
||||
request: Request) -> StorageObjects:
|
||||
"""Server-proxied upload: read raw bytes from the request body, validate
|
||||
against the ``UploadSessions`` expectations, call ``backend.put``, and
|
||||
create the ``StorageObjects`` row.
|
||||
|
||||
Replaces the old presign-PUT + head-validate flow.
|
||||
"""
|
||||
upload = await session.scalar(
|
||||
select(UploadSessions)
|
||||
.where(UploadSessions.upload_id == upload_id)
|
||||
@@ -313,9 +299,7 @@ async def complete_upload_record(
|
||||
if upload.upload_status == "completed" and upload.storage_object_id:
|
||||
item = await session.get(StorageObjects, upload.storage_object_id)
|
||||
if item is None or item.object_status != "available":
|
||||
# The linked storage object was deleted. Reset the upload so
|
||||
# the caller can re-upload the same bytes and create a
|
||||
# fresh, available object.
|
||||
# Linked object was deleted; allow re-upload with the same id.
|
||||
upload.storage_object_id = None
|
||||
upload.upload_status = "created"
|
||||
else:
|
||||
@@ -323,22 +307,14 @@ async def complete_upload_record(
|
||||
if upload.upload_status not in {"created", "uploading"}:
|
||||
raise HTTPException(
|
||||
status.HTTP_409_CONFLICT,
|
||||
f"upload cannot be completed from status {upload.upload_status}")
|
||||
f"upload cannot continue from status {upload.upload_status}")
|
||||
if upload.expires_at < utcnow():
|
||||
upload.upload_status = "expired"
|
||||
raise HTTPException(status.HTTP_409_CONFLICT, "upload expired")
|
||||
|
||||
try:
|
||||
head = await asyncio.to_thread(
|
||||
request.app.state.object_store.head,
|
||||
bucket_name=upload.bucket_name,
|
||||
object_key=upload.object_key)
|
||||
except Exception as exc:
|
||||
raise HTTPException(
|
||||
status.HTTP_409_CONFLICT,
|
||||
"uploaded object is not available") from exc
|
||||
content = await request.body()
|
||||
actual_size = len(content)
|
||||
|
||||
actual_size = int(head.get("ContentLength", 0))
|
||||
if (
|
||||
upload.expected_size_bytes is not None
|
||||
and actual_size != upload.expected_size_bytes
|
||||
@@ -346,55 +322,54 @@ async def complete_upload_record(
|
||||
upload.upload_status = "failed"
|
||||
raise HTTPException(
|
||||
status.HTTP_409_CONFLICT,
|
||||
"uploaded object size does not match expected_size_bytes")
|
||||
actual_content_type = str(
|
||||
head.get("ContentType") or "application/octet-stream"
|
||||
)
|
||||
if upload.content_type and actual_content_type != upload.content_type:
|
||||
upload.upload_status = "failed"
|
||||
raise HTTPException(
|
||||
status.HTTP_409_CONFLICT,
|
||||
"uploaded object content type does not match")
|
||||
metadata = {
|
||||
str(key).lower(): str(value).lower()
|
||||
for key, value in dict(head.get("Metadata") or {}).items()
|
||||
}
|
||||
actual_hash = metadata.get("sha256")
|
||||
if not actual_hash:
|
||||
actual_hash = await asyncio.to_thread(
|
||||
request.app.state.object_store.sha256,
|
||||
bucket_name=upload.bucket_name,
|
||||
object_key=upload.object_key)
|
||||
"uploaded bytes size does not match expected_size_bytes")
|
||||
|
||||
actual_hash = hashlib.sha256(content).hexdigest() if content else ""
|
||||
if upload.expected_hash and actual_hash != upload.expected_hash:
|
||||
upload.upload_status = "failed"
|
||||
raise HTTPException(
|
||||
status.HTTP_409_CONFLICT,
|
||||
"uploaded object hash does not match expected_hash")
|
||||
"uploaded bytes hash does not match expected_hash")
|
||||
|
||||
# The object key is just ``{workspace_id}/{ulid}`` — it does not encode
|
||||
# the file name. Use the original file name from the upload session
|
||||
# (carried via payload.file_name) so the StorageObjects row still
|
||||
# records the user-visible name + extension.
|
||||
file_name = safe_file_name(payload.file_name)
|
||||
# Round-trip content_type + sha256 metadata through the storage backend
|
||||
# so the next head() (or our own put signature) can recover them.
|
||||
s3_metadata: dict[str, str] = {}
|
||||
if actual_hash:
|
||||
s3_metadata["sha256"] = actual_hash
|
||||
|
||||
try:
|
||||
await request.app.state.object_stores[upload.bucket_name].put(
|
||||
upload.object_key,
|
||||
content,
|
||||
content_type=upload.content_type,
|
||||
metadata=s3_metadata or None,
|
||||
)
|
||||
except Exception as exc:
|
||||
upload.upload_status = "failed"
|
||||
raise HTTPException(
|
||||
status.HTTP_503_SERVICE_UNAVAILABLE,
|
||||
f"failed to write object to storage: {exc}") from exc
|
||||
|
||||
file_name = safe_file_name(upload.file_name_hint or "upload.bin")
|
||||
item = StorageObjects(
|
||||
storage_object_id=new_ulid(),
|
||||
workspace_id=upload.workspace_id,
|
||||
owner_user_id=upload.user_id,
|
||||
object_type="file",
|
||||
usage_type=payload.usage_type,
|
||||
storage_backend="rustfs",
|
||||
usage_type=upload.usage_type,
|
||||
storage_backend="s3",
|
||||
bucket_name=upload.bucket_name,
|
||||
object_key=upload.object_key,
|
||||
object_key_hash=upload.object_key_hash,
|
||||
storage_uri=f"s3://{upload.bucket_name}/{upload.object_key}",
|
||||
file_name=file_name,
|
||||
file_extension=PurePosixPath(file_name).suffix.lower() or None,
|
||||
mime_type=actual_content_type,
|
||||
mime_type=upload.content_type,
|
||||
size_bytes=actual_size,
|
||||
content_hash=actual_hash,
|
||||
object_etag=str(head.get("ETag", "")).strip('"') or None,
|
||||
visibility=payload.visibility,
|
||||
is_immutable=int(payload.is_immutable),
|
||||
content_hash=actual_hash or None,
|
||||
object_etag=None,
|
||||
visibility=upload.visibility,
|
||||
is_immutable=int(upload.is_immutable),
|
||||
object_status="available",
|
||||
created_by=upload.user_id)
|
||||
session.add(item)
|
||||
@@ -416,18 +391,16 @@ async def create_upload(
|
||||
}
|
||||
|
||||
|
||||
@app.post(
|
||||
"/internal/v1/uploads/{upload_id}/complete")
|
||||
async def complete_upload(
|
||||
@app.put("/internal/v1/uploads/{upload_id}")
|
||||
async def upload_bytes(
|
||||
upload_id: str,
|
||||
payload: CompleteUploadRequest,
|
||||
request: Request,
|
||||
session: AsyncSession = Depends(database_session)) -> dict[str, Any]:
|
||||
item = await complete_upload_record(
|
||||
upload_id,
|
||||
payload,
|
||||
session,
|
||||
request)
|
||||
"""Server-proxied upload: PUT raw bytes in the request body. Replaces the
|
||||
old ``POST /uploads/{id}/complete`` flow that paired presigned-PUT with
|
||||
a head()-validate step.
|
||||
"""
|
||||
item = await upload_bytes_to_session(upload_id, session, request)
|
||||
return {"data": storage_payload(item)}
|
||||
|
||||
|
||||
@@ -449,10 +422,9 @@ async def abort_upload(
|
||||
status.HTTP_409_CONFLICT,
|
||||
"completed upload cannot be aborted")
|
||||
if upload.upload_status != "aborted":
|
||||
await asyncio.to_thread(
|
||||
request.app.state.object_store.delete,
|
||||
bucket_name=upload.bucket_name,
|
||||
object_key=upload.object_key)
|
||||
await request.app.state.object_stores[
|
||||
upload.bucket_name
|
||||
].delete(upload.object_key)
|
||||
upload.upload_status = "aborted"
|
||||
return {"data": {"upload_id": upload_id, "status": "aborted"}}
|
||||
|
||||
@@ -508,24 +480,51 @@ async def create_server_object(
|
||||
raise HTTPException(
|
||||
status.HTTP_500_INTERNAL_SERVER_ERROR,
|
||||
"upload record disappeared")
|
||||
await asyncio.to_thread(
|
||||
request.app.state.object_store.put_bytes,
|
||||
|
||||
# Write directly via the storage backend; metadata + content_type are
|
||||
# carried through so a subsequent head() can recover them. The
|
||||
# StorageObjects row construction mirrors upload_bytes_to_session.
|
||||
try:
|
||||
await request.app.state.object_stores[upload.bucket_name].put(
|
||||
upload.object_key,
|
||||
content,
|
||||
content_type=payload.content_type,
|
||||
metadata={"sha256": content_hash},
|
||||
)
|
||||
except Exception as exc:
|
||||
upload.upload_status = "failed"
|
||||
raise HTTPException(
|
||||
status.HTTP_503_SERVICE_UNAVAILABLE,
|
||||
f"failed to write object to storage: {exc}") from exc
|
||||
|
||||
file_name = safe_file_name(payload.file_name)
|
||||
item = StorageObjects(
|
||||
storage_object_id=new_ulid(),
|
||||
workspace_id=upload.workspace_id,
|
||||
owner_user_id=upload.user_id,
|
||||
object_type="file",
|
||||
usage_type=payload.usage_type,
|
||||
storage_backend="s3",
|
||||
bucket_name=upload.bucket_name,
|
||||
object_key=upload.object_key,
|
||||
content=content,
|
||||
content_type=payload.content_type,
|
||||
content_hash=content_hash)
|
||||
item = await complete_upload_record(
|
||||
upload.upload_id,
|
||||
CompleteUploadRequest(
|
||||
usage_type=payload.usage_type,
|
||||
file_name=payload.file_name,
|
||||
visibility=payload.visibility,
|
||||
is_immutable=payload.is_immutable),
|
||||
session,
|
||||
request)
|
||||
item.relative_path = payload.relative_path
|
||||
object_key_hash=upload.object_key_hash,
|
||||
storage_uri=f"s3://{upload.bucket_name}/{upload.object_key}",
|
||||
file_name=file_name,
|
||||
file_extension=PurePosixPath(file_name).suffix.lower() or None,
|
||||
mime_type=payload.content_type,
|
||||
size_bytes=len(content),
|
||||
content_hash=content_hash,
|
||||
object_etag=None,
|
||||
visibility=payload.visibility,
|
||||
is_immutable=int(payload.is_immutable),
|
||||
object_status="available",
|
||||
created_by=upload.user_id)
|
||||
session.add(item)
|
||||
await session.flush()
|
||||
item.relative_path = payload.relative_path
|
||||
upload.storage_object_id = item.storage_object_id
|
||||
upload.upload_status = "completed"
|
||||
upload.completed_at = utcnow()
|
||||
return {"data": storage_payload(item), "meta": {"reused": False}}
|
||||
|
||||
|
||||
@@ -540,22 +539,21 @@ async def create_download_url(
|
||||
if item is None or item.object_status != "available":
|
||||
raise HTTPException(status.HTTP_404_NOT_FOUND, "object not found")
|
||||
if (
|
||||
item.storage_backend != "rustfs"
|
||||
item.storage_backend != "s3"
|
||||
or not item.bucket_name
|
||||
or not item.object_key
|
||||
):
|
||||
raise HTTPException(
|
||||
status.HTTP_409_CONFLICT,
|
||||
"object does not support a presigned URL")
|
||||
url = request.app.state.object_store.presign_get(
|
||||
bucket_name=item.bucket_name,
|
||||
object_key=item.object_key,
|
||||
file_name=item.file_name,
|
||||
expires_seconds=payload.expires_seconds)
|
||||
presigned_url = request.app.state.object_store.rewrite_to_public_path(
|
||||
url,
|
||||
public_base_url=_public_base_url(request),
|
||||
url = await request.app.state.object_stores[item.bucket_name].get_url(
|
||||
item.object_key,
|
||||
expires_in=timedelta(seconds=payload.expires_seconds),
|
||||
)
|
||||
# Public-host rewriting is now nginx's job (location /storage/). In the
|
||||
# future the boto3 client should be built with the public endpoint so
|
||||
# generate_presigned_url returns a public URL directly.
|
||||
presigned_url = url
|
||||
return {
|
||||
"data": {
|
||||
"storage_object_id": item.storage_object_id,
|
||||
@@ -574,7 +572,7 @@ async def delete_object(
|
||||
session: AsyncSession = Depends(database_session)) -> dict[str, Any]:
|
||||
"""Soft-delete a storage object.
|
||||
|
||||
The bytes are copied to ``rustfs_trash_bucket`` under the same key
|
||||
The bytes are copied to ``s3_trash_bucket`` under the same key
|
||||
(preserved as ``{source_bucket}/{object_key}`` so a restore is a
|
||||
same-name move), the source key is then deleted from its origin
|
||||
bucket, and the row's ``object_status`` flips to ``"deleted"`` with
|
||||
@@ -583,7 +581,7 @@ async def delete_object(
|
||||
Immutable artifacts (version snapshots, run logs) are not
|
||||
trashed — the policy is enforced by ``is_immutable`` and an
|
||||
explicit 409. The reaper will physically delete trashed objects
|
||||
older than ``rustfs_trash_retention_days`` (out of scope for this
|
||||
older than ``s3_trash_retention_days`` (out of scope for this
|
||||
endpoint; the field is the contract).
|
||||
"""
|
||||
item = await session.scalar(
|
||||
@@ -602,19 +600,17 @@ async def delete_object(
|
||||
"data": {
|
||||
"storage_object_id": storage_object_id,
|
||||
"object_status": item.object_status,
|
||||
"trash_key": item.trash_key,
|
||||
}
|
||||
"trash_key": item.trash_key,
|
||||
}
|
||||
if item.storage_backend == "rustfs" and item.bucket_name and item.object_key:
|
||||
}
|
||||
if item.storage_backend == "s3" and item.bucket_name and item.object_key:
|
||||
trash_key = f"{item.bucket_name}/{item.object_key}"
|
||||
try:
|
||||
await asyncio.to_thread(
|
||||
request.app.state.object_store.move_to_trash,
|
||||
source_bucket=item.bucket_name,
|
||||
source_key=item.object_key,
|
||||
trash_bucket=settings.rustfs_trash_bucket,
|
||||
trash_key=trash_key,
|
||||
)
|
||||
# Cross-backend move: get from source, put to trash, delete source.
|
||||
object_stores = request.app.state.object_stores
|
||||
data = await object_stores[item.bucket_name].get(item.object_key)
|
||||
await object_stores[settings.s3_trash_bucket].put(trash_key, data)
|
||||
await object_stores[item.bucket_name].delete(item.object_key)
|
||||
except Exception as exc:
|
||||
# If the move fails, leave the source intact and surface the
|
||||
# error. We do NOT mark the row as deleted in that case —
|
||||
@@ -631,7 +627,7 @@ async def delete_object(
|
||||
"storage_object_id": storage_object_id,
|
||||
"object_status": item.object_status,
|
||||
"trash_key": item.trash_key,
|
||||
"trash_bucket": settings.rustfs_trash_bucket,
|
||||
"trash_bucket": settings.s3_trash_bucket,
|
||||
}
|
||||
}
|
||||
|
||||
@@ -668,13 +664,10 @@ async def restore_object(
|
||||
status.HTTP_409_CONFLICT,
|
||||
"object has no trash pointer; cannot restore")
|
||||
try:
|
||||
await asyncio.to_thread(
|
||||
request.app.state.object_store.copy,
|
||||
source_bucket=settings.rustfs_trash_bucket,
|
||||
source_key=item.trash_key,
|
||||
dest_bucket=item.bucket_name,
|
||||
dest_key=item.object_key,
|
||||
)
|
||||
# Cross-backend copy: get from trash, put back to source bucket.
|
||||
object_stores = request.app.state.object_stores
|
||||
data = await object_stores[settings.s3_trash_bucket].get(item.trash_key)
|
||||
await object_stores[item.bucket_name].put(item.object_key, data)
|
||||
except Exception as exc:
|
||||
raise HTTPException(
|
||||
status.HTTP_502_BAD_GATEWAY,
|
||||
@@ -722,11 +715,9 @@ async def purge_trash_object(
|
||||
"object is not in trash; refuse to hard-delete live data")
|
||||
if item.trash_key:
|
||||
try:
|
||||
await asyncio.to_thread(
|
||||
request.app.state.object_store.delete,
|
||||
bucket_name=settings.rustfs_trash_bucket,
|
||||
object_key=item.trash_key,
|
||||
)
|
||||
await request.app.state.object_stores[
|
||||
settings.s3_trash_bucket
|
||||
].delete(item.trash_key)
|
||||
except Exception as exc:
|
||||
raise HTTPException(
|
||||
status.HTTP_502_BAD_GATEWAY,
|
||||
|
||||
@@ -1,11 +1,9 @@
|
||||
"""Backend-bound storage client.
|
||||
|
||||
Re-exports :class:`StorageClient` under the same name used by callers in
|
||||
``backend/``. The default client raises :class:`StorageClientError` from
|
||||
``common.storage.client`` so it stays usable from non-FastAPI contexts.
|
||||
Inside FastAPI route handlers we want HTTP-shaped errors, so this module
|
||||
also exposes :class:`BackendStorageClient`, a thin wrapper that translates
|
||||
the framework-agnostic errors into ``HTTPException``.
|
||||
TODO: this HTTP client is dead code post-migration; rewrite to use
|
||||
AsyncStorageBackend directly. The base StorageClient class was removed
|
||||
from common.storage.client, so this module is currently a stub that
|
||||
preserves the import surface but raises NotImplementedError.
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
@@ -14,17 +12,25 @@ from typing import Any
|
||||
|
||||
from fastapi import HTTPException, status
|
||||
|
||||
from common.storage.client import (
|
||||
StorageClient,
|
||||
StorageClientError,
|
||||
StorageRequestFailed,
|
||||
StorageUnavailable,
|
||||
)
|
||||
|
||||
|
||||
__all__ = ["BackendStorageClient", "StorageClient", "StorageClientError"]
|
||||
|
||||
|
||||
class StorageClientError(Exception):
|
||||
pass
|
||||
|
||||
|
||||
class StorageUnavailable(StorageClientError):
|
||||
pass
|
||||
|
||||
|
||||
class StorageRequestFailed(StorageClientError):
|
||||
def __init__(self, status_code: int, detail: Any):
|
||||
self.status_code = status_code
|
||||
self.detail = detail
|
||||
super().__init__(status_code, detail)
|
||||
|
||||
|
||||
def _to_http_exception(exc: StorageClientError) -> HTTPException:
|
||||
if isinstance(exc, StorageUnavailable):
|
||||
return HTTPException(
|
||||
@@ -44,7 +50,7 @@ def _to_http_exception(exc: StorageClientError) -> HTTPException:
|
||||
)
|
||||
|
||||
|
||||
class BackendStorageClient(StorageClient):
|
||||
class BackendStorageClient:
|
||||
"""Storage client that raises ``HTTPException`` for web callers."""
|
||||
|
||||
async def _request(
|
||||
@@ -54,10 +60,10 @@ class BackendStorageClient(StorageClient):
|
||||
*,
|
||||
payload: dict[str, Any] | None = None,
|
||||
) -> dict[str, Any]:
|
||||
try:
|
||||
return await super()._request(method, path, payload=payload)
|
||||
except StorageClientError as exc:
|
||||
raise _to_http_exception(exc) from exc
|
||||
raise NotImplementedError(
|
||||
"TODO: BackendStorageClient is dead code post-migration; "
|
||||
"rewrite to use AsyncStorageBackend directly"
|
||||
)
|
||||
|
||||
|
||||
# Re-bind the imported symbol so existing backend call sites that import
|
||||
|
||||
Reference in New Issue
Block a user