重构模型平台前后端并移除Redis依赖

This commit is contained in:
Winnie
2026-07-30 19:03:00 +08:00
commit c74b2abb48
172 changed files with 40825 additions and 0 deletions
+10
View File
@@ -0,0 +1,10 @@
# Python-generated files
__pycache__/
*.py[oc]
build/
dist/
wheels/
*.egg-info
# Virtual environments
.venv
+1
View File
@@ -0,0 +1 @@
3.12
+13
View File
@@ -0,0 +1,13 @@
FROM python:3.12-slim-bookworm
ENV PYTHONDONTWRITEBYTECODE=1 PYTHONUNBUFFERED=1 PYTHONPATH=/app
WORKDIR /app
COPY --from=ghcr.io/astral-sh/uv:latest /uv /bin/uv
COPY common ./common
COPY backend ./backend
COPY alembic.ini ./
COPY migrations ./migrations
RUN uv pip install --system ./common ./backend
EXPOSE 8000
CMD ["uvicorn", "backend.main:app", "--host", "0.0.0.0", "--port", "8000"]
+5
View File
@@ -0,0 +1,5 @@
# Backend
统一 FastAPI 管理服务。包含用户、Workspace、脚本、稳定版本、调度定义、
立即运行以及 RustFS 对象接口。原 `platform_api``storage_api` 已在此
模块合并,外部 REST 契约保持不变。
+23
View File
@@ -0,0 +1,23 @@
[project]
name = "backend"
version = "0.2.0"
requires-python = ">=3.12"
dependencies = [
"common",
"fastapi==0.116.1",
"uvicorn[standard]==0.35.0",
"httpx==0.28.1",
"croniter==6.2.4",
"alembic==1.18.5",
"cryptography==49.0.0",
]
[tool.uv.sources]
common = { path = "../common" }
[build-system]
requires = ["hatchling"]
build-backend = "hatchling.build"
[tool.hatch.build.targets.wheel]
packages = ["src/backend"]
+1
View File
@@ -0,0 +1 @@
"""Platform API application."""
+218
View File
@@ -0,0 +1,218 @@
from __future__ import annotations
from typing import Any, Literal
from fastapi import APIRouter, Depends, HTTPException, status
from pydantic import BaseModel, ConfigDict, Field
from sqlalchemy import delete, func, or_, select
from sqlalchemy.ext.asyncio import AsyncSession
from common.db.models import Roles, Users, WorkspaceMembers
from common.ids import new_ulid
from backend.dependencies import (
RequestContext,
database_session,
request_context,
)
router = APIRouter(prefix="/api/v1/admin", tags=["admin"])
class EmployeeCreate(BaseModel):
model_config = ConfigDict(extra="forbid")
username: str = Field(min_length=2, max_length=64)
display_name: str = Field(min_length=1, max_length=100)
email: str | None = Field(default=None, max_length=255)
role_code: Literal["admin", "developer"] = "developer"
class EmployeeUpdate(BaseModel):
model_config = ConfigDict(extra="forbid")
display_name: str | None = Field(default=None, min_length=1, max_length=100)
email: str | None = Field(default=None, max_length=255)
role_code: Literal["admin", "developer"] | None = None
status: Literal["active", "disabled", "locked"] | None = None
def require_admin(context: RequestContext) -> None:
if not context.is_admin:
raise HTTPException(status.HTTP_403_FORBIDDEN, "仅管理员可以管理员工")
def employee_payload(user: Users, role: Roles) -> dict[str, Any]:
return {
"user_id": user.user_id,
"username": user.username,
"display_name": user.display_name,
"email": user.email,
"status": user.status,
"role_code": role.role_code,
"role_name": role.role_name,
"created_at": user.created_at.isoformat(),
}
async def member_row(
user_id: str,
context: RequestContext,
session: AsyncSession,
) -> tuple[Users, WorkspaceMembers, Roles]:
row = (
await session.execute(
select(Users, WorkspaceMembers, Roles)
.join(
WorkspaceMembers,
WorkspaceMembers.user_id == Users.user_id,
)
.join(Roles, Roles.role_id == WorkspaceMembers.role_id)
.where(
WorkspaceMembers.workspace_id == context.workspace.workspace_id,
Users.user_id == user_id,
)
)
).first()
if row is None:
raise HTTPException(status.HTTP_404_NOT_FOUND, "员工不存在")
return row
@router.get("/employees")
async def list_employees(
context: RequestContext = Depends(request_context),
session: AsyncSession = Depends(database_session),
) -> dict[str, Any]:
rows = (
await session.execute(
select(Users, Roles)
.join(
WorkspaceMembers,
WorkspaceMembers.user_id == Users.user_id,
)
.join(Roles, Roles.role_id == WorkspaceMembers.role_id)
.where(
WorkspaceMembers.workspace_id == context.workspace.workspace_id,
WorkspaceMembers.member_status == "active",
)
.order_by(Users.created_at, Users.user_id)
)
).all()
return {
"request_id": context.request_id,
"data": [employee_payload(user, role) for user, role in rows],
"meta": {"count": len(rows), "can_manage": context.is_admin},
}
@router.post("/employees", status_code=status.HTTP_201_CREATED)
async def create_employee(
payload: EmployeeCreate,
context: RequestContext = Depends(request_context),
session: AsyncSession = Depends(database_session),
) -> dict[str, Any]:
require_admin(context)
username = payload.username.strip()
display_name = payload.display_name.strip()
duplicate_conditions = [Users.username == username]
if payload.email:
duplicate_conditions.append(Users.email == payload.email.strip())
duplicate = await session.scalar(
select(Users.user_id).where(or_(*duplicate_conditions))
)
if duplicate is not None:
raise HTTPException(status.HTTP_409_CONFLICT, "用户名或邮箱已存在")
role = await session.scalar(
select(Roles).where(Roles.role_code == payload.role_code)
)
if role is None:
raise HTTPException(status.HTTP_422_UNPROCESSABLE_ENTITY, "角色不存在")
user = Users(
user_id=new_ulid(),
username=username,
display_name=display_name,
email=payload.email.strip() if payload.email else None,
password_hash="demo-login-disabled",
status="active",
)
session.add(user)
session.add(
WorkspaceMembers(
workspace_id=context.workspace.workspace_id,
user_id=user.user_id,
role_id=role.role_id,
member_status="active",
)
)
await session.flush()
await session.refresh(user)
return {
"request_id": context.request_id,
"data": employee_payload(user, role),
"meta": {},
}
@router.patch("/employees/{user_id}")
async def update_employee(
user_id: str,
payload: EmployeeUpdate,
context: RequestContext = Depends(request_context),
session: AsyncSession = Depends(database_session),
) -> dict[str, Any]:
require_admin(context)
user, membership, role = await member_row(user_id, context, session)
if payload.display_name is not None:
user.display_name = payload.display_name.strip()
if payload.email is not None:
user.email = payload.email.strip() or None
if payload.status is not None:
user.status = payload.status
if payload.role_code is not None and payload.role_code != role.role_code:
next_role = await session.scalar(
select(Roles).where(Roles.role_code == payload.role_code)
)
if next_role is None:
raise HTTPException(status.HTTP_422_UNPROCESSABLE_ENTITY, "角色不存在")
membership.role_id = next_role.role_id
role = next_role
await session.flush()
await session.refresh(user)
return {
"request_id": context.request_id,
"data": employee_payload(user, role),
"meta": {},
}
@router.delete("/employees/{user_id}")
async def delete_employee(
user_id: str,
context: RequestContext = Depends(request_context),
session: AsyncSession = Depends(database_session),
) -> dict[str, Any]:
require_admin(context)
if user_id == context.user.user_id:
raise HTTPException(status.HTTP_409_CONFLICT, "不能删除当前登录员工")
user, _, role = await member_row(user_id, context, session)
if role.role_code == "admin":
raise HTTPException(status.HTTP_409_CONFLICT, "管理员账号不能删除")
await session.execute(
delete(WorkspaceMembers).where(
WorkspaceMembers.workspace_id == context.workspace.workspace_id,
WorkspaceMembers.user_id == user_id,
)
)
memberships = await session.scalar(
select(func.count())
.select_from(WorkspaceMembers)
.where(WorkspaceMembers.user_id == user_id)
)
if memberships == 0:
user.status = "disabled"
return {
"request_id": context.request_id,
"data": {"user_id": user_id, "deleted": True},
"meta": {},
}
+79
View File
@@ -0,0 +1,79 @@
from __future__ import annotations
from dataclasses import dataclass
from typing import AsyncIterator
from fastapi import Header, HTTPException, Request, status
from sqlalchemy import select
from sqlalchemy.ext.asyncio import AsyncSession
from common.db.models import (
Roles,
Users,
WorkspaceMembers,
Workspaces,
)
from common.ids import new_ulid
@dataclass(frozen=True)
class RequestContext:
request_id: str
user: Users
workspace: Workspaces
role: Roles
@property
def is_admin(self) -> bool:
return self.role.role_code == "admin"
async def database_session(request: Request) -> AsyncIterator[AsyncSession]:
async with request.app.state.session_factory() as session:
try:
yield session
await session.commit()
except Exception:
await session.rollback()
raise
async def request_context(
request: Request,
x_user_id: str = Header(alias="X-User-ID"),
x_workspace_id: str = Header(alias="X-Workspace-ID"),
x_request_id: str | None = Header(default=None, alias="X-Request-ID"),
) -> RequestContext:
async with request.app.state.session_factory() as session:
statement = (
select(Users, Workspaces, Roles)
.join(
WorkspaceMembers,
WorkspaceMembers.user_id == Users.user_id,
)
.join(
Workspaces,
Workspaces.workspace_id == WorkspaceMembers.workspace_id,
)
.join(Roles, Roles.role_id == WorkspaceMembers.role_id)
.where(
Users.user_id == x_user_id,
Users.status == "active",
WorkspaceMembers.workspace_id == x_workspace_id,
WorkspaceMembers.member_status == "active",
Workspaces.status == "active",
)
)
row = (await session.execute(statement)).one_or_none()
if row is None:
raise HTTPException(
status.HTTP_403_FORBIDDEN,
"active workspace membership is required",
)
user, workspace, role = row
return RequestContext(
request_id=x_request_id or new_ulid(),
user=user,
workspace=workspace,
role=role,
)
+110
View File
@@ -0,0 +1,110 @@
from __future__ import annotations
from typing import Any, Awaitable, Callable
from fastapi import APIRouter, Depends, Request, status
from fastapi.responses import JSONResponse
from backend.dependencies import (
RequestContext,
request_context,
)
from backend.runtime_client import RuntimeClientError
from backend.schemas import FileLockTokenRequest
router = APIRouter(tags=["file-locks"])
async def runtime_response(
context: RequestContext,
operation: Callable[[], Awaitable[dict[str, Any]]],
*,
success_status: int = status.HTTP_200_OK,
) -> JSONResponse:
try:
data = await operation()
except RuntimeClientError as exc:
error = exc.detail
if not isinstance(error, dict) or "code" not in error:
error = {
"code": "RUNTIME_REQUEST_FAILED",
"message": str(error),
"retryable": exc.status_code >= 500,
"details": {},
}
return JSONResponse(
status_code=exc.status_code,
content={"request_id": context.request_id, "error": error},
)
return JSONResponse(
status_code=success_status,
content={
"request_id": context.request_id,
"data": data,
"meta": {},
},
)
@router.post(
"/api/v1/files/{storage_object_id}/lock",
status_code=status.HTTP_201_CREATED,
)
async def acquire_file_lock(
storage_object_id: str,
request: Request,
context: RequestContext = Depends(request_context),
) -> JSONResponse:
return await runtime_response(
context,
lambda: request.app.state.runtime_client.acquire_file_lock(
{
"workspace_id": context.workspace.workspace_id,
"storage_object_id": storage_object_id,
"user_id": context.user.user_id,
"request_id": context.request_id,
}
),
success_status=status.HTTP_201_CREATED,
)
@router.post("/api/v1/file-locks/{edit_session_id}/heartbeat")
async def heartbeat_file_lock(
edit_session_id: str,
payload: FileLockTokenRequest,
request: Request,
context: RequestContext = Depends(request_context),
) -> JSONResponse:
return await runtime_response(
context,
lambda: request.app.state.runtime_client.heartbeat_file_lock(
edit_session_id,
{
"workspace_id": context.workspace.workspace_id,
"user_id": context.user.user_id,
"lock_token": payload.lock_token,
},
),
)
@router.delete("/api/v1/file-locks/{edit_session_id}")
async def release_file_lock(
edit_session_id: str,
payload: FileLockTokenRequest,
request: Request,
context: RequestContext = Depends(request_context),
) -> JSONResponse:
return await runtime_response(
context,
lambda: request.app.state.runtime_client.release_file_lock(
edit_session_id,
{
"workspace_id": context.workspace.workspace_id,
"user_id": context.user.user_id,
"lock_token": payload.lock_token,
},
),
)
+84
View File
@@ -0,0 +1,84 @@
from __future__ import annotations
import os
from fastapi import APIRouter, Depends, Request, status
from fastapi.responses import JSONResponse
from backend.dependencies import (
RequestContext,
request_context,
)
from backend.runtime_client import RuntimeClientError
from backend.schemas import (
CreateJupyterAccessTicketRequest,
)
router = APIRouter(tags=["jupyter"])
def cookie_secure() -> bool:
return os.getenv("COOKIE_SECURE", "false").strip().lower() in {
"1",
"true",
"yes",
"on",
}
@router.post(
"/api/v1/jupyter/access-tickets",
status_code=status.HTTP_201_CREATED,
)
async def create_jupyter_access_ticket(
payload: CreateJupyterAccessTicketRequest,
request: Request,
context: RequestContext = Depends(request_context),
) -> JSONResponse:
try:
ticket_data = (
await request.app.state.runtime_client
.create_jupyter_access_ticket(
payload.edit_session_id,
{
"workspace_id": context.workspace.workspace_id,
"user_id": context.user.user_id,
"lock_token": payload.lock_token,
},
)
)
except RuntimeClientError as exc:
error = exc.detail
if not isinstance(error, dict) or "code" not in error:
error = {
"code": "JUPYTER_TICKET_FAILED",
"message": str(error),
"retryable": exc.status_code >= 500,
"details": {},
}
return JSONResponse(
status_code=exc.status_code,
content={"request_id": context.request_id, "error": error},
)
raw_ticket = ticket_data.pop("ticket")
max_age = int(ticket_data.pop("expires_in_seconds"))
response = JSONResponse(
status_code=status.HTTP_201_CREATED,
content={
"request_id": context.request_id,
"data": ticket_data,
"meta": {},
},
)
response.set_cookie(
key="jupyter_access",
value=raw_ticket,
max_age=max_age,
path="/jupyter/",
secure=cookie_secure(),
httponly=True,
samesite="lax",
)
return response
+108
View File
@@ -0,0 +1,108 @@
from __future__ import annotations
import asyncio
import os
from contextlib import asynccontextmanager
from pathlib import Path
from typing import Any, AsyncIterator
import httpx
from fastapi.routing import APIRoute
from common.db import create_database_engine, create_session_factory
from common.service_app import create_service_app
from common.storage import RustFSObjectStore
from backend.admin import router as admin_router
from backend.file_locks import router as file_locks_router
from backend.jupyter import router as jupyter_router
from backend.resources import router as resources_router
from backend.runtime_client import RuntimeClient
from backend.schedule_runs import router as schedule_runs_router
from backend.schedule_client import ScheduleExecutorClient
from backend.schedules import router as schedules_router
from backend.scripts import router as scripts_router
from backend.storage_api import app as storage_app
from backend.storage_client import StorageClient
@asynccontextmanager
async def lifespan(app: Any) -> AsyncIterator[None]:
engine = create_database_engine(os.environ["DATABASE_URL"])
app.state.session_factory = create_session_factory(engine)
workspace_root = Path(
os.getenv("WORKSPACE_ROOT", "/workspace/workspaces")
)
workspace_root.mkdir(parents=True, exist_ok=True)
# Storage API is now part of the backend process. Platform routers keep
# their existing client contract, but calls are dispatched in-process.
app.state.object_store = RustFSObjectStore(
internal_endpoint=os.getenv(
"RUSTFS_INTERNAL_ENDPOINT",
"http://rustfs:9000",
),
public_endpoint=os.getenv(
"RUSTFS_PUBLIC_ENDPOINT",
"http://localhost:9000",
),
access_key=os.environ["RUSTFS_ACCESS_KEY"],
secret_key=os.environ["RUSTFS_SECRET_KEY"],
)
app.state.default_bucket = os.getenv(
"RUSTFS_DEFAULT_BUCKET",
"model-platform",
)
await asyncio.to_thread(
app.state.object_store.ensure_bucket,
app.state.default_bucket,
)
storage_http_client = httpx.AsyncClient(
transport=httpx.ASGITransport(app=app),
base_url="http://backend.internal",
timeout=httpx.Timeout(30.0),
)
app.state.storage_client = StorageClient(
storage_http_client,
os.environ["INTERNAL_SERVICE_TOKEN"],
)
runtime_http_client = httpx.AsyncClient(
base_url=os.getenv("RUNTIME_API_URL", "http://runtime:8000"),
timeout=httpx.Timeout(30.0),
)
app.state.runtime_client = RuntimeClient(
runtime_http_client,
os.environ["INTERNAL_SERVICE_TOKEN"],
)
schedule_http_client = httpx.AsyncClient(
base_url=os.getenv("SCHEDULE_API_URL", "http://schedule:8000"),
timeout=httpx.Timeout(10.0),
)
app.state.schedule_client = ScheduleExecutorClient(
schedule_http_client,
os.environ["INTERNAL_SERVICE_TOKEN"],
)
try:
yield
finally:
await runtime_http_client.aclose()
await schedule_http_client.aclose()
await storage_http_client.aclose()
await engine.dispose()
app = create_service_app(
os.getenv("SERVICE_NAME", "backend"),
lifespan=lifespan,
)
app.include_router(file_locks_router)
app.include_router(jupyter_router)
app.include_router(resources_router)
app.include_router(schedule_runs_router)
app.include_router(schedules_router)
app.include_router(scripts_router)
app.include_router(admin_router)
# Reuse the proven storage endpoints without running another FastAPI service.
for route in storage_app.routes:
if isinstance(route, APIRoute) and route.path.startswith("/internal/"):
app.router.routes.append(route)
+293
View File
@@ -0,0 +1,293 @@
from __future__ import annotations
from datetime import UTC, datetime
from typing import Any
from fastapi import APIRouter, Depends, Header, HTTPException, Query, Request, status
from sqlalchemy import or_, select
from sqlalchemy.ext.asyncio import AsyncSession
from common.db.models import DataResources, StorageObjects
from common.ids import new_ulid
from backend.dependencies import (
RequestContext,
database_session,
request_context,
)
from backend.schemas import (
CompleteResourceUploadRequest,
CreateResourceUploadRequest,
DownloadUrlRequest,
)
router = APIRouter(prefix="/api/v1/data-resources", tags=["data-resources"])
def resource_payload(
resource: DataResources,
storage_object: StorageObjects,
) -> dict[str, Any]:
return {
"resource_id": resource.resource_id,
"workspace_id": resource.workspace_id,
"storage_object_id": resource.storage_object_id,
"owner_user_id": resource.owner_user_id,
"resource_name": resource.resource_name,
"description": resource.description,
"visibility": resource.visibility,
"status": resource.status,
"created_at": resource.created_at.isoformat(),
"updated_at": resource.updated_at.isoformat(),
"file": {
"file_name": storage_object.file_name,
"file_extension": storage_object.file_extension,
"mime_type": storage_object.mime_type,
"size_bytes": storage_object.size_bytes,
"content_hash": storage_object.content_hash,
"object_status": storage_object.object_status,
},
}
def can_view(resource: DataResources, context: RequestContext) -> bool:
return (
resource.owner_user_id == context.user.user_id
or resource.visibility in {"workspace", "public"}
or context.is_admin
)
@router.post("/uploads", status_code=status.HTTP_201_CREATED)
async def create_resource_upload(
payload: CreateResourceUploadRequest,
request: Request,
context: RequestContext = Depends(request_context),
idempotency_key: str = Header(
min_length=8,
max_length=128,
alias="Idempotency-Key",
),
) -> dict[str, Any]:
data = await request.app.state.storage_client.create_upload(
{
"workspace_id": context.workspace.workspace_id,
"user_id": context.user.user_id,
"usage_type": "data_resource",
"file_name": payload.file_name,
"content_type": payload.content_type,
"expected_size_bytes": payload.expected_size_bytes,
"expected_hash": payload.expected_hash,
"idempotency_key": idempotency_key,
"url_scope": "public",
}
)
return {"request_id": context.request_id, "data": data, "meta": {}}
@router.post("/uploads/{upload_id}/complete")
async def complete_resource_upload(
upload_id: str,
payload: CompleteResourceUploadRequest,
request: Request,
context: RequestContext = Depends(request_context),
session: AsyncSession = Depends(database_session),
) -> dict[str, Any]:
storage_data = await request.app.state.storage_client.complete_upload(
upload_id,
{
"usage_type": "data_resource",
"visibility": payload.visibility,
"is_immutable": False,
},
)
if storage_data["workspace_id"] != context.workspace.workspace_id:
raise HTTPException(
status.HTTP_403_FORBIDDEN,
"upload belongs to another workspace",
)
if storage_data["owner_user_id"] != context.user.user_id:
raise HTTPException(
status.HTTP_403_FORBIDDEN,
"upload belongs to another user",
)
existing = await session.scalar(
select(DataResources).where(
DataResources.storage_object_id
== storage_data["storage_object_id"]
)
)
reused = existing is not None
if existing is None:
existing = DataResources(
resource_id=new_ulid(),
workspace_id=context.workspace.workspace_id,
storage_object_id=storage_data["storage_object_id"],
owner_user_id=context.user.user_id,
resource_name=payload.resource_name,
description=payload.description,
visibility=payload.visibility,
status="active",
)
session.add(existing)
await session.flush()
await session.refresh(existing)
storage_object = await session.get(
StorageObjects,
existing.storage_object_id,
)
if storage_object is None:
raise HTTPException(
status.HTTP_409_CONFLICT,
"storage object metadata is missing",
)
return {
"request_id": context.request_id,
"data": resource_payload(existing, storage_object),
"meta": {"reused": reused},
}
@router.get("")
async def list_resources(
context: RequestContext = Depends(request_context),
session: AsyncSession = Depends(database_session),
visibility: str | None = Query(default=None),
keyword: str | None = Query(default=None, max_length=100),
) -> dict[str, Any]:
statement = (
select(DataResources, StorageObjects)
.join(
StorageObjects,
StorageObjects.storage_object_id
== DataResources.storage_object_id,
)
.where(
DataResources.workspace_id == context.workspace.workspace_id,
DataResources.status == "active",
StorageObjects.object_status == "available",
)
.order_by(DataResources.updated_at.desc())
)
if not context.is_admin:
statement = statement.where(
or_(
DataResources.owner_user_id == context.user.user_id,
DataResources.visibility.in_(["workspace", "public"]),
)
)
if visibility:
if visibility not in {"private", "workspace", "public"}:
raise HTTPException(
status.HTTP_422_UNPROCESSABLE_ENTITY,
"invalid visibility",
)
statement = statement.where(DataResources.visibility == visibility)
if keyword:
statement = statement.where(
DataResources.resource_name.like(f"%{keyword.strip()}%")
)
rows = (await session.execute(statement)).all()
return {
"request_id": context.request_id,
"data": [
resource_payload(resource, storage_object)
for resource, storage_object in rows
],
"meta": {"count": len(rows)},
}
async def get_visible_resource(
resource_id: str,
context: RequestContext,
session: AsyncSession,
) -> tuple[DataResources, StorageObjects]:
row = (
await session.execute(
select(DataResources, StorageObjects)
.join(
StorageObjects,
StorageObjects.storage_object_id
== DataResources.storage_object_id,
)
.where(
DataResources.resource_id == resource_id,
DataResources.workspace_id
== context.workspace.workspace_id,
DataResources.status == "active",
)
)
).one_or_none()
if row is None or not can_view(row[0], context):
raise HTTPException(status.HTTP_404_NOT_FOUND, "resource not found")
return row
@router.get("/{resource_id}")
async def get_resource(
resource_id: str,
context: RequestContext = Depends(request_context),
session: AsyncSession = Depends(database_session),
) -> dict[str, Any]:
resource, storage_object = await get_visible_resource(
resource_id,
context,
session,
)
return {
"request_id": context.request_id,
"data": resource_payload(resource, storage_object),
"meta": {},
}
@router.post("/{resource_id}/download-url")
async def resource_download_url(
resource_id: str,
payload: DownloadUrlRequest,
request: Request,
context: RequestContext = Depends(request_context),
session: AsyncSession = Depends(database_session),
) -> dict[str, Any]:
resource, _ = await get_visible_resource(
resource_id,
context,
session,
)
data = await request.app.state.storage_client.create_download_url(
resource.storage_object_id,
payload.expires_seconds,
)
return {"request_id": context.request_id, "data": data, "meta": {}}
@router.delete("/{resource_id}")
async def delete_resource(
resource_id: str,
request: Request,
context: RequestContext = Depends(request_context),
session: AsyncSession = Depends(database_session),
) -> dict[str, Any]:
resource, _ = await get_visible_resource(
resource_id,
context,
session,
)
if (
resource.owner_user_id != context.user.user_id
and not context.is_admin
):
raise HTTPException(
status.HTTP_403_FORBIDDEN,
"resource can only be deleted by its owner or an administrator",
)
await request.app.state.storage_client.delete_object(
resource.storage_object_id
)
resource.status = "deleted"
resource.deleted_at = datetime.now(UTC).replace(tzinfo=None)
return {
"request_id": context.request_id,
"data": {"resource_id": resource_id, "status": "deleted"},
"meta": {},
}
+104
View File
@@ -0,0 +1,104 @@
from __future__ import annotations
from dataclasses import dataclass
from typing import Any
import httpx
@dataclass(frozen=True)
class RuntimeClientError(Exception):
status_code: int
detail: Any
class RuntimeClient:
def __init__(
self,
client: httpx.AsyncClient,
service_token: str,
) -> None:
self.client = client
self.headers = {"X-Service-Token": service_token}
async def _request(
self,
method: str,
path: str,
payload: dict[str, Any],
) -> dict[str, Any]:
try:
response = await self.client.request(
method,
path,
json=payload,
headers=self.headers,
)
except httpx.RequestError as exc:
raise RuntimeClientError(
503,
{
"code": "RUNTIME_UNAVAILABLE",
"message": "Runtime Manager 暂时不可用",
"retryable": True,
"details": {},
},
) from exc
if response.is_error:
try:
detail = response.json().get("detail", response.text)
except ValueError:
detail = response.text
raise RuntimeClientError(response.status_code, detail)
return response.json()
async def acquire_file_lock(
self,
payload: dict[str, Any],
) -> dict[str, Any]:
return (
await self._request(
"POST",
"/internal/v1/file-locks/acquire",
payload,
)
)["data"]
async def heartbeat_file_lock(
self,
edit_session_id: str,
payload: dict[str, Any],
) -> dict[str, Any]:
return (
await self._request(
"POST",
f"/internal/v1/file-locks/{edit_session_id}/heartbeat",
payload,
)
)["data"]
async def release_file_lock(
self,
edit_session_id: str,
payload: dict[str, Any],
) -> dict[str, Any]:
return (
await self._request(
"DELETE",
f"/internal/v1/file-locks/{edit_session_id}",
payload,
)
)["data"]
async def create_jupyter_access_ticket(
self,
edit_session_id: str,
payload: dict[str, Any],
) -> dict[str, Any]:
return (
await self._request(
"POST",
f"/internal/v1/jupyter/access-tickets/{edit_session_id}",
payload,
)
)["data"]
+43
View File
@@ -0,0 +1,43 @@
from __future__ import annotations
import logging
import httpx
LOGGER = logging.getLogger(__name__)
class ScheduleExecutorClient:
"""Best-effort HTTP notification for immediate run dispatch.
MySQL remains the source of truth. If this notification fails, the
executor's database polling loop will still pick up the pending Outbox row.
"""
def __init__(self, client: httpx.AsyncClient, service_token: str) -> None:
self.client = client
self.headers = {"X-Service-Token": service_token}
async def dispatch_run(self, run_id: str) -> bool:
try:
response = await self.client.post(
f"/internal/v1/runs/{run_id}/dispatch",
headers=self.headers,
)
except httpx.RequestError:
LOGGER.warning(
"schedule executor notification failed for run %s",
run_id,
exc_info=True,
)
return False
if response.is_error:
LOGGER.warning(
"schedule executor rejected run %s: %s %s",
run_id,
response.status_code,
response.text[:500],
)
return False
return True
+348
View File
@@ -0,0 +1,348 @@
from __future__ import annotations
import hashlib
from datetime import UTC, datetime
from typing import Any, Literal
from fastapi import APIRouter, Depends, Header, HTTPException, Query, Request, status
from pydantic import Field
from sqlalchemy import select
from sqlalchemy.ext.asyncio import AsyncSession
from common.db.models import (
ScheduleNodeRuns,
ScheduleRuns,
)
from common.eventing import add_outbox_event, utcnow
from common.ids import new_ulid
from backend.dependencies import (
RequestContext,
database_session,
request_context,
)
from backend.schedule_schemas import StrictModel
from backend.schedules import (
graph_rows,
schedule_row,
validate_dag,
)
router = APIRouter(tags=["schedule-runs"])
RunStatus = Literal[
"queued",
"running",
"succeeded",
"failed",
"cancelled",
"timed_out",
]
class RunScheduleRequest(StrictModel):
reason: str = Field(default="manual_run", min_length=1, max_length=255)
def _iso(value: datetime | None) -> str | None:
if value is None:
return None
if value.tzinfo is None:
value = value.replace(tzinfo=UTC)
return value.astimezone(UTC).isoformat()
def _normalized_idempotency_key(
workspace_id: str,
schedule_id: str,
value: str,
) -> str:
normalized = value.strip()
if len(normalized) < 8:
raise HTTPException(
status.HTTP_422_UNPROCESSABLE_ENTITY,
"Idempotency-Key must contain at least 8 characters",
)
digest = hashlib.sha256(
f"{workspace_id}:{schedule_id}:{normalized}".encode("utf-8")
).hexdigest()
return f"run:v1:{digest}"
def _arguments(value: dict[str, Any] | None) -> list[str]:
payload = value or {}
raw = payload.get("_args")
result = [str(item) for item in raw] if isinstance(raw, list) else []
for key, item in payload.items():
if key == "_args":
continue
option = f"--{key.replace('_', '-')}"
if item is True:
result.append(option)
elif item is False or item is None:
continue
elif isinstance(item, list):
for list_item in item:
result.extend((option, str(list_item)))
elif isinstance(item, (str, int, float)):
result.extend((option, str(item)))
else:
raise HTTPException(
status.HTTP_422_UNPROCESSABLE_ENTITY,
f"node argument {key!r} must be a scalar or list",
)
return result
def run_summary(item: ScheduleRuns) -> dict[str, Any]:
return {
"run_id": item.run_id,
"schedule_id": item.schedule_id,
"workspace_id": item.workspace_id,
"workflow_version": item.workflow_version,
"trigger_type": item.trigger_type,
"run_status": item.run_status,
"state_version": item.state_version,
"queued_at": _iso(item.queued_at),
"started_at": _iso(item.started_at),
"finished_at": _iso(item.finished_at),
"duration_ms": item.duration_ms,
"error_code": item.error_code,
"error_message": item.error_message,
"logs_object_id": item.logs_object_id,
"result_object_id": item.result_object_id,
}
def node_run_payload(item: ScheduleNodeRuns) -> dict[str, Any]:
return {
"node_run_id": item.node_run_id,
"run_id": item.run_id,
"node_id": item.node_id,
"versions_id": item.versions_id,
"attempt_no": item.attempt_no,
"node_status": item.node_status,
"state_version": item.state_version,
"started_at": _iso(item.started_at),
"finished_at": _iso(item.finished_at),
"duration_ms": item.duration_ms,
"exit_code": item.exit_code,
"message": item.message,
"logs_object_id": item.logs_object_id,
"result_object_id": item.result_object_id,
}
async def run_detail(
item: ScheduleRuns,
session: AsyncSession,
) -> dict[str, Any]:
node_runs = list(
(
await session.scalars(
select(ScheduleNodeRuns)
.where(ScheduleNodeRuns.run_id == item.run_id)
.order_by(
ScheduleNodeRuns.created_at,
ScheduleNodeRuns.attempt_no,
)
)
).all()
)
return {
**run_summary(item),
"node_runs": [node_run_payload(node_run) for node_run in node_runs],
}
async def _visible_run(
run_id: str,
context: RequestContext,
session: AsyncSession,
) -> ScheduleRuns:
item = await session.scalar(
select(ScheduleRuns).where(
ScheduleRuns.run_id == run_id,
ScheduleRuns.workspace_id == context.workspace.workspace_id,
)
)
if item is None:
raise HTTPException(status.HTTP_404_NOT_FOUND, "schedule run not found")
return item
@router.post(
"/api/v1/schedules/{schedule_id}/run",
status_code=status.HTTP_202_ACCEPTED,
)
async def run_schedule_now(
schedule_id: str,
request: Request,
payload: RunScheduleRequest | None = None,
idempotency_key: str = Header(alias="Idempotency-Key"),
context: RequestContext = Depends(request_context),
session: AsyncSession = Depends(database_session),
) -> dict[str, Any]:
reason = payload.reason if payload is not None else "manual_run"
key = _normalized_idempotency_key(
context.workspace.workspace_id,
schedule_id,
idempotency_key,
)
existing = await session.scalar(
select(ScheduleRuns).where(ScheduleRuns.idempotency_key == key)
)
if existing is not None:
if (
existing.workspace_id != context.workspace.workspace_id
or existing.schedule_id != schedule_id
):
raise HTTPException(
status.HTTP_409_CONFLICT,
"Idempotency-Key belongs to another schedule run",
)
return {
"request_id": context.request_id,
"data": await run_detail(existing, session),
"meta": {"reused": True},
}
schedule = await schedule_row(
schedule_id,
context,
session,
for_update=True,
)
node_rows, edges = await graph_rows(schedule_id, session)
nodes = [row[0] for row in node_rows]
validation = validate_dag(nodes, edges)
if not validation["valid"] or not nodes:
raise HTTPException(
status.HTTP_409_CONFLICT,
detail={
"code": "SCHEDULE_DAG_INVALID",
"message": "schedule must contain a valid non-empty DAG",
"errors": validation["errors"],
},
)
if len(nodes) > 100 or len(edges) > 500:
raise HTTPException(
status.HTTP_409_CONFLICT,
"schedule exceeds the v1 execution size limit",
)
snapshot = {
"schedule_name": schedule.schedule_name,
"workflow_version": schedule.workflow_version,
"max_concurrency": schedule.max_concurrency,
"failure_policy": schedule.failure_policy,
"nodes": [
{
"node_id": node.node_id,
"node_key": node.node_key,
"versions_id": version.versions_id,
"script_type": script.script_type,
"artifact_object_id": version.artifact_object_id,
"artifact_path": version.artifact_path,
"timeout_seconds": node.timeout_seconds,
"retry_count": node.retry_count,
"retry_interval_sec": node.retry_interval_sec,
"arguments": _arguments(node.arguments_json),
}
for node, version, script in node_rows
],
"edges": [
{
"source_node_id": edge.source_node_id,
"target_node_id": edge.target_node_id,
}
for edge in edges
],
}
now = utcnow()
run = ScheduleRuns(
run_id=new_ulid(),
schedule_id=schedule.schedule_id,
workspace_id=schedule.workspace_id,
workflow_version=schedule.workflow_version,
trigger_type="cron" if reason == "cron" else "manual",
idempotency_key=key,
run_status="queued",
state_version=0,
schedule_snapshot=snapshot,
queued_at=now,
triggered_by=context.user.user_id,
)
session.add(run)
schedule.last_run_at = now
await add_outbox_event(
session,
event_type="schedule.run.requested",
producer="platform-api",
trace_id=context.request_id,
aggregate_type="schedule_run",
aggregate_id=run.run_id,
idempotency_key=key,
payload={
"workspace_id": run.workspace_id,
"schedule_id": run.schedule_id,
"run_id": run.run_id,
"workflow_version": run.workflow_version,
"trigger_type": run.trigger_type,
"triggered_by": run.triggered_by,
"schedule_snapshot": snapshot,
},
)
await session.flush()
# Commit before the HTTP push so the executor can read the Outbox row.
# The executor also polls MySQL, so a failed push does not lose the run.
await session.commit()
await request.app.state.schedule_client.dispatch_run(run.run_id)
await session.refresh(run)
return {
"request_id": context.request_id,
"data": await run_detail(run, session),
"meta": {"reused": False},
}
@router.get("/api/v1/schedule-runs")
async def list_schedule_runs(
schedule_id: str | None = Query(default=None),
run_status: RunStatus | None = Query(default=None, alias="status"),
limit: int = Query(default=50, ge=1, le=200),
context: RequestContext = Depends(request_context),
session: AsyncSession = Depends(database_session),
) -> dict[str, Any]:
statement = select(ScheduleRuns).where(
ScheduleRuns.workspace_id == context.workspace.workspace_id,
)
if schedule_id:
statement = statement.where(ScheduleRuns.schedule_id == schedule_id)
if run_status:
statement = statement.where(ScheduleRuns.run_status == run_status)
items = list(
(
await session.scalars(
statement.order_by(ScheduleRuns.queued_at.desc()).limit(limit)
)
).all()
)
return {
"request_id": context.request_id,
"data": [run_summary(item) for item in items],
"meta": {"count": len(items)},
}
@router.get("/api/v1/schedule-runs/{run_id}")
async def get_schedule_run(
run_id: str,
context: RequestContext = Depends(request_context),
session: AsyncSession = Depends(database_session),
) -> dict[str, Any]:
item = await _visible_run(run_id, context, session)
return {
"request_id": context.request_id,
"data": await run_detail(item, session),
"meta": {},
}
+214
View File
@@ -0,0 +1,214 @@
from __future__ import annotations
from datetime import datetime
from typing import Any, Literal
from pydantic import Field, field_validator, model_validator
from backend.schemas import StrictModel
TriggerType = Literal["manual", "cron", "api"]
FailurePolicy = Literal["stop", "continue"]
def _required_text(value: str) -> str:
normalized = value.strip()
if not normalized:
raise ValueError("value must not be blank")
return normalized
class CreateScheduleRequest(StrictModel):
schedule_name: str = Field(min_length=1, max_length=255)
description: str | None = Field(default=None, max_length=1000)
trigger_type: TriggerType = "cron"
cron_expression: str | None = Field(default=None, max_length=128)
timezone: str = Field(default="Asia/Shanghai", min_length=1, max_length=64)
enabled: bool = False
max_concurrency: int = Field(default=1, ge=1, le=64)
failure_policy: FailurePolicy = "stop"
@field_validator("schedule_name", "timezone")
@classmethod
def validate_required_text(cls, value: str) -> str:
return _required_text(value)
@field_validator("description", "cron_expression")
@classmethod
def normalize_optional_text(cls, value: str | None) -> str | None:
if value is None:
return None
normalized = value.strip()
return normalized or None
@model_validator(mode="after")
def validate_trigger(self) -> "CreateScheduleRequest":
if self.trigger_type == "cron" and not self.cron_expression:
raise ValueError("cron_expression is required for cron schedules")
if self.trigger_type != "cron" and self.cron_expression:
raise ValueError(
"cron_expression is only allowed for cron schedules"
)
return self
class UpdateScheduleRequest(StrictModel):
workflow_version: int = Field(ge=1)
schedule_name: str | None = Field(default=None, min_length=1, max_length=255)
description: str | None = Field(default=None, max_length=1000)
trigger_type: TriggerType | None = None
cron_expression: str | None = Field(default=None, max_length=128)
timezone: str | None = Field(default=None, min_length=1, max_length=64)
enabled: bool | None = None
max_concurrency: int | None = Field(default=None, ge=1, le=64)
failure_policy: FailurePolicy | None = None
@field_validator("schedule_name", "timezone")
@classmethod
def validate_optional_required_text(
cls,
value: str | None,
) -> str | None:
return _required_text(value) if value is not None else None
@field_validator("description", "cron_expression")
@classmethod
def normalize_optional_text(cls, value: str | None) -> str | None:
if value is None:
return None
normalized = value.strip()
return normalized or None
@model_validator(mode="after")
def require_change(self) -> "UpdateScheduleRequest":
if self.model_fields_set == {"workflow_version"}:
raise ValueError("at least one schedule field must be updated")
for field in {
"schedule_name",
"trigger_type",
"timezone",
"enabled",
"max_concurrency",
"failure_policy",
}:
if field in self.model_fields_set and getattr(self, field) is None:
raise ValueError(f"{field} cannot be null")
return self
class WorkflowVersionRequest(StrictModel):
workflow_version: int = Field(ge=1)
class CreateScheduleNodeRequest(StrictModel):
workflow_version: int = Field(ge=1)
node_key: str = Field(
min_length=1,
max_length=64,
pattern=r"^[A-Za-z][A-Za-z0-9_-]*$",
)
node_name: str = Field(min_length=1, max_length=255)
versions_id: str = Field(min_length=26, max_length=26)
timeout_seconds: int = Field(default=600, ge=1, le=86_400)
retry_count: int = Field(default=0, ge=0, le=10)
retry_interval_sec: int = Field(default=5, ge=0, le=3600)
position_x: float = Field(default=0, ge=-100_000, le=100_000)
position_y: float = Field(default=0, ge=-100_000, le=100_000)
arguments_json: dict[str, Any] = Field(default_factory=dict, max_length=100)
env_refs_json: dict[str, str] = Field(default_factory=dict, max_length=100)
@field_validator("node_key", "node_name")
@classmethod
def validate_required_text(cls, value: str) -> str:
return _required_text(value)
class UpdateScheduleNodeRequest(StrictModel):
workflow_version: int = Field(ge=1)
node_name: str | None = Field(default=None, min_length=1, max_length=255)
versions_id: str | None = Field(default=None, min_length=26, max_length=26)
timeout_seconds: int | None = Field(default=None, ge=1, le=86_400)
retry_count: int | None = Field(default=None, ge=0, le=10)
retry_interval_sec: int | None = Field(default=None, ge=0, le=3600)
position_x: float | None = Field(
default=None,
ge=-100_000,
le=100_000,
)
position_y: float | None = Field(
default=None,
ge=-100_000,
le=100_000,
)
arguments_json: dict[str, Any] | None = Field(
default=None,
max_length=100,
)
env_refs_json: dict[str, str] | None = Field(
default=None,
max_length=100,
)
@field_validator("node_name")
@classmethod
def validate_optional_required_text(
cls,
value: str | None,
) -> str | None:
return _required_text(value) if value is not None else None
@model_validator(mode="after")
def require_change(self) -> "UpdateScheduleNodeRequest":
if self.model_fields_set == {"workflow_version"}:
raise ValueError("at least one node field must be updated")
for field in self.model_fields_set - {"workflow_version"}:
if getattr(self, field) is None:
raise ValueError(f"{field} cannot be null")
return self
class CreateScheduleEdgeRequest(StrictModel):
workflow_version: int = Field(ge=1)
source_node_id: str = Field(min_length=26, max_length=26)
target_node_id: str = Field(min_length=26, max_length=26)
condition_expr: str | None = Field(default=None, max_length=1000)
@field_validator("condition_expr")
@classmethod
def normalize_condition(cls, value: str | None) -> str | None:
if value is None:
return None
normalized = value.strip()
return normalized or None
@model_validator(mode="after")
def reject_self_edge(self) -> "CreateScheduleEdgeRequest":
if self.source_node_id == self.target_node_id:
raise ValueError("an edge cannot connect a node to itself")
return self
class UpdateScheduleEdgeRequest(StrictModel):
workflow_version: int = Field(ge=1)
condition_expr: str | None = Field(default=None, max_length=1000)
@field_validator("condition_expr")
@classmethod
def normalize_condition(cls, value: str | None) -> str | None:
if value is None:
return None
normalized = value.strip()
return normalized or None
class CronPreviewRequest(StrictModel):
cron_expression: str = Field(min_length=1, max_length=128)
timezone: str = Field(default="Asia/Shanghai", min_length=1, max_length=64)
count: int = Field(default=5, ge=1, le=20)
base_time: datetime | None = None
@field_validator("cron_expression", "timezone")
@classmethod
def validate_required_text(cls, value: str) -> str:
return _required_text(value)
File diff suppressed because it is too large Load Diff
+72
View File
@@ -0,0 +1,72 @@
from __future__ import annotations
from typing import Literal
from pydantic import BaseModel, ConfigDict, Field, field_validator
class StrictModel(BaseModel):
model_config = ConfigDict(extra="forbid")
class CreateResourceUploadRequest(StrictModel):
file_name: str = Field(min_length=1, max_length=255)
content_type: str = Field(min_length=1, max_length=255)
expected_size_bytes: int = Field(ge=0, le=100 * 1024 * 1024)
expected_hash: str | None = Field(default=None, min_length=64, max_length=64)
@field_validator("expected_hash")
@classmethod
def validate_hash(cls, value: str | None) -> str | None:
if value is None:
return None
normalized = value.lower()
if any(character not in "0123456789abcdef" for character in normalized):
raise ValueError("expected_hash must be SHA-256 hex")
return normalized
class CompleteResourceUploadRequest(StrictModel):
resource_name: str = Field(min_length=1, max_length=255)
description: str | None = Field(default=None, max_length=1000)
visibility: Literal["private", "workspace", "public"] = "private"
class CreateScriptRequest(StrictModel):
script_name: str = Field(min_length=1, max_length=255)
script_type: Literal["python", "notebook"]
content: str = Field(max_length=10 * 1024 * 1024)
visibility: Literal["private", "workspace", "public"] = "private"
parent_path: str | None = Field(default=None, max_length=1024)
class CreateWorkspaceDirectoryRequest(StrictModel):
directory_name: str = Field(min_length=1, max_length=255)
parent_path: str = Field(default="", max_length=1024)
class UpdateScriptRequest(StrictModel):
content: str = Field(max_length=10 * 1024 * 1024)
class PublishVersionRequest(StrictModel):
source_object_id: str | None = Field(
default=None,
min_length=26,
max_length=26,
)
release_note: str | None = Field(default=None, max_length=1000)
visibility: Literal["private", "workspace", "public"] = "workspace"
class DownloadUrlRequest(StrictModel):
expires_seconds: int = Field(default=300, ge=30, le=3600)
class FileLockTokenRequest(StrictModel):
lock_token: str = Field(min_length=32, max_length=256)
class CreateJupyterAccessTicketRequest(StrictModel):
edit_session_id: str = Field(min_length=26, max_length=26)
lock_token: str = Field(min_length=32, max_length=256)
+925
View File
@@ -0,0 +1,925 @@
from __future__ import annotations
import hashlib
import json
import mimetypes
import os
import shutil
import tempfile
from datetime import UTC, datetime
from pathlib import Path, PurePosixPath
from typing import Any
from fastapi import (
APIRouter,
Depends,
Header,
HTTPException,
Query,
Request,
status,
)
from sqlalchemy import func, select
from sqlalchemy.ext.asyncio import AsyncSession
from common.db.models import (
EditSessions,
Scripts,
StorageObjects,
Versions,
)
from common.ids import new_ulid
from backend.dependencies import (
RequestContext,
database_session,
request_context,
)
from backend.schemas import (
CreateScriptRequest,
CreateWorkspaceDirectoryRequest,
DownloadUrlRequest,
PublishVersionRequest,
UpdateScriptRequest,
)
router = APIRouter(tags=["scripts"])
def normalize_user_path(value: str, *, allow_empty: bool = True) -> str:
normalized = value.replace("\\", "/").strip().strip("/")
if not normalized and allow_empty:
return ""
pure_path = PurePosixPath(normalized)
if (
pure_path.is_absolute()
or not pure_path.parts
or any(
part in {"", ".", ".."} or any(ord(char) < 32 for char in part)
for part in pure_path.parts
)
):
raise HTTPException(
status.HTTP_422_UNPROCESSABLE_ENTITY,
"invalid workspace path",
)
return pure_path.as_posix()
def safe_directory_name(value: str) -> str:
name = value.strip()
if (
not name
or name in {".", ".."}
or name.startswith(".")
or "/" in name
or "\\" in name
or any(ord(char) < 32 for char in name)
):
raise HTTPException(
status.HTTP_422_UNPROCESSABLE_ENTITY,
"invalid directory_name",
)
return name
def user_relative_path(context: RequestContext, child_path: str = "") -> str:
base = f"users/{context.user.username}"
normalized = normalize_user_path(child_path)
return f"{base}/{normalized}" if normalized else base
def safe_script_name(value: str, script_type: str) -> str:
name = value.replace("\\", "/").rsplit("/", 1)[-1].strip()
if not name or name in {".", ".."} or any(ord(char) < 32 for char in name):
raise HTTPException(
status.HTTP_422_UNPROCESSABLE_ENTITY,
"invalid script_name",
)
expected_suffix = ".py" if script_type == "python" else ".ipynb"
if not name.lower().endswith(expected_suffix):
name += expected_suffix
return name
def validate_script_content(content: str, script_type: str) -> bytes:
encoded = content.encode("utf-8")
if len(encoded) > 10 * 1024 * 1024:
raise HTTPException(
status.HTTP_413_REQUEST_ENTITY_TOO_LARGE,
"script exceeds 10 MiB",
)
if script_type == "notebook":
try:
notebook = json.loads(content)
except json.JSONDecodeError as exc:
raise HTTPException(
status.HTTP_422_UNPROCESSABLE_ENTITY,
"notebook content must be valid JSON",
) from exc
if not isinstance(notebook, dict) or not isinstance(
notebook.get("cells"),
list,
):
raise HTTPException(
status.HTTP_422_UNPROCESSABLE_ENTITY,
"notebook must contain a cells array",
)
return encoded
def workspace_target(
context: RequestContext,
relative_path: str,
) -> Path:
root = Path(
os.getenv("WORKSPACE_ROOT", "/workspace/workspaces")
).resolve()
scoped_root = (root / context.workspace.workspace_code).resolve()
pure_path = PurePosixPath(relative_path)
target = (scoped_root / Path(*pure_path.parts)).resolve()
if scoped_root not in target.parents:
raise HTTPException(
status.HTTP_422_UNPROCESSABLE_ENTITY,
"script path escapes workspace root",
)
return target
def apply_workspace_permissions(path: Path, mode: int) -> None:
os.chmod(path, mode)
if os.name != "nt":
shared_gid = int(os.getenv("WORKSPACE_SHARED_GID", "100"))
os.chown(path, -1, shared_gid)
def atomic_write(target: Path, content: bytes) -> None:
target.parent.mkdir(parents=True, exist_ok=True)
apply_workspace_permissions(target.parent, 0o2770)
descriptor, temporary_name = tempfile.mkstemp(
prefix=f".{target.name}.",
suffix=".tmp",
dir=target.parent,
)
try:
with os.fdopen(descriptor, "wb") as handle:
handle.write(content)
handle.flush()
os.fsync(handle.fileno())
os.replace(temporary_name, target)
apply_workspace_permissions(target, 0o660)
finally:
if os.path.exists(temporary_name):
os.unlink(temporary_name)
def script_payload(
script: Scripts,
storage_object: StorageObjects | dict[str, Any],
) -> dict[str, Any]:
if isinstance(storage_object, dict):
relative_path = storage_object.get("relative_path")
content_hash = storage_object.get("content_hash")
size_bytes = storage_object.get("size_bytes", 0)
else:
relative_path = storage_object.relative_path
content_hash = storage_object.content_hash
size_bytes = storage_object.size_bytes
return {
"script_id": script.script_id,
"workspace_id": script.workspace_id,
"current_object_id": script.current_object_id,
"owner_user_id": script.owner_user_id,
"script_name": script.script_name,
"script_type": script.script_type,
"visibility": script.visibility,
"status": script.status,
"relative_path": relative_path,
"content_hash": content_hash,
"size_bytes": size_bytes,
"created_at": script.created_at.isoformat(),
"updated_at": script.updated_at.isoformat(),
}
def version_payload(version: Versions) -> dict[str, Any]:
return {
"versions_id": version.versions_id,
"workspace_id": version.workspace_id,
"script_id": version.script_id,
"source_object_id": version.source_object_id,
"artifact_object_id": version.artifact_object_id,
"version_no": version.version_no,
"version_label": version.version_label,
"source_path": version.source_path,
"artifact_path": version.artifact_path,
"content_hash": version.content_hash,
"file_size_bytes": version.file_size_bytes,
"visibility": version.visibility,
"release_note": version.release_note,
"created_by": version.created_by,
"created_at": version.created_at.isoformat(),
}
async def get_script_row(
script_id: str,
context: RequestContext,
session: AsyncSession,
*,
for_update: bool = False,
) -> tuple[Scripts, StorageObjects]:
if for_update:
script = await session.scalar(
select(Scripts)
.where(
Scripts.script_id == script_id,
Scripts.workspace_id == context.workspace.workspace_id,
Scripts.status == "active",
)
.with_for_update()
)
if script is None:
raise HTTPException(
status.HTTP_404_NOT_FOUND,
"script not found",
)
storage_object = await session.get(
StorageObjects,
script.current_object_id,
)
if storage_object is None:
raise HTTPException(
status.HTTP_409_CONFLICT,
"script working-copy metadata is missing",
)
row = (script, storage_object)
else:
statement = (
select(Scripts, StorageObjects)
.join(
StorageObjects,
StorageObjects.storage_object_id
== Scripts.current_object_id,
)
.where(
Scripts.script_id == script_id,
Scripts.workspace_id == context.workspace.workspace_id,
Scripts.status == "active",
)
)
row = (await session.execute(statement)).one_or_none()
if row is None:
raise HTTPException(status.HTTP_404_NOT_FOUND, "script not found")
script, storage_object = row
return script, storage_object
async def require_no_active_edit_session(
session: AsyncSession,
storage_object_ids: list[str],
) -> None:
if not storage_object_ids:
return
active_session = await session.scalar(
select(EditSessions).where(
EditSessions.storage_object_id.in_(storage_object_ids),
EditSessions.session_status == "active",
EditSessions.expires_at > datetime.now(UTC).replace(tzinfo=None),
)
)
if active_session is not None:
raise HTTPException(
status.HTTP_409_CONFLICT,
"file is being edited; end the editing session before deletion",
)
async def create_script_record(
*,
name: str,
script_type: str,
content: bytes,
visibility: str,
parent_path: str | None,
request: Request,
context: RequestContext,
session: AsyncSession,
) -> tuple[Scripts, dict[str, Any]]:
folder = (
"scripts" if script_type == "python" else "notebooks"
) if parent_path is None else normalize_user_path(parent_path)
child_path = f"{folder}/{name}" if folder else name
relative_path = user_relative_path(context, child_path)
target = workspace_target(context, relative_path)
existing_script = await session.scalar(
select(Scripts)
.join(
StorageObjects,
StorageObjects.storage_object_id == Scripts.current_object_id,
)
.where(
Scripts.workspace_id == context.workspace.workspace_id,
StorageObjects.relative_path == relative_path,
Scripts.status == "active",
)
)
if existing_script is not None or target.exists():
raise HTTPException(
status.HTTP_409_CONFLICT,
"a file with the same path already exists",
)
atomic_write(target, content)
storage_data = await request.app.state.storage_client.register_workspace_object(
{
"workspace_id": context.workspace.workspace_id,
"user_id": context.user.user_id,
"relative_path": relative_path,
"usage_type": "working_copy",
"visibility": visibility,
}
)
object_id = storage_data["storage_object_id"]
script = await session.scalar(
select(Scripts).where(Scripts.current_object_id == object_id)
)
now = datetime.now(UTC).replace(tzinfo=None)
if script is None:
script = Scripts(
script_id=new_ulid(),
workspace_id=context.workspace.workspace_id,
current_object_id=object_id,
owner_user_id=context.user.user_id,
script_name=name,
script_type=script_type,
visibility=visibility,
status="active",
)
session.add(script)
else:
script.owner_user_id = context.user.user_id
script.script_name = name
script.script_type = script_type
script.visibility = visibility
script.status = "active"
script.deleted_at = None
script.updated_at = now
await session.flush()
await session.refresh(script)
return script, storage_data
@router.post("/api/v1/scripts", status_code=status.HTTP_201_CREATED)
async def create_script(
payload: CreateScriptRequest,
request: Request,
context: RequestContext = Depends(request_context),
session: AsyncSession = Depends(database_session),
) -> dict[str, Any]:
name = safe_script_name(payload.script_name, payload.script_type)
content = validate_script_content(payload.content, payload.script_type)
script, storage_data = await create_script_record(
name=name,
script_type=payload.script_type,
content=content,
visibility=payload.visibility,
parent_path=payload.parent_path,
request=request,
context=context,
session=session,
)
return {
"request_id": context.request_id,
"data": script_payload(script, storage_data),
"meta": {},
}
@router.post(
"/api/v1/scripts/upload",
status_code=status.HTTP_201_CREATED,
)
async def upload_script(
request: Request,
file_name: str = Query(min_length=1, max_length=255),
parent_path: str = Query(default="", max_length=1024),
visibility: str = Query(
default="workspace",
pattern="^(private|workspace|public)$",
),
context: RequestContext = Depends(request_context),
session: AsyncSession = Depends(database_session),
) -> dict[str, Any]:
suffix = Path(file_name).suffix.lower()
if suffix not in {".py", ".ipynb"}:
raise HTTPException(
status.HTTP_422_UNPROCESSABLE_ENTITY,
"only .py and .ipynb files can be uploaded",
)
script_type = "notebook" if suffix == ".ipynb" else "python"
name = safe_script_name(file_name, script_type)
body = await request.body()
if len(body) > 10 * 1024 * 1024:
raise HTTPException(
status.HTTP_413_REQUEST_ENTITY_TOO_LARGE,
"script exceeds 10 MiB",
)
try:
text = body.decode("utf-8")
except UnicodeDecodeError as exc:
raise HTTPException(
status.HTTP_422_UNPROCESSABLE_ENTITY,
"script must use UTF-8 encoding",
) from exc
content = validate_script_content(text, script_type)
script, storage_data = await create_script_record(
name=name,
script_type=script_type,
content=content,
visibility=visibility,
parent_path=parent_path,
request=request,
context=context,
session=session,
)
return {
"request_id": context.request_id,
"data": script_payload(script, storage_data),
"meta": {},
}
@router.get("/api/v1/workspace-tree")
async def get_workspace_tree(
context: RequestContext = Depends(request_context),
) -> dict[str, Any]:
root = workspace_target(context, user_relative_path(context))
root.mkdir(parents=True, exist_ok=True)
apply_workspace_permissions(root, 0o2770)
directories: list[dict[str, str]] = []
for current, names, _files in os.walk(root, followlinks=False):
names[:] = sorted(
name
for name in names
if not name.startswith(".")
and not (Path(current) / name).is_symlink()
)
current_path = Path(current)
if current_path == root:
continue
relative = current_path.relative_to(root).as_posix()
parent = PurePosixPath(relative).parent.as_posix()
directories.append(
{
"path": relative,
"name": current_path.name,
"parent_path": "" if parent == "." else parent,
}
)
return {
"request_id": context.request_id,
"data": {"directories": directories},
"meta": {"directory_count": len(directories)},
}
@router.post(
"/api/v1/workspace-directories",
status_code=status.HTTP_201_CREATED,
)
async def create_workspace_directory(
payload: CreateWorkspaceDirectoryRequest,
context: RequestContext = Depends(request_context),
) -> dict[str, Any]:
name = safe_directory_name(payload.directory_name)
parent = normalize_user_path(payload.parent_path)
child_path = f"{parent}/{name}" if parent else name
target = workspace_target(context, user_relative_path(context, child_path))
parent_target = target.parent
user_root = workspace_target(context, user_relative_path(context))
if (
not parent_target.is_dir()
or (
parent_target != user_root
and user_root not in parent_target.parents
)
):
raise HTTPException(
status.HTTP_404_NOT_FOUND,
"parent directory not found",
)
if target.exists():
raise HTTPException(
status.HTTP_409_CONFLICT,
"a file or directory with the same path already exists",
)
target.mkdir(parents=False)
apply_workspace_permissions(target, 0o2770)
return {
"request_id": context.request_id,
"data": {
"path": child_path,
"name": name,
"parent_path": parent,
},
"meta": {},
}
@router.delete("/api/v1/workspace-directories")
async def delete_workspace_directory(
request: Request,
path: str = Query(min_length=1, max_length=1024),
context: RequestContext = Depends(request_context),
session: AsyncSession = Depends(database_session),
) -> dict[str, Any]:
directory_path = normalize_user_path(path, allow_empty=False)
relative_prefix = user_relative_path(context, directory_path)
target = workspace_target(context, relative_prefix)
if not target.is_dir() or target.is_symlink():
raise HTTPException(
status.HTTP_404_NOT_FOUND,
"directory not found",
)
rows = (
await session.execute(
select(Scripts, StorageObjects)
.join(
StorageObjects,
StorageObjects.storage_object_id == Scripts.current_object_id,
)
.where(
Scripts.workspace_id == context.workspace.workspace_id,
Scripts.owner_user_id == context.user.user_id,
Scripts.status == "active",
StorageObjects.relative_path.startswith(
f"{relative_prefix}/"
),
)
)
).all()
await require_no_active_edit_session(
session,
[script.current_object_id for script, _storage in rows],
)
for script, _storage in rows:
await request.app.state.storage_client.delete_object(
script.current_object_id
)
script.status = "deleted"
script.deleted_at = datetime.now(UTC).replace(tzinfo=None)
shutil.rmtree(target)
return {
"request_id": context.request_id,
"data": {
"path": directory_path,
"status": "deleted",
"deleted_scripts": len(rows),
"versions_preserved": True,
},
"meta": {},
}
@router.get("/api/v1/scripts")
async def list_scripts(
context: RequestContext = Depends(request_context),
session: AsyncSession = Depends(database_session),
) -> dict[str, Any]:
statement = (
select(Scripts, StorageObjects)
.join(
StorageObjects,
StorageObjects.storage_object_id == Scripts.current_object_id,
)
.where(
Scripts.workspace_id == context.workspace.workspace_id,
Scripts.status == "active",
)
.order_by(Scripts.updated_at.desc())
)
rows = (await session.execute(statement)).all()
return {
"request_id": context.request_id,
"data": [
script_payload(script, storage_object)
for script, storage_object in rows
],
"meta": {"count": len(rows)},
}
@router.get("/api/v1/scripts/{script_id}")
async def get_script(
script_id: str,
context: RequestContext = Depends(request_context),
session: AsyncSession = Depends(database_session),
) -> dict[str, Any]:
script, storage_object = await get_script_row(
script_id,
context,
session,
)
return {
"request_id": context.request_id,
"data": script_payload(script, storage_object),
"meta": {},
}
@router.put("/api/v1/scripts/{script_id}")
async def update_script(
script_id: str,
payload: UpdateScriptRequest,
request: Request,
context: RequestContext = Depends(request_context),
session: AsyncSession = Depends(database_session),
) -> dict[str, Any]:
script, storage_object = await get_script_row(
script_id,
context,
session,
for_update=True,
)
if (
script.owner_user_id != context.user.user_id
and not context.is_admin
):
raise HTTPException(
status.HTTP_403_FORBIDDEN,
"script can only be changed by its owner or an administrator",
)
content = validate_script_content(payload.content, script.script_type)
if not storage_object.relative_path:
raise HTTPException(
status.HTTP_409_CONFLICT,
"script has no workspace path",
)
target = workspace_target(context, storage_object.relative_path)
atomic_write(target, content)
storage_data = await request.app.state.storage_client.register_workspace_object(
{
"workspace_id": context.workspace.workspace_id,
"user_id": script.owner_user_id,
"relative_path": storage_object.relative_path,
"usage_type": "working_copy",
"visibility": script.visibility,
}
)
storage_object.content_hash = storage_data["content_hash"]
storage_object.size_bytes = storage_data["size_bytes"]
script.updated_at = datetime.now(UTC).replace(tzinfo=None)
return {
"request_id": context.request_id,
"data": script_payload(script, storage_object),
"meta": {},
}
@router.delete("/api/v1/scripts/{script_id}")
async def delete_script(
script_id: str,
request: Request,
context: RequestContext = Depends(request_context),
session: AsyncSession = Depends(database_session),
) -> dict[str, Any]:
script, storage_object = await get_script_row(
script_id,
context,
session,
for_update=True,
)
if (
script.owner_user_id != context.user.user_id
and not context.is_admin
):
raise HTTPException(
status.HTTP_403_FORBIDDEN,
"script can only be deleted by its owner or an administrator",
)
await require_no_active_edit_session(
session,
[script.current_object_id],
)
if storage_object.relative_path:
target = workspace_target(context, storage_object.relative_path)
if target.is_file():
target.unlink()
await request.app.state.storage_client.delete_object(
script.current_object_id
)
script.status = "deleted"
script.deleted_at = datetime.now(UTC).replace(tzinfo=None)
return {
"request_id": context.request_id,
"data": {
"script_id": script.script_id,
"status": script.status,
"versions_preserved": True,
},
"meta": {},
}
@router.post(
"/api/v1/scripts/{script_id}/versions",
status_code=status.HTTP_201_CREATED,
)
async def publish_version(
script_id: str,
payload: PublishVersionRequest,
request: Request,
context: RequestContext = Depends(request_context),
session: AsyncSession = Depends(database_session),
) -> dict[str, Any]:
script, source_object = await get_script_row(
script_id,
context,
session,
for_update=True,
)
if (
script.owner_user_id != context.user.user_id
and not context.is_admin
):
raise HTTPException(
status.HTTP_403_FORBIDDEN,
"script can only be published by its owner or an administrator",
)
if (
payload.source_object_id
and payload.source_object_id != script.current_object_id
):
raise HTTPException(
status.HTTP_412_PRECONDITION_FAILED,
"source_object_id is not the current working copy",
)
if not source_object.relative_path:
raise HTTPException(
status.HTTP_409_CONFLICT,
"script has no workspace path",
)
target = workspace_target(context, source_object.relative_path)
if not target.is_file():
raise HTTPException(
status.HTTP_409_CONFLICT,
"script working copy is missing",
)
content = target.read_bytes()
content_hash = hashlib.sha256(content).hexdigest()
existing = await session.scalar(
select(Versions).where(
Versions.script_id == script.script_id,
Versions.content_hash == content_hash,
)
)
if existing is not None:
return {
"request_id": context.request_id,
"data": version_payload(existing),
"meta": {"reused": True},
}
content_type = (
mimetypes.guess_type(script.script_name)[0]
or "application/octet-stream"
)
artifact = await request.app.state.storage_client.create_server_object(
workspace_id=context.workspace.workspace_id,
user_id=context.user.user_id,
usage_type="version_artifact",
file_name=script.script_name,
content_type=content_type,
content=content,
visibility=payload.visibility,
is_immutable=True,
idempotency_key=f"version:{script.script_id}:{content_hash}",
)
current_max = await session.scalar(
select(func.max(Versions.version_no)).where(
Versions.script_id == script.script_id
)
)
version_no = int(current_max or 0) + 1
version = Versions(
versions_id=new_ulid(),
workspace_id=context.workspace.workspace_id,
script_id=script.script_id,
source_object_id=script.current_object_id,
artifact_object_id=artifact["storage_object_id"],
version_no=version_no,
version_label=f"v{version_no}.0",
source_path=source_object.relative_path,
artifact_path=artifact["storage_uri"],
content_hash=content_hash,
file_size_bytes=len(content),
visibility=payload.visibility,
release_note=payload.release_note,
created_by=context.user.user_id,
)
session.add(version)
await session.flush()
await session.refresh(version)
return {
"request_id": context.request_id,
"data": version_payload(version),
"meta": {"reused": False},
}
@router.get("/api/v1/scripts/{script_id}/versions")
async def list_versions(
script_id: str,
context: RequestContext = Depends(request_context),
session: AsyncSession = Depends(database_session),
) -> dict[str, Any]:
await get_script_row(script_id, context, session)
versions = (
await session.scalars(
select(Versions)
.where(Versions.script_id == script_id)
.order_by(Versions.version_no.desc())
)
).all()
return {
"request_id": context.request_id,
"data": [version_payload(version) for version in versions],
"meta": {"count": len(versions)},
}
@router.get("/api/v1/versions/{versions_id}")
async def get_version(
versions_id: str,
context: RequestContext = Depends(request_context),
session: AsyncSession = Depends(database_session),
) -> dict[str, Any]:
version = await session.get(Versions, versions_id)
if (
version is None
or version.workspace_id != context.workspace.workspace_id
):
raise HTTPException(status.HTTP_404_NOT_FOUND, "version not found")
return {
"request_id": context.request_id,
"data": version_payload(version),
"meta": {},
}
@router.delete("/api/v1/versions/{versions_id}")
async def delete_version(
versions_id: str,
context: RequestContext = Depends(request_context),
session: AsyncSession = Depends(database_session),
) -> dict[str, Any]:
version = await session.get(Versions, versions_id)
if (
version is None
or version.workspace_id != context.workspace.workspace_id
):
raise HTTPException(status.HTTP_404_NOT_FOUND, "稳定版本不存在")
if (
version.created_by != context.user.user_id
and not context.is_admin
):
raise HTTPException(
status.HTTP_403_FORBIDDEN,
"仅稳定版本发布者或管理员可以删除",
)
version.schedule_hidden_at = datetime.now(UTC).replace(tzinfo=None)
await session.flush()
return {
"request_id": context.request_id,
"data": {
"versions_id": versions_id,
"deleted": True,
"artifact_preserved": True,
},
"meta": {
"message": "已从调度稳定版本列表移除;稳定版本和历史记录保持不变",
},
}
@router.post("/api/v1/versions/{versions_id}/download-url")
async def version_download_url(
versions_id: str,
payload: DownloadUrlRequest,
request: Request,
context: RequestContext = Depends(request_context),
session: AsyncSession = Depends(database_session),
) -> dict[str, Any]:
version = await session.get(Versions, versions_id)
if (
version is None
or version.workspace_id != context.workspace.workspace_id
):
raise HTTPException(status.HTTP_404_NOT_FOUND, "version not found")
data = await request.app.state.storage_client.create_download_url(
version.artifact_object_id,
payload.expires_seconds,
)
return {"request_id": context.request_id, "data": data, "meta": {}}
+681
View File
@@ -0,0 +1,681 @@
from __future__ import annotations
import asyncio
import base64
import binascii
import hashlib
import mimetypes
import os
import secrets
from contextlib import asynccontextmanager
from datetime import UTC, datetime, timedelta
from pathlib import Path, PurePosixPath
from typing import Any, AsyncIterator
from fastapi import Depends, Header, HTTPException, Request, status
from sqlalchemy import select
from sqlalchemy.ext.asyncio import AsyncSession
from common.db import create_database_engine, create_session_factory
from common.db.models import (
StorageObjects,
UploadSessions,
Users,
WorkspaceMembers,
Workspaces,
)
from common.ids import new_ulid
from common.service_app import create_service_app
from common.storage import RustFSObjectStore
from backend.storage_schemas import (
CompleteUploadRequest,
CreateUploadRequest,
DownloadUrlRequest,
RegisterWorkspaceObjectRequest,
ServerObjectRequest,
)
def utcnow() -> datetime:
return datetime.now(UTC).replace(tzinfo=None)
def hash_bytes(value: str) -> bytes:
return hashlib.sha256(value.encode("utf-8")).digest()
def normalized_idempotency_key(
workspace_id: str,
user_id: str,
value: str,
) -> str:
digest = hashlib.sha256(
f"{workspace_id}:{user_id}:{value}".encode("utf-8")
).hexdigest()
return f"v1:{digest}"
def safe_file_name(value: str) -> str:
name = value.replace("\\", "/").rsplit("/", 1)[-1].strip()
if not name or name in {".", ".."} or any(ord(char) < 32 for char in name):
raise HTTPException(status.HTTP_422_UNPROCESSABLE_ENTITY, "invalid file_name")
return name
def storage_payload(item: StorageObjects) -> dict[str, Any]:
return {
"storage_object_id": item.storage_object_id,
"workspace_id": item.workspace_id,
"owner_user_id": item.owner_user_id,
"storage_backend": item.storage_backend,
"usage_type": item.usage_type,
"storage_uri": item.storage_uri,
"relative_path": item.relative_path,
"file_name": item.file_name,
"file_extension": item.file_extension,
"mime_type": item.mime_type,
"size_bytes": item.size_bytes,
"content_hash": item.content_hash,
"visibility": item.visibility,
"is_immutable": bool(item.is_immutable),
"object_status": item.object_status,
"created_at": item.created_at.isoformat(),
"updated_at": item.updated_at.isoformat(),
}
@asynccontextmanager
async def lifespan(app: Any) -> AsyncIterator[None]:
database_url = os.environ["DATABASE_URL"]
engine = create_database_engine(database_url)
app.state.session_factory = create_session_factory(engine)
app.state.object_store = RustFSObjectStore(
internal_endpoint=os.getenv(
"RUSTFS_INTERNAL_ENDPOINT",
"http://rustfs:9000",
),
public_endpoint=os.getenv(
"RUSTFS_PUBLIC_ENDPOINT",
"http://localhost:9000",
),
access_key=os.environ["RUSTFS_ACCESS_KEY"],
secret_key=os.environ["RUSTFS_SECRET_KEY"],
)
app.state.default_bucket = os.getenv(
"RUSTFS_DEFAULT_BUCKET",
"model-platform",
)
await asyncio.to_thread(
app.state.object_store.ensure_bucket,
app.state.default_bucket,
)
try:
yield
finally:
await engine.dispose()
app = create_service_app(
os.getenv("SERVICE_NAME", "storage-api"),
lifespan=lifespan,
)
async def database_session(request: Request) -> AsyncIterator[AsyncSession]:
async with request.app.state.session_factory() as session:
try:
yield session
await session.commit()
except Exception:
await session.rollback()
raise
def verify_internal_service(
x_service_token: str = Header(alias="X-Service-Token"),
) -> None:
expected = os.environ.get("INTERNAL_SERVICE_TOKEN", "")
if not expected or not secrets.compare_digest(expected, x_service_token):
raise HTTPException(
status.HTTP_401_UNAUTHORIZED,
"invalid internal service identity",
)
async def require_workspace_member(
session: AsyncSession,
workspace_id: str,
user_id: str,
) -> Workspaces:
statement = (
select(Workspaces)
.join(
WorkspaceMembers,
WorkspaceMembers.workspace_id == Workspaces.workspace_id,
)
.join(Users, Users.user_id == WorkspaceMembers.user_id)
.where(
Workspaces.workspace_id == workspace_id,
Workspaces.status == "active",
WorkspaceMembers.user_id == user_id,
WorkspaceMembers.member_status == "active",
Users.status == "active",
)
)
workspace = await session.scalar(statement)
if workspace is None:
raise HTTPException(
status.HTTP_403_FORBIDDEN,
"user is not an active workspace member",
)
return workspace
async def create_upload_record(
payload: CreateUploadRequest,
session: AsyncSession,
request: Request,
) -> dict[str, Any]:
workspace = await require_workspace_member(
session,
payload.workspace_id,
payload.user_id,
)
stored_key = normalized_idempotency_key(
payload.workspace_id,
payload.user_id,
payload.idempotency_key,
)
existing = await session.scalar(
select(UploadSessions).where(
UploadSessions.idempotency_key == stored_key
)
)
if existing is not None:
if (
existing.workspace_id != payload.workspace_id
or existing.user_id != payload.user_id
or existing.expected_size_bytes != payload.expected_size_bytes
or existing.expected_hash != payload.expected_hash
or existing.content_type != payload.content_type
):
raise HTTPException(
status.HTTP_409_CONFLICT,
"idempotency key was used with different upload metadata",
)
upload = existing
else:
upload_id = new_ulid()
file_name = safe_file_name(payload.file_name)
bucket_name = (
workspace.artifact_bucket or request.app.state.default_bucket
)
object_key = (
f"workspaces/{payload.workspace_id}/"
f"{payload.usage_type}/{upload_id}/{file_name}"
)
upload = UploadSessions(
upload_id=upload_id,
workspace_id=payload.workspace_id,
user_id=payload.user_id,
idempotency_key=stored_key,
bucket_name=bucket_name,
object_key=object_key,
object_key_hash=hash_bytes(object_key),
upload_status="created",
expires_at=utcnow() + timedelta(minutes=15),
expected_size_bytes=payload.expected_size_bytes,
expected_hash=payload.expected_hash,
content_type=payload.content_type,
)
session.add(upload)
await session.flush()
if upload.upload_status == "completed" and upload.storage_object_id:
storage_object = await session.get(
StorageObjects,
upload.storage_object_id,
)
return {
"upload_id": upload.upload_id,
"status": upload.upload_status,
"storage_object": (
storage_payload(storage_object) if storage_object else None
),
}
if upload.upload_status not in {"created", "uploading"}:
raise HTTPException(
status.HTTP_409_CONFLICT,
f"upload cannot continue from status {upload.upload_status}",
)
url, headers = request.app.state.object_store.presign_put(
bucket_name=upload.bucket_name,
object_key=upload.object_key,
content_type=upload.content_type or "application/octet-stream",
expected_hash=upload.expected_hash,
expires_seconds=900,
public=payload.url_scope == "public",
)
return {
"upload_id": upload.upload_id,
"status": upload.upload_status,
"method": "PUT",
"presigned_url": url,
"required_headers": headers,
"expires_at": upload.expires_at.isoformat(),
}
async def complete_upload_record(
upload_id: str,
payload: CompleteUploadRequest,
session: AsyncSession,
request: Request,
) -> StorageObjects:
upload = await session.scalar(
select(UploadSessions)
.where(UploadSessions.upload_id == upload_id)
.with_for_update()
)
if upload is None:
raise HTTPException(status.HTTP_404_NOT_FOUND, "upload not found")
if upload.upload_status == "completed" and upload.storage_object_id:
item = await session.get(StorageObjects, upload.storage_object_id)
if item is None:
raise HTTPException(
status.HTTP_409_CONFLICT,
"completed upload has no storage object",
)
return item
if upload.upload_status not in {"created", "uploading"}:
raise HTTPException(
status.HTTP_409_CONFLICT,
f"upload cannot be completed from status {upload.upload_status}",
)
if upload.expires_at < utcnow():
upload.upload_status = "expired"
raise HTTPException(status.HTTP_409_CONFLICT, "upload expired")
try:
head = await asyncio.to_thread(
request.app.state.object_store.head,
bucket_name=upload.bucket_name,
object_key=upload.object_key,
)
except Exception as exc:
raise HTTPException(
status.HTTP_409_CONFLICT,
"uploaded object is not available",
) from exc
actual_size = int(head.get("ContentLength", 0))
if (
upload.expected_size_bytes is not None
and actual_size != upload.expected_size_bytes
):
upload.upload_status = "failed"
raise HTTPException(
status.HTTP_409_CONFLICT,
"uploaded object size does not match expected_size_bytes",
)
actual_content_type = str(
head.get("ContentType") or "application/octet-stream"
)
if upload.content_type and actual_content_type != upload.content_type:
upload.upload_status = "failed"
raise HTTPException(
status.HTTP_409_CONFLICT,
"uploaded object content type does not match",
)
metadata = {
str(key).lower(): str(value).lower()
for key, value in dict(head.get("Metadata") or {}).items()
}
actual_hash = metadata.get("sha256")
if not actual_hash:
actual_hash = await asyncio.to_thread(
request.app.state.object_store.sha256,
bucket_name=upload.bucket_name,
object_key=upload.object_key,
)
if upload.expected_hash and actual_hash != upload.expected_hash:
upload.upload_status = "failed"
raise HTTPException(
status.HTTP_409_CONFLICT,
"uploaded object hash does not match expected_hash",
)
file_name = upload.object_key.rsplit("/", 1)[-1]
item = StorageObjects(
storage_object_id=new_ulid(),
workspace_id=upload.workspace_id,
owner_user_id=upload.user_id,
object_type="file",
usage_type=payload.usage_type,
storage_backend="rustfs",
bucket_name=upload.bucket_name,
object_key=upload.object_key,
object_key_hash=upload.object_key_hash,
storage_uri=f"s3://{upload.bucket_name}/{upload.object_key}",
file_name=file_name,
file_extension=Path(file_name).suffix.lower() or None,
mime_type=actual_content_type,
size_bytes=actual_size,
content_hash=actual_hash,
object_etag=str(head.get("ETag", "")).strip('"') or None,
visibility=payload.visibility,
is_immutable=int(payload.is_immutable),
object_status="available",
created_by=upload.user_id,
)
session.add(item)
await session.flush()
await session.refresh(item)
upload.storage_object_id = item.storage_object_id
upload.upload_status = "completed"
upload.completed_at = utcnow()
return item
@app.post("/internal/v1/uploads", dependencies=[Depends(verify_internal_service)])
async def create_upload(
payload: CreateUploadRequest,
request: Request,
session: AsyncSession = Depends(database_session),
) -> dict[str, Any]:
return {
"data": await create_upload_record(payload, session, request),
}
@app.post(
"/internal/v1/uploads/{upload_id}/complete",
dependencies=[Depends(verify_internal_service)],
)
async def complete_upload(
upload_id: str,
payload: CompleteUploadRequest,
request: Request,
session: AsyncSession = Depends(database_session),
) -> dict[str, Any]:
item = await complete_upload_record(
upload_id,
payload,
session,
request,
)
return {"data": storage_payload(item)}
@app.post(
"/internal/v1/uploads/{upload_id}/abort",
dependencies=[Depends(verify_internal_service)],
)
async def abort_upload(
upload_id: str,
request: Request,
session: AsyncSession = Depends(database_session),
) -> dict[str, Any]:
upload = await session.scalar(
select(UploadSessions)
.where(UploadSessions.upload_id == upload_id)
.with_for_update()
)
if upload is None:
raise HTTPException(status.HTTP_404_NOT_FOUND, "upload not found")
if upload.upload_status == "completed":
raise HTTPException(
status.HTTP_409_CONFLICT,
"completed upload cannot be aborted",
)
if upload.upload_status != "aborted":
await asyncio.to_thread(
request.app.state.object_store.delete,
bucket_name=upload.bucket_name,
object_key=upload.object_key,
)
upload.upload_status = "aborted"
return {"data": {"upload_id": upload_id, "status": "aborted"}}
@app.post(
"/internal/v1/objects",
dependencies=[Depends(verify_internal_service)],
)
async def create_server_object(
payload: ServerObjectRequest,
request: Request,
session: AsyncSession = Depends(database_session),
) -> dict[str, Any]:
try:
content = base64.b64decode(payload.content_base64, validate=True)
except (binascii.Error, ValueError) as exc:
raise HTTPException(
status.HTTP_422_UNPROCESSABLE_ENTITY,
"content_base64 is invalid",
) from exc
if len(content) > 100 * 1024 * 1024:
raise HTTPException(
status.HTTP_413_REQUEST_ENTITY_TOO_LARGE,
"object exceeds 100 MiB server-side upload limit",
)
content_hash = hashlib.sha256(content).hexdigest()
upload_result = await create_upload_record(
CreateUploadRequest(
workspace_id=payload.workspace_id,
user_id=payload.user_id,
usage_type=payload.usage_type,
file_name=payload.file_name,
content_type=payload.content_type,
expected_size_bytes=len(content),
expected_hash=content_hash,
idempotency_key=payload.idempotency_key,
url_scope="internal",
),
session,
request,
)
if upload_result.get("status") == "completed":
return {"data": upload_result["storage_object"], "meta": {"reused": True}}
upload = await session.get(UploadSessions, upload_result["upload_id"])
if upload is None:
raise HTTPException(
status.HTTP_500_INTERNAL_SERVER_ERROR,
"upload record disappeared",
)
await asyncio.to_thread(
request.app.state.object_store.put_bytes,
bucket_name=upload.bucket_name,
object_key=upload.object_key,
content=content,
content_type=payload.content_type,
content_hash=content_hash,
)
item = await complete_upload_record(
upload.upload_id,
CompleteUploadRequest(
usage_type=payload.usage_type,
visibility=payload.visibility,
is_immutable=payload.is_immutable,
),
session,
request,
)
return {"data": storage_payload(item), "meta": {"reused": False}}
@app.post(
"/internal/v1/workspace-objects",
dependencies=[Depends(verify_internal_service)],
)
async def register_workspace_object(
payload: RegisterWorkspaceObjectRequest,
session: AsyncSession = Depends(database_session),
) -> dict[str, Any]:
workspace = await require_workspace_member(
session,
payload.workspace_id,
payload.user_id,
)
pure_path = PurePosixPath(payload.relative_path.replace("\\", "/"))
if (
pure_path.is_absolute()
or not pure_path.parts
or any(part in {"", ".", ".."} for part in pure_path.parts)
):
raise HTTPException(
status.HTTP_422_UNPROCESSABLE_ENTITY,
"invalid workspace relative_path",
)
relative_path = pure_path.as_posix()
workspace_root = Path(
os.getenv("WORKSPACE_ROOT", "/workspace/workspaces")
).resolve()
scoped_root = (workspace_root / workspace.workspace_code).resolve()
target = (scoped_root / Path(*pure_path.parts)).resolve()
if scoped_root != target and scoped_root not in target.parents:
raise HTTPException(
status.HTTP_422_UNPROCESSABLE_ENTITY,
"workspace path escapes its root",
)
if not target.is_file():
raise HTTPException(
status.HTTP_404_NOT_FOUND,
"workspace file does not exist",
)
content = await asyncio.to_thread(target.read_bytes)
content_hash = hashlib.sha256(content).hexdigest()
stat_result = target.stat()
path_digest = hash_bytes(relative_path)
item = await session.scalar(
select(StorageObjects).where(
StorageObjects.workspace_id == payload.workspace_id,
StorageObjects.storage_backend == "workspace_fs",
StorageObjects.path_hash == path_digest,
)
)
reused = item is not None
if item is None:
item = StorageObjects(
storage_object_id=new_ulid(),
workspace_id=payload.workspace_id,
owner_user_id=payload.user_id,
object_type="file",
usage_type=payload.usage_type,
storage_backend="workspace_fs",
relative_path=relative_path,
path_hash=path_digest,
storage_uri=target.as_uri(),
file_name=target.name,
visibility=payload.visibility,
is_immutable=0,
object_status="available",
created_by=payload.user_id,
)
session.add(item)
elif item.is_immutable:
raise HTTPException(
status.HTTP_409_CONFLICT,
"immutable workspace object cannot be updated",
)
elif item.owner_user_id != payload.user_id:
raise HTTPException(
status.HTTP_403_FORBIDDEN,
"workspace object belongs to another user",
)
item.usage_type = payload.usage_type
item.file_name = target.name
item.file_extension = target.suffix.lower() or None
item.mime_type = (
mimetypes.guess_type(target.name)[0] or "application/octet-stream"
)
item.size_bytes = stat_result.st_size
item.content_hash = content_hash
item.visibility = payload.visibility
item.object_status = "available"
await session.flush()
await session.refresh(item)
return {"data": storage_payload(item), "meta": {"reused": reused}}
@app.post(
"/internal/v1/objects/{storage_object_id}/download-url",
dependencies=[Depends(verify_internal_service)],
)
async def create_download_url(
storage_object_id: str,
payload: DownloadUrlRequest,
request: Request,
session: AsyncSession = Depends(database_session),
) -> dict[str, Any]:
item = await session.get(StorageObjects, storage_object_id)
if item is None or item.object_status != "available":
raise HTTPException(status.HTTP_404_NOT_FOUND, "object not found")
if (
item.storage_backend != "rustfs"
or not item.bucket_name
or not item.object_key
):
raise HTTPException(
status.HTTP_409_CONFLICT,
"object does not support a presigned URL",
)
url = request.app.state.object_store.presign_get(
bucket_name=item.bucket_name,
object_key=item.object_key,
file_name=item.file_name,
expires_seconds=payload.expires_seconds,
)
return {
"data": {
"storage_object_id": item.storage_object_id,
"presigned_url": url,
"method": "GET",
"expires_in_seconds": payload.expires_seconds,
}
}
@app.delete(
"/internal/v1/objects/{storage_object_id}",
dependencies=[Depends(verify_internal_service)],
)
async def delete_object(
storage_object_id: str,
request: Request,
session: AsyncSession = Depends(database_session),
) -> dict[str, Any]:
item = await session.scalar(
select(StorageObjects)
.where(StorageObjects.storage_object_id == storage_object_id)
.with_for_update()
)
if item is None:
raise HTTPException(status.HTTP_404_NOT_FOUND, "object not found")
if item.is_immutable:
raise HTTPException(
status.HTTP_409_CONFLICT,
"immutable object cannot be deleted",
)
if item.object_status != "deleted":
if item.storage_backend == "rustfs" and item.bucket_name and item.object_key:
await asyncio.to_thread(
request.app.state.object_store.delete,
bucket_name=item.bucket_name,
object_key=item.object_key,
)
item.object_status = "deleted"
item.deleted_at = utcnow()
return {
"data": {
"storage_object_id": storage_object_id,
"object_status": item.object_status,
}
}
@app.get("/internal/health/storage")
async def internal_health() -> dict[str, str]:
return {"status": "ready", "service": "storage-api"}
+116
View File
@@ -0,0 +1,116 @@
from __future__ import annotations
import base64
from typing import Any
import httpx
from fastapi import HTTPException
class StorageClient:
def __init__(
self,
client: httpx.AsyncClient,
service_token: str,
) -> None:
self.client = client
self.headers = {"X-Service-Token": service_token}
async def _request(
self,
method: str,
path: str,
*,
payload: dict[str, Any] | None = None,
) -> dict[str, Any]:
response = await self.client.request(
method,
path,
json=payload,
headers=self.headers,
)
if response.is_error:
try:
detail = response.json().get("detail", response.text)
except ValueError:
detail = response.text
raise HTTPException(response.status_code, detail)
return response.json()
async def create_upload(
self,
payload: dict[str, Any],
) -> dict[str, Any]:
return (await self._request(
"POST",
"/internal/v1/uploads",
payload=payload,
))["data"]
async def complete_upload(
self,
upload_id: str,
payload: dict[str, Any],
) -> dict[str, Any]:
return (await self._request(
"POST",
f"/internal/v1/uploads/{upload_id}/complete",
payload=payload,
))["data"]
async def register_workspace_object(
self,
payload: dict[str, Any],
) -> dict[str, Any]:
return (await self._request(
"POST",
"/internal/v1/workspace-objects",
payload=payload,
))["data"]
async def create_server_object(
self,
*,
workspace_id: str,
user_id: str,
usage_type: str,
file_name: str,
content_type: str,
content: bytes,
visibility: str,
is_immutable: bool,
idempotency_key: str,
) -> dict[str, Any]:
result = await self._request(
"POST",
"/internal/v1/objects",
payload={
"workspace_id": workspace_id,
"user_id": user_id,
"usage_type": usage_type,
"file_name": file_name,
"content_type": content_type,
"content_base64": base64.b64encode(content).decode("ascii"),
"visibility": visibility,
"is_immutable": is_immutable,
"idempotency_key": idempotency_key,
},
)
return result["data"]
async def create_download_url(
self,
storage_object_id: str,
expires_seconds: int,
) -> dict[str, Any]:
return (await self._request(
"POST",
f"/internal/v1/objects/{storage_object_id}/download-url",
payload={"expires_seconds": expires_seconds},
))["data"]
async def delete_object(self, storage_object_id: str) -> dict[str, Any]:
return (await self._request(
"DELETE",
f"/internal/v1/objects/{storage_object_id}",
))["data"]
+79
View File
@@ -0,0 +1,79 @@
from __future__ import annotations
from typing import Literal
from pydantic import BaseModel, ConfigDict, Field, field_validator
class StrictModel(BaseModel):
model_config = ConfigDict(extra="forbid")
class CreateUploadRequest(StrictModel):
workspace_id: str = Field(min_length=26, max_length=26)
user_id: str = Field(min_length=26, max_length=26)
usage_type: Literal[
"data_resource",
"version_artifact",
"snapshot",
"run_log",
"run_result",
]
file_name: str = Field(min_length=1, max_length=255)
content_type: str = Field(min_length=1, max_length=255)
expected_size_bytes: int = Field(ge=0, le=100 * 1024 * 1024)
expected_hash: str | None = Field(default=None, min_length=64, max_length=64)
idempotency_key: str = Field(min_length=8, max_length=128)
url_scope: Literal["public", "internal"] = "public"
@field_validator("expected_hash")
@classmethod
def validate_hash(cls, value: str | None) -> str | None:
if value is None:
return None
normalized = value.lower()
if any(character not in "0123456789abcdef" for character in normalized):
raise ValueError("expected_hash must be lowercase SHA-256 hex")
return normalized
class CompleteUploadRequest(StrictModel):
usage_type: Literal[
"data_resource",
"version_artifact",
"snapshot",
"run_log",
"run_result",
]
visibility: Literal["private", "workspace", "public"] = "private"
is_immutable: bool = False
class ServerObjectRequest(StrictModel):
workspace_id: str = Field(min_length=26, max_length=26)
user_id: str = Field(min_length=26, max_length=26)
usage_type: Literal[
"data_resource",
"version_artifact",
"snapshot",
"run_log",
"run_result",
]
file_name: str = Field(min_length=1, max_length=255)
content_type: str = Field(min_length=1, max_length=255)
content_base64: str = Field(min_length=1)
visibility: Literal["private", "workspace", "public"] = "private"
is_immutable: bool = False
idempotency_key: str = Field(min_length=8, max_length=128)
class RegisterWorkspaceObjectRequest(StrictModel):
workspace_id: str = Field(min_length=26, max_length=26)
user_id: str = Field(min_length=26, max_length=26)
relative_path: str = Field(min_length=1, max_length=1024)
usage_type: Literal["working_copy", "public_script"]
visibility: Literal["private", "workspace", "public"] = "private"
class DownloadUrlRequest(StrictModel):
expires_seconds: int = Field(default=300, ge=30, le=3600)