重构模型平台前后端并移除Redis依赖
This commit is contained in:
@@ -0,0 +1,10 @@
|
||||
# Python-generated files
|
||||
__pycache__/
|
||||
*.py[oc]
|
||||
build/
|
||||
dist/
|
||||
wheels/
|
||||
*.egg-info
|
||||
|
||||
# Virtual environments
|
||||
.venv
|
||||
@@ -0,0 +1 @@
|
||||
3.12
|
||||
@@ -0,0 +1,13 @@
|
||||
FROM python:3.12-slim-bookworm
|
||||
|
||||
ENV PYTHONDONTWRITEBYTECODE=1 PYTHONUNBUFFERED=1 PYTHONPATH=/app
|
||||
WORKDIR /app
|
||||
COPY --from=ghcr.io/astral-sh/uv:latest /uv /bin/uv
|
||||
COPY common ./common
|
||||
COPY backend ./backend
|
||||
COPY alembic.ini ./
|
||||
COPY migrations ./migrations
|
||||
RUN uv pip install --system ./common ./backend
|
||||
|
||||
EXPOSE 8000
|
||||
CMD ["uvicorn", "backend.main:app", "--host", "0.0.0.0", "--port", "8000"]
|
||||
@@ -0,0 +1,5 @@
|
||||
# Backend
|
||||
|
||||
统一 FastAPI 管理服务。包含用户、Workspace、脚本、稳定版本、调度定义、
|
||||
立即运行以及 RustFS 对象接口。原 `platform_api` 与 `storage_api` 已在此
|
||||
模块合并,外部 REST 契约保持不变。
|
||||
@@ -0,0 +1,23 @@
|
||||
[project]
|
||||
name = "backend"
|
||||
version = "0.2.0"
|
||||
requires-python = ">=3.12"
|
||||
dependencies = [
|
||||
"common",
|
||||
"fastapi==0.116.1",
|
||||
"uvicorn[standard]==0.35.0",
|
||||
"httpx==0.28.1",
|
||||
"croniter==6.2.4",
|
||||
"alembic==1.18.5",
|
||||
"cryptography==49.0.0",
|
||||
]
|
||||
|
||||
[tool.uv.sources]
|
||||
common = { path = "../common" }
|
||||
|
||||
[build-system]
|
||||
requires = ["hatchling"]
|
||||
build-backend = "hatchling.build"
|
||||
|
||||
[tool.hatch.build.targets.wheel]
|
||||
packages = ["src/backend"]
|
||||
@@ -0,0 +1 @@
|
||||
"""Platform API application."""
|
||||
@@ -0,0 +1,218 @@
|
||||
from __future__ import annotations
|
||||
|
||||
from typing import Any, Literal
|
||||
|
||||
from fastapi import APIRouter, Depends, HTTPException, status
|
||||
from pydantic import BaseModel, ConfigDict, Field
|
||||
from sqlalchemy import delete, func, or_, select
|
||||
from sqlalchemy.ext.asyncio import AsyncSession
|
||||
|
||||
from common.db.models import Roles, Users, WorkspaceMembers
|
||||
from common.ids import new_ulid
|
||||
from backend.dependencies import (
|
||||
RequestContext,
|
||||
database_session,
|
||||
request_context,
|
||||
)
|
||||
|
||||
|
||||
router = APIRouter(prefix="/api/v1/admin", tags=["admin"])
|
||||
|
||||
|
||||
class EmployeeCreate(BaseModel):
|
||||
model_config = ConfigDict(extra="forbid")
|
||||
|
||||
username: str = Field(min_length=2, max_length=64)
|
||||
display_name: str = Field(min_length=1, max_length=100)
|
||||
email: str | None = Field(default=None, max_length=255)
|
||||
role_code: Literal["admin", "developer"] = "developer"
|
||||
|
||||
|
||||
class EmployeeUpdate(BaseModel):
|
||||
model_config = ConfigDict(extra="forbid")
|
||||
|
||||
display_name: str | None = Field(default=None, min_length=1, max_length=100)
|
||||
email: str | None = Field(default=None, max_length=255)
|
||||
role_code: Literal["admin", "developer"] | None = None
|
||||
status: Literal["active", "disabled", "locked"] | None = None
|
||||
|
||||
|
||||
def require_admin(context: RequestContext) -> None:
|
||||
if not context.is_admin:
|
||||
raise HTTPException(status.HTTP_403_FORBIDDEN, "仅管理员可以管理员工")
|
||||
|
||||
|
||||
def employee_payload(user: Users, role: Roles) -> dict[str, Any]:
|
||||
return {
|
||||
"user_id": user.user_id,
|
||||
"username": user.username,
|
||||
"display_name": user.display_name,
|
||||
"email": user.email,
|
||||
"status": user.status,
|
||||
"role_code": role.role_code,
|
||||
"role_name": role.role_name,
|
||||
"created_at": user.created_at.isoformat(),
|
||||
}
|
||||
|
||||
|
||||
async def member_row(
|
||||
user_id: str,
|
||||
context: RequestContext,
|
||||
session: AsyncSession,
|
||||
) -> tuple[Users, WorkspaceMembers, Roles]:
|
||||
row = (
|
||||
await session.execute(
|
||||
select(Users, WorkspaceMembers, Roles)
|
||||
.join(
|
||||
WorkspaceMembers,
|
||||
WorkspaceMembers.user_id == Users.user_id,
|
||||
)
|
||||
.join(Roles, Roles.role_id == WorkspaceMembers.role_id)
|
||||
.where(
|
||||
WorkspaceMembers.workspace_id == context.workspace.workspace_id,
|
||||
Users.user_id == user_id,
|
||||
)
|
||||
)
|
||||
).first()
|
||||
if row is None:
|
||||
raise HTTPException(status.HTTP_404_NOT_FOUND, "员工不存在")
|
||||
return row
|
||||
|
||||
|
||||
@router.get("/employees")
|
||||
async def list_employees(
|
||||
context: RequestContext = Depends(request_context),
|
||||
session: AsyncSession = Depends(database_session),
|
||||
) -> dict[str, Any]:
|
||||
rows = (
|
||||
await session.execute(
|
||||
select(Users, Roles)
|
||||
.join(
|
||||
WorkspaceMembers,
|
||||
WorkspaceMembers.user_id == Users.user_id,
|
||||
)
|
||||
.join(Roles, Roles.role_id == WorkspaceMembers.role_id)
|
||||
.where(
|
||||
WorkspaceMembers.workspace_id == context.workspace.workspace_id,
|
||||
WorkspaceMembers.member_status == "active",
|
||||
)
|
||||
.order_by(Users.created_at, Users.user_id)
|
||||
)
|
||||
).all()
|
||||
return {
|
||||
"request_id": context.request_id,
|
||||
"data": [employee_payload(user, role) for user, role in rows],
|
||||
"meta": {"count": len(rows), "can_manage": context.is_admin},
|
||||
}
|
||||
|
||||
|
||||
@router.post("/employees", status_code=status.HTTP_201_CREATED)
|
||||
async def create_employee(
|
||||
payload: EmployeeCreate,
|
||||
context: RequestContext = Depends(request_context),
|
||||
session: AsyncSession = Depends(database_session),
|
||||
) -> dict[str, Any]:
|
||||
require_admin(context)
|
||||
username = payload.username.strip()
|
||||
display_name = payload.display_name.strip()
|
||||
duplicate_conditions = [Users.username == username]
|
||||
if payload.email:
|
||||
duplicate_conditions.append(Users.email == payload.email.strip())
|
||||
duplicate = await session.scalar(
|
||||
select(Users.user_id).where(or_(*duplicate_conditions))
|
||||
)
|
||||
if duplicate is not None:
|
||||
raise HTTPException(status.HTTP_409_CONFLICT, "用户名或邮箱已存在")
|
||||
role = await session.scalar(
|
||||
select(Roles).where(Roles.role_code == payload.role_code)
|
||||
)
|
||||
if role is None:
|
||||
raise HTTPException(status.HTTP_422_UNPROCESSABLE_ENTITY, "角色不存在")
|
||||
user = Users(
|
||||
user_id=new_ulid(),
|
||||
username=username,
|
||||
display_name=display_name,
|
||||
email=payload.email.strip() if payload.email else None,
|
||||
password_hash="demo-login-disabled",
|
||||
status="active",
|
||||
)
|
||||
session.add(user)
|
||||
session.add(
|
||||
WorkspaceMembers(
|
||||
workspace_id=context.workspace.workspace_id,
|
||||
user_id=user.user_id,
|
||||
role_id=role.role_id,
|
||||
member_status="active",
|
||||
)
|
||||
)
|
||||
await session.flush()
|
||||
await session.refresh(user)
|
||||
return {
|
||||
"request_id": context.request_id,
|
||||
"data": employee_payload(user, role),
|
||||
"meta": {},
|
||||
}
|
||||
|
||||
|
||||
@router.patch("/employees/{user_id}")
|
||||
async def update_employee(
|
||||
user_id: str,
|
||||
payload: EmployeeUpdate,
|
||||
context: RequestContext = Depends(request_context),
|
||||
session: AsyncSession = Depends(database_session),
|
||||
) -> dict[str, Any]:
|
||||
require_admin(context)
|
||||
user, membership, role = await member_row(user_id, context, session)
|
||||
if payload.display_name is not None:
|
||||
user.display_name = payload.display_name.strip()
|
||||
if payload.email is not None:
|
||||
user.email = payload.email.strip() or None
|
||||
if payload.status is not None:
|
||||
user.status = payload.status
|
||||
if payload.role_code is not None and payload.role_code != role.role_code:
|
||||
next_role = await session.scalar(
|
||||
select(Roles).where(Roles.role_code == payload.role_code)
|
||||
)
|
||||
if next_role is None:
|
||||
raise HTTPException(status.HTTP_422_UNPROCESSABLE_ENTITY, "角色不存在")
|
||||
membership.role_id = next_role.role_id
|
||||
role = next_role
|
||||
await session.flush()
|
||||
await session.refresh(user)
|
||||
return {
|
||||
"request_id": context.request_id,
|
||||
"data": employee_payload(user, role),
|
||||
"meta": {},
|
||||
}
|
||||
|
||||
|
||||
@router.delete("/employees/{user_id}")
|
||||
async def delete_employee(
|
||||
user_id: str,
|
||||
context: RequestContext = Depends(request_context),
|
||||
session: AsyncSession = Depends(database_session),
|
||||
) -> dict[str, Any]:
|
||||
require_admin(context)
|
||||
if user_id == context.user.user_id:
|
||||
raise HTTPException(status.HTTP_409_CONFLICT, "不能删除当前登录员工")
|
||||
user, _, role = await member_row(user_id, context, session)
|
||||
if role.role_code == "admin":
|
||||
raise HTTPException(status.HTTP_409_CONFLICT, "管理员账号不能删除")
|
||||
await session.execute(
|
||||
delete(WorkspaceMembers).where(
|
||||
WorkspaceMembers.workspace_id == context.workspace.workspace_id,
|
||||
WorkspaceMembers.user_id == user_id,
|
||||
)
|
||||
)
|
||||
memberships = await session.scalar(
|
||||
select(func.count())
|
||||
.select_from(WorkspaceMembers)
|
||||
.where(WorkspaceMembers.user_id == user_id)
|
||||
)
|
||||
if memberships == 0:
|
||||
user.status = "disabled"
|
||||
return {
|
||||
"request_id": context.request_id,
|
||||
"data": {"user_id": user_id, "deleted": True},
|
||||
"meta": {},
|
||||
}
|
||||
@@ -0,0 +1,79 @@
|
||||
from __future__ import annotations
|
||||
|
||||
from dataclasses import dataclass
|
||||
from typing import AsyncIterator
|
||||
|
||||
from fastapi import Header, HTTPException, Request, status
|
||||
from sqlalchemy import select
|
||||
from sqlalchemy.ext.asyncio import AsyncSession
|
||||
|
||||
from common.db.models import (
|
||||
Roles,
|
||||
Users,
|
||||
WorkspaceMembers,
|
||||
Workspaces,
|
||||
)
|
||||
from common.ids import new_ulid
|
||||
|
||||
|
||||
@dataclass(frozen=True)
|
||||
class RequestContext:
|
||||
request_id: str
|
||||
user: Users
|
||||
workspace: Workspaces
|
||||
role: Roles
|
||||
|
||||
@property
|
||||
def is_admin(self) -> bool:
|
||||
return self.role.role_code == "admin"
|
||||
|
||||
|
||||
async def database_session(request: Request) -> AsyncIterator[AsyncSession]:
|
||||
async with request.app.state.session_factory() as session:
|
||||
try:
|
||||
yield session
|
||||
await session.commit()
|
||||
except Exception:
|
||||
await session.rollback()
|
||||
raise
|
||||
|
||||
|
||||
async def request_context(
|
||||
request: Request,
|
||||
x_user_id: str = Header(alias="X-User-ID"),
|
||||
x_workspace_id: str = Header(alias="X-Workspace-ID"),
|
||||
x_request_id: str | None = Header(default=None, alias="X-Request-ID"),
|
||||
) -> RequestContext:
|
||||
async with request.app.state.session_factory() as session:
|
||||
statement = (
|
||||
select(Users, Workspaces, Roles)
|
||||
.join(
|
||||
WorkspaceMembers,
|
||||
WorkspaceMembers.user_id == Users.user_id,
|
||||
)
|
||||
.join(
|
||||
Workspaces,
|
||||
Workspaces.workspace_id == WorkspaceMembers.workspace_id,
|
||||
)
|
||||
.join(Roles, Roles.role_id == WorkspaceMembers.role_id)
|
||||
.where(
|
||||
Users.user_id == x_user_id,
|
||||
Users.status == "active",
|
||||
WorkspaceMembers.workspace_id == x_workspace_id,
|
||||
WorkspaceMembers.member_status == "active",
|
||||
Workspaces.status == "active",
|
||||
)
|
||||
)
|
||||
row = (await session.execute(statement)).one_or_none()
|
||||
if row is None:
|
||||
raise HTTPException(
|
||||
status.HTTP_403_FORBIDDEN,
|
||||
"active workspace membership is required",
|
||||
)
|
||||
user, workspace, role = row
|
||||
return RequestContext(
|
||||
request_id=x_request_id or new_ulid(),
|
||||
user=user,
|
||||
workspace=workspace,
|
||||
role=role,
|
||||
)
|
||||
@@ -0,0 +1,110 @@
|
||||
from __future__ import annotations
|
||||
|
||||
from typing import Any, Awaitable, Callable
|
||||
|
||||
from fastapi import APIRouter, Depends, Request, status
|
||||
from fastapi.responses import JSONResponse
|
||||
|
||||
from backend.dependencies import (
|
||||
RequestContext,
|
||||
request_context,
|
||||
)
|
||||
from backend.runtime_client import RuntimeClientError
|
||||
from backend.schemas import FileLockTokenRequest
|
||||
|
||||
|
||||
router = APIRouter(tags=["file-locks"])
|
||||
|
||||
|
||||
async def runtime_response(
|
||||
context: RequestContext,
|
||||
operation: Callable[[], Awaitable[dict[str, Any]]],
|
||||
*,
|
||||
success_status: int = status.HTTP_200_OK,
|
||||
) -> JSONResponse:
|
||||
try:
|
||||
data = await operation()
|
||||
except RuntimeClientError as exc:
|
||||
error = exc.detail
|
||||
if not isinstance(error, dict) or "code" not in error:
|
||||
error = {
|
||||
"code": "RUNTIME_REQUEST_FAILED",
|
||||
"message": str(error),
|
||||
"retryable": exc.status_code >= 500,
|
||||
"details": {},
|
||||
}
|
||||
return JSONResponse(
|
||||
status_code=exc.status_code,
|
||||
content={"request_id": context.request_id, "error": error},
|
||||
)
|
||||
return JSONResponse(
|
||||
status_code=success_status,
|
||||
content={
|
||||
"request_id": context.request_id,
|
||||
"data": data,
|
||||
"meta": {},
|
||||
},
|
||||
)
|
||||
|
||||
|
||||
@router.post(
|
||||
"/api/v1/files/{storage_object_id}/lock",
|
||||
status_code=status.HTTP_201_CREATED,
|
||||
)
|
||||
async def acquire_file_lock(
|
||||
storage_object_id: str,
|
||||
request: Request,
|
||||
context: RequestContext = Depends(request_context),
|
||||
) -> JSONResponse:
|
||||
return await runtime_response(
|
||||
context,
|
||||
lambda: request.app.state.runtime_client.acquire_file_lock(
|
||||
{
|
||||
"workspace_id": context.workspace.workspace_id,
|
||||
"storage_object_id": storage_object_id,
|
||||
"user_id": context.user.user_id,
|
||||
"request_id": context.request_id,
|
||||
}
|
||||
),
|
||||
success_status=status.HTTP_201_CREATED,
|
||||
)
|
||||
|
||||
|
||||
@router.post("/api/v1/file-locks/{edit_session_id}/heartbeat")
|
||||
async def heartbeat_file_lock(
|
||||
edit_session_id: str,
|
||||
payload: FileLockTokenRequest,
|
||||
request: Request,
|
||||
context: RequestContext = Depends(request_context),
|
||||
) -> JSONResponse:
|
||||
return await runtime_response(
|
||||
context,
|
||||
lambda: request.app.state.runtime_client.heartbeat_file_lock(
|
||||
edit_session_id,
|
||||
{
|
||||
"workspace_id": context.workspace.workspace_id,
|
||||
"user_id": context.user.user_id,
|
||||
"lock_token": payload.lock_token,
|
||||
},
|
||||
),
|
||||
)
|
||||
|
||||
|
||||
@router.delete("/api/v1/file-locks/{edit_session_id}")
|
||||
async def release_file_lock(
|
||||
edit_session_id: str,
|
||||
payload: FileLockTokenRequest,
|
||||
request: Request,
|
||||
context: RequestContext = Depends(request_context),
|
||||
) -> JSONResponse:
|
||||
return await runtime_response(
|
||||
context,
|
||||
lambda: request.app.state.runtime_client.release_file_lock(
|
||||
edit_session_id,
|
||||
{
|
||||
"workspace_id": context.workspace.workspace_id,
|
||||
"user_id": context.user.user_id,
|
||||
"lock_token": payload.lock_token,
|
||||
},
|
||||
),
|
||||
)
|
||||
@@ -0,0 +1,84 @@
|
||||
from __future__ import annotations
|
||||
|
||||
import os
|
||||
|
||||
from fastapi import APIRouter, Depends, Request, status
|
||||
from fastapi.responses import JSONResponse
|
||||
|
||||
from backend.dependencies import (
|
||||
RequestContext,
|
||||
request_context,
|
||||
)
|
||||
from backend.runtime_client import RuntimeClientError
|
||||
from backend.schemas import (
|
||||
CreateJupyterAccessTicketRequest,
|
||||
)
|
||||
|
||||
|
||||
router = APIRouter(tags=["jupyter"])
|
||||
|
||||
|
||||
def cookie_secure() -> bool:
|
||||
return os.getenv("COOKIE_SECURE", "false").strip().lower() in {
|
||||
"1",
|
||||
"true",
|
||||
"yes",
|
||||
"on",
|
||||
}
|
||||
|
||||
|
||||
@router.post(
|
||||
"/api/v1/jupyter/access-tickets",
|
||||
status_code=status.HTTP_201_CREATED,
|
||||
)
|
||||
async def create_jupyter_access_ticket(
|
||||
payload: CreateJupyterAccessTicketRequest,
|
||||
request: Request,
|
||||
context: RequestContext = Depends(request_context),
|
||||
) -> JSONResponse:
|
||||
try:
|
||||
ticket_data = (
|
||||
await request.app.state.runtime_client
|
||||
.create_jupyter_access_ticket(
|
||||
payload.edit_session_id,
|
||||
{
|
||||
"workspace_id": context.workspace.workspace_id,
|
||||
"user_id": context.user.user_id,
|
||||
"lock_token": payload.lock_token,
|
||||
},
|
||||
)
|
||||
)
|
||||
except RuntimeClientError as exc:
|
||||
error = exc.detail
|
||||
if not isinstance(error, dict) or "code" not in error:
|
||||
error = {
|
||||
"code": "JUPYTER_TICKET_FAILED",
|
||||
"message": str(error),
|
||||
"retryable": exc.status_code >= 500,
|
||||
"details": {},
|
||||
}
|
||||
return JSONResponse(
|
||||
status_code=exc.status_code,
|
||||
content={"request_id": context.request_id, "error": error},
|
||||
)
|
||||
|
||||
raw_ticket = ticket_data.pop("ticket")
|
||||
max_age = int(ticket_data.pop("expires_in_seconds"))
|
||||
response = JSONResponse(
|
||||
status_code=status.HTTP_201_CREATED,
|
||||
content={
|
||||
"request_id": context.request_id,
|
||||
"data": ticket_data,
|
||||
"meta": {},
|
||||
},
|
||||
)
|
||||
response.set_cookie(
|
||||
key="jupyter_access",
|
||||
value=raw_ticket,
|
||||
max_age=max_age,
|
||||
path="/jupyter/",
|
||||
secure=cookie_secure(),
|
||||
httponly=True,
|
||||
samesite="lax",
|
||||
)
|
||||
return response
|
||||
@@ -0,0 +1,108 @@
|
||||
from __future__ import annotations
|
||||
|
||||
import asyncio
|
||||
import os
|
||||
from contextlib import asynccontextmanager
|
||||
from pathlib import Path
|
||||
from typing import Any, AsyncIterator
|
||||
|
||||
import httpx
|
||||
from fastapi.routing import APIRoute
|
||||
|
||||
from common.db import create_database_engine, create_session_factory
|
||||
from common.service_app import create_service_app
|
||||
from common.storage import RustFSObjectStore
|
||||
from backend.admin import router as admin_router
|
||||
from backend.file_locks import router as file_locks_router
|
||||
from backend.jupyter import router as jupyter_router
|
||||
from backend.resources import router as resources_router
|
||||
from backend.runtime_client import RuntimeClient
|
||||
from backend.schedule_runs import router as schedule_runs_router
|
||||
from backend.schedule_client import ScheduleExecutorClient
|
||||
from backend.schedules import router as schedules_router
|
||||
from backend.scripts import router as scripts_router
|
||||
from backend.storage_api import app as storage_app
|
||||
from backend.storage_client import StorageClient
|
||||
|
||||
|
||||
@asynccontextmanager
|
||||
async def lifespan(app: Any) -> AsyncIterator[None]:
|
||||
engine = create_database_engine(os.environ["DATABASE_URL"])
|
||||
app.state.session_factory = create_session_factory(engine)
|
||||
workspace_root = Path(
|
||||
os.getenv("WORKSPACE_ROOT", "/workspace/workspaces")
|
||||
)
|
||||
workspace_root.mkdir(parents=True, exist_ok=True)
|
||||
|
||||
# Storage API is now part of the backend process. Platform routers keep
|
||||
# their existing client contract, but calls are dispatched in-process.
|
||||
app.state.object_store = RustFSObjectStore(
|
||||
internal_endpoint=os.getenv(
|
||||
"RUSTFS_INTERNAL_ENDPOINT",
|
||||
"http://rustfs:9000",
|
||||
),
|
||||
public_endpoint=os.getenv(
|
||||
"RUSTFS_PUBLIC_ENDPOINT",
|
||||
"http://localhost:9000",
|
||||
),
|
||||
access_key=os.environ["RUSTFS_ACCESS_KEY"],
|
||||
secret_key=os.environ["RUSTFS_SECRET_KEY"],
|
||||
)
|
||||
app.state.default_bucket = os.getenv(
|
||||
"RUSTFS_DEFAULT_BUCKET",
|
||||
"model-platform",
|
||||
)
|
||||
await asyncio.to_thread(
|
||||
app.state.object_store.ensure_bucket,
|
||||
app.state.default_bucket,
|
||||
)
|
||||
storage_http_client = httpx.AsyncClient(
|
||||
transport=httpx.ASGITransport(app=app),
|
||||
base_url="http://backend.internal",
|
||||
timeout=httpx.Timeout(30.0),
|
||||
)
|
||||
app.state.storage_client = StorageClient(
|
||||
storage_http_client,
|
||||
os.environ["INTERNAL_SERVICE_TOKEN"],
|
||||
)
|
||||
runtime_http_client = httpx.AsyncClient(
|
||||
base_url=os.getenv("RUNTIME_API_URL", "http://runtime:8000"),
|
||||
timeout=httpx.Timeout(30.0),
|
||||
)
|
||||
app.state.runtime_client = RuntimeClient(
|
||||
runtime_http_client,
|
||||
os.environ["INTERNAL_SERVICE_TOKEN"],
|
||||
)
|
||||
schedule_http_client = httpx.AsyncClient(
|
||||
base_url=os.getenv("SCHEDULE_API_URL", "http://schedule:8000"),
|
||||
timeout=httpx.Timeout(10.0),
|
||||
)
|
||||
app.state.schedule_client = ScheduleExecutorClient(
|
||||
schedule_http_client,
|
||||
os.environ["INTERNAL_SERVICE_TOKEN"],
|
||||
)
|
||||
try:
|
||||
yield
|
||||
finally:
|
||||
await runtime_http_client.aclose()
|
||||
await schedule_http_client.aclose()
|
||||
await storage_http_client.aclose()
|
||||
await engine.dispose()
|
||||
|
||||
|
||||
app = create_service_app(
|
||||
os.getenv("SERVICE_NAME", "backend"),
|
||||
lifespan=lifespan,
|
||||
)
|
||||
app.include_router(file_locks_router)
|
||||
app.include_router(jupyter_router)
|
||||
app.include_router(resources_router)
|
||||
app.include_router(schedule_runs_router)
|
||||
app.include_router(schedules_router)
|
||||
app.include_router(scripts_router)
|
||||
app.include_router(admin_router)
|
||||
|
||||
# Reuse the proven storage endpoints without running another FastAPI service.
|
||||
for route in storage_app.routes:
|
||||
if isinstance(route, APIRoute) and route.path.startswith("/internal/"):
|
||||
app.router.routes.append(route)
|
||||
@@ -0,0 +1,293 @@
|
||||
from __future__ import annotations
|
||||
|
||||
from datetime import UTC, datetime
|
||||
from typing import Any
|
||||
|
||||
from fastapi import APIRouter, Depends, Header, HTTPException, Query, Request, status
|
||||
from sqlalchemy import or_, select
|
||||
from sqlalchemy.ext.asyncio import AsyncSession
|
||||
|
||||
from common.db.models import DataResources, StorageObjects
|
||||
from common.ids import new_ulid
|
||||
from backend.dependencies import (
|
||||
RequestContext,
|
||||
database_session,
|
||||
request_context,
|
||||
)
|
||||
from backend.schemas import (
|
||||
CompleteResourceUploadRequest,
|
||||
CreateResourceUploadRequest,
|
||||
DownloadUrlRequest,
|
||||
)
|
||||
|
||||
router = APIRouter(prefix="/api/v1/data-resources", tags=["data-resources"])
|
||||
|
||||
|
||||
def resource_payload(
|
||||
resource: DataResources,
|
||||
storage_object: StorageObjects,
|
||||
) -> dict[str, Any]:
|
||||
return {
|
||||
"resource_id": resource.resource_id,
|
||||
"workspace_id": resource.workspace_id,
|
||||
"storage_object_id": resource.storage_object_id,
|
||||
"owner_user_id": resource.owner_user_id,
|
||||
"resource_name": resource.resource_name,
|
||||
"description": resource.description,
|
||||
"visibility": resource.visibility,
|
||||
"status": resource.status,
|
||||
"created_at": resource.created_at.isoformat(),
|
||||
"updated_at": resource.updated_at.isoformat(),
|
||||
"file": {
|
||||
"file_name": storage_object.file_name,
|
||||
"file_extension": storage_object.file_extension,
|
||||
"mime_type": storage_object.mime_type,
|
||||
"size_bytes": storage_object.size_bytes,
|
||||
"content_hash": storage_object.content_hash,
|
||||
"object_status": storage_object.object_status,
|
||||
},
|
||||
}
|
||||
|
||||
|
||||
def can_view(resource: DataResources, context: RequestContext) -> bool:
|
||||
return (
|
||||
resource.owner_user_id == context.user.user_id
|
||||
or resource.visibility in {"workspace", "public"}
|
||||
or context.is_admin
|
||||
)
|
||||
|
||||
|
||||
@router.post("/uploads", status_code=status.HTTP_201_CREATED)
|
||||
async def create_resource_upload(
|
||||
payload: CreateResourceUploadRequest,
|
||||
request: Request,
|
||||
context: RequestContext = Depends(request_context),
|
||||
idempotency_key: str = Header(
|
||||
min_length=8,
|
||||
max_length=128,
|
||||
alias="Idempotency-Key",
|
||||
),
|
||||
) -> dict[str, Any]:
|
||||
data = await request.app.state.storage_client.create_upload(
|
||||
{
|
||||
"workspace_id": context.workspace.workspace_id,
|
||||
"user_id": context.user.user_id,
|
||||
"usage_type": "data_resource",
|
||||
"file_name": payload.file_name,
|
||||
"content_type": payload.content_type,
|
||||
"expected_size_bytes": payload.expected_size_bytes,
|
||||
"expected_hash": payload.expected_hash,
|
||||
"idempotency_key": idempotency_key,
|
||||
"url_scope": "public",
|
||||
}
|
||||
)
|
||||
return {"request_id": context.request_id, "data": data, "meta": {}}
|
||||
|
||||
|
||||
@router.post("/uploads/{upload_id}/complete")
|
||||
async def complete_resource_upload(
|
||||
upload_id: str,
|
||||
payload: CompleteResourceUploadRequest,
|
||||
request: Request,
|
||||
context: RequestContext = Depends(request_context),
|
||||
session: AsyncSession = Depends(database_session),
|
||||
) -> dict[str, Any]:
|
||||
storage_data = await request.app.state.storage_client.complete_upload(
|
||||
upload_id,
|
||||
{
|
||||
"usage_type": "data_resource",
|
||||
"visibility": payload.visibility,
|
||||
"is_immutable": False,
|
||||
},
|
||||
)
|
||||
if storage_data["workspace_id"] != context.workspace.workspace_id:
|
||||
raise HTTPException(
|
||||
status.HTTP_403_FORBIDDEN,
|
||||
"upload belongs to another workspace",
|
||||
)
|
||||
if storage_data["owner_user_id"] != context.user.user_id:
|
||||
raise HTTPException(
|
||||
status.HTTP_403_FORBIDDEN,
|
||||
"upload belongs to another user",
|
||||
)
|
||||
existing = await session.scalar(
|
||||
select(DataResources).where(
|
||||
DataResources.storage_object_id
|
||||
== storage_data["storage_object_id"]
|
||||
)
|
||||
)
|
||||
reused = existing is not None
|
||||
if existing is None:
|
||||
existing = DataResources(
|
||||
resource_id=new_ulid(),
|
||||
workspace_id=context.workspace.workspace_id,
|
||||
storage_object_id=storage_data["storage_object_id"],
|
||||
owner_user_id=context.user.user_id,
|
||||
resource_name=payload.resource_name,
|
||||
description=payload.description,
|
||||
visibility=payload.visibility,
|
||||
status="active",
|
||||
)
|
||||
session.add(existing)
|
||||
await session.flush()
|
||||
await session.refresh(existing)
|
||||
storage_object = await session.get(
|
||||
StorageObjects,
|
||||
existing.storage_object_id,
|
||||
)
|
||||
if storage_object is None:
|
||||
raise HTTPException(
|
||||
status.HTTP_409_CONFLICT,
|
||||
"storage object metadata is missing",
|
||||
)
|
||||
return {
|
||||
"request_id": context.request_id,
|
||||
"data": resource_payload(existing, storage_object),
|
||||
"meta": {"reused": reused},
|
||||
}
|
||||
|
||||
|
||||
@router.get("")
|
||||
async def list_resources(
|
||||
context: RequestContext = Depends(request_context),
|
||||
session: AsyncSession = Depends(database_session),
|
||||
visibility: str | None = Query(default=None),
|
||||
keyword: str | None = Query(default=None, max_length=100),
|
||||
) -> dict[str, Any]:
|
||||
statement = (
|
||||
select(DataResources, StorageObjects)
|
||||
.join(
|
||||
StorageObjects,
|
||||
StorageObjects.storage_object_id
|
||||
== DataResources.storage_object_id,
|
||||
)
|
||||
.where(
|
||||
DataResources.workspace_id == context.workspace.workspace_id,
|
||||
DataResources.status == "active",
|
||||
StorageObjects.object_status == "available",
|
||||
)
|
||||
.order_by(DataResources.updated_at.desc())
|
||||
)
|
||||
if not context.is_admin:
|
||||
statement = statement.where(
|
||||
or_(
|
||||
DataResources.owner_user_id == context.user.user_id,
|
||||
DataResources.visibility.in_(["workspace", "public"]),
|
||||
)
|
||||
)
|
||||
if visibility:
|
||||
if visibility not in {"private", "workspace", "public"}:
|
||||
raise HTTPException(
|
||||
status.HTTP_422_UNPROCESSABLE_ENTITY,
|
||||
"invalid visibility",
|
||||
)
|
||||
statement = statement.where(DataResources.visibility == visibility)
|
||||
if keyword:
|
||||
statement = statement.where(
|
||||
DataResources.resource_name.like(f"%{keyword.strip()}%")
|
||||
)
|
||||
rows = (await session.execute(statement)).all()
|
||||
return {
|
||||
"request_id": context.request_id,
|
||||
"data": [
|
||||
resource_payload(resource, storage_object)
|
||||
for resource, storage_object in rows
|
||||
],
|
||||
"meta": {"count": len(rows)},
|
||||
}
|
||||
|
||||
|
||||
async def get_visible_resource(
|
||||
resource_id: str,
|
||||
context: RequestContext,
|
||||
session: AsyncSession,
|
||||
) -> tuple[DataResources, StorageObjects]:
|
||||
row = (
|
||||
await session.execute(
|
||||
select(DataResources, StorageObjects)
|
||||
.join(
|
||||
StorageObjects,
|
||||
StorageObjects.storage_object_id
|
||||
== DataResources.storage_object_id,
|
||||
)
|
||||
.where(
|
||||
DataResources.resource_id == resource_id,
|
||||
DataResources.workspace_id
|
||||
== context.workspace.workspace_id,
|
||||
DataResources.status == "active",
|
||||
)
|
||||
)
|
||||
).one_or_none()
|
||||
if row is None or not can_view(row[0], context):
|
||||
raise HTTPException(status.HTTP_404_NOT_FOUND, "resource not found")
|
||||
return row
|
||||
|
||||
|
||||
@router.get("/{resource_id}")
|
||||
async def get_resource(
|
||||
resource_id: str,
|
||||
context: RequestContext = Depends(request_context),
|
||||
session: AsyncSession = Depends(database_session),
|
||||
) -> dict[str, Any]:
|
||||
resource, storage_object = await get_visible_resource(
|
||||
resource_id,
|
||||
context,
|
||||
session,
|
||||
)
|
||||
return {
|
||||
"request_id": context.request_id,
|
||||
"data": resource_payload(resource, storage_object),
|
||||
"meta": {},
|
||||
}
|
||||
|
||||
|
||||
@router.post("/{resource_id}/download-url")
|
||||
async def resource_download_url(
|
||||
resource_id: str,
|
||||
payload: DownloadUrlRequest,
|
||||
request: Request,
|
||||
context: RequestContext = Depends(request_context),
|
||||
session: AsyncSession = Depends(database_session),
|
||||
) -> dict[str, Any]:
|
||||
resource, _ = await get_visible_resource(
|
||||
resource_id,
|
||||
context,
|
||||
session,
|
||||
)
|
||||
data = await request.app.state.storage_client.create_download_url(
|
||||
resource.storage_object_id,
|
||||
payload.expires_seconds,
|
||||
)
|
||||
return {"request_id": context.request_id, "data": data, "meta": {}}
|
||||
|
||||
|
||||
@router.delete("/{resource_id}")
|
||||
async def delete_resource(
|
||||
resource_id: str,
|
||||
request: Request,
|
||||
context: RequestContext = Depends(request_context),
|
||||
session: AsyncSession = Depends(database_session),
|
||||
) -> dict[str, Any]:
|
||||
resource, _ = await get_visible_resource(
|
||||
resource_id,
|
||||
context,
|
||||
session,
|
||||
)
|
||||
if (
|
||||
resource.owner_user_id != context.user.user_id
|
||||
and not context.is_admin
|
||||
):
|
||||
raise HTTPException(
|
||||
status.HTTP_403_FORBIDDEN,
|
||||
"resource can only be deleted by its owner or an administrator",
|
||||
)
|
||||
await request.app.state.storage_client.delete_object(
|
||||
resource.storage_object_id
|
||||
)
|
||||
resource.status = "deleted"
|
||||
resource.deleted_at = datetime.now(UTC).replace(tzinfo=None)
|
||||
return {
|
||||
"request_id": context.request_id,
|
||||
"data": {"resource_id": resource_id, "status": "deleted"},
|
||||
"meta": {},
|
||||
}
|
||||
@@ -0,0 +1,104 @@
|
||||
from __future__ import annotations
|
||||
|
||||
from dataclasses import dataclass
|
||||
from typing import Any
|
||||
|
||||
import httpx
|
||||
|
||||
|
||||
@dataclass(frozen=True)
|
||||
class RuntimeClientError(Exception):
|
||||
status_code: int
|
||||
detail: Any
|
||||
|
||||
|
||||
class RuntimeClient:
|
||||
def __init__(
|
||||
self,
|
||||
client: httpx.AsyncClient,
|
||||
service_token: str,
|
||||
) -> None:
|
||||
self.client = client
|
||||
self.headers = {"X-Service-Token": service_token}
|
||||
|
||||
async def _request(
|
||||
self,
|
||||
method: str,
|
||||
path: str,
|
||||
payload: dict[str, Any],
|
||||
) -> dict[str, Any]:
|
||||
try:
|
||||
response = await self.client.request(
|
||||
method,
|
||||
path,
|
||||
json=payload,
|
||||
headers=self.headers,
|
||||
)
|
||||
except httpx.RequestError as exc:
|
||||
raise RuntimeClientError(
|
||||
503,
|
||||
{
|
||||
"code": "RUNTIME_UNAVAILABLE",
|
||||
"message": "Runtime Manager 暂时不可用",
|
||||
"retryable": True,
|
||||
"details": {},
|
||||
},
|
||||
) from exc
|
||||
if response.is_error:
|
||||
try:
|
||||
detail = response.json().get("detail", response.text)
|
||||
except ValueError:
|
||||
detail = response.text
|
||||
raise RuntimeClientError(response.status_code, detail)
|
||||
return response.json()
|
||||
|
||||
async def acquire_file_lock(
|
||||
self,
|
||||
payload: dict[str, Any],
|
||||
) -> dict[str, Any]:
|
||||
return (
|
||||
await self._request(
|
||||
"POST",
|
||||
"/internal/v1/file-locks/acquire",
|
||||
payload,
|
||||
)
|
||||
)["data"]
|
||||
|
||||
async def heartbeat_file_lock(
|
||||
self,
|
||||
edit_session_id: str,
|
||||
payload: dict[str, Any],
|
||||
) -> dict[str, Any]:
|
||||
return (
|
||||
await self._request(
|
||||
"POST",
|
||||
f"/internal/v1/file-locks/{edit_session_id}/heartbeat",
|
||||
payload,
|
||||
)
|
||||
)["data"]
|
||||
|
||||
async def release_file_lock(
|
||||
self,
|
||||
edit_session_id: str,
|
||||
payload: dict[str, Any],
|
||||
) -> dict[str, Any]:
|
||||
return (
|
||||
await self._request(
|
||||
"DELETE",
|
||||
f"/internal/v1/file-locks/{edit_session_id}",
|
||||
payload,
|
||||
)
|
||||
)["data"]
|
||||
|
||||
async def create_jupyter_access_ticket(
|
||||
self,
|
||||
edit_session_id: str,
|
||||
payload: dict[str, Any],
|
||||
) -> dict[str, Any]:
|
||||
return (
|
||||
await self._request(
|
||||
"POST",
|
||||
f"/internal/v1/jupyter/access-tickets/{edit_session_id}",
|
||||
payload,
|
||||
)
|
||||
)["data"]
|
||||
@@ -0,0 +1,43 @@
|
||||
from __future__ import annotations
|
||||
|
||||
import logging
|
||||
|
||||
import httpx
|
||||
|
||||
|
||||
LOGGER = logging.getLogger(__name__)
|
||||
|
||||
|
||||
class ScheduleExecutorClient:
|
||||
"""Best-effort HTTP notification for immediate run dispatch.
|
||||
|
||||
MySQL remains the source of truth. If this notification fails, the
|
||||
executor's database polling loop will still pick up the pending Outbox row.
|
||||
"""
|
||||
|
||||
def __init__(self, client: httpx.AsyncClient, service_token: str) -> None:
|
||||
self.client = client
|
||||
self.headers = {"X-Service-Token": service_token}
|
||||
|
||||
async def dispatch_run(self, run_id: str) -> bool:
|
||||
try:
|
||||
response = await self.client.post(
|
||||
f"/internal/v1/runs/{run_id}/dispatch",
|
||||
headers=self.headers,
|
||||
)
|
||||
except httpx.RequestError:
|
||||
LOGGER.warning(
|
||||
"schedule executor notification failed for run %s",
|
||||
run_id,
|
||||
exc_info=True,
|
||||
)
|
||||
return False
|
||||
if response.is_error:
|
||||
LOGGER.warning(
|
||||
"schedule executor rejected run %s: %s %s",
|
||||
run_id,
|
||||
response.status_code,
|
||||
response.text[:500],
|
||||
)
|
||||
return False
|
||||
return True
|
||||
@@ -0,0 +1,348 @@
|
||||
from __future__ import annotations
|
||||
|
||||
import hashlib
|
||||
from datetime import UTC, datetime
|
||||
from typing import Any, Literal
|
||||
|
||||
from fastapi import APIRouter, Depends, Header, HTTPException, Query, Request, status
|
||||
from pydantic import Field
|
||||
from sqlalchemy import select
|
||||
from sqlalchemy.ext.asyncio import AsyncSession
|
||||
|
||||
from common.db.models import (
|
||||
ScheduleNodeRuns,
|
||||
ScheduleRuns,
|
||||
)
|
||||
from common.eventing import add_outbox_event, utcnow
|
||||
from common.ids import new_ulid
|
||||
from backend.dependencies import (
|
||||
RequestContext,
|
||||
database_session,
|
||||
request_context,
|
||||
)
|
||||
from backend.schedule_schemas import StrictModel
|
||||
from backend.schedules import (
|
||||
graph_rows,
|
||||
schedule_row,
|
||||
validate_dag,
|
||||
)
|
||||
|
||||
|
||||
router = APIRouter(tags=["schedule-runs"])
|
||||
RunStatus = Literal[
|
||||
"queued",
|
||||
"running",
|
||||
"succeeded",
|
||||
"failed",
|
||||
"cancelled",
|
||||
"timed_out",
|
||||
]
|
||||
|
||||
|
||||
class RunScheduleRequest(StrictModel):
|
||||
reason: str = Field(default="manual_run", min_length=1, max_length=255)
|
||||
|
||||
|
||||
def _iso(value: datetime | None) -> str | None:
|
||||
if value is None:
|
||||
return None
|
||||
if value.tzinfo is None:
|
||||
value = value.replace(tzinfo=UTC)
|
||||
return value.astimezone(UTC).isoformat()
|
||||
|
||||
|
||||
def _normalized_idempotency_key(
|
||||
workspace_id: str,
|
||||
schedule_id: str,
|
||||
value: str,
|
||||
) -> str:
|
||||
normalized = value.strip()
|
||||
if len(normalized) < 8:
|
||||
raise HTTPException(
|
||||
status.HTTP_422_UNPROCESSABLE_ENTITY,
|
||||
"Idempotency-Key must contain at least 8 characters",
|
||||
)
|
||||
digest = hashlib.sha256(
|
||||
f"{workspace_id}:{schedule_id}:{normalized}".encode("utf-8")
|
||||
).hexdigest()
|
||||
return f"run:v1:{digest}"
|
||||
|
||||
|
||||
def _arguments(value: dict[str, Any] | None) -> list[str]:
|
||||
payload = value or {}
|
||||
raw = payload.get("_args")
|
||||
result = [str(item) for item in raw] if isinstance(raw, list) else []
|
||||
for key, item in payload.items():
|
||||
if key == "_args":
|
||||
continue
|
||||
option = f"--{key.replace('_', '-')}"
|
||||
if item is True:
|
||||
result.append(option)
|
||||
elif item is False or item is None:
|
||||
continue
|
||||
elif isinstance(item, list):
|
||||
for list_item in item:
|
||||
result.extend((option, str(list_item)))
|
||||
elif isinstance(item, (str, int, float)):
|
||||
result.extend((option, str(item)))
|
||||
else:
|
||||
raise HTTPException(
|
||||
status.HTTP_422_UNPROCESSABLE_ENTITY,
|
||||
f"node argument {key!r} must be a scalar or list",
|
||||
)
|
||||
return result
|
||||
|
||||
|
||||
def run_summary(item: ScheduleRuns) -> dict[str, Any]:
|
||||
return {
|
||||
"run_id": item.run_id,
|
||||
"schedule_id": item.schedule_id,
|
||||
"workspace_id": item.workspace_id,
|
||||
"workflow_version": item.workflow_version,
|
||||
"trigger_type": item.trigger_type,
|
||||
"run_status": item.run_status,
|
||||
"state_version": item.state_version,
|
||||
"queued_at": _iso(item.queued_at),
|
||||
"started_at": _iso(item.started_at),
|
||||
"finished_at": _iso(item.finished_at),
|
||||
"duration_ms": item.duration_ms,
|
||||
"error_code": item.error_code,
|
||||
"error_message": item.error_message,
|
||||
"logs_object_id": item.logs_object_id,
|
||||
"result_object_id": item.result_object_id,
|
||||
}
|
||||
|
||||
|
||||
def node_run_payload(item: ScheduleNodeRuns) -> dict[str, Any]:
|
||||
return {
|
||||
"node_run_id": item.node_run_id,
|
||||
"run_id": item.run_id,
|
||||
"node_id": item.node_id,
|
||||
"versions_id": item.versions_id,
|
||||
"attempt_no": item.attempt_no,
|
||||
"node_status": item.node_status,
|
||||
"state_version": item.state_version,
|
||||
"started_at": _iso(item.started_at),
|
||||
"finished_at": _iso(item.finished_at),
|
||||
"duration_ms": item.duration_ms,
|
||||
"exit_code": item.exit_code,
|
||||
"message": item.message,
|
||||
"logs_object_id": item.logs_object_id,
|
||||
"result_object_id": item.result_object_id,
|
||||
}
|
||||
|
||||
|
||||
async def run_detail(
|
||||
item: ScheduleRuns,
|
||||
session: AsyncSession,
|
||||
) -> dict[str, Any]:
|
||||
node_runs = list(
|
||||
(
|
||||
await session.scalars(
|
||||
select(ScheduleNodeRuns)
|
||||
.where(ScheduleNodeRuns.run_id == item.run_id)
|
||||
.order_by(
|
||||
ScheduleNodeRuns.created_at,
|
||||
ScheduleNodeRuns.attempt_no,
|
||||
)
|
||||
)
|
||||
).all()
|
||||
)
|
||||
return {
|
||||
**run_summary(item),
|
||||
"node_runs": [node_run_payload(node_run) for node_run in node_runs],
|
||||
}
|
||||
|
||||
|
||||
async def _visible_run(
|
||||
run_id: str,
|
||||
context: RequestContext,
|
||||
session: AsyncSession,
|
||||
) -> ScheduleRuns:
|
||||
item = await session.scalar(
|
||||
select(ScheduleRuns).where(
|
||||
ScheduleRuns.run_id == run_id,
|
||||
ScheduleRuns.workspace_id == context.workspace.workspace_id,
|
||||
)
|
||||
)
|
||||
if item is None:
|
||||
raise HTTPException(status.HTTP_404_NOT_FOUND, "schedule run not found")
|
||||
return item
|
||||
|
||||
|
||||
@router.post(
|
||||
"/api/v1/schedules/{schedule_id}/run",
|
||||
status_code=status.HTTP_202_ACCEPTED,
|
||||
)
|
||||
async def run_schedule_now(
|
||||
schedule_id: str,
|
||||
request: Request,
|
||||
payload: RunScheduleRequest | None = None,
|
||||
idempotency_key: str = Header(alias="Idempotency-Key"),
|
||||
context: RequestContext = Depends(request_context),
|
||||
session: AsyncSession = Depends(database_session),
|
||||
) -> dict[str, Any]:
|
||||
reason = payload.reason if payload is not None else "manual_run"
|
||||
key = _normalized_idempotency_key(
|
||||
context.workspace.workspace_id,
|
||||
schedule_id,
|
||||
idempotency_key,
|
||||
)
|
||||
existing = await session.scalar(
|
||||
select(ScheduleRuns).where(ScheduleRuns.idempotency_key == key)
|
||||
)
|
||||
if existing is not None:
|
||||
if (
|
||||
existing.workspace_id != context.workspace.workspace_id
|
||||
or existing.schedule_id != schedule_id
|
||||
):
|
||||
raise HTTPException(
|
||||
status.HTTP_409_CONFLICT,
|
||||
"Idempotency-Key belongs to another schedule run",
|
||||
)
|
||||
return {
|
||||
"request_id": context.request_id,
|
||||
"data": await run_detail(existing, session),
|
||||
"meta": {"reused": True},
|
||||
}
|
||||
|
||||
schedule = await schedule_row(
|
||||
schedule_id,
|
||||
context,
|
||||
session,
|
||||
for_update=True,
|
||||
)
|
||||
node_rows, edges = await graph_rows(schedule_id, session)
|
||||
nodes = [row[0] for row in node_rows]
|
||||
validation = validate_dag(nodes, edges)
|
||||
if not validation["valid"] or not nodes:
|
||||
raise HTTPException(
|
||||
status.HTTP_409_CONFLICT,
|
||||
detail={
|
||||
"code": "SCHEDULE_DAG_INVALID",
|
||||
"message": "schedule must contain a valid non-empty DAG",
|
||||
"errors": validation["errors"],
|
||||
},
|
||||
)
|
||||
if len(nodes) > 100 or len(edges) > 500:
|
||||
raise HTTPException(
|
||||
status.HTTP_409_CONFLICT,
|
||||
"schedule exceeds the v1 execution size limit",
|
||||
)
|
||||
|
||||
snapshot = {
|
||||
"schedule_name": schedule.schedule_name,
|
||||
"workflow_version": schedule.workflow_version,
|
||||
"max_concurrency": schedule.max_concurrency,
|
||||
"failure_policy": schedule.failure_policy,
|
||||
"nodes": [
|
||||
{
|
||||
"node_id": node.node_id,
|
||||
"node_key": node.node_key,
|
||||
"versions_id": version.versions_id,
|
||||
"script_type": script.script_type,
|
||||
"artifact_object_id": version.artifact_object_id,
|
||||
"artifact_path": version.artifact_path,
|
||||
"timeout_seconds": node.timeout_seconds,
|
||||
"retry_count": node.retry_count,
|
||||
"retry_interval_sec": node.retry_interval_sec,
|
||||
"arguments": _arguments(node.arguments_json),
|
||||
}
|
||||
for node, version, script in node_rows
|
||||
],
|
||||
"edges": [
|
||||
{
|
||||
"source_node_id": edge.source_node_id,
|
||||
"target_node_id": edge.target_node_id,
|
||||
}
|
||||
for edge in edges
|
||||
],
|
||||
}
|
||||
now = utcnow()
|
||||
run = ScheduleRuns(
|
||||
run_id=new_ulid(),
|
||||
schedule_id=schedule.schedule_id,
|
||||
workspace_id=schedule.workspace_id,
|
||||
workflow_version=schedule.workflow_version,
|
||||
trigger_type="cron" if reason == "cron" else "manual",
|
||||
idempotency_key=key,
|
||||
run_status="queued",
|
||||
state_version=0,
|
||||
schedule_snapshot=snapshot,
|
||||
queued_at=now,
|
||||
triggered_by=context.user.user_id,
|
||||
)
|
||||
session.add(run)
|
||||
schedule.last_run_at = now
|
||||
await add_outbox_event(
|
||||
session,
|
||||
event_type="schedule.run.requested",
|
||||
producer="platform-api",
|
||||
trace_id=context.request_id,
|
||||
aggregate_type="schedule_run",
|
||||
aggregate_id=run.run_id,
|
||||
idempotency_key=key,
|
||||
payload={
|
||||
"workspace_id": run.workspace_id,
|
||||
"schedule_id": run.schedule_id,
|
||||
"run_id": run.run_id,
|
||||
"workflow_version": run.workflow_version,
|
||||
"trigger_type": run.trigger_type,
|
||||
"triggered_by": run.triggered_by,
|
||||
"schedule_snapshot": snapshot,
|
||||
},
|
||||
)
|
||||
await session.flush()
|
||||
# Commit before the HTTP push so the executor can read the Outbox row.
|
||||
# The executor also polls MySQL, so a failed push does not lose the run.
|
||||
await session.commit()
|
||||
await request.app.state.schedule_client.dispatch_run(run.run_id)
|
||||
await session.refresh(run)
|
||||
return {
|
||||
"request_id": context.request_id,
|
||||
"data": await run_detail(run, session),
|
||||
"meta": {"reused": False},
|
||||
}
|
||||
|
||||
|
||||
@router.get("/api/v1/schedule-runs")
|
||||
async def list_schedule_runs(
|
||||
schedule_id: str | None = Query(default=None),
|
||||
run_status: RunStatus | None = Query(default=None, alias="status"),
|
||||
limit: int = Query(default=50, ge=1, le=200),
|
||||
context: RequestContext = Depends(request_context),
|
||||
session: AsyncSession = Depends(database_session),
|
||||
) -> dict[str, Any]:
|
||||
statement = select(ScheduleRuns).where(
|
||||
ScheduleRuns.workspace_id == context.workspace.workspace_id,
|
||||
)
|
||||
if schedule_id:
|
||||
statement = statement.where(ScheduleRuns.schedule_id == schedule_id)
|
||||
if run_status:
|
||||
statement = statement.where(ScheduleRuns.run_status == run_status)
|
||||
items = list(
|
||||
(
|
||||
await session.scalars(
|
||||
statement.order_by(ScheduleRuns.queued_at.desc()).limit(limit)
|
||||
)
|
||||
).all()
|
||||
)
|
||||
return {
|
||||
"request_id": context.request_id,
|
||||
"data": [run_summary(item) for item in items],
|
||||
"meta": {"count": len(items)},
|
||||
}
|
||||
|
||||
|
||||
@router.get("/api/v1/schedule-runs/{run_id}")
|
||||
async def get_schedule_run(
|
||||
run_id: str,
|
||||
context: RequestContext = Depends(request_context),
|
||||
session: AsyncSession = Depends(database_session),
|
||||
) -> dict[str, Any]:
|
||||
item = await _visible_run(run_id, context, session)
|
||||
return {
|
||||
"request_id": context.request_id,
|
||||
"data": await run_detail(item, session),
|
||||
"meta": {},
|
||||
}
|
||||
@@ -0,0 +1,214 @@
|
||||
from __future__ import annotations
|
||||
|
||||
from datetime import datetime
|
||||
from typing import Any, Literal
|
||||
|
||||
from pydantic import Field, field_validator, model_validator
|
||||
|
||||
from backend.schemas import StrictModel
|
||||
|
||||
|
||||
TriggerType = Literal["manual", "cron", "api"]
|
||||
FailurePolicy = Literal["stop", "continue"]
|
||||
|
||||
|
||||
def _required_text(value: str) -> str:
|
||||
normalized = value.strip()
|
||||
if not normalized:
|
||||
raise ValueError("value must not be blank")
|
||||
return normalized
|
||||
|
||||
|
||||
class CreateScheduleRequest(StrictModel):
|
||||
schedule_name: str = Field(min_length=1, max_length=255)
|
||||
description: str | None = Field(default=None, max_length=1000)
|
||||
trigger_type: TriggerType = "cron"
|
||||
cron_expression: str | None = Field(default=None, max_length=128)
|
||||
timezone: str = Field(default="Asia/Shanghai", min_length=1, max_length=64)
|
||||
enabled: bool = False
|
||||
max_concurrency: int = Field(default=1, ge=1, le=64)
|
||||
failure_policy: FailurePolicy = "stop"
|
||||
|
||||
@field_validator("schedule_name", "timezone")
|
||||
@classmethod
|
||||
def validate_required_text(cls, value: str) -> str:
|
||||
return _required_text(value)
|
||||
|
||||
@field_validator("description", "cron_expression")
|
||||
@classmethod
|
||||
def normalize_optional_text(cls, value: str | None) -> str | None:
|
||||
if value is None:
|
||||
return None
|
||||
normalized = value.strip()
|
||||
return normalized or None
|
||||
|
||||
@model_validator(mode="after")
|
||||
def validate_trigger(self) -> "CreateScheduleRequest":
|
||||
if self.trigger_type == "cron" and not self.cron_expression:
|
||||
raise ValueError("cron_expression is required for cron schedules")
|
||||
if self.trigger_type != "cron" and self.cron_expression:
|
||||
raise ValueError(
|
||||
"cron_expression is only allowed for cron schedules"
|
||||
)
|
||||
return self
|
||||
|
||||
|
||||
class UpdateScheduleRequest(StrictModel):
|
||||
workflow_version: int = Field(ge=1)
|
||||
schedule_name: str | None = Field(default=None, min_length=1, max_length=255)
|
||||
description: str | None = Field(default=None, max_length=1000)
|
||||
trigger_type: TriggerType | None = None
|
||||
cron_expression: str | None = Field(default=None, max_length=128)
|
||||
timezone: str | None = Field(default=None, min_length=1, max_length=64)
|
||||
enabled: bool | None = None
|
||||
max_concurrency: int | None = Field(default=None, ge=1, le=64)
|
||||
failure_policy: FailurePolicy | None = None
|
||||
|
||||
@field_validator("schedule_name", "timezone")
|
||||
@classmethod
|
||||
def validate_optional_required_text(
|
||||
cls,
|
||||
value: str | None,
|
||||
) -> str | None:
|
||||
return _required_text(value) if value is not None else None
|
||||
|
||||
@field_validator("description", "cron_expression")
|
||||
@classmethod
|
||||
def normalize_optional_text(cls, value: str | None) -> str | None:
|
||||
if value is None:
|
||||
return None
|
||||
normalized = value.strip()
|
||||
return normalized or None
|
||||
|
||||
@model_validator(mode="after")
|
||||
def require_change(self) -> "UpdateScheduleRequest":
|
||||
if self.model_fields_set == {"workflow_version"}:
|
||||
raise ValueError("at least one schedule field must be updated")
|
||||
for field in {
|
||||
"schedule_name",
|
||||
"trigger_type",
|
||||
"timezone",
|
||||
"enabled",
|
||||
"max_concurrency",
|
||||
"failure_policy",
|
||||
}:
|
||||
if field in self.model_fields_set and getattr(self, field) is None:
|
||||
raise ValueError(f"{field} cannot be null")
|
||||
return self
|
||||
|
||||
|
||||
class WorkflowVersionRequest(StrictModel):
|
||||
workflow_version: int = Field(ge=1)
|
||||
|
||||
|
||||
class CreateScheduleNodeRequest(StrictModel):
|
||||
workflow_version: int = Field(ge=1)
|
||||
node_key: str = Field(
|
||||
min_length=1,
|
||||
max_length=64,
|
||||
pattern=r"^[A-Za-z][A-Za-z0-9_-]*$",
|
||||
)
|
||||
node_name: str = Field(min_length=1, max_length=255)
|
||||
versions_id: str = Field(min_length=26, max_length=26)
|
||||
timeout_seconds: int = Field(default=600, ge=1, le=86_400)
|
||||
retry_count: int = Field(default=0, ge=0, le=10)
|
||||
retry_interval_sec: int = Field(default=5, ge=0, le=3600)
|
||||
position_x: float = Field(default=0, ge=-100_000, le=100_000)
|
||||
position_y: float = Field(default=0, ge=-100_000, le=100_000)
|
||||
arguments_json: dict[str, Any] = Field(default_factory=dict, max_length=100)
|
||||
env_refs_json: dict[str, str] = Field(default_factory=dict, max_length=100)
|
||||
|
||||
@field_validator("node_key", "node_name")
|
||||
@classmethod
|
||||
def validate_required_text(cls, value: str) -> str:
|
||||
return _required_text(value)
|
||||
|
||||
|
||||
class UpdateScheduleNodeRequest(StrictModel):
|
||||
workflow_version: int = Field(ge=1)
|
||||
node_name: str | None = Field(default=None, min_length=1, max_length=255)
|
||||
versions_id: str | None = Field(default=None, min_length=26, max_length=26)
|
||||
timeout_seconds: int | None = Field(default=None, ge=1, le=86_400)
|
||||
retry_count: int | None = Field(default=None, ge=0, le=10)
|
||||
retry_interval_sec: int | None = Field(default=None, ge=0, le=3600)
|
||||
position_x: float | None = Field(
|
||||
default=None,
|
||||
ge=-100_000,
|
||||
le=100_000,
|
||||
)
|
||||
position_y: float | None = Field(
|
||||
default=None,
|
||||
ge=-100_000,
|
||||
le=100_000,
|
||||
)
|
||||
arguments_json: dict[str, Any] | None = Field(
|
||||
default=None,
|
||||
max_length=100,
|
||||
)
|
||||
env_refs_json: dict[str, str] | None = Field(
|
||||
default=None,
|
||||
max_length=100,
|
||||
)
|
||||
|
||||
@field_validator("node_name")
|
||||
@classmethod
|
||||
def validate_optional_required_text(
|
||||
cls,
|
||||
value: str | None,
|
||||
) -> str | None:
|
||||
return _required_text(value) if value is not None else None
|
||||
|
||||
@model_validator(mode="after")
|
||||
def require_change(self) -> "UpdateScheduleNodeRequest":
|
||||
if self.model_fields_set == {"workflow_version"}:
|
||||
raise ValueError("at least one node field must be updated")
|
||||
for field in self.model_fields_set - {"workflow_version"}:
|
||||
if getattr(self, field) is None:
|
||||
raise ValueError(f"{field} cannot be null")
|
||||
return self
|
||||
|
||||
|
||||
class CreateScheduleEdgeRequest(StrictModel):
|
||||
workflow_version: int = Field(ge=1)
|
||||
source_node_id: str = Field(min_length=26, max_length=26)
|
||||
target_node_id: str = Field(min_length=26, max_length=26)
|
||||
condition_expr: str | None = Field(default=None, max_length=1000)
|
||||
|
||||
@field_validator("condition_expr")
|
||||
@classmethod
|
||||
def normalize_condition(cls, value: str | None) -> str | None:
|
||||
if value is None:
|
||||
return None
|
||||
normalized = value.strip()
|
||||
return normalized or None
|
||||
|
||||
@model_validator(mode="after")
|
||||
def reject_self_edge(self) -> "CreateScheduleEdgeRequest":
|
||||
if self.source_node_id == self.target_node_id:
|
||||
raise ValueError("an edge cannot connect a node to itself")
|
||||
return self
|
||||
|
||||
|
||||
class UpdateScheduleEdgeRequest(StrictModel):
|
||||
workflow_version: int = Field(ge=1)
|
||||
condition_expr: str | None = Field(default=None, max_length=1000)
|
||||
|
||||
@field_validator("condition_expr")
|
||||
@classmethod
|
||||
def normalize_condition(cls, value: str | None) -> str | None:
|
||||
if value is None:
|
||||
return None
|
||||
normalized = value.strip()
|
||||
return normalized or None
|
||||
|
||||
|
||||
class CronPreviewRequest(StrictModel):
|
||||
cron_expression: str = Field(min_length=1, max_length=128)
|
||||
timezone: str = Field(default="Asia/Shanghai", min_length=1, max_length=64)
|
||||
count: int = Field(default=5, ge=1, le=20)
|
||||
base_time: datetime | None = None
|
||||
|
||||
@field_validator("cron_expression", "timezone")
|
||||
@classmethod
|
||||
def validate_required_text(cls, value: str) -> str:
|
||||
return _required_text(value)
|
||||
File diff suppressed because it is too large
Load Diff
@@ -0,0 +1,72 @@
|
||||
from __future__ import annotations
|
||||
|
||||
from typing import Literal
|
||||
|
||||
from pydantic import BaseModel, ConfigDict, Field, field_validator
|
||||
|
||||
|
||||
class StrictModel(BaseModel):
|
||||
model_config = ConfigDict(extra="forbid")
|
||||
|
||||
|
||||
class CreateResourceUploadRequest(StrictModel):
|
||||
file_name: str = Field(min_length=1, max_length=255)
|
||||
content_type: str = Field(min_length=1, max_length=255)
|
||||
expected_size_bytes: int = Field(ge=0, le=100 * 1024 * 1024)
|
||||
expected_hash: str | None = Field(default=None, min_length=64, max_length=64)
|
||||
|
||||
@field_validator("expected_hash")
|
||||
@classmethod
|
||||
def validate_hash(cls, value: str | None) -> str | None:
|
||||
if value is None:
|
||||
return None
|
||||
normalized = value.lower()
|
||||
if any(character not in "0123456789abcdef" for character in normalized):
|
||||
raise ValueError("expected_hash must be SHA-256 hex")
|
||||
return normalized
|
||||
|
||||
|
||||
class CompleteResourceUploadRequest(StrictModel):
|
||||
resource_name: str = Field(min_length=1, max_length=255)
|
||||
description: str | None = Field(default=None, max_length=1000)
|
||||
visibility: Literal["private", "workspace", "public"] = "private"
|
||||
|
||||
|
||||
class CreateScriptRequest(StrictModel):
|
||||
script_name: str = Field(min_length=1, max_length=255)
|
||||
script_type: Literal["python", "notebook"]
|
||||
content: str = Field(max_length=10 * 1024 * 1024)
|
||||
visibility: Literal["private", "workspace", "public"] = "private"
|
||||
parent_path: str | None = Field(default=None, max_length=1024)
|
||||
|
||||
|
||||
class CreateWorkspaceDirectoryRequest(StrictModel):
|
||||
directory_name: str = Field(min_length=1, max_length=255)
|
||||
parent_path: str = Field(default="", max_length=1024)
|
||||
|
||||
|
||||
class UpdateScriptRequest(StrictModel):
|
||||
content: str = Field(max_length=10 * 1024 * 1024)
|
||||
|
||||
|
||||
class PublishVersionRequest(StrictModel):
|
||||
source_object_id: str | None = Field(
|
||||
default=None,
|
||||
min_length=26,
|
||||
max_length=26,
|
||||
)
|
||||
release_note: str | None = Field(default=None, max_length=1000)
|
||||
visibility: Literal["private", "workspace", "public"] = "workspace"
|
||||
|
||||
|
||||
class DownloadUrlRequest(StrictModel):
|
||||
expires_seconds: int = Field(default=300, ge=30, le=3600)
|
||||
|
||||
|
||||
class FileLockTokenRequest(StrictModel):
|
||||
lock_token: str = Field(min_length=32, max_length=256)
|
||||
|
||||
|
||||
class CreateJupyterAccessTicketRequest(StrictModel):
|
||||
edit_session_id: str = Field(min_length=26, max_length=26)
|
||||
lock_token: str = Field(min_length=32, max_length=256)
|
||||
@@ -0,0 +1,925 @@
|
||||
from __future__ import annotations
|
||||
|
||||
import hashlib
|
||||
import json
|
||||
import mimetypes
|
||||
import os
|
||||
import shutil
|
||||
import tempfile
|
||||
from datetime import UTC, datetime
|
||||
from pathlib import Path, PurePosixPath
|
||||
from typing import Any
|
||||
|
||||
from fastapi import (
|
||||
APIRouter,
|
||||
Depends,
|
||||
Header,
|
||||
HTTPException,
|
||||
Query,
|
||||
Request,
|
||||
status,
|
||||
)
|
||||
from sqlalchemy import func, select
|
||||
from sqlalchemy.ext.asyncio import AsyncSession
|
||||
|
||||
from common.db.models import (
|
||||
EditSessions,
|
||||
Scripts,
|
||||
StorageObjects,
|
||||
Versions,
|
||||
)
|
||||
from common.ids import new_ulid
|
||||
from backend.dependencies import (
|
||||
RequestContext,
|
||||
database_session,
|
||||
request_context,
|
||||
)
|
||||
from backend.schemas import (
|
||||
CreateScriptRequest,
|
||||
CreateWorkspaceDirectoryRequest,
|
||||
DownloadUrlRequest,
|
||||
PublishVersionRequest,
|
||||
UpdateScriptRequest,
|
||||
)
|
||||
|
||||
router = APIRouter(tags=["scripts"])
|
||||
|
||||
|
||||
def normalize_user_path(value: str, *, allow_empty: bool = True) -> str:
|
||||
normalized = value.replace("\\", "/").strip().strip("/")
|
||||
if not normalized and allow_empty:
|
||||
return ""
|
||||
pure_path = PurePosixPath(normalized)
|
||||
if (
|
||||
pure_path.is_absolute()
|
||||
or not pure_path.parts
|
||||
or any(
|
||||
part in {"", ".", ".."} or any(ord(char) < 32 for char in part)
|
||||
for part in pure_path.parts
|
||||
)
|
||||
):
|
||||
raise HTTPException(
|
||||
status.HTTP_422_UNPROCESSABLE_ENTITY,
|
||||
"invalid workspace path",
|
||||
)
|
||||
return pure_path.as_posix()
|
||||
|
||||
|
||||
def safe_directory_name(value: str) -> str:
|
||||
name = value.strip()
|
||||
if (
|
||||
not name
|
||||
or name in {".", ".."}
|
||||
or name.startswith(".")
|
||||
or "/" in name
|
||||
or "\\" in name
|
||||
or any(ord(char) < 32 for char in name)
|
||||
):
|
||||
raise HTTPException(
|
||||
status.HTTP_422_UNPROCESSABLE_ENTITY,
|
||||
"invalid directory_name",
|
||||
)
|
||||
return name
|
||||
|
||||
|
||||
def user_relative_path(context: RequestContext, child_path: str = "") -> str:
|
||||
base = f"users/{context.user.username}"
|
||||
normalized = normalize_user_path(child_path)
|
||||
return f"{base}/{normalized}" if normalized else base
|
||||
|
||||
|
||||
def safe_script_name(value: str, script_type: str) -> str:
|
||||
name = value.replace("\\", "/").rsplit("/", 1)[-1].strip()
|
||||
if not name or name in {".", ".."} or any(ord(char) < 32 for char in name):
|
||||
raise HTTPException(
|
||||
status.HTTP_422_UNPROCESSABLE_ENTITY,
|
||||
"invalid script_name",
|
||||
)
|
||||
expected_suffix = ".py" if script_type == "python" else ".ipynb"
|
||||
if not name.lower().endswith(expected_suffix):
|
||||
name += expected_suffix
|
||||
return name
|
||||
|
||||
|
||||
def validate_script_content(content: str, script_type: str) -> bytes:
|
||||
encoded = content.encode("utf-8")
|
||||
if len(encoded) > 10 * 1024 * 1024:
|
||||
raise HTTPException(
|
||||
status.HTTP_413_REQUEST_ENTITY_TOO_LARGE,
|
||||
"script exceeds 10 MiB",
|
||||
)
|
||||
if script_type == "notebook":
|
||||
try:
|
||||
notebook = json.loads(content)
|
||||
except json.JSONDecodeError as exc:
|
||||
raise HTTPException(
|
||||
status.HTTP_422_UNPROCESSABLE_ENTITY,
|
||||
"notebook content must be valid JSON",
|
||||
) from exc
|
||||
if not isinstance(notebook, dict) or not isinstance(
|
||||
notebook.get("cells"),
|
||||
list,
|
||||
):
|
||||
raise HTTPException(
|
||||
status.HTTP_422_UNPROCESSABLE_ENTITY,
|
||||
"notebook must contain a cells array",
|
||||
)
|
||||
return encoded
|
||||
|
||||
|
||||
def workspace_target(
|
||||
context: RequestContext,
|
||||
relative_path: str,
|
||||
) -> Path:
|
||||
root = Path(
|
||||
os.getenv("WORKSPACE_ROOT", "/workspace/workspaces")
|
||||
).resolve()
|
||||
scoped_root = (root / context.workspace.workspace_code).resolve()
|
||||
pure_path = PurePosixPath(relative_path)
|
||||
target = (scoped_root / Path(*pure_path.parts)).resolve()
|
||||
if scoped_root not in target.parents:
|
||||
raise HTTPException(
|
||||
status.HTTP_422_UNPROCESSABLE_ENTITY,
|
||||
"script path escapes workspace root",
|
||||
)
|
||||
return target
|
||||
|
||||
|
||||
def apply_workspace_permissions(path: Path, mode: int) -> None:
|
||||
os.chmod(path, mode)
|
||||
if os.name != "nt":
|
||||
shared_gid = int(os.getenv("WORKSPACE_SHARED_GID", "100"))
|
||||
os.chown(path, -1, shared_gid)
|
||||
|
||||
|
||||
def atomic_write(target: Path, content: bytes) -> None:
|
||||
target.parent.mkdir(parents=True, exist_ok=True)
|
||||
apply_workspace_permissions(target.parent, 0o2770)
|
||||
descriptor, temporary_name = tempfile.mkstemp(
|
||||
prefix=f".{target.name}.",
|
||||
suffix=".tmp",
|
||||
dir=target.parent,
|
||||
)
|
||||
try:
|
||||
with os.fdopen(descriptor, "wb") as handle:
|
||||
handle.write(content)
|
||||
handle.flush()
|
||||
os.fsync(handle.fileno())
|
||||
os.replace(temporary_name, target)
|
||||
apply_workspace_permissions(target, 0o660)
|
||||
finally:
|
||||
if os.path.exists(temporary_name):
|
||||
os.unlink(temporary_name)
|
||||
|
||||
|
||||
def script_payload(
|
||||
script: Scripts,
|
||||
storage_object: StorageObjects | dict[str, Any],
|
||||
) -> dict[str, Any]:
|
||||
if isinstance(storage_object, dict):
|
||||
relative_path = storage_object.get("relative_path")
|
||||
content_hash = storage_object.get("content_hash")
|
||||
size_bytes = storage_object.get("size_bytes", 0)
|
||||
else:
|
||||
relative_path = storage_object.relative_path
|
||||
content_hash = storage_object.content_hash
|
||||
size_bytes = storage_object.size_bytes
|
||||
return {
|
||||
"script_id": script.script_id,
|
||||
"workspace_id": script.workspace_id,
|
||||
"current_object_id": script.current_object_id,
|
||||
"owner_user_id": script.owner_user_id,
|
||||
"script_name": script.script_name,
|
||||
"script_type": script.script_type,
|
||||
"visibility": script.visibility,
|
||||
"status": script.status,
|
||||
"relative_path": relative_path,
|
||||
"content_hash": content_hash,
|
||||
"size_bytes": size_bytes,
|
||||
"created_at": script.created_at.isoformat(),
|
||||
"updated_at": script.updated_at.isoformat(),
|
||||
}
|
||||
|
||||
|
||||
def version_payload(version: Versions) -> dict[str, Any]:
|
||||
return {
|
||||
"versions_id": version.versions_id,
|
||||
"workspace_id": version.workspace_id,
|
||||
"script_id": version.script_id,
|
||||
"source_object_id": version.source_object_id,
|
||||
"artifact_object_id": version.artifact_object_id,
|
||||
"version_no": version.version_no,
|
||||
"version_label": version.version_label,
|
||||
"source_path": version.source_path,
|
||||
"artifact_path": version.artifact_path,
|
||||
"content_hash": version.content_hash,
|
||||
"file_size_bytes": version.file_size_bytes,
|
||||
"visibility": version.visibility,
|
||||
"release_note": version.release_note,
|
||||
"created_by": version.created_by,
|
||||
"created_at": version.created_at.isoformat(),
|
||||
}
|
||||
|
||||
|
||||
async def get_script_row(
|
||||
script_id: str,
|
||||
context: RequestContext,
|
||||
session: AsyncSession,
|
||||
*,
|
||||
for_update: bool = False,
|
||||
) -> tuple[Scripts, StorageObjects]:
|
||||
if for_update:
|
||||
script = await session.scalar(
|
||||
select(Scripts)
|
||||
.where(
|
||||
Scripts.script_id == script_id,
|
||||
Scripts.workspace_id == context.workspace.workspace_id,
|
||||
Scripts.status == "active",
|
||||
)
|
||||
.with_for_update()
|
||||
)
|
||||
if script is None:
|
||||
raise HTTPException(
|
||||
status.HTTP_404_NOT_FOUND,
|
||||
"script not found",
|
||||
)
|
||||
storage_object = await session.get(
|
||||
StorageObjects,
|
||||
script.current_object_id,
|
||||
)
|
||||
if storage_object is None:
|
||||
raise HTTPException(
|
||||
status.HTTP_409_CONFLICT,
|
||||
"script working-copy metadata is missing",
|
||||
)
|
||||
row = (script, storage_object)
|
||||
else:
|
||||
statement = (
|
||||
select(Scripts, StorageObjects)
|
||||
.join(
|
||||
StorageObjects,
|
||||
StorageObjects.storage_object_id
|
||||
== Scripts.current_object_id,
|
||||
)
|
||||
.where(
|
||||
Scripts.script_id == script_id,
|
||||
Scripts.workspace_id == context.workspace.workspace_id,
|
||||
Scripts.status == "active",
|
||||
)
|
||||
)
|
||||
row = (await session.execute(statement)).one_or_none()
|
||||
if row is None:
|
||||
raise HTTPException(status.HTTP_404_NOT_FOUND, "script not found")
|
||||
script, storage_object = row
|
||||
return script, storage_object
|
||||
|
||||
|
||||
async def require_no_active_edit_session(
|
||||
session: AsyncSession,
|
||||
storage_object_ids: list[str],
|
||||
) -> None:
|
||||
if not storage_object_ids:
|
||||
return
|
||||
active_session = await session.scalar(
|
||||
select(EditSessions).where(
|
||||
EditSessions.storage_object_id.in_(storage_object_ids),
|
||||
EditSessions.session_status == "active",
|
||||
EditSessions.expires_at > datetime.now(UTC).replace(tzinfo=None),
|
||||
)
|
||||
)
|
||||
if active_session is not None:
|
||||
raise HTTPException(
|
||||
status.HTTP_409_CONFLICT,
|
||||
"file is being edited; end the editing session before deletion",
|
||||
)
|
||||
|
||||
|
||||
async def create_script_record(
|
||||
*,
|
||||
name: str,
|
||||
script_type: str,
|
||||
content: bytes,
|
||||
visibility: str,
|
||||
parent_path: str | None,
|
||||
request: Request,
|
||||
context: RequestContext,
|
||||
session: AsyncSession,
|
||||
) -> tuple[Scripts, dict[str, Any]]:
|
||||
folder = (
|
||||
"scripts" if script_type == "python" else "notebooks"
|
||||
) if parent_path is None else normalize_user_path(parent_path)
|
||||
child_path = f"{folder}/{name}" if folder else name
|
||||
relative_path = user_relative_path(context, child_path)
|
||||
target = workspace_target(context, relative_path)
|
||||
|
||||
existing_script = await session.scalar(
|
||||
select(Scripts)
|
||||
.join(
|
||||
StorageObjects,
|
||||
StorageObjects.storage_object_id == Scripts.current_object_id,
|
||||
)
|
||||
.where(
|
||||
Scripts.workspace_id == context.workspace.workspace_id,
|
||||
StorageObjects.relative_path == relative_path,
|
||||
Scripts.status == "active",
|
||||
)
|
||||
)
|
||||
if existing_script is not None or target.exists():
|
||||
raise HTTPException(
|
||||
status.HTTP_409_CONFLICT,
|
||||
"a file with the same path already exists",
|
||||
)
|
||||
|
||||
atomic_write(target, content)
|
||||
storage_data = await request.app.state.storage_client.register_workspace_object(
|
||||
{
|
||||
"workspace_id": context.workspace.workspace_id,
|
||||
"user_id": context.user.user_id,
|
||||
"relative_path": relative_path,
|
||||
"usage_type": "working_copy",
|
||||
"visibility": visibility,
|
||||
}
|
||||
)
|
||||
object_id = storage_data["storage_object_id"]
|
||||
script = await session.scalar(
|
||||
select(Scripts).where(Scripts.current_object_id == object_id)
|
||||
)
|
||||
now = datetime.now(UTC).replace(tzinfo=None)
|
||||
if script is None:
|
||||
script = Scripts(
|
||||
script_id=new_ulid(),
|
||||
workspace_id=context.workspace.workspace_id,
|
||||
current_object_id=object_id,
|
||||
owner_user_id=context.user.user_id,
|
||||
script_name=name,
|
||||
script_type=script_type,
|
||||
visibility=visibility,
|
||||
status="active",
|
||||
)
|
||||
session.add(script)
|
||||
else:
|
||||
script.owner_user_id = context.user.user_id
|
||||
script.script_name = name
|
||||
script.script_type = script_type
|
||||
script.visibility = visibility
|
||||
script.status = "active"
|
||||
script.deleted_at = None
|
||||
script.updated_at = now
|
||||
await session.flush()
|
||||
await session.refresh(script)
|
||||
return script, storage_data
|
||||
|
||||
|
||||
@router.post("/api/v1/scripts", status_code=status.HTTP_201_CREATED)
|
||||
async def create_script(
|
||||
payload: CreateScriptRequest,
|
||||
request: Request,
|
||||
context: RequestContext = Depends(request_context),
|
||||
session: AsyncSession = Depends(database_session),
|
||||
) -> dict[str, Any]:
|
||||
name = safe_script_name(payload.script_name, payload.script_type)
|
||||
content = validate_script_content(payload.content, payload.script_type)
|
||||
script, storage_data = await create_script_record(
|
||||
name=name,
|
||||
script_type=payload.script_type,
|
||||
content=content,
|
||||
visibility=payload.visibility,
|
||||
parent_path=payload.parent_path,
|
||||
request=request,
|
||||
context=context,
|
||||
session=session,
|
||||
)
|
||||
return {
|
||||
"request_id": context.request_id,
|
||||
"data": script_payload(script, storage_data),
|
||||
"meta": {},
|
||||
}
|
||||
|
||||
|
||||
@router.post(
|
||||
"/api/v1/scripts/upload",
|
||||
status_code=status.HTTP_201_CREATED,
|
||||
)
|
||||
async def upload_script(
|
||||
request: Request,
|
||||
file_name: str = Query(min_length=1, max_length=255),
|
||||
parent_path: str = Query(default="", max_length=1024),
|
||||
visibility: str = Query(
|
||||
default="workspace",
|
||||
pattern="^(private|workspace|public)$",
|
||||
),
|
||||
context: RequestContext = Depends(request_context),
|
||||
session: AsyncSession = Depends(database_session),
|
||||
) -> dict[str, Any]:
|
||||
suffix = Path(file_name).suffix.lower()
|
||||
if suffix not in {".py", ".ipynb"}:
|
||||
raise HTTPException(
|
||||
status.HTTP_422_UNPROCESSABLE_ENTITY,
|
||||
"only .py and .ipynb files can be uploaded",
|
||||
)
|
||||
script_type = "notebook" if suffix == ".ipynb" else "python"
|
||||
name = safe_script_name(file_name, script_type)
|
||||
body = await request.body()
|
||||
if len(body) > 10 * 1024 * 1024:
|
||||
raise HTTPException(
|
||||
status.HTTP_413_REQUEST_ENTITY_TOO_LARGE,
|
||||
"script exceeds 10 MiB",
|
||||
)
|
||||
try:
|
||||
text = body.decode("utf-8")
|
||||
except UnicodeDecodeError as exc:
|
||||
raise HTTPException(
|
||||
status.HTTP_422_UNPROCESSABLE_ENTITY,
|
||||
"script must use UTF-8 encoding",
|
||||
) from exc
|
||||
content = validate_script_content(text, script_type)
|
||||
script, storage_data = await create_script_record(
|
||||
name=name,
|
||||
script_type=script_type,
|
||||
content=content,
|
||||
visibility=visibility,
|
||||
parent_path=parent_path,
|
||||
request=request,
|
||||
context=context,
|
||||
session=session,
|
||||
)
|
||||
return {
|
||||
"request_id": context.request_id,
|
||||
"data": script_payload(script, storage_data),
|
||||
"meta": {},
|
||||
}
|
||||
|
||||
|
||||
@router.get("/api/v1/workspace-tree")
|
||||
async def get_workspace_tree(
|
||||
context: RequestContext = Depends(request_context),
|
||||
) -> dict[str, Any]:
|
||||
root = workspace_target(context, user_relative_path(context))
|
||||
root.mkdir(parents=True, exist_ok=True)
|
||||
apply_workspace_permissions(root, 0o2770)
|
||||
directories: list[dict[str, str]] = []
|
||||
for current, names, _files in os.walk(root, followlinks=False):
|
||||
names[:] = sorted(
|
||||
name
|
||||
for name in names
|
||||
if not name.startswith(".")
|
||||
and not (Path(current) / name).is_symlink()
|
||||
)
|
||||
current_path = Path(current)
|
||||
if current_path == root:
|
||||
continue
|
||||
relative = current_path.relative_to(root).as_posix()
|
||||
parent = PurePosixPath(relative).parent.as_posix()
|
||||
directories.append(
|
||||
{
|
||||
"path": relative,
|
||||
"name": current_path.name,
|
||||
"parent_path": "" if parent == "." else parent,
|
||||
}
|
||||
)
|
||||
return {
|
||||
"request_id": context.request_id,
|
||||
"data": {"directories": directories},
|
||||
"meta": {"directory_count": len(directories)},
|
||||
}
|
||||
|
||||
|
||||
@router.post(
|
||||
"/api/v1/workspace-directories",
|
||||
status_code=status.HTTP_201_CREATED,
|
||||
)
|
||||
async def create_workspace_directory(
|
||||
payload: CreateWorkspaceDirectoryRequest,
|
||||
context: RequestContext = Depends(request_context),
|
||||
) -> dict[str, Any]:
|
||||
name = safe_directory_name(payload.directory_name)
|
||||
parent = normalize_user_path(payload.parent_path)
|
||||
child_path = f"{parent}/{name}" if parent else name
|
||||
target = workspace_target(context, user_relative_path(context, child_path))
|
||||
parent_target = target.parent
|
||||
user_root = workspace_target(context, user_relative_path(context))
|
||||
if (
|
||||
not parent_target.is_dir()
|
||||
or (
|
||||
parent_target != user_root
|
||||
and user_root not in parent_target.parents
|
||||
)
|
||||
):
|
||||
raise HTTPException(
|
||||
status.HTTP_404_NOT_FOUND,
|
||||
"parent directory not found",
|
||||
)
|
||||
if target.exists():
|
||||
raise HTTPException(
|
||||
status.HTTP_409_CONFLICT,
|
||||
"a file or directory with the same path already exists",
|
||||
)
|
||||
target.mkdir(parents=False)
|
||||
apply_workspace_permissions(target, 0o2770)
|
||||
return {
|
||||
"request_id": context.request_id,
|
||||
"data": {
|
||||
"path": child_path,
|
||||
"name": name,
|
||||
"parent_path": parent,
|
||||
},
|
||||
"meta": {},
|
||||
}
|
||||
|
||||
|
||||
@router.delete("/api/v1/workspace-directories")
|
||||
async def delete_workspace_directory(
|
||||
request: Request,
|
||||
path: str = Query(min_length=1, max_length=1024),
|
||||
context: RequestContext = Depends(request_context),
|
||||
session: AsyncSession = Depends(database_session),
|
||||
) -> dict[str, Any]:
|
||||
directory_path = normalize_user_path(path, allow_empty=False)
|
||||
relative_prefix = user_relative_path(context, directory_path)
|
||||
target = workspace_target(context, relative_prefix)
|
||||
if not target.is_dir() or target.is_symlink():
|
||||
raise HTTPException(
|
||||
status.HTTP_404_NOT_FOUND,
|
||||
"directory not found",
|
||||
)
|
||||
|
||||
rows = (
|
||||
await session.execute(
|
||||
select(Scripts, StorageObjects)
|
||||
.join(
|
||||
StorageObjects,
|
||||
StorageObjects.storage_object_id == Scripts.current_object_id,
|
||||
)
|
||||
.where(
|
||||
Scripts.workspace_id == context.workspace.workspace_id,
|
||||
Scripts.owner_user_id == context.user.user_id,
|
||||
Scripts.status == "active",
|
||||
StorageObjects.relative_path.startswith(
|
||||
f"{relative_prefix}/"
|
||||
),
|
||||
)
|
||||
)
|
||||
).all()
|
||||
await require_no_active_edit_session(
|
||||
session,
|
||||
[script.current_object_id for script, _storage in rows],
|
||||
)
|
||||
for script, _storage in rows:
|
||||
await request.app.state.storage_client.delete_object(
|
||||
script.current_object_id
|
||||
)
|
||||
script.status = "deleted"
|
||||
script.deleted_at = datetime.now(UTC).replace(tzinfo=None)
|
||||
shutil.rmtree(target)
|
||||
return {
|
||||
"request_id": context.request_id,
|
||||
"data": {
|
||||
"path": directory_path,
|
||||
"status": "deleted",
|
||||
"deleted_scripts": len(rows),
|
||||
"versions_preserved": True,
|
||||
},
|
||||
"meta": {},
|
||||
}
|
||||
|
||||
|
||||
@router.get("/api/v1/scripts")
|
||||
async def list_scripts(
|
||||
context: RequestContext = Depends(request_context),
|
||||
session: AsyncSession = Depends(database_session),
|
||||
) -> dict[str, Any]:
|
||||
statement = (
|
||||
select(Scripts, StorageObjects)
|
||||
.join(
|
||||
StorageObjects,
|
||||
StorageObjects.storage_object_id == Scripts.current_object_id,
|
||||
)
|
||||
.where(
|
||||
Scripts.workspace_id == context.workspace.workspace_id,
|
||||
Scripts.status == "active",
|
||||
)
|
||||
.order_by(Scripts.updated_at.desc())
|
||||
)
|
||||
rows = (await session.execute(statement)).all()
|
||||
return {
|
||||
"request_id": context.request_id,
|
||||
"data": [
|
||||
script_payload(script, storage_object)
|
||||
for script, storage_object in rows
|
||||
],
|
||||
"meta": {"count": len(rows)},
|
||||
}
|
||||
|
||||
|
||||
@router.get("/api/v1/scripts/{script_id}")
|
||||
async def get_script(
|
||||
script_id: str,
|
||||
context: RequestContext = Depends(request_context),
|
||||
session: AsyncSession = Depends(database_session),
|
||||
) -> dict[str, Any]:
|
||||
script, storage_object = await get_script_row(
|
||||
script_id,
|
||||
context,
|
||||
session,
|
||||
)
|
||||
return {
|
||||
"request_id": context.request_id,
|
||||
"data": script_payload(script, storage_object),
|
||||
"meta": {},
|
||||
}
|
||||
|
||||
|
||||
@router.put("/api/v1/scripts/{script_id}")
|
||||
async def update_script(
|
||||
script_id: str,
|
||||
payload: UpdateScriptRequest,
|
||||
request: Request,
|
||||
context: RequestContext = Depends(request_context),
|
||||
session: AsyncSession = Depends(database_session),
|
||||
) -> dict[str, Any]:
|
||||
script, storage_object = await get_script_row(
|
||||
script_id,
|
||||
context,
|
||||
session,
|
||||
for_update=True,
|
||||
)
|
||||
if (
|
||||
script.owner_user_id != context.user.user_id
|
||||
and not context.is_admin
|
||||
):
|
||||
raise HTTPException(
|
||||
status.HTTP_403_FORBIDDEN,
|
||||
"script can only be changed by its owner or an administrator",
|
||||
)
|
||||
content = validate_script_content(payload.content, script.script_type)
|
||||
if not storage_object.relative_path:
|
||||
raise HTTPException(
|
||||
status.HTTP_409_CONFLICT,
|
||||
"script has no workspace path",
|
||||
)
|
||||
target = workspace_target(context, storage_object.relative_path)
|
||||
atomic_write(target, content)
|
||||
storage_data = await request.app.state.storage_client.register_workspace_object(
|
||||
{
|
||||
"workspace_id": context.workspace.workspace_id,
|
||||
"user_id": script.owner_user_id,
|
||||
"relative_path": storage_object.relative_path,
|
||||
"usage_type": "working_copy",
|
||||
"visibility": script.visibility,
|
||||
}
|
||||
)
|
||||
storage_object.content_hash = storage_data["content_hash"]
|
||||
storage_object.size_bytes = storage_data["size_bytes"]
|
||||
script.updated_at = datetime.now(UTC).replace(tzinfo=None)
|
||||
return {
|
||||
"request_id": context.request_id,
|
||||
"data": script_payload(script, storage_object),
|
||||
"meta": {},
|
||||
}
|
||||
|
||||
|
||||
@router.delete("/api/v1/scripts/{script_id}")
|
||||
async def delete_script(
|
||||
script_id: str,
|
||||
request: Request,
|
||||
context: RequestContext = Depends(request_context),
|
||||
session: AsyncSession = Depends(database_session),
|
||||
) -> dict[str, Any]:
|
||||
script, storage_object = await get_script_row(
|
||||
script_id,
|
||||
context,
|
||||
session,
|
||||
for_update=True,
|
||||
)
|
||||
if (
|
||||
script.owner_user_id != context.user.user_id
|
||||
and not context.is_admin
|
||||
):
|
||||
raise HTTPException(
|
||||
status.HTTP_403_FORBIDDEN,
|
||||
"script can only be deleted by its owner or an administrator",
|
||||
)
|
||||
await require_no_active_edit_session(
|
||||
session,
|
||||
[script.current_object_id],
|
||||
)
|
||||
if storage_object.relative_path:
|
||||
target = workspace_target(context, storage_object.relative_path)
|
||||
if target.is_file():
|
||||
target.unlink()
|
||||
await request.app.state.storage_client.delete_object(
|
||||
script.current_object_id
|
||||
)
|
||||
script.status = "deleted"
|
||||
script.deleted_at = datetime.now(UTC).replace(tzinfo=None)
|
||||
return {
|
||||
"request_id": context.request_id,
|
||||
"data": {
|
||||
"script_id": script.script_id,
|
||||
"status": script.status,
|
||||
"versions_preserved": True,
|
||||
},
|
||||
"meta": {},
|
||||
}
|
||||
|
||||
|
||||
@router.post(
|
||||
"/api/v1/scripts/{script_id}/versions",
|
||||
status_code=status.HTTP_201_CREATED,
|
||||
)
|
||||
async def publish_version(
|
||||
script_id: str,
|
||||
payload: PublishVersionRequest,
|
||||
request: Request,
|
||||
context: RequestContext = Depends(request_context),
|
||||
session: AsyncSession = Depends(database_session),
|
||||
) -> dict[str, Any]:
|
||||
script, source_object = await get_script_row(
|
||||
script_id,
|
||||
context,
|
||||
session,
|
||||
for_update=True,
|
||||
)
|
||||
if (
|
||||
script.owner_user_id != context.user.user_id
|
||||
and not context.is_admin
|
||||
):
|
||||
raise HTTPException(
|
||||
status.HTTP_403_FORBIDDEN,
|
||||
"script can only be published by its owner or an administrator",
|
||||
)
|
||||
if (
|
||||
payload.source_object_id
|
||||
and payload.source_object_id != script.current_object_id
|
||||
):
|
||||
raise HTTPException(
|
||||
status.HTTP_412_PRECONDITION_FAILED,
|
||||
"source_object_id is not the current working copy",
|
||||
)
|
||||
if not source_object.relative_path:
|
||||
raise HTTPException(
|
||||
status.HTTP_409_CONFLICT,
|
||||
"script has no workspace path",
|
||||
)
|
||||
target = workspace_target(context, source_object.relative_path)
|
||||
if not target.is_file():
|
||||
raise HTTPException(
|
||||
status.HTTP_409_CONFLICT,
|
||||
"script working copy is missing",
|
||||
)
|
||||
content = target.read_bytes()
|
||||
content_hash = hashlib.sha256(content).hexdigest()
|
||||
existing = await session.scalar(
|
||||
select(Versions).where(
|
||||
Versions.script_id == script.script_id,
|
||||
Versions.content_hash == content_hash,
|
||||
)
|
||||
)
|
||||
if existing is not None:
|
||||
return {
|
||||
"request_id": context.request_id,
|
||||
"data": version_payload(existing),
|
||||
"meta": {"reused": True},
|
||||
}
|
||||
content_type = (
|
||||
mimetypes.guess_type(script.script_name)[0]
|
||||
or "application/octet-stream"
|
||||
)
|
||||
artifact = await request.app.state.storage_client.create_server_object(
|
||||
workspace_id=context.workspace.workspace_id,
|
||||
user_id=context.user.user_id,
|
||||
usage_type="version_artifact",
|
||||
file_name=script.script_name,
|
||||
content_type=content_type,
|
||||
content=content,
|
||||
visibility=payload.visibility,
|
||||
is_immutable=True,
|
||||
idempotency_key=f"version:{script.script_id}:{content_hash}",
|
||||
)
|
||||
current_max = await session.scalar(
|
||||
select(func.max(Versions.version_no)).where(
|
||||
Versions.script_id == script.script_id
|
||||
)
|
||||
)
|
||||
version_no = int(current_max or 0) + 1
|
||||
version = Versions(
|
||||
versions_id=new_ulid(),
|
||||
workspace_id=context.workspace.workspace_id,
|
||||
script_id=script.script_id,
|
||||
source_object_id=script.current_object_id,
|
||||
artifact_object_id=artifact["storage_object_id"],
|
||||
version_no=version_no,
|
||||
version_label=f"v{version_no}.0",
|
||||
source_path=source_object.relative_path,
|
||||
artifact_path=artifact["storage_uri"],
|
||||
content_hash=content_hash,
|
||||
file_size_bytes=len(content),
|
||||
visibility=payload.visibility,
|
||||
release_note=payload.release_note,
|
||||
created_by=context.user.user_id,
|
||||
)
|
||||
session.add(version)
|
||||
await session.flush()
|
||||
await session.refresh(version)
|
||||
return {
|
||||
"request_id": context.request_id,
|
||||
"data": version_payload(version),
|
||||
"meta": {"reused": False},
|
||||
}
|
||||
|
||||
|
||||
@router.get("/api/v1/scripts/{script_id}/versions")
|
||||
async def list_versions(
|
||||
script_id: str,
|
||||
context: RequestContext = Depends(request_context),
|
||||
session: AsyncSession = Depends(database_session),
|
||||
) -> dict[str, Any]:
|
||||
await get_script_row(script_id, context, session)
|
||||
versions = (
|
||||
await session.scalars(
|
||||
select(Versions)
|
||||
.where(Versions.script_id == script_id)
|
||||
.order_by(Versions.version_no.desc())
|
||||
)
|
||||
).all()
|
||||
return {
|
||||
"request_id": context.request_id,
|
||||
"data": [version_payload(version) for version in versions],
|
||||
"meta": {"count": len(versions)},
|
||||
}
|
||||
|
||||
|
||||
@router.get("/api/v1/versions/{versions_id}")
|
||||
async def get_version(
|
||||
versions_id: str,
|
||||
context: RequestContext = Depends(request_context),
|
||||
session: AsyncSession = Depends(database_session),
|
||||
) -> dict[str, Any]:
|
||||
version = await session.get(Versions, versions_id)
|
||||
if (
|
||||
version is None
|
||||
or version.workspace_id != context.workspace.workspace_id
|
||||
):
|
||||
raise HTTPException(status.HTTP_404_NOT_FOUND, "version not found")
|
||||
return {
|
||||
"request_id": context.request_id,
|
||||
"data": version_payload(version),
|
||||
"meta": {},
|
||||
}
|
||||
|
||||
|
||||
@router.delete("/api/v1/versions/{versions_id}")
|
||||
async def delete_version(
|
||||
versions_id: str,
|
||||
context: RequestContext = Depends(request_context),
|
||||
session: AsyncSession = Depends(database_session),
|
||||
) -> dict[str, Any]:
|
||||
version = await session.get(Versions, versions_id)
|
||||
if (
|
||||
version is None
|
||||
or version.workspace_id != context.workspace.workspace_id
|
||||
):
|
||||
raise HTTPException(status.HTTP_404_NOT_FOUND, "稳定版本不存在")
|
||||
if (
|
||||
version.created_by != context.user.user_id
|
||||
and not context.is_admin
|
||||
):
|
||||
raise HTTPException(
|
||||
status.HTTP_403_FORBIDDEN,
|
||||
"仅稳定版本发布者或管理员可以删除",
|
||||
)
|
||||
|
||||
version.schedule_hidden_at = datetime.now(UTC).replace(tzinfo=None)
|
||||
await session.flush()
|
||||
return {
|
||||
"request_id": context.request_id,
|
||||
"data": {
|
||||
"versions_id": versions_id,
|
||||
"deleted": True,
|
||||
"artifact_preserved": True,
|
||||
},
|
||||
"meta": {
|
||||
"message": "已从调度稳定版本列表移除;稳定版本和历史记录保持不变",
|
||||
},
|
||||
}
|
||||
|
||||
|
||||
@router.post("/api/v1/versions/{versions_id}/download-url")
|
||||
async def version_download_url(
|
||||
versions_id: str,
|
||||
payload: DownloadUrlRequest,
|
||||
request: Request,
|
||||
context: RequestContext = Depends(request_context),
|
||||
session: AsyncSession = Depends(database_session),
|
||||
) -> dict[str, Any]:
|
||||
version = await session.get(Versions, versions_id)
|
||||
if (
|
||||
version is None
|
||||
or version.workspace_id != context.workspace.workspace_id
|
||||
):
|
||||
raise HTTPException(status.HTTP_404_NOT_FOUND, "version not found")
|
||||
data = await request.app.state.storage_client.create_download_url(
|
||||
version.artifact_object_id,
|
||||
payload.expires_seconds,
|
||||
)
|
||||
return {"request_id": context.request_id, "data": data, "meta": {}}
|
||||
@@ -0,0 +1,681 @@
|
||||
from __future__ import annotations
|
||||
|
||||
import asyncio
|
||||
import base64
|
||||
import binascii
|
||||
import hashlib
|
||||
import mimetypes
|
||||
import os
|
||||
import secrets
|
||||
from contextlib import asynccontextmanager
|
||||
from datetime import UTC, datetime, timedelta
|
||||
from pathlib import Path, PurePosixPath
|
||||
from typing import Any, AsyncIterator
|
||||
|
||||
from fastapi import Depends, Header, HTTPException, Request, status
|
||||
from sqlalchemy import select
|
||||
from sqlalchemy.ext.asyncio import AsyncSession
|
||||
|
||||
from common.db import create_database_engine, create_session_factory
|
||||
from common.db.models import (
|
||||
StorageObjects,
|
||||
UploadSessions,
|
||||
Users,
|
||||
WorkspaceMembers,
|
||||
Workspaces,
|
||||
)
|
||||
from common.ids import new_ulid
|
||||
from common.service_app import create_service_app
|
||||
from common.storage import RustFSObjectStore
|
||||
from backend.storage_schemas import (
|
||||
CompleteUploadRequest,
|
||||
CreateUploadRequest,
|
||||
DownloadUrlRequest,
|
||||
RegisterWorkspaceObjectRequest,
|
||||
ServerObjectRequest,
|
||||
)
|
||||
|
||||
|
||||
def utcnow() -> datetime:
|
||||
return datetime.now(UTC).replace(tzinfo=None)
|
||||
|
||||
|
||||
def hash_bytes(value: str) -> bytes:
|
||||
return hashlib.sha256(value.encode("utf-8")).digest()
|
||||
|
||||
|
||||
def normalized_idempotency_key(
|
||||
workspace_id: str,
|
||||
user_id: str,
|
||||
value: str,
|
||||
) -> str:
|
||||
digest = hashlib.sha256(
|
||||
f"{workspace_id}:{user_id}:{value}".encode("utf-8")
|
||||
).hexdigest()
|
||||
return f"v1:{digest}"
|
||||
|
||||
|
||||
def safe_file_name(value: str) -> str:
|
||||
name = value.replace("\\", "/").rsplit("/", 1)[-1].strip()
|
||||
if not name or name in {".", ".."} or any(ord(char) < 32 for char in name):
|
||||
raise HTTPException(status.HTTP_422_UNPROCESSABLE_ENTITY, "invalid file_name")
|
||||
return name
|
||||
|
||||
|
||||
def storage_payload(item: StorageObjects) -> dict[str, Any]:
|
||||
return {
|
||||
"storage_object_id": item.storage_object_id,
|
||||
"workspace_id": item.workspace_id,
|
||||
"owner_user_id": item.owner_user_id,
|
||||
"storage_backend": item.storage_backend,
|
||||
"usage_type": item.usage_type,
|
||||
"storage_uri": item.storage_uri,
|
||||
"relative_path": item.relative_path,
|
||||
"file_name": item.file_name,
|
||||
"file_extension": item.file_extension,
|
||||
"mime_type": item.mime_type,
|
||||
"size_bytes": item.size_bytes,
|
||||
"content_hash": item.content_hash,
|
||||
"visibility": item.visibility,
|
||||
"is_immutable": bool(item.is_immutable),
|
||||
"object_status": item.object_status,
|
||||
"created_at": item.created_at.isoformat(),
|
||||
"updated_at": item.updated_at.isoformat(),
|
||||
}
|
||||
|
||||
|
||||
@asynccontextmanager
|
||||
async def lifespan(app: Any) -> AsyncIterator[None]:
|
||||
database_url = os.environ["DATABASE_URL"]
|
||||
engine = create_database_engine(database_url)
|
||||
app.state.session_factory = create_session_factory(engine)
|
||||
app.state.object_store = RustFSObjectStore(
|
||||
internal_endpoint=os.getenv(
|
||||
"RUSTFS_INTERNAL_ENDPOINT",
|
||||
"http://rustfs:9000",
|
||||
),
|
||||
public_endpoint=os.getenv(
|
||||
"RUSTFS_PUBLIC_ENDPOINT",
|
||||
"http://localhost:9000",
|
||||
),
|
||||
access_key=os.environ["RUSTFS_ACCESS_KEY"],
|
||||
secret_key=os.environ["RUSTFS_SECRET_KEY"],
|
||||
)
|
||||
app.state.default_bucket = os.getenv(
|
||||
"RUSTFS_DEFAULT_BUCKET",
|
||||
"model-platform",
|
||||
)
|
||||
await asyncio.to_thread(
|
||||
app.state.object_store.ensure_bucket,
|
||||
app.state.default_bucket,
|
||||
)
|
||||
try:
|
||||
yield
|
||||
finally:
|
||||
await engine.dispose()
|
||||
|
||||
|
||||
app = create_service_app(
|
||||
os.getenv("SERVICE_NAME", "storage-api"),
|
||||
lifespan=lifespan,
|
||||
)
|
||||
|
||||
|
||||
async def database_session(request: Request) -> AsyncIterator[AsyncSession]:
|
||||
async with request.app.state.session_factory() as session:
|
||||
try:
|
||||
yield session
|
||||
await session.commit()
|
||||
except Exception:
|
||||
await session.rollback()
|
||||
raise
|
||||
|
||||
|
||||
def verify_internal_service(
|
||||
x_service_token: str = Header(alias="X-Service-Token"),
|
||||
) -> None:
|
||||
expected = os.environ.get("INTERNAL_SERVICE_TOKEN", "")
|
||||
if not expected or not secrets.compare_digest(expected, x_service_token):
|
||||
raise HTTPException(
|
||||
status.HTTP_401_UNAUTHORIZED,
|
||||
"invalid internal service identity",
|
||||
)
|
||||
|
||||
|
||||
async def require_workspace_member(
|
||||
session: AsyncSession,
|
||||
workspace_id: str,
|
||||
user_id: str,
|
||||
) -> Workspaces:
|
||||
statement = (
|
||||
select(Workspaces)
|
||||
.join(
|
||||
WorkspaceMembers,
|
||||
WorkspaceMembers.workspace_id == Workspaces.workspace_id,
|
||||
)
|
||||
.join(Users, Users.user_id == WorkspaceMembers.user_id)
|
||||
.where(
|
||||
Workspaces.workspace_id == workspace_id,
|
||||
Workspaces.status == "active",
|
||||
WorkspaceMembers.user_id == user_id,
|
||||
WorkspaceMembers.member_status == "active",
|
||||
Users.status == "active",
|
||||
)
|
||||
)
|
||||
workspace = await session.scalar(statement)
|
||||
if workspace is None:
|
||||
raise HTTPException(
|
||||
status.HTTP_403_FORBIDDEN,
|
||||
"user is not an active workspace member",
|
||||
)
|
||||
return workspace
|
||||
|
||||
|
||||
async def create_upload_record(
|
||||
payload: CreateUploadRequest,
|
||||
session: AsyncSession,
|
||||
request: Request,
|
||||
) -> dict[str, Any]:
|
||||
workspace = await require_workspace_member(
|
||||
session,
|
||||
payload.workspace_id,
|
||||
payload.user_id,
|
||||
)
|
||||
stored_key = normalized_idempotency_key(
|
||||
payload.workspace_id,
|
||||
payload.user_id,
|
||||
payload.idempotency_key,
|
||||
)
|
||||
existing = await session.scalar(
|
||||
select(UploadSessions).where(
|
||||
UploadSessions.idempotency_key == stored_key
|
||||
)
|
||||
)
|
||||
if existing is not None:
|
||||
if (
|
||||
existing.workspace_id != payload.workspace_id
|
||||
or existing.user_id != payload.user_id
|
||||
or existing.expected_size_bytes != payload.expected_size_bytes
|
||||
or existing.expected_hash != payload.expected_hash
|
||||
or existing.content_type != payload.content_type
|
||||
):
|
||||
raise HTTPException(
|
||||
status.HTTP_409_CONFLICT,
|
||||
"idempotency key was used with different upload metadata",
|
||||
)
|
||||
upload = existing
|
||||
else:
|
||||
upload_id = new_ulid()
|
||||
file_name = safe_file_name(payload.file_name)
|
||||
bucket_name = (
|
||||
workspace.artifact_bucket or request.app.state.default_bucket
|
||||
)
|
||||
object_key = (
|
||||
f"workspaces/{payload.workspace_id}/"
|
||||
f"{payload.usage_type}/{upload_id}/{file_name}"
|
||||
)
|
||||
upload = UploadSessions(
|
||||
upload_id=upload_id,
|
||||
workspace_id=payload.workspace_id,
|
||||
user_id=payload.user_id,
|
||||
idempotency_key=stored_key,
|
||||
bucket_name=bucket_name,
|
||||
object_key=object_key,
|
||||
object_key_hash=hash_bytes(object_key),
|
||||
upload_status="created",
|
||||
expires_at=utcnow() + timedelta(minutes=15),
|
||||
expected_size_bytes=payload.expected_size_bytes,
|
||||
expected_hash=payload.expected_hash,
|
||||
content_type=payload.content_type,
|
||||
)
|
||||
session.add(upload)
|
||||
await session.flush()
|
||||
|
||||
if upload.upload_status == "completed" and upload.storage_object_id:
|
||||
storage_object = await session.get(
|
||||
StorageObjects,
|
||||
upload.storage_object_id,
|
||||
)
|
||||
return {
|
||||
"upload_id": upload.upload_id,
|
||||
"status": upload.upload_status,
|
||||
"storage_object": (
|
||||
storage_payload(storage_object) if storage_object else None
|
||||
),
|
||||
}
|
||||
if upload.upload_status not in {"created", "uploading"}:
|
||||
raise HTTPException(
|
||||
status.HTTP_409_CONFLICT,
|
||||
f"upload cannot continue from status {upload.upload_status}",
|
||||
)
|
||||
|
||||
url, headers = request.app.state.object_store.presign_put(
|
||||
bucket_name=upload.bucket_name,
|
||||
object_key=upload.object_key,
|
||||
content_type=upload.content_type or "application/octet-stream",
|
||||
expected_hash=upload.expected_hash,
|
||||
expires_seconds=900,
|
||||
public=payload.url_scope == "public",
|
||||
)
|
||||
return {
|
||||
"upload_id": upload.upload_id,
|
||||
"status": upload.upload_status,
|
||||
"method": "PUT",
|
||||
"presigned_url": url,
|
||||
"required_headers": headers,
|
||||
"expires_at": upload.expires_at.isoformat(),
|
||||
}
|
||||
|
||||
|
||||
async def complete_upload_record(
|
||||
upload_id: str,
|
||||
payload: CompleteUploadRequest,
|
||||
session: AsyncSession,
|
||||
request: Request,
|
||||
) -> StorageObjects:
|
||||
upload = await session.scalar(
|
||||
select(UploadSessions)
|
||||
.where(UploadSessions.upload_id == upload_id)
|
||||
.with_for_update()
|
||||
)
|
||||
if upload is None:
|
||||
raise HTTPException(status.HTTP_404_NOT_FOUND, "upload not found")
|
||||
if upload.upload_status == "completed" and upload.storage_object_id:
|
||||
item = await session.get(StorageObjects, upload.storage_object_id)
|
||||
if item is None:
|
||||
raise HTTPException(
|
||||
status.HTTP_409_CONFLICT,
|
||||
"completed upload has no storage object",
|
||||
)
|
||||
return item
|
||||
if upload.upload_status not in {"created", "uploading"}:
|
||||
raise HTTPException(
|
||||
status.HTTP_409_CONFLICT,
|
||||
f"upload cannot be completed from status {upload.upload_status}",
|
||||
)
|
||||
if upload.expires_at < utcnow():
|
||||
upload.upload_status = "expired"
|
||||
raise HTTPException(status.HTTP_409_CONFLICT, "upload expired")
|
||||
|
||||
try:
|
||||
head = await asyncio.to_thread(
|
||||
request.app.state.object_store.head,
|
||||
bucket_name=upload.bucket_name,
|
||||
object_key=upload.object_key,
|
||||
)
|
||||
except Exception as exc:
|
||||
raise HTTPException(
|
||||
status.HTTP_409_CONFLICT,
|
||||
"uploaded object is not available",
|
||||
) from exc
|
||||
|
||||
actual_size = int(head.get("ContentLength", 0))
|
||||
if (
|
||||
upload.expected_size_bytes is not None
|
||||
and actual_size != upload.expected_size_bytes
|
||||
):
|
||||
upload.upload_status = "failed"
|
||||
raise HTTPException(
|
||||
status.HTTP_409_CONFLICT,
|
||||
"uploaded object size does not match expected_size_bytes",
|
||||
)
|
||||
actual_content_type = str(
|
||||
head.get("ContentType") or "application/octet-stream"
|
||||
)
|
||||
if upload.content_type and actual_content_type != upload.content_type:
|
||||
upload.upload_status = "failed"
|
||||
raise HTTPException(
|
||||
status.HTTP_409_CONFLICT,
|
||||
"uploaded object content type does not match",
|
||||
)
|
||||
metadata = {
|
||||
str(key).lower(): str(value).lower()
|
||||
for key, value in dict(head.get("Metadata") or {}).items()
|
||||
}
|
||||
actual_hash = metadata.get("sha256")
|
||||
if not actual_hash:
|
||||
actual_hash = await asyncio.to_thread(
|
||||
request.app.state.object_store.sha256,
|
||||
bucket_name=upload.bucket_name,
|
||||
object_key=upload.object_key,
|
||||
)
|
||||
if upload.expected_hash and actual_hash != upload.expected_hash:
|
||||
upload.upload_status = "failed"
|
||||
raise HTTPException(
|
||||
status.HTTP_409_CONFLICT,
|
||||
"uploaded object hash does not match expected_hash",
|
||||
)
|
||||
|
||||
file_name = upload.object_key.rsplit("/", 1)[-1]
|
||||
item = StorageObjects(
|
||||
storage_object_id=new_ulid(),
|
||||
workspace_id=upload.workspace_id,
|
||||
owner_user_id=upload.user_id,
|
||||
object_type="file",
|
||||
usage_type=payload.usage_type,
|
||||
storage_backend="rustfs",
|
||||
bucket_name=upload.bucket_name,
|
||||
object_key=upload.object_key,
|
||||
object_key_hash=upload.object_key_hash,
|
||||
storage_uri=f"s3://{upload.bucket_name}/{upload.object_key}",
|
||||
file_name=file_name,
|
||||
file_extension=Path(file_name).suffix.lower() or None,
|
||||
mime_type=actual_content_type,
|
||||
size_bytes=actual_size,
|
||||
content_hash=actual_hash,
|
||||
object_etag=str(head.get("ETag", "")).strip('"') or None,
|
||||
visibility=payload.visibility,
|
||||
is_immutable=int(payload.is_immutable),
|
||||
object_status="available",
|
||||
created_by=upload.user_id,
|
||||
)
|
||||
session.add(item)
|
||||
await session.flush()
|
||||
await session.refresh(item)
|
||||
upload.storage_object_id = item.storage_object_id
|
||||
upload.upload_status = "completed"
|
||||
upload.completed_at = utcnow()
|
||||
return item
|
||||
|
||||
|
||||
@app.post("/internal/v1/uploads", dependencies=[Depends(verify_internal_service)])
|
||||
async def create_upload(
|
||||
payload: CreateUploadRequest,
|
||||
request: Request,
|
||||
session: AsyncSession = Depends(database_session),
|
||||
) -> dict[str, Any]:
|
||||
return {
|
||||
"data": await create_upload_record(payload, session, request),
|
||||
}
|
||||
|
||||
|
||||
@app.post(
|
||||
"/internal/v1/uploads/{upload_id}/complete",
|
||||
dependencies=[Depends(verify_internal_service)],
|
||||
)
|
||||
async def complete_upload(
|
||||
upload_id: str,
|
||||
payload: CompleteUploadRequest,
|
||||
request: Request,
|
||||
session: AsyncSession = Depends(database_session),
|
||||
) -> dict[str, Any]:
|
||||
item = await complete_upload_record(
|
||||
upload_id,
|
||||
payload,
|
||||
session,
|
||||
request,
|
||||
)
|
||||
return {"data": storage_payload(item)}
|
||||
|
||||
|
||||
@app.post(
|
||||
"/internal/v1/uploads/{upload_id}/abort",
|
||||
dependencies=[Depends(verify_internal_service)],
|
||||
)
|
||||
async def abort_upload(
|
||||
upload_id: str,
|
||||
request: Request,
|
||||
session: AsyncSession = Depends(database_session),
|
||||
) -> dict[str, Any]:
|
||||
upload = await session.scalar(
|
||||
select(UploadSessions)
|
||||
.where(UploadSessions.upload_id == upload_id)
|
||||
.with_for_update()
|
||||
)
|
||||
if upload is None:
|
||||
raise HTTPException(status.HTTP_404_NOT_FOUND, "upload not found")
|
||||
if upload.upload_status == "completed":
|
||||
raise HTTPException(
|
||||
status.HTTP_409_CONFLICT,
|
||||
"completed upload cannot be aborted",
|
||||
)
|
||||
if upload.upload_status != "aborted":
|
||||
await asyncio.to_thread(
|
||||
request.app.state.object_store.delete,
|
||||
bucket_name=upload.bucket_name,
|
||||
object_key=upload.object_key,
|
||||
)
|
||||
upload.upload_status = "aborted"
|
||||
return {"data": {"upload_id": upload_id, "status": "aborted"}}
|
||||
|
||||
|
||||
@app.post(
|
||||
"/internal/v1/objects",
|
||||
dependencies=[Depends(verify_internal_service)],
|
||||
)
|
||||
async def create_server_object(
|
||||
payload: ServerObjectRequest,
|
||||
request: Request,
|
||||
session: AsyncSession = Depends(database_session),
|
||||
) -> dict[str, Any]:
|
||||
try:
|
||||
content = base64.b64decode(payload.content_base64, validate=True)
|
||||
except (binascii.Error, ValueError) as exc:
|
||||
raise HTTPException(
|
||||
status.HTTP_422_UNPROCESSABLE_ENTITY,
|
||||
"content_base64 is invalid",
|
||||
) from exc
|
||||
if len(content) > 100 * 1024 * 1024:
|
||||
raise HTTPException(
|
||||
status.HTTP_413_REQUEST_ENTITY_TOO_LARGE,
|
||||
"object exceeds 100 MiB server-side upload limit",
|
||||
)
|
||||
content_hash = hashlib.sha256(content).hexdigest()
|
||||
upload_result = await create_upload_record(
|
||||
CreateUploadRequest(
|
||||
workspace_id=payload.workspace_id,
|
||||
user_id=payload.user_id,
|
||||
usage_type=payload.usage_type,
|
||||
file_name=payload.file_name,
|
||||
content_type=payload.content_type,
|
||||
expected_size_bytes=len(content),
|
||||
expected_hash=content_hash,
|
||||
idempotency_key=payload.idempotency_key,
|
||||
url_scope="internal",
|
||||
),
|
||||
session,
|
||||
request,
|
||||
)
|
||||
if upload_result.get("status") == "completed":
|
||||
return {"data": upload_result["storage_object"], "meta": {"reused": True}}
|
||||
|
||||
upload = await session.get(UploadSessions, upload_result["upload_id"])
|
||||
if upload is None:
|
||||
raise HTTPException(
|
||||
status.HTTP_500_INTERNAL_SERVER_ERROR,
|
||||
"upload record disappeared",
|
||||
)
|
||||
await asyncio.to_thread(
|
||||
request.app.state.object_store.put_bytes,
|
||||
bucket_name=upload.bucket_name,
|
||||
object_key=upload.object_key,
|
||||
content=content,
|
||||
content_type=payload.content_type,
|
||||
content_hash=content_hash,
|
||||
)
|
||||
item = await complete_upload_record(
|
||||
upload.upload_id,
|
||||
CompleteUploadRequest(
|
||||
usage_type=payload.usage_type,
|
||||
visibility=payload.visibility,
|
||||
is_immutable=payload.is_immutable,
|
||||
),
|
||||
session,
|
||||
request,
|
||||
)
|
||||
return {"data": storage_payload(item), "meta": {"reused": False}}
|
||||
|
||||
|
||||
@app.post(
|
||||
"/internal/v1/workspace-objects",
|
||||
dependencies=[Depends(verify_internal_service)],
|
||||
)
|
||||
async def register_workspace_object(
|
||||
payload: RegisterWorkspaceObjectRequest,
|
||||
session: AsyncSession = Depends(database_session),
|
||||
) -> dict[str, Any]:
|
||||
workspace = await require_workspace_member(
|
||||
session,
|
||||
payload.workspace_id,
|
||||
payload.user_id,
|
||||
)
|
||||
pure_path = PurePosixPath(payload.relative_path.replace("\\", "/"))
|
||||
if (
|
||||
pure_path.is_absolute()
|
||||
or not pure_path.parts
|
||||
or any(part in {"", ".", ".."} for part in pure_path.parts)
|
||||
):
|
||||
raise HTTPException(
|
||||
status.HTTP_422_UNPROCESSABLE_ENTITY,
|
||||
"invalid workspace relative_path",
|
||||
)
|
||||
relative_path = pure_path.as_posix()
|
||||
workspace_root = Path(
|
||||
os.getenv("WORKSPACE_ROOT", "/workspace/workspaces")
|
||||
).resolve()
|
||||
scoped_root = (workspace_root / workspace.workspace_code).resolve()
|
||||
target = (scoped_root / Path(*pure_path.parts)).resolve()
|
||||
if scoped_root != target and scoped_root not in target.parents:
|
||||
raise HTTPException(
|
||||
status.HTTP_422_UNPROCESSABLE_ENTITY,
|
||||
"workspace path escapes its root",
|
||||
)
|
||||
if not target.is_file():
|
||||
raise HTTPException(
|
||||
status.HTTP_404_NOT_FOUND,
|
||||
"workspace file does not exist",
|
||||
)
|
||||
content = await asyncio.to_thread(target.read_bytes)
|
||||
content_hash = hashlib.sha256(content).hexdigest()
|
||||
stat_result = target.stat()
|
||||
path_digest = hash_bytes(relative_path)
|
||||
item = await session.scalar(
|
||||
select(StorageObjects).where(
|
||||
StorageObjects.workspace_id == payload.workspace_id,
|
||||
StorageObjects.storage_backend == "workspace_fs",
|
||||
StorageObjects.path_hash == path_digest,
|
||||
)
|
||||
)
|
||||
reused = item is not None
|
||||
if item is None:
|
||||
item = StorageObjects(
|
||||
storage_object_id=new_ulid(),
|
||||
workspace_id=payload.workspace_id,
|
||||
owner_user_id=payload.user_id,
|
||||
object_type="file",
|
||||
usage_type=payload.usage_type,
|
||||
storage_backend="workspace_fs",
|
||||
relative_path=relative_path,
|
||||
path_hash=path_digest,
|
||||
storage_uri=target.as_uri(),
|
||||
file_name=target.name,
|
||||
visibility=payload.visibility,
|
||||
is_immutable=0,
|
||||
object_status="available",
|
||||
created_by=payload.user_id,
|
||||
)
|
||||
session.add(item)
|
||||
elif item.is_immutable:
|
||||
raise HTTPException(
|
||||
status.HTTP_409_CONFLICT,
|
||||
"immutable workspace object cannot be updated",
|
||||
)
|
||||
elif item.owner_user_id != payload.user_id:
|
||||
raise HTTPException(
|
||||
status.HTTP_403_FORBIDDEN,
|
||||
"workspace object belongs to another user",
|
||||
)
|
||||
item.usage_type = payload.usage_type
|
||||
item.file_name = target.name
|
||||
item.file_extension = target.suffix.lower() or None
|
||||
item.mime_type = (
|
||||
mimetypes.guess_type(target.name)[0] or "application/octet-stream"
|
||||
)
|
||||
item.size_bytes = stat_result.st_size
|
||||
item.content_hash = content_hash
|
||||
item.visibility = payload.visibility
|
||||
item.object_status = "available"
|
||||
await session.flush()
|
||||
await session.refresh(item)
|
||||
return {"data": storage_payload(item), "meta": {"reused": reused}}
|
||||
|
||||
|
||||
@app.post(
|
||||
"/internal/v1/objects/{storage_object_id}/download-url",
|
||||
dependencies=[Depends(verify_internal_service)],
|
||||
)
|
||||
async def create_download_url(
|
||||
storage_object_id: str,
|
||||
payload: DownloadUrlRequest,
|
||||
request: Request,
|
||||
session: AsyncSession = Depends(database_session),
|
||||
) -> dict[str, Any]:
|
||||
item = await session.get(StorageObjects, storage_object_id)
|
||||
if item is None or item.object_status != "available":
|
||||
raise HTTPException(status.HTTP_404_NOT_FOUND, "object not found")
|
||||
if (
|
||||
item.storage_backend != "rustfs"
|
||||
or not item.bucket_name
|
||||
or not item.object_key
|
||||
):
|
||||
raise HTTPException(
|
||||
status.HTTP_409_CONFLICT,
|
||||
"object does not support a presigned URL",
|
||||
)
|
||||
url = request.app.state.object_store.presign_get(
|
||||
bucket_name=item.bucket_name,
|
||||
object_key=item.object_key,
|
||||
file_name=item.file_name,
|
||||
expires_seconds=payload.expires_seconds,
|
||||
)
|
||||
return {
|
||||
"data": {
|
||||
"storage_object_id": item.storage_object_id,
|
||||
"presigned_url": url,
|
||||
"method": "GET",
|
||||
"expires_in_seconds": payload.expires_seconds,
|
||||
}
|
||||
}
|
||||
|
||||
|
||||
@app.delete(
|
||||
"/internal/v1/objects/{storage_object_id}",
|
||||
dependencies=[Depends(verify_internal_service)],
|
||||
)
|
||||
async def delete_object(
|
||||
storage_object_id: str,
|
||||
request: Request,
|
||||
session: AsyncSession = Depends(database_session),
|
||||
) -> dict[str, Any]:
|
||||
item = await session.scalar(
|
||||
select(StorageObjects)
|
||||
.where(StorageObjects.storage_object_id == storage_object_id)
|
||||
.with_for_update()
|
||||
)
|
||||
if item is None:
|
||||
raise HTTPException(status.HTTP_404_NOT_FOUND, "object not found")
|
||||
if item.is_immutable:
|
||||
raise HTTPException(
|
||||
status.HTTP_409_CONFLICT,
|
||||
"immutable object cannot be deleted",
|
||||
)
|
||||
if item.object_status != "deleted":
|
||||
if item.storage_backend == "rustfs" and item.bucket_name and item.object_key:
|
||||
await asyncio.to_thread(
|
||||
request.app.state.object_store.delete,
|
||||
bucket_name=item.bucket_name,
|
||||
object_key=item.object_key,
|
||||
)
|
||||
item.object_status = "deleted"
|
||||
item.deleted_at = utcnow()
|
||||
return {
|
||||
"data": {
|
||||
"storage_object_id": storage_object_id,
|
||||
"object_status": item.object_status,
|
||||
}
|
||||
}
|
||||
|
||||
|
||||
@app.get("/internal/health/storage")
|
||||
async def internal_health() -> dict[str, str]:
|
||||
return {"status": "ready", "service": "storage-api"}
|
||||
@@ -0,0 +1,116 @@
|
||||
from __future__ import annotations
|
||||
|
||||
import base64
|
||||
from typing import Any
|
||||
|
||||
import httpx
|
||||
from fastapi import HTTPException
|
||||
|
||||
|
||||
class StorageClient:
|
||||
def __init__(
|
||||
self,
|
||||
client: httpx.AsyncClient,
|
||||
service_token: str,
|
||||
) -> None:
|
||||
self.client = client
|
||||
self.headers = {"X-Service-Token": service_token}
|
||||
|
||||
async def _request(
|
||||
self,
|
||||
method: str,
|
||||
path: str,
|
||||
*,
|
||||
payload: dict[str, Any] | None = None,
|
||||
) -> dict[str, Any]:
|
||||
response = await self.client.request(
|
||||
method,
|
||||
path,
|
||||
json=payload,
|
||||
headers=self.headers,
|
||||
)
|
||||
if response.is_error:
|
||||
try:
|
||||
detail = response.json().get("detail", response.text)
|
||||
except ValueError:
|
||||
detail = response.text
|
||||
raise HTTPException(response.status_code, detail)
|
||||
return response.json()
|
||||
|
||||
async def create_upload(
|
||||
self,
|
||||
payload: dict[str, Any],
|
||||
) -> dict[str, Any]:
|
||||
return (await self._request(
|
||||
"POST",
|
||||
"/internal/v1/uploads",
|
||||
payload=payload,
|
||||
))["data"]
|
||||
|
||||
async def complete_upload(
|
||||
self,
|
||||
upload_id: str,
|
||||
payload: dict[str, Any],
|
||||
) -> dict[str, Any]:
|
||||
return (await self._request(
|
||||
"POST",
|
||||
f"/internal/v1/uploads/{upload_id}/complete",
|
||||
payload=payload,
|
||||
))["data"]
|
||||
|
||||
async def register_workspace_object(
|
||||
self,
|
||||
payload: dict[str, Any],
|
||||
) -> dict[str, Any]:
|
||||
return (await self._request(
|
||||
"POST",
|
||||
"/internal/v1/workspace-objects",
|
||||
payload=payload,
|
||||
))["data"]
|
||||
|
||||
async def create_server_object(
|
||||
self,
|
||||
*,
|
||||
workspace_id: str,
|
||||
user_id: str,
|
||||
usage_type: str,
|
||||
file_name: str,
|
||||
content_type: str,
|
||||
content: bytes,
|
||||
visibility: str,
|
||||
is_immutable: bool,
|
||||
idempotency_key: str,
|
||||
) -> dict[str, Any]:
|
||||
result = await self._request(
|
||||
"POST",
|
||||
"/internal/v1/objects",
|
||||
payload={
|
||||
"workspace_id": workspace_id,
|
||||
"user_id": user_id,
|
||||
"usage_type": usage_type,
|
||||
"file_name": file_name,
|
||||
"content_type": content_type,
|
||||
"content_base64": base64.b64encode(content).decode("ascii"),
|
||||
"visibility": visibility,
|
||||
"is_immutable": is_immutable,
|
||||
"idempotency_key": idempotency_key,
|
||||
},
|
||||
)
|
||||
return result["data"]
|
||||
|
||||
async def create_download_url(
|
||||
self,
|
||||
storage_object_id: str,
|
||||
expires_seconds: int,
|
||||
) -> dict[str, Any]:
|
||||
return (await self._request(
|
||||
"POST",
|
||||
f"/internal/v1/objects/{storage_object_id}/download-url",
|
||||
payload={"expires_seconds": expires_seconds},
|
||||
))["data"]
|
||||
|
||||
async def delete_object(self, storage_object_id: str) -> dict[str, Any]:
|
||||
return (await self._request(
|
||||
"DELETE",
|
||||
f"/internal/v1/objects/{storage_object_id}",
|
||||
))["data"]
|
||||
@@ -0,0 +1,79 @@
|
||||
from __future__ import annotations
|
||||
|
||||
from typing import Literal
|
||||
|
||||
from pydantic import BaseModel, ConfigDict, Field, field_validator
|
||||
|
||||
|
||||
class StrictModel(BaseModel):
|
||||
model_config = ConfigDict(extra="forbid")
|
||||
|
||||
|
||||
class CreateUploadRequest(StrictModel):
|
||||
workspace_id: str = Field(min_length=26, max_length=26)
|
||||
user_id: str = Field(min_length=26, max_length=26)
|
||||
usage_type: Literal[
|
||||
"data_resource",
|
||||
"version_artifact",
|
||||
"snapshot",
|
||||
"run_log",
|
||||
"run_result",
|
||||
]
|
||||
file_name: str = Field(min_length=1, max_length=255)
|
||||
content_type: str = Field(min_length=1, max_length=255)
|
||||
expected_size_bytes: int = Field(ge=0, le=100 * 1024 * 1024)
|
||||
expected_hash: str | None = Field(default=None, min_length=64, max_length=64)
|
||||
idempotency_key: str = Field(min_length=8, max_length=128)
|
||||
url_scope: Literal["public", "internal"] = "public"
|
||||
|
||||
@field_validator("expected_hash")
|
||||
@classmethod
|
||||
def validate_hash(cls, value: str | None) -> str | None:
|
||||
if value is None:
|
||||
return None
|
||||
normalized = value.lower()
|
||||
if any(character not in "0123456789abcdef" for character in normalized):
|
||||
raise ValueError("expected_hash must be lowercase SHA-256 hex")
|
||||
return normalized
|
||||
|
||||
|
||||
class CompleteUploadRequest(StrictModel):
|
||||
usage_type: Literal[
|
||||
"data_resource",
|
||||
"version_artifact",
|
||||
"snapshot",
|
||||
"run_log",
|
||||
"run_result",
|
||||
]
|
||||
visibility: Literal["private", "workspace", "public"] = "private"
|
||||
is_immutable: bool = False
|
||||
|
||||
|
||||
class ServerObjectRequest(StrictModel):
|
||||
workspace_id: str = Field(min_length=26, max_length=26)
|
||||
user_id: str = Field(min_length=26, max_length=26)
|
||||
usage_type: Literal[
|
||||
"data_resource",
|
||||
"version_artifact",
|
||||
"snapshot",
|
||||
"run_log",
|
||||
"run_result",
|
||||
]
|
||||
file_name: str = Field(min_length=1, max_length=255)
|
||||
content_type: str = Field(min_length=1, max_length=255)
|
||||
content_base64: str = Field(min_length=1)
|
||||
visibility: Literal["private", "workspace", "public"] = "private"
|
||||
is_immutable: bool = False
|
||||
idempotency_key: str = Field(min_length=8, max_length=128)
|
||||
|
||||
|
||||
class RegisterWorkspaceObjectRequest(StrictModel):
|
||||
workspace_id: str = Field(min_length=26, max_length=26)
|
||||
user_id: str = Field(min_length=26, max_length=26)
|
||||
relative_path: str = Field(min_length=1, max_length=1024)
|
||||
usage_type: Literal["working_copy", "public_script"]
|
||||
visibility: Literal["private", "workspace", "public"] = "private"
|
||||
|
||||
|
||||
class DownloadUrlRequest(StrictModel):
|
||||
expires_seconds: int = Field(default=300, ge=30, le=3600)
|
||||
Reference in New Issue
Block a user