This commit is contained in:
tao.chen
2026-07-30 20:02:19 +08:00
parent c1e15758a3
commit ec53edbce5
23 changed files with 191 additions and 1085 deletions
+8 -24
View File
@@ -1,33 +1,17 @@
COMPOSE_PROJECT_NAME=model-platform-refactored COMPOSE_PROJECT_NAME=model-platform-refactored
# Local development ports # ports
NGINX_PORT=8080
GATEWAY_PORT=8081
MYSQL_PORT=3308
REDIS_PORT=6380
RUSTFS_API_PORT=9010
RUSTFS_CONSOLE_PORT=9011
BACKEND_PORT=8010 BACKEND_PORT=8010
RUNTIME_PORT=8012 RUNTIME_PORT=8012
SCHEDULE_PORT=8013 SCHEDULE_PORT=8013
# Jupyter runs on the internal Compose network only in step 14. # MySQL connection URI (async SQLAlchemy driver)
JUPYTER_IMAGE=quay.io/jupyter/base-notebook:2025-12-31 DATABASE_URL=mysql+asyncmy://model_platform:ChangeMe_MySQL_App_2026@mysql:3306/model_platform?charset=utf8mb4
JUPYTER_TOKEN=ChangeMe_Jupyter_Internal_2026
# MySQL 8 local development credentials # Object storage (S3-compatible) endpoints
MYSQL_DATABASE=model_platform RUSTFS_INTERNAL_ENDPOINT=http://rustfs:9000
MYSQL_USER=model_platform RUSTFS_PUBLIC_ENDPOINT=http://localhost:9010
MYSQL_PASSWORD=ChangeMe_MySQL_App_2026
MYSQL_ROOT_PASSWORD=ChangeMe_MySQL_Root_2026
# Redis local development credential
REDIS_PASSWORD=ChangeMe_Redis_2026
# RustFS local development image and credentials
RUSTFS_IMAGE=rustfs/rustfs:latest
RUSTFS_ACCESS_KEY=modelplatform RUSTFS_ACCESS_KEY=modelplatform
RUSTFS_SECRET_KEY=ChangeMe_RustFS_2026 RUSTFS_SECRET_KEY=ChangeMe_RustFS_2026
RUSTFS_API_PORT=9010
# Internal service authentication; replace in every non-local environment. RUSTFS_CONSOLE_PORT=9011
INTERNAL_SERVICE_TOKEN=ChangeMe_Internal_Service_2026
+4 -8
View File
@@ -3,10 +3,11 @@ from __future__ import annotations
from dataclasses import dataclass from dataclasses import dataclass
from typing import AsyncIterator from typing import AsyncIterator
from fastapi import Header, HTTPException, Request, status from fastapi import Depends, Header, HTTPException, Request, status
from sqlalchemy import select from sqlalchemy import select
from sqlalchemy.ext.asyncio import AsyncSession from sqlalchemy.ext.asyncio import AsyncSession
from common.db import session_scope
from common.db.models import ( from common.db.models import (
Roles, Roles,
Users, Users,
@@ -29,13 +30,8 @@ class RequestContext:
async def database_session(request: Request) -> AsyncIterator[AsyncSession]: async def database_session(request: Request) -> AsyncIterator[AsyncSession]:
async with request.app.state.session_factory() as session: async with session_scope(request.app.state.session_factory) as session:
try: yield session
yield session
await session.commit()
except Exception:
await session.rollback()
raise
async def request_context( async def request_context(
-110
View File
@@ -1,110 +0,0 @@
from __future__ import annotations
from typing import Any, Awaitable, Callable
from fastapi import APIRouter, Depends, Request, status
from fastapi.responses import JSONResponse
from backend.dependencies import (
RequestContext,
request_context,
)
from backend.runtime_client import RuntimeClientError
from backend.schemas import FileLockTokenRequest
router = APIRouter(tags=["file-locks"])
async def runtime_response(
context: RequestContext,
operation: Callable[[], Awaitable[dict[str, Any]]],
*,
success_status: int = status.HTTP_200_OK,
) -> JSONResponse:
try:
data = await operation()
except RuntimeClientError as exc:
error = exc.detail
if not isinstance(error, dict) or "code" not in error:
error = {
"code": "RUNTIME_REQUEST_FAILED",
"message": str(error),
"retryable": exc.status_code >= 500,
"details": {},
}
return JSONResponse(
status_code=exc.status_code,
content={"request_id": context.request_id, "error": error},
)
return JSONResponse(
status_code=success_status,
content={
"request_id": context.request_id,
"data": data,
"meta": {},
},
)
@router.post(
"/api/v1/files/{storage_object_id}/lock",
status_code=status.HTTP_201_CREATED,
)
async def acquire_file_lock(
storage_object_id: str,
request: Request,
context: RequestContext = Depends(request_context),
) -> JSONResponse:
return await runtime_response(
context,
lambda: request.app.state.runtime_client.acquire_file_lock(
{
"workspace_id": context.workspace.workspace_id,
"storage_object_id": storage_object_id,
"user_id": context.user.user_id,
"request_id": context.request_id,
}
),
success_status=status.HTTP_201_CREATED,
)
@router.post("/api/v1/file-locks/{edit_session_id}/heartbeat")
async def heartbeat_file_lock(
edit_session_id: str,
payload: FileLockTokenRequest,
request: Request,
context: RequestContext = Depends(request_context),
) -> JSONResponse:
return await runtime_response(
context,
lambda: request.app.state.runtime_client.heartbeat_file_lock(
edit_session_id,
{
"workspace_id": context.workspace.workspace_id,
"user_id": context.user.user_id,
"lock_token": payload.lock_token,
},
),
)
@router.delete("/api/v1/file-locks/{edit_session_id}")
async def release_file_lock(
edit_session_id: str,
payload: FileLockTokenRequest,
request: Request,
context: RequestContext = Depends(request_context),
) -> JSONResponse:
return await runtime_response(
context,
lambda: request.app.state.runtime_client.release_file_lock(
edit_session_id,
{
"workspace_id": context.workspace.workspace_id,
"user_id": context.user.user_id,
"lock_token": payload.lock_token,
},
),
)
+2 -10
View File
@@ -13,7 +13,6 @@ from common.db import create_database_engine, create_session_factory
from common.service_app import create_service_app from common.service_app import create_service_app
from common.storage import RustFSObjectStore from common.storage import RustFSObjectStore
from backend.admin import router as admin_router from backend.admin import router as admin_router
from backend.file_locks import router as file_locks_router
from backend.jupyter import router as jupyter_router from backend.jupyter import router as jupyter_router
from backend.resources import router as resources_router from backend.resources import router as resources_router
from backend.runtime_client import RuntimeClient from backend.runtime_client import RuntimeClient
@@ -60,18 +59,12 @@ async def lifespan(app: Any) -> AsyncIterator[None]:
base_url="http://backend.internal", base_url="http://backend.internal",
timeout=httpx.Timeout(30.0), timeout=httpx.Timeout(30.0),
) )
app.state.storage_client = StorageClient( app.state.storage_client = StorageClient(storage_http_client)
storage_http_client,
os.environ["INTERNAL_SERVICE_TOKEN"],
)
runtime_http_client = httpx.AsyncClient( runtime_http_client = httpx.AsyncClient(
base_url=os.getenv("RUNTIME_API_URL", "http://runtime:8000"), base_url=os.getenv("RUNTIME_API_URL", "http://runtime:8000"),
timeout=httpx.Timeout(30.0), timeout=httpx.Timeout(30.0),
) )
app.state.runtime_client = RuntimeClient( app.state.runtime_client = RuntimeClient(runtime_http_client)
runtime_http_client,
os.environ["INTERNAL_SERVICE_TOKEN"],
)
try: try:
yield yield
finally: finally:
@@ -84,7 +77,6 @@ app = create_service_app(
os.getenv("SERVICE_NAME", "backend"), os.getenv("SERVICE_NAME", "backend"),
lifespan=lifespan, lifespan=lifespan,
) )
app.include_router(file_locks_router)
app.include_router(jupyter_router) app.include_router(jupyter_router)
app.include_router(resources_router) app.include_router(resources_router)
app.include_router(schedule_runs_router) app.include_router(schedule_runs_router)
+27 -76
View File
@@ -1,25 +1,32 @@
from __future__ import annotations from __future__ import annotations
from dataclasses import dataclass
from typing import Any from typing import Any
import httpx import httpx
from common.clients.base import BaseInternalClient, InternalClientError
@dataclass(frozen=True)
class RuntimeClientError(Exception):
status_code: int
detail: Any
class RuntimeClient: class RuntimeClientError(InternalClientError):
def __init__( """Backward-compatible alias for the runtime error type."""
self,
client: httpx.AsyncClient,
service_token: str, _RUNTIME_TRANSPORT_ERROR = RuntimeClientError(
) -> None: 503,
self.client = client {
self.headers = {"X-Service-Token": service_token} "code": "RUNTIME_UNAVAILABLE",
"message": "Runtime Manager 暂时不可用",
"retryable": True,
"details": {},
},
)
class RuntimeClient(BaseInternalClient):
error_class = RuntimeClientError
def __init__(self, client: httpx.AsyncClient) -> None:
super().__init__(client)
async def _request( async def _request(
self, self,
@@ -27,68 +34,12 @@ class RuntimeClient:
path: str, path: str,
payload: dict[str, Any], payload: dict[str, Any],
) -> dict[str, Any]: ) -> dict[str, Any]:
try: return await super()._request(
response = await self.client.request( method,
method, path,
path, payload=payload,
json=payload, on_transport_error=_RUNTIME_TRANSPORT_ERROR,
headers=self.headers, )
)
except httpx.RequestError as exc:
raise RuntimeClientError(
503,
{
"code": "RUNTIME_UNAVAILABLE",
"message": "Runtime Manager 暂时不可用",
"retryable": True,
"details": {},
},
) from exc
if response.is_error:
try:
detail = response.json().get("detail", response.text)
except ValueError:
detail = response.text
raise RuntimeClientError(response.status_code, detail)
return response.json()
async def acquire_file_lock(
self,
payload: dict[str, Any],
) -> dict[str, Any]:
return (
await self._request(
"POST",
"/internal/v1/file-locks/acquire",
payload,
)
)["data"]
async def heartbeat_file_lock(
self,
edit_session_id: str,
payload: dict[str, Any],
) -> dict[str, Any]:
return (
await self._request(
"POST",
f"/internal/v1/file-locks/{edit_session_id}/heartbeat",
payload,
)
)["data"]
async def release_file_lock(
self,
edit_session_id: str,
payload: dict[str, Any],
) -> dict[str, Any]:
return (
await self._request(
"DELETE",
f"/internal/v1/file-locks/{edit_session_id}",
payload,
)
)["data"]
async def get_workspace( async def get_workspace(
self, self,
+1 -1
View File
@@ -20,7 +20,7 @@ from backend.dependencies import (
database_session, database_session,
request_context, request_context,
) )
from backend.schedule_schemas import StrictModel from common.schemas import StrictModel
from backend.schedules import ( from backend.schedules import (
graph_rows, graph_rows,
schedule_row, schedule_row,
+3 -6
View File
@@ -1,15 +1,12 @@
from __future__ import annotations from __future__ import annotations
from datetime import datetime from datetime import datetime
from typing import Any, Literal from typing import Any
from pydantic import Field, field_validator, model_validator from pydantic import Field, field_validator, model_validator
from backend.schemas import StrictModel from common.db.models.schedules import FailurePolicy, TriggerType
from common.schemas import StrictModel
TriggerType = Literal["manual", "cron", "api"]
FailurePolicy = Literal["stop", "continue"]
def _required_text(value: str) -> str: def _required_text(value: str) -> str:
+2 -8
View File
@@ -2,11 +2,9 @@ from __future__ import annotations
from typing import Literal from typing import Literal
from pydantic import BaseModel, ConfigDict, Field, field_validator from pydantic import Field, field_validator
from common.schemas import StrictModel
class StrictModel(BaseModel):
model_config = ConfigDict(extra="forbid")
class CreateResourceUploadRequest(StrictModel): class CreateResourceUploadRequest(StrictModel):
@@ -61,7 +59,3 @@ class PublishVersionRequest(StrictModel):
class DownloadUrlRequest(StrictModel): class DownloadUrlRequest(StrictModel):
expires_seconds: int = Field(default=300, ge=30, le=3600) expires_seconds: int = Field(default=300, ge=30, le=3600)
class FileLockTokenRequest(StrictModel):
lock_token: str = Field(min_length=32, max_length=256)
+73 -162
View File
@@ -12,28 +12,26 @@ from datetime import UTC, datetime, timedelta
from pathlib import Path, PurePosixPath from pathlib import Path, PurePosixPath
from typing import Any, AsyncIterator from typing import Any, AsyncIterator
from fastapi import Depends, Header, HTTPException, Request, status from fastapi import Depends, HTTPException, Request, status
from sqlalchemy import select from sqlalchemy import select
from sqlalchemy.ext.asyncio import AsyncSession from sqlalchemy.ext.asyncio import AsyncSession
from common.db import create_database_engine, create_session_factory from common.db import create_database_engine, create_session_factory, session_scope
from common.db.models import ( from common.db.models import (
StorageObjects, StorageObjects,
UploadSessions, UploadSessions,
Users, Users,
WorkspaceMembers, WorkspaceMembers,
Workspaces, Workspaces)
)
from common.ids import new_ulid from common.ids import new_ulid
from common.service_app import create_service_app from common.service_app import create_service_app
from common.storage import RustFSObjectStore from common.storage import RustFSObjectStore
from backend.storage_schemas import ( from common.storage.schemas import (
CompleteUploadRequest, CompleteUploadRequest,
CreateUploadRequest, CreateUploadRequest,
DownloadUrlRequest, DownloadUrlRequest,
RegisterWorkspaceObjectRequest, RegisterWorkspaceObjectRequest,
ServerObjectRequest, ServerObjectRequest)
)
def utcnow() -> datetime: def utcnow() -> datetime:
@@ -47,8 +45,7 @@ def hash_bytes(value: str) -> bytes:
def normalized_idempotency_key( def normalized_idempotency_key(
workspace_id: str, workspace_id: str,
user_id: str, user_id: str,
value: str, value: str) -> str:
) -> str:
digest = hashlib.sha256( digest = hashlib.sha256(
f"{workspace_id}:{user_id}:{value}".encode("utf-8") f"{workspace_id}:{user_id}:{value}".encode("utf-8")
).hexdigest() ).hexdigest()
@@ -92,23 +89,18 @@ async def lifespan(app: Any) -> AsyncIterator[None]:
app.state.object_store = RustFSObjectStore( app.state.object_store = RustFSObjectStore(
internal_endpoint=os.getenv( internal_endpoint=os.getenv(
"RUSTFS_INTERNAL_ENDPOINT", "RUSTFS_INTERNAL_ENDPOINT",
"http://rustfs:9000", "http://rustfs:9000"),
),
public_endpoint=os.getenv( public_endpoint=os.getenv(
"RUSTFS_PUBLIC_ENDPOINT", "RUSTFS_PUBLIC_ENDPOINT",
"http://localhost:9000", "http://localhost:9000"),
),
access_key=os.environ["RUSTFS_ACCESS_KEY"], access_key=os.environ["RUSTFS_ACCESS_KEY"],
secret_key=os.environ["RUSTFS_SECRET_KEY"], secret_key=os.environ["RUSTFS_SECRET_KEY"])
)
app.state.default_bucket = os.getenv( app.state.default_bucket = os.getenv(
"RUSTFS_DEFAULT_BUCKET", "RUSTFS_DEFAULT_BUCKET",
"model-platform", "model-platform")
)
await asyncio.to_thread( await asyncio.to_thread(
app.state.object_store.ensure_bucket, app.state.object_store.ensure_bucket,
app.state.default_bucket, app.state.default_bucket)
)
try: try:
yield yield
finally: finally:
@@ -117,75 +109,51 @@ async def lifespan(app: Any) -> AsyncIterator[None]:
app = create_service_app( app = create_service_app(
os.getenv("SERVICE_NAME", "storage-api"), os.getenv("SERVICE_NAME", "storage-api"),
lifespan=lifespan, lifespan=lifespan)
)
async def database_session(request: Request) -> AsyncIterator[AsyncSession]: async def database_session(request: Request) -> AsyncIterator[AsyncSession]:
async with request.app.state.session_factory() as session: async with session_scope(request.app.state.session_factory) as session:
try: yield session
yield session
await session.commit()
except Exception:
await session.rollback()
raise
def verify_internal_service(
x_service_token: str = Header(alias="X-Service-Token"),
) -> None:
expected = os.environ.get("INTERNAL_SERVICE_TOKEN", "")
if not expected or not secrets.compare_digest(expected, x_service_token):
raise HTTPException(
status.HTTP_401_UNAUTHORIZED,
"invalid internal service identity",
)
async def require_workspace_member( async def require_workspace_member(
session: AsyncSession, session: AsyncSession,
workspace_id: str, workspace_id: str,
user_id: str, user_id: str) -> Workspaces:
) -> Workspaces:
statement = ( statement = (
select(Workspaces) select(Workspaces)
.join( .join(
WorkspaceMembers, WorkspaceMembers,
WorkspaceMembers.workspace_id == Workspaces.workspace_id, WorkspaceMembers.workspace_id == Workspaces.workspace_id)
)
.join(Users, Users.user_id == WorkspaceMembers.user_id) .join(Users, Users.user_id == WorkspaceMembers.user_id)
.where( .where(
Workspaces.workspace_id == workspace_id, Workspaces.workspace_id == workspace_id,
Workspaces.status == "active", Workspaces.status == "active",
WorkspaceMembers.user_id == user_id, WorkspaceMembers.user_id == user_id,
WorkspaceMembers.member_status == "active", WorkspaceMembers.member_status == "active",
Users.status == "active", Users.status == "active")
)
) )
workspace = await session.scalar(statement) workspace = await session.scalar(statement)
if workspace is None: if workspace is None:
raise HTTPException( raise HTTPException(
status.HTTP_403_FORBIDDEN, status.HTTP_403_FORBIDDEN,
"user is not an active workspace member", "user is not an active workspace member")
)
return workspace return workspace
async def create_upload_record( async def create_upload_record(
payload: CreateUploadRequest, payload: CreateUploadRequest,
session: AsyncSession, session: AsyncSession,
request: Request, request: Request) -> dict[str, Any]:
) -> dict[str, Any]:
workspace = await require_workspace_member( workspace = await require_workspace_member(
session, session,
payload.workspace_id, payload.workspace_id,
payload.user_id, payload.user_id)
)
stored_key = normalized_idempotency_key( stored_key = normalized_idempotency_key(
payload.workspace_id, payload.workspace_id,
payload.user_id, payload.user_id,
payload.idempotency_key, payload.idempotency_key)
)
existing = await session.scalar( existing = await session.scalar(
select(UploadSessions).where( select(UploadSessions).where(
UploadSessions.idempotency_key == stored_key UploadSessions.idempotency_key == stored_key
@@ -201,8 +169,7 @@ async def create_upload_record(
): ):
raise HTTPException( raise HTTPException(
status.HTTP_409_CONFLICT, status.HTTP_409_CONFLICT,
"idempotency key was used with different upload metadata", "idempotency key was used with different upload metadata")
)
upload = existing upload = existing
else: else:
upload_id = new_ulid() upload_id = new_ulid()
@@ -226,16 +193,14 @@ async def create_upload_record(
expires_at=utcnow() + timedelta(minutes=15), expires_at=utcnow() + timedelta(minutes=15),
expected_size_bytes=payload.expected_size_bytes, expected_size_bytes=payload.expected_size_bytes,
expected_hash=payload.expected_hash, expected_hash=payload.expected_hash,
content_type=payload.content_type, content_type=payload.content_type)
)
session.add(upload) session.add(upload)
await session.flush() await session.flush()
if upload.upload_status == "completed" and upload.storage_object_id: if upload.upload_status == "completed" and upload.storage_object_id:
storage_object = await session.get( storage_object = await session.get(
StorageObjects, StorageObjects,
upload.storage_object_id, upload.storage_object_id)
)
return { return {
"upload_id": upload.upload_id, "upload_id": upload.upload_id,
"status": upload.upload_status, "status": upload.upload_status,
@@ -246,8 +211,7 @@ async def create_upload_record(
if upload.upload_status not in {"created", "uploading"}: if upload.upload_status not in {"created", "uploading"}:
raise HTTPException( raise HTTPException(
status.HTTP_409_CONFLICT, status.HTTP_409_CONFLICT,
f"upload cannot continue from status {upload.upload_status}", f"upload cannot continue from status {upload.upload_status}")
)
url, headers = request.app.state.object_store.presign_put( url, headers = request.app.state.object_store.presign_put(
bucket_name=upload.bucket_name, bucket_name=upload.bucket_name,
@@ -255,8 +219,7 @@ async def create_upload_record(
content_type=upload.content_type or "application/octet-stream", content_type=upload.content_type or "application/octet-stream",
expected_hash=upload.expected_hash, expected_hash=upload.expected_hash,
expires_seconds=900, expires_seconds=900,
public=payload.url_scope == "public", public=payload.url_scope == "public")
)
return { return {
"upload_id": upload.upload_id, "upload_id": upload.upload_id,
"status": upload.upload_status, "status": upload.upload_status,
@@ -271,8 +234,7 @@ async def complete_upload_record(
upload_id: str, upload_id: str,
payload: CompleteUploadRequest, payload: CompleteUploadRequest,
session: AsyncSession, session: AsyncSession,
request: Request, request: Request) -> StorageObjects:
) -> StorageObjects:
upload = await session.scalar( upload = await session.scalar(
select(UploadSessions) select(UploadSessions)
.where(UploadSessions.upload_id == upload_id) .where(UploadSessions.upload_id == upload_id)
@@ -285,14 +247,12 @@ async def complete_upload_record(
if item is None: if item is None:
raise HTTPException( raise HTTPException(
status.HTTP_409_CONFLICT, status.HTTP_409_CONFLICT,
"completed upload has no storage object", "completed upload has no storage object")
)
return item return item
if upload.upload_status not in {"created", "uploading"}: if upload.upload_status not in {"created", "uploading"}:
raise HTTPException( raise HTTPException(
status.HTTP_409_CONFLICT, status.HTTP_409_CONFLICT,
f"upload cannot be completed from status {upload.upload_status}", f"upload cannot be completed from status {upload.upload_status}")
)
if upload.expires_at < utcnow(): if upload.expires_at < utcnow():
upload.upload_status = "expired" upload.upload_status = "expired"
raise HTTPException(status.HTTP_409_CONFLICT, "upload expired") raise HTTPException(status.HTTP_409_CONFLICT, "upload expired")
@@ -301,13 +261,11 @@ async def complete_upload_record(
head = await asyncio.to_thread( head = await asyncio.to_thread(
request.app.state.object_store.head, request.app.state.object_store.head,
bucket_name=upload.bucket_name, bucket_name=upload.bucket_name,
object_key=upload.object_key, object_key=upload.object_key)
)
except Exception as exc: except Exception as exc:
raise HTTPException( raise HTTPException(
status.HTTP_409_CONFLICT, status.HTTP_409_CONFLICT,
"uploaded object is not available", "uploaded object is not available") from exc
) from exc
actual_size = int(head.get("ContentLength", 0)) actual_size = int(head.get("ContentLength", 0))
if ( if (
@@ -317,8 +275,7 @@ async def complete_upload_record(
upload.upload_status = "failed" upload.upload_status = "failed"
raise HTTPException( raise HTTPException(
status.HTTP_409_CONFLICT, status.HTTP_409_CONFLICT,
"uploaded object size does not match expected_size_bytes", "uploaded object size does not match expected_size_bytes")
)
actual_content_type = str( actual_content_type = str(
head.get("ContentType") or "application/octet-stream" head.get("ContentType") or "application/octet-stream"
) )
@@ -326,8 +283,7 @@ async def complete_upload_record(
upload.upload_status = "failed" upload.upload_status = "failed"
raise HTTPException( raise HTTPException(
status.HTTP_409_CONFLICT, status.HTTP_409_CONFLICT,
"uploaded object content type does not match", "uploaded object content type does not match")
)
metadata = { metadata = {
str(key).lower(): str(value).lower() str(key).lower(): str(value).lower()
for key, value in dict(head.get("Metadata") or {}).items() for key, value in dict(head.get("Metadata") or {}).items()
@@ -337,14 +293,12 @@ async def complete_upload_record(
actual_hash = await asyncio.to_thread( actual_hash = await asyncio.to_thread(
request.app.state.object_store.sha256, request.app.state.object_store.sha256,
bucket_name=upload.bucket_name, bucket_name=upload.bucket_name,
object_key=upload.object_key, object_key=upload.object_key)
)
if upload.expected_hash and actual_hash != upload.expected_hash: if upload.expected_hash and actual_hash != upload.expected_hash:
upload.upload_status = "failed" upload.upload_status = "failed"
raise HTTPException( raise HTTPException(
status.HTTP_409_CONFLICT, status.HTTP_409_CONFLICT,
"uploaded object hash does not match expected_hash", "uploaded object hash does not match expected_hash")
)
file_name = upload.object_key.rsplit("/", 1)[-1] file_name = upload.object_key.rsplit("/", 1)[-1]
item = StorageObjects( item = StorageObjects(
@@ -367,8 +321,7 @@ async def complete_upload_record(
visibility=payload.visibility, visibility=payload.visibility,
is_immutable=int(payload.is_immutable), is_immutable=int(payload.is_immutable),
object_status="available", object_status="available",
created_by=upload.user_id, created_by=upload.user_id)
)
session.add(item) session.add(item)
await session.flush() await session.flush()
await session.refresh(item) await session.refresh(item)
@@ -378,45 +331,37 @@ async def complete_upload_record(
return item return item
@app.post("/internal/v1/uploads", dependencies=[Depends(verify_internal_service)]) @app.post("/internal/v1/uploads")
async def create_upload( async def create_upload(
payload: CreateUploadRequest, payload: CreateUploadRequest,
request: Request, request: Request,
session: AsyncSession = Depends(database_session), session: AsyncSession = Depends(database_session)) -> dict[str, Any]:
) -> dict[str, Any]:
return { return {
"data": await create_upload_record(payload, session, request), "data": await create_upload_record(payload, session, request),
} }
@app.post( @app.post(
"/internal/v1/uploads/{upload_id}/complete", "/internal/v1/uploads/{upload_id}/complete")
dependencies=[Depends(verify_internal_service)],
)
async def complete_upload( async def complete_upload(
upload_id: str, upload_id: str,
payload: CompleteUploadRequest, payload: CompleteUploadRequest,
request: Request, request: Request,
session: AsyncSession = Depends(database_session), session: AsyncSession = Depends(database_session)) -> dict[str, Any]:
) -> dict[str, Any]:
item = await complete_upload_record( item = await complete_upload_record(
upload_id, upload_id,
payload, payload,
session, session,
request, request)
)
return {"data": storage_payload(item)} return {"data": storage_payload(item)}
@app.post( @app.post(
"/internal/v1/uploads/{upload_id}/abort", "/internal/v1/uploads/{upload_id}/abort")
dependencies=[Depends(verify_internal_service)],
)
async def abort_upload( async def abort_upload(
upload_id: str, upload_id: str,
request: Request, request: Request,
session: AsyncSession = Depends(database_session), session: AsyncSession = Depends(database_session)) -> dict[str, Any]:
) -> dict[str, Any]:
upload = await session.scalar( upload = await session.scalar(
select(UploadSessions) select(UploadSessions)
.where(UploadSessions.upload_id == upload_id) .where(UploadSessions.upload_id == upload_id)
@@ -427,39 +372,32 @@ async def abort_upload(
if upload.upload_status == "completed": if upload.upload_status == "completed":
raise HTTPException( raise HTTPException(
status.HTTP_409_CONFLICT, status.HTTP_409_CONFLICT,
"completed upload cannot be aborted", "completed upload cannot be aborted")
)
if upload.upload_status != "aborted": if upload.upload_status != "aborted":
await asyncio.to_thread( await asyncio.to_thread(
request.app.state.object_store.delete, request.app.state.object_store.delete,
bucket_name=upload.bucket_name, bucket_name=upload.bucket_name,
object_key=upload.object_key, object_key=upload.object_key)
)
upload.upload_status = "aborted" upload.upload_status = "aborted"
return {"data": {"upload_id": upload_id, "status": "aborted"}} return {"data": {"upload_id": upload_id, "status": "aborted"}}
@app.post( @app.post(
"/internal/v1/objects", "/internal/v1/objects")
dependencies=[Depends(verify_internal_service)],
)
async def create_server_object( async def create_server_object(
payload: ServerObjectRequest, payload: ServerObjectRequest,
request: Request, request: Request,
session: AsyncSession = Depends(database_session), session: AsyncSession = Depends(database_session)) -> dict[str, Any]:
) -> dict[str, Any]:
try: try:
content = base64.b64decode(payload.content_base64, validate=True) content = base64.b64decode(payload.content_base64, validate=True)
except (binascii.Error, ValueError) as exc: except (binascii.Error, ValueError) as exc:
raise HTTPException( raise HTTPException(
status.HTTP_422_UNPROCESSABLE_ENTITY, status.HTTP_422_UNPROCESSABLE_ENTITY,
"content_base64 is invalid", "content_base64 is invalid") from exc
) from exc
if len(content) > 100 * 1024 * 1024: if len(content) > 100 * 1024 * 1024:
raise HTTPException( raise HTTPException(
status.HTTP_413_REQUEST_ENTITY_TOO_LARGE, status.HTTP_413_REQUEST_ENTITY_TOO_LARGE,
"object exceeds 100 MiB server-side upload limit", "object exceeds 100 MiB server-side upload limit")
)
content_hash = hashlib.sha256(content).hexdigest() content_hash = hashlib.sha256(content).hexdigest()
upload_result = await create_upload_record( upload_result = await create_upload_record(
CreateUploadRequest( CreateUploadRequest(
@@ -471,11 +409,9 @@ async def create_server_object(
expected_size_bytes=len(content), expected_size_bytes=len(content),
expected_hash=content_hash, expected_hash=content_hash,
idempotency_key=payload.idempotency_key, idempotency_key=payload.idempotency_key,
url_scope="internal", url_scope="internal"),
),
session, session,
request, request)
)
if upload_result.get("status") == "completed": if upload_result.get("status") == "completed":
return {"data": upload_result["storage_object"], "meta": {"reused": True}} return {"data": upload_result["storage_object"], "meta": {"reused": True}}
@@ -483,42 +419,34 @@ async def create_server_object(
if upload is None: if upload is None:
raise HTTPException( raise HTTPException(
status.HTTP_500_INTERNAL_SERVER_ERROR, status.HTTP_500_INTERNAL_SERVER_ERROR,
"upload record disappeared", "upload record disappeared")
)
await asyncio.to_thread( await asyncio.to_thread(
request.app.state.object_store.put_bytes, request.app.state.object_store.put_bytes,
bucket_name=upload.bucket_name, bucket_name=upload.bucket_name,
object_key=upload.object_key, object_key=upload.object_key,
content=content, content=content,
content_type=payload.content_type, content_type=payload.content_type,
content_hash=content_hash, content_hash=content_hash)
)
item = await complete_upload_record( item = await complete_upload_record(
upload.upload_id, upload.upload_id,
CompleteUploadRequest( CompleteUploadRequest(
usage_type=payload.usage_type, usage_type=payload.usage_type,
visibility=payload.visibility, visibility=payload.visibility,
is_immutable=payload.is_immutable, is_immutable=payload.is_immutable),
),
session, session,
request, request)
)
return {"data": storage_payload(item), "meta": {"reused": False}} return {"data": storage_payload(item), "meta": {"reused": False}}
@app.post( @app.post(
"/internal/v1/workspace-objects", "/internal/v1/workspace-objects")
dependencies=[Depends(verify_internal_service)],
)
async def register_workspace_object( async def register_workspace_object(
payload: RegisterWorkspaceObjectRequest, payload: RegisterWorkspaceObjectRequest,
session: AsyncSession = Depends(database_session), session: AsyncSession = Depends(database_session)) -> dict[str, Any]:
) -> dict[str, Any]:
workspace = await require_workspace_member( workspace = await require_workspace_member(
session, session,
payload.workspace_id, payload.workspace_id,
payload.user_id, payload.user_id)
)
pure_path = PurePosixPath(payload.relative_path.replace("\\", "/")) pure_path = PurePosixPath(payload.relative_path.replace("\\", "/"))
if ( if (
pure_path.is_absolute() pure_path.is_absolute()
@@ -527,8 +455,7 @@ async def register_workspace_object(
): ):
raise HTTPException( raise HTTPException(
status.HTTP_422_UNPROCESSABLE_ENTITY, status.HTTP_422_UNPROCESSABLE_ENTITY,
"invalid workspace relative_path", "invalid workspace relative_path")
)
relative_path = pure_path.as_posix() relative_path = pure_path.as_posix()
workspace_root = Path( workspace_root = Path(
os.getenv("WORKSPACE_ROOT", "/workspace/workspaces") os.getenv("WORKSPACE_ROOT", "/workspace/workspaces")
@@ -538,13 +465,11 @@ async def register_workspace_object(
if scoped_root != target and scoped_root not in target.parents: if scoped_root != target and scoped_root not in target.parents:
raise HTTPException( raise HTTPException(
status.HTTP_422_UNPROCESSABLE_ENTITY, status.HTTP_422_UNPROCESSABLE_ENTITY,
"workspace path escapes its root", "workspace path escapes its root")
)
if not target.is_file(): if not target.is_file():
raise HTTPException( raise HTTPException(
status.HTTP_404_NOT_FOUND, status.HTTP_404_NOT_FOUND,
"workspace file does not exist", "workspace file does not exist")
)
content = await asyncio.to_thread(target.read_bytes) content = await asyncio.to_thread(target.read_bytes)
content_hash = hashlib.sha256(content).hexdigest() content_hash = hashlib.sha256(content).hexdigest()
stat_result = target.stat() stat_result = target.stat()
@@ -553,8 +478,7 @@ async def register_workspace_object(
select(StorageObjects).where( select(StorageObjects).where(
StorageObjects.workspace_id == payload.workspace_id, StorageObjects.workspace_id == payload.workspace_id,
StorageObjects.storage_backend == "workspace_fs", StorageObjects.storage_backend == "workspace_fs",
StorageObjects.path_hash == path_digest, StorageObjects.path_hash == path_digest)
)
) )
reused = item is not None reused = item is not None
if item is None: if item is None:
@@ -572,19 +496,16 @@ async def register_workspace_object(
visibility=payload.visibility, visibility=payload.visibility,
is_immutable=0, is_immutable=0,
object_status="available", object_status="available",
created_by=payload.user_id, created_by=payload.user_id)
)
session.add(item) session.add(item)
elif item.is_immutable: elif item.is_immutable:
raise HTTPException( raise HTTPException(
status.HTTP_409_CONFLICT, status.HTTP_409_CONFLICT,
"immutable workspace object cannot be updated", "immutable workspace object cannot be updated")
)
elif item.owner_user_id != payload.user_id: elif item.owner_user_id != payload.user_id:
raise HTTPException( raise HTTPException(
status.HTTP_403_FORBIDDEN, status.HTTP_403_FORBIDDEN,
"workspace object belongs to another user", "workspace object belongs to another user")
)
item.usage_type = payload.usage_type item.usage_type = payload.usage_type
item.file_name = target.name item.file_name = target.name
item.file_extension = target.suffix.lower() or None item.file_extension = target.suffix.lower() or None
@@ -601,15 +522,12 @@ async def register_workspace_object(
@app.post( @app.post(
"/internal/v1/objects/{storage_object_id}/download-url", "/internal/v1/objects/{storage_object_id}/download-url")
dependencies=[Depends(verify_internal_service)],
)
async def create_download_url( async def create_download_url(
storage_object_id: str, storage_object_id: str,
payload: DownloadUrlRequest, payload: DownloadUrlRequest,
request: Request, request: Request,
session: AsyncSession = Depends(database_session), session: AsyncSession = Depends(database_session)) -> dict[str, Any]:
) -> dict[str, Any]:
item = await session.get(StorageObjects, storage_object_id) item = await session.get(StorageObjects, storage_object_id)
if item is None or item.object_status != "available": if item is None or item.object_status != "available":
raise HTTPException(status.HTTP_404_NOT_FOUND, "object not found") raise HTTPException(status.HTTP_404_NOT_FOUND, "object not found")
@@ -620,14 +538,12 @@ async def create_download_url(
): ):
raise HTTPException( raise HTTPException(
status.HTTP_409_CONFLICT, status.HTTP_409_CONFLICT,
"object does not support a presigned URL", "object does not support a presigned URL")
)
url = request.app.state.object_store.presign_get( url = request.app.state.object_store.presign_get(
bucket_name=item.bucket_name, bucket_name=item.bucket_name,
object_key=item.object_key, object_key=item.object_key,
file_name=item.file_name, file_name=item.file_name,
expires_seconds=payload.expires_seconds, expires_seconds=payload.expires_seconds)
)
return { return {
"data": { "data": {
"storage_object_id": item.storage_object_id, "storage_object_id": item.storage_object_id,
@@ -639,14 +555,11 @@ async def create_download_url(
@app.delete( @app.delete(
"/internal/v1/objects/{storage_object_id}", "/internal/v1/objects/{storage_object_id}")
dependencies=[Depends(verify_internal_service)],
)
async def delete_object( async def delete_object(
storage_object_id: str, storage_object_id: str,
request: Request, request: Request,
session: AsyncSession = Depends(database_session), session: AsyncSession = Depends(database_session)) -> dict[str, Any]:
) -> dict[str, Any]:
item = await session.scalar( item = await session.scalar(
select(StorageObjects) select(StorageObjects)
.where(StorageObjects.storage_object_id == storage_object_id) .where(StorageObjects.storage_object_id == storage_object_id)
@@ -657,15 +570,13 @@ async def delete_object(
if item.is_immutable: if item.is_immutable:
raise HTTPException( raise HTTPException(
status.HTTP_409_CONFLICT, status.HTTP_409_CONFLICT,
"immutable object cannot be deleted", "immutable object cannot be deleted")
)
if item.object_status != "deleted": if item.object_status != "deleted":
if item.storage_backend == "rustfs" and item.bucket_name and item.object_key: if item.storage_backend == "rustfs" and item.bucket_name and item.object_key:
await asyncio.to_thread( await asyncio.to_thread(
request.app.state.object_store.delete, request.app.state.object_store.delete,
bucket_name=item.bucket_name, bucket_name=item.bucket_name,
object_key=item.object_key, object_key=item.object_key)
)
item.object_status = "deleted" item.object_status = "deleted"
item.deleted_at = utcnow() item.deleted_at = utcnow()
return { return {
+3 -114
View File
@@ -1,116 +1,5 @@
from __future__ import annotations """Backward-compatible re-export of the shared storage client."""
import base64 from common.storage.client import StorageClient
from typing import Any
import httpx __all__ = ["StorageClient"]
from fastapi import HTTPException
class StorageClient:
def __init__(
self,
client: httpx.AsyncClient,
service_token: str,
) -> None:
self.client = client
self.headers = {"X-Service-Token": service_token}
async def _request(
self,
method: str,
path: str,
*,
payload: dict[str, Any] | None = None,
) -> dict[str, Any]:
response = await self.client.request(
method,
path,
json=payload,
headers=self.headers,
)
if response.is_error:
try:
detail = response.json().get("detail", response.text)
except ValueError:
detail = response.text
raise HTTPException(response.status_code, detail)
return response.json()
async def create_upload(
self,
payload: dict[str, Any],
) -> dict[str, Any]:
return (await self._request(
"POST",
"/internal/v1/uploads",
payload=payload,
))["data"]
async def complete_upload(
self,
upload_id: str,
payload: dict[str, Any],
) -> dict[str, Any]:
return (await self._request(
"POST",
f"/internal/v1/uploads/{upload_id}/complete",
payload=payload,
))["data"]
async def register_workspace_object(
self,
payload: dict[str, Any],
) -> dict[str, Any]:
return (await self._request(
"POST",
"/internal/v1/workspace-objects",
payload=payload,
))["data"]
async def create_server_object(
self,
*,
workspace_id: str,
user_id: str,
usage_type: str,
file_name: str,
content_type: str,
content: bytes,
visibility: str,
is_immutable: bool,
idempotency_key: str,
) -> dict[str, Any]:
result = await self._request(
"POST",
"/internal/v1/objects",
payload={
"workspace_id": workspace_id,
"user_id": user_id,
"usage_type": usage_type,
"file_name": file_name,
"content_type": content_type,
"content_base64": base64.b64encode(content).decode("ascii"),
"visibility": visibility,
"is_immutable": is_immutable,
"idempotency_key": idempotency_key,
},
)
return result["data"]
async def create_download_url(
self,
storage_object_id: str,
expires_seconds: int,
) -> dict[str, Any]:
return (await self._request(
"POST",
f"/internal/v1/objects/{storage_object_id}/download-url",
payload={"expires_seconds": expires_seconds},
))["data"]
async def delete_object(self, storage_object_id: str) -> dict[str, Any]:
return (await self._request(
"DELETE",
f"/internal/v1/objects/{storage_object_id}",
))["data"]
-79
View File
@@ -1,79 +0,0 @@
from __future__ import annotations
from typing import Literal
from pydantic import BaseModel, ConfigDict, Field, field_validator
class StrictModel(BaseModel):
model_config = ConfigDict(extra="forbid")
class CreateUploadRequest(StrictModel):
workspace_id: str = Field(min_length=26, max_length=26)
user_id: str = Field(min_length=26, max_length=26)
usage_type: Literal[
"data_resource",
"version_artifact",
"snapshot",
"run_log",
"run_result",
]
file_name: str = Field(min_length=1, max_length=255)
content_type: str = Field(min_length=1, max_length=255)
expected_size_bytes: int = Field(ge=0, le=100 * 1024 * 1024)
expected_hash: str | None = Field(default=None, min_length=64, max_length=64)
idempotency_key: str = Field(min_length=8, max_length=128)
url_scope: Literal["public", "internal"] = "public"
@field_validator("expected_hash")
@classmethod
def validate_hash(cls, value: str | None) -> str | None:
if value is None:
return None
normalized = value.lower()
if any(character not in "0123456789abcdef" for character in normalized):
raise ValueError("expected_hash must be lowercase SHA-256 hex")
return normalized
class CompleteUploadRequest(StrictModel):
usage_type: Literal[
"data_resource",
"version_artifact",
"snapshot",
"run_log",
"run_result",
]
visibility: Literal["private", "workspace", "public"] = "private"
is_immutable: bool = False
class ServerObjectRequest(StrictModel):
workspace_id: str = Field(min_length=26, max_length=26)
user_id: str = Field(min_length=26, max_length=26)
usage_type: Literal[
"data_resource",
"version_artifact",
"snapshot",
"run_log",
"run_result",
]
file_name: str = Field(min_length=1, max_length=255)
content_type: str = Field(min_length=1, max_length=255)
content_base64: str = Field(min_length=1)
visibility: Literal["private", "workspace", "public"] = "private"
is_immutable: bool = False
idempotency_key: str = Field(min_length=8, max_length=128)
class RegisterWorkspaceObjectRequest(StrictModel):
workspace_id: str = Field(min_length=26, max_length=26)
user_id: str = Field(min_length=26, max_length=26)
relative_path: str = Field(min_length=1, max_length=1024)
usage_type: Literal["working_copy", "public_script"]
visibility: Literal["private", "workspace", "public"] = "private"
class DownloadUrlRequest(StrictModel):
expires_seconds: int = Field(default=300, ge=30, le=3600)
+5 -1
View File
@@ -1,6 +1,6 @@
import datetime import datetime
import decimal import decimal
from typing import Optional from typing import Literal, Optional
from sqlalchemy import DECIMAL, Index, Integer, JSON, String, Text, text from sqlalchemy import DECIMAL, Index, Integer, JSON, String, Text, text
from sqlalchemy.dialects.mysql import BIGINT, CHAR, DATETIME, INTEGER, TINYINT from sqlalchemy.dialects.mysql import BIGINT, CHAR, DATETIME, INTEGER, TINYINT
@@ -9,6 +9,10 @@ from sqlalchemy.orm import Mapped, mapped_column
from common.db.base import Base from common.db.base import Base
TriggerType = Literal["manual", "cron", "api"]
FailurePolicy = Literal["stop", "continue"]
class Schedules(Base): class Schedules(Base):
__tablename__ = "schedules" __tablename__ = "schedules"
__table_args__ = ( __table_args__ = (
+4 -1
View File
@@ -1,3 +1,6 @@
"""Storage building blocks shared by backend and schedule services."""
from common.storage.client import StorageClient
from common.storage.rustfs import RustFSObjectStore from common.storage.rustfs import RustFSObjectStore
__all__ = ["RustFSObjectStore"] __all__ = ["RustFSObjectStore", "StorageClient"]
@@ -1,9 +0,0 @@
"""模型实验开发平台构建脚本。"""
def main() -> None:
print("Hello, Model Platform!")
if __name__ == "__main__":
main()
@@ -1,43 +0,0 @@
{
"cells": [
{
"cell_type": "markdown",
"id": "e139eae2",
"metadata": {},
"source": [
"# 新建模型实验\\n在这里开始数据探索与模型构建。"
]
},
{
"cell_type": "code",
"execution_count": null,
"id": "7a78bce5",
"metadata": {},
"outputs": [],
"source": [
"print('Hello, Model Platform!')"
]
}
],
"metadata": {
"kernelspec": {
"display_name": "Python 3 (ipykernel)",
"language": "python",
"name": "python3"
},
"language_info": {
"codemirror_mode": {
"name": "ipython",
"version": 3
},
"file_extension": ".py",
"mimetype": "text/x-python",
"name": "python",
"nbconvert_exporter": "python",
"pygments_lexer": "ipython3",
"version": "3.13.11"
}
},
"nbformat": 4,
"nbformat_minor": 5
}
@@ -1,67 +0,0 @@
{
"cells": [
{
"cell_type": "code",
"execution_count": 1,
"id": "initial_id",
"metadata": {},
"outputs": [
{
"name": "stdout",
"output_type": "stream",
"text": ["hello world\n"]
}
],
"source": ["print(\"hello world\")"]
},
{
"cell_type": "code",
"execution_count": 1,
"id": "979c21d2489e134d",
"metadata": {},
"outputs": [
{
"name": "stdout",
"output_type": "stream",
"text": ["hello world12323\n"]
}
],
"source": ["print(\"hello world12323\")"]
},
{
"cell_type": "code",
"execution_count": 2,
"id": "e068092a-3b51-4399-9c64-c44d58f4973c",
"metadata": {},
"outputs": [
{
"name": "stdout",
"output_type": "stream",
"text": ["Python 3.12.13\n"]
}
],
"source": ["!python --version"]
},
{
"cell_type": "code",
"execution_count": null,
"id": "1f512d9e-92f7-423e-9176-40fec30dd79c",
"metadata": {},
"outputs": [],
"source": []
}
],
"metadata": {
"kernelspec": {
"display_name": "Python 3 (ipykernel)",
"language": "python",
"name": "python3"
},
"language_info": {
"name": "python",
"version": "3.12.13"
}
},
"nbformat": 4,
"nbformat_minor": 5
}
@@ -1,59 +0,0 @@
{
"cells": [
{
"cell_type": "markdown",
"id": "e139eae2",
"metadata": {},
"source": [
"# 新建模型实验\\n在这里开始数据探索与模型构建。"
]
},
{
"cell_type": "code",
"execution_count": 1,
"id": "7a78bce5",
"metadata": {},
"outputs": [
{
"name": "stdout",
"output_type": "stream",
"text": [
"111\n"
]
}
],
"source": [
"print('111')"
]
},
{
"cell_type": "code",
"execution_count": null,
"id": "42d2764b-5699-4374-afe9-aecda6d2d1cd",
"metadata": {},
"outputs": [],
"source": []
}
],
"metadata": {
"kernelspec": {
"display_name": "Python 3 (ipykernel)",
"language": "python",
"name": "python3"
},
"language_info": {
"codemirror_mode": {
"name": "ipython",
"version": 3
},
"file_extension": ".py",
"mimetype": "text/x-python",
"name": "python",
"nbconvert_exporter": "python",
"pygments_lexer": "ipython3",
"version": "3.13.11"
}
},
"nbformat": 4,
"nbformat_minor": 5
}
@@ -1,9 +0,0 @@
"""模型实验开发平台构建脚本。"""
def main() -> None:
print("Hello, Model Platform!")
if __name__ == "__main__":
main()
-67
View File
@@ -1,67 +0,0 @@
{
"cells": [
{
"cell_type": "code",
"execution_count": 1,
"id": "initial_id",
"metadata": {},
"outputs": [
{
"name": "stdout",
"output_type": "stream",
"text": ["hello world\n"]
}
],
"source": ["print(\"hello world\")"]
},
{
"cell_type": "code",
"execution_count": 1,
"id": "979c21d2489e134d",
"metadata": {},
"outputs": [
{
"name": "stdout",
"output_type": "stream",
"text": ["hello world12323\n"]
}
],
"source": ["print(\"hello world12323\")"]
},
{
"cell_type": "code",
"execution_count": 2,
"id": "e068092a-3b51-4399-9c64-c44d58f4973c",
"metadata": {},
"outputs": [
{
"name": "stdout",
"output_type": "stream",
"text": ["Python 3.12.13\n"]
}
],
"source": ["!python --version"]
},
{
"cell_type": "code",
"execution_count": null,
"id": "1f512d9e-92f7-423e-9176-40fec30dd79c",
"metadata": {},
"outputs": [],
"source": []
}
],
"metadata": {
"kernelspec": {
"display_name": "Python 3 (ipykernel)",
"language": "python",
"name": "python3"
},
"language_info": {
"name": "python",
"version": "3.12.13"
}
},
"nbformat": 4,
"nbformat_minor": 5
}
-60
View File
@@ -1,60 +0,0 @@
version: '3.8'
services:
web:
image: nginx:alpine
ports:
- "8888:80"
restart: unless-stopped
volumes:
- ./default.conf:/etc/nginx/conf.d/default.conf:ro
depends_on:
- backend
backend:
build:
context: .
dockerfile: backend/Dockerfile
ports:
- "8004:8000"
environment:
- RUNTIME_BASE_URL=http://runtime:8001
volumes:
- ./backend:/app/backend:ro
- ./common:/app/common:ro
depends_on:
- runtime
runtime:
build:
context: .
dockerfile: runtime/Dockerfile
cap_add:
- SYS_ADMIN
devices:
- /dev/fuse:/dev/fuse
security_opt:
- apparmor:unconfined
ports:
- "8002:8001"
environment:
- PUBLIC_BASE_URL=http://runtime
# --- Rclone 动态环境变量配置 (对应名称 rustfs) ---
- RCLONE_CONFIG_RUSTFS_TYPE=s3
- RCLONE_CONFIG_RUSTFS_PROVIDER=Other
- RCLONE_CONFIG_RUSTFS_ACCESS_KEY_ID=BdsXeamEnvSDQnk8tRxh
- RCLONE_CONFIG_RUSTFS_SECRET_ACCESS_KEY=mmBVc3RqzbT2VX3ysKGnirYH5kYD3ww3wFtMVvrb
# 替换为你的 RustFS 服务地址(如果是同 docker-compose 网络下的服务,可以直接填服务名:端口)
- RCLONE_CONFIG_RUSTFS_ENDPOINT=http://8.153.151.51:9000
# 自建 S3 建议强制开启 Path-style 访问 (http://endpoint/bucket)
- RCLONE_CONFIG_RUSTFS_ENV_AUTH=false
- RCLONE_CONFIG_RUSTFS_FORCE_PATH_STYLE=true
- RCLONE_CONFIG_RUSTFS_REGION=other
# --- Runtime 逻辑环境变量 ---
- REMOTE_BUCKET=rustfs:workspaces
- WORKSPACES_ROOT=/app/workspaces
volumes:
- ./runtime:/app/runtime:ro
- ./common:/app/common:ro
+41 -144
View File
@@ -1,100 +1,16 @@
name: ${COMPOSE_PROJECT_NAME:-model-platform-refactored} version: '3.8'
x-app-environment: &app-environment
DATABASE_URL: mysql+asyncmy://${MYSQL_USER:-model_platform}:${MYSQL_PASSWORD:-model_platform}@mysql:3306/${MYSQL_DATABASE:-model_platform}?charset=utf8mb4
INTERNAL_SERVICE_TOKEN: ${INTERNAL_SERVICE_TOKEN:-local-internal-token}
JUPYTER_TOKEN: ${JUPYTER_TOKEN:-local-jupyter-token}
WORKSPACE_ROOT: /workspace/workspaces
services: services:
mysql: web:
image: mysql:8.0.36 image: nginx:alpine
restart: unless-stopped restart: unless-stopped
environment:
MYSQL_ROOT_PASSWORD: ${MYSQL_ROOT_PASSWORD:-model_platform_root}
MYSQL_DATABASE: ${MYSQL_DATABASE:-model_platform}
MYSQL_USER: ${MYSQL_USER:-model_platform}
MYSQL_PASSWORD: ${MYSQL_PASSWORD:-model_platform}
command:
- --character-set-server=utf8mb4
- --collation-server=utf8mb4_0900_ai_ci
ports: ports:
- "${MYSQL_PORT:-3308}:3306" - "${GATEWAY_PORT:-8888}:80"
volumes: volumes:
- mysql_data:/var/lib/mysql - ./default.conf:/etc/nginx/conf.d/default.conf:ro
healthcheck:
test: ["CMD-SHELL", "mysqladmin ping -h 127.0.0.1 -u root -p$$MYSQL_ROOT_PASSWORD --silent"]
interval: 10s
timeout: 5s
retries: 15
redis:
image: redis:7.2.5-alpine
restart: unless-stopped
command: ["redis-server", "--appendonly", "yes", "--requirepass", "${REDIS_PASSWORD:-model_platform_redis}"]
ports:
- "${REDIS_PORT:-6380}:6379"
volumes:
- redis_data:/data
healthcheck:
test: ["CMD-SHELL", "redis-cli -a '${REDIS_PASSWORD:-model_platform_redis}' ping | grep PONG"]
interval: 10s
timeout: 5s
retries: 10
rustfs:
image: ${RUSTFS_IMAGE:-rustfs/rustfs:latest}
restart: unless-stopped
environment:
RUSTFS_ADDRESS: ":9000"
RUSTFS_CONSOLE_ENABLE: "true"
RUSTFS_CONSOLE_ADDRESS: ":9001"
RUSTFS_ACCESS_KEY: ${RUSTFS_ACCESS_KEY:-modelplatform}
RUSTFS_SECRET_KEY: ${RUSTFS_SECRET_KEY:-modelplatformsecret}
command: ["/data"]
ports:
- "${RUSTFS_API_PORT:-9010}:9000"
- "${RUSTFS_CONSOLE_PORT:-9011}:9001"
volumes:
- rustfs_data:/data
jupyter:
image: ${JUPYTER_IMAGE:-quay.io/jupyter/base-notebook:2025-12-31}
restart: unless-stopped
environment:
JUPYTER_TOKEN: ${JUPYTER_TOKEN:-local-jupyter-token}
command:
- start-notebook.py
- --ServerApp.base_url=/jupyter/
- --ServerApp.root_dir=/home/jovyan/work
- --ServerApp.ip=0.0.0.0
- --ServerApp.allow_remote_access=True
- --IdentityProvider.token=${JUPYTER_TOKEN:-local-jupyter-token}
- --PasswordIdentityProvider.hashed_password=
volumes:
- ./deploy/data/workspaces:/home/jovyan/work
expose:
- "8888"
migrate:
build:
context: .
dockerfile: backend/Dockerfile
environment:
<<: *app-environment
command:
- uv
- run
- --frozen
- --package
- backend
- alembic
- upgrade
- head
depends_on: depends_on:
mysql: - backend
condition: service_healthy - runtime
restart: "no"
backend: backend:
build: build:
@@ -102,38 +18,39 @@ services:
dockerfile: backend/Dockerfile dockerfile: backend/Dockerfile
restart: unless-stopped restart: unless-stopped
environment: environment:
<<: *app-environment DATABASE_URL: mysql+asyncmy://${MYSQL_USER:-model_platform}:${MYSQL_PASSWORD:-model_platform}@mysql:3306/${MYSQL_DATABASE:-model_platform}?charset=utf8mb4
SERVICE_NAME: backend JWT_SECRET: ${JWT_SECRET:-local-jwt-secret}
READINESS_TARGETS: mysql:3306,redis:6379,rustfs:9000
RUNTIME_API_URL: http://runtime:8000 RUNTIME_API_URL: http://runtime:8000
RUNTIME_BASE_URL: http://runtime:8000
RUSTFS_INTERNAL_ENDPOINT: http://rustfs:9000 RUSTFS_INTERNAL_ENDPOINT: http://rustfs:9000
RUSTFS_PUBLIC_ENDPOINT: http://localhost:${RUSTFS_API_PORT:-9010} RUSTFS_PUBLIC_ENDPOINT: http://localhost:${RUSTFS_API_PORT:-9010}
RUSTFS_ACCESS_KEY: ${RUSTFS_ACCESS_KEY:-modelplatform} RUSTFS_ACCESS_KEY: ${RUSTFS_ACCESS_KEY:-modelplatform}
RUSTFS_SECRET_KEY: ${RUSTFS_SECRET_KEY:-modelplatformsecret} RUSTFS_SECRET_KEY: ${RUSTFS_SECRET_KEY:-modelplatformsecret}
RUSTFS_DEFAULT_BUCKET: model-platform RUSTFS_DEFAULT_BUCKET: model-platform
WORKSPACE_ROOT: /workspace/workspaces
volumes: volumes:
- ./deploy/data/workspaces:/workspace/workspaces - ./deploy/data/workspaces:/workspace/workspaces
- ./backend:/app/backend:ro
- ./common:/app/common:ro
ports: ports:
- "${BACKEND_PORT:-8010}:8000" - "${BACKEND_PORT:-8004}:8000"
depends_on: depends_on:
migrate: - runtime
condition: service_completed_successfully
mysql:
condition: service_healthy
redis:
condition: service_healthy
rustfs:
condition: service_started
runtime: runtime:
build: build:
context: . context: .
dockerfile: runtime/Dockerfile dockerfile: runtime/Dockerfile
restart: unless-stopped restart: unless-stopped
cap_add:
- SYS_ADMIN
devices:
- /dev/fuse:/dev/fuse
security_opt:
- apparmor:unconfined
environment: environment:
<<: *app-environment DATABASE_URL: mysql+asyncmy://${MYSQL_USER:-model_platform}:${MYSQL_PASSWORD:-model_platform}@mysql:3306/${MYSQL_DATABASE:-model_platform}?charset=utf8mb4
SERVICE_NAME: runtime-manager SERVICE_NAME: runtime-manager
READINESS_TARGETS: mysql:3306,redis:6379,jupyter:8888
REDIS_HOST: redis REDIS_HOST: redis
REDIS_PORT: "6379" REDIS_PORT: "6379"
REDIS_PASSWORD: ${REDIS_PASSWORD:-model_platform_redis} REDIS_PASSWORD: ${REDIS_PASSWORD:-model_platform_redis}
@@ -141,15 +58,23 @@ services:
JUPYTER_INTERNAL_URL: http://jupyter:8888/jupyter/ JUPYTER_INTERNAL_URL: http://jupyter:8888/jupyter/
JUPYTER_PROXY_BASE_PATH: /jupyter/ JUPYTER_PROXY_BASE_PATH: /jupyter/
JUPYTER_TICKET_TTL_SECONDS: "300" JUPYTER_TICKET_TTL_SECONDS: "300"
WORKSPACES_ROOT: /workspace/workspaces
PUBLIC_BASE_URL: http://runtime
REMOTE_BUCKET: rustfs:workspaces
RCLONE_CONFIG_RUSTFS_TYPE: s3
RCLONE_CONFIG_RUSTFS_PROVIDER: Other
RCLONE_CONFIG_RUSTFS_ACCESS_KEY_ID: ${RUSTFS_ACCESS_KEY:-modelplatform}
RCLONE_CONFIG_RUSTFS_SECRET_ACCESS_KEY: ${RUSTFS_SECRET_KEY:-modelplatformsecret}
RCLONE_CONFIG_RUSTFS_ENDPOINT: http://rustfs:9000
RCLONE_CONFIG_RUSTFS_ENV_AUTH: "false"
RCLONE_CONFIG_RUSTFS_FORCE_PATH_STYLE: "true"
RCLONE_CONFIG_RUSTFS_REGION: other
volumes:
- ./deploy/data/workspaces:/workspace/workspaces
- ./runtime:/app/runtime:ro
- ./common:/app/common:ro
ports: ports:
- "${RUNTIME_PORT:-8012}:8000" - "${RUNTIME_PORT:-8002}:8001"
depends_on:
mysql:
condition: service_healthy
redis:
condition: service_healthy
jupyter:
condition: service_started
schedule: schedule:
build: build:
@@ -157,9 +82,7 @@ services:
dockerfile: schedule/Dockerfile dockerfile: schedule/Dockerfile
restart: unless-stopped restart: unless-stopped
environment: environment:
<<: *app-environment DATABASE_URL: mysql+asyncmy://${MYSQL_USER:-model_platform}:${MYSQL_PASSWORD:-model_platform}@mysql:3306/${MYSQL_DATABASE:-model_platform}?charset=utf8mb4
SERVICE_NAME: schedule-executor
READINESS_TARGETS: mysql:3306,redis:6379,rustfs:9000,backend:8000
REDIS_HOST: redis REDIS_HOST: redis
REDIS_PORT: "6379" REDIS_PORT: "6379"
REDIS_PASSWORD: ${REDIS_PASSWORD:-model_platform_redis} REDIS_PASSWORD: ${REDIS_PASSWORD:-model_platform_redis}
@@ -167,36 +90,10 @@ services:
RUSTFS_ACCESS_KEY: ${RUSTFS_ACCESS_KEY:-modelplatform} RUSTFS_ACCESS_KEY: ${RUSTFS_ACCESS_KEY:-modelplatform}
RUSTFS_SECRET_KEY: ${RUSTFS_SECRET_KEY:-modelplatformsecret} RUSTFS_SECRET_KEY: ${RUSTFS_SECRET_KEY:-modelplatformsecret}
STORAGE_API_URL: http://backend:8000 STORAGE_API_URL: http://backend:8000
WORKSPACE_ROOT: /workspace/workspaces
volumes: volumes:
- ./deploy/data/workspaces:/workspace/workspaces - ./deploy/data/workspaces:/workspace/workspaces
ports: ports:
- "${SCHEDULE_PORT:-8013}:8000" - "${SCHEDULE_PORT:-8013}:8000"
depends_on: depends_on:
mysql: - backend
condition: service_healthy
redis:
condition: service_healthy
backend:
condition: service_started
gateway:
image: nginx:1.27-alpine
restart: unless-stopped
environment:
INTERNAL_SERVICE_TOKEN: ${INTERNAL_SERVICE_TOKEN:-local-internal-token}
volumes:
- ./nginx/default.conf.template:/etc/nginx/templates/default.conf.template:ro
ports:
- "${GATEWAY_PORT:-8081}:80"
depends_on:
backend:
condition: service_started
runtime:
condition: service_started
jupyter:
condition: service_started
volumes:
mysql_data:
redis_data:
rustfs_data:
+1 -4
View File
@@ -26,10 +26,7 @@ async def lifespan(app: Any) -> AsyncIterator[None]:
session_factory=session_factory, session_factory=session_factory,
redis=redis, redis=redis,
object_store=build_object_store(), object_store=build_object_store(),
storage_client=SchedulerStorageClient( storage_client=SchedulerStorageClient(storage_http_client),
storage_http_client,
os.environ["INTERNAL_SERVICE_TOKEN"],
),
workspace_root=Path( workspace_root=Path(
os.getenv("WORKSPACE_ROOT", "/workspace/workspaces") os.getenv("WORKSPACE_ROOT", "/workspace/workspaces")
), ),
+17 -23
View File
@@ -1,16 +1,13 @@
"""Schedule-specific storage client built on the shared ``StorageClient``."""
from __future__ import annotations from __future__ import annotations
import base64
from typing import Any from typing import Any
import httpx from common.storage.client import StorageClient
class SchedulerStorageClient: class SchedulerStorageClient(StorageClient):
def __init__(self, client: httpx.AsyncClient, service_token: str) -> None:
self.client = client
self.headers = {"X-Service-Token": service_token}
async def create_object( async def create_object(
self, self,
*, *,
@@ -22,20 +19,17 @@ class SchedulerStorageClient:
content: bytes, content: bytes,
idempotency_key: str, idempotency_key: str,
) -> dict[str, Any]: ) -> dict[str, Any]:
response = await self.client.post( return await self.create_server_object(
"/internal/v1/objects", workspace_id=workspace_id,
headers=self.headers, user_id=user_id,
json={ usage_type=usage_type,
"workspace_id": workspace_id, file_name=file_name,
"user_id": user_id, content_type=content_type,
"usage_type": usage_type, content=content,
"file_name": file_name, visibility="workspace",
"content_type": content_type, is_immutable=True,
"content_base64": base64.b64encode(content).decode("ascii"), idempotency_key=idempotency_key,
"visibility": "workspace",
"is_immutable": True,
"idempotency_key": idempotency_key,
},
) )
response.raise_for_status()
return response.json()["data"]
__all__ = ["SchedulerStorageClient"]