diff --git a/backend/src/backend/storage_api.py b/backend/src/backend/storage_api.py index d3b618d..b3f2612 100644 --- a/backend/src/backend/storage_api.py +++ b/backend/src/backend/storage_api.py @@ -1,8 +1,6 @@ from __future__ import annotations -import asyncio -import mimetypes -import secrets +import hashlib from contextlib import asynccontextmanager from datetime import UTC, datetime, timedelta from typing import Any, AsyncIterator @@ -18,14 +16,21 @@ from common.db.models import ( UploadSessions, Users, WorkspaceMembers, - Workspaces) + Workspaces, +) from common.ids import new_ulid from common.service_app import create_service_app -from common.storage import AsyncStorageBackend, PURPOSE_BUCKETS, build_storage_config, create_storage +from common.storage import ( + AsyncStorageBackend, + PURPOSE_BUCKETS, + build_storage_config, + create_storage, +) from common.storage.schemas import ( CreateUploadRequest, DownloadUrlRequest, - ServerObjectRequest) + ServerObjectRequest, +) from backend.services.storage import ( create_download_url_payload, create_server_object_payload, @@ -43,10 +48,7 @@ def hash_bytes(value: str) -> bytes: return hashlib.sha256(value.encode("utf-8")).digest() -def normalized_idempotency_key( - workspace_id: str, - user_id: str, - value: str) -> str: +def normalized_idempotency_key(workspace_id: str, user_id: str, value: str) -> str: digest = hashlib.sha256( f"{workspace_id}:{user_id}:{value}".encode("utf-8") ).hexdigest() @@ -133,9 +135,7 @@ async def lifespan(app: Any) -> AsyncIterator[None]: await engine.dispose() -app = create_service_app( - settings.service_name, - lifespan=lifespan) +app = create_service_app(settings.service_name, lifespan=lifespan) async def database_session(request: Request) -> AsyncIterator[AsyncSession]: @@ -144,46 +144,41 @@ async def database_session(request: Request) -> AsyncIterator[AsyncSession]: async def require_workspace_member( - session: AsyncSession, - workspace_id: str, - user_id: str) -> Workspaces: + session: AsyncSession, workspace_id: str, user_id: str +) -> Workspaces: statement = ( select(Workspaces) .join( - WorkspaceMembers, - WorkspaceMembers.workspace_id == Workspaces.workspace_id) + WorkspaceMembers, WorkspaceMembers.workspace_id == Workspaces.workspace_id + ) .join(Users, Users.user_id == WorkspaceMembers.user_id) .where( Workspaces.workspace_id == workspace_id, Workspaces.status == "active", WorkspaceMembers.user_id == user_id, WorkspaceMembers.member_status == "active", - Users.status == "active") + Users.status == "active", + ) ) workspace = await session.scalar(statement) if workspace is None: raise HTTPException( - status.HTTP_403_FORBIDDEN, - "user is not an active workspace member") + status.HTTP_403_FORBIDDEN, "user is not an active workspace member" + ) return workspace async def create_upload_record( - payload: CreateUploadRequest, - session: AsyncSession, - request: Request) -> dict[str, Any]: + payload: CreateUploadRequest, session: AsyncSession, request: Request +) -> dict[str, Any]: workspace = await require_workspace_member( - session, - payload.workspace_id, - payload.user_id) + session, payload.workspace_id, payload.user_id + ) stored_key = normalized_idempotency_key( - payload.workspace_id, - payload.user_id, - payload.idempotency_key) + payload.workspace_id, payload.user_id, payload.idempotency_key + ) existing = await session.scalar( - select(UploadSessions).where( - UploadSessions.idempotency_key == stored_key - ) + select(UploadSessions).where(UploadSessions.idempotency_key == stored_key) ) if existing is not None: if ( @@ -195,7 +190,8 @@ async def create_upload_record( ): raise HTTPException( status.HTTP_409_CONFLICT, - "idempotency key was used with different upload metadata") + "idempotency key was used with different upload metadata", + ) upload = existing else: upload_id = new_ulid() @@ -203,9 +199,7 @@ async def create_upload_record( # Keep the opaque upload id while preserving the original extension. # Jupyter selects its editor from this suffix, so an extensionless # object would make notebooks look like generic JSON/text files. - file_extension = PurePosixPath( - safe_file_name(payload.file_name) - ).suffix.lower() + file_extension = PurePosixPath(safe_file_name(payload.file_name)).suffix.lower() object_key = f"{payload.workspace_id}/{upload_id}{file_extension}" upload = UploadSessions( upload_id=upload_id, @@ -223,14 +217,13 @@ async def create_upload_record( file_name=payload.file_name, usage_type=payload.usage_type, visibility=payload.visibility, - is_immutable=int(payload.is_immutable)) + is_immutable=int(payload.is_immutable), + ) session.add(upload) await session.flush() if upload.upload_status == "completed" and upload.storage_object_id: - storage_object = await session.get( - StorageObjects, - upload.storage_object_id) + storage_object = await session.get(StorageObjects, upload.storage_object_id) if storage_object is None or storage_object.object_status != "available": # The previously-completed object was deleted (or never # materialized). Treat the idempotency hit as a tombstone @@ -247,7 +240,8 @@ async def create_upload_record( if upload.upload_status not in {"created", "uploading"}: raise HTTPException( status.HTTP_409_CONFLICT, - f"upload cannot continue from status {upload.upload_status}") + f"upload cannot continue from status {upload.upload_status}", + ) # Two-step server-proxied upload: the caller PUTs the raw bytes to # ``upload_path`` after this response, which routes through @@ -283,9 +277,8 @@ def _public_base_url(request: Request) -> str: async def upload_bytes_to_session( - upload_id: str, - session: AsyncSession, - request: Request) -> StorageObjects: + upload_id: str, session: AsyncSession, request: Request +) -> StorageObjects: """Server-proxied upload: read raw bytes from the request body, validate against the ``UploadSessions`` expectations, call ``backend.put``, and create the ``StorageObjects`` row. @@ -310,7 +303,8 @@ async def upload_bytes_to_session( if upload.upload_status not in {"created", "uploading"}: raise HTTPException( status.HTTP_409_CONFLICT, - f"upload cannot continue from status {upload.upload_status}") + f"upload cannot continue from status {upload.upload_status}", + ) if upload.expires_at < utcnow(): upload.upload_status = "expired" raise HTTPException(status.HTTP_409_CONFLICT, "upload expired") @@ -325,14 +319,15 @@ async def upload_bytes_to_session( upload.upload_status = "failed" raise HTTPException( status.HTTP_409_CONFLICT, - "uploaded bytes size does not match expected_size_bytes") + "uploaded bytes size does not match expected_size_bytes", + ) actual_hash = hashlib.sha256(content).hexdigest() if content else "" if upload.expected_hash and actual_hash != upload.expected_hash: upload.upload_status = "failed" raise HTTPException( - status.HTTP_409_CONFLICT, - "uploaded bytes hash does not match expected_hash") + status.HTTP_409_CONFLICT, "uploaded bytes hash does not match expected_hash" + ) # Round-trip content_type + sha256 metadata through the storage backend # so the next head() (or our own put signature) can recover them. @@ -351,7 +346,8 @@ async def upload_bytes_to_session( upload.upload_status = "failed" raise HTTPException( status.HTTP_503_SERVICE_UNAVAILABLE, - f"failed to write object to storage: {exc}") from exc + f"failed to write object to storage: {exc}", + ) from exc file_name = safe_file_name(upload.file_name_hint or "upload.bin") item = StorageObjects( @@ -374,7 +370,8 @@ async def upload_bytes_to_session( visibility=upload.visibility, is_immutable=int(upload.is_immutable), object_status="available", - created_by=upload.user_id) + created_by=upload.user_id, + ) session.add(item) await session.flush() await session.refresh(item) @@ -388,7 +385,8 @@ async def upload_bytes_to_session( async def create_upload( payload: CreateUploadRequest, request: Request, - session: AsyncSession = Depends(database_session)) -> dict[str, Any]: + session: AsyncSession = Depends(database_session), +) -> dict[str, Any]: return { "data": await create_upload_record(payload, session, request), } @@ -396,9 +394,8 @@ async def create_upload( @app.put("/internal/v1/uploads/{upload_id}") async def upload_bytes( - upload_id: str, - request: Request, - session: AsyncSession = Depends(database_session)) -> dict[str, Any]: + upload_id: str, request: Request, session: AsyncSession = Depends(database_session) +) -> dict[str, Any]: """Server-proxied upload: PUT raw bytes in the request body. Replaces the old ``POST /uploads/{id}/complete`` flow that paired presigned-PUT with a head()-validate step. @@ -407,12 +404,10 @@ async def upload_bytes( return {"data": storage_payload(item)} -@app.post( - "/internal/v1/uploads/{upload_id}/abort") +@app.post("/internal/v1/uploads/{upload_id}/abort") async def abort_upload( - upload_id: str, - request: Request, - session: AsyncSession = Depends(database_session)) -> dict[str, Any]: + upload_id: str, request: Request, session: AsyncSession = Depends(database_session) +) -> dict[str, Any]: upload = await session.scalar( select(UploadSessions) .where(UploadSessions.upload_id == upload_id) @@ -422,52 +417,52 @@ async def abort_upload( raise HTTPException(status.HTTP_404_NOT_FOUND, "upload not found") if upload.upload_status == "completed": raise HTTPException( - status.HTTP_409_CONFLICT, - "completed upload cannot be aborted") + status.HTTP_409_CONFLICT, "completed upload cannot be aborted" + ) if upload.upload_status != "aborted": - await request.app.state.object_stores[ - upload.bucket_name - ].delete(upload.object_key) + await request.app.state.object_stores[upload.bucket_name].delete( + upload.object_key + ) upload.upload_status = "aborted" return {"data": {"upload_id": upload_id, "status": "aborted"}} -@app.post( - "/internal/v1/objects") +@app.post("/internal/v1/objects") async def create_server_object( payload: ServerObjectRequest, request: Request, - session: AsyncSession = Depends(database_session)) -> dict[str, Any]: + session: AsyncSession = Depends(database_session), +) -> dict[str, Any]: return await create_server_object_payload(payload, request, session) -@app.post( - "/internal/v1/objects/{storage_object_id}/download-url") +@app.post("/internal/v1/objects/{storage_object_id}/download-url") async def create_download_url( storage_object_id: str, payload: DownloadUrlRequest, request: Request, - session: AsyncSession = Depends(database_session)) -> dict[str, Any]: + session: AsyncSession = Depends(database_session), +) -> dict[str, Any]: item = await session.get(StorageObjects, storage_object_id) return await create_download_url_payload(item, payload, request) -@app.delete( - "/internal/v1/objects/{storage_object_id}") +@app.delete("/internal/v1/objects/{storage_object_id}") async def delete_object( storage_object_id: str, request: Request, - session: AsyncSession = Depends(database_session)) -> dict[str, Any]: + session: AsyncSession = Depends(database_session), +) -> dict[str, Any]: """Soft-delete a storage object. See ``backend.services.storage.soft_delete_object``.""" return await soft_delete_object(storage_object_id, request, session) -@app.post( - "/internal/v1/objects/{storage_object_id}/restore") +@app.post("/internal/v1/objects/{storage_object_id}/restore") async def restore_object( storage_object_id: str, request: Request, - session: AsyncSession = Depends(database_session)) -> dict[str, Any]: + session: AsyncSession = Depends(database_session), +) -> dict[str, Any]: """Restore a soft-deleted object from the trash bucket. Copies the bytes back to the source bucket + key and flips the @@ -486,13 +481,11 @@ async def restore_object( if item is None: raise HTTPException(status.HTTP_404_NOT_FOUND, "object not found") if item.object_status != "deleted": - raise HTTPException( - status.HTTP_409_CONFLICT, - "object is not in trash") + raise HTTPException(status.HTTP_409_CONFLICT, "object is not in trash") if not item.trash_key or not item.bucket_name or not item.object_key: raise HTTPException( - status.HTTP_409_CONFLICT, - "object has no trash pointer; cannot restore") + status.HTTP_409_CONFLICT, "object has no trash pointer; cannot restore" + ) try: # Cross-backend copy: get from trash, put back to source bucket. object_stores = request.app.state.object_stores @@ -513,12 +506,12 @@ async def restore_object( } -@app.post( - "/internal/v1/admin/trash/purge") +@app.post("/internal/v1/admin/trash/purge") async def purge_trash_object( payload: dict[str, Any], request: Request, - session: AsyncSession = Depends(database_session)) -> dict[str, Any]: + session: AsyncSession = Depends(database_session), +) -> dict[str, Any]: """Physically delete a trashed object. Admin / reaper endpoint — given a ``storage_object_id``, deletes @@ -530,8 +523,8 @@ async def purge_trash_object( storage_object_id = (payload or {}).get("storage_object_id", "").strip() if not storage_object_id: raise HTTPException( - status.HTTP_400_BAD_REQUEST, - "storage_object_id is required") + status.HTTP_400_BAD_REQUEST, "storage_object_id is required" + ) item = await session.scalar( select(StorageObjects) .where(StorageObjects.storage_object_id == storage_object_id) @@ -542,12 +535,13 @@ async def purge_trash_object( if item.object_status != "deleted": raise HTTPException( status.HTTP_409_CONFLICT, - "object is not in trash; refuse to hard-delete live data") + "object is not in trash; refuse to hard-delete live data", + ) if item.trash_key: try: - await request.app.state.object_stores[ - settings.s3_trash_bucket - ].delete(item.trash_key) + await request.app.state.object_stores[settings.s3_trash_bucket].delete( + item.trash_key + ) except Exception as exc: raise HTTPException( status.HTTP_502_BAD_GATEWAY, diff --git a/common/src/common/db/models/__init__.py b/common/src/common/db/models/__init__.py index b7fca0d..c03058d 100644 --- a/common/src/common/db/models/__init__.py +++ b/common/src/common/db/models/__init__.py @@ -1,7 +1,6 @@ from common.db.base import Base from common.db.models.events import ConsumerInbox, OutboxEvents from common.db.models.identity import Permissions, RolePermissions, Roles, Users -from common.db.models.runtime import WorkspaceOperations from common.db.models.schedules import ( ScheduleEdges, ScheduleNodeRuns, @@ -32,6 +31,5 @@ __all__ = [ "Users", "Versions", "WorkspaceMembers", - "WorkspaceOperations", "Workspaces", ] diff --git a/common/src/common/db/models/runtime.py b/common/src/common/db/models/runtime.py deleted file mode 100644 index 1299077..0000000 --- a/common/src/common/db/models/runtime.py +++ /dev/null @@ -1,55 +0,0 @@ -import datetime -from typing import Optional - -from sqlalchemy import Index, String, Text, text -from sqlalchemy.dialects.mysql import CHAR, DATETIME, INTEGER, TINYINT -from sqlalchemy.orm import Mapped, mapped_column - -from common.db.base import Base - - -class WorkspaceOperations(Base): - __tablename__ = "workspace_operations" - __table_args__ = ( - Index("fk_workspace_operations_user", "requested_by"), - Index("idx_workspace_operations_runtime", "runtime_id", "created_at"), - Index( - "idx_workspace_operations_workspace", - "workspace_id", - "operation_status", - "created_at", - ), - Index("uk_workspace_operations_request", "request_id", unique=True), - {"comment": "无状态 Backend 的 Workspace/Jupyter 异步操作记录"}, - ) - - operation_id: Mapped[str] = mapped_column(CHAR(26), primary_key=True) - workspace_id: Mapped[str] = mapped_column(CHAR(26), nullable=False) - operation_type: Mapped[str] = mapped_column( - String(24), - nullable=False, - comment="open/close/mount/unmount/start/stop/restart/recycle", - ) - operation_status: Mapped[str] = mapped_column( - String(24), - nullable=False, - server_default=text("'pending'"), - comment="pending/running/succeeded/failed/cancelled", - ) - state_version: Mapped[int] = mapped_column( - INTEGER, nullable=False, server_default=text("0"), comment="乐观锁版本" - ) - requested_by: Mapped[str] = mapped_column(CHAR(26), nullable=False) - created_at: Mapped[datetime.datetime] = mapped_column( - DATETIME(fsp=3), nullable=False, server_default=text("CURRENT_TIMESTAMP(3)") - ) - runtime_id: Mapped[Optional[str]] = mapped_column(CHAR(26)) - request_id: Mapped[Optional[str]] = mapped_column(String(128)) - started_at: Mapped[Optional[datetime.datetime]] = mapped_column(DATETIME(fsp=3)) - finished_at: Mapped[Optional[datetime.datetime]] = mapped_column(DATETIME(fsp=3)) - error_code: Mapped[Optional[str]] = mapped_column(String(64)) - error_message: Mapped[Optional[str]] = mapped_column(Text) - is_deleted: Mapped[int] = mapped_column( - TINYINT(1), nullable=False, server_default=text("0") - ) - deleted_at: Mapped[Optional[datetime.datetime]] = mapped_column(DATETIME(fsp=3)) diff --git a/migrations/versions/8d86e2f82860_initial_baseline_20_active_tables.py b/migrations/versions/8d86e2f82860_initial_baseline_20_active_tables.py deleted file mode 100644 index b9a6129..0000000 --- a/migrations/versions/8d86e2f82860_initial_baseline_20_active_tables.py +++ /dev/null @@ -1,534 +0,0 @@ -"""initial baseline (19 active tables) - -Revision ID: 8d86e2f82860 -Revises: -Create Date: 2026-07-31 13:17:52.047720 -""" - -from collections.abc import Sequence - -from alembic import op -import sqlalchemy as sa -from sqlalchemy.dialects import mysql - -# revision identifiers, used by Alembic. -revision: str = '8d86e2f82860' -down_revision: str | Sequence[str] | None = None -branch_labels: str | Sequence[str] | None = None -depends_on: str | Sequence[str] | None = None - - -def upgrade() -> None: - """Upgrade schema.""" - # ### commands auto generated by Alembic - please adjust! ### - op.create_table('consumer_inbox', - sa.Column('consumer_name', sa.String(length=128), nullable=False), - sa.Column('event_id', mysql.CHAR(length=26), nullable=False), - sa.Column('process_status', sa.String(length=16), server_default=sa.text("'processing'"), nullable=False, comment='processing/succeeded/failed'), - sa.Column('created_at', mysql.DATETIME(fsp=3), server_default=sa.text('CURRENT_TIMESTAMP(3)'), nullable=False), - sa.Column('message_id', sa.String(length=128), nullable=True, comment='Inbox message ID'), - sa.Column('processed_at', mysql.DATETIME(fsp=3), nullable=True), - sa.Column('error_message', sa.String(length=2000), nullable=True), - sa.Column('is_deleted', mysql.TINYINT(display_width=1), server_default=sa.text('0'), nullable=False), - sa.Column('deleted_at', mysql.DATETIME(fsp=3), nullable=True), - sa.PrimaryKeyConstraint('consumer_name', 'event_id'), - comment='消费者幂等 Inbox，防止 Stream 重投导致重复执行' - ) - op.create_index('idx_consumer_inbox_status', 'consumer_inbox', ['consumer_name', 'process_status', 'created_at'], unique=False) - op.create_table('data_resources', - sa.Column('resource_id', mysql.CHAR(length=26), nullable=False), - sa.Column('workspace_id', mysql.CHAR(length=26), nullable=False), - sa.Column('storage_object_id', mysql.CHAR(length=26), nullable=False), - sa.Column('owner_user_id', mysql.CHAR(length=26), nullable=False), - sa.Column('resource_name', sa.String(length=255), nullable=False), - sa.Column('visibility', sa.String(length=16), server_default=sa.text("'private'"), nullable=False), - sa.Column('status', sa.String(length=16), server_default=sa.text("'active'"), nullable=False), - sa.Column('created_at', mysql.DATETIME(fsp=3), server_default=sa.text('CURRENT_TIMESTAMP(3)'), nullable=False), - sa.Column('updated_at', mysql.DATETIME(fsp=3), server_default=sa.text('CURRENT_TIMESTAMP(3) ON UPDATE CURRENT_TIMESTAMP(3)'), nullable=False), - sa.Column('description', sa.String(length=1000), nullable=True), - sa.Column('schema_json', sa.JSON(), nullable=True, comment='字段结构、行数等可选元数据'), - sa.Column('is_deleted', mysql.TINYINT(display_width=1), server_default=sa.text('0'), nullable=False), - sa.Column('deleted_at', mysql.DATETIME(fsp=3), nullable=True), - sa.PrimaryKeyConstraint('resource_id'), - comment='数据资源' - ) - op.create_index('idx_data_resources_owner', 'data_resources', ['owner_user_id', 'status'], unique=False) - op.create_index('idx_data_resources_workspace', 'data_resources', ['workspace_id', 'visibility', 'status'], unique=False) - op.create_index('uk_data_resources_object', 'data_resources', ['storage_object_id'], unique=True) - op.create_table('outbox_events', - sa.Column('event_id', mysql.CHAR(length=26), nullable=False), - sa.Column('aggregate_type', sa.String(length=64), nullable=False), - sa.Column('aggregate_id', sa.String(length=128), nullable=False), - sa.Column('event_type', sa.String(length=128), nullable=False), - sa.Column('schema_version', mysql.SMALLINT(), server_default=sa.text('1'), nullable=False), - sa.Column('payload_json', sa.JSON(), nullable=False), - sa.Column('event_status', sa.String(length=16), server_default=sa.text("'pending'"), nullable=False, comment='pending/published/failed'), - sa.Column('available_at', mysql.DATETIME(fsp=3), server_default=sa.text('CURRENT_TIMESTAMP(3)'), nullable=False), - sa.Column('retry_count', mysql.INTEGER(), server_default=sa.text('0'), nullable=False), - sa.Column('created_at', mysql.DATETIME(fsp=3), server_default=sa.text('CURRENT_TIMESTAMP(3)'), nullable=False), - sa.Column('trace_id', sa.String(length=64), nullable=True), - sa.Column('idempotency_key', sa.String(length=128), nullable=True), - sa.Column('published_at', mysql.DATETIME(fsp=3), nullable=True), - sa.Column('last_error', sa.String(length=2000), nullable=True), - sa.Column('is_deleted', mysql.TINYINT(display_width=1), server_default=sa.text('0'), nullable=False), - sa.Column('deleted_at', mysql.DATETIME(fsp=3), nullable=True), - sa.PrimaryKeyConstraint('event_id'), - comment='事务 Outbox；提交后发布到内部事件总线' - ) - op.create_index('idx_outbox_aggregate', 'outbox_events', ['aggregate_type', 'aggregate_id', 'created_at'], unique=False) - op.create_index('idx_outbox_idempotency', 'outbox_events', ['idempotency_key'], unique=False) - op.create_index('idx_outbox_pending', 'outbox_events', ['event_status', 'available_at', 'created_at'], unique=False) - op.create_table('permissions', - sa.Column('permission_id', mysql.CHAR(length=26), nullable=False), - sa.Column('permission_code', sa.String(length=128), nullable=False), - sa.Column('permission_name', sa.String(length=100), nullable=False), - sa.Column('module_code', sa.String(length=64), nullable=False), - sa.Column('created_at', mysql.DATETIME(fsp=3), server_default=sa.text('CURRENT_TIMESTAMP(3)'), nullable=False), - sa.Column('description', sa.String(length=500), nullable=True), - sa.Column('is_deleted', mysql.TINYINT(display_width=1), server_default=sa.text('0'), nullable=False), - sa.Column('deleted_at', mysql.DATETIME(fsp=3), nullable=True), - sa.PrimaryKeyConstraint('permission_id'), - comment='权限点' - ) - op.create_index('idx_permissions_module', 'permissions', ['module_code'], unique=False) - op.create_index('uk_permissions_code', 'permissions', ['permission_code'], unique=True) - op.create_table('role_permissions', - sa.Column('role_id', mysql.CHAR(length=26), nullable=False), - sa.Column('permission_id', mysql.CHAR(length=26), nullable=False), - sa.Column('created_at', mysql.DATETIME(fsp=3), server_default=sa.text('CURRENT_TIMESTAMP(3)'), nullable=False), - sa.Column('is_deleted', mysql.TINYINT(display_width=1), server_default=sa.text('0'), nullable=False), - sa.Column('deleted_at', mysql.DATETIME(fsp=3), nullable=True), - sa.PrimaryKeyConstraint('role_id', 'permission_id'), - comment='角色权限' - ) - op.create_index('fk_role_permissions_permission', 'role_permissions', ['permission_id'], unique=False) - op.create_table('roles', - sa.Column('role_id', mysql.CHAR(length=26), nullable=False), - sa.Column('role_code', sa.String(length=64), nullable=False), - sa.Column('role_name', sa.String(length=100), nullable=False), - sa.Column('role_scope', sa.String(length=16), nullable=False, comment='platform/workspace'), - sa.Column('is_builtin', mysql.TINYINT(display_width=1), server_default=sa.text('0'), nullable=False), - sa.Column('created_at', mysql.DATETIME(fsp=3), server_default=sa.text('CURRENT_TIMESTAMP(3)'), nullable=False), - sa.Column('updated_at', mysql.DATETIME(fsp=3), server_default=sa.text('CURRENT_TIMESTAMP(3) ON UPDATE CURRENT_TIMESTAMP(3)'), nullable=False), - sa.Column('description', sa.String(length=500), nullable=True), - sa.Column('is_deleted', mysql.TINYINT(display_width=1), server_default=sa.text('0'), nullable=False), - sa.Column('deleted_at', mysql.DATETIME(fsp=3), nullable=True), - sa.PrimaryKeyConstraint('role_id'), - comment='角色' - ) - op.create_index('uk_roles_code', 'roles', ['role_code'], unique=True) - op.create_table('schedule_edges', - sa.Column('edge_id', mysql.CHAR(length=26), nullable=False), - sa.Column('schedule_id', mysql.CHAR(length=26), nullable=False), - sa.Column('source_node_id', mysql.CHAR(length=26), nullable=False), - sa.Column('target_node_id', mysql.CHAR(length=26), nullable=False), - sa.Column('created_at', mysql.DATETIME(fsp=3), server_default=sa.text('CURRENT_TIMESTAMP(3)'), nullable=False), - sa.Column('condition_expr', sa.String(length=1000), nullable=True), - sa.Column('is_deleted', mysql.TINYINT(display_width=1), server_default=sa.text('0'), nullable=False), - sa.Column('deleted_at', mysql.DATETIME(fsp=3), nullable=True), - sa.PrimaryKeyConstraint('edge_id'), - comment='DAG 有向边' - ) - op.create_index('fk_schedule_edges_source', 'schedule_edges', ['source_node_id'], unique=False) - op.create_index('idx_schedule_edges_target', 'schedule_edges', ['target_node_id'], unique=False) - op.create_index('uk_schedule_edges_pair', 'schedule_edges', ['schedule_id', 'source_node_id', 'target_node_id'], unique=True) - op.create_table('schedule_node_runs', - sa.Column('node_run_id', mysql.CHAR(length=26), nullable=False), - sa.Column('run_id', mysql.CHAR(length=26), nullable=False), - sa.Column('node_id', mysql.CHAR(length=26), nullable=False), - sa.Column('versions_id', mysql.CHAR(length=26), nullable=False), - sa.Column('attempt_no', mysql.INTEGER(), server_default=sa.text('1'), nullable=False), - sa.Column('node_status', sa.String(length=24), server_default=sa.text("'queued'"), nullable=False, comment='queued/running/succeeded/failed/skipped/cancelled/timed_out'), - sa.Column('state_version', mysql.INTEGER(), server_default=sa.text('0'), nullable=False, comment='乐观锁版本'), - sa.Column('created_at', mysql.DATETIME(fsp=3), server_default=sa.text('CURRENT_TIMESTAMP(3)'), nullable=False), - sa.Column('started_at', mysql.DATETIME(fsp=3), nullable=True), - sa.Column('finished_at', mysql.DATETIME(fsp=3), nullable=True), - sa.Column('duration_ms', mysql.BIGINT(), nullable=True), - sa.Column('exit_code', sa.Integer(), nullable=True), - sa.Column('message', sa.String(length=2000), nullable=True), - sa.Column('metrics_json', sa.JSON(), nullable=True), - sa.Column('logs_object_id', mysql.CHAR(length=26), nullable=True), - sa.Column('result_object_id', mysql.CHAR(length=26), nullable=True), - sa.Column('is_deleted', mysql.TINYINT(display_width=1), server_default=sa.text('0'), nullable=False), - sa.Column('deleted_at', mysql.DATETIME(fsp=3), nullable=True), - sa.PrimaryKeyConstraint('node_run_id'), - comment='调度节点运行与重试' - ) - op.create_index('fk_node_runs_logs', 'schedule_node_runs', ['logs_object_id'], unique=False) - op.create_index('fk_node_runs_node', 'schedule_node_runs', ['node_id'], unique=False) - op.create_index('fk_node_runs_result', 'schedule_node_runs', ['result_object_id'], unique=False) - op.create_index('idx_node_runs_status', 'schedule_node_runs', ['run_id', 'node_status'], unique=False) - op.create_index('idx_node_runs_version', 'schedule_node_runs', ['versions_id'], unique=False) - op.create_index('uk_node_runs_attempt', 'schedule_node_runs', ['run_id', 'node_id', 'attempt_no'], unique=True) - op.create_table('schedule_nodes', - sa.Column('node_id', mysql.CHAR(length=26), nullable=False), - sa.Column('schedule_id', mysql.CHAR(length=26), nullable=False), - sa.Column('node_key', sa.String(length=64), nullable=False, comment='画布内稳定标识'), - sa.Column('node_name', sa.String(length=255), nullable=False), - sa.Column('versions_id', mysql.CHAR(length=26), nullable=False), - sa.Column('timeout_seconds', mysql.INTEGER(), server_default=sa.text('600'), nullable=False), - sa.Column('retry_count', mysql.INTEGER(), server_default=sa.text('0'), nullable=False), - sa.Column('retry_interval_sec', mysql.INTEGER(), server_default=sa.text('5'), nullable=False), - sa.Column('position_x', sa.DECIMAL(precision=10, scale=2), server_default=sa.text('0.00'), nullable=False), - sa.Column('position_y', sa.DECIMAL(precision=10, scale=2), server_default=sa.text('0.00'), nullable=False), - sa.Column('created_at', mysql.DATETIME(fsp=3), server_default=sa.text('CURRENT_TIMESTAMP(3)'), nullable=False), - sa.Column('updated_at', mysql.DATETIME(fsp=3), server_default=sa.text('CURRENT_TIMESTAMP(3) ON UPDATE CURRENT_TIMESTAMP(3)'), nullable=False), - sa.Column('arguments_json', sa.JSON(), nullable=True), - sa.Column('env_refs_json', sa.JSON(), nullable=True, comment='只存密钥引用，不存明文密钥'), - sa.Column('is_deleted', mysql.TINYINT(display_width=1), server_default=sa.text('0'), nullable=False), - sa.Column('deleted_at', mysql.DATETIME(fsp=3), nullable=True), - sa.PrimaryKeyConstraint('node_id'), - comment='DAG 节点，必须引用稳定版本' - ) - op.create_index('idx_schedule_nodes_version', 'schedule_nodes', ['versions_id'], unique=False) - op.create_index('uk_schedule_nodes_key', 'schedule_nodes', ['schedule_id', 'node_key'], unique=True) - op.create_table('schedule_runs', - sa.Column('run_id', mysql.CHAR(length=26), nullable=False), - sa.Column('schedule_id', mysql.CHAR(length=26), nullable=False), - sa.Column('workspace_id', mysql.CHAR(length=26), nullable=False), - sa.Column('workflow_version', mysql.INTEGER(), nullable=False), - sa.Column('trigger_type', sa.String(length=16), nullable=False, comment='manual/cron/api/retry'), - sa.Column('idempotency_key', sa.String(length=128), nullable=False), - sa.Column('run_status', sa.String(length=24), server_default=sa.text("'queued'"), nullable=False, comment='queued/running/succeeded/failed/cancelled/timed_out'), - sa.Column('state_version', mysql.INTEGER(), server_default=sa.text('0'), nullable=False, comment='乐观锁版本'), - sa.Column('schedule_snapshot', sa.JSON(), nullable=False, comment='执行时 DAG 快照'), - sa.Column('queued_at', mysql.DATETIME(fsp=3), server_default=sa.text('CURRENT_TIMESTAMP(3)'), nullable=False), - sa.Column('created_at', mysql.DATETIME(fsp=3), server_default=sa.text('CURRENT_TIMESTAMP(3)'), nullable=False), - sa.Column('triggered_by', mysql.CHAR(length=26), nullable=True), - sa.Column('started_at', mysql.DATETIME(fsp=3), nullable=True), - sa.Column('finished_at', mysql.DATETIME(fsp=3), nullable=True), - sa.Column('duration_ms', mysql.BIGINT(), nullable=True), - sa.Column('error_code', sa.String(length=64), nullable=True), - sa.Column('error_message', sa.Text(), nullable=True), - sa.Column('logs_object_id', mysql.CHAR(length=26), nullable=True), - sa.Column('result_object_id', mysql.CHAR(length=26), nullable=True), - sa.Column('is_deleted', mysql.TINYINT(display_width=1), server_default=sa.text('0'), nullable=False), - sa.Column('deleted_at', mysql.DATETIME(fsp=3), nullable=True), - sa.PrimaryKeyConstraint('run_id'), - comment='调度运行' - ) - op.create_index('fk_schedule_runs_logs', 'schedule_runs', ['logs_object_id'], unique=False) - op.create_index('fk_schedule_runs_result', 'schedule_runs', ['result_object_id'], unique=False) - op.create_index('fk_schedule_runs_user', 'schedule_runs', ['triggered_by'], unique=False) - op.create_index('idx_schedule_runs_schedule', 'schedule_runs', ['schedule_id', 'created_at'], unique=False) - op.create_index('idx_schedule_runs_status', 'schedule_runs', ['run_status', 'queued_at'], unique=False) - op.create_index('idx_schedule_runs_workspace_status', 'schedule_runs', ['workspace_id', 'run_status', 'queued_at'], unique=False) - op.create_index('uk_schedule_runs_idempotency', 'schedule_runs', ['idempotency_key'], unique=True) - op.create_table('schedules', - sa.Column('schedule_id', mysql.CHAR(length=26), nullable=False), - sa.Column('workspace_id', mysql.CHAR(length=26), nullable=False), - sa.Column('schedule_name', sa.String(length=255), nullable=False), - sa.Column('trigger_type', sa.String(length=16), server_default=sa.text("'cron'"), nullable=False, comment='manual/cron/api'), - sa.Column('timezone', sa.String(length=64), server_default=sa.text("'Asia/Shanghai'"), nullable=False), - sa.Column('enabled', mysql.TINYINT(display_width=1), server_default=sa.text('0'), nullable=False), - sa.Column('workflow_version', mysql.INTEGER(), server_default=sa.text('1'), nullable=False), - sa.Column('max_concurrency', mysql.INTEGER(), server_default=sa.text('1'), nullable=False), - sa.Column('failure_policy', sa.String(length=24), server_default=sa.text("'stop'"), nullable=False, comment='stop/continue'), - sa.Column('created_by', mysql.CHAR(length=26), nullable=False), - sa.Column('updated_by', mysql.CHAR(length=26), nullable=False), - sa.Column('created_at', mysql.DATETIME(fsp=3), server_default=sa.text('CURRENT_TIMESTAMP(3)'), nullable=False), - sa.Column('updated_at', mysql.DATETIME(fsp=3), server_default=sa.text('CURRENT_TIMESTAMP(3) ON UPDATE CURRENT_TIMESTAMP(3)'), nullable=False), - sa.Column('description', sa.String(length=1000), nullable=True), - sa.Column('cron_expression', sa.String(length=128), nullable=True), - sa.Column('last_run_at', mysql.DATETIME(fsp=3), nullable=True), - sa.Column('next_run_at', mysql.DATETIME(fsp=3), nullable=True), - sa.Column('is_deleted', mysql.TINYINT(display_width=1), server_default=sa.text('0'), nullable=False), - sa.Column('deleted_at', mysql.DATETIME(fsp=3), nullable=True), - sa.PrimaryKeyConstraint('schedule_id'), - comment='调度方案' - ) - op.create_index('fk_schedules_created_by', 'schedules', ['created_by'], unique=False) - op.create_index('fk_schedules_updated_by', 'schedules', ['updated_by'], unique=False) - op.create_index('idx_schedules_due', 'schedules', ['enabled', 'next_run_at'], unique=False) - op.create_index('idx_schedules_workspace', 'schedules', ['workspace_id', 'enabled', 'updated_at'], unique=False) - op.create_table('scripts', - sa.Column('script_id', mysql.CHAR(length=26), nullable=False), - sa.Column('workspace_id', mysql.CHAR(length=26), nullable=False), - sa.Column('current_object_id', mysql.CHAR(length=26), nullable=False, comment='当前工作副本'), - sa.Column('owner_user_id', mysql.CHAR(length=26), nullable=False), - sa.Column('script_name', sa.String(length=255), nullable=False), - sa.Column('script_type', sa.String(length=16), nullable=False, comment='python/notebook'), - sa.Column('visibility', sa.String(length=16), server_default=sa.text("'private'"), nullable=False), - sa.Column('status', sa.String(length=16), server_default=sa.text("'active'"), nullable=False), - sa.Column('created_at', mysql.DATETIME(fsp=3), server_default=sa.text('CURRENT_TIMESTAMP(3)'), nullable=False), - sa.Column('updated_at', mysql.DATETIME(fsp=3), server_default=sa.text('CURRENT_TIMESTAMP(3) ON UPDATE CURRENT_TIMESTAMP(3)'), nullable=False), - sa.Column('is_deleted', mysql.TINYINT(display_width=1), server_default=sa.text('0'), nullable=False), - sa.Column('is_locked', mysql.TINYINT(display_width=1), server_default=sa.text('1'), nullable=False), - sa.Column('deleted_at', mysql.DATETIME(fsp=3), nullable=True), - sa.PrimaryKeyConstraint('script_id'), - comment='可执行 Python/Notebook 脚本' - ) - op.create_index('idx_scripts_owner', 'scripts', ['owner_user_id', 'status'], unique=False) - op.create_index('idx_scripts_workspace', 'scripts', ['workspace_id', 'script_type', 'visibility', 'status'], unique=False) - op.create_index('uk_scripts_current_object', 'scripts', ['current_object_id'], unique=True) - op.create_index('uk_scripts_workspace_name', 'scripts', ['workspace_id', 'script_name', 'script_type'], unique=True) - op.create_table('storage_objects', - sa.Column('storage_object_id', mysql.CHAR(length=26), nullable=False), - sa.Column('workspace_id', mysql.CHAR(length=26), nullable=False), - sa.Column('object_type', sa.String(length=16), nullable=False, comment='file/directory'), - sa.Column('usage_type', sa.String(length=32), nullable=False, comment='working_copy/public_script/data_resource/version_artifact/snapshot/run_log/run_result'), - sa.Column('storage_backend', sa.String(length=16), nullable=False, comment='rustfs'), - sa.Column('storage_uri', sa.String(length=1500), nullable=False), - sa.Column('file_name', sa.String(length=255), nullable=False), - sa.Column('size_bytes', mysql.BIGINT(), server_default=sa.text('0'), nullable=False), - sa.Column('visibility', sa.String(length=16), server_default=sa.text("'private'"), nullable=False, comment='private/workspace/public'), - sa.Column('is_immutable', mysql.TINYINT(display_width=1), server_default=sa.text('0'), nullable=False), - sa.Column('object_status', sa.String(length=24), server_default=sa.text("'available'"), nullable=False, comment='uploading/available/deleting/deleted/failed'), - sa.Column('created_by', mysql.CHAR(length=26), nullable=False), - sa.Column('created_at', mysql.DATETIME(fsp=3), server_default=sa.text('CURRENT_TIMESTAMP(3)'), nullable=False), - sa.Column('updated_at', mysql.DATETIME(fsp=3), server_default=sa.text('CURRENT_TIMESTAMP(3) ON UPDATE CURRENT_TIMESTAMP(3)'), nullable=False), - sa.Column('owner_user_id', mysql.CHAR(length=26), nullable=True), - sa.Column('parent_object_id', mysql.CHAR(length=26), nullable=True), - sa.Column('relative_path', sa.String(length=1024), nullable=True, comment='Workspace 相对路径'), - sa.Column('path_hash', sa.BINARY(length=32), nullable=True, comment='SHA-256(relative_path)，由应用写入'), - sa.Column('bucket_name', sa.String(length=128), nullable=True), - sa.Column('object_key', sa.String(length=1024), nullable=True), - sa.Column('object_key_hash', sa.BINARY(length=32), nullable=True, comment='SHA-256(object_key)，由应用写入'), - sa.Column('file_extension', sa.String(length=32), nullable=True), - sa.Column('mime_type', sa.String(length=255), nullable=True), - sa.Column('content_hash', mysql.CHAR(length=64), nullable=True, comment='SHA-256 hex'), - sa.Column('object_etag', sa.String(length=255), nullable=True), - sa.Column('is_deleted', mysql.TINYINT(display_width=1), server_default=sa.text('0'), nullable=False), - sa.Column('deleted_at', mysql.DATETIME(fsp=3), nullable=True), - sa.PrimaryKeyConstraint('storage_object_id'), - comment='Workspace 文件和 RustFS 对象的统一元数据' - ) - op.create_index('fk_storage_created_by', 'storage_objects', ['created_by'], unique=False) - op.create_index('idx_storage_content_hash', 'storage_objects', ['content_hash'], unique=False) - op.create_index('idx_storage_owner', 'storage_objects', ['owner_user_id', 'object_status'], unique=False) - op.create_index('idx_storage_parent', 'storage_objects', ['parent_object_id'], unique=False) - op.create_index('idx_storage_workspace_usage', 'storage_objects', ['workspace_id', 'usage_type', 'object_status'], unique=False) - op.create_index('uk_storage_bucket_key', 'storage_objects', ['storage_backend', 'bucket_name', 'object_key_hash'], unique=True) - op.create_index('uk_storage_workspace_path', 'storage_objects', ['workspace_id', 'storage_backend', 'path_hash'], unique=True) - op.create_table('upload_sessions', - sa.Column('upload_id', mysql.CHAR(length=26), nullable=False), - sa.Column('workspace_id', mysql.CHAR(length=26), nullable=False), - sa.Column('user_id', mysql.CHAR(length=26), nullable=False), - sa.Column('idempotency_key', sa.String(length=128), nullable=False), - sa.Column('bucket_name', sa.String(length=128), nullable=False), - sa.Column('object_key', sa.String(length=1024), nullable=False), - sa.Column('object_key_hash', sa.BINARY(length=32), nullable=False), - sa.Column('upload_status', sa.String(length=24), server_default=sa.text("'created'"), nullable=False, comment='created/uploading/completed/expired/aborted/failed'), - sa.Column('expires_at', mysql.DATETIME(fsp=3), nullable=False), - sa.Column('created_at', mysql.DATETIME(fsp=3), server_default=sa.text('CURRENT_TIMESTAMP(3)'), nullable=False), - sa.Column('updated_at', mysql.DATETIME(fsp=3), server_default=sa.text('CURRENT_TIMESTAMP(3) ON UPDATE CURRENT_TIMESTAMP(3)'), nullable=False), - sa.Column('multipart_upload_id', sa.String(length=255), nullable=True), - sa.Column('expected_size_bytes', mysql.BIGINT(), nullable=True), - sa.Column('expected_hash', mysql.CHAR(length=64), nullable=True), - sa.Column('content_type', sa.String(length=255), nullable=True), - sa.Column('storage_object_id', mysql.CHAR(length=26), nullable=True), - sa.Column('completed_at', mysql.DATETIME(fsp=3), nullable=True), - sa.Column('is_deleted', mysql.TINYINT(display_width=1), server_default=sa.text('0'), nullable=False), - sa.Column('deleted_at', mysql.DATETIME(fsp=3), nullable=True), - sa.PrimaryKeyConstraint('upload_id'), - comment='RustFS 预签名上传会话；URL 本身不持久化' - ) - op.create_index('fk_upload_sessions_storage_object', 'upload_sessions', ['storage_object_id'], unique=False) - op.create_index('fk_upload_sessions_user', 'upload_sessions', ['user_id'], unique=False) - op.create_index('idx_upload_sessions_expiry', 'upload_sessions', ['upload_status', 'expires_at'], unique=False) - op.create_index('idx_upload_sessions_object_key', 'upload_sessions', ['bucket_name', 'object_key_hash'], unique=False) - op.create_index('idx_upload_sessions_workspace', 'upload_sessions', ['workspace_id', 'user_id', 'created_at'], unique=False) - op.create_index('uk_upload_sessions_idempotency', 'upload_sessions', ['idempotency_key'], unique=True) - op.create_table('users', - sa.Column('user_id', mysql.CHAR(length=26), nullable=False), - sa.Column('username', sa.String(length=64), nullable=False), - sa.Column('display_name', sa.String(length=100), nullable=False), - sa.Column('password_hash', sa.String(length=255), nullable=False), - sa.Column('status', sa.String(length=16), server_default=sa.text("'active'"), nullable=False, comment='active/disabled/locked'), - sa.Column('created_at', mysql.DATETIME(fsp=3), server_default=sa.text('CURRENT_TIMESTAMP(3)'), nullable=False), - sa.Column('updated_at', mysql.DATETIME(fsp=3), server_default=sa.text('CURRENT_TIMESTAMP(3) ON UPDATE CURRENT_TIMESTAMP(3)'), nullable=False), - sa.Column('email', sa.String(length=255), nullable=True), - sa.Column('platform_role_id', mysql.CHAR(length=26), nullable=True), - sa.Column('avatar_uri', sa.String(length=1000), nullable=True), - sa.Column('last_login_at', mysql.DATETIME(fsp=3), nullable=True), - sa.Column('is_deleted', mysql.TINYINT(display_width=1), server_default=sa.text('0'), nullable=False), - sa.Column('deleted_at', mysql.DATETIME(fsp=3), nullable=True), - sa.PrimaryKeyConstraint('user_id'), - comment='平台用户' - ) - op.create_index('fk_users_platform_role', 'users', ['platform_role_id'], unique=False) - op.create_index('idx_users_status', 'users', ['status'], unique=False) - op.create_index('uk_users_email', 'users', ['email'], unique=True) - op.create_index('uk_users_username', 'users', ['username'], unique=True) - op.create_table('versions', - sa.Column('versions_id', mysql.CHAR(length=26), nullable=False, comment='稳定版本唯一 ID'), - sa.Column('workspace_id', mysql.CHAR(length=26), nullable=False), - sa.Column('script_id', mysql.CHAR(length=26), nullable=False), - sa.Column('source_object_id', mysql.CHAR(length=26), nullable=False, comment='发布时的源对象'), - sa.Column('artifact_object_id', mysql.CHAR(length=26), nullable=False, comment='RustFS 不可变版本制品'), - sa.Column('version_no', mysql.INTEGER(), nullable=False), - sa.Column('version_label', sa.String(length=32), nullable=False, comment='例如 v1.0'), - sa.Column('source_path', sa.String(length=1024), nullable=False, comment='发布时路径快照'), - sa.Column('artifact_path', sa.String(length=1500), nullable=False), - sa.Column('content_hash', mysql.CHAR(length=64), nullable=False), - sa.Column('file_size_bytes', mysql.BIGINT(), server_default=sa.text('0'), nullable=False), - sa.Column('visibility', sa.String(length=16), server_default=sa.text("'private'"), nullable=False), - sa.Column('created_by', mysql.CHAR(length=26), nullable=False), - sa.Column('created_at', mysql.DATETIME(fsp=3), server_default=sa.text('CURRENT_TIMESTAMP(3)'), nullable=False), - sa.Column('release_note', sa.String(length=1000), nullable=True), - sa.Column('schedule_hidden_at', mysql.DATETIME(fsp=3), nullable=True, comment='从调度稳定版本列表移除的时间；不影响版本和运行历史'), - sa.Column('is_deleted', mysql.TINYINT(display_width=1), server_default=sa.text('0'), nullable=False), - sa.Column('deleted_at', mysql.DATETIME(fsp=3), nullable=True), - sa.PrimaryKeyConstraint('versions_id'), - comment='不可变稳定版本；调度节点必须引用 versions_id' - ) - op.create_index('fk_versions_source_object', 'versions', ['source_object_id'], unique=False) - op.create_index('idx_versions_creator', 'versions', ['created_by', 'created_at'], unique=False) - op.create_index('idx_versions_workspace_created', 'versions', ['workspace_id', 'created_at'], unique=False) - op.create_index('uk_versions_artifact', 'versions', ['artifact_object_id'], unique=True) - op.create_index('uk_versions_script_hash', 'versions', ['script_id', 'content_hash'], unique=True) - op.create_index('uk_versions_script_no', 'versions', ['script_id', 'version_no'], unique=True) - op.create_table('workspace_members', - sa.Column('workspace_id', mysql.CHAR(length=26), nullable=False), - sa.Column('user_id', mysql.CHAR(length=26), nullable=False), - sa.Column('role_id', mysql.CHAR(length=26), nullable=False), - sa.Column('member_status', sa.String(length=16), server_default=sa.text("'active'"), nullable=False), - sa.Column('joined_at', mysql.DATETIME(fsp=3), server_default=sa.text('CURRENT_TIMESTAMP(3)'), nullable=False), - sa.Column('updated_at', mysql.DATETIME(fsp=3), server_default=sa.text('CURRENT_TIMESTAMP(3) ON UPDATE CURRENT_TIMESTAMP(3)'), nullable=False), - sa.Column('is_deleted', mysql.TINYINT(display_width=1), server_default=sa.text('0'), nullable=False), - sa.Column('deleted_at', mysql.DATETIME(fsp=3), nullable=True), - sa.PrimaryKeyConstraint('workspace_id', 'user_id'), - comment='Workspace 成员与角色' - ) - op.create_index('idx_workspace_members_role', 'workspace_members', ['role_id'], unique=False) - op.create_index('idx_workspace_members_user', 'workspace_members', ['user_id', 'member_status'], unique=False) - op.create_table('workspace_operations', - sa.Column('operation_id', mysql.CHAR(length=26), nullable=False), - sa.Column('workspace_id', mysql.CHAR(length=26), nullable=False), - sa.Column('operation_type', sa.String(length=24), nullable=False, comment='open/close/mount/unmount/start/stop/restart/recycle'), - sa.Column('operation_status', sa.String(length=24), server_default=sa.text("'pending'"), nullable=False, comment='pending/running/succeeded/failed/cancelled'), - sa.Column('state_version', mysql.INTEGER(), server_default=sa.text('0'), nullable=False, comment='乐观锁版本'), - sa.Column('requested_by', mysql.CHAR(length=26), nullable=False), - sa.Column('created_at', mysql.DATETIME(fsp=3), server_default=sa.text('CURRENT_TIMESTAMP(3)'), nullable=False), - sa.Column('runtime_id', mysql.CHAR(length=26), nullable=True), - sa.Column('request_id', sa.String(length=128), nullable=True), - sa.Column('started_at', mysql.DATETIME(fsp=3), nullable=True), - sa.Column('finished_at', mysql.DATETIME(fsp=3), nullable=True), - sa.Column('error_code', sa.String(length=64), nullable=True), - sa.Column('error_message', sa.Text(), nullable=True), - sa.Column('is_deleted', mysql.TINYINT(display_width=1), server_default=sa.text('0'), nullable=False), - sa.Column('deleted_at', mysql.DATETIME(fsp=3), nullable=True), - sa.PrimaryKeyConstraint('operation_id'), - comment='无状态 Backend 的 Workspace/Jupyter 异步操作记录' - ) - op.create_index('fk_workspace_operations_user', 'workspace_operations', ['requested_by'], unique=False) - op.create_index('idx_workspace_operations_runtime', 'workspace_operations', ['runtime_id', 'created_at'], unique=False) - op.create_index('idx_workspace_operations_workspace', 'workspace_operations', ['workspace_id', 'operation_status', 'created_at'], unique=False) - op.create_index('uk_workspace_operations_request', 'workspace_operations', ['request_id'], unique=True) - op.create_table('workspaces', - sa.Column('workspace_id', mysql.CHAR(length=26), nullable=False), - sa.Column('workspace_code', sa.String(length=64), nullable=False), - sa.Column('workspace_name', sa.String(length=150), nullable=False), - sa.Column('active_root_uri', sa.String(length=1500), nullable=False, comment='活动工作区，建议 NFS/PVC/file URI'), - sa.Column('quota_bytes', mysql.BIGINT(), server_default=sa.text('0'), nullable=False, comment='0 表示不限额'), - sa.Column('used_bytes', mysql.BIGINT(), server_default=sa.text('0'), nullable=False), - sa.Column('status', sa.String(length=24), server_default=sa.text("'active'"), nullable=False, comment='creating/active/suspended/deleting/deleted'), - sa.Column('created_by', mysql.CHAR(length=26), nullable=False), - sa.Column('created_at', mysql.DATETIME(fsp=3), server_default=sa.text('CURRENT_TIMESTAMP(3)'), nullable=False), - sa.Column('updated_at', mysql.DATETIME(fsp=3), server_default=sa.text('CURRENT_TIMESTAMP(3) ON UPDATE CURRENT_TIMESTAMP(3)'), nullable=False), - sa.Column('description', sa.String(length=1000), nullable=True), - sa.Column('artifact_bucket', sa.String(length=128), nullable=True, comment='RustFS bucket'), - sa.Column('artifact_prefix', sa.String(length=512), nullable=True, comment='RustFS object key prefix'), - sa.Column('is_deleted', mysql.TINYINT(display_width=1), server_default=sa.text('0'), nullable=False), - sa.Column('deleted_at', mysql.DATETIME(fsp=3), nullable=True), - sa.PrimaryKeyConstraint('workspace_id'), - comment='Workspace' - ) - op.create_index('fk_workspaces_created_by', 'workspaces', ['created_by'], unique=False) - op.create_index('idx_workspaces_status', 'workspaces', ['status'], unique=False) - op.create_index('uk_workspaces_code', 'workspaces', ['workspace_code'], unique=True) - # ### end Alembic commands ### - - -def downgrade() -> None: - """Downgrade schema.""" - # ### commands auto generated by Alembic - please adjust! ### - op.drop_index('uk_workspaces_code', table_name='workspaces') - op.drop_index('idx_workspaces_status', table_name='workspaces') - op.drop_index('fk_workspaces_created_by', table_name='workspaces') - op.drop_table('workspaces') - op.drop_index('uk_workspace_operations_request', table_name='workspace_operations') - op.drop_index('idx_workspace_operations_workspace', table_name='workspace_operations') - op.drop_index('idx_workspace_operations_runtime', table_name='workspace_operations') - op.drop_index('fk_workspace_operations_user', table_name='workspace_operations') - op.drop_table('workspace_operations') - op.drop_index('idx_workspace_members_user', table_name='workspace_members') - op.drop_index('idx_workspace_members_role', table_name='workspace_members') - op.drop_table('workspace_members') - op.drop_index('uk_versions_script_no', table_name='versions') - op.drop_index('uk_versions_script_hash', table_name='versions') - op.drop_index('uk_versions_artifact', table_name='versions') - op.drop_index('idx_versions_workspace_created', table_name='versions') - op.drop_index('idx_versions_creator', table_name='versions') - op.drop_index('fk_versions_source_object', table_name='versions') - op.drop_table('versions') - op.drop_index('uk_users_username', table_name='users') - op.drop_index('uk_users_email', table_name='users') - op.drop_index('idx_users_status', table_name='users') - op.drop_index('fk_users_platform_role', table_name='users') - op.drop_table('users') - op.drop_index('uk_upload_sessions_idempotency', table_name='upload_sessions') - op.drop_index('idx_upload_sessions_workspace', table_name='upload_sessions') - op.drop_index('idx_upload_sessions_object_key', table_name='upload_sessions') - op.drop_index('idx_upload_sessions_expiry', table_name='upload_sessions') - op.drop_index('fk_upload_sessions_user', table_name='upload_sessions') - op.drop_index('fk_upload_sessions_storage_object', table_name='upload_sessions') - op.drop_table('upload_sessions') - op.drop_index('uk_storage_workspace_path', table_name='storage_objects') - op.drop_index('uk_storage_bucket_key', table_name='storage_objects') - op.drop_index('idx_storage_workspace_usage', table_name='storage_objects') - op.drop_index('idx_storage_parent', table_name='storage_objects') - op.drop_index('idx_storage_owner', table_name='storage_objects') - op.drop_index('idx_storage_content_hash', table_name='storage_objects') - op.drop_index('fk_storage_created_by', table_name='storage_objects') - op.drop_table('storage_objects') - op.drop_index('uk_scripts_workspace_name', table_name='scripts') - op.drop_index('uk_scripts_current_object', table_name='scripts') - op.drop_index('idx_scripts_workspace', table_name='scripts') - op.drop_index('idx_scripts_owner', table_name='scripts') - op.drop_table('scripts') - op.drop_index('idx_schedules_workspace', table_name='schedules') - op.drop_index('idx_schedules_due', table_name='schedules') - op.drop_index('fk_schedules_updated_by', table_name='schedules') - op.drop_index('fk_schedules_created_by', table_name='schedules') - op.drop_table('schedules') - op.drop_index('uk_schedule_runs_idempotency', table_name='schedule_runs') - op.drop_index('idx_schedule_runs_workspace_status', table_name='schedule_runs') - op.drop_index('idx_schedule_runs_status', table_name='schedule_runs') - op.drop_index('idx_schedule_runs_schedule', table_name='schedule_runs') - op.drop_index('fk_schedule_runs_user', table_name='schedule_runs') - op.drop_index('fk_schedule_runs_result', table_name='schedule_runs') - op.drop_index('fk_schedule_runs_logs', table_name='schedule_runs') - op.drop_table('schedule_runs') - op.drop_index('uk_schedule_nodes_key', table_name='schedule_nodes') - op.drop_index('idx_schedule_nodes_version', table_name='schedule_nodes') - op.drop_table('schedule_nodes') - op.drop_index('uk_node_runs_attempt', table_name='schedule_node_runs') - op.drop_index('idx_node_runs_version', table_name='schedule_node_runs') - op.drop_index('idx_node_runs_status', table_name='schedule_node_runs') - op.drop_index('fk_node_runs_result', table_name='schedule_node_runs') - op.drop_index('fk_node_runs_node', table_name='schedule_node_runs') - op.drop_index('fk_node_runs_logs', table_name='schedule_node_runs') - op.drop_table('schedule_node_runs') - op.drop_index('uk_schedule_edges_pair', table_name='schedule_edges') - op.drop_index('idx_schedule_edges_target', table_name='schedule_edges') - op.drop_index('fk_schedule_edges_source', table_name='schedule_edges') - op.drop_table('schedule_edges') - op.drop_index('uk_roles_code', table_name='roles') - op.drop_table('roles') - op.drop_index('fk_role_permissions_permission', table_name='role_permissions') - op.drop_table('role_permissions') - op.drop_index('uk_permissions_code', table_name='permissions') - op.drop_index('idx_permissions_module', table_name='permissions') - op.drop_table('permissions') - op.drop_index('idx_outbox_pending', table_name='outbox_events') - op.drop_index('idx_outbox_idempotency', table_name='outbox_events') - op.drop_index('idx_outbox_aggregate', table_name='outbox_events') - op.drop_table('outbox_events') - op.drop_index('uk_data_resources_object', table_name='data_resources') - op.drop_index('idx_data_resources_workspace', table_name='data_resources') - op.drop_index('idx_data_resources_owner', table_name='data_resources') - op.drop_table('data_resources') - op.drop_index('idx_consumer_inbox_status', table_name='consumer_inbox') - op.drop_table('consumer_inbox') - # ### end Alembic commands ### diff --git a/migrations/versions/9a1b2c3d4e5f_enable_demo_password_login.py b/migrations/versions/9a1b2c3d4e5f_enable_demo_password_login.py deleted file mode 100644 index 3773fa1..0000000 --- a/migrations/versions/9a1b2c3d4e5f_enable_demo_password_login.py +++ /dev/null @@ -1,57 +0,0 @@ -"""enable password login for the seeded development users - -Revision ID: 9a1b2c3d4e5f -Revises: b71c4f2a9d10 -Create Date: 2026-08-03 16:00:00 -""" - -from collections.abc import Sequence -import os - -from alembic import op -import sqlalchemy as sa - -from common.auth.passwords import hash_password - - -revision: str = "9a1b2c3d4e5f" -down_revision: str | Sequence[str] | None = "b71c4f2a9d10" -branch_labels: str | Sequence[str] | None = None -depends_on: str | Sequence[str] | None = None - - -SEEDED_USER_IDS = ( - "0000000000RF6FG1SDBXG59S13", - "0000000000H2QYCGPCWQM1JSGS", - "0000000000RWG40ESZPGJT629J", - "00000000004CQV7WASJA6N6FW4", -) -DISABLED_PASSWORD = "demo-login-disabled" - - -def upgrade() -> None: - password = os.environ.get("INITIAL_ADMIN_PASSWORD", "admin12345") - users = sa.table( - "users", - sa.column("user_id", sa.String), - sa.column("password_hash", sa.String), - ) - op.execute( - users.update() - .where(users.c.user_id.in_(SEEDED_USER_IDS)) - .where(users.c.password_hash == DISABLED_PASSWORD) - .values(password_hash=hash_password(password)) - ) - - -def downgrade() -> None: - users = sa.table( - "users", - sa.column("user_id", sa.String), - sa.column("password_hash", sa.String), - ) - op.execute( - users.update() - .where(users.c.user_id.in_(SEEDED_USER_IDS)) - .values(password_hash=DISABLED_PASSWORD) - ) diff --git a/migrations/versions/a2b3c4d5e6f7_add_storage_trash_key.py b/migrations/versions/a2b3c4d5e6f7_add_storage_trash_key.py deleted file mode 100644 index de533c6..0000000 --- a/migrations/versions/a2b3c4d5e6f7_add_storage_trash_key.py +++ /dev/null @@ -1,33 +0,0 @@ -"""add the RustFS trash object key - -Revision ID: a2b3c4d5e6f7 -Revises: 9a1b2c3d4e5f -Create Date: 2026-08-03 16:30:00 -""" - -from collections.abc import Sequence - -from alembic import op -import sqlalchemy as sa - - -revision: str = "a2b3c4d5e6f7" -down_revision: str | Sequence[str] | None = "9a1b2c3d4e5f" -branch_labels: str | Sequence[str] | None = None -depends_on: str | Sequence[str] | None = None - - -def upgrade() -> None: - op.add_column( - "storage_objects", - sa.Column( - "trash_key", - sa.String(length=1100), - nullable=True, - comment="Path inside the trash bucket where soft-deleted bytes are stored", - ), - ) - - -def downgrade() -> None: - op.drop_column("storage_objects", "trash_key") diff --git a/migrations/versions/b71c4f2a9d10_seed_demo_context.py b/migrations/versions/b71c4f2a9d10_seed_demo_context.py deleted file mode 100644 index a638590..0000000 --- a/migrations/versions/b71c4f2a9d10_seed_demo_context.py +++ /dev/null @@ -1,168 +0,0 @@ -"""seed the self-hosted demo users and workspaces - -Revision ID: b71c4f2a9d10 -Revises: 8d86e2f82860 -Create Date: 2026-07-31 16:00:00 -""" - -from collections.abc import Sequence - -from alembic import op -import sqlalchemy as sa - - -revision: str = "b71c4f2a9d10" -down_revision: str | Sequence[str] | None = "8d86e2f82860" -branch_labels: str | Sequence[str] | None = None -depends_on: str | Sequence[str] | None = None - - -ADMIN_ROLE_ID = "0000000000000000000000000A" -DEVELOPER_ROLE_ID = "0000000000000000000000000B" - -USERS = ( - ("0000000000RF6FG1SDBXG59S13", "admin-zhang", "张三", ADMIN_ROLE_ID), - ("0000000000H2QYCGPCWQM1JSGS", "admin-li", "李四", ADMIN_ROLE_ID), - ("0000000000RWG40ESZPGJT629J", "dev-wang", "王五", DEVELOPER_ROLE_ID), - ("00000000004CQV7WASJA6N6FW4", "dev-zhao", "赵六", DEVELOPER_ROLE_ID), -) - -WORKSPACES = ( - ("00000000000BM630VT9ARVFZPC", "model-development", "模型开发 Workspace"), - ("0000000000AE0NC0V5T424KK86", "risk-validation", "风险验证 Workspace"), -) - - -def upgrade() -> None: - roles = sa.table( - "roles", - sa.column("role_id", sa.String), - sa.column("role_code", sa.String), - sa.column("role_name", sa.String), - sa.column("role_scope", sa.String), - sa.column("is_builtin", sa.Integer), - sa.column("description", sa.String), - ) - users = sa.table( - "users", - sa.column("user_id", sa.String), - sa.column("username", sa.String), - sa.column("display_name", sa.String), - sa.column("password_hash", sa.String), - sa.column("status", sa.String), - sa.column("email", sa.String), - sa.column("platform_role_id", sa.String), - ) - workspaces = sa.table( - "workspaces", - sa.column("workspace_id", sa.String), - sa.column("workspace_code", sa.String), - sa.column("workspace_name", sa.String), - sa.column("active_root_uri", sa.String), - sa.column("status", sa.String), - sa.column("created_by", sa.String), - sa.column("description", sa.String), - ) - members = sa.table( - "workspace_members", - sa.column("workspace_id", sa.String), - sa.column("user_id", sa.String), - sa.column("role_id", sa.String), - sa.column("member_status", sa.String), - ) - - op.bulk_insert( - roles, - [ - { - "role_id": ADMIN_ROLE_ID, - "role_code": "admin", - "role_name": "管理员", - "role_scope": "workspace", - "is_builtin": 1, - "description": "Self-hosted workspace administrator", - }, - { - "role_id": DEVELOPER_ROLE_ID, - "role_code": "developer", - "role_name": "开发人员", - "role_scope": "workspace", - "is_builtin": 1, - "description": "Self-hosted workspace developer", - }, - ], - ) - op.bulk_insert( - users, - [ - { - "user_id": user_id, - "username": username, - "display_name": display_name, - "password_hash": "demo-login-disabled", - "status": "active", - "email": f"{username}@model-platform.local", - "platform_role_id": role_id, - } - for user_id, username, display_name, role_id in USERS - ], - ) - op.bulk_insert( - workspaces, - [ - { - "workspace_id": workspace_id, - "workspace_code": workspace_code, - "workspace_name": workspace_name, - "active_root_uri": f"s3://workspaces/{workspace_id}/", - "status": "active", - "created_by": USERS[0][0], - "description": "Self-hosted demo workspace", - } - for workspace_id, workspace_code, workspace_name in WORKSPACES - ], - ) - op.bulk_insert( - members, - [ - { - "workspace_id": workspace_id, - "user_id": user_id, - "role_id": role_id, - "member_status": "active", - } - for workspace_id, _, _ in WORKSPACES - for user_id, _, _, role_id in USERS - ], - ) - - -def downgrade() -> None: - connection = op.get_bind() - workspace_ids = [workspace_id for workspace_id, _, _ in WORKSPACES] - user_ids = [user_id for user_id, _, _, _ in USERS] - connection.execute( - sa.text( - "DELETE FROM workspace_members " - "WHERE workspace_id IN :workspace_ids AND user_id IN :user_ids" - ).bindparams( - sa.bindparam("workspace_ids", expanding=True), - sa.bindparam("user_ids", expanding=True), - ), - {"workspace_ids": workspace_ids, "user_ids": user_ids}, - ) - connection.execute( - sa.text("DELETE FROM workspaces WHERE workspace_id IN :workspace_ids") - .bindparams(sa.bindparam("workspace_ids", expanding=True)), - {"workspace_ids": workspace_ids}, - ) - connection.execute( - sa.text("DELETE FROM users WHERE user_id IN :user_ids") - .bindparams(sa.bindparam("user_ids", expanding=True)), - {"user_ids": user_ids}, - ) - connection.execute( - sa.text("DELETE FROM roles WHERE role_id IN :role_ids") - .bindparams(sa.bindparam("role_ids", expanding=True)), - {"role_ids": [ADMIN_ROLE_ID, DEVELOPER_ROLE_ID]}, - ) diff --git a/migrations/versions/c3d4e5f6a7b8_upload_session_object_metadata.py b/migrations/versions/c3d4e5f6a7b8_upload_session_object_metadata.py deleted file mode 100644 index 63e9ea2..0000000 --- a/migrations/versions/c3d4e5f6a7b8_upload_session_object_metadata.py +++ /dev/null @@ -1,73 +0,0 @@ -"""add object metadata columns to upload_sessions - -Stores file_name / usage_type / visibility / is_immutable at session creation -so the server-proxied PUT step can build the StorageObjects row without -re-sending them. Replaces the old CompleteUploadRequest payload that bridged -the presign-PUT and head()-validate steps. - -Revision ID: c3d4e5f6a7b8 -Revises: a2b3c4d5e6f7 -Create Date: 2026-08-05 12:00:00 -""" - -from collections.abc import Sequence - -from alembic import op -import sqlalchemy as sa - - -revision: str = "c3d4e5f6a7b8" -down_revision: str | Sequence[str] | None = "a2b3c4d5e6f7" -branch_labels: str | Sequence[str] | None = None -depends_on: str | Sequence[str] | None = None - - -def upgrade() -> None: - op.add_column( - "upload_sessions", - sa.Column( - "file_name", - sa.String(length=255), - nullable=False, - server_default="", - ), - ) - op.add_column( - "upload_sessions", - sa.Column( - "usage_type", - sa.String(length=32), - nullable=False, - server_default="working_copy", - comment=( - "data_resource/version_artifact/snapshot/run_log/run_result/" - "working_copy/public_script" - ), - ), - ) - op.add_column( - "upload_sessions", - sa.Column( - "visibility", - sa.String(length=16), - nullable=False, - server_default="private", - comment="private/workspace/public", - ), - ) - op.add_column( - "upload_sessions", - sa.Column( - "is_immutable", - sa.TINYINT(1), - nullable=False, - server_default="0", - ), - ) - - -def downgrade() -> None: - op.drop_column("upload_sessions", "is_immutable") - op.drop_column("upload_sessions", "visibility") - op.drop_column("upload_sessions", "usage_type") - op.drop_column("upload_sessions", "file_name") \ No newline at end of file diff --git a/runtime/src/runtime/process.py b/runtime/src/runtime/process.py index 8a16e79..607d2bf 100644 --- a/runtime/src/runtime/process.py +++ b/runtime/src/runtime/process.py @@ -192,9 +192,11 @@ async def start_workspace(ws_id: str) -> dict: port = get_free_port() token = secrets.token_urlsafe(16) + logger.debug(f"starting workspace {ws_id} with token {token}") base_path = f"/jupyter/{ws_id}/" cmd = [ + "uvx", "jupyter", "notebook", f"--port={port}",