refactor: integrate model platform backend

This commit is contained in:
Winnie
2026-07-30 13:43:29 +08:00
parent baee7a60e1
commit 6d6c70cea8
97 changed files with 19724 additions and 2146 deletions
+1 -2
View File
@@ -1,2 +1 @@
def main() -> None:
print("Hello from common!")
"""Shared backend building blocks."""
+17 -5
View File
@@ -1,5 +1,17 @@
# coding=utf-8
"""
@Time :2026/7/29
@Author :tao.chen
"""
"""Shared SQLAlchemy database models and infrastructure."""
from common.db.models import Base
from common.db.session import (
AsyncSessionFactory,
create_database_engine,
create_session_factory,
session_scope,
)
__all__ = [
"AsyncSessionFactory",
"Base",
"create_database_engine",
"create_session_factory",
"session_scope",
]
+878
View File
@@ -0,0 +1,878 @@
from typing import Optional
import datetime
import decimal
from sqlalchemy import BINARY, BigInteger, CHAR, DECIMAL, Double, ForeignKeyConstraint, Index, Integer, JSON, String, Text, text
from sqlalchemy.dialects.mysql import BIGINT, DATETIME, INTEGER, SMALLINT, TINYINT
from sqlalchemy.orm import DeclarativeBase, Mapped, mapped_column, relationship
class Base(DeclarativeBase):
pass
class ConsumerInbox(Base):
__tablename__ = 'consumer_inbox'
__table_args__ = (
Index('idx_consumer_inbox_status', 'consumer_name', 'process_status', 'created_at'),
{'comment': '消费者幂等 Inbox,防止 Stream 重投导致重复执行'}
)
consumer_name: Mapped[str] = mapped_column(String(128), primary_key=True)
event_id: Mapped[str] = mapped_column(CHAR(26), primary_key=True)
process_status: Mapped[str] = mapped_column(String(16), nullable=False, server_default=text("'processing'"), comment='processing/succeeded/failed')
created_at: Mapped[datetime.datetime] = mapped_column(DATETIME(fsp=3), nullable=False, server_default=text('CURRENT_TIMESTAMP(3)'))
message_id: Mapped[Optional[str]] = mapped_column(String(128), comment='Redis Stream message ID')
processed_at: Mapped[Optional[datetime.datetime]] = mapped_column(DATETIME(fsp=3))
error_message: Mapped[Optional[str]] = mapped_column(String(2000))
class OutboxEvents(Base):
__tablename__ = 'outbox_events'
__table_args__ = (
Index('idx_outbox_aggregate', 'aggregate_type', 'aggregate_id', 'created_at'),
Index('idx_outbox_idempotency', 'idempotency_key'),
Index('idx_outbox_pending', 'event_status', 'available_at', 'created_at'),
{'comment': '事务 Outbox;提交后发布到 Redis Streams'}
)
event_id: Mapped[str] = mapped_column(CHAR(26), primary_key=True)
aggregate_type: Mapped[str] = mapped_column(String(64), nullable=False)
aggregate_id: Mapped[str] = mapped_column(String(128), nullable=False)
event_type: Mapped[str] = mapped_column(String(128), nullable=False)
schema_version: Mapped[int] = mapped_column(SMALLINT, nullable=False, server_default=text("1"))
payload_json: Mapped[dict] = mapped_column(JSON, nullable=False)
event_status: Mapped[str] = mapped_column(String(16), nullable=False, server_default=text("'pending'"), comment='pending/published/failed')
available_at: Mapped[datetime.datetime] = mapped_column(DATETIME(fsp=3), nullable=False, server_default=text('CURRENT_TIMESTAMP(3)'))
retry_count: Mapped[int] = mapped_column(INTEGER, nullable=False, server_default=text("0"))
created_at: Mapped[datetime.datetime] = mapped_column(DATETIME(fsp=3), nullable=False, server_default=text('CURRENT_TIMESTAMP(3)'))
trace_id: Mapped[Optional[str]] = mapped_column(String(64))
idempotency_key: Mapped[Optional[str]] = mapped_column(String(128))
published_at: Mapped[Optional[datetime.datetime]] = mapped_column(DATETIME(fsp=3))
last_error: Mapped[Optional[str]] = mapped_column(String(2000))
class Permissions(Base):
__tablename__ = 'permissions'
__table_args__ = (
Index('idx_permissions_module', 'module_code'),
Index('uk_permissions_code', 'permission_code', unique=True),
{'comment': '权限点'}
)
permission_id: Mapped[str] = mapped_column(CHAR(26), primary_key=True)
permission_code: Mapped[str] = mapped_column(String(128), nullable=False)
permission_name: Mapped[str] = mapped_column(String(100), nullable=False)
module_code: Mapped[str] = mapped_column(String(64), nullable=False)
created_at: Mapped[datetime.datetime] = mapped_column(DATETIME(fsp=3), nullable=False, server_default=text('CURRENT_TIMESTAMP(3)'))
description: Mapped[Optional[str]] = mapped_column(String(500))
role_permissions: Mapped[list['RolePermissions']] = relationship('RolePermissions', back_populates='permission')
class Roles(Base):
__tablename__ = 'roles'
__table_args__ = (
Index('uk_roles_code', 'role_code', unique=True),
{'comment': '角色'}
)
role_id: Mapped[str] = mapped_column(CHAR(26), primary_key=True)
role_code: Mapped[str] = mapped_column(String(64), nullable=False)
role_name: Mapped[str] = mapped_column(String(100), nullable=False)
role_scope: Mapped[str] = mapped_column(String(16), nullable=False, comment='platform/workspace')
is_builtin: Mapped[int] = mapped_column(TINYINT(1), nullable=False, server_default=text("0"))
created_at: Mapped[datetime.datetime] = mapped_column(DATETIME(fsp=3), nullable=False, server_default=text('CURRENT_TIMESTAMP(3)'))
updated_at: Mapped[datetime.datetime] = mapped_column(DATETIME(fsp=3), nullable=False, server_default=text('CURRENT_TIMESTAMP(3) ON UPDATE CURRENT_TIMESTAMP(3)'))
description: Mapped[Optional[str]] = mapped_column(String(500))
role_permissions: Mapped[list['RolePermissions']] = relationship('RolePermissions', back_populates='role')
users: Mapped[list['Users']] = relationship('Users', back_populates='platform_role')
workspace_members: Mapped[list['WorkspaceMembers']] = relationship('WorkspaceMembers', back_populates='role')
class RolePermissions(Base):
__tablename__ = 'role_permissions'
__table_args__ = (
ForeignKeyConstraint(['permission_id'], ['permissions.permission_id'], ondelete='CASCADE', name='fk_role_permissions_permission'),
ForeignKeyConstraint(['role_id'], ['roles.role_id'], ondelete='CASCADE', name='fk_role_permissions_role'),
Index('fk_role_permissions_permission', 'permission_id'),
{'comment': '角色权限'}
)
role_id: Mapped[str] = mapped_column(CHAR(26), primary_key=True)
permission_id: Mapped[str] = mapped_column(CHAR(26), primary_key=True)
created_at: Mapped[datetime.datetime] = mapped_column(DATETIME(fsp=3), nullable=False, server_default=text('CURRENT_TIMESTAMP(3)'))
permission: Mapped['Permissions'] = relationship('Permissions', back_populates='role_permissions')
role: Mapped['Roles'] = relationship('Roles', back_populates='role_permissions')
class Users(Base):
__tablename__ = 'users'
__table_args__ = (
ForeignKeyConstraint(['platform_role_id'], ['roles.role_id'], ondelete='SET NULL', name='fk_users_platform_role'),
Index('fk_users_platform_role', 'platform_role_id'),
Index('idx_users_status', 'status'),
Index('uk_users_email', 'email', unique=True),
Index('uk_users_username', 'username', unique=True),
{'comment': '平台用户'}
)
user_id: Mapped[str] = mapped_column(CHAR(26), primary_key=True)
username: Mapped[str] = mapped_column(String(64), nullable=False)
display_name: Mapped[str] = mapped_column(String(100), nullable=False)
password_hash: Mapped[str] = mapped_column(String(255), nullable=False)
status: Mapped[str] = mapped_column(String(16), nullable=False, server_default=text("'active'"), comment='active/disabled/locked')
created_at: Mapped[datetime.datetime] = mapped_column(DATETIME(fsp=3), nullable=False, server_default=text('CURRENT_TIMESTAMP(3)'))
updated_at: Mapped[datetime.datetime] = mapped_column(DATETIME(fsp=3), nullable=False, server_default=text('CURRENT_TIMESTAMP(3) ON UPDATE CURRENT_TIMESTAMP(3)'))
email: Mapped[Optional[str]] = mapped_column(String(255))
platform_role_id: Mapped[Optional[str]] = mapped_column(CHAR(26))
avatar_uri: Mapped[Optional[str]] = mapped_column(String(1000))
last_login_at: Mapped[Optional[datetime.datetime]] = mapped_column(DATETIME(fsp=3))
deleted_at: Mapped[Optional[datetime.datetime]] = mapped_column(DATETIME(fsp=3))
platform_role: Mapped[Optional['Roles']] = relationship('Roles', back_populates='users')
workspaces: Mapped[list['Workspaces']] = relationship('Workspaces', back_populates='users')
audit_logs: Mapped[list['AuditLogs']] = relationship('AuditLogs', back_populates='actor_user')
runtime_instances: Mapped[list['RuntimeInstances']] = relationship('RuntimeInstances', foreign_keys='[RuntimeInstances.owner_user_id]', back_populates='owner_user')
runtime_instances_: Mapped[list['RuntimeInstances']] = relationship('RuntimeInstances', foreign_keys='[RuntimeInstances.started_by]', back_populates='users')
schedules: Mapped[list['Schedules']] = relationship('Schedules', foreign_keys='[Schedules.created_by]', back_populates='users')
schedules_: Mapped[list['Schedules']] = relationship('Schedules', foreign_keys='[Schedules.updated_by]', back_populates='users_')
storage_objects: Mapped[list['StorageObjects']] = relationship('StorageObjects', foreign_keys='[StorageObjects.created_by]', back_populates='users')
storage_objects_: Mapped[list['StorageObjects']] = relationship('StorageObjects', foreign_keys='[StorageObjects.owner_user_id]', back_populates='owner_user')
workspace_members: Mapped[list['WorkspaceMembers']] = relationship('WorkspaceMembers', back_populates='user')
data_resources: Mapped[list['DataResources']] = relationship('DataResources', back_populates='owner_user')
edit_sessions: Mapped[list['EditSessions']] = relationship('EditSessions', back_populates='user')
schedule_runs: Mapped[list['ScheduleRuns']] = relationship('ScheduleRuns', back_populates='users')
scripts: Mapped[list['Scripts']] = relationship('Scripts', back_populates='owner_user')
upload_sessions: Mapped[list['UploadSessions']] = relationship('UploadSessions', back_populates='user')
workspace_operations: Mapped[list['WorkspaceOperations']] = relationship('WorkspaceOperations', back_populates='users')
notebook_snapshots: Mapped[list['NotebookSnapshots']] = relationship('NotebookSnapshots', back_populates='users')
versions: Mapped[list['Versions']] = relationship('Versions', back_populates='users')
experiments: Mapped[list['Experiments']] = relationship('Experiments', back_populates='owner_user')
class Workspaces(Base):
__tablename__ = 'workspaces'
__table_args__ = (
ForeignKeyConstraint(['created_by'], ['users.user_id'], ondelete='RESTRICT', name='fk_workspaces_created_by'),
Index('fk_workspaces_created_by', 'created_by'),
Index('idx_workspaces_status', 'status'),
Index('uk_workspaces_code', 'workspace_code', unique=True),
{'comment': 'Workspace'}
)
workspace_id: Mapped[str] = mapped_column(CHAR(26), primary_key=True)
workspace_code: Mapped[str] = mapped_column(String(64), nullable=False)
workspace_name: Mapped[str] = mapped_column(String(150), nullable=False)
active_root_uri: Mapped[str] = mapped_column(String(1500), nullable=False, comment='活动工作区,建议 NFS/PVC/file URI')
quota_bytes: Mapped[int] = mapped_column(BIGINT, nullable=False, server_default=text("0"), comment='0 表示不限额')
used_bytes: Mapped[int] = mapped_column(BIGINT, nullable=False, server_default=text("0"))
status: Mapped[str] = mapped_column(String(24), nullable=False, server_default=text("'active'"), comment='creating/active/suspended/deleting/deleted')
created_by: Mapped[str] = mapped_column(CHAR(26), nullable=False)
created_at: Mapped[datetime.datetime] = mapped_column(DATETIME(fsp=3), nullable=False, server_default=text('CURRENT_TIMESTAMP(3)'))
updated_at: Mapped[datetime.datetime] = mapped_column(DATETIME(fsp=3), nullable=False, server_default=text('CURRENT_TIMESTAMP(3) ON UPDATE CURRENT_TIMESTAMP(3)'))
description: Mapped[Optional[str]] = mapped_column(String(1000))
artifact_bucket: Mapped[Optional[str]] = mapped_column(String(128), comment='RustFS bucket')
artifact_prefix: Mapped[Optional[str]] = mapped_column(String(512), comment='RustFS object key prefix')
deleted_at: Mapped[Optional[datetime.datetime]] = mapped_column(DATETIME(fsp=3))
users: Mapped['Users'] = relationship('Users', back_populates='workspaces')
audit_logs: Mapped[list['AuditLogs']] = relationship('AuditLogs', back_populates='workspace')
runtime_instances: Mapped[list['RuntimeInstances']] = relationship('RuntimeInstances', back_populates='workspace')
schedules: Mapped[list['Schedules']] = relationship('Schedules', back_populates='workspace')
storage_objects: Mapped[list['StorageObjects']] = relationship('StorageObjects', back_populates='workspace')
workspace_members: Mapped[list['WorkspaceMembers']] = relationship('WorkspaceMembers', back_populates='workspace')
data_resources: Mapped[list['DataResources']] = relationship('DataResources', back_populates='workspace')
edit_sessions: Mapped[list['EditSessions']] = relationship('EditSessions', back_populates='workspace')
schedule_runs: Mapped[list['ScheduleRuns']] = relationship('ScheduleRuns', back_populates='workspace')
scripts: Mapped[list['Scripts']] = relationship('Scripts', back_populates='workspace')
upload_sessions: Mapped[list['UploadSessions']] = relationship('UploadSessions', back_populates='workspace')
workspace_operations: Mapped[list['WorkspaceOperations']] = relationship('WorkspaceOperations', back_populates='workspace')
notebook_snapshots: Mapped[list['NotebookSnapshots']] = relationship('NotebookSnapshots', back_populates='workspace')
versions: Mapped[list['Versions']] = relationship('Versions', back_populates='workspace')
experiments: Mapped[list['Experiments']] = relationship('Experiments', back_populates='workspace')
class AuditLogs(Base):
__tablename__ = 'audit_logs'
__table_args__ = (
ForeignKeyConstraint(['actor_user_id'], ['users.user_id'], ondelete='SET NULL', name='fk_audit_actor'),
ForeignKeyConstraint(['workspace_id'], ['workspaces.workspace_id'], ondelete='SET NULL', name='fk_audit_workspace'),
Index('idx_audit_action_time', 'action_code', 'created_at'),
Index('idx_audit_actor_time', 'actor_user_id', 'created_at'),
Index('idx_audit_workspace_time', 'workspace_id', 'created_at'),
{'comment': '操作审计日志'}
)
audit_id: Mapped[int] = mapped_column(BIGINT, primary_key=True)
action_code: Mapped[str] = mapped_column(String(128), nullable=False)
target_type: Mapped[str] = mapped_column(String(64), nullable=False)
operation_status: Mapped[str] = mapped_column(String(16), nullable=False, server_default=text("'success'"))
created_at: Mapped[datetime.datetime] = mapped_column(DATETIME(fsp=3), nullable=False, server_default=text('CURRENT_TIMESTAMP(3)'))
workspace_id: Mapped[Optional[str]] = mapped_column(CHAR(26))
actor_user_id: Mapped[Optional[str]] = mapped_column(CHAR(26))
target_id: Mapped[Optional[str]] = mapped_column(String(128))
client_ip: Mapped[Optional[str]] = mapped_column(String(45))
user_agent: Mapped[Optional[str]] = mapped_column(String(1000))
detail_json: Mapped[Optional[dict]] = mapped_column(JSON)
actor_user: Mapped[Optional['Users']] = relationship('Users', back_populates='audit_logs')
workspace: Mapped[Optional['Workspaces']] = relationship('Workspaces', back_populates='audit_logs')
class RuntimeInstances(Base):
__tablename__ = 'runtime_instances'
__table_args__ = (
ForeignKeyConstraint(['owner_user_id'], ['users.user_id'], ondelete='SET NULL', name='fk_runtime_owner'),
ForeignKeyConstraint(['started_by'], ['users.user_id'], ondelete='RESTRICT', name='fk_runtime_started_by'),
ForeignKeyConstraint(['workspace_id'], ['workspaces.workspace_id'], ondelete='RESTRICT', name='fk_runtime_workspace'),
Index('fk_runtime_started_by', 'started_by'),
Index('idx_runtime_lease', 'actual_state', 'lease_expires_at'),
Index('idx_runtime_owner_state', 'owner_user_id', 'actual_state'),
Index('idx_runtime_workspace_state', 'workspace_id', 'runtime_type', 'actual_state'),
{'comment': 'Jupyter/未来 VS Code、OpenCode Runtime 实例'}
)
runtime_id: Mapped[str] = mapped_column(CHAR(26), primary_key=True)
workspace_id: Mapped[str] = mapped_column(CHAR(26), nullable=False)
runtime_type: Mapped[str] = mapped_column(String(24), nullable=False, server_default=text("'jupyter'"))
runtime_provider: Mapped[str] = mapped_column(String(24), nullable=False, comment='process/docker/kubernetes')
proxy_base_path: Mapped[str] = mapped_column(String(512), nullable=False)
desired_state: Mapped[str] = mapped_column(String(16), nullable=False, server_default=text("'running'"))
actual_state: Mapped[str] = mapped_column(String(24), nullable=False, server_default=text("'provisioning'"), comment='provisioning/starting/running/unhealthy/stopping/stopped/failed')
state_version: Mapped[int] = mapped_column(INTEGER, nullable=False, server_default=text("0"), comment='乐观锁版本')
started_by: Mapped[str] = mapped_column(CHAR(26), nullable=False)
created_at: Mapped[datetime.datetime] = mapped_column(DATETIME(fsp=3), nullable=False, server_default=text('CURRENT_TIMESTAMP(3)'))
updated_at: Mapped[datetime.datetime] = mapped_column(DATETIME(fsp=3), nullable=False, server_default=text('CURRENT_TIMESTAMP(3) ON UPDATE CURRENT_TIMESTAMP(3)'))
owner_user_id: Mapped[Optional[str]] = mapped_column(CHAR(26), comment='为空表示 Workspace 级 Runtime')
runtime_ref: Mapped[Optional[str]] = mapped_column(String(255), comment='PID/container ID/pod UID')
host_node: Mapped[Optional[str]] = mapped_column(String(255))
internal_url: Mapped[Optional[str]] = mapped_column(String(1000))
started_at: Mapped[Optional[datetime.datetime]] = mapped_column(DATETIME(fsp=3))
last_heartbeat_at: Mapped[Optional[datetime.datetime]] = mapped_column(DATETIME(fsp=3))
lease_expires_at: Mapped[Optional[datetime.datetime]] = mapped_column(DATETIME(fsp=3))
stopped_at: Mapped[Optional[datetime.datetime]] = mapped_column(DATETIME(fsp=3))
error_message: Mapped[Optional[str]] = mapped_column(Text)
owner_user: Mapped[Optional['Users']] = relationship('Users', foreign_keys=[owner_user_id], back_populates='runtime_instances')
users: Mapped['Users'] = relationship('Users', foreign_keys=[started_by], back_populates='runtime_instances_')
workspace: Mapped['Workspaces'] = relationship('Workspaces', back_populates='runtime_instances')
edit_sessions: Mapped[list['EditSessions']] = relationship('EditSessions', back_populates='runtime')
workspace_operations: Mapped[list['WorkspaceOperations']] = relationship('WorkspaceOperations', back_populates='runtime')
class Schedules(Base):
__tablename__ = 'schedules'
__table_args__ = (
ForeignKeyConstraint(['created_by'], ['users.user_id'], ondelete='RESTRICT', name='fk_schedules_created_by'),
ForeignKeyConstraint(['updated_by'], ['users.user_id'], ondelete='RESTRICT', name='fk_schedules_updated_by'),
ForeignKeyConstraint(['workspace_id'], ['workspaces.workspace_id'], ondelete='RESTRICT', name='fk_schedules_workspace'),
Index('fk_schedules_created_by', 'created_by'),
Index('fk_schedules_updated_by', 'updated_by'),
Index('idx_schedules_due', 'enabled', 'next_run_at'),
Index('idx_schedules_workspace', 'workspace_id', 'enabled', 'updated_at'),
{'comment': '调度方案'}
)
schedule_id: Mapped[str] = mapped_column(CHAR(26), primary_key=True)
workspace_id: Mapped[str] = mapped_column(CHAR(26), nullable=False)
schedule_name: Mapped[str] = mapped_column(String(255), nullable=False)
trigger_type: Mapped[str] = mapped_column(String(16), nullable=False, server_default=text("'cron'"), comment='manual/cron/api')
timezone: Mapped[str] = mapped_column(String(64), nullable=False, server_default=text("'Asia/Shanghai'"))
enabled: Mapped[int] = mapped_column(TINYINT(1), nullable=False, server_default=text("0"))
workflow_version: Mapped[int] = mapped_column(INTEGER, nullable=False, server_default=text("1"))
max_concurrency: Mapped[int] = mapped_column(INTEGER, nullable=False, server_default=text("1"))
failure_policy: Mapped[str] = mapped_column(String(24), nullable=False, server_default=text("'stop'"), comment='stop/continue')
created_by: Mapped[str] = mapped_column(CHAR(26), nullable=False)
updated_by: Mapped[str] = mapped_column(CHAR(26), nullable=False)
created_at: Mapped[datetime.datetime] = mapped_column(DATETIME(fsp=3), nullable=False, server_default=text('CURRENT_TIMESTAMP(3)'))
updated_at: Mapped[datetime.datetime] = mapped_column(DATETIME(fsp=3), nullable=False, server_default=text('CURRENT_TIMESTAMP(3) ON UPDATE CURRENT_TIMESTAMP(3)'))
description: Mapped[Optional[str]] = mapped_column(String(1000))
cron_expression: Mapped[Optional[str]] = mapped_column(String(128))
last_run_at: Mapped[Optional[datetime.datetime]] = mapped_column(DATETIME(fsp=3))
next_run_at: Mapped[Optional[datetime.datetime]] = mapped_column(DATETIME(fsp=3))
deleted_at: Mapped[Optional[datetime.datetime]] = mapped_column(DATETIME(fsp=3))
users: Mapped['Users'] = relationship('Users', foreign_keys=[created_by], back_populates='schedules')
users_: Mapped['Users'] = relationship('Users', foreign_keys=[updated_by], back_populates='schedules_')
workspace: Mapped['Workspaces'] = relationship('Workspaces', back_populates='schedules')
schedule_runs: Mapped[list['ScheduleRuns']] = relationship('ScheduleRuns', back_populates='schedule')
schedule_nodes: Mapped[list['ScheduleNodes']] = relationship('ScheduleNodes', back_populates='schedule')
schedule_edges: Mapped[list['ScheduleEdges']] = relationship('ScheduleEdges', back_populates='schedule')
class StorageObjects(Base):
__tablename__ = 'storage_objects'
__table_args__ = (
ForeignKeyConstraint(['created_by'], ['users.user_id'], ondelete='RESTRICT', name='fk_storage_created_by'),
ForeignKeyConstraint(['owner_user_id'], ['users.user_id'], ondelete='SET NULL', name='fk_storage_owner'),
ForeignKeyConstraint(['parent_object_id'], ['storage_objects.storage_object_id'], ondelete='SET NULL', name='fk_storage_parent'),
ForeignKeyConstraint(['workspace_id'], ['workspaces.workspace_id'], ondelete='RESTRICT', name='fk_storage_workspace'),
Index('fk_storage_created_by', 'created_by'),
Index('idx_storage_content_hash', 'content_hash'),
Index('idx_storage_owner', 'owner_user_id', 'object_status'),
Index('idx_storage_parent', 'parent_object_id'),
Index('idx_storage_workspace_usage', 'workspace_id', 'usage_type', 'object_status'),
Index('uk_storage_bucket_key', 'storage_backend', 'bucket_name', 'object_key_hash', unique=True),
Index('uk_storage_workspace_path', 'workspace_id', 'storage_backend', 'path_hash', unique=True),
{'comment': 'Workspace 文件和 RustFS 对象的统一元数据'}
)
storage_object_id: Mapped[str] = mapped_column(CHAR(26), primary_key=True)
workspace_id: Mapped[str] = mapped_column(CHAR(26), nullable=False)
object_type: Mapped[str] = mapped_column(String(16), nullable=False, comment='file/directory')
usage_type: Mapped[str] = mapped_column(String(32), nullable=False, comment='working_copy/public_script/data_resource/version_artifact/snapshot/run_log/run_result')
storage_backend: Mapped[str] = mapped_column(String(16), nullable=False, comment='workspace_fs/rustfs')
storage_uri: Mapped[str] = mapped_column(String(1500), nullable=False)
file_name: Mapped[str] = mapped_column(String(255), nullable=False)
size_bytes: Mapped[int] = mapped_column(BIGINT, nullable=False, server_default=text("0"))
visibility: Mapped[str] = mapped_column(String(16), nullable=False, server_default=text("'private'"), comment='private/workspace/public')
is_immutable: Mapped[int] = mapped_column(TINYINT(1), nullable=False, server_default=text("0"))
object_status: Mapped[str] = mapped_column(String(24), nullable=False, server_default=text("'available'"), comment='uploading/available/deleting/deleted/failed')
created_by: Mapped[str] = mapped_column(CHAR(26), nullable=False)
created_at: Mapped[datetime.datetime] = mapped_column(DATETIME(fsp=3), nullable=False, server_default=text('CURRENT_TIMESTAMP(3)'))
updated_at: Mapped[datetime.datetime] = mapped_column(DATETIME(fsp=3), nullable=False, server_default=text('CURRENT_TIMESTAMP(3) ON UPDATE CURRENT_TIMESTAMP(3)'))
owner_user_id: Mapped[Optional[str]] = mapped_column(CHAR(26))
parent_object_id: Mapped[Optional[str]] = mapped_column(CHAR(26))
relative_path: Mapped[Optional[str]] = mapped_column(String(1024), comment='Workspace 相对路径')
path_hash: Mapped[Optional[bytes]] = mapped_column(BINARY(32), comment='SHA-256(relative_path),由应用写入')
bucket_name: Mapped[Optional[str]] = mapped_column(String(128))
object_key: Mapped[Optional[str]] = mapped_column(String(1024))
object_key_hash: Mapped[Optional[bytes]] = mapped_column(BINARY(32), comment='SHA-256(object_key),由应用写入')
file_extension: Mapped[Optional[str]] = mapped_column(String(32))
mime_type: Mapped[Optional[str]] = mapped_column(String(255))
content_hash: Mapped[Optional[str]] = mapped_column(CHAR(64), comment='SHA-256 hex')
object_etag: Mapped[Optional[str]] = mapped_column(String(255))
deleted_at: Mapped[Optional[datetime.datetime]] = mapped_column(DATETIME(fsp=3))
users: Mapped['Users'] = relationship('Users', foreign_keys=[created_by], back_populates='storage_objects')
owner_user: Mapped[Optional['Users']] = relationship('Users', foreign_keys=[owner_user_id], back_populates='storage_objects_')
parent_object: Mapped[Optional['StorageObjects']] = relationship('StorageObjects', remote_side=[storage_object_id], back_populates='parent_object_reverse')
parent_object_reverse: Mapped[list['StorageObjects']] = relationship('StorageObjects', remote_side=[parent_object_id], back_populates='parent_object')
workspace: Mapped['Workspaces'] = relationship('Workspaces', back_populates='storage_objects')
data_resources: Mapped[list['DataResources']] = relationship('DataResources', back_populates='storage_object')
edit_sessions: Mapped[list['EditSessions']] = relationship('EditSessions', back_populates='storage_object')
schedule_runs: Mapped[list['ScheduleRuns']] = relationship('ScheduleRuns', foreign_keys='[ScheduleRuns.logs_object_id]', back_populates='logs_object')
schedule_runs_: Mapped[list['ScheduleRuns']] = relationship('ScheduleRuns', foreign_keys='[ScheduleRuns.result_object_id]', back_populates='result_object')
scripts: Mapped[list['Scripts']] = relationship('Scripts', back_populates='current_object')
upload_sessions: Mapped[list['UploadSessions']] = relationship('UploadSessions', back_populates='storage_object')
notebook_snapshots: Mapped[list['NotebookSnapshots']] = relationship('NotebookSnapshots', foreign_keys='[NotebookSnapshots.artifact_object_id]', back_populates='artifact_object')
notebook_snapshots_: Mapped[list['NotebookSnapshots']] = relationship('NotebookSnapshots', foreign_keys='[NotebookSnapshots.source_object_id]', back_populates='source_object')
versions: Mapped[list['Versions']] = relationship('Versions', foreign_keys='[Versions.artifact_object_id]', back_populates='artifact_object')
versions_: Mapped[list['Versions']] = relationship('Versions', foreign_keys='[Versions.source_object_id]', back_populates='source_object')
experiments: Mapped[list['Experiments']] = relationship('Experiments', foreign_keys='[Experiments.logs_object_id]', back_populates='logs_object')
experiments_: Mapped[list['Experiments']] = relationship('Experiments', foreign_keys='[Experiments.result_object_id]', back_populates='result_object')
schedule_node_runs: Mapped[list['ScheduleNodeRuns']] = relationship('ScheduleNodeRuns', foreign_keys='[ScheduleNodeRuns.logs_object_id]', back_populates='logs_object')
schedule_node_runs_: Mapped[list['ScheduleNodeRuns']] = relationship('ScheduleNodeRuns', foreign_keys='[ScheduleNodeRuns.result_object_id]', back_populates='result_object')
class WorkspaceMembers(Base):
__tablename__ = 'workspace_members'
__table_args__ = (
ForeignKeyConstraint(['role_id'], ['roles.role_id'], ondelete='RESTRICT', name='fk_workspace_members_role'),
ForeignKeyConstraint(['user_id'], ['users.user_id'], ondelete='RESTRICT', name='fk_workspace_members_user'),
ForeignKeyConstraint(['workspace_id'], ['workspaces.workspace_id'], ondelete='CASCADE', name='fk_workspace_members_workspace'),
Index('idx_workspace_members_role', 'role_id'),
Index('idx_workspace_members_user', 'user_id', 'member_status'),
{'comment': 'Workspace 成员与角色'}
)
workspace_id: Mapped[str] = mapped_column(CHAR(26), primary_key=True)
user_id: Mapped[str] = mapped_column(CHAR(26), primary_key=True)
role_id: Mapped[str] = mapped_column(CHAR(26), nullable=False)
member_status: Mapped[str] = mapped_column(String(16), nullable=False, server_default=text("'active'"))
joined_at: Mapped[datetime.datetime] = mapped_column(DATETIME(fsp=3), nullable=False, server_default=text('CURRENT_TIMESTAMP(3)'))
updated_at: Mapped[datetime.datetime] = mapped_column(DATETIME(fsp=3), nullable=False, server_default=text('CURRENT_TIMESTAMP(3) ON UPDATE CURRENT_TIMESTAMP(3)'))
role: Mapped['Roles'] = relationship('Roles', back_populates='workspace_members')
user: Mapped['Users'] = relationship('Users', back_populates='workspace_members')
workspace: Mapped['Workspaces'] = relationship('Workspaces', back_populates='workspace_members')
class DataResources(Base):
__tablename__ = 'data_resources'
__table_args__ = (
ForeignKeyConstraint(['owner_user_id'], ['users.user_id'], ondelete='RESTRICT', name='fk_data_resources_owner'),
ForeignKeyConstraint(['storage_object_id'], ['storage_objects.storage_object_id'], ondelete='RESTRICT', name='fk_data_resources_object'),
ForeignKeyConstraint(['workspace_id'], ['workspaces.workspace_id'], ondelete='RESTRICT', name='fk_data_resources_workspace'),
Index('idx_data_resources_owner', 'owner_user_id', 'status'),
Index('idx_data_resources_workspace', 'workspace_id', 'visibility', 'status'),
Index('uk_data_resources_object', 'storage_object_id', unique=True),
{'comment': '数据资源'}
)
resource_id: Mapped[str] = mapped_column(CHAR(26), primary_key=True)
workspace_id: Mapped[str] = mapped_column(CHAR(26), nullable=False)
storage_object_id: Mapped[str] = mapped_column(CHAR(26), nullable=False)
owner_user_id: Mapped[str] = mapped_column(CHAR(26), nullable=False)
resource_name: Mapped[str] = mapped_column(String(255), nullable=False)
visibility: Mapped[str] = mapped_column(String(16), nullable=False, server_default=text("'private'"))
status: Mapped[str] = mapped_column(String(16), nullable=False, server_default=text("'active'"))
created_at: Mapped[datetime.datetime] = mapped_column(DATETIME(fsp=3), nullable=False, server_default=text('CURRENT_TIMESTAMP(3)'))
updated_at: Mapped[datetime.datetime] = mapped_column(DATETIME(fsp=3), nullable=False, server_default=text('CURRENT_TIMESTAMP(3) ON UPDATE CURRENT_TIMESTAMP(3)'))
description: Mapped[Optional[str]] = mapped_column(String(1000))
schema_json: Mapped[Optional[dict]] = mapped_column(JSON, comment='字段结构、行数等可选元数据')
deleted_at: Mapped[Optional[datetime.datetime]] = mapped_column(DATETIME(fsp=3))
owner_user: Mapped['Users'] = relationship('Users', back_populates='data_resources')
storage_object: Mapped['StorageObjects'] = relationship('StorageObjects', back_populates='data_resources')
workspace: Mapped['Workspaces'] = relationship('Workspaces', back_populates='data_resources')
experiment_resources: Mapped[list['ExperimentResources']] = relationship('ExperimentResources', back_populates='resource')
class EditSessions(Base):
__tablename__ = 'edit_sessions'
__table_args__ = (
ForeignKeyConstraint(['runtime_id'], ['runtime_instances.runtime_id'], ondelete='SET NULL', name='fk_edit_sessions_runtime'),
ForeignKeyConstraint(['storage_object_id'], ['storage_objects.storage_object_id'], ondelete='RESTRICT', name='fk_edit_sessions_object'),
ForeignKeyConstraint(['user_id'], ['users.user_id'], ondelete='RESTRICT', name='fk_edit_sessions_user'),
ForeignKeyConstraint(['workspace_id'], ['workspaces.workspace_id'], ondelete='RESTRICT', name='fk_edit_sessions_workspace'),
Index('fk_edit_sessions_workspace', 'workspace_id'),
Index('idx_edit_sessions_object', 'storage_object_id', 'session_status', 'expires_at'),
Index('idx_edit_sessions_runtime', 'runtime_id', 'session_status'),
Index('idx_edit_sessions_user', 'user_id', 'session_status'),
{'comment': '编辑会话审计;实时锁状态以 Redis 为准'}
)
edit_session_id: Mapped[str] = mapped_column(CHAR(26), primary_key=True)
workspace_id: Mapped[str] = mapped_column(CHAR(26), nullable=False)
storage_object_id: Mapped[str] = mapped_column(CHAR(26), nullable=False)
user_id: Mapped[str] = mapped_column(CHAR(26), nullable=False)
redis_lock_key: Mapped[str] = mapped_column(String(512), nullable=False)
lock_token_hash: Mapped[bytes] = mapped_column(BINARY(32), nullable=False, comment='不保存原始 token')
session_status: Mapped[str] = mapped_column(String(16), nullable=False, server_default=text("'active'"), comment='active/closed/expired/failed')
started_at: Mapped[datetime.datetime] = mapped_column(DATETIME(fsp=3), nullable=False, server_default=text('CURRENT_TIMESTAMP(3)'))
last_heartbeat_at: Mapped[datetime.datetime] = mapped_column(DATETIME(fsp=3), nullable=False, server_default=text('CURRENT_TIMESTAMP(3)'))
expires_at: Mapped[datetime.datetime] = mapped_column(DATETIME(fsp=3), nullable=False)
runtime_id: Mapped[Optional[str]] = mapped_column(CHAR(26))
jupyter_session_id: Mapped[Optional[str]] = mapped_column(String(255))
ended_at: Mapped[Optional[datetime.datetime]] = mapped_column(DATETIME(fsp=3))
end_reason: Mapped[Optional[str]] = mapped_column(String(64))
runtime: Mapped[Optional['RuntimeInstances']] = relationship('RuntimeInstances', back_populates='edit_sessions')
storage_object: Mapped['StorageObjects'] = relationship('StorageObjects', back_populates='edit_sessions')
user: Mapped['Users'] = relationship('Users', back_populates='edit_sessions')
workspace: Mapped['Workspaces'] = relationship('Workspaces', back_populates='edit_sessions')
class ScheduleRuns(Base):
__tablename__ = 'schedule_runs'
__table_args__ = (
ForeignKeyConstraint(['logs_object_id'], ['storage_objects.storage_object_id'], ondelete='SET NULL', name='fk_schedule_runs_logs'),
ForeignKeyConstraint(['result_object_id'], ['storage_objects.storage_object_id'], ondelete='SET NULL', name='fk_schedule_runs_result'),
ForeignKeyConstraint(['schedule_id'], ['schedules.schedule_id'], ondelete='RESTRICT', name='fk_schedule_runs_schedule'),
ForeignKeyConstraint(['triggered_by'], ['users.user_id'], ondelete='SET NULL', name='fk_schedule_runs_user'),
ForeignKeyConstraint(['workspace_id'], ['workspaces.workspace_id'], ondelete='RESTRICT', name='fk_schedule_runs_workspace'),
Index('fk_schedule_runs_logs', 'logs_object_id'),
Index('fk_schedule_runs_result', 'result_object_id'),
Index('fk_schedule_runs_user', 'triggered_by'),
Index('idx_schedule_runs_schedule', 'schedule_id', 'created_at'),
Index('idx_schedule_runs_status', 'run_status', 'queued_at'),
Index('idx_schedule_runs_workspace_status', 'workspace_id', 'run_status', 'queued_at'),
Index('uk_schedule_runs_idempotency', 'idempotency_key', unique=True),
{'comment': '调度运行'}
)
run_id: Mapped[str] = mapped_column(CHAR(26), primary_key=True)
schedule_id: Mapped[str] = mapped_column(CHAR(26), nullable=False)
workspace_id: Mapped[str] = mapped_column(CHAR(26), nullable=False)
workflow_version: Mapped[int] = mapped_column(INTEGER, nullable=False)
trigger_type: Mapped[str] = mapped_column(String(16), nullable=False, comment='manual/cron/api/retry')
idempotency_key: Mapped[str] = mapped_column(String(128), nullable=False)
run_status: Mapped[str] = mapped_column(String(24), nullable=False, server_default=text("'queued'"), comment='queued/running/succeeded/failed/cancelled/timed_out')
state_version: Mapped[int] = mapped_column(INTEGER, nullable=False, server_default=text("0"), comment='乐观锁版本')
schedule_snapshot: Mapped[dict] = mapped_column(JSON, nullable=False, comment='执行时 DAG 快照')
queued_at: Mapped[datetime.datetime] = mapped_column(DATETIME(fsp=3), nullable=False, server_default=text('CURRENT_TIMESTAMP(3)'))
created_at: Mapped[datetime.datetime] = mapped_column(DATETIME(fsp=3), nullable=False, server_default=text('CURRENT_TIMESTAMP(3)'))
triggered_by: Mapped[Optional[str]] = mapped_column(CHAR(26))
started_at: Mapped[Optional[datetime.datetime]] = mapped_column(DATETIME(fsp=3))
finished_at: Mapped[Optional[datetime.datetime]] = mapped_column(DATETIME(fsp=3))
duration_ms: Mapped[Optional[int]] = mapped_column(BIGINT)
error_code: Mapped[Optional[str]] = mapped_column(String(64))
error_message: Mapped[Optional[str]] = mapped_column(Text)
logs_object_id: Mapped[Optional[str]] = mapped_column(CHAR(26))
result_object_id: Mapped[Optional[str]] = mapped_column(CHAR(26))
logs_object: Mapped[Optional['StorageObjects']] = relationship('StorageObjects', foreign_keys=[logs_object_id], back_populates='schedule_runs')
result_object: Mapped[Optional['StorageObjects']] = relationship('StorageObjects', foreign_keys=[result_object_id], back_populates='schedule_runs_')
schedule: Mapped['Schedules'] = relationship('Schedules', back_populates='schedule_runs')
users: Mapped[Optional['Users']] = relationship('Users', back_populates='schedule_runs')
workspace: Mapped['Workspaces'] = relationship('Workspaces', back_populates='schedule_runs')
experiments: Mapped[list['Experiments']] = relationship('Experiments', back_populates='schedule_run')
schedule_node_runs: Mapped[list['ScheduleNodeRuns']] = relationship('ScheduleNodeRuns', back_populates='run')
class Scripts(Base):
__tablename__ = 'scripts'
__table_args__ = (
ForeignKeyConstraint(['current_object_id'], ['storage_objects.storage_object_id'], ondelete='RESTRICT', name='fk_scripts_current_object'),
ForeignKeyConstraint(['owner_user_id'], ['users.user_id'], ondelete='RESTRICT', name='fk_scripts_owner'),
ForeignKeyConstraint(['workspace_id'], ['workspaces.workspace_id'], ondelete='RESTRICT', name='fk_scripts_workspace'),
Index('idx_scripts_owner', 'owner_user_id', 'status'),
Index('idx_scripts_workspace', 'workspace_id', 'script_type', 'visibility', 'status'),
Index('uk_scripts_current_object', 'current_object_id', unique=True),
{'comment': '可执行 Python/Notebook 脚本'}
)
script_id: Mapped[str] = mapped_column(CHAR(26), primary_key=True)
workspace_id: Mapped[str] = mapped_column(CHAR(26), nullable=False)
current_object_id: Mapped[str] = mapped_column(CHAR(26), nullable=False, comment='当前工作副本')
owner_user_id: Mapped[str] = mapped_column(CHAR(26), nullable=False)
script_name: Mapped[str] = mapped_column(String(255), nullable=False)
script_type: Mapped[str] = mapped_column(String(16), nullable=False, comment='python/notebook')
visibility: Mapped[str] = mapped_column(String(16), nullable=False, server_default=text("'private'"))
status: Mapped[str] = mapped_column(String(16), nullable=False, server_default=text("'active'"))
created_at: Mapped[datetime.datetime] = mapped_column(DATETIME(fsp=3), nullable=False, server_default=text('CURRENT_TIMESTAMP(3)'))
updated_at: Mapped[datetime.datetime] = mapped_column(DATETIME(fsp=3), nullable=False, server_default=text('CURRENT_TIMESTAMP(3) ON UPDATE CURRENT_TIMESTAMP(3)'))
deleted_at: Mapped[Optional[datetime.datetime]] = mapped_column(DATETIME(fsp=3))
current_object: Mapped['StorageObjects'] = relationship('StorageObjects', back_populates='scripts')
owner_user: Mapped['Users'] = relationship('Users', back_populates='scripts')
workspace: Mapped['Workspaces'] = relationship('Workspaces', back_populates='scripts')
notebook_snapshots: Mapped[list['NotebookSnapshots']] = relationship('NotebookSnapshots', back_populates='script')
versions: Mapped[list['Versions']] = relationship('Versions', back_populates='script')
experiments: Mapped[list['Experiments']] = relationship('Experiments', back_populates='script')
class UploadSessions(Base):
__tablename__ = 'upload_sessions'
__table_args__ = (
ForeignKeyConstraint(['storage_object_id'], ['storage_objects.storage_object_id'], ondelete='SET NULL', name='fk_upload_sessions_storage_object'),
ForeignKeyConstraint(['user_id'], ['users.user_id'], ondelete='RESTRICT', name='fk_upload_sessions_user'),
ForeignKeyConstraint(['workspace_id'], ['workspaces.workspace_id'], ondelete='RESTRICT', name='fk_upload_sessions_workspace'),
Index('fk_upload_sessions_storage_object', 'storage_object_id'),
Index('fk_upload_sessions_user', 'user_id'),
Index('idx_upload_sessions_expiry', 'upload_status', 'expires_at'),
Index('idx_upload_sessions_object_key', 'bucket_name', 'object_key_hash'),
Index('idx_upload_sessions_workspace', 'workspace_id', 'user_id', 'created_at'),
Index('uk_upload_sessions_idempotency', 'idempotency_key', unique=True),
{'comment': 'RustFS 预签名上传会话;URL 本身不持久化'}
)
upload_id: Mapped[str] = mapped_column(CHAR(26), primary_key=True)
workspace_id: Mapped[str] = mapped_column(CHAR(26), nullable=False)
user_id: Mapped[str] = mapped_column(CHAR(26), nullable=False)
idempotency_key: Mapped[str] = mapped_column(String(128), nullable=False)
bucket_name: Mapped[str] = mapped_column(String(128), nullable=False)
object_key: Mapped[str] = mapped_column(String(1024), nullable=False)
object_key_hash: Mapped[bytes] = mapped_column(BINARY(32), nullable=False)
upload_status: Mapped[str] = mapped_column(String(24), nullable=False, server_default=text("'created'"), comment='created/uploading/completed/expired/aborted/failed')
expires_at: Mapped[datetime.datetime] = mapped_column(DATETIME(fsp=3), nullable=False)
created_at: Mapped[datetime.datetime] = mapped_column(DATETIME(fsp=3), nullable=False, server_default=text('CURRENT_TIMESTAMP(3)'))
updated_at: Mapped[datetime.datetime] = mapped_column(DATETIME(fsp=3), nullable=False, server_default=text('CURRENT_TIMESTAMP(3) ON UPDATE CURRENT_TIMESTAMP(3)'))
multipart_upload_id: Mapped[Optional[str]] = mapped_column(String(255))
expected_size_bytes: Mapped[Optional[int]] = mapped_column(BIGINT)
expected_hash: Mapped[Optional[str]] = mapped_column(CHAR(64))
content_type: Mapped[Optional[str]] = mapped_column(String(255))
storage_object_id: Mapped[Optional[str]] = mapped_column(CHAR(26))
completed_at: Mapped[Optional[datetime.datetime]] = mapped_column(DATETIME(fsp=3))
storage_object: Mapped[Optional['StorageObjects']] = relationship('StorageObjects', back_populates='upload_sessions')
user: Mapped['Users'] = relationship('Users', back_populates='upload_sessions')
workspace: Mapped['Workspaces'] = relationship('Workspaces', back_populates='upload_sessions')
class WorkspaceOperations(Base):
__tablename__ = 'workspace_operations'
__table_args__ = (
ForeignKeyConstraint(['requested_by'], ['users.user_id'], ondelete='RESTRICT', name='fk_workspace_operations_user'),
ForeignKeyConstraint(['runtime_id'], ['runtime_instances.runtime_id'], ondelete='SET NULL', name='fk_workspace_operations_runtime'),
ForeignKeyConstraint(['workspace_id'], ['workspaces.workspace_id'], ondelete='RESTRICT', name='fk_workspace_operations_workspace'),
Index('fk_workspace_operations_user', 'requested_by'),
Index('idx_workspace_operations_runtime', 'runtime_id', 'created_at'),
Index('idx_workspace_operations_workspace', 'workspace_id', 'operation_status', 'created_at'),
Index('uk_workspace_operations_request', 'request_id', unique=True),
{'comment': '无状态 Backend 的 Workspace/Jupyter 异步操作记录'}
)
operation_id: Mapped[str] = mapped_column(CHAR(26), primary_key=True)
workspace_id: Mapped[str] = mapped_column(CHAR(26), nullable=False)
operation_type: Mapped[str] = mapped_column(String(24), nullable=False, comment='open/close/mount/unmount/start/stop/restart/recycle')
operation_status: Mapped[str] = mapped_column(String(24), nullable=False, server_default=text("'pending'"), comment='pending/running/succeeded/failed/cancelled')
state_version: Mapped[int] = mapped_column(INTEGER, nullable=False, server_default=text("0"), comment='乐观锁版本')
requested_by: Mapped[str] = mapped_column(CHAR(26), nullable=False)
created_at: Mapped[datetime.datetime] = mapped_column(DATETIME(fsp=3), nullable=False, server_default=text('CURRENT_TIMESTAMP(3)'))
runtime_id: Mapped[Optional[str]] = mapped_column(CHAR(26))
request_id: Mapped[Optional[str]] = mapped_column(String(128))
started_at: Mapped[Optional[datetime.datetime]] = mapped_column(DATETIME(fsp=3))
finished_at: Mapped[Optional[datetime.datetime]] = mapped_column(DATETIME(fsp=3))
error_code: Mapped[Optional[str]] = mapped_column(String(64))
error_message: Mapped[Optional[str]] = mapped_column(Text)
users: Mapped['Users'] = relationship('Users', back_populates='workspace_operations')
runtime: Mapped[Optional['RuntimeInstances']] = relationship('RuntimeInstances', back_populates='workspace_operations')
workspace: Mapped['Workspaces'] = relationship('Workspaces', back_populates='workspace_operations')
class NotebookSnapshots(Base):
__tablename__ = 'notebook_snapshots'
__table_args__ = (
ForeignKeyConstraint(['artifact_object_id'], ['storage_objects.storage_object_id'], ondelete='RESTRICT', name='fk_snapshots_artifact_object'),
ForeignKeyConstraint(['created_by'], ['users.user_id'], ondelete='RESTRICT', name='fk_snapshots_created_by'),
ForeignKeyConstraint(['script_id'], ['scripts.script_id'], ondelete='RESTRICT', name='fk_snapshots_script'),
ForeignKeyConstraint(['source_object_id'], ['storage_objects.storage_object_id'], ondelete='RESTRICT', name='fk_snapshots_source_object'),
ForeignKeyConstraint(['workspace_id'], ['workspaces.workspace_id'], ondelete='RESTRICT', name='fk_snapshots_workspace'),
Index('fk_snapshots_created_by', 'created_by'),
Index('fk_snapshots_source_object', 'source_object_id'),
Index('idx_snapshots_workspace_created', 'workspace_id', 'created_at'),
Index('uk_snapshots_artifact', 'artifact_object_id', unique=True),
Index('uk_snapshots_script_hash', 'script_id', 'content_hash', unique=True),
{'comment': 'Notebook 开发快照,append-only'}
)
snapshot_id: Mapped[str] = mapped_column(CHAR(26), primary_key=True)
workspace_id: Mapped[str] = mapped_column(CHAR(26), nullable=False)
script_id: Mapped[str] = mapped_column(CHAR(26), nullable=False)
source_object_id: Mapped[str] = mapped_column(CHAR(26), nullable=False)
artifact_object_id: Mapped[str] = mapped_column(CHAR(26), nullable=False)
snapshot_name: Mapped[str] = mapped_column(String(255), nullable=False)
content_hash: Mapped[str] = mapped_column(CHAR(64), nullable=False)
outputs_stripped: Mapped[int] = mapped_column(TINYINT(1), nullable=False, server_default=text("1"))
created_by: Mapped[str] = mapped_column(CHAR(26), nullable=False)
created_at: Mapped[datetime.datetime] = mapped_column(DATETIME(fsp=3), nullable=False, server_default=text('CURRENT_TIMESTAMP(3)'))
description: Mapped[Optional[str]] = mapped_column(String(1000))
artifact_object: Mapped['StorageObjects'] = relationship('StorageObjects', foreign_keys=[artifact_object_id], back_populates='notebook_snapshots')
users: Mapped['Users'] = relationship('Users', back_populates='notebook_snapshots')
script: Mapped['Scripts'] = relationship('Scripts', back_populates='notebook_snapshots')
source_object: Mapped['StorageObjects'] = relationship('StorageObjects', foreign_keys=[source_object_id], back_populates='notebook_snapshots_')
workspace: Mapped['Workspaces'] = relationship('Workspaces', back_populates='notebook_snapshots')
class Versions(Base):
__tablename__ = 'versions'
__table_args__ = (
ForeignKeyConstraint(['artifact_object_id'], ['storage_objects.storage_object_id'], ondelete='RESTRICT', name='fk_versions_artifact_object'),
ForeignKeyConstraint(['created_by'], ['users.user_id'], ondelete='RESTRICT', name='fk_versions_created_by'),
ForeignKeyConstraint(['script_id'], ['scripts.script_id'], ondelete='RESTRICT', name='fk_versions_script'),
ForeignKeyConstraint(['source_object_id'], ['storage_objects.storage_object_id'], ondelete='RESTRICT', name='fk_versions_source_object'),
ForeignKeyConstraint(['workspace_id'], ['workspaces.workspace_id'], ondelete='RESTRICT', name='fk_versions_workspace'),
Index('fk_versions_source_object', 'source_object_id'),
Index('idx_versions_creator', 'created_by', 'created_at'),
Index('idx_versions_workspace_created', 'workspace_id', 'created_at'),
Index('uk_versions_artifact', 'artifact_object_id', unique=True),
Index('uk_versions_script_hash', 'script_id', 'content_hash', unique=True),
Index('uk_versions_script_no', 'script_id', 'version_no', unique=True),
{'comment': '不可变稳定版本;调度节点必须引用 versions_id'}
)
versions_id: Mapped[str] = mapped_column(CHAR(26), primary_key=True, comment='稳定版本唯一 ID')
workspace_id: Mapped[str] = mapped_column(CHAR(26), nullable=False)
script_id: Mapped[str] = mapped_column(CHAR(26), nullable=False)
source_object_id: Mapped[str] = mapped_column(CHAR(26), nullable=False, comment='发布时的源对象')
artifact_object_id: Mapped[str] = mapped_column(CHAR(26), nullable=False, comment='RustFS 不可变版本制品')
version_no: Mapped[int] = mapped_column(INTEGER, nullable=False)
version_label: Mapped[str] = mapped_column(String(32), nullable=False, comment='例如 v1.0')
source_path: Mapped[str] = mapped_column(String(1024), nullable=False, comment='发布时路径快照')
artifact_path: Mapped[str] = mapped_column(String(1500), nullable=False)
content_hash: Mapped[str] = mapped_column(CHAR(64), nullable=False)
file_size_bytes: Mapped[int] = mapped_column(BIGINT, nullable=False, server_default=text("0"))
visibility: Mapped[str] = mapped_column(String(16), nullable=False, server_default=text("'private'"))
created_by: Mapped[str] = mapped_column(CHAR(26), nullable=False)
created_at: Mapped[datetime.datetime] = mapped_column(DATETIME(fsp=3), nullable=False, server_default=text('CURRENT_TIMESTAMP(3)'))
release_note: Mapped[Optional[str]] = mapped_column(String(1000))
schedule_hidden_at: Mapped[Optional[datetime.datetime]] = mapped_column(
DATETIME(fsp=3),
comment='从调度稳定版本列表移除的时间;不影响版本和运行历史',
)
artifact_object: Mapped['StorageObjects'] = relationship('StorageObjects', foreign_keys=[artifact_object_id], back_populates='versions')
users: Mapped['Users'] = relationship('Users', back_populates='versions')
script: Mapped['Scripts'] = relationship('Scripts', back_populates='versions')
source_object: Mapped['StorageObjects'] = relationship('StorageObjects', foreign_keys=[source_object_id], back_populates='versions_')
workspace: Mapped['Workspaces'] = relationship('Workspaces', back_populates='versions')
experiments: Mapped[list['Experiments']] = relationship('Experiments', back_populates='versions')
schedule_nodes: Mapped[list['ScheduleNodes']] = relationship('ScheduleNodes', back_populates='versions')
schedule_node_runs: Mapped[list['ScheduleNodeRuns']] = relationship('ScheduleNodeRuns', back_populates='versions')
class Experiments(Base):
__tablename__ = 'experiments'
__table_args__ = (
ForeignKeyConstraint(['logs_object_id'], ['storage_objects.storage_object_id'], ondelete='SET NULL', name='fk_experiments_logs'),
ForeignKeyConstraint(['owner_user_id'], ['users.user_id'], ondelete='RESTRICT', name='fk_experiments_owner'),
ForeignKeyConstraint(['parent_experiment_id'], ['experiments.experiment_id'], ondelete='SET NULL', name='fk_experiments_parent'),
ForeignKeyConstraint(['result_object_id'], ['storage_objects.storage_object_id'], ondelete='SET NULL', name='fk_experiments_result'),
ForeignKeyConstraint(['schedule_run_id'], ['schedule_runs.run_id'], ondelete='SET NULL', name='fk_experiments_schedule_run'),
ForeignKeyConstraint(['script_id'], ['scripts.script_id'], ondelete='SET NULL', name='fk_experiments_script'),
ForeignKeyConstraint(['versions_id'], ['versions.versions_id'], ondelete='SET NULL', name='fk_experiments_version'),
ForeignKeyConstraint(['workspace_id'], ['workspaces.workspace_id'], ondelete='RESTRICT', name='fk_experiments_workspace'),
Index('fk_experiments_logs', 'logs_object_id'),
Index('fk_experiments_parent', 'parent_experiment_id'),
Index('fk_experiments_result', 'result_object_id'),
Index('fk_experiments_script', 'script_id'),
Index('idx_experiments_owner', 'owner_user_id', 'created_at'),
Index('idx_experiments_schedule_run', 'schedule_run_id'),
Index('idx_experiments_version', 'versions_id'),
Index('idx_experiments_workspace', 'workspace_id', 'experiment_status', 'created_at'),
{'comment': '实验记录'}
)
experiment_id: Mapped[str] = mapped_column(CHAR(26), primary_key=True)
workspace_id: Mapped[str] = mapped_column(CHAR(26), nullable=False)
owner_user_id: Mapped[str] = mapped_column(CHAR(26), nullable=False)
experiment_name: Mapped[str] = mapped_column(String(255), nullable=False)
source_type: Mapped[str] = mapped_column(String(24), nullable=False, comment='python/notebook/schedule/rerun')
experiment_status: Mapped[str] = mapped_column(String(24), nullable=False, server_default=text("'queued'"))
state_version: Mapped[int] = mapped_column(INTEGER, nullable=False, server_default=text("0"), comment='乐观锁版本')
created_at: Mapped[datetime.datetime] = mapped_column(DATETIME(fsp=3), nullable=False, server_default=text('CURRENT_TIMESTAMP(3)'))
script_id: Mapped[Optional[str]] = mapped_column(CHAR(26))
versions_id: Mapped[Optional[str]] = mapped_column(CHAR(26), comment='工作副本运行时可为空')
schedule_run_id: Mapped[Optional[str]] = mapped_column(CHAR(26))
parent_experiment_id: Mapped[Optional[str]] = mapped_column(CHAR(26))
parameters_json: Mapped[Optional[dict]] = mapped_column(JSON)
environment_json: Mapped[Optional[dict]] = mapped_column(JSON)
result_summary: Mapped[Optional[str]] = mapped_column(String(2000))
logs_object_id: Mapped[Optional[str]] = mapped_column(CHAR(26))
result_object_id: Mapped[Optional[str]] = mapped_column(CHAR(26))
started_at: Mapped[Optional[datetime.datetime]] = mapped_column(DATETIME(fsp=3))
finished_at: Mapped[Optional[datetime.datetime]] = mapped_column(DATETIME(fsp=3))
duration_ms: Mapped[Optional[int]] = mapped_column(BIGINT)
deleted_at: Mapped[Optional[datetime.datetime]] = mapped_column(DATETIME(fsp=3))
logs_object: Mapped[Optional['StorageObjects']] = relationship('StorageObjects', foreign_keys=[logs_object_id], back_populates='experiments')
owner_user: Mapped['Users'] = relationship('Users', back_populates='experiments')
parent_experiment: Mapped[Optional['Experiments']] = relationship('Experiments', remote_side=[experiment_id], back_populates='parent_experiment_reverse')
parent_experiment_reverse: Mapped[list['Experiments']] = relationship('Experiments', remote_side=[parent_experiment_id], back_populates='parent_experiment')
result_object: Mapped[Optional['StorageObjects']] = relationship('StorageObjects', foreign_keys=[result_object_id], back_populates='experiments_')
schedule_run: Mapped[Optional['ScheduleRuns']] = relationship('ScheduleRuns', back_populates='experiments')
script: Mapped[Optional['Scripts']] = relationship('Scripts', back_populates='experiments')
versions: Mapped[Optional['Versions']] = relationship('Versions', back_populates='experiments')
workspace: Mapped['Workspaces'] = relationship('Workspaces', back_populates='experiments')
experiment_metrics: Mapped[list['ExperimentMetrics']] = relationship('ExperimentMetrics', back_populates='experiment')
experiment_resources: Mapped[list['ExperimentResources']] = relationship('ExperimentResources', back_populates='experiment')
class ScheduleNodes(Base):
__tablename__ = 'schedule_nodes'
__table_args__ = (
ForeignKeyConstraint(['schedule_id'], ['schedules.schedule_id'], ondelete='CASCADE', name='fk_schedule_nodes_schedule'),
ForeignKeyConstraint(['versions_id'], ['versions.versions_id'], ondelete='RESTRICT', name='fk_schedule_nodes_version'),
Index('idx_schedule_nodes_version', 'versions_id'),
Index('uk_schedule_nodes_key', 'schedule_id', 'node_key', unique=True),
{'comment': 'DAG 节点,必须引用稳定版本'}
)
node_id: Mapped[str] = mapped_column(CHAR(26), primary_key=True)
schedule_id: Mapped[str] = mapped_column(CHAR(26), nullable=False)
node_key: Mapped[str] = mapped_column(String(64), nullable=False, comment='画布内稳定标识')
node_name: Mapped[str] = mapped_column(String(255), nullable=False)
versions_id: Mapped[str] = mapped_column(CHAR(26), nullable=False)
timeout_seconds: Mapped[int] = mapped_column(INTEGER, nullable=False, server_default=text("600"))
retry_count: Mapped[int] = mapped_column(INTEGER, nullable=False, server_default=text("0"))
retry_interval_sec: Mapped[int] = mapped_column(INTEGER, nullable=False, server_default=text("5"))
position_x: Mapped[decimal.Decimal] = mapped_column(DECIMAL(10, 2), nullable=False, server_default=text("0.00"))
position_y: Mapped[decimal.Decimal] = mapped_column(DECIMAL(10, 2), nullable=False, server_default=text("0.00"))
created_at: Mapped[datetime.datetime] = mapped_column(DATETIME(fsp=3), nullable=False, server_default=text('CURRENT_TIMESTAMP(3)'))
updated_at: Mapped[datetime.datetime] = mapped_column(DATETIME(fsp=3), nullable=False, server_default=text('CURRENT_TIMESTAMP(3) ON UPDATE CURRENT_TIMESTAMP(3)'))
arguments_json: Mapped[Optional[dict]] = mapped_column(JSON)
env_refs_json: Mapped[Optional[dict]] = mapped_column(JSON, comment='只存密钥引用,不存明文密钥')
schedule: Mapped['Schedules'] = relationship('Schedules', back_populates='schedule_nodes')
versions: Mapped['Versions'] = relationship('Versions', back_populates='schedule_nodes')
schedule_edges: Mapped[list['ScheduleEdges']] = relationship('ScheduleEdges', foreign_keys='[ScheduleEdges.source_node_id]', back_populates='source_node')
schedule_edges_: Mapped[list['ScheduleEdges']] = relationship('ScheduleEdges', foreign_keys='[ScheduleEdges.target_node_id]', back_populates='target_node')
schedule_node_runs: Mapped[list['ScheduleNodeRuns']] = relationship('ScheduleNodeRuns', back_populates='node')
class ExperimentMetrics(Base):
__tablename__ = 'experiment_metrics'
__table_args__ = (
ForeignKeyConstraint(['experiment_id'], ['experiments.experiment_id'], ondelete='CASCADE', name='fk_experiment_metrics_experiment'),
Index('idx_experiment_metrics_lookup', 'experiment_id', 'metric_name', 'step_no'),
{'comment': '实验指标,支持筛选和曲线'}
)
metric_id: Mapped[int] = mapped_column(BIGINT, primary_key=True)
experiment_id: Mapped[str] = mapped_column(CHAR(26), nullable=False)
metric_name: Mapped[str] = mapped_column(String(128), nullable=False)
recorded_at: Mapped[datetime.datetime] = mapped_column(DATETIME(fsp=3), nullable=False, server_default=text('CURRENT_TIMESTAMP(3)'))
metric_value: Mapped[Optional[decimal.Decimal]] = mapped_column(Double(asdecimal=True))
metric_text: Mapped[Optional[str]] = mapped_column(String(1000))
step_no: Mapped[Optional[int]] = mapped_column(BigInteger)
experiment: Mapped['Experiments'] = relationship('Experiments', back_populates='experiment_metrics')
class ExperimentResources(Base):
__tablename__ = 'experiment_resources'
__table_args__ = (
ForeignKeyConstraint(['experiment_id'], ['experiments.experiment_id'], ondelete='CASCADE', name='fk_experiment_resources_experiment'),
ForeignKeyConstraint(['resource_id'], ['data_resources.resource_id'], ondelete='RESTRICT', name='fk_experiment_resources_resource'),
Index('fk_experiment_resources_resource', 'resource_id'),
{'comment': '实验与数据资源'}
)
experiment_id: Mapped[str] = mapped_column(CHAR(26), primary_key=True)
resource_id: Mapped[str] = mapped_column(CHAR(26), primary_key=True)
resource_role: Mapped[str] = mapped_column(String(16), primary_key=True, server_default=text("'input'"), comment='input/output')
created_at: Mapped[datetime.datetime] = mapped_column(DATETIME(fsp=3), nullable=False, server_default=text('CURRENT_TIMESTAMP(3)'))
experiment: Mapped['Experiments'] = relationship('Experiments', back_populates='experiment_resources')
resource: Mapped['DataResources'] = relationship('DataResources', back_populates='experiment_resources')
class ScheduleEdges(Base):
__tablename__ = 'schedule_edges'
__table_args__ = (
ForeignKeyConstraint(['schedule_id'], ['schedules.schedule_id'], ondelete='CASCADE', name='fk_schedule_edges_schedule'),
ForeignKeyConstraint(['source_node_id'], ['schedule_nodes.node_id'], ondelete='CASCADE', name='fk_schedule_edges_source'),
ForeignKeyConstraint(['target_node_id'], ['schedule_nodes.node_id'], ondelete='CASCADE', name='fk_schedule_edges_target'),
Index('fk_schedule_edges_source', 'source_node_id'),
Index('idx_schedule_edges_target', 'target_node_id'),
Index('uk_schedule_edges_pair', 'schedule_id', 'source_node_id', 'target_node_id', unique=True),
{'comment': 'DAG 有向边'}
)
edge_id: Mapped[str] = mapped_column(CHAR(26), primary_key=True)
schedule_id: Mapped[str] = mapped_column(CHAR(26), nullable=False)
source_node_id: Mapped[str] = mapped_column(CHAR(26), nullable=False)
target_node_id: Mapped[str] = mapped_column(CHAR(26), nullable=False)
created_at: Mapped[datetime.datetime] = mapped_column(DATETIME(fsp=3), nullable=False, server_default=text('CURRENT_TIMESTAMP(3)'))
condition_expr: Mapped[Optional[str]] = mapped_column(String(1000))
schedule: Mapped['Schedules'] = relationship('Schedules', back_populates='schedule_edges')
source_node: Mapped['ScheduleNodes'] = relationship('ScheduleNodes', foreign_keys=[source_node_id], back_populates='schedule_edges')
target_node: Mapped['ScheduleNodes'] = relationship('ScheduleNodes', foreign_keys=[target_node_id], back_populates='schedule_edges_')
class ScheduleNodeRuns(Base):
__tablename__ = 'schedule_node_runs'
__table_args__ = (
ForeignKeyConstraint(['logs_object_id'], ['storage_objects.storage_object_id'], ondelete='SET NULL', name='fk_node_runs_logs'),
ForeignKeyConstraint(['node_id'], ['schedule_nodes.node_id'], ondelete='RESTRICT', name='fk_node_runs_node'),
ForeignKeyConstraint(['result_object_id'], ['storage_objects.storage_object_id'], ondelete='SET NULL', name='fk_node_runs_result'),
ForeignKeyConstraint(['run_id'], ['schedule_runs.run_id'], ondelete='CASCADE', name='fk_node_runs_run'),
ForeignKeyConstraint(['versions_id'], ['versions.versions_id'], ondelete='RESTRICT', name='fk_node_runs_version'),
Index('fk_node_runs_logs', 'logs_object_id'),
Index('fk_node_runs_node', 'node_id'),
Index('fk_node_runs_result', 'result_object_id'),
Index('idx_node_runs_status', 'run_id', 'node_status'),
Index('idx_node_runs_version', 'versions_id'),
Index('uk_node_runs_attempt', 'run_id', 'node_id', 'attempt_no', unique=True),
{'comment': '调度节点运行与重试'}
)
node_run_id: Mapped[str] = mapped_column(CHAR(26), primary_key=True)
run_id: Mapped[str] = mapped_column(CHAR(26), nullable=False)
node_id: Mapped[str] = mapped_column(CHAR(26), nullable=False)
versions_id: Mapped[str] = mapped_column(CHAR(26), nullable=False)
attempt_no: Mapped[int] = mapped_column(INTEGER, nullable=False, server_default=text("1"))
node_status: Mapped[str] = mapped_column(String(24), nullable=False, server_default=text("'queued'"), comment='queued/running/succeeded/failed/skipped/cancelled/timed_out')
state_version: Mapped[int] = mapped_column(INTEGER, nullable=False, server_default=text("0"), comment='乐观锁版本')
created_at: Mapped[datetime.datetime] = mapped_column(DATETIME(fsp=3), nullable=False, server_default=text('CURRENT_TIMESTAMP(3)'))
started_at: Mapped[Optional[datetime.datetime]] = mapped_column(DATETIME(fsp=3))
finished_at: Mapped[Optional[datetime.datetime]] = mapped_column(DATETIME(fsp=3))
duration_ms: Mapped[Optional[int]] = mapped_column(BIGINT)
exit_code: Mapped[Optional[int]] = mapped_column(Integer)
message: Mapped[Optional[str]] = mapped_column(String(2000))
metrics_json: Mapped[Optional[dict]] = mapped_column(JSON)
logs_object_id: Mapped[Optional[str]] = mapped_column(CHAR(26))
result_object_id: Mapped[Optional[str]] = mapped_column(CHAR(26))
logs_object: Mapped[Optional['StorageObjects']] = relationship('StorageObjects', foreign_keys=[logs_object_id], back_populates='schedule_node_runs')
node: Mapped['ScheduleNodes'] = relationship('ScheduleNodes', back_populates='schedule_node_runs')
result_object: Mapped[Optional['StorageObjects']] = relationship('StorageObjects', foreign_keys=[result_object_id], back_populates='schedule_node_runs_')
run: Mapped['ScheduleRuns'] = relationship('ScheduleRuns', back_populates='schedule_node_runs')
versions: Mapped['Versions'] = relationship('Versions', back_populates='schedule_node_runs')
+51 -5
View File
@@ -1,5 +1,51 @@
# coding=utf-8
"""
@Time :2026/7/29
@Author :tao.chen
"""
from __future__ import annotations
from collections.abc import AsyncIterator
from contextlib import asynccontextmanager
from sqlalchemy.ext.asyncio import (
AsyncEngine,
AsyncSession,
async_sessionmaker,
create_async_engine,
)
AsyncSessionFactory = async_sessionmaker[AsyncSession]
def create_database_engine(
database_url: str,
*,
echo: bool = False,
pool_pre_ping: bool = True,
) -> AsyncEngine:
"""Create an async SQLAlchemy engine without storing global connection state."""
return create_async_engine(
database_url,
echo=echo,
pool_pre_ping=pool_pre_ping,
)
def create_session_factory(engine: AsyncEngine) -> AsyncSessionFactory:
"""Create the shared async session factory used by FastAPI dependencies."""
return async_sessionmaker(
bind=engine,
class_=AsyncSession,
expire_on_commit=False,
autoflush=False,
)
@asynccontextmanager
async def session_scope(
factory: AsyncSessionFactory,
) -> AsyncIterator[AsyncSession]:
"""Commit a unit of work or roll it back when an exception is raised."""
async with factory() as session:
try:
yield session
await session.commit()
except Exception:
await session.rollback()
raise
+71
View File
@@ -0,0 +1,71 @@
from __future__ import annotations
from datetime import UTC, datetime
from typing import Any
from sqlalchemy.ext.asyncio import AsyncSession
from common.db.models import OutboxEvents
from common.ids import new_ulid
STREAM_BY_EVENT_TYPE = {
"schedule.run.requested": "stream:scheduler:commands",
"job.node.execute": "stream:jobs:execute",
"job.node.finished": "stream:jobs:results",
}
def utcnow() -> datetime:
return datetime.now(UTC).replace(tzinfo=None)
def event_time(value: datetime | None = None) -> str:
item = value or utcnow()
if item.tzinfo is None:
item = item.replace(tzinfo=UTC)
return item.astimezone(UTC).isoformat().replace("+00:00", "Z")
async def add_outbox_event(
session: AsyncSession,
*,
event_type: str,
producer: str,
trace_id: str,
aggregate_type: str,
aggregate_id: str,
idempotency_key: str,
payload: dict[str, Any],
available_at: datetime | None = None,
) -> OutboxEvents:
if event_type not in STREAM_BY_EVENT_TYPE:
raise ValueError(f"unsupported event type: {event_type}")
event_id = new_ulid()
envelope = {
"event_id": event_id,
"event_type": event_type,
"schema_version": 1,
"occurred_at": event_time(),
"producer": producer,
"trace_id": trace_id,
"aggregate_type": aggregate_type,
"aggregate_id": aggregate_id,
"idempotency_key": idempotency_key,
"payload": payload,
}
item = OutboxEvents(
event_id=event_id,
aggregate_type=aggregate_type,
aggregate_id=aggregate_id,
event_type=event_type,
schema_version=1,
payload_json=envelope,
event_status="pending",
available_at=available_at or utcnow(),
retry_count=0,
trace_id=trace_id,
idempotency_key=idempotency_key,
)
session.add(item)
return item
+17
View File
@@ -0,0 +1,17 @@
from __future__ import annotations
import secrets
import time
_CROCKFORD32 = "0123456789ABCDEFGHJKMNPQRSTVWXYZ"
def new_ulid() -> str:
"""Return a lexicographically sortable 26-character ULID."""
timestamp_ms = int(time.time_ns() // 1_000_000)
value = (timestamp_ms << 80) | secrets.randbits(80)
encoded = ["0"] * 26
for index in range(25, -1, -1):
encoded[index] = _CROCKFORD32[value & 31]
value >>= 5
return "".join(encoded)
+79
View File
@@ -0,0 +1,79 @@
from __future__ import annotations
import asyncio
import os
from datetime import UTC, datetime
from typing import Any, Callable
from fastapi import FastAPI, Response, status
def _target_list() -> list[str]:
value = os.getenv("READINESS_TARGETS", "")
return [item.strip() for item in value.split(",") if item.strip()]
async def _check_tcp_target(target: str) -> dict[str, Any]:
host, port_text = target.rsplit(":", 1)
try:
reader, writer = await asyncio.wait_for(
asyncio.open_connection(host, int(port_text)),
timeout=1.5,
)
writer.close()
await writer.wait_closed()
return {"target": target, "status": "ok"}
except (OSError, TimeoutError, ValueError) as exc:
return {
"target": target,
"status": "error",
"detail": type(exc).__name__,
}
def create_service_app(
service_name: str,
*,
lifespan: Callable[..., Any] | None = None,
) -> FastAPI:
app = FastAPI(
title=f"{service_name} service",
version="0.1.0",
docs_url="/docs",
redoc_url=None,
lifespan=lifespan,
)
def base_payload(state: str) -> dict[str, str]:
return {
"status": state,
"service": service_name,
"timestamp": datetime.now(UTC).isoformat(),
}
@app.get("/")
async def root() -> dict[str, str]:
return base_payload("running")
@app.get("/health/live")
async def live() -> dict[str, str]:
return base_payload("ok")
@app.get("/api/v1/health")
async def public_health() -> dict[str, str]:
return base_payload("ok")
@app.get("/health/ready")
async def ready(response: Response) -> dict[str, Any]:
checks = await asyncio.gather(
*(_check_tcp_target(target) for target in _target_list())
)
ready_state = all(check["status"] == "ok" for check in checks)
if not ready_state:
response.status_code = status.HTTP_503_SERVICE_UNAVAILABLE
return {
**base_payload("ready" if ready_state else "not_ready"),
"checks": checks,
}
return app
+3 -5
View File
@@ -1,5 +1,3 @@
# coding=utf-8
"""
@Time :2026/7/29
@Author :tao.chen
"""
from common.storage.rustfs import RustFSObjectStore
__all__ = ["RustFSObjectStore"]
+126 -5
View File
@@ -1,5 +1,126 @@
# coding=utf-8
"""
@Time :2026/7/29
@Author :tao.chen
"""
from __future__ import annotations
import hashlib
from typing import Any, BinaryIO
import boto3
from botocore.client import Config
from botocore.exceptions import ClientError
class RustFSObjectStore:
def __init__(
self,
*,
internal_endpoint: str,
public_endpoint: str,
access_key: str,
secret_key: str,
) -> None:
common = {
"aws_access_key_id": access_key,
"aws_secret_access_key": secret_key,
"region_name": "us-east-1",
"config": Config(
signature_version="s3v4",
s3={"addressing_style": "path"},
),
}
self.internal = boto3.client(
"s3",
endpoint_url=internal_endpoint.rstrip("/"),
**common,
)
self.public = boto3.client(
"s3",
endpoint_url=public_endpoint.rstrip("/"),
**common,
)
def ensure_bucket(self, bucket_name: str) -> None:
try:
self.internal.head_bucket(Bucket=bucket_name)
except ClientError as exc:
code = str(exc.response.get("Error", {}).get("Code", ""))
if code not in {"404", "NoSuchBucket", "NotFound"}:
raise
self.internal.create_bucket(Bucket=bucket_name)
def presign_put(
self,
*,
bucket_name: str,
object_key: str,
content_type: str,
expected_hash: str | None,
expires_seconds: int,
public: bool,
) -> tuple[str, dict[str, str]]:
params: dict[str, Any] = {
"Bucket": bucket_name,
"Key": object_key,
"ContentType": content_type,
}
headers = {"Content-Type": content_type}
if expected_hash:
params["Metadata"] = {"sha256": expected_hash}
headers["x-amz-meta-sha256"] = expected_hash
client = self.public if public else self.internal
url = client.generate_presigned_url(
"put_object",
Params=params,
ExpiresIn=expires_seconds,
)
return url, headers
def presign_get(
self,
*,
bucket_name: str,
object_key: str,
file_name: str,
expires_seconds: int,
) -> str:
return self.public.generate_presigned_url(
"get_object",
Params={
"Bucket": bucket_name,
"Key": object_key,
"ResponseContentDisposition": (
f'attachment; filename="{file_name.encode("ascii", "ignore").decode() or "download"}"'
),
},
ExpiresIn=expires_seconds,
)
def put_bytes(
self,
*,
bucket_name: str,
object_key: str,
content: bytes,
content_type: str,
content_hash: str,
) -> None:
self.internal.put_object(
Bucket=bucket_name,
Key=object_key,
Body=content,
ContentType=content_type,
Metadata={"sha256": content_hash},
)
def head(self, *, bucket_name: str, object_key: str) -> dict[str, Any]:
return self.internal.head_object(Bucket=bucket_name, Key=object_key)
def sha256(self, *, bucket_name: str, object_key: str) -> str:
response = self.internal.get_object(Bucket=bucket_name, Key=object_key)
body: BinaryIO = response["Body"]
digest = hashlib.sha256()
while chunk := body.read(1024 * 1024):
digest.update(chunk)
body.close()
return digest.hexdigest()
def delete(self, *, bucket_name: str, object_key: str) -> None:
self.internal.delete_object(Bucket=bucket_name, Key=object_key)