343 lines
14 KiB
Python
343 lines
14 KiB
Python
"""Centralised configuration via pydantic-settings.
|
|
|
|
All Python services (backend / schedule / runtime) import ``settings`` from
|
|
this module. The instance is a process-wide singleton (lru_cache wrapped),
|
|
so each field is parsed from the environment exactly once at first access.
|
|
|
|
Rules for adding a new variable:
|
|
|
|
1. Add the field here with a sensible default that lets local dev
|
|
boot without the env var set.
|
|
2. Use ``settings.<name>`` at the call site. Never re-introduce
|
|
``os.environ`` / ``os.getenv`` for application config — they
|
|
bypass this central registry.
|
|
3. Document the env var name in ``.env.example`` so operators know
|
|
it exists.
|
|
"""
|
|
|
|
from __future__ import annotations
|
|
|
|
import base64
|
|
import os
|
|
from functools import lru_cache
|
|
from typing import Annotated, Any
|
|
from venv import logger
|
|
|
|
from cryptography.hazmat.primitives.ciphers.aead import AESGCM
|
|
from pydantic import Field, field_validator, model_validator
|
|
from pydantic_settings import BaseSettings, NoDecode, SettingsConfigDict
|
|
from sqlalchemy.engine import make_url
|
|
|
|
|
|
# ── AES 解密核心函数 ────────────────────────────────────────────────────────
|
|
def _decrypt_value(cipher_text: str) -> str:
|
|
"""解密 ENC(...) 格式的字符串。密钥从环境变量 APP_CONFIG_SECRET_KEY 获取。"""
|
|
if not (cipher_text.startswith("ENC(") and cipher_text.endswith(")")):
|
|
return cipher_text
|
|
|
|
secret_key = os.getenv("APP_CONFIG_SECRET_KEY")
|
|
if not secret_key:
|
|
raise RuntimeError(
|
|
"致命错误: 检测到配置项包含 ENC(...) 密文，但系统环境变量 "
|
|
"APP_CONFIG_SECRET_KEY 未设置！"
|
|
)
|
|
|
|
raw_payload = cipher_text[4:-1]
|
|
try:
|
|
data = base64.b64decode(raw_payload)
|
|
nonce, ciphertext = data[:12], data[12:]
|
|
# 将传入的 key 补全或截断为 32 字节 (AES-256)
|
|
key_bytes = secret_key.encode("utf-8").ljust(32, b"\0")[:32]
|
|
cipher = AESGCM(key_bytes)
|
|
decrypted = cipher.decrypt(nonce, ciphertext, None)
|
|
return decrypted.decode("utf-8")
|
|
except Exception as e:
|
|
raise ValueError(f"配置项解密失败，请检查密钥或密文正确性: {e}") from e
|
|
|
|
|
|
class Settings(BaseSettings):
|
|
# ── database ──────────────────────────────────────────────────
|
|
database_url: str = Field(
|
|
default=(
|
|
"mysql+asyncmy://model_platform:model_platform@mysql:3306/"
|
|
"model_platform?charset=utf8mb4"
|
|
),
|
|
description="SQLAlchemy async URI for the platform MySQL.",
|
|
)
|
|
operations_database_url: str | None = Field(
|
|
default=None,
|
|
description=(
|
|
"SQLAlchemy async URI for the isolated model_operations database. "
|
|
"When omitted it reuses DATABASE_URL credentials and switches only "
|
|
"the schema name to model_operations."
|
|
),
|
|
)
|
|
platform_read_database_url: str | None = Field(
|
|
default=None,
|
|
description="Read-only model_platform connection used by operations APIs.",
|
|
)
|
|
deploy_database_url: str | None = Field(
|
|
default=None,
|
|
description="Read-only model_deploy connection used by operations APIs.",
|
|
)
|
|
|
|
# ── operations Redis accelerator ─────────────────────────────
|
|
redis_url: str | None = Field(
|
|
default=None,
|
|
description=(
|
|
"Optional Redis URL for operations query cache and event Streams. "
|
|
"MySQL remains the source of truth when Redis is unavailable."
|
|
),
|
|
)
|
|
redis_socket_connect_timeout_seconds: float = Field(default=2.0, gt=0, le=30)
|
|
redis_socket_timeout_seconds: float = Field(default=2.0, gt=0, le=30)
|
|
operations_cache_ttl_seconds: int = Field(default=300, ge=10, le=86400)
|
|
operations_redis_prefix: str = Field(default="model-platform:operations")
|
|
operations_event_stream: str = Field(
|
|
default="model-platform:operations:events"
|
|
)
|
|
operations_event_stream_maxlen: int = Field(default=10000, ge=100)
|
|
operations_data_mode: str = Field(
|
|
default="database",
|
|
description="Operations data source: database or mock.",
|
|
)
|
|
|
|
# ── JWT ───────────────────────────────────────────────────────
|
|
jwt_secret: str = Field(
|
|
default="dev-only-not-for-production",
|
|
description="HS256 secret used by backend's jupyter auth_request.",
|
|
)
|
|
cookie_force_secure: bool = Field(
|
|
default=False,
|
|
description=(
|
|
"Force the ``Secure`` flag on the session cookie even when the "
|
|
"inbound request scheme is plain HTTP. Enable this when running "
|
|
"behind a TLS-terminating reverse proxy that strips or rewrites "
|
|
"``X-Forwarded-Proto`` — without it the cookie is written without "
|
|
"the Secure flag and modern browsers will refuse to send it back "
|
|
"over HTTPS."
|
|
),
|
|
)
|
|
|
|
# ── service identity ──────────────────────────────────────────
|
|
service_name: str = Field(
|
|
default="service",
|
|
description="Service label surfaced in lifespan / health checks.",
|
|
)
|
|
|
|
# ── logging ───────────────────────────────────────────────────
|
|
log_level: str = Field(
|
|
default="DEBUG",
|
|
description=(
|
|
"loguru stderr sink level. One of DEBUG/INFO/WARNING/ERROR/CRITICAL; "
|
|
"anything else falls back to INFO inside configure_logging()."
|
|
),
|
|
)
|
|
audit_log_dir: str = Field(
|
|
default="data/logs/audit",
|
|
description=(
|
|
"审计日志目录。每天一个文件 audit-YYYY-MM-DD.log。"
|
|
"路径相对于 backend 进程 cwd（容器内通常为 /app）。"
|
|
),
|
|
)
|
|
audit_log_retention_days: int = Field(
|
|
default=30,
|
|
description="审计日志保留天数；过期文件启动时清理。设 0 关闭清理。",
|
|
)
|
|
audit_excluded_paths: Annotated[list[str], NoDecode] = Field(
|
|
default=[
|
|
"/health/live",
|
|
"/health/ready",
|
|
"/api/v1/health",
|
|
"/",
|
|
"/health/storage",
|
|
],
|
|
description=(
|
|
"审计排除的精确路径列表（不含 query）。命中即不写审计行。"
|
|
"诊断日志（method/path/status/ms）仍写 stderr。"
|
|
"环境变量 AUDIT_EXCLUDED_PATHS 用逗号分隔，例如"
|
|
" '/health/live,/api/v1/health'。"
|
|
),
|
|
)
|
|
|
|
@field_validator("audit_excluded_paths", mode="before")
|
|
@classmethod
|
|
def _split_audit_paths(cls, v):
|
|
if isinstance(v, str):
|
|
return [s.strip() for s in v.split(",") if s.strip()]
|
|
return v
|
|
|
|
# ── runtime container endpoint ───────────────────────────────
|
|
runtime_api_url: str = Field(
|
|
default="http://runtime:8000",
|
|
description="Backend → Runtime HTTP endpoint.",
|
|
)
|
|
rclone_rc_url: str = Field(
|
|
default="http://runtime:5572",
|
|
description="Backend → rclone RC HTTP endpoint (VFS cache invalidation).",
|
|
)
|
|
|
|
# ── object storage backend selection ─────────────────────────
|
|
storage_backend: str = Field(
|
|
default="s3",
|
|
description=(
|
|
"Which storage backend the deployment uses. ``s3`` (default) "
|
|
"reads the ``s3_*`` settings and connects to an S3-compatible "
|
|
"service. ``local`` uses on-disk filesystems under "
|
|
"``local_storage_base_dir`` — useful for dev / single-node / "
|
|
"air-gapped deployments."
|
|
),
|
|
)
|
|
local_storage_base_dir: str = Field(
|
|
default="/data",
|
|
description=(
|
|
"Root directory for the local-filesystem storage backend. The 4 "
|
|
"buckets become subdirectories: ``<root>/workspace``, "
|
|
"``<root>/version``, ``<root>/run_log``, ``<root>/trash``. "
|
|
"Default ``/data``; this directory must be a shared Docker "
|
|
"volume between the backend and runtime containers in local mode."
|
|
),
|
|
)
|
|
|
|
# ── S3-compatible object storage ─────────────────────────────
|
|
s3_endpoint: str = Field(
|
|
default="http://s3:9000",
|
|
description="S3 endpoint for the object-storage upstream.",
|
|
)
|
|
s3_access_key: str = Field(
|
|
default="modelplatform",
|
|
description="S3 access key for S3-compatible storage.",
|
|
)
|
|
s3_secret_key: str = Field(
|
|
default="modelplatformsecret",
|
|
description="S3 secret key for S3-compatible storage.",
|
|
)
|
|
s3_workspace_bucket: str = Field(
|
|
default="workspace",
|
|
description=(
|
|
"Bucket for workspace files (notebooks / scripts / working "
|
|
"copies). Layout: s3://<bucket>/<workspace_id>/..."
|
|
),
|
|
)
|
|
s3_version_bucket: str = Field(
|
|
default="version",
|
|
description="Bucket for immutable script-version artifacts.",
|
|
)
|
|
s3_run_log_bucket: str = Field(
|
|
default="run-log",
|
|
description="Bucket for schedule run logs.",
|
|
)
|
|
s3_trash_bucket: str = Field(
|
|
default="trash",
|
|
description=(
|
|
"Bucket for soft-deleted objects. The source bucket key is "
|
|
"preserved as a prefix so a restore is a same-key move. "
|
|
"Trash is reaped on a schedule out of band."
|
|
),
|
|
)
|
|
s3_trash_retention_days: int = Field(
|
|
default=30,
|
|
description=(
|
|
"How long a trashed object is retained before reaping. "
|
|
"Tracked in the database (StorageObjects.deleted_at) so the "
|
|
"reaper can run as a single SQL sweep."
|
|
),
|
|
)
|
|
|
|
# ── schedule → backend API ────────────────────────────────────
|
|
backend_api_url: str = Field(
|
|
default="http://backend:8000",
|
|
description="Schedule → Backend HTTP base URL (cron post-back).",
|
|
)
|
|
|
|
# ── service-to-service auth for /internal/v1/* (P0-1 fix) ─────
|
|
# Shared secret between backend and the schedule worker. The schedule
|
|
# posts the value in the ``X-Internal-Service-Token`` header when it
|
|
# uploads ``run_log`` / ``run_result`` artifacts. Backend's storage
|
|
# API rejects requests whose header does not match this value.
|
|
# Override via ``INTERNAL_SERVICE_TOKEN``; the placeholder default is
|
|
# safe for local dev with the matching schedule config but must be
|
|
# replaced in any non-dev deployment.
|
|
internal_service_token: str = Field(
|
|
default="dev-only-internal-token-not-for-production",
|
|
description=(
|
|
"Shared secret for service-to-service auth on /internal/v1/*. "
|
|
"Set identically on backend and schedule via INTERNAL_SERVICE_TOKEN."
|
|
),
|
|
)
|
|
|
|
# ── runtime public base URL ──────────────────────────────────
|
|
public_base_url: str = Field(
|
|
default="http://runtime",
|
|
description="Public base URL for the runtime container.",
|
|
)
|
|
|
|
# ── schedule execution tuning ────────────────────────────────
|
|
schedule_event_namespace: str = Field(
|
|
default="model-platform-local",
|
|
description=(
|
|
"Namespace prepended to schedule outbox event types. Each "
|
|
"deployment that shares a database must use a unique value."
|
|
),
|
|
)
|
|
schedule_execution_concurrency: int = Field(
|
|
default=4,
|
|
description=(
|
|
"Max concurrent notebooks running in the schedule worker. "
|
|
"Each notebook is dispatched as an asyncio task bounded by "
|
|
"a semaphore; the polling loop is never blocked."
|
|
),
|
|
)
|
|
|
|
# ── readiness probes ──────────────────────────────────────────
|
|
readiness_targets: str = Field(default="")
|
|
|
|
# ── 全局密文拦截器 ───────────────────────────────────────────
|
|
@model_validator(mode="before")
|
|
@classmethod
|
|
def _decrypt_encrypted_fields(cls, values: dict[str, Any]) -> dict[str, Any]:
|
|
"""在 Pydantic 赋值前自动扫描所有 str 类型的环境变量，解密 ENC(...)"""
|
|
if not isinstance(values, dict):
|
|
return values
|
|
|
|
for key, val in values.items():
|
|
if isinstance(val, str) and val.startswith("ENC("):
|
|
values[key] = _decrypt_value(val)
|
|
# print(values[key])
|
|
return values
|
|
|
|
@model_validator(mode="after")
|
|
def _derive_operations_database_url(self) -> "Settings":
|
|
"""Default the operations database to the platform server credentials."""
|
|
if not self.operations_database_url:
|
|
platform_url = make_url(self.database_url)
|
|
self.operations_database_url = platform_url.set(
|
|
database="model_operations"
|
|
).render_as_string(hide_password=False)
|
|
if not self.platform_read_database_url:
|
|
self.platform_read_database_url = self.database_url
|
|
if not self.deploy_database_url:
|
|
platform_url = make_url(self.database_url)
|
|
self.deploy_database_url = platform_url.set(
|
|
database="model_deploy"
|
|
).render_as_string(hide_password=False)
|
|
return self
|
|
|
|
model_config = SettingsConfigDict(
|
|
env_file=".env",
|
|
env_file_encoding="utf-8",
|
|
case_sensitive=False,
|
|
extra="ignore",
|
|
)
|
|
|
|
|
|
@lru_cache
|
|
def get_settings() -> Settings:
|
|
"""Construct (and cache) the singleton Settings instance."""
|
|
return Settings()
|
|
|
|
|
|
settings: Settings = get_settings()
|
|
|
|
__all__ = ["Settings", "get_settings", "settings"]
|