refactor
This commit is contained in:
@@ -9,13 +9,6 @@ from common.db.models import OutboxEvents
|
||||
from common.ids import new_ulid
|
||||
|
||||
|
||||
STREAM_BY_EVENT_TYPE = {
|
||||
"schedule.run.requested": "stream:scheduler:commands",
|
||||
"job.node.execute": "stream:jobs:execute",
|
||||
"job.node.finished": "stream:jobs:results",
|
||||
}
|
||||
|
||||
|
||||
def utcnow() -> datetime:
|
||||
return datetime.now(UTC).replace(tzinfo=None)
|
||||
|
||||
@@ -39,28 +32,14 @@ async def add_outbox_event(
|
||||
payload: dict[str, Any],
|
||||
available_at: datetime | None = None,
|
||||
) -> OutboxEvents:
|
||||
if event_type not in STREAM_BY_EVENT_TYPE:
|
||||
raise ValueError(f"unsupported event type: {event_type}")
|
||||
event_id = new_ulid()
|
||||
envelope = {
|
||||
"event_id": event_id,
|
||||
"event_type": event_type,
|
||||
"schema_version": 1,
|
||||
"occurred_at": event_time(),
|
||||
"producer": producer,
|
||||
"trace_id": trace_id,
|
||||
"aggregate_type": aggregate_type,
|
||||
"aggregate_id": aggregate_id,
|
||||
"idempotency_key": idempotency_key,
|
||||
"payload": payload,
|
||||
}
|
||||
item = OutboxEvents(
|
||||
event_id=event_id,
|
||||
aggregate_type=aggregate_type,
|
||||
aggregate_id=aggregate_id,
|
||||
event_type=event_type,
|
||||
schema_version=1,
|
||||
payload_json=envelope,
|
||||
payload_json=payload,
|
||||
event_status="pending",
|
||||
available_at=available_at or utcnow(),
|
||||
retry_count=0,
|
||||
@@ -69,3 +48,7 @@ async def add_outbox_event(
|
||||
)
|
||||
session.add(item)
|
||||
return item
|
||||
|
||||
|
||||
__all__ = ["add_outbox_event", "event_time", "utcnow"]
|
||||
|
||||
|
||||
@@ -0,0 +1,59 @@
|
||||
"""Shared APScheduler jobstore helpers.
|
||||
|
||||
The same ``apscheduler_jobs`` table is consumed by the FastAPI backend
|
||||
(to enqueue cron jobs through ``add_job``) and by the schedule executor
|
||||
(to run them). Both sides agree on the table name and the URL form
|
||||
that :class:`SQLAlchemyJobStore` expects, so this module is the single
|
||||
source of truth for that contract.
|
||||
|
||||
``apscheduler`` is declared as an optional peer dependency: only the
|
||||
schedule service imports it at runtime. This module therefore exposes
|
||||
the table name and URL helpers but defers actual jobstore construction
|
||||
to the caller.
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
from typing import TYPE_CHECKING
|
||||
|
||||
if TYPE_CHECKING: # pragma: no cover - typing only
|
||||
from apscheduler.jobstores.sqlalchemy import SQLAlchemyJobStore
|
||||
|
||||
|
||||
JOBSTORE_TABLE = "apscheduler_jobs"
|
||||
|
||||
|
||||
def to_sync_database_url(database_url: str) -> str:
|
||||
"""Rewrite ``mysql+asyncmy://`` to ``mysql+pymysql://``.
|
||||
|
||||
APScheduler's ``SQLAlchemyJobStore`` uses a synchronous engine; the
|
||||
project's default async URL needs to be downgraded before it can
|
||||
drive the jobstore.
|
||||
"""
|
||||
return database_url.replace("mysql+asyncmy://", "mysql+pymysql://", 1)
|
||||
|
||||
|
||||
def build_sqlalchemy_jobstore(
|
||||
database_url: str,
|
||||
*,
|
||||
tablename: str = JOBSTORE_TABLE,
|
||||
) -> "SQLAlchemyJobStore":
|
||||
"""Instantiate a :class:`SQLAlchemyJobStore` for the canonical table.
|
||||
|
||||
The caller is responsible for ensuring APScheduler and its sync
|
||||
driver (``pymysql``) are installed; ``common`` does not depend on
|
||||
either.
|
||||
"""
|
||||
from apscheduler.jobstores.sqlalchemy import SQLAlchemyJobStore
|
||||
|
||||
return SQLAlchemyJobStore(
|
||||
url=to_sync_database_url(database_url),
|
||||
tablename=tablename,
|
||||
)
|
||||
|
||||
|
||||
__all__ = [
|
||||
"JOBSTORE_TABLE",
|
||||
"build_sqlalchemy_jobstore",
|
||||
"to_sync_database_url",
|
||||
]
|
||||
@@ -0,0 +1,14 @@
|
||||
"""Shared Pydantic base models for service-level request validation."""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
from pydantic import BaseModel, ConfigDict
|
||||
|
||||
|
||||
class StrictModel(BaseModel):
|
||||
"""Base model that rejects unknown fields."""
|
||||
|
||||
model_config = ConfigDict(extra="forbid")
|
||||
|
||||
|
||||
__all__ = ["StrictModel"]
|
||||
@@ -2,6 +2,7 @@ from __future__ import annotations
|
||||
|
||||
import hashlib
|
||||
from typing import Any, BinaryIO
|
||||
from urllib.parse import urlsplit, urlunsplit
|
||||
|
||||
import boto3
|
||||
from botocore.client import Config
|
||||
@@ -13,7 +14,6 @@ class RustFSObjectStore:
|
||||
self,
|
||||
*,
|
||||
internal_endpoint: str,
|
||||
public_endpoint: str,
|
||||
access_key: str,
|
||||
secret_key: str,
|
||||
) -> None:
|
||||
@@ -31,11 +31,7 @@ class RustFSObjectStore:
|
||||
endpoint_url=internal_endpoint.rstrip("/"),
|
||||
**common,
|
||||
)
|
||||
self.public = boto3.client(
|
||||
"s3",
|
||||
endpoint_url=public_endpoint.rstrip("/"),
|
||||
**common,
|
||||
)
|
||||
self._internal_endpoint = internal_endpoint.rstrip("/")
|
||||
|
||||
def ensure_bucket(self, bucket_name: str) -> None:
|
||||
try:
|
||||
@@ -54,7 +50,6 @@ class RustFSObjectStore:
|
||||
content_type: str,
|
||||
expected_hash: str | None,
|
||||
expires_seconds: int,
|
||||
public: bool,
|
||||
) -> tuple[str, dict[str, str]]:
|
||||
params: dict[str, Any] = {
|
||||
"Bucket": bucket_name,
|
||||
@@ -65,8 +60,7 @@ class RustFSObjectStore:
|
||||
if expected_hash:
|
||||
params["Metadata"] = {"sha256": expected_hash}
|
||||
headers["x-amz-meta-sha256"] = expected_hash
|
||||
client = self.public if public else self.internal
|
||||
url = client.generate_presigned_url(
|
||||
url = self.internal.generate_presigned_url(
|
||||
"put_object",
|
||||
Params=params,
|
||||
ExpiresIn=expires_seconds,
|
||||
@@ -81,7 +75,7 @@ class RustFSObjectStore:
|
||||
file_name: str,
|
||||
expires_seconds: int,
|
||||
) -> str:
|
||||
return self.public.generate_presigned_url(
|
||||
return self.internal.generate_presigned_url(
|
||||
"get_object",
|
||||
Params={
|
||||
"Bucket": bucket_name,
|
||||
@@ -93,6 +87,36 @@ class RustFSObjectStore:
|
||||
ExpiresIn=expires_seconds,
|
||||
)
|
||||
|
||||
def rewrite_to_public_path(
|
||||
self,
|
||||
url: str,
|
||||
*,
|
||||
public_base_url: str,
|
||||
) -> str:
|
||||
"""Rewrite the host of a presigned URL to the public edge.
|
||||
|
||||
Replaces the scheme + host (and strips any trailing slash) with
|
||||
``public_base_url``; the bucket prefix is moved under the
|
||||
``/storage/`` path so Nginx can forward the call to RustFS
|
||||
without exposing its port.
|
||||
"""
|
||||
parsed = urlsplit(url)
|
||||
public = urlsplit(public_base_url.rstrip("/"))
|
||||
netloc = public.netloc
|
||||
prefix = public.path.rstrip("/")
|
||||
# Presigned URLs are generated against the internal S3 endpoint
|
||||
# and always start with ``/{bucket}/...``; relocate the bucket
|
||||
# segment under ``/storage/...`` so the public edge can route
|
||||
# the request to the right bucket.
|
||||
rewritten_path = f"{prefix}/storage{parsed.path}"
|
||||
return urlunsplit((
|
||||
public.scheme,
|
||||
netloc,
|
||||
rewritten_path,
|
||||
parsed.query,
|
||||
parsed.fragment,
|
||||
))
|
||||
|
||||
def put_bytes(
|
||||
self,
|
||||
*,
|
||||
|
||||
Reference in New Issue
Block a user