fix: artifact_bucket

This commit is contained in:
tao.chen
2026-08-20 10:26:44 +08:00
parent d8a31bde95
commit e5633cc95a
6 changed files with 209 additions and 9 deletions
+16 -7
View File
@@ -186,16 +186,25 @@ class SchedulerService:
return await self.orchestrator.dispatch_run(run_id)
def build_object_store() -> Any:
def build_object_store(bucket_name: str | None = None) -> Any:
"""Construct the AsyncStorageBackend the worker reads version artifacts from.
Respects ``settings.storage_backend``: in ``local`` mode points at the
local ``${local_storage_base_dir}/version`` directory (the same place
``backend.storage_api`` writes to), in ``s3`` mode points at
``settings.s3_version_bucket``. The worker only reads version artifacts
via this store (run logs / results still go through the backend's
HTTP storage API), so pointing at the version bucket/directory is the
correct resolution regardless of the artifact's workspace.
``backend.storage_api`` writes to), in ``s3`` mode points at the
bucket named by ``bucket_name`` (default ``settings.s3_version_bucket``).
The bucket override exists because some workspaces configure a custom
``Workspaces.artifact_bucket``; their version artifacts are uploaded
into that bucket by the backend, so the worker must read from the
same place — not the global version bucket. Run logs / results still
go through the backend's HTTP storage API (which honors the override
on its own), so this factory only matters for the artifact-download
path.
For local mode the override is ignored: custom bucket subdirectories
only make sense in S3 mode, and the version base_dir is the right
destination either way.
"""
if settings.storage_backend == "local":
return create_storage(
@@ -211,7 +220,7 @@ def build_object_store() -> Any:
{
"type": "s3",
"mode": "async",
"bucket": settings.s3_version_bucket,
"bucket": bucket_name or settings.s3_version_bucket,
"endpoint_url": settings.s3_endpoint,
"aws_access_key_id": settings.s3_access_key,
"aws_secret_access_key": settings.s3_secret_key,
+29 -1
View File
@@ -58,6 +58,30 @@ class NodeExecutor:
self.session_factory = session_factory
self.object_store = object_store
self.storage_client = storage_client
# Per-bucket object store cache. The injected ``object_store`` is
# the default version-bucket store; workspaces that override
# ``Workspaces.artifact_bucket`` need a store bound to that custom
# bucket (P0-3 fix). Build lazily so the common (no-override) path
# incurs no extra cost.
self._bucket_stores: dict[str, Any] = {
settings.s3_version_bucket: object_store,
}
def _store_for(self, bucket_name: str) -> Any:
"""Return the AsyncStorageBackend bound to ``bucket_name``.
Caches per-bucket stores on first use; the default version bucket
always reuses the injected ``object_store`` so the common path
stays zero-allocation.
"""
store = self._bucket_stores.get(bucket_name)
if store is not None:
return store
from schedule.service import build_object_store
store = build_object_store(bucket_name=bucket_name)
self._bucket_stores[bucket_name] = store
return store
async def handle_node_execute(
self,
@@ -380,7 +404,11 @@ class NodeExecutor:
object_key: str,
content_hash: str,
) -> bytes:
content = await self.object_store.get(object_key)
# Honor the artifact's actual bucket (P0-3 fix): the artifact may
# live in ``Workspaces.artifact_bucket`` rather than the global
# version bucket the default ``object_store`` is bound to.
store = self._store_for(bucket_name)
content = await store.get(object_key)
if hashlib.sha256(content).hexdigest() != content_hash:
raise ValueError("stable version artifact hash mismatch")
logger.debug(