fix: artifact_bucket
This commit is contained in:
@@ -186,16 +186,25 @@ class SchedulerService:
|
||||
return await self.orchestrator.dispatch_run(run_id)
|
||||
|
||||
|
||||
def build_object_store() -> Any:
|
||||
def build_object_store(bucket_name: str | None = None) -> Any:
|
||||
"""Construct the AsyncStorageBackend the worker reads version artifacts from.
|
||||
|
||||
Respects ``settings.storage_backend``: in ``local`` mode points at the
|
||||
local ``${local_storage_base_dir}/version`` directory (the same place
|
||||
``backend.storage_api`` writes to), in ``s3`` mode points at
|
||||
``settings.s3_version_bucket``. The worker only reads version artifacts
|
||||
via this store (run logs / results still go through the backend's
|
||||
HTTP storage API), so pointing at the version bucket/directory is the
|
||||
correct resolution regardless of the artifact's workspace.
|
||||
``backend.storage_api`` writes to), in ``s3`` mode points at the
|
||||
bucket named by ``bucket_name`` (default ``settings.s3_version_bucket``).
|
||||
|
||||
The bucket override exists because some workspaces configure a custom
|
||||
``Workspaces.artifact_bucket``; their version artifacts are uploaded
|
||||
into that bucket by the backend, so the worker must read from the
|
||||
same place — not the global version bucket. Run logs / results still
|
||||
go through the backend's HTTP storage API (which honors the override
|
||||
on its own), so this factory only matters for the artifact-download
|
||||
path.
|
||||
|
||||
For local mode the override is ignored: custom bucket subdirectories
|
||||
only make sense in S3 mode, and the version base_dir is the right
|
||||
destination either way.
|
||||
"""
|
||||
if settings.storage_backend == "local":
|
||||
return create_storage(
|
||||
@@ -211,7 +220,7 @@ def build_object_store() -> Any:
|
||||
{
|
||||
"type": "s3",
|
||||
"mode": "async",
|
||||
"bucket": settings.s3_version_bucket,
|
||||
"bucket": bucket_name or settings.s3_version_bucket,
|
||||
"endpoint_url": settings.s3_endpoint,
|
||||
"aws_access_key_id": settings.s3_access_key,
|
||||
"aws_secret_access_key": settings.s3_secret_key,
|
||||
|
||||
@@ -58,6 +58,30 @@ class NodeExecutor:
|
||||
self.session_factory = session_factory
|
||||
self.object_store = object_store
|
||||
self.storage_client = storage_client
|
||||
# Per-bucket object store cache. The injected ``object_store`` is
|
||||
# the default version-bucket store; workspaces that override
|
||||
# ``Workspaces.artifact_bucket`` need a store bound to that custom
|
||||
# bucket (P0-3 fix). Build lazily so the common (no-override) path
|
||||
# incurs no extra cost.
|
||||
self._bucket_stores: dict[str, Any] = {
|
||||
settings.s3_version_bucket: object_store,
|
||||
}
|
||||
|
||||
def _store_for(self, bucket_name: str) -> Any:
|
||||
"""Return the AsyncStorageBackend bound to ``bucket_name``.
|
||||
|
||||
Caches per-bucket stores on first use; the default version bucket
|
||||
always reuses the injected ``object_store`` so the common path
|
||||
stays zero-allocation.
|
||||
"""
|
||||
store = self._bucket_stores.get(bucket_name)
|
||||
if store is not None:
|
||||
return store
|
||||
from schedule.service import build_object_store
|
||||
|
||||
store = build_object_store(bucket_name=bucket_name)
|
||||
self._bucket_stores[bucket_name] = store
|
||||
return store
|
||||
|
||||
async def handle_node_execute(
|
||||
self,
|
||||
@@ -380,7 +404,11 @@ class NodeExecutor:
|
||||
object_key: str,
|
||||
content_hash: str,
|
||||
) -> bytes:
|
||||
content = await self.object_store.get(object_key)
|
||||
# Honor the artifact's actual bucket (P0-3 fix): the artifact may
|
||||
# live in ``Workspaces.artifact_bucket`` rather than the global
|
||||
# version bucket the default ``object_store`` is bound to.
|
||||
store = self._store_for(bucket_name)
|
||||
content = await store.get(object_key)
|
||||
if hashlib.sha256(content).hexdigest() != content_hash:
|
||||
raise ValueError("stable version artifact hash mismatch")
|
||||
logger.debug(
|
||||
|
||||
Reference in New Issue
Block a user