feat(pending): app_name field and date-sharded pending storage
- Add optional app_name to PendingSubmission and prepare_submit_job, passed through PrepareSubmitJobRequest for user-provided tracking. - Rewrite PendingStore to shard records by UTC date under data/pending_jobs/YYYY-MM-DD.json instead of a single monolithic pending_jobs.json. - Legacy single-file pending_jobs.json remains readable; first save() migrates it and renames the old file to avoid double reads. - Tests cover date sharding, multi-date list, legacy read, legacy migration, and app_name round-trip.
This commit is contained in:
@@ -33,6 +33,10 @@ class SaveConnectionRequest(BaseModel):
|
||||
|
||||
class PrepareSubmitJobRequest(BaseModel):
|
||||
connection: str
|
||||
app_name: str | None = Field(
|
||||
default=None,
|
||||
description="Optional human-readable application name for tracking the pending submission.",
|
||||
)
|
||||
script_path: str = Field(
|
||||
...,
|
||||
description=(
|
||||
|
||||
@@ -68,6 +68,7 @@ def prepare_submit_job(
|
||||
executor_memory: str = "4G",
|
||||
executor_cores: int = 2,
|
||||
num_executors: int = 2,
|
||||
app_name: str | None = None,
|
||||
) -> dict[str, object]:
|
||||
"""Snapshot connection params and persist a PendingSubmission. Does NOT submit."""
|
||||
logger.debug(
|
||||
@@ -109,6 +110,7 @@ def prepare_submit_job(
|
||||
pending_id = _new_pending_id()
|
||||
pending = PendingSubmission(
|
||||
pending_id=pending_id,
|
||||
app_name=app_name,
|
||||
connection=connection,
|
||||
master=conn.master,
|
||||
deploy_mode=conn.deploy_mode,
|
||||
|
||||
Reference in New Issue
Block a user