feat(pending): app_name field and date-sharded pending storage
- Add optional app_name to PendingSubmission and prepare_submit_job, passed through PrepareSubmitJobRequest for user-provided tracking. - Rewrite PendingStore to shard records by UTC date under data/pending_jobs/YYYY-MM-DD.json instead of a single monolithic pending_jobs.json. - Legacy single-file pending_jobs.json remains readable; first save() migrates it and renames the old file to avoid double reads. - Tests cover date sharding, multi-date list, legacy read, legacy migration, and app_name round-trip.
This commit is contained in:
@@ -74,6 +74,24 @@ def test_prepare_persists_pending_with_snapshot(monkeypatch, real_script):
|
||||
assert p.script_path == str(real_script)
|
||||
|
||||
|
||||
def test_prepare_persists_app_name(monkeypatch, real_script):
|
||||
monkeypatch.setattr(submit, "run_spark_submit", lambda cmd: None)
|
||||
submit.prepare_submit_job(
|
||||
connection="prod", script_path=str(real_script), app_name="my-etl-job"
|
||||
)
|
||||
p = submit.pending_store.get(_last_pending_id())
|
||||
assert p is not None
|
||||
assert p.app_name == "my-etl-job"
|
||||
|
||||
|
||||
def test_prepare_app_name_defaults_to_none(monkeypatch, real_script):
|
||||
monkeypatch.setattr(submit, "run_spark_submit", lambda cmd: None)
|
||||
submit.prepare_submit_job(connection="prod", script_path=str(real_script))
|
||||
p = submit.pending_store.get(_last_pending_id())
|
||||
assert p is not None
|
||||
assert p.app_name is None
|
||||
|
||||
|
||||
def test_prepare_raises_for_unknown_connection(real_script):
|
||||
with pytest.raises(KeyError, match="missing"):
|
||||
submit.prepare_submit_job(connection="missing", script_path=str(real_script))
|
||||
|
||||
Reference in New Issue
Block a user