feat(pending): app_name field and date-sharded pending storage

- Add optional app_name to PendingSubmission and prepare_submit_job,
  passed through PrepareSubmitJobRequest for user-provided tracking.
- Rewrite PendingStore to shard records by UTC date under
  data/pending_jobs/YYYY-MM-DD.json instead of a single monolithic
  pending_jobs.json.
- Legacy single-file pending_jobs.json remains readable; first save()
  migrates it and renames the old file to avoid double reads.
- Tests cover date sharding, multi-date list, legacy read, legacy
  migration, and app_name round-trip.
This commit is contained in:
Claude
2026-06-26 14:36:24 +08:00
parent c44e9bcc55
commit da9ba657f3
7 changed files with 163 additions and 35 deletions
+18
View File
@@ -74,6 +74,24 @@ def test_prepare_persists_pending_with_snapshot(monkeypatch, real_script):
assert p.script_path == str(real_script)
def test_prepare_persists_app_name(monkeypatch, real_script):
monkeypatch.setattr(submit, "run_spark_submit", lambda cmd: None)
submit.prepare_submit_job(
connection="prod", script_path=str(real_script), app_name="my-etl-job"
)
p = submit.pending_store.get(_last_pending_id())
assert p is not None
assert p.app_name == "my-etl-job"
def test_prepare_app_name_defaults_to_none(monkeypatch, real_script):
monkeypatch.setattr(submit, "run_spark_submit", lambda cmd: None)
submit.prepare_submit_job(connection="prod", script_path=str(real_script))
p = submit.pending_store.get(_last_pending_id())
assert p is not None
assert p.app_name is None
def test_prepare_raises_for_unknown_connection(real_script):
with pytest.raises(KeyError, match="missing"):
submit.prepare_submit_job(connection="missing", script_path=str(real_script))