Claude
a613c7bdb8
Harden confirm_submit_job for unreliable test environments and transient spark-submit failures: - Add confirm_max_retries and confirm_retry_delay_seconds settings (env-configurable). - SUBMITTED pending returns cached result without re-running spark-submit. - FAILED pending is reset to PENDING and retried. - CANCELLED pending is still rejected. - On success, persist pending as SUBMITTED before creating the in-memory Job so a job_store failure cannot leave the record as PENDING while the YARN app is already running. - Persist tracking_url on PendingSubmission for idempotent returns. Tests cover retry-then-success, max-retries-exceeded, idempotency, FAILED reset, and pending-saved-before-job-store.
129 lines
4.2 KiB
Python
129 lines
4.2 KiB
Python
# coding=utf-8
|
|
import json
|
|
from datetime import datetime
|
|
from pathlib import Path
|
|
|
|
import pytest
|
|
|
|
from spark_executor.core import pending_store
|
|
from spark_executor.models import PendingSubmission
|
|
|
|
|
|
def _pending(pid: str = "p_1", created_at: datetime | None = None) -> PendingSubmission:
|
|
return PendingSubmission(
|
|
pending_id=pid,
|
|
app_name="test",
|
|
connection="prod",
|
|
master="yarn",
|
|
deploy_mode="cluster",
|
|
script_path="/tmp/j.py",
|
|
queue="default",
|
|
executor_memory="4G",
|
|
executor_cores=2,
|
|
num_executors=2,
|
|
spark_conf={},
|
|
extra_args={},
|
|
created_at=created_at or datetime(2026, 6, 24),
|
|
)
|
|
|
|
|
|
@pytest.fixture(autouse=True)
|
|
def _fresh(tmp_path: Path, monkeypatch):
|
|
monkeypatch.setattr(pending_store, "DEFAULT_DATA_DIR", str(tmp_path))
|
|
monkeypatch.setattr(pending_store, "store", pending_store.PendingStore())
|
|
|
|
|
|
def test_save_then_get_roundtrip():
|
|
pending_store.store.save(_pending())
|
|
out = pending_store.store.get("p_1")
|
|
assert out is not None
|
|
assert out.connection == "prod"
|
|
assert out.master == "yarn"
|
|
|
|
|
|
def test_save_persists_to_date_file(tmp_path: Path):
|
|
pending_store.store.save(_pending())
|
|
path = tmp_path / "pending_jobs" / "2026-06-24.json"
|
|
assert path.is_file()
|
|
|
|
|
|
def test_get_missing_returns_none():
|
|
assert pending_store.store.get("missing") is None
|
|
|
|
|
|
def test_list_all_empty_when_file_absent():
|
|
assert pending_store.store.list_all() == []
|
|
|
|
|
|
def test_list_all_returns_saved():
|
|
pending_store.store.save(_pending("a"))
|
|
pending_store.store.save(_pending("b"))
|
|
assert {p.pending_id for p in pending_store.store.list_all()} == {"a", "b"}
|
|
|
|
|
|
def test_list_all_spans_multiple_date_files():
|
|
pending_store.store.save(_pending("a", created_at=datetime(2026, 6, 24)))
|
|
pending_store.store.save(_pending("b", created_at=datetime(2026, 6, 25)))
|
|
assert {p.pending_id for p in pending_store.store.list_all()} == {"a", "b"}
|
|
|
|
|
|
def test_get_searches_date_files():
|
|
pending_store.store.save(_pending("a", created_at=datetime(2026, 6, 24)))
|
|
assert pending_store.store.get("a") is not None
|
|
|
|
|
|
def test_delete_removes_from_date_file_and_deletes_empty_file():
|
|
pending_store.store.save(_pending("a", created_at=datetime(2026, 6, 24)))
|
|
date_file = pending_store.store.dir_path / "2026-06-24.json"
|
|
assert date_file.is_file()
|
|
assert pending_store.store.delete("a") is True
|
|
assert pending_store.store.get("a") is None
|
|
assert not date_file.exists()
|
|
|
|
|
|
def _legacy_record_text() -> str:
|
|
return (
|
|
'{"p_legacy": {"pending_id": "p_legacy", "app_name": "legacy", '
|
|
'"connection": "prod", "master": "yarn", "deploy_mode": "cluster", '
|
|
'"script_path": "/tmp/j.py", "queue": "default", "executor_memory": "4G", '
|
|
'"executor_cores": 2, "num_executors": 2, "spark_conf": {}, '
|
|
'"created_at": "2026-06-23T00:00:00"}}'
|
|
)
|
|
|
|
|
|
def test_legacy_file_is_readable(tmp_path: Path):
|
|
legacy = tmp_path / "pending_jobs.json"
|
|
legacy.write_text(_legacy_record_text(), encoding="utf-8")
|
|
assert pending_store.store.get("p_legacy") is not None
|
|
assert any(p.pending_id == "p_legacy" for p in pending_store.store.list_all())
|
|
|
|
|
|
def test_save_migrates_legacy_file(tmp_path: Path):
|
|
legacy = tmp_path / "pending_jobs.json"
|
|
legacy.write_text(_legacy_record_text(), encoding="utf-8")
|
|
pending_store.store.save(_pending("p_new", created_at=datetime(2026, 6, 24)))
|
|
assert not legacy.exists()
|
|
assert pending_store.store.get("p_legacy") is not None
|
|
migrated = pending_store.store.dir_path / "2026-06-23.json"
|
|
assert migrated.is_file()
|
|
|
|
|
|
def test_save_upserts_existing():
|
|
pending_store.store.save(_pending("a"))
|
|
updated = _pending("a")
|
|
updated.status = "SUBMITTED"
|
|
updated.job_id = "j_1"
|
|
pending_store.store.save(updated)
|
|
assert pending_store.store.get("a").status == "SUBMITTED"
|
|
assert pending_store.store.get("a").job_id == "j_1"
|
|
|
|
|
|
def test_delete_returns_true_when_present():
|
|
pending_store.store.save(_pending())
|
|
assert pending_store.store.delete("p_1") is True
|
|
assert pending_store.store.get("p_1") is None
|
|
|
|
|
|
def test_delete_returns_false_when_absent():
|
|
assert pending_store.store.delete("nope") is False
|