feat(submit): configurable confirm retries and idempotent confirm

Harden confirm_submit_job for unreliable test environments and transient spark-submit failures: - Add confirm_max_retries and confirm_retry_delay_seconds settings (env-configurable). - SUBMITTED pending returns cached result without re-running spark-submit. - FAILED pending is reset to PENDING and retried. - CANCELLED pending is still rejected. - On success, persist pending as SUBMITTED before creating the in-memory Job so a job_store failure cannot leave the record as PENDING while the YARN app is already running. - Persist tracking_url on PendingSubmission for idempotent returns. Tests cover retry-then-success, max-retries-exceeded, idempotency, FAILED reset, and pending-saved-before-job-store.
2026-06-26 15:51:51 +08:00
parent 9e7b425f09
commit a613c7bdb8
5 changed files with 240 additions and 29 deletions
@@ -1,4 +1,5 @@
 # coding=utf-8
+import json
 from datetime import datetime
 from pathlib import Path

@@ -1,13 +1,20 @@
 # coding=utf-8
 from pathlib import Path
 from unittest.mock import patch
+import time

 import pytest

+from common.config import settings
 from spark_executor.core import connection_store, pending_store
 from spark_executor.core.pending_store import PendingStore
 from spark_executor.models import Connection
 from spark_executor.tools import submit
+from spark_executor.core.spark_submit import SparkSubmitError
+
+
+def _spark_proc(stderr: str = "tracking URL: http://rm:8088/proxy/application_17400000001/\n"):
+    return type("P", (), {"returncode": 0, "stderr": stderr})()


@pytest.fixture(autouse=True)
@@ -331,7 +338,6 @@ def test_confirm_refuses_non_pending_status(monkeypatch, real_script):


 def test_confirm_marks_failed_on_spark_submit_error(monkeypatch, real_script):
-    from spark_executor.core.spark_submit import SparkSubmitError
    submit.prepare_submit_job(
        connection="prod",
        script_path=str(real_script),
@@ -345,6 +351,7 @@ def test_confirm_marks_failed_on_spark_submit_error(monkeypatch, real_script):
    def _raise(_cmd):
        raise SparkSubmitError("boom")
    monkeypatch.setattr(submit, "run_spark_submit", _raise)
+    settings.confirm_max_retries = 0
    with pytest.raises(SparkSubmitError):
        submit.confirm_submit_job(pending_id=pid)
    p = submit.pending_store.get(pid)
@@ -352,6 +359,142 @@ def test_confirm_marks_failed_on_spark_submit_error(monkeypatch, real_script):
    assert "boom" in (p.error or "")


+# --- confirm retry / idempotency ---
+
+def test_confirm_retries_spark_submit_failure_then_succeeds(monkeypatch, real_script):
+    monkeypatch.setattr(time, "sleep", lambda _s: None)
+    settings.confirm_max_retries = 2
+    submit.prepare_submit_job(
+        connection="prod",
+        script_path=str(real_script),
+        queue="default",
+        executor_memory="4G",
+        executor_cores=2,
+        num_executors=2,
+        app_name="test-app",
+    )
+    pid = _last_pending_id()
+    calls = []
+    def _maybe_fail(cmd):
+        calls.append(cmd)
+        if len(calls) < 3:
+            raise SparkSubmitError("transient")
+        return _spark_proc()
+    monkeypatch.setattr(submit, "run_spark_submit", _maybe_fail)
+    result = submit.confirm_submit_job(pending_id=pid)
+    assert result.application_id == "application_17400000001"
+    assert len(calls) == 3
+    p = submit.pending_store.get(pid)
+    assert p.status == "SUBMITTED"
+
+
+def test_confirm_exhausts_retries_and_sets_failed(monkeypatch, real_script):
+    monkeypatch.setattr(time, "sleep", lambda _s: None)
+    settings.confirm_max_retries = 2
+    submit.prepare_submit_job(
+        connection="prod",
+        script_path=str(real_script),
+        queue="default",
+        executor_memory="4G",
+        executor_cores=2,
+        num_executors=2,
+        app_name="test-app",
+    )
+    pid = _last_pending_id()
+    calls = []
+    def _always_fail(cmd):
+        calls.append(cmd)
+        raise SparkSubmitError("persistent")
+    monkeypatch.setattr(submit, "run_spark_submit", _always_fail)
+    with pytest.raises(SparkSubmitError, match="persistent"):
+        submit.confirm_submit_job(pending_id=pid)
+    assert len(calls) == 3
+    p = submit.pending_store.get(pid)
+    assert p.status == "FAILED"
+    assert "persistent" in (p.error or "")
+
+
+def test_confirm_idempotent_when_submitted(monkeypatch, real_script):
+    submit.prepare_submit_job(
+        connection="prod",
+        script_path=str(real_script),
+        queue="default",
+        executor_memory="4G",
+        executor_cores=2,
+        num_executors=2,
+        app_name="test-app",
+    )
+    pid = _last_pending_id()
+    monkeypatch.setattr(submit, "run_spark_submit", lambda _cmd: _spark_proc())
+    first = submit.confirm_submit_job(pending_id=pid)
+
+    call_count = [0]
+    def _counting_run(cmd):
+        call_count[0] += 1
+        return _spark_proc()
+    monkeypatch.setattr(submit, "run_spark_submit", _counting_run)
+    second = submit.confirm_submit_job(pending_id=pid)
+    assert call_count[0] == 0
+    assert second.job_id == first.job_id
+    assert second.application_id == first.application_id
+    assert second.tracking_url == first.tracking_url
+
+
+def test_confirm_resets_failed_and_retries(monkeypatch, real_script):
+    monkeypatch.setattr(time, "sleep", lambda _s: None)
+    submit.prepare_submit_job(
+        connection="prod",
+        script_path=str(real_script),
+        queue="default",
+        executor_memory="4G",
+        executor_cores=2,
+        num_executors=2,
+        app_name="test-app",
+    )
+    pid = _last_pending_id()
+    monkeypatch.setattr(
+        submit,
+        "run_spark_submit",
+        lambda _cmd: (_ for _ in ()).throw(SparkSubmitError("boom")),
+    )
+    with pytest.raises(SparkSubmitError):
+        submit.confirm_submit_job(pending_id=pid)
+    p = submit.pending_store.get(pid)
+    assert p.status == "FAILED"
+    assert "boom" in (p.error or "")
+
+    monkeypatch.setattr(submit, "run_spark_submit", lambda _cmd: _spark_proc())
+    result = submit.confirm_submit_job(pending_id=pid)
+    assert result.application_id == "application_17400000001"
+    p = submit.pending_store.get(pid)
+    assert p.status == "SUBMITTED"
+    assert p.error is None
+
+
+def test_confirm_updates_pending_before_job_store(monkeypatch, real_script):
+    submit.prepare_submit_job(
+        connection="prod",
+        script_path=str(real_script),
+        queue="default",
+        executor_memory="4G",
+        executor_cores=2,
+        num_executors=2,
+        app_name="test-app",
+    )
+    pid = _last_pending_id()
+    monkeypatch.setattr(submit, "run_spark_submit", lambda _cmd: _spark_proc())
+
+    def _exploding_put(job):
+        raise RuntimeError("job_store unavailable")
+    monkeypatch.setattr(submit.job_store, "put", _exploding_put)
+    with pytest.raises(RuntimeError, match="job_store unavailable"):
+        submit.confirm_submit_job(pending_id=pid)
+    p = submit.pending_store.get(pid)
+    assert p.status == "SUBMITTED"
+    assert p.application_id == "application_17400000001"
+    assert p.tracking_url is not None
+
+
 # --- list_pending_jobs ---

 def test_list_pending_jobs_empty():