feat(submit): configurable confirm retries and idempotent confirm

Harden confirm_submit_job for unreliable test environments and transient
spark-submit failures:

- Add confirm_max_retries and confirm_retry_delay_seconds settings
  (env-configurable).
- SUBMITTED pending returns cached result without re-running spark-submit.
- FAILED pending is reset to PENDING and retried.
- CANCELLED pending is still rejected.
- On success, persist pending as SUBMITTED before creating the in-memory Job
  so a job_store failure cannot leave the record as PENDING while the YARN
  app is already running.
- Persist tracking_url on PendingSubmission for idempotent returns.

Tests cover retry-then-success, max-retries-exceeded, idempotency,
FAILED reset, and pending-saved-before-job-store.

spark_executor/tools/submit.py

@@ -5,9 +5,11 @@
 """
 import os
 import secrets
+import time
 import uuid
 from datetime import datetime
 
+from common.config import settings
 from common.logging import logger
 from common.sql_guard import validate_pyspark_code
 from spark_executor.core.connection_store import store as conn_store
@@ -148,10 +150,29 @@ def confirm_submit_job(*, pending_id: str) -> SubmitResult:
     pending = pending_store.get(pending_id)
     if pending is None:
         raise KeyError(f"Unknown pending_id: {pending_id}")
+
+    if pending.status == "SUBMITTED":
+        logger.info(
+            f"confirm_submit_job idempotent pending_id={pending_id} "
+            f"job_id={pending.job_id} application_id={pending.application_id}"
+        )
+        return SubmitResult(
+            job_id=pending.job_id,
+            application_id=pending.application_id,
+            tracking_url=pending.tracking_url,
+        )
+    if pending.status == "CANCELLED":
+        raise ValueError(f"pending_id {pending_id} is CANCELLED, cannot confirm")
+    if pending.status == "FAILED":
+        pending.status = "PENDING"
+        pending.error = None
+        pending_store.save(pending)
+        logger.info(f"confirm_submit_job reset pending_id={pending_id} from FAILED to PENDING")
     if pending.status != "PENDING":
         raise ValueError(
             f"pending_id {pending_id} is in status {pending.status!r}, not PENDING"
         )
+
     # Defense in depth: re-verify the script still exists. A user could
     # delete the file between prepare and confirm (or an external cleanup
     # job could remove it). 400 via the ValueError -> 400 handler.
@@ -172,43 +193,72 @@ def confirm_submit_job(*, pending_id: str) -> SubmitResult:
         f"confirm_submit_job start pending_id={pending_id} "
         f"application_target={pending.master} script_path={pending.script_path}"
     )
-    try:
-        result = run_spark_submit(cmd)
-    except SparkSubmitError as exc:
-        pending.status = "FAILED"
-        pending.error = str(exc)
+
+    last_error: SparkSubmitError | None = None
+    max_attempts = settings.confirm_max_retries + 1
+    for attempt in range(1, max_attempts + 1):
+        try:
+            result = run_spark_submit(cmd)
+        except SparkSubmitError as exc:
+            last_error = exc
+            logger.warning(
+                f"confirm_submit_job attempt {attempt}/{max_attempts} failed "
+                f"pending_id={pending_id} err={exc}"
+            )
+            if attempt < max_attempts:
+                time.sleep(settings.confirm_retry_delay_seconds)
+            continue
+        except Exception:
+            # Unexpected failure (not a spark-submit error): fail fast without retry.
+            logger.exception(
+                f"confirm_submit_job unexpected error pending_id={pending_id}"
+            )
+            raise
+
+        application_id, tracking_url = parse_spark_submit_output(result.stderr)
+        job_id = uuid.uuid4().hex[:12]
+
+        # Persist pending as SUBMITTED before creating the in-memory Job so a
+        # job_store failure cannot leave the record as PENDING while the YARN
+        # app is already running.
+        pending.status = "SUBMITTED"
+        pending.job_id = job_id
+        pending.application_id = application_id
+        pending.tracking_url = tracking_url
         pending_store.save(pending)
-        logger.error(f"confirm_submit_job failed pending_id={pending_id} err={exc}")
-        raise
 
-    application_id, tracking_url = parse_spark_submit_output(result.stderr)
-    job_id = uuid.uuid4().hex[:12]
+        job_store.put(
+            Job(
+                job_id=job_id,
+                application_id=application_id,
+                script_path=pending.script_path,
+                queue=pending.queue,
+                submit_time=datetime.utcnow(),
+                connection=pending.connection,
+                yarn_rm_url=pending.yarn_rm_url,
+            )
+        )
 
-    job_store.put(
-        Job(
+        logger.info(
+            f"confirm_submit_job ok pending_id={pending_id} job_id={job_id} "
+            f"application_id={application_id}"
+        )
+        return SubmitResult(
             job_id=job_id,
             application_id=application_id,
-            script_path=pending.script_path,
-            queue=pending.queue,
-            submit_time=datetime.utcnow(),
-            connection=pending.connection,
-            yarn_rm_url=pending.yarn_rm_url,
+            tracking_url=tracking_url,
         )
-    )
 
-    pending.status = "SUBMITTED"
-    pending.job_id = job_id
-    pending.application_id = application_id
+    # Exhausted all retries.
+    assert last_error is not None
+    pending.status = "FAILED"
+    pending.error = str(last_error)
     pending_store.save(pending)
-    logger.info(
-        f"confirm_submit_job ok pending_id={pending_id} job_id={job_id} "
-        f"application_id={application_id}"
-    )
-    return SubmitResult(
-        job_id=job_id,
-        application_id=application_id,
-        tracking_url=tracking_url,
+    logger.error(
+        f"confirm_submit_job failed pending_id={pending_id} after {max_attempts} attempts "
+        f"err={last_error}"
     )
+    raise last_error
 
 
 def list_pending_jobs() -> list[dict[str, object]]: