refactor(submit): drop spark-submit retry loop; confirm raises on failure

confirm_submit_job no longer retries on SparkSubmitError. A failed
single attempt is recorded as-is:

  - pending.status = 'FAILED' with the error message in pending.error
  - the function re-raises the SparkSubmitError so the caller (and the
    agent) sees the failure immediately
  - the agent can call get_pending_job to see the persisted FAILED
    state, including the captured error

The 'retry' code path was hiding real failures behind transient
recovery: a spark-submit that died because the script is broken looks
the same as one that died because YARN was momentarily unreachable.
Without retries, the agent gets a clean FAILED state and a clear
exception to act on, instead of a delayed, ambiguous result.

Manual recovery is still possible: re-calling confirm_submit_job on a
FAILED pending resets it to PENDING and runs a single fresh attempt
(see test_confirm_resets_failed_then_succeeds). The reset path is
useful for 'I fixed the script, try again' flows; it just is no longer
the default on every failure.

Removed:
  - the for-attempt loop and time.sleep in submit.py
  - import time in submit.py and test_submit_tool.py
  - confirm_max_retries and confirm_retry_delay_seconds from
    common/config.py (env vars, snapshot, reload)
  - 2 retry tests (test_confirm_retries_spark_submit_failure_then_succeeds,
    test_confirm_exhausts_retries_and_sets_failed)
  - test_confirm_marks_failed_on_spark_submit_error updated to assert
    the new single-attempt + raise contract
  - test_confirm_resets_failed_and_retries renamed to
    test_confirm_resets_failed_then_succeeds (no more retry loop to
    exercise)

Full suite 244 passed (3 fewer than before, matching the removed
tests).

Co-Authored-By: Claude Fable 5 <noreply@anthropic.com>

spark_executor/tools/submit.py

@@ -5,7 +5,6 @@
 """
 import os
 import secrets
-import time
 import uuid
 from datetime import datetime
 
@@ -145,7 +144,16 @@ job_store: JobStore = JobStore()
 
 
 def confirm_submit_job(*, pending_id: str) -> SubmitResult:
-    """Actually invoke spark-submit for a previously-prepared PendingSubmission."""
+    """Invoke spark-submit for a previously-prepared PendingSubmission.
+
+    Raises SparkSubmitError if the single spark-submit attempt fails;
+    before raising, the pending is marked FAILED with the error message
+    in pending.error so the agent can inspect it via get_pending_job.
+
+    There is no automatic retry — a failed confirm is recorded as-is
+    and the user is expected to investigate. Re-confirming a FAILED
+    pending resets it to PENDING for a single fresh attempt.
+    """
     logger.debug(f"confirm_submit_job enter pending_id={pending_id}")
     pending = pending_store.get(pending_id)
     if pending is None:
@@ -164,6 +172,9 @@ def confirm_submit_job(*, pending_id: str) -> SubmitResult:
     if pending.status == "CANCELLED":
         raise ValueError(f"pending_id {pending_id} is CANCELLED, cannot confirm")
     if pending.status == "FAILED":
+        # Reset for a manual retry: the user has presumably fixed the
+        # underlying issue (yarn RM was down, file restored) and is
+        # asking us to try once more.
         pending.status = "PENDING"
         pending.error = None
         pending_store.save(pending)
@@ -194,71 +205,50 @@ def confirm_submit_job(*, pending_id: str) -> SubmitResult:
         f"application_target={pending.master} script_path={pending.script_path}"
     )
 
-    last_error: SparkSubmitError | None = None
-    max_attempts = settings.confirm_max_retries + 1
-    for attempt in range(1, max_attempts + 1):
-        try:
-            result = run_spark_submit(cmd)
-        except SparkSubmitError as exc:
-            last_error = exc
-            logger.warning(
-                f"confirm_submit_job attempt {attempt}/{max_attempts} failed "
-                f"pending_id={pending_id} err={exc}"
-            )
-            if attempt < max_attempts:
-                time.sleep(settings.confirm_retry_delay_seconds)
-            continue
-        except Exception:
-            # Unexpected failure (not a spark-submit error): fail fast without retry.
-            logger.exception(
-                f"confirm_submit_job unexpected error pending_id={pending_id}"
-            )
-            raise
-
-        application_id, tracking_url = parse_spark_submit_output(result.stderr)
-        job_id = uuid.uuid4().hex[:12]
-
-        # Persist pending as SUBMITTED before creating the in-memory Job so a
-        # job_store failure cannot leave the record as PENDING while the YARN
-        # app is already running.
-        pending.status = "SUBMITTED"
-        pending.job_id = job_id
-        pending.application_id = application_id
-        pending.tracking_url = tracking_url
+    try:
+        result = run_spark_submit(cmd)
+    except SparkSubmitError as exc:
+        pending.status = "FAILED"
+        pending.error = str(exc)
         pending_store.save(pending)
-
-        job_store.put(
-            Job(
-                job_id=job_id,
-                application_id=application_id,
-                script_path=pending.script_path,
-                queue=pending.queue,
-                submit_time=datetime.utcnow(),
-                connection=pending.connection,
-                yarn_rm_url=pending.yarn_rm_url,
-            )
+        logger.error(
+            f"confirm_submit_job failed pending_id={pending_id} err={exc}"
         )
+        raise
 
-        logger.info(
-            f"confirm_submit_job ok pending_id={pending_id} job_id={job_id} "
-            f"application_id={application_id}"
-        )
-        return SubmitResult(
+    application_id, tracking_url = parse_spark_submit_output(result.stderr)
+    job_id = uuid.uuid4().hex[:12]
+
+    # Persist pending as SUBMITTED before creating the in-memory Job so a
+    # job_store failure cannot leave the record as PENDING while the YARN
+    # app is already running.
+    pending.status = "SUBMITTED"
+    pending.job_id = job_id
+    pending.application_id = application_id
+    pending.tracking_url = tracking_url
+    pending_store.save(pending)
+
+    job_store.put(
+        Job(
             job_id=job_id,
             application_id=application_id,
-            tracking_url=tracking_url,
+            script_path=pending.script_path,
+            queue=pending.queue,
+            submit_time=datetime.utcnow(),
+            connection=pending.connection,
+            yarn_rm_url=pending.yarn_rm_url,
         )
-
-    # Exhausted all retries.
-    assert last_error is not None
-    pending.status = "FAILED"
-    pending.error = str(last_error)
-    pending_store.save(pending)
-    logger.error(
-        f"confirm_submit_job failed pending_id={pending_id} after {max_attempts} attempts "
-        f"err={last_error}"
     )
-    raise last_error
+
+    logger.info(
+        f"confirm_submit_job ok pending_id={pending_id} job_id={job_id} "
+        f"application_id={application_id}"
+    )
+    return SubmitResult(
+        job_id=job_id,
+        application_id=application_id,
+        tracking_url=tracking_url,
+    )
 
 
 def list_pending_jobs() -> list[dict[str, object]]: