feat(submit): configurable confirm retries and idempotent confirm

Harden confirm_submit_job for unreliable test environments and transient
spark-submit failures:

- Add confirm_max_retries and confirm_retry_delay_seconds settings
  (env-configurable).
- SUBMITTED pending returns cached result without re-running spark-submit.
- FAILED pending is reset to PENDING and retried.
- CANCELLED pending is still rejected.
- On success, persist pending as SUBMITTED before creating the in-memory Job
  so a job_store failure cannot leave the record as PENDING while the YARN
  app is already running.
- Persist tracking_url on PendingSubmission for idempotent returns.

Tests cover retry-then-success, max-retries-exceeded, idempotency,
FAILED reset, and pending-saved-before-job-store.

common/config.py

@@ -76,6 +76,13 @@ class Settings:
     # The binary is resolved against $PATH (set by docker-entrypoint.sh).
     spark_submit_bin: str = "spark-submit"
 
+    # --- Confirm retry policy ---
+    # confirm_submit_job retries spark-submit on SparkSubmitError to tolerate
+    # transient client/network issues in test environments or unstable clusters.
+    # Max retries (not counting the first attempt) and delay between attempts.
+    confirm_max_retries: int = 3
+    confirm_retry_delay_seconds: float = 5.0
+
     # --- SSL/TLS defaults for YARN REST calls ---
     ssl_verify_default: bool = True
     ssl_ca_bundle_default: str | None = None
@@ -106,6 +113,12 @@ class Settings:
             ssl_ca_bundle_default=(
                 os.environ.get("SPARK_EXECUTOR_SSL_CA_BUNDLE_DEFAULT") or None
             ),
+            confirm_max_retries=int(
+                os.environ.get("SPARK_EXECUTOR_CONFIRM_MAX_RETRIES", "3")
+            ),
+            confirm_retry_delay_seconds=float(
+                os.environ.get("SPARK_EXECUTOR_CONFIRM_RETRY_DELAY_SECONDS", "5.0")
+            ),
         )
 
     def reload(self) -> "Settings":
@@ -123,6 +136,8 @@ class Settings:
         self.spark_submit_bin = fresh.spark_submit_bin
         self.ssl_verify_default = fresh.ssl_verify_default
         self.ssl_ca_bundle_default = fresh.ssl_ca_bundle_default
+        self.confirm_max_retries = fresh.confirm_max_retries
+        self.confirm_retry_delay_seconds = fresh.confirm_retry_delay_seconds
         return self