executor: parse tracking_url and error on missing application_id

Co-Authored-By: Claude <noreply@anthropic.com>

internal/executor/spark_submit.go

@@ -23,10 +23,13 @@ import (
 // application application_xxx" line and a few progress lines.
 const stdoutCap = 10 << 20
 
-// appIDPattern matches the YARN line printed on successful submission:
-//
-//	"Submitted application application_12345_0001"
-var appIDPattern = regexp.MustCompile(`application_\d+_\d+`)
+// submittedAppPattern matches the explicit success line, including the
+// application id captured in group 1. This mirrors the Python reference.
+var submittedAppPattern = regexp.MustCompile(`Submitted application (\S+)`)
+
+// trackingURLPattern matches "tracking URL: <url>" on stderr. The URL tail
+// is used as a fallback source for the application id.
+var trackingURLPattern = regexp.MustCompile(`tracking URL:\s+(\S+)`)
 
 // SparkSubmitOpts is the resolved command line for a single spark-submit run.
 //
@@ -43,11 +46,12 @@ type SparkSubmitOpts struct {
 
 // Result is what spark_submit returns to the LLM.
 type Result struct {
-	AppID      string `json:"app_id,omitempty"`
-	ExitCode   int    `json:"exit_code"`
-	StdoutTail string `json:"stdout_tail"`
-	StderrTail string `json:"stderr_tail"`
-	DurationMS int64  `json:"duration_ms"`
+	AppID       string `json:"app_id,omitempty"`
+	TrackingURL string `json:"tracking_url,omitempty"`
+	ExitCode    int    `json:"exit_code"`
+	StdoutTail  string `json:"stdout_tail"`
+	StderrTail  string `json:"stderr_tail"`
+	DurationMS  int64  `json:"duration_ms"`
 }
 
 // ErrBinaryEmpty is returned when the cluster has no spark-submit binary
@@ -90,27 +94,70 @@ func Run(ctx context.Context, opts SparkSubmitOpts) (*Result, error) {
 		DurationMS: dur.Milliseconds(),
 	}
 
-	// app_id: prefer stdout, fall back to stderr. Both are searched only on
-	// the tail (cap-bounded) so we don't pay for the full output.
-	if id := matchAppID(res.StdoutTail); id != "" {
-		res.AppID = id
-	} else if id := matchAppID(res.StderrTail); id != "" {
-		res.AppID = id
-	}
+	appID, trackingURL, ok := parseSparkSubmitOutput(res.StdoutTail, res.StderrTail)
+	res.AppID = appID
+	res.TrackingURL = trackingURL
 
+	if !ok {
+		return res, fmt.Errorf("executor: spark-submit ran but no application_id in output (exit_code=%d)", res.ExitCode)
+	}
 	if err != nil {
-		// Wrap the underlying error but keep the parsed result so the LLM
-		// can see exit_code and stderr even on failure.
 		return res, fmt.Errorf("executor: spark-submit failed: %w", err)
 	}
 	return res, nil
 }
 
-func matchAppID(s string) string {
-	if s == "" {
+// parseSparkSubmitOutput extracts the application id and tracking URL from
+// spark-submit output. It mirrors the Python reference implementation: first
+// look for "Submitted application <id>", then fall back to "tracking URL:",
+// deriving the id from the last path segment. The returned ok is false when
+// no application id could be determined.
+func parseSparkSubmitOutput(stdout, stderr string) (appID, trackingURL string, ok bool) {
+	// Stage 1: explicit success line, prefer stdout then stderr.
+	var trackingFound bool
+	if m := submittedAppPattern.FindStringSubmatch(stdout); m != nil {
+		appID = m[1]
+		ok = true
+	} else if m := submittedAppPattern.FindStringSubmatch(stderr); m != nil {
+		appID = m[1]
+		ok = true
+	}
+
+	// Stage 2: tracking URL fallback (stderr only, like the Python reference).
+	if !ok {
+		if m := trackingURLPattern.FindStringSubmatch(stderr); m != nil {
+			trackingURL = m[1]
+			trackingFound = true
+			appID = extractAppIDFromTrackingURL(trackingURL)
+			ok = appID != ""
+			if !ok {
+				trackingURL = ""
+			}
+		}
+	}
+
+	// Tracking URL is independent: if present anywhere on stderr, capture it.
+	if trackingURL == "" && !trackingFound {
+		if m := trackingURLPattern.FindStringSubmatch(stderr); m != nil {
+			trackingURL = m[1]
+		}
+	}
+	return
+}
+
+// extractAppIDFromTrackingURL returns the last path segment if it starts with
+// "application_", otherwise an empty string.
+func extractAppIDFromTrackingURL(rawURL string) string {
+	trimmed := strings.TrimRight(rawURL, "/")
+	idx := strings.LastIndex(trimmed, "/")
+	if idx < 0 {
 		return ""
 	}
-	return appIDPattern.FindString(s)
+	tail := trimmed[idx+1:]
+	if strings.HasPrefix(tail, "application_") {
+		return tail
+	}
+	return ""
 }
 
 // exitCodeFromErr returns 0 on nil, the process's exit code on *exec.ExitError,