admin: submissions page, download endpoint, uploads search by app_id

Co-Authored-By: Claude <noreply@anthropic.com>
tools: persist spark_submit calls to SubmissionRepo
2026-07-14 13:45:11 +08:00 · 2026-07-14 13:45:07 +08:00 · 2026-07-14 13:45:03 +08:00 · 2026-07-14 13:45:01 +08:00
20 changed files with 1363 additions and 87 deletions
@@ -10,6 +10,7 @@ import (
 	"os"
 	"path/filepath"
 	"strconv"
+	"strings"

 	"github.com/gin-gonic/gin"

@@ -22,7 +23,7 @@ import (

 // Mount attaches the /admin sub-router to r, protecting every route with
 // bearer-token admin authentication.
-func Mount(r *gin.Engine, repo *storage.ClusterRepo, uploadRepo *storage.UploadRepo, auditRepo *audit.Repo, uploadStore *uploads.Store, adminTokens []string) {
+func Mount(r *gin.Engine, repo *storage.ClusterRepo, uploadRepo *storage.UploadRepo, submissionRepo *storage.SubmissionRepo, auditRepo *audit.Repo, uploadStore *uploads.Store, adminTokens []string) {
 	// HTML page is public — its own modal prompts for the token.
 	// All other /admin/* endpoints (API + OpenAPI docs) still require auth.
 	gPublic := r.Group("/admin")
@@ -38,7 +39,10 @@ func Mount(r *gin.Engine, repo *storage.ClusterRepo, uploadRepo *storage.UploadR
 	g.GET("/audit", listAudit(auditRepo))
 	g.GET("/uploads", uploadsWebHandler)
 	g.GET("/uploads/api", listUploads(uploadRepo))
+	g.GET("/uploads/:id/download", downloadUpload(uploadRepo, uploadStore))
 	g.DELETE("/uploads/:id", deleteUpload(uploadRepo, auditRepo, uploadStore))
+	g.GET("/submissions", submissionsWebHandler)
+	g.GET("/submissions/api", listSubmissions(submissionRepo))
 	g.GET("/docs", DocsHandler)
 	g.GET("/docs/spec", OpenAPISpecHandler)
 }
@@ -300,6 +304,81 @@ func listUploads(repo *storage.UploadRepo) gin.HandlerFunc {
 	}
 }

+func listSubmissions(repo *storage.SubmissionRepo) gin.HandlerFunc {
+	return func(c *gin.Context) {
+		limitStr := c.DefaultQuery("limit", "100")
+		limit, err := strconv.Atoi(limitStr)
+		if err != nil {
+			c.JSON(http.StatusBadRequest, gin.H{"error": "invalid limit"})
+			return
+		}
+		offsetStr := c.DefaultQuery("offset", "0")
+		offset, err := strconv.Atoi(offsetStr)
+		if err != nil {
+			c.JSON(http.StatusBadRequest, gin.H{"error": "invalid offset"})
+			return
+		}
+
+		subs, err := repo.List(c.Request.Context(), storage.SubmissionFilter{
+			Search:    c.Query("search"),
+			FileID:    c.Query("file_id"),
+			ClusterID: c.Query("cluster_id"),
+			AppID:     c.Query("app_id"),
+			Limit:     limit,
+			Offset:    offset,
+		})
+		if err != nil {
+			c.JSON(http.StatusInternalServerError, gin.H{"error": err.Error()})
+			return
+		}
+		c.JSON(http.StatusOK, subs)
+	}
+}
+
+func downloadUpload(repo *storage.UploadRepo, store *uploads.Store) gin.HandlerFunc {
+	return func(c *gin.Context) {
+		id := c.Param("id")
+		ctx := c.Request.Context()
+
+		meta, err := repo.Get(ctx, id)
+		if err != nil {
+			if errors.Is(err, storage.ErrNotFound) {
+				c.JSON(http.StatusNotFound, gin.H{"error": "upload not found"})
+				return
+			}
+			c.JSON(http.StatusInternalServerError, gin.H{"error": err.Error()})
+			return
+		}
+
+		dataPath, err := store.AbsPath(id)
+		if err != nil {
+			c.JSON(http.StatusBadRequest, gin.H{"error": err.Error()})
+			return
+		}
+		data, err := os.ReadFile(dataPath)
+		if err != nil {
+			if os.IsNotExist(err) {
+				c.JSON(http.StatusNotFound, gin.H{"error": "upload file missing"})
+				return
+			}
+			c.JSON(http.StatusInternalServerError, gin.H{"error": err.Error()})
+			return
+		}
+
+		c.Header("Content-Disposition", "attachment; filename=\""+escapeDispositionFilename(meta.Name)+"\"")
+		c.Data(http.StatusOK, "application/octet-stream", data)
+	}
+}
+
+// escapeDispositionFilename escapes quotes and backslashes for the
+// Content-Disposition filename parameter. The upload validator already rejects
+// names containing quotes or newlines; this is defense in depth.
+func escapeDispositionFilename(name string) string {
+	name = strings.ReplaceAll(name, "\\", "\\\\")
+	name = strings.ReplaceAll(name, "\"", "\\\"")
+	return name
+}
+
 func deleteUpload(repo *storage.UploadRepo, auditRepo *audit.Repo, store *uploads.Store) gin.HandlerFunc {
 	return func(c *gin.Context) {
 		id := c.Param("id")
@@ -35,7 +35,7 @@ func newTestAdmin(t *testing.T) (*gin.Engine, *storage.ClusterRepo, *audit.Repo)
 	if err != nil {
 		t.Fatalf("new upload store: %v", err)
 	}
-	Mount(r, db.Clusters(), db.Uploads(), audit.NewRepo(db), &uploadStore, []string{"good-token"})
+	Mount(r, db.Clusters(), db.Uploads(), db.Submissions(), audit.NewRepo(db), &uploadStore, []string{"good-token"})
 	return r, db.Clusters(), audit.NewRepo(db)
 }

@@ -0,0 +1,155 @@
+package admin
+
+import (
+	"context"
+	"encoding/json"
+	"net/http"
+	"os"
+	"path/filepath"
+	"strings"
+	"testing"
+	"time"
+
+	"github.com/gin-gonic/gin"
+
+	"spark-mcp-go/internal/storage"
+	"spark-mcp-go/internal/uploads"
+)
+
+func newTestAdminSubmissions(t *testing.T) (*gin.Engine, *storage.SubmissionRepo, *storage.UploadRepo, *uploads.Store) {
+	t.Helper()
+	db, err := storage.Open(":memory:")
+	if err != nil {
+		t.Fatalf("open: %v", err)
+	}
+	t.Cleanup(func() { _ = db.Close() })
+
+	uploadStore, err := uploads.New(filepath.Join(t.TempDir(), "uploads"))
+	if err != nil {
+		t.Fatalf("new upload store: %v", err)
+	}
+
+	r := gin.New()
+	Mount(r, db.Clusters(), db.Uploads(), db.Submissions(), nil, &uploadStore, []string{"good-token"})
+	return r, db.Submissions(), db.Uploads(), &uploadStore
+}
+
+func TestSubmissionsWeb_RequiresAuth(t *testing.T) {
+	r, _, _, _ := newTestAdminSubmissions(t)
+
+	w := doReq(t, r, "GET", "/admin/submissions", "", nil)
+	if w.Code != http.StatusUnauthorized {
+		t.Errorf("GET /admin/submissions without auth: got %d, want %d", w.Code, http.StatusUnauthorized)
+	}
+}
+
+func TestSubmissionsWeb_WithAuth(t *testing.T) {
+	r, _, _, _ := newTestAdminSubmissions(t)
+
+	w := doReq(t, r, "GET", "/admin/submissions", "good-token", nil)
+	if w.Code != http.StatusOK {
+		t.Fatalf("GET /admin/submissions with auth: got %d, want %d", w.Code, http.StatusOK)
+	}
+	ct := w.Header().Get("Content-Type")
+	if !strings.Contains(ct, "text/html") {
+		t.Errorf("Content-Type = %q, want text/html", ct)
+	}
+	body := w.Body.String()
+	if !strings.Contains(body, "href=\"/admin/uploads\"") && !strings.Contains(body, "href='/admin/uploads'") {
+		t.Errorf("body missing uploads nav link")
+	}
+	if !strings.Contains(body, "href=\"/admin/submissions\"") && !strings.Contains(body, "href='/admin/submissions'") {
+		t.Errorf("body missing submissions nav link")
+	}
+	if !strings.Contains(body, "submissions-body") {
+		t.Errorf("body missing submissions table")
+	}
+}
+
+func TestListSubmissions_SearchByAppID(t *testing.T) {
+	r, subRepo, uploadRepo, _ := newTestAdminSubmissions(t)
+	ctx := context.Background()
+
+	uploadedAt := time.Unix(0, time.Now().UnixNano())
+	if err := uploadRepo.Create(ctx, "00000000000000000000000000000001", "alpha.py", 10, "deadbeef", uploadedAt); err != nil {
+		t.Fatalf("create upload: %v", err)
+	}
+	if err := uploadRepo.Create(ctx, "00000000000000000000000000000002", "beta.py", 20, "cafebabe", uploadedAt.Add(time.Second)); err != nil {
+		t.Fatalf("create upload: %v", err)
+	}
+
+	submissions := []storage.Submission{
+		{FileID: "00000000000000000000000000000001", AppID: "application_111_0001", TrackingURL: "http://a", ClusterID: "c1", ExitCode: 0, DurationMS: 100, SubmittedAt: time.Now()},
+		{FileID: "00000000000000000000000000000002", AppID: "application_222_0001", TrackingURL: "http://b", ClusterID: "c1", ExitCode: 0, DurationMS: 100, SubmittedAt: time.Now().Add(time.Second)},
+	}
+	for _, s := range submissions {
+		if err := subRepo.Create(ctx, s); err != nil {
+			t.Fatalf("create submission: %v", err)
+		}
+	}
+
+	w := doReq(t, r, "GET", "/admin/submissions/api?search=application_111", "good-token", nil)
+	if w.Code != http.StatusOK {
+		t.Fatalf("search: got status %d: %s", w.Code, w.Body.String())
+	}
+	var list []map[string]any
+	if err := json.Unmarshal(w.Body.Bytes(), &list); err != nil {
+		t.Fatalf("unmarshal search: %v", err)
+	}
+	if len(list) != 1 {
+		t.Errorf("search result: got %d, want 1", len(list))
+	}
+	if len(list) > 0 && list[0]["app_id"] != "application_111_0001" {
+		t.Errorf("app_id=%v, want application_111_0001", list[0]["app_id"])
+	}
+
+	w = doReq(t, r, "GET", "/admin/submissions/api?search=beta.py", "good-token", nil)
+	if w.Code != http.StatusOK {
+		t.Fatalf("search by name: got status %d: %s", w.Code, w.Body.String())
+	}
+	if err := json.Unmarshal(w.Body.Bytes(), &list); err != nil {
+		t.Fatalf("unmarshal search: %v", err)
+	}
+	if len(list) != 1 {
+		t.Errorf("search by name result: got %d, want 1", len(list))
+	}
+}
+
+func TestDownloadUpload(t *testing.T) {
+	r, _, uploadRepo, store := newTestAdminSubmissions(t)
+	ctx := context.Background()
+
+	id := "00000000000000000000000000000003"
+	payload := []byte("hello payload")
+	if err := uploadRepo.Create(ctx, id, "report.csv", int64(len(payload)), "feedface", time.Now()); err != nil {
+		t.Fatalf("create upload: %v", err)
+	}
+	dataPath, err := store.AbsPath(id)
+	if err != nil {
+		t.Fatalf("abspath: %v", err)
+	}
+	if err := os.WriteFile(dataPath, payload, 0o640); err != nil {
+		t.Fatalf("write data: %v", err)
+	}
+
+	w := doReq(t, r, "GET", "/admin/uploads/"+id+"/download", "good-token", nil)
+	if w.Code != http.StatusOK {
+		t.Fatalf("download: got status %d: %s", w.Code, w.Body.String())
+	}
+	if string(w.Body.Bytes()) != string(payload) {
+		t.Errorf("body = %q, want %q", w.Body.Bytes(), payload)
+	}
+	cd := w.Header().Get("Content-Disposition")
+	if !strings.Contains(cd, `filename="report.csv"`) {
+		t.Errorf("Content-Disposition = %q, want filename=\"report.csv\"", cd)
+	}
+}
+
+func TestDownloadUpload_NotFound(t *testing.T) {
+	r, _, _, _ := newTestAdminSubmissions(t)
+
+	w := doReq(t, r, "GET", "/admin/uploads/00000000000000000000000000000099/download", "good-token", nil)
+	if w.Code != http.StatusNotFound {
+		t.Errorf("download unknown: got %d, want %d", w.Code, http.StatusNotFound)
+	}
+}
@@ -18,7 +18,7 @@ import (
 	"spark-mcp-go/internal/uploads"
 )

-func newTestAdminUploads(t *testing.T) (*gin.Engine, *storage.UploadRepo, *audit.Repo, *uploads.Store) {
+func newTestAdminUploads(t *testing.T) (*gin.Engine, *storage.UploadRepo, *storage.SubmissionRepo, *audit.Repo, *uploads.Store) {
 	t.Helper()
 	db, err := storage.Open(":memory:")
 	if err != nil {
@@ -32,12 +32,12 @@ func newTestAdminUploads(t *testing.T) (*gin.Engine, *storage.UploadRepo, *audit
 	}

 	r := gin.New()
-	Mount(r, db.Clusters(), db.Uploads(), audit.NewRepo(db), &uploadStore, []string{"good-token"})
-	return r, db.Uploads(), audit.NewRepo(db), &uploadStore
+	Mount(r, db.Clusters(), db.Uploads(), db.Submissions(), audit.NewRepo(db), &uploadStore, []string{"good-token"})
+	return r, db.Uploads(), db.Submissions(), audit.NewRepo(db), &uploadStore
 }

 func TestUploadsWeb_RequiresAuth(t *testing.T) {
-	r, _, _, _ := newTestAdminUploads(t)
+	r, _, _, _, _ := newTestAdminUploads(t)

 	w := doReq(t, r, "GET", "/admin/uploads", "", nil)
 	if w.Code != http.StatusUnauthorized {
@@ -62,7 +62,7 @@ func TestUploadsWeb_RequiresAuth(t *testing.T) {
 }

 func TestListUploads(t *testing.T) {
-	r, repo, _, _ := newTestAdminUploads(t)
+	r, repo, _, _, _ := newTestAdminUploads(t)
 	ctx := context.Background()

 	uploadedAt := time.Unix(0, time.Now().UnixNano())
@@ -98,7 +98,7 @@ func TestListUploads(t *testing.T) {
 }

 func TestDeleteUpload(t *testing.T) {
-	r, repo, auditRepo, store := newTestAdminUploads(t)
+	r, repo, _, auditRepo, store := newTestAdminUploads(t)
 	ctx := context.Background()

 	id := "00000000000000000000000000000003"
@@ -152,3 +152,45 @@ func TestDeleteUpload(t *testing.T) {
 		t.Errorf("missing upload.delete audit entry")
 	}
 }
+
+func TestListUploads_SearchByAppID(t *testing.T) {
+	r, uploadRepo, subRepo, _, _ := newTestAdminUploads(t)
+	ctx := context.Background()
+
+	uploadedAt := time.Unix(0, time.Now().UnixNano())
+	if err := uploadRepo.Create(ctx, "00000000000000000000000000000010", "gamma.py", 10, "deadbeef", uploadedAt); err != nil {
+		t.Fatalf("create upload: %v", err)
+	}
+	if err := uploadRepo.Create(ctx, "00000000000000000000000000000011", "delta.py", 20, "cafebabe", uploadedAt.Add(time.Second)); err != nil {
+		t.Fatalf("create upload: %v", err)
+	}
+
+	// Only gamma.py has an associated submission; searching by app_id should
+	// still return it even though the filename doesn't match the query.
+	if err := subRepo.Create(ctx, storage.Submission{
+		FileID:      "00000000000000000000000000000010",
+		AppID:       "application_gamma_0001",
+		TrackingURL: "http://gamma",
+		ClusterID:   "c1",
+		ExitCode:    0,
+		DurationMS:  100,
+		SubmittedAt: time.Now(),
+	}); err != nil {
+		t.Fatalf("create submission: %v", err)
+	}
+
+	w := doReq(t, r, "GET", "/admin/uploads/api?search=application_gamma", "good-token", nil)
+	if w.Code != http.StatusOK {
+		t.Fatalf("search by app_id: got status %d: %s", w.Code, w.Body.String())
+	}
+	var list []map[string]any
+	if err := json.Unmarshal(w.Body.Bytes(), &list); err != nil {
+		t.Fatalf("unmarshal search: %v", err)
+	}
+	if len(list) != 1 {
+		t.Errorf("search by app_id result: got %d, want 1", len(list))
+	}
+	if len(list) > 0 && list[0]["file_id"] != "00000000000000000000000000000010" {
+		t.Errorf("file_id=%v, want upload 10", list[0]["file_id"])
+	}
+}
@@ -27,3 +27,12 @@ func uploadsWebHandler(c *gin.Context) {
 	}
 	c.Data(http.StatusOK, "text/html; charset=utf-8", data)
 }
+
+func submissionsWebHandler(c *gin.Context) {
+	data, err := webFS.ReadFile("web/submissions.html")
+	if err != nil {
+		c.String(http.StatusInternalServerError, "submissions.html: %v", err)
+		return
+	}
+	c.Data(http.StatusOK, "text/html; charset=utf-8", data)
+}
@@ -120,6 +120,7 @@
     <nav>
       <a href='/admin' class='active'>Clusters</a>
       <a href='/admin/uploads'>Uploads</a>
+        <a href='/admin/submissions'>Submissions</a>
     </nav>
    </div>
    <div class='auth'>
@@ -0,0 +1,367 @@
+<!DOCTYPE html>
+<html lang='en'>
+<head>
+  <meta charset='utf-8'>
+  <meta name='viewport' content='width=device-width, initial-scale=1'>
+  <title>Submissions - spark-mcp-go Admin</title>
+  <style>
+    :root {
+      --bg: #0d1117;
+      --panel: #161b22;
+      --border: #30363d;
+      --text: #c9d1d9;
+      --muted: #8b949e;
+      --accent: #58a6ff;
+      --danger: #f85149;
+      --ok: #3fb950;
+    }
+    * { box-sizing: border-box; }
+    body {
+      margin: 0;
+      font-family: ui-monospace, SFMono-Regular, Menlo, Monaco, Consolas, 'Liberation Mono', monospace;
+      background: var(--bg);
+      color: var(--text);
+      line-height: 1.5;
+    }
+    header {
+      display: flex;
+      align-items: center;
+      justify-content: space-between;
+      gap: 1rem;
+      padding: 1rem;
+      border-bottom: 1px solid var(--border);
+      background: var(--panel);
+      flex-wrap: wrap;
+    }
+    header h1 { margin: 0; font-size: 1.1rem; color: var(--accent); }
+    nav { display: flex; gap: .75rem; }
+    nav a { color: var(--muted); text-decoration: none; }
+    nav a:hover { color: var(--text); }
+    nav a.active { color: var(--accent); }
+    .auth { display: flex; align-items: center; gap: .5rem; flex-wrap: wrap; }
+    input[type='text'], input[type='password'] {
+      background: var(--bg);
+      color: var(--text);
+      border: 1px solid var(--border);
+      border-radius: 4px;
+      padding: .4rem .5rem;
+      font: inherit;
+    }
+    input[type='text'] { min-width: 260px; }
+    button {
+      background: var(--accent);
+      color: #fff;
+      border: none;
+      border-radius: 4px;
+      padding: .4rem .8rem;
+      font: inherit;
+      cursor: pointer;
+    }
+    button:hover { opacity: .9; }
+    button.danger { background: var(--danger); }
+    button.secondary { background: var(--border); color: var(--text); }
+    main { padding: 1rem; max-width: 1400px; margin: 0 auto; }
+    .toolbar { display: flex; gap: .5rem; margin-bottom: 1rem; align-items: center; flex-wrap: wrap; }
+    .toolbar span { color: var(--muted); }
+    .error {
+      background: rgba(248, 81, 73, .15);
+      border: 1px solid var(--danger);
+      color: var(--danger);
+      padding: .75rem;
+      border-radius: 4px;
+      margin-bottom: 1rem;
+      white-space: pre-wrap;
+    }
+    .hidden { display: none; }
+    table { width: 100%; border-collapse: collapse; }
+    th, td { text-align: left; padding: .5rem; border-bottom: 1px solid var(--border); }
+    th { color: var(--muted); font-weight: 600; user-select: none; }
+    th.sortable { cursor: pointer; }
+    th.sortable:hover { color: var(--text); }
+    td { vertical-align: middle; font-size: .9rem; }
+    td.mono { font-family: ui-monospace, SFMono-Regular, Menlo, Monaco, Consolas, monospace; }
+    td.wrap { word-break: break-all; }
+    .actions { display: flex; gap: .4rem; }
+    dialog.modal {
+      border: 1px solid var(--border);
+      background: var(--panel);
+      color: var(--text);
+      border-radius: 6px;
+      padding: 1.5rem;
+      min-width: 360px;
+      max-width: 90vw;
+    }
+    dialog.modal::backdrop { background: rgba(0, 0, 0, .65); }
+    dialog.modal h2 { margin: 0 0 .5rem; font-size: 1.1rem; color: var(--accent); }
+    dialog.modal form { display: flex; flex-direction: column; gap: .75rem; }
+    dialog.modal label { color: var(--muted); font-size: .9rem; }
+    dialog.modal input { width: 100%; }
+    .modal-error { color: var(--danger); font-size: .9rem; margin: 0; }
+    .modal-actions { display: flex; gap: .5rem; justify-content: flex-end; }
+  </style>
+</head>
+<body>
+  <header>
+    <div style='display:flex;align-items:center;gap:1rem;'>
+      <h1>spark-mcp-go Admin</h1>
+      <nav>
+        <a href='/admin'>Clusters</a>
+        <a href='/admin/uploads'>Uploads</a>
+        <a href='/admin/submissions' class='active'>Submissions</a>
+      </nav>
+    </div>
+    <div class='auth'>
+      <button id='set-token' class='secondary'>Set Token</button>
+      <button id='logout' class='danger'>Logout</button>
+    </div>
+  </header>
+
+  <main>
+    <div id='error' class='error hidden'></div>
+
+    <section class='toolbar'>
+      <input id='search' type='text' placeholder='Search by file name or app id...'>
+      <button id='refresh' class='secondary'>Refresh</button>
+      <span id='count'></span>
+    </section>
+
+    <section>
+      <table>
+        <thead>
+          <tr>
+            <th class='sortable' data-key='id'>ID</th>
+            <th class='sortable' data-key='file_id'>File ID</th>
+            <th class='sortable' data-key='file_name'>File Name</th>
+            <th class='sortable' data-key='app_id'>App ID</th>
+            <th>Tracking URL</th>
+            <th class='sortable' data-key='cluster_id'>Cluster</th>
+            <th class='sortable' data-key='exit_code'>Exit</th>
+            <th class='sortable' data-key='duration_ms'>Duration (ms)</th>
+            <th class='sortable' data-key='submitted_at'>Submitted At</th>
+          </tr>
+        </thead>
+        <tbody id='submissions-body'>
+          <tr><td colspan='9'>Loading...</td></tr>
+        </tbody>
+      </table>
+    </section>
+  </main>
+
+  <dialog id='token-modal' class='modal'>
+    <h2>Admin Token Required</h2>
+    <p id='modal-error' class='modal-error hidden'></p>
+    <form id='token-form'>
+      <label for='modal-token'>Bearer token</label>
+      <input id='modal-token' type='password' autocomplete='off' required>
+      <div class='modal-actions'>
+        <button type='submit' id='modal-save'>Save</button>
+        <button type='button' id='modal-cancel' class='secondary hidden'>Cancel</button>
+      </div>
+    </form>
+  </dialog>
+
+  <script>
+    const $ = (sel) => document.querySelector(sel);
+    const API = '/admin/submissions';
+
+    let submissions = [];
+    let sortKey = 'submitted_at';
+    let sortDir = -1;
+    let pendingRetry = null;
+    let cancelBlocker = null;
+
+    function headers() {
+      return {
+        'Authorization': 'Bearer ' + (localStorage.getItem('adminToken') || ''),
+        'Content-Type': 'application/json'
+      };
+    }
+
+    function showError(msg) {
+      const el = $('#error');
+      el.textContent = msg || '';
+      el.classList.toggle('hidden', !msg);
+    }
+
+    function showModalError(msg) {
+      const el = $('#modal-error');
+      el.textContent = msg || '';
+      el.classList.toggle('hidden', !msg);
+    }
+
+    function openTokenModal(opts = {}) {
+      const dialog = $('#token-modal');
+      const cancelBtn = $('#modal-cancel');
+      showModalError(opts.error || '');
+      cancelBtn.classList.toggle('hidden', !opts.allowCancel);
+      if (cancelBlocker) {
+        dialog.removeEventListener('cancel', cancelBlocker);
+        cancelBlocker = null;
+      }
+      if (!opts.allowCancel) {
+        cancelBlocker = (e) => e.preventDefault();
+        dialog.addEventListener('cancel', cancelBlocker);
+      }
+      dialog.showModal();
+      $('#modal-token').focus();
+    }
+
+    function closeTokenModal() {
+      const dialog = $('#token-modal');
+      showModalError('');
+      if (cancelBlocker) {
+        dialog.removeEventListener('cancel', cancelBlocker);
+        cancelBlocker = null;
+      }
+      dialog.close();
+      $('#modal-token').value = '';
+    }
+
+    function saveTokenFromModal(e) {
+      e.preventDefault();
+      localStorage.setItem('adminToken', $('#modal-token').value.trim());
+      closeTokenModal();
+      if (pendingRetry) {
+        const fn = pendingRetry;
+        pendingRetry = null;
+        fn();
+      } else {
+        loadSubmissions();
+      }
+    }
+
+    async function api(method, path, body) {
+      const opts = { method, headers: headers() };
+      if (body !== undefined) opts.body = JSON.stringify(body);
+      const res = await fetch(API + path, opts);
+      if (!res.ok) {
+        if (res.status === 401) {
+          localStorage.removeItem('adminToken');
+          pendingRetry = () => api(method, path, body);
+          openTokenModal({
+            error: 'Token rejected by server (401). Enter a valid token.',
+            allowCancel: false
+          });
+          throw new Error('401 Unauthorized');
+        }
+        let detail = res.statusText;
+        try {
+          const j = await res.json();
+          if (j.error) detail = j.error;
+        } catch (_) {}
+        throw new Error(`${res.status} ${detail}`);
+      }
+      if (res.status === 204) return null;
+      return res.json();
+    }
+
+    function formatTime(ts) {
+      if (!ts) return '';
+      const d = new Date(ts);
+      return isNaN(d) ? ts : d.toISOString();
+    }
+
+    function truncate(s, n) {
+      if (!s || s.length <= n) return s || '';
+      return s.slice(0, n) + '…';
+    }
+
+    function escapeHtml(s) {
+      return String(s)
+        .replace(/&/g, '&amp;')
+        .replace(/</g, '&lt;')
+        .replace(/>/g, '&gt;')
+        .replace(/"/g, '&quot;')
+        .replace(/'/g, '&#39;');
+    }
+
+    async function loadSubmissions() {
+      showError('');
+      try {
+        const search = $('#search').value.trim();
+        const path = search ? `/api?search=${encodeURIComponent(search)}` : '/api';
+        submissions = await api('GET', path);
+        renderList();
+      } catch (e) {
+        showError('Failed to load submissions: ' + e.message);
+      }
+    }
+
+    function compare(a, b) {
+      let av = a[sortKey];
+      let bv = b[sortKey];
+      if (sortKey === 'id' || sortKey === 'exit_code' || sortKey === 'duration_ms') {
+        av = Number(av); bv = Number(bv);
+      } else if (sortKey === 'submitted_at') {
+        av = new Date(av).getTime(); bv = new Date(bv).getTime();
+      } else {
+        av = String(av).toLowerCase(); bv = String(bv).toLowerCase();
+      }
+      if (av < bv) return -sortDir;
+      if (av > bv) return sortDir;
+      return 0;
+    }
+
+    function renderList() {
+      const tbody = $('#submissions-body');
+      const list = submissions.slice().sort(compare);
+      $('#count').textContent = list.length + ' submission' + (list.length === 1 ? '' : 's');
+      tbody.innerHTML = '';
+      if (!list.length) {
+        tbody.innerHTML = '<tr><td colspan="9">No submissions.</td></tr>';
+        return;
+      }
+      for (const s of list) {
+        const tr = document.createElement('tr');
+        const fileLink = s.file_id ? `<a href='/admin/uploads/${encodeURIComponent(s.file_id)}'>${escapeHtml(truncate(s.file_id, 8))}</a>` : escapeHtml(truncate(s.file_id, 8));
+        tr.innerHTML = `
+          <td class='mono'>${escapeHtml(s.id)}</td>
+          <td class='mono' title='${escapeHtml(s.file_id)}'>${fileLink}</td>
+          <td>${escapeHtml(s.file_name || '')}</td>
+          <td class='mono'>${escapeHtml(s.app_id)}</td>
+          <td class='wrap'>${escapeHtml(s.tracking_url || '')}</td>
+          <td>${escapeHtml(s.cluster_id)}</td>
+          <td>${escapeHtml(s.exit_code)}</td>
+          <td>${escapeHtml(s.duration_ms)}</td>
+          <td>${escapeHtml(formatTime(s.submitted_at))}</td>
+        `;
+        tbody.appendChild(tr);
+      }
+    }
+
+    function handleSort(e) {
+      const key = e.target.dataset.key;
+      if (!key) return;
+      if (sortKey === key) {
+        sortDir = -sortDir;
+      } else {
+        sortKey = key;
+        sortDir = key === 'submitted_at' ? -1 : 1;
+      }
+      document.querySelectorAll('th.sortable').forEach(th => {
+        th.textContent = th.textContent.replace(/ [▲▼]$/, '');
+      });
+      const marker = sortDir > 0 ? ' ▲' : ' ▼';
+      e.target.textContent = e.target.textContent.replace(/ [▲▼]$/, '') + marker;
+      renderList();
+    }
+
+    $('#search').addEventListener('input', loadSubmissions);
+    $('#refresh').addEventListener('click', loadSubmissions);
+    $('#set-token').addEventListener('click', () => openTokenModal({ allowCancel: true }));
+    $('#logout').addEventListener('click', () => {
+      localStorage.removeItem('adminToken');
+      openTokenModal({ error: 'Token cleared. Enter a new token to continue.', allowCancel: false });
+    });
+    $('#token-form').addEventListener('submit', saveTokenFromModal);
+    $('#modal-cancel').addEventListener('click', closeTokenModal);
+    document.querySelectorAll('th.sortable').forEach(th => th.addEventListener('click', handleSort));
+
+    if (!localStorage.getItem('adminToken')) {
+      openTokenModal({ allowCancel: false });
+    } else {
+      loadSubmissions();
+    }
+  </script>
+</body>
+</html>
@@ -106,6 +106,7 @@
     <nav>
       <a href='/admin'>Clusters</a>
       <a href='/admin/uploads' class='active'>Uploads</a>
+        <a href='/admin/submissions'>Submissions</a>
     </nav>
    </div>
    <div class='auth'>
@@ -23,10 +23,13 @@ import (
 // application application_xxx" line and a few progress lines.
 const stdoutCap = 10 << 20

-// appIDPattern matches the YARN line printed on successful submission:
-//
-//	"Submitted application application_12345_0001"
-var appIDPattern = regexp.MustCompile(`application_\d+_\d+`)
+// submittedAppPattern matches the explicit success line, including the
+// application id captured in group 1. This mirrors the Python reference.
+var submittedAppPattern = regexp.MustCompile(`Submitted application (\S+)`)
+
+// trackingURLPattern matches "tracking URL: <url>" on stderr. The URL tail
+// is used as a fallback source for the application id.
+var trackingURLPattern = regexp.MustCompile(`tracking URL:\s+(\S+)`)

 // SparkSubmitOpts is the resolved command line for a single spark-submit run.
 //
@@ -44,6 +47,7 @@ type SparkSubmitOpts struct {
 // Result is what spark_submit returns to the LLM.
 type Result struct {
 	AppID       string `json:"app_id,omitempty"`
+	TrackingURL string `json:"tracking_url,omitempty"`
 	ExitCode    int    `json:"exit_code"`
 	StdoutTail  string `json:"stdout_tail"`
 	StderrTail  string `json:"stderr_tail"`
@@ -90,27 +94,70 @@ func Run(ctx context.Context, opts SparkSubmitOpts) (*Result, error) {
 		DurationMS: dur.Milliseconds(),
 	}

-	// app_id: prefer stdout, fall back to stderr. Both are searched only on
-	// the tail (cap-bounded) so we don't pay for the full output.
-	if id := matchAppID(res.StdoutTail); id != "" {
-		res.AppID = id
-	} else if id := matchAppID(res.StderrTail); id != "" {
-		res.AppID = id
-	}
+	appID, trackingURL, ok := parseSparkSubmitOutput(res.StdoutTail, res.StderrTail)
+	res.AppID = appID
+	res.TrackingURL = trackingURL

+	if !ok {
+		return res, fmt.Errorf("executor: spark-submit ran but no application_id in output (exit_code=%d)", res.ExitCode)
+	}
 	if err != nil {
-		// Wrap the underlying error but keep the parsed result so the LLM
-		// can see exit_code and stderr even on failure.
 		return res, fmt.Errorf("executor: spark-submit failed: %w", err)
 	}
 	return res, nil
 }

-func matchAppID(s string) string {
-	if s == "" {
+// parseSparkSubmitOutput extracts the application id and tracking URL from
+// spark-submit output. It mirrors the Python reference implementation: first
+// look for "Submitted application <id>", then fall back to "tracking URL:",
+// deriving the id from the last path segment. The returned ok is false when
+// no application id could be determined.
+func parseSparkSubmitOutput(stdout, stderr string) (appID, trackingURL string, ok bool) {
+	// Stage 1: explicit success line, prefer stdout then stderr.
+	var trackingFound bool
+	if m := submittedAppPattern.FindStringSubmatch(stdout); m != nil {
+		appID = m[1]
+		ok = true
+	} else if m := submittedAppPattern.FindStringSubmatch(stderr); m != nil {
+		appID = m[1]
+		ok = true
+	}
+
+	// Stage 2: tracking URL fallback (stderr only, like the Python reference).
+	if !ok {
+		if m := trackingURLPattern.FindStringSubmatch(stderr); m != nil {
+			trackingURL = m[1]
+			trackingFound = true
+			appID = extractAppIDFromTrackingURL(trackingURL)
+			ok = appID != ""
+			if !ok {
+				trackingURL = ""
+			}
+		}
+	}
+
+	// Tracking URL is independent: if present anywhere on stderr, capture it.
+	if trackingURL == "" && !trackingFound {
+		if m := trackingURLPattern.FindStringSubmatch(stderr); m != nil {
+			trackingURL = m[1]
+		}
+	}
+	return
+}
+
+// extractAppIDFromTrackingURL returns the last path segment if it starts with
+// "application_", otherwise an empty string.
+func extractAppIDFromTrackingURL(rawURL string) string {
+	trimmed := strings.TrimRight(rawURL, "/")
+	idx := strings.LastIndex(trimmed, "/")
+	if idx < 0 {
 		return ""
 	}
-	return appIDPattern.FindString(s)
+	tail := trimmed[idx+1:]
+	if strings.HasPrefix(tail, "application_") {
+		return tail
+	}
+	return ""
 }

 // exitCodeFromErr returns 0 on nil, the process's exit code on *exec.ExitError,
@@ -0,0 +1,79 @@
+package executor
+
+import "testing"
+
+func TestParseAppID_FromStdout(t *testing.T) {
+	stdout := "some prefix\nSubmitted application application_123_456\n"
+	stderr := "random log\n"
+	appID, trackingURL, ok := parseSparkSubmitOutput(stdout, stderr)
+	if !ok {
+		t.Fatal("expected ok")
+	}
+	if appID != "application_123_456" {
+		t.Errorf("appID=%q, want application_123_456", appID)
+	}
+	if trackingURL != "" {
+		t.Errorf("trackingURL=%q, want empty", trackingURL)
+	}
+}
+
+func TestParseAppID_FromStderr(t *testing.T) {
+	stdout := ""
+	stderr := "Submitted application application_123_456\n"
+	appID, trackingURL, ok := parseSparkSubmitOutput(stdout, stderr)
+	if !ok {
+		t.Fatal("expected ok")
+	}
+	if appID != "application_123_456" {
+		t.Errorf("appID=%q, want application_123_456", appID)
+	}
+	if trackingURL != "" {
+		t.Errorf("trackingURL=%q, want empty", trackingURL)
+	}
+}
+
+func TestParseAppID_FromTrackingURL(t *testing.T) {
+	stdout := ""
+	stderr := "Log line\ntracking URL: http://rm.example.com:8088/proxy/application_123_456/\n"
+	appID, trackingURL, ok := parseSparkSubmitOutput(stdout, stderr)
+	if !ok {
+		t.Fatal("expected ok")
+	}
+	if appID != "application_123_456" {
+		t.Errorf("appID=%q, want application_123_456", appID)
+	}
+	wantURL := "http://rm.example.com:8088/proxy/application_123_456/"
+	if trackingURL != wantURL {
+		t.Errorf("trackingURL=%q, want %q", trackingURL, wantURL)
+	}
+}
+
+func TestParseAppID_NoAppID(t *testing.T) {
+	stdout := ""
+	stderr := "some random log"
+	appID, trackingURL, ok := parseSparkSubmitOutput(stdout, stderr)
+	if ok {
+		t.Fatal("expected not ok")
+	}
+	if appID != "" {
+		t.Errorf("appID=%q, want empty", appID)
+	}
+	if trackingURL != "" {
+		t.Errorf("trackingURL=%q, want empty", trackingURL)
+	}
+}
+
+func TestParseAppID_BadTrackingURL(t *testing.T) {
+	stdout := ""
+	stderr := "tracking URL: http://rm.example.com:8088/proxy/not-an-app/\n"
+	appID, trackingURL, ok := parseSparkSubmitOutput(stdout, stderr)
+	if ok {
+		t.Fatal("expected not ok")
+	}
+	if appID != "" {
+		t.Errorf("appID=%q, want empty", appID)
+	}
+	if trackingURL != "" {
+		t.Errorf("trackingURL=%q, want empty", trackingURL)
+	}
+}
@@ -30,7 +30,7 @@ func TestRun_FakeBinary(t *testing.T) {

 	res, err := Run(ctx, SparkSubmitOpts{
 		Binary:  "/bin/echo",
-		Args:    []string{"--master", "yarn; rm -rf /tmp/this-should-not-exist"},
+		Args:    []string{"--master", "yarn; rm -rf /tmp/this-should-not-exist", "Submitted application application_12345_0001"},
 		Timeout: 3 * time.Second,
 	})
 	if err != nil {
@@ -43,8 +43,8 @@ func TestRun_FakeBinary(t *testing.T) {
 	if !strings.Contains(res.StdoutTail, "yarn; rm -rf /tmp/this-should-not-exist") {
 		t.Errorf("stdout missing literal arg, got: %q", res.StdoutTail)
 	}
-	if res.AppID != "" {
-		t.Errorf("AppID should be empty for echo, got %q", res.AppID)
+	if res.AppID != "application_12345_0001" {
+		t.Errorf("AppID = %q, want application_12345_0001", res.AppID)
 	}
 }

@@ -66,22 +66,26 @@ func TestRun_NonZeroExit(t *testing.T) {
 	}
 }

-// TestMatchAppID locks down the regex so a YARN output tweak doesn't
+// TestSubmittedAppPattern locks down the regex so a YARN output tweak doesn't
 // silently break app_id extraction.
-func TestMatchAppID(t *testing.T) {
+func TestSubmittedAppPattern(t *testing.T) {
 	cases := []struct {
 		in, want string
 	}{
 		{"Submitted application application_12345_0001", "application_12345_0001"},
-		{"... some prefix application_999_42 ... tail", "application_999_42"},
+		{"Submitted application application_999_42 extra", "application_999_42"},
 		{"no app id here", ""},
 		{"", ""},
-		{"application_1_2 application_3_4", "application_1_2"}, // first match
+		{"Submitted application app_1_2 Submitted application app_3_4", "app_1_2"}, // first match
 	}
 	for _, tc := range cases {
-		got := matchAppID(tc.in)
+		m := submittedAppPattern.FindStringSubmatch(tc.in)
+		var got string
+		if m != nil {
+			got = m[1]
+		}
 		if got != tc.want {
-			t.Errorf("matchAppID(%q) = %q, want %q", tc.in, got, tc.want)
+			t.Errorf("submittedAppPattern(%q) = %q, want %q", tc.in, got, tc.want)
 		}
 	}
 }
@@ -16,6 +16,7 @@ type Deps struct {
 	Logger             *slog.Logger
 	AuditRepo          *audit.Repo
 	ClusterRepo        *storage.ClusterRepo
+	SubmissionRepo     *storage.SubmissionRepo
 	SparkSubmitTimeout time.Duration
 	HTTPClient         *httpclient.Client
 	MaxResponseBytes   int64
@@ -25,7 +25,7 @@ import (

 // testDepsWithDataDir returns dependencies backed by an in-memory DB and a
 // temporary data directory.
-func testDepsWithDataDir(t *testing.T) (*Deps, *storage.ClusterRepo, *audit.Repo) {
+func testDepsWithDataDir(t *testing.T) (*Deps, *storage.ClusterRepo, *storage.SubmissionRepo, *audit.Repo) {
 	t.Helper()
 	db, err := storage.Open(":memory:")
 	if err != nil {
@@ -46,10 +46,11 @@ func testDepsWithDataDir(t *testing.T) (*Deps, *storage.ClusterRepo, *audit.Repo
 		}),
 		AuditRepo:        audit.NewRepo(db),
 		ClusterRepo:      db.Clusters(),
+		SubmissionRepo:   db.Submissions(),
 		MaxResponseBytes: 1 << 20,
 		DataDir:          dataDir,
 		UploadStore:      uploadStore,
-	}, db.Clusters(), audit.NewRepo(db)
+	}, db.Clusters(), db.Submissions(), audit.NewRepo(db)
 }

 // createCluster creates a cluster in the repository with the given fields.
@@ -100,7 +101,7 @@ func TestFetchURL(t *testing.T) {
 	}))
 	defer srv.Close()

-	deps, repo, _ := testDepsWithDataDir(t)
+	deps, repo, _, _ := testDepsWithDataDir(t)
 	createCluster(t, repo, &cluster.Cluster{
 		ID:           "cluster-a",
 		Name:         "Cluster A",
@@ -294,7 +295,7 @@ func TestFetchURL_RedirectPreservesAuth(t *testing.T) {
 	}))
 	defer server1.Close()

-	deps, repo, _ := testDepsWithDataDir(t)
+	deps, repo, _, _ := testDepsWithDataDir(t)
 	createCluster(t, repo, &cluster.Cluster{
 		ID:           "cluster-redirect",
 		Name:         "Redirect",
@@ -323,7 +324,7 @@ func TestFetchURL_RedirectPreservesAuth(t *testing.T) {
 }

 func TestUploadFile(t *testing.T) {
-	deps, _, auditRepo := testDepsWithDataDir(t)
+	deps, _, _, auditRepo := testDepsWithDataDir(t)

 	req := newToolRequest(UploadFileName, map[string]any{
 		"filename": "hello.txt",
@@ -387,7 +388,7 @@ func TestUploadFile(t *testing.T) {
 }

 func TestUploadFile_PathTraversal(t *testing.T) {
-	deps, _, _ := testDepsWithDataDir(t)
+	deps, _, _, _ := testDepsWithDataDir(t)

 	cases := []string{
 		"../../../etc/passwd",
@@ -418,7 +419,7 @@ func TestUploadFile_PathTraversal(t *testing.T) {
 }

 func TestUploadFile_Base64(t *testing.T) {
-	deps, _, _ := testDepsWithDataDir(t)
+	deps, _, _, _ := testDepsWithDataDir(t)

 	req := newToolRequest(UploadFileName, map[string]any{
 		"filename": "hello.bin",
@@ -6,10 +6,12 @@ import (
 	"log/slog"
 	"sort"
 	"strconv"
+	"time"

 	"github.com/mark3labs/mcp-go/mcp"

 	"spark-mcp-go/internal/executor"
+	"spark-mcp-go/internal/storage"
 )

 const SparkSubmitName = "spark_submit"
@@ -98,7 +100,8 @@ func (d *Deps) SparkSubmitHandler(ctx context.Context, req mcp.CallToolRequest)
 	// letting spark-submit produce a confusing FileNotFoundException later.
 	// Placed after all required-string parses so error messages surface in
 	// field order (cluster_id, master, ..., script_path, queue, ...).
-	if _, err := d.UploadStore.Validate(scriptPath); err != nil {
+	fileID, err := d.UploadStore.Validate(scriptPath)
+	if err != nil {
 		if d.Logger != nil {
 			d.Logger.Warn("spark_submit.unminted_path_rejected", slog.String("path", scriptPath), slog.String("err", err.Error()))
 		}
@@ -173,17 +176,11 @@ func (d *Deps) SparkSubmitHandler(ctx context.Context, req mcp.CallToolRequest)
 	})
 	if err != nil {
 		callLog.WithError(err)
-		// Even on error, result carries ExitCode/Stderr for the LLM.
 		if result != nil {
-			callLog.WithResult(map[string]any{
-				"binary":      binary,
-				"argv":        cmd,
-				"app_id":      result.AppID,
-				"exit_code":   result.ExitCode,
-				"duration_ms": result.DurationMS,
-			})
+			callLog.WithResult(sparkSubmitResultLog(binary, cmd, result))
 			return textResult(encodeJSON(map[string]any{
 				"app_id":       result.AppID,
+				"tracking_url": result.TrackingURL,
 				"exit_code":    result.ExitCode,
 				"stdout_tail":  result.StdoutTail,
 				"stderr_tail":  result.StderrTail,
@@ -194,14 +191,43 @@ func (d *Deps) SparkSubmitHandler(ctx context.Context, req mcp.CallToolRequest)
 		return errResult("spark_submit: " + err.Error()), nil
 	}

-	callLog.WithResult(map[string]any{
+	if result.AppID != "" && d.SubmissionRepo != nil {
+		sub := storage.Submission{
+			FileID:      fileID,
+			AppID:       result.AppID,
+			TrackingURL: result.TrackingURL,
+			ClusterID:   clusterID,
+			ExitCode:    result.ExitCode,
+			DurationMS:  result.DurationMS,
+			SubmittedAt: time.Now(),
+		}
+		if createErr := d.SubmissionRepo.Create(ctx, sub); createErr != nil {
+			if d.Logger != nil {
+				d.Logger.Warn("spark_submit.persist_failed", slog.String("file_id", fileID), slog.String("app_id", result.AppID), slog.String("err", createErr.Error()))
+			} else {
+				slog.Default().Warn("spark_submit.persist_failed", slog.String("file_id", fileID), slog.String("app_id", result.AppID), slog.String("err", createErr.Error()))
+			}
+		}
+	}
+
+	callLog.WithResult(sparkSubmitResultLog(binary, cmd, result))
+	return textResult(encodeJSON(result)), nil
+}
+
+// sparkSubmitResultLog builds the standard result map used for both logging
+// and the admin audit trail.
+func sparkSubmitResultLog(binary string, cmd []string, result *executor.Result) map[string]any {
+	m := map[string]any{
 		"binary":      binary,
 		"argv":        cmd,
 		"app_id":      result.AppID,
 		"exit_code":   result.ExitCode,
 		"duration_ms": result.DurationMS,
-	})
-	return textResult(encodeJSON(result)), nil
+	}
+	if result.TrackingURL != "" {
+		m["tracking_url"] = result.TrackingURL
+	}
+	return m
 }

 // SparkSubmitCommandOpts holds the structured arguments used to build the
@@ -10,11 +10,15 @@ import (

 	"github.com/mark3labs/mcp-go/mcp"

+	"spark-mcp-go/internal/audit"
 	"spark-mcp-go/internal/cluster"
+	"spark-mcp-go/internal/httpclient"
+	"spark-mcp-go/internal/storage"
+	"spark-mcp-go/internal/uploads"
 )

 func TestSparkSubmit_StructuredCommand(t *testing.T) {
-	deps, repo, _ := testDepsWithDataDir(t)
+	deps, repo, _, _ := testDepsWithDataDir(t)
 	deps.SparkSubmitTimeout = 5 * time.Second
 	store := deps.UploadStore

@@ -25,7 +29,7 @@ func TestSparkSubmit_StructuredCommand(t *testing.T) {

 	echoed := filepath.Join(t.TempDir(), "echoed")
 	echoScript := filepath.Join(t.TempDir(), "echo-args.sh")
-	if err := os.WriteFile(echoScript, []byte("#!/bin/sh\nfor arg do\n  echo \"$arg\" >> \"$ECHO_FILE\"\ndone\n"), 0o755); err != nil {
+	if err := os.WriteFile(echoScript, []byte("#!/bin/sh\nfor arg do\n  echo \"$arg\" >> \"$ECHO_FILE\"\ndone\necho 'Submitted application application_test_0001'\n"), 0o755); err != nil {
 		t.Fatalf("write echo script: %v", err)
 	}

@@ -97,8 +101,107 @@ func TestSparkSubmit_StructuredCommand(t *testing.T) {
 	}
 }

+func TestSparkSubmit_PersistsSubmission(t *testing.T) {
+	// Open a dedicated DB so we can insert upload_files metadata for the join.
+	db, err := storage.Open(":memory:")
+	if err != nil {
+		t.Fatalf("open db: %v", err)
+	}
+	t.Cleanup(func() { _ = db.Close() })
+
+	dataDir := t.TempDir()
+	store, err := uploads.New(filepath.Join(dataDir, "uploads"))
+	if err != nil {
+		t.Fatalf("new upload store: %v", err)
+	}
+
+	deps := &Deps{
+		HTTPClient:         httpclient.New(httpclient.Config{Timeout: 5 * time.Second, MaxResponseBytes: 1 << 20}),
+		AuditRepo:          audit.NewRepo(db),
+		ClusterRepo:        db.Clusters(),
+		SubmissionRepo:     db.Submissions(),
+		MaxResponseBytes:   1 << 20,
+		DataDir:            dataDir,
+		UploadStore:        store,
+		SparkSubmitTimeout: 5 * time.Second,
+	}
+	repo := db.Clusters()
+	subRepo := db.Submissions()
+
+	fileID, _, _, _, absPath, err := store.Save([]byte("print('hi')\n"), "hello.py")
+	if err != nil {
+		t.Fatalf("save upload: %v", err)
+	}
+	if err := db.Uploads().Create(context.Background(), fileID, "hello.py", 14, "dummy-sha", time.Now()); err != nil {
+		t.Fatalf("create upload record: %v", err)
+	}
+
+	echoScript := filepath.Join(t.TempDir(), "echo-submit.sh")
+	if err := os.WriteFile(echoScript, []byte("#!/bin/sh\necho 'tracking URL: http://rm.example.com:8088/proxy/application_persist_0001/' >&2\necho 'Submitted application application_persist_0001'\n"), 0o755); err != nil {
+		t.Fatalf("write echo script: %v", err)
+	}
+
+	createCluster(t, repo, &cluster.Cluster{
+		ID:                    "cluster-persist",
+		Name:                  "Persist",
+		IsActive:              true,
+		AuthType:              cluster.AuthNone,
+		SparkSubmitExecuteBin: echoScript,
+	})
+
+	req := newToolRequest(SparkSubmitName, map[string]any{
+		"cluster_id":      "cluster-persist",
+		"master":          "yarn",
+		"deploy_mode":     "cluster",
+		"script_path":     absPath,
+		"queue":           "default",
+		"executor_memory": "2G",
+		"executor_cores":  1,
+		"num_executors":   4,
+	})
+	res, err := deps.SparkSubmitHandler(context.Background(), req)
+	if err != nil {
+		t.Fatalf("handler error: %v", err)
+	}
+	if res.IsError {
+		t.Fatalf("unexpected error result: %v", res.Content)
+	}
+
+	payload := resultJSON(t, res)
+	if payload["app_id"] != "application_persist_0001" {
+		t.Errorf("app_id=%v, want application_persist_0001", payload["app_id"])
+	}
+	if payload["tracking_url"] != "http://rm.example.com:8088/proxy/application_persist_0001/" {
+		t.Errorf("tracking_url=%v, want tracking URL", payload["tracking_url"])
+	}
+
+	subs, err := subRepo.List(context.Background(), storage.SubmissionFilter{Limit: 10})
+	if err != nil {
+		t.Fatalf("list submissions: %v", err)
+	}
+	if len(subs) != 1 {
+		t.Fatalf("got %d submissions, want 1", len(subs))
+	}
+	s := subs[0]
+	if s.FileID != fileID {
+		t.Errorf("file_id=%q, want %q", s.FileID, fileID)
+	}
+	if s.AppID != "application_persist_0001" {
+		t.Errorf("app_id=%q, want application_persist_0001", s.AppID)
+	}
+	if s.TrackingURL != "http://rm.example.com:8088/proxy/application_persist_0001/" {
+		t.Errorf("tracking_url=%q, want tracking URL", s.TrackingURL)
+	}
+	if s.ClusterID != "cluster-persist" {
+		t.Errorf("cluster_id=%q, want cluster-persist", s.ClusterID)
+	}
+	if s.FileName != "hello.py" {
+		t.Errorf("file_name=%q, want hello.py", s.FileName)
+	}
+}
+
 func TestSparkSubmit_MissingRequiredField(t *testing.T) {
-	deps, _, _ := testDepsWithDataDir(t)
+	deps, _, _, _ := testDepsWithDataDir(t)
 	store := deps.UploadStore
 	_, _, _, _, mintedPath, err := store.Save([]byte("# dummy\n"), "dummy.py")
 	if err != nil {
@@ -131,7 +234,7 @@ func TestSparkSubmit_MissingRequiredField(t *testing.T) {
 }

 func TestSparkSubmit_BadSparkConfValue(t *testing.T) {
-	deps, _, _ := testDepsWithDataDir(t)
+	deps, _, _, _ := testDepsWithDataDir(t)
 	store := deps.UploadStore
 	_, _, _, _, mintedPath, err := store.Save([]byte("# dummy\n"), "dummy.py")
 	if err != nil {
@@ -166,7 +269,7 @@ func TestSparkSubmit_BadSparkConfValue(t *testing.T) {
 }

 func TestSparkSubmit_RejectsNonMintedPath(t *testing.T) {
-	deps, _, _ := testDepsWithDataDir(t)
+	deps, _, _, _ := testDepsWithDataDir(t)

 	unminted := filepath.Join(t.TempDir(), "unminted.py")
 	if err := os.WriteFile(unminted, []byte("print('not from upload_file')\n"), 0o644); err != nil {
@@ -200,7 +303,7 @@ func TestSparkSubmit_RejectsNonMintedPath(t *testing.T) {
 }

 func TestSparkSubmit_EmptyMaster(t *testing.T) {
-	deps, _, _ := testDepsWithDataDir(t)
+	deps, _, _, _ := testDepsWithDataDir(t)
 	store := deps.UploadStore
 	_, _, _, _, mintedPath, err := store.Save([]byte("# dummy\n"), "dummy.py")
 	if err != nil {
@@ -47,6 +47,19 @@ CREATE TABLE IF NOT EXISTS upload_files (
  uploaded_at  INTEGER NOT NULL
 );
 CREATE INDEX IF NOT EXISTS idx_uploads_ts ON upload_files(uploaded_at DESC);
+CREATE TABLE IF NOT EXISTS spark_submissions (
+  id            INTEGER PRIMARY KEY AUTOINCREMENT,
+  file_id       TEXT NOT NULL,
+  app_id        TEXT NOT NULL,
+  tracking_url  TEXT,
+  cluster_id    TEXT NOT NULL,
+  exit_code     INTEGER NOT NULL,
+  duration_ms   INTEGER NOT NULL,
+  submitted_at  INTEGER NOT NULL
+);
+CREATE INDEX IF NOT EXISTS idx_submissions_app_id ON spark_submissions(app_id);
+CREATE INDEX IF NOT EXISTS idx_submissions_submitted_at ON spark_submissions(submitted_at DESC);
+CREATE INDEX IF NOT EXISTS idx_submissions_file_id ON spark_submissions(file_id);
 `

 // DB is the storage handle.
@@ -0,0 +1,194 @@
+package storage
+
+import (
+	"context"
+	"database/sql"
+	"errors"
+	"fmt"
+	"strings"
+	"time"
+)
+
+// Submission is one persisted spark_submit invocation.
+type Submission struct {
+	ID          int64     `json:"id"`
+	FileID      string    `json:"file_id"`
+	AppID       string    `json:"app_id"`
+	TrackingURL string    `json:"tracking_url,omitempty"`
+	ClusterID   string    `json:"cluster_id"`
+	ExitCode    int       `json:"exit_code"`
+	DurationMS  int64     `json:"duration_ms"`
+	SubmittedAt time.Time `json:"submitted_at"`
+}
+
+// SubmissionWithFile joins a submission with the original upload file name.
+// FileName is empty when the referenced upload has been deleted.
+type SubmissionWithFile struct {
+	Submission
+	FileName string `json:"file_name"`
+}
+
+// SubmissionFilter controls the rows returned by SubmissionRepo.List.
+type SubmissionFilter struct {
+	Search    string
+	FileID    string
+	ClusterID string
+	AppID     string
+	Limit     int
+	Offset    int
+}
+
+// SubmissionRepo provides CRUD operations for spark_submissions.
+type SubmissionRepo struct {
+	db *DB
+}
+
+// Submissions returns a repository bound to this DB handle.
+func (d *DB) Submissions() *SubmissionRepo {
+	return &SubmissionRepo{db: d}
+}
+
+// Create inserts a new submission record.
+func (r *SubmissionRepo) Create(ctx context.Context, s Submission) error {
+	if s.SubmittedAt.IsZero() {
+		s.SubmittedAt = time.Now()
+	}
+	_, err := r.db.sqlDB.ExecContext(ctx, `
+		INSERT INTO spark_submissions (file_id, app_id, tracking_url, cluster_id, exit_code, duration_ms, submitted_at)
+		VALUES (?, ?, ?, ?, ?, ?, ?)`,
+		s.FileID, s.AppID, s.TrackingURL, s.ClusterID, s.ExitCode, s.DurationMS, s.SubmittedAt.UnixNano(),
+	)
+	if err != nil {
+		return fmt.Errorf("storage: create submission: %w", err)
+	}
+	return nil
+}
+
+// Get retrieves a submission by id.
+func (r *SubmissionRepo) Get(ctx context.Context, id int64) (Submission, error) {
+	row := r.db.sqlDB.QueryRowContext(ctx, `
+		SELECT id, file_id, app_id, tracking_url, cluster_id, exit_code, duration_ms, submitted_at
+		FROM spark_submissions
+		WHERE id = ?`, id)
+	s, err := scanSubmission(row.Scan)
+	if errors.Is(err, sql.ErrNoRows) {
+		return Submission{}, ErrNotFound
+	}
+	if err != nil {
+		return Submission{}, fmt.Errorf("storage: get submission %d: %w", id, err)
+	}
+	return s, nil
+}
+
+// List returns submissions ordered by submitted_at descending.
+//
+// Search matches app_id or the joined upload file name. Empty filter strings
+// mean no restriction. Limit defaults to 100 and is capped at 500.
+func (r *SubmissionRepo) List(ctx context.Context, filter SubmissionFilter) ([]SubmissionWithFile, error) {
+	if filter.Limit <= 0 {
+		filter.Limit = 100
+	}
+	if filter.Limit > 500 {
+		filter.Limit = 500
+	}
+	if filter.Offset < 0 {
+		filter.Offset = 0
+	}
+
+	args := []any{}
+	var where []string
+	if filter.Search != "" {
+		like := "%" + filter.Search + "%"
+		args = append(args, like, like)
+		where = append(where, "(s.app_id LIKE ? OR COALESCE(u.name, '') LIKE ?)")
+	}
+	if filter.FileID != "" {
+		args = append(args, filter.FileID)
+		where = append(where, "s.file_id = ?")
+	}
+	if filter.ClusterID != "" {
+		args = append(args, filter.ClusterID)
+		where = append(where, "s.cluster_id = ?")
+	}
+	if filter.AppID != "" {
+		args = append(args, filter.AppID)
+		where = append(where, "s.app_id = ?")
+	}
+
+	// WHERE args are in the order the conditions were appended; append LIMIT/OFFSET last.
+	args = append(args, filter.Limit, filter.Offset)
+
+	q := `SELECT s.id, s.file_id, s.app_id, s.tracking_url, s.cluster_id, s.exit_code, s.duration_ms, s.submitted_at, COALESCE(u.name, '') AS file_name
+		FROM spark_submissions s
+		LEFT JOIN upload_files u ON s.file_id = u.file_id`
+	if len(where) > 0 {
+		q += "\n\t\tWHERE " + strings.Join(where, " AND ")
+	}
+	q += "\n\t\tORDER BY s.submitted_at DESC\n\t\tLIMIT ? OFFSET ?"
+
+	rows, err := r.db.sqlDB.QueryContext(ctx, q, args...)
+	if err != nil {
+		return nil, fmt.Errorf("storage: list submissions: %w", err)
+	}
+	defer rows.Close()
+
+	var out []SubmissionWithFile
+	for rows.Next() {
+		swf, err := scanSubmissionWithFile(rows.Scan)
+		if err != nil {
+			return nil, fmt.Errorf("storage: list submissions: %w", err)
+		}
+		out = append(out, swf)
+	}
+	if err := rows.Err(); err != nil {
+		return nil, fmt.Errorf("storage: list submissions: %w", err)
+	}
+	if out == nil {
+		out = []SubmissionWithFile{}
+	}
+	return out, nil
+}
+
+// Delete removes a submission by id.
+func (r *SubmissionRepo) Delete(ctx context.Context, id int64) error {
+	res, err := r.db.sqlDB.ExecContext(ctx, `DELETE FROM spark_submissions WHERE id = ?`, id)
+	if err != nil {
+		return fmt.Errorf("storage: delete submission %d: %w", id, err)
+	}
+	n, err := res.RowsAffected()
+	if err != nil {
+		return fmt.Errorf("storage: delete submission %d: rows affected: %w", id, err)
+	}
+	if n == 0 {
+		return ErrNotFound
+	}
+	return nil
+}
+
+func scanSubmission(scan scanFunc) (Submission, error) {
+	var s Submission
+	var trackingURL sql.NullString
+	var submittedAt int64
+	if err := scan(&s.ID, &s.FileID, &s.AppID, &trackingURL, &s.ClusterID, &s.ExitCode, &s.DurationMS, &submittedAt); err != nil {
+		return Submission{}, err
+	}
+	if trackingURL.Valid {
+		s.TrackingURL = trackingURL.String
+	}
+	s.SubmittedAt = time.Unix(0, submittedAt)
+	return s, nil
+}
+
+func scanSubmissionWithFile(scan scanFunc) (SubmissionWithFile, error) {
+	var swf SubmissionWithFile
+	var trackingURL sql.NullString
+	var submittedAt int64
+	if err := scan(&swf.ID, &swf.FileID, &swf.AppID, &trackingURL, &swf.ClusterID, &swf.ExitCode, &swf.DurationMS, &submittedAt, &swf.FileName); err != nil {
+		return SubmissionWithFile{}, err
+	}
+	if trackingURL.Valid {
+		swf.TrackingURL = trackingURL.String
+	}
+	swf.SubmittedAt = time.Unix(0, submittedAt)
+	return swf, nil
+}
@@ -0,0 +1,151 @@
+package storage
+
+import (
+	"context"
+	"testing"
+	"time"
+)
+
+func setupSubmissionTest(t *testing.T) (*SubmissionRepo, *UploadRepo, context.Context) {
+	db, err := Open(":memory:")
+	if err != nil {
+		t.Fatalf("open: %v", err)
+	}
+	t.Cleanup(func() { _ = db.Close() })
+	return db.Submissions(), db.Uploads(), context.Background()
+}
+
+func TestSubmissionRepo_RoundTrip(t *testing.T) {
+	repo, uploadRepo, ctx := setupSubmissionTest(t)
+
+	if err := uploadRepo.Create(ctx, "00000000000000000000000000000001", "alpha.py", 10, "deadbeef", time.Now()); err != nil {
+		t.Fatalf("create upload: %v", err)
+	}
+
+	s := Submission{
+		FileID:      "00000000000000000000000000000001",
+		AppID:       "application_1_0001",
+		TrackingURL: "http://rm.example.com:8088/proxy/application_1_0001/",
+		ClusterID:   "cluster-a",
+		ExitCode:    0,
+		DurationMS:  1234,
+		SubmittedAt: time.Unix(0, time.Now().UnixNano()),
+	}
+	if err := repo.Create(ctx, s); err != nil {
+		t.Fatalf("create: %v", err)
+	}
+
+	list, err := repo.List(ctx, SubmissionFilter{Limit: 10})
+	if err != nil {
+		t.Fatalf("list: %v", err)
+	}
+	if len(list) != 1 {
+		t.Fatalf("len(list)=%d, want 1", len(list))
+	}
+	swf := list[0]
+	if swf.FileName != "alpha.py" {
+		t.Errorf("file_name=%q, want alpha.py", swf.FileName)
+	}
+	if swf.AppID != s.AppID {
+		t.Errorf("app_id=%q, want %q", swf.AppID, s.AppID)
+	}
+
+	id := swf.ID
+	got, err := repo.Get(ctx, id)
+	if err != nil {
+		t.Fatalf("get: %v", err)
+	}
+	if got.ID != id || got.AppID != s.AppID {
+		t.Errorf("got=%+v, want id=%d app_id=%s", got, id, s.AppID)
+	}
+
+	if err := repo.Delete(ctx, id); err != nil {
+		t.Fatalf("delete: %v", err)
+	}
+	if _, err := repo.Get(ctx, id); !isErrNotFound(err) {
+		t.Errorf("expected ErrNotFound after delete, got %v", err)
+	}
+}
+
+func TestSubmissionRepo_SearchByAppID(t *testing.T) {
+	repo, uploadRepo, ctx := setupSubmissionTest(t)
+
+	if err := uploadRepo.Create(ctx, "00000000000000000000000000000001", "alpha.py", 10, "deadbeef", time.Now()); err != nil {
+		t.Fatalf("create upload alpha: %v", err)
+	}
+	if err := uploadRepo.Create(ctx, "00000000000000000000000000000002", "beta.py", 10, "cafebabe", time.Now()); err != nil {
+		t.Fatalf("create upload beta: %v", err)
+	}
+	if err := repo.Create(ctx, Submission{FileID: "00000000000000000000000000000001", AppID: "application_alpha_1", ClusterID: "c", ExitCode: 0, DurationMS: 1, SubmittedAt: time.Now()}); err != nil {
+		t.Fatalf("create alpha: %v", err)
+	}
+	if err := repo.Create(ctx, Submission{FileID: "00000000000000000000000000000002", AppID: "application_beta_1", ClusterID: "c", ExitCode: 0, DurationMS: 1, SubmittedAt: time.Now()}); err != nil {
+		t.Fatalf("create beta: %v", err)
+	}
+
+	list, err := repo.List(ctx, SubmissionFilter{Search: "alpha", Limit: 10})
+	if err != nil {
+		t.Fatalf("search: %v", err)
+	}
+	if len(list) != 1 || list[0].AppID != "application_alpha_1" || list[0].FileName != "alpha.py" {
+		t.Errorf("got %+v, want 1 alpha submission", list)
+	}
+}
+
+func TestSubmissionRepo_SearchByFileName(t *testing.T) {
+	db, err := Open(":memory:")
+	if err != nil {
+		t.Fatalf("open: %v", err)
+	}
+	defer db.Close()
+	repo := db.Submissions()
+	uploadRepo := db.Uploads()
+	ctx := context.Background()
+
+	if err := uploadRepo.Create(ctx, "00000000000000000000000000000001", "report.py", 10, "deadbeef", time.Now()); err != nil {
+		t.Fatalf("create upload: %v", err)
+	}
+	if err := repo.Create(ctx, Submission{FileID: "00000000000000000000000000000001", AppID: "application_x_1", ClusterID: "c", ExitCode: 0, DurationMS: 1, SubmittedAt: time.Now()}); err != nil {
+		t.Fatalf("create: %v", err)
+	}
+
+	list, err := repo.List(ctx, SubmissionFilter{Search: "report", Limit: 10})
+	if err != nil {
+		t.Fatalf("search: %v", err)
+	}
+	if len(list) != 1 || list[0].FileName != "report.py" {
+		t.Errorf("got %+v, want 1 report submission", list)
+	}
+}
+
+func TestSubmissionRepo_FilterByFileID(t *testing.T) {
+	db, err := Open(":memory:")
+	if err != nil {
+		t.Fatalf("open: %v", err)
+	}
+	defer db.Close()
+	repo := db.Submissions()
+	ctx := context.Background()
+
+	if err := repo.Create(ctx, Submission{FileID: "f1", AppID: "a1", ClusterID: "c", ExitCode: 0, DurationMS: 1, SubmittedAt: time.Now()}); err != nil {
+		t.Fatalf("create f1: %v", err)
+	}
+	if err := repo.Create(ctx, Submission{FileID: "f2", AppID: "a2", ClusterID: "c", ExitCode: 0, DurationMS: 1, SubmittedAt: time.Now()}); err != nil {
+		t.Fatalf("create f2: %v", err)
+	}
+
+	list, err := repo.List(ctx, SubmissionFilter{FileID: "f1", Limit: 10})
+	if err != nil {
+		t.Fatalf("list: %v", err)
+	}
+	if len(list) != 1 || list[0].FileID != "f1" {
+		t.Errorf("got %+v, want f1 submission", list)
+	}
+}
+
+func isErrNotFound(err error) bool {
+	if err == nil {
+		return false
+	}
+	return err.Error() == ErrNotFound.Error()
+}
@@ -68,7 +68,8 @@ func (r *UploadRepo) Get(ctx context.Context, fileID string) (UploadMeta, error)

 // List returns upload index records ordered by uploaded_at descending.
 //
-// If search is non-empty, name is filtered with a case-insensitive LIKE.
+// If search is non-empty, file_name or any associated spark_submit app_id is
+// filtered with a case-insensitive LIKE.
 // A limit of zero or less falls back to 100; values above 500 are capped.
 func (r *UploadRepo) List(ctx context.Context, search string, limit int) ([]UploadMeta, error) {
 	if limit <= 0 {
@@ -82,18 +83,19 @@ func (r *UploadRepo) List(ctx context.Context, search string, limit int) ([]Uplo
 	var err error
 	if search != "" {
 		rows, err = r.db.sqlDB.QueryContext(ctx, `
-			SELECT file_id, name, size, sha256, uploaded_at
-			FROM upload_files
-			WHERE name LIKE ?
-			ORDER BY uploaded_at DESC
+			SELECT DISTINCT u.file_id, u.name, u.size, u.sha256, u.uploaded_at
+			FROM upload_files u
+			LEFT JOIN spark_submissions s ON u.file_id = s.file_id
+			WHERE u.name LIKE ? OR s.app_id LIKE ?
+			ORDER BY u.uploaded_at DESC
 			LIMIT ?`,
-			"%"+search+"%", limit,
+			"%"+search+"%", "%"+search+"%", limit,
 		)
 	} else {
 		rows, err = r.db.sqlDB.QueryContext(ctx, `
-			SELECT file_id, name, size, sha256, uploaded_at
-			FROM upload_files
-			ORDER BY uploaded_at DESC
+			SELECT u.file_id, u.name, u.size, u.sha256, u.uploaded_at
+			FROM upload_files u
+			ORDER BY u.uploaded_at DESC
 			LIMIT ?`, limit)
 	}
 	if err != nil {
@@ -105,7 +105,7 @@ func run() error {
 		c.JSON(http.StatusOK, gin.H{"ok": true, "version": "0.0.0"})
 	})

-	admin.Mount(r, db.Clusters(), db.Uploads(), audit.NewRepo(db), &uploadStore, cfg.AdminTokens)
+	admin.Mount(r, db.Clusters(), db.Uploads(), db.Submissions(), audit.NewRepo(db), &uploadStore, cfg.AdminTokens)

 	mcpHandler, err := mcpsrv.Handler(&tools.Deps{
 		Logger:             logger.With("component", "mcp"),
@@ -119,6 +119,7 @@ func run() error {
 		MaxResponseBytes: cfg.MaxResponseBytes,
 		DataDir:          cfg.DataDir,
 		UploadStore:      uploadStore,
+		SubmissionRepo:   db.Submissions(),
 		AnalyzerThresholds: analyzer.Thresholds{
 			DataSkewRatio:       cfg.AnalyzerDataSkewRatio,
 			GCPressureRatio:     cfg.AnalyzerGCPressureRatio,
Author	SHA1	Message	Date
tao.chenandClaude	25f93dfb59	admin: submissions page, download endpoint, uploads search by app_id Co-Authored-By: Claude <noreply@anthropic.com>	2026-07-14 13:45:11 +08:00
tao.chenandClaude	573397dbb4	tools: persist spark_submit calls to SubmissionRepo Co-Authored-By: Claude <noreply@anthropic.com>	2026-07-14 13:45:07 +08:00
tao.chenandClaude	a2232cecc4	storage: add spark_submissions table and SubmissionRepo Co-Authored-By: Claude <noreply@anthropic.com>	2026-07-14 13:45:03 +08:00
tao.chenandClaude	d6e3952a4b	executor: parse tracking_url and error on missing application_id Co-Authored-By: Claude <noreply@anthropic.com>	2026-07-14 13:45:01 +08:00