fix: validate script_path exists + tell agent to call generate_job_file

Two problems with the prior prepare_submit_job flow:

  1. The agent could pass a script_path that only existed in its own
     context (LLM-generated code not yet on disk) or a path on the host
     filesystem that's invisible inside the container. The new check
     surfaces this as a 400 with a clear remediation hint instead of
     letting spark-submit fail later with an opaque FileNotFoundError -> 500.

  2. The container-isolation issue: any path the agent gives is interpreted
     inside the container. The two ways a file can legitimately exist there
     are (a) generate_job_file(code=...) just wrote it to
     SPARK_EXECUTOR_JOBS_DIR (the default ./data/jobs/ is the only
     gitignored dir that survives restarts), or (b) a host dir was
     mounted via -v. The error message spells both out so an agent can
     self-correct.

Implementation:
  - submit.py: new _check_script_path() that raises ValueError (-> 400)
    when the path is missing, empty, or a directory. Called in both
    prepare_submit_job and confirm_submit_job (defense in depth).
  - prepare_submit_job checks the connection FIRST (KeyError -> 404)
    before the script (ValueError -> 400), so an agent with both problems
    sees the more fundamental 'unknown connection' error first.
  - server.py / requests.py: route description and Pydantic field
    description spell out the generate_job_file pattern so an LLM
    reading the tool schema learns the right next step.

8 new tests; 8 existing tests adjusted to create real files (they used
synthetic /tmp/*.py paths that don't exist).

tests/integration/test_mcp_routes.py

@@ -76,12 +76,15 @@ def test_save_connection_accepts_dict_spark_conf_in_body():
     assert r.json() == {"name": "prod", "status": "SAVED"}
 
 
-def test_prepare_submit_job_works_via_body():
+def test_prepare_submit_job_works_via_body(tmp_path):
     c = TestClient(app)
     c.post("/save_connection", json={"name": "prod", "master": "yarn"})
+    # The script must exist (Stage 2 cleanup) — write a real file first.
+    script = tmp_path / "demo.py"
+    script.write_text("print('hi')\n")
     r = c.post(
         "/prepare_submit_job",
-        json={"connection": "prod", "script_path": "/tmp/demo.py", "queue": "research"},
+        json={"connection": "prod", "script_path": str(script), "queue": "research"},
     )
     assert r.status_code == 200, r.text
     body = r.json()
@@ -91,12 +94,29 @@ def test_prepare_submit_job_works_via_body():
     assert body["parameters"]["master"] == "yarn"  # snapshotted from connection
 
 
-def test_list_and_get_pending_job_roundtrip_via_body():
+def test_prepare_rejects_nonexistent_script_path_with_400():
+    """MCP clients should see a clean 400 with a remediation hint, not a 500."""
     c = TestClient(app)
     c.post("/save_connection", json={"name": "prod", "master": "yarn"})
+    r = c.post(
+        "/prepare_submit_job",
+        json={"connection": "prod", "script_path": "/nope/does_not_exist.py"},
+    )
+    assert r.status_code == 400
+    detail = r.json()["detail"]
+    # The message must guide the agent to the right next step
+    assert "does not exist" in detail
+    assert "generate_job_file" in detail
+
+
+def test_list_and_get_pending_job_roundtrip_via_body(tmp_path):
+    c = TestClient(app)
+    c.post("/save_connection", json={"name": "prod", "master": "yarn"})
+    script = tmp_path / "a.py"
+    script.write_text("print('hi')\n")
     prep = c.post(
         "/prepare_submit_job",
-        json={"connection": "prod", "script_path": "/tmp/a.py"},
+        json={"connection": "prod", "script_path": str(script)},
     ).json()
     pid = prep["pending_id"]
 
@@ -104,7 +124,7 @@ def test_list_and_get_pending_job_roundtrip_via_body():
     assert any(p["pending_id"] == pid for p in listed)
 
     got = c.post("/get_pending_job", json={"pending_id": pid}).json()
-    assert got["script_path"] == "/tmp/a.py"
+    assert got["script_path"] == str(script)
     assert got["status"] == "PENDING"
 
 
@@ -164,13 +184,15 @@ def test_unknown_connection_in_prepare_returns_404():
     assert "nope" in r.json()["detail"]
 
 
-def test_confirm_non_pending_returns_400():
+def test_confirm_non_pending_returns_400(tmp_path):
     """A CANCELLED pending should refuse confirm; FastAPI should surface ValueError as 400."""
     c = TestClient(app)
     c.post("/save_connection", json={"name": "prod", "master": "yarn"})
+    script = tmp_path / "x.py"
+    script.write_text("print('hi')\n")
     prep = c.post(
         "/prepare_submit_job",
-        json={"connection": "prod", "script_path": "/tmp/x.py"},
+        json={"connection": "prod", "script_path": str(script)},
     ).json()
     pid = prep["pending_id"]
     c.post("/cancel_pending_job", json={"pending_id": pid})
@@ -179,13 +201,15 @@ def test_confirm_non_pending_returns_400():
     assert "CANCELLED" in r.json()["detail"]
 
 
-def test_cancel_already_submitted_returns_400():
+def test_cancel_already_submitted_returns_400(tmp_path):
     """Cancelling a SUBMITTED pending should refuse with ValueError -> 400."""
     c = TestClient(app)
     c.post("/save_connection", json={"name": "prod", "master": "yarn"})
+    script = tmp_path / "x.py"
+    script.write_text("print('hi')\n")
     prep = c.post(
         "/prepare_submit_job",
-        json={"connection": "prod", "script_path": "/tmp/x.py"},
+        json={"connection": "prod", "script_path": str(script)},
     ).json()
     pid = prep["pending_id"]
     # Simulate a SUBMITTED state by mutating the pending directly