feat: add external job tools + improve LLM-facing tool descriptions

Add 3 new MCP tools for inspecting YARN applications NOT submitted
through this service: get_external_job_logs, get_external_job_status,
get_external_job_result. Each takes application_id + connection_name
and queries YARN directly, bypassing the local JobStore.

- spark_executor/tools/external_jobs.py: 3 tool functions
- spark_executor/tools/requests.py: 3 new Pydantic body models
  (ExternalJobLogsRequest, ExternalJobStatusRequest,
  ExternalJobResultRequest)
- spark_executor/server.py: 3 new POST routes with explicit operation_id
- tests/unit/test_external_jobs.py: 7 unit tests
- tests/integration/test_mcp_routes.py: assert 20 tool routes
- README.md: list the 3 new tools

To make the LLM pick the right tool and not guess at field values,
also:

- Add Pydantic field descriptions for 22 fields across 8 request models
  (SaveConnectionRequest, UpdatePendingJobRequest, GetJobLogsRequest,
  JobIdRequest, PendingIdRequest, ConnectionNameRequest, plus the new
  ExternalJob*Request models).
- Update 12 route descriptions with cross-references, prerequisite
  context, and 400 behavior notes.
- Refactor _unknown_job_error: an input that looks like a YARN
  application_id (starts with 'application_') now returns HTTP 400
  (ValueError) with a hint message naming the right external tool;
  other not-found cases still return 404 (KeyError). This catches the
  common LLM mistake of passing application_id to the internal
  get_job_* / kill_job tools.
- 4 new unit tests for the 400 behavior.

Tests: 356 passed (up from 242).

Co-Authored-By: Claude Fable 5 <noreply@anthropic.com>

tests/integration/test_mcp_routes.py

@@ -64,6 +64,16 @@ def test_seventeen_tool_routes_registered():
     assert "/update_pending_job" in paths
 
 
+def test_twenty_tool_routes_registered():
+    paths = {r.path for r in app.routes}
+    for path in (
+        "/get_external_job_logs",
+        "/get_external_job_status",
+        "/get_external_job_result",
+    ):
+        assert path in paths, f"missing MCP tool route: {path}"
+
+
 # --- operation_id: pin clean MCP tool names (no auto-generated suffixes) ---
 #
 # fastapi-mcp uses each route's OpenAPI `operationId` as the MCP tool name

tests/unit/test_external_jobs.py

@@ -0,0 +1,133 @@
+# coding=utf-8
+import json
+from unittest.mock import patch
+
+import pytest
+
+from spark_executor.core import connection_store
+from spark_executor.models import Connection
+from spark_executor.tools import connections, external_jobs
+
+
+def _fresh_stores():
+    """Reset connection store singletons for a single test."""
+    store = connection_store.ConnectionStore()
+    connection_store.store = store
+    connections.store = store
+    external_jobs.conn_store = store
+
+
+def test_get_external_job_logs_returns_tailed():
+    _fresh_stores()
+    external_jobs.conn_store.save(
+        Connection(name="prod", master="yarn", yarn_rm_url="http://rm:8088")
+    )
+    long_log = "LOG" * 3000
+    with patch(
+        "spark_executor.tools.external_jobs.get_application_logs",
+        return_value=long_log,
+    ) as m:
+        out = external_jobs.get_external_job_logs(
+            application_id="application_1", connection_name="prod", tail_chars=100
+        )
+    assert out == long_log[-100:]
+    args = m.call_args.args
+    assert args[0] == "application_1"
+    assert args[1].yarn_rm_url == "http://rm:8088"
+
+
+def test_get_external_job_logs_returns_full_when_short():
+    _fresh_stores()
+    external_jobs.conn_store.save(
+        Connection(name="prod", master="yarn", yarn_rm_url="http://rm:8088")
+    )
+    short_log = "short log"
+    with patch(
+        "spark_executor.tools.external_jobs.get_application_logs",
+        return_value=short_log,
+    ):
+        out = external_jobs.get_external_job_logs(
+            application_id="application_1", connection_name="prod", tail_chars=5000
+        )
+    assert out == short_log
+
+
+def test_get_external_job_logs_raises_when_connection_missing():
+    _fresh_stores()
+    with pytest.raises(KeyError, match="Connection not found"):
+        external_jobs.get_external_job_logs(
+            application_id="application_1", connection_name="missing"
+        )
+
+
+def test_get_external_job_status_returns_state():
+    _fresh_stores()
+    external_jobs.conn_store.save(
+        Connection(name="prod", master="yarn", yarn_rm_url="http://rm:8088")
+    )
+    raw = json.dumps({"app": {"state": "RUNNING"}})
+    with patch(
+        "spark_executor.tools.external_jobs.get_application_status",
+        return_value=("RUNNING", raw),
+    ) as m:
+        out = external_jobs.get_external_job_status(
+            application_id="application_1", connection_name="prod"
+        )
+    assert out.application_id == "application_1"
+    assert out.state == "RUNNING"
+    assert out.raw == raw
+    args = m.call_args.args
+    assert args[0] == "application_1"
+    assert args[1].yarn_rm_url == "http://rm:8088"
+
+
+def test_get_external_job_status_raises_when_connection_missing():
+    _fresh_stores()
+    with pytest.raises(KeyError, match="Connection not found"):
+        external_jobs.get_external_job_status(
+            application_id="application_1", connection_name="missing"
+        )
+
+
+def test_get_external_job_result_parses_app_fields():
+    _fresh_stores()
+    external_jobs.conn_store.save(
+        Connection(name="prod", master="yarn", yarn_rm_url="http://rm:8088")
+    )
+    raw = json.dumps(
+        {
+            "app": {
+                "state": "FINISHED",
+                "finalStatus": "SUCCEEDED",
+                "diagnostics": "",
+                "trackingUrl": "http://rm:8088/proxy/application_1",
+                "startedTime": 100,
+                "finishedTime": 200,
+            }
+        }
+    )
+    with patch(
+        "spark_executor.tools.external_jobs.get_application_status",
+        return_value=("FINISHED", raw),
+    ) as m:
+        out = external_jobs.get_external_job_result(
+            application_id="application_1", connection_name="prod"
+        )
+    assert out.application_id == "application_1"
+    assert out.state == "FINISHED"
+    assert out.final_status == "SUCCEEDED"
+    assert out.diagnostics == ""
+    assert out.tracking_url == "http://rm:8088/proxy/application_1"
+    assert out.started_time == 100
+    assert out.finished_time == 200
+    args = m.call_args.args
+    assert args[0] == "application_1"
+    assert args[1].yarn_rm_url == "http://rm:8088"
+
+
+def test_get_external_job_result_raises_when_connection_missing():
+    _fresh_stores()
+    with pytest.raises(KeyError, match="Connection not found"):
+        external_jobs.get_external_job_result(
+            application_id="application_1", connection_name="missing"
+        )

tests/unit/test_kill_tool.py

@@ -63,6 +63,14 @@ def test_kill_job_raises_for_unknown_job():
         kill.kill_job("missing")
 
 
+def test_kill_job_raises_400_for_external_application_id(fresh_stores):
+    """Input that looks like a YARN application_id but is not in the local
+    JobStore must raise ValueError (-> 400) — kill_job has no external
+    equivalent, so the error points to the YARN CLI / UI."""
+    with pytest.raises(ValueError, match="YARN CLI"):
+        kill.kill_job("application_17400000001_0001")
+
+
 def test_kill_job_raises_when_connection_missing():
     _fresh_stores()
     kill.store.put(

tests/unit/test_logs_tool.py

@@ -68,6 +68,14 @@ def test_get_job_logs_raises_for_unknown_job():
         logs.get_job_logs("missing")
 
 
+def test_get_job_logs_raises_400_for_external_application_id(fresh_stores):
+    """Input that looks like a YARN application_id but is not in the local
+    JobStore must raise ValueError (-> 400) with a hint to use the
+    external tool, NOT a generic KeyError (-> 404)."""
+    with pytest.raises(ValueError, match="get_external_job_logs"):
+        logs.get_job_logs("application_17400000001_0001")
+
+
 def test_get_job_logs_raises_when_connection_missing(fresh_stores):
     logs.store.put(
         Job(

tests/unit/test_result_tool.py

@@ -136,6 +136,14 @@ def test_result_raises_keyerror_for_unknown_job():
     assert "application_id" in msg
 
 
+def test_result_raises_400_for_external_application_id(fresh_stores):
+    """Input that looks like a YARN application_id but is not in the local
+    JobStore must raise ValueError (-> 400) with a hint to use the
+    external tool, NOT a generic KeyError (-> 404)."""
+    with pytest.raises(ValueError, match="get_external_job_result"):
+        result.get_job_result("application_17400000001_0001")
+
+
 def test_result_raises_when_connection_missing():
     _fresh_stores()
     result.store.put(

tests/unit/test_status_tool.py

@@ -66,6 +66,14 @@ def test_get_job_status_raises_for_unknown_job():
         status.get_job_status("missing")
 
 
+def test_get_job_status_raises_400_for_external_application_id(fresh_stores):
+    """Input that looks like a YARN application_id but is not in the local
+    JobStore must raise ValueError (-> 400) with a hint to use the
+    external tool, NOT a generic KeyError (-> 404)."""
+    with pytest.raises(ValueError, match="get_external_job_status"):
+        status.get_job_status("application_17400000001_0001")
+
+
 def test_get_job_status_raises_when_connection_missing():
     _fresh_stores()
     status.store.put(