refactor(mcp): rename generate_job_file to write_job_file to match what it does
The MCP tool was named `generate_job_file` from Stage 2 but it does
NOT generate PySpark code — the calling LLM writes the code in its own
context, and this tool only persists it to a file under
SPARK_EXECUTOR_JOBS_DIR so `spark-submit` can see it. The misleading
`generate_` prefix sent agents (and humans) looking for a code
generator that doesn't exist.
This commit folds three related polish changes into one (split later
with rebase -i if you want them as separate history):
1. The rename itself:
- `tools/generate.py` → `tools/write_job.py`
- `generate_job_file` → `write_job_file`
- `GenerateJobFileRequest` → `WriteJobFileRequest`
- `/generate_job_file` route → `/write_job_file`
- `operation_id="generate_job_file"` → `operation_id="write_job_file"`
The internal helper `core.job_writer.write_job_file` (which just
writes bytes to disk with no SQL guard) is imported with an
`_write_to_disk` alias to avoid the name collision with the
MCP-exposed function in the same module.
The description for the tool now explicitly states 'this tool
does NOT generate PySpark code. The calling LLM is expected to
have already written the code; this tool only persists it.'
2. Skill for LLM agents operating the service
(`docs/superpowers/skills/spark-executor-mcp-operate/SKILL.md`,
449 lines). Covers the 16 tools, the two-step prepare/confirm
flow, the dual-ID contract (job_id vs application_id), the
PendingSubmission state machine, the Connection profile, the
job-file workflow, the error reference, common pitfalls, and a
full end-to-end word-count example.
3. Default `executor_memory` lowered 4G → 2G
(`_DEFAULTS_TO_CONFIRM` in `server.py`). Mirrors the matching
change in `test_mcp_routes.py` and the 5 unit tests that
reference the default. Aligns with the lighter workloads the
service is sized for in its current container profile.
Also tracked in git for the first time:
- `docs/superpowers/plans/2026-06-24-spark-executor-mcp.md`
(the original Stage 1/2/3 design plan, updated to use the new
tool name throughout).
Test rename:
- `tests/unit/test_generate_tool.py` → `test_write_job_tool.py`
- the new test file picks up an extra assertion that the SQL guard
rejects a `DROP TABLE` statement at write time.
243 tests pass (was 242; +1 new SQL-guard assertion). Zero regressions.
Co-Authored-By: Claude Fable 5 <noreply@anthropic.com>
This commit is contained in:
Claude
@@ -52,7 +52,7 @@ def test_sixteen_tool_routes_registered():
|
||||
"/get_connection",
|
||||
"/delete_connection",
|
||||
# LLM-driven PySpark generation (3) — Stage 2
|
||||
"/generate_job_file",
|
||||
"/write_job_file",
|
||||
"/read_job_file",
|
||||
"/update_job_file",
|
||||
):
|
||||
@@ -146,7 +146,7 @@ def test_prepare_submit_job_works_via_body(tmp_path):
|
||||
"connection": "prod",
|
||||
"script_path": str(script),
|
||||
"queue": "research",
|
||||
"executor_memory": "4G",
|
||||
"executor_memory": "2G",
|
||||
"executor_cores": 2,
|
||||
"num_executors": 2,
|
||||
"app_name": "test-app",
|
||||
@@ -177,7 +177,7 @@ def test_prepare_rejects_unconfirmed_defaults(tmp_path):
|
||||
detail = r.json()["detail"]
|
||||
assert "Please confirm default values" in detail
|
||||
assert "queue='default'" in detail
|
||||
assert "executor_memory='4G'" in detail
|
||||
assert "executor_memory='2G'" in detail
|
||||
assert "executor_cores=2" in detail
|
||||
assert "num_executors=2" in detail
|
||||
|
||||
@@ -193,7 +193,7 @@ def test_prepare_accepts_confirmed_defaults(tmp_path):
|
||||
"connection": "prod",
|
||||
"script_path": str(script),
|
||||
"queue": "default",
|
||||
"executor_memory": "4G",
|
||||
"executor_memory": "2G",
|
||||
"executor_cores": 2,
|
||||
"num_executors": 2,
|
||||
"app_name": "test-app",
|
||||
@@ -203,7 +203,7 @@ def test_prepare_accepts_confirmed_defaults(tmp_path):
|
||||
body = r.json()
|
||||
assert body["status"] == "PENDING"
|
||||
assert body["parameters"]["queue"] == "default"
|
||||
assert body["parameters"]["executor_memory"] == "4G"
|
||||
assert body["parameters"]["executor_memory"] == "2G"
|
||||
assert body["parameters"]["executor_cores"] == 2
|
||||
assert body["parameters"]["num_executors"] == 2
|
||||
|
||||
@@ -218,7 +218,7 @@ def test_prepare_rejects_nonexistent_script_path_with_400():
|
||||
"connection": "prod",
|
||||
"script_path": "/nope/does_not_exist.py",
|
||||
"queue": "default",
|
||||
"executor_memory": "4G",
|
||||
"executor_memory": "2G",
|
||||
"executor_cores": 2,
|
||||
"num_executors": 2,
|
||||
"app_name": "test-app",
|
||||
@@ -228,12 +228,12 @@ def test_prepare_rejects_nonexistent_script_path_with_400():
|
||||
detail = r.json()["detail"]
|
||||
# The message must guide the agent to the right next step
|
||||
assert "does not exist" in detail
|
||||
assert "generate_job_file" in detail
|
||||
assert "write_job_file" in detail
|
||||
|
||||
|
||||
def test_prepare_rejects_sql_policy_violation_with_400(tmp_path):
|
||||
"""SQL guard: even if the file exists, prepare_submit_job must reject
|
||||
code containing DROP/DELETE/etc. (defense in depth — generate_job_file
|
||||
code containing DROP/DELETE/etc. (defense in depth — write_job_file
|
||||
also enforces this, but a host-mounted file might bypass that)."""
|
||||
c = TestClient(app)
|
||||
c.post("/save_connection", json={"name": "prod", "master": "yarn"})
|
||||
@@ -245,7 +245,7 @@ def test_prepare_rejects_sql_policy_violation_with_400(tmp_path):
|
||||
"connection": "prod",
|
||||
"script_path": str(bad_script),
|
||||
"queue": "default",
|
||||
"executor_memory": "4G",
|
||||
"executor_memory": "2G",
|
||||
"executor_cores": 2,
|
||||
"num_executors": 2,
|
||||
"app_name": "test-app",
|
||||
@@ -257,14 +257,14 @@ def test_prepare_rejects_sql_policy_violation_with_400(tmp_path):
|
||||
assert "SELECT" in detail and "INSERT" in detail # the policy is explained
|
||||
|
||||
|
||||
def test_generate_rejects_sql_policy_violation_with_400(tmp_path):
|
||||
"""Same policy at generate_job_file — agent gets immediate feedback
|
||||
def test_write_rejects_sql_policy_violation_with_400(tmp_path):
|
||||
"""Same policy at write_job_file — agent gets immediate feedback
|
||||
before the file is even written to disk."""
|
||||
from common import config
|
||||
config.settings.jobs_dir = str(tmp_path / "jobs")
|
||||
c = TestClient(app)
|
||||
r = c.post(
|
||||
"/generate_job_file",
|
||||
"/write_job_file",
|
||||
json={"code": 'spark.sql("DELETE FROM events")\n'},
|
||||
)
|
||||
assert r.status_code == 400
|
||||
@@ -331,7 +331,7 @@ def test_list_and_get_pending_job_roundtrip_via_body(tmp_path):
|
||||
"connection": "prod",
|
||||
"script_path": str(script),
|
||||
"queue": "default",
|
||||
"executor_memory": "4G",
|
||||
"executor_memory": "2G",
|
||||
"executor_cores": 2,
|
||||
"num_executors": 2,
|
||||
"app_name": "test-app",
|
||||
@@ -354,15 +354,15 @@ def test_list_connections_works_with_empty_body():
|
||||
assert r.json() == []
|
||||
|
||||
|
||||
# --- Stage 2: generate_job_file ---
|
||||
# --- Stage 2: write_job_file ---
|
||||
|
||||
def test_generate_job_file_accepts_code_string_in_body(tmp_path, monkeypatch):
|
||||
def test_write_job_file_accepts_code_string_in_body(tmp_path, monkeypatch):
|
||||
"""End-to-end: a Pydantic body model lets tools/call pass a code string
|
||||
that FastAPI would reject if it were a query parameter (length limits)."""
|
||||
monkeypatch.chdir(tmp_path)
|
||||
c = TestClient(app)
|
||||
code = "print('from MCP integration test')\n" * 100 # > FastAPI query limit
|
||||
r = c.post("/generate_job_file", json={"code": code})
|
||||
r = c.post("/write_job_file", json={"code": code})
|
||||
assert r.status_code == 200, r.text
|
||||
p = r.json()["script_path"]
|
||||
assert os.path.isfile(p)
|
||||
@@ -401,7 +401,7 @@ def test_unknown_connection_in_prepare_returns_404():
|
||||
"connection": "nope",
|
||||
"script_path": "/tmp/x.py",
|
||||
"queue": "default",
|
||||
"executor_memory": "4G",
|
||||
"executor_memory": "2G",
|
||||
"executor_cores": 2,
|
||||
"num_executors": 2,
|
||||
"app_name": "test-app",
|
||||
@@ -423,7 +423,7 @@ def test_confirm_non_pending_returns_400(tmp_path):
|
||||
"connection": "prod",
|
||||
"script_path": str(script),
|
||||
"queue": "default",
|
||||
"executor_memory": "4G",
|
||||
"executor_memory": "2G",
|
||||
"executor_cores": 2,
|
||||
"num_executors": 2,
|
||||
"app_name": "test-app",
|
||||
@@ -448,7 +448,7 @@ def test_cancel_already_submitted_returns_400(tmp_path):
|
||||
"connection": "prod",
|
||||
"script_path": str(script),
|
||||
"queue": "default",
|
||||
"executor_memory": "4G",
|
||||
"executor_memory": "2G",
|
||||
"executor_cores": 2,
|
||||
"num_executors": 2,
|
||||
"app_name": "test-app",
|
||||
@@ -477,7 +477,7 @@ def test_update_pending_job_route(tmp_path):
|
||||
"connection": "prod",
|
||||
"script_path": str(script),
|
||||
"queue": "default",
|
||||
"executor_memory": "4G",
|
||||
"executor_memory": "2G",
|
||||
"executor_cores": 2,
|
||||
"num_executors": 2,
|
||||
"app_name": "test-app",
|
||||
@@ -496,7 +496,7 @@ def test_update_pending_job_route(tmp_path):
|
||||
got = c.post("/get_pending_job", json={"pending_id": pid}).json()
|
||||
assert got["queue"] == "research"
|
||||
# untouched fields are preserved
|
||||
assert got["executor_memory"] == "4G"
|
||||
assert got["executor_memory"] == "2G"
|
||||
assert got["app_name"] == "test-app"
|
||||
|
||||
|
||||
@@ -511,7 +511,7 @@ def test_update_pending_job_route_rejects_submitted(tmp_path, monkeypatch):
|
||||
"connection": "prod",
|
||||
"script_path": str(script),
|
||||
"queue": "default",
|
||||
"executor_memory": "4G",
|
||||
"executor_memory": "2G",
|
||||
"executor_cores": 2,
|
||||
"num_executors": 2,
|
||||
"app_name": "test-app",
|
||||
|
||||
Reference in New Issue
Block a user