tao.chen/mcp-server
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

refactor(mcp): rename generate_job_file to write_job_file to match what it does

Browse Source 
The MCP tool was named `generate_job_file` from Stage 2 but it does
NOT generate PySpark code — the calling LLM writes the code in its own
context, and this tool only persists it to a file under
SPARK_EXECUTOR_JOBS_DIR so `spark-submit` can see it. The misleading
`generate_` prefix sent agents (and humans) looking for a code
generator that doesn't exist.

This commit folds three related polish changes into one (split later
with rebase -i if you want them as separate history):

  1. The rename itself:
     - `tools/generate.py`  →  `tools/write_job.py`
     - `generate_job_file`  →  `write_job_file`
     - `GenerateJobFileRequest`  →  `WriteJobFileRequest`
     - `/generate_job_file` route  →  `/write_job_file`
     - `operation_id="generate_job_file"`  →  `operation_id="write_job_file"`
     The internal helper `core.job_writer.write_job_file` (which just
     writes bytes to disk with no SQL guard) is imported with an
     `_write_to_disk` alias to avoid the name collision with the
     MCP-exposed function in the same module.
     The description for the tool now explicitly states 'this tool
     does NOT generate PySpark code. The calling LLM is expected to
     have already written the code; this tool only persists it.'

  2. Skill for LLM agents operating the service
     (`docs/superpowers/skills/spark-executor-mcp-operate/SKILL.md`,
     449 lines). Covers the 16 tools, the two-step prepare/confirm
     flow, the dual-ID contract (job_id vs application_id), the
     PendingSubmission state machine, the Connection profile, the
     job-file workflow, the error reference, common pitfalls, and a
     full end-to-end word-count example.

  3. Default `executor_memory` lowered 4G → 2G
     (`_DEFAULTS_TO_CONFIRM` in `server.py`). Mirrors the matching
     change in `test_mcp_routes.py` and the 5 unit tests that
     reference the default. Aligns with the lighter workloads the
     service is sized for in its current container profile.

Also tracked in git for the first time:
  - `docs/superpowers/plans/2026-06-24-spark-executor-mcp.md`
    (the original Stage 1/2/3 design plan, updated to use the new
    tool name throughout).

Test rename:
  - `tests/unit/test_generate_tool.py`  →  `test_write_job_tool.py`
  - the new test file picks up an extra assertion that the SQL guard
    rejects a `DROP TABLE` statement at write time.

243 tests pass (was 242; +1 new SQL-guard assertion). Zero regressions.

Co-Authored-By: Claude Fable 5 <noreply@anthropic.com>
This commit is contained in:
Claude
2026-06-29 19:13:29 +08:00
co-authored by Claude Fable 5
parent 0fef77c0b8
commit 523e6a9c76
18 changed files with 3661 additions and 154 deletions
Download Patch File Download Diff File Expand all files Collapse all files
tests/unit/test_submit_tool.py
+33 -33
View File
@@ -56,7 +56,7 @@ def test_prepare_does_not_invoke_spark_submit(monkeypatch, real_script):
connection="prod",
script_path=str(real_script),
queue="default",
executor_memory="4G",
executor_memory="2G",
executor_cores=2,
num_executors=2,
app_name="test-app",
@@ -81,7 +81,7 @@ def test_prepare_persists_pending_with_snapshot(monkeypatch, real_script):
connection="prod",
script_path=str(real_script),
queue="research",
executor_memory="4G",
executor_memory="2G",
executor_cores=2,
num_executors=2,
app_name="test-app",
@@ -103,7 +103,7 @@ def test_prepare_persists_app_name(monkeypatch, real_script):
connection="prod",
script_path=str(real_script),
queue="default",
executor_memory="4G",
executor_memory="2G",
executor_cores=2,
num_executors=2,
app_name="my-etl-job",
@@ -125,7 +125,7 @@ def test_prepare_accepts_restored_defaults(monkeypatch, real_script):
connection="prod",
script_path=str(real_script),
queue="default",
executor_memory="4G",
executor_memory="2G",
executor_cores=2,
num_executors=2,
app_name="test-app",
@@ -133,7 +133,7 @@ def test_prepare_accepts_restored_defaults(monkeypatch, real_script):
assert out["status"] == "PENDING"
p = submit.pending_store.get(_last_pending_id())
assert p.queue == "default"
assert p.executor_memory == "4G"
assert p.executor_memory == "2G"
assert p.executor_cores == 2
assert p.num_executors == 2
@@ -144,7 +144,7 @@ def test_prepare_snapshots_extra_args(monkeypatch, real_script):
connection="prod",
script_path=str(real_script),
queue="default",
executor_memory="4G",
executor_memory="2G",
executor_cores=2,
num_executors=2,
app_name="test-app",
@@ -173,7 +173,7 @@ def test_prepare_raises_for_unknown_connection(real_script):
connection="missing",
script_path=str(real_script),
queue="default",
executor_memory="4G",
executor_memory="2G",
executor_cores=2,
num_executors=2,
app_name="test-app",
@@ -189,7 +189,7 @@ def test_prepare_rejects_nonexistent_script_path(tmp_path):
connection="prod",
script_path=missing,
queue="default",
executor_memory="4G",
executor_memory="2G",
executor_cores=2,
num_executors=2,
app_name="test-app",
@@ -203,7 +203,7 @@ def test_prepare_rejects_directory_as_script_path(tmp_path):
connection="prod",
script_path=str(tmp_path),
queue="default",
executor_memory="4G",
executor_memory="2G",
executor_cores=2,
num_executors=2,
app_name="test-app",
@@ -216,21 +216,21 @@ def test_prepare_rejects_empty_script_path():
connection="prod",
script_path="",
queue="default",
executor_memory="4G",
executor_memory="2G",
executor_cores=2,
num_executors=2,
app_name="test-app",
)
def test_prepare_error_message_mentions_generate_job_file(tmp_path):
"""The agent must be told to call generate_job_file first."""
with pytest.raises(ValueError, match="generate_job_file"):
def test_prepare_error_message_mentions_write_job_file(tmp_path):
"""The agent must be told to call write_job_file first."""
with pytest.raises(ValueError, match="write_job_file"):
submit.prepare_submit_job(
connection="prod",
script_path=str(tmp_path / "x.py"),
queue="default",
executor_memory="4G",
executor_memory="2G",
executor_cores=2,
num_executors=2,
app_name="test-app",
@@ -245,7 +245,7 @@ def test_confirm_rejects_if_script_was_deleted_after_prepare(tmp_path, monkeypat
connection="prod",
script_path=str(script),
queue="default",
executor_memory="4G",
executor_memory="2G",
executor_cores=2,
num_executors=2,
app_name="test-app",
@@ -264,7 +264,7 @@ def test_prepare_snapshots_connection_at_prepare_time(monkeypatch, real_script):
connection="prod",
script_path=str(real_script),
queue="default",
executor_memory="4G",
executor_memory="2G",
executor_cores=2,
num_executors=2,
app_name="test-app",
@@ -287,7 +287,7 @@ def test_confirm_invokes_spark_submit_and_marks_submitted(monkeypatch, real_scri
connection="prod",
script_path=str(real_script),
queue="default",
executor_memory="4G",
executor_memory="2G",
executor_cores=2,
num_executors=2,
app_name="test-app",
@@ -323,7 +323,7 @@ def test_confirm_refuses_non_pending_status(monkeypatch, real_script):
connection="prod",
script_path=str(real_script),
queue="default",
executor_memory="4G",
executor_memory="2G",
executor_cores=2,
num_executors=2,
app_name="test-app",
@@ -342,7 +342,7 @@ def test_confirm_marks_failed_on_spark_submit_error(monkeypatch, real_script):
connection="prod",
script_path=str(real_script),
queue="default",
executor_memory="4G",
executor_memory="2G",
executor_cores=2,
num_executors=2,
app_name="test-app",
@@ -368,7 +368,7 @@ def test_confirm_retries_spark_submit_failure_then_succeeds(monkeypatch, real_sc
connection="prod",
script_path=str(real_script),
queue="default",
executor_memory="4G",
executor_memory="2G",
executor_cores=2,
num_executors=2,
app_name="test-app",
@@ -395,7 +395,7 @@ def test_confirm_exhausts_retries_and_sets_failed(monkeypatch, real_script):
connection="prod",
script_path=str(real_script),
queue="default",
executor_memory="4G",
executor_memory="2G",
executor_cores=2,
num_executors=2,
app_name="test-app",
@@ -419,7 +419,7 @@ def test_confirm_idempotent_when_submitted(monkeypatch, real_script):
connection="prod",
script_path=str(real_script),
queue="default",
executor_memory="4G",
executor_memory="2G",
executor_cores=2,
num_executors=2,
app_name="test-app",
@@ -446,7 +446,7 @@ def test_confirm_resets_failed_and_retries(monkeypatch, real_script):
connection="prod",
script_path=str(real_script),
queue="default",
executor_memory="4G",
executor_memory="2G",
executor_cores=2,
num_executors=2,
app_name="test-app",
@@ -476,7 +476,7 @@ def test_confirm_updates_pending_before_job_store(monkeypatch, real_script):
connection="prod",
script_path=str(real_script),
queue="default",
executor_memory="4G",
executor_memory="2G",
executor_cores=2,
num_executors=2,
app_name="test-app",
@@ -510,7 +510,7 @@ def test_list_pending_jobs_returns_all(monkeypatch, tmp_path):
connection="prod",
script_path=str(a),
queue="default",
executor_memory="4G",
executor_memory="2G",
executor_cores=2,
num_executors=2,
app_name="test-app",
@@ -519,7 +519,7 @@ def test_list_pending_jobs_returns_all(monkeypatch, tmp_path):
connection="prod",
script_path=str(b),
queue="default",
executor_memory="4G",
executor_memory="2G",
executor_cores=2,
num_executors=2,
app_name="test-app",
@@ -537,7 +537,7 @@ def test_get_pending_job_returns_dump(monkeypatch, real_script):
connection="prod",
script_path=str(real_script),
queue="default",
executor_memory="4G",
executor_memory="2G",
executor_cores=2,
num_executors=2,
app_name="test-app",
@@ -562,7 +562,7 @@ def test_cancel_pending_job_marks_cancelled(monkeypatch, real_script):
connection="prod",
script_path=str(real_script),
queue="default",
executor_memory="4G",
executor_memory="2G",
executor_cores=2,
num_executors=2,
app_name="test-app",
@@ -583,7 +583,7 @@ def test_cancel_pending_job_refuses_submitted(monkeypatch, real_script):
connection="prod",
script_path=str(real_script),
queue="default",
executor_memory="4G",
executor_memory="2G",
executor_cores=2,
num_executors=2,
app_name="test-app",
@@ -604,7 +604,7 @@ def test_update_pending_updates_resource_params(monkeypatch, real_script):
connection="prod",
script_path=str(real_script),
queue="default",
executor_memory="4G",
executor_memory="2G",
executor_cores=2,
num_executors=2,
app_name="test-app",
@@ -639,7 +639,7 @@ def test_update_pending_rejects_non_pending_status(monkeypatch, real_script):
connection="prod",
script_path=str(real_script),
queue="default",
executor_memory="4G",
executor_memory="2G",
executor_cores=2,
num_executors=2,
app_name="test-app",
@@ -657,7 +657,7 @@ def test_update_pending_rejects_bad_script_path(tmp_path, real_script):
connection="prod",
script_path=str(real_script),
queue="default",
executor_memory="4G",
executor_memory="2G",
executor_cores=2,
num_executors=2,
app_name="test-app",
@@ -673,7 +673,7 @@ def test_update_pending_rejects_sql_violation_script(tmp_path, real_script):
connection="prod",
script_path=str(real_script),
queue="default",
executor_memory="4G",
executor_memory="2G",
executor_cores=2,
num_executors=2,
app_name="test-app",