Claude
4b07617af0
Remove defaults from queue, executor_memory, executor_cores, num_executors, and app_name in prepare_submit_job. All must now be explicitly confirmed by the caller. Also add extra_args to the PendingSubmission snapshot so users can confirm non-conf spark-submit flags (e.g. --jars, --py-files) at prepare time; confirm_submit_job passes them through to build_spark_submit_command. This is an intentional breaking change to the MCP tool contract: callers can no longer rely on implicit defaults.
129 lines
3.8 KiB
Python
129 lines
3.8 KiB
Python
# coding=utf-8
|
|
"""
|
|
@Time :2026/6/24
|
|
@Author :tao.chen
|
|
|
|
Pydantic request models for the FastAPI route layer. The underlying tool
|
|
functions in tools/*.py still take keyword arguments; these models exist only
|
|
so fastapi-mcp can call the routes via tools/call (which sends args as a
|
|
JSON body) without 422-ing on dict-typed parameters like spark_conf.
|
|
"""
|
|
from pydantic import BaseModel, Field
|
|
|
|
|
|
class EmptyRequest(BaseModel):
|
|
"""Used for tools that take no arguments (list_connections, list_pending_jobs)."""
|
|
pass
|
|
|
|
|
|
class SaveConnectionRequest(BaseModel):
|
|
name: str
|
|
master: str
|
|
deploy_mode: str = "cluster"
|
|
yarn_rm_url: str | None = None
|
|
spark_conf: dict[str, str] | None = None
|
|
ssl_verify: bool | None = None
|
|
ssl_ca_bundle: str | None = None
|
|
auth_type: str = "none"
|
|
auth_user: str | None = None
|
|
auth_password: str | None = None
|
|
auth_principal: str | None = None
|
|
auth_keytab: str | None = None
|
|
|
|
|
|
class PrepareSubmitJobRequest(BaseModel):
|
|
connection: str
|
|
app_name: str = Field(
|
|
...,
|
|
description="Human-readable application name for tracking the pending submission.",
|
|
)
|
|
script_path: str = Field(
|
|
...,
|
|
description=(
|
|
"Absolute path to the PySpark script inside the container's "
|
|
"filesystem. Must point at an existing regular file. For "
|
|
"LLM-generated code, call generate_job_file(code=...) first "
|
|
"and pass the returned script_path here. For pre-existing "
|
|
"files on the host, mount them via a docker volume and pass "
|
|
"the in-container path. Returns 400 with a remediation hint "
|
|
"if the path is missing or not a file."
|
|
),
|
|
)
|
|
queue: str = Field(
|
|
...,
|
|
description="YARN queue to submit to. Must be explicitly confirmed by the caller.",
|
|
)
|
|
executor_memory: str = Field(
|
|
...,
|
|
description="Executor memory, e.g. '4G'. Must be explicitly confirmed by the caller.",
|
|
)
|
|
executor_cores: int = Field(
|
|
...,
|
|
description="Number of cores per executor. Must be explicitly confirmed by the caller.",
|
|
)
|
|
num_executors: int = Field(
|
|
...,
|
|
description="Total number of executors. Must be explicitly confirmed by the caller.",
|
|
)
|
|
extra_args: dict[str, str] | None = Field(
|
|
default=None,
|
|
description="Additional spark-submit flags (e.g. jars, py-files) confirmed at prepare time.",
|
|
)
|
|
|
|
|
|
class PendingIdRequest(BaseModel):
|
|
pending_id: str
|
|
|
|
|
|
class JobIdRequest(BaseModel):
|
|
job_id: str
|
|
|
|
|
|
class GetJobLogsRequest(BaseModel):
|
|
job_id: str
|
|
tail_chars: int = 5000
|
|
|
|
|
|
class ConnectionNameRequest(BaseModel):
|
|
name: str
|
|
|
|
|
|
class GenerateJobFileRequest(BaseModel):
|
|
code: str = Field(
|
|
...,
|
|
description=(
|
|
"Full PySpark source code to write to disk. Will be passed verbatim "
|
|
"to spark-submit after the agent calls prepare_submit_job on the "
|
|
"returned path."
|
|
),
|
|
)
|
|
|
|
|
|
class ReadJobFileRequest(BaseModel):
|
|
script_path: str = Field(
|
|
...,
|
|
description=(
|
|
"Absolute path to a PySpark script inside the container's "
|
|
"filesystem. Must point at an existing regular file."
|
|
),
|
|
)
|
|
|
|
|
|
class UpdateJobFileRequest(BaseModel):
|
|
script_path: str = Field(
|
|
...,
|
|
description=(
|
|
"Absolute path to an existing PySpark script inside the "
|
|
"container's filesystem. Must be under SPARK_EXECUTOR_JOBS_DIR "
|
|
"(the same dir generate_job_file writes to) — protects against "
|
|
"overwriting host-mounted configs or other critical files."
|
|
),
|
|
)
|
|
content: str = Field(
|
|
...,
|
|
description=(
|
|
"New file content (replaces the file in full; no merge/diff). "
|
|
"Maximum 1 MB to keep the MCP response bounded."
|
|
),
|
|
)
|