ClaudeandClaude Fable 5
627e70f697
Add a new MCP tool that lets the agent fetch URLs on the cluster's
network (YARN tracking UI, Spark History Server, NodeManager web UIs)
when the agent is on a different network and cannot reach those hosts
directly.
The MCP service runs on the YARN RM node, so it can reach every host
the cluster knows about — the agent just needs a way to ask.
Security: SSRF guard via host suffix overlap
- URL host must share >= 2 labels of suffix with the named
Connection's yarn_rm_url host (e.g. yarn_rm_url='rm.prod.internal'
allows 'http://nm01.prod.internal/...')
- IP literals (10.0.0.1, ::1) rejected
- Non-http(s) schemes (file://, gopher://, ftp://) rejected
- Connection with no yarn_rm_url cannot use this tool
- Reuses Connection.auth_for_httpx() and verify_for_httpx() so the
agent does not need cluster credentials
- Response body capped at 1 MB (truncated=true if larger)
- 30s timeout, follows redirects, loguru INFO audit log on every call
- spark_executor/tools/fetch_url.py: new tool + 2 helpers
(_host_suffix_overlap, _validate_url_host)
- spark_executor/models.py: FetchUrlResult Pydantic model
- spark_executor/tools/requests.py: FetchUrlRequest with descriptions
- spark_executor/server.py: /fetch_url route, operation_id='fetch_url'
- tests/unit/test_fetch_url.py: 13 unit tests covering all guards,
truncation, auth/SSL pass-through, redirect follow
- tests/integration/test_mcp_routes.py: assert 21 tool routes
- README.md: 1 row in Spark Executor 工具 table
Tests: 369 passed (up from 356).
Co-Authored-By: Claude Fable 5 <noreply@anthropic.com>
116 lines
3.2 KiB
Python
116 lines
3.2 KiB
Python
# coding=utf-8
|
|
"""
|
|
@Time :2026/6/24
|
|
@Author :tao.chen
|
|
"""
|
|
from datetime import datetime
|
|
|
|
from pydantic import BaseModel, Field, field_validator
|
|
|
|
|
|
class Job(BaseModel):
|
|
job_id: str
|
|
application_id: str
|
|
script_path: str
|
|
queue: str
|
|
submit_time: datetime
|
|
connection: str
|
|
yarn_rm_url: str | None = None
|
|
|
|
|
|
class JobStatus(BaseModel):
|
|
application_id: str
|
|
state: str
|
|
raw: str = Field(default="")
|
|
|
|
|
|
class JobResult(BaseModel):
|
|
application_id: str
|
|
state: str
|
|
final_status: str | None = None
|
|
diagnostics: str | None = None
|
|
tracking_url: str | None = None
|
|
started_time: int | None = None
|
|
finished_time: int | None = None
|
|
|
|
|
|
class SubmitResult(BaseModel):
|
|
job_id: str
|
|
application_id: str
|
|
tracking_url: str | None = None
|
|
|
|
|
|
class FetchUrlResult(BaseModel):
|
|
url: str
|
|
status_code: int
|
|
content_type: str
|
|
body: str
|
|
truncated: bool = False
|
|
|
|
|
|
class Connection(BaseModel):
|
|
name: str
|
|
# Defaults to "yarn" because that's the literal string spark-submit wants
|
|
# for --master when targeting YARN. Override for Standalone (spark://...),
|
|
# Kubernetes (k8s://...), or local mode.
|
|
master: str = "yarn"
|
|
deploy_mode: str = "cluster"
|
|
yarn_rm_url: str | None = None
|
|
spark_conf: dict[str, str] = Field(default_factory=dict)
|
|
# None means "fall back to Settings.ssl_verify_default". Explicit True/False
|
|
# overrides the global default for this connection.
|
|
ssl_verify: bool | None = None
|
|
ssl_ca_bundle: str | None = None
|
|
# Authentication for YARN REST calls.
|
|
auth_type: str = "none" # "none" | "simple" | "basic" | "kerberos"
|
|
auth_user: str | None = None
|
|
auth_password: str | None = None
|
|
# Display/audit only for kerberos; actual SPNEGO uses the system cache.
|
|
auth_principal: str | None = None
|
|
auth_keytab: str | None = None
|
|
|
|
@field_validator("master")
|
|
@classmethod
|
|
def _check_master(cls, v: str) -> str:
|
|
"""Catch common typos like 'yarn-cluster' or 'http://...'. """
|
|
if v == "yarn":
|
|
return v
|
|
if v.startswith(("spark://", "k8s://", "mesos://", "local")):
|
|
return v
|
|
raise ValueError(
|
|
f"master must be 'yarn', 'spark://...', 'k8s://...', 'mesos://...', "
|
|
f"or 'local[/N]'; got {v!r}"
|
|
)
|
|
|
|
@field_validator("auth_type")
|
|
@classmethod
|
|
def _check_auth_type(cls, v: str) -> str:
|
|
if v not in {"none", "simple", "basic", "kerberos"}:
|
|
raise ValueError(
|
|
f"auth_type must be one of none/simple/basic/kerberos; got {v!r}"
|
|
)
|
|
return v
|
|
|
|
|
|
class PendingSubmission(BaseModel):
|
|
pending_id: str
|
|
app_name: str | None = None
|
|
|
|
connection: str
|
|
master: str
|
|
deploy_mode: str
|
|
yarn_rm_url: str | None = None
|
|
script_path: str
|
|
queue: str
|
|
executor_memory: str
|
|
executor_cores: int
|
|
num_executors: int
|
|
spark_conf: dict[str, str] = Field(default_factory=dict)
|
|
extra_args: dict[str, str] = Field(default_factory=dict)
|
|
created_at: datetime
|
|
status: str = "PENDING" # PENDING | SUBMITTED | CANCELLED | FAILED
|
|
error: str | None = None
|
|
job_id: str | None = None
|
|
application_id: str | None = None
|
|
tracking_url: str | None = None
|