Claude
70400d3ed1
`get_job_status` returns YARN state + the raw response blob, so the terminal fields (finalStatus, diagnostics, trackingUrl, startedTime, finishedTime) are buried inside `raw` and not surfaced in a structured form. Add a new tool that parses them. `get_job_result(job_id)` reuses `yarn_client.get_application_status` and extracts: - finalStatus (SUCCEEDED / FAILED / KILLED / UNDEFINED) - diagnostics (YARN final message) - tracking_url (Spark Web UI) - started_time / finished_time (epoch ms) All five fields are optional: running jobs have no `finishedTime`, and older YARN versions (CDH 5 / H2) may omit some fields. Missing fields stay None — never raise. Coexistence with `get_job_status` is intentional: the latter is for polling the running YARN state, the former is the terminal view. Tests: 4 cases — happy path, running job (no finishedTime), bare-minimum raw (all optionals None), unknown job_id raises KeyError. uv run pytest -> 171 passed.
84 lines
2.1 KiB
Python
84 lines
2.1 KiB
Python
# coding=utf-8
|
|
"""
|
|
@Time :2026/6/24
|
|
@Author :tao.chen
|
|
"""
|
|
from datetime import datetime
|
|
|
|
from pydantic import BaseModel, Field, field_validator
|
|
|
|
|
|
class Job(BaseModel):
|
|
job_id: str
|
|
application_id: str
|
|
script_path: str
|
|
queue: str
|
|
submit_time: datetime
|
|
connection: str
|
|
yarn_rm_url: str | None = None
|
|
|
|
|
|
class JobStatus(BaseModel):
|
|
application_id: str
|
|
state: str
|
|
raw: str = Field(default="")
|
|
|
|
|
|
class JobResult(BaseModel):
|
|
application_id: str
|
|
state: str
|
|
final_status: str | None = None
|
|
diagnostics: str | None = None
|
|
tracking_url: str | None = None
|
|
started_time: int | None = None
|
|
finished_time: int | None = None
|
|
|
|
|
|
class SubmitResult(BaseModel):
|
|
job_id: str
|
|
application_id: str
|
|
tracking_url: str | None = None
|
|
|
|
|
|
class Connection(BaseModel):
|
|
name: str
|
|
# Defaults to "yarn" because that's the literal string spark-submit wants
|
|
# for --master when targeting YARN. Override for Standalone (spark://...),
|
|
# Kubernetes (k8s://...), or local mode.
|
|
master: str = "yarn"
|
|
deploy_mode: str = "cluster"
|
|
yarn_rm_url: str | None = None
|
|
spark_conf: dict[str, str] = Field(default_factory=dict)
|
|
|
|
@field_validator("master")
|
|
@classmethod
|
|
def _check_master(cls, v: str) -> str:
|
|
"""Catch common typos like 'yarn-cluster' or 'http://...'. """
|
|
if v == "yarn":
|
|
return v
|
|
if v.startswith(("spark://", "k8s://", "mesos://", "local")):
|
|
return v
|
|
raise ValueError(
|
|
f"master must be 'yarn', 'spark://...', 'k8s://...', 'mesos://...', "
|
|
f"or 'local[/N]'; got {v!r}"
|
|
)
|
|
|
|
|
|
class PendingSubmission(BaseModel):
|
|
pending_id: str
|
|
connection: str
|
|
master: str
|
|
deploy_mode: str
|
|
yarn_rm_url: str | None = None
|
|
script_path: str
|
|
queue: str
|
|
executor_memory: str
|
|
executor_cores: int
|
|
num_executors: int
|
|
spark_conf: dict[str, str] = Field(default_factory=dict)
|
|
created_at: datetime
|
|
status: str = "PENDING" # PENDING | SUBMITTED | CANCELLED | FAILED
|
|
error: str | None = None
|
|
job_id: str | None = None
|
|
application_id: str | None = None
|