feat: add list_applications tool for YARN application enumeration
Add a new MCP tool that queries YARN's /ws/v1/cluster/apps endpoint
through a named Connection, returning a list of ApplicationSummary
records. Bypasses the local JobStore — useful for enumerating apps
that were not submitted through this service.
API:
list_applications(
connection_name: str, # required, which YARN cluster
state: str | None = None, # YARN state filter: NEW/NEW_SAVING/
# SUBMITTED/ACCEPTED/RUNNING/
# FINISHED/FAILED/KILLED
queue: str | None = None, # YARN queue filter
limit: int = 100, # cap on returned apps (YARN has no
# offset-based pagination; combine
# state/queue filters for big clusters)
) -> list[ApplicationSummary]
Implementation:
- yarn_client.list_applications(config, *, state, queue, limit) -> list[dict]
Returns raw YARN app dicts; raises YarnError on 4xx/5xx; returns
[] on 404 (no apps match). Uses the existing _request helper,
which now accepts a "params" kwarg for query strings (one-line
additive change).
- external_jobs.list_applications(connection_name, state, queue, limit)
-> list[ApplicationSummary]. Looks up the Connection, builds the
YarnClientConfig, calls the yarn_client function, maps each raw
YARN dict to ApplicationSummary (mirroring the manual field-mapping
style of get_job_result). The yarn_client function is imported
as "list_applications_yarn" to avoid name collision.
- ApplicationSummary: 12-field Pydantic model with snake_case names
(application_id, name, user, queue, state, final_status,
application_type, application_tags, started_time, finished_time,
tracking_url, progress). Unused YARN fields (memorySeconds,
vcoreSeconds, preemptedResource*, etc.) are not exposed.
- ListApplicationsRequest: Pydantic body model with Field(description=)
for LLM-facing schema.
- /list_applications route registered with operation_id=
"list_applications", placed next to the other external YARN tools.
Tests:
- 8 new unit tests in test_external_jobs.py (happy path, state/queue/
limit pass-through, default limit, empty list, missing connection,
full field mapping).
- test_mcp_routes.py: assert 23 tool routes.
- README: list_applications row added to the Spark Executor table.
Tests: 390 passed (was 382, +8 net).
Co-Authored-By: Claude Fable 5 <noreply@anthropic.com>
This commit is contained in:
Claude
@@ -64,12 +64,13 @@ def test_seventeen_tool_routes_registered():
|
||||
assert "/update_pending_job" in paths
|
||||
|
||||
|
||||
def test_twenty_two_tool_routes_registered():
|
||||
def test_twenty_three_tool_routes_registered():
|
||||
paths = {r.path for r in app.routes}
|
||||
for path in (
|
||||
"/get_external_job_logs",
|
||||
"/get_external_job_status",
|
||||
"/get_external_job_result",
|
||||
"/list_applications",
|
||||
"/fetch_url",
|
||||
"/update_connection",
|
||||
):
|
||||
|
||||
@@ -5,8 +5,9 @@ from unittest.mock import patch
|
||||
import pytest
|
||||
|
||||
from spark_executor.core import connection_store
|
||||
from spark_executor.models import Connection
|
||||
from spark_executor.models import ApplicationSummary, Connection
|
||||
from spark_executor.tools import connections, external_jobs
|
||||
from spark_executor.tools.requests import ListApplicationsRequest
|
||||
|
||||
|
||||
def _fresh_stores():
|
||||
@@ -17,6 +18,13 @@ def _fresh_stores():
|
||||
external_jobs.conn_store = store
|
||||
|
||||
|
||||
@pytest.fixture
|
||||
def fresh_stores(tmp_path, monkeypatch):
|
||||
"""Reset connection store singletons to an isolated tmp_path."""
|
||||
monkeypatch.setattr(connection_store, "DEFAULT_DATA_DIR", str(tmp_path))
|
||||
_fresh_stores()
|
||||
|
||||
|
||||
def test_get_external_job_logs_returns_tailed():
|
||||
_fresh_stores()
|
||||
external_jobs.conn_store.save(
|
||||
@@ -131,3 +139,154 @@ def test_get_external_job_result_raises_when_connection_missing():
|
||||
external_jobs.get_external_job_result(
|
||||
application_id="application_1", connection_name="missing"
|
||||
)
|
||||
|
||||
|
||||
def test_list_applications_returns_summaries(fresh_stores):
|
||||
external_jobs.conn_store.save(
|
||||
Connection(name="prod", master="yarn", yarn_rm_url="http://rm:8088")
|
||||
)
|
||||
with patch(
|
||||
"spark_executor.tools.external_jobs.list_applications_yarn",
|
||||
return_value=[
|
||||
{
|
||||
"id": "application_1",
|
||||
"name": "app-one",
|
||||
"user": "alice",
|
||||
"queue": "default",
|
||||
"state": "RUNNING",
|
||||
"finalStatus": "UNDEFINED",
|
||||
"applicationType": "SPARK",
|
||||
"applicationTags": "tag1",
|
||||
"startedTime": 1000,
|
||||
"finishedTime": 0,
|
||||
"trackingUrl": "http://rm:8088/proxy/application_1",
|
||||
"progress": 75.0,
|
||||
},
|
||||
{
|
||||
"id": "application_2",
|
||||
"name": "app-two",
|
||||
"user": "bob",
|
||||
"queue": "research",
|
||||
"state": "FINISHED",
|
||||
"finalStatus": "SUCCEEDED",
|
||||
"applicationType": "SPARK",
|
||||
"applicationTags": "",
|
||||
"startedTime": 2000,
|
||||
"finishedTime": 3000,
|
||||
"trackingUrl": "http://rm:8088/proxy/application_2",
|
||||
"progress": 100.0,
|
||||
},
|
||||
],
|
||||
) as m:
|
||||
out = external_jobs.list_applications("prod")
|
||||
assert len(out) == 2
|
||||
assert all(isinstance(item, ApplicationSummary) for item in out)
|
||||
assert out[0].application_id == "application_1"
|
||||
assert out[1].application_id == "application_2"
|
||||
args = m.call_args.args
|
||||
assert args[0].yarn_rm_url == "http://rm:8088"
|
||||
|
||||
|
||||
def test_list_applications_passes_state_filter(fresh_stores):
|
||||
external_jobs.conn_store.save(
|
||||
Connection(name="prod", master="yarn", yarn_rm_url="http://rm:8088")
|
||||
)
|
||||
with patch(
|
||||
"spark_executor.tools.external_jobs.list_applications_yarn",
|
||||
return_value=[],
|
||||
) as m:
|
||||
external_jobs.list_applications("prod", state="RUNNING")
|
||||
assert m.call_args.kwargs == {"state": "RUNNING", "queue": None, "limit": 100}
|
||||
|
||||
|
||||
def test_list_applications_passes_queue_filter(fresh_stores):
|
||||
external_jobs.conn_store.save(
|
||||
Connection(name="prod", master="yarn", yarn_rm_url="http://rm:8088")
|
||||
)
|
||||
with patch(
|
||||
"spark_executor.tools.external_jobs.list_applications_yarn",
|
||||
return_value=[],
|
||||
) as m:
|
||||
external_jobs.list_applications("prod", queue="research")
|
||||
assert m.call_args.kwargs == {"state": None, "queue": "research", "limit": 100}
|
||||
|
||||
|
||||
def test_list_applications_passes_limit_filter(fresh_stores):
|
||||
external_jobs.conn_store.save(
|
||||
Connection(name="prod", master="yarn", yarn_rm_url="http://rm:8088")
|
||||
)
|
||||
with patch(
|
||||
"spark_executor.tools.external_jobs.list_applications_yarn",
|
||||
return_value=[],
|
||||
) as m:
|
||||
external_jobs.list_applications("prod", limit=50)
|
||||
assert m.call_args.kwargs == {"state": None, "queue": None, "limit": 50}
|
||||
|
||||
|
||||
def test_list_applications_default_limit_is_100(fresh_stores):
|
||||
external_jobs.conn_store.save(
|
||||
Connection(name="prod", master="yarn", yarn_rm_url="http://rm:8088")
|
||||
)
|
||||
with patch(
|
||||
"spark_executor.tools.external_jobs.list_applications_yarn",
|
||||
return_value=[],
|
||||
) as m:
|
||||
external_jobs.list_applications("prod")
|
||||
assert m.call_args.kwargs == {"state": None, "queue": None, "limit": 100}
|
||||
assert ListApplicationsRequest(connection_name="prod").limit == 100
|
||||
|
||||
|
||||
def test_list_applications_returns_empty_list_when_no_apps(fresh_stores):
|
||||
external_jobs.conn_store.save(
|
||||
Connection(name="prod", master="yarn", yarn_rm_url="http://rm:8088")
|
||||
)
|
||||
with patch(
|
||||
"spark_executor.tools.external_jobs.list_applications_yarn",
|
||||
return_value=[],
|
||||
):
|
||||
out = external_jobs.list_applications("prod")
|
||||
assert out == []
|
||||
|
||||
|
||||
def test_list_applications_raises_for_missing_connection(fresh_stores):
|
||||
with pytest.raises(KeyError, match="Connection not found"):
|
||||
external_jobs.list_applications("missing")
|
||||
|
||||
|
||||
def test_list_applications_maps_yarn_json_to_summary(fresh_stores):
|
||||
external_jobs.conn_store.save(
|
||||
Connection(name="prod", master="yarn", yarn_rm_url="http://rm:8088")
|
||||
)
|
||||
yarn_app = {
|
||||
"id": "application_42",
|
||||
"name": "mapped-app",
|
||||
"user": "carol",
|
||||
"queue": "prod",
|
||||
"state": "ACCEPTED",
|
||||
"finalStatus": "UNDEFINED",
|
||||
"applicationType": "MAPREDUCE",
|
||||
"applicationTags": "batch",
|
||||
"startedTime": 12345,
|
||||
"finishedTime": 0,
|
||||
"trackingUrl": "http://rm:8088/proxy/application_42",
|
||||
"progress": 12.5,
|
||||
}
|
||||
with patch(
|
||||
"spark_executor.tools.external_jobs.list_applications_yarn",
|
||||
return_value=[yarn_app],
|
||||
):
|
||||
out = external_jobs.list_applications("prod")
|
||||
assert len(out) == 1
|
||||
summary = out[0]
|
||||
assert summary.application_id == "application_42"
|
||||
assert summary.name == "mapped-app"
|
||||
assert summary.user == "carol"
|
||||
assert summary.queue == "prod"
|
||||
assert summary.state == "ACCEPTED"
|
||||
assert summary.final_status == "UNDEFINED"
|
||||
assert summary.application_type == "MAPREDUCE"
|
||||
assert summary.application_tags == "batch"
|
||||
assert summary.started_time == 12345
|
||||
assert summary.finished_time == 0
|
||||
assert summary.tracking_url == "http://rm:8088/proxy/application_42"
|
||||
assert summary.progress == 12.5
|
||||
|
||||
Reference in New Issue
Block a user