tao.chen/mcp-server
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

fix(yarn_client): use amContainerLogs for AM container log fallback

Browse Source 
The previous _fetch_logs_via_nodemanager walked
/apps/{appid}/appattempts -> /apps/{appid}/appattempts/{id}/containers,
but YARN ResourceManager does not expose the second hop. Container
info lives on NodeManager and is not reachable from RM REST, so the
fallback returned _logs_unavailable_error on every cluster without
/aggregated-logs (Hadoop 2.x, log aggregation disabled).

Replace the walk with: GET /apps/{appid} -> read app.amContainerLogs
-> GET {amContainerLogs}/stdout. The URL is provided directly by the
app response, so no container enumeration is needed.

Returns AM (driver) container logs only. Executor container logs
still require yarn.log-aggregation-enable=true (primary /aggregated-logs
path); the existing _logs_unavailable_error message already says that,
so the contract is preserved.

Co-Authored-By: Claude Fable 5 <noreply@anthropic.com>
This commit is contained in:
Claude
2026-06-30 10:33:01 +08:00
co-authored by Claude Fable 5
parent dfca612f8c
commit fd0036ca80
2 changed files with 44 additions and 93 deletions
Download Patch File Download Diff File Expand all files Collapse all files
tests/unit/test_yarn_client.py
+19 -51
View File
@@ -125,52 +125,40 @@ def test_logs_returns_text_on_h3_aggregated_endpoint():
assert m.call_args.kwargs["verify"] is True
def test_logs_falls_back_to_nodemanager_on_404():
def test_logs_falls_back_to_am_container_on_404():
responses = [
_resp(404), # aggregated-logs H3 endpoint missing
_resp(200, json_data={"appAttempts": {"appAttempt": [{"id": "attempt_1"}]}}),
_resp(404), # aggregated-logs endpoint missing
_resp(
200,
json_data={
"containers": {
"container": [
{
"id": "container_1",
"user": "hdfs",
"nodeHttpAddress": "nm1:8042",
}
]
"app": {
"amContainerLogs": "http://nm1:8042/node/containerlogs/container_1/hdfs",
}
},
),
_resp(200, text="container log content"),
_resp(200, text="am container log content"),
]
with patch(
"spark_executor.core.yarn_client.httpx.request", side_effect=responses
) as m:
out = get_application_logs("application_1", YarnClientConfig(yarn_rm_url=RM))
assert "container_1" in out
assert "container log content" in out
assert out == "am container log content"
calls = m.call_args_list
assert calls[0].args == (
"GET",
f"{RM}/ws/v1/cluster/apps/application_1/aggregated-logs",
)
assert calls[1].args == ("GET", f"{RM}/ws/v1/cluster/apps/application_1/appattempts")
assert calls[1].args == ("GET", f"{RM}/ws/v1/cluster/apps/application_1")
assert calls[2].args == (
"GET",
f"{RM}/ws/v1/cluster/apps/application_1/appattempts/attempt_1/containers",
)
assert calls[3].args == (
"GET",
"http://nm1:8042/node/containerlogs/container_1/hdfs/",
"http://nm1:8042/node/containerlogs/container_1/hdfs/stdout",
)
def test_logs_raises_on_404_when_nm_also_empty():
def test_logs_raises_on_404_when_am_container_field_missing():
responses = [
_resp(404), # aggregated-logs H3 endpoint missing
_resp(200, json_data={"appAttempts": {"appAttempt": []}}),
_resp(404), # aggregated-logs endpoint missing
_resp(200, json_data={"app": {}}),
]
with patch(
"spark_executor.core.yarn_client.httpx.request", side_effect=responses
@@ -179,47 +167,27 @@ def test_logs_raises_on_404_when_nm_also_empty():
get_application_logs("application_1", YarnClientConfig(yarn_rm_url=RM))
def test_logs_handles_multiple_containers():
def test_logs_returns_am_container_stdout():
responses = [
_resp(404), # aggregated-logs H3 endpoint missing
_resp(200, json_data={"appAttempts": {"appAttempt": [{"id": "attempt_1"}]}}),
_resp(404), # aggregated-logs endpoint missing
_resp(
200,
json_data={
"containers": {
"container": [
{
"id": "container_1",
"user": "hdfs",
"nodeHttpAddress": "nm1:8042",
},
{
"id": "container_2",
"user": "hdfs",
"nodeHttpAddress": "nm2:8042",
},
]
"app": {
"amContainerLogs": "http://nm1:8042/node/containerlogs/container_1/hdfs",
}
},
),
_resp(200, text="log one"),
_resp(200, text="log two"),
_resp(200, text="driver stdout"),
]
with patch(
"spark_executor.core.yarn_client.httpx.request", side_effect=responses
) as m:
out = get_application_logs("application_1", YarnClientConfig(yarn_rm_url=RM))
assert "log one" in out
assert "log two" in out
assert out.index("log one") < out.index("log two")
calls = m.call_args_list
assert calls[3].args == (
assert out == "driver stdout"
assert m.call_args_list[2].args == (
"GET",
"http://nm1:8042/node/containerlogs/container_1/hdfs/",
)
assert calls[4].args == (
"GET",
"http://nm2:8042/node/containerlogs/container_2/hdfs/",
"http://nm1:8042/node/containerlogs/container_1/hdfs/stdout",
)