feat(yarn_client): H2 fallback to NM container logs when /aggregated-logs 404/501
`/ws/v1/cluster/apps/{id}/aggregated-logs` is a Hadoop 3+ endpoint; on
CDH 5 / Hadoop 2.6 it returns 404 and the old code surfaced a hard error
to the user. Add capability detection in `get_application_logs`: on 404 or
501 from the aggregated-logs endpoint, walk `appattempts` -> `containers`
-> NodeManager `/node/containerlogs/{id}/{user}/` and concatenate the
results.
5xx on the aggregated-logs endpoint still raises immediately (no fallback
attempted for genuine server errors). The H3 fast path is unchanged.
Constraints honored:
- No `subprocess` / `yarn` CLI. The whole point of yarn_client.py is to
avoid that dependency, so the fallback stays on `httpx` + REST.
- No new pyproject.toml / uv.lock dependencies.
- Public signature of `get_application_logs` unchanged; `tools/logs.py`
needs no changes.
- No version detection — pure capability detection (try, react to status).
Tests: replaced the old "raise on 404" test with 5 new cases (H3 fast
path, NM fallback, NM path empty, multiple containers, 5xx immediate
raise). `uv run pytest` -> 167 passed.
This commit is contained in:
@@ -96,27 +96,123 @@ def test_status_rejects_non_http_url():
|
||||
|
||||
# --- get_application_logs ---
|
||||
|
||||
def test_logs_returns_text():
|
||||
def test_logs_returns_text_on_h3_aggregated_endpoint():
|
||||
fake = _resp(200, text="log line 1\nlog line 2\n")
|
||||
with patch("spark_executor.core.yarn_client.httpx.request", return_value=fake) as m:
|
||||
out = get_application_logs("application_1", RM)
|
||||
assert out == "log line 1\nlog line 2\n"
|
||||
assert m.call_args.args == ("GET", f"{RM}/ws/v1/cluster/apps/application_1/aggregated-logs")
|
||||
assert m.call_args.args == (
|
||||
"GET",
|
||||
f"{RM}/ws/v1/cluster/apps/application_1/aggregated-logs",
|
||||
)
|
||||
|
||||
|
||||
def test_logs_raises_on_404_with_explanation():
|
||||
with patch("spark_executor.core.yarn_client.httpx.request", return_value=_resp(404)):
|
||||
def test_logs_falls_back_to_nodemanager_on_404():
|
||||
responses = [
|
||||
_resp(404), # aggregated-logs H3 endpoint missing
|
||||
_resp(200, json_data={"appAttempts": {"appAttempt": [{"id": "attempt_1"}]}}),
|
||||
_resp(
|
||||
200,
|
||||
json_data={
|
||||
"containers": {
|
||||
"container": [
|
||||
{
|
||||
"id": "container_1",
|
||||
"user": "hdfs",
|
||||
"nodeHttpAddress": "nm1:8042",
|
||||
}
|
||||
]
|
||||
}
|
||||
},
|
||||
),
|
||||
_resp(200, text="container log content"),
|
||||
]
|
||||
with patch(
|
||||
"spark_executor.core.yarn_client.httpx.request", side_effect=responses
|
||||
) as m:
|
||||
out = get_application_logs("application_1", RM)
|
||||
assert "container_1" in out
|
||||
assert "container log content" in out
|
||||
calls = m.call_args_list
|
||||
assert calls[0].args == (
|
||||
"GET",
|
||||
f"{RM}/ws/v1/cluster/apps/application_1/aggregated-logs",
|
||||
)
|
||||
assert calls[1].args == ("GET", f"{RM}/ws/v1/cluster/apps/application_1/appattempts")
|
||||
assert calls[2].args == (
|
||||
"GET",
|
||||
f"{RM}/ws/v1/cluster/apps/application_1/appattempts/attempt_1/containers",
|
||||
)
|
||||
assert calls[3].args == (
|
||||
"GET",
|
||||
"http://nm1:8042/node/containerlogs/container_1/hdfs/",
|
||||
)
|
||||
|
||||
|
||||
def test_logs_raises_on_404_when_nm_also_empty():
|
||||
responses = [
|
||||
_resp(404), # aggregated-logs H3 endpoint missing
|
||||
_resp(200, json_data={"appAttempts": {"appAttempt": []}}),
|
||||
]
|
||||
with patch(
|
||||
"spark_executor.core.yarn_client.httpx.request", side_effect=responses
|
||||
):
|
||||
with pytest.raises(YarnError, match="log-aggregation-enable"):
|
||||
get_application_logs("application_1", RM)
|
||||
|
||||
|
||||
def test_logs_raises_on_5xx():
|
||||
def test_logs_handles_multiple_containers():
|
||||
responses = [
|
||||
_resp(404), # aggregated-logs H3 endpoint missing
|
||||
_resp(200, json_data={"appAttempts": {"appAttempt": [{"id": "attempt_1"}]}}),
|
||||
_resp(
|
||||
200,
|
||||
json_data={
|
||||
"containers": {
|
||||
"container": [
|
||||
{
|
||||
"id": "container_1",
|
||||
"user": "hdfs",
|
||||
"nodeHttpAddress": "nm1:8042",
|
||||
},
|
||||
{
|
||||
"id": "container_2",
|
||||
"user": "hdfs",
|
||||
"nodeHttpAddress": "nm2:8042",
|
||||
},
|
||||
]
|
||||
}
|
||||
},
|
||||
),
|
||||
_resp(200, text="log one"),
|
||||
_resp(200, text="log two"),
|
||||
]
|
||||
with patch(
|
||||
"spark_executor.core.yarn_client.httpx.request", side_effect=responses
|
||||
) as m:
|
||||
out = get_application_logs("application_1", RM)
|
||||
assert "log one" in out
|
||||
assert "log two" in out
|
||||
assert out.index("log one") < out.index("log two")
|
||||
calls = m.call_args_list
|
||||
assert calls[3].args == (
|
||||
"GET",
|
||||
"http://nm1:8042/node/containerlogs/container_1/hdfs/",
|
||||
)
|
||||
assert calls[4].args == (
|
||||
"GET",
|
||||
"http://nm2:8042/node/containerlogs/container_2/hdfs/",
|
||||
)
|
||||
|
||||
|
||||
def test_logs_5xx_on_aggregated_endpoint_raises_immediately():
|
||||
with patch(
|
||||
"spark_executor.core.yarn_client.httpx.request",
|
||||
return_value=_resp(500, text="boom"),
|
||||
):
|
||||
with pytest.raises(YarnError):
|
||||
) as m:
|
||||
with pytest.raises(YarnError, match="500"):
|
||||
get_application_logs("application_1", RM)
|
||||
assert m.call_count == 1
|
||||
|
||||
|
||||
# --- kill_application ---
|
||||
|
||||
Reference in New Issue
Block a user