revert(yarn_client): drop HTML directory-listing fallback
The previous commit (32ee167) added a regex-based HTML directory-listing parser as a fallback when the NodeManager wraps /stdout in HTML (Knox, wrapped vendor YARN builds). On review, the parser is too brittle to trust in production: - Apache-style listings vary across Hadoop distributions; some add extra rows, icons, or anchors that change the regex hit set. - Some clusters return the listing but 4xx each individual log file (auth layer in the way), so the fallback silently returns empty. - The behavior the agent observes for the same application_id becomes environment-dependent in ways that are hard to reason about. Revert to the simple, predictable path: GET /apps/{appid}, read amContainerLogs, GET {amContainerLogs}/stdout, return its text. If the cluster wraps /stdout in HTML, the agent gets a clear 'log-aggregation-enable' error and can ask the user to enable log aggregation or use a different fetch mechanism — much easier to debug than a half-parsed directory listing. Removes: - _looks_like_html, _parse_log_listing_html helpers - The bare-URL fallback path and its try/except wrapper - 4 unit tests that exercised the parser Full suite back to 243 passed (matchesfd0036c). Co-Authored-By: Claude Fable 5 <noreply@anthropic.com>
This commit is contained in:
Claude
@@ -12,7 +12,6 @@ from spark_executor.core.yarn_client import (
|
||||
YarnConfigError,
|
||||
YarnError,
|
||||
YarnClientConfig,
|
||||
_parse_log_listing_html,
|
||||
get_application_logs,
|
||||
get_application_status,
|
||||
kill_application,
|
||||
@@ -202,139 +201,6 @@ def test_logs_5xx_on_aggregated_endpoint_raises_immediately():
|
||||
assert m.call_count == 1
|
||||
|
||||
|
||||
def test_logs_falls_back_to_directory_listing_when_stdout_returns_html():
|
||||
"""Knox / wrapped YARN: GET {amContainerLogs}/stdout returns an HTML
|
||||
directory listing instead of raw log bytes. The fallback should fetch the
|
||||
bare URL, parse the listing, and pull each log file individually."""
|
||||
listing = (
|
||||
"<html><body>"
|
||||
'<a href="../">Parent Directory</a>'
|
||||
'<a href="?C=N;O=D">Name</a>'
|
||||
'<a href="stdout">stdout</a>'
|
||||
'<a href="stderr">stderr</a>'
|
||||
'<a href="syslog">syslog</a>'
|
||||
"</body></html>"
|
||||
)
|
||||
html_stdout = (
|
||||
"<!doctype html><html><body>wrapper around /stdout endpoint</body></html>"
|
||||
)
|
||||
responses = [
|
||||
_resp(404), # aggregated-logs missing
|
||||
_resp(
|
||||
200,
|
||||
json_data={
|
||||
"app": {
|
||||
"amContainerLogs": "http://nm1:8042/node/containerlogs/container_1/hdfs",
|
||||
}
|
||||
},
|
||||
),
|
||||
_resp(200, text=html_stdout), # /stdout -> HTML wrapper
|
||||
_resp(200, text=listing), # bare URL -> directory listing
|
||||
_resp(200, text="driver stdout line\n"), # fetch stdout
|
||||
_resp(200, text="driver stderr line\n"), # fetch stderr
|
||||
_resp(200, text="syslog line\n"), # fetch syslog
|
||||
]
|
||||
with patch(
|
||||
"spark_executor.core.yarn_client.httpx.request", side_effect=responses
|
||||
) as m:
|
||||
out = get_application_logs("application_1", YarnClientConfig(yarn_rm_url=RM))
|
||||
|
||||
# Header + body for each file in document order.
|
||||
assert "=== stdout ===" in out
|
||||
assert "=== stderr ===" in out
|
||||
assert "=== syslog ===" in out
|
||||
assert "driver stdout line" in out
|
||||
assert "driver stderr line" in out
|
||||
assert "syslog line" in out
|
||||
# stdout must come before stderr (document order preserved).
|
||||
assert out.index("=== stdout ===") < out.index("=== stderr ===") < out.index("=== syslog ===")
|
||||
|
||||
# 7 HTTP calls: aggregated-logs, app, /stdout (HTML), bare URL, stdout file, stderr file, syslog file
|
||||
assert m.call_count == 7
|
||||
assert m.call_args_list[2].args == (
|
||||
"GET",
|
||||
"http://nm1:8042/node/containerlogs/container_1/hdfs/stdout",
|
||||
)
|
||||
assert m.call_args_list[3].args == (
|
||||
"GET",
|
||||
"http://nm1:8042/node/containerlogs/container_1/hdfs",
|
||||
)
|
||||
assert m.call_args_list[4].args == (
|
||||
"GET",
|
||||
"http://nm1:8042/node/containerlogs/container_1/hdfs/stdout",
|
||||
)
|
||||
assert m.call_args_list[5].args == (
|
||||
"GET",
|
||||
"http://nm1:8042/node/containerlogs/container_1/hdfs/stderr",
|
||||
)
|
||||
assert m.call_args_list[6].args == (
|
||||
"GET",
|
||||
"http://nm1:8042/node/containerlogs/container_1/hdfs/syslog",
|
||||
)
|
||||
|
||||
|
||||
def test_logs_html_filter_strips_sort_links_and_parent_dir():
|
||||
"""_parse_log_listing_html must drop ../, sort-toggle query strings, and
|
||||
absolute URLs, while keeping plain log-file names."""
|
||||
html = (
|
||||
'<a href="../">Parent</a>'
|
||||
'<a href="?C=N;O=D">Name</a>'
|
||||
'<a href="?C=M;O=A">Last modified</a>'
|
||||
'<a href="?C=S;O=A">Size</a>'
|
||||
'<a href="?C=D;O=A">Description</a>'
|
||||
'<a href="http://example.com/somewhere">absolute</a>'
|
||||
'<a href="/absolute/path">rooted</a>'
|
||||
'<a href="stdout">stdout</a>'
|
||||
'<a href="stderr">stderr</a>'
|
||||
'<a href="syslog">syslog</a>'
|
||||
'<a href="stdout">stdout-dup</a>' # duplicate
|
||||
)
|
||||
assert _parse_log_listing_html(html) == ["stdout", "stderr", "syslog"]
|
||||
|
||||
|
||||
def test_logs_html_filter_handles_prelaunch_files():
|
||||
"""Some YARN versions list prelaunch.out / prelaunch.err / launch_container.sh
|
||||
in the directory listing alongside stdout/stderr. Keep them — we want to
|
||||
surface every log file the container produced."""
|
||||
html = (
|
||||
'<a href="prelaunch.out">prelaunch.out</a>'
|
||||
'<a href="prelaunch.err">prelaunch.err</a>'
|
||||
'<a href="launch_container.sh">launch_container.sh</a>'
|
||||
'<a href="directory-info">directory-info</a>'
|
||||
)
|
||||
assert _parse_log_listing_html(html) == [
|
||||
"prelaunch.out",
|
||||
"prelaunch.err",
|
||||
"launch_container.sh",
|
||||
"directory-info",
|
||||
]
|
||||
|
||||
|
||||
def test_logs_raises_when_directory_listing_empty():
|
||||
"""If /stdout returns HTML AND the directory listing has no parseable log
|
||||
file names, raise _logs_unavailable_error."""
|
||||
html_stdout = "<!doctype html><html>wrapper</html>"
|
||||
empty_listing = '<html><body><a href="../">Parent</a></body></html>'
|
||||
responses = [
|
||||
_resp(404), # aggregated-logs missing
|
||||
_resp(
|
||||
200,
|
||||
json_data={
|
||||
"app": {
|
||||
"amContainerLogs": "http://nm1:8042/node/containerlogs/container_1/hdfs",
|
||||
}
|
||||
},
|
||||
),
|
||||
_resp(200, text=html_stdout), # /stdout -> HTML
|
||||
_resp(200, text=empty_listing), # bare URL -> only parent link
|
||||
]
|
||||
with patch(
|
||||
"spark_executor.core.yarn_client.httpx.request", side_effect=responses
|
||||
):
|
||||
with pytest.raises(YarnError, match="log-aggregation-enable"):
|
||||
get_application_logs("application_1", YarnClientConfig(yarn_rm_url=RM))
|
||||
|
||||
|
||||
# --- kill_application ---
|
||||
|
||||
def test_kill_sends_put_with_killed_state():
|
||||
|
||||
Reference in New Issue
Block a user