fix(yarn_client): fall back to directory listing when /stdout is HTML

Some YARN deployments (Knox-proxied, wrapped vendor builds) return an HTML directory listing page for /node/containerlogs/.../stdout instead of the raw log bytes. The previous fix assumed a clean text/plain response and failed every log fetch on those clusters. Two new private helpers in yarn_client.py: - _looks_like_html(text): cheap prolog sniff (<!doctype html, <html, <?xml) on the first ~200 chars. - _parse_log_listing_html(html): extract plain log-file names via a href="..." regex; drop ../, absolute paths, sort toggles (?C=N;O=D), and absolute URLs. _fetch_logs_via_am_container now does: 1. GET /apps/{appid} (unchanged). 2. Read app.amContainerLogs. 3. Fast path: GET {amContainerLogs}/stdout. If 2xx and NOT HTML, return. 4. Fallback: GET {amContainerLogs} (bare URL), parse the HTML listing, fetch each log file individually, and concatenate with === filename === headers in document order. 5. Raise _logs_unavailable_error if both paths yield nothing. Tests: 3 new (test_logs_falls_back_to_directory_listing_when_stdout_returns_html, test_logs_html_filter_strips_sort_links_and_parent_dir, test_logs_html_filter_handles_prelaunch_files, test_logs_raises_when_directory_listing_empty) — full suite 247 passed. Co-Authored-By: Claude Fable 5 <noreply@anthropic.com>
2026-06-30 10:56:42 +08:00
parent fd0036ca80
commit 32ee167b59
2 changed files with 206 additions and 10 deletions
@@ -14,9 +14,12 @@ Endpoints used (YARN 2.6+):

 When aggregated logs are unavailable (HTTP 404/501, e.g. log aggregation
 disabled), fall back to the amContainerLogs field reported by the app
-endpoint and fetch the AM (driver) container's /stdout directly from the
-NodeManager. This returns driver logs only; executor container logs still
-require yarn.log-aggregation-enable=true (the primary /aggregated-logs path).
+endpoint and fetch the AM (driver) container's /stdout from the
+NodeManager. Some clusters (Knox-proxied, wrapped YARN builds) wrap
+/stdout in an HTML directory listing; in that case we fetch the bare
+{amContainerLogs} URL, parse the listing, and pull each log file. This
+returns driver logs only; executor container logs still require
+yarn.log-aggregation-enable=true (the primary /aggregated-logs path).

 The ResourceManager URL is passed in per call (snapshotted on the Job at
 confirm_submit_job time) and falls back to the YARN_RESOURCE_MANAGER_URL env
@@ -26,6 +29,8 @@ field was designed for, but no longer requires the `yarn` CLI to interpret it.
 import json
 from dataclasses import dataclass

+import re
+
 import httpx
 import httpx_kerberos

@@ -163,11 +168,50 @@ def _logs_unavailable_error(application_id: str) -> YarnError:
    )


+def _looks_like_html(text: str) -> bool:
+    """Cheap heuristic: True if the body starts with an HTML/XHTML prolog.
+
+    Intentionally lightweight (no full parser) — used only to decide whether
+    a NodeManager log response is raw text or an HTML directory listing.
+    """
+    head = text.lstrip()[:200].lower()
+    return head.startswith(("<!doctype html", "<html", "<?xml"))
+
+
+def _parse_log_listing_html(html: str) -> list[str]:
+    """Extract plain log-file names from a NodeManager directory-listing page.
+
+    Keeps simple filenames (stdout, stderr, syslog, prelaunch.err,
+    launch_container.sh, ...); drops parent-dir links, absolute URLs, and
+    sort-toggle query strings. Returns hrefs in document order, deduplicated.
+    """
+    hrefs = re.findall(r'href="([^"]*)"', html, re.IGNORECASE)
+    seen: set[str] = set()
+    out: list[str] = []
+    for href in hrefs:
+        if href in ("../", "/") or href.startswith("/"):
+            continue
+        if "://" in href or "?" in href:
+            continue
+        if href in seen:
+            continue
+        seen.add(href)
+        out.append(href)
+    return out
+
+
 def _fetch_logs_via_am_container(application_id: str, config: YarnClientConfig) -> str:
    """
-    Fetch ApplicationMaster (driver) container stdout as a fallback when the
+    Fetch ApplicationMaster (driver) container logs as a fallback when the
    aggregated-logs endpoint is unavailable.

+    Fast path: GET {amContainerLogs}/stdout and return its text directly when
+    the NodeManager serves raw log bytes (modern YARN).
+
+    Fallback path: when /stdout returns an HTML directory listing (Knox /
+    wrapped YARN), GET the bare {amContainerLogs} URL, parse the listing for
+    log-file names, and fetch each one individually.
+
    Returns AM (driver) container logs only. Executor container logs require
    yarn.log-aggregation-enable=true, which is covered by the primary
    /aggregated-logs path.
@@ -183,12 +227,30 @@ def _fetch_logs_via_am_container(application_id: str, config: YarnClientConfig)
    if not am_container_logs:
        raise _logs_unavailable_error(application_id)

-    log_url = f"{am_container_logs.rstrip('/')}/stdout"
-    resp = _request("GET", log_url, timeout=60.0, verify=config.verify_for_httpx(), auth=config.auth_for_httpx())
-    if resp.status_code >= 400:
-        logger.error(f"YARN GET {log_url} -> {resp.status_code}: {resp.text[:500]}")
-        raise _logs_unavailable_error(application_id)
-    return resp.text
+    am_container_logs = am_container_logs.rstrip("/")
+    verify = config.verify_for_httpx()
+    auth = config.auth_for_httpx()
+
+    # Fast path: modern YARN serves raw log bytes at /stdout.
+    stdout_url = f"{am_container_logs}/stdout"
+    resp = _request("GET", stdout_url, timeout=60.0, verify=verify, auth=auth)
+    if resp.status_code < 400 and not _looks_like_html(resp.text):
+        return resp.text
+
+    # Fallback path: Knox / wrapped YARN serves an HTML directory listing for
+    # every URL on the NodeManager log endpoint. Parse it and fetch each file.
+    resp = _request("GET", am_container_logs, timeout=60.0, verify=verify, auth=auth)
+    if resp.status_code < 400:
+        parts: list[str] = []
+        for filename in _parse_log_listing_html(resp.text):
+            file_url = f"{am_container_logs}/{filename}"
+            file_resp = _request("GET", file_url, timeout=60.0, verify=verify, auth=auth)
+            if file_resp.status_code < 400 and not _looks_like_html(file_resp.text):
+                parts.append(f"=== {filename} ===\n{file_resp.text}")
+        if parts:
+            return "\n\n".join(parts)
+
+    raise _logs_unavailable_error(application_id)


 def get_application_logs(application_id: str, config: YarnClientConfig) -> str: