fix(yarn_client): manually follow 3xx Location redirects on log fetch

httpx follows redirects by default, but it does NOT forward the
Authorization header across host boundaries. In YARN deployments where
the ResourceManager 307-redirects the NodeManager log fetch to a
different host (load balancer, Knox, NM selection), the follow-up
request lands unauthenticated and returns 401/403.

Replace the _request call in _fetch_logs_via_am_container with
_request_following_redirects, which:
  - Walks the 3xx Location chain (up to 3 hops).
  - Re-applies auth + verify on every hop.
  - Resolves relative Location URLs against the current request URL.
  - Raises YarnError on 3xx-without-Location (misconfigured server) and
    on hop count overflow (redirect loop protection).

301/302/303/307/308 are all treated as 'follow the Location' per
RFC 7231 — the method/body handling for the rest is up to httpx when
we re-issue the request.

Tests: 4 new (cross-host 307, no-Location 307, two-hop chain, relative
Location). Full suite 247 passed.

Co-Authored-By: Claude Fable 5 <noreply@anthropic.com>

spark_executor/core/yarn_client.py

@@ -15,9 +15,11 @@ Endpoints used (YARN 2.6+):
 When aggregated logs are unavailable (HTTP 404/501, e.g. log aggregation
 disabled), fall back to the amContainerLogs field reported by the app
 endpoint and fetch the AM (driver) container's /stdout directly from
-the NodeManager. This returns driver logs only; executor container logs
-still require yarn.log-aggregation-enable=true (the primary
-/aggregated-logs path).
+the NodeManager. The log fetch follows 3xx Location redirects manually
+(httpx's default redirect-following drops Authorization across host
+boundaries, which breaks RM→NM 307 redirects in many deployments).
+This returns driver logs only; executor container logs still require
+yarn.log-aggregation-enable=true (the primary /aggregated-logs path).
 
 The ResourceManager URL is passed in per call (snapshotted on the Job at
 confirm_submit_job time) and falls back to the YARN_RESOURCE_MANAGER_URL env
@@ -26,6 +28,7 @@ field was designed for, but no longer requires the `yarn` CLI to interpret it.
 """
 import json
 from dataclasses import dataclass
+from urllib.parse import urljoin
 
 import httpx
 import httpx_kerberos
@@ -137,6 +140,46 @@ def _request(method: str, url: str, *, json_body: dict | None = None,
     return resp
 
 
+# Status codes that indicate "follow the Location header". 307/308 preserve
+# the method+body; the rest collapse to GET for the redirect target per
+# RFC 7231 — httpx will do the right thing for us on the second request.
+_REDIRECT_STATUSES = frozenset({301, 302, 303, 307, 308})
+_MAX_LOG_REDIRECTS = 3
+
+
+def _request_following_redirects(
+    method: str, url: str, *, max_redirects: int = _MAX_LOG_REDIRECTS,
+    **kwargs,
+) -> httpx.Response:
+    """Like `_request` but follows 3xx Location redirects up to `max_redirects`.
+
+    httpx follows redirects by default, but it does NOT forward the
+    `Authorization` header across host boundaries — so when the YARN RM
+    307-redirects us to a different NodeManager host, the follow-up
+    request lands unauthenticated and gets 401/403. Doing it ourselves
+    keeps the auth + verify config on every hop.
+
+    Relative `Location` values are resolved against the current request
+    URL. A 3xx without a `Location` header is returned as-is (the caller
+    will treat it as a non-2xx response and surface a clean error).
+    """
+    for hop in range(max_redirects + 1):
+        resp = _request(method, url, **kwargs)
+        if resp.status_code not in _REDIRECT_STATUSES:
+            return resp
+        location = resp.headers.get("Location") or resp.headers.get("location")
+        if not location:
+            raise YarnError(
+                f"YARN {method} returned {resp.status_code} with no Location header at {url}"
+            )
+        next_url = urljoin(url, location)
+        logger.debug(f"YARN {method} {url} -> {resp.status_code}, following to {next_url}")
+        url = next_url
+    raise YarnError(
+        f"YARN {method} exceeded {max_redirects} redirects (last url: {url})"
+    )
+
+
 def get_application_status(application_id: str, config: YarnClientConfig) -> tuple[str, str]:
     """Return (state, raw_json_text) for an application, or raise YarnError."""
     url = f"{_base_url(config.yarn_rm_url)}/ws/v1/cluster/apps/{application_id}"
@@ -185,7 +228,10 @@ def _fetch_logs_via_am_container(application_id: str, config: YarnClientConfig)
         raise _logs_unavailable_error(application_id)
 
     log_url = f"{am_container_logs.rstrip('/')}/stdout"
-    resp = _request("GET", log_url, timeout=60.0, verify=config.verify_for_httpx(), auth=config.auth_for_httpx())
+    resp = _request_following_redirects(
+        "GET", log_url, timeout=60.0,
+        verify=config.verify_for_httpx(), auth=config.auth_for_httpx(),
+    )
     if resp.status_code >= 400:
         logger.error(f"YARN GET {log_url} -> {resp.status_code}: {resp.text[:500]}")
         raise _logs_unavailable_error(application_id)