From c44e9bcc55779523a20df4ffdc51ac81ced4582f Mon Sep 17 00:00:00 2001 From: Claude Date: Fri, 26 Jun 2026 14:27:50 +0800 Subject: [PATCH] fix(yarn_client): explicitly request JSON from YARN REST API YARN ResourceManager REST can return either JSON or XML depending on the Accept / Content-Type headers. We always parse responses with resp.json(), so an ambiguous or XML response would break the client. Add Accept: application/json to every _request call so the response format is pinned unambiguously. This is compatible with both Hadoop 2.x/CDH 5 and Hadoop 3.x RM endpoints. Tests: assert that httpx.request receives headers={"Accept": "application/json"}. --- spark_executor/core/yarn_client.py | 5 ++++- tests/unit/test_yarn_client.py | 7 +++++++ 2 files changed, 11 insertions(+), 1 deletion(-) diff --git a/spark_executor/core/yarn_client.py b/spark_executor/core/yarn_client.py index 5a0b8e4..fc32c60 100644 --- a/spark_executor/core/yarn_client.py +++ b/spark_executor/core/yarn_client.py @@ -114,9 +114,12 @@ def _base_url(yarn_rm_url: str | None) -> str: def _request(method: str, url: str, *, json_body: dict | None = None, timeout: float = 30.0, verify: bool | str = True, auth: httpx.Auth | None = None) -> httpx.Response: + headers = {"Accept": "application/json"} logger.debug(f"YARN {method} {url}" + (f" body={json_body}" if json_body else "")) try: - resp = httpx.request(method, url, json=json_body, timeout=timeout, verify=verify, auth=auth) + resp = httpx.request( + method, url, json=json_body, headers=headers, timeout=timeout, verify=verify, auth=auth + ) except httpx.HTTPError as exc: logger.error(f"YARN {method} {url} failed: {exc}") raise YarnError(f"YARN connection failed: {exc}") from exc diff --git a/tests/unit/test_yarn_client.py b/tests/unit/test_yarn_client.py index 8e8ea22..7a3024f 100644 --- a/tests/unit/test_yarn_client.py +++ b/tests/unit/test_yarn_client.py @@ -62,6 +62,13 @@ def test_status_parses_app_state(): assert m.call_args.kwargs["verify"] is True +def test_request_sends_accept_json_header(): + fake = _resp(200, json_data={"app": {"id": "application_1", "state": "RUNNING"}}) + with patch("spark_executor.core.yarn_client.httpx.request", return_value=fake) as m: + get_application_status("application_1", YarnClientConfig(yarn_rm_url=RM)) + assert m.call_args.kwargs["headers"] == {"Accept": "application/json"} + + def test_status_raises_on_404(): with patch("spark_executor.core.yarn_client.httpx.request", return_value=_resp(404)): with pytest.raises(YarnError, match="not found"):