ClaudeandClaude Fable 5
a5b9539663
Two changes:
1) Drop every check except the allowlist lookup.
Old _validate_url_host did: scheme check, host-presence check,
IP-literal check, empty-allowlist check, then glob match.
New _validate_url_host does: parse host, return on glob match,
raise on miss. That's it. The only remaining structural check is
'the URL must have a host' (otherwise the glob has nothing to
test against).
Security implication: scheme (file://, gopher://, ftp://) and
IP literals (10.0.0.1, ::1) are NO LONGER rejected by the
validator. The allowlist is the single source of truth. If the
user writes ['*.*.*.*'], they have opted in to 4-label hosts
including IP literals; if they write ['ccam*'], they get ccam1-
ccam99 and nothing else. The default ['ccam*'] / [] pattern is
tight by construction.
Removed: import ipaddress, the scheme/IP rejection branches, the
'allowlist empty' explicit branch (the empty list naturally
matches nothing).
2) Rename allowed_url_hosts -> url_allowlist.
The previous name was a verbose double-negative ('allowed ... hosts').
The new name is short, modern (allowlist > whitelist), and matches
the pattern of the field (URL hosts allowed). Renamed in:
- Connection (models.py)
- SaveConnectionRequest, UpdateConnectionRequest, FetchUrlRequest
(requests.py)
- _validate_url_host, _host_matches_any_glob parameters
(fetch_url.py)
- save_connection / update_connection call sites and error
messages (connections.py, fetch_url.py)
- Route descriptions (server.py)
- All test files
- README
No backward-compat alias: the field was added in 0291f36 and
hasn't shipped, so no production migration. Local dev data
(data/connections.json, gitignored) with allowed_url_hosts set
will be silently dropped by Pydantic v2 (default for extra
fields is ignore) — those connections lose their allowlist and
fetch_url will reject everything until re-saved.
Test cleanup:
- Removed 9 obsolete tests (scheme/IP/suffix rejection)
- Renamed allowed_url_hosts -> url_allowlist in 11 surviving tests
- Added 4 new tests documenting the 'allowlist is the only gate'
model: IP literal accepted, HTTPS accepted, no-host rejected,
empty allowlist rejected, error message mentions url_allowlist
-1 obsolete test, net -4 from 386 -> 382 tests passing.
Co-Authored-By: Claude Fable 5 <noreply@anthropic.com>
105 lines
3.6 KiB
Python
105 lines
3.6 KiB
Python
# coding=utf-8
|
|
"""
|
|
@Time :2026/7/9
|
|
@Author :tao.chen
|
|
|
|
Generic HTTP GET proxy for the agent. Lets the agent fetch URLs on the
|
|
cluster's network when it cannot reach those hosts directly. Security:
|
|
the host is checked against an explicit fnmatch glob allowlist configured
|
|
on the connection (`url_allowlist`). Empty or missing allowlist means no
|
|
URL access; the allowlist is the only gate — scheme and IP-literal checks
|
|
are intentionally NOT performed. Reuses the connection's saved auth/SSL
|
|
config so the agent doesn't need cluster credentials.
|
|
"""
|
|
import fnmatch
|
|
from urllib.parse import urlparse
|
|
|
|
import httpx
|
|
|
|
from common.logging import logger
|
|
from spark_executor.core.yarn_client import YarnClientConfig
|
|
from spark_executor.models import FetchUrlResult
|
|
from spark_executor.tools.connections import store as conn_store
|
|
|
|
_MAX_BODY_BYTES = 1_000_000 # 1 MB cap on response body
|
|
_REQUEST_TIMEOUT_SECONDS = 30
|
|
|
|
|
|
def _host_matches_any_glob(host: str, allowlist: list[str]) -> bool:
|
|
"""True if `host` matches any of the fnmatch glob patterns.
|
|
|
|
fnmatch is case-sensitive on Linux (our deployment target). `*` in a
|
|
pattern does NOT cross '.' boundaries, so 'ccam*' matches 'ccam50' but
|
|
NOT 'ccam50.evil.com' (which is what we want — domain boundary matters
|
|
for SSRF).
|
|
"""
|
|
host_labels = host.split(".")
|
|
for p in allowlist:
|
|
pat_labels = p.split(".")
|
|
if len(pat_labels) != len(host_labels):
|
|
continue
|
|
if all(fnmatch.fnmatchcase(h, pat) for h, pat in zip(host_labels, pat_labels)):
|
|
return True
|
|
return False
|
|
|
|
|
|
def _validate_url_host(url: str, allowlist: list[str] | None) -> None:
|
|
"""Reject the URL unless its host matches a glob in `allowlist`.
|
|
|
|
The only check. The url_allowlist is the single source of truth for
|
|
what fetch_url is allowed to access — no scheme, IP-literal, or
|
|
"must be the same as yarn_rm_url" guardrails. The caller is
|
|
responsible for writing a tight allowlist.
|
|
|
|
The only structural check: the URL must have a host (otherwise
|
|
the glob match has nothing to test). Anything else is the
|
|
allowlist's job.
|
|
"""
|
|
host = urlparse(url).hostname
|
|
if not host:
|
|
raise ValueError(f"URL has no host: {url!r}")
|
|
if _host_matches_any_glob(host, allowlist or []):
|
|
return
|
|
raise ValueError(
|
|
f"URL host {host!r} is not in Connection.url_allowlist {allowlist!r}. "
|
|
f"Add the host pattern to url_allowlist (or use a broader glob) "
|
|
f"and try again."
|
|
)
|
|
|
|
|
|
def fetch_url(url: str, connection_name: str) -> FetchUrlResult:
|
|
"""Proxy an HTTP GET to url using the auth/SSL settings of connection_name."""
|
|
logger.debug(f"fetch_url enter url={url} connection_name={connection_name}")
|
|
conn = conn_store.get(connection_name)
|
|
if conn is None:
|
|
raise KeyError(f"Connection not found: {connection_name}")
|
|
|
|
_validate_url_host(url, conn.url_allowlist)
|
|
|
|
config = YarnClientConfig.from_connection(conn)
|
|
resp = httpx.get(
|
|
url,
|
|
auth=config.auth_for_httpx(),
|
|
verify=config.verify_for_httpx(),
|
|
timeout=_REQUEST_TIMEOUT_SECONDS,
|
|
follow_redirects=True,
|
|
)
|
|
|
|
body = resp.text[:_MAX_BODY_BYTES]
|
|
truncated = len(resp.text) > _MAX_BODY_BYTES
|
|
|
|
logger.info(
|
|
f"fetch_url ok url={url} connection_name={connection_name} "
|
|
f"status_code={resp.status_code} "
|
|
f"content_type={resp.headers.get('content-type', '')} "
|
|
f"body_bytes={len(body)} truncated={truncated}"
|
|
)
|
|
|
|
return FetchUrlResult(
|
|
url=url,
|
|
status_code=resp.status_code,
|
|
content_type=resp.headers.get("content-type", ""),
|
|
body=body,
|
|
truncated=truncated,
|
|
)
|