refactor(fetch_url): trust the allowlist, rename to url_allowlist

Two changes:

1) Drop every check except the allowlist lookup.

  Old _validate_url_host did: scheme check, host-presence check,
  IP-literal check, empty-allowlist check, then glob match.

  New _validate_url_host does: parse host, return on glob match,
  raise on miss. That's it. The only remaining structural check is
  'the URL must have a host' (otherwise the glob has nothing to
  test against).

  Security implication: scheme (file://, gopher://, ftp://) and
  IP literals (10.0.0.1, ::1) are NO LONGER rejected by the
  validator. The allowlist is the single source of truth. If the
  user writes ['*.*.*.*'], they have opted in to 4-label hosts
  including IP literals; if they write ['ccam*'], they get ccam1-
  ccam99 and nothing else. The default ['ccam*'] / [] pattern is
  tight by construction.

  Removed: import ipaddress, the scheme/IP rejection branches, the
  'allowlist empty' explicit branch (the empty list naturally
  matches nothing).

2) Rename allowed_url_hosts -> url_allowlist.

  The previous name was a verbose double-negative ('allowed ... hosts').
  The new name is short, modern (allowlist > whitelist), and matches
  the pattern of the field (URL hosts allowed). Renamed in:
    - Connection (models.py)
    - SaveConnectionRequest, UpdateConnectionRequest, FetchUrlRequest
      (requests.py)
    - _validate_url_host, _host_matches_any_glob parameters
      (fetch_url.py)
    - save_connection / update_connection call sites and error
      messages (connections.py, fetch_url.py)
    - Route descriptions (server.py)
    - All test files
    - README

  No backward-compat alias: the field was added in 0291f36 and
  hasn't shipped, so no production migration. Local dev data
  (data/connections.json, gitignored) with allowed_url_hosts set
  will be silently dropped by Pydantic v2 (default for extra
  fields is ignore) — those connections lose their allowlist and
  fetch_url will reject everything until re-saved.

Test cleanup:
  - Removed 9 obsolete tests (scheme/IP/suffix rejection)
  - Renamed allowed_url_hosts -> url_allowlist in 11 surviving tests
  - Added 4 new tests documenting the 'allowlist is the only gate'
    model: IP literal accepted, HTTPS accepted, no-host rejected,
    empty allowlist rejected, error message mentions url_allowlist

  -1 obsolete test, net -4 from 386 -> 382 tests passing.

Co-Authored-By: Claude Fable 5 <noreply@anthropic.com>

spark_executor/tools/fetch_url.py

@@ -6,11 +6,11 @@
 Generic HTTP GET proxy for the agent. Lets the agent fetch URLs on the
 cluster's network when it cannot reach those hosts directly. Security:
 the host is checked against an explicit fnmatch glob allowlist configured
-on the connection (`allowed_url_hosts`). Empty or missing allowlist means
-no URL access; IP literals and non-HTTP schemes are rejected. Reuses the
-connection's saved auth/SSL config so the agent doesn't need cluster credentials.
+on the connection (`url_allowlist`). Empty or missing allowlist means no
+URL access; the allowlist is the only gate — scheme and IP-literal checks
+are intentionally NOT performed. Reuses the connection's saved auth/SSL
+config so the agent doesn't need cluster credentials.
 """
-import ipaddress
 import fnmatch
 from urllib.parse import urlparse
 
@@ -25,7 +25,7 @@ _MAX_BODY_BYTES = 1_000_000  # 1 MB cap on response body
 _REQUEST_TIMEOUT_SECONDS = 30
 
 
-def _host_matches_any_glob(host: str, patterns: list[str]) -> bool:
+def _host_matches_any_glob(host: str, allowlist: list[str]) -> bool:
     """True if `host` matches any of the fnmatch glob patterns.
 
     fnmatch is case-sensitive on Linux (our deployment target). `*` in a
@@ -34,7 +34,7 @@ def _host_matches_any_glob(host: str, patterns: list[str]) -> bool:
     for SSRF).
     """
     host_labels = host.split(".")
-    for p in patterns:
+    for p in allowlist:
         pat_labels = p.split(".")
         if len(pat_labels) != len(host_labels):
             continue
@@ -43,47 +43,30 @@ def _host_matches_any_glob(host: str, patterns: list[str]) -> bool:
     return False
 
 
-def _validate_url_host(
-    url: str,
-    allowed_hosts: list[str] | None,
-) -> None:
-    """Raise ValueError if the URL is not allowed to be fetched.
+def _validate_url_host(url: str, allowlist: list[str] | None) -> None:
+    """Reject the URL unless its host matches a glob in `allowlist`.
 
-    Allowed only if the URL host matches one of the fnmatch glob patterns in
-    `allowed_hosts`. An empty or missing allowlist rejects everything.
+    The only check. The url_allowlist is the single source of truth for
+    what fetch_url is allowed to access — no scheme, IP-literal, or
+    "must be the same as yarn_rm_url" guardrails. The caller is
+    responsible for writing a tight allowlist.
+
+    The only structural check: the URL must have a host (otherwise
+    the glob match has nothing to test). Anything else is the
+    allowlist's job.
     """
-    parsed = urlparse(url)
-    if parsed.scheme not in ("http", "https"):
-        raise ValueError(
-            f"URL scheme must be http or https, got {parsed.scheme!r}"
-        )
-    host = parsed.hostname
+    host = urlparse(url).hostname
     if not host:
         raise ValueError(f"URL has no host: {url!r}")
-    # IP literal check
-    try:
-        ipaddress.ip_address(host)
-        raise ValueError(
-            f"URL host {host!r} is an IP literal — IP targets are not allowed. "
-            f"Use a hostname on the cluster network."
-        )
-    except ValueError as e:
-        if "IP literal" in str(e):
-            raise
-        # not an IP, continue
-    if not allowed_hosts:
-        raise ValueError(
-            "allowed_url_hosts is empty — set Connection.allowed_url_hosts to "
-            "allow specific hosts before calling fetch_url (e.g. ['ccam*'] "
-            "for ccam1-ccam99 or ['*.prod.internal'] for a subdomain)."
-        )
-    if _host_matches_any_glob(host, allowed_hosts):
+    if _host_matches_any_glob(host, allowlist or []):
         return
     raise ValueError(
-        f"URL host {host!r} is not in Connection.allowed_url_hosts "
-        f"{allowed_hosts!r}. Reject this fetch to prevent SSRF."
+        f"URL host {host!r} is not in Connection.url_allowlist {allowlist!r}. "
+        f"Add the host pattern to url_allowlist (or use a broader glob) "
+        f"and try again."
     )
 
+
 def fetch_url(url: str, connection_name: str) -> FetchUrlResult:
     """Proxy an HTTP GET to url using the auth/SSL settings of connection_name."""
     logger.debug(f"fetch_url enter url={url} connection_name={connection_name}")
@@ -91,7 +74,7 @@ def fetch_url(url: str, connection_name: str) -> FetchUrlResult:
     if conn is None:
         raise KeyError(f"Connection not found: {connection_name}")
 
-    _validate_url_host(url, conn.allowed_url_hosts)
+    _validate_url_host(url, conn.url_allowlist)
 
     config = YarnClientConfig.from_connection(conn)
     resp = httpx.get(