tao.chen/mcp-server
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

refactor(fetch_url): trust the allowlist, rename to url_allowlist

Browse Source 
Two changes:

1) Drop every check except the allowlist lookup.

  Old _validate_url_host did: scheme check, host-presence check,
  IP-literal check, empty-allowlist check, then glob match.

  New _validate_url_host does: parse host, return on glob match,
  raise on miss. That's it. The only remaining structural check is
  'the URL must have a host' (otherwise the glob has nothing to
  test against).

  Security implication: scheme (file://, gopher://, ftp://) and
  IP literals (10.0.0.1, ::1) are NO LONGER rejected by the
  validator. The allowlist is the single source of truth. If the
  user writes ['*.*.*.*'], they have opted in to 4-label hosts
  including IP literals; if they write ['ccam*'], they get ccam1-
  ccam99 and nothing else. The default ['ccam*'] / [] pattern is
  tight by construction.

  Removed: import ipaddress, the scheme/IP rejection branches, the
  'allowlist empty' explicit branch (the empty list naturally
  matches nothing).

2) Rename allowed_url_hosts -> url_allowlist.

  The previous name was a verbose double-negative ('allowed ... hosts').
  The new name is short, modern (allowlist > whitelist), and matches
  the pattern of the field (URL hosts allowed). Renamed in:
    - Connection (models.py)
    - SaveConnectionRequest, UpdateConnectionRequest, FetchUrlRequest
      (requests.py)
    - _validate_url_host, _host_matches_any_glob parameters
      (fetch_url.py)
    - save_connection / update_connection call sites and error
      messages (connections.py, fetch_url.py)
    - Route descriptions (server.py)
    - All test files
    - README

  No backward-compat alias: the field was added in 0291f36 and
  hasn't shipped, so no production migration. Local dev data
  (data/connections.json, gitignored) with allowed_url_hosts set
  will be silently dropped by Pydantic v2 (default for extra
  fields is ignore) — those connections lose their allowlist and
  fetch_url will reject everything until re-saved.

Test cleanup:
  - Removed 9 obsolete tests (scheme/IP/suffix rejection)
  - Renamed allowed_url_hosts -> url_allowlist in 11 surviving tests
  - Added 4 new tests documenting the 'allowlist is the only gate'
    model: IP literal accepted, HTTPS accepted, no-host rejected,
    empty allowlist rejected, error message mentions url_allowlist

  -1 obsolete test, net -4 from 386 -> 382 tests passing.

Co-Authored-By: Claude Fable 5 <noreply@anthropic.com>
This commit is contained in:
Claude
2026-07-09 12:09:41 +08:00
co-authored by Claude Fable 5
parent 6d7387b022
commit a5b9539663
8 changed files with 138 additions and 186 deletions
Download Patch File Download Diff File Expand all files Collapse all files
tests/unit/test_connection_tools.py
+7 -7
View File
@@ -148,10 +148,10 @@ def test_update_connection_with_no_fields_is_noop(_fresh_store):
assert out["deploy_mode"] == "client"
def test_update_connection_allows_url_hosts_change(_fresh_store):
def test_update_connection_changes_url_allowlist(_fresh_store):
connections.save_connection(name="prod", master="yarn")
out = connections.update_connection(name="prod", allowed_url_hosts=["ccam*"])
assert out["allowed_url_hosts"] == ["ccam*"]
out = connections.update_connection(name="prod", url_allowlist=["ccam*"])
assert out["url_allowlist"] == ["ccam*"]
def test_update_connection_replaces_not_merges_dict(_fresh_store):
@@ -160,14 +160,14 @@ def test_update_connection_replaces_not_merges_dict(_fresh_store):
assert out["spark_conf"] == {"b": "2"}
def test_update_connection_replaces_not_merges_list(_fresh_store):
def test_update_connection_replaces_not_merges_url_allowlist(_fresh_store):
connections.save_connection(
name="prod",
master="yarn",
allowed_url_hosts=["ccam*"],
url_allowlist=["ccam*"],
)
out = connections.update_connection(name="prod", allowed_url_hosts=["nm*"])
assert out["allowed_url_hosts"] == ["nm*"]
out = connections.update_connection(name="prod", url_allowlist=["nm*"])
assert out["url_allowlist"] == ["nm*"]
def test_update_connection_raises_for_unknown_name(_fresh_store):
tests/unit/test_fetch_url.py
+76 -105
View File
@@ -26,14 +26,14 @@ def fresh_stores(tmp_path: Path, monkeypatch):
_fresh_stores(tmp_path, monkeypatch)
yield
def test_fetch_url_returns_body_and_status(tmp_path, monkeypatch):
_fresh_stores(tmp_path, monkeypatch)
def test_fetch_url_returns_body_and_status(fresh_stores):
fetch_url.conn_store.save(
Connection(
name="prod",
master="yarn",
yarn_rm_url="http://rm.prod.internal:8088",
allowed_url_hosts=["*.prod.internal"],
url_allowlist=["*.prod.internal"],
)
)
resp = httpx.Response(200, text="hello", headers={"content-type": "text/html"})
@@ -47,89 +47,18 @@ def test_fetch_url_returns_body_and_status(tmp_path, monkeypatch):
assert m.call_count == 1
def test_fetch_url_raises_for_missing_connection(tmp_path, monkeypatch):
_fresh_stores(tmp_path, monkeypatch)
def test_fetch_url_raises_for_missing_connection(fresh_stores):
with pytest.raises(KeyError, match="Connection not found"):
fetch_url.fetch_url("http://rm.prod.internal:8088/", "missing")
def test_fetch_url_rejects_non_http_scheme(tmp_path, monkeypatch):
_fresh_stores(tmp_path, monkeypatch)
fetch_url.conn_store.save(
Connection(name="prod", master="yarn", yarn_rm_url="http://rm.prod.internal:8088")
)
with pytest.raises(ValueError, match="scheme"):
fetch_url.fetch_url("file:///etc/passwd", "prod")
def test_fetch_url_rejects_ftp_scheme(tmp_path, monkeypatch):
_fresh_stores(tmp_path, monkeypatch)
fetch_url.conn_store.save(
Connection(name="prod", master="yarn", yarn_rm_url="http://rm.prod.internal:8088")
)
with pytest.raises(ValueError, match="scheme"):
fetch_url.fetch_url("ftp://rm.prod.internal/foo", "prod")
def test_fetch_url_rejects_ip_literal(tmp_path, monkeypatch):
_fresh_stores(tmp_path, monkeypatch)
fetch_url.conn_store.save(
Connection(name="prod", master="yarn", yarn_rm_url="http://rm.prod.internal:8088")
)
with pytest.raises(ValueError, match="IP literal"):
fetch_url.fetch_url("http://10.0.0.1/secrets", "prod")
def test_fetch_url_rejects_ipv6_literal(tmp_path, monkeypatch):
_fresh_stores(tmp_path, monkeypatch)
fetch_url.conn_store.save(
Connection(name="prod", master="yarn", yarn_rm_url="http://rm.prod.internal:8088")
)
with pytest.raises(ValueError, match="IP literal"):
fetch_url.fetch_url("http://[::1]:8080/", "prod")
def test_fetch_url_rejects_external_host(tmp_path, monkeypatch):
_fresh_stores(tmp_path, monkeypatch)
def test_fetch_url_truncates_body_over_1mb(fresh_stores):
fetch_url.conn_store.save(
Connection(
name="prod",
master="yarn",
yarn_rm_url="http://rm.prod.internal:8088",
allowed_url_hosts=["*.prod.internal"],
)
)
with pytest.raises(ValueError, match="is not in Connection.allowed_url_hosts"):
fetch_url.fetch_url("http://evil.com/foo", "prod")
def test_fetch_url_rejects_unrelated_single_label_host(tmp_path, monkeypatch):
"""Without a common suffix rule, a single-label RM host no longer helps."""
_fresh_stores(tmp_path, monkeypatch)
fetch_url.conn_store.save(
Connection(name="prod", master="yarn", yarn_rm_url="http://rm:8088")
)
with pytest.raises(ValueError, match="allowed_url_hosts is empty"):
fetch_url.fetch_url("http://other-rm/", "prod")
def test_fetch_url_rejects_when_allowed_url_hosts_empty(tmp_path, monkeypatch):
_fresh_stores(tmp_path, monkeypatch)
fetch_url.conn_store.save(
Connection(name="prod", master="yarn", allowed_url_hosts=[])
)
with pytest.raises(ValueError, match="allowed_url_hosts is empty"):
fetch_url.fetch_url("http://anything.com/", "prod")
def test_fetch_url_truncates_body_over_1mb(tmp_path, monkeypatch):
_fresh_stores(tmp_path, monkeypatch)
fetch_url.conn_store.save(
Connection(
name="prod",
master="yarn",
yarn_rm_url="http://rm.prod.internal:8088",
allowed_url_hosts=["*.prod.internal"],
url_allowlist=["*.prod.internal"],
)
)
big = "x" * (fetch_url._MAX_BODY_BYTES + 1)
@@ -140,8 +69,7 @@ def test_fetch_url_truncates_body_over_1mb(tmp_path, monkeypatch):
assert len(out.body) == fetch_url._MAX_BODY_BYTES
def test_fetch_url_passes_auth_from_connection(tmp_path, monkeypatch):
_fresh_stores(tmp_path, monkeypatch)
def test_fetch_url_passes_auth_from_connection(fresh_stores):
fetch_url.conn_store.save(
Connection(
name="auth",
@@ -150,7 +78,7 @@ def test_fetch_url_passes_auth_from_connection(tmp_path, monkeypatch):
auth_type="basic",
auth_user="u",
auth_password="p",
allowed_url_hosts=["*.prod.internal"],
url_allowlist=["*.prod.internal"],
)
)
resp = httpx.Response(200, text="ok")
@@ -159,19 +87,19 @@ def test_fetch_url_passes_auth_from_connection(tmp_path, monkeypatch):
auth = m.call_args.kwargs["auth"]
assert isinstance(auth, httpx.BasicAuth)
import base64
creds = base64.b64decode(auth._auth_header.split()[1]).decode()
assert creds == "u:p"
def test_fetch_url_passes_ssl_verify_from_connection(tmp_path, monkeypatch):
_fresh_stores(tmp_path, monkeypatch)
def test_fetch_url_passes_ssl_verify_from_connection(fresh_stores):
fetch_url.conn_store.save(
Connection(
name="insecure",
master="yarn",
yarn_rm_url="http://rm.prod.internal:8088",
ssl_verify=False,
allowed_url_hosts=["*.prod.internal"],
url_allowlist=["*.prod.internal"],
)
)
resp = httpx.Response(200, text="ok")
@@ -180,14 +108,13 @@ def test_fetch_url_passes_ssl_verify_from_connection(tmp_path, monkeypatch):
assert m.call_args.kwargs["verify"] is False
def test_fetch_url_follows_redirects(tmp_path, monkeypatch):
_fresh_stores(tmp_path, monkeypatch)
def test_fetch_url_follows_redirects(fresh_stores):
fetch_url.conn_store.save(
Connection(
name="prod",
master="yarn",
yarn_rm_url="http://rm.prod.internal:8088",
allowed_url_hosts=["*.prod.internal"],
url_allowlist=["*.prod.internal"],
)
)
resp = httpx.Response(200, text="ok")
@@ -195,13 +122,14 @@ def test_fetch_url_follows_redirects(tmp_path, monkeypatch):
fetch_url.fetch_url("http://nm01.prod.internal:8042/node", "prod")
assert m.call_args.kwargs["follow_redirects"] is True
def test_fetch_url_allows_host_matching_glob_pattern(fresh_stores):
fetch_url.conn_store.save(
Connection(
name="ccam",
master="yarn",
yarn_rm_url="http://ccam1:8088",
allowed_url_hosts=["ccam*"],
url_allowlist=["ccam*"],
)
)
resp = httpx.Response(200, text="hello")
@@ -211,53 +139,59 @@ def test_fetch_url_allows_host_matching_glob_pattern(fresh_stores):
assert out.body == "hello"
assert m.call_count == 1
def test_fetch_url_allows_host_matching_any_of_multiple_globs(fresh_stores):
fetch_url.conn_store.save(
Connection(
name="prod",
master="yarn",
yarn_rm_url="http://rm.prod.internal:8088",
allowed_url_hosts=["ccam*", "*.prod.internal"],
url_allowlist=["ccam*", "*.prod.internal"],
)
)
resp = httpx.Response(200, text="hello")
with patch("spark_executor.tools.fetch_url.httpx.get", return_value=resp) as m:
out = fetch_url.fetch_url("http://history.prod.internal:18080/api/v1/info", "prod")
out = fetch_url.fetch_url(
"http://history.prod.internal:18080/api/v1/info", "prod"
)
assert out.status_code == 200
assert out.body == "hello"
assert m.call_count == 1
def test_fetch_url_glob_does_not_match_unrelated_host(fresh_stores):
fetch_url.conn_store.save(
Connection(
name="ccam",
master="yarn",
yarn_rm_url="http://ccam1:8088",
allowed_url_hosts=["ccam*"],
url_allowlist=["ccam*"],
)
)
with pytest.raises(ValueError, match="is not in Connection.allowed_url_hosts"):
with pytest.raises(ValueError, match="is not in Connection.url_allowlist"):
fetch_url.fetch_url("http://evil.com/foo", "ccam")
def test_fetch_url_glob_does_not_cross_dot_boundary(fresh_stores):
fetch_url.conn_store.save(
Connection(
name="ccam",
master="yarn",
yarn_rm_url="http://ccam1:8088",
allowed_url_hosts=["ccam*"],
url_allowlist=["ccam*"],
)
)
with pytest.raises(ValueError, match="is not in Connection.allowed_url_hosts"):
with pytest.raises(ValueError, match="is not in Connection.url_allowlist"):
fetch_url.fetch_url("http://ccam50.evil.com/", "ccam")
def test_fetch_url_glob_match_does_not_require_suffix_overlap(fresh_stores):
fetch_url.conn_store.save(
Connection(
name="prod",
master="yarn",
yarn_rm_url="http://rm:8088",
allowed_url_hosts=["ccam*"],
url_allowlist=["ccam*"],
)
)
resp = httpx.Response(200, text="hello")
@@ -267,30 +201,55 @@ def test_fetch_url_glob_match_does_not_require_suffix_overlap(fresh_stores):
assert out.body == "hello"
assert m.call_count == 1
def test_fetch_url_empty_allowed_hosts_rejects_everything(fresh_stores):
def test_fetch_url_accepts_ip_literal_when_in_url_allowlist(fresh_stores):
fetch_url.conn_store.save(
Connection(
name="prod",
master="yarn",
yarn_rm_url="http://rm.prod.internal",
allowed_url_hosts=[],
url_allowlist=["*.*.*.*"],
)
)
with pytest.raises(ValueError, match="allowed_url_hosts is empty"):
fetch_url.fetch_url("http://nm.prod.internal/", "prod")
resp = httpx.Response(200, text="hello")
with patch("spark_executor.tools.fetch_url.httpx.get", return_value=resp) as m:
out = fetch_url.fetch_url("http://10.0.0.1/secret", "prod")
assert out.status_code == 200
assert out.body == "hello"
assert m.call_count == 1
def test_fetch_url_omitted_allowed_url_hosts_defaults_to_empty_and_rejects(fresh_stores):
def test_fetch_url_accepts_https_when_in_url_allowlist(fresh_stores):
fetch_url.conn_store.save(
Connection(
name="prod",
master="yarn",
yarn_rm_url="http://rm.prod.internal",
url_allowlist=["ccam*.example.com"],
)
)
with pytest.raises(ValueError, match="allowed_url_hosts is empty"):
fetch_url.fetch_url("http://nm.prod.internal/", "prod")
resp = httpx.Response(200, text="hello")
with patch("spark_executor.tools.fetch_url.httpx.get", return_value=resp) as m:
out = fetch_url.fetch_url("https://ccam1.example.com/secure", "prod")
assert out.status_code == 200
assert out.body == "hello"
assert m.call_count == 1
def test_fetch_url_error_message_hints_at_allowed_url_hosts(fresh_stores):
def test_fetch_url_rejects_when_url_has_no_host(fresh_stores):
with pytest.raises(ValueError, match="URL has no host"):
fetch_url._validate_url_host("", ["*"])
with pytest.raises(ValueError, match="URL has no host"):
fetch_url._validate_url_host("not-a-url", ["*"])
def test_fetch_url_rejects_when_url_allowlist_empty(fresh_stores):
fetch_url.conn_store.save(
Connection(name="prod", master="yarn", url_allowlist=[])
)
with pytest.raises(ValueError, match="is not in Connection.url_allowlist"):
fetch_url.fetch_url("http://anything.com/", "prod")
def test_fetch_url_error_message_mentions_url_allowlist(fresh_stores):
fetch_url.conn_store.save(
Connection(
name="prod",
@@ -298,5 +257,17 @@ def test_fetch_url_error_message_hints_at_allowed_url_hosts(fresh_stores):
yarn_rm_url="http://rm.prod.internal:8088",
)
)
with pytest.raises(ValueError, match="set Connection.allowed_url_hosts"):
with pytest.raises(ValueError, match="url_allowlist"):
fetch_url.fetch_url("http://evil.com/foo", "prod")
def test_fetch_url_omitted_url_allowlist_defaults_to_empty_and_rejects(fresh_stores):
fetch_url.conn_store.save(
Connection(
name="prod",
master="yarn",
yarn_rm_url="http://rm.prod.internal",
)
)
with pytest.raises(ValueError, match="is not in Connection.url_allowlist"):
fetch_url.fetch_url("http://nm.prod.internal/", "prod")