ClaudeandClaude Fable 5
a5b9539663
Two changes:
1) Drop every check except the allowlist lookup.
Old _validate_url_host did: scheme check, host-presence check,
IP-literal check, empty-allowlist check, then glob match.
New _validate_url_host does: parse host, return on glob match,
raise on miss. That's it. The only remaining structural check is
'the URL must have a host' (otherwise the glob has nothing to
test against).
Security implication: scheme (file://, gopher://, ftp://) and
IP literals (10.0.0.1, ::1) are NO LONGER rejected by the
validator. The allowlist is the single source of truth. If the
user writes ['*.*.*.*'], they have opted in to 4-label hosts
including IP literals; if they write ['ccam*'], they get ccam1-
ccam99 and nothing else. The default ['ccam*'] / [] pattern is
tight by construction.
Removed: import ipaddress, the scheme/IP rejection branches, the
'allowlist empty' explicit branch (the empty list naturally
matches nothing).
2) Rename allowed_url_hosts -> url_allowlist.
The previous name was a verbose double-negative ('allowed ... hosts').
The new name is short, modern (allowlist > whitelist), and matches
the pattern of the field (URL hosts allowed). Renamed in:
- Connection (models.py)
- SaveConnectionRequest, UpdateConnectionRequest, FetchUrlRequest
(requests.py)
- _validate_url_host, _host_matches_any_glob parameters
(fetch_url.py)
- save_connection / update_connection call sites and error
messages (connections.py, fetch_url.py)
- Route descriptions (server.py)
- All test files
- README
No backward-compat alias: the field was added in 0291f36 and
hasn't shipped, so no production migration. Local dev data
(data/connections.json, gitignored) with allowed_url_hosts set
will be silently dropped by Pydantic v2 (default for extra
fields is ignore) — those connections lose their allowlist and
fetch_url will reject everything until re-saved.
Test cleanup:
- Removed 9 obsolete tests (scheme/IP/suffix rejection)
- Renamed allowed_url_hosts -> url_allowlist in 11 surviving tests
- Added 4 new tests documenting the 'allowlist is the only gate'
model: IP literal accepted, HTTPS accepted, no-host rejected,
empty allowlist rejected, error message mentions url_allowlist
-1 obsolete test, net -4 from 386 -> 382 tests passing.
Co-Authored-By: Claude Fable 5 <noreply@anthropic.com>
274 lines
9.1 KiB
Python
274 lines
9.1 KiB
Python
# coding=utf-8
|
|
from pathlib import Path
|
|
from unittest.mock import patch
|
|
|
|
import httpx
|
|
import pytest
|
|
|
|
from spark_executor.core import connection_store
|
|
from spark_executor.core.connection_store import ConnectionStore
|
|
from spark_executor.models import Connection
|
|
from spark_executor.tools import connections, fetch_url
|
|
|
|
|
|
def _fresh_stores(tmp_path, monkeypatch):
|
|
"""Reset connection store singletons for a single test."""
|
|
monkeypatch.setattr(connection_store, "DEFAULT_DATA_DIR", str(tmp_path))
|
|
store = ConnectionStore()
|
|
monkeypatch.setattr(connection_store, "store", store)
|
|
connections.store = store
|
|
fetch_url.conn_store = store
|
|
|
|
|
|
@pytest.fixture
|
|
def fresh_stores(tmp_path: Path, monkeypatch):
|
|
"""Reset connection store singletons for a single test."""
|
|
_fresh_stores(tmp_path, monkeypatch)
|
|
yield
|
|
|
|
|
|
def test_fetch_url_returns_body_and_status(fresh_stores):
|
|
fetch_url.conn_store.save(
|
|
Connection(
|
|
name="prod",
|
|
master="yarn",
|
|
yarn_rm_url="http://rm.prod.internal:8088",
|
|
url_allowlist=["*.prod.internal"],
|
|
)
|
|
)
|
|
resp = httpx.Response(200, text="hello", headers={"content-type": "text/html"})
|
|
with patch("spark_executor.tools.fetch_url.httpx.get", return_value=resp) as m:
|
|
out = fetch_url.fetch_url("http://nm01.prod.internal:8042/node", "prod")
|
|
assert out.url == "http://nm01.prod.internal:8042/node"
|
|
assert out.status_code == 200
|
|
assert out.content_type == "text/html"
|
|
assert out.body == "hello"
|
|
assert out.truncated is False
|
|
assert m.call_count == 1
|
|
|
|
|
|
def test_fetch_url_raises_for_missing_connection(fresh_stores):
|
|
with pytest.raises(KeyError, match="Connection not found"):
|
|
fetch_url.fetch_url("http://rm.prod.internal:8088/", "missing")
|
|
|
|
|
|
def test_fetch_url_truncates_body_over_1mb(fresh_stores):
|
|
fetch_url.conn_store.save(
|
|
Connection(
|
|
name="prod",
|
|
master="yarn",
|
|
yarn_rm_url="http://rm.prod.internal:8088",
|
|
url_allowlist=["*.prod.internal"],
|
|
)
|
|
)
|
|
big = "x" * (fetch_url._MAX_BODY_BYTES + 1)
|
|
resp = httpx.Response(200, text=big)
|
|
with patch("spark_executor.tools.fetch_url.httpx.get", return_value=resp):
|
|
out = fetch_url.fetch_url("http://nm01.prod.internal/big", "prod")
|
|
assert out.truncated is True
|
|
assert len(out.body) == fetch_url._MAX_BODY_BYTES
|
|
|
|
|
|
def test_fetch_url_passes_auth_from_connection(fresh_stores):
|
|
fetch_url.conn_store.save(
|
|
Connection(
|
|
name="auth",
|
|
master="yarn",
|
|
yarn_rm_url="http://rm.prod.internal:8088",
|
|
auth_type="basic",
|
|
auth_user="u",
|
|
auth_password="p",
|
|
url_allowlist=["*.prod.internal"],
|
|
)
|
|
)
|
|
resp = httpx.Response(200, text="ok")
|
|
with patch("spark_executor.tools.fetch_url.httpx.get", return_value=resp) as m:
|
|
fetch_url.fetch_url("http://nm01.prod.internal:8042/node", "auth")
|
|
auth = m.call_args.kwargs["auth"]
|
|
assert isinstance(auth, httpx.BasicAuth)
|
|
import base64
|
|
|
|
creds = base64.b64decode(auth._auth_header.split()[1]).decode()
|
|
assert creds == "u:p"
|
|
|
|
|
|
def test_fetch_url_passes_ssl_verify_from_connection(fresh_stores):
|
|
fetch_url.conn_store.save(
|
|
Connection(
|
|
name="insecure",
|
|
master="yarn",
|
|
yarn_rm_url="http://rm.prod.internal:8088",
|
|
ssl_verify=False,
|
|
url_allowlist=["*.prod.internal"],
|
|
)
|
|
)
|
|
resp = httpx.Response(200, text="ok")
|
|
with patch("spark_executor.tools.fetch_url.httpx.get", return_value=resp) as m:
|
|
fetch_url.fetch_url("http://nm01.prod.internal:8042/node", "insecure")
|
|
assert m.call_args.kwargs["verify"] is False
|
|
|
|
|
|
def test_fetch_url_follows_redirects(fresh_stores):
|
|
fetch_url.conn_store.save(
|
|
Connection(
|
|
name="prod",
|
|
master="yarn",
|
|
yarn_rm_url="http://rm.prod.internal:8088",
|
|
url_allowlist=["*.prod.internal"],
|
|
)
|
|
)
|
|
resp = httpx.Response(200, text="ok")
|
|
with patch("spark_executor.tools.fetch_url.httpx.get", return_value=resp) as m:
|
|
fetch_url.fetch_url("http://nm01.prod.internal:8042/node", "prod")
|
|
assert m.call_args.kwargs["follow_redirects"] is True
|
|
|
|
|
|
def test_fetch_url_allows_host_matching_glob_pattern(fresh_stores):
|
|
fetch_url.conn_store.save(
|
|
Connection(
|
|
name="ccam",
|
|
master="yarn",
|
|
yarn_rm_url="http://ccam1:8088",
|
|
url_allowlist=["ccam*"],
|
|
)
|
|
)
|
|
resp = httpx.Response(200, text="hello")
|
|
with patch("spark_executor.tools.fetch_url.httpx.get", return_value=resp) as m:
|
|
out = fetch_url.fetch_url("http://ccam50:8088/foo", "ccam")
|
|
assert out.status_code == 200
|
|
assert out.body == "hello"
|
|
assert m.call_count == 1
|
|
|
|
|
|
def test_fetch_url_allows_host_matching_any_of_multiple_globs(fresh_stores):
|
|
fetch_url.conn_store.save(
|
|
Connection(
|
|
name="prod",
|
|
master="yarn",
|
|
yarn_rm_url="http://rm.prod.internal:8088",
|
|
url_allowlist=["ccam*", "*.prod.internal"],
|
|
)
|
|
)
|
|
resp = httpx.Response(200, text="hello")
|
|
with patch("spark_executor.tools.fetch_url.httpx.get", return_value=resp) as m:
|
|
out = fetch_url.fetch_url(
|
|
"http://history.prod.internal:18080/api/v1/info", "prod"
|
|
)
|
|
assert out.status_code == 200
|
|
assert out.body == "hello"
|
|
assert m.call_count == 1
|
|
|
|
|
|
def test_fetch_url_glob_does_not_match_unrelated_host(fresh_stores):
|
|
fetch_url.conn_store.save(
|
|
Connection(
|
|
name="ccam",
|
|
master="yarn",
|
|
yarn_rm_url="http://ccam1:8088",
|
|
url_allowlist=["ccam*"],
|
|
)
|
|
)
|
|
with pytest.raises(ValueError, match="is not in Connection.url_allowlist"):
|
|
fetch_url.fetch_url("http://evil.com/foo", "ccam")
|
|
|
|
|
|
def test_fetch_url_glob_does_not_cross_dot_boundary(fresh_stores):
|
|
fetch_url.conn_store.save(
|
|
Connection(
|
|
name="ccam",
|
|
master="yarn",
|
|
yarn_rm_url="http://ccam1:8088",
|
|
url_allowlist=["ccam*"],
|
|
)
|
|
)
|
|
with pytest.raises(ValueError, match="is not in Connection.url_allowlist"):
|
|
fetch_url.fetch_url("http://ccam50.evil.com/", "ccam")
|
|
|
|
|
|
def test_fetch_url_glob_match_does_not_require_suffix_overlap(fresh_stores):
|
|
fetch_url.conn_store.save(
|
|
Connection(
|
|
name="prod",
|
|
master="yarn",
|
|
yarn_rm_url="http://rm:8088",
|
|
url_allowlist=["ccam*"],
|
|
)
|
|
)
|
|
resp = httpx.Response(200, text="hello")
|
|
with patch("spark_executor.tools.fetch_url.httpx.get", return_value=resp) as m:
|
|
out = fetch_url.fetch_url("http://ccam50:8088/", "prod")
|
|
assert out.status_code == 200
|
|
assert out.body == "hello"
|
|
assert m.call_count == 1
|
|
|
|
|
|
def test_fetch_url_accepts_ip_literal_when_in_url_allowlist(fresh_stores):
|
|
fetch_url.conn_store.save(
|
|
Connection(
|
|
name="prod",
|
|
master="yarn",
|
|
url_allowlist=["*.*.*.*"],
|
|
)
|
|
)
|
|
resp = httpx.Response(200, text="hello")
|
|
with patch("spark_executor.tools.fetch_url.httpx.get", return_value=resp) as m:
|
|
out = fetch_url.fetch_url("http://10.0.0.1/secret", "prod")
|
|
assert out.status_code == 200
|
|
assert out.body == "hello"
|
|
assert m.call_count == 1
|
|
|
|
|
|
def test_fetch_url_accepts_https_when_in_url_allowlist(fresh_stores):
|
|
fetch_url.conn_store.save(
|
|
Connection(
|
|
name="prod",
|
|
master="yarn",
|
|
url_allowlist=["ccam*.example.com"],
|
|
)
|
|
)
|
|
resp = httpx.Response(200, text="hello")
|
|
with patch("spark_executor.tools.fetch_url.httpx.get", return_value=resp) as m:
|
|
out = fetch_url.fetch_url("https://ccam1.example.com/secure", "prod")
|
|
assert out.status_code == 200
|
|
assert out.body == "hello"
|
|
assert m.call_count == 1
|
|
|
|
|
|
def test_fetch_url_rejects_when_url_has_no_host(fresh_stores):
|
|
with pytest.raises(ValueError, match="URL has no host"):
|
|
fetch_url._validate_url_host("", ["*"])
|
|
with pytest.raises(ValueError, match="URL has no host"):
|
|
fetch_url._validate_url_host("not-a-url", ["*"])
|
|
|
|
|
|
def test_fetch_url_rejects_when_url_allowlist_empty(fresh_stores):
|
|
fetch_url.conn_store.save(
|
|
Connection(name="prod", master="yarn", url_allowlist=[])
|
|
)
|
|
with pytest.raises(ValueError, match="is not in Connection.url_allowlist"):
|
|
fetch_url.fetch_url("http://anything.com/", "prod")
|
|
|
|
|
|
def test_fetch_url_error_message_mentions_url_allowlist(fresh_stores):
|
|
fetch_url.conn_store.save(
|
|
Connection(
|
|
name="prod",
|
|
master="yarn",
|
|
yarn_rm_url="http://rm.prod.internal:8088",
|
|
)
|
|
)
|
|
with pytest.raises(ValueError, match="url_allowlist"):
|
|
fetch_url.fetch_url("http://evil.com/foo", "prod")
|
|
|
|
|
|
def test_fetch_url_omitted_url_allowlist_defaults_to_empty_and_rejects(fresh_stores):
|
|
fetch_url.conn_store.save(
|
|
Connection(
|
|
name="prod",
|
|
master="yarn",
|
|
yarn_rm_url="http://rm.prod.internal",
|
|
)
|
|
)
|
|
with pytest.raises(ValueError, match="is not in Connection.url_allowlist"):
|
|
fetch_url.fetch_url("http://nm.prod.internal/", "prod")
|