Phase 3+4: 共享 HTTP 客户端 + admin API + audit log

Phase 3 (httpclient):
- shared Client (Timeout + MaxResponseBytes 截断)
- SSRF: 16 段私网 CIDR (含 169.254/::ffff:0:0/96 IPv4-mapped)
  + hostname allowlist (精确/后缀/*. 通配)
  + scheme 校验 (仅 http/https)
- auth 适配器: none / simple (YARN user.name) / basic (SetBasicAuth)
- DoWithRedirect: 跨主机跳转保留 Authorization, max 5 默认
- 19 个测试全绿 (httptest 模拟)

Phase 4 (admin + audit):
- internal/audit: Entry + Repo.Insert/List + MarshalDetails, snake_case JSON
- internal/middleware: AdminAuth/AgentAuth (constant-time 比对)
- internal/admin: 6 端点 (GET/POST/PUT/DELETE /admin/clusters, GET /admin/audit)
  + 写操作触发 audit_log (含 before/after diff)
  + AuthPassword 空=保留旧密码
  + 校验: 必填字段 + auth_type 枚举 + rate_limit>=0
- main.go: 挂载 storage.Open + admin.Mount
- L fix: audit.Entry 加 json tag (smoke test 发现大写 key 不规范)

Co-Authored-By: Claude <noreply@anthropic.com>
This commit is contained in:
tao.chen
2026-07-10 13:54:24 +08:00
co-authored by Claude
parent a4e2472716
commit 2c39091706
14 changed files with 1528 additions and 2 deletions
+53
View File
@@ -0,0 +1,53 @@
package httpclient
import (
"errors"
"fmt"
"net/http"
"net/url"
"spark-mcp-go/internal/cluster"
)
// ApplyAuth rewrites req according to the cluster's configured AuthType.
// - none / empty: no-op
// - simple: appends ?user.name=<username> (YARN SimpleAuth)
// - basic: attaches HTTP Basic credentials
//
// baseURL is kept for future use (e.g. reconstructing absolute URLs in Phase 5)
// and is intentionally unused in this version.
func ApplyAuth(req *http.Request, baseURL string, c *cluster.Cluster) error {
if baseURL == "" {
// baseURL is reserved for Phase 5 host rewriting.
}
switch c.AuthType {
case cluster.AuthNone, "":
return nil
case cluster.AuthSimple:
u, err := url.Parse(req.URL.String())
if err != nil {
return fmt.Errorf("httpclient: simple auth parse url: %w", err)
}
q := u.Query()
user := c.AuthUsername
if user == "" {
user = "yarn"
}
q.Set("user.name", user)
u.RawQuery = q.Encode()
req.URL = u
return nil
case cluster.AuthBasic:
if c.AuthUsername == "" || c.AuthPassword == "" {
return errors.New("httpclient: basic auth requires username and password")
}
req.SetBasicAuth(c.AuthUsername, c.AuthPassword)
return nil
default:
return fmt.Errorf("httpclient: unknown auth type %q", c.AuthType)
}
}
+99
View File
@@ -0,0 +1,99 @@
package httpclient
import (
"encoding/base64"
"net/http"
"strings"
"testing"
"spark-mcp-go/internal/cluster"
)
func TestApplyAuth_None(t *testing.T) {
req, _ := http.NewRequest(http.MethodGet, "http://example.com/foo", nil)
c := &cluster.Cluster{AuthType: cluster.AuthNone}
if err := ApplyAuth(req, "", c); err != nil {
t.Fatalf("apply auth: %v", err)
}
if req.Header.Get("Authorization") != "" {
t.Errorf("expected no Authorization header")
}
if req.URL.Query().Get("user.name") != "" {
t.Errorf("expected no user.name query parameter")
}
}
func TestApplyAuth_Simple_DefaultUser(t *testing.T) {
req, _ := http.NewRequest(http.MethodGet, "http://rm.example.com/ws/v1/cluster/apps", nil)
c := &cluster.Cluster{AuthType: cluster.AuthSimple}
if err := ApplyAuth(req, "", c); err != nil {
t.Fatalf("apply auth: %v", err)
}
if got := req.URL.Query().Get("user.name"); got != "yarn" {
t.Errorf("user.name=%q, want yarn", got)
}
}
func TestApplyAuth_Simple_CustomUser(t *testing.T) {
req, _ := http.NewRequest(http.MethodGet, "http://rm.example.com/ws/v1/cluster/apps", nil)
c := &cluster.Cluster{AuthType: cluster.AuthSimple, AuthUsername: "alice"}
if err := ApplyAuth(req, "", c); err != nil {
t.Fatalf("apply auth: %v", err)
}
if got := req.URL.Query().Get("user.name"); got != "alice" {
t.Errorf("user.name=%q, want alice", got)
}
}
func TestApplyAuth_Simple_PreservesExistingQuery(t *testing.T) {
req, _ := http.NewRequest(http.MethodGet, "http://rm.example.com/ws/v1/cluster/apps?foo=bar", nil)
c := &cluster.Cluster{AuthType: cluster.AuthSimple}
if err := ApplyAuth(req, "", c); err != nil {
t.Fatalf("apply auth: %v", err)
}
q := req.URL.Query()
if q.Get("foo") != "bar" {
t.Errorf("foo=%q, want bar", q.Get("foo"))
}
if q.Get("user.name") != "yarn" {
t.Errorf("user.name=%q, want yarn", q.Get("user.name"))
}
raw := req.URL.RawQuery
if !strings.Contains(raw, "foo=bar") || !strings.Contains(raw, "user.name=yarn") {
t.Errorf("raw query %q missing expected parameters", raw)
}
}
func TestApplyAuth_Basic(t *testing.T) {
req, _ := http.NewRequest(http.MethodGet, "http://rm.example.com/foo", nil)
c := &cluster.Cluster{AuthType: cluster.AuthBasic, AuthUsername: "u", AuthPassword: "p"}
if err := ApplyAuth(req, "", c); err != nil {
t.Fatalf("apply auth: %v", err)
}
want := "Basic " + base64.StdEncoding.EncodeToString([]byte("u:p"))
if got := req.Header.Get("Authorization"); got != want {
t.Errorf("Authorization=%q, want %q", got, want)
}
}
func TestApplyAuth_Basic_MissingCreds(t *testing.T) {
req, _ := http.NewRequest(http.MethodGet, "http://rm.example.com/foo", nil)
c := &cluster.Cluster{AuthType: cluster.AuthBasic, AuthUsername: "", AuthPassword: "p"}
if err := ApplyAuth(req, "", c); err == nil {
t.Error("expected error for missing username")
}
req2, _ := http.NewRequest(http.MethodGet, "http://rm.example.com/foo", nil)
c2 := &cluster.Cluster{AuthType: cluster.AuthBasic, AuthUsername: "u", AuthPassword: ""}
if err := ApplyAuth(req2, "", c2); err == nil {
t.Error("expected error for missing password")
}
}
func TestApplyAuth_UnknownAuthType(t *testing.T) {
req, _ := http.NewRequest(http.MethodGet, "http://rm.example.com/foo", nil)
c := &cluster.Cluster{AuthType: "kerberos"}
if err := ApplyAuth(req, "", c); err == nil {
t.Error("expected error for unknown auth type")
}
}
+60
View File
@@ -0,0 +1,60 @@
package httpclient
import (
"context"
"io"
"net/http"
"time"
)
// Config tunes the shared HTTP client.
type Config struct {
Timeout time.Duration // e.g. 30s
MaxResponseBytes int64 // e.g. 1 << 20
}
// Client is a thin, SSRF-aware wrapper around net/http.Client.
type Client struct {
cfg Config
hc *http.Client
}
// New creates a Client with redirect following disabled.
// Redirect handling is intentionally left to DoWithRedirect.
func New(cfg Config) *Client {
return &Client{
cfg: cfg,
hc: &http.Client{
Timeout: cfg.Timeout,
CheckRedirect: func(req *http.Request, via []*http.Request) error {
return http.ErrUseLastResponse
},
},
}
}
// Do executes a single HTTP request. The response body is truncated to
// MaxResponseBytes. The request URL is validated by CheckURL before sending;
// hosts listed in allowedHosts bypass SSRF checks.
//
// TODO(prod): implement IP-pinned dialer to fully close TOCTOU between the
// SSRF check here and the actual TCP dial performed by net/http.
func (c *Client) Do(ctx context.Context, req *http.Request, allowedHosts ...string) (*http.Response, error) {
if err := CheckURL(req.URL.String(), allowedHosts...); err != nil {
return nil, err
}
resp, err := c.hc.Do(req.WithContext(ctx))
if err != nil {
return nil, err
}
resp.Body = struct {
io.Reader
io.Closer
}{
Reader: io.LimitReader(resp.Body, c.cfg.MaxResponseBytes),
Closer: resp.Body,
}
return resp, nil
}
+56
View File
@@ -0,0 +1,56 @@
package httpclient
import (
"context"
"io"
"net/http"
"net/http/httptest"
"strings"
"testing"
"time"
)
func TestClient_Timeout(t *testing.T) {
srv := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
time.Sleep(2 * time.Second)
w.WriteHeader(http.StatusOK)
}))
defer srv.Close()
c := New(Config{Timeout: 100 * time.Millisecond, MaxResponseBytes: 1024})
req, err := http.NewRequest(http.MethodGet, srv.URL, nil)
if err != nil {
t.Fatalf("new request: %v", err)
}
_, err = c.Do(context.Background(), req)
if err == nil {
t.Fatal("expected timeout error, got nil")
}
}
func TestClient_MaxResponseBytes(t *testing.T) {
srv := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
w.WriteHeader(http.StatusOK)
w.Write([]byte(strings.Repeat("a", 2048)))
}))
defer srv.Close()
c := New(Config{Timeout: 5 * time.Second, MaxResponseBytes: 512})
req, err := http.NewRequest(http.MethodGet, srv.URL, nil)
if err != nil {
t.Fatalf("new request: %v", err)
}
resp, err := c.Do(context.Background(), req, "127.0.0.1")
if err != nil {
t.Fatalf("do: %v", err)
}
defer resp.Body.Close()
body, err := io.ReadAll(resp.Body)
if err != nil {
t.Fatalf("read body: %v", err)
}
if len(body) > 512 {
t.Fatalf("expected at most 512 bytes, got %d", len(body))
}
}
+62
View File
@@ -0,0 +1,62 @@
package httpclient
import (
"context"
"errors"
"fmt"
"net/http"
)
const defaultMaxRedirects = 5
// DoWithRedirect 手动跟随 3xx,跨主机跳转保留 Authorization header。
// maxRedirects == 0 禁止 redirect(等同 Do)
// maxRedirects < 0 用 defaultMaxRedirects
func (c *Client) DoWithRedirect(ctx context.Context, req *http.Request, maxRedirects int, allowedHosts ...string) (*http.Response, error) {
if maxRedirects == 0 {
return c.Do(ctx, req, allowedHosts...)
}
if maxRedirects < 0 {
maxRedirects = defaultMaxRedirects
}
current := req
remaining := maxRedirects
for {
resp, err := c.Do(ctx, current, allowedHosts...)
if err != nil {
return nil, err
}
if resp.StatusCode < 300 || resp.StatusCode >= 400 {
return resp, nil
}
// 3xx
if remaining == 0 {
_ = resp.Body.Close()
return nil, errors.New("httpclient: too many redirects")
}
remaining--
loc := resp.Header.Get("Location")
if loc == "" {
_ = resp.Body.Close()
return nil, fmt.Errorf("httpclient: %d %s without Location header", resp.StatusCode, resp.Status)
}
next, err := current.URL.Parse(loc)
if err != nil {
_ = resp.Body.Close()
return nil, fmt.Errorf("httpclient: parse redirect Location %q: %w", loc, err)
}
_ = resp.Body.Close()
newReq, err := http.NewRequestWithContext(ctx, current.Method, next.String(), nil)
if err != nil {
return nil, fmt.Errorf("httpclient: build redirect request: %w", err)
}
// **保留** header (Authorization 等)
newReq.Header = current.Header.Clone()
newReq.Body = nil
current = newReq
}
}
+142
View File
@@ -0,0 +1,142 @@
package httpclient
import (
"context"
"encoding/base64"
"io"
"net/http"
"net/http/httptest"
"strings"
"testing"
"time"
)
func TestDoWithRedirect_SameHost(t *testing.T) {
srv := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
switch r.URL.Path {
case "/start":
w.Header().Set("Location", "/final")
w.WriteHeader(http.StatusFound)
case "/final":
w.WriteHeader(http.StatusOK)
w.Write([]byte("ok"))
default:
w.WriteHeader(http.StatusNotFound)
}
}))
defer srv.Close()
c := New(Config{Timeout: 5 * time.Second, MaxResponseBytes: 1024})
req, err := http.NewRequest(http.MethodGet, srv.URL+"/start", nil)
if err != nil {
t.Fatalf("new request: %v", err)
}
resp, err := c.DoWithRedirect(context.Background(), req, 5, "127.0.0.1")
if err != nil {
t.Fatalf("do with redirect: %v", err)
}
defer resp.Body.Close()
if resp.StatusCode != http.StatusOK {
t.Fatalf("status=%d, want 200", resp.StatusCode)
}
body, _ := io.ReadAll(resp.Body)
if string(body) != "ok" {
t.Fatalf("body=%q, want ok", string(body))
}
}
func TestDoWithRedirect_CrossHost_PreservesAuth(t *testing.T) {
srv2 := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
w.WriteHeader(http.StatusOK)
w.Write([]byte(r.Header.Get("Authorization")))
}))
defer srv2.Close()
srv1 := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
if r.URL.Path == "/start" {
w.Header().Set("Location", srv2.URL+"/final")
w.WriteHeader(http.StatusTemporaryRedirect)
return
}
w.WriteHeader(http.StatusNotFound)
}))
defer srv1.Close()
c := New(Config{Timeout: 5 * time.Second, MaxResponseBytes: 1024})
req, err := http.NewRequest(http.MethodGet, srv1.URL+"/start", nil)
if err != nil {
t.Fatalf("new request: %v", err)
}
req.SetBasicAuth("u", "p")
resp, err := c.DoWithRedirect(context.Background(), req, 5, "127.0.0.1")
if err != nil {
t.Fatalf("do with redirect: %v", err)
}
defer resp.Body.Close()
body, err := io.ReadAll(resp.Body)
if err != nil {
t.Fatalf("read body: %v", err)
}
want := "Basic " + base64.StdEncoding.EncodeToString([]byte("u:p"))
if string(body) != want {
t.Fatalf("body=%q, want %q", string(body), want)
}
}
func TestDoWithRedirect_MaxLimit(t *testing.T) {
srv := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
w.Header().Set("Location", "/loop")
w.WriteHeader(http.StatusFound)
}))
defer srv.Close()
c := New(Config{Timeout: 5 * time.Second, MaxResponseBytes: 1024})
req, _ := http.NewRequest(http.MethodGet, srv.URL+"/loop", nil)
_, err := c.DoWithRedirect(context.Background(), req, 2, "127.0.0.1")
if err == nil {
t.Fatal("expected too many redirects error")
}
if !strings.Contains(err.Error(), "too many redirects") {
t.Fatalf("expected too many redirects, got %v", err)
}
}
func TestDoWithRedirect_NoLocation(t *testing.T) {
srv := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
w.WriteHeader(http.StatusFound)
}))
defer srv.Close()
c := New(Config{Timeout: 5 * time.Second, MaxResponseBytes: 1024})
req, _ := http.NewRequest(http.MethodGet, srv.URL, nil)
_, err := c.DoWithRedirect(context.Background(), req, 5, "127.0.0.1")
if err == nil {
t.Fatal("expected error for missing Location")
}
if !strings.Contains(err.Error(), "without Location header") {
t.Fatalf("expected missing Location error, got %v", err)
}
}
func TestDoWithRedirect_ZeroDisallows(t *testing.T) {
srv := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
w.Header().Set("Location", "/elsewhere")
w.WriteHeader(http.StatusFound)
}))
defer srv.Close()
c := New(Config{Timeout: 5 * time.Second, MaxResponseBytes: 1024})
req, _ := http.NewRequest(http.MethodGet, srv.URL, nil)
resp, err := c.DoWithRedirect(context.Background(), req, 0, "127.0.0.1")
if err != nil {
t.Fatalf("do: %v", err)
}
defer resp.Body.Close()
if resp.StatusCode != http.StatusFound {
t.Fatalf("status=%d, want 302", resp.StatusCode)
}
}
+127
View File
@@ -0,0 +1,127 @@
package httpclient
import (
"fmt"
"net"
"net/url"
"strings"
)
// privateCIDRs lists IPv4 and IPv6 ranges that must not be reached by the
// outbound HTTP client (SSRF prevention). Includes RFC 1918, loopback, link-
// local, multicast, documentation, and IPv4-mapped IPv6 ranges.
var privateCIDRs = []string{
"0.0.0.0/8", "10.0.0.0/8", "100.64.0.0/10",
"127.0.0.0/8", "169.254.0.0/16",
"172.16.0.0/12", "192.0.0.0/24", "192.168.0.0/16",
"198.18.0.0/15", "224.0.0.0/4", "240.0.0.0/4",
"::1/128", "fc00::/7", "fe80::/10", "::ffff:0:0/96",
}
// privateNets holds the parsed CIDR blocks from privateCIDRs.
var privateNets []*net.IPNet
func init() {
for _, cidr := range privateCIDRs {
_, ipNet, err := net.ParseCIDR(cidr)
if err != nil {
// net.ParseCIDR only fails on malformed input; the list is static.
panic(fmt.Sprintf("httpclient: unable to parse private CIDR %q: %v", cidr, err))
}
privateNets = append(privateNets, ipNet)
}
}
// CheckURL validates that rawURL is an http(s) URL and that its hostname does
// not resolve to a private/reserved IP address. Hosts matching any entry in
// allowedHosts bypass the IP check.
//
// allowedHosts entries support exact matches, suffix matches (e.g.
// "example.com" matches "rm.example.com"), and wildcard suffixes (e.g.
// "*.example.com").
func CheckURL(rawURL string, allowedHosts ...string) error {
u, err := url.Parse(rawURL)
if err != nil {
return fmt.Errorf("httpclient: parse url: %w", err)
}
if !strings.EqualFold(u.Scheme, "http") && !strings.EqualFold(u.Scheme, "https") {
return fmt.Errorf("httpclient: unsupported url scheme %q", u.Scheme)
}
host := strings.ToLower(u.Hostname())
if host == "" {
return fmt.Errorf("httpclient: url has no hostname")
}
for _, allowed := range allowedHosts {
if matchAllowedHost(host, allowed) {
return nil
}
}
ips, err := net.LookupIP(host)
if err != nil {
return fmt.Errorf("httpclient: lookup host %q: %w", host, err)
}
if len(ips) == 0 {
return fmt.Errorf("httpclient: host %q resolved to no IPs", host)
}
for _, ip := range ips {
if isPrivateIP(ip) {
return fmt.Errorf("httpclient: host %q resolves to private IP %s", host, ip)
}
}
return nil
}
// isPrivateIP reports whether ip falls inside any of the reserved CIDR blocks.
func isPrivateIP(ip net.IP) bool {
if v4 := ip.To4(); v4 != nil {
for _, ipNet := range privateNets {
if isIPv4MappedNet(ipNet) {
continue
}
if ipNet.Contains(v4) {
return true
}
}
return false
}
for _, ipNet := range privateNets {
if ipNet.Contains(ip) {
return true
}
}
return false
}
// isIPv4MappedNet reports whether n is an IPv4-mapped IPv6 CIDR such as
// ::ffff:0:0/96. These must not be used to classify plain IPv4 addresses.
func isIPv4MappedNet(n *net.IPNet) bool {
return len(n.IP) == net.IPv6len && n.IP.To4() != nil && n.IP[10] == 0xff && n.IP[11] == 0xff
}
// matchAllowedHost reports whether host matches pattern. Patterns may be exact
// hostnames, domain suffixes ("example.com" matches "rm.example.com"), or
// wildcard suffixes ("*.example.com").
func matchAllowedHost(host, pattern string) bool {
pattern = strings.ToLower(strings.TrimSpace(pattern))
if pattern == "" {
return false
}
if pattern == host {
return true
}
if strings.HasPrefix(pattern, "*.") {
pattern = pattern[2:]
}
if strings.HasPrefix(pattern, ".") {
pattern = pattern[1:]
}
return host == pattern || strings.HasSuffix(host, "."+pattern)
}
+79
View File
@@ -0,0 +1,79 @@
package httpclient
import (
"net"
"net/http"
"net/http/httptest"
"testing"
)
func TestCheckURL_RejectsLoopback(t *testing.T) {
err := CheckURL("http://127.0.0.1:8080/path")
if err == nil {
t.Fatal("expected error for loopback URL")
}
}
func TestCheckURL_RejectsLinkLocal(t *testing.T) {
err := CheckURL("http://169.254.169.254/latest/meta-data")
if err == nil {
t.Fatal("expected error for link-local URL")
}
}
func TestCheckURL_AllowsAllowlisted(t *testing.T) {
err := CheckURL("http://127.0.0.1:8080", "127.0.0.1")
if err != nil {
t.Fatalf("expected allowlisted host to pass: %v", err)
}
if !matchAllowedHost("rm.example.com", "*.example.com") {
t.Fatal("expected *.example.com to match rm.example.com")
}
}
func TestCheckURL_RejectsBadScheme(t *testing.T) {
cases := []string{"file:///etc/passwd", "gopher://x", "ftp://x"}
for _, raw := range cases {
err := CheckURL(raw)
if err == nil {
t.Errorf("expected error for %q", raw)
}
}
}
func TestCheckURL_RejectsEmptyHost(t *testing.T) {
err := CheckURL("http:///path")
if err == nil {
t.Fatal("expected error for URL with empty host")
}
}
func TestCheckURL_AcceptsPublic(t *testing.T) {
srv := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
w.WriteHeader(http.StatusOK)
}))
defer srv.Close()
err := CheckURL(srv.URL, "127.0.0.1")
if err != nil {
t.Fatalf("expected allowlisted server URL to pass: %v", err)
}
}
func TestIsPrivateIP(t *testing.T) {
cases := []struct {
ip string
want bool
}{
{"10.1.2.3", true},
{"8.8.8.8", false},
{"::1", true},
{"2001:db8::1", false},
}
for _, tc := range cases {
got := isPrivateIP(net.ParseIP(tc.ip))
if got != tc.want {
t.Errorf("isPrivateIP(%q) = %v, want %v", tc.ip, got, tc.want)
}
}
}