Phase 3+4: 共享 HTTP 客户端 + admin API + audit log

Phase 3 (httpclient):
- shared Client (Timeout + MaxResponseBytes 截断)
- SSRF: 16 段私网 CIDR (含 169.254/::ffff:0:0/96 IPv4-mapped)
  + hostname allowlist (精确/后缀/*. 通配)
  + scheme 校验 (仅 http/https)
- auth 适配器: none / simple (YARN user.name) / basic (SetBasicAuth)
- DoWithRedirect: 跨主机跳转保留 Authorization, max 5 默认
- 19 个测试全绿 (httptest 模拟)

Phase 4 (admin + audit):
- internal/audit: Entry + Repo.Insert/List + MarshalDetails, snake_case JSON
- internal/middleware: AdminAuth/AgentAuth (constant-time 比对)
- internal/admin: 6 端点 (GET/POST/PUT/DELETE /admin/clusters, GET /admin/audit)
  + 写操作触发 audit_log (含 before/after diff)
  + AuthPassword 空=保留旧密码
  + 校验: 必填字段 + auth_type 枚举 + rate_limit>=0
- main.go: 挂载 storage.Open + admin.Mount
- L fix: audit.Entry 加 json tag (smoke test 发现大写 key 不规范)

Co-Authored-By: Claude <noreply@anthropic.com>
This commit is contained in:
tao.chen
2026-07-10 13:54:24 +08:00
co-authored by Claude
parent a4e2472716
commit 2c39091706
14 changed files with 1528 additions and 2 deletions
+251
View File
@@ -0,0 +1,251 @@
// Package admin exposes the admin HTTP API for cluster management and audit
// log access. All routes are protected by middleware.AdminAuth.
package admin
import (
"errors"
"net/http"
"strconv"
"github.com/gin-gonic/gin"
"spark-mcp-go/internal/audit"
"spark-mcp-go/internal/cluster"
"spark-mcp-go/internal/middleware"
"spark-mcp-go/internal/storage"
)
// Mount attaches the /admin sub-router to r, protecting every route with
// bearer-token admin authentication.
func Mount(r *gin.Engine, repo *storage.ClusterRepo, auditRepo *audit.Repo, adminTokens []string) {
g := r.Group("/admin", middleware.AdminAuth(adminTokens))
g.GET("/clusters", listClusters(repo))
g.POST("/clusters", createCluster(repo, auditRepo))
g.GET("/clusters/:id", getCluster(repo))
g.PUT("/clusters/:id", updateCluster(repo, auditRepo))
g.DELETE("/clusters/:id", deleteCluster(repo, auditRepo))
g.GET("/audit", listAudit(auditRepo))
}
func listClusters(repo *storage.ClusterRepo) gin.HandlerFunc {
return func(c *gin.Context) {
clusters, err := repo.List(c.Request.Context())
if err != nil {
c.JSON(http.StatusInternalServerError, gin.H{"error": err.Error()})
return
}
c.JSON(http.StatusOK, clusters)
}
}
func createCluster(repo *storage.ClusterRepo, auditRepo *audit.Repo) gin.HandlerFunc {
return func(c *gin.Context) {
var cl cluster.Cluster
if err := c.ShouldBindJSON(&cl); err != nil {
c.JSON(http.StatusBadRequest, gin.H{"error": err.Error()})
return
}
if err := validateClusterCreate(&cl); err != nil {
c.JSON(http.StatusBadRequest, gin.H{"error": err.Error()})
return
}
ctx := c.Request.Context()
if err := repo.Create(ctx, &cl); err != nil {
c.JSON(http.StatusInternalServerError, gin.H{"error": err.Error()})
return
}
details, err := audit.MarshalDetails(clusterFields(&cl))
if err == nil {
_ = auditRepo.Insert(ctx, &audit.Entry{
Actor: actor(c),
Action: audit.ActionClusterCreate,
ClusterID: cl.ID,
Details: details,
})
}
c.JSON(http.StatusCreated, &cl)
}
}
func getCluster(repo *storage.ClusterRepo) gin.HandlerFunc {
return func(c *gin.Context) {
cl, err := repo.Get(c.Request.Context(), c.Param("id"))
if err != nil {
if errors.Is(err, storage.ErrNotFound) {
c.JSON(http.StatusNotFound, gin.H{"error": "cluster not found"})
return
}
c.JSON(http.StatusInternalServerError, gin.H{"error": err.Error()})
return
}
c.JSON(http.StatusOK, cl)
}
}
func updateCluster(repo *storage.ClusterRepo, auditRepo *audit.Repo) gin.HandlerFunc {
return func(c *gin.Context) {
id := c.Param("id")
ctx := c.Request.Context()
old, err := repo.Get(ctx, id)
if err != nil {
if errors.Is(err, storage.ErrNotFound) {
c.JSON(http.StatusNotFound, gin.H{"error": "cluster not found"})
return
}
c.JSON(http.StatusInternalServerError, gin.H{"error": err.Error()})
return
}
var cl cluster.Cluster
if err := c.ShouldBindJSON(&cl); err != nil {
c.JSON(http.StatusBadRequest, gin.H{"error": err.Error()})
return
}
cl.ID = id
if cl.AuthPassword == "" {
cl.AuthPassword = old.AuthPassword
}
if err := repo.Update(ctx, &cl); err != nil {
if errors.Is(err, storage.ErrNotFound) {
c.JSON(http.StatusNotFound, gin.H{"error": "cluster not found"})
return
}
c.JSON(http.StatusInternalServerError, gin.H{"error": err.Error()})
return
}
details, err := audit.MarshalDetails(map[string]any{
"before": clusterFields(old),
"after": clusterFields(&cl),
})
if err == nil {
_ = auditRepo.Insert(ctx, &audit.Entry{
Actor: actor(c),
Action: audit.ActionClusterUpdate,
ClusterID: cl.ID,
Details: details,
})
}
c.JSON(http.StatusOK, &cl)
}
}
func deleteCluster(repo *storage.ClusterRepo, auditRepo *audit.Repo) gin.HandlerFunc {
return func(c *gin.Context) {
id := c.Param("id")
ctx := c.Request.Context()
old, err := repo.Get(ctx, id)
if err != nil {
if errors.Is(err, storage.ErrNotFound) {
c.JSON(http.StatusNotFound, gin.H{"error": "cluster not found"})
return
}
c.JSON(http.StatusInternalServerError, gin.H{"error": err.Error()})
return
}
if err := repo.Delete(ctx, id); err != nil {
if errors.Is(err, storage.ErrNotFound) {
c.JSON(http.StatusNotFound, gin.H{"error": "cluster not found"})
return
}
c.JSON(http.StatusInternalServerError, gin.H{"error": err.Error()})
return
}
details, err := audit.MarshalDetails(map[string]any{
"id": id,
"name": old.Name,
})
if err == nil {
_ = auditRepo.Insert(ctx, &audit.Entry{
Actor: actor(c),
Action: audit.ActionClusterDelete,
ClusterID: id,
Details: details,
})
}
c.Status(http.StatusNoContent)
}
}
func listAudit(auditRepo *audit.Repo) gin.HandlerFunc {
return func(c *gin.Context) {
limitStr := c.DefaultQuery("limit", "100")
limit, err := strconv.Atoi(limitStr)
if err != nil {
c.JSON(http.StatusBadRequest, gin.H{"error": "invalid limit"})
return
}
entries, err := auditRepo.List(c.Request.Context(), limit)
if err != nil {
c.JSON(http.StatusInternalServerError, gin.H{"error": err.Error()})
return
}
c.JSON(http.StatusOK, entries)
}
}
func actor(c *gin.Context) string {
tok, _ := c.Get("admin_token")
s, _ := tok.(string)
if len(s) > 8 {
s = s[:8]
}
return "admin:" + s
}
func validateClusterCreate(c *cluster.Cluster) error {
if c.ID == "" {
return errors.New("id is required")
}
if c.Name == "" {
return errors.New("name is required")
}
if c.RMURL == "" {
return errors.New("rm_url is required")
}
if c.SHSURL == "" {
return errors.New("shs_url is required")
}
if c.SparkSubmitExecuteBin == "" {
return errors.New("spark_submit_execute_bin is required")
}
switch c.AuthType {
case cluster.AuthNone, cluster.AuthSimple, cluster.AuthBasic:
default:
return errors.New("auth_type must be one of none, simple, basic")
}
if c.RateLimitPerMin < 0 {
return errors.New("rate_limit_per_min must be >= 0")
}
return nil
}
func clusterFields(c *cluster.Cluster) map[string]any {
return map[string]any{
"id": c.ID,
"name": c.Name,
"rm_url": c.RMURL,
"shs_url": c.SHSURL,
"spark_submit_execute_bin": c.SparkSubmitExecuteBin,
"is_active": c.IsActive,
"auth_type": c.AuthType,
"auth_username": c.AuthUsername,
"ssl_verify": c.SSLVerify,
"ssl_ca_bundle": c.SSLCABundle,
"url_allowlist": c.URLAllowlist,
"default_submit_args": c.DefaultSubmitArgs,
"rate_limit_per_min": c.RateLimitPerMin,
}
}
+394
View File
@@ -0,0 +1,394 @@
package admin
import (
"bytes"
"context"
"encoding/json"
"io"
"net/http"
"net/http/httptest"
"strings"
"testing"
"github.com/gin-gonic/gin"
"spark-mcp-go/internal/audit"
"spark-mcp-go/internal/storage"
)
func init() {
gin.SetMode(gin.TestMode)
}
func newTestAdmin(t *testing.T) (*gin.Engine, *storage.ClusterRepo, *audit.Repo) {
t.Helper()
db, err := storage.Open(":memory:")
if err != nil {
t.Fatalf("open: %v", err)
}
t.Cleanup(func() { _ = db.Close() })
r := gin.New()
Mount(r, db.Clusters(), audit.NewRepo(db), []string{"good-token"})
return r, db.Clusters(), audit.NewRepo(db)
}
func doReq(t *testing.T, r *gin.Engine, method, path, token string, body any) *httptest.ResponseRecorder {
t.Helper()
var rdr io.Reader
if body != nil {
b, _ := json.Marshal(body)
rdr = bytes.NewReader(b)
}
req := httptest.NewRequest(method, path, rdr)
if token != "" {
req.Header.Set("Authorization", "Bearer "+token)
}
if body != nil {
req.Header.Set("Content-Type", "application/json")
}
w := httptest.NewRecorder()
r.ServeHTTP(w, req)
return w
}
func fullClusterMap(id string) map[string]any {
return map[string]any{
"id": id,
"name": id + "-name",
"rm_url": "http://rm.example.com:8088",
"shs_url": "http://shs.example.com:18080",
"spark_submit_execute_bin": "/usr/bin/spark-submit",
"is_active": true,
"auth_type": "basic",
"auth_username": "admin",
"auth_password": "real-secret",
"ssl_verify": false,
"ssl_ca_bundle": "",
"url_allowlist": []string{"*.example.com"},
"default_submit_args": []string{"--master", "yarn"},
"rate_limit_per_min": 10,
}
}
func TestMount_RequiresAuth(t *testing.T) {
r, _, _ := newTestAdmin(t)
tests := []struct {
name string
method string
path string
token string
want int
}{
{"no header", "GET", "/admin/clusters", "", http.StatusUnauthorized},
{"wrong scheme", "GET", "/admin/clusters", "Basic xxx", http.StatusUnauthorized},
{"wrong token", "GET", "/admin/clusters", "wrong", http.StatusUnauthorized},
{"valid token", "GET", "/admin/clusters", "good-token", http.StatusOK},
}
for _, tt := range tests {
t.Run(tt.name, func(t *testing.T) {
w := doReq(t, r, tt.method, tt.path, tt.token, nil)
if w.Code != tt.want {
t.Errorf("got status %d, want %d", w.Code, tt.want)
}
})
}
}
func TestListClusters(t *testing.T) {
r, _, _ := newTestAdmin(t)
doReq(t, r, "POST", "/admin/clusters", "good-token", fullClusterMap("c1"))
doReq(t, r, "POST", "/admin/clusters", "good-token", fullClusterMap("c2"))
w := doReq(t, r, "GET", "/admin/clusters", "good-token", nil)
if w.Code != http.StatusOK {
t.Fatalf("got status %d, want %d", w.Code, http.StatusOK)
}
var list []map[string]any
if err := json.Unmarshal(w.Body.Bytes(), &list); err != nil {
t.Fatalf("unmarshal list: %v", err)
}
if len(list) != 2 {
t.Errorf("got %d clusters, want 2", len(list))
}
for _, cl := range list {
if _, ok := cl["auth_password"]; ok {
t.Errorf("response must not contain auth_password")
}
}
body := w.Body.String()
if strings.Contains(body, "real-secret") {
t.Errorf("response body leaks auth_password")
}
}
func TestCreateCluster(t *testing.T) {
r, _, auditRepo := newTestAdmin(t)
w := doReq(t, r, "POST", "/admin/clusters", "good-token", fullClusterMap("c1"))
if w.Code != http.StatusCreated {
t.Fatalf("got status %d, want %d", w.Code, http.StatusCreated)
}
var cl map[string]any
if err := json.Unmarshal(w.Body.Bytes(), &cl); err != nil {
t.Fatalf("unmarshal cluster: %v", err)
}
if cl["id"] != "c1" || cl["name"] != "c1-name" {
t.Errorf("unexpected cluster fields: id=%v name=%v", cl["id"], cl["name"])
}
entries, err := auditRepo.List(context.Background(), 100)
if err != nil {
t.Fatalf("list audit: %v", err)
}
if len(entries) < 1 {
t.Fatalf("got %d audit entries, want at least 1", len(entries))
}
found := false
for _, e := range entries {
if e.Action == audit.ActionClusterCreate && e.Actor == "admin:good-tok" {
found = true
break
}
}
if !found {
t.Errorf("missing create audit entry from admin:good-tok")
}
}
func TestCreateCluster_ValidationErrors(t *testing.T) {
r, _, _ := newTestAdmin(t)
tests := []struct {
name string
body map[string]any
}{
{
"missing id",
func() map[string]any {
m := fullClusterMap("c1")
delete(m, "id")
return m
}(),
},
{
"missing name",
func() map[string]any {
m := fullClusterMap("c1")
delete(m, "name")
return m
}(),
},
{
"invalid auth_type",
func() map[string]any {
m := fullClusterMap("c1")
m["auth_type"] = "kerberos"
return m
}(),
},
{
"negative rate limit",
func() map[string]any {
m := fullClusterMap("c1")
m["rate_limit_per_min"] = -1
return m
}(),
},
}
for _, tt := range tests {
t.Run(tt.name, func(t *testing.T) {
w := doReq(t, r, "POST", "/admin/clusters", "good-token", tt.body)
if w.Code != http.StatusBadRequest {
t.Errorf("got status %d, want %d", w.Code, http.StatusBadRequest)
}
})
}
}
func TestGetCluster(t *testing.T) {
r, _, _ := newTestAdmin(t)
doReq(t, r, "POST", "/admin/clusters", "good-token", fullClusterMap("c1"))
w := doReq(t, r, "GET", "/admin/clusters/c1", "good-token", nil)
if w.Code != http.StatusOK {
t.Fatalf("got status %d, want %d", w.Code, http.StatusOK)
}
var cl map[string]any
if err := json.Unmarshal(w.Body.Bytes(), &cl); err != nil {
t.Fatalf("unmarshal cluster: %v", err)
}
if cl["id"] != "c1" || cl["name"] != "c1-name" {
t.Errorf("unexpected cluster fields: id=%v name=%v", cl["id"], cl["name"])
}
w = doReq(t, r, "GET", "/admin/clusters/not-found", "good-token", nil)
if w.Code != http.StatusNotFound {
t.Errorf("got status %d, want %d", w.Code, http.StatusNotFound)
}
var errBody map[string]any
if err := json.Unmarshal(w.Body.Bytes(), &errBody); err != nil {
t.Fatalf("unmarshal error body: %v", err)
}
if errBody["error"] != "cluster not found" {
t.Errorf("unexpected error message: %v", errBody["error"])
}
}
func TestUpdateCluster(t *testing.T) {
r, _, auditRepo := newTestAdmin(t)
doReq(t, r, "POST", "/admin/clusters", "good-token", fullClusterMap("c1"))
update := map[string]any{
"name": "updated",
"rate_limit_per_min": 99,
}
w := doReq(t, r, "PUT", "/admin/clusters/c1", "good-token", update)
if w.Code != http.StatusOK {
t.Fatalf("got status %d, want %d: %s", w.Code, http.StatusOK, w.Body.String())
}
var cl map[string]any
if err := json.Unmarshal(w.Body.Bytes(), &cl); err != nil {
t.Fatalf("unmarshal cluster: %v", err)
}
if cl["name"] != "updated" || cl["rate_limit_per_min"] != float64(99) {
t.Errorf("unexpected update response: name=%v rate_limit_per_min=%v", cl["name"], cl["rate_limit_per_min"])
}
w = doReq(t, r, "GET", "/admin/clusters/c1", "good-token", nil)
if w.Code != http.StatusOK {
t.Fatalf("get after update: got status %d", w.Code)
}
if err := json.Unmarshal(w.Body.Bytes(), &cl); err != nil {
t.Fatalf("unmarshal cluster after update: %v", err)
}
if cl["name"] != "updated" || cl["rate_limit_per_min"] != float64(99) {
t.Errorf("cluster not updated: name=%v rate_limit_per_min=%v", cl["name"], cl["rate_limit_per_min"])
}
entries, err := auditRepo.List(context.Background(), 100)
if err != nil {
t.Fatalf("list audit: %v", err)
}
found := false
for _, e := range entries {
if e.Action == audit.ActionClusterUpdate {
found = true
var details map[string]any
if err := json.Unmarshal([]byte(e.Details), &details); err != nil {
t.Fatalf("unmarshal audit details: %v", err)
}
if _, ok := details["before"]; !ok {
t.Errorf("audit details missing before")
}
if _, ok := details["after"]; !ok {
t.Errorf("audit details missing after")
}
}
}
if !found {
t.Errorf("missing update audit entry")
}
}
func TestUpdateCluster_PreservesEmptyPassword(t *testing.T) {
r, repo, auditRepo := newTestAdmin(t)
doReq(t, r, "POST", "/admin/clusters", "good-token", fullClusterMap("c1"))
// AuthPassword has json:"-", so POST cannot receive it via JSON.
// Seed the stored password directly through the repo.
cl, err := repo.Get(context.Background(), "c1")
if err != nil {
t.Fatalf("get cluster from repo: %v", err)
}
cl.AuthPassword = "real-secret"
if err := repo.Update(context.Background(), cl); err != nil {
t.Fatalf("seed password: %v", err)
}
// Discard the audit row produced by the seed update so it does not confuse later checks.
_, _ = auditRepo.List(context.Background(), 100)
update := map[string]any{
"name": "renamed",
}
w := doReq(t, r, "PUT", "/admin/clusters/c1", "good-token", update)
if w.Code != http.StatusOK {
t.Fatalf("got status %d, want %d: %s", w.Code, http.StatusOK, w.Body.String())
}
if strings.Contains(w.Body.String(), "real-secret") {
t.Errorf("update response leaks auth_password")
}
cl, err = repo.Get(context.Background(), "c1")
if err != nil {
t.Fatalf("get cluster from repo: %v", err)
}
if cl.AuthPassword != "real-secret" {
t.Errorf("password changed: got %q, want %q", cl.AuthPassword, "real-secret")
}
}
func TestDeleteCluster(t *testing.T) {
r, _, auditRepo := newTestAdmin(t)
doReq(t, r, "POST", "/admin/clusters", "good-token", fullClusterMap("c1"))
w := doReq(t, r, "DELETE", "/admin/clusters/c1", "good-token", nil)
if w.Code != http.StatusNoContent {
t.Fatalf("got status %d, want %d", w.Code, http.StatusNoContent)
}
w = doReq(t, r, "GET", "/admin/clusters/c1", "good-token", nil)
if w.Code != http.StatusNotFound {
t.Errorf("get after delete: got status %d, want %d", w.Code, http.StatusNotFound)
}
entries, err := auditRepo.List(context.Background(), 100)
if err != nil {
t.Fatalf("list audit: %v", err)
}
found := false
for _, e := range entries {
if e.Action == audit.ActionClusterDelete {
found = true
break
}
}
if !found {
t.Errorf("missing delete audit entry")
}
}
func TestListAudit(t *testing.T) {
r, _, _ := newTestAdmin(t)
doReq(t, r, "POST", "/admin/clusters", "good-token", fullClusterMap("c1"))
doReq(t, r, "PUT", "/admin/clusters/c1", "good-token", map[string]any{"name": "updated"})
doReq(t, r, "DELETE", "/admin/clusters/c1", "good-token", nil)
w := doReq(t, r, "GET", "/admin/audit", "good-token", nil)
if w.Code != http.StatusOK {
t.Fatalf("got status %d, want %d", w.Code, http.StatusOK)
}
var entries []*audit.Entry
if err := json.Unmarshal(w.Body.Bytes(), &entries); err != nil {
t.Fatalf("unmarshal audit list: %v", err)
}
if len(entries) != 3 {
t.Fatalf("got %d audit entries, want 3", len(entries))
}
wantActions := []string{string(audit.ActionClusterDelete), string(audit.ActionClusterUpdate), string(audit.ActionClusterCreate)}
for i, want := range wantActions {
got := string(entries[i].Action)
if got != want {
t.Errorf("entry[%d].action=%s, want %s", i, got, want)
}
}
}