修复 spark_submit 参数重复、上传校验绕过等 8 处缺陷
对应代码审查发现的问题（#1, #2, #4-#8, #10）： #1 CRITICAL：spark_submit 在 argv 中重复拼接 binary。此前 cmd := []string{binary} 后又把 cmd 作为 Args 传给 executor.Run， 而 executor 会再拼一次 Binary，导致 OS argv 为 [binary, binary, ...]， spark-submit 会把自身当作应用 jar。现在 cmd 从 --master 开始， executor.Run 使用 Binary + Args，argv 正确。 #2：Validate 曾接受 .meta.json 路径本身。现在显式拒绝 sidecar 路径， 要求传入数据文件路径。 #4：Sweep 对 sidecar 损坏的数据文件跳过清理。现在损坏 sidecar 会回退 到数据文件 mtime，超期即删除。 #5：upload_file 描述仍引用已移除的 args 字段，已改为引用 script_path 及结构化字段。 #6：Deps.UploadStore 改为值类型 uploads.Store，避免 nil 绕过上传校验； 移除 spark_submit/upload_file 中的 nil 检查。 #7：master/queue/executor_memory 增加空字符串校验。 #8：提取 buildSparkSubmitCommand 构建 argv，消除双写参数的结构性根因。 #10：Validate 失败时记录 slog.Warn("spark_submit.unminted_path_rejected")。 新增测试： - TestSparkSubmit_StructuredCommand：断言 argv 首行为 --master，末行 仍为 script_path。 - TestSparkSubmit_EmptyMaster：空 master 返回错误。 - TestStore_Validate_RejectsSidecarPath：拒绝 .meta.json 路径。 - TestStore_Sweep_DeletesDataWithCorruptSidecar：损坏 sidecar 的数据文件 被清理。 未在本提交处理： - #9 cluster.DefaultSubmitArgs 弃用留作后续批次。 Co-Authored-By: tao.chen <93983997+taochen-ct@users.noreply.github.com>
This commit is contained in:
tao.chen
@@ -3,6 +3,7 @@ package tools
|
||||
import (
|
||||
"context"
|
||||
"fmt"
|
||||
"log/slog"
|
||||
"sort"
|
||||
"strconv"
|
||||
|
||||
@@ -63,11 +64,13 @@ func NewSparkSubmitTool() mcp.Tool {
|
||||
|
||||
// SparkSubmitHandler runs spark-submit against the requested cluster.
|
||||
func (d *Deps) SparkSubmitHandler(ctx context.Context, req mcp.CallToolRequest) (*mcp.CallToolResult, error) {
|
||||
args := req.GetArguments()
|
||||
|
||||
clusterID, err := req.RequireString("cluster_id")
|
||||
if err != nil {
|
||||
return errResult("spark_submit: " + err.Error()), nil
|
||||
}
|
||||
master, err := req.RequireString("master")
|
||||
master, err := requireNonEmptyString(args, "master")
|
||||
if err != nil {
|
||||
return errResult("spark_submit: " + err.Error()), nil
|
||||
}
|
||||
@@ -80,27 +83,26 @@ func (d *Deps) SparkSubmitHandler(ctx context.Context, req mcp.CallToolRequest)
|
||||
return errResult("spark_submit: " + err.Error()), nil
|
||||
}
|
||||
|
||||
executorMemory, err := req.RequireString("executor_memory")
|
||||
queue, err := requireNonEmptyString(args, "queue")
|
||||
if err != nil {
|
||||
return errResult("spark_submit: " + err.Error()), nil
|
||||
}
|
||||
queue, err := req.RequireString("queue")
|
||||
executorMemory, err := requireNonEmptyString(args, "executor_memory")
|
||||
if err != nil {
|
||||
return errResult("spark_submit: " + err.Error()), nil
|
||||
}
|
||||
|
||||
args := req.GetArguments()
|
||||
|
||||
// Reject paths that didn't come through upload_file on this server.
|
||||
// Cluster-local paths and arbitrary host paths are unreachable from the MCP
|
||||
// server's spark-submit, so we fail fast with a clear message rather than
|
||||
// letting spark-submit produce a confusing FileNotFoundException later.
|
||||
// Placed after all required-string parses so error messages surface in
|
||||
// field order (cluster_id, master, ..., script_path, queue, ...).
|
||||
if d.UploadStore != nil {
|
||||
if _, err := d.UploadStore.Validate(scriptPath); err != nil {
|
||||
return errResult(fmt.Sprintf("spark_submit: script_path %q is not from upload_file on this server; upload the file first via upload_file and pass back the path field", scriptPath)), nil
|
||||
if _, err := d.UploadStore.Validate(scriptPath); err != nil {
|
||||
if d.Logger != nil {
|
||||
d.Logger.Warn("spark_submit.unminted_path_rejected", slog.String("path", scriptPath), slog.String("err", err.Error()))
|
||||
}
|
||||
return errResult(fmt.Sprintf("spark_submit: script_path %q is not from upload_file on this server; upload the file first via upload_file and pass back the path field", scriptPath)), nil
|
||||
}
|
||||
|
||||
// deploy_mode is required for forward compatibility but the builder does not
|
||||
@@ -149,25 +151,16 @@ func (d *Deps) SparkSubmitHandler(ctx context.Context, req mcp.CallToolRequest)
|
||||
}
|
||||
|
||||
binary := cl.SparkSubmitExecuteBin
|
||||
|
||||
// Build argv inline, mirroring the Python reference's output order exactly.
|
||||
// DefaultSubmitArgs is no longer prepended; the field is currently unused at
|
||||
// runtime but kept for backward compatibility with the admin API. Removal is
|
||||
// a separate follow-up.
|
||||
cmd := []string{binary}
|
||||
cmd = append(cmd, "--master", master)
|
||||
cmd = append(cmd, "--queue", queue)
|
||||
cmd = append(cmd, "--executor-memory", executorMemory)
|
||||
cmd = append(cmd, "--executor-cores", strconv.Itoa(executorCores))
|
||||
cmd = append(cmd, "--num-executors", strconv.Itoa(numExecutors))
|
||||
|
||||
for _, k := range sortedStringKeys(sparkConf) {
|
||||
cmd = append(cmd, "--conf", k+"="+sparkConf[k])
|
||||
}
|
||||
for _, k := range sortedStringKeys(extraArgs) {
|
||||
cmd = append(cmd, "--"+k, extraArgs[k])
|
||||
}
|
||||
cmd = append(cmd, scriptPath)
|
||||
cmd := buildSparkSubmitCommand(SparkSubmitCommandOpts{
|
||||
Master: master,
|
||||
Queue: queue,
|
||||
ExecutorMemory: executorMemory,
|
||||
ExecutorCores: executorCores,
|
||||
NumExecutors: numExecutors,
|
||||
SparkConf: sparkConf,
|
||||
ExtraArgs: extraArgs,
|
||||
ScriptPath: scriptPath,
|
||||
})
|
||||
|
||||
if d.Logger != nil {
|
||||
d.Logger.Debug("spark_submit.built", "cluster_id", clusterID, "argv", cmd)
|
||||
@@ -207,6 +200,40 @@ func (d *Deps) SparkSubmitHandler(ctx context.Context, req mcp.CallToolRequest)
|
||||
return textResult(encodeJSON(result)), nil
|
||||
}
|
||||
|
||||
// SparkSubmitCommandOpts holds the structured arguments used to build the
|
||||
// spark-submit argv. The returned argv does NOT include the binary itself.
|
||||
type SparkSubmitCommandOpts struct {
|
||||
Master string
|
||||
Queue string
|
||||
ExecutorMemory string
|
||||
ExecutorCores int
|
||||
NumExecutors int
|
||||
SparkConf map[string]string
|
||||
ExtraArgs map[string]string
|
||||
ScriptPath string
|
||||
}
|
||||
|
||||
// buildSparkSubmitCommand builds the post-binary argv for spark-submit.
|
||||
// DefaultSubmitArgs is no longer prepended; the field is currently unused at
|
||||
// runtime but kept for backward compatibility with the admin API. Removal is
|
||||
// a separate follow-up.
|
||||
func buildSparkSubmitCommand(opts SparkSubmitCommandOpts) []string {
|
||||
cmd := []string{"--master", opts.Master}
|
||||
cmd = append(cmd, "--queue", opts.Queue)
|
||||
cmd = append(cmd, "--executor-memory", opts.ExecutorMemory)
|
||||
cmd = append(cmd, "--executor-cores", strconv.Itoa(opts.ExecutorCores))
|
||||
cmd = append(cmd, "--num-executors", strconv.Itoa(opts.NumExecutors))
|
||||
|
||||
for _, k := range sortedStringKeys(opts.SparkConf) {
|
||||
cmd = append(cmd, "--conf", k+"="+opts.SparkConf[k])
|
||||
}
|
||||
for _, k := range sortedStringKeys(opts.ExtraArgs) {
|
||||
cmd = append(cmd, "--"+k, opts.ExtraArgs[k])
|
||||
}
|
||||
cmd = append(cmd, opts.ScriptPath)
|
||||
return cmd
|
||||
}
|
||||
|
||||
// requireNonNegativeInt extracts an integer parameter from the request and
|
||||
// rejects negative or non-integer values.
|
||||
func requireNonNegativeInt(args map[string]any, key string) (int, error) {
|
||||
@@ -261,3 +288,17 @@ func sortedStringKeys(m map[string]string) []string {
|
||||
sort.Strings(keys)
|
||||
return keys
|
||||
}
|
||||
|
||||
// requireNonEmptyString extracts a required string parameter and rejects empty
|
||||
// values. MCP's RequireString already enforces presence/type; this enforces
|
||||
// that the field is not blank.
|
||||
func requireNonEmptyString(args map[string]any, key string) (string, error) {
|
||||
s, ok := args[key].(string)
|
||||
if !ok {
|
||||
return "", fmt.Errorf("%s is required", key)
|
||||
}
|
||||
if s == "" {
|
||||
return "", fmt.Errorf("%s is required", key)
|
||||
}
|
||||
return s, nil
|
||||
}
|
||||
|
||||
Reference in New Issue
Block a user