uploads,tools,config,main: 将 upload_file 返回路径改为服务器端绝对路径并透传给 spark_submit
之前 upload_file 把文件写到 DataDir/uploads/<filename>,返回相对路径
uploads/hello.txt。MCP 服务器与 agent 不在同一台机器,agent 无法构造服务
器本地路径,因此 spark_submit 无法定位脚本。
改为由新的 internal/uploads 包 mint 一个 32 字符十六进制 file_id,文件落盘为
DataDir/uploads/<file_id>,元数据写入 <file_id>.meta.json(原始文件名只存在
sidecar 里)。upload_file 返回 {file_id, path, name, size, sha256},其中 path
是绝对路径。spark_submit 的 description 明确要求 LLM 直接把 upload_file 返回
的 path 放进 args,不要自己构造路径。
为什么只在描述里约束而不在代码里拒绝非 mint 的绝对路径:集群本地已有路径
(如 /opt/spark/examples/pi.py)是合法的 spark-submit 参数,代码不能替
LLM 拒绝。
测试锁定:
- internal/uploads: Save 往返、AbsPath/Validate 非法路径、Sweep 过期/未过期/
孤立 sidecar
- internal/mcp/tools: upload_file 新响应字段、spark_submit 透传 mint 路径与
集群本地路径
刻意未做:S3/HDFS 上传、给 spark_submit 新增 file_id 参数、在代码层面拒绝
集群本地绝对路径。
破坏性变更:upload_file 响应从相对 path 改为绝对 path,并新增 file_id/name/
sha256 字段。该工具尚无外部调用者。
Co-Authored-By: Claude <noreply@anthropic.com>
This commit is contained in:
@@ -29,6 +29,7 @@ type Config struct {
|
||||
AnalyzerDataSkewRatio float64
|
||||
AnalyzerGCPressureRatio float64
|
||||
AnalyzerBottleneckShuffleGB float64
|
||||
UploadTTL time.Duration
|
||||
}
|
||||
|
||||
// Load reads configuration from environment variables and returns a populated Config.
|
||||
@@ -44,6 +45,7 @@ func Load() (*Config, error) {
|
||||
HTTPClientTimeout: 30 * time.Second,
|
||||
MaxResponseBytes: 1 << 20,
|
||||
SparkSubmitTimeout: 60 * time.Second,
|
||||
UploadTTL: 168 * time.Hour,
|
||||
LogDir: "./data/logs",
|
||||
LogLevel: "info",
|
||||
LogFormat: "text",
|
||||
@@ -80,6 +82,11 @@ func Load() (*Config, error) {
|
||||
return nil, err
|
||||
}
|
||||
|
||||
cfg.UploadTTL, err = parseDuration("UPLOAD_TTL", cfg.UploadTTL)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
|
||||
cfg.LogDir = envString("LOG_DIR", cfg.LogDir)
|
||||
cfg.LogLevel = envString("LOG_LEVEL", cfg.LogLevel)
|
||||
cfg.LogFormat = envString("LOG_FORMAT", cfg.LogFormat)
|
||||
@@ -110,6 +117,10 @@ func Load() (*Config, error) {
|
||||
return nil, err
|
||||
}
|
||||
|
||||
if err := validateUploadTTL(cfg.UploadTTL); err != nil {
|
||||
return nil, err
|
||||
}
|
||||
|
||||
cfg.SQLitePath = filepath.Join(cfg.DataDir, "spark-mcp.db")
|
||||
|
||||
return cfg, nil
|
||||
@@ -212,6 +223,16 @@ func validateGinMode(mode string) error {
|
||||
return fmt.Errorf("config: invalid GIN_MODE %q, want debug/release/test", mode)
|
||||
}
|
||||
|
||||
func validateUploadTTL(d time.Duration) error {
|
||||
if d <= 0 {
|
||||
return fmt.Errorf("config: UPLOAD_TTL must be positive")
|
||||
}
|
||||
if d > 8760*time.Hour {
|
||||
return fmt.Errorf("config: UPLOAD_TTL must be at most 8760h (1 year)")
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
||||
// loadDotenv reads KEY=VALUE pairs from path and sets them via os.Setenv only
|
||||
// when the variable is not already defined. Missing files are ignored.
|
||||
func loadDotenv(path string) error {
|
||||
@@ -280,7 +301,7 @@ func (c *Config) String() string {
|
||||
|
||||
return fmt.Sprintf(
|
||||
"ListenAddr=%s DataDir=%s SQLitePath=%s AdminTokens=%s AgentToken=%s "+
|
||||
"HTTPClientTimeout=%s MaxResponseBytes=%d SparkSubmitTimeout=%s "+
|
||||
"HTTPClientTimeout=%s MaxResponseBytes=%d SparkSubmitTimeout=%s UploadTTL=%s "+
|
||||
"LogDir=%s LogLevel=%s LogFormat=%s AnalyzerDataSkewRatio=%.1f "+
|
||||
"AnalyzerGCPressureRatio=%.1f AnalyzerBottleneckShuffleGB=%.1f GinMode=%s",
|
||||
c.ListenAddr,
|
||||
@@ -291,6 +312,7 @@ func (c *Config) String() string {
|
||||
c.HTTPClientTimeout,
|
||||
c.MaxResponseBytes,
|
||||
c.SparkSubmitTimeout,
|
||||
c.UploadTTL,
|
||||
c.LogDir,
|
||||
c.LogLevel,
|
||||
c.LogFormat,
|
||||
|
||||
Reference in New Issue
Block a user