feat: add docker-compose.yml + .env.example

docker-compose.yml:
- Builds the Dockerfile, exposes :8000
- Mounts ./data (persistent) and ./hadoop-conf (ro) for cluster configs
- Forwards YARN_RESOURCE_MANAGER_URL via env var (fallback for REST client)
- Healthcheck via /openapi.json (cheap liveness probe; MCP initialize
  handshake would be more accurate but requires session-id plumbing)
- restart: unless-stopped for production
- Resource limit section commented out (server itself is lightweight)

.env.example:
- Documents the env vars compose expects
- .env is gitignored
This commit is contained in:
Claude
2026-06-24 18:20:16 +08:00
parent 565661371b
commit ce8dc1e2ba
2 changed files with 83 additions and 0 deletions
+15
View File
@@ -0,0 +1,15 @@
# Spark Executor MCP — environment template
# Copy to .env and edit. .env is gitignored.
# YARN ResourceManager URL the REST client falls back to when a Connection
# was saved without yarn_rm_url, or a Job lacks a snapshot (e.g. if the
# pending record was created before yarn_rm_url was a field).
#
# Examples:
# YARN_RESOURCE_MANAGER_URL=http://yarn-rm.prod.internal:8088
# YARN_RESOURCE_MANAGER_URL=https://yarn-rm.staging.example.com:8088
YARN_RESOURCE_MANAGER_URL=
# Optional: JVM flags forwarded to spark-submit. Useful for proxies, custom
# truststores, or driver memory caps.
# SPARK_SUBMIT_OPTS=-Dhttps.proxyHost=proxy.corp -Dhttps.proxyPort=3128
+68
View File
@@ -0,0 +1,68 @@
# spark-executor-mcp — production runtime
#
# Bring up with:
# docker compose up -d --build
#
# Prerequisites (one-time):
# 1. Put your YARN/Hadoop client configs in ./hadoop-conf/ (must contain
# core-site.xml + yarn-site.xml + hdfs-site.xml matching the target
# cluster). The Dockerfile's RUN mkdir -p already creates the dir, but
# it will be empty until you populate it.
# 2. Set YARN_RESOURCE_MANAGER_URL in .env (or export it in your shell).
# This is the fallback when a Connection was saved without yarn_rm_url.
#
# After it starts, the MCP endpoint is at:
# http://localhost:8000/spark-executor-mcp (initialize -> tools/list -> tools/call)
services:
spark-executor-mcp:
build:
context: .
dockerfile: Dockerfile
image: spark-executor-mcp:latest
container_name: spark-executor-mcp
restart: unless-stopped
ports:
- "8000:8000"
environment:
# Stream Python output to stdout/stderr line-by-line (live loguru output).
PYTHONUNBUFFERED: "1"
# Fallback YARN RM URL used by the REST client when a Connection's
# yarn_rm_url is not set or a Job lacks a snapshot. Leave empty if
# you always set yarn_rm_url per Connection via save_connection.
YARN_RESOURCE_MANAGER_URL: ${YARN_RESOURCE_MANAGER_URL:-}
# Optional: pass JVM options to spark-submit (e.g. for proxies, memory).
# SPARK_SUBMIT_OPTS: "-Dhttps.proxyHost=..."
volumes:
# Persist Connections, PendingSubmissions, and loguru logs across
# container restarts. Gitignored.
- ./data:/app/data
# Real Hadoop/YARN client configs read by spark-submit at submit time.
# Read-only so the running container cannot mutate cluster config.
- ./hadoop-conf:/etc/hadoop/conf:ro
healthcheck:
# The MCP endpoint requires an initialize handshake for real liveness;
# /openapi.json (the main app's spec) is a cheap proxy for "process up".
# Replace with a /health route once added to main.py (see note below).
test: ["CMD", "python", "-c", "import httpx; httpx.get('http://localhost:8000/openapi.json', timeout=5).raise_for_status()"]
interval: 30s
timeout: 5s
retries: 3
start_period: 15s
# No resource limits — spark-submit talks to YARN, which does the actual
# heavy lifting. The MCP server itself is lightweight (FastAPI + httpx).
# Uncomment to cap if needed:
# deploy:
# resources:
# limits:
# cpus: "1.0"
# memory: 1G