Claude
5970fadcd2
Three problems with the prior Dockerfile:
1. JAVA_HOME and SPARK_HOME were hardcoded at build time (ENV ...),
so docker-compose / .env overrides had no effect.
2. PATH was constructed at build time using those hardcoded values,
so even if you could override the env vars, PATH would still
reference the old literal paths (e.g. /usr/lib/jvm/java-17-openjdk-amd64/bin
hardcoded into PATH at image build).
3. There was no .env.example entry for either, so operators had no
template to follow.
Fix:
- docker-entrypoint.sh: re-derives PATH from the (possibly overridden)
JAVA_HOME and SPARK_HOME at every container start. Strips any stale
JDK/Spark bin dirs from PATH first so a restart with a new override
actually changes which / resolve.
- Dockerfile: COPY + ENTRYPOINT [entrypoint.sh], CMD [gunicorn main:app].
The existing ENV JAVA_HOME and ENV SPARK_HOME stay as the defaults
so the image works out of the box; users override via .env.
- .env.example: new 'Java + Spark install paths' section, default
values match the Dockerfile, comments explain the entrypoint
re-derivation.
- docker-compose.yml: forwards JAVA_HOME and SPARK_HOME with
container-side defaults matching the Dockerfile.
Verified:
- 116/116 tests still pass
- Direct entrypoint run with JAVA_HOME=/fake/jdk shows PATH rebuilt as
/app/.venv/bin:/fake/jdk/bin:/fake/spark/bin:... (override took effect)
Note: uses awk instead of 'paste -sd:' for portability across macOS
(BSD paste doesn't support -s) and Linux (GNU does).
92 lines
3.6 KiB
YAML
92 lines
3.6 KiB
YAML
# spark-executor-mcp — production runtime
|
|
#
|
|
# Bring up with:
|
|
# docker compose up -d --build
|
|
#
|
|
# Prerequisites (one-time):
|
|
# 1. Put your YARN/Hadoop client configs in ./hadoop-conf/ (must contain
|
|
# core-site.xml + yarn-site.xml + hdfs-site.xml matching the target
|
|
# cluster). The Dockerfile's RUN mkdir -p already creates the dir, but
|
|
# it will be empty until you populate it.
|
|
# 2. Set YARN_RESOURCE_MANAGER_URL in .env (or export it in your shell).
|
|
# This is the fallback when a Connection was saved without yarn_rm_url.
|
|
#
|
|
# After it starts, the MCP endpoint is at:
|
|
# http://localhost:8000/spark-executor-mcp (initialize -> tools/list -> tools/call)
|
|
|
|
services:
|
|
mcp-server:
|
|
build:
|
|
context: .
|
|
dockerfile: Dockerfile
|
|
image: mcp-server:latest
|
|
container_name: mcp-tools
|
|
|
|
restart: unless-stopped
|
|
|
|
ports:
|
|
- "8000:8000"
|
|
|
|
environment:
|
|
# Stream Python output to stdout/stderr line-by-line (live loguru output).
|
|
PYTHONUNBUFFERED: "1"
|
|
|
|
# --- common/config.py knobs (see .env.example for full docs) ---
|
|
# Base dir for connections.json, pending_jobs.json, loguru logs/, jobs/.
|
|
# Defaults to ./data inside the container (mounted from host via volumes below).
|
|
SPARK_EXECUTOR_DATA_DIR: ${SPARK_EXECUTOR_DATA_DIR:-/app/data}
|
|
|
|
# Where generate_job_file writes LLM-generated PySpark code.
|
|
# Defaults to <SPARK_EXECUTOR_DATA_DIR>/jobs.
|
|
SPARK_EXECUTOR_JOBS_DIR: ${SPARK_EXECUTOR_JOBS_DIR:-/app/data/jobs}
|
|
|
|
# Base dir for loguru file output (debug/ + info/ subdirs are
|
|
# auto-created). Defaults to <SPARK_EXECUTOR_DATA_DIR>/logs.
|
|
SPARK_EXECUTOR_LOG_DIR: ${SPARK_EXECUTOR_LOG_DIR:-/app/data/logs}
|
|
|
|
# Fallback YARN RM URL used by the REST client when a Connection's
|
|
# yarn_rm_url is not set or a Job lacks a snapshot. Leave empty if
|
|
# you always set yarn_rm_url per Connection via save_connection.
|
|
YARN_RESOURCE_MANAGER_URL: ${YARN_RESOURCE_MANAGER_URL:-}
|
|
|
|
# Loguru verbosity for stderr + info file. DEBUG | INFO.
|
|
SPARK_EXECUTOR_LOG_LEVEL: ${SPARK_EXECUTOR_LOG_LEVEL:-DEBUG}
|
|
|
|
# --- Runtime paths (consumed by docker-entrypoint.sh, not common/config.py) ---
|
|
# Override to point at a different JDK install or pre-mounted Spark
|
|
# distribution. The entrypoint re-derives PATH from these at every
|
|
# container start, so overrides actually take effect.
|
|
# JAVA_HOME=/usr/lib/jvm/java-17-openjdk-amd64
|
|
# SPARK_HOME=/opt/spark
|
|
JAVA_HOME: ${JAVA_HOME:-/usr/lib/jvm/java-17-openjdk-amd64}
|
|
SPARK_HOME: ${SPARK_HOME:-/opt/spark}
|
|
|
|
# --- gunicorn.conf.py knobs (NOT read by common/config.py) ---
|
|
# 2 workers is a good default for a small MCP service; raise for
|
|
# high-concurrency deploys.
|
|
GUNICORN_WORKERS: ${GUNICORN_WORKERS:-2}
|
|
GUNICORN_TIMEOUT: ${GUNICORN_TIMEOUT:-120}
|
|
GUNICORN_BIND: ${GUNICORN_BIND:-0.0.0.0:8000}
|
|
|
|
# Optional: pass JVM options to spark-submit (e.g. for proxies, memory).
|
|
# SPARK_SUBMIT_OPTS: "-Dhttps.proxyHost=..."
|
|
|
|
volumes:
|
|
# Persist Connections, PendingSubmissions, and loguru logs across
|
|
# container restarts. Gitignored.
|
|
- ./data:/app/data
|
|
|
|
# Real Hadoop/YARN client configs read by spark-submit at submit time.
|
|
# Read-only so the running container cannot mutate cluster config.
|
|
- ./hadoop-conf:/etc/hadoop/conf:ro
|
|
|
|
|
|
# No resource limits — spark-submit talks to YARN, which does the actual
|
|
# heavy lifting. The MCP server itself is lightweight (FastAPI + httpx).
|
|
# Uncomment to cap if needed:
|
|
# deploy:
|
|
# resources:
|
|
# limits:
|
|
# cpus: "1.0"
|
|
# memory: 1G
|