Claude
52624aacff
.env.example now lists every env var read by the app, with a comment
explaining the meaning and the default. The previously-missing ones
(from the common/config.py refactor) are added:
- SPARK_EXECUTOR_DATA_DIR
- SPARK_EXECUTOR_JOBS_DIR
- SPARK_EXECUTOR_LOG_LEVEL
docker-compose.yml forwards all of them with sensible container-side
defaults (SPARK_EXECUTOR_DATA_DIR=/app/data, which is the volume
mount point from the host).
Split the env block in compose into two commented sections
('common/config.py knobs' vs 'gunicorn.conf.py knobs') so it's clear
which file each one is consumed by.
The GUNICORN_LOGLEVEL entry is also added (was missing). All other
GUNICORN_* knobs were already documented.
116/116 still pass.
79 lines
2.9 KiB
YAML
79 lines
2.9 KiB
YAML
# spark-executor-mcp — production runtime
|
|
#
|
|
# Bring up with:
|
|
# docker compose up -d --build
|
|
#
|
|
# Prerequisites (one-time):
|
|
# 1. Put your YARN/Hadoop client configs in ./hadoop-conf/ (must contain
|
|
# core-site.xml + yarn-site.xml + hdfs-site.xml matching the target
|
|
# cluster). The Dockerfile's RUN mkdir -p already creates the dir, but
|
|
# it will be empty until you populate it.
|
|
# 2. Set YARN_RESOURCE_MANAGER_URL in .env (or export it in your shell).
|
|
# This is the fallback when a Connection was saved without yarn_rm_url.
|
|
#
|
|
# After it starts, the MCP endpoint is at:
|
|
# http://localhost:8000/spark-executor-mcp (initialize -> tools/list -> tools/call)
|
|
|
|
services:
|
|
mcp-server:
|
|
build:
|
|
context: .
|
|
dockerfile: Dockerfile
|
|
image: mcp-server:latest
|
|
container_name: mcp-tools
|
|
|
|
restart: unless-stopped
|
|
|
|
ports:
|
|
- "8000:8000"
|
|
|
|
environment:
|
|
# Stream Python output to stdout/stderr line-by-line (live loguru output).
|
|
PYTHONUNBUFFERED: "1"
|
|
|
|
# --- common/config.py knobs (see .env.example for full docs) ---
|
|
# Base dir for connections.json, pending_jobs.json, loguru logs/, jobs/.
|
|
# Defaults to ./data inside the container (mounted from host via volumes below).
|
|
SPARK_EXECUTOR_DATA_DIR: ${SPARK_EXECUTOR_DATA_DIR:-/app/data}
|
|
|
|
# Where generate_job_file writes LLM-generated PySpark code.
|
|
# Defaults to <SPARK_EXECUTOR_DATA_DIR>/jobs.
|
|
SPARK_EXECUTOR_JOBS_DIR: ${SPARK_EXECUTOR_JOBS_DIR:-/app/data/jobs}
|
|
|
|
# Fallback YARN RM URL used by the REST client when a Connection's
|
|
# yarn_rm_url is not set or a Job lacks a snapshot. Leave empty if
|
|
# you always set yarn_rm_url per Connection via save_connection.
|
|
YARN_RESOURCE_MANAGER_URL: ${YARN_RESOURCE_MANAGER_URL:-}
|
|
|
|
# Loguru verbosity for stderr + info file. DEBUG | INFO.
|
|
SPARK_EXECUTOR_LOG_LEVEL: ${SPARK_EXECUTOR_LOG_LEVEL:-DEBUG}
|
|
|
|
# --- gunicorn.conf.py knobs (NOT read by common/config.py) ---
|
|
# 2 workers is a good default for a small MCP service; raise for
|
|
# high-concurrency deploys.
|
|
GUNICORN_WORKERS: ${GUNICORN_WORKERS:-2}
|
|
GUNICORN_TIMEOUT: ${GUNICORN_TIMEOUT:-120}
|
|
GUNICORN_BIND: ${GUNICORN_BIND:-0.0.0.0:8000}
|
|
|
|
# Optional: pass JVM options to spark-submit (e.g. for proxies, memory).
|
|
# SPARK_SUBMIT_OPTS: "-Dhttps.proxyHost=..."
|
|
|
|
volumes:
|
|
# Persist Connections, PendingSubmissions, and loguru logs across
|
|
# container restarts. Gitignored.
|
|
- ./data:/app/data
|
|
|
|
# Real Hadoop/YARN client configs read by spark-submit at submit time.
|
|
# Read-only so the running container cannot mutate cluster config.
|
|
- ./hadoop-conf:/etc/hadoop/conf:ro
|
|
|
|
|
|
# No resource limits — spark-submit talks to YARN, which does the actual
|
|
# heavy lifting. The MCP server itself is lightweight (FastAPI + httpx).
|
|
# Uncomment to cap if needed:
|
|
# deploy:
|
|
# resources:
|
|
# limits:
|
|
# cpus: "1.0"
|
|
# memory: 1G
|