diff --git a/.env.example b/.env.example new file mode 100644 index 0000000..4e1e364 --- /dev/null +++ b/.env.example @@ -0,0 +1,15 @@ +# Spark Executor MCP — environment template +# Copy to .env and edit. .env is gitignored. + +# YARN ResourceManager URL the REST client falls back to when a Connection +# was saved without yarn_rm_url, or a Job lacks a snapshot (e.g. if the +# pending record was created before yarn_rm_url was a field). +# +# Examples: +# YARN_RESOURCE_MANAGER_URL=http://yarn-rm.prod.internal:8088 +# YARN_RESOURCE_MANAGER_URL=https://yarn-rm.staging.example.com:8088 +YARN_RESOURCE_MANAGER_URL= + +# Optional: JVM flags forwarded to spark-submit. Useful for proxies, custom +# truststores, or driver memory caps. +# SPARK_SUBMIT_OPTS=-Dhttps.proxyHost=proxy.corp -Dhttps.proxyPort=3128 diff --git a/docker-compose.yml b/docker-compose.yml new file mode 100644 index 0000000..9a27022 --- /dev/null +++ b/docker-compose.yml @@ -0,0 +1,68 @@ +# spark-executor-mcp — production runtime +# +# Bring up with: +# docker compose up -d --build +# +# Prerequisites (one-time): +# 1. Put your YARN/Hadoop client configs in ./hadoop-conf/ (must contain +# core-site.xml + yarn-site.xml + hdfs-site.xml matching the target +# cluster). The Dockerfile's RUN mkdir -p already creates the dir, but +# it will be empty until you populate it. +# 2. Set YARN_RESOURCE_MANAGER_URL in .env (or export it in your shell). +# This is the fallback when a Connection was saved without yarn_rm_url. +# +# After it starts, the MCP endpoint is at: +# http://localhost:8000/spark-executor-mcp (initialize -> tools/list -> tools/call) + +services: + spark-executor-mcp: + build: + context: . + dockerfile: Dockerfile + image: spark-executor-mcp:latest + container_name: spark-executor-mcp + + restart: unless-stopped + + ports: + - "8000:8000" + + environment: + # Stream Python output to stdout/stderr line-by-line (live loguru output). + PYTHONUNBUFFERED: "1" + + # Fallback YARN RM URL used by the REST client when a Connection's + # yarn_rm_url is not set or a Job lacks a snapshot. Leave empty if + # you always set yarn_rm_url per Connection via save_connection. + YARN_RESOURCE_MANAGER_URL: ${YARN_RESOURCE_MANAGER_URL:-} + + # Optional: pass JVM options to spark-submit (e.g. for proxies, memory). + # SPARK_SUBMIT_OPTS: "-Dhttps.proxyHost=..." + + volumes: + # Persist Connections, PendingSubmissions, and loguru logs across + # container restarts. Gitignored. + - ./data:/app/data + + # Real Hadoop/YARN client configs read by spark-submit at submit time. + # Read-only so the running container cannot mutate cluster config. + - ./hadoop-conf:/etc/hadoop/conf:ro + + healthcheck: + # The MCP endpoint requires an initialize handshake for real liveness; + # /openapi.json (the main app's spec) is a cheap proxy for "process up". + # Replace with a /health route once added to main.py (see note below). + test: ["CMD", "python", "-c", "import httpx; httpx.get('http://localhost:8000/openapi.json', timeout=5).raise_for_status()"] + interval: 30s + timeout: 5s + retries: 3 + start_period: 15s + + # No resource limits — spark-submit talks to YARN, which does the actual + # heavy lifting. The MCP server itself is lightweight (FastAPI + httpx). + # Uncomment to cap if needed: + # deploy: + # resources: + # limits: + # cpus: "1.0" + # memory: 1G