feat: add docker-compose.yml + .env.example
docker-compose.yml: - Builds the Dockerfile, exposes :8000 - Mounts ./data (persistent) and ./hadoop-conf (ro) for cluster configs - Forwards YARN_RESOURCE_MANAGER_URL via env var (fallback for REST client) - Healthcheck via /openapi.json (cheap liveness probe; MCP initialize handshake would be more accurate but requires session-id plumbing) - restart: unless-stopped for production - Resource limit section commented out (server itself is lightweight) .env.example: - Documents the env vars compose expects - .env is gitignored
This commit is contained in:
@@ -0,0 +1,68 @@
|
||||
# spark-executor-mcp — production runtime
|
||||
#
|
||||
# Bring up with:
|
||||
# docker compose up -d --build
|
||||
#
|
||||
# Prerequisites (one-time):
|
||||
# 1. Put your YARN/Hadoop client configs in ./hadoop-conf/ (must contain
|
||||
# core-site.xml + yarn-site.xml + hdfs-site.xml matching the target
|
||||
# cluster). The Dockerfile's RUN mkdir -p already creates the dir, but
|
||||
# it will be empty until you populate it.
|
||||
# 2. Set YARN_RESOURCE_MANAGER_URL in .env (or export it in your shell).
|
||||
# This is the fallback when a Connection was saved without yarn_rm_url.
|
||||
#
|
||||
# After it starts, the MCP endpoint is at:
|
||||
# http://localhost:8000/spark-executor-mcp (initialize -> tools/list -> tools/call)
|
||||
|
||||
services:
|
||||
spark-executor-mcp:
|
||||
build:
|
||||
context: .
|
||||
dockerfile: Dockerfile
|
||||
image: spark-executor-mcp:latest
|
||||
container_name: spark-executor-mcp
|
||||
|
||||
restart: unless-stopped
|
||||
|
||||
ports:
|
||||
- "8000:8000"
|
||||
|
||||
environment:
|
||||
# Stream Python output to stdout/stderr line-by-line (live loguru output).
|
||||
PYTHONUNBUFFERED: "1"
|
||||
|
||||
# Fallback YARN RM URL used by the REST client when a Connection's
|
||||
# yarn_rm_url is not set or a Job lacks a snapshot. Leave empty if
|
||||
# you always set yarn_rm_url per Connection via save_connection.
|
||||
YARN_RESOURCE_MANAGER_URL: ${YARN_RESOURCE_MANAGER_URL:-}
|
||||
|
||||
# Optional: pass JVM options to spark-submit (e.g. for proxies, memory).
|
||||
# SPARK_SUBMIT_OPTS: "-Dhttps.proxyHost=..."
|
||||
|
||||
volumes:
|
||||
# Persist Connections, PendingSubmissions, and loguru logs across
|
||||
# container restarts. Gitignored.
|
||||
- ./data:/app/data
|
||||
|
||||
# Real Hadoop/YARN client configs read by spark-submit at submit time.
|
||||
# Read-only so the running container cannot mutate cluster config.
|
||||
- ./hadoop-conf:/etc/hadoop/conf:ro
|
||||
|
||||
healthcheck:
|
||||
# The MCP endpoint requires an initialize handshake for real liveness;
|
||||
# /openapi.json (the main app's spec) is a cheap proxy for "process up".
|
||||
# Replace with a /health route once added to main.py (see note below).
|
||||
test: ["CMD", "python", "-c", "import httpx; httpx.get('http://localhost:8000/openapi.json', timeout=5).raise_for_status()"]
|
||||
interval: 30s
|
||||
timeout: 5s
|
||||
retries: 3
|
||||
start_period: 15s
|
||||
|
||||
# No resource limits — spark-submit talks to YARN, which does the actual
|
||||
# heavy lifting. The MCP server itself is lightweight (FastAPI + httpx).
|
||||
# Uncomment to cap if needed:
|
||||
# deploy:
|
||||
# resources:
|
||||
# limits:
|
||||
# cpus: "1.0"
|
||||
# memory: 1G
|
||||
Reference in New Issue
Block a user