60 lines
2.0 KiB
YAML
60 lines
2.0 KiB
YAML
# spark-executor-mcp — production runtime
|
|
#
|
|
# Bring up with:
|
|
# docker compose up -d --build
|
|
#
|
|
# Prerequisites (one-time):
|
|
# 1. Put your YARN/Hadoop client configs in ./hadoop-conf/ (must contain
|
|
# core-site.xml + yarn-site.xml + hdfs-site.xml matching the target
|
|
# cluster). The Dockerfile's RUN mkdir -p already creates the dir, but
|
|
# it will be empty until you populate it.
|
|
# 2. Set YARN_RESOURCE_MANAGER_URL in .env (or export it in your shell).
|
|
# This is the fallback when a Connection was saved without yarn_rm_url.
|
|
#
|
|
# After it starts, the MCP endpoint is at:
|
|
# http://localhost:8000/spark-executor-mcp (initialize -> tools/list -> tools/call)
|
|
|
|
services:
|
|
mcp-server:
|
|
build:
|
|
context: .
|
|
dockerfile: Dockerfile
|
|
image: mcp-server:latest
|
|
container_name: mcp-tools
|
|
|
|
restart: unless-stopped
|
|
|
|
ports:
|
|
- "8000:8000"
|
|
|
|
environment:
|
|
# Stream Python output to stdout/stderr line-by-line (live loguru output).
|
|
PYTHONUNBUFFERED: "1"
|
|
|
|
# Fallback YARN RM URL used by the REST client when a Connection's
|
|
# yarn_rm_url is not set or a Job lacks a snapshot. Leave empty if
|
|
# you always set yarn_rm_url per Connection via save_connection.
|
|
YARN_RESOURCE_MANAGER_URL: ${YARN_RESOURCE_MANAGER_URL:-}
|
|
|
|
# Optional: pass JVM options to spark-submit (e.g. for proxies, memory).
|
|
# SPARK_SUBMIT_OPTS: "-Dhttps.proxyHost=..."
|
|
|
|
volumes:
|
|
# Persist Connections, PendingSubmissions, and loguru logs across
|
|
# container restarts. Gitignored.
|
|
- ./data:/app/data
|
|
|
|
# Real Hadoop/YARN client configs read by spark-submit at submit time.
|
|
# Read-only so the running container cannot mutate cluster config.
|
|
- ./hadoop-conf:/etc/hadoop/conf:ro
|
|
|
|
|
|
# No resource limits — spark-submit talks to YARN, which does the actual
|
|
# heavy lifting. The MCP server itself is lightweight (FastAPI + httpx).
|
|
# Uncomment to cap if needed:
|
|
# deploy:
|
|
# resources:
|
|
# limits:
|
|
# cpus: "1.0"
|
|
# memory: 1G
|