diff --git a/.env.example b/.env.example index 05e5310..ca5f153 100644 --- a/.env.example +++ b/.env.example @@ -1,23 +1,55 @@ # Spark Executor MCP — environment template # Copy to .env and edit. .env is gitignored. +# +# All env vars in this file are read by common/config.py (the single source +# of truth for application config). The only exception is the GUNICORN_* +# block at the bottom — those are read by gunicorn.conf.py. -# YARN ResourceManager URL the REST client falls back to when a Connection -# was saved without yarn_rm_url, or a Job lacks a snapshot (e.g. if the -# pending record was created before yarn_rm_url was a field). +# --- Data persistence --- +# Base directory for connections.json, pending_jobs.json, loguru logs/, +# and (by default) jobs/. Mount this from the host in production so +# state survives container restarts. The default ./data/ is fine in dev. +# +# SPARK_EXECUTOR_DATA_DIR=./data +# SPARK_EXECUTOR_DATA_DIR=/var/lib/spark-executor/data + +# --- Job files (LLM-generated PySpark) --- +# Where generate_job_file writes PySpark source. Defaults to +# /jobs. Override to point at a larger disk +# (e.g. /var/spark-jobs) when the data volume is small. +# +# SPARK_EXECUTOR_JOBS_DIR=./data/jobs +# SPARK_EXECUTOR_JOBS_DIR=/var/spark-jobs + +# --- YARN REST client --- +# Fallback URL when a Job's yarn_rm_url (snapshotted from its Connection +# at prepare_submit_job time) is unset. Set this OR per-Connection via +# save_connection. # # Examples: # YARN_RESOURCE_MANAGER_URL=http://yarn-rm.prod.internal:8088 # YARN_RESOURCE_MANAGER_URL=https://yarn-rm.staging.example.com:8088 YARN_RESOURCE_MANAGER_URL= -# Optional: JVM flags forwarded to spark-submit. Useful for proxies, custom -# truststores, or driver memory caps. +# --- Loguru --- +# Verbosity for stderr + the info-level file sink. The debug-level file +# sink always captures full DEBUG (audit trail regardless of level). +# DEBUG - default; full verbosity +# INFO - quieter; recommended for production +# +# SPARK_EXECUTOR_LOG_LEVEL=DEBUG +# SPARK_EXECUTOR_LOG_LEVEL=INFO + +# --- Optional: JVM flags forwarded to spark-submit --- +# Useful for proxies, custom truststores, or driver memory caps. # SPARK_SUBMIT_OPTS=-Dhttps.proxyHost=proxy.corp -Dhttps.proxyPort=3128 -# Gunicorn process model (see gunicorn.conf.py). Defaults shown. +# --- Gunicorn process model (see gunicorn.conf.py; NOT read by common/config.py) --- +# Defaults shown. These are read by gunicorn directly, not by the app. # GUNICORN_WORKERS=2 # GUNICORN_THREADS=1 -# GUNICORN_TIMEOUT=120 # generous; yarn logs can be slow +# GUNICORN_TIMEOUT=120 # generous; yarn logs can be slow # GUNICORN_GRACEFUL_TIMEOUT=30 # GUNICORN_KEEPALIVE=5 # GUNICORN_BIND=0.0.0.0:8000 +# GUNICORN_LOGLEVEL=info diff --git a/docker-compose.yml b/docker-compose.yml index bc651c5..0ee6553 100644 --- a/docker-compose.yml +++ b/docker-compose.yml @@ -31,12 +31,24 @@ services: # Stream Python output to stdout/stderr line-by-line (live loguru output). PYTHONUNBUFFERED: "1" + # --- common/config.py knobs (see .env.example for full docs) --- + # Base dir for connections.json, pending_jobs.json, loguru logs/, jobs/. + # Defaults to ./data inside the container (mounted from host via volumes below). + SPARK_EXECUTOR_DATA_DIR: ${SPARK_EXECUTOR_DATA_DIR:-/app/data} + + # Where generate_job_file writes LLM-generated PySpark code. + # Defaults to /jobs. + SPARK_EXECUTOR_JOBS_DIR: ${SPARK_EXECUTOR_JOBS_DIR:-/app/data/jobs} + # Fallback YARN RM URL used by the REST client when a Connection's # yarn_rm_url is not set or a Job lacks a snapshot. Leave empty if # you always set yarn_rm_url per Connection via save_connection. YARN_RESOURCE_MANAGER_URL: ${YARN_RESOURCE_MANAGER_URL:-} - # Gunicorn tuning (see gunicorn.conf.py for full list of knobs). + # Loguru verbosity for stderr + info file. DEBUG | INFO. + SPARK_EXECUTOR_LOG_LEVEL: ${SPARK_EXECUTOR_LOG_LEVEL:-DEBUG} + + # --- gunicorn.conf.py knobs (NOT read by common/config.py) --- # 2 workers is a good default for a small MCP service; raise for # high-concurrency deploys. GUNICORN_WORKERS: ${GUNICORN_WORKERS:-2}