# Spark Executor MCP — environment template # Copy to .env and edit. .env is gitignored. # # All env vars in this file are read by common/config.py (the single source # of truth for application config). The only exception is the GUNICORN_* # block at the bottom — those are read by gunicorn.conf.py. # --- Data persistence --- # Base directory for connections.json, pending_jobs.json, loguru logs/, # and (by default) jobs/. Mount this from the host in production so # state survives container restarts. The default ./data/ is fine in dev. # # SPARK_EXECUTOR_DATA_DIR=./data # SPARK_EXECUTOR_DATA_DIR=/var/lib/spark-executor/data # --- Job files (LLM-generated PySpark) --- # Where generate_job_file writes PySpark source. Defaults to # /jobs. Override to point at a larger disk # (e.g. /var/spark-jobs) when the data volume is small. # # SPARK_EXECUTOR_JOBS_DIR=./data/jobs # SPARK_EXECUTOR_JOBS_DIR=/var/spark-jobs # --- YARN REST client --- # Fallback URL when a Job's yarn_rm_url (snapshotted from its Connection # at prepare_submit_job time) is unset. Set this OR per-Connection via # save_connection. # # Examples: # YARN_RESOURCE_MANAGER_URL=http://yarn-rm.prod.internal:8088 # YARN_RESOURCE_MANAGER_URL=https://yarn-rm.staging.example.com:8088 YARN_RESOURCE_MANAGER_URL= # --- Loguru file sinks --- # Base directory for loguru output. Subdirs debug/ and info/ are created # automatically; rotated daily, gzipped, kept 30 days. Defaults to # /logs. Override to point at a dedicated log # volume (e.g. /var/log/spark-executor) or a network mount. # # SPARK_EXECUTOR_LOG_DIR=./data/logs # SPARK_EXECUTOR_LOG_DIR=/var/log/spark-executor # --- Loguru verbosity --- # For stderr + the info-level file sink. The debug-level file sink # always captures full DEBUG (audit trail regardless of level). # DEBUG - default; full verbosity # INFO - quieter; recommended for production # # SPARK_EXECUTOR_LOG_LEVEL=DEBUG # SPARK_EXECUTOR_LOG_LEVEL=INFO # --- Optional: JVM flags forwarded to spark-submit --- # Useful for proxies, custom truststores, or driver memory caps. # SPARK_SUBMIT_OPTS=-Dhttps.proxyHost=proxy.corp -Dhttps.proxyPort=3128 # --- Java + Spark install paths --- # Where the openjdk-17-jre-headless JDK lives, and where the Spark # distribution was extracted during the image build. Defaults match the # Dockerfile's ENTRYPOINT script. Override if you mount a different # Java (e.g. /usr/lib/jvm/java-17-openjdk-arm64 on some ARM hosts) or # a pre-installed Spark from a host volume (e.g. /opt/spark-3.5.1-bin-hadoop3). # # The container's docker-entrypoint.sh re-derives PATH from these values # at every start, so overriding them here actually changes which `java` # and `spark-submit` binaries the gunicorn process picks up. # # JAVA_HOME=/usr/lib/jvm/java-17-openjdk-amd64 # SPARK_HOME=/opt/spark # --- Gunicorn process model (see gunicorn.conf.py; NOT read by common/config.py) --- # Defaults shown. These are read by gunicorn directly, not by the app. # GUNICORN_WORKERS=2 # GUNICORN_THREADS=1 # GUNICORN_TIMEOUT=120 # generous; yarn logs can be slow # GUNICORN_GRACEFUL_TIMEOUT=30 # GUNICORN_KEEPALIVE=5 # GUNICORN_BIND=0.0.0.0:8000 # GUNICORN_LOGLEVEL=info