docs: document every env var in .env.example + forward in compose

.env.example now lists every env var read by the app, with a comment
explaining the meaning and the default. The previously-missing ones
(from the common/config.py refactor) are added:
  - SPARK_EXECUTOR_DATA_DIR
  - SPARK_EXECUTOR_JOBS_DIR
  - SPARK_EXECUTOR_LOG_LEVEL

docker-compose.yml forwards all of them with sensible container-side
defaults (SPARK_EXECUTOR_DATA_DIR=/app/data, which is the volume
mount point from the host).

Split the env block in compose into two commented sections
('common/config.py knobs' vs 'gunicorn.conf.py knobs') so it's clear
which file each one is consumed by.

The GUNICORN_LOGLEVEL entry is also added (was missing). All other
GUNICORN_* knobs were already documented.

116/116 still pass.

.env.example

@@ -1,23 +1,55 @@
 # Spark Executor MCP — environment template
 # Copy to .env and edit. .env is gitignored.
+#
+# All env vars in this file are read by common/config.py (the single source
+# of truth for application config). The only exception is the GUNICORN_*
+# block at the bottom — those are read by gunicorn.conf.py.
 
-# YARN ResourceManager URL the REST client falls back to when a Connection
-# was saved without yarn_rm_url, or a Job lacks a snapshot (e.g. if the
-# pending record was created before yarn_rm_url was a field).
+# --- Data persistence ---
+# Base directory for connections.json, pending_jobs.json, loguru logs/,
+# and (by default) jobs/. Mount this from the host in production so
+# state survives container restarts. The default ./data/ is fine in dev.
+#
+# SPARK_EXECUTOR_DATA_DIR=./data
+# SPARK_EXECUTOR_DATA_DIR=/var/lib/spark-executor/data
+
+# --- Job files (LLM-generated PySpark) ---
+# Where generate_job_file writes PySpark source. Defaults to
+# <SPARK_EXECUTOR_DATA_DIR>/jobs. Override to point at a larger disk
+# (e.g. /var/spark-jobs) when the data volume is small.
+#
+# SPARK_EXECUTOR_JOBS_DIR=./data/jobs
+# SPARK_EXECUTOR_JOBS_DIR=/var/spark-jobs
+
+# --- YARN REST client ---
+# Fallback URL when a Job's yarn_rm_url (snapshotted from its Connection
+# at prepare_submit_job time) is unset. Set this OR per-Connection via
+# save_connection.
 #
 # Examples:
 #   YARN_RESOURCE_MANAGER_URL=http://yarn-rm.prod.internal:8088
 #   YARN_RESOURCE_MANAGER_URL=https://yarn-rm.staging.example.com:8088
 YARN_RESOURCE_MANAGER_URL=
 
-# Optional: JVM flags forwarded to spark-submit. Useful for proxies, custom
-# truststores, or driver memory caps.
+# --- Loguru ---
+# Verbosity for stderr + the info-level file sink. The debug-level file
+# sink always captures full DEBUG (audit trail regardless of level).
+#   DEBUG - default; full verbosity
+#   INFO  - quieter; recommended for production
+#
+# SPARK_EXECUTOR_LOG_LEVEL=DEBUG
+# SPARK_EXECUTOR_LOG_LEVEL=INFO
+
+# --- Optional: JVM flags forwarded to spark-submit ---
+# Useful for proxies, custom truststores, or driver memory caps.
 # SPARK_SUBMIT_OPTS=-Dhttps.proxyHost=proxy.corp -Dhttps.proxyPort=3128
 
-# Gunicorn process model (see gunicorn.conf.py). Defaults shown.
+# --- Gunicorn process model (see gunicorn.conf.py; NOT read by common/config.py) ---
+# Defaults shown. These are read by gunicorn directly, not by the app.
 # GUNICORN_WORKERS=2
 # GUNICORN_THREADS=1
 # GUNICORN_TIMEOUT=120         # generous; yarn logs can be slow
 # GUNICORN_GRACEFUL_TIMEOUT=30
 # GUNICORN_KEEPALIVE=5
 # GUNICORN_BIND=0.0.0.0:8000
+# GUNICORN_LOGLEVEL=info

docker-compose.yml

@@ -31,12 +31,24 @@ services:
       # Stream Python output to stdout/stderr line-by-line (live loguru output).
       PYTHONUNBUFFERED: "1"
 
+      # --- common/config.py knobs (see .env.example for full docs) ---
+      # Base dir for connections.json, pending_jobs.json, loguru logs/, jobs/.
+      # Defaults to ./data inside the container (mounted from host via volumes below).
+      SPARK_EXECUTOR_DATA_DIR: ${SPARK_EXECUTOR_DATA_DIR:-/app/data}
+
+      # Where generate_job_file writes LLM-generated PySpark code.
+      # Defaults to <SPARK_EXECUTOR_DATA_DIR>/jobs.
+      SPARK_EXECUTOR_JOBS_DIR: ${SPARK_EXECUTOR_JOBS_DIR:-/app/data/jobs}
+
       # Fallback YARN RM URL used by the REST client when a Connection's
       # yarn_rm_url is not set or a Job lacks a snapshot. Leave empty if
       # you always set yarn_rm_url per Connection via save_connection.
       YARN_RESOURCE_MANAGER_URL: ${YARN_RESOURCE_MANAGER_URL:-}
 
-      # Gunicorn tuning (see gunicorn.conf.py for full list of knobs).
+      # Loguru verbosity for stderr + info file. DEBUG | INFO.
+      SPARK_EXECUTOR_LOG_LEVEL: ${SPARK_EXECUTOR_LOG_LEVEL:-DEBUG}
+
+      # --- gunicorn.conf.py knobs (NOT read by common/config.py) ---
       # 2 workers is a good default for a small MCP service; raise for
       # high-concurrency deploys.
       GUNICORN_WORKERS: ${GUNICORN_WORKERS:-2}