refactor: env-configurable loguru file paths

common/logging.py used hardcoded 'data/logs/{debug,info}/' paths,
which forced logs into the same dir as connections.json. Operators
couldn't put logs on a dedicated volume or a different disk for
retention policy reasons.

Now the base log dir comes from settings.log_dir (env var
SPARK_EXECUTOR_LOG_DIR, default <data_dir>/logs). Subdirs debug/ and
info/ are auto-created. Same behavior in dev (./data/logs/), but in
prod you can do:

  SPARK_EXECUTOR_LOG_DIR=/var/log/spark-executor   # dedicated volume
  SPARK_EXECUTOR_LOG_DIR=/mnt/slow-storage/logs   # cold storage for old logs

Other changes:
  - .env.example: new SPARK_EXECUTOR_LOG_DIR entry, split into its own
    'Loguru file sinks' section
  - docker-compose.yml: forwards SPARK_EXECUTOR_LOG_DIR with the
    container-side default /app/data/logs (the existing ./data:/app/data
    volume mount already covers this)
  - common/config.py: Settings gets a new log_dir field, defaults
    derived from data_dir; reload() also resets it

116/116 still pass. Live smoke verified: with
SPARK_EXECUTOR_LOG_DIR=/tmp/spark-logs, both
  /tmp/spark-logs/info/2026-06-25.log
  /tmp/spark-logs/debug/2026-06-25.log
are created on the first request.

.env.example

@@ -31,9 +31,18 @@
 #   YARN_RESOURCE_MANAGER_URL=https://yarn-rm.staging.example.com:8088
 YARN_RESOURCE_MANAGER_URL=
 
-# --- Loguru ---
-# Verbosity for stderr + the info-level file sink. The debug-level file
-# sink always captures full DEBUG (audit trail regardless of level).
+# --- Loguru file sinks ---
+# Base directory for loguru output. Subdirs debug/ and info/ are created
+# automatically; rotated daily, gzipped, kept 30 days. Defaults to
+# <SPARK_EXECUTOR_DATA_DIR>/logs. Override to point at a dedicated log
+# volume (e.g. /var/log/spark-executor) or a network mount.
+#
+# SPARK_EXECUTOR_LOG_DIR=./data/logs
+# SPARK_EXECUTOR_LOG_DIR=/var/log/spark-executor
+
+# --- Loguru verbosity ---
+# For stderr + the info-level file sink. The debug-level file sink
+# always captures full DEBUG (audit trail regardless of level).
 #   DEBUG - default; full verbosity
 #   INFO  - quieter; recommended for production
 #

common/config.py

@@ -49,6 +49,13 @@ class Settings:
     # SPARK_EXECUTOR_JOBS_DIR (e.g. /var/spark-jobs on a big-disk host).
     jobs_dir: str = ""
 
+    # --- Loguru file sinks ---
+    # Base directory for loguru file output. Debug and info subdirs are
+    # derived as <log_dir>/debug and <log_dir>/info. Defaults to
+    # <data_dir>/logs; override via SPARK_EXECUTOR_LOG_DIR to put logs
+    # on a dedicated volume (e.g. /var/log/spark-executor).
+    log_dir: str = ""
+
     # --- YARN REST client (fallback for Job.yarn_rm_url snapshot) ---
     # Set in the env OR per-Connection via save_connection.
     yarn_resource_manager_url: str | None = None
@@ -64,9 +71,13 @@ class Settings:
         jobs_dir = os.environ.get(
             "SPARK_EXECUTOR_JOBS_DIR", os.path.join(data_dir, "jobs")
         )
+        log_dir = os.environ.get(
+            "SPARK_EXECUTOR_LOG_DIR", os.path.join(data_dir, "logs")
+        )
         return cls(
             data_dir=data_dir,
             jobs_dir=jobs_dir,
+            log_dir=log_dir,
             # `or None` collapses empty string to None for the URL fallback
             yarn_resource_manager_url=os.environ.get("YARN_RESOURCE_MANAGER_URL") or None,
             log_level=os.environ.get("SPARK_EXECUTOR_LOG_LEVEL", "DEBUG"),
@@ -81,6 +92,7 @@ class Settings:
         fresh = self.from_env()
         self.data_dir = fresh.data_dir
         self.jobs_dir = fresh.jobs_dir
+        self.log_dir = fresh.log_dir
         self.yarn_resource_manager_url = fresh.yarn_resource_manager_url
         self.log_level = fresh.log_level
         return self

common/logging.py

@@ -22,8 +22,12 @@ from loguru import logger
 
 from common.config import settings
 
-Path("data/logs/debug").mkdir(parents=True, exist_ok=True)
-Path("data/logs/info").mkdir(parents=True, exist_ok=True)
+# Loguru file sinks — paths derived from settings.log_dir (env-var driven
+# via SPARK_EXECUTOR_LOG_DIR, default <data_dir>/logs).
+DEBUG_LOG_DIR = Path(settings.log_dir) / "debug"
+INFO_LOG_DIR = Path(settings.log_dir) / "info"
+DEBUG_LOG_DIR.mkdir(parents=True, exist_ok=True)
+INFO_LOG_DIR.mkdir(parents=True, exist_ok=True)
 
 logger.remove()
 logger.add(
@@ -40,7 +44,7 @@ logger.add(
 )
 
 logger.add(
-    "data/logs/debug/{time:YYYY-MM-DD}.log",
+    str(DEBUG_LOG_DIR / "{time:YYYY-MM-DD}.log"),
     level="DEBUG",  # always full at the file level for audit
     enqueue=True,
     retention="30 days",
@@ -49,7 +53,7 @@ logger.add(
 )
 
 logger.add(
-    "data/logs/info/{time:YYYY-MM-DD}.log",
+    str(INFO_LOG_DIR / "{time:YYYY-MM-DD}.log"),
     level=settings.log_level,
     enqueue=True,
     retention="30 days",

docker-compose.yml

@@ -40,6 +40,10 @@ services:
       # Defaults to <SPARK_EXECUTOR_DATA_DIR>/jobs.
       SPARK_EXECUTOR_JOBS_DIR: ${SPARK_EXECUTOR_JOBS_DIR:-/app/data/jobs}
 
+      # Base dir for loguru file output (debug/ + info/ subdirs are
+      # auto-created). Defaults to <SPARK_EXECUTOR_DATA_DIR>/logs.
+      SPARK_EXECUTOR_LOG_DIR: ${SPARK_EXECUTOR_LOG_DIR:-/app/data/logs}
+
       # Fallback YARN RM URL used by the REST client when a Connection's
       # yarn_rm_url is not set or a Job lacks a snapshot. Leave empty if
       # you always set yarn_rm_url per Connection via save_connection.