diff --git a/.env.example b/.env.example index ca5f153..c5a70f6 100644 --- a/.env.example +++ b/.env.example @@ -31,9 +31,18 @@ # YARN_RESOURCE_MANAGER_URL=https://yarn-rm.staging.example.com:8088 YARN_RESOURCE_MANAGER_URL= -# --- Loguru --- -# Verbosity for stderr + the info-level file sink. The debug-level file -# sink always captures full DEBUG (audit trail regardless of level). +# --- Loguru file sinks --- +# Base directory for loguru output. Subdirs debug/ and info/ are created +# automatically; rotated daily, gzipped, kept 30 days. Defaults to +# /logs. Override to point at a dedicated log +# volume (e.g. /var/log/spark-executor) or a network mount. +# +# SPARK_EXECUTOR_LOG_DIR=./data/logs +# SPARK_EXECUTOR_LOG_DIR=/var/log/spark-executor + +# --- Loguru verbosity --- +# For stderr + the info-level file sink. The debug-level file sink +# always captures full DEBUG (audit trail regardless of level). # DEBUG - default; full verbosity # INFO - quieter; recommended for production # diff --git a/common/config.py b/common/config.py index 55f7bcb..b416e2b 100644 --- a/common/config.py +++ b/common/config.py @@ -49,6 +49,13 @@ class Settings: # SPARK_EXECUTOR_JOBS_DIR (e.g. /var/spark-jobs on a big-disk host). jobs_dir: str = "" + # --- Loguru file sinks --- + # Base directory for loguru file output. Debug and info subdirs are + # derived as /debug and /info. Defaults to + # /logs; override via SPARK_EXECUTOR_LOG_DIR to put logs + # on a dedicated volume (e.g. /var/log/spark-executor). + log_dir: str = "" + # --- YARN REST client (fallback for Job.yarn_rm_url snapshot) --- # Set in the env OR per-Connection via save_connection. yarn_resource_manager_url: str | None = None @@ -64,9 +71,13 @@ class Settings: jobs_dir = os.environ.get( "SPARK_EXECUTOR_JOBS_DIR", os.path.join(data_dir, "jobs") ) + log_dir = os.environ.get( + "SPARK_EXECUTOR_LOG_DIR", os.path.join(data_dir, "logs") + ) return cls( data_dir=data_dir, jobs_dir=jobs_dir, + log_dir=log_dir, # `or None` collapses empty string to None for the URL fallback yarn_resource_manager_url=os.environ.get("YARN_RESOURCE_MANAGER_URL") or None, log_level=os.environ.get("SPARK_EXECUTOR_LOG_LEVEL", "DEBUG"), @@ -81,6 +92,7 @@ class Settings: fresh = self.from_env() self.data_dir = fresh.data_dir self.jobs_dir = fresh.jobs_dir + self.log_dir = fresh.log_dir self.yarn_resource_manager_url = fresh.yarn_resource_manager_url self.log_level = fresh.log_level return self diff --git a/common/logging.py b/common/logging.py index 1d828cc..8489697 100644 --- a/common/logging.py +++ b/common/logging.py @@ -22,8 +22,12 @@ from loguru import logger from common.config import settings -Path("data/logs/debug").mkdir(parents=True, exist_ok=True) -Path("data/logs/info").mkdir(parents=True, exist_ok=True) +# Loguru file sinks — paths derived from settings.log_dir (env-var driven +# via SPARK_EXECUTOR_LOG_DIR, default /logs). +DEBUG_LOG_DIR = Path(settings.log_dir) / "debug" +INFO_LOG_DIR = Path(settings.log_dir) / "info" +DEBUG_LOG_DIR.mkdir(parents=True, exist_ok=True) +INFO_LOG_DIR.mkdir(parents=True, exist_ok=True) logger.remove() logger.add( @@ -40,7 +44,7 @@ logger.add( ) logger.add( - "data/logs/debug/{time:YYYY-MM-DD}.log", + str(DEBUG_LOG_DIR / "{time:YYYY-MM-DD}.log"), level="DEBUG", # always full at the file level for audit enqueue=True, retention="30 days", @@ -49,7 +53,7 @@ logger.add( ) logger.add( - "data/logs/info/{time:YYYY-MM-DD}.log", + str(INFO_LOG_DIR / "{time:YYYY-MM-DD}.log"), level=settings.log_level, enqueue=True, retention="30 days", diff --git a/docker-compose.yml b/docker-compose.yml index 0ee6553..5db5570 100644 --- a/docker-compose.yml +++ b/docker-compose.yml @@ -40,6 +40,10 @@ services: # Defaults to /jobs. SPARK_EXECUTOR_JOBS_DIR: ${SPARK_EXECUTOR_JOBS_DIR:-/app/data/jobs} + # Base dir for loguru file output (debug/ + info/ subdirs are + # auto-created). Defaults to /logs. + SPARK_EXECUTOR_LOG_DIR: ${SPARK_EXECUTOR_LOG_DIR:-/app/data/logs} + # Fallback YARN RM URL used by the REST client when a Connection's # yarn_rm_url is not set or a Job lacks a snapshot. Leave empty if # you always set yarn_rm_url per Connection via save_connection.