Claude
13e39b39f3
common/logging.py used hardcoded 'data/logs/{debug,info}/' paths,
which forced logs into the same dir as connections.json. Operators
couldn't put logs on a dedicated volume or a different disk for
retention policy reasons.
Now the base log dir comes from settings.log_dir (env var
SPARK_EXECUTOR_LOG_DIR, default <data_dir>/logs). Subdirs debug/ and
info/ are auto-created. Same behavior in dev (./data/logs/), but in
prod you can do:
SPARK_EXECUTOR_LOG_DIR=/var/log/spark-executor # dedicated volume
SPARK_EXECUTOR_LOG_DIR=/mnt/slow-storage/logs # cold storage for old logs
Other changes:
- .env.example: new SPARK_EXECUTOR_LOG_DIR entry, split into its own
'Loguru file sinks' section
- docker-compose.yml: forwards SPARK_EXECUTOR_LOG_DIR with the
container-side default /app/data/logs (the existing ./data:/app/data
volume mount already covers this)
- common/config.py: Settings gets a new log_dir field, defaults
derived from data_dir; reload() also resets it
116/116 still pass. Live smoke verified: with
SPARK_EXECUTOR_LOG_DIR=/tmp/spark-logs, both
/tmp/spark-logs/info/2026-06-25.log
/tmp/spark-logs/debug/2026-06-25.log
are created on the first request.
62 lines
1.8 KiB
Python
62 lines
1.8 KiB
Python
# coding=utf-8
|
|
"""
|
|
@Time :2026/6/24
|
|
@Author :tao.chen
|
|
|
|
Process-wide loguru configuration. Import `logger` from here in every
|
|
module instead of instantiating new loggers.
|
|
|
|
Log level is controlled via `SPARK_EXECUTOR_LOG_LEVEL` (see common/config.py):
|
|
DEBUG - default; full verbosity
|
|
INFO - quieter; recommended for production
|
|
|
|
Levels used in this project:
|
|
DEBUG - entry/exit of public tools, subprocess commands, file I/O paths
|
|
INFO - business events (job submitted, status changed, connection saved)
|
|
WARNING - recoverable problems (transient YARN issues, retry-able)
|
|
ERROR - raised exceptions (caller will see the traceback)
|
|
"""
|
|
import sys
|
|
from pathlib import Path
|
|
from loguru import logger
|
|
|
|
from common.config import settings
|
|
|
|
# Loguru file sinks — paths derived from settings.log_dir (env-var driven
|
|
# via SPARK_EXECUTOR_LOG_DIR, default <data_dir>/logs).
|
|
DEBUG_LOG_DIR = Path(settings.log_dir) / "debug"
|
|
INFO_LOG_DIR = Path(settings.log_dir) / "info"
|
|
DEBUG_LOG_DIR.mkdir(parents=True, exist_ok=True)
|
|
INFO_LOG_DIR.mkdir(parents=True, exist_ok=True)
|
|
|
|
logger.remove()
|
|
logger.add(
|
|
sys.stderr,
|
|
level=settings.log_level,
|
|
format=(
|
|
"<green>{time:HH:mm:ss.SSS}</green> | "
|
|
"<level>{level: <7}</level> | "
|
|
"<cyan>{name}</cyan>:<cyan>{function}</cyan>:<cyan>{line}</cyan> - "
|
|
"<level>{message}</level>"
|
|
),
|
|
enqueue=True,
|
|
colorize=True,
|
|
)
|
|
|
|
logger.add(
|
|
str(DEBUG_LOG_DIR / "{time:YYYY-MM-DD}.log"),
|
|
level="DEBUG", # always full at the file level for audit
|
|
enqueue=True,
|
|
retention="30 days",
|
|
compression="gz",
|
|
colorize=True,
|
|
)
|
|
|
|
logger.add(
|
|
str(INFO_LOG_DIR / "{time:YYYY-MM-DD}.log"),
|
|
level=settings.log_level,
|
|
enqueue=True,
|
|
retention="30 days",
|
|
compression="gz",
|
|
encoding="utf-8",
|
|
) |