Claude
939f6842d3
Delete compression="gz" from both debug and info file sinks so logs are kept as plain text files. Simplifies log inspection and avoids the small CPU overhead of nightly gzip compression.
60 lines
1.7 KiB
Python
60 lines
1.7 KiB
Python
# coding=utf-8
|
|
"""
|
|
@Time :2026/6/24
|
|
@Author :tao.chen
|
|
|
|
Process-wide loguru configuration. Import `logger` from here in every
|
|
module instead of instantiating new loggers.
|
|
|
|
Log level is controlled via `SPARK_EXECUTOR_LOG_LEVEL` (see common/config.py):
|
|
DEBUG - default; full verbosity
|
|
INFO - quieter; recommended for production
|
|
|
|
Levels used in this project:
|
|
DEBUG - entry/exit of public tools, subprocess commands, file I/O paths
|
|
INFO - business events (job submitted, status changed, connection saved)
|
|
WARNING - recoverable problems (transient YARN issues, retry-able)
|
|
ERROR - raised exceptions (caller will see the traceback)
|
|
"""
|
|
import sys
|
|
from pathlib import Path
|
|
from loguru import logger
|
|
|
|
from common.config import settings
|
|
|
|
# Loguru file sinks — paths derived from settings.log_dir (env-var driven
|
|
# via SPARK_EXECUTOR_LOG_DIR, default <data_dir>/logs).
|
|
DEBUG_LOG_DIR = Path(settings.log_dir) / "debug"
|
|
INFO_LOG_DIR = Path(settings.log_dir) / "info"
|
|
DEBUG_LOG_DIR.mkdir(parents=True, exist_ok=True)
|
|
INFO_LOG_DIR.mkdir(parents=True, exist_ok=True)
|
|
|
|
logger.remove()
|
|
logger.add(
|
|
sys.stderr,
|
|
level=settings.log_level,
|
|
format=(
|
|
"<green>{time:HH:mm:ss.SSS}</green> | "
|
|
"<level>{level: <7}</level> | "
|
|
"<cyan>{name}</cyan>:<cyan>{function}</cyan>:<cyan>{line}</cyan> - "
|
|
"<level>{message}</level>"
|
|
),
|
|
enqueue=True,
|
|
colorize=True,
|
|
)
|
|
|
|
logger.add(
|
|
str(DEBUG_LOG_DIR / "{time:YYYY-MM-DD}.log"),
|
|
level="DEBUG", # always full at the file level for audit
|
|
enqueue=True,
|
|
retention="30 days",
|
|
colorize=True,
|
|
)
|
|
|
|
logger.add(
|
|
str(INFO_LOG_DIR / "{time:YYYY-MM-DD}.log"),
|
|
level=settings.log_level,
|
|
enqueue=True,
|
|
retention="30 days",
|
|
encoding="utf-8",
|
|
) |