feat: structured DEBUG/INFO logging via loguru
- common/logging.py: improve format to timestamp|LEVEL|module:func:line - message - core/ layer: DEBUG log every subprocess invocation (cmd, rc, byte counts), JSON load/dump events, parsed application_id. ERROR log on failures. - tools/ layer: DEBUG log every public tool entry with key parameters, INFO log on business outcomes (saved/submitted/killed/...). - New tests/unit/test_logging.py: capture loguru output via in-memory sink and assert DEBUG + INFO messages are emitted for representative flows.
This commit is contained in:
@@ -5,6 +5,8 @@
|
||||
"""
|
||||
import re
|
||||
|
||||
from common.logging import logger
|
||||
|
||||
_APP_ID_RE = re.compile(r"Submitted application (\S+)")
|
||||
_TRACKING_URL_RE = re.compile(r"tracking URL:\s+(\S+)")
|
||||
|
||||
@@ -20,14 +22,20 @@ def parse_spark_submit_output(stderr: str) -> tuple[str, str | None]:
|
||||
else:
|
||||
url_match = _TRACKING_URL_RE.search(stderr)
|
||||
if not url_match:
|
||||
logger.error("Could not find application_id in spark-submit output")
|
||||
raise ValueError("Could not find application_id in spark-submit output")
|
||||
# tracking URL is of the form http://rm:8088/proxy/application_xxx/
|
||||
tracking_url = url_match.group(1)
|
||||
tail = tracking_url.rstrip("/").rsplit("/", 1)[-1]
|
||||
if not tail.startswith("application_"):
|
||||
logger.error("Could not find application_id in spark-submit output (bad tracking URL)")
|
||||
raise ValueError("Could not find application_id in spark-submit output")
|
||||
application_id = tail
|
||||
|
||||
url_match = _TRACKING_URL_RE.search(stderr)
|
||||
tracking_url = url_match.group(1) if url_match else None
|
||||
logger.debug(
|
||||
f"parse_spark_submit_output -> application_id={application_id} "
|
||||
f"tracking_url={tracking_url}"
|
||||
)
|
||||
return application_id, tracking_url
|
||||
|
||||
Reference in New Issue
Block a user