feat: structured DEBUG/INFO logging via loguru
- common/logging.py: improve format to timestamp|LEVEL|module:func:line - message - core/ layer: DEBUG log every subprocess invocation (cmd, rc, byte counts), JSON load/dump events, parsed application_id. ERROR log on failures. - tools/ layer: DEBUG log every public tool entry with key parameters, INFO log on business outcomes (saved/submitted/killed/...). - New tests/unit/test_logging.py: capture loguru output via in-memory sink and assert DEBUG + INFO messages are emitted for representative flows.
This commit is contained in:
@@ -0,0 +1,63 @@
|
||||
# coding=utf-8
|
||||
import io
|
||||
from pathlib import Path
|
||||
|
||||
import pytest
|
||||
|
||||
from spark_executor.core import connection_store, pending_store
|
||||
from spark_executor.core.connection_store import ConnectionStore
|
||||
from spark_executor.core.pending_store import PendingStore
|
||||
from spark_executor.models import Connection
|
||||
from spark_executor.tools import connections, submit
|
||||
|
||||
|
||||
@pytest.fixture(autouse=True)
|
||||
def _fresh_stores(tmp_path: Path, monkeypatch):
|
||||
monkeypatch.setattr(connection_store, "DEFAULT_DATA_DIR", str(tmp_path))
|
||||
monkeypatch.setattr(connection_store, "store", ConnectionStore())
|
||||
monkeypatch.setattr(pending_store, "DEFAULT_DATA_DIR", str(tmp_path))
|
||||
monkeypatch.setattr(pending_store, "store", PendingStore())
|
||||
connections.store = connection_store.store
|
||||
submit.conn_store = connection_store.store
|
||||
submit.pending_store = pending_store.store
|
||||
submit.job_store = submit.job_store.__class__() # fresh in-memory job store
|
||||
|
||||
|
||||
@pytest.fixture
|
||||
def log_capture():
|
||||
"""Attach an in-memory sink to loguru so tests can assert on emitted lines."""
|
||||
from common.logging import logger
|
||||
buf = io.StringIO()
|
||||
handler_id = logger.add(buf, level="DEBUG", format="{level}|{message}")
|
||||
yield buf
|
||||
logger.remove(handler_id)
|
||||
|
||||
|
||||
def test_save_connection_emits_info_log(log_capture):
|
||||
connections.save_connection(name="prod", master="yarn")
|
||||
text = log_capture.getvalue()
|
||||
assert "INFO" in text
|
||||
assert "save_connection enter" in text
|
||||
assert "DEBUG" in text
|
||||
assert "connection saved" in text
|
||||
|
||||
|
||||
def test_prepare_submit_job_emits_debug_and_info(log_capture):
|
||||
connections.save_connection(name="prod", master="yarn", deploy_mode="cluster")
|
||||
log_capture.truncate(0); log_capture.seek(0)
|
||||
submit.prepare_submit_job(connection="prod", script_path="/tmp/j.py", queue="research")
|
||||
text = log_capture.getvalue()
|
||||
assert "DEBUG|prepare_submit_job enter" in text
|
||||
assert "INFO|prepare_submit_job ok" in text
|
||||
assert "script_path=/tmp/j.py" in text
|
||||
assert "queue=research" in text
|
||||
|
||||
|
||||
def test_get_unknown_pending_job_emits_debug(log_capture):
|
||||
log_capture.truncate(0); log_capture.seek(0)
|
||||
import pytest as _pytest
|
||||
with _pytest.raises(KeyError):
|
||||
submit.get_pending_job("p_doesnotexist")
|
||||
text = log_capture.getvalue()
|
||||
assert "DEBUG|get_pending_job enter" in text
|
||||
assert "p_doesnotexist" in text
|
||||
Reference in New Issue
Block a user