feat: structured DEBUG/INFO logging via loguru

- common/logging.py: improve format to timestamp|LEVEL|module:func:line - message
- core/ layer: DEBUG log every subprocess invocation (cmd, rc, byte counts),
  JSON load/dump events, parsed application_id. ERROR log on failures.
- tools/ layer: DEBUG log every public tool entry with key parameters,
  INFO log on business outcomes (saved/submitted/killed/...).
- New tests/unit/test_logging.py: capture loguru output via in-memory sink
  and assert DEBUG + INFO messages are emitted for representative flows.
This commit is contained in:
Claude
2026-06-24 15:02:54 +08:00
parent fad296591c
commit 5c7dcf57cc
12 changed files with 178 additions and 19 deletions
+63
View File
@@ -0,0 +1,63 @@
# coding=utf-8
import io
from pathlib import Path
import pytest
from spark_executor.core import connection_store, pending_store
from spark_executor.core.connection_store import ConnectionStore
from spark_executor.core.pending_store import PendingStore
from spark_executor.models import Connection
from spark_executor.tools import connections, submit
@pytest.fixture(autouse=True)
def _fresh_stores(tmp_path: Path, monkeypatch):
monkeypatch.setattr(connection_store, "DEFAULT_DATA_DIR", str(tmp_path))
monkeypatch.setattr(connection_store, "store", ConnectionStore())
monkeypatch.setattr(pending_store, "DEFAULT_DATA_DIR", str(tmp_path))
monkeypatch.setattr(pending_store, "store", PendingStore())
connections.store = connection_store.store
submit.conn_store = connection_store.store
submit.pending_store = pending_store.store
submit.job_store = submit.job_store.__class__() # fresh in-memory job store
@pytest.fixture
def log_capture():
"""Attach an in-memory sink to loguru so tests can assert on emitted lines."""
from common.logging import logger
buf = io.StringIO()
handler_id = logger.add(buf, level="DEBUG", format="{level}|{message}")
yield buf
logger.remove(handler_id)
def test_save_connection_emits_info_log(log_capture):
connections.save_connection(name="prod", master="yarn")
text = log_capture.getvalue()
assert "INFO" in text
assert "save_connection enter" in text
assert "DEBUG" in text
assert "connection saved" in text
def test_prepare_submit_job_emits_debug_and_info(log_capture):
connections.save_connection(name="prod", master="yarn", deploy_mode="cluster")
log_capture.truncate(0); log_capture.seek(0)
submit.prepare_submit_job(connection="prod", script_path="/tmp/j.py", queue="research")
text = log_capture.getvalue()
assert "DEBUG|prepare_submit_job enter" in text
assert "INFO|prepare_submit_job ok" in text
assert "script_path=/tmp/j.py" in text
assert "queue=research" in text
def test_get_unknown_pending_job_emits_debug(log_capture):
log_capture.truncate(0); log_capture.seek(0)
import pytest as _pytest
with _pytest.raises(KeyError):
submit.get_pending_job("p_doesnotexist")
text = log_capture.getvalue()
assert "DEBUG|get_pending_job enter" in text
assert "p_doesnotexist" in text