Claude
5c7dcf57cc
- common/logging.py: improve format to timestamp|LEVEL|module:func:line - message - core/ layer: DEBUG log every subprocess invocation (cmd, rc, byte counts), JSON load/dump events, parsed application_id. ERROR log on failures. - tools/ layer: DEBUG log every public tool entry with key parameters, INFO log on business outcomes (saved/submitted/killed/...). - New tests/unit/test_logging.py: capture loguru output via in-memory sink and assert DEBUG + INFO messages are emitted for representative flows.
64 lines
2.3 KiB
Python
64 lines
2.3 KiB
Python
# coding=utf-8
|
|
import io
|
|
from pathlib import Path
|
|
|
|
import pytest
|
|
|
|
from spark_executor.core import connection_store, pending_store
|
|
from spark_executor.core.connection_store import ConnectionStore
|
|
from spark_executor.core.pending_store import PendingStore
|
|
from spark_executor.models import Connection
|
|
from spark_executor.tools import connections, submit
|
|
|
|
|
|
@pytest.fixture(autouse=True)
|
|
def _fresh_stores(tmp_path: Path, monkeypatch):
|
|
monkeypatch.setattr(connection_store, "DEFAULT_DATA_DIR", str(tmp_path))
|
|
monkeypatch.setattr(connection_store, "store", ConnectionStore())
|
|
monkeypatch.setattr(pending_store, "DEFAULT_DATA_DIR", str(tmp_path))
|
|
monkeypatch.setattr(pending_store, "store", PendingStore())
|
|
connections.store = connection_store.store
|
|
submit.conn_store = connection_store.store
|
|
submit.pending_store = pending_store.store
|
|
submit.job_store = submit.job_store.__class__() # fresh in-memory job store
|
|
|
|
|
|
@pytest.fixture
|
|
def log_capture():
|
|
"""Attach an in-memory sink to loguru so tests can assert on emitted lines."""
|
|
from common.logging import logger
|
|
buf = io.StringIO()
|
|
handler_id = logger.add(buf, level="DEBUG", format="{level}|{message}")
|
|
yield buf
|
|
logger.remove(handler_id)
|
|
|
|
|
|
def test_save_connection_emits_info_log(log_capture):
|
|
connections.save_connection(name="prod", master="yarn")
|
|
text = log_capture.getvalue()
|
|
assert "INFO" in text
|
|
assert "save_connection enter" in text
|
|
assert "DEBUG" in text
|
|
assert "connection saved" in text
|
|
|
|
|
|
def test_prepare_submit_job_emits_debug_and_info(log_capture):
|
|
connections.save_connection(name="prod", master="yarn", deploy_mode="cluster")
|
|
log_capture.truncate(0); log_capture.seek(0)
|
|
submit.prepare_submit_job(connection="prod", script_path="/tmp/j.py", queue="research")
|
|
text = log_capture.getvalue()
|
|
assert "DEBUG|prepare_submit_job enter" in text
|
|
assert "INFO|prepare_submit_job ok" in text
|
|
assert "script_path=/tmp/j.py" in text
|
|
assert "queue=research" in text
|
|
|
|
|
|
def test_get_unknown_pending_job_emits_debug(log_capture):
|
|
log_capture.truncate(0); log_capture.seek(0)
|
|
import pytest as _pytest
|
|
with _pytest.raises(KeyError):
|
|
submit.get_pending_job("p_doesnotexist")
|
|
text = log_capture.getvalue()
|
|
assert "DEBUG|get_pending_job enter" in text
|
|
assert "p_doesnotexist" in text
|