tao.chen/mcp-server
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

feat: structured DEBUG/INFO logging via loguru

Browse Source 
- common/logging.py: improve format to timestamp|LEVEL|module:func:line - message
- core/ layer: DEBUG log every subprocess invocation (cmd, rc, byte counts),
  JSON load/dump events, parsed application_id. ERROR log on failures.
- tools/ layer: DEBUG log every public tool entry with key parameters,
  INFO log on business outcomes (saved/submitted/killed/...).
- New tests/unit/test_logging.py: capture loguru output via in-memory sink
  and assert DEBUG + INFO messages are emitted for representative flows.
This commit is contained in:
Claude
2026-06-24 15:02:54 +08:00
parent fad296591c
commit 5c7dcf57cc
12 changed files with 178 additions and 19 deletions
Download Patch File Download Diff File Expand all files Collapse all files
spark_executor/core/connection_store.py
+7 -1
View File
@@ -9,6 +9,7 @@ import tempfile
from pathlib import Path
from threading import Lock
from common.logging import logger
from spark_executor.models import Connection
DEFAULT_DATA_DIR = os.environ.get("SPARK_EXECUTOR_DATA_DIR", "./data")
@@ -37,8 +38,10 @@ class ConnectionStore:
def _load(self) -> dict[str, Connection]:
if not self.path.exists():
logger.debug(f"ConnectionStore._load: file {self.path} absent, returning empty")
return {}
raw = json.loads(self.path.read_text(encoding="utf-8"))
logger.debug(f"ConnectionStore._load: loaded {len(raw)} records from {self.path}")
return {name: Connection.model_validate(c) for name, c in raw.items()}
def _dump(self, records: dict[str, Connection]) -> None:
@@ -52,6 +55,7 @@ class ConnectionStore:
with os.fdopen(fd, "w", encoding="utf-8") as f:
json.dump(payload, f, indent=2, ensure_ascii=False)
os.replace(tmp_path, self.path)
logger.debug(f"ConnectionStore._dump: wrote {len(records)} records to {self.path}")
except Exception:
if os.path.exists(tmp_path):
os.unlink(tmp_path)
@@ -70,6 +74,7 @@ class ConnectionStore:
records = self._load()
records[conn.name] = conn
self._dump(records)
logger.info(f"connection saved name={conn.name} master={conn.master}")
def delete(self, name: str) -> bool:
with self._lock:
@@ -78,7 +83,8 @@ class ConnectionStore:
return False
del records[name]
self._dump(records)
return True
logger.info(f"connection deleted name={name}")
return True
# Module-level singleton; replaced in tests.
spark_executor/core/log_parser.py
+8
View File
@@ -5,6 +5,8 @@
"""
import re
from common.logging import logger
_APP_ID_RE = re.compile(r"Submitted application (\S+)")
_TRACKING_URL_RE = re.compile(r"tracking URL:\s+(\S+)")
@@ -20,14 +22,20 @@ def parse_spark_submit_output(stderr: str) -> tuple[str, str | None]:
else:
url_match = _TRACKING_URL_RE.search(stderr)
if not url_match:
logger.error("Could not find application_id in spark-submit output")
raise ValueError("Could not find application_id in spark-submit output")
# tracking URL is of the form http://rm:8088/proxy/application_xxx/
tracking_url = url_match.group(1)
tail = tracking_url.rstrip("/").rsplit("/", 1)[-1]
if not tail.startswith("application_"):
logger.error("Could not find application_id in spark-submit output (bad tracking URL)")
raise ValueError("Could not find application_id in spark-submit output")
application_id = tail
url_match = _TRACKING_URL_RE.search(stderr)
tracking_url = url_match.group(1) if url_match else None
logger.debug(
f"parse_spark_submit_output -> application_id={application_id} "
f"tracking_url={tracking_url}"
)
return application_id, tracking_url
spark_executor/core/pending_store.py
+10 -1
View File
@@ -9,6 +9,7 @@ import tempfile
from pathlib import Path
from threading import Lock
from common.logging import logger
from spark_executor.models import PendingSubmission
DEFAULT_DATA_DIR = os.environ.get("SPARK_EXECUTOR_DATA_DIR", "./data")
@@ -33,8 +34,10 @@ class PendingStore:
def _load(self) -> dict[str, PendingSubmission]:
if not self.path.exists():
logger.debug(f"PendingStore._load: file {self.path} absent, returning empty")
return {}
raw = json.loads(self.path.read_text(encoding="utf-8"))
logger.debug(f"PendingStore._load: loaded {len(raw)} records from {self.path}")
return {pid: PendingSubmission.model_validate(p) for pid, p in raw.items()}
def _dump(self, records: dict[str, PendingSubmission]) -> None:
@@ -47,6 +50,7 @@ class PendingStore:
with os.fdopen(fd, "w", encoding="utf-8") as f:
json.dump(payload, f, indent=2, ensure_ascii=False, default=str)
os.replace(tmp_path, self.path)
logger.debug(f"PendingStore._dump: wrote {len(records)} records to {self.path}")
except Exception:
if os.path.exists(tmp_path):
os.unlink(tmp_path)
@@ -65,6 +69,10 @@ class PendingStore:
records = self._load()
records[pending.pending_id] = pending
self._dump(records)
logger.info(
f"pending saved pending_id={pending.pending_id} "
f"status={pending.status} connection={pending.connection}"
)
def delete(self, pending_id: str) -> bool:
with self._lock:
@@ -73,7 +81,8 @@ class PendingStore:
return False
del records[pending_id]
self._dump(records)
return True
logger.info(f"pending deleted pending_id={pending_id}")
return True
# Module-level singleton; replaced in tests.
spark_executor/core/spark_submit.py
+9
View File
@@ -5,6 +5,8 @@
"""
import subprocess
from common.logging import logger
class SparkSubmitError(Exception):
"""Raised when spark-submit exits with a non-zero return code."""
@@ -33,12 +35,19 @@ def build_spark_submit_command(
for key, value in (spark_conf or {}).items():
cmd.extend(["--conf", f"{key}={value}"])
cmd.append(script_path)
logger.debug(f"build_spark_submit_command -> {cmd}")
return cmd
def run_spark_submit(cmd: list[str]) -> "subprocess.CompletedProcess[str]":
logger.debug(f"run_spark_submit exec: {cmd}")
result = subprocess.run(cmd, capture_output=True, text=True, errors="replace")
logger.debug(
f"run_spark_submit done rc={result.returncode} "
f"stdout_len={len(result.stdout)} stderr_len={len(result.stderr)}"
)
if result.returncode != 0:
logger.error(f"spark-submit failed (rc={result.returncode}): {result.stderr[:500]}")
raise SparkSubmitError(
f"spark-submit failed (rc={result.returncode}): {result.stderr}"
)
spark_executor/core/yarn_client.py
+25 -2
View File
@@ -6,6 +6,8 @@
import re
import subprocess
from common.logging import logger
class YarnError(Exception):
"""Raised when a yarn CLI invocation fails."""
@@ -15,33 +17,54 @@ _STATE_RE = re.compile(r"State\s*:\s*(\S+)")
def _run(cmd: list[str]) -> "subprocess.CompletedProcess[str]":
return subprocess.run(cmd, capture_output=True, text=True, errors="replace")
logger.debug(f"yarn _run exec: {cmd}")
result = subprocess.run(cmd, capture_output=True, text=True, errors="replace")
logger.debug(
f"yarn _run done rc={result.returncode} "
f"stdout_len={len(result.stdout)} stderr_len={len(result.stderr)}"
)
return result
def get_application_status(application_id: str) -> tuple[str, str]:
proc = _run(["yarn", "application", "-status", application_id])
if proc.returncode != 0:
logger.error(
f"yarn application -status failed (rc={proc.returncode}) for "
f"{application_id}: {proc.stderr[:500]}"
)
raise YarnError(
f"yarn application -status failed (rc={proc.returncode}): {proc.stderr}"
)
match = _STATE_RE.search(proc.stdout)
if not match:
raise YarnError(f"Could not parse YARN state from output: {proc.stdout!r}")
return match.group(1), proc.stdout
state = match.group(1)
logger.info(f"yarn status {application_id} -> {state}")
return state, proc.stdout
def get_application_logs(application_id: str) -> str:
proc = _run(["yarn", "logs", "-applicationId", application_id])
if proc.returncode != 0:
logger.error(
f"yarn logs failed (rc={proc.returncode}) for {application_id}: {proc.stderr[:500]}"
)
raise YarnError(
f"yarn logs failed (rc={proc.returncode}): {proc.stderr}"
)
logger.info(f"yarn logs {application_id} -> {len(proc.stdout)} chars")
return proc.stdout
def kill_application(application_id: str) -> None:
proc = _run(["yarn", "application", "-kill", application_id])
if proc.returncode != 0:
logger.error(
f"yarn application -kill failed (rc={proc.returncode}) for "
f"{application_id}: {proc.stderr[:500]}"
)
raise YarnError(
f"yarn application -kill failed (rc={proc.returncode}): {proc.stderr}"
)
logger.info(f"yarn kill {application_id} -> ok")