tao.chen/mcp-server
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

feat: structured DEBUG/INFO logging via loguru

Browse Source 
- common/logging.py: improve format to timestamp|LEVEL|module:func:line - message
- core/ layer: DEBUG log every subprocess invocation (cmd, rc, byte counts),
  JSON load/dump events, parsed application_id. ERROR log on failures.
- tools/ layer: DEBUG log every public tool entry with key parameters,
  INFO log on business outcomes (saved/submitted/killed/...).
- New tests/unit/test_logging.py: capture loguru output via in-memory sink
  and assert DEBUG + INFO messages are emitted for representative flows.
This commit is contained in:
Claude
2026-06-24 15:02:54 +08:00
parent fad296591c
commit 5c7dcf57cc
12 changed files with 178 additions and 19 deletions
Download Patch File Download Diff File Expand all files Collapse all files
spark_executor/tools/connections.py
+7 -2
View File
@@ -16,6 +16,10 @@ def save_connection(
yarn_rm_url: str | None = None,
spark_conf: dict[str, str] | None = None,
) -> dict[str, str]:
logger.debug(
f"save_connection enter name={name} master={master} deploy_mode={deploy_mode} "
f"yarn_rm_url={yarn_rm_url} spark_conf_keys={list((spark_conf or {}).keys())}"
)
conn = Connection(
name=name,
master=master,
@@ -24,15 +28,16 @@ def save_connection(
spark_conf=spark_conf or {},
)
store.save(conn)
logger.info(f"save_connection name={name} master={master}")
return {"name": name, "status": "SAVED"}
def list_connections() -> list[dict[str, object]]:
logger.debug("list_connections enter")
return [c.model_dump() for c in store.list_all()]
def get_connection(name: str) -> dict[str, object]:
logger.debug(f"get_connection enter name={name}")
conn = store.get(name)
if conn is None:
raise KeyError(f"Unknown connection: {name}")
@@ -40,8 +45,8 @@ def get_connection(name: str) -> dict[str, object]:
def delete_connection(name: str) -> dict[str, str]:
logger.debug(f"delete_connection enter name={name}")
removed = store.delete(name)
if not removed:
raise KeyError(f"Unknown connection: {name}")
logger.info(f"delete_connection name={name}")
return {"name": name, "status": "DELETED"}
spark_executor/tools/kill.py
+2 -1
View File
@@ -11,11 +11,12 @@ store = JobStore()
def kill_job(job_id: str) -> dict[str, str]:
logger.debug(f"kill_job enter job_id={job_id}")
job = store.get(job_id)
if job is None:
raise KeyError(f"Unknown job_id: {job_id}")
kill_application(job.application_id)
logger.info(f"kill_job job_id={job_id} application_id={job.application_id}")
logger.info(f"kill_job ok job_id={job_id} application_id={job.application_id}")
return {
"job_id": job_id,
"application_id": job.application_id,
spark_executor/tools/logs.py
+3 -2
View File
@@ -11,13 +11,14 @@ store = JobStore()
def get_job_logs(job_id: str, tail_chars: int = 5000) -> str:
logger.debug(f"get_job_logs enter job_id={job_id} tail_chars={tail_chars}")
job = store.get(job_id)
if job is None:
raise KeyError(f"Unknown job_id: {job_id}")
full = get_application_logs(job.application_id)
tailed = full[-tail_chars:] if len(full) > tail_chars else full
logger.info(
f"get_job_logs job_id={job_id} application_id={job.application_id} "
f"chars={len(tailed)}"
f"get_job_logs ok job_id={job_id} application_id={job.application_id} "
f"full_chars={len(full)} returned_chars={len(tailed)}"
)
return tailed
spark_executor/tools/status.py
+2 -1
View File
@@ -12,9 +12,10 @@ store = JobStore()
def get_job_status(job_id: str) -> JobStatus:
logger.debug(f"get_job_status enter job_id={job_id}")
job = store.get(job_id)
if job is None:
raise KeyError(f"Unknown job_id: {job_id}")
state, raw = get_application_status(job.application_id)
logger.info(f"get_job_status job_id={job_id} state={state}")
logger.info(f"get_job_status ok job_id={job_id} application_id={job.application_id} state={state}")
return JobStatus(application_id=job.application_id, state=state, raw=raw)
spark_executor/tools/submit.py
+21 -7
View File
@@ -4,6 +4,7 @@
@Author :tao.chen
"""
import secrets
import uuid
from datetime import datetime
from common.logging import logger
@@ -17,8 +18,6 @@ from spark_executor.core.spark_submit import (
run_spark_submit,
)
from spark_executor.models import Job, PendingSubmission, SubmitResult
import uuid
from datetime import datetime as _datetime
def _new_pending_id() -> str:
@@ -35,6 +34,11 @@ def prepare_submit_job(
num_executors: int = 2,
) -> dict[str, object]:
"""Snapshot connection params and persist a PendingSubmission. Does NOT submit."""
logger.debug(
f"prepare_submit_job enter connection={connection} script_path={script_path} "
f"queue={queue} executor_memory={executor_memory} executor_cores={executor_cores} "
f"num_executors={num_executors}"
)
conn = conn_store.get(connection)
if conn is None:
raise KeyError(f"Unknown connection: {connection}")
@@ -56,7 +60,8 @@ def prepare_submit_job(
)
pending_store.save(pending)
logger.info(
f"prepare_submit_job pending_id={pending_id} connection={connection} master={conn.master}"
f"prepare_submit_job ok pending_id={pending_id} connection={connection} "
f"master={conn.master} script_path={script_path}"
)
return {
"pending_id": pending_id,
@@ -71,6 +76,7 @@ job_store: JobStore = JobStore()
def confirm_submit_job(*, pending_id: str) -> SubmitResult:
"""Actually invoke spark-submit for a previously-prepared PendingSubmission."""
logger.debug(f"confirm_submit_job enter pending_id={pending_id}")
pending = pending_store.get(pending_id)
if pending is None:
raise KeyError(f"Unknown pending_id: {pending_id}")
@@ -89,13 +95,17 @@ def confirm_submit_job(*, pending_id: str) -> SubmitResult:
num_executors=pending.num_executors,
spark_conf=pending.spark_conf,
)
logger.info(f"confirm_submit_job pending_id={pending_id} cmd={cmd}")
logger.info(
f"confirm_submit_job start pending_id={pending_id} "
f"application_target={pending.master} script_path={pending.script_path}"
)
try:
result = run_spark_submit(cmd)
except SparkSubmitError as exc:
pending.status = "FAILED"
pending.error = str(exc)
pending_store.save(pending)
logger.error(f"confirm_submit_job failed pending_id={pending_id} err={exc}")
raise
application_id, tracking_url = parse_spark_submit_output(result.stderr)
@@ -107,7 +117,7 @@ def confirm_submit_job(*, pending_id: str) -> SubmitResult:
application_id=application_id,
script_path=pending.script_path,
queue=pending.queue,
submit_time=_datetime.utcnow(),
submit_time=datetime.utcnow(),
connection=pending.connection,
)
)
@@ -117,7 +127,8 @@ def confirm_submit_job(*, pending_id: str) -> SubmitResult:
pending.application_id = application_id
pending_store.save(pending)
logger.info(
f"confirm_submit_job pending_id={pending_id} job_id={job_id} application_id={application_id}"
f"confirm_submit_job ok pending_id={pending_id} job_id={job_id} "
f"application_id={application_id}"
)
return SubmitResult(
job_id=job_id,
@@ -127,10 +138,12 @@ def confirm_submit_job(*, pending_id: str) -> SubmitResult:
def list_pending_jobs() -> list[dict[str, object]]:
logger.debug("list_pending_jobs enter")
return [p.model_dump() for p in pending_store.list_all()]
def get_pending_job(pending_id: str) -> dict[str, object]:
logger.debug(f"get_pending_job enter pending_id={pending_id}")
p = pending_store.get(pending_id)
if p is None:
raise KeyError(f"Unknown pending_id: {pending_id}")
@@ -138,6 +151,7 @@ def get_pending_job(pending_id: str) -> dict[str, object]:
def cancel_pending_job(pending_id: str) -> dict[str, str]:
logger.debug(f"cancel_pending_job enter pending_id={pending_id}")
p = pending_store.get(pending_id)
if p is None:
raise KeyError(f"Unknown pending_id: {pending_id}")
@@ -147,5 +161,5 @@ def cancel_pending_job(pending_id: str) -> dict[str, str]:
)
p.status = "CANCELLED"
pending_store.save(p)
logger.info(f"cancel_pending_job pending_id={pending_id}")
logger.info(f"cancel_pending_job ok pending_id={pending_id}")
return {"pending_id": pending_id, "status": "CANCELLED"}