Claude
4b07617af0
Remove defaults from queue, executor_memory, executor_cores, num_executors, and app_name in prepare_submit_job. All must now be explicitly confirmed by the caller. Also add extra_args to the PendingSubmission snapshot so users can confirm non-conf spark-submit flags (e.g. --jars, --py-files) at prepare time; confirm_submit_job passes them through to build_spark_submit_command. This is an intentional breaking change to the MCP tool contract: callers can no longer rely on implicit defaults.
74 lines
2.5 KiB
Python
74 lines
2.5 KiB
Python
# coding=utf-8
|
|
import io
|
|
from pathlib import Path
|
|
|
|
import pytest
|
|
|
|
from spark_executor.core import connection_store, pending_store
|
|
from spark_executor.core.connection_store import ConnectionStore
|
|
from spark_executor.core.pending_store import PendingStore
|
|
from spark_executor.models import Connection
|
|
from spark_executor.tools import connections, submit
|
|
|
|
|
|
@pytest.fixture(autouse=True)
|
|
def _fresh_stores(tmp_path: Path, monkeypatch):
|
|
monkeypatch.setattr(connection_store, "DEFAULT_DATA_DIR", str(tmp_path))
|
|
monkeypatch.setattr(connection_store, "store", ConnectionStore())
|
|
monkeypatch.setattr(pending_store, "DEFAULT_DATA_DIR", str(tmp_path))
|
|
monkeypatch.setattr(pending_store, "store", PendingStore())
|
|
connections.store = connection_store.store
|
|
submit.conn_store = connection_store.store
|
|
submit.pending_store = pending_store.store
|
|
submit.job_store = submit.job_store.__class__() # fresh in-memory job store
|
|
|
|
|
|
@pytest.fixture
|
|
def log_capture():
|
|
"""Attach an in-memory sink to loguru so tests can assert on emitted lines."""
|
|
from common.logging import logger
|
|
buf = io.StringIO()
|
|
handler_id = logger.add(buf, level="DEBUG", format="{level}|{message}")
|
|
yield buf
|
|
logger.remove(handler_id)
|
|
|
|
|
|
def test_save_connection_emits_info_log(log_capture):
|
|
connections.save_connection(name="prod", master="yarn")
|
|
text = log_capture.getvalue()
|
|
assert "INFO" in text
|
|
assert "save_connection enter" in text
|
|
assert "DEBUG" in text
|
|
assert "connection saved" in text
|
|
|
|
|
|
def test_prepare_submit_job_emits_debug_and_info(log_capture, tmp_path):
|
|
connections.save_connection(name="prod", master="yarn", deploy_mode="cluster")
|
|
script = tmp_path / "demo.py"
|
|
script.write_text("print('hi')\n")
|
|
log_capture.truncate(0); log_capture.seek(0)
|
|
submit.prepare_submit_job(
|
|
connection="prod",
|
|
script_path=str(script),
|
|
queue="research",
|
|
executor_memory="4G",
|
|
executor_cores=2,
|
|
num_executors=2,
|
|
app_name="test-app",
|
|
)
|
|
text = log_capture.getvalue()
|
|
assert "DEBUG|prepare_submit_job enter" in text
|
|
assert "INFO|prepare_submit_job ok" in text
|
|
assert f"script_path={script}" in text
|
|
assert "queue=research" in text
|
|
|
|
|
|
def test_get_unknown_pending_job_emits_debug(log_capture):
|
|
log_capture.truncate(0); log_capture.seek(0)
|
|
import pytest as _pytest
|
|
with _pytest.raises(KeyError):
|
|
submit.get_pending_job("p_doesnotexist")
|
|
text = log_capture.getvalue()
|
|
assert "DEBUG|get_pending_job enter" in text
|
|
assert "p_doesnotexist" in text
|