Claude
f8e367d43c
Before: SPARK_EXECUTOR_DATA_DIR, SPARK_EXECUTOR_JOBS_DIR, and
YARN_RESOURCE_MANAGER_URL were each read directly via os.environ.get()
inside the module that used them. Log level was hardcoded DEBUG in
common/logging.py. There was no single file showing what the full set
of env vars the app reads is.
After: common/config.py defines a single Settings dataclass that
reads all env vars at import time and exposes them as fields on a
module-level singleton. App code uses "from common.config import
settings; settings.data_dir" etc. New SPARK_EXECUTOR_LOG_LEVEL env
var controls stderr + info file verbosity (debug file always gets full
DEBUG).
Improvements:
- One file lists every env var the app reads (was: grep the codebase)
- Tests can monkeypatch fields on the settings singleton directly
instead of monkeypatching the env + reloading
- Adding a new env var means adding one field in config.py, not
editing 3+ call sites
- settings.reload() method for tests that prefer env-var style
Out of scope (kept where they are):
- GUNICORN_* env vars live in gunicorn.conf.py (gunicorn concept)
- PYTHONUNBUFFERED in Dockerfile (Python runtime flag)
- SPARK_SUBMIT_OPTS not in config (JVM flag, not Python)
Test changes:
- test_job_writer.py: settings.jobs_dir instead of monkeypatching
SPARK_EXECUTOR_JOBS_DIR
- test_yarn_client.py: settings.yarn_resource_manager_url instead of
monkeypatching YARN_RESOURCE_MANAGER_URL
- test_generate_tool.py: same as job_writer
- Each test file gets an autouse fixture that snapshots+restores
settings so one test mutation does not leak into the next
116/116 still pass. Live verified: SPARK_EXECUTOR_LOG_LEVEL=INFO
suppresses DEBUG loguru output as expected.
99 lines
3.2 KiB
Python
99 lines
3.2 KiB
Python
# coding=utf-8
|
|
import os
|
|
from pathlib import Path
|
|
|
|
import pytest
|
|
|
|
from common import config
|
|
from spark_executor.core.job_writer import (
|
|
ENV_JOBS_DIR,
|
|
resolve_jobs_dir,
|
|
write_job_file,
|
|
)
|
|
|
|
|
|
@pytest.fixture(autouse=True)
|
|
def _restore_settings():
|
|
"""job_writer uses settings.jobs_dir which is captured at module import.
|
|
Tests that set env vars then call settings.reload() — this fixture saves
|
|
and restores the original settings so one test doesn't leak into another."""
|
|
snapshot = config.Settings(
|
|
data_dir=config.settings.data_dir,
|
|
jobs_dir=config.settings.jobs_dir,
|
|
yarn_resource_manager_url=config.settings.yarn_resource_manager_url,
|
|
log_level=config.settings.log_level,
|
|
)
|
|
yield
|
|
config.settings.data_dir = snapshot.data_dir
|
|
config.settings.jobs_dir = snapshot.jobs_dir
|
|
config.settings.yarn_resource_manager_url = snapshot.yarn_resource_manager_url
|
|
config.settings.log_level = snapshot.log_level
|
|
|
|
|
|
# --- resolve_jobs_dir (priority: arg > settings.jobs_dir) ---
|
|
|
|
def test_resolve_explicit_arg_wins(tmp_path: Path, monkeypatch):
|
|
config.settings.jobs_dir = "/from/settings"
|
|
assert resolve_jobs_dir(str(tmp_path)) == str(tmp_path)
|
|
|
|
|
|
def test_resolve_uses_settings_jobs_dir_when_no_arg(monkeypatch, tmp_path: Path):
|
|
config.settings.jobs_dir = str(tmp_path)
|
|
assert resolve_jobs_dir() == str(tmp_path)
|
|
|
|
|
|
def test_resolve_default_when_settings_unset(monkeypatch):
|
|
config.settings.jobs_dir = "./data/jobs"
|
|
assert resolve_jobs_dir() == "./data/jobs"
|
|
|
|
|
|
# --- write_job_file ---
|
|
|
|
def test_write_creates_file_with_code_and_returns_abs_path(tmp_path: Path):
|
|
out = write_job_file("print('hi')\n", jobs_dir=str(tmp_path))
|
|
assert os.path.isfile(out)
|
|
assert os.path.isabs(out)
|
|
assert out.startswith(str(tmp_path))
|
|
assert out.endswith(".py")
|
|
with open(out) as f:
|
|
assert f.read() == "print('hi')\n"
|
|
|
|
|
|
def test_write_creates_jobs_dir_if_missing(tmp_path: Path):
|
|
target = tmp_path / "newdir"
|
|
assert not target.exists()
|
|
out = write_job_file("x = 1\n", jobs_dir=str(target))
|
|
assert target.is_dir()
|
|
assert os.path.isfile(out)
|
|
|
|
|
|
def test_write_uses_settings_jobs_dir_when_no_arg(tmp_path: Path):
|
|
config.settings.jobs_dir = str(tmp_path)
|
|
out = write_job_file("settings-driven\n")
|
|
assert out.startswith(str(tmp_path))
|
|
with open(out) as f:
|
|
assert f.read() == "settings-driven\n"
|
|
|
|
|
|
def test_write_uses_default_jobs_dir(tmp_path: Path):
|
|
"""When settings.jobs_dir is the default './data/jobs', writes resolve
|
|
relative to cwd. chdir to tmp so the test doesn't pollute the project."""
|
|
config.settings.jobs_dir = "./data/jobs"
|
|
old_cwd = os.getcwd()
|
|
try:
|
|
os.chdir(tmp_path)
|
|
out = write_job_file("default\n")
|
|
assert os.path.isabs(out)
|
|
expected_dir = tmp_path / "data" / "jobs"
|
|
assert expected_dir.is_dir()
|
|
assert str(out).startswith(str(expected_dir))
|
|
finally:
|
|
os.chdir(old_cwd)
|
|
|
|
|
|
def test_write_returns_unique_paths_for_concurrent_calls(tmp_path: Path):
|
|
"""Two writes in the same second must still get distinct filenames (via random suffix)."""
|
|
out1 = write_job_file("a", jobs_dir=str(tmp_path))
|
|
out2 = write_job_file("b", jobs_dir=str(tmp_path))
|
|
assert out1 != out2
|