Claude
9340061b30
spark_executor/core/job_writer.py provides write_job_file(code, jobs_dir=None) that resolves the output directory in this order: 1. explicit jobs_dir argument (for tests + programmatic override) 2. SPARK_EXECUTOR_JOBS_DIR environment variable (operator override) 3. DEFAULT_JOBS_DIR = './data/jobs' (gitignored, persists via volume mount) Filenames are job_<UTC-stamp>_<random-hex>.py — same-second writes still get distinct names via the random suffix, so concurrent agents won't clobber each other. 8 tests cover: arg>env>default precedence, dir auto-creation, env-var override, default fallback in a tmp cwd, and uniqueness under concurrent calls.
79 lines
2.6 KiB
Python
79 lines
2.6 KiB
Python
# coding=utf-8
|
|
import os
|
|
from pathlib import Path
|
|
|
|
from spark_executor.core.job_writer import (
|
|
DEFAULT_JOBS_DIR,
|
|
ENV_JOBS_DIR,
|
|
resolve_jobs_dir,
|
|
write_job_file,
|
|
)
|
|
|
|
|
|
# --- resolve_jobs_dir (priority: arg > env > default) ---
|
|
|
|
def test_resolve_explicit_arg_wins(tmp_path: Path, monkeypatch):
|
|
monkeypatch.setenv(ENV_JOBS_DIR, "/from/env")
|
|
assert resolve_jobs_dir(str(tmp_path)) == str(tmp_path)
|
|
|
|
|
|
def test_resolve_env_var_used_when_no_arg(monkeypatch, tmp_path: Path):
|
|
monkeypatch.setenv(ENV_JOBS_DIR, str(tmp_path))
|
|
assert resolve_jobs_dir() == str(tmp_path)
|
|
|
|
|
|
def test_resolve_default_when_neither_set(monkeypatch):
|
|
monkeypatch.delenv(ENV_JOBS_DIR, raising=False)
|
|
assert resolve_jobs_dir() == DEFAULT_JOBS_DIR
|
|
assert resolve_jobs_dir() == "./data/jobs"
|
|
|
|
|
|
# --- write_job_file ---
|
|
|
|
def test_write_creates_file_with_code_and_returns_abs_path(tmp_path: Path):
|
|
out = write_job_file("print('hi')\n", jobs_dir=str(tmp_path))
|
|
assert os.path.isfile(out)
|
|
assert os.path.isabs(out)
|
|
assert out.startswith(str(tmp_path))
|
|
assert out.endswith(".py")
|
|
with open(out) as f:
|
|
assert f.read() == "print('hi')\n"
|
|
|
|
|
|
def test_write_creates_jobs_dir_if_missing(tmp_path: Path):
|
|
target = tmp_path / "newdir"
|
|
assert not target.exists()
|
|
out = write_job_file("x = 1\n", jobs_dir=str(target))
|
|
assert target.is_dir()
|
|
assert os.path.isfile(out)
|
|
|
|
|
|
def test_write_uses_env_var_when_no_arg(monkeypatch, tmp_path: Path):
|
|
monkeypatch.setenv(ENV_JOBS_DIR, str(tmp_path))
|
|
out = write_job_file("env-driven\n")
|
|
assert out.startswith(str(tmp_path))
|
|
with open(out) as f:
|
|
assert f.read() == "env-driven\n"
|
|
|
|
|
|
def test_write_uses_default_when_neither_set(monkeypatch, tmp_path: Path):
|
|
"""Default is ./data/jobs relative to cwd. Run in a tmp dir so the test
|
|
doesn't pollute the real project and stays self-contained."""
|
|
monkeypatch.delenv(ENV_JOBS_DIR, raising=False)
|
|
monkeypatch.chdir(tmp_path) # pytest built-in: chdir for this test only
|
|
out = write_job_file("default\n")
|
|
assert os.path.isabs(out)
|
|
# Should resolve to <tmp_path>/data/jobs/job_*.py
|
|
expected_dir = tmp_path / "data" / "jobs"
|
|
assert expected_dir.is_dir()
|
|
assert str(out).startswith(str(expected_dir))
|
|
with open(out) as f:
|
|
assert f.read() == "default\n"
|
|
|
|
|
|
def test_write_returns_unique_paths_for_concurrent_calls(tmp_path: Path):
|
|
"""Two writes in the same second must still get distinct filenames (via random suffix)."""
|
|
out1 = write_job_file("a", jobs_dir=str(tmp_path))
|
|
out2 = write_job_file("b", jobs_dir=str(tmp_path))
|
|
assert out1 != out2
|