refactor: unify env-var config in common/config.py

Before: SPARK_EXECUTOR_DATA_DIR, SPARK_EXECUTOR_JOBS_DIR, and
YARN_RESOURCE_MANAGER_URL were each read directly via os.environ.get()
inside the module that used them. Log level was hardcoded DEBUG in
common/logging.py. There was no single file showing what the full set
of env vars the app reads is.

After: common/config.py defines a single Settings dataclass that
reads all env vars at import time and exposes them as fields on a
module-level singleton. App code uses "from common.config import
settings; settings.data_dir" etc. New SPARK_EXECUTOR_LOG_LEVEL env
var controls stderr + info file verbosity (debug file always gets full
DEBUG).

Improvements:
  - One file lists every env var the app reads (was: grep the codebase)
  - Tests can monkeypatch fields on the settings singleton directly
    instead of monkeypatching the env + reloading
  - Adding a new env var means adding one field in config.py, not
    editing 3+ call sites
  - settings.reload() method for tests that prefer env-var style

Out of scope (kept where they are):
  - GUNICORN_* env vars live in gunicorn.conf.py (gunicorn concept)
  - PYTHONUNBUFFERED in Dockerfile (Python runtime flag)
  - SPARK_SUBMIT_OPTS not in config (JVM flag, not Python)

Test changes:
  - test_job_writer.py: settings.jobs_dir instead of monkeypatching
    SPARK_EXECUTOR_JOBS_DIR
  - test_yarn_client.py: settings.yarn_resource_manager_url instead of
    monkeypatching YARN_RESOURCE_MANAGER_URL
  - test_generate_tool.py: same as job_writer
  - Each test file gets an autouse fixture that snapshots+restores
    settings so one test mutation does not leak into the next

116/116 still pass. Live verified: SPARK_EXECUTOR_LOG_LEVEL=INFO
suppresses DEBUG loguru output as expected.

tests/unit/test_job_writer.py

@@ -2,29 +2,48 @@
 import os
 from pathlib import Path
 
+import pytest
+
+from common import config
 from spark_executor.core.job_writer import (
-    DEFAULT_JOBS_DIR,
     ENV_JOBS_DIR,
     resolve_jobs_dir,
     write_job_file,
 )
 
 
-# --- resolve_jobs_dir (priority: arg > env > default) ---
+@pytest.fixture(autouse=True)
+def _restore_settings():
+    """job_writer uses settings.jobs_dir which is captured at module import.
+    Tests that set env vars then call settings.reload() — this fixture saves
+    and restores the original settings so one test doesn't leak into another."""
+    snapshot = config.Settings(
+        data_dir=config.settings.data_dir,
+        jobs_dir=config.settings.jobs_dir,
+        yarn_resource_manager_url=config.settings.yarn_resource_manager_url,
+        log_level=config.settings.log_level,
+    )
+    yield
+    config.settings.data_dir = snapshot.data_dir
+    config.settings.jobs_dir = snapshot.jobs_dir
+    config.settings.yarn_resource_manager_url = snapshot.yarn_resource_manager_url
+    config.settings.log_level = snapshot.log_level
+
+
+# --- resolve_jobs_dir (priority: arg > settings.jobs_dir) ---
 
 def test_resolve_explicit_arg_wins(tmp_path: Path, monkeypatch):
-    monkeypatch.setenv(ENV_JOBS_DIR, "/from/env")
+    config.settings.jobs_dir = "/from/settings"
     assert resolve_jobs_dir(str(tmp_path)) == str(tmp_path)
 
 
-def test_resolve_env_var_used_when_no_arg(monkeypatch, tmp_path: Path):
-    monkeypatch.setenv(ENV_JOBS_DIR, str(tmp_path))
+def test_resolve_uses_settings_jobs_dir_when_no_arg(monkeypatch, tmp_path: Path):
+    config.settings.jobs_dir = str(tmp_path)
     assert resolve_jobs_dir() == str(tmp_path)
 
 
-def test_resolve_default_when_neither_set(monkeypatch):
-    monkeypatch.delenv(ENV_JOBS_DIR, raising=False)
-    assert resolve_jobs_dir() == DEFAULT_JOBS_DIR
+def test_resolve_default_when_settings_unset(monkeypatch):
+    config.settings.jobs_dir = "./data/jobs"
     assert resolve_jobs_dir() == "./data/jobs"
 
 
@@ -48,27 +67,28 @@ def test_write_creates_jobs_dir_if_missing(tmp_path: Path):
     assert os.path.isfile(out)
 
 
-def test_write_uses_env_var_when_no_arg(monkeypatch, tmp_path: Path):
-    monkeypatch.setenv(ENV_JOBS_DIR, str(tmp_path))
-    out = write_job_file("env-driven\n")
+def test_write_uses_settings_jobs_dir_when_no_arg(tmp_path: Path):
+    config.settings.jobs_dir = str(tmp_path)
+    out = write_job_file("settings-driven\n")
     assert out.startswith(str(tmp_path))
     with open(out) as f:
-        assert f.read() == "env-driven\n"
+        assert f.read() == "settings-driven\n"
 
 
-def test_write_uses_default_when_neither_set(monkeypatch, tmp_path: Path):
-    """Default is ./data/jobs relative to cwd. Run in a tmp dir so the test
-    doesn't pollute the real project and stays self-contained."""
-    monkeypatch.delenv(ENV_JOBS_DIR, raising=False)
-    monkeypatch.chdir(tmp_path)  # pytest built-in: chdir for this test only
-    out = write_job_file("default\n")
-    assert os.path.isabs(out)
-    # Should resolve to <tmp_path>/data/jobs/job_*.py
-    expected_dir = tmp_path / "data" / "jobs"
-    assert expected_dir.is_dir()
-    assert str(out).startswith(str(expected_dir))
-    with open(out) as f:
-        assert f.read() == "default\n"
+def test_write_uses_default_jobs_dir(tmp_path: Path):
+    """When settings.jobs_dir is the default './data/jobs', writes resolve
+    relative to cwd. chdir to tmp so the test doesn't pollute the project."""
+    config.settings.jobs_dir = "./data/jobs"
+    old_cwd = os.getcwd()
+    try:
+        os.chdir(tmp_path)
+        out = write_job_file("default\n")
+        assert os.path.isabs(out)
+        expected_dir = tmp_path / "data" / "jobs"
+        assert expected_dir.is_dir()
+        assert str(out).startswith(str(expected_dir))
+    finally:
+        os.chdir(old_cwd)
 
 
 def test_write_returns_unique_paths_for_concurrent_calls(tmp_path: Path):