mcp-server/common/config.py

# coding=utf-8
"""
@Time   :2026/6/24
@Author :tao.chen

Single source of truth for environment-driven application configuration.

Why this module exists: every prior `os.environ.get("SPARK_EXECUTOR_*")`
was scattered across `connection_store.py`, `pending_store.py`,
`yarn_client.py`, `job_writer.py`. The same env var name appeared in
multiple files, and there was no single place to see "what's
configurable?". This file is that place.

All application code should:
    from common.config import settings
    ... settings.data_dir / settings.yarn_resource_manager_url / ...

...rather than `os.environ.get(...)` directly. This way:
  - The env var name appears in exactly one place.
  - Ops can audit the full config surface from one file.
  - Tests can monkeypatch fields on the `settings` singleton directly
    (more reliable than monkeypatching env + reloading).

NOT in this module:
  - Gunicorn-specific env vars (workers, threads, bind, timeout) live in
    gunicorn.conf.py because they configure gunicorn, not the app.
  - `PYTHONUNBUFFERED` — Python runtime flag, set in Dockerfile.
  - `SPARK_SUBMIT_OPTS` — JVM flags, not Python config; users pass them
    to spark-submit directly.
"""
import os
from dataclasses import dataclass


@dataclass
class Settings:
    """Application settings loaded from environment variables at import time.

    Mutable so tests can reassign fields directly. For tests that use
    `monkeypatch.setenv(...)`, call `settings.reload()` to re-read from
    the (now-patched) environment.
    """

    # --- Data persistence (./data/ in dev, /var/lib/... in prod) ---
    data_dir: str = "./data"

    # --- Job files (LLM-generated PySpark) ---
    # Defaults to <data_dir>/jobs; can be pointed at a larger disk via
    # SPARK_EXECUTOR_JOBS_DIR (e.g. /var/spark-jobs on a big-disk host).
    jobs_dir: str = ""

    # --- Loguru file sinks ---
    # Base directory for loguru file output. Debug and info subdirs are
    # derived as <log_dir>/debug and <log_dir>/info. Defaults to
    # <data_dir>/logs; override via SPARK_EXECUTOR_LOG_DIR to put logs
    # on a dedicated volume (e.g. /var/log/spark-executor).
    log_dir: str = ""

    # --- YARN REST client (fallback for Job.yarn_rm_url snapshot) ---
    # Set in the env OR per-Connection via save_connection.
    yarn_resource_manager_url: str | None = None

    # --- Loguru ---
    # Controls stderr verbosity and the info-level file sink. The debug
    # file sink always captures DEBUG (full audit trail).
    log_level: str = "DEBUG"

    # --- Spark CLI binary name ---
    # The program name used in cmd[0] when invoking the Spark client.
    # Defaults to 'spark-submit' (the standard Spark 2.x / 3.x / 4.x CLI).
    # Override via SPARK_EXECUTOR_SPARK_SUBMIT_BIN for:
    #   - 'spark2-submit' on a system where both Spark 1.x and 2.x are
    #     installed and spark-submit points to the wrong one
    #   - a custom wrapper script (e.g. '/usr/local/bin/spark-submit-wrapper')
    #   - 'pyspark' if you want to launch via the PySpark entrypoint
    # The binary is resolved against $PATH (set by docker-entrypoint.sh).
    spark_submit_bin: str = "spark-submit"

    # --- Confirm retry policy ---
    # confirm_submit_job retries spark-submit on SparkSubmitError to tolerate
    # transient client/network issues in test environments or unstable clusters.
    # Max retries (not counting the first attempt) and delay between attempts.
    confirm_max_retries: int = 3
    confirm_retry_delay_seconds: float = 5.0

    # --- SSL/TLS defaults for YARN REST calls ---
    ssl_verify_default: bool = True
    ssl_ca_bundle_default: str | None = None

    @classmethod
    def from_env(cls) -> "Settings":
        data_dir = os.environ.get("SPARK_EXECUTOR_DATA_DIR", "./data")
        jobs_dir = os.environ.get(
            "SPARK_EXECUTOR_JOBS_DIR", os.path.join(data_dir, "jobs")
        )
        log_dir = os.environ.get(
            "SPARK_EXECUTOR_LOG_DIR", os.path.join(data_dir, "logs")
        )
        return cls(
            data_dir=data_dir,
            jobs_dir=jobs_dir,
            log_dir=log_dir,
            # `or None` collapses empty string to None for the URL fallback
            yarn_resource_manager_url=os.environ.get("YARN_RESOURCE_MANAGER_URL") or None,
            log_level=os.environ.get("SPARK_EXECUTOR_LOG_LEVEL", "DEBUG"),
            spark_submit_bin=os.environ.get(
                "SPARK_EXECUTOR_SPARK_SUBMIT_BIN", "spark-submit"
            ),
            ssl_verify_default=(
                os.environ.get("SPARK_EXECUTOR_SSL_VERIFY_DEFAULT", "true").lower()
                not in ("false", "0", "no", "off")
            ),
            ssl_ca_bundle_default=(
                os.environ.get("SPARK_EXECUTOR_SSL_CA_BUNDLE_DEFAULT") or None
            ),
            confirm_max_retries=int(
                os.environ.get("SPARK_EXECUTOR_CONFIRM_MAX_RETRIES", "3")
            ),
            confirm_retry_delay_seconds=float(
                os.environ.get("SPARK_EXECUTOR_CONFIRM_RETRY_DELAY_SECONDS", "5.0")
            ),
        )

    def reload(self) -> "Settings":
        """Re-read from environment, mutate in place, return self (chainable).

        Useful in tests that `monkeypatch.setenv(...)` and want the
        singleton to pick up the new value without a re-import.
        """
        fresh = self.from_env()
        self.data_dir = fresh.data_dir
        self.jobs_dir = fresh.jobs_dir
        self.log_dir = fresh.log_dir
        self.yarn_resource_manager_url = fresh.yarn_resource_manager_url
        self.log_level = fresh.log_level
        self.spark_submit_bin = fresh.spark_submit_bin
        self.ssl_verify_default = fresh.ssl_verify_default
        self.ssl_ca_bundle_default = fresh.ssl_ca_bundle_default
        self.confirm_max_retries = fresh.confirm_max_retries
        self.confirm_retry_delay_seconds = fresh.confirm_retry_delay_seconds
        return self


# Module-level singleton. Loaded once at import time.
settings = Settings.from_env()