Claude
ad557b5984
Add per-Connection ssl_verify / ssl_ca_bundle plus global defaults so CDH 5 / on-prem clusters with self-signed certs or custom CA bundles can be queried without patching code. - Connection gets ssl_verify (bool|None) and ssl_ca_bundle (str|None) - Settings gets ssl_verify_default and ssl_ca_bundle_default - New YarnClientConfig dataclass carries the resolved verify= value - _request passes verify= through to httpx.request - All public yarn_client functions now take YarnClientConfig instead of a bare yarn_rm_url string; tool call sites resolve the Connection - SaveConnectionRequest exposes the two new fields Tests cover per-connection CA bundle, per-connection verify=False, global default fallback, and connection-not-found error.
131 lines
5.2 KiB
Python
131 lines
5.2 KiB
Python
# coding=utf-8
|
|
"""
|
|
@Time :2026/6/24
|
|
@Author :tao.chen
|
|
|
|
Single source of truth for environment-driven application configuration.
|
|
|
|
Why this module exists: every prior `os.environ.get("SPARK_EXECUTOR_*")`
|
|
was scattered across `connection_store.py`, `pending_store.py`,
|
|
`yarn_client.py`, `job_writer.py`. The same env var name appeared in
|
|
multiple files, and there was no single place to see "what's
|
|
configurable?". This file is that place.
|
|
|
|
All application code should:
|
|
from common.config import settings
|
|
... settings.data_dir / settings.yarn_resource_manager_url / ...
|
|
|
|
...rather than `os.environ.get(...)` directly. This way:
|
|
- The env var name appears in exactly one place.
|
|
- Ops can audit the full config surface from one file.
|
|
- Tests can monkeypatch fields on the `settings` singleton directly
|
|
(more reliable than monkeypatching env + reloading).
|
|
|
|
NOT in this module:
|
|
- Gunicorn-specific env vars (workers, threads, bind, timeout) live in
|
|
gunicorn.conf.py because they configure gunicorn, not the app.
|
|
- `PYTHONUNBUFFERED` — Python runtime flag, set in Dockerfile.
|
|
- `SPARK_SUBMIT_OPTS` — JVM flags, not Python config; users pass them
|
|
to spark-submit directly.
|
|
"""
|
|
import os
|
|
from dataclasses import dataclass
|
|
|
|
|
|
@dataclass
|
|
class Settings:
|
|
"""Application settings loaded from environment variables at import time.
|
|
|
|
Mutable so tests can reassign fields directly. For tests that use
|
|
`monkeypatch.setenv(...)`, call `settings.reload()` to re-read from
|
|
the (now-patched) environment.
|
|
"""
|
|
|
|
# --- Data persistence (./data/ in dev, /var/lib/... in prod) ---
|
|
data_dir: str = "./data"
|
|
|
|
# --- Job files (LLM-generated PySpark) ---
|
|
# Defaults to <data_dir>/jobs; can be pointed at a larger disk via
|
|
# SPARK_EXECUTOR_JOBS_DIR (e.g. /var/spark-jobs on a big-disk host).
|
|
jobs_dir: str = ""
|
|
|
|
# --- Loguru file sinks ---
|
|
# Base directory for loguru file output. Debug and info subdirs are
|
|
# derived as <log_dir>/debug and <log_dir>/info. Defaults to
|
|
# <data_dir>/logs; override via SPARK_EXECUTOR_LOG_DIR to put logs
|
|
# on a dedicated volume (e.g. /var/log/spark-executor).
|
|
log_dir: str = ""
|
|
|
|
# --- YARN REST client (fallback for Job.yarn_rm_url snapshot) ---
|
|
# Set in the env OR per-Connection via save_connection.
|
|
yarn_resource_manager_url: str | None = None
|
|
|
|
# --- Loguru ---
|
|
# Controls stderr verbosity and the info-level file sink. The debug
|
|
# file sink always captures DEBUG (full audit trail).
|
|
log_level: str = "DEBUG"
|
|
|
|
# --- Spark CLI binary name ---
|
|
# The program name used in cmd[0] when invoking the Spark client.
|
|
# Defaults to 'spark-submit' (the standard Spark 2.x / 3.x / 4.x CLI).
|
|
# Override via SPARK_EXECUTOR_SPARK_SUBMIT_BIN for:
|
|
# - 'spark2-submit' on a system where both Spark 1.x and 2.x are
|
|
# installed and spark-submit points to the wrong one
|
|
# - a custom wrapper script (e.g. '/usr/local/bin/spark-submit-wrapper')
|
|
# - 'pyspark' if you want to launch via the PySpark entrypoint
|
|
# The binary is resolved against $PATH (set by docker-entrypoint.sh).
|
|
spark_submit_bin: str = "spark-submit"
|
|
|
|
# --- SSL/TLS defaults for YARN REST calls ---
|
|
ssl_verify_default: bool = True
|
|
ssl_ca_bundle_default: str | None = None
|
|
|
|
@classmethod
|
|
def from_env(cls) -> "Settings":
|
|
data_dir = os.environ.get("SPARK_EXECUTOR_DATA_DIR", "./data")
|
|
jobs_dir = os.environ.get(
|
|
"SPARK_EXECUTOR_JOBS_DIR", os.path.join(data_dir, "jobs")
|
|
)
|
|
log_dir = os.environ.get(
|
|
"SPARK_EXECUTOR_LOG_DIR", os.path.join(data_dir, "logs")
|
|
)
|
|
return cls(
|
|
data_dir=data_dir,
|
|
jobs_dir=jobs_dir,
|
|
log_dir=log_dir,
|
|
# `or None` collapses empty string to None for the URL fallback
|
|
yarn_resource_manager_url=os.environ.get("YARN_RESOURCE_MANAGER_URL") or None,
|
|
log_level=os.environ.get("SPARK_EXECUTOR_LOG_LEVEL", "DEBUG"),
|
|
spark_submit_bin=os.environ.get(
|
|
"SPARK_EXECUTOR_SPARK_SUBMIT_BIN", "spark-submit"
|
|
),
|
|
ssl_verify_default=(
|
|
os.environ.get("SPARK_EXECUTOR_SSL_VERIFY_DEFAULT", "true").lower()
|
|
not in ("false", "0", "no", "off")
|
|
),
|
|
ssl_ca_bundle_default=(
|
|
os.environ.get("SPARK_EXECUTOR_SSL_CA_BUNDLE_DEFAULT") or None
|
|
),
|
|
)
|
|
|
|
def reload(self) -> "Settings":
|
|
"""Re-read from environment, mutate in place, return self (chainable).
|
|
|
|
Useful in tests that `monkeypatch.setenv(...)` and want the
|
|
singleton to pick up the new value without a re-import.
|
|
"""
|
|
fresh = self.from_env()
|
|
self.data_dir = fresh.data_dir
|
|
self.jobs_dir = fresh.jobs_dir
|
|
self.log_dir = fresh.log_dir
|
|
self.yarn_resource_manager_url = fresh.yarn_resource_manager_url
|
|
self.log_level = fresh.log_level
|
|
self.spark_submit_bin = fresh.spark_submit_bin
|
|
self.ssl_verify_default = fresh.ssl_verify_default
|
|
self.ssl_ca_bundle_default = fresh.ssl_ca_bundle_default
|
|
return self
|
|
|
|
|
|
# Module-level singleton. Loaded once at import time.
|
|
settings = Settings.from_env()
|