Claude
1c9e4a321d
Hardcoding 'spark-submit' as cmd[0] in build_spark_submit_command
breaks for hosts where:
- both Spark 1.x and 2.x/3.x are installed and 'spark-submit' resolves
to the wrong one (use 'spark2-submit' or 'spark3-submit' explicitly)
- the user wants to launch via the PySpark entrypoint ('pyspark')
- a custom wrapper script sits on PATH (e.g. a credentials-injecting
'spark-submit-wrapper')
New env var SPARK_EXECUTOR_SPARK_SUBMIT_BIN. Default is 'spark-submit'
(preserves the current behavior for everyone). Override in .env /
docker-compose.yml to change.
common/config.py:
- new Settings.spark_submit_bin field
- env-var resolution in from_env() with default 'spark-submit'
- included in reload() so tests work
spark_executor/core/spark_submit.py:
- cmd[0] reads settings.spark_submit_bin (was hardcoded 'spark-submit')
.env.example: new section with the override and example values.
docker-compose.yml: forwards the var with the standard 'spark-submit'
default.
Tests: 2 new (settings.spark_submit_bin='spark2-submit', ='pyspark')
plus existing tests updated to use the settings-restore fixture so
mutations don't leak between tests.
165/163 still pass.
60 lines
1.8 KiB
Python
60 lines
1.8 KiB
Python
# coding=utf-8
|
|
"""
|
|
@Time :2026/6/24
|
|
@Author :tao.chen
|
|
"""
|
|
import subprocess
|
|
|
|
from common.config import settings
|
|
from common.logging import logger
|
|
|
|
|
|
class SparkSubmitError(Exception):
|
|
"""Raised when spark-submit exits with a non-zero return code."""
|
|
|
|
|
|
def build_spark_submit_command(
|
|
*,
|
|
master: str,
|
|
deploy_mode: str,
|
|
script_path: str,
|
|
queue: str,
|
|
executor_memory: str,
|
|
executor_cores: int,
|
|
num_executors: int,
|
|
spark_conf: dict[str, str] | None = None,
|
|
) -> list[str]:
|
|
# cmd[0] is the Spark CLI binary name, configurable via
|
|
# SPARK_EXECUTOR_SPARK_SUBMIT_BIN. Defaults to 'spark-submit' but
|
|
# can be 'spark2-submit' (mixed-version hosts), 'pyspark' (PySpark
|
|
# entrypoint), or a path to a wrapper script.
|
|
cmd = [
|
|
settings.spark_submit_bin,
|
|
"--master", master,
|
|
"--deploy-mode", deploy_mode,
|
|
"--queue", queue,
|
|
"--executor-memory", executor_memory,
|
|
"--executor-cores", str(executor_cores),
|
|
"--num-executors", str(num_executors),
|
|
]
|
|
for key, value in (spark_conf or {}).items():
|
|
cmd.extend(["--conf", f"{key}={value}"])
|
|
cmd.append(script_path)
|
|
logger.debug(f"build_spark_submit_command -> {cmd}")
|
|
return cmd
|
|
|
|
|
|
def run_spark_submit(cmd: list[str]) -> "subprocess.CompletedProcess[str]":
|
|
logger.debug(f"run_spark_submit exec: {cmd}")
|
|
result = subprocess.run(cmd, capture_output=True, text=True, errors="replace")
|
|
logger.debug(
|
|
f"run_spark_submit done rc={result.returncode} "
|
|
f"stdout_len={len(result.stdout)} stderr_len={len(result.stderr)}"
|
|
)
|
|
if result.returncode != 0:
|
|
logger.error(f"spark-submit failed (rc={result.returncode}): {result.stderr[:500]}")
|
|
raise SparkSubmitError(
|
|
f"spark-submit failed (rc={result.returncode}): {result.stderr}"
|
|
)
|
|
return result
|