feat: env-configurable spark-submit binary name
Hardcoding 'spark-submit' as cmd[0] in build_spark_submit_command
breaks for hosts where:
- both Spark 1.x and 2.x/3.x are installed and 'spark-submit' resolves
to the wrong one (use 'spark2-submit' or 'spark3-submit' explicitly)
- the user wants to launch via the PySpark entrypoint ('pyspark')
- a custom wrapper script sits on PATH (e.g. a credentials-injecting
'spark-submit-wrapper')
New env var SPARK_EXECUTOR_SPARK_SUBMIT_BIN. Default is 'spark-submit'
(preserves the current behavior for everyone). Override in .env /
docker-compose.yml to change.
common/config.py:
- new Settings.spark_submit_bin field
- env-var resolution in from_env() with default 'spark-submit'
- included in reload() so tests work
spark_executor/core/spark_submit.py:
- cmd[0] reads settings.spark_submit_bin (was hardcoded 'spark-submit')
.env.example: new section with the override and example values.
docker-compose.yml: forwards the var with the standard 'spark-submit'
default.
Tests: 2 new (settings.spark_submit_bin='spark2-submit', ='pyspark')
plus existing tests updated to use the settings-restore fixture so
mutations don't leak between tests.
165/163 still pass.
This commit is contained in:
@@ -3,6 +3,7 @@ from unittest.mock import MagicMock, patch
|
||||
|
||||
import pytest
|
||||
|
||||
from common import config
|
||||
from spark_executor.core.spark_submit import (
|
||||
SparkSubmitError,
|
||||
build_spark_submit_command,
|
||||
@@ -10,6 +11,25 @@ from spark_executor.core.spark_submit import (
|
||||
)
|
||||
|
||||
|
||||
@pytest.fixture(autouse=True)
|
||||
def _restore_settings():
|
||||
snapshot = config.Settings(
|
||||
data_dir=config.settings.data_dir,
|
||||
jobs_dir=config.settings.jobs_dir,
|
||||
log_dir=config.settings.log_dir,
|
||||
yarn_resource_manager_url=config.settings.yarn_resource_manager_url,
|
||||
log_level=config.settings.log_level,
|
||||
spark_submit_bin=config.settings.spark_submit_bin,
|
||||
)
|
||||
yield
|
||||
config.settings.data_dir = snapshot.data_dir
|
||||
config.settings.jobs_dir = snapshot.jobs_dir
|
||||
config.settings.log_dir = snapshot.log_dir
|
||||
config.settings.yarn_resource_manager_url = snapshot.yarn_resource_manager_url
|
||||
config.settings.log_level = snapshot.log_level
|
||||
config.settings.spark_submit_bin = snapshot.spark_submit_bin
|
||||
|
||||
|
||||
def test_build_command_uses_provided_master_and_deploy_mode():
|
||||
cmd = build_spark_submit_command(
|
||||
master="yarn",
|
||||
@@ -20,7 +40,7 @@ def test_build_command_uses_provided_master_and_deploy_mode():
|
||||
executor_cores=2,
|
||||
num_executors=2,
|
||||
)
|
||||
assert cmd[:2] == ["spark-submit", "--master"]
|
||||
assert cmd[:2] == ["spark-submit", "--master"] # default binary
|
||||
assert "yarn" in cmd
|
||||
assert "cluster" in cmd
|
||||
assert "--queue" in cmd and "default" in cmd
|
||||
@@ -63,6 +83,37 @@ def test_build_command_appends_spark_conf_entries():
|
||||
assert cmd[-1] == "/tmp/j.py"
|
||||
|
||||
|
||||
def test_build_command_uses_settings_spark_submit_bin():
|
||||
"""The Spark CLI binary is env-configurable via settings.spark_submit_bin
|
||||
(env var SPARK_EXECUTOR_SPARK_SUBMIT_BIN). Default is 'spark-submit';
|
||||
override to 'spark2-submit', 'pyspark', etc. for mixed-version hosts."""
|
||||
config.settings.spark_submit_bin = "spark2-submit"
|
||||
cmd = build_spark_submit_command(
|
||||
master="yarn",
|
||||
deploy_mode="cluster",
|
||||
script_path="/tmp/j.py",
|
||||
queue="default",
|
||||
executor_memory="4G",
|
||||
executor_cores=2,
|
||||
num_executors=2,
|
||||
)
|
||||
assert cmd[0] == "spark2-submit"
|
||||
|
||||
|
||||
def test_build_command_supports_pyspark_binary():
|
||||
config.settings.spark_submit_bin = "pyspark"
|
||||
cmd = build_spark_submit_command(
|
||||
master="yarn",
|
||||
deploy_mode="cluster",
|
||||
script_path="/tmp/j.py",
|
||||
queue="default",
|
||||
executor_memory="4G",
|
||||
executor_cores=2,
|
||||
num_executors=2,
|
||||
)
|
||||
assert cmd[0] == "pyspark"
|
||||
|
||||
|
||||
def test_run_spark_submit_returns_completed_process(monkeypatch):
|
||||
fake = MagicMock()
|
||||
fake.returncode = 0
|
||||
|
||||
Reference in New Issue
Block a user