feat(spark_submit): extra_args kwarg for non-conf spark-submit flags
The function had hardcoded --master / --deploy-mode / --queue /
--executor-memory / --executor-cores / --num-executors plus a
`spark_conf` dict for --conf, with no way to pass other flags like
--jars, --py-files, --files, --driver-memory, --name, etc.
Add `extra_args: dict[str, str] | None = None` that emits `--{key}
{value}` pairs, placed after the --conf loop and before the script
path. Default None preserves the existing cmd shape exactly.
Structured (dict) instead of raw list[str] so LLM agents writing
tool-call payloads can't accidentally pack flag+value into a single
string. Key order in the dict is preserved.
Tests: 4 cases — None default, single pair, multiple pairs in order,
no collision with spark_conf. uv run pytest -> 175 passed.
This commit is contained in:
@@ -23,6 +23,7 @@ def build_spark_submit_command(
|
||||
executor_cores: int,
|
||||
num_executors: int,
|
||||
spark_conf: dict[str, str] | None = None,
|
||||
extra_args: dict[str, str] | None = None,
|
||||
) -> list[str]:
|
||||
# cmd[0] is the Spark CLI binary name, configurable via
|
||||
# SPARK_EXECUTOR_SPARK_SUBMIT_BIN. Defaults to 'spark-submit' but
|
||||
@@ -39,6 +40,8 @@ def build_spark_submit_command(
|
||||
]
|
||||
for key, value in (spark_conf or {}).items():
|
||||
cmd.extend(["--conf", f"{key}={value}"])
|
||||
for key, value in (extra_args or {}).items():
|
||||
cmd.extend([f"--{key}", str(value)])
|
||||
cmd.append(script_path)
|
||||
logger.debug(f"build_spark_submit_command -> {cmd}")
|
||||
return cmd
|
||||
|
||||
Reference in New Issue
Block a user