feat: add spark-submit command builder and runner

This commit is contained in:
Claude
2026-06-24 14:35:07 +08:00
parent e6902b00a0
commit b4b2ad534a
2 changed files with 131 additions and 0 deletions
+45
View File
@@ -0,0 +1,45 @@
# coding=utf-8
"""
@Time :2026/6/24
@Author :tao.chen
"""
import subprocess
class SparkSubmitError(Exception):
"""Raised when spark-submit exits with a non-zero return code."""
def build_spark_submit_command(
*,
master: str,
deploy_mode: str,
script_path: str,
queue: str,
executor_memory: str,
executor_cores: int,
num_executors: int,
spark_conf: dict[str, str] | None = None,
) -> list[str]:
cmd = [
"spark-submit",
"--master", master,
"--deploy-mode", deploy_mode,
"--queue", queue,
"--executor-memory", executor_memory,
"--executor-cores", str(executor_cores),
"--num-executors", str(num_executors),
]
for key, value in (spark_conf or {}).items():
cmd.extend(["--conf", f"{key}={value}"])
cmd.append(script_path)
return cmd
def run_spark_submit(cmd: list[str]) -> "subprocess.CompletedProcess[str]":
result = subprocess.run(cmd, capture_output=True, text=True, errors="replace")
if result.returncode != 0:
raise SparkSubmitError(
f"spark-submit failed (rc={result.returncode}): {result.stderr}"
)
return result