diff --git a/spark_executor/models.py b/spark_executor/models.py index 704f9de..ce27faf 100644 --- a/spark_executor/models.py +++ b/spark_executor/models.py @@ -5,7 +5,7 @@ """ from datetime import datetime -from pydantic import BaseModel, Field +from pydantic import BaseModel, Field, field_validator class Job(BaseModel): @@ -32,11 +32,27 @@ class SubmitResult(BaseModel): class Connection(BaseModel): name: str - master: str + # Defaults to "yarn" because that's the literal string spark-submit wants + # for --master when targeting YARN. Override for Standalone (spark://...), + # Kubernetes (k8s://...), or local mode. + master: str = "yarn" deploy_mode: str = "cluster" yarn_rm_url: str | None = None spark_conf: dict[str, str] = Field(default_factory=dict) + @field_validator("master") + @classmethod + def _check_master(cls, v: str) -> str: + """Catch common typos like 'yarn-cluster' or 'http://...'. """ + if v == "yarn": + return v + if v.startswith(("spark://", "k8s://", "mesos://", "local")): + return v + raise ValueError( + f"master must be 'yarn', 'spark://...', 'k8s://...', 'mesos://...', " + f"or 'local[/N]'; got {v!r}" + ) + class PendingSubmission(BaseModel): pending_id: str diff --git a/tests/unit/test_models.py b/tests/unit/test_models.py index d16dfb9..21ea59c 100644 --- a/tests/unit/test_models.py +++ b/tests/unit/test_models.py @@ -44,11 +44,18 @@ def test_submit_result_tracking_url_optional(): def test_connection_defaults(): c = Connection(name="prod", master="yarn") + assert c.master == "yarn" assert c.deploy_mode == "cluster" assert c.yarn_rm_url is None assert c.spark_conf == {} +def test_connection_master_defaults_to_yarn(): + """YARN users should not have to write master='yarn' explicitly.""" + c = Connection(name="prod") + assert c.master == "yarn" + + def test_connection_with_spark_conf(): c = Connection( name="dev", @@ -59,6 +66,24 @@ def test_connection_with_spark_conf(): assert c.spark_conf["spark.sql.shuffle.partitions"] == "200" +def test_connection_accepts_other_masters(): + """Standalone, k8s, and local should pass validation.""" + Connection(name="standalone", master="spark://host:7077") + Connection(name="k8s", master="k8s://https://api.example.com:443") + Connection(name="local", master="local[4]") + + +def test_connection_rejects_bad_master(): + """Common typos like 'yarn-cluster' or 'http://...' should fail loudly.""" + import pytest + with pytest.raises(ValueError, match="must be 'yarn'"): + Connection(name="bad", master="yarn-cluster") + with pytest.raises(ValueError, match="must be"): + Connection(name="bad", master="http://rm:8088") + with pytest.raises(ValueError, match="must be"): + Connection(name="bad", master="") + + def test_pending_submission_defaults_to_pending_status(): p = PendingSubmission( pending_id="p_1",