From 516541a4dfe7b31c007a11c6e0f18a0648207986 Mon Sep 17 00:00:00 2001 From: Claude Date: Thu, 25 Jun 2026 10:05:27 +0800 Subject: [PATCH] feat: Connection.master defaults to 'yarn' MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit YARN is the dominant use case, and the value of master for YARN is the literal string 'yarn' (not a URL) — there is no useful variation. Making it a default saves every YARN user from typing the same value every time: save_connection(name='prod') # master defaults to 'yarn' save_connection(name='dev', master='spark://10.0.0.5:7077') # override Added a field_validator on master that catches common typos ('yarn-cluster', 'http://...', empty string) at save time with a clear error message instead of letting them reach spark-submit. Backward compatible: existing Connection records without an explicit master field default to 'yarn' on load (Pydantic applies defaults on deserialization), so already-saved data still works. 3 new tests cover the default, the validator, and that other cluster managers (spark://, k8s://, local[N]) still pass. --- spark_executor/models.py | 20 ++++++++++++++++++-- tests/unit/test_models.py | 25 +++++++++++++++++++++++++ 2 files changed, 43 insertions(+), 2 deletions(-) diff --git a/spark_executor/models.py b/spark_executor/models.py index 704f9de..ce27faf 100644 --- a/spark_executor/models.py +++ b/spark_executor/models.py @@ -5,7 +5,7 @@ """ from datetime import datetime -from pydantic import BaseModel, Field +from pydantic import BaseModel, Field, field_validator class Job(BaseModel): @@ -32,11 +32,27 @@ class SubmitResult(BaseModel): class Connection(BaseModel): name: str - master: str + # Defaults to "yarn" because that's the literal string spark-submit wants + # for --master when targeting YARN. Override for Standalone (spark://...), + # Kubernetes (k8s://...), or local mode. + master: str = "yarn" deploy_mode: str = "cluster" yarn_rm_url: str | None = None spark_conf: dict[str, str] = Field(default_factory=dict) + @field_validator("master") + @classmethod + def _check_master(cls, v: str) -> str: + """Catch common typos like 'yarn-cluster' or 'http://...'. """ + if v == "yarn": + return v + if v.startswith(("spark://", "k8s://", "mesos://", "local")): + return v + raise ValueError( + f"master must be 'yarn', 'spark://...', 'k8s://...', 'mesos://...', " + f"or 'local[/N]'; got {v!r}" + ) + class PendingSubmission(BaseModel): pending_id: str diff --git a/tests/unit/test_models.py b/tests/unit/test_models.py index d16dfb9..21ea59c 100644 --- a/tests/unit/test_models.py +++ b/tests/unit/test_models.py @@ -44,11 +44,18 @@ def test_submit_result_tracking_url_optional(): def test_connection_defaults(): c = Connection(name="prod", master="yarn") + assert c.master == "yarn" assert c.deploy_mode == "cluster" assert c.yarn_rm_url is None assert c.spark_conf == {} +def test_connection_master_defaults_to_yarn(): + """YARN users should not have to write master='yarn' explicitly.""" + c = Connection(name="prod") + assert c.master == "yarn" + + def test_connection_with_spark_conf(): c = Connection( name="dev", @@ -59,6 +66,24 @@ def test_connection_with_spark_conf(): assert c.spark_conf["spark.sql.shuffle.partitions"] == "200" +def test_connection_accepts_other_masters(): + """Standalone, k8s, and local should pass validation.""" + Connection(name="standalone", master="spark://host:7077") + Connection(name="k8s", master="k8s://https://api.example.com:443") + Connection(name="local", master="local[4]") + + +def test_connection_rejects_bad_master(): + """Common typos like 'yarn-cluster' or 'http://...' should fail loudly.""" + import pytest + with pytest.raises(ValueError, match="must be 'yarn'"): + Connection(name="bad", master="yarn-cluster") + with pytest.raises(ValueError, match="must be"): + Connection(name="bad", master="http://rm:8088") + with pytest.raises(ValueError, match="must be"): + Connection(name="bad", master="") + + def test_pending_submission_defaults_to_pending_status(): p = PendingSubmission( pending_id="p_1",