feat: Connection.master defaults to 'yarn'
YARN is the dominant use case, and the value of master for YARN is the
literal string 'yarn' (not a URL) — there is no useful variation. Making
it a default saves every YARN user from typing the same value every time:
save_connection(name='prod') # master defaults to 'yarn'
save_connection(name='dev', master='spark://10.0.0.5:7077') # override
Added a field_validator on master that catches common typos
('yarn-cluster', 'http://...', empty string) at save time with a clear
error message instead of letting them reach spark-submit.
Backward compatible: existing Connection records without an explicit
master field default to 'yarn' on load (Pydantic applies defaults on
deserialization), so already-saved data still works.
3 new tests cover the default, the validator, and that other cluster
managers (spark://, k8s://, local[N]) still pass.
This commit is contained in:
@@ -44,11 +44,18 @@ def test_submit_result_tracking_url_optional():
|
||||
|
||||
def test_connection_defaults():
|
||||
c = Connection(name="prod", master="yarn")
|
||||
assert c.master == "yarn"
|
||||
assert c.deploy_mode == "cluster"
|
||||
assert c.yarn_rm_url is None
|
||||
assert c.spark_conf == {}
|
||||
|
||||
|
||||
def test_connection_master_defaults_to_yarn():
|
||||
"""YARN users should not have to write master='yarn' explicitly."""
|
||||
c = Connection(name="prod")
|
||||
assert c.master == "yarn"
|
||||
|
||||
|
||||
def test_connection_with_spark_conf():
|
||||
c = Connection(
|
||||
name="dev",
|
||||
@@ -59,6 +66,24 @@ def test_connection_with_spark_conf():
|
||||
assert c.spark_conf["spark.sql.shuffle.partitions"] == "200"
|
||||
|
||||
|
||||
def test_connection_accepts_other_masters():
|
||||
"""Standalone, k8s, and local should pass validation."""
|
||||
Connection(name="standalone", master="spark://host:7077")
|
||||
Connection(name="k8s", master="k8s://https://api.example.com:443")
|
||||
Connection(name="local", master="local[4]")
|
||||
|
||||
|
||||
def test_connection_rejects_bad_master():
|
||||
"""Common typos like 'yarn-cluster' or 'http://...' should fail loudly."""
|
||||
import pytest
|
||||
with pytest.raises(ValueError, match="must be 'yarn'"):
|
||||
Connection(name="bad", master="yarn-cluster")
|
||||
with pytest.raises(ValueError, match="must be"):
|
||||
Connection(name="bad", master="http://rm:8088")
|
||||
with pytest.raises(ValueError, match="must be"):
|
||||
Connection(name="bad", master="")
|
||||
|
||||
|
||||
def test_pending_submission_defaults_to_pending_status():
|
||||
p = PendingSubmission(
|
||||
pending_id="p_1",
|
||||
|
||||
Reference in New Issue
Block a user