From 0e608231687f84775aad6ce68ec965406e58abc5 Mon Sep 17 00:00:00 2001 From: Claude Date: Fri, 26 Jun 2026 18:23:19 +0800 Subject: [PATCH] fix(docker): honor SPARK_EXECUTOR_SPARK_SUBMIT_BIN and persist PATH for docker exec docker-entrypoint.sh now: - Reads SPARK_EXECUTOR_SPARK_SUBMIT_BIN with default spark-submit and validates that the configured binary resolves in PATH. - If the configured binary is an absolute path, prepends its directory to PATH so subprocess.run can find it. - Still validates JAVA_HOME/bin/java and SPARK_HOME/bin/spark-submit. - Writes /etc/profile.d/spark_executor_env.sh with the resolved JAVA_HOME, SPARK_HOME, SPARK_EXECUTOR_SPARK_SUBMIT_BIN, and PATH so interactive `docker exec` shells see the same environment as the main gunicorn process. This fixes two reported issues: 1. SPARK_EXECUTOR_SPARK_SUBMIT_BIN was ignored because the binary was not guaranteed to be on PATH. 2. `docker exec` shells did not see JAVA_HOME/SPARK_HOME in PATH. --- docker-entrypoint.sh | 70 +++++++++++++++++++++++++++++++------------- 1 file changed, 50 insertions(+), 20 deletions(-) diff --git a/docker-entrypoint.sh b/docker-entrypoint.sh index 7401cd7..16aa756 100755 --- a/docker-entrypoint.sh +++ b/docker-entrypoint.sh @@ -1,31 +1,28 @@ #!/bin/sh # docker-entrypoint.sh # -# Spark 3.1.2 + JDK 11 container entrypoint. +# Spark Executor MCP runtime entrypoint. # -# Three responsibilities, in order: -# 1. Resolve JAVA_HOME and SPARK_HOME from env (with sensible defaults -# matching the Dockerfile). -# 2. Validate the resolved paths exist and contain the expected -# binaries. Fail fast at container start with a clear error -# message, instead of letting a job submission die later with an -# opaque "no such file" or "command not found". -# 3. Update PATH so the (possibly overridden) JAVA_HOME/bin and -# SPARK_HOME/bin are prepended — overrides via docker-compose / .env -# take effect on the very next container start, without rebuilding -# the image. +# Responsibilities: +# 1. Resolve JAVA_HOME, SPARK_HOME, and SPARK_EXECUTOR_SPARK_SUBMIT_BIN +# from env (with sensible defaults matching the Dockerfile). +# 2. Validate the resolved paths exist and contain the expected binaries. +# Fail fast at container start with a clear error message instead of +# letting a job submission die later with an opaque "no such file". +# 3. Update PATH so the (possibly overridden) JDK + Spark bin dirs win, +# and so SPARK_EXECUTOR_SPARK_SUBMIT_BIN resolves correctly. +# 4. Persist the resolved environment to /etc/profile.d so `docker exec` +# shells (interactive bash) see the same JAVA_HOME/SPARK_HOME/PATH. # -# Compared to the previous version this drops the awk-based PATH -# stripping (too brittle — would also strip the new JAVA_HOME/bin if -# it happened to be under /usr/lib/jvm/) and instead just prepends. -# Whatever was in the old PATH is preserved; the new paths win -# because they come first. +# Whatever was in the old PATH is preserved; the new paths win because they +# come first. set -e # Defaults match the Dockerfile's build-time ENV # (openjdk-17-jre-headless + Spark 3.5.8) : "${JAVA_HOME:=/usr/lib/jvm/java-17-openjdk-amd64}" : "${SPARK_HOME:=/opt/spark}" +: "${SPARK_EXECUTOR_SPARK_SUBMIT_BIN:=spark-submit}" # --- Validation (fail fast with a clear error) --- if [ ! -x "${JAVA_HOME}/bin/java" ]; then @@ -43,19 +40,52 @@ fi # Export the resolved values (so subprocesses see them) export JAVA_HOME export SPARK_HOME +export SPARK_EXECUTOR_SPARK_SUBMIT_BIN # --- Update PATH --- -# Prepend the project venv and the (possibly overridden) JDK + Spark -# bin dirs. Order matters: /app/.venv/bin first (project tools win), +# Order matters: /app/.venv/bin first (project tools win), # then JAVA_HOME/bin (overrides any system java), then SPARK_HOME/bin, # then whatever was already on PATH. -export PATH="/app/.venv/bin:${JAVA_HOME}/bin:${SPARK_HOME}/bin:${PATH}" +SPARK_BIN_DIR="${SPARK_HOME}/bin" +JAVA_BIN_DIR="${JAVA_HOME}/bin" +export PATH="/app/.venv/bin:${JAVA_BIN_DIR}:${SPARK_BIN_DIR}:${PATH}" + +# If the configured spark-submit binary is an absolute path, make sure its +# directory is on PATH so subprocess.run / os.exec can resolve it. +case "${SPARK_EXECUTOR_SPARK_SUBMIT_BIN}" in + /*) + CUSTOM_BIN_DIR=$(dirname "${SPARK_EXECUTOR_SPARK_SUBMIT_BIN}") + if [ -n "${CUSTOM_BIN_DIR}" ] && [ "${CUSTOM_BIN_DIR}" != "." ]; then + export PATH="${CUSTOM_BIN_DIR}:${PATH}" + fi + ;; +esac + +# Validate that the configured spark-submit binary is actually reachable. +if ! command -v "${SPARK_EXECUTOR_SPARK_SUBMIT_BIN}" >/dev/null 2>&1; then + echo "[entrypoint] FATAL: SPARK_EXECUTOR_SPARK_SUBMIT_BIN=${SPARK_EXECUTOR_SPARK_SUBMIT_BIN} is not in PATH" >&2 + echo "[entrypoint] Hint: use an absolute path, or put the binary on PATH" >&2 + exit 1 +fi + +# --- Persist resolved env for `docker exec` shells --- +# Non-login /bin/sh won't source this automatically, but interactive bash will. +# For non-interactive exec, users can: source /etc/profile.d/spark_executor_env.sh +mkdir -p /etc/profile.d +cat > /etc/profile.d/spark_executor_env.sh <&2 echo "[entrypoint] SPARK_HOME=${SPARK_HOME}" >&2 +echo "[entrypoint] SPARK_EXECUTOR_SPARK_SUBMIT_BIN=${SPARK_EXECUTOR_SPARK_SUBMIT_BIN}" >&2 echo "[entrypoint] java: $(command -v java)" >&2 echo "[entrypoint] spark-submit: $(command -v spark-submit)" >&2 +echo "[entrypoint] configured spark bin: $(command -v "${SPARK_EXECUTOR_SPARK_SUBMIT_BIN}")" >&2 echo "[entrypoint] java version: $(java -version 2>&1 | head -1)" >&2 # Run whatever CMD was passed (gunicorn main:app, or python main.py, etc.)