diff --git a/docker-entrypoint.sh b/docker-entrypoint.sh index 7401cd7..16aa756 100755 --- a/docker-entrypoint.sh +++ b/docker-entrypoint.sh @@ -1,31 +1,28 @@ #!/bin/sh # docker-entrypoint.sh # -# Spark 3.1.2 + JDK 11 container entrypoint. +# Spark Executor MCP runtime entrypoint. # -# Three responsibilities, in order: -# 1. Resolve JAVA_HOME and SPARK_HOME from env (with sensible defaults -# matching the Dockerfile). -# 2. Validate the resolved paths exist and contain the expected -# binaries. Fail fast at container start with a clear error -# message, instead of letting a job submission die later with an -# opaque "no such file" or "command not found". -# 3. Update PATH so the (possibly overridden) JAVA_HOME/bin and -# SPARK_HOME/bin are prepended — overrides via docker-compose / .env -# take effect on the very next container start, without rebuilding -# the image. +# Responsibilities: +# 1. Resolve JAVA_HOME, SPARK_HOME, and SPARK_EXECUTOR_SPARK_SUBMIT_BIN +# from env (with sensible defaults matching the Dockerfile). +# 2. Validate the resolved paths exist and contain the expected binaries. +# Fail fast at container start with a clear error message instead of +# letting a job submission die later with an opaque "no such file". +# 3. Update PATH so the (possibly overridden) JDK + Spark bin dirs win, +# and so SPARK_EXECUTOR_SPARK_SUBMIT_BIN resolves correctly. +# 4. Persist the resolved environment to /etc/profile.d so `docker exec` +# shells (interactive bash) see the same JAVA_HOME/SPARK_HOME/PATH. # -# Compared to the previous version this drops the awk-based PATH -# stripping (too brittle — would also strip the new JAVA_HOME/bin if -# it happened to be under /usr/lib/jvm/) and instead just prepends. -# Whatever was in the old PATH is preserved; the new paths win -# because they come first. +# Whatever was in the old PATH is preserved; the new paths win because they +# come first. set -e # Defaults match the Dockerfile's build-time ENV # (openjdk-17-jre-headless + Spark 3.5.8) : "${JAVA_HOME:=/usr/lib/jvm/java-17-openjdk-amd64}" : "${SPARK_HOME:=/opt/spark}" +: "${SPARK_EXECUTOR_SPARK_SUBMIT_BIN:=spark-submit}" # --- Validation (fail fast with a clear error) --- if [ ! -x "${JAVA_HOME}/bin/java" ]; then @@ -43,19 +40,52 @@ fi # Export the resolved values (so subprocesses see them) export JAVA_HOME export SPARK_HOME +export SPARK_EXECUTOR_SPARK_SUBMIT_BIN # --- Update PATH --- -# Prepend the project venv and the (possibly overridden) JDK + Spark -# bin dirs. Order matters: /app/.venv/bin first (project tools win), +# Order matters: /app/.venv/bin first (project tools win), # then JAVA_HOME/bin (overrides any system java), then SPARK_HOME/bin, # then whatever was already on PATH. -export PATH="/app/.venv/bin:${JAVA_HOME}/bin:${SPARK_HOME}/bin:${PATH}" +SPARK_BIN_DIR="${SPARK_HOME}/bin" +JAVA_BIN_DIR="${JAVA_HOME}/bin" +export PATH="/app/.venv/bin:${JAVA_BIN_DIR}:${SPARK_BIN_DIR}:${PATH}" + +# If the configured spark-submit binary is an absolute path, make sure its +# directory is on PATH so subprocess.run / os.exec can resolve it. +case "${SPARK_EXECUTOR_SPARK_SUBMIT_BIN}" in + /*) + CUSTOM_BIN_DIR=$(dirname "${SPARK_EXECUTOR_SPARK_SUBMIT_BIN}") + if [ -n "${CUSTOM_BIN_DIR}" ] && [ "${CUSTOM_BIN_DIR}" != "." ]; then + export PATH="${CUSTOM_BIN_DIR}:${PATH}" + fi + ;; +esac + +# Validate that the configured spark-submit binary is actually reachable. +if ! command -v "${SPARK_EXECUTOR_SPARK_SUBMIT_BIN}" >/dev/null 2>&1; then + echo "[entrypoint] FATAL: SPARK_EXECUTOR_SPARK_SUBMIT_BIN=${SPARK_EXECUTOR_SPARK_SUBMIT_BIN} is not in PATH" >&2 + echo "[entrypoint] Hint: use an absolute path, or put the binary on PATH" >&2 + exit 1 +fi + +# --- Persist resolved env for `docker exec` shells --- +# Non-login /bin/sh won't source this automatically, but interactive bash will. +# For non-interactive exec, users can: source /etc/profile.d/spark_executor_env.sh +mkdir -p /etc/profile.d +cat > /etc/profile.d/spark_executor_env.sh <&2 echo "[entrypoint] SPARK_HOME=${SPARK_HOME}" >&2 +echo "[entrypoint] SPARK_EXECUTOR_SPARK_SUBMIT_BIN=${SPARK_EXECUTOR_SPARK_SUBMIT_BIN}" >&2 echo "[entrypoint] java: $(command -v java)" >&2 echo "[entrypoint] spark-submit: $(command -v spark-submit)" >&2 +echo "[entrypoint] configured spark bin: $(command -v "${SPARK_EXECUTOR_SPARK_SUBMIT_BIN}")" >&2 echo "[entrypoint] java version: $(java -version 2>&1 | head -1)" >&2 # Run whatever CMD was passed (gunicorn main:app, or python main.py, etc.)