Claude
0e60823168
docker-entrypoint.sh now: - Reads SPARK_EXECUTOR_SPARK_SUBMIT_BIN with default spark-submit and validates that the configured binary resolves in PATH. - If the configured binary is an absolute path, prepends its directory to PATH so subprocess.run can find it. - Still validates JAVA_HOME/bin/java and SPARK_HOME/bin/spark-submit. - Writes /etc/profile.d/spark_executor_env.sh with the resolved JAVA_HOME, SPARK_HOME, SPARK_EXECUTOR_SPARK_SUBMIT_BIN, and PATH so interactive `docker exec` shells see the same environment as the main gunicorn process. This fixes two reported issues: 1. SPARK_EXECUTOR_SPARK_SUBMIT_BIN was ignored because the binary was not guaranteed to be on PATH. 2. `docker exec` shells did not see JAVA_HOME/SPARK_HOME in PATH.
93 lines
4.0 KiB
Bash
Executable File
93 lines
4.0 KiB
Bash
Executable File
#!/bin/sh
|
|
# docker-entrypoint.sh
|
|
#
|
|
# Spark Executor MCP runtime entrypoint.
|
|
#
|
|
# Responsibilities:
|
|
# 1. Resolve JAVA_HOME, SPARK_HOME, and SPARK_EXECUTOR_SPARK_SUBMIT_BIN
|
|
# from env (with sensible defaults matching the Dockerfile).
|
|
# 2. Validate the resolved paths exist and contain the expected binaries.
|
|
# Fail fast at container start with a clear error message instead of
|
|
# letting a job submission die later with an opaque "no such file".
|
|
# 3. Update PATH so the (possibly overridden) JDK + Spark bin dirs win,
|
|
# and so SPARK_EXECUTOR_SPARK_SUBMIT_BIN resolves correctly.
|
|
# 4. Persist the resolved environment to /etc/profile.d so `docker exec`
|
|
# shells (interactive bash) see the same JAVA_HOME/SPARK_HOME/PATH.
|
|
#
|
|
# Whatever was in the old PATH is preserved; the new paths win because they
|
|
# come first.
|
|
set -e
|
|
|
|
# Defaults match the Dockerfile's build-time ENV
|
|
# (openjdk-17-jre-headless + Spark 3.5.8)
|
|
: "${JAVA_HOME:=/usr/lib/jvm/java-17-openjdk-amd64}"
|
|
: "${SPARK_HOME:=/opt/spark}"
|
|
: "${SPARK_EXECUTOR_SPARK_SUBMIT_BIN:=spark-submit}"
|
|
|
|
# --- Validation (fail fast with a clear error) ---
|
|
if [ ! -x "${JAVA_HOME}/bin/java" ]; then
|
|
echo "[entrypoint] FATAL: JAVA_HOME=${JAVA_HOME} but ${JAVA_HOME}/bin/java is missing or not executable" >&2
|
|
echo "[entrypoint] Hint: set JAVA_HOME to a directory containing bin/java (e.g. /usr/lib/jvm/java-11-openjdk-amd64)" >&2
|
|
exit 1
|
|
fi
|
|
|
|
if [ ! -x "${SPARK_HOME}/bin/spark-submit" ]; then
|
|
echo "[entrypoint] FATAL: SPARK_HOME=${SPARK_HOME} but ${SPARK_HOME}/bin/spark-submit is missing or not executable" >&2
|
|
echo "[entrypoint] Hint: set SPARK_HOME to the Spark install root (e.g. /opt/spark)" >&2
|
|
exit 1
|
|
fi
|
|
|
|
# Export the resolved values (so subprocesses see them)
|
|
export JAVA_HOME
|
|
export SPARK_HOME
|
|
export SPARK_EXECUTOR_SPARK_SUBMIT_BIN
|
|
|
|
# --- Update PATH ---
|
|
# Order matters: /app/.venv/bin first (project tools win),
|
|
# then JAVA_HOME/bin (overrides any system java), then SPARK_HOME/bin,
|
|
# then whatever was already on PATH.
|
|
SPARK_BIN_DIR="${SPARK_HOME}/bin"
|
|
JAVA_BIN_DIR="${JAVA_HOME}/bin"
|
|
export PATH="/app/.venv/bin:${JAVA_BIN_DIR}:${SPARK_BIN_DIR}:${PATH}"
|
|
|
|
# If the configured spark-submit binary is an absolute path, make sure its
|
|
# directory is on PATH so subprocess.run / os.exec can resolve it.
|
|
case "${SPARK_EXECUTOR_SPARK_SUBMIT_BIN}" in
|
|
/*)
|
|
CUSTOM_BIN_DIR=$(dirname "${SPARK_EXECUTOR_SPARK_SUBMIT_BIN}")
|
|
if [ -n "${CUSTOM_BIN_DIR}" ] && [ "${CUSTOM_BIN_DIR}" != "." ]; then
|
|
export PATH="${CUSTOM_BIN_DIR}:${PATH}"
|
|
fi
|
|
;;
|
|
esac
|
|
|
|
# Validate that the configured spark-submit binary is actually reachable.
|
|
if ! command -v "${SPARK_EXECUTOR_SPARK_SUBMIT_BIN}" >/dev/null 2>&1; then
|
|
echo "[entrypoint] FATAL: SPARK_EXECUTOR_SPARK_SUBMIT_BIN=${SPARK_EXECUTOR_SPARK_SUBMIT_BIN} is not in PATH" >&2
|
|
echo "[entrypoint] Hint: use an absolute path, or put the binary on PATH" >&2
|
|
exit 1
|
|
fi
|
|
|
|
# --- Persist resolved env for `docker exec` shells ---
|
|
# Non-login /bin/sh won't source this automatically, but interactive bash will.
|
|
# For non-interactive exec, users can: source /etc/profile.d/spark_executor_env.sh
|
|
mkdir -p /etc/profile.d
|
|
cat > /etc/profile.d/spark_executor_env.sh <<EOF
|
|
export JAVA_HOME="${JAVA_HOME}"
|
|
export SPARK_HOME="${SPARK_HOME}"
|
|
export SPARK_EXECUTOR_SPARK_SUBMIT_BIN="${SPARK_EXECUTOR_SPARK_SUBMIT_BIN}"
|
|
export PATH="${PATH}"
|
|
EOF
|
|
|
|
# --- Log the effective resolution so docker logs show what was picked ---
|
|
echo "[entrypoint] JAVA_HOME=${JAVA_HOME}" >&2
|
|
echo "[entrypoint] SPARK_HOME=${SPARK_HOME}" >&2
|
|
echo "[entrypoint] SPARK_EXECUTOR_SPARK_SUBMIT_BIN=${SPARK_EXECUTOR_SPARK_SUBMIT_BIN}" >&2
|
|
echo "[entrypoint] java: $(command -v java)" >&2
|
|
echo "[entrypoint] spark-submit: $(command -v spark-submit)" >&2
|
|
echo "[entrypoint] configured spark bin: $(command -v "${SPARK_EXECUTOR_SPARK_SUBMIT_BIN}")" >&2
|
|
echo "[entrypoint] java version: $(java -version 2>&1 | head -1)" >&2
|
|
|
|
# Run whatever CMD was passed (gunicorn main:app, or python main.py, etc.)
|
|
exec "$@"
|