fix(docker): honor SPARK_EXECUTOR_SPARK_SUBMIT_BIN and persist PATH for docker exec

docker-entrypoint.sh now:
- Reads SPARK_EXECUTOR_SPARK_SUBMIT_BIN with default spark-submit and
  validates that the configured binary resolves in PATH.
- If the configured binary is an absolute path, prepends its directory
  to PATH so subprocess.run can find it.
- Still validates JAVA_HOME/bin/java and SPARK_HOME/bin/spark-submit.
- Writes /etc/profile.d/spark_executor_env.sh with the resolved
  JAVA_HOME, SPARK_HOME, SPARK_EXECUTOR_SPARK_SUBMIT_BIN, and PATH so
  interactive `docker exec` shells see the same environment as the
  main gunicorn process.

This fixes two reported issues:
1. SPARK_EXECUTOR_SPARK_SUBMIT_BIN was ignored because the binary was
   not guaranteed to be on PATH.
2. `docker exec` shells did not see JAVA_HOME/SPARK_HOME in PATH.

docker-entrypoint.sh

@@ -1,31 +1,28 @@
 #!/bin/sh
 # docker-entrypoint.sh
 #
-# Spark 3.1.2 + JDK 11 container entrypoint.
+# Spark Executor MCP runtime entrypoint.
 #
-# Three responsibilities, in order:
+# Responsibilities:
-#   1. Resolve JAVA_HOME and SPARK_HOME from env (with sensible defaults
+#   1. Resolve JAVA_HOME, SPARK_HOME, and SPARK_EXECUTOR_SPARK_SUBMIT_BIN
-#      matching the Dockerfile).
+#      from env (with sensible defaults matching the Dockerfile).
-#   2. Validate the resolved paths exist and contain the expected
+#   2. Validate the resolved paths exist and contain the expected binaries.
-#      binaries. Fail fast at container start with a clear error
+#      Fail fast at container start with a clear error message instead of
-#      message, instead of letting a job submission die later with an
+#      letting a job submission die later with an opaque "no such file".
-#      opaque "no such file" or "command not found".
+#   3. Update PATH so the (possibly overridden) JDK + Spark bin dirs win,
-#   3. Update PATH so the (possibly overridden) JAVA_HOME/bin and
+#      and so SPARK_EXECUTOR_SPARK_SUBMIT_BIN resolves correctly.
-#      SPARK_HOME/bin are prepended — overrides via docker-compose / .env
+#   4. Persist the resolved environment to /etc/profile.d so `docker exec`
-#      take effect on the very next container start, without rebuilding
+#      shells (interactive bash) see the same JAVA_HOME/SPARK_HOME/PATH.
-#      the image.
 #
-# Compared to the previous version this drops the awk-based PATH
+# Whatever was in the old PATH is preserved; the new paths win because they
-# stripping (too brittle — would also strip the new JAVA_HOME/bin if
+# come first.
-# it happened to be under /usr/lib/jvm/) and instead just prepends.
-# Whatever was in the old PATH is preserved; the new paths win
-# because they come first.
 set -e
 
 # Defaults match the Dockerfile's build-time ENV
 # (openjdk-17-jre-headless + Spark 3.5.8)
 : "${JAVA_HOME:=/usr/lib/jvm/java-17-openjdk-amd64}"
 : "${SPARK_HOME:=/opt/spark}"
+: "${SPARK_EXECUTOR_SPARK_SUBMIT_BIN:=spark-submit}"
 
 # --- Validation (fail fast with a clear error) ---
 if [ ! -x "${JAVA_HOME}/bin/java" ]; then
@@ -43,19 +40,52 @@ fi
 # Export the resolved values (so subprocesses see them)
 export JAVA_HOME
 export SPARK_HOME
+export SPARK_EXECUTOR_SPARK_SUBMIT_BIN
 
 # --- Update PATH ---
-# Prepend the project venv and the (possibly overridden) JDK + Spark
+# Order matters: /app/.venv/bin first (project tools win),
-# bin dirs. Order matters: /app/.venv/bin first (project tools win),
 # then JAVA_HOME/bin (overrides any system java), then SPARK_HOME/bin,
 # then whatever was already on PATH.
-export PATH="/app/.venv/bin:${JAVA_HOME}/bin:${SPARK_HOME}/bin:${PATH}"
+SPARK_BIN_DIR="${SPARK_HOME}/bin"
+JAVA_BIN_DIR="${JAVA_HOME}/bin"
+export PATH="/app/.venv/bin:${JAVA_BIN_DIR}:${SPARK_BIN_DIR}:${PATH}"
+
+# If the configured spark-submit binary is an absolute path, make sure its
+# directory is on PATH so subprocess.run / os.exec can resolve it.
+case "${SPARK_EXECUTOR_SPARK_SUBMIT_BIN}" in
+    /*)
+        CUSTOM_BIN_DIR=$(dirname "${SPARK_EXECUTOR_SPARK_SUBMIT_BIN}")
+        if [ -n "${CUSTOM_BIN_DIR}" ] && [ "${CUSTOM_BIN_DIR}" != "." ]; then
+            export PATH="${CUSTOM_BIN_DIR}:${PATH}"
+        fi
+        ;;
+esac
+
+# Validate that the configured spark-submit binary is actually reachable.
+if ! command -v "${SPARK_EXECUTOR_SPARK_SUBMIT_BIN}" >/dev/null 2>&1; then
+    echo "[entrypoint] FATAL: SPARK_EXECUTOR_SPARK_SUBMIT_BIN=${SPARK_EXECUTOR_SPARK_SUBMIT_BIN} is not in PATH" >&2
+    echo "[entrypoint] Hint: use an absolute path, or put the binary on PATH" >&2
+    exit 1
+fi
+
+# --- Persist resolved env for `docker exec` shells ---
+# Non-login /bin/sh won't source this automatically, but interactive bash will.
+# For non-interactive exec, users can: source /etc/profile.d/spark_executor_env.sh
+mkdir -p /etc/profile.d
+cat > /etc/profile.d/spark_executor_env.sh <<EOF
+export JAVA_HOME="${JAVA_HOME}"
+export SPARK_HOME="${SPARK_HOME}"
+export SPARK_EXECUTOR_SPARK_SUBMIT_BIN="${SPARK_EXECUTOR_SPARK_SUBMIT_BIN}"
+export PATH="${PATH}"
+EOF
 
 # --- Log the effective resolution so docker logs show what was picked ---
 echo "[entrypoint] JAVA_HOME=${JAVA_HOME}" >&2
 echo "[entrypoint] SPARK_HOME=${SPARK_HOME}" >&2
+echo "[entrypoint] SPARK_EXECUTOR_SPARK_SUBMIT_BIN=${SPARK_EXECUTOR_SPARK_SUBMIT_BIN}" >&2
 echo "[entrypoint] java:          $(command -v java)" >&2
 echo "[entrypoint] spark-submit:  $(command -v spark-submit)" >&2
+echo "[entrypoint] configured spark bin: $(command -v "${SPARK_EXECUTOR_SPARK_SUBMIT_BIN}")" >&2
 echo "[entrypoint] java version:  $(java -version 2>&1 | head -1)" >&2
 
 # Run whatever CMD was passed (gunicorn main:app, or python main.py, etc.)