diff --git a/.env.example b/.env.example index 3f96207..6a25873 100644 --- a/.env.example +++ b/.env.example @@ -54,17 +54,17 @@ YARN_RESOURCE_MANAGER_URL= # SPARK_SUBMIT_OPTS=-Dhttps.proxyHost=proxy.corp -Dhttps.proxyPort=3128 # --- Java + Spark install paths --- -# Where the openjdk-17-jre-headless JDK lives, and where the Spark +# Where the openjdk-11-jre-headless JDK lives, and where the Spark # distribution was extracted during the image build. Defaults match the # Dockerfile's ENTRYPOINT script. Override if you mount a different # Java (e.g. /usr/lib/jvm/java-17-openjdk-arm64 on some ARM hosts) or -# a pre-installed Spark from a host volume (e.g. /opt/spark-3.5.1-bin-hadoop3). +# a pre-installed Spark from a host volume (e.g. /opt/spark-3.1.2-bin-hadoop2.7). # # The container's docker-entrypoint.sh re-derives PATH from these values # at every start, so overriding them here actually changes which `java` # and `spark-submit` binaries the gunicorn process picks up. # -# JAVA_HOME=/usr/lib/jvm/java-17-openjdk-amd64 +# JAVA_HOME=/usr/lib/jvm/java-11-openjdk-amd64 # SPARK_HOME=/opt/spark # --- Gunicorn process model (see gunicorn.conf.py; NOT read by common/config.py) --- diff --git a/Dockerfile b/Dockerfile index 2e99985..f7c1e23 100644 --- a/Dockerfile +++ b/Dockerfile @@ -12,7 +12,7 @@ COPY --from=ghcr.io/astral-sh/uv:latest /uv /uvx /usr/local/bin/ # --- Spark + Hadoop config (matches the original Dockerfile) --- -ARG SPARK_VERSION=4.1.2 +ARG SPARK_VERSION=3.1.2 RUN sed -i 's|deb.debian.org|mirrors.tuna.tsinghua.edu.cn|g' /etc/apt/sources.list.d/debian.sources && \ apt-get update && \ @@ -20,18 +20,23 @@ RUN sed -i 's|deb.debian.org|mirrors.tuna.tsinghua.edu.cn|g' /etc/apt/sources.li curl \ ca-certificates \ tar \ - openjdk-17-jre-headless && \ + openjdk-11-jre-headless && \ curl -L \ - https://mirrors.tuna.tsinghua.edu.cn/apache/spark/spark-${SPARK_VERSION}/spark-${SPARK_VERSION}-bin-hadoop3.tgz \ + https://mirrors.tuna.tsinghua.edu.cn/apache/spark/spark-${SPARK_VERSION}/spark-${SPARK_VERSION}-bin-hadoop2.7.tgz \ -o /tmp/spark.tgz && \ mkdir -p /opt && \ tar -xzf /tmp/spark.tgz -C /opt && \ - mv /opt/spark-${SPARK_VERSION}-bin-hadoop3 /opt/spark && \ + mv /opt/spark-${SPARK_VERSION}-bin-hadoop2.7 /opt/spark && \ rm -f /tmp/spark.tgz && \ apt-get clean && \ - rm -rf /var/lib/apt/lists/* + rm -rf /var/lib/apt/lists/* && \ + # Defensive: rewrite the first line of every bin/* script to use a + # known-good shebang. Some Spark distributions have shipped with + # 'bach' (typo for 'bash') in the shebang, which makes the kernel + # refuse to exec the script at all. Idempotent and cheap. + find /opt/spark/bin -type f -exec sed -i '1s|^.*$|#!/usr/bin/env bash|' {} + -ENV JAVA_HOME=/usr/lib/jvm/java-17-openjdk-amd64 +ENV JAVA_HOME=/usr/lib/jvm/java-11-openjdk-amd64 ENV SPARK_HOME=/opt/spark ENV PATH=${JAVA_HOME}/bin:${SPARK_HOME}/bin:${PATH} diff --git a/docker-compose.yml b/docker-compose.yml index 1cfecbc..b75a8e3 100644 --- a/docker-compose.yml +++ b/docker-compose.yml @@ -56,9 +56,9 @@ services: # Override to point at a different JDK install or pre-mounted Spark # distribution. The entrypoint re-derives PATH from these at every # container start, so overrides actually take effect. - # JAVA_HOME=/usr/lib/jvm/java-17-openjdk-amd64 + # JAVA_HOME=/usr/lib/jvm/java-11-openjdk-amd64 # SPARK_HOME=/opt/spark - JAVA_HOME: ${JAVA_HOME:-/usr/lib/jvm/java-17-openjdk-amd64} + JAVA_HOME: ${JAVA_HOME:-/usr/lib/jvm/java-11-openjdk-amd64} SPARK_HOME: ${SPARK_HOME:-/opt/spark} # --- gunicorn.conf.py knobs (NOT read by common/config.py) --- diff --git a/docker-entrypoint.sh b/docker-entrypoint.sh index 89ba269..291242f 100755 --- a/docker-entrypoint.sh +++ b/docker-entrypoint.sh @@ -1,38 +1,61 @@ #!/bin/sh # docker-entrypoint.sh # -# Re-derives PATH from JAVA_HOME and SPARK_HOME at container start, so that -# overriding either via docker-compose / .env actually changes which -# `java` and `spark-submit` binaries the gunicorn process picks up. +# Spark 3.1.2 + JDK 11 container entrypoint. # -# Without this, the image's built-in PATH (set at build time from the -# Dockerfile's ENV) would still reference the old literal paths even -# after the env vars are overridden at runtime. +# Three responsibilities, in order: +# 1. Resolve JAVA_HOME and SPARK_HOME from env (with sensible defaults +# matching the Dockerfile). +# 2. Validate the resolved paths exist and contain the expected +# binaries. Fail fast at container start with a clear error +# message, instead of letting a job submission die later with an +# opaque "no such file" or "command not found". +# 3. Update PATH so the (possibly overridden) JAVA_HOME/bin and +# SPARK_HOME/bin are prepended — overrides via docker-compose / .env +# take effect on the very next container start, without rebuilding +# the image. # -# Usage in Dockerfile: -# ENTRYPOINT ["/usr/local/bin/docker-entrypoint.sh"] -# CMD ["gunicorn", "main:app"] +# Compared to the previous version this drops the awk-based PATH +# stripping (too brittle — would also strip the new JAVA_HOME/bin if +# it happened to be under /usr/lib/jvm/) and instead just prepends. +# Whatever was in the old PATH is preserved; the new paths win +# because they come first. set -e -# Defaults if not set (matches Dockerfile's build-time defaults) -: "${JAVA_HOME:=/usr/lib/jvm/java-17-openjdk-amd64}" +# Defaults match the Dockerfile's build-time ENV +: "${JAVA_HOME:=/usr/lib/jvm/java-11-openjdk-amd64}" : "${SPARK_HOME:=/opt/spark}" -# Project venv always takes precedence -VENV_BIN="/app/.venv/bin" +# --- Validation (fail fast with a clear error) --- +if [ ! -x "${JAVA_HOME}/bin/java" ]; then + echo "[entrypoint] FATAL: JAVA_HOME=${JAVA_HOME} but ${JAVA_HOME}/bin/java is missing or not executable" >&2 + echo "[entrypoint] Hint: set JAVA_HOME to a directory containing bin/java (e.g. /usr/lib/jvm/java-11-openjdk-amd64)" >&2 + exit 1 +fi -# Strip any stale JAVA_HOME / SPARK_HOME bin dirs from PATH, then prepend -# the current ones so overrides actually take effect on a container restart. -# Uses awk instead of `paste -sd:` for portability (macOS paste is BSD and -# doesn't support -s). -PATH_CLEAN=$(echo "$PATH" | tr ':' '\n' | grep -v "/usr/lib/jvm/" | grep -v "/opt/spark/bin" | awk '{ printf "%s%s", $0, (NR==1?":":"") }' | sed 's/:$//') -export PATH="${VENV_BIN}:${JAVA_HOME}/bin:${SPARK_HOME}/bin:${PATH_CLEAN}" +if [ ! -x "${SPARK_HOME}/bin/spark-submit" ]; then + echo "[entrypoint] FATAL: SPARK_HOME=${SPARK_HOME} but ${SPARK_HOME}/bin/spark-submit is missing or not executable" >&2 + echo "[entrypoint] Hint: set SPARK_HOME to the Spark install root (e.g. /opt/spark)" >&2 + exit 1 +fi + +# Export the resolved values (so subprocesses see them) export JAVA_HOME export SPARK_HOME +# --- Update PATH --- +# Prepend the project venv and the (possibly overridden) JDK + Spark +# bin dirs. Order matters: /app/.venv/bin first (project tools win), +# then JAVA_HOME/bin (overrides any system java), then SPARK_HOME/bin, +# then whatever was already on PATH. +export PATH="/app/.venv/bin:${JAVA_HOME}/bin:${SPARK_HOME}/bin:${PATH}" + +# --- Log the effective resolution so docker logs show what was picked --- echo "[entrypoint] JAVA_HOME=${JAVA_HOME}" >&2 echo "[entrypoint] SPARK_HOME=${SPARK_HOME}" >&2 -echo "[entrypoint] PATH=${PATH}" >&2 +echo "[entrypoint] java: $(command -v java)" >&2 +echo "[entrypoint] spark-submit: $(command -v spark-submit)" >&2 +echo "[entrypoint] java version: $(java -version 2>&1 | head -1)" >&2 -# Run whatever CMD was passed (gunicorn main:app, or `python main.py`, etc.) +# Run whatever CMD was passed (gunicorn main:app, or python main.py, etc.) exec "$@"