diff --git a/.env.example b/.env.example index c5a70f6..3f96207 100644 --- a/.env.example +++ b/.env.example @@ -53,6 +53,20 @@ YARN_RESOURCE_MANAGER_URL= # Useful for proxies, custom truststores, or driver memory caps. # SPARK_SUBMIT_OPTS=-Dhttps.proxyHost=proxy.corp -Dhttps.proxyPort=3128 +# --- Java + Spark install paths --- +# Where the openjdk-17-jre-headless JDK lives, and where the Spark +# distribution was extracted during the image build. Defaults match the +# Dockerfile's ENTRYPOINT script. Override if you mount a different +# Java (e.g. /usr/lib/jvm/java-17-openjdk-arm64 on some ARM hosts) or +# a pre-installed Spark from a host volume (e.g. /opt/spark-3.5.1-bin-hadoop3). +# +# The container's docker-entrypoint.sh re-derives PATH from these values +# at every start, so overriding them here actually changes which `java` +# and `spark-submit` binaries the gunicorn process picks up. +# +# JAVA_HOME=/usr/lib/jvm/java-17-openjdk-amd64 +# SPARK_HOME=/opt/spark + # --- Gunicorn process model (see gunicorn.conf.py; NOT read by common/config.py) --- # Defaults shown. These are read by gunicorn directly, not by the app. # GUNICORN_WORKERS=2 diff --git a/Dockerfile b/Dockerfile index 0bae808..2e99985 100644 --- a/Dockerfile +++ b/Dockerfile @@ -62,6 +62,13 @@ RUN uv sync --index-url=https://pypi.tuna.tsinghua.edu.cn/simple/ --frozen --no- ENV PATH=/app/.venv/bin:$PATH ENV PYTHONUNBUFFERED=1 +# Entrypoint re-derives PATH from JAVA_HOME and SPARK_HOME at container +# start, so overriding either via docker-compose / .env actually changes +# which `java` and `spark-submit` binaries the gunicorn process picks up. +COPY docker-entrypoint.sh /usr/local/bin/docker-entrypoint.sh +RUN chmod +x /usr/local/bin/docker-entrypoint.sh +ENTRYPOINT ["/usr/local/bin/docker-entrypoint.sh"] + # gunicorn is the prod entrypoint — multiple ASGI workers, graceful # shutdown, stdout/stderr logs. Config knobs are env-var driven (see # gunicorn.conf.py). diff --git a/docker-compose.yml b/docker-compose.yml index 5db5570..1cfecbc 100644 --- a/docker-compose.yml +++ b/docker-compose.yml @@ -52,6 +52,15 @@ services: # Loguru verbosity for stderr + info file. DEBUG | INFO. SPARK_EXECUTOR_LOG_LEVEL: ${SPARK_EXECUTOR_LOG_LEVEL:-DEBUG} + # --- Runtime paths (consumed by docker-entrypoint.sh, not common/config.py) --- + # Override to point at a different JDK install or pre-mounted Spark + # distribution. The entrypoint re-derives PATH from these at every + # container start, so overrides actually take effect. + # JAVA_HOME=/usr/lib/jvm/java-17-openjdk-amd64 + # SPARK_HOME=/opt/spark + JAVA_HOME: ${JAVA_HOME:-/usr/lib/jvm/java-17-openjdk-amd64} + SPARK_HOME: ${SPARK_HOME:-/opt/spark} + # --- gunicorn.conf.py knobs (NOT read by common/config.py) --- # 2 workers is a good default for a small MCP service; raise for # high-concurrency deploys. diff --git a/docker-entrypoint.sh b/docker-entrypoint.sh new file mode 100755 index 0000000..89ba269 --- /dev/null +++ b/docker-entrypoint.sh @@ -0,0 +1,38 @@ +#!/bin/sh +# docker-entrypoint.sh +# +# Re-derives PATH from JAVA_HOME and SPARK_HOME at container start, so that +# overriding either via docker-compose / .env actually changes which +# `java` and `spark-submit` binaries the gunicorn process picks up. +# +# Without this, the image's built-in PATH (set at build time from the +# Dockerfile's ENV) would still reference the old literal paths even +# after the env vars are overridden at runtime. +# +# Usage in Dockerfile: +# ENTRYPOINT ["/usr/local/bin/docker-entrypoint.sh"] +# CMD ["gunicorn", "main:app"] +set -e + +# Defaults if not set (matches Dockerfile's build-time defaults) +: "${JAVA_HOME:=/usr/lib/jvm/java-17-openjdk-amd64}" +: "${SPARK_HOME:=/opt/spark}" + +# Project venv always takes precedence +VENV_BIN="/app/.venv/bin" + +# Strip any stale JAVA_HOME / SPARK_HOME bin dirs from PATH, then prepend +# the current ones so overrides actually take effect on a container restart. +# Uses awk instead of `paste -sd:` for portability (macOS paste is BSD and +# doesn't support -s). +PATH_CLEAN=$(echo "$PATH" | tr ':' '\n' | grep -v "/usr/lib/jvm/" | grep -v "/opt/spark/bin" | awk '{ printf "%s%s", $0, (NR==1?":":"") }' | sed 's/:$//') +export PATH="${VENV_BIN}:${JAVA_HOME}/bin:${SPARK_HOME}/bin:${PATH_CLEAN}" +export JAVA_HOME +export SPARK_HOME + +echo "[entrypoint] JAVA_HOME=${JAVA_HOME}" >&2 +echo "[entrypoint] SPARK_HOME=${SPARK_HOME}" >&2 +echo "[entrypoint] PATH=${PATH}" >&2 + +# Run whatever CMD was passed (gunicorn main:app, or `python main.py`, etc.) +exec "$@"