update: Dockerfile

Dockerfile

@@ -12,32 +12,26 @@ COPY --from=ghcr.io/astral-sh/uv:latest /uv /uvx /usr/local/bin/
 
 # --- Spark + Hadoop config (matches the original Dockerfile) ---
 
-ARG SPARK_VERSION=3.1.2
+ARG SPARK_VERSION=3.5.8
 
 RUN sed -i 's|deb.debian.org|mirrors.tuna.tsinghua.edu.cn|g' /etc/apt/sources.list.d/debian.sources && \
     apt-get update && \
-    apt-cache search openjdk && \
     apt-get install -y --no-install-recommends \
         curl \
         ca-certificates \
         tar \
-        openjdk-11-jre-headless && \
+        openjdk-17-jre-headless && \
     curl -L \
-      https://mirrors.tuna.tsinghua.edu.cn/apache/spark/spark-${SPARK_VERSION}/spark-${SPARK_VERSION}-bin-hadoop2.7.tgz \
+      https://mirrors.tuna.tsinghua.edu.cn/apache/spark/spark-${SPARK_VERSION}/spark-${SPARK_VERSION}-bin-hadoop3.tgz \
       -o /tmp/spark.tgz && \
     mkdir -p /opt && \
     tar -xzf /tmp/spark.tgz -C /opt && \
     mv /opt/spark-${SPARK_VERSION}-bin-hadoop2.7 /opt/spark && \
     rm -f /tmp/spark.tgz && \
     apt-get clean && \
-    rm -rf /var/lib/apt/lists/* && \
+    rm -rf /var/lib/apt/lists/*
-    # Defensive: rewrite the first line of every bin/* script to use a
-    # known-good shebang. Some Spark distributions have shipped with
-    # 'bach' (typo for 'bash') in the shebang, which makes the kernel
-    # refuse to exec the script at all. Idempotent and cheap.
-    find /opt/spark/bin -type f -exec sed -i '1s|^.*$|#!/usr/bin/env bash|' {} +
 
-ENV JAVA_HOME=/usr/lib/jvm/java-11-openjdk-amd64
+ENV JAVA_HOME=/usr/lib/jvm/java-17-openjdk-amd64
 ENV SPARK_HOME=/opt/spark
 ENV PATH=${JAVA_HOME}/bin:${SPARK_HOME}/bin:${PATH}