diff --git a/.github/workflows/build-amd64-releases.yml b/.github/workflows/build-amd64-releases.yml index 20cafe1fb..15b61726e 100644 --- a/.github/workflows/build-amd64-releases.yml +++ b/.github/workflows/build-amd64-releases.yml @@ -37,13 +37,13 @@ jobs: runs-on: ${{ matrix.runner }} strategy: matrix: - sparkver: [spark-3.0, spark-3.1, spark-3.2, spark-3.3, spark-3.4, spark-3.5] + sparkver: [spark-3.0, spark-3.1, spark-3.2, spark-3.3, spark-3.4, spark-3.5, spark-4.0, spark-4.1] scalaver: [ 2.12, 2.13 ] javaver: [ 8, 21 ] auronver: [8.0.0-SNAPSHOT] runner: [ ubuntu-24.04 ] exclude: - # Only build on scala-2.13 for spark-3.5 + # Only build on scala-2.13 for spark-3.5+ - sparkver: spark-3.0 scalaver: '2.13' - sparkver: spark-3.1 @@ -64,6 +64,32 @@ jobs: javaver: '21' - sparkver: spark-3.4 javaver: '21' + # Spark 4.x only supports scala-2.13 and java 17+ + - sparkver: spark-4.0 + scalaver: '2.12' + - sparkver: spark-4.1 + scalaver: '2.12' + - sparkver: spark-4.0 + javaver: '8' + - sparkver: spark-4.1 + javaver: '8' + # Spark 4.x JDK 21 builds are handled by explicit rockylinux8 include entries below. + - sparkver: spark-4.0 + scalaver: '2.13' + javaver: '21' + - sparkver: spark-4.1 + scalaver: '2.13' + javaver: '21' + include: + # Spark 4.x uses rockylinux8 image (with JDK 21) + - sparkver: spark-4.0 + scalaver: '2.13' + javaver: '21' + image: rockylinux8 + - sparkver: spark-4.1 + scalaver: '2.13' + javaver: '21' + image: rockylinux8 steps: - uses: actions/checkout@v6 @@ -97,11 +123,13 @@ jobs: key: auron-${{matrix.sparkver}}_${{matrix.scalaver}}-release-${{ env.osclassfier }}-${{ matrix.auronver }}.jar - name: Build auron-${{ matrix.sparkver }}_${{ matrix.scalaver }} + env: + AURON_JAVA_VERSION: ${{ matrix.javaver }} run: | sed -i 's/docker-compose -f/docker compose -f/g' ./auron-build.sh ./auron-build.sh \ --docker true \ - --image centos7 \ + --image ${{ matrix.image || 'centos7' }} \ --release \ --sparkver ${{ env.sparkver_short }} \ --scalaver ${{ matrix.scalaver }} diff --git a/.github/workflows/build-arm-releases.yml b/.github/workflows/build-arm-releases.yml index fb278144e..817193ae6 100644 --- a/.github/workflows/build-arm-releases.yml +++ b/.github/workflows/build-arm-releases.yml @@ -37,7 +37,7 @@ jobs: runs-on: ${{ matrix.runner }} strategy: matrix: - sparkver: [spark-3.0, spark-3.1, spark-3.2, spark-3.3, spark-3.4, spark-3.5] + sparkver: [spark-3.0, spark-3.1, spark-3.2, spark-3.3, spark-3.4, spark-3.5, spark-4.0, spark-4.1] auronver: [8.0.0-SNAPSHOT] scalaver: [2.12, 2.13] javaver: [ 8, 21 ] @@ -64,6 +64,15 @@ jobs: javaver: '21' - sparkver: spark-3.4 javaver: '21' + # Spark 4.x only supports scala-2.13 and java 17+ + - sparkver: spark-4.0 + scalaver: '2.12' + - sparkver: spark-4.1 + scalaver: '2.12' + - sparkver: spark-4.0 + javaver: '8' + - sparkver: spark-4.1 + javaver: '8' steps: - uses: actions/checkout@v6 diff --git a/.github/workflows/build-macos-releases.yml b/.github/workflows/build-macos-releases.yml index 0a2746991..e47e9d089 100644 --- a/.github/workflows/build-macos-releases.yml +++ b/.github/workflows/build-macos-releases.yml @@ -37,7 +37,7 @@ jobs: runs-on: ${{ matrix.runner }} strategy: matrix: - sparkver: [spark-3.0, spark-3.1, spark-3.2, spark-3.3, spark-3.4, spark-3.5] + sparkver: [spark-3.0, spark-3.1, spark-3.2, spark-3.3, spark-3.4, spark-3.5, spark-4.0, spark-4.1] auronver: [7.0.0-SNAPSHOT] scalaver: [2.12, 2.13] javaver: [8, 21] @@ -64,6 +64,15 @@ jobs: javaver: '21' - sparkver: spark-3.4 javaver: '21' + # Spark 4.x only supports scala-2.13 and java 17+ + - sparkver: spark-4.0 + scalaver: '2.12' + - sparkver: spark-4.1 + scalaver: '2.12' + - sparkver: spark-4.0 + javaver: '8' + - sparkver: spark-4.1 + javaver: '8' steps: - uses: actions/checkout@v6 diff --git a/dev/docker-build/docker-compose.yml b/dev/docker-build/docker-compose.yml index e9aa9f0dd..7bfb0f816 100644 --- a/dev/docker-build/docker-compose.yml +++ b/dev/docker-build/docker-compose.yml @@ -32,6 +32,7 @@ services: environment: RUSTFLAGS: "-C target-cpu=skylake" AURON_BUILD_ARGS: "${AURON_BUILD_ARGS}" + AURON_JAVA_VERSION: "${AURON_JAVA_VERSION:-8}" command: > bash -c ' source ~/.bashrc && diff --git a/dev/docker-build/rockylinux8/Dockerfile b/dev/docker-build/rockylinux8/Dockerfile index e519c4f4c..c38aa72b6 100644 --- a/dev/docker-build/rockylinux8/Dockerfile +++ b/dev/docker-build/rockylinux8/Dockerfile @@ -63,14 +63,34 @@ RUN curl https://sh.rustup.rs -sSf -o rustup-init && \ ENV PATH="/root/.cargo/bin:${PATH}" # --------------------------------------------------------------------- -# Install Java (OpenJDK 8) +# Install Java (OpenJDK 8 and OpenJDK 21) # --------------------------------------------------------------------- -RUN dnf -y install java-1.8.0-openjdk java-1.8.0-openjdk-devel && \ +RUN dnf -y install java-1.8.0-openjdk java-1.8.0-openjdk-devel \ + java-21-openjdk java-21-openjdk-devel && \ dnf clean all && \ rm -rf /var/cache/dnf -ENV JAVA_HOME="/usr/lib/jvm/java-1.8.0-openjdk" + +# Create stable symlinks (Rocky 8 installs to versioned directories) +RUN ln -sfn /usr/lib/jvm/java-1.8.0-openjdk /usr/lib/jvm/java-8 && \ + ln -sfn $(rpm -ql java-21-openjdk-devel | grep '/bin/javac$' | sed 's|/bin/javac||') /usr/lib/jvm/java-21 + +ENV JAVA_HOME="/usr/lib/jvm/java-8" ENV PATH="${JAVA_HOME}/bin:${PATH}" +# Entrypoint script to switch JDK at runtime based on AURON_JAVA_VERSION env var +RUN echo '#!/bin/bash' > /usr/local/bin/entrypoint.sh && \ + echo 'source ~/.bashrc 2>/dev/null || true' >> /usr/local/bin/entrypoint.sh && \ + echo 'if [ "$AURON_JAVA_VERSION" = "21" ]; then' >> /usr/local/bin/entrypoint.sh && \ + echo ' export JAVA_HOME="/usr/lib/jvm/java-21"' >> /usr/local/bin/entrypoint.sh && \ + echo 'else' >> /usr/local/bin/entrypoint.sh && \ + echo ' export JAVA_HOME="/usr/lib/jvm/java-8"' >> /usr/local/bin/entrypoint.sh && \ + echo 'fi' >> /usr/local/bin/entrypoint.sh && \ + echo 'export PATH="$JAVA_HOME/bin:$PATH"' >> /usr/local/bin/entrypoint.sh && \ + echo 'exec "$@"' >> /usr/local/bin/entrypoint.sh && \ + chmod +x /usr/local/bin/entrypoint.sh + +ENTRYPOINT ["/usr/local/bin/entrypoint.sh"] + # --------------------------------------------------------------------- # Set working directory # --------------------------------------------------------------------- @@ -79,4 +99,4 @@ WORKDIR /auron # --------------------------------------------------------------------- # Default command # --------------------------------------------------------------------- -CMD ["/bin/bash"] \ No newline at end of file +CMD ["/bin/bash"]