velox_backend_enhanced.yml

# Licensed to the Apache Software Foundation (ASF) under one or more
# contributor license agreements.  See the NOTICE file distributed with
# this work for additional information regarding copyright ownership.
# The ASF licenses this file to You under the Apache License, Version 2.0
# (the "License"); you may not use this file except in compliance with
# the License.  You may obtain a copy of the License at
#
#    http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.

name: Velox Backend (Enhanced)

on:
  pull_request:
    paths:
      - '.github/workflows/velox_backend_enhanced.yml'
      - 'pom.xml'
      - 'backends-velox/**'
      - 'gluten-uniffle/**'
      - 'gluten-celeborn/**'
      - 'gluten-ras/**'
      - 'gluten-core/**'
      - 'gluten-substrait/**'
      - 'gluten-arrow/**'
      - 'gluten-delta/**'
      - 'gluten-iceberg/**'
      - 'gluten-hudi/**'
      - 'gluten-ut/**'
      - 'package/**'
      - 'shims/**'
      - 'tools/gluten-it/**'
      - 'ep/build-velox/**'
      - 'cpp/**'
      - 'dev/**'
      - 'build/mvn'

env:
  ACTIONS_ALLOW_USE_UNSECURE_NODE_VERSION: true
  MVN_CMD: 'build/mvn -ntp'
  WGET_CMD: 'wget -nv'
  CCACHE_DIR: "${{ github.workspace }}/.ccache"
  # spark.sql.ansi.enabled defaults to false.
  SPARK_ANSI_SQL_MODE: false

concurrency:
  group: ${{ github.repository }}-${{ github.head_ref || github.sha }}-${{ github.workflow }}
  cancel-in-progress: true

jobs:
  build-native-lib-centos-7:
    runs-on: ubuntu-22.04
    steps:
      - uses: actions/checkout@v4
      - name: Get Ccache
        uses: actions/cache/restore@v4
        with:
          path: '${{ env.CCACHE_DIR }}'
          key: ccache-enhanced-centos7-release-default-${{github.sha}}
          restore-keys: |
            ccache-enhanced-centos7-release-default
      - name: Build Gluten native libraries
        run: |
          docker pull apache/gluten:vcpkg-centos-7
          docker run -v $GITHUB_WORKSPACE:/work -w /work apache/gluten:vcpkg-centos-7 bash -c "
            set -e
            yum install tzdata -y
            df -a
            cd /work
            export CCACHE_DIR=/work/.ccache
            mkdir -p /work/.ccache
            bash dev/ci-velox-buildstatic-centos-7-enhanced-features.sh
            ccache -s
            mkdir -p /work/.m2/repository/org/apache/arrow/
            cp -r /root/.m2/repository/org/apache/arrow/* /work/.m2/repository/org/apache/arrow/
          "

      - name: "Save ccache"
        uses: actions/cache/save@v4
        id: ccache
        with:
          path: '${{ env.CCACHE_DIR }}'
          key: ccache-enhanced-centos7-release-default-${{github.sha}}
      - uses: actions/upload-artifact@v4
        with:
          name: velox-native-lib-enhanced-centos-7-${{github.sha}}
          path: ./cpp/build/releases/
          if-no-files-found: error
      - uses: actions/upload-artifact@v4
        with:
          name: arrow-jars-enhanced-centos-7-${{github.sha}}
          path: .m2/repository/org/apache/arrow/
          if-no-files-found: error

  spark-test-spark34:
    needs: build-native-lib-centos-7
    runs-on: ubuntu-22.04
    container: apache/gluten:centos-8-jdk8
    steps:
      - uses: actions/checkout@v4
      - name: Download All Artifacts
        uses: actions/download-artifact@v4
        with:
          name: velox-native-lib-enhanced-centos-7-${{github.sha}}
          path: ./cpp/build/releases
      - name: Download Arrow Jars
        uses: actions/download-artifact@v4
        with:
          name: arrow-jars-enhanced-centos-7-${{github.sha}}
          path: /root/.m2/repository/org/apache/arrow/
      - name: Prepare spark.test.home for Spark 3.4.4 (other tests)
        run: |
          dnf module -y install python39 && \
          alternatives --set python3 /usr/bin/python3.9 && \
          pip3 install setuptools==77.0.3 && \
          pip3 install pyspark==3.4.4 cython && \
          pip3 install pandas==2.2.3 pyarrow==20.0.0
      - name: Build and Run unit test for Spark 3.4.4 (other tests)
        run: |
          cd $GITHUB_WORKSPACE/
          export SPARK_SCALA_VERSION=2.12
          yum install -y java-17-openjdk-devel
          export JAVA_HOME=/usr/lib/jvm/java-17-openjdk
          export PATH=$JAVA_HOME/bin:$PATH
          java -version
          export SPARK_HOME=/opt/shims/spark34/spark_home/
          ls -l $SPARK_HOME
          $MVN_CMD clean test -Pspark-3.4 -Pjava-17 -Pbackends-velox -Piceberg -Piceberg-test -Pdelta -Phudi \
          -DtagsToExclude=org.apache.spark.tags.ExtendedSQLTest,org.apache.gluten.tags.UDFTest,org.apache.gluten.tags.SkipTest \
          -DargLine="-Dspark.test.home=$SPARK_HOME"
      - name: Upload test report
        if: always()
        uses: actions/upload-artifact@v4
        with:
          name: ${{ github.job }}-report
          path: '**/surefire-reports/TEST-*.xml'
      - name: Upload unit tests log files
        if: ${{ !success() }}
        uses: actions/upload-artifact@v4
        with:
          name: ${{ github.job }}-test-log
          path: |
            **/target/*.log
            **/gluten-ut/**/hs_err_*.log
            **/gluten-ut/**/core.*
      - name: Upload golden files
        if: failure()
        uses: actions/upload-artifact@v4
        with:
          name: ${{ github.job }}-golden-files
          path: /tmp/tpch-approved-plan/**

  spark-test-spark35:
    needs: build-native-lib-centos-7
    runs-on: ubuntu-22.04
    container: apache/gluten:centos-8-jdk8
    steps:
      - uses: actions/checkout@v4
      - name: Download All Artifacts
        uses: actions/download-artifact@v4
        with:
          name: velox-native-lib-enhanced-centos-7-${{github.sha}}
          path: ./cpp/build/releases
      - name: Download Arrow Jars
        uses: actions/download-artifact@v4
        with:
          name: arrow-jars-enhanced-centos-7-${{github.sha}}
          path: /root/.m2/repository/org/apache/arrow/
      - name: Prepare spark.test.home for Spark 3.4.4 (other tests)
        run: |
          dnf module -y install python39 && \
          alternatives --set python3 /usr/bin/python3.9 && \
          pip3 install setuptools==77.0.3 && \
          pip3 install pyspark==3.4.4 cython && \
          pip3 install pandas==2.2.3 pyarrow==20.0.0
      - name: Build and Run unit test for Spark 3.5.5 (other tests)
        run: |
          cd $GITHUB_WORKSPACE/
          export SPARK_SCALA_VERSION=2.12
          yum install -y java-17-openjdk-devel
          export JAVA_HOME=/usr/lib/jvm/java-17-openjdk
          export PATH=$JAVA_HOME/bin:$PATH
          java -version
          export SPARK_HOME=/opt/shims/spark35/spark_home/
          ls -l $SPARK_HOME
          $MVN_CMD clean test -Pspark-3.5 -Pjava-17 -Pbackends-velox -Piceberg -Pdelta -Phudi \
          -DtagsToExclude=org.apache.spark.tags.ExtendedSQLTest,org.apache.gluten.tags.UDFTest,org.apache.gluten.tags.SkipTest \
          -DargLine="-Dspark.test.home=$SPARK_HOME"
      - name: Upload test report
        if: always()
        uses: actions/upload-artifact@v4
        with:
          name: ${{ github.job }}-report
          path: '**/surefire-reports/TEST-*.xml'
      - name: Upload unit tests log files
        if: ${{ !success() }}
        uses: actions/upload-artifact@v4
        with:
          name: ${{ github.job }}-test-log
          path: |
            **/target/*.log
            **/gluten-ut/**/hs_err_*.log
            **/gluten-ut/**/core.*
      - name: Upload golden files
        if: failure()
        uses: actions/upload-artifact@v4
        with:
          name: ${{ github.job }}-golden-files
          path: /tmp/tpch-approved-plan/**

  spark-test-spark35-slow:
    needs: build-native-lib-centos-7
    runs-on: ubuntu-22.04
    container: apache/gluten:centos-8-jdk8
    steps:
      - uses: actions/checkout@v4
      - name: Download All Artifacts
        uses: actions/download-artifact@v4
        with:
          name: velox-native-lib-enhanced-centos-7-${{github.sha}}
          path: ./cpp/build/releases
      - name: Download Arrow Jars
        uses: actions/download-artifact@v4
        with:
          name: arrow-jars-enhanced-centos-7-${{github.sha}}
          path: /root/.m2/repository/org/apache/arrow/
      - name: Prepare Spark Resources for Spark 3.5.5
        run: |
          rm -rf /opt/shims/spark35
          bash .github/workflows/util/install-resources.sh 3.5
      - name: Build and Run unit test for Spark 3.5.5 (slow tests)
        run: |
          cd $GITHUB_WORKSPACE/
          yum install -y java-17-openjdk-devel
          export JAVA_HOME=/usr/lib/jvm/java-17-openjdk
          export PATH=$JAVA_HOME/bin:$PATH
          java -version
          $MVN_CMD clean test -Pspark-3.5 -Pjava-17 -Pbackends-velox -Piceberg -Pdelta -Phudi -Ppaimon -Pspark-ut \
          -DargLine="-Dspark.test.home=/opt/shims/spark35/spark_home/" \
          -DtagsToInclude=org.apache.spark.tags.ExtendedSQLTest
      - name: Upload test report
        if: always()
        uses: actions/upload-artifact@v4
        with:
          name: ${{ github.job }}-report
          path: '**/surefire-reports/TEST-*.xml'
      - name: Upload unit tests log files
        if: ${{ !success() }}
        uses: actions/upload-artifact@v4
        with:
          name: ${{ github.job }}-test-log
          path: |
            **/target/*.log
            **/gluten-ut/**/hs_err_*.log
            **/gluten-ut/**/core.*

  spark-test-spark40:
    needs: build-native-lib-centos-7
    runs-on: ubuntu-22.04
    env:
      SPARK_TESTING: true
    container: apache/gluten:centos-8-jdk17
    steps:
      - uses: actions/checkout@v4
      - name: Download All Artifacts
        uses: actions/download-artifact@v4
        with:
          name: velox-native-lib-enhanced-centos-7-${{github.sha}}
          path: ./cpp/build/releases/
      - name: Download Arrow Jars
        uses: actions/download-artifact@v4
        with:
          name: arrow-jars-enhanced-centos-7-${{github.sha}}
          path: /root/.m2/repository/org/apache/arrow/
      - name: Prepare
        run: |
          dnf module -y install python39 && \
          alternatives --set python3 /usr/bin/python3.9 && \
          pip3 install setuptools==77.0.3 && \
          pip3 install pyspark==3.5.5 cython && \
          pip3 install pandas==2.2.3 pyarrow==20.0.0
      - name: Prepare Spark Resources for Spark 4.0.1 #TODO remove after image update
        run: |
          rm -rf /opt/shims/spark40
          bash .github/workflows/util/install-resources.sh 4.0
          mv /opt/shims/spark40/spark_home/assembly/target/scala-2.12 /opt/shims/spark40/spark_home/assembly/target/scala-2.13
      - name: Build and Run unit test for Spark 4.0.0 with scala-2.13 (other tests)
        run: |
          cd $GITHUB_WORKSPACE/
          export SPARK_SCALA_VERSION=2.13
          yum install -y java-17-openjdk-devel
          export JAVA_HOME=/usr/lib/jvm/java-17-openjdk
          export PATH=$JAVA_HOME/bin:$PATH
          java -version
          $MVN_CMD clean test -Pspark-4.0 -Pscala-2.13 -Pjava-17 -Pbackends-velox -Piceberg \
          -Pspark-ut -DargLine="-Dspark.test.home=/opt/shims/spark40/spark_home/" \
          -DtagsToExclude=org.apache.spark.tags.ExtendedSQLTest,org.apache.gluten.tags.UDFTest,org.apache.gluten.tags.EnhancedFeaturesTest,org.apache.gluten.tags.SkipTest
      - name: Upload test report
        if: always()
        uses: actions/upload-artifact@v4
        with:
          name: ${{ github.job }}-report
          path: '**/surefire-reports/TEST-*.xml'
      - name: Upload unit tests log files
        if: ${{ !success() }}
        uses: actions/upload-artifact@v4
        with:
          name: ${{ github.job }}-test-log
          path: |
            **/target/*.log
            **/gluten-ut/**/hs_err_*.log
            **/gluten-ut/**/core.*