diff --git a/.github/actions/java-test/action.yaml b/.github/actions/java-test/action.yaml index 53775da0df..9f74fe39f8 100644 --- a/.github/actions/java-test/action.yaml +++ b/.github/actions/java-test/action.yaml @@ -18,6 +18,9 @@ name: "Java Test" description: "Run Java tests" inputs: + suites: + description: 'suites' + default: '' maven_opts: description: 'Maven options passed to the mvn command' required: false @@ -54,7 +57,9 @@ runs: - name: Run tests shell: bash run: | - MAVEN_OPTS="-XX:+UnlockDiagnosticVMOptions -XX:+ShowMessageBoxOnError -XX:+HeapDumpOnOutOfMemoryError -XX:ErrorFile=./hs_err_pid%p.log" SPARK_HOME=`pwd` ./mvnw -B clean install ${{ inputs.maven_opts }} + MAVEN_SUITES="$(echo "${{ inputs.suites }}" | paste -sd, -)" + echo "Running with MAVEN_SUITES=$MAVEN_SUITES" + MAVEN_OPTS="-DwildcardSuites=$MAVEN_SUITES -XX:+UnlockDiagnosticVMOptions -XX:+ShowMessageBoxOnError -XX:+HeapDumpOnOutOfMemoryError -XX:ErrorFile=./hs_err_pid%p.log" SPARK_HOME=`pwd` ./mvnw -B clean install ${{ inputs.maven_opts }} ${{ env.MAVEN_SUITES }} - name: Upload crash logs if: failure() uses: actions/upload-artifact@v4 diff --git a/.github/workflows/benchmark-tpch.yml b/.github/workflows/benchmark-tpch.yml index 3d4fbb750a..9cb5bc6512 100644 --- a/.github/workflows/benchmark-tpch.yml +++ b/.github/workflows/benchmark-tpch.yml @@ -22,16 +22,16 @@ concurrency: cancel-in-progress: true on: - push: - paths-ignore: - - "doc/**" - - "docs/**" - - "**.md" - pull_request: - paths-ignore: - - "doc/**" - - "docs/**" - - "**.md" +# push: +# paths-ignore: +# - "doc/**" +# - "docs/**" +# - "**.md" +# pull_request: +# paths-ignore: +# - "doc/**" +# - "docs/**" +# - "**.md" # manual trigger # https://docs.github.com/en/actions/managing-workflow-runs/manually-running-a-workflow workflow_dispatch: diff --git a/.github/workflows/benchmark.yml b/.github/workflows/benchmark.yml index de1ad57739..712a71582a 100644 --- a/.github/workflows/benchmark.yml +++ b/.github/workflows/benchmark.yml @@ -22,16 +22,16 @@ concurrency: cancel-in-progress: true on: - push: - paths-ignore: - - "doc/**" - - "docs/**" - - "**.md" - pull_request: - paths-ignore: - - "doc/**" - - "docs/**" - - "**.md" +# push: +# paths-ignore: +# - "doc/**" +# - "docs/**" +# - "**.md" +# pull_request: +# paths-ignore: +# - "doc/**" +# - "docs/**" +# - "**.md" # manual trigger # https://docs.github.com/en/actions/managing-workflow-runs/manually-running-a-workflow workflow_dispatch: diff --git a/.github/workflows/miri.yml b/.github/workflows/miri.yml index 923598ee3b..aef91951dd 100644 --- a/.github/workflows/miri.yml +++ b/.github/workflows/miri.yml @@ -18,16 +18,16 @@ name: Run Miri Safety Checks on: - push: - paths-ignore: - - "doc/**" - - "docs/**" - - "**.md" - pull_request: - paths-ignore: - - "doc/**" - - "docs/**" - - "**.md" +# push: +# paths-ignore: +# - "doc/**" +# - "docs/**" +# - "**.md" +# pull_request: +# paths-ignore: +# - "doc/**" +# - "docs/**" +# - "**.md" # manual trigger # https://docs.github.com/en/actions/managing-workflow-runs/manually-running-a-workflow workflow_dispatch: diff --git a/.github/workflows/pr_build.yml b/.github/workflows/pr_build.yml deleted file mode 100644 index 308dd2d68c..0000000000 --- a/.github/workflows/pr_build.yml +++ /dev/null @@ -1,317 +0,0 @@ -# Licensed to the Apache Software Foundation (ASF) under one -# or more contributor license agreements. See the NOTICE file -# distributed with this work for additional information -# regarding copyright ownership. The ASF licenses this file -# to you under the Apache License, Version 2.0 (the -# "License"); you may not use this file except in compliance -# with the License. You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, -# software distributed under the License is distributed on an -# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY -# KIND, either express or implied. See the License for the -# specific language governing permissions and limitations -# under the License. - -name: PR Build - -concurrency: - group: ${{ github.repository }}-${{ github.head_ref || github.sha }}-${{ github.workflow }} - cancel-in-progress: true - -on: - push: - paths-ignore: - - "doc/**" - - "docs/**" - - "**.md" - pull_request: - paths-ignore: - - "doc/**" - - "docs/**" - - "**.md" - # manual trigger - # https://docs.github.com/en/actions/managing-workflow-runs/manually-running-a-workflow - workflow_dispatch: - -env: - RUST_VERSION: stable - -jobs: - - # run Comet test with full matrix of Java and Scala versions for the latest supported 3.5.x release - linux-test: - strategy: - matrix: - os: [ubuntu-latest] - java_version: [11, 17] - test-target: [rust, java] - spark-version: ['3.5'] - scala-version: ['2.12', '2.13'] - is_push_event: - - ${{ github.event_name == 'push' }} - exclude: # exclude java 11 for pull_request event - - java_version: 11 - is_push_event: false - fail-fast: false - name: ${{ matrix.os }}/java ${{ matrix.java_version }}-spark-${{matrix.spark-version}}-scala-${{matrix.scala-version}}/${{ matrix.test-target }} - runs-on: ${{ matrix.os }} - container: - image: amd64/rust - steps: - - uses: actions/checkout@v4 - - name: Setup Rust & Java toolchain - uses: ./.github/actions/setup-builder - with: - rust-version: ${{env.RUST_VERSION}} - jdk-version: ${{ matrix.java_version }} - - if: matrix.test-target == 'rust' - name: Rust test steps - uses: ./.github/actions/rust-test - - if: matrix.test-target == 'java' - name: Java test steps - uses: ./.github/actions/java-test - with: - maven_opts: -Pspark-${{ matrix.spark-version }},scala-${{ matrix.scala-version }} - # upload test reports only for java 17 - upload-test-reports: ${{ matrix.java_version == '17' }} - - # run Comet tests for older 3.5.x releases with one java and scala versions - linux-test-35-minor-versions: - strategy: - matrix: - os: [ubuntu-latest] - java_version: [17] - test-target: [java] - spark-version: ['3.5'] - spark-patch-version: ['3.5.2', '3.5.3', '3.5.4'] - scala-version: ['2.12'] - is_push_event: - - ${{ github.event_name == 'push' }} - fail-fast: false - name: ${{ matrix.os }}/java ${{ matrix.java_version }}-spark-${{matrix.spark-patch-version}}-scala-${{matrix.scala-version}}/${{ matrix.test-target }} - runs-on: ${{ matrix.os }} - container: - image: amd64/rust - steps: - - uses: actions/checkout@v4 - - name: Setup Rust & Java toolchain - uses: ./.github/actions/setup-builder - with: - rust-version: ${{env.RUST_VERSION}} - jdk-version: ${{ matrix.java_version }} - - if: matrix.test-target == 'rust' - name: Rust test steps - uses: ./.github/actions/rust-test - - if: matrix.test-target == 'java' - name: Java test steps - uses: ./.github/actions/java-test - with: - maven_opts: "-Pspark-${{ matrix.spark-version }},scala-${{ matrix.scala-version }} -Dspark.version=${{ matrix.spark-patch-version }}" - # upload test reports only for java 17 - upload-test-reports: ${{ matrix.java_version == '17' }} - - linux-test-native-datafusion-scan: - strategy: - matrix: - os: [ubuntu-latest] - scan_impl: ['native_datafusion', 'native_iceberg_compat'] - java_version: [17] - test-target: [rust, java] - spark-version: ['3.5'] - scala-version: ['2.12'] - is_push_event: - - ${{ github.event_name == 'push' }} - fail-fast: false - name: ${{ matrix.os }}/java ${{ matrix.java_version }}-spark-${{matrix.spark-version}}-scala-${{matrix.scala-version}}/${{ matrix.test-target }}-${{ matrix.scan_impl }} - runs-on: ${{ matrix.os }} - env: - COMET_PARQUET_SCAN_IMPL: ${{ matrix.scan_impl }} - container: - image: amd64/rust - steps: - - uses: actions/checkout@v4 - - name: Setup Rust & Java toolchain - uses: ./.github/actions/setup-builder - with: - rust-version: ${{env.RUST_VERSION}} - jdk-version: ${{ matrix.java_version }} - - if: matrix.test-target == 'rust' - name: Rust test steps - uses: ./.github/actions/rust-test - - if: matrix.test-target == 'java' - name: Java test steps - uses: ./.github/actions/java-test - with: - maven_opts: -Pspark-${{ matrix.spark-version }},scala-${{ matrix.scala-version }} - # upload test reports only for java 17 - upload-test-reports: ${{ matrix.java_version == '17' }} - - linux-test-with-old-spark: - strategy: - matrix: - os: [ubuntu-latest] - java_version: [11, 17] - test-target: [java] - spark-version: ['3.4'] - scala-version: ['2.12', '2.13'] - fail-fast: false - name: ${{ matrix.os }}/java ${{ matrix.java_version }}-spark-${{matrix.spark-version}}-scala-${{matrix.scala-version}}/${{ matrix.test-target }} - runs-on: ${{ matrix.os }} - container: - image: amd64/rust - steps: - - uses: actions/checkout@v4 - - name: Setup Rust & Java toolchain - uses: ./.github/actions/setup-builder - with: - rust-version: ${{env.RUST_VERSION}} - jdk-version: ${{ matrix.java_version }} - - name: Java test steps - uses: ./.github/actions/java-test - with: - maven_opts: -Pspark-${{ matrix.spark-version }},scala-${{ matrix.scala-version }} - - macos-test: - strategy: - matrix: - os: [macos-13] - java_version: [11, 17] - test-target: [rust, java] - spark-version: ['3.4', '3.5'] - scala-version: ['2.12', '2.13'] - fail-fast: false - if: github.event_name == 'push' - name: ${{ matrix.os }}/java ${{ matrix.java_version }}-spark-${{matrix.spark-version}}-scala-${{matrix.scala-version}}/${{ matrix.test-target }} - runs-on: ${{ matrix.os }} - steps: - - uses: actions/checkout@v4 - - name: Setup Rust & Java toolchain - uses: ./.github/actions/setup-macos-builder - with: - rust-version: ${{env.RUST_VERSION}} - jdk-version: ${{ matrix.java_version }} - - if: matrix.test-target == 'rust' - name: Rust test steps - uses: ./.github/actions/rust-test - - if: matrix.test-target == 'java' - name: Java test steps - uses: ./.github/actions/java-test - with: - maven_opts: -Pspark-${{ matrix.spark-version }},scala-${{ matrix.scala-version }} - - macos-aarch64-test: - strategy: - matrix: - java_version: [11, 17] - test-target: [rust, java] - spark-version: ['3.4', '3.5'] - scala-version: ['2.12', '2.13'] - is_push_event: - - ${{ github.event_name == 'push' }} - exclude: # exclude java 11 for pull_request event - - java_version: 11 - is_push_event: false - fail-fast: false - name: macos-14(Silicon)/java ${{ matrix.java_version }}-spark-${{matrix.spark-version}}-scala-${{matrix.scala-version}}/${{ matrix.test-target }} - runs-on: macos-14 - steps: - - uses: actions/checkout@v4 - - name: Setup Rust & Java toolchain - uses: ./.github/actions/setup-macos-builder - with: - rust-version: ${{env.RUST_VERSION}} - jdk-version: ${{ matrix.java_version }} - jdk-architecture: aarch64 - protoc-architecture: aarch_64 - - if: matrix.test-target == 'rust' - name: Rust test steps - uses: ./.github/actions/rust-test - - if: matrix.test-target == 'java' - name: Java test steps - uses: ./.github/actions/java-test - with: - maven_opts: -Pspark-${{ matrix.spark-version }},scala-${{ matrix.scala-version }} - - macos-test-with-spark4_0: - strategy: - matrix: - os: [macos-13] - java_version: [17] - test-target: [java] - spark-version: ['4.0'] - fail-fast: false - if: github.event_name == 'push' - name: ${{ matrix.os }}/java ${{ matrix.java_version }}-spark-${{matrix.spark-version}}/${{ matrix.test-target }} - runs-on: ${{ matrix.os }} - steps: - - uses: actions/checkout@v4 - - name: Setup Rust & Java toolchain - uses: ./.github/actions/setup-macos-builder - with: - rust-version: ${{env.RUST_VERSION}} - jdk-version: ${{ matrix.java_version }} - - name: Java test steps - uses: ./.github/actions/java-test - with: - maven_opts: -Pspark-${{ matrix.spark-version }} - upload-test-reports: true - - macos-aarch64-test-with-spark4_0: - strategy: - matrix: - java_version: [17] - test-target: [java] - spark-version: ['4.0'] - is_push_event: - - ${{ github.event_name == 'push' }} - exclude: # exclude java 11 for pull_request event - - java_version: 11 - is_push_event: false - fail-fast: false - name: macos-14(Silicon)/java ${{ matrix.java_version }}-spark-${{matrix.spark-version}}/${{ matrix.test-target }} - runs-on: macos-14 - steps: - - uses: actions/checkout@v4 - - name: Setup Rust & Java toolchain - uses: ./.github/actions/setup-macos-builder - with: - rust-version: ${{env.RUST_VERSION}} - jdk-version: ${{ matrix.java_version }} - jdk-architecture: aarch64 - protoc-architecture: aarch_64 - - name: Java test steps - uses: ./.github/actions/java-test - with: - maven_opts: -Pspark-${{ matrix.spark-version }} - upload-test-reports: true - - macos-aarch64-test-with-old-spark: - strategy: - matrix: - java_version: [17] - test-target: [java] - spark-version: ['3.4'] - scala-version: ['2.12', '2.13'] - exclude: - - java_version: 8 - fail-fast: false - name: macos-14(Silicon)/java ${{ matrix.java_version }}-spark-${{matrix.spark-version}}-scala-${{matrix.scala-version}}/${{ matrix.test-target }} - runs-on: macos-14 - steps: - - uses: actions/checkout@v4 - - name: Setup Rust & Java toolchain - uses: ./.github/actions/setup-macos-builder - with: - rust-version: ${{env.RUST_VERSION}} - jdk-version: ${{ matrix.java_version }} - jdk-architecture: aarch64 - protoc-architecture: aarch_64 - - if: matrix.test-target == 'java' - name: Java test steps - uses: ./.github/actions/java-test - with: - maven_opts: -Pspark-${{ matrix.spark-version }},scala-${{ matrix.scala-version }} - diff --git a/.github/workflows/pr_build_linux_spark4.yml b/.github/workflows/pr_build_linux_spark4.yml deleted file mode 100644 index dc0a96c8d8..0000000000 --- a/.github/workflows/pr_build_linux_spark4.yml +++ /dev/null @@ -1,72 +0,0 @@ -# Licensed to the Apache Software Foundation (ASF) under one -# or more contributor license agreements. See the NOTICE file -# distributed with this work for additional information -# regarding copyright ownership. The ASF licenses this file -# to you under the Apache License, Version 2.0 (the -# "License"); you may not use this file except in compliance -# with the License. You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, -# software distributed under the License is distributed on an -# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY -# KIND, either express or implied. See the License for the -# specific language governing permissions and limitations -# under the License. - -name: PR Build (Linux + Spark 4) - -concurrency: - group: ${{ github.repository }}-${{ github.head_ref || github.sha }}-${{ github.workflow }} - cancel-in-progress: true - -on: - -# temporarily disabled on PRs due to https://github.com/apache/datafusion-comet/issues/1786 -# push: -# paths-ignore: -# - "doc/**" -# - "docs/**" -# - "**.md" -# pull_request: -# paths-ignore: -# - "doc/**" -# - "docs/**" -# - "**.md" - # manual trigger - # https://docs.github.com/en/actions/managing-workflow-runs/manually-running-a-workflow - workflow_dispatch: - -env: - RUST_VERSION: stable - -jobs: - - linux-test-with-spark4_0: - strategy: - matrix: - os: [ubuntu-latest] - java_version: [17] - test-target: [java] - spark-version: ['4.0'] - is_push_event: - - ${{ github.event_name == 'push' }} - fail-fast: false - name: ${{ matrix.os }}/java ${{ matrix.java_version }}-spark-${{matrix.spark-version}}/${{ matrix.test-target }} - runs-on: ${{ matrix.os }} - container: - image: amd64/rust - steps: - - uses: actions/checkout@v4 - - name: Setup Rust & Java toolchain - uses: ./.github/actions/setup-builder - with: - rust-version: ${{env.RUST_VERSION}} - jdk-version: ${{ matrix.java_version }} - - name: Java test steps - uses: ./.github/actions/java-test - with: - maven_opts: -Pspark-${{ matrix.spark-version }} - upload-test-reports: true - diff --git a/.github/workflows/pr_build_new.yml b/.github/workflows/pr_build_new.yml new file mode 100644 index 0000000000..b151a58a13 --- /dev/null +++ b/.github/workflows/pr_build_new.yml @@ -0,0 +1,108 @@ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. + +name: PR Build + +concurrency: + group: ${{ github.repository }}-${{ github.head_ref || github.sha }}-${{ github.workflow }} + cancel-in-progress: true + +on: + push: + paths-ignore: + - "doc/**" + - "docs/**" + - "**.md" + pull_request: + paths-ignore: + - "doc/**" + - "docs/**" + - "**.md" + # manual trigger + # https://docs.github.com/en/actions/managing-workflow-runs/manually-running-a-workflow + workflow_dispatch: + +env: + RUST_VERSION: stable + +jobs: + + linux-test-with-spark4_0: + strategy: + matrix: + os: [ubuntu-latest] + java_version: [17] + test-target: [java] + spark-version: ['4.0'] + suites: + - name: "fuzz" + value: | + org.apache.comet.CometFuzzTestSuite + org.apache.comet.DataGeneratorSuite + - name: "shuffle" + value: | + org.apache.comet.exec.CometShuffleSuite + org.apache.comet.exec.CometShuffle4_0Suite + org.apache.comet.exec.CometNativeShuffleSuite + org.apache.comet.exec.CometShuffleEncryptionSuite + org.apache.comet.exec.CometShuffleManagerSuite + org.apache.comet.exec.CometAsyncShuffleSuite + org.apache.comet.exec.DisableAQECometShuffleSuite + org.apache.comet.exec.DisableAQECometAsyncShuffleSuite + - name: "parquet" + value: | + org.apache.comet.parquet.ParquetReadV1Suite + org.apache.comet.parquet.ParquetReadV2Suite + org.apache.spark.sql.comet.ParquetDatetimeRebaseV1Suite + org.apache.spark.sql.comet.ParquetDatetimeRebaseV2Suite + org.apache.spark.sql.comet.ParquetEncryptionITCase + org.apache.comet.exec.CometNativeReaderSuite + - name: "exec" + value: | + org.apache.comet.exec.CometExecSuite + org.apache.comet.exec.CometExec3_4PlusSuite + org.apache.comet.exec.CometJoinSuite + org.apache.comet.exec.CometAggregateSuite + org.apache.comet.CometArrayExpressionSuite + org.apache.comet.CometCastSuite + org.apache.comet.CometExpressionSuite + org.apache.comet.CometExpressionCoverageSuite + org.apache.comet.CometNativeSuite + org.apache.comet.CometSparkSessionExtensionsSuite + org.apache.spark.CometPluginsSuite + org.apache.spark.sql.CometTPCDSQuerySuite + org.apache.spark.sql.CometTPCHQuerySuite + org.apache.spark.sql.comet.CometPlanStabilitySuite + org.apache.spark.sql.comet.CometTaskMetricsSuite + fail-fast: false + name: ${{ matrix.os }}/java ${{ matrix.java_version }}-spark-${{matrix.spark-version}}/${{ matrix.test-target }} [${{ matrix.suites.name }}] + runs-on: ${{ matrix.os }} + container: + image: amd64/rust + steps: + - uses: actions/checkout@v4 + - name: Setup Rust & Java toolchain + uses: ./.github/actions/setup-builder + with: + rust-version: ${{env.RUST_VERSION}} + jdk-version: ${{ matrix.java_version }} + - name: Java test steps + uses: ./.github/actions/java-test + with: + suites: ${{ matrix.suites.value }} + maven_opts: "-Pspark-${{ matrix.spark-version }}" + upload-test-reports: true diff --git a/.github/workflows/spark_sql_test.yml b/.github/workflows/spark_sql_test.yml index 58defe6608..cb917062bb 100644 --- a/.github/workflows/spark_sql_test.yml +++ b/.github/workflows/spark_sql_test.yml @@ -22,16 +22,16 @@ concurrency: cancel-in-progress: true on: - push: - paths-ignore: - - "doc/**" - - "docs/**" - - "**.md" - pull_request: - paths-ignore: - - "doc/**" - - "docs/**" - - "**.md" +# push: +# paths-ignore: +# - "doc/**" +# - "docs/**" +# - "**.md" +# pull_request: +# paths-ignore: +# - "doc/**" +# - "docs/**" +# - "**.md" # manual trigger # https://docs.github.com/en/actions/managing-workflow-runs/manually-running-a-workflow workflow_dispatch: diff --git a/.github/workflows/spark_sql_test_ansi.yml b/.github/workflows/spark_sql_test_ansi.yml index 170c10644b..466d1f4eef 100644 --- a/.github/workflows/spark_sql_test_ansi.yml +++ b/.github/workflows/spark_sql_test_ansi.yml @@ -22,14 +22,14 @@ concurrency: cancel-in-progress: true on: - push: - paths-ignore: - - "docs/**" - - "**.md" - pull_request: - paths-ignore: - - "docs/**" - - "**.md" +# push: +# paths-ignore: +# - "docs/**" +# - "**.md" +# pull_request: +# paths-ignore: +# - "docs/**" +# - "**.md" # manual trigger # https://docs.github.com/en/actions/managing-workflow-runs/manually-running-a-workflow workflow_dispatch: