Skip to content

Commit 5a36451

Browse files
andygroveandygrove
authored
chore: Add manually-triggered CI jobs for testing Spark SQL with native scans (#1624)
1 parent a723974 commit 5a36451

File tree

2 files changed

+142
-0
lines changed

2 files changed

+142
-0
lines changed

‎.github/workflows/spark_sql_test_native_datafusion.yml‎

Lines changed: 71 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,71 @@
1+
# Licensed to the Apache Software Foundation (ASF) under one
2+
# or more contributor license agreements. See the NOTICE file
3+
# distributed with this work for additional information
4+
# regarding copyright ownership. The ASF licenses this file
5+
# to you under the Apache License, Version 2.0 (the
6+
# "License"); you may not use this file except in compliance
7+
# with the License. You may obtain a copy of the License at
8+
#
9+
# http://www.apache.org/licenses/LICENSE-2.0
10+
#
11+
# Unless required by applicable law or agreed to in writing,
12+
# software distributed under the License is distributed on an
13+
# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
14+
# KIND, either express or implied. See the License for the
15+
# specific language governing permissions and limitations
16+
# under the License.
17+
18+
name: Spark SQL Tests (native_datafusion)
19+
20+
concurrency:
21+
group: ${{ github.repository }}-${{ github.head_ref || github.sha }}-${{ github.workflow }}
22+
cancel-in-progress: true
23+
24+
on:
25+
# manual trigger
26+
# https://docs.github.com/en/actions/managing-workflow-runs/manually-running-a-workflow
27+
workflow_dispatch:
28+
29+
env:
30+
RUST_VERSION: stable
31+
32+
jobs:
33+
spark-sql-catalyst-native-iceberg-compat:
34+
strategy:
35+
matrix:
36+
os: [ubuntu-24.04]
37+
java-version: [11]
38+
spark-version: [{short: '3.4', full: '3.4.3'}, {short: '3.5', full: '3.5.5'}]
39+
module:
40+
- {name: "catalyst", args1: "catalyst/test", args2: ""}
41+
- {name: "sql/core-1", args1: "", args2: sql/testOnly * -- -l org.apache.spark.tags.ExtendedSQLTest -l org.apache.spark.tags.SlowSQLTest}
42+
- {name: "sql/core-2", args1: "", args2: "sql/testOnly * -- -n org.apache.spark.tags.ExtendedSQLTest"}
43+
- {name: "sql/core-3", args1: "", args2: "sql/testOnly * -- -n org.apache.spark.tags.SlowSQLTest"}
44+
- {name: "sql/hive-1", args1: "", args2: "hive/testOnly * -- -l org.apache.spark.tags.ExtendedHiveTest -l org.apache.spark.tags.SlowHiveTest"}
45+
- {name: "sql/hive-2", args1: "", args2: "hive/testOnly * -- -n org.apache.spark.tags.ExtendedHiveTest"}
46+
- {name: "sql/hive-3", args1: "", args2: "hive/testOnly * -- -n org.apache.spark.tags.SlowHiveTest"}
47+
fail-fast: false
48+
name: spark-sql-native-iceberg-compat-${{ matrix.module.name }}/${{ matrix.os }}/spark-${{ matrix.spark-version.full }}/java-${{ matrix.java-version }}
49+
runs-on: ${{ matrix.os }}
50+
container:
51+
image: amd64/rust
52+
steps:
53+
- uses: actions/checkout@v4
54+
- name: Setup Rust & Java toolchain
55+
uses: ./.github/actions/setup-builder
56+
with:
57+
rust-version: ${{env.RUST_VERSION}}
58+
jdk-version: ${{ matrix.java-version }}
59+
- name: Setup Spark
60+
uses: ./.github/actions/setup-spark-builder
61+
with:
62+
spark-version: ${{ matrix.spark-version.full }}
63+
spark-short-version: ${{ matrix.spark-version.short }}
64+
- name: Run Spark tests
65+
run: |
66+
cd apache-spark
67+
rm -rf /root/.m2/repository/org/apache/parquet # somehow parquet cache requires cleanups
68+
ENABLE_COMET=true ENABLE_COMET_SHUFFLE=true COMET_PARQUET_SCAN_IMPL=native_datafusion build/sbt ${{ matrix.module.args1 }} "${{ matrix.module.args2 }}"
69+
env:
70+
LC_ALL: "C.UTF-8"
71+

‎.github/workflows/spark_sql_test_native_iceberg_compat.yml‎

Lines changed: 71 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,71 @@
1+
# Licensed to the Apache Software Foundation (ASF) under one
2+
# or more contributor license agreements. See the NOTICE file
3+
# distributed with this work for additional information
4+
# regarding copyright ownership. The ASF licenses this file
5+
# to you under the Apache License, Version 2.0 (the
6+
# "License"); you may not use this file except in compliance
7+
# with the License. You may obtain a copy of the License at
8+
#
9+
# http://www.apache.org/licenses/LICENSE-2.0
10+
#
11+
# Unless required by applicable law or agreed to in writing,
12+
# software distributed under the License is distributed on an
13+
# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
14+
# KIND, either express or implied. See the License for the
15+
# specific language governing permissions and limitations
16+
# under the License.
17+
18+
name: Spark SQL Tests (native_iceberg_compat)
19+
20+
concurrency:
21+
group: ${{ github.repository }}-${{ github.head_ref || github.sha }}-${{ github.workflow }}
22+
cancel-in-progress: true
23+
24+
on:
25+
# manual trigger
26+
# https://docs.github.com/en/actions/managing-workflow-runs/manually-running-a-workflow
27+
workflow_dispatch:
28+
29+
env:
30+
RUST_VERSION: stable
31+
32+
jobs:
33+
spark-sql-catalyst-native-iceberg-compat:
34+
strategy:
35+
matrix:
36+
os: [ubuntu-24.04]
37+
java-version: [11]
38+
spark-version: [{short: '3.4', full: '3.4.3'}, {short: '3.5', full: '3.5.5'}]
39+
module:
40+
- {name: "catalyst", args1: "catalyst/test", args2: ""}
41+
- {name: "sql/core-1", args1: "", args2: sql/testOnly * -- -l org.apache.spark.tags.ExtendedSQLTest -l org.apache.spark.tags.SlowSQLTest}
42+
- {name: "sql/core-2", args1: "", args2: "sql/testOnly * -- -n org.apache.spark.tags.ExtendedSQLTest"}
43+
- {name: "sql/core-3", args1: "", args2: "sql/testOnly * -- -n org.apache.spark.tags.SlowSQLTest"}
44+
- {name: "sql/hive-1", args1: "", args2: "hive/testOnly * -- -l org.apache.spark.tags.ExtendedHiveTest -l org.apache.spark.tags.SlowHiveTest"}
45+
- {name: "sql/hive-2", args1: "", args2: "hive/testOnly * -- -n org.apache.spark.tags.ExtendedHiveTest"}
46+
- {name: "sql/hive-3", args1: "", args2: "hive/testOnly * -- -n org.apache.spark.tags.SlowHiveTest"}
47+
fail-fast: false
48+
name: spark-sql-native-iceberg-compat-${{ matrix.module.name }}/${{ matrix.os }}/spark-${{ matrix.spark-version.full }}/java-${{ matrix.java-version }}
49+
runs-on: ${{ matrix.os }}
50+
container:
51+
image: amd64/rust
52+
steps:
53+
- uses: actions/checkout@v4
54+
- name: Setup Rust & Java toolchain
55+
uses: ./.github/actions/setup-builder
56+
with:
57+
rust-version: ${{env.RUST_VERSION}}
58+
jdk-version: ${{ matrix.java-version }}
59+
- name: Setup Spark
60+
uses: ./.github/actions/setup-spark-builder
61+
with:
62+
spark-version: ${{ matrix.spark-version.full }}
63+
spark-short-version: ${{ matrix.spark-version.short }}
64+
- name: Run Spark tests
65+
run: |
66+
cd apache-spark
67+
rm -rf /root/.m2/repository/org/apache/parquet # somehow parquet cache requires cleanups
68+
ENABLE_COMET=true ENABLE_COMET_SHUFFLE=true COMET_PARQUET_SCAN_IMPL=native_iceberg_compat build/sbt ${{ matrix.module.args1 }} "${{ matrix.module.args2 }}"
69+
env:
70+
LC_ALL: "C.UTF-8"
71+

0 commit comments

Comments
 (0)