Skip to content

Commit 4b8782a

Browse files
hsiang-chsiang-c
authored
chore: Comet + Iceberg (1.8.1) CI (#1715)
## Which issue does this PR close? Closes #. #1685 ## Rationale for this change Run Iceberg Spark' tests as part of Comet CI ## What changes are included in this PR? - Added a new action `setup-spark-local-jar/action.yaml` that locally publishes Spark w/ Comet jars - Produced a git diff for Iceberg based on this [instruction](https://github.com/apache/datafusion-comet/blob/main/docs/source/user-guide/iceberg.md) - Make Iceberg Spark module depend on the local Spark SNAPSHOT jar - Target Iceberg version 1.8.1, Spark 3.5.6 and Scala 2.13 - Run Iceberg Spark's tests, based on Iceberg's GitHub workflow: https://github.com/apache/iceberg/blob/main/.github/workflows/spark-ci.yml
1 parent dad564f commit 4b8782a

File tree

5 files changed

+463
-0
lines changed

5 files changed

+463
-0
lines changed

‎.github/actions/setup-iceberg-builder/action.yaml‎

Lines changed: 50 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,50 @@
1+
# Licensed to the Apache Software Foundation (ASF) under one
2+
# or more contributor license agreements. See the NOTICE file
3+
# distributed with this work for additional information
4+
# regarding copyright ownership. The ASF licenses this file
5+
# to you under the Apache License, Version 2.0 (the
6+
# "License"); you may not use this file except in compliance
7+
# with the License. You may obtain a copy of the License at
8+
#
9+
# http://www.apache.org/licenses/LICENSE-2.0
10+
#
11+
# Unless required by applicable law or agreed to in writing,
12+
# software distributed under the License is distributed on an
13+
# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
14+
# KIND, either express or implied. See the License for the
15+
# specific language governing permissions and limitations
16+
# under the License.
17+
18+
name: Setup Iceberg Builder
19+
description: 'Setup Apache Iceberg to run Spark SQL tests'
20+
inputs:
21+
iceberg-version:
22+
description: 'The Apache Iceberg version (e.g., 1.8.1) to build'
23+
required: true
24+
scala-version:
25+
description: 'The Scala short version (e.g., 2.13) to build'
26+
required: true
27+
spark-short-version:
28+
description: 'The Apache Spark short version (e.g., 3.5) to build'
29+
required: true
30+
runs:
31+
using: "composite"
32+
steps:
33+
- name: Clone Iceberg repo
34+
uses: actions/checkout@v4
35+
with:
36+
repository: apache/iceberg
37+
path: apache-iceberg
38+
ref: apache-iceberg-${{inputs.iceberg-version}}
39+
fetch-depth: 1
40+
41+
- name: Setup Iceberg for Comet
42+
shell: bash
43+
run: |
44+
cd apache-iceberg
45+
git apply ../dev/diffs/iceberg/${{inputs.iceberg-version}}.diff
46+
47+
- name: Build Comet
48+
shell: bash
49+
run: |
50+
PROFILES="-Pspark-${{inputs.spark-short-version}} -Pscala-${{inputs.scala-version}}" make release

‎.github/actions/setup-spark-local-jar/action.yaml‎

Lines changed: 48 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,48 @@
1+
# Licensed to the Apache Software Foundation (ASF) under one
2+
# or more contributor license agreements. See the NOTICE file
3+
# distributed with this work for additional information
4+
# regarding copyright ownership. The ASF licenses this file
5+
# to you under the Apache License, Version 2.0 (the
6+
# "License"); you may not use this file except in compliance
7+
# with the License. You may obtain a copy of the License at
8+
#
9+
# http://www.apache.org/licenses/LICENSE-2.0
10+
#
11+
# Unless required by applicable law or agreed to in writing,
12+
# software distributed under the License is distributed on an
13+
# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
14+
# KIND, either express or implied. See the License for the
15+
# specific language governing permissions and limitations
16+
# under the License.
17+
18+
name: Setup Spark Local Jar
19+
description: 'Build comet-patched Apache Spark for Iceberg Spark tests'
20+
inputs:
21+
spark-short-version:
22+
description: 'The Apache Spark short version (e.g., 3.5) to build'
23+
required: true
24+
spark-version:
25+
description: 'The Apache Spark version (e.g., 3.5.6) to build'
26+
required: true
27+
scala-version:
28+
description: 'The Scala short version (e.g., 2.13) to build'
29+
required: true
30+
runs:
31+
using: "composite"
32+
steps:
33+
- name: Clone Spark repo
34+
uses: actions/checkout@v4
35+
with:
36+
repository: apache/spark
37+
path: apache-spark
38+
ref: v${{inputs.spark-version}}
39+
fetch-depth: 1
40+
41+
- name: Publish local Spark snapshot w/ Comet
42+
shell: bash
43+
run: |
44+
cd apache-spark
45+
git apply ../dev/diffs/${{inputs.spark-version}}.diff
46+
./dev/change-scala-version.sh ${{inputs.scala-version}}
47+
./build/mvn versions:set -DnewVersion=${{inputs.spark-version}}-SNAPSHOT
48+
./build/mvn -Pscala-${{inputs.scala-version}} -Phive -Phive-thriftserver -DskipTests -Denforcer.skip=true clean install

‎.github/workflows/iceberg_spark_test.yml‎

Lines changed: 85 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,85 @@
1+
# Licensed to the Apache Software Foundation (ASF) under one
2+
# or more contributor license agreements. See the NOTICE file
3+
# distributed with this work for additional information
4+
# regarding copyright ownership. The ASF licenses this file
5+
# to you under the Apache License, Version 2.0 (the
6+
# "License"); you may not use this file except in compliance
7+
# with the License. You may obtain a copy of the License at
8+
#
9+
# http://www.apache.org/licenses/LICENSE-2.0
10+
#
11+
# Unless required by applicable law or agreed to in writing,
12+
# software distributed under the License is distributed on an
13+
# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
14+
# KIND, either express or implied. See the License for the
15+
# specific language governing permissions and limitations
16+
# under the License.
17+
18+
name: Iceberg Spark SQL Tests
19+
20+
concurrency:
21+
group: ${{ github.repository }}-${{ github.head_ref || github.sha }}-${{ github.workflow }}
22+
cancel-in-progress: true
23+
24+
on:
25+
push:
26+
paths-ignore:
27+
- "doc/**"
28+
- "docs/**"
29+
- "**.md"
30+
pull_request:
31+
paths-ignore:
32+
- "doc/**"
33+
- "docs/**"
34+
- "**.md"
35+
# manual trigger
36+
# https://docs.github.com/en/actions/managing-workflow-runs/manually-running-a-workflow
37+
workflow_dispatch:
38+
39+
env:
40+
RUST_VERSION: stable
41+
42+
jobs:
43+
iceberg-spark-sql:
44+
strategy:
45+
matrix:
46+
os: [ubuntu-24.04]
47+
java-version: [11, 17]
48+
iceberg-version: [{short: '1.8', full: '1.8.1'}]
49+
spark-version: [{short: '3.5', full: '3.5.6'}]
50+
scala-version: ['2.13']
51+
fail-fast: false
52+
name: iceberg-spark-sql/${{ matrix.os }}/iceberg-${{ matrix.iceberg-version.full }}/spark-${{ matrix.spark-version.full }}/scala-${{ matrix.scala-version }}/java-${{ matrix.java-version }}
53+
runs-on: ${{ matrix.os }}
54+
container:
55+
image: amd64/rust
56+
env:
57+
SPARK_LOCAL_IP: localhost
58+
steps:
59+
- uses: actions/checkout@v4
60+
- name: Setup Rust & Java toolchain
61+
uses: ./.github/actions/setup-builder
62+
with:
63+
rust-version: ${{env.RUST_VERSION}}
64+
jdk-version: ${{ matrix.java-version }}
65+
- name: Setup Iceberg
66+
uses: ./.github/actions/setup-iceberg-builder
67+
with:
68+
iceberg-version: ${{ matrix.iceberg-version.full }}
69+
scala-version: ${{ matrix.scala-version }}
70+
spark-short-version: ${{ matrix.spark-version.short }}
71+
- name: Build local Spark jar with comet patch
72+
uses: ./.github/actions/setup-spark-local-jar
73+
with:
74+
spark-short-version: ${{ matrix.spark-version.short }}
75+
spark-version: ${{ matrix.spark-version.full }}
76+
scala-version: ${{ matrix.scala-version }}
77+
- name: Run Iceberg Spark tests
78+
run: |
79+
cd apache-iceberg
80+
rm -rf /root/.m2/repository/org/apache/parquet # somehow parquet cache requires cleanups
81+
ENABLE_COMET=true ./gradlew -DsparkVersions=${{ matrix.spark-version.short }} -DscalaVersion=${{ matrix.scala-version }} -DflinkVersions= -DkafkaVersions= \
82+
:iceberg-spark:iceberg-spark-${{ matrix.spark-version.short }}_${{ matrix.scala-version }}:check \
83+
:iceberg-spark:iceberg-spark-extensions-${{ matrix.spark-version.short }}_${{ matrix.scala-version }}:check \
84+
:iceberg-spark:iceberg-spark-runtime-${{ matrix.spark-version.short }}_${{ matrix.scala-version }}:check \
85+
-Pquick=true -x javadoc

0 commit comments

Comments
 (0)