Skip to content

Commit 3a498c4

Browse files
hsiang-chsiang-c
authored
fix: [iceberg] Switch to OSS Spark and run Iceberg Spark tests in parallel (#1987)
1 parent c777fa8 commit 3a498c4

File tree

5 files changed

+1728
-282
lines changed

5 files changed

+1728
-282
lines changed

‎.github/actions/setup-iceberg-builder/action.yaml‎

Lines changed: 0 additions & 11 deletions
Original file line numberDiff line numberDiff line change
@@ -21,12 +21,6 @@ inputs:
2121
iceberg-version:
2222
description: 'The Apache Iceberg version (e.g., 1.8.1) to build'
2323
required: true
24-
scala-version:
25-
description: 'The Scala short version (e.g., 2.13) to build'
26-
required: true
27-
spark-short-version:
28-
description: 'The Apache Spark short version (e.g., 3.5) to build'
29-
required: true
3024
runs:
3125
using: "composite"
3226
steps:
@@ -43,8 +37,3 @@ runs:
4337
run: |
4438
cd apache-iceberg
4539
git apply ../dev/diffs/iceberg/${{inputs.iceberg-version}}.diff
46-
47-
- name: Build Comet
48-
shell: bash
49-
run: |
50-
PROFILES="-Pspark-${{inputs.spark-short-version}} -Pscala-${{inputs.scala-version}}" make release

‎.github/actions/setup-spark-local-jar/action.yaml‎

Lines changed: 0 additions & 48 deletions
This file was deleted.

‎.github/workflows/iceberg_spark_test.yml‎

Lines changed: 85 additions & 13 deletions
Original file line numberDiff line numberDiff line change
@@ -40,7 +40,7 @@ env:
4040
RUST_VERSION: stable
4141

4242
jobs:
43-
iceberg-spark-sql:
43+
iceberg-spark:
4444
if: contains(github.event.pull_request.title, '[iceberg]')
4545
strategy:
4646
matrix:
@@ -50,7 +50,7 @@ jobs:
5050
spark-version: [{short: '3.5', full: '3.5.6'}]
5151
scala-version: ['2.13']
5252
fail-fast: false
53-
name: iceberg-spark-sql/${{ matrix.os }}/iceberg-${{ matrix.iceberg-version.full }}/spark-${{ matrix.spark-version.full }}/scala-${{ matrix.scala-version }}/java-${{ matrix.java-version }}
53+
name: iceberg-spark/${{ matrix.os }}/iceberg-${{ matrix.iceberg-version.full }}/spark-${{ matrix.spark-version.full }}/scala-${{ matrix.scala-version }}/java-${{ matrix.java-version }}
5454
runs-on: ${{ matrix.os }}
5555
container:
5656
image: amd64/rust
@@ -63,24 +63,96 @@ jobs:
6363
with:
6464
rust-version: ${{env.RUST_VERSION}}
6565
jdk-version: ${{ matrix.java-version }}
66+
- name: Build Comet
67+
shell: bash
68+
run: |
69+
PROFILES="-Pspark-${{matrix.spark-version.short}} -Pscala-${{matrix.scala-version}}" make release
6670
- name: Setup Iceberg
6771
uses: ./.github/actions/setup-iceberg-builder
6872
with:
6973
iceberg-version: ${{ matrix.iceberg-version.full }}
70-
scala-version: ${{ matrix.scala-version }}
71-
spark-short-version: ${{ matrix.spark-version.short }}
72-
- name: Build local Spark jar with comet patch
73-
uses: ./.github/actions/setup-spark-local-jar
74-
with:
75-
spark-short-version: ${{ matrix.spark-version.short }}
76-
spark-version: ${{ matrix.spark-version.full }}
77-
scala-version: ${{ matrix.scala-version }}
7874
- name: Run Iceberg Spark tests
7975
run: |
8076
cd apache-iceberg
8177
rm -rf /root/.m2/repository/org/apache/parquet # somehow parquet cache requires cleanups
8278
ENABLE_COMET=true ./gradlew -DsparkVersions=${{ matrix.spark-version.short }} -DscalaVersion=${{ matrix.scala-version }} -DflinkVersions= -DkafkaVersions= \
83-
:iceberg-spark:iceberg-spark-${{ matrix.spark-version.short }}_${{ matrix.scala-version }}:check \
84-
:iceberg-spark:iceberg-spark-extensions-${{ matrix.spark-version.short }}_${{ matrix.scala-version }}:check \
85-
:iceberg-spark:iceberg-spark-runtime-${{ matrix.spark-version.short }}_${{ matrix.scala-version }}:check \
79+
:iceberg-spark:iceberg-spark-${{ matrix.spark-version.short }}_${{ matrix.scala-version }}:test \
8680
-Pquick=true -x javadoc
81+
82+
iceberg-spark-extensions:
83+
if: contains(github.event.pull_request.title, '[iceberg]')
84+
strategy:
85+
matrix:
86+
os: [ubuntu-24.04]
87+
java-version: [11, 17]
88+
iceberg-version: [{short: '1.8', full: '1.8.1'}]
89+
spark-version: [{short: '3.5', full: '3.5.6'}]
90+
scala-version: ['2.13']
91+
fail-fast: false
92+
name: iceberg-spark-extensions/${{ matrix.os }}/iceberg-${{ matrix.iceberg-version.full }}/spark-${{ matrix.spark-version.full }}/scala-${{ matrix.scala-version }}/java-${{ matrix.java-version }}
93+
runs-on: ${{ matrix.os }}
94+
container:
95+
image: amd64/rust
96+
env:
97+
SPARK_LOCAL_IP: localhost
98+
steps:
99+
- uses: actions/checkout@v4
100+
- name: Setup Rust & Java toolchain
101+
uses: ./.github/actions/setup-builder
102+
with:
103+
rust-version: ${{env.RUST_VERSION}}
104+
jdk-version: ${{ matrix.java-version }}
105+
- name: Build Comet
106+
shell: bash
107+
run: |
108+
PROFILES="-Pspark-${{matrix.spark-version.short}} -Pscala-${{matrix.scala-version}}" make release
109+
- name: Setup Iceberg
110+
uses: ./.github/actions/setup-iceberg-builder
111+
with:
112+
iceberg-version: ${{ matrix.iceberg-version.full }}
113+
- name: Run Iceberg Spark extensions tests
114+
run: |
115+
cd apache-iceberg
116+
rm -rf /root/.m2/repository/org/apache/parquet # somehow parquet cache requires cleanups
117+
ENABLE_COMET=true ./gradlew -DsparkVersions=${{ matrix.spark-version.short }} -DscalaVersion=${{ matrix.scala-version }} -DflinkVersions= -DkafkaVersions= \
118+
:iceberg-spark:iceberg-spark-extensions-${{ matrix.spark-version.short }}_${{ matrix.scala-version }}:test \
119+
-Pquick=true -x javadoc
120+
121+
iceberg-spark-runtime:
122+
if: contains(github.event.pull_request.title, '[iceberg]')
123+
strategy:
124+
matrix:
125+
os: [ubuntu-24.04]
126+
java-version: [11, 17]
127+
iceberg-version: [{short: '1.8', full: '1.8.1'}]
128+
spark-version: [{short: '3.5', full: '3.5.6'}]
129+
scala-version: ['2.13']
130+
fail-fast: false
131+
name: iceberg-spark-runtime/${{ matrix.os }}/iceberg-${{ matrix.iceberg-version.full }}/spark-${{ matrix.spark-version.full }}/scala-${{ matrix.scala-version }}/java-${{ matrix.java-version }}
132+
runs-on: ${{ matrix.os }}
133+
container:
134+
image: amd64/rust
135+
env:
136+
SPARK_LOCAL_IP: localhost
137+
steps:
138+
- uses: actions/checkout@v4
139+
- name: Setup Rust & Java toolchain
140+
uses: ./.github/actions/setup-builder
141+
with:
142+
rust-version: ${{env.RUST_VERSION}}
143+
jdk-version: ${{ matrix.java-version }}
144+
- name: Build Comet
145+
shell: bash
146+
run: |
147+
PROFILES="-Pspark-${{matrix.spark-version.short}} -Pscala-${{matrix.scala-version}}" make release
148+
- name: Setup Iceberg
149+
uses: ./.github/actions/setup-iceberg-builder
150+
with:
151+
iceberg-version: ${{ matrix.iceberg-version.full }}
152+
- name: Run Iceberg Spark runtime tests
153+
run: |
154+
cd apache-iceberg
155+
rm -rf /root/.m2/repository/org/apache/parquet # somehow parquet cache requires cleanups
156+
ENABLE_COMET=true ./gradlew -DsparkVersions=${{ matrix.spark-version.short }} -DscalaVersion=${{ matrix.scala-version }} -DflinkVersions= -DkafkaVersions= \
157+
:iceberg-spark:iceberg-spark-runtime-${{ matrix.spark-version.short }}_${{ matrix.scala-version }}:integrationTest \
158+
-Pquick=true -x javadoc

0 commit comments

Comments
 (0)