Skip to content

Commit e255a07

Browse files
hsiang-chsiang-c
committed
chore: Comet + Iceberg (1.8.1) CI
1 parent a93d972 commit e255a07

File tree

5 files changed

+485
-0
lines changed

5 files changed

+485
-0
lines changed

‎.github/actions/setup-iceberg-builder/action.yaml‎

Lines changed: 63 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,63 @@
1+
# Licensed to the Apache Software Foundation (ASF) under one
2+
# or more contributor license agreements. See the NOTICE file
3+
# distributed with this work for additional information
4+
# regarding copyright ownership. The ASF licenses this file
5+
# to you under the Apache License, Version 2.0 (the
6+
# "License"); you may not use this file except in compliance
7+
# with the License. You may obtain a copy of the License at
8+
#
9+
# http://www.apache.org/licenses/LICENSE-2.0
10+
#
11+
# Unless required by applicable law or agreed to in writing,
12+
# software distributed under the License is distributed on an
13+
# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
14+
# KIND, either express or implied. See the License for the
15+
# specific language governing permissions and limitations
16+
# under the License.
17+
18+
name: Setup Iceberg Builder
19+
description: 'Setup Apache Iceberg to run Spark SQL tests'
20+
inputs:
21+
iceberg-short-version:
22+
description: 'The Apache Iceberg short version (e.g., 1.8) to build'
23+
required: true
24+
iceberg-version:
25+
description: 'The Apache Iceberg version (e.g., 1.8.1) to build'
26+
required: true
27+
scala-version:
28+
description: 'The Scala short version (e.g., 2.13) to build'
29+
required: true
30+
spark-short-version:
31+
description: 'The Apache Spark short version (e.g., 3.5) to build'
32+
required: true
33+
runs:
34+
using: "composite"
35+
steps:
36+
- name: Clone Iceberg repo
37+
uses: actions/checkout@v4
38+
with:
39+
repository: apache/iceberg
40+
path: apache-iceberg
41+
ref: apache-iceberg-${{inputs.iceberg-version}}
42+
fetch-depth: 1
43+
44+
- name: Setup Iceberg for Comet
45+
shell: bash
46+
run: |
47+
cd apache-iceberg
48+
git apply ../dev/diffs/iceberg/${{inputs.iceberg-version}}.diff
49+
50+
- name: Cache Maven dependencies
51+
uses: actions/cache@v4
52+
with:
53+
path: |
54+
~/.m2/repository
55+
/root/.m2/repository
56+
key: ${{ runner.os }}-iceberg-spark-sql-${{ hashFiles('spark/**/pom.xml', 'common/**/pom.xml') }}
57+
restore-keys: |
58+
${{ runner.os }}-iceberg-spark-sql-
59+
60+
- name: Build Comet
61+
shell: bash
62+
run: |
63+
PROFILES="-Pspark-${{inputs.spark-short-version}} -Pscala-${{inputs.scala-version}}" make release

‎.github/workflows/iceberg_spark_test.yml‎

Lines changed: 81 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,81 @@
1+
# Licensed to the Apache Software Foundation (ASF) under one
2+
# or more contributor license agreements. See the NOTICE file
3+
# distributed with this work for additional information
4+
# regarding copyright ownership. The ASF licenses this file
5+
# to you under the Apache License, Version 2.0 (the
6+
# "License"); you may not use this file except in compliance
7+
# with the License. You may obtain a copy of the License at
8+
#
9+
# http://www.apache.org/licenses/LICENSE-2.0
10+
#
11+
# Unless required by applicable law or agreed to in writing,
12+
# software distributed under the License is distributed on an
13+
# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
14+
# KIND, either express or implied. See the License for the
15+
# specific language governing permissions and limitations
16+
# under the License.
17+
18+
name: Iceberg Spark SQL Tests
19+
20+
concurrency:
21+
group: ${{ github.repository }}-${{ github.head_ref || github.sha }}-${{ github.workflow }}
22+
cancel-in-progress: true
23+
24+
on:
25+
push:
26+
paths-ignore:
27+
- "doc/**"
28+
- "docs/**"
29+
- "**.md"
30+
pull_request:
31+
paths-ignore:
32+
- "doc/**"
33+
- "docs/**"
34+
- "**.md"
35+
# manual trigger
36+
# https://docs.github.com/en/actions/managing-workflow-runs/manually-running-a-workflow
37+
workflow_dispatch:
38+
39+
env:
40+
RUST_VERSION: stable
41+
42+
jobs:
43+
iceberg-spark-sql:
44+
strategy:
45+
matrix:
46+
os: [ubuntu-24.04]
47+
java-version: [11, 17]
48+
iceberg-version: [{short: '1.8', full: '1.8.1'}]
49+
spark-version: [{short: '3.5', full: '3.5.4'}]
50+
scala-version: [ '2.12', '2.13']
51+
fail-fast: false
52+
name: iceberg-spark-sql/${{ matrix.os }}/iceberg-${{ matrix.iceberg-version.full }}/spark-${{ matrix.spark-version.full }}/scala-${{ matrix.scala-version }}/java-${{ matrix.java-version }}
53+
runs-on: ${{ matrix.os }}
54+
container:
55+
image: amd64/rust
56+
env:
57+
SPARK_LOCAL_IP: localhost
58+
steps:
59+
- uses: actions/checkout@v4
60+
- name: Setup Rust & Java toolchain
61+
uses: ./.github/actions/setup-builder
62+
with:
63+
rust-version: ${{env.RUST_VERSION}}
64+
jdk-version: ${{ matrix.java-version }}
65+
- name: Setup Iceberg
66+
uses: ./.github/actions/setup-iceberg-builder
67+
with:
68+
iceberg-version: ${{ matrix.iceberg-version.full }}
69+
iceberg-short-version: ${{ matrix.iceberg-version.short }}
70+
scala-version: ${{ matrix.scala-version }}
71+
spark-short-version: ${{ matrix.spark-version.short }}
72+
- run: echo -e "$(ip addr show eth0 | grep "inet\b" | awk '{print $2}' | cut -d/ -f1)\t$(hostname -f) $(hostname -s)" | sudo tee -a /etc/hosts
73+
- name: Run Iceberg Spark tests
74+
run: |
75+
cd apache-iceberg
76+
rm -rf /root/.m2/repository/org/apache/parquet # somehow parquet cache requires cleanups
77+
ENABLE_COMET=true ENABLE_COMET_SHUFFLE=true ../../gradlew -DsparkVersions=${{ matrix.spark-version.short }} -DscalaVersion=${{ matrix.scala-version }} -DflinkVersions= -DkafkaVersions= \
78+
:iceberg-spark:iceberg-spark-${{ matrix.spark-version.short }}_${{ matrix.scala-version }}:check \
79+
:iceberg-spark:iceberg-spark-extensions-${{ matrix.spark-version.short }}_${{ matrix.scala-version }}:check \
80+
:iceberg-spark:iceberg-spark-runtime-${{ matrix.spark-version.short }}_${{ matrix.scala-version }}:check \
81+
-Pquick=true -x javadoc

‎.github/workflows/iceberg_spark_test_native_datafusion.yml‎

Lines changed: 81 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,81 @@
1+
# Licensed to the Apache Software Foundation (ASF) under one
2+
# or more contributor license agreements. See the NOTICE file
3+
# distributed with this work for additional information
4+
# regarding copyright ownership. The ASF licenses this file
5+
# to you under the Apache License, Version 2.0 (the
6+
# "License"); you may not use this file except in compliance
7+
# with the License. You may obtain a copy of the License at
8+
#
9+
# http://www.apache.org/licenses/LICENSE-2.0
10+
#
11+
# Unless required by applicable law or agreed to in writing,
12+
# software distributed under the License is distributed on an
13+
# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
14+
# KIND, either express or implied. See the License for the
15+
# specific language governing permissions and limitations
16+
# under the License.
17+
18+
name: Iceberg Spark SQL Tests (native_datafusion)
19+
20+
concurrency:
21+
group: ${{ github.repository }}-${{ github.head_ref || github.sha }}-${{ github.workflow }}
22+
cancel-in-progress: true
23+
24+
on:
25+
push:
26+
paths-ignore:
27+
- "doc/**"
28+
- "docs/**"
29+
- "**.md"
30+
pull_request:
31+
paths-ignore:
32+
- "doc/**"
33+
- "docs/**"
34+
- "**.md"
35+
# manual trigger
36+
# https://docs.github.com/en/actions/managing-workflow-runs/manually-running-a-workflow
37+
workflow_dispatch:
38+
39+
env:
40+
RUST_VERSION: stable
41+
42+
jobs:
43+
iceberg-spark-sql-native-datafusion:
44+
strategy:
45+
matrix:
46+
os: [ubuntu-24.04]
47+
java-version: [11, 17]
48+
iceberg-version: [{short: '1.8', full: '1.8.1'}]
49+
spark-version: [{short: '3.5', full: '3.5.4'}]
50+
scala-version: [ '2.12', '2.13']
51+
fail-fast: false
52+
name: iceberg-spark-sql-native-datafusion/${{ matrix.os }}/iceberg-${{ matrix.iceberg-version.full }}/spark-${{ matrix.spark-version.full }}/scala-${{ matrix.scala-version }}/java-${{ matrix.java-version }}
53+
runs-on: ${{ matrix.os }}
54+
container:
55+
image: amd64/rust
56+
env:
57+
SPARK_LOCAL_IP: localhost
58+
steps:
59+
- uses: actions/checkout@v4
60+
- name: Setup Rust & Java toolchain
61+
uses: ./.github/actions/setup-builder
62+
with:
63+
rust-version: ${{env.RUST_VERSION}}
64+
jdk-version: ${{ matrix.java-version }}
65+
- name: Setup Iceberg
66+
uses: ./.github/actions/setup-iceberg-builder
67+
with:
68+
iceberg-version: ${{ matrix.iceberg-version.full }}
69+
iceberg-short-version: ${{ matrix.iceberg-version.short }}
70+
scala-version: ${{ matrix.scala-version }}
71+
spark-short-version: ${{ matrix.spark-version.short }}
72+
- run: echo -e "$(ip addr show eth0 | grep "inet\b" | awk '{print $2}' | cut -d/ -f1)\t$(hostname -f) $(hostname -s)" | sudo tee -a /etc/hosts
73+
- name: Run Iceberg Spark tests
74+
run: |
75+
cd apache-iceberg
76+
rm -rf /root/.m2/repository/org/apache/parquet # somehow parquet cache requires cleanups
77+
ENABLE_COMET=true ENABLE_COMET_SHUFFLE=true COMET_PARQUET_SCAN_IMPL=native_datafusion ../../gradlew -DsparkVersions=${{ matrix.spark-version.short }} -DscalaVersion=${{ matrix.scala-version }} -DflinkVersions= -DkafkaVersions= \
78+
:iceberg-spark:iceberg-spark-${{ matrix.spark-version.short }}_${{ matrix.scala-version }}:check \
79+
:iceberg-spark:iceberg-spark-extensions-${{ matrix.spark-version.short }}_${{ matrix.scala-version }}:check \
80+
:iceberg-spark:iceberg-spark-runtime-${{ matrix.spark-version.short }}_${{ matrix.scala-version }}:check \
81+
-Pquick=true -x javadoc

‎.github/workflows/iceberg_spark_test_native_iceberg_compat.yml‎

Lines changed: 81 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,81 @@
1+
# Licensed to the Apache Software Foundation (ASF) under one
2+
# or more contributor license agreements. See the NOTICE file
3+
# distributed with this work for additional information
4+
# regarding copyright ownership. The ASF licenses this file
5+
# to you under the Apache License, Version 2.0 (the
6+
# "License"); you may not use this file except in compliance
7+
# with the License. You may obtain a copy of the License at
8+
#
9+
# http://www.apache.org/licenses/LICENSE-2.0
10+
#
11+
# Unless required by applicable law or agreed to in writing,
12+
# software distributed under the License is distributed on an
13+
# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
14+
# KIND, either express or implied. See the License for the
15+
# specific language governing permissions and limitations
16+
# under the License.
17+
18+
name: Iceberg Spark SQL Tests (native_iceberg_compat)
19+
20+
concurrency:
21+
group: ${{ github.repository }}-${{ github.head_ref || github.sha }}-${{ github.workflow }}
22+
cancel-in-progress: true
23+
24+
on:
25+
push:
26+
paths-ignore:
27+
- "doc/**"
28+
- "docs/**"
29+
- "**.md"
30+
pull_request:
31+
paths-ignore:
32+
- "doc/**"
33+
- "docs/**"
34+
- "**.md"
35+
# manual trigger
36+
# https://docs.github.com/en/actions/managing-workflow-runs/manually-running-a-workflow
37+
workflow_dispatch:
38+
39+
env:
40+
RUST_VERSION: stable
41+
42+
jobs:
43+
iceberg-spark-sql-native-iceberg-compat:
44+
strategy:
45+
matrix:
46+
os: [ubuntu-24.04]
47+
java-version: [11, 17]
48+
iceberg-version: [{short: '1.8', full: '1.8.1'}]
49+
spark-version: [{short: '3.5', full: '3.5.4'}]
50+
scala-version: [ '2.12', '2.13']
51+
fail-fast: false
52+
name: iceberg-spark-sql-native-iceberg-compat/${{ matrix.os }}/iceberg-${{ matrix.iceberg-version.full }}/spark-${{ matrix.spark-version.full }}/scala-${{ matrix.scala-version }}/java-${{ matrix.java-version }}
53+
runs-on: ${{ matrix.os }}
54+
container:
55+
image: amd64/rust
56+
env:
57+
SPARK_LOCAL_IP: localhost
58+
steps:
59+
- uses: actions/checkout@v4
60+
- name: Setup Rust & Java toolchain
61+
uses: ./.github/actions/setup-builder
62+
with:
63+
rust-version: ${{env.RUST_VERSION}}
64+
jdk-version: ${{ matrix.java-version }}
65+
- name: Setup Iceberg
66+
uses: ./.github/actions/setup-iceberg-builder
67+
with:
68+
iceberg-version: ${{ matrix.iceberg-version.full }}
69+
iceberg-short-version: ${{ matrix.iceberg-version.short }}
70+
scala-version: ${{ matrix.scala-version }}
71+
spark-short-version: ${{ matrix.spark-version.short }}
72+
- run: echo -e "$(ip addr show eth0 | grep "inet\b" | awk '{print $2}' | cut -d/ -f1)\t$(hostname -f) $(hostname -s)" | sudo tee -a /etc/hosts
73+
- name: Run Iceberg Spark tests
74+
run: |
75+
cd apache-iceberg
76+
rm -rf /root/.m2/repository/org/apache/parquet # somehow parquet cache requires cleanups
77+
ENABLE_COMET=true ENABLE_COMET_SHUFFLE=true COMET_PARQUET_SCAN_IMPL=native_iceberg_compat ../../gradlew -DsparkVersions=${{ matrix.spark-version.short }} -DscalaVersion=${{ matrix.scala-version }} -DflinkVersions= -DkafkaVersions= \
78+
:iceberg-spark:iceberg-spark-${{ matrix.spark-version.short }}_${{ matrix.scala-version }}:check \
79+
:iceberg-spark:iceberg-spark-extensions-${{ matrix.spark-version.short }}_${{ matrix.scala-version }}:check \
80+
:iceberg-spark:iceberg-spark-runtime-${{ matrix.spark-version.short }}_${{ matrix.scala-version }}:check \
81+
-Pquick=true -x javadoc

0 commit comments

Comments
 (0)