Skip to content

Commit bdd2f32

Browse files
koukou
committed
XXX
1 parent 1d4b757 commit bdd2f32

File tree

11 files changed

+377
-341
lines changed

11 files changed

+377
-341
lines changed

‎.github/workflows/rc.yml‎

Lines changed: 266 additions & 22 deletions
Original file line numberDiff line numberDiff line change
@@ -32,8 +32,8 @@ concurrency:
3232
permissions:
3333
contents: read
3434
jobs:
35-
archive:
36-
name: Archive
35+
source:
36+
name: Source
3737
runs-on: ubuntu-latest
3838
timeout-minutes: 5
3939
steps:
@@ -70,15 +70,260 @@ jobs:
7070
- name: Audit
7171
run: |
7272
dev/release/run_rat.sh "${TAR_GZ}"
73-
- uses: actions/upload-artifact@6f51ac03b9356f520e9adb1b1b7802705f340c2b # v4.5.0
73+
- name: Upload source archive
74+
uses: actions/upload-artifact@65c4c4a1ddee5b72f698fdd19549f0f0fb45cf08 # v4.6.0
7475
with:
75-
name: archive
76+
name: release-source
7677
path: |
7778
apache-arrow-java-*
79+
jni-ubuntu:
80+
name: JNI ${{ matrix.platform.runs_on }} ${{ matrix.platform.arch }}
81+
runs-on: ${{ matrix.platform.runs_on }}
82+
needs:
83+
- source
84+
strategy:
85+
fail-fast: false
86+
matrix:
87+
platform:
88+
- runs_on: ubuntu-latest
89+
arch: "x86_64"
90+
archery_arch: "amd64"
91+
env:
92+
# architecture name used for archery build
93+
ARCH: ${{ matrix.platform.archery_arch }}
94+
permissions:
95+
contents: read
96+
packages: write
97+
steps:
98+
- name: Download source archive
99+
uses: actions/download-artifact@fa0a91b85d4f404e444e00e005971372dc801d16 # v4.1.8
100+
with:
101+
name: release-source
102+
- name: Extract Download the latest Apache Arrow C++
103+
run: |
104+
tar -xf apache-arrow-java-*.tar.gz --strip-components=1
105+
- name: Download the latest Apache Arrow C++
106+
run: |
107+
ci/scripts/download_cpp.sh
108+
- name: Checkout apache/arrow-testing
109+
uses: actions/checkout@11bd71901bbe5b1630ceea73d27597364c9af683 # v4.2.2
110+
with:
111+
repository: apache/arrow-testing
112+
path: arrow/testing
113+
- name: Checkout apache/parquet-testing
114+
uses: actions/checkout@11bd71901bbe5b1630ceea73d27597364c9af683 # v4.2.2
115+
with:
116+
repository: apache/parquet-testing
117+
path: arrow/cpp/submodules/parquet-testing
118+
- uses: docker/login-action@9780b0c442fbb1117ed29e0efdff1e18412f7567 # v3.3.0
119+
with:
120+
registry: ghcr.io
121+
username: ${{ github.actor }}
122+
password: ${{ secrets.GITHUB_TOKEN }}
123+
- name: Prepare ccache
124+
run: |
125+
echo "CCACHE_DIR=${PWD}/ccache" >> ${GITHUB_ENV}
126+
- name: Cache ccache
127+
uses: actions/cache@1bd1e32a3bdc45362d1e726936510720a7c30a57 # v4.2.0
128+
with:
129+
path: ccache
130+
key: jni-linux-${{ matrix.platform.arch }}-${{ hashFiles('arrow/cpp/**') }}
131+
restore-keys: jni-linux-${{ matrix.platform.arch }}-
132+
- name: Build C++ libraries
133+
run: |
134+
docker compose run vcpkg-jni
135+
- name: Push Docker image
136+
if: success() && github.event_name == 'push' && github.repository == 'apache/arrow-java' && github.ref_name == 'main'
137+
run: |
138+
docker compose push vcpkg-jni
139+
- name: Compress into single artifact to keep directory structure
140+
run: tar -cvzf jni-linux-${{ matrix.platform.arch }}.tar.gz jni/
141+
- name: Upload artifacts
142+
uses: actions/upload-artifact@65c4c4a1ddee5b72f698fdd19549f0f0fb45cf08 # v4.6.0
143+
with:
144+
name: jni-linux-${{ matrix.platform.arch }}
145+
path: jni-linux-${{ matrix.platform.arch }}.tar.gz
146+
jni-macos:
147+
name: JNI ${{ matrix.platform.runs_on }} ${{ matrix.platform.arch }}
148+
runs-on: ${{ matrix.platform.runs_on }}
149+
needs:
150+
- source
151+
strategy:
152+
fail-fast: false
153+
matrix:
154+
platform:
155+
- { runs_on: macos-13, arch: "x86_64"}
156+
- { runs_on: macos-14, arch: "aarch_64" }
157+
env:
158+
MACOSX_DEPLOYMENT_TARGET: "14.0"
159+
steps:
160+
- name: Download source archive
161+
uses: actions/download-artifact@fa0a91b85d4f404e444e00e005971372dc801d16 # v4.1.8
162+
with:
163+
name: release-source
164+
- name: Extract Download the latest Apache Arrow C++
165+
run: |
166+
tar -xf apache-arrow-java-*.tar.gz --strip-components=1
167+
# We need 19.0.0 for latest Boost support
168+
- name: Download the latest RC Apache Arrow C++
169+
run: |
170+
ci/scripts/download_cpp.sh latest-rc
171+
- name: Checkout apache/arrow-testing
172+
uses: actions/checkout@11bd71901bbe5b1630ceea73d27597364c9af683 # v4.2.2
173+
with:
174+
repository: apache/arrow-testing
175+
path: arrow/testing
176+
- name: Checkout apache/parquet-testing
177+
uses: actions/checkout@11bd71901bbe5b1630ceea73d27597364c9af683 # v4.2.2
178+
with:
179+
repository: apache/parquet-testing
180+
path: arrow/cpp/submodules/parquet-testing
181+
- name: Set up Python
182+
uses: actions/setup-python@v5
183+
with:
184+
cache: 'pip'
185+
python-version: 3.12
186+
- name: Install Archery
187+
run: pip install -e arrow/dev/archery[all]
188+
- name: Install dependencies
189+
run: |
190+
# We want to use llvm@14 to avoid shared z3
191+
# dependency. llvm@14 doesn't depend on z3 and llvm depends
192+
# on z3. And Homebrew's z3 provides only shared library. It
193+
# doesn't provides static z3 because z3's CMake doesn't accept
194+
# building both shared and static libraries at once.
195+
# See also: Z3_BUILD_LIBZ3_SHARED in
196+
# https://github.com/Z3Prover/z3/blob/master/README-CMake.md
197+
#
198+
# If llvm is installed, Apache Arrow C++ uses llvm rather than
199+
# llvm@14 because llvm is newer than llvm@14.
200+
brew uninstall llvm || :
201+
202+
# Ensure updating python@XXX with the "--overwrite" option.
203+
# If python@XXX is updated without "--overwrite", it causes
204+
# a conflict error. Because Python 3 installed not by
205+
# Homebrew exists in /usr/local on GitHub Actions. If
206+
# Homebrew's python@XXX is updated without "--overwrite", it
207+
# tries to replace /usr/local/bin/2to3 and so on and causes
208+
# a conflict error.
209+
brew update
210+
for python_package in $(brew list | grep python@); do
211+
brew install --overwrite ${python_package}
212+
done
213+
brew install --overwrite python
214+
215+
if [ "$(uname -m)" = "arm64" ]; then
216+
# pkg-config formula is deprecated but it's still installed
217+
# in GitHub Actions runner now. We can remove this once
218+
# pkg-config formula is removed from GitHub Actions runner.
219+
brew uninstall pkg-config || :
220+
brew uninstall [email protected] || :
221+
fi
222+
223+
brew bundle --file=arrow/cpp/Brewfile
224+
# We want to link aws-sdk-cpp statically but Homebrew's
225+
# aws-sdk-cpp provides only shared library. If we have
226+
# Homebrew's aws-sdk-cpp, our build mix Homebrew's
227+
# aws-sdk-cpp and bundled aws-sdk-cpp. We uninstall Homebrew's
228+
# aws-sdk-cpp to ensure using only bundled aws-sdk-cpp.
229+
brew uninstall aws-sdk-cpp
230+
# We want to use bundled RE2 for static linking. If
231+
# Homebrew's RE2 is installed, its header file may be used.
232+
# We uninstall Homebrew's RE2 to ensure using bundled RE2.
233+
brew uninstall grpc || : # gRPC depends on RE2
234+
brew uninstall [email protected] || : # gRPC 1.54 may be installed too
235+
brew uninstall re2
236+
# We want to use bundled Protobuf for static linking. If
237+
# Homebrew's Protobuf is installed, its library file may be
238+
# used on test We uninstall Homebrew's Protobuf to ensure using
239+
# bundled Protobuf.
240+
brew uninstall protobuf
241+
242+
brew bundle --file=Brewfile
243+
- name: Prepare ccache
244+
run: |
245+
echo "CCACHE_DIR=${PWD}/ccache" >> ${GITHUB_ENV}
246+
- name: Cache ccache
247+
uses: actions/cache@1bd1e32a3bdc45362d1e726936510720a7c30a57 # v4.2.0
248+
with:
249+
path: ccache
250+
key: jni-macos-${{ matrix.platform.arch }}-${{ hashFiles('arrow/cpp/**') }}
251+
restore-keys: jni-macos-${{ matrix.platform.arch }}-
252+
- name: Build C++ libraries
253+
run: |
254+
set -e
255+
# make brew Java available to CMake
256+
export JAVA_HOME=$(brew --prefix openjdk@11)/libexec/openjdk.jdk/Contents/Home
257+
ci/scripts/jni_macos_build.sh . arrow build jni
258+
- name: Compress into single artifact to keep directory structure
259+
run: tar -cvzf jni-macos-${{ matrix.platform.arch }}.tar.gz jni/
260+
- name: Upload artifacts
261+
uses: actions/upload-artifact@65c4c4a1ddee5b72f698fdd19549f0f0fb45cf08 # v4.6.0
262+
with:
263+
name: jni-macos-${{ matrix.platform.arch }}
264+
path: jni-macos-${{ matrix.platform.arch }}.tar.gz
265+
binaries:
266+
name: Binaries
267+
runs-on: ubuntu-latest
268+
needs:
269+
- jni-ubuntu
270+
- jni-macos
271+
steps:
272+
- name: Download artifacts
273+
uses: actions/download-artifact@fa0a91b85d4f404e444e00e005971372dc801d16 # v4.1.8
274+
with:
275+
path: artifacts
276+
- name: Decompress artifacts
277+
run: |
278+
mv artifacts/*/*.tar.gz .
279+
tar -xf apache-arrow-java-*.tar.gz --strip-components=1
280+
tar -xvzf jni-linux-x86_64.tar.gz
281+
# tar -xvzf jni-linux-aarch_64.tar.gz
282+
tar -xvzf jni-macos-x86_64.tar.gz
283+
tar -xvzf jni-macos-aarch_64.tar.gz
284+
# tar -xvzf jni-windows.tar.gz
285+
- name: Test that shared libraries exist
286+
run: |
287+
set -x
288+
289+
test -f jni/arrow_cdata_jni/x86_64/libarrow_cdata_jni.so
290+
test -f jni/arrow_dataset_jni/x86_64/libarrow_dataset_jni.so
291+
test -f jni/arrow_orc_jni/x86_64/libarrow_orc_jni.so
292+
test -f jni/gandiva_jni/x86_64/libgandiva_jni.so
293+
294+
# test -f jni/arrow_cdata_jni/aarch_64/libarrow_cdata_jni.so
295+
# test -f jni/arrow_dataset_jni/aarch_64/libarrow_dataset_jni.so
296+
# test -f jni/arrow_orc_jni/aarch_64/libarrow_orc_jni.so
297+
# test -f jni/gandiva_jni/aarch_64/libgandiva_jni.so
298+
299+
test -f jni/arrow_cdata_jni/x86_64/libarrow_cdata_jni.dylib
300+
test -f jni/arrow_dataset_jni/x86_64/libarrow_dataset_jni.dylib
301+
test -f jni/arrow_orc_jni/x86_64/libarrow_orc_jni.dylib
302+
test -f jni/gandiva_jni/x86_64/libgandiva_jni.dylib
303+
304+
test -f jni/arrow_cdata_jni/aarch_64/libarrow_cdata_jni.dylib
305+
test -f jni/arrow_dataset_jni/aarch_64/libarrow_dataset_jni.dylib
306+
test -f jni/arrow_orc_jni/aarch_64/libarrow_orc_jni.dylib
307+
test -f jni/gandiva_jni/aarch_64/libgandiva_jni.dylib
308+
309+
# test -f jni/arrow_cdata_jni/x86_64/arrow_cdata_jni.dll
310+
# test -f jni/arrow_dataset_jni/x86_64/arrow_dataset_jni.dll
311+
# test -f jni/arrow_orc_jni/x86_64/arrow_orc_jni.dll
312+
- name: Build bundled JAR
313+
env:
314+
MAVEN_ARGS: >-
315+
--no-transfer-progress
316+
run: |
317+
ci/scripts/jni_full_build.sh jni binaries
318+
- name: Upload artifacts
319+
uses: actions/upload-artifact@65c4c4a1ddee5b72f698fdd19549f0f0fb45cf08 # v4.6.0
320+
with:
321+
name: release-binaries
322+
path: binaries/*
78323
verify:
79324
name: Verify
80325
needs:
81-
- archive
326+
- binaries
82327
runs-on: ${{ matrix.os }}
83328
strategy:
84329
fail-fast: false
@@ -87,13 +332,10 @@ jobs:
87332
- macos-latest
88333
- ubuntu-latest
89334
steps:
90-
- name: Checkout
91-
uses: actions/checkout@11bd71901bbe5b1630ceea73d27597364c9af683 # v4.2.2
92-
with:
93-
submodules: recursive
94-
- uses: actions/download-artifact@fa0a91b85d4f404e444e00e005971372dc801d16 # v4.1.8
335+
- name: Download release artifacts
336+
uses: actions/download-artifact@fa0a91b85d4f404e444e00e005971372dc801d16 # v4.1.8
95337
with:
96-
name: archive
338+
pattern: release-*
97339
- name: Verify
98340
run: |
99341
tar_gz=$(echo apache-arrow-java-*.tar.gz)
@@ -105,9 +347,13 @@ jobs:
105347
else
106348
rc=$(date +%Y%m%d)
107349
fi
108-
VERIFY_DEFAULT=0 \
109-
VERIFY_SOURCE=1 \
110-
dev/release/verify_rc.sh "${version}" "${rc}"
350+
tar -xf ${tar_gz}
351+
export VERIFY_DEFAULT=0
352+
export VERIFY_BINARY=1
353+
export VERIFY_SOURCE=1
354+
cd apache-arrow-java-*
355+
mv ../${tar_gz} ./
356+
dev/release/verify_rc.sh "${version}" "${rc}"
111357
upload:
112358
name: Upload
113359
if: github.ref_type == 'tag'
@@ -117,13 +363,11 @@ jobs:
117363
permissions:
118364
contents: write
119365
steps:
120-
- name: Checkout
121-
uses: actions/checkout@11bd71901bbe5b1630ceea73d27597364c9af683 # v4.2.2
122-
with:
123-
submodules: recursive
124-
- uses: actions/download-artifact@fa0a91b85d4f404e444e00e005971372dc801d16 # v4.1.8
366+
- name: Download release artifacts
367+
uses: actions/download-artifact@fa0a91b85d4f404e444e00e005971372dc801d16 # v4.1.8
125368
with:
126-
name: archive
369+
pattern: release-*
370+
path: artifacts
127371
- name: Upload
128372
run: |
129373
# GH-499: How to create release notes?
@@ -133,9 +377,9 @@ jobs:
133377
gh release create ${GITHUB_REF_NAME} \
134378
--generate-notes \
135379
--prerelease \
380+
--repo ${{ github.repository }} \
136381
--title "Apache Arrow Java ${version} RC${rc}" \
137382
--verify-tag \
138-
apache-arrow-java-*.tar.gz \
139-
apache-arrow-java-*.tar.gz.sha*
383+
artifacts/*
140384
env:
141385
GH_TOKEN: ${{ secrets.GITHUB_TOKEN }}

0 commit comments

Comments
 (0)