Skip to content

Commit fe21b3d

Browse files
koukou
committed
XXX
1 parent 1d4b757 commit fe21b3d

File tree

10 files changed

+358
-338
lines changed

10 files changed

+358
-338
lines changed

‎.github/workflows/rc.yml‎

Lines changed: 248 additions & 22 deletions
Original file line numberDiff line numberDiff line change
@@ -32,8 +32,8 @@ concurrency:
3232
permissions:
3333
contents: read
3434
jobs:
35-
archive:
36-
name: Archive
35+
source:
36+
name: Source
3737
runs-on: ubuntu-latest
3838
timeout-minutes: 5
3939
steps:
@@ -70,15 +70,242 @@ jobs:
7070
- name: Audit
7171
run: |
7272
dev/release/run_rat.sh "${TAR_GZ}"
73-
- uses: actions/upload-artifact@6f51ac03b9356f520e9adb1b1b7802705f340c2b # v4.5.0
73+
- name: Upload source archive
74+
uses: actions/upload-artifact@65c4c4a1ddee5b72f698fdd19549f0f0fb45cf08 # v4.6.0
7475
with:
75-
name: archive
76+
name: release-source
7677
path: |
7778
apache-arrow-java-*
79+
jni-ubuntu:
80+
name: JNI ${{ matrix.platform.runs_on }} ${{ matrix.platform.arch }}
81+
runs-on: ${{ matrix.platform.runs_on }}
82+
needs:
83+
- source
84+
strategy:
85+
fail-fast: false
86+
matrix:
87+
platform:
88+
- runs_on: ubuntu-latest
89+
arch: "x86_64"
90+
archery_arch: "amd64"
91+
env:
92+
# architecture name used for archery build
93+
ARCH: ${{ matrix.platform.archery_arch }}
94+
permissions:
95+
contents: read
96+
packages: write
97+
steps:
98+
- name: Download source archive
99+
uses: actions/download-artifact@fa0a91b85d4f404e444e00e005971372dc801d16 # v4.1.8
100+
with:
101+
name: release-source
102+
- name: Extract Download the latest Apache Arrow C++
103+
run: |
104+
tar -xf apache-arrow-java-*.tar.gz --strip-components=1
105+
- name: Download the latest Apache Arrow C++
106+
run: |
107+
ci/scripts/download_cpp.sh
108+
- name: Checkout apache/arrow-testing
109+
uses: actions/checkout@11bd71901bbe5b1630ceea73d27597364c9af683 # v4.2.2
110+
with:
111+
repository: apache/arrow-testing
112+
path: arrow/testing
113+
- name: Checkout apache/parquet-testing
114+
uses: actions/checkout@11bd71901bbe5b1630ceea73d27597364c9af683 # v4.2.2
115+
with:
116+
repository: apache/parquet-testing
117+
path: arrow/cpp/submodules/parquet-testing
118+
- uses: docker/login-action@9780b0c442fbb1117ed29e0efdff1e18412f7567 # v3.3.0
119+
with:
120+
registry: ghcr.io
121+
username: ${{ github.actor }}
122+
password: ${{ secrets.GITHUB_TOKEN }}
123+
- name: Build C++ libraries
124+
run: |
125+
docker compose run vcpkg-jni
126+
- name: Push Docker image
127+
if: success() && github.event_name == 'push' && github.repository == 'apache/arrow-java' && github.ref_name == 'main'
128+
run: |
129+
docker compose push vcpkg-jni
130+
- name: Compress into single artifact to keep directory structure
131+
run: tar -cvzf jni-linux-${{ matrix.platform.arch }}.tar.gz dist/
132+
- name: Upload artifacts
133+
uses: actions/upload-artifact@v4
134+
with:
135+
name: jni-linux-${{ matrix.platform.arch }}
136+
path: jni-linux-${{ matrix.platform.arch }}.tar.gz
137+
jni-macos:
138+
name: JNI ${{ matrix.platform.runs_on }} ${{ matrix.platform.arch }}
139+
runs-on: ${{ matrix.platform.runs_on }}
140+
needs:
141+
- source
142+
strategy:
143+
fail-fast: false
144+
matrix:
145+
platform:
146+
- { runs_on: macos-13, arch: "x86_64"}
147+
- { runs_on: macos-14, arch: "aarch_64" }
148+
env:
149+
MACOSX_DEPLOYMENT_TARGET: "14.0"
150+
steps:
151+
- name: Download source archive
152+
uses: actions/download-artifact@fa0a91b85d4f404e444e00e005971372dc801d16 # v4.1.8
153+
with:
154+
name: release-source
155+
- name: Extract Download the latest Apache Arrow C++
156+
run: |
157+
tar -xf apache-arrow-java-*.tar.gz --strip-components=1
158+
# We need 19.0.0 for latest Boost support
159+
- name: Download the latest RC Apache Arrow C++
160+
run: |
161+
ci/scripts/download_cpp.sh latest-rc
162+
- name: Checkout apache/arrow-testing
163+
uses: actions/checkout@11bd71901bbe5b1630ceea73d27597364c9af683 # v4.2.2
164+
with:
165+
repository: apache/arrow-testing
166+
path: arrow/testing
167+
- name: Checkout apache/parquet-testing
168+
uses: actions/checkout@11bd71901bbe5b1630ceea73d27597364c9af683 # v4.2.2
169+
with:
170+
repository: apache/parquet-testing
171+
path: arrow/cpp/submodules/parquet-testing
172+
- name: Set up Python
173+
uses: actions/setup-python@v5
174+
with:
175+
cache: 'pip'
176+
python-version: 3.12
177+
- name: Install Archery
178+
run: pip install -e arrow/dev/archery[all]
179+
- name: Install dependencies
180+
run: |
181+
# We want to use llvm@14 to avoid shared z3
182+
# dependency. llvm@14 doesn't depend on z3 and llvm depends
183+
# on z3. And Homebrew's z3 provides only shared library. It
184+
# doesn't provides static z3 because z3's CMake doesn't accept
185+
# building both shared and static libraries at once.
186+
# See also: Z3_BUILD_LIBZ3_SHARED in
187+
# https://github.com/Z3Prover/z3/blob/master/README-CMake.md
188+
#
189+
# If llvm is installed, Apache Arrow C++ uses llvm rather than
190+
# llvm@14 because llvm is newer than llvm@14.
191+
brew uninstall llvm || :
192+
193+
# Ensure updating python@XXX with the "--overwrite" option.
194+
# If python@XXX is updated without "--overwrite", it causes
195+
# a conflict error. Because Python 3 installed not by
196+
# Homebrew exists in /usr/local on GitHub Actions. If
197+
# Homebrew's python@XXX is updated without "--overwrite", it
198+
# tries to replace /usr/local/bin/2to3 and so on and causes
199+
# a conflict error.
200+
brew update
201+
for python_package in $(brew list | grep python@); do
202+
brew install --overwrite ${python_package}
203+
done
204+
brew install --overwrite python
205+
206+
if [ "$(uname -m)" = "arm64" ]; then
207+
# pkg-config formula is deprecated but it's still installed
208+
# in GitHub Actions runner now. We can remove this once
209+
# pkg-config formula is removed from GitHub Actions runner.
210+
brew uninstall pkg-config || :
211+
brew uninstall [email protected] || :
212+
fi
213+
214+
brew bundle --file=arrow/cpp/Brewfile
215+
# We want to link aws-sdk-cpp statically but Homebrew's
216+
# aws-sdk-cpp provides only shared library. If we have
217+
# Homebrew's aws-sdk-cpp, our build mix Homebrew's
218+
# aws-sdk-cpp and bundled aws-sdk-cpp. We uninstall Homebrew's
219+
# aws-sdk-cpp to ensure using only bundled aws-sdk-cpp.
220+
brew uninstall aws-sdk-cpp
221+
# We want to use bundled RE2 for static linking. If
222+
# Homebrew's RE2 is installed, its header file may be used.
223+
# We uninstall Homebrew's RE2 to ensure using bundled RE2.
224+
brew uninstall grpc || : # gRPC depends on RE2
225+
brew uninstall [email protected] || : # gRPC 1.54 may be installed too
226+
brew uninstall re2
227+
# We want to use bundled Protobuf for static linking. If
228+
# Homebrew's Protobuf is installed, its library file may be
229+
# used on test We uninstall Homebrew's Protobuf to ensure using
230+
# bundled Protobuf.
231+
brew uninstall protobuf
232+
233+
brew bundle --file=Brewfile
234+
- name: Build C++ libraries
235+
run: |
236+
set -e
237+
# make brew Java available to CMake
238+
export JAVA_HOME=$(brew --prefix openjdk@11)/libexec/openjdk.jdk/Contents/Home
239+
ci/scripts/jni_macos_build.sh . arrow build jni
240+
- name: Compress into single artifact to keep directory structure
241+
run: tar -cvzf jni-macos-${{ matrix.platform.arch }}.tar.gz jni/
242+
- name: Upload artifacts
243+
uses: actions/upload-artifact@65c4c4a1ddee5b72f698fdd19549f0f0fb45cf08 # v4.6.0
244+
with:
245+
name: jni-macos-${{ matrix.platform.arch }}
246+
path: jni-macos-${{ matrix.platform.arch }}.tar.gz
247+
binaries:
248+
name: Binaries
249+
runs-on: ubuntu-latest
250+
needs:
251+
- jni-ubuntu
252+
- jni-macos
253+
steps:
254+
- name: Download artifacts
255+
uses: actions/download-artifact@fa0a91b85d4f404e444e00e005971372dc801d16 # v4.1.8
256+
with:
257+
path: artifacts
258+
- name: Decompress artifacts
259+
run: |
260+
mv artifacts/*/*.tar.gz .
261+
tar -xf apache-arrow-java-*.tar.gz --strip-components=1
262+
tar -xvzf jni-linux-x86_64.tar.gz
263+
# tar -xvzf jni-linux-aarch_64.tar.gz
264+
tar -xvzf jni-macos-x86_64.tar.gz
265+
tar -xvzf jni-macos-aarch_64.tar.gz
266+
# tar -xvzf jni-windows.tar.gz
267+
- name: Test that shared libraries exist
268+
run: |
269+
set -x
270+
271+
test -f dist/arrow_cdata_jni/x86_64/libarrow_cdata_jni.so
272+
test -f dist/arrow_dataset_jni/x86_64/libarrow_dataset_jni.so
273+
test -f dist/arrow_orc_jni/x86_64/libarrow_orc_jni.so
274+
test -f dist/gandiva_jni/x86_64/libgandiva_jni.so
275+
276+
# test -f dist/arrow_cdata_jni/aarch_64/libarrow_cdata_jni.so
277+
# test -f dist/arrow_dataset_jni/aarch_64/libarrow_dataset_jni.so
278+
# test -f dist/arrow_orc_jni/aarch_64/libarrow_orc_jni.so
279+
# test -f dist/gandiva_jni/aarch_64/libgandiva_jni.so
280+
281+
test -f dist/arrow_cdata_jni/x86_64/libarrow_cdata_jni.dylib
282+
test -f dist/arrow_dataset_jni/x86_64/libarrow_dataset_jni.dylib
283+
test -f dist/arrow_orc_jni/x86_64/libarrow_orc_jni.dylib
284+
test -f dist/gandiva_jni/x86_64/libgandiva_jni.dylib
285+
286+
test -f dist/arrow_cdata_jni/aarch_64/libarrow_cdata_jni.dylib
287+
test -f dist/arrow_dataset_jni/aarch_64/libarrow_dataset_jni.dylib
288+
test -f dist/arrow_orc_jni/aarch_64/libarrow_orc_jni.dylib
289+
test -f dist/gandiva_jni/aarch_64/libgandiva_jni.dylib
290+
291+
# test -f dist/arrow_cdata_jni/x86_64/arrow_cdata_jni.dll
292+
# test -f dist/arrow_dataset_jni/x86_64/arrow_dataset_jni.dll
293+
# test -f dist/arrow_orc_jni/x86_64/arrow_orc_jni.dll
294+
- name: Build bundled JAR
295+
env:
296+
MAVEN_ARGS: >-
297+
--no-transfer-progress
298+
run: |
299+
ci/scripts/jni_full_build.sh jni binaries
300+
- name: Upload artifacts
301+
uses: actions/upload-artifact@65c4c4a1ddee5b72f698fdd19549f0f0fb45cf08 # v4.6.0
302+
with:
303+
name: release-binaries
304+
path: binaries/*
78305
verify:
79306
name: Verify
80307
needs:
81-
- archive
308+
- binaries
82309
runs-on: ${{ matrix.os }}
83310
strategy:
84311
fail-fast: false
@@ -87,13 +314,10 @@ jobs:
87314
- macos-latest
88315
- ubuntu-latest
89316
steps:
90-
- name: Checkout
91-
uses: actions/checkout@11bd71901bbe5b1630ceea73d27597364c9af683 # v4.2.2
92-
with:
93-
submodules: recursive
94-
- uses: actions/download-artifact@fa0a91b85d4f404e444e00e005971372dc801d16 # v4.1.8
317+
- name: Download release artifacts
318+
uses: actions/download-artifact@fa0a91b85d4f404e444e00e005971372dc801d16 # v4.1.8
95319
with:
96-
name: archive
320+
pattern: release-*
97321
- name: Verify
98322
run: |
99323
tar_gz=$(echo apache-arrow-java-*.tar.gz)
@@ -105,9 +329,13 @@ jobs:
105329
else
106330
rc=$(date +%Y%m%d)
107331
fi
108-
VERIFY_DEFAULT=0 \
109-
VERIFY_SOURCE=1 \
110-
dev/release/verify_rc.sh "${version}" "${rc}"
332+
tar -xf ${tar_gz}
333+
export VERIFY_DEFAULT=0
334+
export VERIFY_BINARY=1
335+
export VERIFY_SOURCE=1
336+
cd apache-arrow-java-*
337+
mv ../${tar_gz} ./
338+
dev/release/verify_rc.sh "${version}" "${rc}"
111339
upload:
112340
name: Upload
113341
if: github.ref_type == 'tag'
@@ -117,13 +345,11 @@ jobs:
117345
permissions:
118346
contents: write
119347
steps:
120-
- name: Checkout
121-
uses: actions/checkout@11bd71901bbe5b1630ceea73d27597364c9af683 # v4.2.2
122-
with:
123-
submodules: recursive
124-
- uses: actions/download-artifact@fa0a91b85d4f404e444e00e005971372dc801d16 # v4.1.8
348+
- name: Download release artifacts
349+
uses: actions/download-artifact@fa0a91b85d4f404e444e00e005971372dc801d16 # v4.1.8
125350
with:
126-
name: archive
351+
pattern: release-*
352+
path: artifacts
127353
- name: Upload
128354
run: |
129355
# GH-499: How to create release notes?
@@ -133,9 +359,9 @@ jobs:
133359
gh release create ${GITHUB_REF_NAME} \
134360
--generate-notes \
135361
--prerelease \
362+
--repo ${{ github.repository }} \
136363
--title "Apache Arrow Java ${version} RC${rc}" \
137364
--verify-tag \
138-
apache-arrow-java-*.tar.gz \
139-
apache-arrow-java-*.tar.gz.sha*
365+
artifacts/*
140366
env:
141367
GH_TOKEN: ${{ secrets.GITHUB_TOKEN }}

0 commit comments

Comments
 (0)