Skip to content

Commit da043e1

Browse files
koukou
committed
XXX
1 parent 1d4b757 commit da043e1

File tree

9 files changed

+322
-347
lines changed

9 files changed

+322
-347
lines changed

‎.github/workflows/rc.yml‎

Lines changed: 247 additions & 22 deletions
Original file line numberDiff line numberDiff line change
@@ -32,8 +32,8 @@ concurrency:
3232
permissions:
3333
contents: read
3434
jobs:
35-
archive:
36-
name: Archive
35+
source:
36+
name: Source
3737
runs-on: ubuntu-latest
3838
timeout-minutes: 5
3939
steps:
@@ -70,15 +70,241 @@ jobs:
7070
- name: Audit
7171
run: |
7272
dev/release/run_rat.sh "${TAR_GZ}"
73-
- uses: actions/upload-artifact@6f51ac03b9356f520e9adb1b1b7802705f340c2b # v4.5.0
73+
- name: Upload source archive
74+
uses: actions/upload-artifact@65c4c4a1ddee5b72f698fdd19549f0f0fb45cf08 # v4.6.0
7475
with:
75-
name: archive
76+
name: release-source
7677
path: |
7778
apache-arrow-java-*
79+
jni-ubuntu:
80+
name: JNI ${{ matrix.platform.runs_on }} ${{ matrix.platform.arch }}
81+
runs-on: ${{ matrix.platform.runs_on }}
82+
needs:
83+
- source
84+
strategy:
85+
fail-fast: false
86+
matrix:
87+
platform:
88+
- runs_on: ubuntu-latest
89+
arch: "x86_64"
90+
archery_arch: "amd64"
91+
env:
92+
# architecture name used for archery build
93+
ARCH: ${{ matrix.platform.archery_arch }}
94+
permissions:
95+
contents: read
96+
packages: write
97+
steps:
98+
- name: Download source archive
99+
uses: actions/download-artifact@fa0a91b85d4f404e444e00e005971372dc801d16 # v4.1.8
100+
with:
101+
name: release-source
102+
- name: Extract Download the latest Apache Arrow C++
103+
run: |
104+
tar -xf apache-arrow-java-*.tar.gz --strip-components=1
105+
- name: Download the latest Apache Arrow C++
106+
run: |
107+
ci/scripts/download_cpp.sh
108+
- name: Checkout apache/arrow-testing
109+
uses: actions/checkout@11bd71901bbe5b1630ceea73d27597364c9af683 # v4.2.2
110+
with:
111+
repo: apache/arrow-testing
112+
path: arrow/testing
113+
- name: Checkout apache/parquet-testing
114+
uses: actions/checkout@11bd71901bbe5b1630ceea73d27597364c9af683 # v4.2.2
115+
with:
116+
repo: apache/parquet-testing
117+
path: cpp/submodules/parquet-testing
118+
- uses: docker/login-action@9780b0c442fbb1117ed29e0efdff1e18412f7567 # v3.3.0
119+
with:
120+
registry: ghcr.io
121+
username: ${{ github.actor }}
122+
password: ${{ secrets.GITHUB_TOKEN }}
123+
- name: Build C++ libraries
124+
run: |
125+
docker compose run vcpkg-jni
126+
- name: Push Docker image
127+
if: success() && github.event_name == 'push' && github.repository == 'apache/arrow-java' && github.ref_name == 'main'
128+
run: |
129+
docker compose push vcpkg-jni
130+
- name: Compress into single artifact to keep directory structure
131+
run: tar -cvzf jni-linux-${{ matrix.platform.arch }}.tar.gz dist/
132+
- name: Upload artifacts
133+
uses: actions/upload-artifact@v4
134+
with:
135+
name: jni-linux-${{ matrix.platform.arch }}
136+
path: jni-linux-${{ matrix.platform.arch }}.tar.gz
137+
jni-macos:
138+
name: JNI ${{ matrix.platform.runs_on }} ${{ matrix.platform.arch }}
139+
runs-on: ${{ matrix.platform.runs_on }}
140+
needs:
141+
- source
142+
strategy:
143+
fail-fast: false
144+
matrix:
145+
platform:
146+
- { runs_on: macos-13, arch: "x86_64"}
147+
- { runs_on: macos-14, arch: "aarch_64" }
148+
env:
149+
MACOSX_DEPLOYMENT_TARGET: "14.0"
150+
steps:
151+
- name: Download source archive
152+
uses: actions/download-artifact@fa0a91b85d4f404e444e00e005971372dc801d16 # v4.1.8
153+
with:
154+
name: release-source
155+
- name: Extract Download the latest Apache Arrow C++
156+
run: |
157+
tar -xf apache-arrow-java-*.tar.gz --strip-components=1
158+
- name: Download the latest Apache Arrow C++
159+
run: |
160+
ci/scripts/download_cpp.sh
161+
- name: Checkout apache/arrow-testing
162+
uses: actions/checkout@11bd71901bbe5b1630ceea73d27597364c9af683 # v4.2.2
163+
with:
164+
repo: apache/arrow-testing
165+
path: arrow/testing
166+
- name: Checkout apache/parquet-testing
167+
uses: actions/checkout@11bd71901bbe5b1630ceea73d27597364c9af683 # v4.2.2
168+
with:
169+
repo: apache/parquet-testing
170+
path: cpp/submodules/parquet-testing
171+
- name: Set up Python
172+
uses: actions/setup-python@v5
173+
with:
174+
cache: 'pip'
175+
python-version: 3.12
176+
- name: Install Archery
177+
run: pip install -e arrow/dev/archery[all]
178+
- name: Install dependencies
179+
run: |
180+
# We want to use llvm@14 to avoid shared z3
181+
# dependency. llvm@14 doesn't depend on z3 and llvm depends
182+
# on z3. And Homebrew's z3 provides only shared library. It
183+
# doesn't provides static z3 because z3's CMake doesn't accept
184+
# building both shared and static libraries at once.
185+
# See also: Z3_BUILD_LIBZ3_SHARED in
186+
# https://github.com/Z3Prover/z3/blob/master/README-CMake.md
187+
#
188+
# If llvm is installed, Apache Arrow C++ uses llvm rather than
189+
# llvm@14 because llvm is newer than llvm@14.
190+
brew uninstall llvm || :
191+
192+
# Ensure updating python@XXX with the "--overwrite" option.
193+
# If python@XXX is updated without "--overwrite", it causes
194+
# a conflict error. Because Python 3 installed not by
195+
# Homebrew exists in /usr/local on GitHub Actions. If
196+
# Homebrew's python@XXX is updated without "--overwrite", it
197+
# tries to replace /usr/local/bin/2to3 and so on and causes
198+
# a conflict error.
199+
brew update
200+
for python_package in $(brew list | grep python@); do
201+
brew install --overwrite ${python_package}
202+
done
203+
brew install --overwrite python
204+
205+
if [ "$(uname -m)" = "arm64" ]; then
206+
# pkg-config formula is deprecated but it's still installed
207+
# in GitHub Actions runner now. We can remove this once
208+
# pkg-config formula is removed from GitHub Actions runner.
209+
brew uninstall pkg-config || :
210+
brew uninstall [email protected] || :
211+
fi
212+
213+
brew bundle --file=arrow/cpp/Brewfile
214+
# We want to link aws-sdk-cpp statically but Homebrew's
215+
# aws-sdk-cpp provides only shared library. If we have
216+
# Homebrew's aws-sdk-cpp, our build mix Homebrew's
217+
# aws-sdk-cpp and bundled aws-sdk-cpp. We uninstall Homebrew's
218+
# aws-sdk-cpp to ensure using only bundled aws-sdk-cpp.
219+
brew uninstall aws-sdk-cpp
220+
# We want to use bundled RE2 for static linking. If
221+
# Homebrew's RE2 is installed, its header file may be used.
222+
# We uninstall Homebrew's RE2 to ensure using bundled RE2.
223+
brew uninstall grpc || : # gRPC depends on RE2
224+
brew uninstall [email protected] || : # gRPC 1.54 may be installed too
225+
brew uninstall re2
226+
# We want to use bundled Protobuf for static linking. If
227+
# Homebrew's Protobuf is installed, its library file may be
228+
# used on test We uninstall Homebrew's Protobuf to ensure using
229+
# bundled Protobuf.
230+
brew uninstall protobuf
231+
232+
brew bundle --file=Brewfile
233+
- name: Build C++ libraries
234+
run: |
235+
set -e
236+
# make brew Java available to CMake
237+
export JAVA_HOME=$(brew --prefix openjdk@11)/libexec/openjdk.jdk/Contents/Home
238+
ci/scripts/jni_macos_build.sh . arrow build jni
239+
- name: Compress into single artifact to keep directory structure
240+
run: tar -cvzf jni-macos-${{ matrix.platform.arch }}.tar.gz jni/
241+
- name: Upload artifacts
242+
uses: actions/upload-artifact@65c4c4a1ddee5b72f698fdd19549f0f0fb45cf08 # v4.6.0
243+
with:
244+
name: jni-macos-${{ matrix.platform.arch }}
245+
path: jni-macos-${{ matrix.platform.arch }}.tar.gz
246+
binaries:
247+
name: Binaries
248+
runs-on: ubuntu-latest
249+
needs:
250+
- jni-ubuntu
251+
- jni-macos
252+
steps:
253+
- name: Download artifacts
254+
uses: actions/download-artifact@fa0a91b85d4f404e444e00e005971372dc801d16 # v4.1.8
255+
with:
256+
path: artifacts
257+
- name: Decompress artifacts
258+
run: |
259+
mv artifacts/*/*.tar.gz .
260+
tar -xf apache-arrow-java-*.tar.gz --strip-components=1
261+
tar -xvzf jni-linux-x86_64.tar.gz
262+
# tar -xvzf jni-linux-aarch_64.tar.gz
263+
tar -xvzf jni-macos-x86_64.tar.gz
264+
tar -xvzf jni-macos-aarch_64.tar.gz
265+
# tar -xvzf jni-windows.tar.gz
266+
- name: Test that shared libraries exist
267+
run: |
268+
set -x
269+
270+
test -f dist/arrow_cdata_jni/x86_64/libarrow_cdata_jni.so
271+
test -f dist/arrow_dataset_jni/x86_64/libarrow_dataset_jni.so
272+
test -f dist/arrow_orc_jni/x86_64/libarrow_orc_jni.so
273+
test -f dist/gandiva_jni/x86_64/libgandiva_jni.so
274+
275+
# test -f dist/arrow_cdata_jni/aarch_64/libarrow_cdata_jni.so
276+
# test -f dist/arrow_dataset_jni/aarch_64/libarrow_dataset_jni.so
277+
# test -f dist/arrow_orc_jni/aarch_64/libarrow_orc_jni.so
278+
# test -f dist/gandiva_jni/aarch_64/libgandiva_jni.so
279+
280+
test -f dist/arrow_cdata_jni/x86_64/libarrow_cdata_jni.dylib
281+
test -f dist/arrow_dataset_jni/x86_64/libarrow_dataset_jni.dylib
282+
test -f dist/arrow_orc_jni/x86_64/libarrow_orc_jni.dylib
283+
test -f dist/gandiva_jni/x86_64/libgandiva_jni.dylib
284+
285+
test -f dist/arrow_cdata_jni/aarch_64/libarrow_cdata_jni.dylib
286+
test -f dist/arrow_dataset_jni/aarch_64/libarrow_dataset_jni.dylib
287+
test -f dist/arrow_orc_jni/aarch_64/libarrow_orc_jni.dylib
288+
test -f dist/gandiva_jni/aarch_64/libgandiva_jni.dylib
289+
290+
# test -f dist/arrow_cdata_jni/x86_64/arrow_cdata_jni.dll
291+
# test -f dist/arrow_dataset_jni/x86_64/arrow_dataset_jni.dll
292+
# test -f dist/arrow_orc_jni/x86_64/arrow_orc_jni.dll
293+
- name: Build bundled JAR
294+
env:
295+
MAVEN_ARGS: >-
296+
--no-transfer-progress
297+
run: |
298+
ci/scripts/jni_full_build.sh jni binaries
299+
- name: Upload artifacts
300+
uses: actions/upload-artifact@65c4c4a1ddee5b72f698fdd19549f0f0fb45cf08 # v4.6.0
301+
with:
302+
name: release-binaries
303+
path: binaries/*
78304
verify:
79305
name: Verify
80306
needs:
81-
- archive
307+
- binaries
82308
runs-on: ${{ matrix.os }}
83309
strategy:
84310
fail-fast: false
@@ -87,13 +313,10 @@ jobs:
87313
- macos-latest
88314
- ubuntu-latest
89315
steps:
90-
- name: Checkout
91-
uses: actions/checkout@11bd71901bbe5b1630ceea73d27597364c9af683 # v4.2.2
92-
with:
93-
submodules: recursive
94-
- uses: actions/download-artifact@fa0a91b85d4f404e444e00e005971372dc801d16 # v4.1.8
316+
- name: Download release artifacts
317+
uses: actions/download-artifact@fa0a91b85d4f404e444e00e005971372dc801d16 # v4.1.8
95318
with:
96-
name: archive
319+
pattern: release-*
97320
- name: Verify
98321
run: |
99322
tar_gz=$(echo apache-arrow-java-*.tar.gz)
@@ -105,9 +328,13 @@ jobs:
105328
else
106329
rc=$(date +%Y%m%d)
107330
fi
108-
VERIFY_DEFAULT=0 \
109-
VERIFY_SOURCE=1 \
110-
dev/release/verify_rc.sh "${version}" "${rc}"
331+
tar -xf ${tar_gz}
332+
export VERIFY_DEFAULT=0
333+
export VERIFY_BINARY=1
334+
export VERIFY_SOURCE=1
335+
cd apache-arrow-java-*
336+
mv ../${tar_gz} ./
337+
dev/release/verify_rc.sh "${version}" "${rc}"
111338
upload:
112339
name: Upload
113340
if: github.ref_type == 'tag'
@@ -117,13 +344,11 @@ jobs:
117344
permissions:
118345
contents: write
119346
steps:
120-
- name: Checkout
121-
uses: actions/checkout@11bd71901bbe5b1630ceea73d27597364c9af683 # v4.2.2
122-
with:
123-
submodules: recursive
124-
- uses: actions/download-artifact@fa0a91b85d4f404e444e00e005971372dc801d16 # v4.1.8
347+
- name: Download release artifacts
348+
uses: actions/download-artifact@fa0a91b85d4f404e444e00e005971372dc801d16 # v4.1.8
125349
with:
126-
name: archive
350+
pattern: release-*
351+
path: artifacts
127352
- name: Upload
128353
run: |
129354
# GH-499: How to create release notes?
@@ -133,9 +358,9 @@ jobs:
133358
gh release create ${GITHUB_REF_NAME} \
134359
--generate-notes \
135360
--prerelease \
361+
--repo ${{ github.repository }} \
136362
--title "Apache Arrow Java ${version} RC${rc}" \
137363
--verify-tag \
138-
apache-arrow-java-*.tar.gz \
139-
apache-arrow-java-*.tar.gz.sha*
364+
artifacts/*
140365
env:
141366
GH_TOKEN: ${{ secrets.GITHUB_TOKEN }}

0 commit comments

Comments
 (0)