Skip to content

Commit 3fd85df

Browse files
koukou
committed
XXX
1 parent 1d4b757 commit 3fd85df

File tree

8 files changed

+316
-78
lines changed

8 files changed

+316
-78
lines changed

‎.github/workflows/rc.yml‎

Lines changed: 241 additions & 20 deletions
Original file line numberDiff line numberDiff line change
@@ -70,15 +70,237 @@ jobs:
7070
- name: Audit
7171
run: |
7272
dev/release/run_rat.sh "${TAR_GZ}"
73-
- uses: actions/upload-artifact@6f51ac03b9356f520e9adb1b1b7802705f340c2b # v4.5.0
73+
- name: Upload source archive
74+
uses: actions/upload-artifact@65c4c4a1ddee5b72f698fdd19549f0f0fb45cf08 # v4.6.0
7475
with:
75-
name: archive
76+
name: release-source
7677
path: |
7778
apache-arrow-java-*
79+
jni-ubuntu:
80+
name: JNI ${{ matrix.platform.runs_on }} ${{ matrix.platform.arch }}
81+
runs-on: ${{ matrix.platform.runs_on }}
82+
strategy:
83+
fail-fast: false
84+
matrix:
85+
platform:
86+
- runs_on: ubuntu-latest
87+
arch: "x86_64"
88+
archery_arch: "amd64"
89+
env:
90+
# architecture name used for archery build
91+
ARCH: ${{ matrix.platform.archery_arch }}
92+
permissions:
93+
contents: read
94+
packages: write
95+
steps:
96+
- name: Download source archive
97+
uses: actions/download-artifact@fa0a91b85d4f404e444e00e005971372dc801d16 # v4.1.8
98+
with:
99+
name: release-source
100+
- name: Extract Download the latest Apache Arrow C++
101+
run: |
102+
tar -xf apache-arrow-java-*.tar.gz --strip-components=1
103+
- name: Download the latest Apache Arrow C++
104+
run: |
105+
ci/scripts/download_cpp.sh
106+
- name: Checkout apache/arrow-testing
107+
uses: actions/checkout@11bd71901bbe5b1630ceea73d27597364c9af683 # v4.2.2
108+
with:
109+
repo: apache/arrow-testing
110+
path: arrow/testing
111+
- name: Checkout apache/parquet-testing
112+
uses: actions/checkout@11bd71901bbe5b1630ceea73d27597364c9af683 # v4.2.2
113+
with:
114+
repo: apache/parquet-testing
115+
path: cpp/submodules/parquet-testing
116+
- uses: docker/login-action@9780b0c442fbb1117ed29e0efdff1e18412f7567 # v3.3.0
117+
with:
118+
registry: ghcr.io
119+
username: ${{ github.actor }}
120+
password: ${{ secrets.GITHUB_TOKEN }}
121+
- name: Build C++ libraries
122+
run: |
123+
docker compose run vcpkg-jni
124+
- name: Push Docker image
125+
if: success() && github.event_name == 'push' && github.repository == 'apache/arrow-java' && github.ref_name == 'main'
126+
run: |
127+
docker compose push vcpkg-jni
128+
- name: Compress into single artifact to keep directory structure
129+
run: tar -cvzf jni-linux-${{ matrix.platform.arch }}.tar.gz dist/
130+
- name: Upload artifacts
131+
uses: actions/upload-artifact@v4
132+
with:
133+
name: jni-linux-${{ matrix.platform.arch }}
134+
path: jni-linux-${{ matrix.platform.arch }}.tar.gz
135+
jni-macos:
136+
name: JNI ${{ matrix.platform.runs_on }} ${{ matrix.platform.arch }}
137+
runs-on: ${{ matrix.platform.runs_on }}
138+
strategy:
139+
fail-fast: false
140+
matrix:
141+
platform:
142+
- { runs_on: macos-13, arch: "x86_64"}
143+
- { runs_on: macos-14, arch: "aarch_64" }
144+
env:
145+
MACOSX_DEPLOYMENT_TARGET: "14.0"
146+
steps:
147+
- name: Download source archive
148+
uses: actions/download-artifact@fa0a91b85d4f404e444e00e005971372dc801d16 # v4.1.8
149+
with:
150+
name: release-source
151+
- name: Extract Download the latest Apache Arrow C++
152+
run: |
153+
tar -xf apache-arrow-java-*.tar.gz --strip-components=1
154+
- name: Download the latest Apache Arrow C++
155+
run: |
156+
ci/scripts/download_cpp.sh
157+
- name: Checkout apache/arrow-testing
158+
uses: actions/checkout@11bd71901bbe5b1630ceea73d27597364c9af683 # v4.2.2
159+
with:
160+
repo: apache/arrow-testing
161+
path: arrow/testing
162+
- name: Checkout apache/parquet-testing
163+
uses: actions/checkout@11bd71901bbe5b1630ceea73d27597364c9af683 # v4.2.2
164+
with:
165+
repo: apache/parquet-testing
166+
path: cpp/submodules/parquet-testing
167+
- name: Set up Python
168+
uses: actions/setup-python@v5
169+
with:
170+
cache: 'pip'
171+
python-version: 3.12
172+
- name: Install Archery
173+
run: pip install -e arrow/dev/archery[all]
174+
- name: Install dependencies
175+
run: |
176+
# We want to use llvm@14 to avoid shared z3
177+
# dependency. llvm@14 doesn't depend on z3 and llvm depends
178+
# on z3. And Homebrew's z3 provides only shared library. It
179+
# doesn't provides static z3 because z3's CMake doesn't accept
180+
# building both shared and static libraries at once.
181+
# See also: Z3_BUILD_LIBZ3_SHARED in
182+
# https://github.com/Z3Prover/z3/blob/master/README-CMake.md
183+
#
184+
# If llvm is installed, Apache Arrow C++ uses llvm rather than
185+
# llvm@14 because llvm is newer than llvm@14.
186+
brew uninstall llvm || :
187+
188+
# Ensure updating python@XXX with the "--overwrite" option.
189+
# If python@XXX is updated without "--overwrite", it causes
190+
# a conflict error. Because Python 3 installed not by
191+
# Homebrew exists in /usr/local on GitHub Actions. If
192+
# Homebrew's python@XXX is updated without "--overwrite", it
193+
# tries to replace /usr/local/bin/2to3 and so on and causes
194+
# a conflict error.
195+
brew update
196+
for python_package in $(brew list | grep python@); do
197+
brew install --overwrite ${python_package}
198+
done
199+
brew install --overwrite python
200+
201+
if [ "$(uname -m)" = "arm64" ]; then
202+
# pkg-config formula is deprecated but it's still installed
203+
# in GitHub Actions runner now. We can remove this once
204+
# pkg-config formula is removed from GitHub Actions runner.
205+
brew uninstall pkg-config || :
206+
brew uninstall [email protected] || :
207+
fi
208+
209+
brew bundle --file=arrow/cpp/Brewfile
210+
# We want to link aws-sdk-cpp statically but Homebrew's
211+
# aws-sdk-cpp provides only shared library. If we have
212+
# Homebrew's aws-sdk-cpp, our build mix Homebrew's
213+
# aws-sdk-cpp and bundled aws-sdk-cpp. We uninstall Homebrew's
214+
# aws-sdk-cpp to ensure using only bundled aws-sdk-cpp.
215+
brew uninstall aws-sdk-cpp
216+
# We want to use bundled RE2 for static linking. If
217+
# Homebrew's RE2 is installed, its header file may be used.
218+
# We uninstall Homebrew's RE2 to ensure using bundled RE2.
219+
brew uninstall grpc || : # gRPC depends on RE2
220+
brew uninstall [email protected] || : # gRPC 1.54 may be installed too
221+
brew uninstall re2
222+
# We want to use bundled Protobuf for static linking. If
223+
# Homebrew's Protobuf is installed, its library file may be
224+
# used on test We uninstall Homebrew's Protobuf to ensure using
225+
# bundled Protobuf.
226+
brew uninstall protobuf
227+
228+
brew bundle --file=Brewfile
229+
- name: Build C++ libraries
230+
run: |
231+
set -e
232+
# make brew Java available to CMake
233+
export JAVA_HOME=$(brew --prefix openjdk@11)/libexec/openjdk.jdk/Contents/Home
234+
ci/scripts/jni_macos_build.sh . arrow build jni
235+
- name: Compress into single artifact to keep directory structure
236+
run: tar -cvzf jni-macos-${{ matrix.platform.arch }}.tar.gz jni/
237+
- name: Upload artifacts
238+
uses: actions/upload-artifact@65c4c4a1ddee5b72f698fdd19549f0f0fb45cf08 # v4.6.0
239+
with:
240+
name: jni-macos-${{ matrix.platform.arch }}
241+
path: jni-macos-${{ matrix.platform.arch }}.tar.gz
242+
jar:
243+
name: Build JAR files
244+
runs-on: ubuntu-latest
245+
needs:
246+
- jni-ubuntu
247+
- jni-macos
248+
steps:
249+
- name: Download artifacts
250+
uses: actions/download-artifact@fa0a91b85d4f404e444e00e005971372dc801d16 # v4.1.8
251+
with:
252+
path: artifacts
253+
- name: Decompress artifacts
254+
run: |
255+
mv artifacts/*/*.tar.gz .
256+
tar -xf apache-arrow-java-*.tar.gz --strip-components=1
257+
tar -xvzf jni-linux-x86_64.tar.gz
258+
# tar -xvzf jni-linux-aarch_64.tar.gz
259+
tar -xvzf jni-macos-x86_64.tar.gz
260+
tar -xvzf jni-macos-aarch_64.tar.gz
261+
# tar -xvzf jni-windows.tar.gz
262+
- name: Test that shared libraries exist
263+
run: |
264+
set -x
265+
266+
test -f dist/arrow_cdata_jni/x86_64/libarrow_cdata_jni.so
267+
test -f dist/arrow_dataset_jni/x86_64/libarrow_dataset_jni.so
268+
test -f dist/arrow_orc_jni/x86_64/libarrow_orc_jni.so
269+
test -f dist/gandiva_jni/x86_64/libgandiva_jni.so
270+
271+
# test -f dist/arrow_cdata_jni/aarch_64/libarrow_cdata_jni.so
272+
# test -f dist/arrow_dataset_jni/aarch_64/libarrow_dataset_jni.so
273+
# test -f dist/arrow_orc_jni/aarch_64/libarrow_orc_jni.so
274+
# test -f dist/gandiva_jni/aarch_64/libgandiva_jni.so
275+
276+
test -f dist/arrow_cdata_jni/x86_64/libarrow_cdata_jni.dylib
277+
test -f dist/arrow_dataset_jni/x86_64/libarrow_dataset_jni.dylib
278+
test -f dist/arrow_orc_jni/x86_64/libarrow_orc_jni.dylib
279+
test -f dist/gandiva_jni/x86_64/libgandiva_jni.dylib
280+
281+
test -f dist/arrow_cdata_jni/aarch_64/libarrow_cdata_jni.dylib
282+
test -f dist/arrow_dataset_jni/aarch_64/libarrow_dataset_jni.dylib
283+
test -f dist/arrow_orc_jni/aarch_64/libarrow_orc_jni.dylib
284+
test -f dist/gandiva_jni/aarch_64/libgandiva_jni.dylib
285+
286+
# test -f dist/arrow_cdata_jni/x86_64/arrow_cdata_jni.dll
287+
# test -f dist/arrow_dataset_jni/x86_64/arrow_dataset_jni.dll
288+
# test -f dist/arrow_orc_jni/x86_64/arrow_orc_jni.dll
289+
- name: Build bundled JAR
290+
env:
291+
MAVEN_ARGS: >-
292+
--no-transfer-progress
293+
run: |
294+
ci/scripts/jni_full_build.sh jni binaries
295+
- name: Upload artifacts
296+
uses: actions/upload-artifact@65c4c4a1ddee5b72f698fdd19549f0f0fb45cf08 # v4.6.0
297+
with:
298+
name: release-binaries
299+
path: binaries/*
78300
verify:
79301
name: Verify
80302
needs:
81-
- archive
303+
- jar
82304
runs-on: ${{ matrix.os }}
83305
strategy:
84306
fail-fast: false
@@ -87,13 +309,10 @@ jobs:
87309
- macos-latest
88310
- ubuntu-latest
89311
steps:
90-
- name: Checkout
91-
uses: actions/checkout@11bd71901bbe5b1630ceea73d27597364c9af683 # v4.2.2
92-
with:
93-
submodules: recursive
94-
- uses: actions/download-artifact@fa0a91b85d4f404e444e00e005971372dc801d16 # v4.1.8
312+
- name: Download release artifacts
313+
uses: actions/download-artifact@fa0a91b85d4f404e444e00e005971372dc801d16 # v4.1.8
95314
with:
96-
name: archive
315+
pattern: release-*
97316
- name: Verify
98317
run: |
99318
tar_gz=$(echo apache-arrow-java-*.tar.gz)
@@ -105,9 +324,13 @@ jobs:
105324
else
106325
rc=$(date +%Y%m%d)
107326
fi
108-
VERIFY_DEFAULT=0 \
109-
VERIFY_SOURCE=1 \
110-
dev/release/verify_rc.sh "${version}" "${rc}"
327+
tar -xf ${tar_gz}
328+
export VERIFY_DEFAULT=0
329+
export VERIFY_BINARY=1
330+
export VERIFY_SOURCE=1
331+
cd apache-arrow-java-*
332+
mv ../${tar_gz} ./
333+
dev/release/verify_rc.sh "${version}" "${rc}"
111334
upload:
112335
name: Upload
113336
if: github.ref_type == 'tag'
@@ -117,13 +340,11 @@ jobs:
117340
permissions:
118341
contents: write
119342
steps:
120-
- name: Checkout
121-
uses: actions/checkout@11bd71901bbe5b1630ceea73d27597364c9af683 # v4.2.2
122-
with:
123-
submodules: recursive
124-
- uses: actions/download-artifact@fa0a91b85d4f404e444e00e005971372dc801d16 # v4.1.8
343+
- name: Download release artifacts
344+
uses: actions/download-artifact@fa0a91b85d4f404e444e00e005971372dc801d16 # v4.1.8
125345
with:
126-
name: archive
346+
pattern: release-*
347+
path: artifacts
127348
- name: Upload
128349
run: |
129350
# GH-499: How to create release notes?
@@ -133,9 +354,9 @@ jobs:
133354
gh release create ${GITHUB_REF_NAME} \
134355
--generate-notes \
135356
--prerelease \
357+
--repo ${{ github.repository }} \
136358
--title "Apache Arrow Java ${version} RC${rc}" \
137359
--verify-tag \
138-
apache-arrow-java-*.tar.gz \
139-
apache-arrow-java-*.tar.gz.sha*
360+
artifacts/*
140361
env:
141362
GH_TOKEN: ${{ secrets.GITHUB_TOKEN }}

‎ci/scripts/build.sh‎

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -1,4 +1,5 @@
11
#!/usr/bin/env bash
2+
#
23
# Licensed to the Apache Software Foundation (ASF) under one
34
# or more contributor license agreements. See the NOTICE file
45
# distributed with this work for additional information

‎ci/scripts/download_cpp.sh‎

Lines changed: 30 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,30 @@
1+
#!/usr/bin/env bash
2+
#
3+
# Licensed to the Apache Software Foundation (ASF) under one
4+
# or more contributor license agreements. See the NOTICE file
5+
# distributed with this work for additional information
6+
# regarding copyright ownership. The ASF licenses this file
7+
# to you under the Apache License, Version 2.0 (the
8+
# "License"); you may not use this file except in compliance
9+
# with the License. You may obtain a copy of the License at
10+
#
11+
# http://www.apache.org/licenses/LICENSE-2.0
12+
#
13+
# Unless required by applicable law or agreed to in writing,
14+
# software distributed under the License is distributed on an
15+
# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
16+
# KIND, either express or implied. See the License for the
17+
# specific language governing permissions and limitations
18+
# under the License.
19+
20+
set -euxo pipefail
21+
22+
latest_version=$(curl \
23+
https://raw.githubusercontent.com/apache/arrow-site/refs/heads/main/_data/versions.yml |
24+
grep '^ number:' |
25+
sed -E -e "s/^ number: '|'$//g")
26+
wget \
27+
--output-document=apache-arrow-${latest_version}.tar.gz \
28+
"https://www.apache.org/dyn/closer.lua?action=download&filename=arrow/arrow-${latest_version}/apache-arrow-${latest_version}.tar.gz"
29+
tar xf apache-arrow-${latest_version}.tar.gz
30+
mv apache-arrow-${latest_version} arrow

0 commit comments

Comments
 (0)