Skip to content

Commit a5c44d7

Browse files
author
unknown
committed
1 parent 44bd61e commit a5c44d7

File tree

5 files changed

+370
-151
lines changed

5 files changed

+370
-151
lines changed

‎.circleci/config.yml‎

Lines changed: 16 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,16 @@
1+
2+
version: 2
3+
4+
jobs:
5+
build:
6+
machine: true
7+
8+
workflows:
9+
version: 2
10+
build:
11+
jobs:
12+
- build:
13+
filters:
14+
branches:
15+
only:
16+
- /.*-circle-.*/

‎.github/workflows/crossbow.yml‎

Lines changed: 345 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,345 @@
1+
# Licensed to the Apache Software Foundation (ASF) under one
2+
# or more contributor license agreements. See the NOTICE file
3+
# distributed with this work for additional information
4+
# regarding copyright ownership. The ASF licenses this file
5+
# to you under the Apache License, Version 2.0 (the
6+
# "License"); you may not use this file except in compliance
7+
# with the License. You may obtain a copy of the License at
8+
#
9+
# http://www.apache.org/licenses/LICENSE-2.0
10+
#
11+
# Unless required by applicable law or agreed to in writing,
12+
# software distributed under the License is distributed on an
13+
# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
14+
# KIND, either express or implied. See the License for the
15+
# specific language governing permissions and limitations
16+
# under the License.
17+
18+
19+
# NOTE: must set "Crossbow" as name to have the badge links working in the
20+
# github comment reports!
21+
name: Crossbow
22+
on:
23+
push:
24+
branches:
25+
- "*-github-*"
26+
27+
env:
28+
ARCHERY_DEBUG: 1
29+
30+
31+
jobs:
32+
33+
build-cpp-ubuntu:
34+
name: Build C++ libraries Ubuntu ${{ matrix.platform.arch }}
35+
runs-on: ${{ matrix.platform.runs_on }}
36+
env:
37+
# architecture name used for archery build
38+
ARCH: ${{ matrix.platform.archery_arch }}
39+
ARCH_ALIAS: ${{ matrix.platform.archery_arch_alias }}
40+
ARCH_SHORT: ${{ matrix.platform.archery_arch_short }}
41+
strategy:
42+
fail-fast: false
43+
matrix:
44+
platform:
45+
- runs_on: ["ubuntu-latest"]
46+
arch: "x86_64"
47+
archery_arch: "amd64"
48+
archery_arch_alias: "x86_64"
49+
archery_arch_short: "amd64"
50+
- runs_on: ["buildjet-8vcpu-ubuntu-2204-arm"]
51+
arch: "aarch_64"
52+
archery_arch: "arm64v8"
53+
archery_arch_alias: "aarch64"
54+
archery_arch_short: "arm64"
55+
steps:
56+
- name: Checkout Arrow
57+
uses: actions/checkout@v4
58+
with:
59+
fetch-depth: 1
60+
path: arrow
61+
repository: akravchukdremio/arrow
62+
ref: 021b3b923a12e6aeaa672a0d44e2fbb267fc4a95
63+
submodules: recursive
64+
65+
- name: Free up disk space
66+
if: runner.os == 'Linux' && runner.arch == 'X64'
67+
shell: bash
68+
run: |
69+
arrow/ci/scripts/util_free_space.sh
70+
71+
- name: Set up Python by actions/setup-python
72+
if: |
73+
!(runner.os == 'Linux' && runner.arch != 'X64')
74+
uses: actions/setup-python@v4
75+
with:
76+
cache: 'pip'
77+
python-version: 3.12
78+
- name: Set up Python by apt
79+
if: runner.os == 'Linux' && runner.arch != 'X64'
80+
run: |
81+
sudo apt update
82+
sudo apt-get install -y python3-pip
83+
pip install -U pip
84+
echo "$HOME/.local/bin" >>"$GITHUB_PATH"
85+
- name: Install Archery
86+
shell: bash
87+
run: pip install -e arrow/dev/archery[all]
88+
89+
- name: Free up disk space
90+
run: |
91+
df -h
92+
sudo rm -rf /opt/hostedtoolcache/CodeQL || :
93+
df -h
94+
- name: Set Swap Space
95+
continue-on-error: true
96+
shell: bash
97+
run: |
98+
sudo fallocate -l 16G /swapfile
99+
sudo chmod 600 /swapfile
100+
sudo mkswap /swapfile
101+
sudo swapon /swapfile
102+
- name: Build C++ libraries
103+
env:
104+
AWS_ACCESS_KEY_ID: "${{ secrets.AWS_ACCESS_KEY_ID }}"
105+
AWS_SECRET_ACCESS_KEY: "${{ secrets.AWS_SECRET_ACCESS_KEY }}"
106+
SCCACHE_BUCKET: "${{ secrets.SCCACHE_BUCKET }}"
107+
SCCACHE_REGION: "${{ secrets.SCCACHE_REGION }}"
108+
SCCACHE_S3_KEY_PREFIX: "sccache"
109+
110+
run: |
111+
archery docker run \
112+
-e ARROW_JAVA_BUILD=OFF \
113+
-e ARROW_JAVA_TEST=OFF \
114+
java-jni-manylinux-2014
115+
- name: Compress into single artifact to keep directory structure
116+
run: tar -cvzf arrow-shared-libs-linux-${{ matrix.platform.arch }}.tar.gz arrow/java-dist/
117+
- name: Upload artifacts
118+
uses: actions/upload-artifact@v4
119+
with:
120+
name: ubuntu-shared-lib-${{ matrix.platform.arch }}
121+
path: arrow-shared-libs-linux-${{ matrix.platform.arch }}.tar.gz
122+
123+
build-cpp-macos:
124+
name: Build C++ libraries macOS ${{ matrix.platform.arch }}
125+
runs-on: ${{ matrix.platform.runs_on }}
126+
strategy:
127+
fail-fast: false
128+
matrix:
129+
platform:
130+
- { runs_on: ["macos-15-intel"], arch: "x86_64"}
131+
env:
132+
MACOSX_DEPLOYMENT_TARGET: "12.0"
133+
steps:
134+
- name: Checkout Arrow
135+
uses: actions/checkout@v4
136+
with:
137+
fetch-depth: 1
138+
path: arrow
139+
repository: akravchukdremio/arrow
140+
ref: 021b3b923a12e6aeaa672a0d44e2fbb267fc4a95
141+
submodules: recursive
142+
143+
- name: Set up Python
144+
uses: actions/setup-python@v4
145+
with:
146+
cache: 'pip'
147+
python-version: 3.12
148+
- name: Install Archery
149+
shell: bash
150+
run: pip install -e arrow/dev/archery[all]
151+
- name: Install dependencies
152+
run: |
153+
# We want to use llvm@14 to avoid shared z3
154+
# dependency. llvm@14 doesn't depend on z3 and llvm depends
155+
# on z3. And Homebrew's z3 provides only shared library. It
156+
# doesn't provides static z3 because z3's CMake doesn't accept
157+
# building both shared and static libraries at once.
158+
# See also: Z3_BUILD_LIBZ3_SHARED in
159+
# https://github.com/Z3Prover/z3/blob/master/README-CMake.md
160+
#
161+
# If llvm is installed, Apache Arrow C++ uses llvm rather than
162+
# llvm@14 because llvm is newer than llvm@14.
163+
brew uninstall llvm || :
164+
165+
# Ensure updating python@XXX with the "--overwrite" option.
166+
# If python@XXX is updated without "--overwrite", it causes
167+
# a conflict error. Because Python 3 installed not by
168+
# Homebrew exists in /usr/local on GitHub Actions. If
169+
# Homebrew's python@XXX is updated without "--overwrite", it
170+
# tries to replace /usr/local/bin/2to3 and so on and causes
171+
# a conflict error.
172+
# brew update
173+
for python_package in $(brew list | grep python@); do
174+
brew install --overwrite ${python_package}
175+
done
176+
brew install --overwrite python
177+
178+
brew bundle --file=arrow/cpp/Brewfile
179+
# We want to link aws-sdk-cpp statically but Homebrew's
180+
# aws-sdk-cpp provides only shared library. If we have
181+
# Homebrew's aws-sdk-cpp, our build mix Homebrew's
182+
# aws-sdk-cpp and bundled aws-sdk-cpp. We uninstall Homebrew's
183+
# aws-sdk-cpp to ensure using only bundled aws-sdk-cpp.
184+
brew uninstall aws-sdk-cpp
185+
# We want to use bundled RE2 for static linking. If
186+
# Homebrew's RE2 is installed, its header file may be used.
187+
# We uninstall Homebrew's RE2 to ensure using bundled RE2.
188+
brew uninstall grpc || : # gRPC depends on RE2
189+
brew uninstall re2
190+
# We want to use bundled Protobuf for static linking. If
191+
# Homebrew's Protobuf is installed, its library file may be
192+
# used on test We uninstall Homebrew's Protobuf to ensure using
193+
# bundled Protobuf.
194+
brew uninstall protobuf
195+
# fix cmake and boost versionsAdd commentMore actions
196+
brew uninstall -f boost || true
197+
brew uninstall -f cmake || true
198+
mkdir -p homebrew-custom/Formula
199+
curl -o homebrew-custom/Formula/cmake.rb https://raw.githubusercontent.com/Homebrew/homebrew-core/f68532bfe5cb87474093df8a839c3818c6aa44dd/Formula/c/cmake.rb
200+
curl -o homebrew-custom/Formula/boost.rb https://raw.githubusercontent.com/Homebrew/homebrew-core/23f9c56c5075dd56b4471e2c93f89f6400b49ddd/Formula/b/boost.rb
201+
brew tap-new local/homebrew-custom
202+
cp ./homebrew-custom/Formula/*.rb "$(brew --repo local/homebrew-custom)/Formula/"
203+
brew install -v local/homebrew-custom/cmake
204+
brew install -v local/homebrew-custom/boost
205+
brew pin cmake
206+
brew pin boost
207+
#
208+
209+
210+
brew bundle --file=arrow/java/Brewfile
211+
- name: Build C++ libraries
212+
env:
213+
ARROW_USE_CCACHE: "ON"
214+
run: |
215+
set -e
216+
# make brew Java available to CMake
217+
export JAVA_HOME=$(brew --prefix openjdk@11)/libexec/openjdk.jdk/Contents/Home
218+
arrow/ci/scripts/java_jni_macos_build.sh \
219+
$GITHUB_WORKSPACE/arrow \
220+
$GITHUB_WORKSPACE/arrow/cpp-build \
221+
$GITHUB_WORKSPACE/arrow/java-dist
222+
- name: Compress into single artifact to keep directory structure
223+
run: tar -cvzf arrow-shared-libs-macos-${{ matrix.platform.arch }}.tar.gz arrow/java-dist/
224+
- name: Upload artifacts
225+
uses: actions/upload-artifact@v4
226+
with:
227+
name: macos-shared-lib-${{ matrix.platform.arch }}
228+
path: arrow-shared-libs-macos-${{ matrix.platform.arch }}.tar.gz
229+
230+
package-jars:
231+
name: Build jar files
232+
runs-on: ${{ matrix.platform.runs_on }}
233+
strategy:
234+
fail-fast: false
235+
matrix:
236+
platform:
237+
- { runs_on: ["macos-15-intel"], arch: "x86_64"}
238+
needs:
239+
- build-cpp-ubuntu
240+
- build-cpp-macos
241+
steps:
242+
- name: Checkout Arrow
243+
uses: actions/checkout@v4
244+
with:
245+
fetch-depth: 0
246+
path: arrow
247+
repository: akravchukdremio/arrow
248+
ref: 021b3b923a12e6aeaa672a0d44e2fbb267fc4a95
249+
submodules: recursive
250+
251+
- name: Download Libraries
252+
uses: actions/download-artifact@v4
253+
with:
254+
path: artifacts
255+
- name: Decompress artifacts
256+
run: |
257+
mv artifacts/*/*.tar.gz .
258+
tar -xvzf arrow-shared-libs-linux-x86_64.tar.gz
259+
tar -xvzf arrow-shared-libs-linux-aarch_64.tar.gz
260+
tar -xvzf arrow-shared-libs-macos-x86_64.tar.gz
261+
- name: Test that shared libraries exist
262+
run: |
263+
set -x
264+
265+
test -f arrow/java-dist/arrow_cdata_jni/x86_64/libarrow_cdata_jni.so
266+
test -f arrow/java-dist/arrow_dataset_jni/x86_64/libarrow_dataset_jni.so
267+
test -f arrow/java-dist/arrow_orc_jni/x86_64/libarrow_orc_jni.so
268+
test -f arrow/java-dist/gandiva_jni/x86_64/libgandiva_jni.so
269+
270+
test -f arrow/java-dist/arrow_cdata_jni/aarch_64/libarrow_cdata_jni.so
271+
test -f arrow/java-dist/arrow_dataset_jni/aarch_64/libarrow_dataset_jni.so
272+
test -f arrow/java-dist/arrow_orc_jni/aarch_64/libarrow_orc_jni.so
273+
test -f arrow/java-dist/gandiva_jni/aarch_64/libgandiva_jni.so
274+
275+
test -f arrow/java-dist/arrow_cdata_jni/x86_64/libarrow_cdata_jni.dylib
276+
test -f arrow/java-dist/arrow_dataset_jni/x86_64/libarrow_dataset_jni.dylib
277+
test -f arrow/java-dist/arrow_orc_jni/x86_64/libarrow_orc_jni.dylib
278+
test -f arrow/java-dist/gandiva_jni/x86_64/libgandiva_jni.dylib
279+
- name: Build bundled jar
280+
env:
281+
MAVEN_ARGS: >-
282+
--no-transfer-progress
283+
run: |
284+
set -e
285+
pushd arrow/java
286+
mvn versions:set -DnewVersion=18.1.2-SNAPSHOT
287+
mvn versions:set -DnewVersion=18.1.2-SNAPSHOT -f bom
288+
popd
289+
arrow/ci/scripts/java_full_build.sh \
290+
$GITHUB_WORKSPACE/arrow \
291+
$GITHUB_WORKSPACE/arrow/java-dist
292+
- name: Set up Python by actions/setup-python
293+
if: |
294+
!(runner.os == 'Linux' && runner.arch != 'X64')
295+
uses: actions/setup-python@v4
296+
with:
297+
python-version: 3.12
298+
- name: Set up Python by apt
299+
if: runner.os == 'Linux' && runner.arch != 'X64'
300+
run: |
301+
sudo apt update
302+
sudo apt install -y \
303+
libgit2-dev \
304+
libpython3-dev \
305+
python3-pip
306+
sudo python3 -m pip install --upgrade pip
307+
- name: Checkout Crossbow
308+
uses: actions/checkout@v4
309+
with:
310+
path: crossbow
311+
ref: nightly-334
312+
- name: Setup Crossbow
313+
shell: bash
314+
run: |
315+
python3 -m pip install -e arrow/dev/archery[crossbow]
316+
echo "$HOME/.local/bin" >> $GITHUB_PATH
317+
- name: Upload artifacts
318+
shell: bash
319+
run: |
320+
archery crossbow \
321+
--queue-path $(pwd)/crossbow \
322+
--queue-remote https://github.com/Vijeth-test/arrow-build \
323+
upload-artifacts \
324+
--sha nightly-334-github-java-jars \
325+
--tag nightly-334-github-java-jars \
326+
"arrow/java-dist/*.jar" \
327+
"arrow/java-dist/*.json" \
328+
"arrow/java-dist/*.pom" \
329+
"arrow/java-dist/*.xml" \
330+
"arrow/java-dist/*.zip"
331+
env:
332+
CROSSBOW_GITHUB_TOKEN: ${{ secrets.CROSSBOW_GITHUB_TOKEN || secrets.GITHUB_TOKEN }}
333+
- name: Verify uploaded artifacts
334+
shell: bash
335+
run: |
336+
archery crossbow \
337+
--queue-path $(pwd)/crossbow \
338+
--queue-remote https://github.com/Vijeth-test/arrow-build \
339+
status \
340+
--task-filter 'java-jars' \
341+
--no-fetch \
342+
--validate \
343+
nightly-334
344+
env:
345+
CROSSBOW_GITHUB_TOKEN: ${{ secrets.CROSSBOW_GITHUB_TOKEN || secrets.GITHUB_TOKEN }}

0 commit comments

Comments
 (0)