Skip to content

Commit 7e85651

Browse files
hcho3hcho3
authored
[CI] Add CI pipeline to build libxgboost4j.so targeting Linux ARM64 (dmlc#10487) (dmlc#10489)
1 parent b48bf52 commit 7e85651

9 files changed

+158
-43
lines changed

‎dev/prepare_jvm_release.py

Lines changed: 9 additions & 16 deletions
Original file line numberDiff line numberDiff line change
@@ -134,6 +134,7 @@ def main():
134134
print("====Creating directories to hold native binaries====")
135135
for os_ident, arch in [
136136
("linux", "x86_64"),
137+
("linux", "aarch64"),
137138
("windows", "x86_64"),
138139
("macos", "x86_64"),
139140
("macos", "aarch64"),
@@ -156,6 +157,14 @@ def main():
156157
url=f"{nightly_bucket_prefix}/{git_branch}/libxgboost4j/xgboost4j_{commit_hash}.dll",
157158
filename="xgboost4j/src/main/resources/lib/windows/x86_64/xgboost4j.dll",
158159
)
160+
retrieve(
161+
url=f"{nightly_bucket_prefix}/{git_branch}/libxgboost4j/libxgboost4j_linux_x86_64_{commit_hash}.so",
162+
filename="xgboost4j/src/main/resources/lib/linux/x86_64/libxgboost4j.so",
163+
)
164+
retrieve(
165+
url=f"{nightly_bucket_prefix}/{git_branch}/libxgboost4j/libxgboost4j_linux_arm64_{commit_hash}.so",
166+
filename="xgboost4j/src/main/resources/lib/linux/aarch64/libxgboost4j.so",
167+
)
159168
retrieve(
160169
url=f"{nightly_bucket_prefix}/{git_branch}/libxgboost4j/libxgboost4j_{commit_hash}.dylib",
161170
filename="xgboost4j/src/main/resources/lib/macos/x86_64/libxgboost4j.dylib",
@@ -166,22 +175,6 @@ def main():
166175
)
167176

168177
with tempfile.TemporaryDirectory() as tempdir:
169-
# libxgboost4j.so for Linux x86_64, CPU only
170-
zip_path = os.path.join(tempdir, "xgboost4j_2.12.jar")
171-
extract_dir = os.path.join(tempdir, "xgboost4j")
172-
retrieve(
173-
url=f"{maven_repo_prefix}/xgboost4j_2.12/{version}/"
174-
f"xgboost4j_2.12-{version}.jar",
175-
filename=zip_path,
176-
)
177-
os.mkdir(extract_dir)
178-
with zipfile.ZipFile(zip_path, "r") as t:
179-
t.extractall(extract_dir)
180-
cp(
181-
os.path.join(extract_dir, "lib", "linux", "x86_64", "libxgboost4j.so"),
182-
"xgboost4j/src/main/resources/lib/linux/x86_64/libxgboost4j.so",
183-
)
184-
185178
# libxgboost4j.so for Linux x86_64, GPU support
186179
zip_path = os.path.join(tempdir, "xgboost4j-gpu_2.12.jar")
187180
extract_dir = os.path.join(tempdir, "xgboost4j-gpu")

‎tests/buildkite/build-jvm-linux-arm64-manylinux2014.sh

Lines changed: 29 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,29 @@
1+
#!/bin/bash
2+
3+
set -euo pipefail
4+
5+
source tests/buildkite/conftest.sh
6+
7+
command_wrapper="tests/ci_build/ci_build.sh jvm_manylinux2014_aarch64"
8+
9+
# Build XGBoost4J binary
10+
echo "--- Build libxgboost4j.so (targeting glibc 2.17)"
11+
set -x
12+
mkdir build
13+
$command_wrapper bash -c \
14+
"cd build && cmake .. -DJVM_BINDINGS=ON -DUSE_OPENMP=ON && make -j$(nproc)"
15+
ldd lib/libxgboost4j.so
16+
objdump -T lib/libxgboost4j.so | grep GLIBC_ | sed 's/.*GLIBC_\([.0-9]*\).*/\1/g' | sort -Vu
17+
18+
echo "--- Upload libxgboost4j.so"
19+
pushd lib
20+
libname=libxgboost4j_linux_arm64_${BUILDKITE_COMMIT}.so
21+
mv -v libxgboost4j.so ${libname}
22+
buildkite-agent artifact upload ${libname}
23+
#if [[ ($is_pull_request == 0) && ($is_release_branch == 1) ]]
24+
#then
25+
aws s3 cp ${libname} \
26+
s3://xgboost-nightly-builds/${BRANCH_NAME}/libxgboost4j/ \
27+
--acl public-read --no-progress
28+
#fi
29+
popd

‎tests/buildkite/build-jvm-linux-x86_64-manylinux2014.sh

Lines changed: 29 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,29 @@
1+
#!/bin/bash
2+
3+
set -euo pipefail
4+
5+
source tests/buildkite/conftest.sh
6+
7+
command_wrapper="tests/ci_build/ci_build.sh jvm_manylinux2014_x86_64"
8+
9+
# Build XGBoost4J binary
10+
echo "--- Build libxgboost4j.so (targeting glibc 2.17)"
11+
set -x
12+
mkdir build
13+
$command_wrapper bash -c \
14+
"cd build && cmake .. -GNinja -DJVM_BINDINGS=ON -DUSE_OPENMP=ON && ninja -v"
15+
ldd lib/libxgboost4j.so
16+
objdump -T lib/libxgboost4j.so | grep GLIBC_ | sed 's/.*GLIBC_\([.0-9]*\).*/\1/g' | sort -Vu
17+
18+
echo "--- Upload libxgboost4j.so"
19+
pushd lib
20+
libname=libxgboost4j_linux_x86_64_${BUILDKITE_COMMIT}.so
21+
mv -v libxgboost4j.so ${libname}
22+
buildkite-agent artifact upload ${libname}
23+
#if [[ ($is_pull_request == 0) && ($is_release_branch == 1) ]]
24+
#then
25+
aws s3 cp ${libname} \
26+
s3://xgboost-nightly-builds/${BRANCH_NAME}/libxgboost4j/ \
27+
--acl public-read --no-progress
28+
#fi
29+
popd

‎tests/buildkite/build-jvm-macos-m1.sh

Lines changed: 41 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,41 @@
1+
#!/bin/bash
2+
3+
set -euo pipefail
4+
5+
source tests/buildkite/conftest.sh
6+
7+
# Display system info
8+
echo "--- Display system information"
9+
set -x
10+
system_profiler SPSoftwareDataType
11+
sysctl -n machdep.cpu.brand_string
12+
uname -m
13+
set +x
14+
15+
# Build XGBoost4J binary
16+
echo "--- Build libxgboost4j.dylib"
17+
set -x
18+
mkdir build
19+
pushd build
20+
export JAVA_HOME=$(/usr/libexec/java_home)
21+
cmake .. -GNinja -DJVM_BINDINGS=ON -DUSE_OPENMP=ON -DCMAKE_OSX_DEPLOYMENT_TARGET=10.15
22+
ninja -v
23+
popd
24+
rm -rf build
25+
otool -L lib/libxgboost.dylib
26+
set +x
27+
28+
echo "--- Upload libxgboost4j.dylib"
29+
set -x
30+
pushd lib
31+
libname=libxgboost4j_m1_${BUILDKITE_COMMIT}.dylib
32+
mv -v libxgboost4j.dylib ${libname}
33+
buildkite-agent artifact upload ${libname}
34+
#if [[ ($is_pull_request == 0) && ($is_release_branch == 1) ]]
35+
#then
36+
aws s3 cp ${libname} \
37+
s3://xgboost-nightly-builds/${BRANCH_NAME}/libxgboost4j/ \
38+
--acl public-read --no-progress
39+
#fi
40+
popd
41+
set +x

‎tests/buildkite/pipeline-mac-m1.yml

Lines changed: 5 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -1,6 +1,11 @@
11
steps:
22
- block: ":rocket: Run this test job"
33
if: build.pull_request.id != null || build.branch =~ /^dependabot\//
4+
- label: ":macos: Build libxgboost4j.dylib for MacOS M1"
5+
command: "tests/buildkite/build-jvm-macos-m1.sh"
6+
key: mac-m1-jvm
7+
agents:
8+
queue: mac-mini-m1
49
- label: ":macos: Build and Test XGBoost for MacOS M1 with Clang 11"
510
command: "tests/buildkite/test-macos-m1-clang11.sh"
611
key: mac-m1-appleclang11

‎tests/buildkite/pipeline.yml

Lines changed: 10 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -56,6 +56,16 @@ steps:
5656
key: build-jvm-packages
5757
agents:
5858
queue: linux-amd64-cpu
59+
- label: ":console: Build libxgboost4j.so for Linux ARM64 (targeting glibc 2.17)"
60+
command: "tests/buildkite/build-jvm-linux-arm64-manylinux2014.sh"
61+
key: build-jvm-linux-arm64-manylinux2014
62+
agents:
63+
queue: linux-arm64-cpu
64+
- label: ":console: Build libxgboost4j.so for Linux x86_64 (targeting glibc 2.17)"
65+
command: "tests/buildkite/build-jvm-linux-x86_64-manylinux2014.sh"
66+
key: build-jvm-linux-x86_64-manylinux2014
67+
agents:
68+
queue: linux-amd64-cpu
5969
- label: ":console: Build JVM package doc"
6070
command: "tests/buildkite/build-jvm-doc.sh"
6171
key: build-jvm-doc

‎tests/buildkite/test-macos-m1-clang11.sh

Lines changed: 1 addition & 27 deletions
Original file line numberDiff line numberDiff line change
@@ -12,33 +12,6 @@ sysctl -n machdep.cpu.brand_string
1212
uname -m
1313
set +x
1414

15-
# Build XGBoost4J binary
16-
echo "--- Build libxgboost4j.dylib"
17-
set -x
18-
mkdir build
19-
pushd build
20-
export JAVA_HOME=$(/usr/libexec/java_home)
21-
cmake .. -GNinja -DJVM_BINDINGS=ON -DUSE_OPENMP=ON -DCMAKE_OSX_DEPLOYMENT_TARGET=10.15
22-
ninja -v
23-
popd
24-
rm -rf build
25-
otool -L lib/libxgboost.dylib
26-
set +x
27-
28-
echo "--- Upload Python wheel"
29-
set -x
30-
pushd lib
31-
mv -v libxgboost4j.dylib libxgboost4j_m1_${BUILDKITE_COMMIT}.dylib
32-
buildkite-agent artifact upload libxgboost4j_m1_${BUILDKITE_COMMIT}.dylib
33-
if [[ ($is_pull_request == 0) && ($is_release_branch == 1) ]]
34-
then
35-
aws s3 cp libxgboost4j_m1_${BUILDKITE_COMMIT}.dylib \
36-
s3://xgboost-nightly-builds/${BRANCH_NAME}/libxgboost4j/ \
37-
--acl public-read --no-progress
38-
fi
39-
popd
40-
set +x
41-
4215
# Ensure that XGBoost can be built with Clang 11
4316
echo "--- Build and Test XGBoost with MacOS M1, Clang 11"
4417
set -x
@@ -49,3 +22,4 @@ cmake .. -GNinja -DCMAKE_C_COMPILER=${LLVM11_PATH}/bin/clang \
4922
-DCMAKE_CXX_COMPILER=${LLVM11_PATH}/bin/clang++ -DGOOGLE_TEST=ON \
5023
-DUSE_DMLC_GTEST=ON
5124
ninja -v
25+
./testxgboost

‎tests/ci_build/Dockerfile.jvm_manylinux2014_aarch64

Lines changed: 17 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,17 @@
1+
FROM quay.io/pypa/manylinux2014_aarch64
2+
3+
RUN yum update -y && yum install -y java-1.8.0-openjdk-devel
4+
5+
# Install lightweight sudo (not bound to TTY)
6+
ENV GOSU_VERSION 1.10
7+
RUN set -ex; \
8+
curl -o /usr/local/bin/gosu -L "https://github.com/tianon/gosu/releases/download/$GOSU_VERSION/gosu-arm64" && \
9+
chmod +x /usr/local/bin/gosu && \
10+
gosu nobody true
11+
12+
# Default entry-point to use if running locally
13+
# It will preserve attributes of created files
14+
COPY entrypoint.sh /scripts/
15+
16+
WORKDIR /workspace
17+
ENTRYPOINT ["/scripts/entrypoint.sh"]

‎tests/ci_build/Dockerfile.jvm_manylinux2014_x86_64

Lines changed: 17 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,17 @@
1+
FROM quay.io/pypa/manylinux2014_x86_64
2+
3+
RUN yum update -y && yum install -y java-1.8.0-openjdk-devel ninja-build
4+
5+
# Install lightweight sudo (not bound to TTY)
6+
ENV GOSU_VERSION 1.10
7+
RUN set -ex; \
8+
curl -o /usr/local/bin/gosu -L "https://github.com/tianon/gosu/releases/download/$GOSU_VERSION/gosu-amd64" && \
9+
chmod +x /usr/local/bin/gosu && \
10+
gosu nobody true
11+
12+
# Default entry-point to use if running locally
13+
# It will preserve attributes of created files
14+
COPY entrypoint.sh /scripts/
15+
16+
WORKDIR /workspace
17+
ENTRYPOINT ["/scripts/entrypoint.sh"]

0 commit comments

Comments
 (0)