Skip to content

Commit b48bf52

Browse files
hcho3hcho3
authored
[CI] Set up pipeline to build manylinux2014 wheels (dmlc#10478) (dmlc#10482)
* [CI] Set up pipeline to build manylinux2014 wheels (dmlc#10478) * [CI] Fix S3 upload for manylinux2014 wheels (dmlc#10483)
1 parent 3e42c1f commit b48bf52

10 files changed

+169
-7
lines changed

‎doc/install.rst

Lines changed: 20 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -27,6 +27,26 @@ Pre-built binary wheels are uploaded to PyPI (Python Package Index) for each rel
2727
You might need to run the command with ``--user`` flag or use ``virtualenv`` if you run
2828
into permission errors.
2929

30+
.. note:: Parts of the Python package now require glibc 2.28+
31+
32+
Starting from 2.1.0, XGBoost Python package will be distributed in two variants:
33+
34+
* ``manylinux_2_28``: for recent Linux distros with glibc 2.28 or newer. This variant comes with all features enabled.
35+
* ``manylinux2014``: for old Linux distros with glibc older than 2.28. This variant does not support GPU algorithms or federated learning.
36+
37+
The ``pip`` package manager will automatically choose the correct variant depending on your system.
38+
39+
Starting from **May 31, 2025**, we will stop distributing the ``manylinux2014`` variant and exclusively
40+
distribute the ``manylinux_2_28`` variant. We made this decision so that our CI/CD pipeline won't have
41+
depend on software components that reached end-of-life (such as CentOS 7). We strongly encourage
42+
everyone to migrate to recent Linux distros in order to use future versions of XGBoost.
43+
44+
Note. If you want to use GPU algorithms or federated learning on an older Linux distro, you have
45+
two alternatives:
46+
47+
1. Upgrade to a recent Linux distro with glibc 2.28+. OR
48+
2. Build XGBoost from the source.
49+
3050
.. note:: Windows users need to install Visual C++ Redistributable
3151

3252
XGBoost requires DLLs from `Visual C++ Redistributable

‎tests/buildkite/build-cuda-with-rmm.sh

Lines changed: 2 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -42,15 +42,15 @@ $command_wrapper python tests/ci_build/rename_whl.py \
4242
--platform-tag ${WHEEL_TAG}
4343

4444
echo "--- Audit binary wheel to ensure it's compliant with ${WHEEL_TAG} standard"
45-
tests/ci_build/ci_build.sh auditwheel_x86_64 auditwheel repair \
45+
tests/ci_build/ci_build.sh manylinux_2_28_x86_64 auditwheel repair \
4646
--plat ${WHEEL_TAG} python-package/dist/*.whl
4747
$command_wrapper python tests/ci_build/rename_whl.py \
4848
--wheel-path wheelhouse/*.whl \
4949
--commit-hash ${BUILDKITE_COMMIT} \
5050
--platform-tag ${WHEEL_TAG}
5151
mv -v wheelhouse/*.whl python-package/dist/
5252
# Make sure that libgomp.so is vendored in the wheel
53-
tests/ci_build/ci_build.sh auditwheel_x86_64 bash -c \
53+
tests/ci_build/ci_build.sh manylinux_2_28_x86_64 bash -c \
5454
"unzip -l python-package/dist/*.whl | grep libgomp || exit -1"
5555

5656
echo "--- Upload Python wheel"

‎tests/buildkite/build-cuda.sh

Lines changed: 2 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -41,15 +41,15 @@ $command_wrapper python tests/ci_build/rename_whl.py \
4141
--platform-tag ${WHEEL_TAG}
4242

4343
echo "--- Audit binary wheel to ensure it's compliant with ${WHEEL_TAG} standard"
44-
tests/ci_build/ci_build.sh auditwheel_x86_64 auditwheel repair \
44+
tests/ci_build/ci_build.sh manylinux_2_28_x86_64 auditwheel repair \
4545
--plat ${WHEEL_TAG} python-package/dist/*.whl
4646
$command_wrapper python tests/ci_build/rename_whl.py \
4747
--wheel-path wheelhouse/*.whl \
4848
--commit-hash ${BUILDKITE_COMMIT} \
4949
--platform-tag ${WHEEL_TAG}
5050
mv -v wheelhouse/*.whl python-package/dist/
5151
# Make sure that libgomp.so is vendored in the wheel
52-
tests/ci_build/ci_build.sh auditwheel_x86_64 bash -c \
52+
tests/ci_build/ci_build.sh manylinux_2_28_x86_64 bash -c \
5353
"unzip -l python-package/dist/*.whl | grep libgomp || exit -1"
5454

5555
echo "--- Upload Python wheel"

‎tests/buildkite/build-manylinux2014-aarch64.sh

Lines changed: 33 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,33 @@
1+
#!/bin/bash
2+
3+
set -euo pipefail
4+
5+
source tests/buildkite/conftest.sh
6+
7+
WHEEL_TAG=manylinux2014_aarch64
8+
command_wrapper="tests/ci_build/ci_build.sh manylinux2014_aarch64"
9+
python_bin="/opt/python/cp310-cp310/bin/python"
10+
11+
echo "--- Build binary wheel for ${WHEEL_TAG}"
12+
# Patch to add warning about manylinux2014 variant
13+
patch -p0 < tests/buildkite/manylinux2014_warning.patch
14+
$command_wrapper bash -c \
15+
"cd python-package && ${python_bin} -m pip wheel --no-deps -vvv . --wheel-dir dist/"
16+
git checkout python-package/xgboost/core.py # discard the patch
17+
18+
$command_wrapper auditwheel repair --plat ${WHEEL_TAG} python-package/dist/*.whl
19+
$command_wrapper ${python_bin} tests/ci_build/rename_whl.py \
20+
--wheel-path wheelhouse/*.whl \
21+
--commit-hash ${BUILDKITE_COMMIT} \
22+
--platform-tag ${WHEEL_TAG}
23+
rm -rf python-package/dist/
24+
mkdir python-package/dist/
25+
mv -v wheelhouse/*.whl python-package/dist/
26+
27+
echo "--- Upload Python wheel"
28+
buildkite-agent artifact upload python-package/dist/*.whl
29+
if [[ ($is_pull_request == 0) && ($is_release_branch == 1) ]]
30+
then
31+
aws s3 cp python-package/dist/*.whl s3://xgboost-nightly-builds/${BRANCH_NAME}/ \
32+
--acl public-read --no-progress
33+
fi

‎tests/buildkite/build-manylinux2014-x86_64.sh

Lines changed: 33 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,33 @@
1+
#!/bin/bash
2+
3+
set -euo pipefail
4+
5+
source tests/buildkite/conftest.sh
6+
7+
WHEEL_TAG=manylinux2014_x86_64
8+
command_wrapper="tests/ci_build/ci_build.sh manylinux2014_x86_64"
9+
python_bin="/opt/python/cp310-cp310/bin/python"
10+
11+
echo "--- Build binary wheel for ${WHEEL_TAG}"
12+
# Patch to add warning about manylinux2014 variant
13+
patch -p0 < tests/buildkite/manylinux2014_warning.patch
14+
$command_wrapper bash -c \
15+
"cd python-package && ${python_bin} -m pip wheel --no-deps -vvv . --wheel-dir dist/"
16+
git checkout python-package/xgboost/core.py # discard the patch
17+
18+
$command_wrapper auditwheel repair --plat ${WHEEL_TAG} python-package/dist/*.whl
19+
$command_wrapper ${python_bin} tests/ci_build/rename_whl.py \
20+
--wheel-path wheelhouse/*.whl \
21+
--commit-hash ${BUILDKITE_COMMIT} \
22+
--platform-tag ${WHEEL_TAG}
23+
rm -rf python-package/dist/
24+
mkdir python-package/dist/
25+
mv -v wheelhouse/*.whl python-package/dist/
26+
27+
echo "--- Upload Python wheel"
28+
buildkite-agent artifact upload python-package/dist/*.whl
29+
if [[ ($is_pull_request == 0) && ($is_release_branch == 1) ]]
30+
then
31+
aws s3 cp python-package/dist/*.whl s3://xgboost-nightly-builds/${BRANCH_NAME}/ \
32+
--acl public-read --no-progress
33+
fi

‎tests/buildkite/manylinux2014_warning.patch

Lines changed: 37 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,37 @@
1+
diff --git python-package/pyproject.toml python-package/pyproject.toml
2+
index a273d8c13..dee49686a 100644
3+
--- python-package/pyproject.toml
4+
+++ python-package/pyproject.toml
5+
@@ -30,8 +30,7 @@ classifiers = [
6+
]
7+
dependencies = [
8+
"numpy",
9+
- "scipy",
10+
- "nvidia-nccl-cu12 ; platform_system == 'Linux' and platform_machine != 'aarch64'"
11+
+ "scipy"
12+
]
13+
14+
[project.urls]
15+
diff --git python-package/xgboost/core.py python-package/xgboost/core.py
16+
index e8bc735e6..030972ef2 100644
17+
--- python-package/xgboost/core.py
18+
+++ python-package/xgboost/core.py
19+
@@ -262,6 +262,18 @@ Likely cause:
20+
)
21+
raise ValueError(msg)
22+
23+
+ warnings.warn(
24+
+ "Your system has an old version of glibc (< 2.28). We will stop supporting "
25+
+ "Linux distros with glibc older than 2.28 after **May 31, 2025**. "
26+
+ "Please upgrade to a recent Linux distro (with glibc 2.28+) to use "
27+
+ "future versions of XGBoost.\n"
28+
+ "Note: You have installed the 'manylinux2014' variant of XGBoost. Certain "
29+
+ "features such as GPU algorithms or federated learning are not available. "
30+
+ "To use these features, please upgrade to a recent Linux distro with glibc "
31+
+ "2.28+, and install the 'manylinux_2_28' variant.",
32+
+ FutureWarning
33+
+ )
34+
+
35+
return lib
36+
37+

‎tests/buildkite/pipeline.yml

Lines changed: 12 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -26,18 +26,17 @@ steps:
2626
key: run-clang-tidy
2727
agents:
2828
queue: linux-amd64-cpu
29-
- wait
3029
- label: ":console: Build CPU"
3130
command: "tests/buildkite/build-cpu.sh"
3231
key: build-cpu
3332
agents:
3433
queue: linux-amd64-cpu
35-
- label: ":console: Build CPU ARM64"
34+
- label: ":console: Build CPU ARM64 + manylinux_2_28_aarch64 wheel"
3635
command: "tests/buildkite/build-cpu-arm64.sh"
3736
key: build-cpu-arm64
3837
agents:
3938
queue: linux-arm64-cpu
40-
- label: ":console: Build CUDA"
39+
- label: ":console: Build CUDA + manylinux_2_28_x86_64 wheel"
4140
command: "tests/buildkite/build-cuda.sh"
4241
key: build-cuda
4342
agents:
@@ -62,6 +61,16 @@ steps:
6261
key: build-jvm-doc
6362
agents:
6463
queue: linux-amd64-cpu
64+
- label: ":console: Build manylinux2014_x86_64 wheel"
65+
command: "tests/buildkite/build-manylinux2014-x86_64.sh"
66+
key: build-manylinux2014-x86_64
67+
agents:
68+
queue: linux-amd64-cpu
69+
- label: ":console: Build manylinux2014_aarch64 wheel"
70+
command: "tests/buildkite/build-manylinux2014-aarch64.sh"
71+
key: build-manylinux2014-aarch64
72+
agents:
73+
queue: linux-arm64-cpu
6574
- wait
6675
#### -------- TEST --------
6776
- label: ":console: Test Python package, CPU"

‎tests/ci_build/Dockerfile.manylinux2014_aarch64

Lines changed: 15 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,15 @@
1+
FROM quay.io/pypa/manylinux2014_aarch64
2+
3+
# Install lightweight sudo (not bound to TTY)
4+
ENV GOSU_VERSION 1.10
5+
RUN set -ex; \
6+
curl -o /usr/local/bin/gosu -L "https://github.com/tianon/gosu/releases/download/$GOSU_VERSION/gosu-arm64" && \
7+
chmod +x /usr/local/bin/gosu && \
8+
gosu nobody true
9+
10+
# Default entry-point to use if running locally
11+
# It will preserve attributes of created files
12+
COPY entrypoint.sh /scripts/
13+
14+
WORKDIR /workspace
15+
ENTRYPOINT ["/scripts/entrypoint.sh"]

‎tests/ci_build/Dockerfile.manylinux2014_x86_64

Lines changed: 15 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,15 @@
1+
FROM quay.io/pypa/manylinux2014_x86_64
2+
3+
# Install lightweight sudo (not bound to TTY)
4+
ENV GOSU_VERSION 1.10
5+
RUN set -ex; \
6+
curl -o /usr/local/bin/gosu -L "https://github.com/tianon/gosu/releases/download/$GOSU_VERSION/gosu-amd64" && \
7+
chmod +x /usr/local/bin/gosu && \
8+
gosu nobody true
9+
10+
# Default entry-point to use if running locally
11+
# It will preserve attributes of created files
12+
COPY entrypoint.sh /scripts/
13+
14+
WORKDIR /workspace
15+
ENTRYPOINT ["/scripts/entrypoint.sh"]

‎tests/ci_build/Dockerfile.auditwheel_x86_64 renamed to ‎tests/ci_build/Dockerfile.manylinux_2_28_x86_64

File renamed without changes.

0 commit comments

Comments
 (0)