Skip to content

Commit 0635a2f

Browse files
jackluo923kirkrodriguesLinZhihao-723
authored
feat(core): Add support for building and running on manylinux_2_28; Add manylinux_2_28 dependency container images. (#1041)
Co-authored-by: Kirk Rodrigues <[email protected]> Co-authored-by: Lin Zhihao <[email protected]>
1 parent bb8244c commit 0635a2f

File tree

10 files changed

+264
-18
lines changed

10 files changed

+264
-18
lines changed

‎components/core/CMakeLists.txt‎

Lines changed: 10 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -72,7 +72,15 @@ if (CLP_USE_STATIC_LIBS)
7272
if (APPLE)
7373
set(CLP_STATIC_LIBS_UNSUPPORTED_PLATFORM "macOS")
7474
elseif (EXISTS "/etc/centos-release")
75-
set(CLP_STATIC_LIBS_UNSUPPORTED_PLATFORM "CentOS")
75+
# NOTE:
76+
# 1. We don't support static linking on any CentOS-based distro except manylinux_2_28 (which
77+
# shows up as "AlmaLinux").
78+
# 2. A release called "AlmaLinux" doesn't guarantee we're running on a manylinux distro, but
79+
# we can improve this check when someone reports an issue.
80+
file(READ "/etc/centos-release" CENTOS_RELEASE_CONTENT)
81+
if(NOT "${CENTOS_RELEASE_CONTENT}" MATCHES "AlmaLinux")
82+
set(CLP_STATIC_LIBS_UNSUPPORTED_PLATFORM "CentOS")
83+
endif()
7684
endif()
7785

7886
if (DEFINED CLP_STATIC_LIBS_UNSUPPORTED_PLATFORM)
@@ -212,7 +220,7 @@ endif()
212220
# Find and setup libcurl
213221
# By default, CURL does not provide static libraries
214222
if(CLP_NEED_CURL)
215-
find_package(CURL 7.68.0 REQUIRED)
223+
find_package(CURL 7.61.1 REQUIRED)
216224
if(CURL_FOUND)
217225
message(STATUS "Found CURL ${CURL_VERSION_STRING}")
218226
else()

‎components/core/tools/docker-images/clp-env-base-manylinux_2_28-aarch64/Dockerfile‎

Lines changed: 14 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,14 @@
1+
FROM quay.io/pypa/manylinux_2_28_aarch64
2+
3+
WORKDIR /root
4+
5+
RUN mkdir -p ./tools/scripts/lib_install
6+
COPY ./tools/scripts/lib_install ./tools/scripts/lib_install
7+
8+
RUN ./tools/scripts/lib_install/manylinux_2_28/install-all.sh
9+
10+
# Remove cached files
11+
RUN dnf clean all && rm -rf /var/cache/dnf /tmp/* /var/tmp/*
12+
13+
# NOTE: Don't flatten the image or else we'll lose any environment modifications from the base
14+
# image.

‎components/core/tools/docker-images/clp-env-base-manylinux_2_28-aarch64/build.sh‎

Lines changed: 27 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,27 @@
1+
#!/usr/bin/env bash
2+
3+
set -eu
4+
set -o pipefail
5+
6+
script_dir="$( cd "$( dirname "${BASH_SOURCE[0]}" )" &> /dev/null && pwd )"
7+
component_root="${script_dir}/../../../"
8+
9+
build_cmd=(
10+
docker buildx build
11+
--platform linux/arm64
12+
--tag clp-core-dependencies-aarch64-manylinux_2_28:dev
13+
"$component_root"
14+
--file "${script_dir}/Dockerfile"
15+
--load
16+
)
17+
18+
if command -v git >/dev/null && git -C "$script_dir" rev-parse --is-inside-work-tree >/dev/null ;
19+
then
20+
build_cmd+=(
21+
--label "org.opencontainers.image.revision=$(git -C "$script_dir" rev-parse HEAD)"
22+
--label "org.opencontainers.image.source=$(git -C "$script_dir" remote get-url origin)"
23+
)
24+
fi
25+
26+
echo "Running: ${build_cmd[*]}"
27+
"${build_cmd[@]}"

‎components/core/tools/docker-images/clp-env-base-manylinux_2_28-x86_64/Dockerfile‎

Lines changed: 14 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,14 @@
1+
FROM quay.io/pypa/manylinux_2_28_x86_64
2+
3+
WORKDIR /root
4+
5+
RUN mkdir -p ./tools/scripts/lib_install
6+
COPY ./tools/scripts/lib_install ./tools/scripts/lib_install
7+
8+
RUN ./tools/scripts/lib_install/manylinux_2_28/install-all.sh
9+
10+
# Remove cached files
11+
RUN dnf clean all && rm -rf /var/cache/dnf /tmp/* /var/tmp/*
12+
13+
# NOTE: Don't flatten the image or else we'll lose any environment modifications from the base
14+
# image.

‎components/core/tools/docker-images/clp-env-base-manylinux_2_28-x86_64/build.sh‎

Lines changed: 27 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,27 @@
1+
#!/usr/bin/env bash
2+
3+
set -eu
4+
set -o pipefail
5+
6+
script_dir="$( cd "$( dirname "${BASH_SOURCE[0]}" )" &> /dev/null && pwd )"
7+
component_root="${script_dir}/../../../"
8+
9+
build_cmd=(
10+
docker buildx build
11+
--platform linux/amd64
12+
--tag clp-core-dependencies-x86-manylinux_2_28:dev
13+
"$component_root"
14+
--file "${script_dir}/Dockerfile"
15+
--load
16+
)
17+
18+
if command -v git >/dev/null && git -C "$script_dir" rev-parse --is-inside-work-tree >/dev/null ;
19+
then
20+
build_cmd+=(
21+
--label "org.opencontainers.image.revision=$(git -C "$script_dir" rev-parse HEAD)"
22+
--label "org.opencontainers.image.source=$(git -C "$script_dir" remote get-url origin)"
23+
)
24+
fi
25+
26+
echo "Running: ${build_cmd[*]}"
27+
"${build_cmd[@]}"

‎components/core/tools/scripts/lib_install/libarchive.sh‎

Lines changed: 2 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -63,8 +63,8 @@ fi
6363
cd ${extracted_dir}
6464
mkdir -p cmake-build-release
6565
cd cmake-build-release
66-
# NOTE: Disable expat so the static libarchive doesn't look for it at link time
67-
cmake -DENABLE_EXPAT=OFF ../
66+
# NOTE: Disable Expat and OpenSSL so the static libarchive doesn't look for them at link time.
67+
cmake -DENABLE_EXPAT=OFF -DENABLE_OPENSSL=OFF ../
6868
make -j${num_cpus}
6969

7070
# Check if checkinstall is installed

‎components/core/tools/scripts/lib_install/manylinux_2_28/install-all.sh‎

Lines changed: 12 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,12 @@
1+
#!/usr/bin/env bash
2+
3+
set -eu
4+
set -o pipefail
5+
6+
script_dir="$( cd "$( dirname "${BASH_SOURCE[0]}" )" &> /dev/null && pwd )"
7+
8+
"${script_dir}/install-prebuilt-packages.sh"
9+
"${script_dir}/install-packages-from-source.sh"
10+
11+
# TODO: https://github.com/y-scope/clp/issues/795
12+
"${script_dir}/../check-cmake-version.sh"

‎components/core/tools/scripts/lib_install/manylinux_2_28/install-packages-from-source.sh‎

Lines changed: 22 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,22 @@
1+
#!/usr/bin/env bash
2+
3+
set -eu
4+
set -o pipefail
5+
6+
script_dir="$( cd "$( dirname "${BASH_SOURCE[0]}" )" &> /dev/null && pwd )"
7+
lib_install_scripts_dir="${script_dir}/.."
8+
9+
# NOTE: The remaining installation scripts depend on boost, so we install it beforehand.
10+
"${lib_install_scripts_dir}/install-boost.sh" 1.87.0
11+
12+
# NOTE:
13+
# 1. libarchive may statically link with LZMA, LZ4, and Zstandard, so we install them beforehand.
14+
# 2. The versions of libarchive, LZMA, LZ4, and Zstandard available in manylinux_2_28's package
15+
# repositories are either dated or don't include static libraries, so we install more recent
16+
# versions from source.
17+
"${lib_install_scripts_dir}/liblzma.sh" 5.8.1
18+
"${lib_install_scripts_dir}/lz4.sh" 1.10.0
19+
"${lib_install_scripts_dir}/zstandard.sh" 1.5.7
20+
"${lib_install_scripts_dir}/libarchive.sh" 3.8.0
21+
22+
"${lib_install_scripts_dir}/msgpack.sh" 7.0.0

‎components/core/tools/scripts/lib_install/manylinux_2_28/install-prebuilt-packages.sh‎

Lines changed: 45 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,45 @@
1+
#!/usr/bin/env bash
2+
3+
set -eu
4+
set -o pipefail
5+
6+
dnf install -y \
7+
gcc-c++ \
8+
java-11-openjdk \
9+
jq \
10+
libcurl-devel \
11+
mariadb-connector-c-devel \
12+
openssl-devel \
13+
zlib-devel \
14+
zlib-static
15+
16+
# Determine architecture for `task` release to install
17+
rpm_arch=$(rpm --eval "%{_arch}")
18+
case "$rpm_arch" in
19+
"x86_64")
20+
task_pkg_arch="amd64"
21+
;;
22+
"aarch64")
23+
task_pkg_arch="arm64"
24+
;;
25+
*)
26+
echo "Error: Unsupported architecture - $rpm_arch"
27+
exit 1
28+
;;
29+
esac
30+
31+
# Install `task`
32+
# NOTE: We lock `task` to a version < 3.43 to avoid https://github.com/y-scope/clp/issues/872
33+
task_pkg_path=$(mktemp -t --suffix ".rpm" task-pkg.XXXXXXXXXX) || exit 1
34+
curl \
35+
--fail \
36+
--location \
37+
--output "$task_pkg_path" \
38+
--show-error \
39+
"https://github.com/go-task/task/releases/download/v3.42.1/task_linux_${task_pkg_arch}.rpm"
40+
dnf install --assumeyes "$task_pkg_path"
41+
rm "$task_pkg_path"
42+
43+
# Downgrade to CMake v3 to work around https://github.com/y-scope/clp/issues/795
44+
pipx uninstall cmake
45+
pipx install cmake~=3.31

‎docs/src/dev-guide/tooling-containers.md‎

Lines changed: 91 additions & 14 deletions
Original file line numberDiff line numberDiff line change
@@ -1,38 +1,115 @@
11
# Containers
22

3-
We publish (to [GitHub packages][gh-packages]) several Docker container images useful for building
4-
and running CLP:
3+
We maintain several Docker container images that are useful for building and running CLP. All images
4+
can be built and used locally, but some are available to download from
5+
[GitHub Packages][gh-packages].
56

6-
* An [image][core-deps-centos-stream-9] containing the dependencies necessary to build CLP core in a
7-
Centos Stream 9 x86 environment.
7+
To build an image locally, run the `build.sh` script in the image's directory.
8+
9+
## clp-core-dependencies-&lt;arch&gt;-manylinux_2_28
10+
11+
Images containing the dependencies necessary to build CLP core in a [manylinux_2_28][manylinux_2_28]
12+
environment (aarch64 or x86).
13+
14+
Binaries built on manylinux_2_28 (based on AlmaLinux 8) are expected to be compatible with other
15+
distros using glibc 2.28+, including:
16+
17+
* CentOS/RHEL 8+
18+
* Debian 10+
19+
* Fedora 29+
20+
* Ubuntu 18.10+
21+
22+
### clp-core-dependencies-aarch64-manylinux_2_28
23+
24+
* Path:
825

926
```text
10-
ghcr.io/y-scope/clp/clp-core-dependencies-x86-centos-stream-9:main
27+
components/core/tools/docker-images/clp-env-base-manylinux_2_28-aarch64
1128
```
1229

13-
* An [image][core-deps-ubuntu-jammy] containing the dependencies necessary to build CLP core in an
14-
Ubuntu Jammy x86 environment.
30+
### clp-core-dependencies-x86-manylinux_2_28
31+
32+
* Path:
1533

1634
```text
17-
ghcr.io/y-scope/clp/clp-core-dependencies-x86-ubuntu-jammy:main
35+
components/core/tools/docker-images/clp-env-base-manylinux_2_28-x86_64
36+
```
37+
38+
## clp-core-dependencies-x86-centos-stream-9
39+
40+
An image containing the dependencies necessary to build CLP core in a CentOS Stream 9 x86
41+
environment.
42+
43+
* [GitHub Packages page][core-deps-centos-stream-9]
44+
* Pull command:
45+
46+
```bash
47+
docker pull ghcr.io/y-scope/clp/clp-core-dependencies-x86-centos-stream-9:main
1848
```
1949

20-
* An [image][core-ubuntu-jammy] containing the CLP core binaries (`clg`, `clp`, `clp-s`, `glt`,
21-
etc.) built in an Ubuntu Jammy x86 environment.
50+
* Path:
2251

2352
```text
24-
ghcr.io/y-scope/clp/clp-core-x86-ubuntu-jammy:main
53+
components/core/tools/docker-images/clp-env-base-centos-stream-9
54+
```
55+
56+
## clp-core-dependencies-x86-ubuntu-jammy
57+
58+
An image containing the dependencies necessary to build CLP core in an Ubuntu Jammy x86
59+
environment.
60+
61+
* [GitHub Packages page][core-deps-ubuntu-jammy]
62+
* Pull command:
63+
64+
```bash
65+
docker pull ghcr.io/y-scope/clp/clp-core-dependencies-x86-ubuntu-jammy:main
66+
```
67+
68+
* Path:
69+
70+
```text
71+
components/core/tools/docker-images/clp-env-base-ubuntu-jammy
72+
```
73+
74+
## clp-core-x86-ubuntu-jammy
75+
76+
An image containing the CLP core binaries (`clg`, `clp`, `clp-s`, `glt`, etc.) built in an Ubuntu
77+
Jammy x86 environment.
78+
79+
* [GitHub Packages page][core-ubuntu-jammy]
80+
* Pull command:
81+
82+
```bash
83+
docker pull ghcr.io/y-scope/clp/clp-core-x86-ubuntu-jammy:main
84+
```
85+
86+
* Path:
87+
88+
```text
89+
components/core/tools/docker-images/clp-core-ubuntu-jammy
90+
```
91+
92+
## clp-execution-x86-ubuntu-jammy
93+
94+
An image containing the dependencies necessary to run the CLP package in an Ubuntu Jammy x86
95+
environment.
96+
97+
* [GitHub Packages page][exe-ubuntu-jammy]
98+
* Pull command:
99+
100+
```bash
101+
docker pull ghcr.io/y-scope/clp/clp-execution-x86-ubuntu-jammy:main
25102
```
26103

27-
* An [image][exe-ubuntu-jammy] containing the dependencies necessary to run the CLP package in an
28-
Ubuntu Jammy x86 environment.
104+
* Path:
29105

30106
```text
31-
ghcr.io/y-scope/clp/clp-execution-x86-ubuntu-jammy:main
107+
tools/docker-images/clp-execution-base-ubuntu-jammy
32108
```
33109

34110
[core-deps-centos-stream-9]: https://github.com/y-scope/clp/pkgs/container/clp%2Fclp-core-dependencies-x86-centos-stream-9
35111
[core-deps-ubuntu-jammy]: https://github.com/y-scope/clp/pkgs/container/clp%2Fclp-core-dependencies-x86-ubuntu-jammy
36112
[core-ubuntu-jammy]: https://github.com/y-scope/clp/pkgs/container/clp%2Fclp-core-x86-ubuntu-jammy
37113
[exe-ubuntu-jammy]: https://github.com/y-scope/clp/pkgs/container/clp%2Fclp-execution-x86-ubuntu-jammy
38114
[gh-packages]: https://github.com/orgs/y-scope/packages?repo_name=clp
115+
[manylinux_2_28]: https://github.com/pypa/manylinux?tab=readme-ov-file#manylinux_2_28-almalinux-8-based

0 commit comments

Comments
 (0)