diff --git a/components/core/CMakeLists.txt b/components/core/CMakeLists.txt index 5d7cea3af0..6b4c966a29 100644 --- a/components/core/CMakeLists.txt +++ b/components/core/CMakeLists.txt @@ -72,7 +72,15 @@ if (CLP_USE_STATIC_LIBS) if (APPLE) set(CLP_STATIC_LIBS_UNSUPPORTED_PLATFORM "macOS") elseif (EXISTS "/etc/centos-release") - set(CLP_STATIC_LIBS_UNSUPPORTED_PLATFORM "CentOS") + # NOTE: + # 1. We don't support static linking on any CentOS-based distro except manylinux_2_28 (which + # shows up as "AlmaLinux"). + # 2. A release called "AlmaLinux" doesn't guarantee we're running on a manylinux distro, but + # we can improve this check when someone reports an issue. + file(READ "/etc/centos-release" CENTOS_RELEASE_CONTENT) + if(NOT "${CENTOS_RELEASE_CONTENT}" MATCHES "AlmaLinux") + set(CLP_STATIC_LIBS_UNSUPPORTED_PLATFORM "CentOS") + endif() endif() if (DEFINED CLP_STATIC_LIBS_UNSUPPORTED_PLATFORM) @@ -212,7 +220,7 @@ endif() # Find and setup libcurl # By default, CURL does not provide static libraries if(CLP_NEED_CURL) - find_package(CURL 7.68.0 REQUIRED) + find_package(CURL 7.61.1 REQUIRED) if(CURL_FOUND) message(STATUS "Found CURL ${CURL_VERSION_STRING}") else() diff --git a/components/core/tools/docker-images/clp-env-base-manylinux_2_28-aarch64/Dockerfile b/components/core/tools/docker-images/clp-env-base-manylinux_2_28-aarch64/Dockerfile new file mode 100644 index 0000000000..06df2a2961 --- /dev/null +++ b/components/core/tools/docker-images/clp-env-base-manylinux_2_28-aarch64/Dockerfile @@ -0,0 +1,14 @@ +FROM quay.io/pypa/manylinux_2_28_aarch64 + +WORKDIR /root + +RUN mkdir -p ./tools/scripts/lib_install +COPY ./tools/scripts/lib_install ./tools/scripts/lib_install + +RUN ./tools/scripts/lib_install/manylinux_2_28/install-all.sh + +# Remove cached files +RUN dnf clean all && rm -rf /var/cache/dnf /tmp/* /var/tmp/* + +# NOTE: Don't flatten the image or else we'll lose any environment modifications from the base +# image. diff --git a/components/core/tools/docker-images/clp-env-base-manylinux_2_28-aarch64/build.sh b/components/core/tools/docker-images/clp-env-base-manylinux_2_28-aarch64/build.sh new file mode 100755 index 0000000000..e31049522a --- /dev/null +++ b/components/core/tools/docker-images/clp-env-base-manylinux_2_28-aarch64/build.sh @@ -0,0 +1,27 @@ +#!/usr/bin/env bash + +set -eu +set -o pipefail + +script_dir="$( cd "$( dirname "${BASH_SOURCE[0]}" )" &> /dev/null && pwd )" +component_root="${script_dir}/../../../" + +build_cmd=( + docker buildx build + --platform linux/arm64 + --tag clp-core-dependencies-aarch64-manylinux_2_28:dev + "$component_root" + --file "${script_dir}/Dockerfile" + --load +) + +if command -v git >/dev/null && git -C "$script_dir" rev-parse --is-inside-work-tree >/dev/null ; +then + build_cmd+=( + --label "org.opencontainers.image.revision=$(git -C "$script_dir" rev-parse HEAD)" + --label "org.opencontainers.image.source=$(git -C "$script_dir" remote get-url origin)" + ) +fi + +echo "Running: ${build_cmd[*]}" +"${build_cmd[@]}" diff --git a/components/core/tools/docker-images/clp-env-base-manylinux_2_28-x86_64/Dockerfile b/components/core/tools/docker-images/clp-env-base-manylinux_2_28-x86_64/Dockerfile new file mode 100644 index 0000000000..1185c7d031 --- /dev/null +++ b/components/core/tools/docker-images/clp-env-base-manylinux_2_28-x86_64/Dockerfile @@ -0,0 +1,14 @@ +FROM quay.io/pypa/manylinux_2_28_x86_64 + +WORKDIR /root + +RUN mkdir -p ./tools/scripts/lib_install +COPY ./tools/scripts/lib_install ./tools/scripts/lib_install + +RUN ./tools/scripts/lib_install/manylinux_2_28/install-all.sh + +# Remove cached files +RUN dnf clean all && rm -rf /var/cache/dnf /tmp/* /var/tmp/* + +# NOTE: Don't flatten the image or else we'll lose any environment modifications from the base +# image. diff --git a/components/core/tools/docker-images/clp-env-base-manylinux_2_28-x86_64/build.sh b/components/core/tools/docker-images/clp-env-base-manylinux_2_28-x86_64/build.sh new file mode 100755 index 0000000000..b7212f02c2 --- /dev/null +++ b/components/core/tools/docker-images/clp-env-base-manylinux_2_28-x86_64/build.sh @@ -0,0 +1,27 @@ +#!/usr/bin/env bash + +set -eu +set -o pipefail + +script_dir="$( cd "$( dirname "${BASH_SOURCE[0]}" )" &> /dev/null && pwd )" +component_root="${script_dir}/../../../" + +build_cmd=( + docker buildx build + --platform linux/amd64 + --tag clp-core-dependencies-x86-manylinux_2_28:dev + "$component_root" + --file "${script_dir}/Dockerfile" + --load +) + +if command -v git >/dev/null && git -C "$script_dir" rev-parse --is-inside-work-tree >/dev/null ; +then + build_cmd+=( + --label "org.opencontainers.image.revision=$(git -C "$script_dir" rev-parse HEAD)" + --label "org.opencontainers.image.source=$(git -C "$script_dir" remote get-url origin)" + ) +fi + +echo "Running: ${build_cmd[*]}" +"${build_cmd[@]}" diff --git a/components/core/tools/scripts/lib_install/libarchive.sh b/components/core/tools/scripts/lib_install/libarchive.sh index f96bd50a14..dd8c2b733e 100755 --- a/components/core/tools/scripts/lib_install/libarchive.sh +++ b/components/core/tools/scripts/lib_install/libarchive.sh @@ -63,8 +63,8 @@ fi cd ${extracted_dir} mkdir -p cmake-build-release cd cmake-build-release -# NOTE: Disable expat so the static libarchive doesn't look for it at link time -cmake -DENABLE_EXPAT=OFF ../ +# NOTE: Disable Expat and OpenSSL so the static libarchive doesn't look for them at link time. +cmake -DENABLE_EXPAT=OFF -DENABLE_OPENSSL=OFF ../ make -j${num_cpus} # Check if checkinstall is installed diff --git a/components/core/tools/scripts/lib_install/manylinux_2_28/install-all.sh b/components/core/tools/scripts/lib_install/manylinux_2_28/install-all.sh new file mode 100755 index 0000000000..089cd6bc66 --- /dev/null +++ b/components/core/tools/scripts/lib_install/manylinux_2_28/install-all.sh @@ -0,0 +1,12 @@ +#!/usr/bin/env bash + +set -eu +set -o pipefail + +script_dir="$( cd "$( dirname "${BASH_SOURCE[0]}" )" &> /dev/null && pwd )" + +"${script_dir}/install-prebuilt-packages.sh" +"${script_dir}/install-packages-from-source.sh" + +# TODO: https://github.com/y-scope/clp/issues/795 +"${script_dir}/../check-cmake-version.sh" diff --git a/components/core/tools/scripts/lib_install/manylinux_2_28/install-packages-from-source.sh b/components/core/tools/scripts/lib_install/manylinux_2_28/install-packages-from-source.sh new file mode 100755 index 0000000000..d31343c8ee --- /dev/null +++ b/components/core/tools/scripts/lib_install/manylinux_2_28/install-packages-from-source.sh @@ -0,0 +1,22 @@ +#!/usr/bin/env bash + +set -eu +set -o pipefail + +script_dir="$( cd "$( dirname "${BASH_SOURCE[0]}" )" &> /dev/null && pwd )" +lib_install_scripts_dir="${script_dir}/.." + +# NOTE: The remaining installation scripts depend on boost, so we install it beforehand. +"${lib_install_scripts_dir}/install-boost.sh" 1.87.0 + +# NOTE: +# 1. libarchive may statically link with LZMA, LZ4, and Zstandard, so we install them beforehand. +# 2. The versions of libarchive, LZMA, LZ4, and Zstandard available in manylinux_2_28's package +# repositories are either dated or don't include static libraries, so we install more recent +# versions from source. +"${lib_install_scripts_dir}/liblzma.sh" 5.8.1 +"${lib_install_scripts_dir}/lz4.sh" 1.10.0 +"${lib_install_scripts_dir}/zstandard.sh" 1.5.7 +"${lib_install_scripts_dir}/libarchive.sh" 3.8.0 + +"${lib_install_scripts_dir}/msgpack.sh" 7.0.0 diff --git a/components/core/tools/scripts/lib_install/manylinux_2_28/install-prebuilt-packages.sh b/components/core/tools/scripts/lib_install/manylinux_2_28/install-prebuilt-packages.sh new file mode 100755 index 0000000000..57ea2caab0 --- /dev/null +++ b/components/core/tools/scripts/lib_install/manylinux_2_28/install-prebuilt-packages.sh @@ -0,0 +1,45 @@ +#!/usr/bin/env bash + +set -eu +set -o pipefail + +dnf install -y \ + gcc-c++ \ + java-11-openjdk \ + jq \ + libcurl-devel \ + mariadb-connector-c-devel \ + openssl-devel \ + zlib-devel \ + zlib-static + +# Determine architecture for `task` release to install +rpm_arch=$(rpm --eval "%{_arch}") +case "$rpm_arch" in + "x86_64") + task_pkg_arch="amd64" + ;; + "aarch64") + task_pkg_arch="arm64" + ;; + *) + echo "Error: Unsupported architecture - $rpm_arch" + exit 1 + ;; +esac + +# Install `task` +# NOTE: We lock `task` to a version < 3.43 to avoid https://github.com/y-scope/clp/issues/872 +task_pkg_path=$(mktemp -t --suffix ".rpm" task-pkg.XXXXXXXXXX) || exit 1 +curl \ + --fail \ + --location \ + --output "$task_pkg_path" \ + --show-error \ + "https://github.com/go-task/task/releases/download/v3.42.1/task_linux_${task_pkg_arch}.rpm" +dnf install --assumeyes "$task_pkg_path" +rm "$task_pkg_path" + +# Downgrade to CMake v3 to work around https://github.com/y-scope/clp/issues/795 +pipx uninstall cmake +pipx install cmake~=3.31 diff --git a/docs/src/dev-guide/tooling-containers.md b/docs/src/dev-guide/tooling-containers.md index 2fdc9c356d..1e43246344 100644 --- a/docs/src/dev-guide/tooling-containers.md +++ b/docs/src/dev-guide/tooling-containers.md @@ -1,34 +1,110 @@ # Containers -We publish (to [GitHub packages][gh-packages]) several Docker container images useful for building -and running CLP: +We maintain several Docker container images that are useful for building and running CLP. All images +can be built and used locally, but some are available to download from +[GitHub Packages][gh-packages]. -* An [image][core-deps-centos-stream-9] containing the dependencies necessary to build CLP core in a - Centos Stream 9 x86 environment. +To build an image locally, run the `build.sh` script in the image's directory. + +## clp-core-dependencies-<arch>-manylinux_2_28 + +Images containing the dependencies necessary to build CLP core in a [manylinux_2_28][manylinux_2_28] +environment (aarch64 or x86). + +Binaries built on manylinux_2_28 (based on AlmaLinux 8) are expected to be compatible with other +distros using glibc 2.28+, including: + +* CentOS/RHEL 8+ +* Debian 10+ +* Fedora 29+ +* Ubuntu 18.10+ + +### clp-core-dependencies-aarch64-manylinux_2_28 + +* Path: ```text - ghcr.io/y-scope/clp/clp-core-dependencies-x86-centos-stream-9:main + components/core/tools/docker-images/clp-env-base-manylinux_2_28-aarch64 ``` -* An [image][core-deps-ubuntu-jammy] containing the dependencies necessary to build CLP core in an - Ubuntu Jammy x86 environment. +### clp-core-dependencies-x86-manylinux_2_28 + +* Path: ```text - ghcr.io/y-scope/clp/clp-core-dependencies-x86-ubuntu-jammy:main + components/core/tools/docker-images/clp-env-base-manylinux_2_28-x86_64 + ``` + +## clp-core-dependencies-x86-centos-stream-9 + +An image containing the dependencies necessary to build CLP core in a CentOS Stream 9 x86 +environment. + +* [GitHub Packages page][core-deps-centos-stream-9] +* Pull command: + + ```bash + docker pull ghcr.io/y-scope/clp/clp-core-dependencies-x86-centos-stream-9:main ``` -* An [image][core-ubuntu-jammy] containing the CLP core binaries (`clg`, `clp`, `clp-s`, `glt`, - etc.) built in an Ubuntu Jammy x86 environment. +* Path: ```text - ghcr.io/y-scope/clp/clp-core-x86-ubuntu-jammy:main + components/core/tools/docker-images/clp-env-base-centos-stream-9 + ``` + +## clp-core-dependencies-x86-ubuntu-jammy + +An image containing the dependencies necessary to build CLP core in an Ubuntu Jammy x86 +environment. + +* [GitHub Packages page][core-deps-ubuntu-jammy] +* Pull command: + + ```bash + docker pull ghcr.io/y-scope/clp/clp-core-dependencies-x86-ubuntu-jammy:main + ``` + +* Path: + + ```text + components/core/tools/docker-images/clp-env-base-ubuntu-jammy + ``` + +## clp-core-x86-ubuntu-jammy + +An image containing the CLP core binaries (`clg`, `clp`, `clp-s`, `glt`, etc.) built in an Ubuntu +Jammy x86 environment. + +* [GitHub Packages page][core-ubuntu-jammy] +* Pull command: + + ```bash + docker pull ghcr.io/y-scope/clp/clp-core-x86-ubuntu-jammy:main + ``` + +* Path: + + ```text + components/core/tools/docker-images/clp-core-ubuntu-jammy + ``` + +## clp-execution-x86-ubuntu-jammy + +An image containing the dependencies necessary to run the CLP package in an Ubuntu Jammy x86 +environment. + +* [GitHub Packages page][exe-ubuntu-jammy] +* Pull command: + + ```bash + docker pull ghcr.io/y-scope/clp/clp-execution-x86-ubuntu-jammy:main ``` -* An [image][exe-ubuntu-jammy] containing the dependencies necessary to run the CLP package in an - Ubuntu Jammy x86 environment. +* Path: ```text - ghcr.io/y-scope/clp/clp-execution-x86-ubuntu-jammy:main + tools/docker-images/clp-execution-base-ubuntu-jammy ``` [core-deps-centos-stream-9]: https://github.com/y-scope/clp/pkgs/container/clp%2Fclp-core-dependencies-x86-centos-stream-9 @@ -36,3 +112,4 @@ and running CLP: [core-ubuntu-jammy]: https://github.com/y-scope/clp/pkgs/container/clp%2Fclp-core-x86-ubuntu-jammy [exe-ubuntu-jammy]: https://github.com/y-scope/clp/pkgs/container/clp%2Fclp-execution-x86-ubuntu-jammy [gh-packages]: https://github.com/orgs/y-scope/packages?repo_name=clp +[manylinux_2_28]: https://github.com/pypa/manylinux?tab=readme-ov-file#manylinux_2_28-almalinux-8-based