From 64b7d8dbccf29e42f79abc1d6c8c8acd8af95765 Mon Sep 17 00:00:00 2001 From: Jack Luo Date: Thu, 26 Jun 2025 16:06:56 -0400 Subject: [PATCH 01/28] feat: Add support for manylinux_2_28 and use it to build x86_64 or aarch64 binaries --- components/core/CMakeLists.txt | 9 +- .../Dockerfile | 23 +++ .../build.sh | 31 ++++ .../Dockerfile | 23 +++ .../build.sh | 31 ++++ .../tools/scripts/lib_install/libarchive.sh | 4 +- .../lib_install/manylinux_2_28/install-all.sh | 15 ++ .../install-packages-from-source.sh | 19 +++ .../install-prebuilt-packages.sh | 64 ++++++++ components/core/tools/scripts/utils/README.md | 13 +- .../scripts/utils/build-and-run-unit-tests.py | 58 ++----- .../tools/scripts/utils/build-with-docker.py | 143 ++++++++++++++++++ components/core/tools/scripts/utils/build.py | 117 ++++++++++++++ docs/src/dev-guide/components-core/index.md | 92 ++++++----- .../manylinux_2_28-deps-install.md | 24 +++ docs/src/dev-guide/tooling-containers.md | 11 ++ 16 files changed, 588 insertions(+), 89 deletions(-) create mode 100644 components/core/tools/docker-images/clp-env-base-manylinux_2_28_aarch64/Dockerfile create mode 100755 components/core/tools/docker-images/clp-env-base-manylinux_2_28_aarch64/build.sh create mode 100644 components/core/tools/docker-images/clp-env-base-manylinux_2_28_x86_64/Dockerfile create mode 100755 components/core/tools/docker-images/clp-env-base-manylinux_2_28_x86_64/build.sh create mode 100755 components/core/tools/scripts/lib_install/manylinux_2_28/install-all.sh create mode 100755 components/core/tools/scripts/lib_install/manylinux_2_28/install-packages-from-source.sh create mode 100755 components/core/tools/scripts/lib_install/manylinux_2_28/install-prebuilt-packages.sh create mode 100644 components/core/tools/scripts/utils/build-with-docker.py create mode 100644 components/core/tools/scripts/utils/build.py create mode 100644 docs/src/dev-guide/components-core/manylinux_2_28-deps-install.md diff --git a/components/core/CMakeLists.txt b/components/core/CMakeLists.txt index 5d7cea3af0..3ae14354a5 100644 --- a/components/core/CMakeLists.txt +++ b/components/core/CMakeLists.txt @@ -72,7 +72,12 @@ if (CLP_USE_STATIC_LIBS) if (APPLE) set(CLP_STATIC_LIBS_UNSUPPORTED_PLATFORM "macOS") elseif (EXISTS "/etc/centos-release") - set(CLP_STATIC_LIBS_UNSUPPORTED_PLATFORM "CentOS") + # For now, only manylinux_2_28 (AlmaLinux 8 based) supports static linking + file(READ "/etc/centos-release" CENTOS_RELEASE_CONTENT) + string(FIND "${CENTOS_RELEASE_CONTENT}" "AlmaLinux" ALMALINUX_FOUND) + if(ALMALINUX_FOUND EQUAL -1) + set(CLP_STATIC_LIBS_UNSUPPORTED_PLATFORM "CentOS") + endif() endif() if (DEFINED CLP_STATIC_LIBS_UNSUPPORTED_PLATFORM) @@ -212,7 +217,7 @@ endif() # Find and setup libcurl # By default, CURL does not provide static libraries if(CLP_NEED_CURL) - find_package(CURL 7.68.0 REQUIRED) + find_package(CURL 7.61.1 REQUIRED) if(CURL_FOUND) message(STATUS "Found CURL ${CURL_VERSION_STRING}") else() diff --git a/components/core/tools/docker-images/clp-env-base-manylinux_2_28_aarch64/Dockerfile b/components/core/tools/docker-images/clp-env-base-manylinux_2_28_aarch64/Dockerfile new file mode 100644 index 0000000000..7558829c2a --- /dev/null +++ b/components/core/tools/docker-images/clp-env-base-manylinux_2_28_aarch64/Dockerfile @@ -0,0 +1,23 @@ +FROM quay.io/pypa/manylinux_2_28_aarch64 + +# Binaries built on manylinux_2_28 (AlmaLinux 8 based) is expected to be +# compatible with other distros using glibc 2.28 or later, including: +# - Debian 10+ +# - Ubuntu 18.10+ +# - Fedora 29+ +# - CentOS/RHEL 8+ + +WORKDIR /root + +RUN mkdir -p ./tools/scripts/lib_install +ADD ./tools/scripts/lib_install ./tools/scripts/lib_install + +RUN ./tools/scripts/lib_install/manylinux_2_28/install-all.sh + +# Remove cached files +RUN dnf clean all && rm -rf /var/cache/dnf /tmp/* /var/tmp/* + +# DO NOT FLATTEN THE IMAGE!!! +# The consequence of flattening the image is that we lose shell startup scripts +# and environment modification from /etc/profile.d, /etc/bashrc, etc. +# These are important for manylinux, otherwise, default system compilers are used, etc. diff --git a/components/core/tools/docker-images/clp-env-base-manylinux_2_28_aarch64/build.sh b/components/core/tools/docker-images/clp-env-base-manylinux_2_28_aarch64/build.sh new file mode 100755 index 0000000000..262d3f01b9 --- /dev/null +++ b/components/core/tools/docker-images/clp-env-base-manylinux_2_28_aarch64/build.sh @@ -0,0 +1,31 @@ +#!/bin/bash + +set -euo pipefail + +# Get the directory this script is in +script_dir="$( cd "$( dirname "${BASH_SOURCE[0]}" )" &> /dev/null && pwd )" +component_root=${script_dir}/../../../ + +# Build aarch64 image, will automatically use QEMU emulate if not on native platform +build_cmd=( + docker buildx build + --platform linux/arm64 + --tag clp-core-dependencies-manylinux_2_28_aarch64:dev + "$component_root" + --file "${script_dir}/Dockerfile" + --load +) + +# If in a git repo, add labels for revision and source +if command -v git >/dev/null && git -C "$script_dir" rev-parse --is-inside-work-tree >/dev/null ; +then + build_cmd+=( + --label "org.opencontainers.image.revision=$(git -C "$script_dir" rev-parse HEAD)" + --label "org.opencontainers.image.source=$(git -C "$script_dir" remote get-url origin)" + ) +fi + +echo "Running: ${build_cmd[*]}" + +# Execute the docker buildx command +"${build_cmd[@]}" diff --git a/components/core/tools/docker-images/clp-env-base-manylinux_2_28_x86_64/Dockerfile b/components/core/tools/docker-images/clp-env-base-manylinux_2_28_x86_64/Dockerfile new file mode 100644 index 0000000000..a0d30bc24e --- /dev/null +++ b/components/core/tools/docker-images/clp-env-base-manylinux_2_28_x86_64/Dockerfile @@ -0,0 +1,23 @@ +FROM quay.io/pypa/manylinux_2_28_x86_64 + +# Binaries built on manylinux_2_28 (AlmaLinux 8 based) is expected to be +# compatible with other distros using glibc 2.28 or later, including: +# - Debian 10+ +# - Ubuntu 18.10+ +# - Fedora 29+ +# - CentOS/RHEL 8+ + +WORKDIR /root + +RUN mkdir -p ./tools/scripts/lib_install +ADD ./tools/scripts/lib_install ./tools/scripts/lib_install + +RUN ./tools/scripts/lib_install/manylinux_2_28/install-all.sh + +# Remove cached files +RUN dnf clean all && rm -rf /var/cache/dnf /tmp/* /var/tmp/* + +# DO NOT FLATTEN THE IMAGE!!! +# The consequence of flattening the image is that we lose shell startup scripts +# and environment modification from /etc/profile.d, /etc/bashrc, etc. +# These are important for manylinux, otherwise, default system compilers are used, etc. diff --git a/components/core/tools/docker-images/clp-env-base-manylinux_2_28_x86_64/build.sh b/components/core/tools/docker-images/clp-env-base-manylinux_2_28_x86_64/build.sh new file mode 100755 index 0000000000..2e08001b8f --- /dev/null +++ b/components/core/tools/docker-images/clp-env-base-manylinux_2_28_x86_64/build.sh @@ -0,0 +1,31 @@ +#!/bin/bash + +set -euo pipefail + +# Get the directory this script is in +script_dir="$( cd "$( dirname "${BASH_SOURCE[0]}" )" &> /dev/null && pwd )" +component_root=${script_dir}/../../../ + +# Build x86_64 image, will automatically use QEMU emulate if not on native platform +build_cmd=( + docker buildx build + --platform linux/amd64 + --tag clp-core-dependencies-manylinux_2_28_x86_64:dev + "$component_root" + --file "${script_dir}/Dockerfile" + --load +) + +# If in a git repo, add labels for revision and source +if command -v git >/dev/null && git -C "$script_dir" rev-parse --is-inside-work-tree >/dev/null ; +then + build_cmd+=( + --label "org.opencontainers.image.revision=$(git -C "$script_dir" rev-parse HEAD)" + --label "org.opencontainers.image.source=$(git -C "$script_dir" remote get-url origin)" + ) +fi + +echo "Running: ${build_cmd[*]}" + +# Execute the docker buildx command +"${build_cmd[@]}" diff --git a/components/core/tools/scripts/lib_install/libarchive.sh b/components/core/tools/scripts/lib_install/libarchive.sh index f96bd50a14..3a74578ed4 100755 --- a/components/core/tools/scripts/lib_install/libarchive.sh +++ b/components/core/tools/scripts/lib_install/libarchive.sh @@ -63,8 +63,8 @@ fi cd ${extracted_dir} mkdir -p cmake-build-release cd cmake-build-release -# NOTE: Disable expat so the static libarchive doesn't look for it at link time -cmake -DENABLE_EXPAT=OFF ../ +# NOTE: Disable expat and openssl so the static libarchive doesn't look for it at link time +cmake -DENABLE_EXPAT=OFF -DENABLE_OPENSSL=OFF ../ make -j${num_cpus} # Check if checkinstall is installed diff --git a/components/core/tools/scripts/lib_install/manylinux_2_28/install-all.sh b/components/core/tools/scripts/lib_install/manylinux_2_28/install-all.sh new file mode 100755 index 0000000000..a44de8130c --- /dev/null +++ b/components/core/tools/scripts/lib_install/manylinux_2_28/install-all.sh @@ -0,0 +1,15 @@ +#!/usr/bin/env bash + +# Exit on any error +set -e + +# Error on undefined variable +set -u + +script_dir="$( cd "$( dirname "${BASH_SOURCE[0]}" )" &> /dev/null && pwd )" + +"${script_dir}/install-prebuilt-packages.sh" +"${script_dir}/install-packages-from-source.sh" + +# TODO: https://github.com/y-scope/clp/issues/795 +"${script_dir}/../check-cmake-version.sh" diff --git a/components/core/tools/scripts/lib_install/manylinux_2_28/install-packages-from-source.sh b/components/core/tools/scripts/lib_install/manylinux_2_28/install-packages-from-source.sh new file mode 100755 index 0000000000..2b04f6c3b1 --- /dev/null +++ b/components/core/tools/scripts/lib_install/manylinux_2_28/install-packages-from-source.sh @@ -0,0 +1,19 @@ +#!/usr/bin/env bash + +# Exit on any error +set -e + +# Error on undefined variable +set -u + +script_dir="$( cd "$( dirname "${BASH_SOURCE[0]}" )" &> /dev/null && pwd )" +lib_install_scripts_dir="${script_dir}/.." + +# NOTE: The remaining installation scripts depend on boost, so we install it beforehand. +"${lib_install_scripts_dir}"/install-boost.sh 1.87.0 + +"$lib_install_scripts_dir"/liblzma.sh 5.8.1 +"$lib_install_scripts_dir"/lz4.sh 1.10.0 +"$lib_install_scripts_dir"/zstandard.sh 1.5.7 +"$lib_install_scripts_dir"/libarchive.sh 3.8.0 +"${lib_install_scripts_dir}"/msgpack.sh 7.0.0 diff --git a/components/core/tools/scripts/lib_install/manylinux_2_28/install-prebuilt-packages.sh b/components/core/tools/scripts/lib_install/manylinux_2_28/install-prebuilt-packages.sh new file mode 100755 index 0000000000..67c293eab6 --- /dev/null +++ b/components/core/tools/scripts/lib_install/manylinux_2_28/install-prebuilt-packages.sh @@ -0,0 +1,64 @@ +#!/usr/bin/env bash + +# Exit on any error +set -e + +# Error on undefined variable +set -u + +dnf install -y \ + gcc-c++ \ + java-11-openjdk \ + libcurl-devel \ + mariadb-connector-c-devel \ + openssl-devel \ + zlib-devel \ + zlib-static \ + jq + +# Determine architecture for `task` release to install +rpm_arch=$(rpm --eval "%{_arch}") +case "$rpm_arch" in + "x86_64") + task_pkg_arch="amd64" + ;; + "aarch64") + task_pkg_arch="arm64" + ;; + *) + echo "Error: Unsupported architecture - $rpm_arch" + exit 1 + ;; +esac + +# Install `task` +# NOTE: We lock `task` to a version < 3.43 to avoid https://github.com/y-scope/clp/issues/872 +task_pkg_path="$(mktemp -t --suffix ".rpm" task-pkg.XXXXXXXXXX)" +curl \ + --fail \ + --location \ + --output "$task_pkg_path" \ + --show-error \ + "https://github.com/go-task/task/releases/download/v3.42.1/task_linux_${task_pkg_arch}.rpm" +dnf install --assumeyes "$task_pkg_path" +rm "$task_pkg_path" + +# The bundled CMake 4.x version is too new; downgrading to CMake 3.27 is +# necessary to resolve build issues with third-party libraries. Even after +# we added the "-DCMAKE_POLICY_VERSION_MINIMUM=3.5" flag, compilation issues +# in libraries such as googletest-src and yaml-cpp still persist. +# See sample error below: +# CMake Error at _deps/googletest-src/CMakeLists.txt:4 (cmake_minimum_required): +# Support for CMake versions below 3.5 has been dropped. +# +# Adjust the VERSION argument’s minimum value, or use the ... syntax +# to indicate the required CMake version range. +# +# Alternatively, you can try adding -DCMAKE_POLICY_VERSION_MINIMUM=3.5 +export PLATFORM=$(uname -m) +export CMAKE_VERSION=3.27.9 +export CMAKE_RELEASE_URL=https://github.com/Kitware/CMake/releases/download/v${CMAKE_VERSION}/cmake-${CMAKE_VERSION}-linux-${PLATFORM}.tar.gz +echo "Downgrading CMAKE to 3.27.9" +curl -L ${CMAKE_RELEASE_URL} | tar -xzf - +mv cmake-${CMAKE_VERSION}-linux-${PLATFORM} /opt/cmake-${CMAKE_VERSION} +ln -sf /opt/cmake-${CMAKE_VERSION}/bin/cmake /usr/local/bin/cmake diff --git a/components/core/tools/scripts/utils/README.md b/components/core/tools/scripts/utils/README.md index 53e5e7dadd..52c514dafd 100644 --- a/components/core/tools/scripts/utils/README.md +++ b/components/core/tools/scripts/utils/README.md @@ -1,8 +1,15 @@ This directory contains uncategorized utility scripts. - -* `build-and-run-unit-tests.sh` can be used to build all executables and run - the unit tests. +* `build.py` can be used to trigger a local platform build +* `build-with-docker.py` can be used to build all targets within a docker + image for x86_64 and arm64 + * manylinux_2_28 based images are used and the output binaries are + expected to be compatible with Debian 10+, Ubuntu 18.10+, Fedora 29+, + CentOS/RHEL 8+ and other platforms. + * Command to enable docker emulation: + `docker run --rm --privileged multiarch/qemu-user-static --reset -p yes` +* `build-and-run-unit-tests.py` can be used to perform a local platform + build of all executables and run the unit tests. * `run-in-container.sh` can be used to run a command (e.g., `make`), in a container containing the core component's dependencies. * All commands are run from the root of the core component. diff --git a/components/core/tools/scripts/utils/build-and-run-unit-tests.py b/components/core/tools/scripts/utils/build-and-run-unit-tests.py index 7c4b136171..1930479af5 100644 --- a/components/core/tools/scripts/utils/build-and-run-unit-tests.py +++ b/components/core/tools/scripts/utils/build-and-run-unit-tests.py @@ -1,5 +1,6 @@ import argparse import logging +import os import subprocess import sys from pathlib import Path @@ -21,45 +22,13 @@ logger = logging.getLogger(__name__) -def _config_cmake_project(src_dir: Path, build_dir: Path, use_shared_libs: bool): - cmd = [ - "cmake", - "-S", - str(src_dir), - "-B", - str(build_dir), - ] - if use_shared_libs: - cmd.append("-DCLP_USE_STATIC_LIBS=OFF") - subprocess.run(cmd, check=True) - - -def _build_project(build_dir: Path, num_jobs: Optional[int]): - """ - :param build_dir: - :param num_jobs: Max number of jobs to run when building. - """ - - cmd = [ - "cmake", - "--build", - str(build_dir), - "--parallel", - ] - if num_jobs is not None: - cmd.append(str(num_jobs)) - subprocess.run(cmd, check=True) - - def _run_unit_tests(build_dir: Path, test_spec: Optional[str]): """ :param build_dir: :param test_spec: Catch2 test specification. """ - cmd = [ - "./unitTest", - ] + cmd = ["./unitTest"] if test_spec is not None: cmd.append(test_spec) subprocess.run(cmd, cwd=build_dir, check=True) @@ -69,9 +38,6 @@ def main(argv: List[str]) -> int: args_parser = argparse.ArgumentParser( description="Builds the CLP-core's binaries and runs its unit tests." ) - args_parser.add_argument( - "--source-dir", required=True, help="Directory containing the main CMakeLists.txt." - ) args_parser.add_argument("--build-dir", required=True, help="Build output directory.") args_parser.add_argument( "--use-shared-libs", @@ -79,21 +45,27 @@ def main(argv: List[str]) -> int: help="Build targets by linking against shared libraries.", ) args_parser.add_argument( - "--num-jobs", type=int, help="Max number of jobs to run when building." + "--num-jobs", type=int, default=os.cpu_count(), help="Max number of jobs to run when building." ) args_parser.add_argument("--test-spec", help="Catch2 test specification.") parsed_args = args_parser.parse_args(argv[1:]) - src_dir: Path = Path(parsed_args.source_dir) build_dir: Path = Path(parsed_args.build_dir) - use_shared_libs: bool = parsed_args.use_shared_libs - num_jobs: Optional[int] = parsed_args.num_jobs test_spec: Optional[str] = parsed_args.test_spec - _config_cmake_project(src_dir, build_dir, use_shared_libs) - _build_project(build_dir, num_jobs) - _run_unit_tests(build_dir, test_spec) + build_cmd = [ + "python3", "build.py", + "--build-dir", str(build_dir), + "--num-jobs", str(parsed_args.num_jobs) + ] + if parsed_args.use_shared_libs: + build_cmd.append("--use-shared-libs") + logger.info("Starting build process...") + subprocess.run(build_cmd, check=True) + logger.info("Build completed. Running unit tests...") + _run_unit_tests(build_dir, test_spec) + logger.info("All unit tests completed successfully.") return 0 diff --git a/components/core/tools/scripts/utils/build-with-docker.py b/components/core/tools/scripts/utils/build-with-docker.py new file mode 100644 index 0000000000..68b5f8041d --- /dev/null +++ b/components/core/tools/scripts/utils/build-with-docker.py @@ -0,0 +1,143 @@ +import argparse +import logging +import os +import pathlib +import subprocess +import sys +from pathlib import Path +from typing import List, Optional + +# Set up console logging +logging_console_handler = logging.StreamHandler() +logging_formatter = logging.Formatter( + "%(asctime)s.%(msecs)03d %(levelname)s [%(module)s] %(message)s", datefmt="%Y-%m-%dT%H:%M:%S" +) +logging_console_handler.setFormatter(logging_formatter) + +# Set up root logger +root_logger = logging.getLogger() +root_logger.setLevel(logging.INFO) +root_logger.addHandler(logging_console_handler) + +# Create logger +logger = logging.getLogger(__name__) + +# Number of parent directories to reach project root from this script +PROJECT_ROOT_DEPTH = 6 + +# --- Platform Configurations --- +PLATFORM_CONFIGS = { + "linux/amd64": { + "build_script": "clp-env-base-manylinux_2_28_x86_64/build.sh", + "docker_image": "clp-core-dependencies-manylinux_2_28_x86_64" + }, + "linux/arm64": { + "build_script": "clp-env-base-manylinux_2_28_aarch64/build.sh", + "docker_image": "clp-core-dependencies-manylinux_2_28_aarch64" + } +} + + +def _build_clp_env_base_image(target_platform: str) -> None: + """ + Builds the CLP base Docker image for the specified platform. + """ + docker_images_path = Path(__file__).resolve().parents[2] / "docker-images" + config = PLATFORM_CONFIGS.get(target_platform) + if not config: + raise NotImplementedError(f"Unsupported target platform: {target_platform}") + build_script_path = docker_images_path / config["build_script"] + logger.info(f"Building CLP env base image using script: {build_script_path}") + try: + subprocess.run([str(build_script_path)], check=True) + except subprocess.CalledProcessError as e: + logger.error(f"Failed to build CLP env base image: {e}") + sys.exit(e.returncode) + + +def _find_project_root() -> Path: + root = Path(__file__).resolve() + for _ in range(PROJECT_ROOT_DEPTH): + root = root.parent + return root + + +def _build_clp_manylinux_2_28_binaries( + output_path: Path, + target_platform: str, + use_shared_libs: bool = False +) -> None: + """ + Builds the CLP-core binaries for the specified target platform + inside Docker using manylinux_2_28 based image + """ + output_path.mkdir(parents=True, exist_ok=True) + config = PLATFORM_CONFIGS.get(target_platform) + if not config: + raise NotImplementedError(f"Unsupported target platform: {target_platform}") + + project_root = _find_project_root() + build_script = Path("/clp/components/core/tools/scripts/utils/build.py") + + cmd = [ + "docker", "run", + "--user", f"{os.getuid()}:{os.getgid()}", + "--platform", target_platform, + "-v", f"{project_root}:/clp", + "-v", f"{output_path}:/output", + f"{config['docker_image']}:dev", + "python3", str(build_script), + "--build-dir", "/output" + ] + if use_shared_libs: + cmd.append("--use-shared-libs") + + logger.info("Running Docker build command: %s", " ".join(map(str, cmd))) + try: + subprocess.run(cmd, check=True) + except subprocess.CalledProcessError as e: + logger.error(f"Docker build failed: {e}") + sys.exit(e.returncode) + + +def main(argv: List[str]) -> int: + args_parser = argparse.ArgumentParser( + description="Builds the CLP-core's binaries" + ) + args_parser.add_argument( + "--output-dir", + required=True, + help="Directory to put the compiled CLP-core binaries in." + ) + args_parser.add_argument( + "--use-shared-libs", + action="store_true", + help="Build targets by linking against shared libraries." + ) + args_parser.add_argument( + "--target-platform", + default="linux/amd64", + choices=["linux/amd64", "linux/arm64"], + help="Target platform of the compiled binary: linux/amd64 or linux/arm64" + ) + + parsed_args = args_parser.parse_args(argv[1:]) + output_dir = Path(parsed_args.output_dir).resolve() + + logger.info(f"Target platform: {parsed_args.target_platform}") + logger.info(f"Output directory: {output_dir}") + logger.info(f"Use shared libs: {parsed_args.use_shared_libs}") + + _build_clp_env_base_image(parsed_args.target_platform) + _build_clp_manylinux_2_28_binaries( + output_dir, + parsed_args.target_platform, + parsed_args.use_shared_libs + ) + + logger.info("Build process completed successfully.") + return 0 + + +if "__main__" == __name__: + sys.exit(main(sys.argv)) diff --git a/components/core/tools/scripts/utils/build.py b/components/core/tools/scripts/utils/build.py new file mode 100644 index 0000000000..8ed4c082b8 --- /dev/null +++ b/components/core/tools/scripts/utils/build.py @@ -0,0 +1,117 @@ +import argparse +import logging +import os +import subprocess +import sys +from pathlib import Path +from typing import List, Optional + + +# Set up console logging +logging_console_handler = logging.StreamHandler() +logging_formatter = logging.Formatter( + "%(asctime)s.%(msecs)03d %(levelname)s [%(module)s] %(message)s", datefmt="%Y-%m-%dT%H:%M:%S" +) +logging_console_handler.setFormatter(logging_formatter) + +# Set up root logger +root_logger = logging.getLogger() +root_logger.setLevel(logging.INFO) +root_logger.addHandler(logging_console_handler) + +# Create logger +logger = logging.getLogger(__name__) + +# Number of parent directories to reach project root from this script +PROJECT_ROOT_DEPTH = 6 + +def _run_subprocess(cmd: List[str], cwd: Optional[Path] = None) -> None: + logger.info(f"Running: {' '.join(cmd)} in {cwd if cwd else Path.cwd()}") + try: + subprocess.run(cmd, check=True, cwd=str(cwd) if cwd else None) + except subprocess.CalledProcessError as e: + logger.error(f"Command failed: {' '.join(cmd)} (in {cwd})\nReturn code: {e.returncode}") + sys.exit(e.returncode) + + +def _config_cmake_project(src_dir: Path, build_dir: Path, use_shared_libs: bool): + cmd = [ + "cmake", + "-S", str(src_dir), + "-B", str(build_dir), + ] + if use_shared_libs: + cmd.append("-DCLP_USE_STATIC_LIBS=OFF") + subprocess.run(cmd, check=True) + + +def _build_project(build_dir: Path, num_jobs: Optional[int]): + """ + :param build_dir: + :param num_jobs: Max number of jobs to run when building. + """ + cmd = [ + "cmake", + "--build", + str(build_dir), + "--parallel", + ] + if num_jobs is not None: + cmd.append(str(num_jobs)) + subprocess.run(cmd, check=True, cwd=str(build_dir)) + + +def _find_project_root() -> Path: + root = Path(__file__).resolve() + for _ in range(PROJECT_ROOT_DEPTH): + root = root.parent + return root + + +def main(argv: List[str]) -> int: + args_parser = argparse.ArgumentParser( + description="Builds standard CLP-core binaries in the current local environment" + ) + args_parser.add_argument("--build-dir", required=True, help="Build output directory.") + args_parser.add_argument( + "--use-shared-libs", + action="store_true", + help="Build targets by linking against shared libraries.", + ) + args_parser.add_argument( + "--num-jobs", + type=int, + default=os.cpu_count(), + help="Max number of jobs to run when building." + ) + + parsed_args = args_parser.parse_args(argv[1:]) + project_root = _find_project_root() + logger.info(f"Inferred project root: {project_root}") + + # Initialize core dependencies + logger.info("Cleaning project...") + _run_subprocess(["task", "clean"], cwd=project_root) + logger.info("Installing core dependencies...") + _run_subprocess(["task", "deps:core"], cwd=project_root) + + # Create build directory + build_path = Path(parsed_args.build_dir) + if not build_path.exists(): + logger.info(f"Creating build directory: {build_path}") + build_path.mkdir(parents=True, exist_ok=True) + + # Configure and build core + core_path = project_root / "components" / "core" + logger.info(f"Configuring CMake project in {core_path} -> {build_path}") + _config_cmake_project(core_path, build_path, parsed_args.use_shared_libs) + + logger.info(f"Building project in {build_path} with {parsed_args.num_jobs} jobs") + _build_project(build_path, parsed_args.num_jobs) + + logger.info("Build completed successfully.") + return 0 + + +if "__main__" == __name__: + sys.exit(main(sys.argv)) diff --git a/docs/src/dev-guide/components-core/index.md b/docs/src/dev-guide/components-core/index.md index 64d33cd5f8..ac78214503 100644 --- a/docs/src/dev-guide/components-core/index.md +++ b/docs/src/dev-guide/components-core/index.md @@ -15,9 +15,56 @@ CLP core is the low-level component that performs compression, decompression, an To build, we require some source dependencies, packages from package managers, and libraries built from source. +### Build Environment + +A handful of packages and libraries are required to build CLP. There are two options to use them: + +* Install them on your machine and build CLP natively +* Build CLP within a prebuilt docker container that contains the libraries; + However, this won't work if you need additional libraries that aren't already in the container. +* If you just want to quickly build a CLP binary compatible with most platforms, you can run the following script: + ```shell + # Build CLP binary using manylinux_2_28 for x86_64 or aarch64 platforms + /path/to/clp/components/core/tools/scripts/utils/build-with-docker.py \ + --output \ + [--platform linux/amd64|linux/arm64] + ``` + * `--output `: Directory to place the built CLP binary. + * `--platform linux/amd64|linux/arm64` (optional): Target platform. Defaults to linux/amd64 if not specified. + + +#### Native Build Environment + +See the relevant README for your OS: + +* [manylinux_2_28](manylinux_2_28) +* [CentOS Stream 9](centos-stream-9-deps-install) +* [macOS](macos-deps-install) +* [Ubuntu 22.04](ubuntu-jammy-deps-install) + +Want to build natively on an OS not listed here? You can file a [feature request][feature-req]. + +#### Docker Build Environments + +You can use these commands to start a container in which you can manually build and run CLP: + +* Start a build container + ```shell + # Make sure to change /path/to/clp/ and /path/to/my/logs below + docker run --rm -it \ + --name 'clp-build-env' \ + -u$(id -u):$(id -g) \ + -v$(readlink -f /path/to/clp):/mnt/clp \ + -v$(readlink -f /path/to/my/logs):/mnt/logs \ + ghcr.io/y-scope/clp/clp-core-dependencies-x86-ubuntu-jammy:main \ + /bin/bash -l + + cd /mnt/clp + ``` + ### Set up -To initialize the project, run: +After the build environment is initialized, we begin by initializing the project: ```shell tools/scripts/deps-download/init.sh @@ -51,48 +98,12 @@ The task will download, build, and install (within the build directory) the foll | [yscope-log-viewer](https://github.com/y-scope/yscope-log-viewer.git) | 969ff35 | | [ystdlib-cpp](https://github.com/y-scope/ystdlib-cpp.git) | d80cf86 | -### Environment -A handful of packages and libraries are required to build CLP. There are two options to use them: - -* Install them on your machine and build CLP natively -* Build CLP within a prebuilt docker container that contains the libraries; - However, this won't work if you need additional libraries that aren't already in the container. - -#### Native Environment - -See the relevant README for your OS: - -* [CentOS Stream 9](centos-stream-9-deps-install) -* [macOS](macos-deps-install) -* [Ubuntu 22.04](ubuntu-jammy-deps-install) - -Want to build natively on an OS not listed here? You can file a [feature request][feature-req]. - -#### Docker Environment - -You can use these commands to start a container in which you can build and run CLP: - -```shell -# Make sure to change /path/to/clp/components/core and /path/to/my/logs below -docker run --rm -it \ - --name 'clp-build-env' \ - -u$(id -u):$(id -g) \ - -v$(readlink -f /path/to/clp/components/core):/mnt/clp \ - -v$(readlink -f /path/to/my/logs):/mnt/logs \ - ghcr.io/y-scope/clp/clp-core-dependencies-x86-ubuntu-jammy:main \ - /bin/bash -l - -cd /mnt/clp -``` - -Make sure to change `/path/to/clp/components/core` and `/path/to/my/logs` to -the relevant paths on your machine. - -## Build +### Build * Configure the cmake project: ```shell + cd components/core mkdir build cd build cmake ../ @@ -103,6 +114,9 @@ the relevant paths on your machine. make -j ``` +> Make sure to change `/path/to/clp/` and `/path/to/my/logs` to +the relevant paths on your machine. + :::{toctree} :hidden: diff --git a/docs/src/dev-guide/components-core/manylinux_2_28-deps-install.md b/docs/src/dev-guide/components-core/manylinux_2_28-deps-install.md new file mode 100644 index 0000000000..f133b92906 --- /dev/null +++ b/docs/src/dev-guide/components-core/manylinux_2_28-deps-install.md @@ -0,0 +1,24 @@ +# manylinux_2_28 setup + +To install the dependencies required to build clp-core, follow the steps below. +These same steps are used by our Docker containers. + +## Installing dependencies + +:::{caution} +Before you run any commands below, you should review the scripts to ensure they will not install +any dependencies or apply any configurations that you don't expect. +::: + +To install all dependencies, run the following with elevated privileges: + +:::{note} +The packages built from source ([install-packages-from-source.sh][src-install-script]) are installed +without using a packager. So if you ever need to uninstall them, you will need to do so manually. +::: + +```shell +components/core/tools/scripts/lib_install/manylinux_2_28/install-all.sh +``` + +[src-install-script]: https://github.com/y-scope/clp/blob/main/components/core/tools/scripts/lib_install/manylinux_2_28/install-packages-from-source.sh diff --git a/docs/src/dev-guide/tooling-containers.md b/docs/src/dev-guide/tooling-containers.md index 2fdc9c356d..5d95e98815 100644 --- a/docs/src/dev-guide/tooling-containers.md +++ b/docs/src/dev-guide/tooling-containers.md @@ -3,6 +3,15 @@ We publish (to [GitHub packages][gh-packages]) several Docker container images useful for building and running CLP: +* Various manylinux_2_28 based images + ([x86_64](core-deps-manylinux_2_28_x86_64), [aarch64](core-deps-manylinux_2_28_aarch64)) + containing the dependencies necessary to build CLP core in manylinux_2_28 environments + + ```text + ghcr.io/y-scope/clp/clp-core-dependencies-manylinux_2_28_x86_64:main + ghcr.io/y-scope/clp/clp-core-dependencies-manylinux_2_28_aarch64:main + ``` + * An [image][core-deps-centos-stream-9] containing the dependencies necessary to build CLP core in a Centos Stream 9 x86 environment. @@ -31,6 +40,8 @@ and running CLP: ghcr.io/y-scope/clp/clp-execution-x86-ubuntu-jammy:main ``` +[core-deps-manylinux_2_28_x86_64]: https://github.com/y-scope/clp/pkgs/container/clp%2Fclp-core-dependencies-manylinux_2_28_x86_64 +[core-deps-manylinux_2_28_aarch64]: https://github.com/y-scope/clp/pkgs/container/clp%2Fclp-core-dependencies-manylinux_2_28_aarch64 [core-deps-centos-stream-9]: https://github.com/y-scope/clp/pkgs/container/clp%2Fclp-core-dependencies-x86-centos-stream-9 [core-deps-ubuntu-jammy]: https://github.com/y-scope/clp/pkgs/container/clp%2Fclp-core-dependencies-x86-ubuntu-jammy [core-ubuntu-jammy]: https://github.com/y-scope/clp/pkgs/container/clp%2Fclp-core-x86-ubuntu-jammy From 917cecd66299f117e251cdca1b191d2cb7a901f9 Mon Sep 17 00:00:00 2001 From: Jack Luo Date: Thu, 26 Jun 2025 16:27:12 -0400 Subject: [PATCH 02/28] fix: resolved python lint issues --- .../scripts/utils/build-and-run-unit-tests.py | 14 ++++-- .../tools/scripts/utils/build-with-docker.py | 47 +++++++++---------- components/core/tools/scripts/utils/build.py | 10 ++-- 3 files changed, 39 insertions(+), 32 deletions(-) diff --git a/components/core/tools/scripts/utils/build-and-run-unit-tests.py b/components/core/tools/scripts/utils/build-and-run-unit-tests.py index 1930479af5..0e5cad23be 100644 --- a/components/core/tools/scripts/utils/build-and-run-unit-tests.py +++ b/components/core/tools/scripts/utils/build-and-run-unit-tests.py @@ -45,7 +45,10 @@ def main(argv: List[str]) -> int: help="Build targets by linking against shared libraries.", ) args_parser.add_argument( - "--num-jobs", type=int, default=os.cpu_count(), help="Max number of jobs to run when building." + "--num-jobs", + type=int, + default=os.cpu_count(), + help="Max number of jobs to run when building.", ) args_parser.add_argument("--test-spec", help="Catch2 test specification.") @@ -54,9 +57,12 @@ def main(argv: List[str]) -> int: test_spec: Optional[str] = parsed_args.test_spec build_cmd = [ - "python3", "build.py", - "--build-dir", str(build_dir), - "--num-jobs", str(parsed_args.num_jobs) + "python3", + "build.py", + "--build-dir", + str(build_dir), + "--num-jobs", + str(parsed_args.num_jobs), ] if parsed_args.use_shared_libs: build_cmd.append("--use-shared-libs") diff --git a/components/core/tools/scripts/utils/build-with-docker.py b/components/core/tools/scripts/utils/build-with-docker.py index 68b5f8041d..9c74322006 100644 --- a/components/core/tools/scripts/utils/build-with-docker.py +++ b/components/core/tools/scripts/utils/build-with-docker.py @@ -29,12 +29,12 @@ PLATFORM_CONFIGS = { "linux/amd64": { "build_script": "clp-env-base-manylinux_2_28_x86_64/build.sh", - "docker_image": "clp-core-dependencies-manylinux_2_28_x86_64" + "docker_image": "clp-core-dependencies-manylinux_2_28_x86_64", }, "linux/arm64": { "build_script": "clp-env-base-manylinux_2_28_aarch64/build.sh", - "docker_image": "clp-core-dependencies-manylinux_2_28_aarch64" - } + "docker_image": "clp-core-dependencies-manylinux_2_28_aarch64", + }, } @@ -63,9 +63,7 @@ def _find_project_root() -> Path: def _build_clp_manylinux_2_28_binaries( - output_path: Path, - target_platform: str, - use_shared_libs: bool = False + output_path: Path, target_platform: str, use_shared_libs: bool = False ) -> None: """ Builds the CLP-core binaries for the specified target platform @@ -80,14 +78,21 @@ def _build_clp_manylinux_2_28_binaries( build_script = Path("/clp/components/core/tools/scripts/utils/build.py") cmd = [ - "docker", "run", - "--user", f"{os.getuid()}:{os.getgid()}", - "--platform", target_platform, - "-v", f"{project_root}:/clp", - "-v", f"{output_path}:/output", + "docker", + "run", + "--user", + f"{os.getuid()}:{os.getgid()}", + "--platform", + target_platform, + "-v", + f"{project_root}:/clp", + "-v", + f"{output_path}:/output", f"{config['docker_image']}:dev", - "python3", str(build_script), - "--build-dir", "/output" + "python3", + str(build_script), + "--build-dir", + "/output", ] if use_shared_libs: cmd.append("--use-shared-libs") @@ -101,24 +106,20 @@ def _build_clp_manylinux_2_28_binaries( def main(argv: List[str]) -> int: - args_parser = argparse.ArgumentParser( - description="Builds the CLP-core's binaries" - ) + args_parser = argparse.ArgumentParser(description="Builds the CLP-core's binaries") args_parser.add_argument( - "--output-dir", - required=True, - help="Directory to put the compiled CLP-core binaries in." + "--output-dir", required=True, help="Directory to put the compiled CLP-core binaries in." ) args_parser.add_argument( "--use-shared-libs", action="store_true", - help="Build targets by linking against shared libraries." + help="Build targets by linking against shared libraries.", ) args_parser.add_argument( "--target-platform", default="linux/amd64", choices=["linux/amd64", "linux/arm64"], - help="Target platform of the compiled binary: linux/amd64 or linux/arm64" + help="Target platform of the compiled binary: linux/amd64 or linux/arm64", ) parsed_args = args_parser.parse_args(argv[1:]) @@ -130,9 +131,7 @@ def main(argv: List[str]) -> int: _build_clp_env_base_image(parsed_args.target_platform) _build_clp_manylinux_2_28_binaries( - output_dir, - parsed_args.target_platform, - parsed_args.use_shared_libs + output_dir, parsed_args.target_platform, parsed_args.use_shared_libs ) logger.info("Build process completed successfully.") diff --git a/components/core/tools/scripts/utils/build.py b/components/core/tools/scripts/utils/build.py index 8ed4c082b8..4f1b0b8a80 100644 --- a/components/core/tools/scripts/utils/build.py +++ b/components/core/tools/scripts/utils/build.py @@ -6,7 +6,6 @@ from pathlib import Path from typing import List, Optional - # Set up console logging logging_console_handler = logging.StreamHandler() logging_formatter = logging.Formatter( @@ -25,6 +24,7 @@ # Number of parent directories to reach project root from this script PROJECT_ROOT_DEPTH = 6 + def _run_subprocess(cmd: List[str], cwd: Optional[Path] = None) -> None: logger.info(f"Running: {' '.join(cmd)} in {cwd if cwd else Path.cwd()}") try: @@ -37,8 +37,10 @@ def _run_subprocess(cmd: List[str], cwd: Optional[Path] = None) -> None: def _config_cmake_project(src_dir: Path, build_dir: Path, use_shared_libs: bool): cmd = [ "cmake", - "-S", str(src_dir), - "-B", str(build_dir), + "-S", + str(src_dir), + "-B", + str(build_dir), ] if use_shared_libs: cmd.append("-DCLP_USE_STATIC_LIBS=OFF") @@ -82,7 +84,7 @@ def main(argv: List[str]) -> int: "--num-jobs", type=int, default=os.cpu_count(), - help="Max number of jobs to run when building." + help="Max number of jobs to run when building.", ) parsed_args = args_parser.parse_args(argv[1:]) From 53f5974df00df7a5db282993dee69736863d8450 Mon Sep 17 00:00:00 2001 From: Jack Luo Date: Thu, 26 Jun 2025 16:39:01 -0400 Subject: [PATCH 03/28] fix: resolved docs issues --- docs/src/dev-guide/components-core/index.md | 6 +++--- docs/src/dev-guide/tooling-containers.md | 4 ++-- 2 files changed, 5 insertions(+), 5 deletions(-) diff --git a/docs/src/dev-guide/components-core/index.md b/docs/src/dev-guide/components-core/index.md index ac78214503..ffd5a3cd98 100644 --- a/docs/src/dev-guide/components-core/index.md +++ b/docs/src/dev-guide/components-core/index.md @@ -4,7 +4,7 @@ CLP core is the low-level component that performs compression, decompression, an ## Requirements -* We have built and tested CLP on the OSes listed [below](#native-environment). +* We have built and tested CLP on the OSes listed [below](#native-build-environment). * If you have trouble building for another OS, file an issue, and we may be able to help. * A recent compiler that fully supports C++20 features such as * std::span @@ -37,7 +37,7 @@ A handful of packages and libraries are required to build CLP. There are two opt See the relevant README for your OS: -* [manylinux_2_28](manylinux_2_28) +* [manylinux_2_28](manylinux_2_28-deps-install) * [CentOS Stream 9](centos-stream-9-deps-install) * [macOS](macos-deps-install) * [Ubuntu 22.04](ubuntu-jammy-deps-install) @@ -119,7 +119,7 @@ the relevant paths on your machine. :::{toctree} :hidden: - +manylinux_2_28-deps-install centos-stream-9-deps-install macos-deps-install ubuntu-jammy-deps-install diff --git a/docs/src/dev-guide/tooling-containers.md b/docs/src/dev-guide/tooling-containers.md index 5d95e98815..643bfb6b81 100644 --- a/docs/src/dev-guide/tooling-containers.md +++ b/docs/src/dev-guide/tooling-containers.md @@ -3,8 +3,8 @@ We publish (to [GitHub packages][gh-packages]) several Docker container images useful for building and running CLP: -* Various manylinux_2_28 based images - ([x86_64](core-deps-manylinux_2_28_x86_64), [aarch64](core-deps-manylinux_2_28_aarch64)) +* Various manylinux_2_28 based images + ([x86_64][core-deps-manylinux_2_28_x86_64], [aarch64][core-deps-manylinux_2_28_aarch64]) containing the dependencies necessary to build CLP core in manylinux_2_28 environments ```text From 73563d551a575254015d9c62eed601dcaf8d17c1 Mon Sep 17 00:00:00 2001 From: Jack Luo Date: Thu, 26 Jun 2025 16:52:59 -0400 Subject: [PATCH 04/28] fix: resolved issues raised by code-rabbit --- .../manylinux_2_28/install-packages-from-source.sh | 3 +++ .../manylinux_2_28/install-prebuilt-packages.sh | 14 +++++++++----- components/core/tools/scripts/utils/build.py | 6 ++++++ docs/src/dev-guide/components-core/index.md | 8 +++++--- docs/src/dev-guide/tooling-containers.md | 2 +- 5 files changed, 24 insertions(+), 9 deletions(-) diff --git a/components/core/tools/scripts/lib_install/manylinux_2_28/install-packages-from-source.sh b/components/core/tools/scripts/lib_install/manylinux_2_28/install-packages-from-source.sh index 2b04f6c3b1..25f0949b43 100755 --- a/components/core/tools/scripts/lib_install/manylinux_2_28/install-packages-from-source.sh +++ b/components/core/tools/scripts/lib_install/manylinux_2_28/install-packages-from-source.sh @@ -3,6 +3,9 @@ # Exit on any error set -e +# Exit on failures hidden inside pipes (e.g., curl | tar) +set -o pipefail + # Error on undefined variable set -u diff --git a/components/core/tools/scripts/lib_install/manylinux_2_28/install-prebuilt-packages.sh b/components/core/tools/scripts/lib_install/manylinux_2_28/install-prebuilt-packages.sh index 67c293eab6..8f526b4b46 100755 --- a/components/core/tools/scripts/lib_install/manylinux_2_28/install-prebuilt-packages.sh +++ b/components/core/tools/scripts/lib_install/manylinux_2_28/install-prebuilt-packages.sh @@ -3,6 +3,9 @@ # Exit on any error set -e +# Exit on failures hidden inside pipes (e.g., curl | tar) +set -o pipefail + # Error on undefined variable set -u @@ -33,7 +36,8 @@ esac # Install `task` # NOTE: We lock `task` to a version < 3.43 to avoid https://github.com/y-scope/clp/issues/872 -task_pkg_path="$(mktemp -t --suffix ".rpm" task-pkg.XXXXXXXXXX)" +tmp_rpm=$(mktemp -t --suffix ".rpm" task-pkg.XXXXXXXXXX) || exit 1 +task_pkg_path="$tmp_rpm" curl \ --fail \ --location \ @@ -41,7 +45,7 @@ curl \ --show-error \ "https://github.com/go-task/task/releases/download/v3.42.1/task_linux_${task_pkg_arch}.rpm" dnf install --assumeyes "$task_pkg_path" -rm "$task_pkg_path" +rm -rf "$task_pkg_path" # The bundled CMake 4.x version is too new; downgrading to CMake 3.27 is # necessary to resolve build issues with third-party libraries. Even after @@ -59,6 +63,6 @@ export PLATFORM=$(uname -m) export CMAKE_VERSION=3.27.9 export CMAKE_RELEASE_URL=https://github.com/Kitware/CMake/releases/download/v${CMAKE_VERSION}/cmake-${CMAKE_VERSION}-linux-${PLATFORM}.tar.gz echo "Downgrading CMAKE to 3.27.9" -curl -L ${CMAKE_RELEASE_URL} | tar -xzf - -mv cmake-${CMAKE_VERSION}-linux-${PLATFORM} /opt/cmake-${CMAKE_VERSION} -ln -sf /opt/cmake-${CMAKE_VERSION}/bin/cmake /usr/local/bin/cmake +curl -L "${CMAKE_RELEASE_URL}" | tar -xzf - +mv "cmake-${CMAKE_VERSION}-linux-${PLATFORM}" "/opt/cmake-${CMAKE_VERSION}" +ln -sf "/opt/cmake-${CMAKE_VERSION}/bin/cmake" /usr/local/bin/cmake diff --git a/components/core/tools/scripts/utils/build.py b/components/core/tools/scripts/utils/build.py index 4f1b0b8a80..ea3779d08d 100644 --- a/components/core/tools/scripts/utils/build.py +++ b/components/core/tools/scripts/utils/build.py @@ -1,3 +1,9 @@ +""" +Centralized build utility for CLP-core binaries. + +This script automates the process of cleaning, installing dependencies, configuring CMake, +and building the CLP-core binaries in the local environment. +""" import argparse import logging import os diff --git a/docs/src/dev-guide/components-core/index.md b/docs/src/dev-guide/components-core/index.md index ffd5a3cd98..1f6ec72958 100644 --- a/docs/src/dev-guide/components-core/index.md +++ b/docs/src/dev-guide/components-core/index.md @@ -17,7 +17,7 @@ from source. ### Build Environment -A handful of packages and libraries are required to build CLP. There are two options to use them: +A handful of packages and libraries are required to build CLP. There are three options to use them: * Install them on your machine and build CLP natively * Build CLP within a prebuilt docker container that contains the libraries; @@ -49,6 +49,7 @@ Want to build natively on an OS not listed here? You can file a [feature request You can use these commands to start a container in which you can manually build and run CLP: * Start a build container + ```shell # Make sure to change /path/to/clp/ and /path/to/my/logs below docker run --rm -it \ @@ -102,6 +103,7 @@ The task will download, build, and install (within the build directory) the foll ### Build * Configure the cmake project: + ```shell cd components/core mkdir build @@ -110,12 +112,12 @@ The task will download, build, and install (within the build directory) the foll ``` * Build: + ```shell make -j ``` -> Make sure to change `/path/to/clp/` and `/path/to/my/logs` to -the relevant paths on your machine. +> Make sure to change `/path/to/clp/` and `/path/to/my/logs` to the relevant paths on your machine. :::{toctree} :hidden: diff --git a/docs/src/dev-guide/tooling-containers.md b/docs/src/dev-guide/tooling-containers.md index 643bfb6b81..1cf74ea510 100644 --- a/docs/src/dev-guide/tooling-containers.md +++ b/docs/src/dev-guide/tooling-containers.md @@ -6,7 +6,7 @@ and running CLP: * Various manylinux_2_28 based images ([x86_64][core-deps-manylinux_2_28_x86_64], [aarch64][core-deps-manylinux_2_28_aarch64]) containing the dependencies necessary to build CLP core in manylinux_2_28 environments - + ```text ghcr.io/y-scope/clp/clp-core-dependencies-manylinux_2_28_x86_64:main ghcr.io/y-scope/clp/clp-core-dependencies-manylinux_2_28_aarch64:main From 0b81039eaef42e6054d0a3ee70a6bf163b403f1c Mon Sep 17 00:00:00 2001 From: Jack Luo Date: Thu, 26 Jun 2025 16:58:43 -0400 Subject: [PATCH 05/28] fix: resolved more python linting issue --- components/core/tools/scripts/utils/build.py | 1 + 1 file changed, 1 insertion(+) diff --git a/components/core/tools/scripts/utils/build.py b/components/core/tools/scripts/utils/build.py index ea3779d08d..a450bebee8 100644 --- a/components/core/tools/scripts/utils/build.py +++ b/components/core/tools/scripts/utils/build.py @@ -4,6 +4,7 @@ This script automates the process of cleaning, installing dependencies, configuring CMake, and building the CLP-core binaries in the local environment. """ + import argparse import logging import os From 5ab594674aa9a802b2b6ad663bccb8646b3c7bc2 Mon Sep 17 00:00:00 2001 From: Jack Luo Date: Thu, 26 Jun 2025 17:14:04 -0400 Subject: [PATCH 06/28] fix: resolved issues with github workflow --- .github/workflows/clp-core-build-macos.yaml | 1 - .github/workflows/clp-core-build.yaml | 2 -- 2 files changed, 3 deletions(-) diff --git a/.github/workflows/clp-core-build-macos.yaml b/.github/workflows/clp-core-build-macos.yaml index 51eb6985d0..971da5e680 100644 --- a/.github/workflows/clp-core-build-macos.yaml +++ b/.github/workflows/clp-core-build-macos.yaml @@ -84,7 +84,6 @@ jobs: run: >- python3 ./tools/scripts/utils/build-and-run-unit-tests.py ${{matrix.use_shared_libs == true && '--use-shared-libs' || ''}} - --source-dir . --build-dir build --num-jobs $(getconf _NPROCESSORS_ONLN) --test-spec "~[Stopwatch]" diff --git a/.github/workflows/clp-core-build.yaml b/.github/workflows/clp-core-build.yaml index ad0ea8fbf2..a734d3592a 100644 --- a/.github/workflows/clp-core-build.yaml +++ b/.github/workflows/clp-core-build.yaml @@ -182,7 +182,6 @@ jobs: CLP_CORE_MAX_PARALLELISM_PER_BUILD_TASK=$(getconf _NPROCESSORS_ONLN) task deps:core && python3 /mnt/repo/components/core/tools/scripts/utils/build-and-run-unit-tests.py ${{matrix.use_shared_libs == true && '--use-shared-libs' || ''}} - --source-dir /mnt/repo/components/core --build-dir /mnt/repo/components/core/build --num-jobs $(getconf _NPROCESSORS_ONLN) @@ -225,7 +224,6 @@ jobs: CLP_CORE_MAX_PARALLELISM_PER_BUILD_TASK=$(getconf _NPROCESSORS_ONLN) task deps:core && python3 /mnt/repo/components/core/tools/scripts/utils/build-and-run-unit-tests.py ${{matrix.use_shared_libs == true && '--use-shared-libs' || ''}} - --source-dir /mnt/repo/components/core --build-dir /mnt/repo/components/core/build --num-jobs $(getconf _NPROCESSORS_ONLN) From 117ff00868fba5c5faf9b9059e1b786a362419b7 Mon Sep 17 00:00:00 2001 From: Jack Luo Date: Fri, 27 Jun 2025 11:01:14 -0400 Subject: [PATCH 07/28] fix: reduced the scope of the PR --- components/core/tools/scripts/utils/README.md | 13 +- .../scripts/utils/build-and-run-unit-tests.py | 64 +++++--- .../tools/scripts/utils/build-with-docker.py | 142 ------------------ components/core/tools/scripts/utils/build.py | 126 ---------------- docs/src/dev-guide/components-core/index.md | 98 +++++------- .../manylinux_2_28-deps-install.md | 24 --- docs/src/dev-guide/tooling-containers.md | 11 -- 7 files changed, 87 insertions(+), 391 deletions(-) delete mode 100644 components/core/tools/scripts/utils/build-with-docker.py delete mode 100644 components/core/tools/scripts/utils/build.py delete mode 100644 docs/src/dev-guide/components-core/manylinux_2_28-deps-install.md diff --git a/components/core/tools/scripts/utils/README.md b/components/core/tools/scripts/utils/README.md index 52c514dafd..53e5e7dadd 100644 --- a/components/core/tools/scripts/utils/README.md +++ b/components/core/tools/scripts/utils/README.md @@ -1,15 +1,8 @@ This directory contains uncategorized utility scripts. -* `build.py` can be used to trigger a local platform build -* `build-with-docker.py` can be used to build all targets within a docker - image for x86_64 and arm64 - * manylinux_2_28 based images are used and the output binaries are - expected to be compatible with Debian 10+, Ubuntu 18.10+, Fedora 29+, - CentOS/RHEL 8+ and other platforms. - * Command to enable docker emulation: - `docker run --rm --privileged multiarch/qemu-user-static --reset -p yes` -* `build-and-run-unit-tests.py` can be used to perform a local platform - build of all executables and run the unit tests. + +* `build-and-run-unit-tests.sh` can be used to build all executables and run + the unit tests. * `run-in-container.sh` can be used to run a command (e.g., `make`), in a container containing the core component's dependencies. * All commands are run from the root of the core component. diff --git a/components/core/tools/scripts/utils/build-and-run-unit-tests.py b/components/core/tools/scripts/utils/build-and-run-unit-tests.py index 0e5cad23be..7c4b136171 100644 --- a/components/core/tools/scripts/utils/build-and-run-unit-tests.py +++ b/components/core/tools/scripts/utils/build-and-run-unit-tests.py @@ -1,6 +1,5 @@ import argparse import logging -import os import subprocess import sys from pathlib import Path @@ -22,13 +21,45 @@ logger = logging.getLogger(__name__) +def _config_cmake_project(src_dir: Path, build_dir: Path, use_shared_libs: bool): + cmd = [ + "cmake", + "-S", + str(src_dir), + "-B", + str(build_dir), + ] + if use_shared_libs: + cmd.append("-DCLP_USE_STATIC_LIBS=OFF") + subprocess.run(cmd, check=True) + + +def _build_project(build_dir: Path, num_jobs: Optional[int]): + """ + :param build_dir: + :param num_jobs: Max number of jobs to run when building. + """ + + cmd = [ + "cmake", + "--build", + str(build_dir), + "--parallel", + ] + if num_jobs is not None: + cmd.append(str(num_jobs)) + subprocess.run(cmd, check=True) + + def _run_unit_tests(build_dir: Path, test_spec: Optional[str]): """ :param build_dir: :param test_spec: Catch2 test specification. """ - cmd = ["./unitTest"] + cmd = [ + "./unitTest", + ] if test_spec is not None: cmd.append(test_spec) subprocess.run(cmd, cwd=build_dir, check=True) @@ -38,6 +69,9 @@ def main(argv: List[str]) -> int: args_parser = argparse.ArgumentParser( description="Builds the CLP-core's binaries and runs its unit tests." ) + args_parser.add_argument( + "--source-dir", required=True, help="Directory containing the main CMakeLists.txt." + ) args_parser.add_argument("--build-dir", required=True, help="Build output directory.") args_parser.add_argument( "--use-shared-libs", @@ -45,33 +79,21 @@ def main(argv: List[str]) -> int: help="Build targets by linking against shared libraries.", ) args_parser.add_argument( - "--num-jobs", - type=int, - default=os.cpu_count(), - help="Max number of jobs to run when building.", + "--num-jobs", type=int, help="Max number of jobs to run when building." ) args_parser.add_argument("--test-spec", help="Catch2 test specification.") parsed_args = args_parser.parse_args(argv[1:]) + src_dir: Path = Path(parsed_args.source_dir) build_dir: Path = Path(parsed_args.build_dir) + use_shared_libs: bool = parsed_args.use_shared_libs + num_jobs: Optional[int] = parsed_args.num_jobs test_spec: Optional[str] = parsed_args.test_spec - build_cmd = [ - "python3", - "build.py", - "--build-dir", - str(build_dir), - "--num-jobs", - str(parsed_args.num_jobs), - ] - if parsed_args.use_shared_libs: - build_cmd.append("--use-shared-libs") - - logger.info("Starting build process...") - subprocess.run(build_cmd, check=True) - logger.info("Build completed. Running unit tests...") + _config_cmake_project(src_dir, build_dir, use_shared_libs) + _build_project(build_dir, num_jobs) _run_unit_tests(build_dir, test_spec) - logger.info("All unit tests completed successfully.") + return 0 diff --git a/components/core/tools/scripts/utils/build-with-docker.py b/components/core/tools/scripts/utils/build-with-docker.py deleted file mode 100644 index 9c74322006..0000000000 --- a/components/core/tools/scripts/utils/build-with-docker.py +++ /dev/null @@ -1,142 +0,0 @@ -import argparse -import logging -import os -import pathlib -import subprocess -import sys -from pathlib import Path -from typing import List, Optional - -# Set up console logging -logging_console_handler = logging.StreamHandler() -logging_formatter = logging.Formatter( - "%(asctime)s.%(msecs)03d %(levelname)s [%(module)s] %(message)s", datefmt="%Y-%m-%dT%H:%M:%S" -) -logging_console_handler.setFormatter(logging_formatter) - -# Set up root logger -root_logger = logging.getLogger() -root_logger.setLevel(logging.INFO) -root_logger.addHandler(logging_console_handler) - -# Create logger -logger = logging.getLogger(__name__) - -# Number of parent directories to reach project root from this script -PROJECT_ROOT_DEPTH = 6 - -# --- Platform Configurations --- -PLATFORM_CONFIGS = { - "linux/amd64": { - "build_script": "clp-env-base-manylinux_2_28_x86_64/build.sh", - "docker_image": "clp-core-dependencies-manylinux_2_28_x86_64", - }, - "linux/arm64": { - "build_script": "clp-env-base-manylinux_2_28_aarch64/build.sh", - "docker_image": "clp-core-dependencies-manylinux_2_28_aarch64", - }, -} - - -def _build_clp_env_base_image(target_platform: str) -> None: - """ - Builds the CLP base Docker image for the specified platform. - """ - docker_images_path = Path(__file__).resolve().parents[2] / "docker-images" - config = PLATFORM_CONFIGS.get(target_platform) - if not config: - raise NotImplementedError(f"Unsupported target platform: {target_platform}") - build_script_path = docker_images_path / config["build_script"] - logger.info(f"Building CLP env base image using script: {build_script_path}") - try: - subprocess.run([str(build_script_path)], check=True) - except subprocess.CalledProcessError as e: - logger.error(f"Failed to build CLP env base image: {e}") - sys.exit(e.returncode) - - -def _find_project_root() -> Path: - root = Path(__file__).resolve() - for _ in range(PROJECT_ROOT_DEPTH): - root = root.parent - return root - - -def _build_clp_manylinux_2_28_binaries( - output_path: Path, target_platform: str, use_shared_libs: bool = False -) -> None: - """ - Builds the CLP-core binaries for the specified target platform - inside Docker using manylinux_2_28 based image - """ - output_path.mkdir(parents=True, exist_ok=True) - config = PLATFORM_CONFIGS.get(target_platform) - if not config: - raise NotImplementedError(f"Unsupported target platform: {target_platform}") - - project_root = _find_project_root() - build_script = Path("/clp/components/core/tools/scripts/utils/build.py") - - cmd = [ - "docker", - "run", - "--user", - f"{os.getuid()}:{os.getgid()}", - "--platform", - target_platform, - "-v", - f"{project_root}:/clp", - "-v", - f"{output_path}:/output", - f"{config['docker_image']}:dev", - "python3", - str(build_script), - "--build-dir", - "/output", - ] - if use_shared_libs: - cmd.append("--use-shared-libs") - - logger.info("Running Docker build command: %s", " ".join(map(str, cmd))) - try: - subprocess.run(cmd, check=True) - except subprocess.CalledProcessError as e: - logger.error(f"Docker build failed: {e}") - sys.exit(e.returncode) - - -def main(argv: List[str]) -> int: - args_parser = argparse.ArgumentParser(description="Builds the CLP-core's binaries") - args_parser.add_argument( - "--output-dir", required=True, help="Directory to put the compiled CLP-core binaries in." - ) - args_parser.add_argument( - "--use-shared-libs", - action="store_true", - help="Build targets by linking against shared libraries.", - ) - args_parser.add_argument( - "--target-platform", - default="linux/amd64", - choices=["linux/amd64", "linux/arm64"], - help="Target platform of the compiled binary: linux/amd64 or linux/arm64", - ) - - parsed_args = args_parser.parse_args(argv[1:]) - output_dir = Path(parsed_args.output_dir).resolve() - - logger.info(f"Target platform: {parsed_args.target_platform}") - logger.info(f"Output directory: {output_dir}") - logger.info(f"Use shared libs: {parsed_args.use_shared_libs}") - - _build_clp_env_base_image(parsed_args.target_platform) - _build_clp_manylinux_2_28_binaries( - output_dir, parsed_args.target_platform, parsed_args.use_shared_libs - ) - - logger.info("Build process completed successfully.") - return 0 - - -if "__main__" == __name__: - sys.exit(main(sys.argv)) diff --git a/components/core/tools/scripts/utils/build.py b/components/core/tools/scripts/utils/build.py deleted file mode 100644 index a450bebee8..0000000000 --- a/components/core/tools/scripts/utils/build.py +++ /dev/null @@ -1,126 +0,0 @@ -""" -Centralized build utility for CLP-core binaries. - -This script automates the process of cleaning, installing dependencies, configuring CMake, -and building the CLP-core binaries in the local environment. -""" - -import argparse -import logging -import os -import subprocess -import sys -from pathlib import Path -from typing import List, Optional - -# Set up console logging -logging_console_handler = logging.StreamHandler() -logging_formatter = logging.Formatter( - "%(asctime)s.%(msecs)03d %(levelname)s [%(module)s] %(message)s", datefmt="%Y-%m-%dT%H:%M:%S" -) -logging_console_handler.setFormatter(logging_formatter) - -# Set up root logger -root_logger = logging.getLogger() -root_logger.setLevel(logging.INFO) -root_logger.addHandler(logging_console_handler) - -# Create logger -logger = logging.getLogger(__name__) - -# Number of parent directories to reach project root from this script -PROJECT_ROOT_DEPTH = 6 - - -def _run_subprocess(cmd: List[str], cwd: Optional[Path] = None) -> None: - logger.info(f"Running: {' '.join(cmd)} in {cwd if cwd else Path.cwd()}") - try: - subprocess.run(cmd, check=True, cwd=str(cwd) if cwd else None) - except subprocess.CalledProcessError as e: - logger.error(f"Command failed: {' '.join(cmd)} (in {cwd})\nReturn code: {e.returncode}") - sys.exit(e.returncode) - - -def _config_cmake_project(src_dir: Path, build_dir: Path, use_shared_libs: bool): - cmd = [ - "cmake", - "-S", - str(src_dir), - "-B", - str(build_dir), - ] - if use_shared_libs: - cmd.append("-DCLP_USE_STATIC_LIBS=OFF") - subprocess.run(cmd, check=True) - - -def _build_project(build_dir: Path, num_jobs: Optional[int]): - """ - :param build_dir: - :param num_jobs: Max number of jobs to run when building. - """ - cmd = [ - "cmake", - "--build", - str(build_dir), - "--parallel", - ] - if num_jobs is not None: - cmd.append(str(num_jobs)) - subprocess.run(cmd, check=True, cwd=str(build_dir)) - - -def _find_project_root() -> Path: - root = Path(__file__).resolve() - for _ in range(PROJECT_ROOT_DEPTH): - root = root.parent - return root - - -def main(argv: List[str]) -> int: - args_parser = argparse.ArgumentParser( - description="Builds standard CLP-core binaries in the current local environment" - ) - args_parser.add_argument("--build-dir", required=True, help="Build output directory.") - args_parser.add_argument( - "--use-shared-libs", - action="store_true", - help="Build targets by linking against shared libraries.", - ) - args_parser.add_argument( - "--num-jobs", - type=int, - default=os.cpu_count(), - help="Max number of jobs to run when building.", - ) - - parsed_args = args_parser.parse_args(argv[1:]) - project_root = _find_project_root() - logger.info(f"Inferred project root: {project_root}") - - # Initialize core dependencies - logger.info("Cleaning project...") - _run_subprocess(["task", "clean"], cwd=project_root) - logger.info("Installing core dependencies...") - _run_subprocess(["task", "deps:core"], cwd=project_root) - - # Create build directory - build_path = Path(parsed_args.build_dir) - if not build_path.exists(): - logger.info(f"Creating build directory: {build_path}") - build_path.mkdir(parents=True, exist_ok=True) - - # Configure and build core - core_path = project_root / "components" / "core" - logger.info(f"Configuring CMake project in {core_path} -> {build_path}") - _config_cmake_project(core_path, build_path, parsed_args.use_shared_libs) - - logger.info(f"Building project in {build_path} with {parsed_args.num_jobs} jobs") - _build_project(build_path, parsed_args.num_jobs) - - logger.info("Build completed successfully.") - return 0 - - -if "__main__" == __name__: - sys.exit(main(sys.argv)) diff --git a/docs/src/dev-guide/components-core/index.md b/docs/src/dev-guide/components-core/index.md index 1f6ec72958..64d33cd5f8 100644 --- a/docs/src/dev-guide/components-core/index.md +++ b/docs/src/dev-guide/components-core/index.md @@ -4,7 +4,7 @@ CLP core is the low-level component that performs compression, decompression, an ## Requirements -* We have built and tested CLP on the OSes listed [below](#native-build-environment). +* We have built and tested CLP on the OSes listed [below](#native-environment). * If you have trouble building for another OS, file an issue, and we may be able to help. * A recent compiler that fully supports C++20 features such as * std::span @@ -15,57 +15,9 @@ CLP core is the low-level component that performs compression, decompression, an To build, we require some source dependencies, packages from package managers, and libraries built from source. -### Build Environment - -A handful of packages and libraries are required to build CLP. There are three options to use them: - -* Install them on your machine and build CLP natively -* Build CLP within a prebuilt docker container that contains the libraries; - However, this won't work if you need additional libraries that aren't already in the container. -* If you just want to quickly build a CLP binary compatible with most platforms, you can run the following script: - ```shell - # Build CLP binary using manylinux_2_28 for x86_64 or aarch64 platforms - /path/to/clp/components/core/tools/scripts/utils/build-with-docker.py \ - --output \ - [--platform linux/amd64|linux/arm64] - ``` - * `--output `: Directory to place the built CLP binary. - * `--platform linux/amd64|linux/arm64` (optional): Target platform. Defaults to linux/amd64 if not specified. - - -#### Native Build Environment - -See the relevant README for your OS: - -* [manylinux_2_28](manylinux_2_28-deps-install) -* [CentOS Stream 9](centos-stream-9-deps-install) -* [macOS](macos-deps-install) -* [Ubuntu 22.04](ubuntu-jammy-deps-install) - -Want to build natively on an OS not listed here? You can file a [feature request][feature-req]. - -#### Docker Build Environments - -You can use these commands to start a container in which you can manually build and run CLP: - -* Start a build container - - ```shell - # Make sure to change /path/to/clp/ and /path/to/my/logs below - docker run --rm -it \ - --name 'clp-build-env' \ - -u$(id -u):$(id -g) \ - -v$(readlink -f /path/to/clp):/mnt/clp \ - -v$(readlink -f /path/to/my/logs):/mnt/logs \ - ghcr.io/y-scope/clp/clp-core-dependencies-x86-ubuntu-jammy:main \ - /bin/bash -l - - cd /mnt/clp - ``` - ### Set up -After the build environment is initialized, we begin by initializing the project: +To initialize the project, run: ```shell tools/scripts/deps-download/init.sh @@ -99,29 +51,61 @@ The task will download, build, and install (within the build directory) the foll | [yscope-log-viewer](https://github.com/y-scope/yscope-log-viewer.git) | 969ff35 | | [ystdlib-cpp](https://github.com/y-scope/ystdlib-cpp.git) | d80cf86 | +### Environment -### Build +A handful of packages and libraries are required to build CLP. There are two options to use them: -* Configure the cmake project: +* Install them on your machine and build CLP natively +* Build CLP within a prebuilt docker container that contains the libraries; + However, this won't work if you need additional libraries that aren't already in the container. + +#### Native Environment + +See the relevant README for your OS: + +* [CentOS Stream 9](centos-stream-9-deps-install) +* [macOS](macos-deps-install) +* [Ubuntu 22.04](ubuntu-jammy-deps-install) + +Want to build natively on an OS not listed here? You can file a [feature request][feature-req]. + +#### Docker Environment + +You can use these commands to start a container in which you can build and run CLP: + +```shell +# Make sure to change /path/to/clp/components/core and /path/to/my/logs below +docker run --rm -it \ + --name 'clp-build-env' \ + -u$(id -u):$(id -g) \ + -v$(readlink -f /path/to/clp/components/core):/mnt/clp \ + -v$(readlink -f /path/to/my/logs):/mnt/logs \ + ghcr.io/y-scope/clp/clp-core-dependencies-x86-ubuntu-jammy:main \ + /bin/bash -l + +cd /mnt/clp +``` +Make sure to change `/path/to/clp/components/core` and `/path/to/my/logs` to +the relevant paths on your machine. + +## Build + +* Configure the cmake project: ```shell - cd components/core mkdir build cd build cmake ../ ``` * Build: - ```shell make -j ``` -> Make sure to change `/path/to/clp/` and `/path/to/my/logs` to the relevant paths on your machine. - :::{toctree} :hidden: -manylinux_2_28-deps-install + centos-stream-9-deps-install macos-deps-install ubuntu-jammy-deps-install diff --git a/docs/src/dev-guide/components-core/manylinux_2_28-deps-install.md b/docs/src/dev-guide/components-core/manylinux_2_28-deps-install.md deleted file mode 100644 index f133b92906..0000000000 --- a/docs/src/dev-guide/components-core/manylinux_2_28-deps-install.md +++ /dev/null @@ -1,24 +0,0 @@ -# manylinux_2_28 setup - -To install the dependencies required to build clp-core, follow the steps below. -These same steps are used by our Docker containers. - -## Installing dependencies - -:::{caution} -Before you run any commands below, you should review the scripts to ensure they will not install -any dependencies or apply any configurations that you don't expect. -::: - -To install all dependencies, run the following with elevated privileges: - -:::{note} -The packages built from source ([install-packages-from-source.sh][src-install-script]) are installed -without using a packager. So if you ever need to uninstall them, you will need to do so manually. -::: - -```shell -components/core/tools/scripts/lib_install/manylinux_2_28/install-all.sh -``` - -[src-install-script]: https://github.com/y-scope/clp/blob/main/components/core/tools/scripts/lib_install/manylinux_2_28/install-packages-from-source.sh diff --git a/docs/src/dev-guide/tooling-containers.md b/docs/src/dev-guide/tooling-containers.md index 1cf74ea510..2fdc9c356d 100644 --- a/docs/src/dev-guide/tooling-containers.md +++ b/docs/src/dev-guide/tooling-containers.md @@ -3,15 +3,6 @@ We publish (to [GitHub packages][gh-packages]) several Docker container images useful for building and running CLP: -* Various manylinux_2_28 based images - ([x86_64][core-deps-manylinux_2_28_x86_64], [aarch64][core-deps-manylinux_2_28_aarch64]) - containing the dependencies necessary to build CLP core in manylinux_2_28 environments - - ```text - ghcr.io/y-scope/clp/clp-core-dependencies-manylinux_2_28_x86_64:main - ghcr.io/y-scope/clp/clp-core-dependencies-manylinux_2_28_aarch64:main - ``` - * An [image][core-deps-centos-stream-9] containing the dependencies necessary to build CLP core in a Centos Stream 9 x86 environment. @@ -40,8 +31,6 @@ and running CLP: ghcr.io/y-scope/clp/clp-execution-x86-ubuntu-jammy:main ``` -[core-deps-manylinux_2_28_x86_64]: https://github.com/y-scope/clp/pkgs/container/clp%2Fclp-core-dependencies-manylinux_2_28_x86_64 -[core-deps-manylinux_2_28_aarch64]: https://github.com/y-scope/clp/pkgs/container/clp%2Fclp-core-dependencies-manylinux_2_28_aarch64 [core-deps-centos-stream-9]: https://github.com/y-scope/clp/pkgs/container/clp%2Fclp-core-dependencies-x86-centos-stream-9 [core-deps-ubuntu-jammy]: https://github.com/y-scope/clp/pkgs/container/clp%2Fclp-core-dependencies-x86-ubuntu-jammy [core-ubuntu-jammy]: https://github.com/y-scope/clp/pkgs/container/clp%2Fclp-core-x86-ubuntu-jammy From 9112825db5eca24c541a7fbe68fb42d649684519 Mon Sep 17 00:00:00 2001 From: Jack Luo Date: Fri, 27 Jun 2025 11:03:55 -0400 Subject: [PATCH 08/28] Revert "fix: resolved issues with github workflow" This reverts commit 5ab594674aa9a802b2b6ad663bccb8646b3c7bc2. --- .github/workflows/clp-core-build-macos.yaml | 1 + .github/workflows/clp-core-build.yaml | 2 ++ 2 files changed, 3 insertions(+) diff --git a/.github/workflows/clp-core-build-macos.yaml b/.github/workflows/clp-core-build-macos.yaml index 971da5e680..51eb6985d0 100644 --- a/.github/workflows/clp-core-build-macos.yaml +++ b/.github/workflows/clp-core-build-macos.yaml @@ -84,6 +84,7 @@ jobs: run: >- python3 ./tools/scripts/utils/build-and-run-unit-tests.py ${{matrix.use_shared_libs == true && '--use-shared-libs' || ''}} + --source-dir . --build-dir build --num-jobs $(getconf _NPROCESSORS_ONLN) --test-spec "~[Stopwatch]" diff --git a/.github/workflows/clp-core-build.yaml b/.github/workflows/clp-core-build.yaml index a734d3592a..ad0ea8fbf2 100644 --- a/.github/workflows/clp-core-build.yaml +++ b/.github/workflows/clp-core-build.yaml @@ -182,6 +182,7 @@ jobs: CLP_CORE_MAX_PARALLELISM_PER_BUILD_TASK=$(getconf _NPROCESSORS_ONLN) task deps:core && python3 /mnt/repo/components/core/tools/scripts/utils/build-and-run-unit-tests.py ${{matrix.use_shared_libs == true && '--use-shared-libs' || ''}} + --source-dir /mnt/repo/components/core --build-dir /mnt/repo/components/core/build --num-jobs $(getconf _NPROCESSORS_ONLN) @@ -224,6 +225,7 @@ jobs: CLP_CORE_MAX_PARALLELISM_PER_BUILD_TASK=$(getconf _NPROCESSORS_ONLN) task deps:core && python3 /mnt/repo/components/core/tools/scripts/utils/build-and-run-unit-tests.py ${{matrix.use_shared_libs == true && '--use-shared-libs' || ''}} + --source-dir /mnt/repo/components/core --build-dir /mnt/repo/components/core/build --num-jobs $(getconf _NPROCESSORS_ONLN) From b18c2f38d50a3e0d17cf36de2010dc63566f03c6 Mon Sep 17 00:00:00 2001 From: Jack Luo Date: Fri, 27 Jun 2025 13:08:35 -0400 Subject: [PATCH 09/28] fix: resolved issues raised by coderabbit --- components/core/CMakeLists.txt | 4 +--- .../clp-env-base-manylinux_2_28_aarch64/Dockerfile | 4 ++-- .../clp-env-base-manylinux_2_28_x86_64/Dockerfile | 4 ++-- .../tools/scripts/lib_install/manylinux_2_28/install-all.sh | 3 +++ 4 files changed, 8 insertions(+), 7 deletions(-) diff --git a/components/core/CMakeLists.txt b/components/core/CMakeLists.txt index 3ae14354a5..9c8fd3aa0d 100644 --- a/components/core/CMakeLists.txt +++ b/components/core/CMakeLists.txt @@ -73,9 +73,7 @@ if (CLP_USE_STATIC_LIBS) set(CLP_STATIC_LIBS_UNSUPPORTED_PLATFORM "macOS") elseif (EXISTS "/etc/centos-release") # For now, only manylinux_2_28 (AlmaLinux 8 based) supports static linking - file(READ "/etc/centos-release" CENTOS_RELEASE_CONTENT) - string(FIND "${CENTOS_RELEASE_CONTENT}" "AlmaLinux" ALMALINUX_FOUND) - if(ALMALINUX_FOUND EQUAL -1) + if(NOT "${CENTOS_RELEASE_CONTENT}" MATCHES "AlmaLinux") set(CLP_STATIC_LIBS_UNSUPPORTED_PLATFORM "CentOS") endif() endif() diff --git a/components/core/tools/docker-images/clp-env-base-manylinux_2_28_aarch64/Dockerfile b/components/core/tools/docker-images/clp-env-base-manylinux_2_28_aarch64/Dockerfile index 7558829c2a..20cc094d27 100644 --- a/components/core/tools/docker-images/clp-env-base-manylinux_2_28_aarch64/Dockerfile +++ b/components/core/tools/docker-images/clp-env-base-manylinux_2_28_aarch64/Dockerfile @@ -1,4 +1,4 @@ -FROM quay.io/pypa/manylinux_2_28_aarch64 +FROM quay.io/pypa/manylinux_2_28_aarch64:2025.06.25-1 # Binaries built on manylinux_2_28 (AlmaLinux 8 based) is expected to be # compatible with other distros using glibc 2.28 or later, including: @@ -10,7 +10,7 @@ FROM quay.io/pypa/manylinux_2_28_aarch64 WORKDIR /root RUN mkdir -p ./tools/scripts/lib_install -ADD ./tools/scripts/lib_install ./tools/scripts/lib_install +COPY ./tools/scripts/lib_install ./tools/scripts/lib_install RUN ./tools/scripts/lib_install/manylinux_2_28/install-all.sh diff --git a/components/core/tools/docker-images/clp-env-base-manylinux_2_28_x86_64/Dockerfile b/components/core/tools/docker-images/clp-env-base-manylinux_2_28_x86_64/Dockerfile index a0d30bc24e..62edc89b7c 100644 --- a/components/core/tools/docker-images/clp-env-base-manylinux_2_28_x86_64/Dockerfile +++ b/components/core/tools/docker-images/clp-env-base-manylinux_2_28_x86_64/Dockerfile @@ -1,4 +1,4 @@ -FROM quay.io/pypa/manylinux_2_28_x86_64 +FROM quay.io/pypa/manylinux_2_28_x86_64:2025.06.25-1 # Binaries built on manylinux_2_28 (AlmaLinux 8 based) is expected to be # compatible with other distros using glibc 2.28 or later, including: @@ -10,7 +10,7 @@ FROM quay.io/pypa/manylinux_2_28_x86_64 WORKDIR /root RUN mkdir -p ./tools/scripts/lib_install -ADD ./tools/scripts/lib_install ./tools/scripts/lib_install +COPY ./tools/scripts/lib_install ./tools/scripts/lib_install RUN ./tools/scripts/lib_install/manylinux_2_28/install-all.sh diff --git a/components/core/tools/scripts/lib_install/manylinux_2_28/install-all.sh b/components/core/tools/scripts/lib_install/manylinux_2_28/install-all.sh index a44de8130c..de8d5f1300 100755 --- a/components/core/tools/scripts/lib_install/manylinux_2_28/install-all.sh +++ b/components/core/tools/scripts/lib_install/manylinux_2_28/install-all.sh @@ -3,6 +3,9 @@ # Exit on any error set -e +# Exit on failures hidden inside pipes (e.g., curl | tar) +set -o pipefail + # Error on undefined variable set -u From ab09ee3f1f0b6453631ff3d68764b8fff31b6cdd Mon Sep 17 00:00:00 2001 From: Jack Luo Date: Sun, 29 Jun 2025 12:03:18 -0400 Subject: [PATCH 10/28] fix: added back a line that wasn't supposed to be removed --- components/core/CMakeLists.txt | 1 + 1 file changed, 1 insertion(+) diff --git a/components/core/CMakeLists.txt b/components/core/CMakeLists.txt index 9c8fd3aa0d..b92c567a2c 100644 --- a/components/core/CMakeLists.txt +++ b/components/core/CMakeLists.txt @@ -73,6 +73,7 @@ if (CLP_USE_STATIC_LIBS) set(CLP_STATIC_LIBS_UNSUPPORTED_PLATFORM "macOS") elseif (EXISTS "/etc/centos-release") # For now, only manylinux_2_28 (AlmaLinux 8 based) supports static linking + file(READ "/etc/centos-release" CENTOS_RELEASE_CONTENT) if(NOT "${CENTOS_RELEASE_CONTENT}" MATCHES "AlmaLinux") set(CLP_STATIC_LIBS_UNSUPPORTED_PLATFORM "CentOS") endif() From af887736754326a1e8d5a937c29d964cc95258b3 Mon Sep 17 00:00:00 2001 From: Kirk Rodrigues <2454684+kirkrodrigues@users.noreply.github.com> Date: Sun, 29 Jun 2025 15:41:47 -0400 Subject: [PATCH 11/28] Restructure docs before adding new, unpublished container images. --- docs/src/dev-guide/tooling-containers.md | 75 +++++++++++++++++++----- 1 file changed, 61 insertions(+), 14 deletions(-) diff --git a/docs/src/dev-guide/tooling-containers.md b/docs/src/dev-guide/tooling-containers.md index 2fdc9c356d..e3fa81a651 100644 --- a/docs/src/dev-guide/tooling-containers.md +++ b/docs/src/dev-guide/tooling-containers.md @@ -1,34 +1,81 @@ # Containers -We publish (to [GitHub packages][gh-packages]) several Docker container images useful for building -and running CLP: +We maintain several Docker container images that are useful for building and running CLP. All images +can be built and used locally, but some are available to download from +[GitHub Packages][gh-packages]. -* An [image][core-deps-centos-stream-9] containing the dependencies necessary to build CLP core in a - Centos Stream 9 x86 environment. +To build an image locally, run the `build.sh` script in the image's directory. + +## clp-core-dependencies-x86-centos-stream-9 + +An image containing the dependencies necessary to build CLP core in a CentOS Stream 9 x86 +environment. + +* [GitHub Packages page][core-deps-centos-stream-9] +* Pull command: + + ```bash + docker pull ghcr.io/y-scope/clp/clp-core-dependencies-x86-centos-stream-9:main + ``` + +* Path: ```text - ghcr.io/y-scope/clp/clp-core-dependencies-x86-centos-stream-9:main + components/core/tools/docker-images/clp-env-base-centos-stream-9 + ``` + +## clp-core-dependencies-x86-ubuntu-jammy + +An image containing the dependencies necessary to build CLP core in an Ubuntu Jammy x86 +environment. + +* [GitHub Packages page][core-deps-ubuntu-jammy] +* Pull command: + + ```bash + docker pull ghcr.io/y-scope/clp/clp-core-dependencies-x86-ubuntu-jammy:main ``` -* An [image][core-deps-ubuntu-jammy] containing the dependencies necessary to build CLP core in an - Ubuntu Jammy x86 environment. +* Path: ```text - ghcr.io/y-scope/clp/clp-core-dependencies-x86-ubuntu-jammy:main + components/core/tools/docker-images/clp-env-base-ubuntu-jammy ``` -* An [image][core-ubuntu-jammy] containing the CLP core binaries (`clg`, `clp`, `clp-s`, `glt`, - etc.) built in an Ubuntu Jammy x86 environment. +## clp-core-x86-ubuntu-jammy + +An image containing the CLP core binaries (`clg`, `clp`, `clp-s`, `glt`, etc.) built in an Ubuntu +Jammy x86 environment. + +* [GitHub Packages page][core-ubuntu-jammy] +* Pull command: + + ```bash + docker pull ghcr.io/y-scope/clp/clp-core-x86-ubuntu-jammy:main + ``` + +* Path: ```text - ghcr.io/y-scope/clp/clp-core-x86-ubuntu-jammy:main + components/core/tools/docker-images/clp-core-ubuntu-jammy + ``` + +## clp-execution-x86-ubuntu-jammy + +An image containing the dependencies necessary to run the CLP package in an Ubuntu Jammy x86 +environment. + +* [GitHub Packages page][exe-ubuntu-jammy] +* Pull command: + + ```bash + docker pull ghcr.io/y-scope/clp/clp-execution-x86-ubuntu-jammy:main ``` -* An [image][exe-ubuntu-jammy] containing the dependencies necessary to run the CLP package in an - Ubuntu Jammy x86 environment. +* Path: ```text - ghcr.io/y-scope/clp/clp-execution-x86-ubuntu-jammy:main + tools/docker-images/clp-execution-base-ubuntu-jammy ``` [core-deps-centos-stream-9]: https://github.com/y-scope/clp/pkgs/container/clp%2Fclp-core-dependencies-x86-centos-stream-9 From 8378fdb2358852f925613647fb514cf5b5c0f4e0 Mon Sep 17 00:00:00 2001 From: Kirk Rodrigues <2454684+kirkrodrigues@users.noreply.github.com> Date: Sun, 29 Jun 2025 15:43:34 -0400 Subject: [PATCH 12/28] refactor: Remove unnecessary comments from build scripts; Make style consistent. --- .../clp-env-base-manylinux_2_28_aarch64/build.sh | 12 ++++-------- .../clp-env-base-manylinux_2_28_x86_64/build.sh | 12 ++++-------- 2 files changed, 8 insertions(+), 16 deletions(-) diff --git a/components/core/tools/docker-images/clp-env-base-manylinux_2_28_aarch64/build.sh b/components/core/tools/docker-images/clp-env-base-manylinux_2_28_aarch64/build.sh index 262d3f01b9..f5ad4f945e 100755 --- a/components/core/tools/docker-images/clp-env-base-manylinux_2_28_aarch64/build.sh +++ b/components/core/tools/docker-images/clp-env-base-manylinux_2_28_aarch64/build.sh @@ -1,12 +1,11 @@ -#!/bin/bash +#!/usr/bin/env bash -set -euo pipefail +set -eu +set -o pipefail -# Get the directory this script is in script_dir="$( cd "$( dirname "${BASH_SOURCE[0]}" )" &> /dev/null && pwd )" -component_root=${script_dir}/../../../ +component_root="${script_dir}/../../../" -# Build aarch64 image, will automatically use QEMU emulate if not on native platform build_cmd=( docker buildx build --platform linux/arm64 @@ -16,7 +15,6 @@ build_cmd=( --load ) -# If in a git repo, add labels for revision and source if command -v git >/dev/null && git -C "$script_dir" rev-parse --is-inside-work-tree >/dev/null ; then build_cmd+=( @@ -26,6 +24,4 @@ then fi echo "Running: ${build_cmd[*]}" - -# Execute the docker buildx command "${build_cmd[@]}" diff --git a/components/core/tools/docker-images/clp-env-base-manylinux_2_28_x86_64/build.sh b/components/core/tools/docker-images/clp-env-base-manylinux_2_28_x86_64/build.sh index 2e08001b8f..9583b69b5a 100755 --- a/components/core/tools/docker-images/clp-env-base-manylinux_2_28_x86_64/build.sh +++ b/components/core/tools/docker-images/clp-env-base-manylinux_2_28_x86_64/build.sh @@ -1,12 +1,11 @@ -#!/bin/bash +#!/usr/bin/env bash -set -euo pipefail +set -eu +set -o pipefail -# Get the directory this script is in script_dir="$( cd "$( dirname "${BASH_SOURCE[0]}" )" &> /dev/null && pwd )" -component_root=${script_dir}/../../../ +component_root="${script_dir}/../../../" -# Build x86_64 image, will automatically use QEMU emulate if not on native platform build_cmd=( docker buildx build --platform linux/amd64 @@ -16,7 +15,6 @@ build_cmd=( --load ) -# If in a git repo, add labels for revision and source if command -v git >/dev/null && git -C "$script_dir" rev-parse --is-inside-work-tree >/dev/null ; then build_cmd+=( @@ -26,6 +24,4 @@ then fi echo "Running: ${build_cmd[*]}" - -# Execute the docker buildx command "${build_cmd[@]}" From 9c5c624f930b9ed5053913187f0e95ff2779857f Mon Sep 17 00:00:00 2001 From: Kirk Rodrigues <2454684+kirkrodrigues@users.noreply.github.com> Date: Sun, 29 Jun 2025 15:45:02 -0400 Subject: [PATCH 13/28] Use latest conventions in install-all.sh --- .../scripts/lib_install/manylinux_2_28/install-all.sh | 8 +------- 1 file changed, 1 insertion(+), 7 deletions(-) diff --git a/components/core/tools/scripts/lib_install/manylinux_2_28/install-all.sh b/components/core/tools/scripts/lib_install/manylinux_2_28/install-all.sh index de8d5f1300..089cd6bc66 100755 --- a/components/core/tools/scripts/lib_install/manylinux_2_28/install-all.sh +++ b/components/core/tools/scripts/lib_install/manylinux_2_28/install-all.sh @@ -1,14 +1,8 @@ #!/usr/bin/env bash -# Exit on any error -set -e - -# Exit on failures hidden inside pipes (e.g., curl | tar) +set -eu set -o pipefail -# Error on undefined variable -set -u - script_dir="$( cd "$( dirname "${BASH_SOURCE[0]}" )" &> /dev/null && pwd )" "${script_dir}/install-prebuilt-packages.sh" From c31a6c0898625abd301cb12fc2e60df0e06b16d0 Mon Sep 17 00:00:00 2001 From: Kirk Rodrigues <2454684+kirkrodrigues@users.noreply.github.com> Date: Sun, 29 Jun 2025 15:45:54 -0400 Subject: [PATCH 14/28] Use latest conventions in install-packages-from-source.sh; Add comments to explain choice of source packages. --- .../install-packages-from-source.sh | 26 +++++++++---------- 1 file changed, 13 insertions(+), 13 deletions(-) diff --git a/components/core/tools/scripts/lib_install/manylinux_2_28/install-packages-from-source.sh b/components/core/tools/scripts/lib_install/manylinux_2_28/install-packages-from-source.sh index 25f0949b43..3ceafcac16 100755 --- a/components/core/tools/scripts/lib_install/manylinux_2_28/install-packages-from-source.sh +++ b/components/core/tools/scripts/lib_install/manylinux_2_28/install-packages-from-source.sh @@ -1,22 +1,22 @@ #!/usr/bin/env bash -# Exit on any error -set -e - -# Exit on failures hidden inside pipes (e.g., curl | tar) +set -eu set -o pipefail -# Error on undefined variable -set -u - script_dir="$( cd "$( dirname "${BASH_SOURCE[0]}" )" &> /dev/null && pwd )" lib_install_scripts_dir="${script_dir}/.." # NOTE: The remaining installation scripts depend on boost, so we install it beforehand. -"${lib_install_scripts_dir}"/install-boost.sh 1.87.0 +"${lib_install_scripts_dir}/install-boost.sh" 1.87.0 + +# NOTE: +# 1. libarchive may statically link with LZMA, LZ4, and Zstandard, so we install them beforehand. +# 2. The versions of libarchive, LZMA, LZ4, and Zstandard available in the manylinux 2.28 package +# repositories are either dated or don't include static libraries, so we install more recent +# versions from source. +"${lib_install_scripts_dir}/liblzma.sh" 5.8.1 +"${lib_install_scripts_dir}/lz4.sh" 1.10.0 +"${lib_install_scripts_dir}/zstandard.sh" 1.5.7 +"${lib_install_scripts_dir}/libarchive.sh" 3.8.0 -"$lib_install_scripts_dir"/liblzma.sh 5.8.1 -"$lib_install_scripts_dir"/lz4.sh 1.10.0 -"$lib_install_scripts_dir"/zstandard.sh 1.5.7 -"$lib_install_scripts_dir"/libarchive.sh 3.8.0 -"${lib_install_scripts_dir}"/msgpack.sh 7.0.0 +"${lib_install_scripts_dir}/msgpack.sh" 7.0.0 From dfa89e99ecc768807bb28f0352a469a5ae957340 Mon Sep 17 00:00:00 2001 From: Kirk Rodrigues <2454684+kirkrodrigues@users.noreply.github.com> Date: Sun, 29 Jun 2025 15:47:23 -0400 Subject: [PATCH 15/28] Use latest conventions in install-prebuilt-packages.sh; Remove unnecessary comments; Use pipx to downgrade CMake to v3. --- .../install-prebuilt-packages.sh | 39 ++++--------------- 1 file changed, 8 insertions(+), 31 deletions(-) diff --git a/components/core/tools/scripts/lib_install/manylinux_2_28/install-prebuilt-packages.sh b/components/core/tools/scripts/lib_install/manylinux_2_28/install-prebuilt-packages.sh index 8f526b4b46..04cf58a19a 100755 --- a/components/core/tools/scripts/lib_install/manylinux_2_28/install-prebuilt-packages.sh +++ b/components/core/tools/scripts/lib_install/manylinux_2_28/install-prebuilt-packages.sh @@ -1,23 +1,17 @@ #!/usr/bin/env bash -# Exit on any error -set -e - -# Exit on failures hidden inside pipes (e.g., curl | tar) +set -eu set -o pipefail -# Error on undefined variable -set -u - dnf install -y \ gcc-c++ \ java-11-openjdk \ + jq \ libcurl-devel \ mariadb-connector-c-devel \ openssl-devel \ zlib-devel \ - zlib-static \ - jq + zlib-static # Determine architecture for `task` release to install rpm_arch=$(rpm --eval "%{_arch}") @@ -36,8 +30,7 @@ esac # Install `task` # NOTE: We lock `task` to a version < 3.43 to avoid https://github.com/y-scope/clp/issues/872 -tmp_rpm=$(mktemp -t --suffix ".rpm" task-pkg.XXXXXXXXXX) || exit 1 -task_pkg_path="$tmp_rpm" +task_pkg_path=$(mktemp -t --suffix ".rpm" task-pkg.XXXXXXXXXX) || exit 1 curl \ --fail \ --location \ @@ -45,24 +38,8 @@ curl \ --show-error \ "https://github.com/go-task/task/releases/download/v3.42.1/task_linux_${task_pkg_arch}.rpm" dnf install --assumeyes "$task_pkg_path" -rm -rf "$task_pkg_path" +rm "$task_pkg_path" -# The bundled CMake 4.x version is too new; downgrading to CMake 3.27 is -# necessary to resolve build issues with third-party libraries. Even after -# we added the "-DCMAKE_POLICY_VERSION_MINIMUM=3.5" flag, compilation issues -# in libraries such as googletest-src and yaml-cpp still persist. -# See sample error below: -# CMake Error at _deps/googletest-src/CMakeLists.txt:4 (cmake_minimum_required): -# Support for CMake versions below 3.5 has been dropped. -# -# Adjust the VERSION argument’s minimum value, or use the ... syntax -# to indicate the required CMake version range. -# -# Alternatively, you can try adding -DCMAKE_POLICY_VERSION_MINIMUM=3.5 -export PLATFORM=$(uname -m) -export CMAKE_VERSION=3.27.9 -export CMAKE_RELEASE_URL=https://github.com/Kitware/CMake/releases/download/v${CMAKE_VERSION}/cmake-${CMAKE_VERSION}-linux-${PLATFORM}.tar.gz -echo "Downgrading CMAKE to 3.27.9" -curl -L "${CMAKE_RELEASE_URL}" | tar -xzf - -mv "cmake-${CMAKE_VERSION}-linux-${PLATFORM}" "/opt/cmake-${CMAKE_VERSION}" -ln -sf "/opt/cmake-${CMAKE_VERSION}/bin/cmake" /usr/local/bin/cmake +# Downgrade to CMake v3 to work around https://github.com/y-scope/clp/issues/795 +pipx uninstall cmake +pipx install cmake~=3.0 From c74423767712fcdd2a7b50656765ca388a601ab6 Mon Sep 17 00:00:00 2001 From: Kirk Rodrigues <2454684+kirkrodrigues@users.noreply.github.com> Date: Sun, 29 Jun 2025 15:57:45 -0400 Subject: [PATCH 16/28] Simplify comment. --- .../clp-env-base-manylinux_2_28_aarch64/Dockerfile | 6 ++---- .../clp-env-base-manylinux_2_28_x86_64/Dockerfile | 6 ++---- 2 files changed, 4 insertions(+), 8 deletions(-) diff --git a/components/core/tools/docker-images/clp-env-base-manylinux_2_28_aarch64/Dockerfile b/components/core/tools/docker-images/clp-env-base-manylinux_2_28_aarch64/Dockerfile index 20cc094d27..773ef785f1 100644 --- a/components/core/tools/docker-images/clp-env-base-manylinux_2_28_aarch64/Dockerfile +++ b/components/core/tools/docker-images/clp-env-base-manylinux_2_28_aarch64/Dockerfile @@ -17,7 +17,5 @@ RUN ./tools/scripts/lib_install/manylinux_2_28/install-all.sh # Remove cached files RUN dnf clean all && rm -rf /var/cache/dnf /tmp/* /var/tmp/* -# DO NOT FLATTEN THE IMAGE!!! -# The consequence of flattening the image is that we lose shell startup scripts -# and environment modification from /etc/profile.d, /etc/bashrc, etc. -# These are important for manylinux, otherwise, default system compilers are used, etc. +# NOTE: Don't flatten the image or else we'll lose any environment modifications from the base +# image. diff --git a/components/core/tools/docker-images/clp-env-base-manylinux_2_28_x86_64/Dockerfile b/components/core/tools/docker-images/clp-env-base-manylinux_2_28_x86_64/Dockerfile index 62edc89b7c..bf73cd8b30 100644 --- a/components/core/tools/docker-images/clp-env-base-manylinux_2_28_x86_64/Dockerfile +++ b/components/core/tools/docker-images/clp-env-base-manylinux_2_28_x86_64/Dockerfile @@ -17,7 +17,5 @@ RUN ./tools/scripts/lib_install/manylinux_2_28/install-all.sh # Remove cached files RUN dnf clean all && rm -rf /var/cache/dnf /tmp/* /var/tmp/* -# DO NOT FLATTEN THE IMAGE!!! -# The consequence of flattening the image is that we lose shell startup scripts -# and environment modification from /etc/profile.d, /etc/bashrc, etc. -# These are important for manylinux, otherwise, default system compilers are used, etc. +# NOTE: Don't flatten the image or else we'll lose any environment modifications from the base +# image. From 8cb738f93f40856a65eb8b430a16e14e7c5c8928 Mon Sep 17 00:00:00 2001 From: Kirk Rodrigues <2454684+kirkrodrigues@users.noreply.github.com> Date: Sun, 29 Jun 2025 16:02:57 -0400 Subject: [PATCH 17/28] Rename images for consistency. --- .../Dockerfile | 0 .../build.sh | 2 +- .../Dockerfile | 0 .../build.sh | 2 +- 4 files changed, 2 insertions(+), 2 deletions(-) rename components/core/tools/docker-images/{clp-env-base-manylinux_2_28_aarch64 => clp-env-base-manylinux-2.28-aarch64}/Dockerfile (100%) rename components/core/tools/docker-images/{clp-env-base-manylinux_2_28_aarch64 => clp-env-base-manylinux-2.28-aarch64}/build.sh (91%) rename components/core/tools/docker-images/{clp-env-base-manylinux_2_28_x86_64 => clp-env-base-manylinux-2.28-x86_64}/Dockerfile (100%) rename components/core/tools/docker-images/{clp-env-base-manylinux_2_28_x86_64 => clp-env-base-manylinux-2.28-x86_64}/build.sh (92%) diff --git a/components/core/tools/docker-images/clp-env-base-manylinux_2_28_aarch64/Dockerfile b/components/core/tools/docker-images/clp-env-base-manylinux-2.28-aarch64/Dockerfile similarity index 100% rename from components/core/tools/docker-images/clp-env-base-manylinux_2_28_aarch64/Dockerfile rename to components/core/tools/docker-images/clp-env-base-manylinux-2.28-aarch64/Dockerfile diff --git a/components/core/tools/docker-images/clp-env-base-manylinux_2_28_aarch64/build.sh b/components/core/tools/docker-images/clp-env-base-manylinux-2.28-aarch64/build.sh similarity index 91% rename from components/core/tools/docker-images/clp-env-base-manylinux_2_28_aarch64/build.sh rename to components/core/tools/docker-images/clp-env-base-manylinux-2.28-aarch64/build.sh index f5ad4f945e..551d7e3c26 100755 --- a/components/core/tools/docker-images/clp-env-base-manylinux_2_28_aarch64/build.sh +++ b/components/core/tools/docker-images/clp-env-base-manylinux-2.28-aarch64/build.sh @@ -9,7 +9,7 @@ component_root="${script_dir}/../../../" build_cmd=( docker buildx build --platform linux/arm64 - --tag clp-core-dependencies-manylinux_2_28_aarch64:dev + --tag clp-core-dependencies-aarch64-manylinux-2.28:dev "$component_root" --file "${script_dir}/Dockerfile" --load diff --git a/components/core/tools/docker-images/clp-env-base-manylinux_2_28_x86_64/Dockerfile b/components/core/tools/docker-images/clp-env-base-manylinux-2.28-x86_64/Dockerfile similarity index 100% rename from components/core/tools/docker-images/clp-env-base-manylinux_2_28_x86_64/Dockerfile rename to components/core/tools/docker-images/clp-env-base-manylinux-2.28-x86_64/Dockerfile diff --git a/components/core/tools/docker-images/clp-env-base-manylinux_2_28_x86_64/build.sh b/components/core/tools/docker-images/clp-env-base-manylinux-2.28-x86_64/build.sh similarity index 92% rename from components/core/tools/docker-images/clp-env-base-manylinux_2_28_x86_64/build.sh rename to components/core/tools/docker-images/clp-env-base-manylinux-2.28-x86_64/build.sh index 9583b69b5a..bf6f00eecd 100755 --- a/components/core/tools/docker-images/clp-env-base-manylinux_2_28_x86_64/build.sh +++ b/components/core/tools/docker-images/clp-env-base-manylinux-2.28-x86_64/build.sh @@ -9,7 +9,7 @@ component_root="${script_dir}/../../../" build_cmd=( docker buildx build --platform linux/amd64 - --tag clp-core-dependencies-manylinux_2_28_x86_64:dev + --tag clp-core-dependencies-x86-manylinux-2.28:dev "$component_root" --file "${script_dir}/Dockerfile" --load From a35809d97d3268f10344dfafed9c5e87175900c1 Mon Sep 17 00:00:00 2001 From: Kirk Rodrigues <2454684+kirkrodrigues@users.noreply.github.com> Date: Sun, 29 Jun 2025 16:13:50 -0400 Subject: [PATCH 18/28] Add new images to docs. --- .../Dockerfile | 7 ----- .../Dockerfile | 7 ----- docs/src/dev-guide/tooling-containers.md | 29 +++++++++++++++++++ 3 files changed, 29 insertions(+), 14 deletions(-) diff --git a/components/core/tools/docker-images/clp-env-base-manylinux-2.28-aarch64/Dockerfile b/components/core/tools/docker-images/clp-env-base-manylinux-2.28-aarch64/Dockerfile index 773ef785f1..4691c6f3f5 100644 --- a/components/core/tools/docker-images/clp-env-base-manylinux-2.28-aarch64/Dockerfile +++ b/components/core/tools/docker-images/clp-env-base-manylinux-2.28-aarch64/Dockerfile @@ -1,12 +1,5 @@ FROM quay.io/pypa/manylinux_2_28_aarch64:2025.06.25-1 -# Binaries built on manylinux_2_28 (AlmaLinux 8 based) is expected to be -# compatible with other distros using glibc 2.28 or later, including: -# - Debian 10+ -# - Ubuntu 18.10+ -# - Fedora 29+ -# - CentOS/RHEL 8+ - WORKDIR /root RUN mkdir -p ./tools/scripts/lib_install diff --git a/components/core/tools/docker-images/clp-env-base-manylinux-2.28-x86_64/Dockerfile b/components/core/tools/docker-images/clp-env-base-manylinux-2.28-x86_64/Dockerfile index bf73cd8b30..f71ff91152 100644 --- a/components/core/tools/docker-images/clp-env-base-manylinux-2.28-x86_64/Dockerfile +++ b/components/core/tools/docker-images/clp-env-base-manylinux-2.28-x86_64/Dockerfile @@ -1,12 +1,5 @@ FROM quay.io/pypa/manylinux_2_28_x86_64:2025.06.25-1 -# Binaries built on manylinux_2_28 (AlmaLinux 8 based) is expected to be -# compatible with other distros using glibc 2.28 or later, including: -# - Debian 10+ -# - Ubuntu 18.10+ -# - Fedora 29+ -# - CentOS/RHEL 8+ - WORKDIR /root RUN mkdir -p ./tools/scripts/lib_install diff --git a/docs/src/dev-guide/tooling-containers.md b/docs/src/dev-guide/tooling-containers.md index e3fa81a651..0b2b2bc776 100644 --- a/docs/src/dev-guide/tooling-containers.md +++ b/docs/src/dev-guide/tooling-containers.md @@ -6,6 +6,35 @@ can be built and used locally, but some are available to download from To build an image locally, run the `build.sh` script in the image's directory. +## clp-core-dependencies-<arch>-manylinux-2.28 + +Images containing the dependencies necessary to build CLP core in a manylinux 2.28 environment +(aarch64 or x86). + +Binaries built on manylinux 2.28 (based on AlmaLinux 8) are expected to be compatible with other +distros using glibc 2.28+, including: + +* CentOS/RHEL 8+ +* Debian 10+ +* Fedora 29+ +* Ubuntu 18.10+ + +### clp-core-dependencies-aarch64-manylinux-2.28 + +* Path: + + ```text + components/core/tools/docker-images/clp-env-base-manylinux-2.28-aarch64 + ``` + +### clp-core-dependencies-x86-manylinux-2.28 + +* Path: + + ```text + components/core/tools/docker-images/clp-env-base-manylinux-2.28-x86_64 + ``` + ## clp-core-dependencies-x86-centos-stream-9 An image containing the dependencies necessary to build CLP core in a CentOS Stream 9 x86 From b1716df37a4a9a3111b4b59b879fcec13bebef59 Mon Sep 17 00:00:00 2001 From: Kirk Rodrigues <2454684+kirkrodrigues@users.noreply.github.com> Date: Sun, 29 Jun 2025 16:16:22 -0400 Subject: [PATCH 19/28] Edit comment. --- components/core/tools/scripts/lib_install/libarchive.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/components/core/tools/scripts/lib_install/libarchive.sh b/components/core/tools/scripts/lib_install/libarchive.sh index 3a74578ed4..dd8c2b733e 100755 --- a/components/core/tools/scripts/lib_install/libarchive.sh +++ b/components/core/tools/scripts/lib_install/libarchive.sh @@ -63,7 +63,7 @@ fi cd ${extracted_dir} mkdir -p cmake-build-release cd cmake-build-release -# NOTE: Disable expat and openssl so the static libarchive doesn't look for it at link time +# NOTE: Disable Expat and OpenSSL so the static libarchive doesn't look for them at link time. cmake -DENABLE_EXPAT=OFF -DENABLE_OPENSSL=OFF ../ make -j${num_cpus} From e916ecba73569718180a3608d91a885c2068ee9b Mon Sep 17 00:00:00 2001 From: Kirk Rodrigues <2454684+kirkrodrigues@users.noreply.github.com> Date: Sun, 29 Jun 2025 16:18:32 -0400 Subject: [PATCH 20/28] Don't lock base image version so that we can tell if something breaks in the latest base image. --- .../clp-env-base-manylinux-2.28-aarch64/Dockerfile | 2 +- .../docker-images/clp-env-base-manylinux-2.28-x86_64/Dockerfile | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/components/core/tools/docker-images/clp-env-base-manylinux-2.28-aarch64/Dockerfile b/components/core/tools/docker-images/clp-env-base-manylinux-2.28-aarch64/Dockerfile index 4691c6f3f5..06df2a2961 100644 --- a/components/core/tools/docker-images/clp-env-base-manylinux-2.28-aarch64/Dockerfile +++ b/components/core/tools/docker-images/clp-env-base-manylinux-2.28-aarch64/Dockerfile @@ -1,4 +1,4 @@ -FROM quay.io/pypa/manylinux_2_28_aarch64:2025.06.25-1 +FROM quay.io/pypa/manylinux_2_28_aarch64 WORKDIR /root diff --git a/components/core/tools/docker-images/clp-env-base-manylinux-2.28-x86_64/Dockerfile b/components/core/tools/docker-images/clp-env-base-manylinux-2.28-x86_64/Dockerfile index f71ff91152..1185c7d031 100644 --- a/components/core/tools/docker-images/clp-env-base-manylinux-2.28-x86_64/Dockerfile +++ b/components/core/tools/docker-images/clp-env-base-manylinux-2.28-x86_64/Dockerfile @@ -1,4 +1,4 @@ -FROM quay.io/pypa/manylinux_2_28_x86_64:2025.06.25-1 +FROM quay.io/pypa/manylinux_2_28_x86_64 WORKDIR /root From e1809e86b23dc546b80c318a54942ee33da351f9 Mon Sep 17 00:00:00 2001 From: Kirk Rodrigues <2454684+kirkrodrigues@users.noreply.github.com> Date: Sun, 29 Jun 2025 16:32:21 -0400 Subject: [PATCH 21/28] Improve comment. --- components/core/CMakeLists.txt | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) diff --git a/components/core/CMakeLists.txt b/components/core/CMakeLists.txt index b92c567a2c..af3f4e8631 100644 --- a/components/core/CMakeLists.txt +++ b/components/core/CMakeLists.txt @@ -72,7 +72,11 @@ if (CLP_USE_STATIC_LIBS) if (APPLE) set(CLP_STATIC_LIBS_UNSUPPORTED_PLATFORM "macOS") elseif (EXISTS "/etc/centos-release") - # For now, only manylinux_2_28 (AlmaLinux 8 based) supports static linking + # NOTE: + # 1. We don't support static linking on any CentOS-based distro except manylinux 2.28 (which + # shows up as "AlmaLinux"). + # 2. A released called "AlmaLinux" doesn't guarantee we're running on a manylinux distro, + # but we can improve this check when someone reports an issue. file(READ "/etc/centos-release" CENTOS_RELEASE_CONTENT) if(NOT "${CENTOS_RELEASE_CONTENT}" MATCHES "AlmaLinux") set(CLP_STATIC_LIBS_UNSUPPORTED_PLATFORM "CentOS") From 502a31cbace4215ef169ac06a1ad6bdb44ce39ce Mon Sep 17 00:00:00 2001 From: Kirk Rodrigues <2454684+kirkrodrigues@users.noreply.github.com> Date: Sun, 29 Jun 2025 16:53:13 -0400 Subject: [PATCH 22/28] Rename manylinux scripts directory. --- .../clp-env-base-manylinux-2.28-aarch64/Dockerfile | 2 +- .../docker-images/clp-env-base-manylinux-2.28-x86_64/Dockerfile | 2 +- .../{manylinux_2_28 => manylinux-2.28}/install-all.sh | 0 .../install-packages-from-source.sh | 0 .../install-prebuilt-packages.sh | 0 5 files changed, 2 insertions(+), 2 deletions(-) rename components/core/tools/scripts/lib_install/{manylinux_2_28 => manylinux-2.28}/install-all.sh (100%) rename components/core/tools/scripts/lib_install/{manylinux_2_28 => manylinux-2.28}/install-packages-from-source.sh (100%) rename components/core/tools/scripts/lib_install/{manylinux_2_28 => manylinux-2.28}/install-prebuilt-packages.sh (100%) diff --git a/components/core/tools/docker-images/clp-env-base-manylinux-2.28-aarch64/Dockerfile b/components/core/tools/docker-images/clp-env-base-manylinux-2.28-aarch64/Dockerfile index 06df2a2961..42c7320014 100644 --- a/components/core/tools/docker-images/clp-env-base-manylinux-2.28-aarch64/Dockerfile +++ b/components/core/tools/docker-images/clp-env-base-manylinux-2.28-aarch64/Dockerfile @@ -5,7 +5,7 @@ WORKDIR /root RUN mkdir -p ./tools/scripts/lib_install COPY ./tools/scripts/lib_install ./tools/scripts/lib_install -RUN ./tools/scripts/lib_install/manylinux_2_28/install-all.sh +RUN ./tools/scripts/lib_install/manylinux-2.28/install-all.sh # Remove cached files RUN dnf clean all && rm -rf /var/cache/dnf /tmp/* /var/tmp/* diff --git a/components/core/tools/docker-images/clp-env-base-manylinux-2.28-x86_64/Dockerfile b/components/core/tools/docker-images/clp-env-base-manylinux-2.28-x86_64/Dockerfile index 1185c7d031..1617ccd2ff 100644 --- a/components/core/tools/docker-images/clp-env-base-manylinux-2.28-x86_64/Dockerfile +++ b/components/core/tools/docker-images/clp-env-base-manylinux-2.28-x86_64/Dockerfile @@ -5,7 +5,7 @@ WORKDIR /root RUN mkdir -p ./tools/scripts/lib_install COPY ./tools/scripts/lib_install ./tools/scripts/lib_install -RUN ./tools/scripts/lib_install/manylinux_2_28/install-all.sh +RUN ./tools/scripts/lib_install/manylinux-2.28/install-all.sh # Remove cached files RUN dnf clean all && rm -rf /var/cache/dnf /tmp/* /var/tmp/* diff --git a/components/core/tools/scripts/lib_install/manylinux_2_28/install-all.sh b/components/core/tools/scripts/lib_install/manylinux-2.28/install-all.sh similarity index 100% rename from components/core/tools/scripts/lib_install/manylinux_2_28/install-all.sh rename to components/core/tools/scripts/lib_install/manylinux-2.28/install-all.sh diff --git a/components/core/tools/scripts/lib_install/manylinux_2_28/install-packages-from-source.sh b/components/core/tools/scripts/lib_install/manylinux-2.28/install-packages-from-source.sh similarity index 100% rename from components/core/tools/scripts/lib_install/manylinux_2_28/install-packages-from-source.sh rename to components/core/tools/scripts/lib_install/manylinux-2.28/install-packages-from-source.sh diff --git a/components/core/tools/scripts/lib_install/manylinux_2_28/install-prebuilt-packages.sh b/components/core/tools/scripts/lib_install/manylinux-2.28/install-prebuilt-packages.sh similarity index 100% rename from components/core/tools/scripts/lib_install/manylinux_2_28/install-prebuilt-packages.sh rename to components/core/tools/scripts/lib_install/manylinux-2.28/install-prebuilt-packages.sh From f96a13a5b32f340a4614705ac92bdb6188c2ed94 Mon Sep 17 00:00:00 2001 From: Kirk Rodrigues <2454684+kirkrodrigues@users.noreply.github.com> Date: Sun, 29 Jun 2025 16:54:48 -0400 Subject: [PATCH 23/28] Update more docs. --- docs/src/dev-guide/components-core/index.md | 2 ++ .../manylinux-2.28-deps-install.md | 17 +++++++++++++++++ 2 files changed, 19 insertions(+) create mode 100644 docs/src/dev-guide/components-core/manylinux-2.28-deps-install.md diff --git a/docs/src/dev-guide/components-core/index.md b/docs/src/dev-guide/components-core/index.md index 64d33cd5f8..05845f05d5 100644 --- a/docs/src/dev-guide/components-core/index.md +++ b/docs/src/dev-guide/components-core/index.md @@ -65,6 +65,7 @@ See the relevant README for your OS: * [CentOS Stream 9](centos-stream-9-deps-install) * [macOS](macos-deps-install) +* [manylinux 2.28](manylinux-2.28-deps-install) * [Ubuntu 22.04](ubuntu-jammy-deps-install) Want to build natively on an OS not listed here? You can file a [feature request][feature-req]. @@ -108,6 +109,7 @@ the relevant paths on your machine. centos-stream-9-deps-install macos-deps-install +manylinux-2.28-deps-install ubuntu-jammy-deps-install regex-utils ::: diff --git a/docs/src/dev-guide/components-core/manylinux-2.28-deps-install.md b/docs/src/dev-guide/components-core/manylinux-2.28-deps-install.md new file mode 100644 index 0000000000..c2cec5c58f --- /dev/null +++ b/docs/src/dev-guide/components-core/manylinux-2.28-deps-install.md @@ -0,0 +1,17 @@ +# manylinux 2.28 setup + +To install the dependencies required to build clp-core, follow the steps below. These same steps are +used by our Docker containers. + +## Installing dependencies + +:::{caution} +Before you run any commands below, you should review the scripts to ensure they will not install +any dependencies or apply any configurations that you don't expect. +::: + +To install all dependencies, run the following with elevated privileges: + +```shell +components/core/tools/scripts/lib_install/manylinux-2.28/install-all.sh +``` From 10dc855a38f5a5a87da0e4b204465df316473931 Mon Sep 17 00:00:00 2001 From: Kirk Rodrigues <2454684+kirkrodrigues@users.noreply.github.com> Date: Sun, 29 Jun 2025 22:24:42 -0400 Subject: [PATCH 24/28] Set minimum CMake version to 3.31. --- .../lib_install/manylinux-2.28/install-prebuilt-packages.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/components/core/tools/scripts/lib_install/manylinux-2.28/install-prebuilt-packages.sh b/components/core/tools/scripts/lib_install/manylinux-2.28/install-prebuilt-packages.sh index 04cf58a19a..57ea2caab0 100755 --- a/components/core/tools/scripts/lib_install/manylinux-2.28/install-prebuilt-packages.sh +++ b/components/core/tools/scripts/lib_install/manylinux-2.28/install-prebuilt-packages.sh @@ -42,4 +42,4 @@ rm "$task_pkg_path" # Downgrade to CMake v3 to work around https://github.com/y-scope/clp/issues/795 pipx uninstall cmake -pipx install cmake~=3.0 +pipx install cmake~=3.31 From 02d257d55a5a337f094bb407fe5a8575c39c63f2 Mon Sep 17 00:00:00 2001 From: Kirk Rodrigues <2454684+kirkrodrigues@users.noreply.github.com> Date: Sun, 29 Jun 2025 22:26:33 -0400 Subject: [PATCH 25/28] Add link to manylinux 2.28. --- docs/src/dev-guide/tooling-containers.md | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/docs/src/dev-guide/tooling-containers.md b/docs/src/dev-guide/tooling-containers.md index 0b2b2bc776..ae5915cd98 100644 --- a/docs/src/dev-guide/tooling-containers.md +++ b/docs/src/dev-guide/tooling-containers.md @@ -8,8 +8,8 @@ To build an image locally, run the `build.sh` script in the image's directory. ## clp-core-dependencies-<arch>-manylinux-2.28 -Images containing the dependencies necessary to build CLP core in a manylinux 2.28 environment -(aarch64 or x86). +Images containing the dependencies necessary to build CLP core in a [manylinux 2.28][manylinux-2.28] +environment (aarch64 or x86). Binaries built on manylinux 2.28 (based on AlmaLinux 8) are expected to be compatible with other distros using glibc 2.28+, including: @@ -112,3 +112,4 @@ environment. [core-ubuntu-jammy]: https://github.com/y-scope/clp/pkgs/container/clp%2Fclp-core-x86-ubuntu-jammy [exe-ubuntu-jammy]: https://github.com/y-scope/clp/pkgs/container/clp%2Fclp-execution-x86-ubuntu-jammy [gh-packages]: https://github.com/orgs/y-scope/packages?repo_name=clp +[manylinux-2.28]: https://github.com/pypa/manylinux?tab=readme-ov-file#manylinux_2_28-almalinux-8-based From 31e987497f7acc13e099e7b1a6cf09e473552a45 Mon Sep 17 00:00:00 2001 From: kirkrodrigues <2454684+kirkrodrigues@users.noreply.github.com> Date: Sun, 29 Jun 2025 22:27:15 -0400 Subject: [PATCH 26/28] Update components/core/CMakeLists.txt Co-authored-by: Lin Zhihao <59785146+LinZhihao-723@users.noreply.github.com> --- components/core/CMakeLists.txt | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/components/core/CMakeLists.txt b/components/core/CMakeLists.txt index af3f4e8631..d36def7bdf 100644 --- a/components/core/CMakeLists.txt +++ b/components/core/CMakeLists.txt @@ -75,8 +75,8 @@ if (CLP_USE_STATIC_LIBS) # NOTE: # 1. We don't support static linking on any CentOS-based distro except manylinux 2.28 (which # shows up as "AlmaLinux"). - # 2. A released called "AlmaLinux" doesn't guarantee we're running on a manylinux distro, - # but we can improve this check when someone reports an issue. + # 2. A release called "AlmaLinux" doesn't guarantee we're running on a manylinux distro, but + # we can improve this check when someone reports an issue. file(READ "/etc/centos-release" CENTOS_RELEASE_CONTENT) if(NOT "${CENTOS_RELEASE_CONTENT}" MATCHES "AlmaLinux") set(CLP_STATIC_LIBS_UNSUPPORTED_PLATFORM "CentOS") From 8fd2fa04b09d1b3f29ed2abd789c96a80744d1e7 Mon Sep 17 00:00:00 2001 From: Kirk Rodrigues <2454684+kirkrodrigues@users.noreply.github.com> Date: Sun, 29 Jun 2025 22:51:35 -0400 Subject: [PATCH 27/28] Revert "Update more docs." This reverts commit f96a13a5b32f340a4614705ac92bdb6188c2ed94. --- docs/src/dev-guide/components-core/index.md | 2 -- .../manylinux-2.28-deps-install.md | 17 ----------------- 2 files changed, 19 deletions(-) delete mode 100644 docs/src/dev-guide/components-core/manylinux-2.28-deps-install.md diff --git a/docs/src/dev-guide/components-core/index.md b/docs/src/dev-guide/components-core/index.md index 05845f05d5..64d33cd5f8 100644 --- a/docs/src/dev-guide/components-core/index.md +++ b/docs/src/dev-guide/components-core/index.md @@ -65,7 +65,6 @@ See the relevant README for your OS: * [CentOS Stream 9](centos-stream-9-deps-install) * [macOS](macos-deps-install) -* [manylinux 2.28](manylinux-2.28-deps-install) * [Ubuntu 22.04](ubuntu-jammy-deps-install) Want to build natively on an OS not listed here? You can file a [feature request][feature-req]. @@ -109,7 +108,6 @@ the relevant paths on your machine. centos-stream-9-deps-install macos-deps-install -manylinux-2.28-deps-install ubuntu-jammy-deps-install regex-utils ::: diff --git a/docs/src/dev-guide/components-core/manylinux-2.28-deps-install.md b/docs/src/dev-guide/components-core/manylinux-2.28-deps-install.md deleted file mode 100644 index c2cec5c58f..0000000000 --- a/docs/src/dev-guide/components-core/manylinux-2.28-deps-install.md +++ /dev/null @@ -1,17 +0,0 @@ -# manylinux 2.28 setup - -To install the dependencies required to build clp-core, follow the steps below. These same steps are -used by our Docker containers. - -## Installing dependencies - -:::{caution} -Before you run any commands below, you should review the scripts to ensure they will not install -any dependencies or apply any configurations that you don't expect. -::: - -To install all dependencies, run the following with elevated privileges: - -```shell -components/core/tools/scripts/lib_install/manylinux-2.28/install-all.sh -``` From 57cda645f9a3f53b1b6bd1785b3973bd9aa8a02b Mon Sep 17 00:00:00 2001 From: Kirk Rodrigues <2454684+kirkrodrigues@users.noreply.github.com> Date: Mon, 30 Jun 2025 06:22:54 -0400 Subject: [PATCH 28/28] Replace manylinux 2.28 with manylinux_2_28. --- components/core/CMakeLists.txt | 2 +- .../Dockerfile | 2 +- .../build.sh | 2 +- .../Dockerfile | 2 +- .../build.sh | 2 +- .../install-all.sh | 0 .../install-packages-from-source.sh | 2 +- .../install-prebuilt-packages.sh | 0 docs/src/dev-guide/tooling-containers.md | 16 ++++++++-------- 9 files changed, 14 insertions(+), 14 deletions(-) rename components/core/tools/docker-images/{clp-env-base-manylinux-2.28-aarch64 => clp-env-base-manylinux_2_28-aarch64}/Dockerfile (84%) rename components/core/tools/docker-images/{clp-env-base-manylinux-2.28-aarch64 => clp-env-base-manylinux_2_28-aarch64}/build.sh (91%) rename components/core/tools/docker-images/{clp-env-base-manylinux-2.28-x86_64 => clp-env-base-manylinux_2_28-x86_64}/Dockerfile (84%) rename components/core/tools/docker-images/{clp-env-base-manylinux-2.28-x86_64 => clp-env-base-manylinux_2_28-x86_64}/build.sh (92%) rename components/core/tools/scripts/lib_install/{manylinux-2.28 => manylinux_2_28}/install-all.sh (100%) rename components/core/tools/scripts/lib_install/{manylinux-2.28 => manylinux_2_28}/install-packages-from-source.sh (96%) rename components/core/tools/scripts/lib_install/{manylinux-2.28 => manylinux_2_28}/install-prebuilt-packages.sh (100%) diff --git a/components/core/CMakeLists.txt b/components/core/CMakeLists.txt index d36def7bdf..6b4c966a29 100644 --- a/components/core/CMakeLists.txt +++ b/components/core/CMakeLists.txt @@ -73,7 +73,7 @@ if (CLP_USE_STATIC_LIBS) set(CLP_STATIC_LIBS_UNSUPPORTED_PLATFORM "macOS") elseif (EXISTS "/etc/centos-release") # NOTE: - # 1. We don't support static linking on any CentOS-based distro except manylinux 2.28 (which + # 1. We don't support static linking on any CentOS-based distro except manylinux_2_28 (which # shows up as "AlmaLinux"). # 2. A release called "AlmaLinux" doesn't guarantee we're running on a manylinux distro, but # we can improve this check when someone reports an issue. diff --git a/components/core/tools/docker-images/clp-env-base-manylinux-2.28-aarch64/Dockerfile b/components/core/tools/docker-images/clp-env-base-manylinux_2_28-aarch64/Dockerfile similarity index 84% rename from components/core/tools/docker-images/clp-env-base-manylinux-2.28-aarch64/Dockerfile rename to components/core/tools/docker-images/clp-env-base-manylinux_2_28-aarch64/Dockerfile index 42c7320014..06df2a2961 100644 --- a/components/core/tools/docker-images/clp-env-base-manylinux-2.28-aarch64/Dockerfile +++ b/components/core/tools/docker-images/clp-env-base-manylinux_2_28-aarch64/Dockerfile @@ -5,7 +5,7 @@ WORKDIR /root RUN mkdir -p ./tools/scripts/lib_install COPY ./tools/scripts/lib_install ./tools/scripts/lib_install -RUN ./tools/scripts/lib_install/manylinux-2.28/install-all.sh +RUN ./tools/scripts/lib_install/manylinux_2_28/install-all.sh # Remove cached files RUN dnf clean all && rm -rf /var/cache/dnf /tmp/* /var/tmp/* diff --git a/components/core/tools/docker-images/clp-env-base-manylinux-2.28-aarch64/build.sh b/components/core/tools/docker-images/clp-env-base-manylinux_2_28-aarch64/build.sh similarity index 91% rename from components/core/tools/docker-images/clp-env-base-manylinux-2.28-aarch64/build.sh rename to components/core/tools/docker-images/clp-env-base-manylinux_2_28-aarch64/build.sh index 551d7e3c26..e31049522a 100755 --- a/components/core/tools/docker-images/clp-env-base-manylinux-2.28-aarch64/build.sh +++ b/components/core/tools/docker-images/clp-env-base-manylinux_2_28-aarch64/build.sh @@ -9,7 +9,7 @@ component_root="${script_dir}/../../../" build_cmd=( docker buildx build --platform linux/arm64 - --tag clp-core-dependencies-aarch64-manylinux-2.28:dev + --tag clp-core-dependencies-aarch64-manylinux_2_28:dev "$component_root" --file "${script_dir}/Dockerfile" --load diff --git a/components/core/tools/docker-images/clp-env-base-manylinux-2.28-x86_64/Dockerfile b/components/core/tools/docker-images/clp-env-base-manylinux_2_28-x86_64/Dockerfile similarity index 84% rename from components/core/tools/docker-images/clp-env-base-manylinux-2.28-x86_64/Dockerfile rename to components/core/tools/docker-images/clp-env-base-manylinux_2_28-x86_64/Dockerfile index 1617ccd2ff..1185c7d031 100644 --- a/components/core/tools/docker-images/clp-env-base-manylinux-2.28-x86_64/Dockerfile +++ b/components/core/tools/docker-images/clp-env-base-manylinux_2_28-x86_64/Dockerfile @@ -5,7 +5,7 @@ WORKDIR /root RUN mkdir -p ./tools/scripts/lib_install COPY ./tools/scripts/lib_install ./tools/scripts/lib_install -RUN ./tools/scripts/lib_install/manylinux-2.28/install-all.sh +RUN ./tools/scripts/lib_install/manylinux_2_28/install-all.sh # Remove cached files RUN dnf clean all && rm -rf /var/cache/dnf /tmp/* /var/tmp/* diff --git a/components/core/tools/docker-images/clp-env-base-manylinux-2.28-x86_64/build.sh b/components/core/tools/docker-images/clp-env-base-manylinux_2_28-x86_64/build.sh similarity index 92% rename from components/core/tools/docker-images/clp-env-base-manylinux-2.28-x86_64/build.sh rename to components/core/tools/docker-images/clp-env-base-manylinux_2_28-x86_64/build.sh index bf6f00eecd..b7212f02c2 100755 --- a/components/core/tools/docker-images/clp-env-base-manylinux-2.28-x86_64/build.sh +++ b/components/core/tools/docker-images/clp-env-base-manylinux_2_28-x86_64/build.sh @@ -9,7 +9,7 @@ component_root="${script_dir}/../../../" build_cmd=( docker buildx build --platform linux/amd64 - --tag clp-core-dependencies-x86-manylinux-2.28:dev + --tag clp-core-dependencies-x86-manylinux_2_28:dev "$component_root" --file "${script_dir}/Dockerfile" --load diff --git a/components/core/tools/scripts/lib_install/manylinux-2.28/install-all.sh b/components/core/tools/scripts/lib_install/manylinux_2_28/install-all.sh similarity index 100% rename from components/core/tools/scripts/lib_install/manylinux-2.28/install-all.sh rename to components/core/tools/scripts/lib_install/manylinux_2_28/install-all.sh diff --git a/components/core/tools/scripts/lib_install/manylinux-2.28/install-packages-from-source.sh b/components/core/tools/scripts/lib_install/manylinux_2_28/install-packages-from-source.sh similarity index 96% rename from components/core/tools/scripts/lib_install/manylinux-2.28/install-packages-from-source.sh rename to components/core/tools/scripts/lib_install/manylinux_2_28/install-packages-from-source.sh index 3ceafcac16..d31343c8ee 100755 --- a/components/core/tools/scripts/lib_install/manylinux-2.28/install-packages-from-source.sh +++ b/components/core/tools/scripts/lib_install/manylinux_2_28/install-packages-from-source.sh @@ -11,7 +11,7 @@ lib_install_scripts_dir="${script_dir}/.." # NOTE: # 1. libarchive may statically link with LZMA, LZ4, and Zstandard, so we install them beforehand. -# 2. The versions of libarchive, LZMA, LZ4, and Zstandard available in the manylinux 2.28 package +# 2. The versions of libarchive, LZMA, LZ4, and Zstandard available in manylinux_2_28's package # repositories are either dated or don't include static libraries, so we install more recent # versions from source. "${lib_install_scripts_dir}/liblzma.sh" 5.8.1 diff --git a/components/core/tools/scripts/lib_install/manylinux-2.28/install-prebuilt-packages.sh b/components/core/tools/scripts/lib_install/manylinux_2_28/install-prebuilt-packages.sh similarity index 100% rename from components/core/tools/scripts/lib_install/manylinux-2.28/install-prebuilt-packages.sh rename to components/core/tools/scripts/lib_install/manylinux_2_28/install-prebuilt-packages.sh diff --git a/docs/src/dev-guide/tooling-containers.md b/docs/src/dev-guide/tooling-containers.md index ae5915cd98..1e43246344 100644 --- a/docs/src/dev-guide/tooling-containers.md +++ b/docs/src/dev-guide/tooling-containers.md @@ -6,12 +6,12 @@ can be built and used locally, but some are available to download from To build an image locally, run the `build.sh` script in the image's directory. -## clp-core-dependencies-<arch>-manylinux-2.28 +## clp-core-dependencies-<arch>-manylinux_2_28 -Images containing the dependencies necessary to build CLP core in a [manylinux 2.28][manylinux-2.28] +Images containing the dependencies necessary to build CLP core in a [manylinux_2_28][manylinux_2_28] environment (aarch64 or x86). -Binaries built on manylinux 2.28 (based on AlmaLinux 8) are expected to be compatible with other +Binaries built on manylinux_2_28 (based on AlmaLinux 8) are expected to be compatible with other distros using glibc 2.28+, including: * CentOS/RHEL 8+ @@ -19,20 +19,20 @@ distros using glibc 2.28+, including: * Fedora 29+ * Ubuntu 18.10+ -### clp-core-dependencies-aarch64-manylinux-2.28 +### clp-core-dependencies-aarch64-manylinux_2_28 * Path: ```text - components/core/tools/docker-images/clp-env-base-manylinux-2.28-aarch64 + components/core/tools/docker-images/clp-env-base-manylinux_2_28-aarch64 ``` -### clp-core-dependencies-x86-manylinux-2.28 +### clp-core-dependencies-x86-manylinux_2_28 * Path: ```text - components/core/tools/docker-images/clp-env-base-manylinux-2.28-x86_64 + components/core/tools/docker-images/clp-env-base-manylinux_2_28-x86_64 ``` ## clp-core-dependencies-x86-centos-stream-9 @@ -112,4 +112,4 @@ environment. [core-ubuntu-jammy]: https://github.com/y-scope/clp/pkgs/container/clp%2Fclp-core-x86-ubuntu-jammy [exe-ubuntu-jammy]: https://github.com/y-scope/clp/pkgs/container/clp%2Fclp-execution-x86-ubuntu-jammy [gh-packages]: https://github.com/orgs/y-scope/packages?repo_name=clp -[manylinux-2.28]: https://github.com/pypa/manylinux?tab=readme-ov-file#manylinux_2_28-almalinux-8-based +[manylinux_2_28]: https://github.com/pypa/manylinux?tab=readme-ov-file#manylinux_2_28-almalinux-8-based