From acc4a62fa0387c62acd6a4b7ff83d6bf6e7ff81c Mon Sep 17 00:00:00 2001 From: Bingran Hu Date: Sun, 24 Aug 2025 15:19:11 -0400 Subject: [PATCH 01/69] Add manylinux deps image build --- .github/workflows/clp-core-build.yaml | 38 +++++++++++++++++++++++++++ 1 file changed, 38 insertions(+) diff --git a/.github/workflows/clp-core-build.yaml b/.github/workflows/clp-core-build.yaml index 6a04c11798..0cb3177277 100644 --- a/.github/workflows/clp-core-build.yaml +++ b/.github/workflows/clp-core-build.yaml @@ -76,6 +76,18 @@ jobs: - "components/core/tools/scripts/lib_install/*.sh" - "components/core/tools/docker-images/clp-env-base-centos-stream-9/**" - "components/core/tools/scripts/lib_install/centos-stream-9/**" + manylinux_2_28_image: + - ".github/actions/**" + - ".github/workflows/clp-core-build.yaml" + - "components/core/tools/scripts/lib_install/*.sh" + - "components/core/tools/docker-images/clp-env-base-manylinux_2_28-x86_64/**" + - "components/core/tools/scripts/lib_install/manylinux_2_28/**" + musllinux_1_2_image: + - ".github/actions/**" + - ".github/workflows/clp-core-build.yaml" + - "components/core/tools/scripts/lib_install/*.sh" + - "components/core/tools/docker-images/clp-env-base-musllinux_1_2-x86_64/**" + - "components/core/tools/scripts/lib_install/musllinux_1_2/**" ubuntu_jammy_image: - ".github/actions/**" - ".github/workflows/clp-core-build.yaml" @@ -121,6 +133,32 @@ jobs: ${{github.event_name != 'pull_request' && github.ref == 'refs/heads/main'}} token: "${{secrets.GITHUB_TOKEN}}" + manylinux-2-28-deps-image: + name: "manylinux-2-28-deps-image" + if: "needs.filter-relevant-changes.outputs.manylinux_2_28_image_changed == 'true'" + needs: "filter-relevant-changes" + runs-on: "ubuntu-24.04" + steps: + - uses: "actions/checkout@11bd71901bbe5b1630ceea73d27597364c9af683" + with: + submodules: "recursive" + + - name: "Work around actions/runner-images/issues/6775" + run: "chown $(id -u):$(id -g) -R ." + shell: "bash" + + - uses: "./.github/actions/clp-core-build-containers" + env: + OS_NAME: "manylinux_2_28" + with: + image_name: "${{env.DEPS_IMAGE_NAME_PREFIX}}${{env.OS_NAME}}" + docker_context: "components/core" + docker_file: "components/core/tools/docker-images/clp-env-base-${{env.OS_NAME}}\ + /Dockerfile" + push_deps_image: >- + ${{github.event_name != 'pull_request' && github.ref == 'refs/heads/main'}} + token: "${{secrets.GITHUB_TOKEN}}" + ubuntu-jammy-deps-image: name: "ubuntu-jammy-deps-image" if: "needs.filter-relevant-changes.outputs.ubuntu_jammy_image_changed == 'true'" From 81d679903420932e0523032ad9f94a91ea42c97a Mon Sep 17 00:00:00 2001 From: Bingran Hu Date: Sun, 24 Aug 2025 15:34:08 -0400 Subject: [PATCH 02/69] Small fix --- .../tools/scripts/lib_install/manylinux_2_28/install-all.sh | 3 +-- .../lib_install/manylinux_2_28/install-prebuilt-packages.sh | 5 +++-- 2 files changed, 4 insertions(+), 4 deletions(-) diff --git a/components/core/tools/scripts/lib_install/manylinux_2_28/install-all.sh b/components/core/tools/scripts/lib_install/manylinux_2_28/install-all.sh index 089cd6bc66..f834003b7c 100755 --- a/components/core/tools/scripts/lib_install/manylinux_2_28/install-all.sh +++ b/components/core/tools/scripts/lib_install/manylinux_2_28/install-all.sh @@ -1,7 +1,6 @@ #!/usr/bin/env bash -set -eu -set -o pipefail +set -euo pipefail script_dir="$( cd "$( dirname "${BASH_SOURCE[0]}" )" &> /dev/null && pwd )" diff --git a/components/core/tools/scripts/lib_install/manylinux_2_28/install-prebuilt-packages.sh b/components/core/tools/scripts/lib_install/manylinux_2_28/install-prebuilt-packages.sh index 57ea2caab0..875540ebe6 100755 --- a/components/core/tools/scripts/lib_install/manylinux_2_28/install-prebuilt-packages.sh +++ b/components/core/tools/scripts/lib_install/manylinux_2_28/install-prebuilt-packages.sh @@ -41,5 +41,6 @@ dnf install --assumeyes "$task_pkg_path" rm "$task_pkg_path" # Downgrade to CMake v3 to work around https://github.com/y-scope/clp/issues/795 -pipx uninstall cmake -pipx install cmake~=3.31 +if ! command -v cmake ; then + pipx install cmake~=3.31 +fi From bdb75bb31d838e06573f9f53803bf4e85df17a08 Mon Sep 17 00:00:00 2001 From: Bingran Hu Date: Sun, 24 Aug 2025 15:48:44 -0400 Subject: [PATCH 03/69] Create filter outputs --- .github/workflows/clp-core-build.yaml | 2 ++ 1 file changed, 2 insertions(+) diff --git a/.github/workflows/clp-core-build.yaml b/.github/workflows/clp-core-build.yaml index 0cb3177277..e47d506881 100644 --- a/.github/workflows/clp-core-build.yaml +++ b/.github/workflows/clp-core-build.yaml @@ -45,6 +45,8 @@ jobs: runs-on: "ubuntu-24.04" outputs: centos_stream_9_image_changed: "${{steps.filter.outputs.centos_stream_9_image}}" + manylinux_2_28_image_changed: "${{steps.filter.outputs.manylinux_2_28_image}}" + musllinux_1_2_image_changed: "${{steps.filter.outputs.musllinux_1_2_image}}" ubuntu_jammy_image_changed: "${{steps.filter.outputs.ubuntu_jammy_image}}" clp_changed: "${{steps.filter.outputs.clp}}" steps: From 373d43f4eee8ebc01a06afaa05d20c189373accb Mon Sep 17 00:00:00 2001 From: Bingran Hu Date: Sun, 24 Aug 2025 16:04:07 -0400 Subject: [PATCH 04/69] Update docker dir --- .github/workflows/clp-core-build.yaml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/clp-core-build.yaml b/.github/workflows/clp-core-build.yaml index e47d506881..34aa1e5579 100644 --- a/.github/workflows/clp-core-build.yaml +++ b/.github/workflows/clp-core-build.yaml @@ -155,7 +155,7 @@ jobs: with: image_name: "${{env.DEPS_IMAGE_NAME_PREFIX}}${{env.OS_NAME}}" docker_context: "components/core" - docker_file: "components/core/tools/docker-images/clp-env-base-${{env.OS_NAME}}\ + docker_file: "components/core/tools/docker-images/clp-env-base-${{env.OS_NAME}}-x86_64\ /Dockerfile" push_deps_image: >- ${{github.event_name != 'pull_request' && github.ref == 'refs/heads/main'}} From 7f8706e42c49d4d94e1fc375ee80db9a0bbb6796 Mon Sep 17 00:00:00 2001 From: Bingran Hu Date: Mon, 25 Aug 2025 00:49:08 -0400 Subject: [PATCH 05/69] Fix cmake version --- .../manylinux_2_28/install-prebuilt-packages.sh | 8 +++++--- 1 file changed, 5 insertions(+), 3 deletions(-) diff --git a/components/core/tools/scripts/lib_install/manylinux_2_28/install-prebuilt-packages.sh b/components/core/tools/scripts/lib_install/manylinux_2_28/install-prebuilt-packages.sh index 875540ebe6..6cde2fef18 100755 --- a/components/core/tools/scripts/lib_install/manylinux_2_28/install-prebuilt-packages.sh +++ b/components/core/tools/scripts/lib_install/manylinux_2_28/install-prebuilt-packages.sh @@ -40,7 +40,9 @@ curl \ dnf install --assumeyes "$task_pkg_path" rm "$task_pkg_path" -# Downgrade to CMake v3 to work around https://github.com/y-scope/clp/issues/795 -if ! command -v cmake ; then - pipx install cmake~=3.31 +# Install CMake v3.31.x as ANTLR and yaml-cpp do not yet support CMake v4+. +# See also: https://github.com/y-scope/clp/issues/795 +if command -v cmake ; then + pipx uninstall cmake fi +pipx install "cmake~=3.31" From 39d7abb138dd7f2010031a2b6f730657eb5dcc13 Mon Sep 17 00:00:00 2001 From: Bingran Hu Date: Mon, 25 Aug 2025 01:13:57 -0400 Subject: [PATCH 06/69] Add remaining workflows --- .github/workflows/clp-core-build.yaml | 104 ++++++++++++++++++++++++++ 1 file changed, 104 insertions(+) diff --git a/.github/workflows/clp-core-build.yaml b/.github/workflows/clp-core-build.yaml index 34aa1e5579..abdb1d7661 100644 --- a/.github/workflows/clp-core-build.yaml +++ b/.github/workflows/clp-core-build.yaml @@ -161,6 +161,32 @@ jobs: ${{github.event_name != 'pull_request' && github.ref == 'refs/heads/main'}} token: "${{secrets.GITHUB_TOKEN}}" + musllinux-1-2-deps-image: + name: "musllinux-1-2-deps-image" + if: "needs.filter-relevant-changes.outputs.musllinux_1_2_image_changed == 'true'" + needs: "filter-relevant-changes" + runs-on: "ubuntu-24.04" + steps: + - uses: "actions/checkout@11bd71901bbe5b1630ceea73d27597364c9af683" + with: + submodules: "recursive" + + - name: "Work around actions/runner-images/issues/6775" + run: "chown $(id -u):$(id -g) -R ." + shell: "bash" + + - uses: "./.github/actions/clp-core-build-containers" + env: + OS_NAME: "musllinux_1_2" + with: + image_name: "${{env.DEPS_IMAGE_NAME_PREFIX}}${{env.OS_NAME}}" + docker_context: "components/core" + docker_file: "components/core/tools/docker-images/clp-env-base-${{env.OS_NAME}}-x86_64\ + /Dockerfile" + push_deps_image: >- + ${{github.event_name != 'pull_request' && github.ref == 'refs/heads/main'}} + token: "${{secrets.GITHUB_TOKEN}}" + ubuntu-jammy-deps-image: name: "ubuntu-jammy-deps-image" if: "needs.filter-relevant-changes.outputs.ubuntu_jammy_image_changed == 'true'" @@ -226,6 +252,84 @@ jobs: --build-dir /mnt/repo/components/core/build --num-jobs $(getconf _NPROCESSORS_ONLN) + manylinux-2-28-binaries: + # Run if the ancestor jobs succeeded OR they were skipped and clp was changed. + if: >- + success() + || (!cancelled() && !failure() && needs.filter-relevant-changes.outputs.clp_changed == 'true') + needs: + - "manylinux-2-28-deps-image" + - "filter-relevant-changes" + strategy: + matrix: + use_shared_libs: [true, false] + name: "manylinux-2-28-${{matrix.use_shared_libs == true && 'dynamic' || 'static'}}-linked-bins" + continue-on-error: true + runs-on: "ubuntu-24.04" + steps: + - uses: "actions/checkout@11bd71901bbe5b1630ceea73d27597364c9af683" + with: + submodules: "recursive" + + - name: "Work around actions/runner-images/issues/6775" + run: "chown $(id -u):$(id -g) -R ." + shell: "bash" + + - uses: "./.github/actions/run-on-image" + env: + OS_NAME: "manylinux_2_28" + with: + image_name: "${{env.DEPS_IMAGE_NAME_PREFIX}}${{env.OS_NAME}}" + use_published_image: >- + ${{needs.filter-relevant-changes.outputs.manylinux_2_28_image_changed == 'false' + || (github.event_name != 'pull_request' && github.ref == 'refs/heads/main')}} + run_command: >- + CLP_CORE_MAX_PARALLELISM_PER_BUILD_TASK=$(getconf _NPROCESSORS_ONLN) task deps:core + && python3 /mnt/repo/components/core/tools/scripts/utils/build-and-run-unit-tests.py + ${{matrix.use_shared_libs == true && '--use-shared-libs' || ''}} + --source-dir /mnt/repo/components/core + --build-dir /mnt/repo/components/core/build + --num-jobs $(getconf _NPROCESSORS_ONLN) + + musllinux-1-2-binaries: + # Run if the ancestor jobs succeeded OR they were skipped and clp was changed. + if: >- + success() + || (!cancelled() && !failure() && needs.filter-relevant-changes.outputs.clp_changed == 'true') + needs: + - "musllinux-1-2-deps-image" + - "filter-relevant-changes" + strategy: + matrix: + use_shared_libs: [true, false] + name: "musllinux-1-2-${{matrix.use_shared_libs == true && 'dynamic' || 'static'}}-linked-bins" + continue-on-error: true + runs-on: "ubuntu-24.04" + steps: + - uses: "actions/checkout@11bd71901bbe5b1630ceea73d27597364c9af683" + with: + submodules: "recursive" + + - name: "Work around actions/runner-images/issues/6775" + run: "chown $(id -u):$(id -g) -R ." + shell: "bash" + + - uses: "./.github/actions/run-on-image" + env: + OS_NAME: "musllinux_1_2" + with: + image_name: "${{env.DEPS_IMAGE_NAME_PREFIX}}${{env.OS_NAME}}" + use_published_image: >- + ${{needs.filter-relevant-changes.outputs.musllinux_1_2_image_changed == 'false' + || (github.event_name != 'pull_request' && github.ref == 'refs/heads/main')}} + run_command: >- + CLP_CORE_MAX_PARALLELISM_PER_BUILD_TASK=$(getconf _NPROCESSORS_ONLN) task deps:core + && python3 /mnt/repo/components/core/tools/scripts/utils/build-and-run-unit-tests.py + ${{matrix.use_shared_libs == true && '--use-shared-libs' || ''}} + --source-dir /mnt/repo/components/core + --build-dir /mnt/repo/components/core/build + --num-jobs $(getconf _NPROCESSORS_ONLN) + ubuntu-jammy-binaries: # Run if the ancestor jobs succeeded OR they were skipped and clp was changed. if: >- From 916a4b62144196e67b762ac7246b6ef8eec62809 Mon Sep 17 00:00:00 2001 From: Bingran Hu Date: Mon, 25 Aug 2025 01:37:28 -0400 Subject: [PATCH 07/69] Remove unrelated changes --- .../scripts/lib_install/manylinux_2_28/install-all.sh | 3 ++- .../manylinux_2_28/install-prebuilt-packages.sh | 9 +++------ 2 files changed, 5 insertions(+), 7 deletions(-) diff --git a/components/core/tools/scripts/lib_install/manylinux_2_28/install-all.sh b/components/core/tools/scripts/lib_install/manylinux_2_28/install-all.sh index f834003b7c..089cd6bc66 100755 --- a/components/core/tools/scripts/lib_install/manylinux_2_28/install-all.sh +++ b/components/core/tools/scripts/lib_install/manylinux_2_28/install-all.sh @@ -1,6 +1,7 @@ #!/usr/bin/env bash -set -euo pipefail +set -eu +set -o pipefail script_dir="$( cd "$( dirname "${BASH_SOURCE[0]}" )" &> /dev/null && pwd )" diff --git a/components/core/tools/scripts/lib_install/manylinux_2_28/install-prebuilt-packages.sh b/components/core/tools/scripts/lib_install/manylinux_2_28/install-prebuilt-packages.sh index 6cde2fef18..57ea2caab0 100755 --- a/components/core/tools/scripts/lib_install/manylinux_2_28/install-prebuilt-packages.sh +++ b/components/core/tools/scripts/lib_install/manylinux_2_28/install-prebuilt-packages.sh @@ -40,9 +40,6 @@ curl \ dnf install --assumeyes "$task_pkg_path" rm "$task_pkg_path" -# Install CMake v3.31.x as ANTLR and yaml-cpp do not yet support CMake v4+. -# See also: https://github.com/y-scope/clp/issues/795 -if command -v cmake ; then - pipx uninstall cmake -fi -pipx install "cmake~=3.31" +# Downgrade to CMake v3 to work around https://github.com/y-scope/clp/issues/795 +pipx uninstall cmake +pipx install cmake~=3.31 From 06a902031d15ae93d86c5e37f3fac0da0f001a1b Mon Sep 17 00:00:00 2001 From: Bingran Hu Date: Mon, 25 Aug 2025 02:06:52 -0400 Subject: [PATCH 08/69] Update docs --- docs/src/dev-guide/components-core/index.md | 4 ++++ .../manylinux-2-28-deps-install.md | 24 +++++++++++++++++++ .../musllinux-1-2-deps-install.md | 24 +++++++++++++++++++ 3 files changed, 52 insertions(+) create mode 100644 docs/src/dev-guide/components-core/manylinux-2-28-deps-install.md create mode 100644 docs/src/dev-guide/components-core/musllinux-1-2-deps-install.md diff --git a/docs/src/dev-guide/components-core/index.md b/docs/src/dev-guide/components-core/index.md index 272fcc0269..c8905189e1 100644 --- a/docs/src/dev-guide/components-core/index.md +++ b/docs/src/dev-guide/components-core/index.md @@ -70,6 +70,8 @@ A handful of packages and libraries are required to build CLP. There are two opt See the relevant README for your OS: * [CentOS Stream 9](centos-stream-9-deps-install) +* [Manylinux 2.28](manylinux-2-28-deps-install) +* [Musllinux 1.2](musllinux-1-2-deps-install) * [macOS](macos-deps-install) * [Ubuntu 22.04](ubuntu-jammy-deps-install) @@ -113,6 +115,8 @@ the relevant paths on your machine. :hidden: centos-stream-9-deps-install +manylinux-2-28-deps-install +musllinux-1-2-deps-install macos-deps-install ubuntu-jammy-deps-install regex-utils diff --git a/docs/src/dev-guide/components-core/manylinux-2-28-deps-install.md b/docs/src/dev-guide/components-core/manylinux-2-28-deps-install.md new file mode 100644 index 0000000000..dc7dd99d7d --- /dev/null +++ b/docs/src/dev-guide/components-core/manylinux-2-28-deps-install.md @@ -0,0 +1,24 @@ +# Manylinux 2.28 + +To install the dependencies required to build clp-core, follow the steps below. +These same steps are used by our Docker containers. + +## Installing dependencies + +:::{caution} +Before you run any commands below, you should review the scripts to ensure they will not install +any dependencies or apply any configurations that you don't expect. +::: + +To install all dependencies, run the following with elevated privileges: + +:::{note} +The packages built from source ([install-packages-from-source.sh][src-install-script]) are installed +without using a packager. So if you ever need to uninstall them, you will need to do so manually. +::: + +```shell +components/core/tools/scripts/lib_install/manylinux_2_28/install-all.sh +``` + +[src-install-script]: https://github.com/y-scope/clp/blob/main/components/core/tools/scripts/lib_install/manylinux_2_28/install-packages-from-source.sh diff --git a/docs/src/dev-guide/components-core/musllinux-1-2-deps-install.md b/docs/src/dev-guide/components-core/musllinux-1-2-deps-install.md new file mode 100644 index 0000000000..fda0370ad6 --- /dev/null +++ b/docs/src/dev-guide/components-core/musllinux-1-2-deps-install.md @@ -0,0 +1,24 @@ +# Musllinux 1.2 + +To install the dependencies required to build clp-core, follow the steps below. +These same steps are used by our Docker containers. + +## Installing dependencies + +:::{caution} +Before you run any commands below, you should review the scripts to ensure they will not install +any dependencies or apply any configurations that you don't expect. +::: + +To install all dependencies, run the following with elevated privileges: + +:::{note} +The packages built from source ([install-packages-from-source.sh][src-install-script]) are installed +without using a packager. So if you ever need to uninstall them, you will need to do so manually. +::: + +```shell +components/core/tools/scripts/lib_install/musllinux_1_2/install-all.sh +``` + +[src-install-script]: https://github.com/y-scope/clp/blob/main/components/core/tools/scripts/lib_install/musllinux_1_2/install-packages-from-source.sh From b6f143671cc0178a4b5e4ace4670193922da9733 Mon Sep 17 00:00:00 2001 From: Bingran Hu Date: Mon, 25 Aug 2025 03:50:41 -0400 Subject: [PATCH 09/69] Add manylinux arm64 deps img workflow --- .github/workflows/clp-core-build.yaml | 26 ++++++++++++++++++++++++++ 1 file changed, 26 insertions(+) diff --git a/.github/workflows/clp-core-build.yaml b/.github/workflows/clp-core-build.yaml index abdb1d7661..a0c4b4d253 100644 --- a/.github/workflows/clp-core-build.yaml +++ b/.github/workflows/clp-core-build.yaml @@ -135,6 +135,32 @@ jobs: ${{github.event_name != 'pull_request' && github.ref == 'refs/heads/main'}} token: "${{secrets.GITHUB_TOKEN}}" + manylinux-2-28-aarch64-deps-image: + name: "manylinux-2-28-aarch64-deps-image" + if: "needs.filter-relevant-changes.outputs.manylinux_2_28_image_changed == 'true'" + needs: "filter-relevant-changes" + runs-on: "ubuntu-24.04-arm" + steps: + - uses: "actions/checkout@11bd71901bbe5b1630ceea73d27597364c9af683" + with: + submodules: "recursive" + + - name: "Work around actions/runner-images/issues/6775" + run: "chown $(id -u):$(id -g) -R ." + shell: "bash" + + - uses: "./.github/actions/clp-core-build-containers" + env: + OS_NAME: "manylinux_2_28-aarch64" + with: + image_name: "${{env.DEPS_IMAGE_NAME_PREFIX}}${{env.OS_NAME}}" + docker_context: "components/core" + docker_file: "components/core/tools/docker-images/clp-env-base-${{env.OS_NAME}}\ + /Dockerfile" + push_deps_image: >- + ${{github.event_name != 'pull_request' && github.ref == 'refs/heads/main'}} + token: "${{secrets.GITHUB_TOKEN}}" + manylinux-2-28-deps-image: name: "manylinux-2-28-deps-image" if: "needs.filter-relevant-changes.outputs.manylinux_2_28_image_changed == 'true'" From 93bee22e1a0fc46d74303270ed56871e7655e2a1 Mon Sep 17 00:00:00 2001 From: Bingran Hu Date: Mon, 25 Aug 2025 04:47:16 -0400 Subject: [PATCH 10/69] Add both x86_64 and arm64 versions for two platforms --- .github/workflows/clp-core-build.yaml | 180 ++++++++++++++++++++++---- 1 file changed, 153 insertions(+), 27 deletions(-) diff --git a/.github/workflows/clp-core-build.yaml b/.github/workflows/clp-core-build.yaml index a0c4b4d253..d3d18365d5 100644 --- a/.github/workflows/clp-core-build.yaml +++ b/.github/workflows/clp-core-build.yaml @@ -45,8 +45,10 @@ jobs: runs-on: "ubuntu-24.04" outputs: centos_stream_9_image_changed: "${{steps.filter.outputs.centos_stream_9_image}}" - manylinux_2_28_image_changed: "${{steps.filter.outputs.manylinux_2_28_image}}" - musllinux_1_2_image_changed: "${{steps.filter.outputs.musllinux_1_2_image}}" + manylinux_2_28_aarch64_image_changed: "${{steps.filter.outputs.manylinux_2_28_aarch64_image}}" + manylinux_2_28_x86_64_image_changed: "${{steps.filter.outputs.manylinux_2_28_x86_64_image}}" + musllinux_1_2_aarch64_image_changed: "${{steps.filter.outputs.musllinux_1_2_aarch64_image}}" + musllinux_1_2_x86_64_image_changed: "${{steps.filter.outputs.musllinux_1_2_x86_64_image}}" ubuntu_jammy_image_changed: "${{steps.filter.outputs.ubuntu_jammy_image}}" clp_changed: "${{steps.filter.outputs.clp}}" steps: @@ -78,13 +80,25 @@ jobs: - "components/core/tools/scripts/lib_install/*.sh" - "components/core/tools/docker-images/clp-env-base-centos-stream-9/**" - "components/core/tools/scripts/lib_install/centos-stream-9/**" - manylinux_2_28_image: + manylinux_2_28_aarch64_image: + - ".github/actions/**" + - ".github/workflows/clp-core-build.yaml" + - "components/core/tools/scripts/lib_install/*.sh" + - "components/core/tools/docker-images/clp-env-base-manylinux_2_28-aarch64/**" + - "components/core/tools/scripts/lib_install/manylinux_2_28/**" + manylinux_2_28_x86_64_image: - ".github/actions/**" - ".github/workflows/clp-core-build.yaml" - "components/core/tools/scripts/lib_install/*.sh" - "components/core/tools/docker-images/clp-env-base-manylinux_2_28-x86_64/**" + musllinux_1_2_aarch64_image: + - ".github/actions/**" + - ".github/workflows/clp-core-build.yaml" + - "components/core/tools/scripts/lib_install/*.sh" + - "components/core/tools/docker-images/clp-env-base-musllinux_1_2-aarch64/**" + - "components/core/tools/scripts/lib_install/musllinux_1_2/**" - "components/core/tools/scripts/lib_install/manylinux_2_28/**" - musllinux_1_2_image: + musllinux_1_2_x86_64_image: - ".github/actions/**" - ".github/workflows/clp-core-build.yaml" - "components/core/tools/scripts/lib_install/*.sh" @@ -135,9 +149,9 @@ jobs: ${{github.event_name != 'pull_request' && github.ref == 'refs/heads/main'}} token: "${{secrets.GITHUB_TOKEN}}" - manylinux-2-28-aarch64-deps-image: - name: "manylinux-2-28-aarch64-deps-image" - if: "needs.filter-relevant-changes.outputs.manylinux_2_28_image_changed == 'true'" + manylinux_2_28-aarch64-deps-image: + name: "manylinux_2_28-aarch64-deps-image" + if: "needs.filter-relevant-changes.outputs.manylinux_2_28_aarch64_image_changed == 'true'" needs: "filter-relevant-changes" runs-on: "ubuntu-24.04-arm" steps: @@ -161,9 +175,9 @@ jobs: ${{github.event_name != 'pull_request' && github.ref == 'refs/heads/main'}} token: "${{secrets.GITHUB_TOKEN}}" - manylinux-2-28-deps-image: - name: "manylinux-2-28-deps-image" - if: "needs.filter-relevant-changes.outputs.manylinux_2_28_image_changed == 'true'" + manylinux_2_28-x86_64-deps-image: + name: "manylinux_2_28-x86_64-deps-image" + if: "needs.filter-relevant-changes.outputs.manylinux_2_28_x86_64_image_changed == 'true'" needs: "filter-relevant-changes" runs-on: "ubuntu-24.04" steps: @@ -177,19 +191,45 @@ jobs: - uses: "./.github/actions/clp-core-build-containers" env: - OS_NAME: "manylinux_2_28" + OS_NAME: "manylinux_2_28-x86_64" with: image_name: "${{env.DEPS_IMAGE_NAME_PREFIX}}${{env.OS_NAME}}" docker_context: "components/core" - docker_file: "components/core/tools/docker-images/clp-env-base-${{env.OS_NAME}}-x86_64\ + docker_file: "components/core/tools/docker-images/clp-env-base-${{env.OS_NAME}}\ /Dockerfile" push_deps_image: >- ${{github.event_name != 'pull_request' && github.ref == 'refs/heads/main'}} token: "${{secrets.GITHUB_TOKEN}}" - musllinux-1-2-deps-image: - name: "musllinux-1-2-deps-image" - if: "needs.filter-relevant-changes.outputs.musllinux_1_2_image_changed == 'true'" + musllinux_1_2-aarch64-deps-image: + name: "musllinux_1_2-aarch64-deps-image" + if: "needs.filter-relevant-changes.outputs.musllinux_1_2_aarch64_image_changed == 'true'" + needs: "filter-relevant-changes" + runs-on: "ubuntu-24.04-arm" + steps: + - uses: "actions/checkout@11bd71901bbe5b1630ceea73d27597364c9af683" + with: + submodules: "recursive" + + - name: "Work around actions/runner-images/issues/6775" + run: "chown $(id -u):$(id -g) -R ." + shell: "bash" + + - uses: "./.github/actions/clp-core-build-containers" + env: + OS_NAME: "musllinux_1_2-aarch64" + with: + image_name: "${{env.DEPS_IMAGE_NAME_PREFIX}}${{env.OS_NAME}}" + docker_context: "components/core" + docker_file: "components/core/tools/docker-images/clp-env-base-${{env.OS_NAME}}\ + /Dockerfile" + push_deps_image: >- + ${{github.event_name != 'pull_request' && github.ref == 'refs/heads/main'}} + token: "${{secrets.GITHUB_TOKEN}}" + + musllinux_1_2-x86_64-deps-image: + name: "musllinux_1_2-x86_64-deps-image" + if: "needs.filter-relevant-changes.outputs.musllinux_1_2_x86_64_image_changed == 'true'" needs: "filter-relevant-changes" runs-on: "ubuntu-24.04" steps: @@ -203,11 +243,11 @@ jobs: - uses: "./.github/actions/clp-core-build-containers" env: - OS_NAME: "musllinux_1_2" + OS_NAME: "musllinux_1_2-x86_64" with: image_name: "${{env.DEPS_IMAGE_NAME_PREFIX}}${{env.OS_NAME}}" docker_context: "components/core" - docker_file: "components/core/tools/docker-images/clp-env-base-${{env.OS_NAME}}-x86_64\ + docker_file: "components/core/tools/docker-images/clp-env-base-${{env.OS_NAME}}\ /Dockerfile" push_deps_image: >- ${{github.event_name != 'pull_request' && github.ref == 'refs/heads/main'}} @@ -278,18 +318,61 @@ jobs: --build-dir /mnt/repo/components/core/build --num-jobs $(getconf _NPROCESSORS_ONLN) - manylinux-2-28-binaries: + manylinux_2_28-aarch64-binaries: # Run if the ancestor jobs succeeded OR they were skipped and clp was changed. if: >- success() || (!cancelled() && !failure() && needs.filter-relevant-changes.outputs.clp_changed == 'true') needs: - - "manylinux-2-28-deps-image" + - "manylinux_2_28-aarch64-deps-image" - "filter-relevant-changes" strategy: matrix: use_shared_libs: [true, false] - name: "manylinux-2-28-${{matrix.use_shared_libs == true && 'dynamic' || 'static'}}-linked-bins" + name: >- + manylinux_2_28-aarch64-${{matrix.use_shared_libs == true && 'dynamic' || + 'static'}}-linked-bins + continue-on-error: true + runs-on: "ubuntu-24.04-arm" + steps: + - uses: "actions/checkout@11bd71901bbe5b1630ceea73d27597364c9af683" + with: + submodules: "recursive" + + - name: "Work around actions/runner-images/issues/6775" + run: "chown $(id -u):$(id -g) -R ." + shell: "bash" + + - uses: "./.github/actions/run-on-image" + env: + OS_NAME: "manylinux_2_28-aarch64" + with: + image_name: "${{env.DEPS_IMAGE_NAME_PREFIX}}${{env.OS_NAME}}" + use_published_image: >- + ${{needs.filter-relevant-changes.outputs.manylinux_2_28_aarch64_image_changed == 'false' + || (github.event_name != 'pull_request' && github.ref == 'refs/heads/main')}} + run_command: >- + CLP_CORE_MAX_PARALLELISM_PER_BUILD_TASK=$(getconf _NPROCESSORS_ONLN) task deps:core + && python3 /mnt/repo/components/core/tools/scripts/utils/build-and-run-unit-tests.py + ${{matrix.use_shared_libs == true && '--use-shared-libs' || ''}} + --source-dir /mnt/repo/components/core + --build-dir /mnt/repo/components/core/build + --num-jobs $(getconf _NPROCESSORS_ONLN) + + manylinux_2_28-x86_64-binaries: + # Run if the ancestor jobs succeeded OR they were skipped and clp was changed. + if: >- + success() + || (!cancelled() && !failure() && needs.filter-relevant-changes.outputs.clp_changed == 'true') + needs: + - "manylinux_2_28-x86_64-deps-image" + - "filter-relevant-changes" + strategy: + matrix: + use_shared_libs: [true, false] + name: >- + manylinux_2_28-x86_64-${{matrix.use_shared_libs == true && 'dynamic' || + 'static'}}-linked-bins continue-on-error: true runs-on: "ubuntu-24.04" steps: @@ -303,11 +386,52 @@ jobs: - uses: "./.github/actions/run-on-image" env: - OS_NAME: "manylinux_2_28" + OS_NAME: "manylinux_2_28-x86_64" + with: + image_name: "${{env.DEPS_IMAGE_NAME_PREFIX}}${{env.OS_NAME}}" + use_published_image: >- + ${{needs.filter-relevant-changes.outputs.manylinux_2_28_x86_64_image_changed == 'false' + || (github.event_name != 'pull_request' && github.ref == 'refs/heads/main')}} + run_command: >- + CLP_CORE_MAX_PARALLELISM_PER_BUILD_TASK=$(getconf _NPROCESSORS_ONLN) task deps:core + && python3 /mnt/repo/components/core/tools/scripts/utils/build-and-run-unit-tests.py + ${{matrix.use_shared_libs == true && '--use-shared-libs' || ''}} + --source-dir /mnt/repo/components/core + --build-dir /mnt/repo/components/core/build + --num-jobs $(getconf _NPROCESSORS_ONLN) + + musllinux_1_2-aarch64-binaries: + # Run if the ancestor jobs succeeded OR they were skipped and clp was changed. + if: >- + success() + || (!cancelled() && !failure() && needs.filter-relevant-changes.outputs.clp_changed == 'true') + needs: + - "musllinux_1_2-aarch64-deps-image" + - "filter-relevant-changes" + strategy: + matrix: + use_shared_libs: [true, false] + name: >- + musllinux_1_2-aarch64-${{matrix.use_shared_libs == true && 'dynamic' || + 'static'}}-linked-bins + continue-on-error: true + runs-on: "ubuntu-24.04-arm" + steps: + - uses: "actions/checkout@11bd71901bbe5b1630ceea73d27597364c9af683" + with: + submodules: "recursive" + + - name: "Work around actions/runner-images/issues/6775" + run: "chown $(id -u):$(id -g) -R ." + shell: "bash" + + - uses: "./.github/actions/run-on-image" + env: + OS_NAME: "musllinux_1_2-aarch64" with: image_name: "${{env.DEPS_IMAGE_NAME_PREFIX}}${{env.OS_NAME}}" use_published_image: >- - ${{needs.filter-relevant-changes.outputs.manylinux_2_28_image_changed == 'false' + ${{needs.filter-relevant-changes.outputs.musllinux_1_2_aarch64_image_changed == 'false' || (github.event_name != 'pull_request' && github.ref == 'refs/heads/main')}} run_command: >- CLP_CORE_MAX_PARALLELISM_PER_BUILD_TASK=$(getconf _NPROCESSORS_ONLN) task deps:core @@ -317,18 +441,20 @@ jobs: --build-dir /mnt/repo/components/core/build --num-jobs $(getconf _NPROCESSORS_ONLN) - musllinux-1-2-binaries: + musllinux_1_2-x86_64-binaries: # Run if the ancestor jobs succeeded OR they were skipped and clp was changed. if: >- success() || (!cancelled() && !failure() && needs.filter-relevant-changes.outputs.clp_changed == 'true') needs: - - "musllinux-1-2-deps-image" + - "musllinux_1_2-x86_64-deps-image" - "filter-relevant-changes" strategy: matrix: use_shared_libs: [true, false] - name: "musllinux-1-2-${{matrix.use_shared_libs == true && 'dynamic' || 'static'}}-linked-bins" + name: >- + musllinux_1_2-x86_64-${{matrix.use_shared_libs == true && 'dynamic' || + 'static'}}-linked-bins continue-on-error: true runs-on: "ubuntu-24.04" steps: @@ -342,11 +468,11 @@ jobs: - uses: "./.github/actions/run-on-image" env: - OS_NAME: "musllinux_1_2" + OS_NAME: "musllinux_1_2-x86_64" with: image_name: "${{env.DEPS_IMAGE_NAME_PREFIX}}${{env.OS_NAME}}" use_published_image: >- - ${{needs.filter-relevant-changes.outputs.musllinux_1_2_image_changed == 'false' + ${{needs.filter-relevant-changes.outputs.musllinux_1_2_x86_64_image_changed == 'false' || (github.event_name != 'pull_request' && github.ref == 'refs/heads/main')}} run_command: >- CLP_CORE_MAX_PARALLELISM_PER_BUILD_TASK=$(getconf _NPROCESSORS_ONLN) task deps:core From 22cfcccb30e2db70b634dd0759e1c3ce7ec3e961 Mon Sep 17 00:00:00 2001 From: Bingran Hu Date: Mon, 25 Aug 2025 04:50:56 -0400 Subject: [PATCH 11/69] fox --- .github/workflows/clp-core-build.yaml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/clp-core-build.yaml b/.github/workflows/clp-core-build.yaml index d3d18365d5..f73139a221 100644 --- a/.github/workflows/clp-core-build.yaml +++ b/.github/workflows/clp-core-build.yaml @@ -91,13 +91,13 @@ jobs: - ".github/workflows/clp-core-build.yaml" - "components/core/tools/scripts/lib_install/*.sh" - "components/core/tools/docker-images/clp-env-base-manylinux_2_28-x86_64/**" + - "components/core/tools/scripts/lib_install/manylinux_2_28/**" musllinux_1_2_aarch64_image: - ".github/actions/**" - ".github/workflows/clp-core-build.yaml" - "components/core/tools/scripts/lib_install/*.sh" - "components/core/tools/docker-images/clp-env-base-musllinux_1_2-aarch64/**" - "components/core/tools/scripts/lib_install/musllinux_1_2/**" - - "components/core/tools/scripts/lib_install/manylinux_2_28/**" musllinux_1_2_x86_64_image: - ".github/actions/**" - ".github/workflows/clp-core-build.yaml" From 8adc4e6fbe8813fa14270c1eee20957b1cb9f132 Mon Sep 17 00:00:00 2001 From: Bingran Hu Date: Mon, 25 Aug 2025 05:28:12 -0400 Subject: [PATCH 12/69] use matrix --- .github/workflows/clp-core-build.yaml | 87 +++++++++------------------ 1 file changed, 27 insertions(+), 60 deletions(-) diff --git a/.github/workflows/clp-core-build.yaml b/.github/workflows/clp-core-build.yaml index f73139a221..b575e2ace8 100644 --- a/.github/workflows/clp-core-build.yaml +++ b/.github/workflows/clp-core-build.yaml @@ -30,7 +30,8 @@ on: env: BINARIES_ARTIFACT_NAME_PREFIX: "clp-core-binaries-" - DEPS_IMAGE_NAME_PREFIX: "clp-core-dependencies-x86-" + DEPS_IMAGE_NAME_PREFIX_X86: "clp-core-dependencies-x86-" + DEPS_IMAGE_NAME_PREFIX_ARM: "clp-core-dependencies-arm-" concurrency: group: "${{github.workflow}}-${{github.ref}}" @@ -123,63 +124,29 @@ jobs: - "taskfiles/**" - "tools/scripts/deps-download/**" - centos-stream-9-deps-image: - name: "centos-stream-9-deps-image" - if: "needs.filter-relevant-changes.outputs.centos_stream_9_image_changed == 'true'" - needs: "filter-relevant-changes" - runs-on: "ubuntu-24.04" - steps: - - uses: "actions/checkout@11bd71901bbe5b1630ceea73d27597364c9af683" - with: - submodules: "recursive" - - - name: "Work around actions/runner-images/issues/6775" - run: "chown $(id -u):$(id -g) -R ." - shell: "bash" - - - uses: "./.github/actions/clp-core-build-containers" - env: - OS_NAME: "centos-stream-9" - with: - image_name: "${{env.DEPS_IMAGE_NAME_PREFIX}}${{env.OS_NAME}}" - docker_context: "components/core" - docker_file: "components/core/tools/docker-images/clp-env-base-${{env.OS_NAME}}\ - /Dockerfile" - push_deps_image: >- - ${{github.event_name != 'pull_request' && github.ref == 'refs/heads/main'}} - token: "${{secrets.GITHUB_TOKEN}}" - - manylinux_2_28-aarch64-deps-image: - name: "manylinux_2_28-aarch64-deps-image" - if: "needs.filter-relevant-changes.outputs.manylinux_2_28_aarch64_image_changed == 'true'" - needs: "filter-relevant-changes" - runs-on: "ubuntu-24.04-arm" - steps: - - uses: "actions/checkout@11bd71901bbe5b1630ceea73d27597364c9af683" - with: - submodules: "recursive" - - - name: "Work around actions/runner-images/issues/6775" - run: "chown $(id -u):$(id -g) -R ." - shell: "bash" - - - uses: "./.github/actions/clp-core-build-containers" - env: - OS_NAME: "manylinux_2_28-aarch64" - with: - image_name: "${{env.DEPS_IMAGE_NAME_PREFIX}}${{env.OS_NAME}}" - docker_context: "components/core" - docker_file: "components/core/tools/docker-images/clp-env-base-${{env.OS_NAME}}\ - /Dockerfile" - push_deps_image: >- - ${{github.event_name != 'pull_request' && github.ref == 'refs/heads/main'}} - token: "${{secrets.GITHUB_TOKEN}}" - - manylinux_2_28-x86_64-deps-image: - name: "manylinux_2_28-x86_64-deps-image" - if: "needs.filter-relevant-changes.outputs.manylinux_2_28_x86_64_image_changed == 'true'" + deps-image: + strategy: + fail-fast: false + matrix: + include: + - arch: "x86_64" + os_name: "centos-stream-9" + - arch: "aarch64" + os_name: "manylinux_2_28-aarch64" + - arch: "x86_64" + os_name: "manylinux_2_28-x86_64" + name: "${{ matrix.os_name }}-deps-image" + if: >- + ${{ + (matrix.os_name == 'centos-stream-9' && + needs.filter-relevant-changes.outputs.centos_stream_9_image_changed == 'true') || + (matrix.os_name == 'manylinux_2_28-aarch64' && + needs.filter-relevant-changes.outputs.manylinux_2_28_aarch64_image_changed == 'true') || + (matrix.os_name == 'manylinux_2_28-x86_64' && + needs.filter-relevant-changes.outputs.manylinux_2_28_x86_64_image_changed == 'true') + }} needs: "filter-relevant-changes" - runs-on: "ubuntu-24.04" + runs-on: "${{ matrix.arch == 'x86_64' && ubuntu-24.04 || ubuntu-24.04-arm }}" steps: - uses: "actions/checkout@11bd71901bbe5b1630ceea73d27597364c9af683" with: @@ -190,10 +157,10 @@ jobs: shell: "bash" - uses: "./.github/actions/clp-core-build-containers" - env: - OS_NAME: "manylinux_2_28-x86_64" with: - image_name: "${{env.DEPS_IMAGE_NAME_PREFIX}}${{env.OS_NAME}}" + image_name: >- + ${{matrix.arch == 'x86_64' && env.DEPS_IMAGE_NAME_PREFIX_X86 || + env.DEPS_IMAGE_NAME_PREFIX_ARM}}${{env.OS_NAME}} docker_context: "components/core" docker_file: "components/core/tools/docker-images/clp-env-base-${{env.OS_NAME}}\ /Dockerfile" From ff98538150da7acf592020877c02886a48423c4a Mon Sep 17 00:00:00 2001 From: Bingran Hu Date: Mon, 25 Aug 2025 05:51:48 -0400 Subject: [PATCH 13/69] Update vars --- .github/workflows/clp-core-build.yaml | 20 ++++++++++---------- 1 file changed, 10 insertions(+), 10 deletions(-) diff --git a/.github/workflows/clp-core-build.yaml b/.github/workflows/clp-core-build.yaml index b575e2ace8..8a3dfa192d 100644 --- a/.github/workflows/clp-core-build.yaml +++ b/.github/workflows/clp-core-build.yaml @@ -186,7 +186,7 @@ jobs: env: OS_NAME: "musllinux_1_2-aarch64" with: - image_name: "${{env.DEPS_IMAGE_NAME_PREFIX}}${{env.OS_NAME}}" + image_name: "${{env.DEPS_IMAGE_NAME_PREFIX_ARM}}${{env.OS_NAME}}" docker_context: "components/core" docker_file: "components/core/tools/docker-images/clp-env-base-${{env.OS_NAME}}\ /Dockerfile" @@ -212,7 +212,7 @@ jobs: env: OS_NAME: "musllinux_1_2-x86_64" with: - image_name: "${{env.DEPS_IMAGE_NAME_PREFIX}}${{env.OS_NAME}}" + image_name: "${{env.DEPS_IMAGE_NAME_PREFIX_X86}}${{env.OS_NAME}}" docker_context: "components/core" docker_file: "components/core/tools/docker-images/clp-env-base-${{env.OS_NAME}}\ /Dockerfile" @@ -238,7 +238,7 @@ jobs: env: OS_NAME: "ubuntu-jammy" with: - image_name: "${{env.DEPS_IMAGE_NAME_PREFIX}}${{env.OS_NAME}}" + image_name: "${{env.DEPS_IMAGE_NAME_PREFIX_X86}}${{env.OS_NAME}}" docker_context: "components/core" docker_file: "components/core/tools/docker-images/clp-env-base-${{env.OS_NAME}}\ /Dockerfile" @@ -273,7 +273,7 @@ jobs: env: OS_NAME: "centos-stream-9" with: - image_name: "${{env.DEPS_IMAGE_NAME_PREFIX}}${{env.OS_NAME}}" + image_name: "${{env.DEPS_IMAGE_NAME_PREFIX_X86}}${{env.OS_NAME}}" use_published_image: >- ${{needs.filter-relevant-changes.outputs.centos_stream_9_image_changed == 'false' || (github.event_name != 'pull_request' && github.ref == 'refs/heads/main')}} @@ -314,7 +314,7 @@ jobs: env: OS_NAME: "manylinux_2_28-aarch64" with: - image_name: "${{env.DEPS_IMAGE_NAME_PREFIX}}${{env.OS_NAME}}" + image_name: "${{env.DEPS_IMAGE_NAME_PREFIX_ARM}}${{env.OS_NAME}}" use_published_image: >- ${{needs.filter-relevant-changes.outputs.manylinux_2_28_aarch64_image_changed == 'false' || (github.event_name != 'pull_request' && github.ref == 'refs/heads/main')}} @@ -355,7 +355,7 @@ jobs: env: OS_NAME: "manylinux_2_28-x86_64" with: - image_name: "${{env.DEPS_IMAGE_NAME_PREFIX}}${{env.OS_NAME}}" + image_name: "${{env.DEPS_IMAGE_NAME_PREFIX_X86}}${{env.OS_NAME}}" use_published_image: >- ${{needs.filter-relevant-changes.outputs.manylinux_2_28_x86_64_image_changed == 'false' || (github.event_name != 'pull_request' && github.ref == 'refs/heads/main')}} @@ -396,7 +396,7 @@ jobs: env: OS_NAME: "musllinux_1_2-aarch64" with: - image_name: "${{env.DEPS_IMAGE_NAME_PREFIX}}${{env.OS_NAME}}" + image_name: "${{env.DEPS_IMAGE_NAME_PREFIX_ARM}}${{env.OS_NAME}}" use_published_image: >- ${{needs.filter-relevant-changes.outputs.musllinux_1_2_aarch64_image_changed == 'false' || (github.event_name != 'pull_request' && github.ref == 'refs/heads/main')}} @@ -437,7 +437,7 @@ jobs: env: OS_NAME: "musllinux_1_2-x86_64" with: - image_name: "${{env.DEPS_IMAGE_NAME_PREFIX}}${{env.OS_NAME}}" + image_name: "${{env.DEPS_IMAGE_NAME_PREFIX_X86}}${{env.OS_NAME}}" use_published_image: >- ${{needs.filter-relevant-changes.outputs.musllinux_1_2_x86_64_image_changed == 'false' || (github.event_name != 'pull_request' && github.ref == 'refs/heads/main')}} @@ -480,7 +480,7 @@ jobs: - uses: "./.github/actions/run-on-image" with: - image_name: "${{env.DEPS_IMAGE_NAME_PREFIX}}${{env.OS_NAME}}" + image_name: "${{env.DEPS_IMAGE_NAME_PREFIX_X86}}${{env.OS_NAME}}" use_published_image: >- ${{needs.filter-relevant-changes.outputs.ubuntu_jammy_image_changed == 'false' || (github.event_name != 'pull_request' && github.ref == 'refs/heads/main')}} @@ -614,7 +614,7 @@ jobs: env: OS_NAME: "ubuntu-jammy" with: - image_name: "${{env.DEPS_IMAGE_NAME_PREFIX}}${{env.OS_NAME}}" + image_name: "${{env.DEPS_IMAGE_NAME_PREFIX_X86}}${{env.OS_NAME}}" use_published_image: >- ${{needs.filter-relevant-changes.outputs.ubuntu_jammy_image_changed == 'false' || (github.event_name != 'pull_request' && github.ref == 'refs/heads/main')}} From bff70646832837fdf335648c458626d9cbae2a16 Mon Sep 17 00:00:00 2001 From: Bingran Hu Date: Mon, 25 Aug 2025 05:59:04 -0400 Subject: [PATCH 14/69] Fix grammar --- .github/workflows/clp-core-build.yaml | 15 ++++++++------- 1 file changed, 8 insertions(+), 7 deletions(-) diff --git a/.github/workflows/clp-core-build.yaml b/.github/workflows/clp-core-build.yaml index 8a3dfa192d..e0be79545c 100644 --- a/.github/workflows/clp-core-build.yaml +++ b/.github/workflows/clp-core-build.yaml @@ -146,7 +146,8 @@ jobs: needs.filter-relevant-changes.outputs.manylinux_2_28_x86_64_image_changed == 'true') }} needs: "filter-relevant-changes" - runs-on: "${{ matrix.arch == 'x86_64' && ubuntu-24.04 || ubuntu-24.04-arm }}" + runs-on: >- + ${{ matrix.arch == 'x86_64' && 'ubuntu-24.04' || 'ubuntu-24.04-arm' }} steps: - uses: "actions/checkout@11bd71901bbe5b1630ceea73d27597364c9af683" with: @@ -160,10 +161,10 @@ jobs: with: image_name: >- ${{matrix.arch == 'x86_64' && env.DEPS_IMAGE_NAME_PREFIX_X86 || - env.DEPS_IMAGE_NAME_PREFIX_ARM}}${{env.OS_NAME}} + env.DEPS_IMAGE_NAME_PREFIX_ARM}}${{matrix.os_name}} docker_context: "components/core" - docker_file: "components/core/tools/docker-images/clp-env-base-${{env.OS_NAME}}\ - /Dockerfile" + docker_file: >- + components/core/tools/docker-images/clp-env-base-${{matrix.os_name}}/Dockerfile push_deps_image: >- ${{github.event_name != 'pull_request' && github.ref == 'refs/heads/main'}} token: "${{secrets.GITHUB_TOKEN}}" @@ -252,7 +253,7 @@ jobs: success() || (!cancelled() && !failure() && needs.filter-relevant-changes.outputs.clp_changed == 'true') needs: - - "centos-stream-9-deps-image" + - "deps-image" - "filter-relevant-changes" strategy: matrix: @@ -291,7 +292,7 @@ jobs: success() || (!cancelled() && !failure() && needs.filter-relevant-changes.outputs.clp_changed == 'true') needs: - - "manylinux_2_28-aarch64-deps-image" + - "deps-image" - "filter-relevant-changes" strategy: matrix: @@ -332,7 +333,7 @@ jobs: success() || (!cancelled() && !failure() && needs.filter-relevant-changes.outputs.clp_changed == 'true') needs: - - "manylinux_2_28-x86_64-deps-image" + - "deps-image" - "filter-relevant-changes" strategy: matrix: From 6224601a30ed7433fef915961c4b5190288362a2 Mon Sep 17 00:00:00 2001 From: Bingran Hu Date: Mon, 25 Aug 2025 06:00:54 -0400 Subject: [PATCH 15/69] revert optimizations --- .github/workflows/clp-core-build.yaml | 118 ++++++++++++++++---------- 1 file changed, 75 insertions(+), 43 deletions(-) diff --git a/.github/workflows/clp-core-build.yaml b/.github/workflows/clp-core-build.yaml index e0be79545c..f73139a221 100644 --- a/.github/workflows/clp-core-build.yaml +++ b/.github/workflows/clp-core-build.yaml @@ -30,8 +30,7 @@ on: env: BINARIES_ARTIFACT_NAME_PREFIX: "clp-core-binaries-" - DEPS_IMAGE_NAME_PREFIX_X86: "clp-core-dependencies-x86-" - DEPS_IMAGE_NAME_PREFIX_ARM: "clp-core-dependencies-arm-" + DEPS_IMAGE_NAME_PREFIX: "clp-core-dependencies-x86-" concurrency: group: "${{github.workflow}}-${{github.ref}}" @@ -124,30 +123,11 @@ jobs: - "taskfiles/**" - "tools/scripts/deps-download/**" - deps-image: - strategy: - fail-fast: false - matrix: - include: - - arch: "x86_64" - os_name: "centos-stream-9" - - arch: "aarch64" - os_name: "manylinux_2_28-aarch64" - - arch: "x86_64" - os_name: "manylinux_2_28-x86_64" - name: "${{ matrix.os_name }}-deps-image" - if: >- - ${{ - (matrix.os_name == 'centos-stream-9' && - needs.filter-relevant-changes.outputs.centos_stream_9_image_changed == 'true') || - (matrix.os_name == 'manylinux_2_28-aarch64' && - needs.filter-relevant-changes.outputs.manylinux_2_28_aarch64_image_changed == 'true') || - (matrix.os_name == 'manylinux_2_28-x86_64' && - needs.filter-relevant-changes.outputs.manylinux_2_28_x86_64_image_changed == 'true') - }} + centos-stream-9-deps-image: + name: "centos-stream-9-deps-image" + if: "needs.filter-relevant-changes.outputs.centos_stream_9_image_changed == 'true'" needs: "filter-relevant-changes" - runs-on: >- - ${{ matrix.arch == 'x86_64' && 'ubuntu-24.04' || 'ubuntu-24.04-arm' }} + runs-on: "ubuntu-24.04" steps: - uses: "actions/checkout@11bd71901bbe5b1630ceea73d27597364c9af683" with: @@ -158,13 +138,65 @@ jobs: shell: "bash" - uses: "./.github/actions/clp-core-build-containers" + env: + OS_NAME: "centos-stream-9" with: - image_name: >- - ${{matrix.arch == 'x86_64' && env.DEPS_IMAGE_NAME_PREFIX_X86 || - env.DEPS_IMAGE_NAME_PREFIX_ARM}}${{matrix.os_name}} + image_name: "${{env.DEPS_IMAGE_NAME_PREFIX}}${{env.OS_NAME}}" docker_context: "components/core" - docker_file: >- - components/core/tools/docker-images/clp-env-base-${{matrix.os_name}}/Dockerfile + docker_file: "components/core/tools/docker-images/clp-env-base-${{env.OS_NAME}}\ + /Dockerfile" + push_deps_image: >- + ${{github.event_name != 'pull_request' && github.ref == 'refs/heads/main'}} + token: "${{secrets.GITHUB_TOKEN}}" + + manylinux_2_28-aarch64-deps-image: + name: "manylinux_2_28-aarch64-deps-image" + if: "needs.filter-relevant-changes.outputs.manylinux_2_28_aarch64_image_changed == 'true'" + needs: "filter-relevant-changes" + runs-on: "ubuntu-24.04-arm" + steps: + - uses: "actions/checkout@11bd71901bbe5b1630ceea73d27597364c9af683" + with: + submodules: "recursive" + + - name: "Work around actions/runner-images/issues/6775" + run: "chown $(id -u):$(id -g) -R ." + shell: "bash" + + - uses: "./.github/actions/clp-core-build-containers" + env: + OS_NAME: "manylinux_2_28-aarch64" + with: + image_name: "${{env.DEPS_IMAGE_NAME_PREFIX}}${{env.OS_NAME}}" + docker_context: "components/core" + docker_file: "components/core/tools/docker-images/clp-env-base-${{env.OS_NAME}}\ + /Dockerfile" + push_deps_image: >- + ${{github.event_name != 'pull_request' && github.ref == 'refs/heads/main'}} + token: "${{secrets.GITHUB_TOKEN}}" + + manylinux_2_28-x86_64-deps-image: + name: "manylinux_2_28-x86_64-deps-image" + if: "needs.filter-relevant-changes.outputs.manylinux_2_28_x86_64_image_changed == 'true'" + needs: "filter-relevant-changes" + runs-on: "ubuntu-24.04" + steps: + - uses: "actions/checkout@11bd71901bbe5b1630ceea73d27597364c9af683" + with: + submodules: "recursive" + + - name: "Work around actions/runner-images/issues/6775" + run: "chown $(id -u):$(id -g) -R ." + shell: "bash" + + - uses: "./.github/actions/clp-core-build-containers" + env: + OS_NAME: "manylinux_2_28-x86_64" + with: + image_name: "${{env.DEPS_IMAGE_NAME_PREFIX}}${{env.OS_NAME}}" + docker_context: "components/core" + docker_file: "components/core/tools/docker-images/clp-env-base-${{env.OS_NAME}}\ + /Dockerfile" push_deps_image: >- ${{github.event_name != 'pull_request' && github.ref == 'refs/heads/main'}} token: "${{secrets.GITHUB_TOKEN}}" @@ -187,7 +219,7 @@ jobs: env: OS_NAME: "musllinux_1_2-aarch64" with: - image_name: "${{env.DEPS_IMAGE_NAME_PREFIX_ARM}}${{env.OS_NAME}}" + image_name: "${{env.DEPS_IMAGE_NAME_PREFIX}}${{env.OS_NAME}}" docker_context: "components/core" docker_file: "components/core/tools/docker-images/clp-env-base-${{env.OS_NAME}}\ /Dockerfile" @@ -213,7 +245,7 @@ jobs: env: OS_NAME: "musllinux_1_2-x86_64" with: - image_name: "${{env.DEPS_IMAGE_NAME_PREFIX_X86}}${{env.OS_NAME}}" + image_name: "${{env.DEPS_IMAGE_NAME_PREFIX}}${{env.OS_NAME}}" docker_context: "components/core" docker_file: "components/core/tools/docker-images/clp-env-base-${{env.OS_NAME}}\ /Dockerfile" @@ -239,7 +271,7 @@ jobs: env: OS_NAME: "ubuntu-jammy" with: - image_name: "${{env.DEPS_IMAGE_NAME_PREFIX_X86}}${{env.OS_NAME}}" + image_name: "${{env.DEPS_IMAGE_NAME_PREFIX}}${{env.OS_NAME}}" docker_context: "components/core" docker_file: "components/core/tools/docker-images/clp-env-base-${{env.OS_NAME}}\ /Dockerfile" @@ -253,7 +285,7 @@ jobs: success() || (!cancelled() && !failure() && needs.filter-relevant-changes.outputs.clp_changed == 'true') needs: - - "deps-image" + - "centos-stream-9-deps-image" - "filter-relevant-changes" strategy: matrix: @@ -274,7 +306,7 @@ jobs: env: OS_NAME: "centos-stream-9" with: - image_name: "${{env.DEPS_IMAGE_NAME_PREFIX_X86}}${{env.OS_NAME}}" + image_name: "${{env.DEPS_IMAGE_NAME_PREFIX}}${{env.OS_NAME}}" use_published_image: >- ${{needs.filter-relevant-changes.outputs.centos_stream_9_image_changed == 'false' || (github.event_name != 'pull_request' && github.ref == 'refs/heads/main')}} @@ -292,7 +324,7 @@ jobs: success() || (!cancelled() && !failure() && needs.filter-relevant-changes.outputs.clp_changed == 'true') needs: - - "deps-image" + - "manylinux_2_28-aarch64-deps-image" - "filter-relevant-changes" strategy: matrix: @@ -315,7 +347,7 @@ jobs: env: OS_NAME: "manylinux_2_28-aarch64" with: - image_name: "${{env.DEPS_IMAGE_NAME_PREFIX_ARM}}${{env.OS_NAME}}" + image_name: "${{env.DEPS_IMAGE_NAME_PREFIX}}${{env.OS_NAME}}" use_published_image: >- ${{needs.filter-relevant-changes.outputs.manylinux_2_28_aarch64_image_changed == 'false' || (github.event_name != 'pull_request' && github.ref == 'refs/heads/main')}} @@ -333,7 +365,7 @@ jobs: success() || (!cancelled() && !failure() && needs.filter-relevant-changes.outputs.clp_changed == 'true') needs: - - "deps-image" + - "manylinux_2_28-x86_64-deps-image" - "filter-relevant-changes" strategy: matrix: @@ -356,7 +388,7 @@ jobs: env: OS_NAME: "manylinux_2_28-x86_64" with: - image_name: "${{env.DEPS_IMAGE_NAME_PREFIX_X86}}${{env.OS_NAME}}" + image_name: "${{env.DEPS_IMAGE_NAME_PREFIX}}${{env.OS_NAME}}" use_published_image: >- ${{needs.filter-relevant-changes.outputs.manylinux_2_28_x86_64_image_changed == 'false' || (github.event_name != 'pull_request' && github.ref == 'refs/heads/main')}} @@ -397,7 +429,7 @@ jobs: env: OS_NAME: "musllinux_1_2-aarch64" with: - image_name: "${{env.DEPS_IMAGE_NAME_PREFIX_ARM}}${{env.OS_NAME}}" + image_name: "${{env.DEPS_IMAGE_NAME_PREFIX}}${{env.OS_NAME}}" use_published_image: >- ${{needs.filter-relevant-changes.outputs.musllinux_1_2_aarch64_image_changed == 'false' || (github.event_name != 'pull_request' && github.ref == 'refs/heads/main')}} @@ -438,7 +470,7 @@ jobs: env: OS_NAME: "musllinux_1_2-x86_64" with: - image_name: "${{env.DEPS_IMAGE_NAME_PREFIX_X86}}${{env.OS_NAME}}" + image_name: "${{env.DEPS_IMAGE_NAME_PREFIX}}${{env.OS_NAME}}" use_published_image: >- ${{needs.filter-relevant-changes.outputs.musllinux_1_2_x86_64_image_changed == 'false' || (github.event_name != 'pull_request' && github.ref == 'refs/heads/main')}} @@ -481,7 +513,7 @@ jobs: - uses: "./.github/actions/run-on-image" with: - image_name: "${{env.DEPS_IMAGE_NAME_PREFIX_X86}}${{env.OS_NAME}}" + image_name: "${{env.DEPS_IMAGE_NAME_PREFIX}}${{env.OS_NAME}}" use_published_image: >- ${{needs.filter-relevant-changes.outputs.ubuntu_jammy_image_changed == 'false' || (github.event_name != 'pull_request' && github.ref == 'refs/heads/main')}} @@ -615,7 +647,7 @@ jobs: env: OS_NAME: "ubuntu-jammy" with: - image_name: "${{env.DEPS_IMAGE_NAME_PREFIX_X86}}${{env.OS_NAME}}" + image_name: "${{env.DEPS_IMAGE_NAME_PREFIX}}${{env.OS_NAME}}" use_published_image: >- ${{needs.filter-relevant-changes.outputs.ubuntu_jammy_image_changed == 'false' || (github.event_name != 'pull_request' && github.ref == 'refs/heads/main')}} From 828e63467e9399e697758657e4dd2685130a8065 Mon Sep 17 00:00:00 2001 From: Bingran Hu Date: Mon, 25 Aug 2025 06:04:23 -0400 Subject: [PATCH 16/69] Fix image prefixes with arch type --- .github/workflows/clp-core-build.yaml | 29 ++++++++++++++------------- 1 file changed, 15 insertions(+), 14 deletions(-) diff --git a/.github/workflows/clp-core-build.yaml b/.github/workflows/clp-core-build.yaml index f73139a221..655ed6ea58 100644 --- a/.github/workflows/clp-core-build.yaml +++ b/.github/workflows/clp-core-build.yaml @@ -30,7 +30,8 @@ on: env: BINARIES_ARTIFACT_NAME_PREFIX: "clp-core-binaries-" - DEPS_IMAGE_NAME_PREFIX: "clp-core-dependencies-x86-" + DEPS_IMAGE_NAME_PREFIX_X86: "clp-core-dependencies-x86-" + DEPS_IMAGE_NAME_PREFIX_ARM: "clp-core-dependencies-arm-" concurrency: group: "${{github.workflow}}-${{github.ref}}" @@ -141,7 +142,7 @@ jobs: env: OS_NAME: "centos-stream-9" with: - image_name: "${{env.DEPS_IMAGE_NAME_PREFIX}}${{env.OS_NAME}}" + image_name: "${{env.DEPS_IMAGE_NAME_PREFIX_X86}}${{env.OS_NAME}}" docker_context: "components/core" docker_file: "components/core/tools/docker-images/clp-env-base-${{env.OS_NAME}}\ /Dockerfile" @@ -167,7 +168,7 @@ jobs: env: OS_NAME: "manylinux_2_28-aarch64" with: - image_name: "${{env.DEPS_IMAGE_NAME_PREFIX}}${{env.OS_NAME}}" + image_name: "${{env.DEPS_IMAGE_NAME_PREFIX_ARM}}${{env.OS_NAME}}" docker_context: "components/core" docker_file: "components/core/tools/docker-images/clp-env-base-${{env.OS_NAME}}\ /Dockerfile" @@ -193,7 +194,7 @@ jobs: env: OS_NAME: "manylinux_2_28-x86_64" with: - image_name: "${{env.DEPS_IMAGE_NAME_PREFIX}}${{env.OS_NAME}}" + image_name: "${{env.DEPS_IMAGE_NAME_PREFIX_X86}}${{env.OS_NAME}}" docker_context: "components/core" docker_file: "components/core/tools/docker-images/clp-env-base-${{env.OS_NAME}}\ /Dockerfile" @@ -219,7 +220,7 @@ jobs: env: OS_NAME: "musllinux_1_2-aarch64" with: - image_name: "${{env.DEPS_IMAGE_NAME_PREFIX}}${{env.OS_NAME}}" + image_name: "${{env.DEPS_IMAGE_NAME_PREFIX_ARM}}${{env.OS_NAME}}" docker_context: "components/core" docker_file: "components/core/tools/docker-images/clp-env-base-${{env.OS_NAME}}\ /Dockerfile" @@ -245,7 +246,7 @@ jobs: env: OS_NAME: "musllinux_1_2-x86_64" with: - image_name: "${{env.DEPS_IMAGE_NAME_PREFIX}}${{env.OS_NAME}}" + image_name: "${{env.DEPS_IMAGE_NAME_PREFIX_X86}}${{env.OS_NAME}}" docker_context: "components/core" docker_file: "components/core/tools/docker-images/clp-env-base-${{env.OS_NAME}}\ /Dockerfile" @@ -271,7 +272,7 @@ jobs: env: OS_NAME: "ubuntu-jammy" with: - image_name: "${{env.DEPS_IMAGE_NAME_PREFIX}}${{env.OS_NAME}}" + image_name: "${{env.DEPS_IMAGE_NAME_PREFIX_X86}}${{env.OS_NAME}}" docker_context: "components/core" docker_file: "components/core/tools/docker-images/clp-env-base-${{env.OS_NAME}}\ /Dockerfile" @@ -306,7 +307,7 @@ jobs: env: OS_NAME: "centos-stream-9" with: - image_name: "${{env.DEPS_IMAGE_NAME_PREFIX}}${{env.OS_NAME}}" + image_name: "${{env.DEPS_IMAGE_NAME_PREFIX_X86}}${{env.OS_NAME}}" use_published_image: >- ${{needs.filter-relevant-changes.outputs.centos_stream_9_image_changed == 'false' || (github.event_name != 'pull_request' && github.ref == 'refs/heads/main')}} @@ -347,7 +348,7 @@ jobs: env: OS_NAME: "manylinux_2_28-aarch64" with: - image_name: "${{env.DEPS_IMAGE_NAME_PREFIX}}${{env.OS_NAME}}" + image_name: "${{env.DEPS_IMAGE_NAME_PREFIX_ARM}}${{env.OS_NAME}}" use_published_image: >- ${{needs.filter-relevant-changes.outputs.manylinux_2_28_aarch64_image_changed == 'false' || (github.event_name != 'pull_request' && github.ref == 'refs/heads/main')}} @@ -388,7 +389,7 @@ jobs: env: OS_NAME: "manylinux_2_28-x86_64" with: - image_name: "${{env.DEPS_IMAGE_NAME_PREFIX}}${{env.OS_NAME}}" + image_name: "${{env.DEPS_IMAGE_NAME_PREFIX_X86}}${{env.OS_NAME}}" use_published_image: >- ${{needs.filter-relevant-changes.outputs.manylinux_2_28_x86_64_image_changed == 'false' || (github.event_name != 'pull_request' && github.ref == 'refs/heads/main')}} @@ -429,7 +430,7 @@ jobs: env: OS_NAME: "musllinux_1_2-aarch64" with: - image_name: "${{env.DEPS_IMAGE_NAME_PREFIX}}${{env.OS_NAME}}" + image_name: "${{env.DEPS_IMAGE_NAME_PREFIX_ARM}}${{env.OS_NAME}}" use_published_image: >- ${{needs.filter-relevant-changes.outputs.musllinux_1_2_aarch64_image_changed == 'false' || (github.event_name != 'pull_request' && github.ref == 'refs/heads/main')}} @@ -470,7 +471,7 @@ jobs: env: OS_NAME: "musllinux_1_2-x86_64" with: - image_name: "${{env.DEPS_IMAGE_NAME_PREFIX}}${{env.OS_NAME}}" + image_name: "${{env.DEPS_IMAGE_NAME_PREFIX_X86}}${{env.OS_NAME}}" use_published_image: >- ${{needs.filter-relevant-changes.outputs.musllinux_1_2_x86_64_image_changed == 'false' || (github.event_name != 'pull_request' && github.ref == 'refs/heads/main')}} @@ -513,7 +514,7 @@ jobs: - uses: "./.github/actions/run-on-image" with: - image_name: "${{env.DEPS_IMAGE_NAME_PREFIX}}${{env.OS_NAME}}" + image_name: "${{env.DEPS_IMAGE_NAME_PREFIX_X86}}${{env.OS_NAME}}" use_published_image: >- ${{needs.filter-relevant-changes.outputs.ubuntu_jammy_image_changed == 'false' || (github.event_name != 'pull_request' && github.ref == 'refs/heads/main')}} @@ -647,7 +648,7 @@ jobs: env: OS_NAME: "ubuntu-jammy" with: - image_name: "${{env.DEPS_IMAGE_NAME_PREFIX}}${{env.OS_NAME}}" + image_name: "${{env.DEPS_IMAGE_NAME_PREFIX_X86}}${{env.OS_NAME}}" use_published_image: >- ${{needs.filter-relevant-changes.outputs.ubuntu_jammy_image_changed == 'false' || (github.event_name != 'pull_request' && github.ref == 'refs/heads/main')}} From 386e5dcf472a538dccfa4c8b997f51cccd337c8d Mon Sep 17 00:00:00 2001 From: Bingran Hu Date: Mon, 25 Aug 2025 06:04:57 -0400 Subject: [PATCH 17/69] Alphabeticize --- .github/workflows/clp-core-build.yaml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/clp-core-build.yaml b/.github/workflows/clp-core-build.yaml index 655ed6ea58..18f3cfc20a 100644 --- a/.github/workflows/clp-core-build.yaml +++ b/.github/workflows/clp-core-build.yaml @@ -30,8 +30,8 @@ on: env: BINARIES_ARTIFACT_NAME_PREFIX: "clp-core-binaries-" - DEPS_IMAGE_NAME_PREFIX_X86: "clp-core-dependencies-x86-" DEPS_IMAGE_NAME_PREFIX_ARM: "clp-core-dependencies-arm-" + DEPS_IMAGE_NAME_PREFIX_X86: "clp-core-dependencies-x86-" concurrency: group: "${{github.workflow}}-${{github.ref}}" From 6f5579b5a26be9fdd9fa9be3a53606b7838c9c1b Mon Sep 17 00:00:00 2001 From: Bingran Hu Date: Mon, 25 Aug 2025 06:15:29 -0400 Subject: [PATCH 18/69] Remove doc updates since manylinux and musllinux are not common OS platforms that people have access to --- docs/src/dev-guide/components-core/index.md | 21 +++++++--------- .../manylinux-2-28-deps-install.md | 24 ------------------- .../musllinux-1-2-deps-install.md | 24 ------------------- 3 files changed, 8 insertions(+), 61 deletions(-) delete mode 100644 docs/src/dev-guide/components-core/manylinux-2-28-deps-install.md delete mode 100644 docs/src/dev-guide/components-core/musllinux-1-2-deps-install.md diff --git a/docs/src/dev-guide/components-core/index.md b/docs/src/dev-guide/components-core/index.md index 048a46ec83..272fcc0269 100644 --- a/docs/src/dev-guide/components-core/index.md +++ b/docs/src/dev-guide/components-core/index.md @@ -38,23 +38,22 @@ The task will download, build, and install (within the build directory) the foll | [abseil-cpp](https://github.com/abseil/abseil-cpp) | 20250512.0 | | [ANTLR](https://www.antlr.org) | v4.13.2 | | [Boost](https://github.com/boostorg/boost) | v1.87.0 | -| [Catch2](https://github.com/catchorg/Catch2) | v3.8.0 | -| [date](https://github.com/HowardHinnant/date) | v3.0.1 | +| [Catch2](https://github.com/catchorg/Catch2.git) | v3.8.0 | +| [date](https://github.com/HowardHinnant/date.git) | v3.0.1 | | [fmt](https://github.com/fmtlib/fmt) | v10.2.1 | +| [json](https://github.com/nlohmann/json.git) | v3.11.3 | +| [log-surgeon](https://github.com/y-scope/log-surgeon) | f801a3f | | [liblzma](https://github.com/tukaani-project/xz) | v5.8.1 | -| [log-surgeon](https://github.com/y-scope/log-surgeon) | a82ad13 | | [lz4](https://github.com/lz4/lz4) | v1.10.0 | -| [microsoft.gsl](https://github.com/microsoft/GSL) | v4.0.0 | | [mongo-cxx-driver](https://github.com/mongodb/mongo-cxx-driver) | r3.10.2 | | [msgpack-cxx](https://github.com/msgpack/msgpack-c/tree/cpp_master) | v7.0.0 | -| [nlohmann_json](https://github.com/nlohmann/json) | v3.11.3 | | [simdjson](https://github.com/simdjson/simdjson) | v3.13.0 | | [spdlog](https://github.com/gabime/spdlog) | v1.14.1 | | [SQLite3](https://www.sqlite.org/download.html) | v3.36.0 | -| [utfcpp](https://github.com/nemtrif/utfcpp) | v4.0.6 | -| [yaml-cpp](https://github.com/jbeder/yaml-cpp) | v0.7.0 | -| [yscope-log-viewer](https://github.com/y-scope/yscope-log-viewer) | 3abe4cc | -| [ystdlib-cpp](https://github.com/y-scope/ystdlib-cpp) | d80cf86 | +| [uftcpp](https://github.com/nemtrif/utfcpp.git) | v4.0.6 | +| [yaml-cpp](https://github.com/jbeder/yaml-cpp.git) | v0.7.0 | +| [yscope-log-viewer](https://github.com/y-scope/yscope-log-viewer.git) | 969ff35 | +| [ystdlib-cpp](https://github.com/y-scope/ystdlib-cpp.git) | d80cf86 | | [zlib](https://github.com/madler/zlib) | v1.3.1 | | [zstd](https://github.com/facebook/zstd) | v1.5.7 | @@ -71,8 +70,6 @@ A handful of packages and libraries are required to build CLP. There are two opt See the relevant README for your OS: * [CentOS Stream 9](centos-stream-9-deps-install) -* [Manylinux 2.28](manylinux-2-28-deps-install) -* [Musllinux 1.2](musllinux-1-2-deps-install) * [macOS](macos-deps-install) * [Ubuntu 22.04](ubuntu-jammy-deps-install) @@ -116,8 +113,6 @@ the relevant paths on your machine. :hidden: centos-stream-9-deps-install -manylinux-2-28-deps-install -musllinux-1-2-deps-install macos-deps-install ubuntu-jammy-deps-install regex-utils diff --git a/docs/src/dev-guide/components-core/manylinux-2-28-deps-install.md b/docs/src/dev-guide/components-core/manylinux-2-28-deps-install.md deleted file mode 100644 index dc7dd99d7d..0000000000 --- a/docs/src/dev-guide/components-core/manylinux-2-28-deps-install.md +++ /dev/null @@ -1,24 +0,0 @@ -# Manylinux 2.28 - -To install the dependencies required to build clp-core, follow the steps below. -These same steps are used by our Docker containers. - -## Installing dependencies - -:::{caution} -Before you run any commands below, you should review the scripts to ensure they will not install -any dependencies or apply any configurations that you don't expect. -::: - -To install all dependencies, run the following with elevated privileges: - -:::{note} -The packages built from source ([install-packages-from-source.sh][src-install-script]) are installed -without using a packager. So if you ever need to uninstall them, you will need to do so manually. -::: - -```shell -components/core/tools/scripts/lib_install/manylinux_2_28/install-all.sh -``` - -[src-install-script]: https://github.com/y-scope/clp/blob/main/components/core/tools/scripts/lib_install/manylinux_2_28/install-packages-from-source.sh diff --git a/docs/src/dev-guide/components-core/musllinux-1-2-deps-install.md b/docs/src/dev-guide/components-core/musllinux-1-2-deps-install.md deleted file mode 100644 index fda0370ad6..0000000000 --- a/docs/src/dev-guide/components-core/musllinux-1-2-deps-install.md +++ /dev/null @@ -1,24 +0,0 @@ -# Musllinux 1.2 - -To install the dependencies required to build clp-core, follow the steps below. -These same steps are used by our Docker containers. - -## Installing dependencies - -:::{caution} -Before you run any commands below, you should review the scripts to ensure they will not install -any dependencies or apply any configurations that you don't expect. -::: - -To install all dependencies, run the following with elevated privileges: - -:::{note} -The packages built from source ([install-packages-from-source.sh][src-install-script]) are installed -without using a packager. So if you ever need to uninstall them, you will need to do so manually. -::: - -```shell -components/core/tools/scripts/lib_install/musllinux_1_2/install-all.sh -``` - -[src-install-script]: https://github.com/y-scope/clp/blob/main/components/core/tools/scripts/lib_install/musllinux_1_2/install-packages-from-source.sh From bc976fe68e6ef7e9d716be027fe129dc858e4791 Mon Sep 17 00:00:00 2001 From: Bingran Hu Date: Tue, 26 Aug 2025 05:16:45 -0400 Subject: [PATCH 19/69] Add docs and remove aarch64 builds --- .github/workflows/clp-core-build.yaml | 181 ++------------------- docs/src/dev-guide/tooling-containers.md | 16 ++ docs/src/dev-guide/tooling-gh-workflows.md | 14 ++ 3 files changed, 46 insertions(+), 165 deletions(-) diff --git a/.github/workflows/clp-core-build.yaml b/.github/workflows/clp-core-build.yaml index 18f3cfc20a..bc3bc2f7d2 100644 --- a/.github/workflows/clp-core-build.yaml +++ b/.github/workflows/clp-core-build.yaml @@ -30,8 +30,7 @@ on: env: BINARIES_ARTIFACT_NAME_PREFIX: "clp-core-binaries-" - DEPS_IMAGE_NAME_PREFIX_ARM: "clp-core-dependencies-arm-" - DEPS_IMAGE_NAME_PREFIX_X86: "clp-core-dependencies-x86-" + DEPS_IMAGE_NAME_PREFIX: "clp-core-dependencies-x86-" concurrency: group: "${{github.workflow}}-${{github.ref}}" @@ -46,9 +45,7 @@ jobs: runs-on: "ubuntu-24.04" outputs: centos_stream_9_image_changed: "${{steps.filter.outputs.centos_stream_9_image}}" - manylinux_2_28_aarch64_image_changed: "${{steps.filter.outputs.manylinux_2_28_aarch64_image}}" manylinux_2_28_x86_64_image_changed: "${{steps.filter.outputs.manylinux_2_28_x86_64_image}}" - musllinux_1_2_aarch64_image_changed: "${{steps.filter.outputs.musllinux_1_2_aarch64_image}}" musllinux_1_2_x86_64_image_changed: "${{steps.filter.outputs.musllinux_1_2_x86_64_image}}" ubuntu_jammy_image_changed: "${{steps.filter.outputs.ubuntu_jammy_image}}" clp_changed: "${{steps.filter.outputs.clp}}" @@ -81,24 +78,12 @@ jobs: - "components/core/tools/scripts/lib_install/*.sh" - "components/core/tools/docker-images/clp-env-base-centos-stream-9/**" - "components/core/tools/scripts/lib_install/centos-stream-9/**" - manylinux_2_28_aarch64_image: - - ".github/actions/**" - - ".github/workflows/clp-core-build.yaml" - - "components/core/tools/scripts/lib_install/*.sh" - - "components/core/tools/docker-images/clp-env-base-manylinux_2_28-aarch64/**" - - "components/core/tools/scripts/lib_install/manylinux_2_28/**" manylinux_2_28_x86_64_image: - ".github/actions/**" - ".github/workflows/clp-core-build.yaml" - "components/core/tools/scripts/lib_install/*.sh" - "components/core/tools/docker-images/clp-env-base-manylinux_2_28-x86_64/**" - "components/core/tools/scripts/lib_install/manylinux_2_28/**" - musllinux_1_2_aarch64_image: - - ".github/actions/**" - - ".github/workflows/clp-core-build.yaml" - - "components/core/tools/scripts/lib_install/*.sh" - - "components/core/tools/docker-images/clp-env-base-musllinux_1_2-aarch64/**" - - "components/core/tools/scripts/lib_install/musllinux_1_2/**" musllinux_1_2_x86_64_image: - ".github/actions/**" - ".github/workflows/clp-core-build.yaml" @@ -142,33 +127,7 @@ jobs: env: OS_NAME: "centos-stream-9" with: - image_name: "${{env.DEPS_IMAGE_NAME_PREFIX_X86}}${{env.OS_NAME}}" - docker_context: "components/core" - docker_file: "components/core/tools/docker-images/clp-env-base-${{env.OS_NAME}}\ - /Dockerfile" - push_deps_image: >- - ${{github.event_name != 'pull_request' && github.ref == 'refs/heads/main'}} - token: "${{secrets.GITHUB_TOKEN}}" - - manylinux_2_28-aarch64-deps-image: - name: "manylinux_2_28-aarch64-deps-image" - if: "needs.filter-relevant-changes.outputs.manylinux_2_28_aarch64_image_changed == 'true'" - needs: "filter-relevant-changes" - runs-on: "ubuntu-24.04-arm" - steps: - - uses: "actions/checkout@11bd71901bbe5b1630ceea73d27597364c9af683" - with: - submodules: "recursive" - - - name: "Work around actions/runner-images/issues/6775" - run: "chown $(id -u):$(id -g) -R ." - shell: "bash" - - - uses: "./.github/actions/clp-core-build-containers" - env: - OS_NAME: "manylinux_2_28-aarch64" - with: - image_name: "${{env.DEPS_IMAGE_NAME_PREFIX_ARM}}${{env.OS_NAME}}" + image_name: "${{env.DEPS_IMAGE_NAME_PREFIX}}${{env.OS_NAME}}" docker_context: "components/core" docker_file: "components/core/tools/docker-images/clp-env-base-${{env.OS_NAME}}\ /Dockerfile" @@ -192,37 +151,11 @@ jobs: - uses: "./.github/actions/clp-core-build-containers" env: - OS_NAME: "manylinux_2_28-x86_64" + OS_NAME: "manylinux_2_28" with: - image_name: "${{env.DEPS_IMAGE_NAME_PREFIX_X86}}${{env.OS_NAME}}" + image_name: "${{env.DEPS_IMAGE_NAME_PREFIX}}${{env.OS_NAME}}" docker_context: "components/core" - docker_file: "components/core/tools/docker-images/clp-env-base-${{env.OS_NAME}}\ - /Dockerfile" - push_deps_image: >- - ${{github.event_name != 'pull_request' && github.ref == 'refs/heads/main'}} - token: "${{secrets.GITHUB_TOKEN}}" - - musllinux_1_2-aarch64-deps-image: - name: "musllinux_1_2-aarch64-deps-image" - if: "needs.filter-relevant-changes.outputs.musllinux_1_2_aarch64_image_changed == 'true'" - needs: "filter-relevant-changes" - runs-on: "ubuntu-24.04-arm" - steps: - - uses: "actions/checkout@11bd71901bbe5b1630ceea73d27597364c9af683" - with: - submodules: "recursive" - - - name: "Work around actions/runner-images/issues/6775" - run: "chown $(id -u):$(id -g) -R ." - shell: "bash" - - - uses: "./.github/actions/clp-core-build-containers" - env: - OS_NAME: "musllinux_1_2-aarch64" - with: - image_name: "${{env.DEPS_IMAGE_NAME_PREFIX_ARM}}${{env.OS_NAME}}" - docker_context: "components/core" - docker_file: "components/core/tools/docker-images/clp-env-base-${{env.OS_NAME}}\ + docker_file: "components/core/tools/docker-images/clp-env-base-${{env.OS_NAME}}-x86_64\ /Dockerfile" push_deps_image: >- ${{github.event_name != 'pull_request' && github.ref == 'refs/heads/main'}} @@ -244,11 +177,11 @@ jobs: - uses: "./.github/actions/clp-core-build-containers" env: - OS_NAME: "musllinux_1_2-x86_64" + OS_NAME: "musllinux_1_2" with: - image_name: "${{env.DEPS_IMAGE_NAME_PREFIX_X86}}${{env.OS_NAME}}" + image_name: "${{env.DEPS_IMAGE_NAME_PREFIX}}${{env.OS_NAME}}" docker_context: "components/core" - docker_file: "components/core/tools/docker-images/clp-env-base-${{env.OS_NAME}}\ + docker_file: "components/core/tools/docker-images/clp-env-base-${{env.OS_NAME}}-x86_64\ /Dockerfile" push_deps_image: >- ${{github.event_name != 'pull_request' && github.ref == 'refs/heads/main'}} @@ -272,7 +205,7 @@ jobs: env: OS_NAME: "ubuntu-jammy" with: - image_name: "${{env.DEPS_IMAGE_NAME_PREFIX_X86}}${{env.OS_NAME}}" + image_name: "${{env.DEPS_IMAGE_NAME_PREFIX}}${{env.OS_NAME}}" docker_context: "components/core" docker_file: "components/core/tools/docker-images/clp-env-base-${{env.OS_NAME}}\ /Dockerfile" @@ -307,7 +240,7 @@ jobs: env: OS_NAME: "centos-stream-9" with: - image_name: "${{env.DEPS_IMAGE_NAME_PREFIX_X86}}${{env.OS_NAME}}" + image_name: "${{env.DEPS_IMAGE_NAME_PREFIX}}${{env.OS_NAME}}" use_published_image: >- ${{needs.filter-relevant-changes.outputs.centos_stream_9_image_changed == 'false' || (github.event_name != 'pull_request' && github.ref == 'refs/heads/main')}} @@ -319,47 +252,6 @@ jobs: --build-dir /mnt/repo/components/core/build --num-jobs $(getconf _NPROCESSORS_ONLN) - manylinux_2_28-aarch64-binaries: - # Run if the ancestor jobs succeeded OR they were skipped and clp was changed. - if: >- - success() - || (!cancelled() && !failure() && needs.filter-relevant-changes.outputs.clp_changed == 'true') - needs: - - "manylinux_2_28-aarch64-deps-image" - - "filter-relevant-changes" - strategy: - matrix: - use_shared_libs: [true, false] - name: >- - manylinux_2_28-aarch64-${{matrix.use_shared_libs == true && 'dynamic' || - 'static'}}-linked-bins - continue-on-error: true - runs-on: "ubuntu-24.04-arm" - steps: - - uses: "actions/checkout@11bd71901bbe5b1630ceea73d27597364c9af683" - with: - submodules: "recursive" - - - name: "Work around actions/runner-images/issues/6775" - run: "chown $(id -u):$(id -g) -R ." - shell: "bash" - - - uses: "./.github/actions/run-on-image" - env: - OS_NAME: "manylinux_2_28-aarch64" - with: - image_name: "${{env.DEPS_IMAGE_NAME_PREFIX_ARM}}${{env.OS_NAME}}" - use_published_image: >- - ${{needs.filter-relevant-changes.outputs.manylinux_2_28_aarch64_image_changed == 'false' - || (github.event_name != 'pull_request' && github.ref == 'refs/heads/main')}} - run_command: >- - CLP_CORE_MAX_PARALLELISM_PER_BUILD_TASK=$(getconf _NPROCESSORS_ONLN) task deps:core - && python3 /mnt/repo/components/core/tools/scripts/utils/build-and-run-unit-tests.py - ${{matrix.use_shared_libs == true && '--use-shared-libs' || ''}} - --source-dir /mnt/repo/components/core - --build-dir /mnt/repo/components/core/build - --num-jobs $(getconf _NPROCESSORS_ONLN) - manylinux_2_28-x86_64-binaries: # Run if the ancestor jobs succeeded OR they were skipped and clp was changed. if: >- @@ -387,9 +279,9 @@ jobs: - uses: "./.github/actions/run-on-image" env: - OS_NAME: "manylinux_2_28-x86_64" + OS_NAME: "manylinux_2_28" with: - image_name: "${{env.DEPS_IMAGE_NAME_PREFIX_X86}}${{env.OS_NAME}}" + image_name: "${{env.DEPS_IMAGE_NAME_PREFIX}}${{env.OS_NAME}}" use_published_image: >- ${{needs.filter-relevant-changes.outputs.manylinux_2_28_x86_64_image_changed == 'false' || (github.event_name != 'pull_request' && github.ref == 'refs/heads/main')}} @@ -401,47 +293,6 @@ jobs: --build-dir /mnt/repo/components/core/build --num-jobs $(getconf _NPROCESSORS_ONLN) - musllinux_1_2-aarch64-binaries: - # Run if the ancestor jobs succeeded OR they were skipped and clp was changed. - if: >- - success() - || (!cancelled() && !failure() && needs.filter-relevant-changes.outputs.clp_changed == 'true') - needs: - - "musllinux_1_2-aarch64-deps-image" - - "filter-relevant-changes" - strategy: - matrix: - use_shared_libs: [true, false] - name: >- - musllinux_1_2-aarch64-${{matrix.use_shared_libs == true && 'dynamic' || - 'static'}}-linked-bins - continue-on-error: true - runs-on: "ubuntu-24.04-arm" - steps: - - uses: "actions/checkout@11bd71901bbe5b1630ceea73d27597364c9af683" - with: - submodules: "recursive" - - - name: "Work around actions/runner-images/issues/6775" - run: "chown $(id -u):$(id -g) -R ." - shell: "bash" - - - uses: "./.github/actions/run-on-image" - env: - OS_NAME: "musllinux_1_2-aarch64" - with: - image_name: "${{env.DEPS_IMAGE_NAME_PREFIX_ARM}}${{env.OS_NAME}}" - use_published_image: >- - ${{needs.filter-relevant-changes.outputs.musllinux_1_2_aarch64_image_changed == 'false' - || (github.event_name != 'pull_request' && github.ref == 'refs/heads/main')}} - run_command: >- - CLP_CORE_MAX_PARALLELISM_PER_BUILD_TASK=$(getconf _NPROCESSORS_ONLN) task deps:core - && python3 /mnt/repo/components/core/tools/scripts/utils/build-and-run-unit-tests.py - ${{matrix.use_shared_libs == true && '--use-shared-libs' || ''}} - --source-dir /mnt/repo/components/core - --build-dir /mnt/repo/components/core/build - --num-jobs $(getconf _NPROCESSORS_ONLN) - musllinux_1_2-x86_64-binaries: # Run if the ancestor jobs succeeded OR they were skipped and clp was changed. if: >- @@ -469,9 +320,9 @@ jobs: - uses: "./.github/actions/run-on-image" env: - OS_NAME: "musllinux_1_2-x86_64" + OS_NAME: "musllinux_1_2" with: - image_name: "${{env.DEPS_IMAGE_NAME_PREFIX_X86}}${{env.OS_NAME}}" + image_name: "${{env.DEPS_IMAGE_NAME_PREFIX}}${{env.OS_NAME}}" use_published_image: >- ${{needs.filter-relevant-changes.outputs.musllinux_1_2_x86_64_image_changed == 'false' || (github.event_name != 'pull_request' && github.ref == 'refs/heads/main')}} @@ -514,7 +365,7 @@ jobs: - uses: "./.github/actions/run-on-image" with: - image_name: "${{env.DEPS_IMAGE_NAME_PREFIX_X86}}${{env.OS_NAME}}" + image_name: "${{env.DEPS_IMAGE_NAME_PREFIX}}${{env.OS_NAME}}" use_published_image: >- ${{needs.filter-relevant-changes.outputs.ubuntu_jammy_image_changed == 'false' || (github.event_name != 'pull_request' && github.ref == 'refs/heads/main')}} @@ -648,7 +499,7 @@ jobs: env: OS_NAME: "ubuntu-jammy" with: - image_name: "${{env.DEPS_IMAGE_NAME_PREFIX_X86}}${{env.OS_NAME}}" + image_name: "${{env.DEPS_IMAGE_NAME_PREFIX}}${{env.OS_NAME}}" use_published_image: >- ${{needs.filter-relevant-changes.outputs.ubuntu_jammy_image_changed == 'false' || (github.event_name != 'pull_request' && github.ref == 'refs/heads/main')}} diff --git a/docs/src/dev-guide/tooling-containers.md b/docs/src/dev-guide/tooling-containers.md index fcef664133..36ee7664aa 100644 --- a/docs/src/dev-guide/tooling-containers.md +++ b/docs/src/dev-guide/tooling-containers.md @@ -29,6 +29,13 @@ distros using glibc 2.28+, including: ### clp-core-dependencies-x86-manylinux_2_28 +* [GitHub Packages page][core-deps-manylinux_2_28-x86_64] +* Pull command: + + ```bash + docker pull ghcr.io/y-scope/clp/clp-core-dependencies-x86-manylinux_2_28:main + ``` + * Path: ```text @@ -55,6 +62,13 @@ other distros using musl 1.2, including: ### clp-core-dependencies-x86-musllinux_1_2 +* [GitHub Packages page][core-deps-musllinux_1_2-x86_64] +* Pull command: + + ```bash + docker pull ghcr.io/y-scope/clp/clp-core-dependencies-x86-musllinux_1_2:main + ``` + * Path: ```text @@ -134,6 +148,8 @@ environment. ``` [core-deps-centos-stream-9]: https://github.com/y-scope/clp/pkgs/container/clp%2Fclp-core-dependencies-x86-centos-stream-9 +[core-deps-manylinux_2_28-x86_64]: https://github.com/y-scope/clp/pkgs/container/clp%2Fclp-core-dependencies-x86-manylinux_2_28 +[core-deps-musllinux_1_2-x86_64]: https://github.com/y-scope/clp/pkgs/container/clp%2Fclp-core-dependencies-x86-musllinux_1_2 [core-deps-ubuntu-jammy]: https://github.com/y-scope/clp/pkgs/container/clp%2Fclp-core-dependencies-x86-ubuntu-jammy [core-ubuntu-jammy]: https://github.com/y-scope/clp/pkgs/container/clp%2Fclp-core-x86-ubuntu-jammy [exe-ubuntu-jammy]: https://github.com/y-scope/clp/pkgs/container/clp%2Fclp-execution-x86-ubuntu-jammy diff --git a/docs/src/dev-guide/tooling-gh-workflows.md b/docs/src/dev-guide/tooling-gh-workflows.md index bc38f75257..a66de10dfc 100644 --- a/docs/src/dev-guide/tooling-gh-workflows.md +++ b/docs/src/dev-guide/tooling-gh-workflows.md @@ -29,10 +29,16 @@ shown below. }%% flowchart TD filter-relevant-changes --> centos-stream-9-deps-image + filter-relevant-changes --> manylinux_2_28-x86_64-deps-image + filter-relevant-changes --> musllinux_1_2-x86_64-deps-image filter-relevant-changes --> ubuntu-jammy-deps-image filter-relevant-changes --> centos-stream-9-binaries + filter-relevant-changes --> manylinux_2_28-x86_64-binaries + filter-relevant-changes --> musllinux_1_2-x86_64-binaries filter-relevant-changes --> ubuntu-jammy-binaries centos-stream-9-deps-image --> centos-stream-9-binaries + manylinux_2_28-x86_64-deps-image --> manylinux_2_28-x86_64-binaries + musllinux_1_2-x86_64-deps-image --> musllinux_1_2-x86_64-binaries ubuntu-jammy-deps-image --> ubuntu-jammy-binaries ubuntu-jammy-binaries --> ubuntu-jammy-binaries-image ::: @@ -43,10 +49,18 @@ Arrows between jobs indicate a dependency. The jobs are as follows: the following jobs should run. * `centos-stream-9-deps-image`: Builds a container image containing the dependencies necessary to build CLP-core in a CentOS Stream 9 x86 environment. +* `manylinux_2_28-x86_64-deps-image`: Builds a container image containing the dependencies necessary + to build CLP-core in a Manylinux 2.28 x86 environment. +* `musllinux_1_2-x86_64-deps-image`: Builds a container image containing the dependencies necessary + to build CLP-core in a Musllinux 1.2 x86 environment. * `ubuntu-jammy-deps-image`: Builds a container image containing the dependencies necessary to build CLP-core in an Ubuntu Jammy x86 environment. * `centos-stream-9-binaries`: Builds the CLP-core binaries in the built CentOS Stream 9 container and runs core's unit tests. +* `manylinux_2_28-x86_64-binaries`: Builds the CLP-core binaries in the built Manylinux 2.28 + container and runs core's unit tests. +* `musllinux_1_2-x86_64-binaries`: Builds the CLP-core binaries in the built Musllinux 1.2 container + and runs core's unit tests. * `ubuntu-jammy-binaries`: Builds the CLP-core binaries in the built Ubuntu Jammy container and runs core's unit tests. * `ubuntu-jammy-binaries-image`: Builds an Ubuntu Jammy container image containing CLP-core's From 8d463fd2303f8aa9e42f8a4da0fd8c6061d9cfa9 Mon Sep 17 00:00:00 2001 From: Bingran Hu Date: Tue, 26 Aug 2025 01:22:17 -0800 Subject: [PATCH 20/69] cleanup version changes --- docs/src/dev-guide/components-core/index.md | 17 +++++++++-------- 1 file changed, 9 insertions(+), 8 deletions(-) diff --git a/docs/src/dev-guide/components-core/index.md b/docs/src/dev-guide/components-core/index.md index 272fcc0269..5cb964f521 100644 --- a/docs/src/dev-guide/components-core/index.md +++ b/docs/src/dev-guide/components-core/index.md @@ -38,22 +38,23 @@ The task will download, build, and install (within the build directory) the foll | [abseil-cpp](https://github.com/abseil/abseil-cpp) | 20250512.0 | | [ANTLR](https://www.antlr.org) | v4.13.2 | | [Boost](https://github.com/boostorg/boost) | v1.87.0 | -| [Catch2](https://github.com/catchorg/Catch2.git) | v3.8.0 | -| [date](https://github.com/HowardHinnant/date.git) | v3.0.1 | +| [Catch2](https://github.com/catchorg/Catch2) | v3.8.0 | +| [date](https://github.com/HowardHinnant/date) | v3.0.1 | | [fmt](https://github.com/fmtlib/fmt) | v10.2.1 | -| [json](https://github.com/nlohmann/json.git) | v3.11.3 | -| [log-surgeon](https://github.com/y-scope/log-surgeon) | f801a3f | | [liblzma](https://github.com/tukaani-project/xz) | v5.8.1 | +| [log-surgeon](https://github.com/y-scope/log-surgeon) | a82ad13 | | [lz4](https://github.com/lz4/lz4) | v1.10.0 | +| [microsoft.gsl](https://github.com/microsoft/GSL) | v4.0.0 | | [mongo-cxx-driver](https://github.com/mongodb/mongo-cxx-driver) | r3.10.2 | | [msgpack-cxx](https://github.com/msgpack/msgpack-c/tree/cpp_master) | v7.0.0 | +| [nlohmann_json](https://github.com/nlohmann/json) | v3.11.3 | | [simdjson](https://github.com/simdjson/simdjson) | v3.13.0 | | [spdlog](https://github.com/gabime/spdlog) | v1.14.1 | | [SQLite3](https://www.sqlite.org/download.html) | v3.36.0 | -| [uftcpp](https://github.com/nemtrif/utfcpp.git) | v4.0.6 | -| [yaml-cpp](https://github.com/jbeder/yaml-cpp.git) | v0.7.0 | -| [yscope-log-viewer](https://github.com/y-scope/yscope-log-viewer.git) | 969ff35 | -| [ystdlib-cpp](https://github.com/y-scope/ystdlib-cpp.git) | d80cf86 | +| [utfcpp](https://github.com/nemtrif/utfcpp) | v4.0.6 | +| [yaml-cpp](https://github.com/jbeder/yaml-cpp) | v0.7.0 | +| [yscope-log-viewer](https://github.com/y-scope/yscope-log-viewer) | 3abe4cc | +| [ystdlib-cpp](https://github.com/y-scope/ystdlib-cpp) | d80cf86 | | [zlib](https://github.com/madler/zlib) | v1.3.1 | | [zstd](https://github.com/facebook/zstd) | v1.5.7 | From d68ddba9a9fcb9f012d58c829860fa04f472f760 Mon Sep 17 00:00:00 2001 From: Bingran Hu Date: Wed, 27 Aug 2025 09:22:17 +0800 Subject: [PATCH 21/69] Apply suggestions from code review Co-authored-by: kirkrodrigues <2454684+kirkrodrigues@users.noreply.github.com> --- .github/workflows/clp-core-build.yaml | 6 ++---- docs/src/dev-guide/tooling-gh-workflows.md | 10 +++++----- 2 files changed, 7 insertions(+), 9 deletions(-) diff --git a/.github/workflows/clp-core-build.yaml b/.github/workflows/clp-core-build.yaml index bc3bc2f7d2..4f29cec4f3 100644 --- a/.github/workflows/clp-core-build.yaml +++ b/.github/workflows/clp-core-build.yaml @@ -264,8 +264,7 @@ jobs: matrix: use_shared_libs: [true, false] name: >- - manylinux_2_28-x86_64-${{matrix.use_shared_libs == true && 'dynamic' || - 'static'}}-linked-bins + manylinux_2_28-x86_64-${{matrix.use_shared_libs == true && 'dynamic' || 'static'}}-linked-bins continue-on-error: true runs-on: "ubuntu-24.04" steps: @@ -305,8 +304,7 @@ jobs: matrix: use_shared_libs: [true, false] name: >- - musllinux_1_2-x86_64-${{matrix.use_shared_libs == true && 'dynamic' || - 'static'}}-linked-bins + musllinux_1_2-x86_64-${{matrix.use_shared_libs == true && 'dynamic' || 'static'}}-linked-bins continue-on-error: true runs-on: "ubuntu-24.04" steps: diff --git a/docs/src/dev-guide/tooling-gh-workflows.md b/docs/src/dev-guide/tooling-gh-workflows.md index a66de10dfc..7ac57344c0 100644 --- a/docs/src/dev-guide/tooling-gh-workflows.md +++ b/docs/src/dev-guide/tooling-gh-workflows.md @@ -27,7 +27,7 @@ shown below. } } }%% -flowchart TD +flowchart LR filter-relevant-changes --> centos-stream-9-deps-image filter-relevant-changes --> manylinux_2_28-x86_64-deps-image filter-relevant-changes --> musllinux_1_2-x86_64-deps-image @@ -50,16 +50,16 @@ Arrows between jobs indicate a dependency. The jobs are as follows: * `centos-stream-9-deps-image`: Builds a container image containing the dependencies necessary to build CLP-core in a CentOS Stream 9 x86 environment. * `manylinux_2_28-x86_64-deps-image`: Builds a container image containing the dependencies necessary - to build CLP-core in a Manylinux 2.28 x86 environment. + to build CLP-core in a manylinux_2_28 x86 environment. * `musllinux_1_2-x86_64-deps-image`: Builds a container image containing the dependencies necessary - to build CLP-core in a Musllinux 1.2 x86 environment. + to build CLP-core in a musllinux_1_2 x86 environment. * `ubuntu-jammy-deps-image`: Builds a container image containing the dependencies necessary to build CLP-core in an Ubuntu Jammy x86 environment. * `centos-stream-9-binaries`: Builds the CLP-core binaries in the built CentOS Stream 9 container and runs core's unit tests. -* `manylinux_2_28-x86_64-binaries`: Builds the CLP-core binaries in the built Manylinux 2.28 +* `manylinux_2_28-x86_64-binaries`: Builds the CLP-core binaries in the built manylinux_2_28 container and runs core's unit tests. -* `musllinux_1_2-x86_64-binaries`: Builds the CLP-core binaries in the built Musllinux 1.2 container +* `musllinux_1_2-x86_64-binaries`: Builds the CLP-core binaries in the built musllinux_1_2 container and runs core's unit tests. * `ubuntu-jammy-binaries`: Builds the CLP-core binaries in the built Ubuntu Jammy container and runs core's unit tests. From d3b9b24f80c0da7bf3980be9ecf24e109a0ff49f Mon Sep 17 00:00:00 2001 From: Bingran Hu Date: Wed, 27 Aug 2025 23:03:05 -0400 Subject: [PATCH 22/69] Install cmake 3.23 --- .../install-prebuilt-packages.sh | 18 +++++++++++++++++- .../install-prebuilt-packages.sh | 11 +++++++---- .../musllinux_1_2/install-prebuilt-packages.sh | 11 +++++++---- .../ubuntu-jammy/install-prebuilt-packages.sh | 15 ++++++++++++++- 4 files changed, 45 insertions(+), 10 deletions(-) diff --git a/components/core/tools/scripts/lib_install/centos-stream-9/install-prebuilt-packages.sh b/components/core/tools/scripts/lib_install/centos-stream-9/install-prebuilt-packages.sh index f91035b61c..9bc7f2da03 100755 --- a/components/core/tools/scripts/lib_install/centos-stream-9/install-prebuilt-packages.sh +++ b/components/core/tools/scripts/lib_install/centos-stream-9/install-prebuilt-packages.sh @@ -7,7 +7,6 @@ set -e set -u dnf install -y \ - cmake \ diffutils \ gcc-c++ \ git \ @@ -23,6 +22,23 @@ dnf install -y \ python3-pip \ unzip +if ! command -v pipx; then + python3 -m pip install pipx +fi + +if [ "$(id -u)" -eq 0 ]; then + # Running as root: install pipx softwares into system directories + export PIPX_HOME=/opt/_internal/pipx + export PIPX_BIN_DIR=/usr/local/bin +fi +pipx ensurepath + +# ystdlib requires CMake v3.23; ANTLR and yaml-cpp do not yet support CMake v4+. +# See also: https://github.com/y-scope/clp/issues/795 +if ! command -v cmake ; then + pipx install "cmake~=3.23" +fi + # Determine architecture for `task` release to install rpm_arch=$(rpm --eval "%{_arch}") case "$rpm_arch" in diff --git a/components/core/tools/scripts/lib_install/manylinux_2_28/install-prebuilt-packages.sh b/components/core/tools/scripts/lib_install/manylinux_2_28/install-prebuilt-packages.sh index 57ea2caab0..e90845a951 100755 --- a/components/core/tools/scripts/lib_install/manylinux_2_28/install-prebuilt-packages.sh +++ b/components/core/tools/scripts/lib_install/manylinux_2_28/install-prebuilt-packages.sh @@ -13,6 +13,13 @@ dnf install -y \ zlib-devel \ zlib-static +# ystdlib requires CMake v3.23; ANTLR and yaml-cpp do not yet support CMake v4+. +# See also: https://github.com/y-scope/clp/issues/795 +pipx uninstall cmake +if ! command -v cmake ; then + pipx install "cmake~=3.23" +fi + # Determine architecture for `task` release to install rpm_arch=$(rpm --eval "%{_arch}") case "$rpm_arch" in @@ -39,7 +46,3 @@ curl \ "https://github.com/go-task/task/releases/download/v3.42.1/task_linux_${task_pkg_arch}.rpm" dnf install --assumeyes "$task_pkg_path" rm "$task_pkg_path" - -# Downgrade to CMake v3 to work around https://github.com/y-scope/clp/issues/795 -pipx uninstall cmake -pipx install cmake~=3.31 diff --git a/components/core/tools/scripts/lib_install/musllinux_1_2/install-prebuilt-packages.sh b/components/core/tools/scripts/lib_install/musllinux_1_2/install-prebuilt-packages.sh index e48fa79f59..0b3446eee9 100755 --- a/components/core/tools/scripts/lib_install/musllinux_1_2/install-prebuilt-packages.sh +++ b/components/core/tools/scripts/lib_install/musllinux_1_2/install-prebuilt-packages.sh @@ -14,6 +14,13 @@ apk update && apk add --no-cache \ zlib-dev \ zlib-static +# ystdlib requires CMake v3.23; ANTLR and yaml-cpp do not yet support CMake v4+. +# See also: https://github.com/y-scope/clp/issues/795 +pipx uninstall cmake +if ! command -v cmake ; then + pipx install "cmake~=3.23" +fi + # Determine architecture for `task` release to install arch=$(uname -m) case "$arch" in @@ -41,7 +48,3 @@ curl \ tar -C /usr/local/bin -xzf "$task_pkg_path" task chmod +x /usr/local/bin/task rm "$task_pkg_path" - -# Downgrade to CMake v3 to work around https://github.com/y-scope/clp/issues/795 -pipx uninstall cmake -pipx install cmake~=3.31 diff --git a/components/core/tools/scripts/lib_install/ubuntu-jammy/install-prebuilt-packages.sh b/components/core/tools/scripts/lib_install/ubuntu-jammy/install-prebuilt-packages.sh index 03a3c26e8c..342c7d091b 100755 --- a/components/core/tools/scripts/lib_install/ubuntu-jammy/install-prebuilt-packages.sh +++ b/components/core/tools/scripts/lib_install/ubuntu-jammy/install-prebuilt-packages.sh @@ -10,7 +10,6 @@ apt-get update DEBIAN_FRONTEND=noninteractive apt-get install --no-install-recommends -y \ ca-certificates \ checkinstall \ - cmake \ curl \ build-essential \ git \ @@ -21,6 +20,7 @@ DEBIAN_FRONTEND=noninteractive apt-get install --no-install-recommends -y \ libmariadb-dev \ libssl-dev \ openjdk-11-jdk \ + pipx \ pkg-config \ python3 \ python3-pip \ @@ -28,6 +28,19 @@ DEBIAN_FRONTEND=noninteractive apt-get install --no-install-recommends -y \ software-properties-common \ unzip +if [ "$(id -u)" -eq 0 ]; then + # Running as root: install pipx softwares into system directories + export PIPX_HOME=/opt/_internal/pipx + export PIPX_BIN_DIR=/usr/local/bin +fi +pipx ensurepath + +# ystdlib requires CMake v3.23; ANTLR and yaml-cpp do not yet support CMake v4+. +# See also: https://github.com/y-scope/clp/issues/795 +if ! command -v cmake ; then + pipx install "cmake~=3.23" +fi + # Install `task` # NOTE: We lock `task` to a version < 3.43 to avoid https://github.com/y-scope/clp/issues/872 task_pkg_arch=$(dpkg --print-architecture) From e6043ad00b523f3fc29f6d579473fe1a3085f37e Mon Sep 17 00:00:00 2001 From: Bingran Hu Date: Thu, 28 Aug 2025 01:18:58 -0400 Subject: [PATCH 23/69] switch task and uv to pipx install too --- .../clp-env-base-centos-stream-9/Dockerfile | 2 -- .../clp-env-base-ubuntu-jammy/Dockerfile | 2 -- .../install-prebuilt-packages.sh | 33 +++++------------ .../install-prebuilt-packages.sh | 35 ++++++------------ .../install-prebuilt-packages.sh | 36 ++++++------------- .../ubuntu-jammy/install-prebuilt-packages.sh | 19 +++++----- 6 files changed, 40 insertions(+), 87 deletions(-) diff --git a/components/core/tools/docker-images/clp-env-base-centos-stream-9/Dockerfile b/components/core/tools/docker-images/clp-env-base-centos-stream-9/Dockerfile index 85ac66522f..3959d441f6 100644 --- a/components/core/tools/docker-images/clp-env-base-centos-stream-9/Dockerfile +++ b/components/core/tools/docker-images/clp-env-base-centos-stream-9/Dockerfile @@ -7,8 +7,6 @@ ADD ./tools/scripts/lib_install ./tools/scripts/lib_install RUN ./tools/scripts/lib_install/centos-stream-9/install-all.sh -RUN pip3 install --no-cache-dir uv - # Remove cached files RUN dnf clean all \ && rm -rf /tmp/* /var/tmp/* diff --git a/components/core/tools/docker-images/clp-env-base-ubuntu-jammy/Dockerfile b/components/core/tools/docker-images/clp-env-base-ubuntu-jammy/Dockerfile index 6aed574188..d0fb69004f 100644 --- a/components/core/tools/docker-images/clp-env-base-ubuntu-jammy/Dockerfile +++ b/components/core/tools/docker-images/clp-env-base-ubuntu-jammy/Dockerfile @@ -7,8 +7,6 @@ ADD ./tools/scripts/lib_install ./tools/scripts/lib_install RUN ./tools/scripts/lib_install/ubuntu-jammy/install-all.sh -RUN pip3 install --no-cache-dir uv - # Remove cached files RUN apt-get clean \ && rm -rf /var/lib/apt/lists/* /tmp/* /var/tmp/* diff --git a/components/core/tools/scripts/lib_install/centos-stream-9/install-prebuilt-packages.sh b/components/core/tools/scripts/lib_install/centos-stream-9/install-prebuilt-packages.sh index 9bc7f2da03..ae1cd5bd3e 100755 --- a/components/core/tools/scripts/lib_install/centos-stream-9/install-prebuilt-packages.sh +++ b/components/core/tools/scripts/lib_install/centos-stream-9/install-prebuilt-packages.sh @@ -33,35 +33,20 @@ if [ "$(id -u)" -eq 0 ]; then fi pipx ensurepath +# Install `cmake` # ystdlib requires CMake v3.23; ANTLR and yaml-cpp do not yet support CMake v4+. # See also: https://github.com/y-scope/clp/issues/795 if ! command -v cmake ; then pipx install "cmake~=3.23" fi -# Determine architecture for `task` release to install -rpm_arch=$(rpm --eval "%{_arch}") -case "$rpm_arch" in - "x86_64") - task_pkg_arch="amd64" - ;; - "aarch64") - task_pkg_arch="arm64" - ;; - *) - echo "Error: Unsupported architecture - $rpm_arch" - exit 1 - ;; -esac - # Install `task` # NOTE: We lock `task` to a version < 3.43 to avoid https://github.com/y-scope/clp/issues/872 -task_pkg_path="$(mktemp -t --suffix ".rpm" task-pkg.XXXXXXXXXX)" -curl \ - --fail \ - --location \ - --output "$task_pkg_path" \ - --show-error \ - "https://github.com/go-task/task/releases/download/v3.42.1/task_linux_${task_pkg_arch}.rpm" -dnf install --assumeyes "$task_pkg_path" -rm "$task_pkg_path" +if ! command -v task ; then + pipx install "go-task-bin>=3.40,<3.43" +fi + +# Install `uv` +if ! command -v uv ; then + pix install "uv>=0.8" +fi diff --git a/components/core/tools/scripts/lib_install/manylinux_2_28/install-prebuilt-packages.sh b/components/core/tools/scripts/lib_install/manylinux_2_28/install-prebuilt-packages.sh index e90845a951..b4053812c5 100755 --- a/components/core/tools/scripts/lib_install/manylinux_2_28/install-prebuilt-packages.sh +++ b/components/core/tools/scripts/lib_install/manylinux_2_28/install-prebuilt-packages.sh @@ -13,6 +13,7 @@ dnf install -y \ zlib-devel \ zlib-static +# Install `cmake` # ystdlib requires CMake v3.23; ANTLR and yaml-cpp do not yet support CMake v4+. # See also: https://github.com/y-scope/clp/issues/795 pipx uninstall cmake @@ -20,29 +21,15 @@ if ! command -v cmake ; then pipx install "cmake~=3.23" fi -# Determine architecture for `task` release to install -rpm_arch=$(rpm --eval "%{_arch}") -case "$rpm_arch" in - "x86_64") - task_pkg_arch="amd64" - ;; - "aarch64") - task_pkg_arch="arm64" - ;; - *) - echo "Error: Unsupported architecture - $rpm_arch" - exit 1 - ;; -esac - # Install `task` # NOTE: We lock `task` to a version < 3.43 to avoid https://github.com/y-scope/clp/issues/872 -task_pkg_path=$(mktemp -t --suffix ".rpm" task-pkg.XXXXXXXXXX) || exit 1 -curl \ - --fail \ - --location \ - --output "$task_pkg_path" \ - --show-error \ - "https://github.com/go-task/task/releases/download/v3.42.1/task_linux_${task_pkg_arch}.rpm" -dnf install --assumeyes "$task_pkg_path" -rm "$task_pkg_path" +pipx uninstall go-task-bin +if ! command -v task ; then + pipx install "go-task-bin>=3.40,<3.43" +fi + +# Install `uv` +pipx uninstall uv +if ! command -v uv ; then + pix install "uv>=0.8" +fi diff --git a/components/core/tools/scripts/lib_install/musllinux_1_2/install-prebuilt-packages.sh b/components/core/tools/scripts/lib_install/musllinux_1_2/install-prebuilt-packages.sh index 0b3446eee9..aa0273c517 100755 --- a/components/core/tools/scripts/lib_install/musllinux_1_2/install-prebuilt-packages.sh +++ b/components/core/tools/scripts/lib_install/musllinux_1_2/install-prebuilt-packages.sh @@ -14,6 +14,7 @@ apk update && apk add --no-cache \ zlib-dev \ zlib-static +# Install `cmake` # ystdlib requires CMake v3.23; ANTLR and yaml-cpp do not yet support CMake v4+. # See also: https://github.com/y-scope/clp/issues/795 pipx uninstall cmake @@ -21,30 +22,15 @@ if ! command -v cmake ; then pipx install "cmake~=3.23" fi -# Determine architecture for `task` release to install -arch=$(uname -m) -case "$arch" in - "x86_64") - task_pkg_arch="amd64" - ;; - "aarch64") - task_pkg_arch="arm64" - ;; - *) - echo "Error: Unsupported architecture - $arch" - exit 1 - ;; -esac - # Install `task` # NOTE: We lock `task` to a version < 3.43 to avoid https://github.com/y-scope/clp/issues/872 -task_pkg_path=$(mktemp -t task-pkg.XXXXXXXXXX).tar.gz || exit 1 -curl \ - --fail \ - --location \ - --output "$task_pkg_path" \ - --show-error \ - "https://github.com/go-task/task/releases/download/v3.42.1/task_linux_${task_pkg_arch}.tar.gz" -tar -C /usr/local/bin -xzf "$task_pkg_path" task -chmod +x /usr/local/bin/task -rm "$task_pkg_path" +pipx uninstall go-task-bin +if ! command -v task ; then + pipx install "go-task-bin>=3.40,<3.43" +fi + +# Install `uv` +pipx uninstall uv +if ! command -v uv ; then + pix install "uv>=0.8" +fi diff --git a/components/core/tools/scripts/lib_install/ubuntu-jammy/install-prebuilt-packages.sh b/components/core/tools/scripts/lib_install/ubuntu-jammy/install-prebuilt-packages.sh index 342c7d091b..2d1892a0e4 100755 --- a/components/core/tools/scripts/lib_install/ubuntu-jammy/install-prebuilt-packages.sh +++ b/components/core/tools/scripts/lib_install/ubuntu-jammy/install-prebuilt-packages.sh @@ -35,6 +35,7 @@ if [ "$(id -u)" -eq 0 ]; then fi pipx ensurepath +# Install `cmake` # ystdlib requires CMake v3.23; ANTLR and yaml-cpp do not yet support CMake v4+. # See also: https://github.com/y-scope/clp/issues/795 if ! command -v cmake ; then @@ -43,13 +44,11 @@ fi # Install `task` # NOTE: We lock `task` to a version < 3.43 to avoid https://github.com/y-scope/clp/issues/872 -task_pkg_arch=$(dpkg --print-architecture) -task_pkg_path="$(mktemp -t --suffix ".deb" task-pkg.XXXXXXXXXX)" -curl \ - --fail \ - --location \ - --output "$task_pkg_path" \ - --show-error \ - "https://github.com/go-task/task/releases/download/v3.42.1/task_linux_${task_pkg_arch}.deb" -dpkg --install "$task_pkg_path" -rm "$task_pkg_path" +if ! command -v task ; then + pipx install "go-task-bin>=3.40,<3.43" +fi + +# Install `uv` +if ! command -v uv ; then + pix install "uv>=0.8" +fi From 2e2e9c8c78163f984bdfb9f96376cd86cbbaf566 Mon Sep 17 00:00:00 2001 From: Bingran Hu Date: Thu, 28 Aug 2025 05:52:35 -0400 Subject: [PATCH 24/69] Typo fix --- .../centos-stream-9/install-prebuilt-packages.sh | 9 +++------ .../core/tools/scripts/lib_install/macos/install-all.sh | 7 ++----- .../manylinux_2_28/install-prebuilt-packages.sh | 6 +++--- .../musllinux_1_2/install-prebuilt-packages.sh | 6 +++--- .../ubuntu-jammy/install-prebuilt-packages.sh | 9 +++------ 5 files changed, 14 insertions(+), 23 deletions(-) diff --git a/components/core/tools/scripts/lib_install/centos-stream-9/install-prebuilt-packages.sh b/components/core/tools/scripts/lib_install/centos-stream-9/install-prebuilt-packages.sh index ae1cd5bd3e..139bc317da 100755 --- a/components/core/tools/scripts/lib_install/centos-stream-9/install-prebuilt-packages.sh +++ b/components/core/tools/scripts/lib_install/centos-stream-9/install-prebuilt-packages.sh @@ -1,10 +1,7 @@ #!/usr/bin/env bash -# Exit on any error -set -e - -# Error on undefined variable -set -u +# Exit on any error, use of undefined variables, or failure within a pipeline +set -euo pipefail dnf install -y \ diffutils \ @@ -48,5 +45,5 @@ fi # Install `uv` if ! command -v uv ; then - pix install "uv>=0.8" + pipx install "uv>=0.8" fi diff --git a/components/core/tools/scripts/lib_install/macos/install-all.sh b/components/core/tools/scripts/lib_install/macos/install-all.sh index 78327c4956..43198d7959 100755 --- a/components/core/tools/scripts/lib_install/macos/install-all.sh +++ b/components/core/tools/scripts/lib_install/macos/install-all.sh @@ -1,10 +1,7 @@ #!/usr/bin/env bash -# Exit on any error -set -e - -# Error on undefined variable -set -u +# Exit on any error, use of undefined variables, or failure within a pipeline +set -euo pipefail brew update diff --git a/components/core/tools/scripts/lib_install/manylinux_2_28/install-prebuilt-packages.sh b/components/core/tools/scripts/lib_install/manylinux_2_28/install-prebuilt-packages.sh index b4053812c5..74dbb77d48 100755 --- a/components/core/tools/scripts/lib_install/manylinux_2_28/install-prebuilt-packages.sh +++ b/components/core/tools/scripts/lib_install/manylinux_2_28/install-prebuilt-packages.sh @@ -1,7 +1,7 @@ #!/usr/bin/env bash -set -eu -set -o pipefail +# Exit on any error, use of undefined variables, or failure within a pipeline +set -euo pipefail dnf install -y \ gcc-c++ \ @@ -31,5 +31,5 @@ fi # Install `uv` pipx uninstall uv if ! command -v uv ; then - pix install "uv>=0.8" + pipx install "uv>=0.8" fi diff --git a/components/core/tools/scripts/lib_install/musllinux_1_2/install-prebuilt-packages.sh b/components/core/tools/scripts/lib_install/musllinux_1_2/install-prebuilt-packages.sh index aa0273c517..03e514fb68 100755 --- a/components/core/tools/scripts/lib_install/musllinux_1_2/install-prebuilt-packages.sh +++ b/components/core/tools/scripts/lib_install/musllinux_1_2/install-prebuilt-packages.sh @@ -1,7 +1,7 @@ #!/usr/bin/env bash -set -eu -set -o pipefail +# Exit on any error, use of undefined variables, or failure within a pipeline +set -euo pipefail apk update && apk add --no-cache \ bzip2-dev \ @@ -32,5 +32,5 @@ fi # Install `uv` pipx uninstall uv if ! command -v uv ; then - pix install "uv>=0.8" + pipx install "uv>=0.8" fi diff --git a/components/core/tools/scripts/lib_install/ubuntu-jammy/install-prebuilt-packages.sh b/components/core/tools/scripts/lib_install/ubuntu-jammy/install-prebuilt-packages.sh index 2d1892a0e4..9349b86b9a 100755 --- a/components/core/tools/scripts/lib_install/ubuntu-jammy/install-prebuilt-packages.sh +++ b/components/core/tools/scripts/lib_install/ubuntu-jammy/install-prebuilt-packages.sh @@ -1,10 +1,7 @@ #!/usr/bin/env bash -# Exit on any error -set -e - -# Error on undefined variable -set -u +# Exit on any error, use of undefined variables, or failure within a pipeline +set -euo pipefail apt-get update DEBIAN_FRONTEND=noninteractive apt-get install --no-install-recommends -y \ @@ -50,5 +47,5 @@ fi # Install `uv` if ! command -v uv ; then - pix install "uv>=0.8" + pipx install "uv>=0.8" fi From 4141831a7a303ca9bac1fd76c1adb16498d61c68 Mon Sep 17 00:00:00 2001 From: Bingran Hu Date: Thu, 28 Aug 2025 12:50:23 -0400 Subject: [PATCH 25/69] Allow pipx uninstall not found --- .../lib_install/manylinux_2_28/install-prebuilt-packages.sh | 6 +++--- .../lib_install/musllinux_1_2/install-prebuilt-packages.sh | 6 +++--- 2 files changed, 6 insertions(+), 6 deletions(-) diff --git a/components/core/tools/scripts/lib_install/manylinux_2_28/install-prebuilt-packages.sh b/components/core/tools/scripts/lib_install/manylinux_2_28/install-prebuilt-packages.sh index 74dbb77d48..19b5d73116 100755 --- a/components/core/tools/scripts/lib_install/manylinux_2_28/install-prebuilt-packages.sh +++ b/components/core/tools/scripts/lib_install/manylinux_2_28/install-prebuilt-packages.sh @@ -16,20 +16,20 @@ dnf install -y \ # Install `cmake` # ystdlib requires CMake v3.23; ANTLR and yaml-cpp do not yet support CMake v4+. # See also: https://github.com/y-scope/clp/issues/795 -pipx uninstall cmake +pipx uninstall cmake || true if ! command -v cmake ; then pipx install "cmake~=3.23" fi # Install `task` # NOTE: We lock `task` to a version < 3.43 to avoid https://github.com/y-scope/clp/issues/872 -pipx uninstall go-task-bin +pipx uninstall go-task-bin || true if ! command -v task ; then pipx install "go-task-bin>=3.40,<3.43" fi # Install `uv` -pipx uninstall uv +pipx uninstall uv || true if ! command -v uv ; then pipx install "uv>=0.8" fi diff --git a/components/core/tools/scripts/lib_install/musllinux_1_2/install-prebuilt-packages.sh b/components/core/tools/scripts/lib_install/musllinux_1_2/install-prebuilt-packages.sh index 03e514fb68..b5fa25b085 100755 --- a/components/core/tools/scripts/lib_install/musllinux_1_2/install-prebuilt-packages.sh +++ b/components/core/tools/scripts/lib_install/musllinux_1_2/install-prebuilt-packages.sh @@ -17,20 +17,20 @@ apk update && apk add --no-cache \ # Install `cmake` # ystdlib requires CMake v3.23; ANTLR and yaml-cpp do not yet support CMake v4+. # See also: https://github.com/y-scope/clp/issues/795 -pipx uninstall cmake +pipx uninstall cmake || true if ! command -v cmake ; then pipx install "cmake~=3.23" fi # Install `task` # NOTE: We lock `task` to a version < 3.43 to avoid https://github.com/y-scope/clp/issues/872 -pipx uninstall go-task-bin +pipx uninstall go-task-bin || true if ! command -v task ; then pipx install "go-task-bin>=3.40,<3.43" fi # Install `uv` -pipx uninstall uv +pipx uninstall uv || true if ! command -v uv ; then pipx install "uv>=0.8" fi From fc2fd9ad48e909834d4b54b2cd82e7f81f195302 Mon Sep 17 00:00:00 2001 From: Bingran Hu Date: Fri, 29 Aug 2025 01:36:08 -0400 Subject: [PATCH 26/69] add lib version checks --- .../centos-stream-9/install-all.sh | 10 ++--- .../scripts/lib_install/macos/install-all.sh | 6 +-- .../lib_install/manylinux_2_28/install-all.sh | 7 ++-- .../lib_install/musllinux_1_2/install-all.sh | 7 ++-- .../lib_install/ubuntu-jammy/install-all.sh | 10 ++--- .../check-build-tool-versions.sh | 39 +++++++++++++++++++ .../check-cmake-version.sh | 9 ++--- .../check-go-task-version.sh | 18 +++++++++ .../lib_version_checks/check-uv-version.sh | 14 +++++++ .../print-go-task-version.yaml | 6 +++ 10 files changed, 95 insertions(+), 31 deletions(-) create mode 100755 components/core/tools/scripts/lib_version_checks/check-build-tool-versions.sh rename components/core/tools/scripts/{lib_install => lib_version_checks}/check-cmake-version.sh (64%) create mode 100755 components/core/tools/scripts/lib_version_checks/check-go-task-version.sh create mode 100755 components/core/tools/scripts/lib_version_checks/check-uv-version.sh create mode 100644 components/core/tools/scripts/lib_version_checks/print-go-task-version.yaml diff --git a/components/core/tools/scripts/lib_install/centos-stream-9/install-all.sh b/components/core/tools/scripts/lib_install/centos-stream-9/install-all.sh index a44de8130c..a7c183456e 100755 --- a/components/core/tools/scripts/lib_install/centos-stream-9/install-all.sh +++ b/components/core/tools/scripts/lib_install/centos-stream-9/install-all.sh @@ -1,15 +1,11 @@ #!/usr/bin/env bash -# Exit on any error -set -e - -# Error on undefined variable -set -u +# Exit on any error, use of undefined variables, or failure within a pipeline +set -euo pipefail script_dir="$( cd "$( dirname "${BASH_SOURCE[0]}" )" &> /dev/null && pwd )" "${script_dir}/install-prebuilt-packages.sh" "${script_dir}/install-packages-from-source.sh" -# TODO: https://github.com/y-scope/clp/issues/795 -"${script_dir}/../check-cmake-version.sh" +"${script_dir}/../../lib_version_checks/check-build-tool-versions.sh" diff --git a/components/core/tools/scripts/lib_install/macos/install-all.sh b/components/core/tools/scripts/lib_install/macos/install-all.sh index 43198d7959..46face74c2 100755 --- a/components/core/tools/scripts/lib_install/macos/install-all.sh +++ b/components/core/tools/scripts/lib_install/macos/install-all.sh @@ -3,6 +3,8 @@ # Exit on any error, use of undefined variables, or failure within a pipeline set -euo pipefail +script_dir="$( cd "$( dirname "${BASH_SOURCE[0]}" )" &> /dev/null && pwd )" + brew update # Install CMake v3.31.6 as ANTLR and yaml-cpp do not yet support CMake v4+. @@ -42,6 +44,4 @@ if ! command -v pkg-config ; then brew install pkg-config fi -# TODO: https://github.com/y-scope/clp/issues/795 -script_dir="$( cd "$( dirname "${BASH_SOURCE[0]}" )" &> /dev/null && pwd )" -"${script_dir}/../check-cmake-version.sh" +"${script_dir}/../../lib_version_checks/check-build-tool-versions.sh" diff --git a/components/core/tools/scripts/lib_install/manylinux_2_28/install-all.sh b/components/core/tools/scripts/lib_install/manylinux_2_28/install-all.sh index 089cd6bc66..a7c183456e 100755 --- a/components/core/tools/scripts/lib_install/manylinux_2_28/install-all.sh +++ b/components/core/tools/scripts/lib_install/manylinux_2_28/install-all.sh @@ -1,12 +1,11 @@ #!/usr/bin/env bash -set -eu -set -o pipefail +# Exit on any error, use of undefined variables, or failure within a pipeline +set -euo pipefail script_dir="$( cd "$( dirname "${BASH_SOURCE[0]}" )" &> /dev/null && pwd )" "${script_dir}/install-prebuilt-packages.sh" "${script_dir}/install-packages-from-source.sh" -# TODO: https://github.com/y-scope/clp/issues/795 -"${script_dir}/../check-cmake-version.sh" +"${script_dir}/../../lib_version_checks/check-build-tool-versions.sh" diff --git a/components/core/tools/scripts/lib_install/musllinux_1_2/install-all.sh b/components/core/tools/scripts/lib_install/musllinux_1_2/install-all.sh index 089cd6bc66..a7c183456e 100755 --- a/components/core/tools/scripts/lib_install/musllinux_1_2/install-all.sh +++ b/components/core/tools/scripts/lib_install/musllinux_1_2/install-all.sh @@ -1,12 +1,11 @@ #!/usr/bin/env bash -set -eu -set -o pipefail +# Exit on any error, use of undefined variables, or failure within a pipeline +set -euo pipefail script_dir="$( cd "$( dirname "${BASH_SOURCE[0]}" )" &> /dev/null && pwd )" "${script_dir}/install-prebuilt-packages.sh" "${script_dir}/install-packages-from-source.sh" -# TODO: https://github.com/y-scope/clp/issues/795 -"${script_dir}/../check-cmake-version.sh" +"${script_dir}/../../lib_version_checks/check-build-tool-versions.sh" diff --git a/components/core/tools/scripts/lib_install/ubuntu-jammy/install-all.sh b/components/core/tools/scripts/lib_install/ubuntu-jammy/install-all.sh index a44de8130c..a7c183456e 100755 --- a/components/core/tools/scripts/lib_install/ubuntu-jammy/install-all.sh +++ b/components/core/tools/scripts/lib_install/ubuntu-jammy/install-all.sh @@ -1,15 +1,11 @@ #!/usr/bin/env bash -# Exit on any error -set -e - -# Error on undefined variable -set -u +# Exit on any error, use of undefined variables, or failure within a pipeline +set -euo pipefail script_dir="$( cd "$( dirname "${BASH_SOURCE[0]}" )" &> /dev/null && pwd )" "${script_dir}/install-prebuilt-packages.sh" "${script_dir}/install-packages-from-source.sh" -# TODO: https://github.com/y-scope/clp/issues/795 -"${script_dir}/../check-cmake-version.sh" +"${script_dir}/../../lib_version_checks/check-build-tool-versions.sh" diff --git a/components/core/tools/scripts/lib_version_checks/check-build-tool-versions.sh b/components/core/tools/scripts/lib_version_checks/check-build-tool-versions.sh new file mode 100755 index 0000000000..f647c24c36 --- /dev/null +++ b/components/core/tools/scripts/lib_version_checks/check-build-tool-versions.sh @@ -0,0 +1,39 @@ +#!/usr/bin/env bash + +# Exit on any error, use of undefined variables, or failure within a pipeline +set -euo pipefail + +script_dir="$( cd "$( dirname "${BASH_SOURCE[0]}" )" &> /dev/null && pwd )" + +failures=0 + +run_check() { + local script="$1" + local desc="$2" + + if output=$("$script_dir/$script" 2>&1); then + echo "[OK] $desc" + else + echo "[FAIL] $desc" + echo "$output" + failures=$((failures + 1)) + fi +} + +# Run all checks +# TODO: https://github.com/y-scope/clp/issues/795 +run_check check-cmake-version.sh "CMake version" + +# TODO: https://github.com/y-scope/clp/issues/872 +run_check check-go-task-version.sh "Go Task version" + +run_check check-uv-version.sh "uv version" + +# Print summary +echo +if [ "$failures" -gt 0 ]; then + echo "$failures check(s) failed." + exit 1 +else + echo "All checks passed." +fi diff --git a/components/core/tools/scripts/lib_install/check-cmake-version.sh b/components/core/tools/scripts/lib_version_checks/check-cmake-version.sh similarity index 64% rename from components/core/tools/scripts/lib_install/check-cmake-version.sh rename to components/core/tools/scripts/lib_version_checks/check-cmake-version.sh index d942b9ce98..488821c0f7 100755 --- a/components/core/tools/scripts/lib_install/check-cmake-version.sh +++ b/components/core/tools/scripts/lib_version_checks/check-cmake-version.sh @@ -1,10 +1,7 @@ #!/usr/bin/env bash -# Exit on any error -set -e - -# Error on undefined variable -set -u +# Exit on any error, use of undefined variables, or failure within a pipeline +set -euo pipefail # Get the installed cmake version string cmake_version=$(cmake -E capabilities | jq --raw-output ".version.string") @@ -12,6 +9,6 @@ cmake_major_version=$(cmake -E capabilities | jq --raw-output ".version.major") # Check if version is 4.0 or higher if [[ "$cmake_major_version" -ge "4" ]]; then - echo "CMake version $cmake_version is currently unsupported (>= 4.0)." + echo "Error: CMake version $cmake_version is currently unsupported (>= 4.0)." exit 1 fi diff --git a/components/core/tools/scripts/lib_version_checks/check-go-task-version.sh b/components/core/tools/scripts/lib_version_checks/check-go-task-version.sh new file mode 100755 index 0000000000..44138b280c --- /dev/null +++ b/components/core/tools/scripts/lib_version_checks/check-go-task-version.sh @@ -0,0 +1,18 @@ +#!/usr/bin/env bash + +# Exit on any error, use of undefined variables, or failure within a pipeline +set -euo pipefail + +script_dir="$( cd "$( dirname "${BASH_SOURCE[0]}" )" &> /dev/null && pwd )" + +# Get the installed go-task version string +task_version=$(task --silent --taskfile ${script_dir}/print-go-task-version.yaml version) +IFS=. read task_major_version task_minor_version _ <<< "$task_version" + +# Check version constraints +if [ "$task_major_version" -ne "3" ] || \ + [ "$task_minor_version" -lt "40" ] || \ + [ "$task_minor_version" -ge "43" ]; then + echo "Error: Task version ${task_version} is currently unsupported (must be 3.40 <= ver < 3.43)." + exit 1 +fi diff --git a/components/core/tools/scripts/lib_version_checks/check-uv-version.sh b/components/core/tools/scripts/lib_version_checks/check-uv-version.sh new file mode 100755 index 0000000000..6740c177ed --- /dev/null +++ b/components/core/tools/scripts/lib_version_checks/check-uv-version.sh @@ -0,0 +1,14 @@ +#!/usr/bin/env bash + +# Exit on any error, use of undefined variables, or failure within a pipeline +set -euo pipefail + +# Get the installed uv version string +uv_version=$(uv self version --output-format json | jq --raw-output ".version") +IFS=. read uv_major_version uv_minor_version _ <<< "$uv_version" + +# Check if version is 4.0 or higher +if [ "$uv_major_version" -lt "1" ] && [ "$uv_minor_version" -lt "8" ]; then + echo "Error: uv version $uv_version is currently unsupported (< 0.8)." + exit 1 +fi diff --git a/components/core/tools/scripts/lib_version_checks/print-go-task-version.yaml b/components/core/tools/scripts/lib_version_checks/print-go-task-version.yaml new file mode 100644 index 0000000000..bebb85d2ff --- /dev/null +++ b/components/core/tools/scripts/lib_version_checks/print-go-task-version.yaml @@ -0,0 +1,6 @@ +version: "3" + +tasks: + version: + cmd: >- + echo {{.TASK_VERSION}} | sed "s/^v//" From 54e18e0322af111e7446e119604793d07aedbbf6 Mon Sep 17 00:00:00 2001 From: Bingran Hu Date: Fri, 29 Aug 2025 01:48:58 -0400 Subject: [PATCH 27/69] reorder dir structure --- .../tools/scripts/lib_install/centos-stream-9/install-all.sh | 2 +- .../lib_version_checks/check-build-tool-versions.sh | 0 .../{ => lib_install}/lib_version_checks/check-cmake-version.sh | 0 .../lib_version_checks/check-go-task-version.sh | 0 .../{ => lib_install}/lib_version_checks/check-uv-version.sh | 0 .../lib_version_checks/print-go-task-version.yaml | 0 components/core/tools/scripts/lib_install/macos/install-all.sh | 2 +- .../tools/scripts/lib_install/manylinux_2_28/install-all.sh | 2 +- .../core/tools/scripts/lib_install/musllinux_1_2/install-all.sh | 2 +- .../core/tools/scripts/lib_install/ubuntu-jammy/install-all.sh | 2 +- 10 files changed, 5 insertions(+), 5 deletions(-) rename components/core/tools/scripts/{ => lib_install}/lib_version_checks/check-build-tool-versions.sh (100%) rename components/core/tools/scripts/{ => lib_install}/lib_version_checks/check-cmake-version.sh (100%) rename components/core/tools/scripts/{ => lib_install}/lib_version_checks/check-go-task-version.sh (100%) rename components/core/tools/scripts/{ => lib_install}/lib_version_checks/check-uv-version.sh (100%) rename components/core/tools/scripts/{ => lib_install}/lib_version_checks/print-go-task-version.yaml (100%) diff --git a/components/core/tools/scripts/lib_install/centos-stream-9/install-all.sh b/components/core/tools/scripts/lib_install/centos-stream-9/install-all.sh index a7c183456e..8ea47b541e 100755 --- a/components/core/tools/scripts/lib_install/centos-stream-9/install-all.sh +++ b/components/core/tools/scripts/lib_install/centos-stream-9/install-all.sh @@ -8,4 +8,4 @@ script_dir="$( cd "$( dirname "${BASH_SOURCE[0]}" )" &> /dev/null && pwd )" "${script_dir}/install-prebuilt-packages.sh" "${script_dir}/install-packages-from-source.sh" -"${script_dir}/../../lib_version_checks/check-build-tool-versions.sh" +"${script_dir}/../lib_version_checks/check-build-tool-versions.sh" diff --git a/components/core/tools/scripts/lib_version_checks/check-build-tool-versions.sh b/components/core/tools/scripts/lib_install/lib_version_checks/check-build-tool-versions.sh similarity index 100% rename from components/core/tools/scripts/lib_version_checks/check-build-tool-versions.sh rename to components/core/tools/scripts/lib_install/lib_version_checks/check-build-tool-versions.sh diff --git a/components/core/tools/scripts/lib_version_checks/check-cmake-version.sh b/components/core/tools/scripts/lib_install/lib_version_checks/check-cmake-version.sh similarity index 100% rename from components/core/tools/scripts/lib_version_checks/check-cmake-version.sh rename to components/core/tools/scripts/lib_install/lib_version_checks/check-cmake-version.sh diff --git a/components/core/tools/scripts/lib_version_checks/check-go-task-version.sh b/components/core/tools/scripts/lib_install/lib_version_checks/check-go-task-version.sh similarity index 100% rename from components/core/tools/scripts/lib_version_checks/check-go-task-version.sh rename to components/core/tools/scripts/lib_install/lib_version_checks/check-go-task-version.sh diff --git a/components/core/tools/scripts/lib_version_checks/check-uv-version.sh b/components/core/tools/scripts/lib_install/lib_version_checks/check-uv-version.sh similarity index 100% rename from components/core/tools/scripts/lib_version_checks/check-uv-version.sh rename to components/core/tools/scripts/lib_install/lib_version_checks/check-uv-version.sh diff --git a/components/core/tools/scripts/lib_version_checks/print-go-task-version.yaml b/components/core/tools/scripts/lib_install/lib_version_checks/print-go-task-version.yaml similarity index 100% rename from components/core/tools/scripts/lib_version_checks/print-go-task-version.yaml rename to components/core/tools/scripts/lib_install/lib_version_checks/print-go-task-version.yaml diff --git a/components/core/tools/scripts/lib_install/macos/install-all.sh b/components/core/tools/scripts/lib_install/macos/install-all.sh index 46face74c2..c3622e3811 100755 --- a/components/core/tools/scripts/lib_install/macos/install-all.sh +++ b/components/core/tools/scripts/lib_install/macos/install-all.sh @@ -44,4 +44,4 @@ if ! command -v pkg-config ; then brew install pkg-config fi -"${script_dir}/../../lib_version_checks/check-build-tool-versions.sh" +"${script_dir}/../lib_version_checks/check-build-tool-versions.sh" diff --git a/components/core/tools/scripts/lib_install/manylinux_2_28/install-all.sh b/components/core/tools/scripts/lib_install/manylinux_2_28/install-all.sh index a7c183456e..8ea47b541e 100755 --- a/components/core/tools/scripts/lib_install/manylinux_2_28/install-all.sh +++ b/components/core/tools/scripts/lib_install/manylinux_2_28/install-all.sh @@ -8,4 +8,4 @@ script_dir="$( cd "$( dirname "${BASH_SOURCE[0]}" )" &> /dev/null && pwd )" "${script_dir}/install-prebuilt-packages.sh" "${script_dir}/install-packages-from-source.sh" -"${script_dir}/../../lib_version_checks/check-build-tool-versions.sh" +"${script_dir}/../lib_version_checks/check-build-tool-versions.sh" diff --git a/components/core/tools/scripts/lib_install/musllinux_1_2/install-all.sh b/components/core/tools/scripts/lib_install/musllinux_1_2/install-all.sh index a7c183456e..8ea47b541e 100755 --- a/components/core/tools/scripts/lib_install/musllinux_1_2/install-all.sh +++ b/components/core/tools/scripts/lib_install/musllinux_1_2/install-all.sh @@ -8,4 +8,4 @@ script_dir="$( cd "$( dirname "${BASH_SOURCE[0]}" )" &> /dev/null && pwd )" "${script_dir}/install-prebuilt-packages.sh" "${script_dir}/install-packages-from-source.sh" -"${script_dir}/../../lib_version_checks/check-build-tool-versions.sh" +"${script_dir}/../lib_version_checks/check-build-tool-versions.sh" diff --git a/components/core/tools/scripts/lib_install/ubuntu-jammy/install-all.sh b/components/core/tools/scripts/lib_install/ubuntu-jammy/install-all.sh index a7c183456e..8ea47b541e 100755 --- a/components/core/tools/scripts/lib_install/ubuntu-jammy/install-all.sh +++ b/components/core/tools/scripts/lib_install/ubuntu-jammy/install-all.sh @@ -8,4 +8,4 @@ script_dir="$( cd "$( dirname "${BASH_SOURCE[0]}" )" &> /dev/null && pwd )" "${script_dir}/install-prebuilt-packages.sh" "${script_dir}/install-packages-from-source.sh" -"${script_dir}/../../lib_version_checks/check-build-tool-versions.sh" +"${script_dir}/../lib_version_checks/check-build-tool-versions.sh" From 856c06f4b5938242f5788403c83a3687fb925742 Mon Sep 17 00:00:00 2001 From: Bingran Hu Date: Fri, 29 Aug 2025 02:23:24 -0400 Subject: [PATCH 28/69] Move PIPX path setup to dockerfile --- .../docker-images/clp-env-base-centos-stream-9/Dockerfile | 3 +++ .../docker-images/clp-env-base-ubuntu-jammy/Dockerfile | 3 +++ .../centos-stream-9/install-prebuilt-packages.sh | 6 ------ .../lib_install/ubuntu-jammy/install-prebuilt-packages.sh | 5 ----- 4 files changed, 6 insertions(+), 11 deletions(-) diff --git a/components/core/tools/docker-images/clp-env-base-centos-stream-9/Dockerfile b/components/core/tools/docker-images/clp-env-base-centos-stream-9/Dockerfile index 3959d441f6..6c54690414 100644 --- a/components/core/tools/docker-images/clp-env-base-centos-stream-9/Dockerfile +++ b/components/core/tools/docker-images/clp-env-base-centos-stream-9/Dockerfile @@ -5,6 +5,9 @@ WORKDIR /root RUN mkdir -p ./tools/scripts/lib_install ADD ./tools/scripts/lib_install ./tools/scripts/lib_install +ENV PIPX_BIN_DIR=/usr/local/bin +ENV PIPX_HOME=/opt/_internal/pipx + RUN ./tools/scripts/lib_install/centos-stream-9/install-all.sh # Remove cached files diff --git a/components/core/tools/docker-images/clp-env-base-ubuntu-jammy/Dockerfile b/components/core/tools/docker-images/clp-env-base-ubuntu-jammy/Dockerfile index d0fb69004f..1bd42907df 100644 --- a/components/core/tools/docker-images/clp-env-base-ubuntu-jammy/Dockerfile +++ b/components/core/tools/docker-images/clp-env-base-ubuntu-jammy/Dockerfile @@ -5,6 +5,9 @@ WORKDIR /root RUN mkdir -p ./tools/scripts/lib_install ADD ./tools/scripts/lib_install ./tools/scripts/lib_install +ENV PIPX_BIN_DIR=/usr/local/bin +ENV PIPX_HOME=/opt/_internal/pipx + RUN ./tools/scripts/lib_install/ubuntu-jammy/install-all.sh # Remove cached files diff --git a/components/core/tools/scripts/lib_install/centos-stream-9/install-prebuilt-packages.sh b/components/core/tools/scripts/lib_install/centos-stream-9/install-prebuilt-packages.sh index 139bc317da..8f3dc3656b 100755 --- a/components/core/tools/scripts/lib_install/centos-stream-9/install-prebuilt-packages.sh +++ b/components/core/tools/scripts/lib_install/centos-stream-9/install-prebuilt-packages.sh @@ -22,12 +22,6 @@ dnf install -y \ if ! command -v pipx; then python3 -m pip install pipx fi - -if [ "$(id -u)" -eq 0 ]; then - # Running as root: install pipx softwares into system directories - export PIPX_HOME=/opt/_internal/pipx - export PIPX_BIN_DIR=/usr/local/bin -fi pipx ensurepath # Install `cmake` diff --git a/components/core/tools/scripts/lib_install/ubuntu-jammy/install-prebuilt-packages.sh b/components/core/tools/scripts/lib_install/ubuntu-jammy/install-prebuilt-packages.sh index 9349b86b9a..31931ec88a 100755 --- a/components/core/tools/scripts/lib_install/ubuntu-jammy/install-prebuilt-packages.sh +++ b/components/core/tools/scripts/lib_install/ubuntu-jammy/install-prebuilt-packages.sh @@ -25,11 +25,6 @@ DEBIAN_FRONTEND=noninteractive apt-get install --no-install-recommends -y \ software-properties-common \ unzip -if [ "$(id -u)" -eq 0 ]; then - # Running as root: install pipx softwares into system directories - export PIPX_HOME=/opt/_internal/pipx - export PIPX_BIN_DIR=/usr/local/bin -fi pipx ensurepath # Install `cmake` From e0950c10f1d2bd1fd4250fe16535f96b87454c0f Mon Sep 17 00:00:00 2001 From: Bingran Hu Date: Fri, 29 Aug 2025 02:26:27 -0400 Subject: [PATCH 29/69] Improve docs --- docs/README.md | 4 +++- docs/src/dev-docs/components-core/index.md | 4 +++- 2 files changed, 6 insertions(+), 2 deletions(-) diff --git a/docs/README.md b/docs/README.md index 92ada1132a..6913900591 100644 --- a/docs/README.md +++ b/docs/README.md @@ -14,7 +14,8 @@ this project: * [Node.js] >= 16 to be able to [view the output](#viewing-the-output) * Python 3.10 or later * [Task] >= 3.40.0 and < 3.43.0 - * We constrain the version due to unresolved [issues][clp-issue-872]. + * Minimum version 3.40.0 is required for [yscope-dev-utils]. + * We constrain the version to < 3.43.0 due to unresolved [issues][clp-issue-872]. ## Build Commands @@ -46,3 +47,4 @@ the address it binds to (usually http://localhost:8080). [http-server]: https://www.npmjs.com/package/http-server [Node.js]: https://nodejs.org/en/download/current [Task]: https://taskfile.dev/ +[yscope-dev-utils]: https://github.com/y-scope/yscope-dev-utils diff --git a/docs/src/dev-docs/components-core/index.md b/docs/src/dev-docs/components-core/index.md index 5cb964f521..5041dda17a 100644 --- a/docs/src/dev-docs/components-core/index.md +++ b/docs/src/dev-docs/components-core/index.md @@ -10,7 +10,8 @@ CLP core is the low-level component that performs compression, decompression, an * std::span * std::source_location * [Task] >= 3.40.0 and < 3.43.0 - * We constrain the version due to unresolved [issues][clp-issue-872]. + * Minimum version 3.40.0 is required for [yscope-dev-utils]. + * We constrain the version to < 3.43.0 due to unresolved [issues][clp-issue-872]. To build, we require some source dependencies, packages from package managers, and libraries built from source. @@ -122,3 +123,4 @@ regex-utils [clp-issue-872]: https://github.com/y-scope/clp/issues/872 [feature-req]: https://github.com/y-scope/clp/issues/new?assignees=&labels=enhancement&template=feature-request.yml [Task]: https://taskfile.dev/ +[yscope-dev-utils]: https://github.com/y-scope/yscope-dev-utils From b833729a11245f40327b9f366aa0df92e96b471e Mon Sep 17 00:00:00 2001 From: Bingran Hu Date: Fri, 29 Aug 2025 02:37:28 -0400 Subject: [PATCH 30/69] lint cmake --- .../lib_install/lib_version_checks/check-cmake-version.sh | 4 ++-- .../lib_version_checks/check-go-task-version.sh | 8 ++++---- .../lib_install/lib_version_checks/check-uv-version.sh | 6 +++--- 3 files changed, 9 insertions(+), 9 deletions(-) diff --git a/components/core/tools/scripts/lib_install/lib_version_checks/check-cmake-version.sh b/components/core/tools/scripts/lib_install/lib_version_checks/check-cmake-version.sh index 488821c0f7..b3524589cd 100755 --- a/components/core/tools/scripts/lib_install/lib_version_checks/check-cmake-version.sh +++ b/components/core/tools/scripts/lib_install/lib_version_checks/check-cmake-version.sh @@ -8,7 +8,7 @@ cmake_version=$(cmake -E capabilities | jq --raw-output ".version.string") cmake_major_version=$(cmake -E capabilities | jq --raw-output ".version.major") # Check if version is 4.0 or higher -if [[ "$cmake_major_version" -ge "4" ]]; then - echo "Error: CMake version $cmake_version is currently unsupported (>= 4.0)." +if [[ "${cmake_major_version}" -ge "4" ]]; then + echo "Error: CMake version ${cmake_version} is currently unsupported (>= 4.0)." exit 1 fi diff --git a/components/core/tools/scripts/lib_install/lib_version_checks/check-go-task-version.sh b/components/core/tools/scripts/lib_install/lib_version_checks/check-go-task-version.sh index 44138b280c..fcdedf2de9 100755 --- a/components/core/tools/scripts/lib_install/lib_version_checks/check-go-task-version.sh +++ b/components/core/tools/scripts/lib_install/lib_version_checks/check-go-task-version.sh @@ -7,12 +7,12 @@ script_dir="$( cd "$( dirname "${BASH_SOURCE[0]}" )" &> /dev/null && pwd )" # Get the installed go-task version string task_version=$(task --silent --taskfile ${script_dir}/print-go-task-version.yaml version) -IFS=. read task_major_version task_minor_version _ <<< "$task_version" +IFS=. read task_major_version task_minor_version _ <<< "${task_version}" # Check version constraints -if [ "$task_major_version" -ne "3" ] || \ - [ "$task_minor_version" -lt "40" ] || \ - [ "$task_minor_version" -ge "43" ]; then +if [ "${task_major_version}" -ne "3" ] || \ + [ "${task_minor_version}" -lt "40" ] || \ + [ "${task_minor_version}" -ge "43" ]; then echo "Error: Task version ${task_version} is currently unsupported (must be 3.40 <= ver < 3.43)." exit 1 fi diff --git a/components/core/tools/scripts/lib_install/lib_version_checks/check-uv-version.sh b/components/core/tools/scripts/lib_install/lib_version_checks/check-uv-version.sh index 6740c177ed..4302696fab 100755 --- a/components/core/tools/scripts/lib_install/lib_version_checks/check-uv-version.sh +++ b/components/core/tools/scripts/lib_install/lib_version_checks/check-uv-version.sh @@ -5,10 +5,10 @@ set -euo pipefail # Get the installed uv version string uv_version=$(uv self version --output-format json | jq --raw-output ".version") -IFS=. read uv_major_version uv_minor_version _ <<< "$uv_version" +IFS=. read uv_major_version uv_minor_version _ <<< "${uv_version}" # Check if version is 4.0 or higher -if [ "$uv_major_version" -lt "1" ] && [ "$uv_minor_version" -lt "8" ]; then - echo "Error: uv version $uv_version is currently unsupported (< 0.8)." +if [ "${uv_major_version}" -lt "1" ] && [ "${uv_minor_version}" -lt "8" ]; then + echo "Error: uv version ${uv_version} is currently unsupported (< 0.8)." exit 1 fi From 778fdac1384cbd16ccacf1d22264d8e68d05b86e Mon Sep 17 00:00:00 2001 From: Bingran Hu Date: Fri, 29 Aug 2025 02:42:11 -0400 Subject: [PATCH 31/69] lock to cmake 3.23 instead of 3.31 --- .../lib_install/centos-stream-9/install-prebuilt-packages.sh | 2 +- components/core/tools/scripts/lib_install/macos/install-all.sh | 2 +- .../lib_install/manylinux_2_28/install-prebuilt-packages.sh | 2 +- .../lib_install/musllinux_1_2/install-prebuilt-packages.sh | 2 +- .../lib_install/ubuntu-jammy/install-prebuilt-packages.sh | 2 +- 5 files changed, 5 insertions(+), 5 deletions(-) diff --git a/components/core/tools/scripts/lib_install/centos-stream-9/install-prebuilt-packages.sh b/components/core/tools/scripts/lib_install/centos-stream-9/install-prebuilt-packages.sh index 8f3dc3656b..2a56b89ba1 100755 --- a/components/core/tools/scripts/lib_install/centos-stream-9/install-prebuilt-packages.sh +++ b/components/core/tools/scripts/lib_install/centos-stream-9/install-prebuilt-packages.sh @@ -28,7 +28,7 @@ pipx ensurepath # ystdlib requires CMake v3.23; ANTLR and yaml-cpp do not yet support CMake v4+. # See also: https://github.com/y-scope/clp/issues/795 if ! command -v cmake ; then - pipx install "cmake~=3.23" + pipx install "cmake>=3.23,<3.24" fi # Install `task` diff --git a/components/core/tools/scripts/lib_install/macos/install-all.sh b/components/core/tools/scripts/lib_install/macos/install-all.sh index c3622e3811..7933864aa7 100755 --- a/components/core/tools/scripts/lib_install/macos/install-all.sh +++ b/components/core/tools/scripts/lib_install/macos/install-all.sh @@ -12,7 +12,7 @@ brew update if command -v cmake ; then brew uninstall --force cmake fi -pipx install "cmake~=3.31" +pipx install "cmake>=3.23,<3.24" # Install a version of `task` < 3.43 to avoid https://github.com/y-scope/clp/issues/872 if command -v task ; then diff --git a/components/core/tools/scripts/lib_install/manylinux_2_28/install-prebuilt-packages.sh b/components/core/tools/scripts/lib_install/manylinux_2_28/install-prebuilt-packages.sh index 19b5d73116..9f53f85a4d 100755 --- a/components/core/tools/scripts/lib_install/manylinux_2_28/install-prebuilt-packages.sh +++ b/components/core/tools/scripts/lib_install/manylinux_2_28/install-prebuilt-packages.sh @@ -18,7 +18,7 @@ dnf install -y \ # See also: https://github.com/y-scope/clp/issues/795 pipx uninstall cmake || true if ! command -v cmake ; then - pipx install "cmake~=3.23" + pipx install "cmake>=3.23,<3.24" fi # Install `task` diff --git a/components/core/tools/scripts/lib_install/musllinux_1_2/install-prebuilt-packages.sh b/components/core/tools/scripts/lib_install/musllinux_1_2/install-prebuilt-packages.sh index b5fa25b085..ccab782c15 100755 --- a/components/core/tools/scripts/lib_install/musllinux_1_2/install-prebuilt-packages.sh +++ b/components/core/tools/scripts/lib_install/musllinux_1_2/install-prebuilt-packages.sh @@ -19,7 +19,7 @@ apk update && apk add --no-cache \ # See also: https://github.com/y-scope/clp/issues/795 pipx uninstall cmake || true if ! command -v cmake ; then - pipx install "cmake~=3.23" + pipx install "cmake>=3.23,<3.24" fi # Install `task` diff --git a/components/core/tools/scripts/lib_install/ubuntu-jammy/install-prebuilt-packages.sh b/components/core/tools/scripts/lib_install/ubuntu-jammy/install-prebuilt-packages.sh index 31931ec88a..fcf3e85ec8 100755 --- a/components/core/tools/scripts/lib_install/ubuntu-jammy/install-prebuilt-packages.sh +++ b/components/core/tools/scripts/lib_install/ubuntu-jammy/install-prebuilt-packages.sh @@ -31,7 +31,7 @@ pipx ensurepath # ystdlib requires CMake v3.23; ANTLR and yaml-cpp do not yet support CMake v4+. # See also: https://github.com/y-scope/clp/issues/795 if ! command -v cmake ; then - pipx install "cmake~=3.23" + pipx install "cmake>=3.23,<3.24" fi # Install `task` From 26a3a1094c779f8ba1c9b2bd06653d52c622c394 Mon Sep 17 00:00:00 2001 From: Bingran Hu Date: Fri, 29 Aug 2025 02:57:07 -0400 Subject: [PATCH 32/69] Update docs --- docs/src/dev-docs/components-core/index.md | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/docs/src/dev-docs/components-core/index.md b/docs/src/dev-docs/components-core/index.md index 5041dda17a..fc57f5eda0 100644 --- a/docs/src/dev-docs/components-core/index.md +++ b/docs/src/dev-docs/components-core/index.md @@ -9,6 +9,9 @@ CLP core is the low-level component that performs compression, decompression, an * A recent compiler that fully supports C++20 features such as * std::span * std::source_location +* [CMake] >= 3.23.0 and < 4.0.0 + * Minimum version 3.23.0 is required for [yscope-dev-utils]. + * We constrain the version to < 4.0.0 due to unresolved [issues][clp-issue-795]. * [Task] >= 3.40.0 and < 3.43.0 * Minimum version 3.40.0 is required for [yscope-dev-utils]. * We constrain the version to < 3.43.0 due to unresolved [issues][clp-issue-872]. @@ -120,6 +123,8 @@ ubuntu-jammy-deps-install regex-utils ::: +[CMake]: https://cmake.org/ +[clp-issue-795]: https://github.com/y-scope/clp/issues/795 [clp-issue-872]: https://github.com/y-scope/clp/issues/872 [feature-req]: https://github.com/y-scope/clp/issues/new?assignees=&labels=enhancement&template=feature-request.yml [Task]: https://taskfile.dev/ From 8fec8babe18c4bced4832109bfbe5c0b8be68858 Mon Sep 17 00:00:00 2001 From: Bingran Hu Date: Fri, 29 Aug 2025 03:51:43 -0400 Subject: [PATCH 33/69] Address coderabbitai review --- .../install-prebuilt-packages.sh | 8 ++-- .../scripts/lib_install/install-cmake.sh | 47 ------------------- .../check-build-tool-versions.sh | 17 +++---- .../check-go-task-version.sh | 4 +- .../lib_version_checks/check-uv-version.sh | 4 +- .../print-go-task-version.yaml | 2 +- .../scripts/lib_install/macos/install-all.sh | 40 ++++++++-------- .../install-prebuilt-packages.sh | 6 +-- .../install-prebuilt-packages.sh | 6 +-- .../ubuntu-jammy/install-prebuilt-packages.sh | 6 +-- 10 files changed, 47 insertions(+), 93 deletions(-) delete mode 100755 components/core/tools/scripts/lib_install/install-cmake.sh diff --git a/components/core/tools/scripts/lib_install/centos-stream-9/install-prebuilt-packages.sh b/components/core/tools/scripts/lib_install/centos-stream-9/install-prebuilt-packages.sh index 2a56b89ba1..bdfdaf8030 100755 --- a/components/core/tools/scripts/lib_install/centos-stream-9/install-prebuilt-packages.sh +++ b/components/core/tools/scripts/lib_install/centos-stream-9/install-prebuilt-packages.sh @@ -19,7 +19,7 @@ dnf install -y \ python3-pip \ unzip -if ! command -v pipx; then +if ! command -v pipx >/dev/null 2>&1; then python3 -m pip install pipx fi pipx ensurepath @@ -27,17 +27,17 @@ pipx ensurepath # Install `cmake` # ystdlib requires CMake v3.23; ANTLR and yaml-cpp do not yet support CMake v4+. # See also: https://github.com/y-scope/clp/issues/795 -if ! command -v cmake ; then +if ! command -v cmake >/dev/null 2>&1; then pipx install "cmake>=3.23,<3.24" fi # Install `task` # NOTE: We lock `task` to a version < 3.43 to avoid https://github.com/y-scope/clp/issues/872 -if ! command -v task ; then +if ! command -v task >/dev/null 2>&1; then pipx install "go-task-bin>=3.40,<3.43" fi # Install `uv` -if ! command -v uv ; then +if ! command -v uv >/dev/null 2>&1; then pipx install "uv>=0.8" fi diff --git a/components/core/tools/scripts/lib_install/install-cmake.sh b/components/core/tools/scripts/lib_install/install-cmake.sh deleted file mode 100755 index 140feabf1e..0000000000 --- a/components/core/tools/scripts/lib_install/install-cmake.sh +++ /dev/null @@ -1,47 +0,0 @@ -#!/usr/bin/env bash - -# Exit on error -set -e - -cUsage="Usage: ${BASH_SOURCE[0]} " -if [ "$#" -lt 1 ] ; then - echo $cUsage - exit -fi -version=$1 - -echo "Checking for elevated privileges..." -if [ ${EUID:-$(id -u)} -ne 0 ] ; then - sudo echo "Script can elevate privileges." -fi - -# Get number of cpu cores -num_cpus=$(grep -c ^processor /proc/cpuinfo) - -package_name=cmake - -# Create temp dir for installation -temp_dir=/tmp/${package_name}-installation -mkdir -p $temp_dir - -cd $temp_dir - -# Download source -tar_filename=cmake-${version}.tar.gz -curl -fsSL https://github.com/Kitware/CMake/releases/download/v${version}/${tar_filename} -o ${tar_filename} -tar xzf ${tar_filename} -cd cmake-${version} - -# Build -./bootstrap -make -j${num_cpus} - -# Install -if [ ${EUID:-$(id -u)} -ne 0 ] ; then - sudo make install -else - make install -fi - -# Clean up -rm -rf $temp_dir diff --git a/components/core/tools/scripts/lib_install/lib_version_checks/check-build-tool-versions.sh b/components/core/tools/scripts/lib_install/lib_version_checks/check-build-tool-versions.sh index f647c24c36..24d5b9d496 100755 --- a/components/core/tools/scripts/lib_install/lib_version_checks/check-build-tool-versions.sh +++ b/components/core/tools/scripts/lib_install/lib_version_checks/check-build-tool-versions.sh @@ -8,14 +8,15 @@ script_dir="$( cd "$( dirname "${BASH_SOURCE[0]}" )" &> /dev/null && pwd )" failures=0 run_check() { - local script="$1" - local desc="$2" + local script_name="$1" + local label="$2" + local script_output - if output=$("$script_dir/$script" 2>&1); then - echo "[OK] $desc" + if script_output=$("${script_dir}/${script_name}" 2>&1); then + echo "[OK] ${label}" else - echo "[FAIL] $desc" - echo "$output" + echo "[FAIL] ${label}" + echo "${script_output}" failures=$((failures + 1)) fi } @@ -31,8 +32,8 @@ run_check check-uv-version.sh "uv version" # Print summary echo -if [ "$failures" -gt 0 ]; then - echo "$failures check(s) failed." +if [ "${failures}" -gt 0 ]; then + echo "${failures} check(s) failed." exit 1 else echo "All checks passed." diff --git a/components/core/tools/scripts/lib_install/lib_version_checks/check-go-task-version.sh b/components/core/tools/scripts/lib_install/lib_version_checks/check-go-task-version.sh index fcdedf2de9..35ef32232b 100755 --- a/components/core/tools/scripts/lib_install/lib_version_checks/check-go-task-version.sh +++ b/components/core/tools/scripts/lib_install/lib_version_checks/check-go-task-version.sh @@ -6,8 +6,8 @@ set -euo pipefail script_dir="$( cd "$( dirname "${BASH_SOURCE[0]}" )" &> /dev/null && pwd )" # Get the installed go-task version string -task_version=$(task --silent --taskfile ${script_dir}/print-go-task-version.yaml version) -IFS=. read task_major_version task_minor_version _ <<< "${task_version}" +task_version=$(task --silent --taskfile "${script_dir}/print-go-task-version.yaml" version) +IFS=. read -r task_major_version task_minor_version _ <<< "${task_version}" # Check version constraints if [ "${task_major_version}" -ne "3" ] || \ diff --git a/components/core/tools/scripts/lib_install/lib_version_checks/check-uv-version.sh b/components/core/tools/scripts/lib_install/lib_version_checks/check-uv-version.sh index 4302696fab..3b66c92039 100755 --- a/components/core/tools/scripts/lib_install/lib_version_checks/check-uv-version.sh +++ b/components/core/tools/scripts/lib_install/lib_version_checks/check-uv-version.sh @@ -5,9 +5,9 @@ set -euo pipefail # Get the installed uv version string uv_version=$(uv self version --output-format json | jq --raw-output ".version") -IFS=. read uv_major_version uv_minor_version _ <<< "${uv_version}" +IFS=. read -r uv_major_version uv_minor_version _ <<< "${uv_version}" -# Check if version is 4.0 or higher +# Check version constraints if [ "${uv_major_version}" -lt "1" ] && [ "${uv_minor_version}" -lt "8" ]; then echo "Error: uv version ${uv_version} is currently unsupported (< 0.8)." exit 1 diff --git a/components/core/tools/scripts/lib_install/lib_version_checks/print-go-task-version.yaml b/components/core/tools/scripts/lib_install/lib_version_checks/print-go-task-version.yaml index bebb85d2ff..36bf9cbe2d 100644 --- a/components/core/tools/scripts/lib_install/lib_version_checks/print-go-task-version.yaml +++ b/components/core/tools/scripts/lib_install/lib_version_checks/print-go-task-version.yaml @@ -3,4 +3,4 @@ version: "3" tasks: version: cmd: >- - echo {{.TASK_VERSION}} | sed "s/^v//" + echo {{ .TASK_VERSION | trimPrefix "v" }} diff --git a/components/core/tools/scripts/lib_install/macos/install-all.sh b/components/core/tools/scripts/lib_install/macos/install-all.sh index 7933864aa7..c63edfb2f7 100755 --- a/components/core/tools/scripts/lib_install/macos/install-all.sh +++ b/components/core/tools/scripts/lib_install/macos/install-all.sh @@ -6,25 +6,6 @@ set -euo pipefail script_dir="$( cd "$( dirname "${BASH_SOURCE[0]}" )" &> /dev/null && pwd )" brew update - -# Install CMake v3.31.6 as ANTLR and yaml-cpp do not yet support CMake v4+. -# See also: https://github.com/y-scope/clp/issues/795 -if command -v cmake ; then - brew uninstall --force cmake -fi -pipx install "cmake>=3.23,<3.24" - -# Install a version of `task` < 3.43 to avoid https://github.com/y-scope/clp/issues/872 -if command -v task ; then - brew uninstall --force task -fi -pipx install "go-task-bin>=3.40,<3.43" - -# Install uv -if ! command -v uv ; then - pipx install "uv>=0.8" -fi - brew install \ coreutils \ gcc \ @@ -33,6 +14,7 @@ brew install \ llvm@16 \ lz4 \ mariadb-connector-c \ + pipx \ xz \ zstd @@ -40,8 +22,26 @@ brew install \ # NOTE: We might expect that pkg-config is installed through brew, so trying to install it again # would be harmless; however, in certain environments, like the macOS GitHub hosted runner, # pkg-config is installed by other means, meaning a brew install would cause conflicts. -if ! command -v pkg-config ; then +if ! command -v pkg-config >/dev/null 2>&1; then brew install pkg-config fi +# Install CMake v3.31.6 as ANTLR and yaml-cpp do not yet support CMake v4+. +# See also: https://github.com/y-scope/clp/issues/795 +if command -v cmake >/dev/null 2>&1; then + brew uninstall --force cmake +fi +pipx install "cmake>=3.23,<3.24" + +# Install a version of `task` < 3.43 to avoid https://github.com/y-scope/clp/issues/872 +if command -v task >/dev/null 2>&1; then + brew uninstall --force task +fi +pipx install "go-task-bin>=3.40,<3.43" + +# Install uv +if ! command -v uv >/dev/null 2>&1; then + pipx install "uv>=0.8" +fi + "${script_dir}/../lib_version_checks/check-build-tool-versions.sh" diff --git a/components/core/tools/scripts/lib_install/manylinux_2_28/install-prebuilt-packages.sh b/components/core/tools/scripts/lib_install/manylinux_2_28/install-prebuilt-packages.sh index 9f53f85a4d..68167048ce 100755 --- a/components/core/tools/scripts/lib_install/manylinux_2_28/install-prebuilt-packages.sh +++ b/components/core/tools/scripts/lib_install/manylinux_2_28/install-prebuilt-packages.sh @@ -17,19 +17,19 @@ dnf install -y \ # ystdlib requires CMake v3.23; ANTLR and yaml-cpp do not yet support CMake v4+. # See also: https://github.com/y-scope/clp/issues/795 pipx uninstall cmake || true -if ! command -v cmake ; then +if ! command -v cmake >/dev/null 2>&1; then pipx install "cmake>=3.23,<3.24" fi # Install `task` # NOTE: We lock `task` to a version < 3.43 to avoid https://github.com/y-scope/clp/issues/872 pipx uninstall go-task-bin || true -if ! command -v task ; then +if ! command -v task >/dev/null 2>&1; then pipx install "go-task-bin>=3.40,<3.43" fi # Install `uv` pipx uninstall uv || true -if ! command -v uv ; then +if ! command -v uv >/dev/null 2>&1; then pipx install "uv>=0.8" fi diff --git a/components/core/tools/scripts/lib_install/musllinux_1_2/install-prebuilt-packages.sh b/components/core/tools/scripts/lib_install/musllinux_1_2/install-prebuilt-packages.sh index ccab782c15..a8c285e6ee 100755 --- a/components/core/tools/scripts/lib_install/musllinux_1_2/install-prebuilt-packages.sh +++ b/components/core/tools/scripts/lib_install/musllinux_1_2/install-prebuilt-packages.sh @@ -18,19 +18,19 @@ apk update && apk add --no-cache \ # ystdlib requires CMake v3.23; ANTLR and yaml-cpp do not yet support CMake v4+. # See also: https://github.com/y-scope/clp/issues/795 pipx uninstall cmake || true -if ! command -v cmake ; then +if ! command -v cmake >/dev/null 2>&1; then pipx install "cmake>=3.23,<3.24" fi # Install `task` # NOTE: We lock `task` to a version < 3.43 to avoid https://github.com/y-scope/clp/issues/872 pipx uninstall go-task-bin || true -if ! command -v task ; then +if ! command -v task >/dev/null 2>&1; then pipx install "go-task-bin>=3.40,<3.43" fi # Install `uv` pipx uninstall uv || true -if ! command -v uv ; then +if ! command -v uv >/dev/null 2>&1; then pipx install "uv>=0.8" fi diff --git a/components/core/tools/scripts/lib_install/ubuntu-jammy/install-prebuilt-packages.sh b/components/core/tools/scripts/lib_install/ubuntu-jammy/install-prebuilt-packages.sh index fcf3e85ec8..1cb3e6a3a0 100755 --- a/components/core/tools/scripts/lib_install/ubuntu-jammy/install-prebuilt-packages.sh +++ b/components/core/tools/scripts/lib_install/ubuntu-jammy/install-prebuilt-packages.sh @@ -30,17 +30,17 @@ pipx ensurepath # Install `cmake` # ystdlib requires CMake v3.23; ANTLR and yaml-cpp do not yet support CMake v4+. # See also: https://github.com/y-scope/clp/issues/795 -if ! command -v cmake ; then +if ! command -v cmake >/dev/null 2>&1; then pipx install "cmake>=3.23,<3.24" fi # Install `task` # NOTE: We lock `task` to a version < 3.43 to avoid https://github.com/y-scope/clp/issues/872 -if ! command -v task ; then +if ! command -v task >/dev/null 2>&1; then pipx install "go-task-bin>=3.40,<3.43" fi # Install `uv` -if ! command -v uv ; then +if ! command -v uv >/dev/null 2>&1; then pipx install "uv>=0.8" fi From bbf21fc0f15fa895378810d167c0c04a4d80fa11 Mon Sep 17 00:00:00 2001 From: Bingran Hu Date: Sun, 31 Aug 2025 08:55:05 -0400 Subject: [PATCH 34/69] Fix comment --- components/core/tools/scripts/lib_install/macos/install-all.sh | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/components/core/tools/scripts/lib_install/macos/install-all.sh b/components/core/tools/scripts/lib_install/macos/install-all.sh index c63edfb2f7..bb99fd77e8 100755 --- a/components/core/tools/scripts/lib_install/macos/install-all.sh +++ b/components/core/tools/scripts/lib_install/macos/install-all.sh @@ -26,7 +26,8 @@ if ! command -v pkg-config >/dev/null 2>&1; then brew install pkg-config fi -# Install CMake v3.31.6 as ANTLR and yaml-cpp do not yet support CMake v4+. +# Install `cmake` +# ystdlib requires CMake v3.23; ANTLR and yaml-cpp do not yet support CMake v4+. # See also: https://github.com/y-scope/clp/issues/795 if command -v cmake >/dev/null 2>&1; then brew uninstall --force cmake From 8601b81a0556333b005906f0d9aabe68d1f4e758 Mon Sep 17 00:00:00 2001 From: Bingran Hu Date: Sun, 31 Aug 2025 10:27:43 -0400 Subject: [PATCH 35/69] change deps core to sequential --- taskfiles/deps/main.yaml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/taskfiles/deps/main.yaml b/taskfiles/deps/main.yaml index 8eaf638d52..501c4c2a2b 100644 --- a/taskfiles/deps/main.yaml +++ b/taskfiles/deps/main.yaml @@ -64,7 +64,7 @@ tasks: core-all-parallel: internal: true run: "once" - deps: + cmds: - task: "absl" - task: "antlr-jar" - task: "antlr-runtime" From 330c0aea5715c08d1cfb9a5ae47cc97b0f7c3cea Mon Sep 17 00:00:00 2001 From: Bingran Hu Date: Sun, 31 Aug 2025 13:49:51 -0400 Subject: [PATCH 36/69] Bump to 4.1.1 --- docs/src/dev-docs/components-core/index.md | 2 +- taskfiles/deps/main.yaml | 7 +++---- 2 files changed, 4 insertions(+), 5 deletions(-) diff --git a/docs/src/dev-docs/components-core/index.md b/docs/src/dev-docs/components-core/index.md index 5cb964f521..7a5bb96c86 100644 --- a/docs/src/dev-docs/components-core/index.md +++ b/docs/src/dev-docs/components-core/index.md @@ -45,7 +45,7 @@ The task will download, build, and install (within the build directory) the foll | [log-surgeon](https://github.com/y-scope/log-surgeon) | a82ad13 | | [lz4](https://github.com/lz4/lz4) | v1.10.0 | | [microsoft.gsl](https://github.com/microsoft/GSL) | v4.0.0 | -| [mongo-cxx-driver](https://github.com/mongodb/mongo-cxx-driver) | r3.10.2 | +| [mongo-cxx-driver](https://github.com/mongodb/mongo-cxx-driver) | r4.1.1 | | [msgpack-cxx](https://github.com/msgpack/msgpack-c/tree/cpp_master) | v7.0.0 | | [nlohmann_json](https://github.com/nlohmann/json) | v3.11.3 | | [simdjson](https://github.com/simdjson/simdjson) | v3.13.0 | diff --git a/taskfiles/deps/main.yaml b/taskfiles/deps/main.yaml index 8eaf638d52..b3ea5ac1b7 100644 --- a/taskfiles/deps/main.yaml +++ b/taskfiles/deps/main.yaml @@ -352,9 +352,8 @@ tasks: TARBALL_URL: "https://github.com/microsoft/GSL/archive/refs/tags/v4.0.0.tar.gz" mongocxx: - internal: true vars: - VERSION: "r3.10.2" + VERSION: "r4.1.1" run: "once" cmds: - task: "utils:install-remote-cmake-lib" @@ -364,10 +363,10 @@ tasks: - "-DBUILD_SHARED_LIBS_WITH_STATIC_MONGOC=ON" - "-DCMAKE_BUILD_TYPE=Release" - "-DCMAKE_INSTALL_MESSAGE=LAZY" - - "-DENABLE_TESTS=OFF" + - "-DENABLE_EXAMPLES=OFF" - "-DENABLE_UNINSTALL=OFF" LIB_NAME: "mongocxx" - TARBALL_SHA256: "52b99b2866019b5ea25d15c5a39e2a88c70fe1259c40f1091deff8bfae0194be" + TARBALL_SHA256: "19dff3cf834a3e09229260f22a0325820a7e30c78b294db91794dd934776b33a" TARBALL_URL: "https://github.com/mongodb/mongo-cxx-driver/releases/download/{{.VERSION}}\ /mongo-cxx-driver-{{.VERSION}}.tar.gz" From a09f934ed6811cf9ee367ce1b2d441103e4b8e44 Mon Sep 17 00:00:00 2001 From: Bingran Hu Date: Sun, 31 Aug 2025 13:59:47 -0400 Subject: [PATCH 37/69] change deps core to sequential --- taskfiles/deps/main.yaml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/taskfiles/deps/main.yaml b/taskfiles/deps/main.yaml index b3ea5ac1b7..7c6dea8abc 100644 --- a/taskfiles/deps/main.yaml +++ b/taskfiles/deps/main.yaml @@ -64,7 +64,7 @@ tasks: core-all-parallel: internal: true run: "once" - deps: + cmds: - task: "absl" - task: "antlr-jar" - task: "antlr-runtime" From 0e946749ab729df086baea99f3cc45ffa34b70f5 Mon Sep 17 00:00:00 2001 From: Bingran Hu Date: Mon, 1 Sep 2025 10:31:41 +0800 Subject: [PATCH 38/69] remove using simdjson --- components/core/src/clp_s/JsonParser.hpp | 2 -- components/core/src/clp_s/search/QueryRunner.hpp | 2 -- 2 files changed, 4 deletions(-) diff --git a/components/core/src/clp_s/JsonParser.hpp b/components/core/src/clp_s/JsonParser.hpp index a5718a1a57..c69af68fd1 100644 --- a/components/core/src/clp_s/JsonParser.hpp +++ b/components/core/src/clp_s/JsonParser.hpp @@ -31,8 +31,6 @@ #include "Utils.hpp" #include "ZstdCompressor.hpp" -using namespace simdjson; - namespace clp_s { struct JsonParserOption { std::vector input_paths; diff --git a/components/core/src/clp_s/search/QueryRunner.hpp b/components/core/src/clp_s/search/QueryRunner.hpp index 6c37142bd9..5de87954ec 100644 --- a/components/core/src/clp_s/search/QueryRunner.hpp +++ b/components/core/src/clp_s/search/QueryRunner.hpp @@ -32,8 +32,6 @@ #include "ast/Literal.hpp" #include "SchemaMatch.hpp" -using namespace simdjson; - namespace clp_s::search { /** * This class is a core component of the log search system responsible for executing parsed queries From 5c49e2b6ab69bd13e1a55d9b5776a0011559c6f8 Mon Sep 17 00:00:00 2001 From: Bingran Hu Date: Mon, 1 Sep 2025 11:27:53 +0800 Subject: [PATCH 39/69] resolve simdjson issues --- components/core/src/clp_s/JsonParser.cpp | 74 ++++++++++--------- components/core/src/clp_s/JsonParser.hpp | 6 +- .../core/src/clp_s/search/QueryRunner.cpp | 74 +++++++++---------- .../core/src/clp_s/search/QueryRunner.hpp | 10 +-- 4 files changed, 84 insertions(+), 80 deletions(-) diff --git a/components/core/src/clp_s/JsonParser.cpp b/components/core/src/clp_s/JsonParser.cpp index 732fac7687..3613f349fa 100644 --- a/components/core/src/clp_s/JsonParser.cpp +++ b/components/core/src/clp_s/JsonParser.cpp @@ -137,15 +137,15 @@ JsonParser::JsonParser(JsonParserOption const& option) m_archive_writer->open(m_archive_options); } -void JsonParser::parse_obj_in_array(ondemand::object line, int32_t parent_node_id) { - ondemand::object_iterator it = line.begin(); +void JsonParser::parse_obj_in_array(simdjson::ondemand::object line, int32_t parent_node_id) { + simdjson::ondemand::object_iterator it = line.begin(); if (it == line.end()) { m_current_schema.insert_unordered(parent_node_id); return; } - std::stack object_stack; - std::stack object_it_stack; + std::stack object_stack; + std::stack object_it_stack; std::stack node_id_stack; node_id_stack.push(parent_node_id); @@ -153,8 +153,8 @@ void JsonParser::parse_obj_in_array(ondemand::object line, int32_t parent_node_i object_it_stack.push(std::move(it)); size_t object_start = m_current_schema.start_unordered_object(NodeType::Object); - ondemand::field cur_field; - ondemand::value cur_value; + simdjson::ondemand::field cur_field; + simdjson::ondemand::value cur_value; std::string_view cur_key; int32_t node_id; while (true) { @@ -177,7 +177,7 @@ void JsonParser::parse_obj_in_array(ondemand::object line, int32_t parent_node_i cur_value = cur_field.value(); switch (cur_value.type()) { - case ondemand::json_type::object: { + case simdjson::ondemand::json_type::object: { node_id = m_archive_writer ->add_node(node_id_stack.top(), NodeType::Object, cur_key); object_stack.push(std::move(cur_value.get_object())); @@ -192,7 +192,7 @@ void JsonParser::parse_obj_in_array(ondemand::object line, int32_t parent_node_i } break; } - case ondemand::json_type::array: { + case simdjson::ondemand::json_type::array: { node_id = m_archive_writer->add_node( node_id_stack.top(), NodeType::StructuredArray, @@ -201,8 +201,8 @@ void JsonParser::parse_obj_in_array(ondemand::object line, int32_t parent_node_i parse_array(cur_value.get_array(), node_id); break; } - case ondemand::json_type::number: { - ondemand::number number_value = cur_value.get_number(); + case simdjson::ondemand::json_type::number: { + simdjson::ondemand::number number_value = cur_value.get_number(); if (true == number_value.is_double()) { double double_value = number_value.get_double(); m_current_parsed_message.add_unordered_value(double_value); @@ -222,7 +222,7 @@ void JsonParser::parse_obj_in_array(ondemand::object line, int32_t parent_node_i m_current_schema.insert_unordered(node_id); break; } - case ondemand::json_type::string: { + case simdjson::ondemand::json_type::string: { std::string_view value = cur_value.get_string(true); if (value.find(' ') != std::string::npos) { node_id = m_archive_writer @@ -235,7 +235,7 @@ void JsonParser::parse_obj_in_array(ondemand::object line, int32_t parent_node_i m_current_schema.insert_unordered(node_id); break; } - case ondemand::json_type::boolean: { + case simdjson::ondemand::json_type::boolean: { bool value = cur_value.get_bool(); m_current_parsed_message.add_unordered_value(value); node_id = m_archive_writer @@ -243,7 +243,7 @@ void JsonParser::parse_obj_in_array(ondemand::object line, int32_t parent_node_i m_current_schema.insert_unordered(node_id); break; } - case ondemand::json_type::null: { + case simdjson::ondemand::json_type::null: { node_id = m_archive_writer ->add_node(node_id_stack.top(), NodeType::NullValue, cur_key); m_current_schema.insert_unordered(node_id); @@ -255,32 +255,32 @@ void JsonParser::parse_obj_in_array(ondemand::object line, int32_t parent_node_i } } -void JsonParser::parse_array(ondemand::array array, int32_t parent_node_id) { - ondemand::array_iterator it = array.begin(); +void JsonParser::parse_array(simdjson::ondemand::array array, int32_t parent_node_id) { + simdjson::ondemand::array_iterator it = array.begin(); if (it == array.end()) { m_current_schema.insert_unordered(parent_node_id); return; } size_t array_start = m_current_schema.start_unordered_object(NodeType::StructuredArray); - ondemand::value cur_value; + simdjson::ondemand::value cur_value; int32_t node_id; for (; it != array.end(); ++it) { cur_value = *it; switch (cur_value.type()) { - case ondemand::json_type::object: { + case simdjson::ondemand::json_type::object: { node_id = m_archive_writer->add_node(parent_node_id, NodeType::Object, ""); parse_obj_in_array(std::move(cur_value.get_object()), node_id); break; } - case ondemand::json_type::array: { + case simdjson::ondemand::json_type::array: { node_id = m_archive_writer->add_node(parent_node_id, NodeType::StructuredArray, ""); parse_array(std::move(cur_value.get_array()), node_id); break; } - case ondemand::json_type::number: { - ondemand::number number_value = cur_value.get_number(); + case simdjson::ondemand::json_type::number: { + simdjson::ondemand::number number_value = cur_value.get_number(); if (true == number_value.is_double()) { double double_value = number_value.get_double(); m_current_parsed_message.add_unordered_value(double_value); @@ -298,7 +298,7 @@ void JsonParser::parse_array(ondemand::array array, int32_t parent_node_id) { m_current_schema.insert_unordered(node_id); break; } - case ondemand::json_type::string: { + case simdjson::ondemand::json_type::string: { std::string_view value = cur_value.get_string(true); if (value.find(' ') != std::string::npos) { node_id = m_archive_writer->add_node(parent_node_id, NodeType::ClpString, ""); @@ -309,14 +309,14 @@ void JsonParser::parse_array(ondemand::array array, int32_t parent_node_id) { m_current_schema.insert_unordered(node_id); break; } - case ondemand::json_type::boolean: { + case simdjson::ondemand::json_type::boolean: { bool value = cur_value.get_bool(); m_current_parsed_message.add_unordered_value(value); node_id = m_archive_writer->add_node(parent_node_id, NodeType::Boolean, ""); m_current_schema.insert_unordered(node_id); break; } - case ondemand::json_type::null: { + case simdjson::ondemand::json_type::null: { node_id = m_archive_writer->add_node(parent_node_id, NodeType::NullValue, ""); m_current_schema.insert_unordered(node_id); break; @@ -326,13 +326,17 @@ void JsonParser::parse_array(ondemand::array array, int32_t parent_node_id) { m_current_schema.end_unordered_object(array_start); } -void JsonParser::parse_line(ondemand::value line, int32_t parent_node_id, std::string const& key) { +void JsonParser::parse_line( + simdjson::ondemand::value line, + int32_t parent_node_id, + std::string const& key +) { int32_t node_id; - std::stack object_stack; + std::stack object_stack; std::stack node_id_stack; - std::stack object_it_stack; + std::stack object_it_stack; - ondemand::field cur_field; + simdjson::ondemand::field cur_field; std::string_view cur_key = key; node_id_stack.push(parent_node_id); @@ -345,12 +349,12 @@ void JsonParser::parse_line(ondemand::value line, int32_t parent_node_id, std::s } switch (line.type()) { - case ondemand::json_type::object: { + case simdjson::ondemand::json_type::object: { node_id = m_archive_writer ->add_node(node_id_stack.top(), NodeType::Object, cur_key); object_stack.push(std::move(line.get_object())); auto objref = object_stack.top(); - auto it = ondemand::object_iterator(objref.begin()); + auto it = simdjson::ondemand::object_iterator(objref.begin()); if (it == objref.end()) { m_current_schema.insert_ordered(node_id); object_stack.pop(); @@ -361,7 +365,7 @@ void JsonParser::parse_line(ondemand::value line, int32_t parent_node_id, std::s continue; } } - case ondemand::json_type::array: { + case simdjson::ondemand::json_type::array: { if (m_structurize_arrays) { node_id = m_archive_writer->add_node( node_id_stack.top(), @@ -382,9 +386,9 @@ void JsonParser::parse_line(ondemand::value line, int32_t parent_node_id, std::s } break; } - case ondemand::json_type::number: { + case simdjson::ondemand::json_type::number: { NodeType type; - ondemand::number number_value = line.get_number(); + simdjson::ondemand::number number_value = line.get_number(); auto const matches_timestamp = m_archive_writer->matches_timestamp(node_id_stack.top(), cur_key); if (false == number_value.is_double()) { @@ -420,7 +424,7 @@ void JsonParser::parse_line(ondemand::value line, int32_t parent_node_id, std::s m_current_schema.insert_ordered(node_id); break; } - case ondemand::json_type::string: { + case simdjson::ondemand::json_type::string: { std::string_view value = line.get_string(true); auto const matches_timestamp = m_archive_writer->matches_timestamp(node_id_stack.top(), cur_key); @@ -451,7 +455,7 @@ void JsonParser::parse_line(ondemand::value line, int32_t parent_node_id, std::s m_current_schema.insert_ordered(node_id); break; } - case ondemand::json_type::boolean: { + case simdjson::ondemand::json_type::boolean: { bool value = line.get_bool(); node_id = m_archive_writer ->add_node(node_id_stack.top(), NodeType::Boolean, cur_key); @@ -459,7 +463,7 @@ void JsonParser::parse_line(ondemand::value line, int32_t parent_node_id, std::s m_current_schema.insert_ordered(node_id); break; } - case ondemand::json_type::null: { + case simdjson::ondemand::json_type::null: { node_id = m_archive_writer ->add_node(node_id_stack.top(), NodeType::NullValue, cur_key); m_current_schema.insert_ordered(node_id); diff --git a/components/core/src/clp_s/JsonParser.hpp b/components/core/src/clp_s/JsonParser.hpp index c69af68fd1..968c639803 100644 --- a/components/core/src/clp_s/JsonParser.hpp +++ b/components/core/src/clp_s/JsonParser.hpp @@ -89,7 +89,7 @@ class JsonParser { * @param key the key of the node * @throw simdjson::simdjson_error when encountering invalid fields while parsing line */ - void parse_line(ondemand::value line, int32_t parent_node_id, std::string const& key); + void parse_line(simdjson::ondemand::value line, int32_t parent_node_id, std::string const& key); /** * Determines the archive node type based on the IR node type and value. @@ -172,14 +172,14 @@ class JsonParser { * @param line the JSON array * @param parent_node_id the parent node id */ - void parse_array(ondemand::array line, int32_t parent_node_id); + void parse_array(simdjson::ondemand::array line, int32_t parent_node_id); /** * Parses an object within an array in a JSON line * @param line the JSON object * @param parent_node_id the parent node id */ - void parse_obj_in_array(ondemand::object line, int32_t parent_node_id); + void parse_obj_in_array(simdjson::ondemand::object line, int32_t parent_node_id); /** * Splits the archive if the size of the archive exceeds the maximum size diff --git a/components/core/src/clp_s/search/QueryRunner.cpp b/components/core/src/clp_s/search/QueryRunner.cpp index 4b874853a7..b721e02593 100644 --- a/components/core/src/clp_s/search/QueryRunner.cpp +++ b/components/core/src/clp_s/search/QueryRunner.cpp @@ -493,7 +493,7 @@ bool QueryRunner::evaluate_array_filter( value.reserve(value.size() + simdjson::SIMDJSON_PADDING); } auto obj = m_array_parser.iterate(value); - ondemand::array array = obj.get_array(); + simdjson::ondemand::array array = obj.get_array(); // pre-evaluate whether we can match strings or numbers to eliminate // duplicate effort on every item @@ -509,7 +509,7 @@ bool QueryRunner::evaluate_array_filter( } bool QueryRunner::evaluate_array_filter_value( - ondemand::value& item, + simdjson::ondemand::value& item, FilterOperation op, DescriptorList const& unresolved_tokens, size_t cur_idx, @@ -517,8 +517,8 @@ bool QueryRunner::evaluate_array_filter_value( ) const { bool match = false; switch (item.type()) { - case ondemand::json_type::object: { - ondemand::object nested_object = item.get_object(); + case simdjson::ondemand::json_type::object: { + simdjson::ondemand::object nested_object = item.get_object(); if (evaluate_array_filter_object( nested_object, op, @@ -530,14 +530,14 @@ bool QueryRunner::evaluate_array_filter_value( match = true; } } break; - case ondemand::json_type::array: { - ondemand::array nested_array = item.get_array(); + case simdjson::ondemand::json_type::array: { + simdjson::ondemand::array nested_array = item.get_array(); if (evaluate_array_filter_array(nested_array, op, unresolved_tokens, cur_idx, operand)) { match = true; } } break; - case ondemand::json_type::string: { + case simdjson::ondemand::json_type::string: { if (true == m_maybe_string && unresolved_tokens.size() == cur_idx && clp::string_utils::wildcard_match_unsafe( item.get_string().value(), @@ -548,11 +548,11 @@ bool QueryRunner::evaluate_array_filter_value( match = op == FilterOperation::EQ; } } break; - case ondemand::json_type::number: { + case simdjson::ondemand::json_type::number: { if (false == m_maybe_number || unresolved_tokens.size() != cur_idx) { break; } - ondemand::number number = item.get_number(); + simdjson::ondemand::number number = item.get_number(); if (number.is_double()) { double tmp_double; operand->as_float(tmp_double, op); @@ -570,7 +570,7 @@ bool QueryRunner::evaluate_array_filter_value( match = eval(op, number.get_int64(), tmp_uint); } } break; - case ondemand::json_type::boolean: { + case simdjson::ondemand::json_type::boolean: { if (unresolved_tokens.size() != cur_idx || op == FilterOperation::EXISTS || op == FilterOperation::NEXISTS) { @@ -581,7 +581,7 @@ bool QueryRunner::evaluate_array_filter_value( match = true; } } break; - case ondemand::json_type::null: { + case simdjson::ondemand::json_type::null: { if (op != FilterOperation::EXISTS && op != FilterOperation::NEXISTS && operand->as_null(op)) { @@ -593,13 +593,13 @@ bool QueryRunner::evaluate_array_filter_value( } bool QueryRunner::evaluate_array_filter_array( - ondemand::array& array, + simdjson::ondemand::array& array, FilterOperation op, DescriptorList const& unresolved_tokens, size_t cur_idx, std::shared_ptr const& operand ) const { - for (ondemand::value item : array) { + for (simdjson::ondemand::value item : array) { if (evaluate_array_filter_value(item, op, unresolved_tokens, cur_idx, operand)) { return true; } @@ -608,7 +608,7 @@ bool QueryRunner::evaluate_array_filter_array( } bool QueryRunner::evaluate_array_filter_object( - ondemand::object& object, + simdjson::ondemand::object& object, FilterOperation op, DescriptorList const& unresolved_tokens, size_t cur_idx, @@ -630,7 +630,7 @@ bool QueryRunner::evaluate_array_filter_object( return op == FilterOperation::EXISTS; } - ondemand::value item = field.value(); + simdjson::ondemand::value item = field.value(); return evaluate_array_filter_value(item, op, unresolved_tokens, cur_idx, operand); } return false; @@ -645,7 +645,7 @@ bool QueryRunner::evaluate_wildcard_array_filter( value.reserve(value.size() + simdjson::SIMDJSON_PADDING); } auto obj = m_array_parser.iterate(value); - ondemand::array array = obj.get_array(); + simdjson::ondemand::array array = obj.get_array(); // pre-evaluate whether we can match strings or numbers to eliminate // duplicate effort on every item @@ -656,26 +656,26 @@ bool QueryRunner::evaluate_wildcard_array_filter( } bool QueryRunner::evaluate_wildcard_array_filter( - ondemand::array& array, + simdjson::ondemand::array& array, FilterOperation op, std::shared_ptr const& operand ) const { bool match = false; for (auto item : array) { switch (item.type()) { - case ondemand::json_type::object: { - ondemand::object nested_object = item.get_object(); + case simdjson::ondemand::json_type::object: { + simdjson::ondemand::object nested_object = item.get_object(); if (evaluate_wildcard_array_filter(nested_object, op, operand)) { match = true; } } break; - case ondemand::json_type::array: { - ondemand::array nested_array = item.get_array(); + case simdjson::ondemand::json_type::array: { + simdjson::ondemand::array nested_array = item.get_array(); if (evaluate_wildcard_array_filter(nested_array, op, operand)) { match = true; } } break; - case ondemand::json_type::string: { + case simdjson::ondemand::json_type::string: { if (false == m_maybe_string) { break; } @@ -689,11 +689,11 @@ bool QueryRunner::evaluate_wildcard_array_filter( } break; } break; - case ondemand::json_type::number: { + case simdjson::ondemand::json_type::number: { if (false == m_maybe_number) { break; } - ondemand::number number = item.get_number(); + simdjson::ondemand::number number = item.get_number(); if (number.is_double()) { double tmp_double; operand->as_float(tmp_double, op); @@ -708,13 +708,13 @@ bool QueryRunner::evaluate_wildcard_array_filter( match |= eval(op, number.get_int64(), tmp_int); } } break; - case ondemand::json_type::boolean: { + case simdjson::ondemand::json_type::boolean: { bool tmp; if (operand->as_bool(tmp, op) && eval(op, item.get_bool(), tmp)) { match = true; } } break; - case ondemand::json_type::null: + case simdjson::ondemand::json_type::null: if (operand->as_null(op)) { match |= op == FilterOperation::EQ; } @@ -729,27 +729,27 @@ bool QueryRunner::evaluate_wildcard_array_filter( } bool QueryRunner::evaluate_wildcard_array_filter( - ondemand::object& object, + simdjson::ondemand::object& object, FilterOperation op, std::shared_ptr const& operand ) const { bool match = false; for (auto field : object) { - ondemand::value item = field.value(); + simdjson::ondemand::value item = field.value(); switch (item.type()) { - case ondemand::json_type::object: { - ondemand::object nested_object = item.get_object(); + case simdjson::ondemand::json_type::object: { + simdjson::ondemand::object nested_object = item.get_object(); if (evaluate_wildcard_array_filter(nested_object, op, operand)) { match = true; } } break; - case ondemand::json_type::array: { - ondemand::array nested_array = item.get_array(); + case simdjson::ondemand::json_type::array: { + simdjson::ondemand::array nested_array = item.get_array(); if (evaluate_wildcard_array_filter(nested_array, op, operand)) { match = true; } } break; - case ondemand::json_type::string: { + case simdjson::ondemand::json_type::string: { if (false == m_maybe_string) { break; } @@ -763,11 +763,11 @@ bool QueryRunner::evaluate_wildcard_array_filter( } break; } break; - case ondemand::json_type::number: { + case simdjson::ondemand::json_type::number: { if (false == m_maybe_number) { break; } - ondemand::number number = item.get_number(); + simdjson::ondemand::number number = item.get_number(); if (number.is_double()) { double tmp_double; operand->as_float(tmp_double, op); @@ -782,13 +782,13 @@ bool QueryRunner::evaluate_wildcard_array_filter( match |= eval(op, number.get_int64(), tmp_int); } } break; - case ondemand::json_type::boolean: { + case simdjson::ondemand::json_type::boolean: { bool tmp; if (operand->as_bool(tmp, op) && eval(op, item.get_bool(), tmp)) { match = true; } } break; - case ondemand::json_type::null: + case simdjson::ondemand::json_type::null: if (operand->as_null(op)) { match |= op == FilterOperation::EQ; } diff --git a/components/core/src/clp_s/search/QueryRunner.hpp b/components/core/src/clp_s/search/QueryRunner.hpp index 5de87954ec..9bd32cb518 100644 --- a/components/core/src/clp_s/search/QueryRunner.hpp +++ b/components/core/src/clp_s/search/QueryRunner.hpp @@ -296,7 +296,7 @@ class QueryRunner : public FilterClass { * @return true if the expression evaluates to true, false otherwise */ inline auto evaluate_array_filter_value( - ondemand::value& item, + simdjson::ondemand::value& item, ast::FilterOperation op, ast::DescriptorList const& unresolved_tokens, size_t cur_idx, @@ -313,7 +313,7 @@ class QueryRunner : public FilterClass { * @return true if the expression evaluates to true, false otherwise */ auto evaluate_array_filter_array( - ondemand::array& array, + simdjson::ondemand::array& array, ast::FilterOperation op, ast::DescriptorList const& unresolved_tokens, size_t cur_idx, @@ -330,7 +330,7 @@ class QueryRunner : public FilterClass { * @return true if the expression evaluates to true, false otherwise */ auto evaluate_array_filter_object( - ondemand::object& object, + simdjson::ondemand::object& object, ast::FilterOperation op, ast::DescriptorList const& unresolved_tokens, size_t cur_idx, @@ -358,7 +358,7 @@ class QueryRunner : public FilterClass { * @return true if the expression evaluates to true, false otherwise */ auto evaluate_wildcard_array_filter( - ondemand::array& array, + simdjson::ondemand::array& array, ast::FilterOperation op, std::shared_ptr const& operand ) const -> bool; @@ -371,7 +371,7 @@ class QueryRunner : public FilterClass { * @return true if the expression evaluates to true, false otherwise */ auto evaluate_wildcard_array_filter( - ondemand::object& object, + simdjson::ondemand::object& object, ast::FilterOperation op, std::shared_ptr const& operand ) const -> bool; From 627d02426f5923a1dc1d5eb9d77e372071aa3bdf Mon Sep 17 00:00:00 2001 From: Bingran Hu Date: Mon, 1 Sep 2025 11:30:56 +0800 Subject: [PATCH 40/69] switch back to parallel deps core --- taskfiles/deps/main.yaml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/taskfiles/deps/main.yaml b/taskfiles/deps/main.yaml index 7c6dea8abc..b3ea5ac1b7 100644 --- a/taskfiles/deps/main.yaml +++ b/taskfiles/deps/main.yaml @@ -64,7 +64,7 @@ tasks: core-all-parallel: internal: true run: "once" - cmds: + deps: - task: "absl" - task: "antlr-jar" - task: "antlr-runtime" From a5930d63fe6873aed9b74a787f31c7e10db350b5 Mon Sep 17 00:00:00 2001 From: Bingran Hu Date: Mon, 1 Sep 2025 00:31:57 -0400 Subject: [PATCH 41/69] move brew uninstall macos into github actions --- .github/workflows/clp-core-build-macos.yaml | 5 ++++- .../tools/scripts/lib_install/macos/install-all.sh | 10 ++++------ taskfiles/deps/main.yaml | 2 +- 3 files changed, 9 insertions(+), 8 deletions(-) diff --git a/.github/workflows/clp-core-build-macos.yaml b/.github/workflows/clp-core-build-macos.yaml index ebf567cdb7..be061f88ca 100644 --- a/.github/workflows/clp-core-build-macos.yaml +++ b/.github/workflows/clp-core-build-macos.yaml @@ -64,7 +64,10 @@ jobs: rm -f /usr/local/bin/go* rm -f /usr/local/bin/idle3* rm -f /usr/local/bin/pydoc3* - rm -f /usr/local/bin/python3* + + - name: "Remove preinstalled brew programs which will conflict with pipx's installs" + run: | + brew uninstall --force cmake - name: "Install dependencies" run: "./components/core/tools/scripts/lib_install/macos/install-all.sh" diff --git a/components/core/tools/scripts/lib_install/macos/install-all.sh b/components/core/tools/scripts/lib_install/macos/install-all.sh index bb99fd77e8..54004938de 100755 --- a/components/core/tools/scripts/lib_install/macos/install-all.sh +++ b/components/core/tools/scripts/lib_install/macos/install-all.sh @@ -29,16 +29,14 @@ fi # Install `cmake` # ystdlib requires CMake v3.23; ANTLR and yaml-cpp do not yet support CMake v4+. # See also: https://github.com/y-scope/clp/issues/795 -if command -v cmake >/dev/null 2>&1; then - brew uninstall --force cmake +if ! command -v cmake >/dev/null 2>&1; then + pipx install "cmake>=3.23,<3.24" fi -pipx install "cmake>=3.23,<3.24" # Install a version of `task` < 3.43 to avoid https://github.com/y-scope/clp/issues/872 -if command -v task >/dev/null 2>&1; then - brew uninstall --force task +if ! command -v task >/dev/null 2>&1; then + pipx install "go-task-bin>=3.40,<3.43" fi -pipx install "go-task-bin>=3.40,<3.43" # Install uv if ! command -v uv >/dev/null 2>&1; then diff --git a/taskfiles/deps/main.yaml b/taskfiles/deps/main.yaml index 501c4c2a2b..8eaf638d52 100644 --- a/taskfiles/deps/main.yaml +++ b/taskfiles/deps/main.yaml @@ -64,7 +64,7 @@ tasks: core-all-parallel: internal: true run: "once" - cmds: + deps: - task: "absl" - task: "antlr-jar" - task: "antlr-runtime" From e7a42bc0c3f030f678e6a4b848f3121aa9b3e7cc Mon Sep 17 00:00:00 2001 From: Bingran Hu Date: Mon, 1 Sep 2025 16:23:09 +0800 Subject: [PATCH 42/69] Update .github/workflows/clp-core-build-macos.yaml Co-authored-by: coderabbitai[bot] <136622811+coderabbitai[bot]@users.noreply.github.com> --- .github/workflows/clp-core-build-macos.yaml | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/.github/workflows/clp-core-build-macos.yaml b/.github/workflows/clp-core-build-macos.yaml index be061f88ca..41239cae83 100644 --- a/.github/workflows/clp-core-build-macos.yaml +++ b/.github/workflows/clp-core-build-macos.yaml @@ -67,8 +67,9 @@ jobs: - name: "Remove preinstalled brew programs which will conflict with pipx's installs" run: | - brew uninstall --force cmake - + if brew list --formula cmake >/dev/null 2>&1; then + brew uninstall --force cmake + fi - name: "Install dependencies" run: "./components/core/tools/scripts/lib_install/macos/install-all.sh" From 58b86887b7c67b3abe77158b17487a9682f2aae9 Mon Sep 17 00:00:00 2001 From: Bingran Hu Date: Mon, 1 Sep 2025 16:24:05 +0800 Subject: [PATCH 43/69] minor fix --- .github/workflows/clp-core-build-macos.yaml | 1 + 1 file changed, 1 insertion(+) diff --git a/.github/workflows/clp-core-build-macos.yaml b/.github/workflows/clp-core-build-macos.yaml index 41239cae83..2372aa71ba 100644 --- a/.github/workflows/clp-core-build-macos.yaml +++ b/.github/workflows/clp-core-build-macos.yaml @@ -64,6 +64,7 @@ jobs: rm -f /usr/local/bin/go* rm -f /usr/local/bin/idle3* rm -f /usr/local/bin/pydoc3* + rm -f /usr/local/bin/python3* - name: "Remove preinstalled brew programs which will conflict with pipx's installs" run: | From c359e3093bb17dabb15c7ae9232155447e6dd4e9 Mon Sep 17 00:00:00 2001 From: Bingran Hu Date: Mon, 1 Sep 2025 17:32:26 +0800 Subject: [PATCH 44/69] Refactor to ask user to uninstall manually and fail-fast for build tools version checking --- .../Dockerfile | 4 ++ .../Dockerfile | 4 ++ .../Dockerfile | 4 ++ .../Dockerfile | 4 ++ .../centos-stream-9/install-all.sh | 2 - .../install-prebuilt-packages.sh | 20 +--------- .../check-build-tool-versions.sh | 40 ------------------- .../scripts/lib_install/macos/install-all.sh | 20 +--------- .../lib_install/manylinux_2_28/install-all.sh | 2 - .../install-prebuilt-packages.sh | 22 +--------- .../lib_install/musllinux_1_2/install-all.sh | 2 - .../install-prebuilt-packages.sh | 22 +--------- .../lib_install/pipx_install/install-all.sh | 10 +++++ .../lib_install/pipx_install/install-cmake.sh | 24 +++++++++++ .../pipx_install/install-go-task.sh | 23 +++++++++++ .../lib_install/pipx_install/install-uv.sh | 23 +++++++++++ .../lib_install/ubuntu-jammy/install-all.sh | 2 - .../ubuntu-jammy/install-prebuilt-packages.sh | 23 ++--------- 18 files changed, 108 insertions(+), 143 deletions(-) delete mode 100755 components/core/tools/scripts/lib_install/lib_version_checks/check-build-tool-versions.sh create mode 100755 components/core/tools/scripts/lib_install/pipx_install/install-all.sh create mode 100755 components/core/tools/scripts/lib_install/pipx_install/install-cmake.sh create mode 100755 components/core/tools/scripts/lib_install/pipx_install/install-go-task.sh create mode 100755 components/core/tools/scripts/lib_install/pipx_install/install-uv.sh diff --git a/components/core/tools/docker-images/clp-env-base-manylinux_2_28-aarch64/Dockerfile b/components/core/tools/docker-images/clp-env-base-manylinux_2_28-aarch64/Dockerfile index 06df2a2961..406410b00d 100644 --- a/components/core/tools/docker-images/clp-env-base-manylinux_2_28-aarch64/Dockerfile +++ b/components/core/tools/docker-images/clp-env-base-manylinux_2_28-aarch64/Dockerfile @@ -5,6 +5,10 @@ WORKDIR /root RUN mkdir -p ./tools/scripts/lib_install COPY ./tools/scripts/lib_install ./tools/scripts/lib_install +RUN pipx uninstall cmake || true +RUN pipx uninstall go-task-bin || true +RUN pipx uninstall uv || true + RUN ./tools/scripts/lib_install/manylinux_2_28/install-all.sh # Remove cached files diff --git a/components/core/tools/docker-images/clp-env-base-manylinux_2_28-x86_64/Dockerfile b/components/core/tools/docker-images/clp-env-base-manylinux_2_28-x86_64/Dockerfile index 1185c7d031..a4383f0781 100644 --- a/components/core/tools/docker-images/clp-env-base-manylinux_2_28-x86_64/Dockerfile +++ b/components/core/tools/docker-images/clp-env-base-manylinux_2_28-x86_64/Dockerfile @@ -5,6 +5,10 @@ WORKDIR /root RUN mkdir -p ./tools/scripts/lib_install COPY ./tools/scripts/lib_install ./tools/scripts/lib_install +RUN pipx uninstall cmake || true +RUN pipx uninstall go-task-bin || true +RUN pipx uninstall uv || true + RUN ./tools/scripts/lib_install/manylinux_2_28/install-all.sh # Remove cached files diff --git a/components/core/tools/docker-images/clp-env-base-musllinux_1_2-aarch64/Dockerfile b/components/core/tools/docker-images/clp-env-base-musllinux_1_2-aarch64/Dockerfile index 0aab83227d..5ef42a1d07 100644 --- a/components/core/tools/docker-images/clp-env-base-musllinux_1_2-aarch64/Dockerfile +++ b/components/core/tools/docker-images/clp-env-base-musllinux_1_2-aarch64/Dockerfile @@ -5,6 +5,10 @@ WORKDIR /root RUN mkdir -p ./tools/scripts/lib_install COPY ./tools/scripts/lib_install ./tools/scripts/lib_install +RUN pipx uninstall cmake || true +RUN pipx uninstall go-task-bin || true +RUN pipx uninstall uv || true + RUN ./tools/scripts/lib_install/musllinux_1_2/install-all.sh # Remove cached files diff --git a/components/core/tools/docker-images/clp-env-base-musllinux_1_2-x86_64/Dockerfile b/components/core/tools/docker-images/clp-env-base-musllinux_1_2-x86_64/Dockerfile index ecdcd46e07..09bc5889d4 100644 --- a/components/core/tools/docker-images/clp-env-base-musllinux_1_2-x86_64/Dockerfile +++ b/components/core/tools/docker-images/clp-env-base-musllinux_1_2-x86_64/Dockerfile @@ -5,6 +5,10 @@ WORKDIR /root RUN mkdir -p ./tools/scripts/lib_install COPY ./tools/scripts/lib_install ./tools/scripts/lib_install +RUN pipx uninstall cmake || true +RUN pipx uninstall go-task-bin || true +RUN pipx uninstall uv || true + RUN ./tools/scripts/lib_install/musllinux_1_2/install-all.sh # Remove cached files diff --git a/components/core/tools/scripts/lib_install/centos-stream-9/install-all.sh b/components/core/tools/scripts/lib_install/centos-stream-9/install-all.sh index 8ea47b541e..0b710aedb2 100755 --- a/components/core/tools/scripts/lib_install/centos-stream-9/install-all.sh +++ b/components/core/tools/scripts/lib_install/centos-stream-9/install-all.sh @@ -7,5 +7,3 @@ script_dir="$( cd "$( dirname "${BASH_SOURCE[0]}" )" &> /dev/null && pwd )" "${script_dir}/install-prebuilt-packages.sh" "${script_dir}/install-packages-from-source.sh" - -"${script_dir}/../lib_version_checks/check-build-tool-versions.sh" diff --git a/components/core/tools/scripts/lib_install/centos-stream-9/install-prebuilt-packages.sh b/components/core/tools/scripts/lib_install/centos-stream-9/install-prebuilt-packages.sh index bdfdaf8030..03f7bc8b2c 100755 --- a/components/core/tools/scripts/lib_install/centos-stream-9/install-prebuilt-packages.sh +++ b/components/core/tools/scripts/lib_install/centos-stream-9/install-prebuilt-packages.sh @@ -22,22 +22,6 @@ dnf install -y \ if ! command -v pipx >/dev/null 2>&1; then python3 -m pip install pipx fi -pipx ensurepath -# Install `cmake` -# ystdlib requires CMake v3.23; ANTLR and yaml-cpp do not yet support CMake v4+. -# See also: https://github.com/y-scope/clp/issues/795 -if ! command -v cmake >/dev/null 2>&1; then - pipx install "cmake>=3.23,<3.24" -fi - -# Install `task` -# NOTE: We lock `task` to a version < 3.43 to avoid https://github.com/y-scope/clp/issues/872 -if ! command -v task >/dev/null 2>&1; then - pipx install "go-task-bin>=3.40,<3.43" -fi - -# Install `uv` -if ! command -v uv >/dev/null 2>&1; then - pipx install "uv>=0.8" -fi +# Install `cmake`, `go-task` and `uv` +"${script_dir}/../pipx_install/install-all.sh" diff --git a/components/core/tools/scripts/lib_install/lib_version_checks/check-build-tool-versions.sh b/components/core/tools/scripts/lib_install/lib_version_checks/check-build-tool-versions.sh deleted file mode 100755 index 24d5b9d496..0000000000 --- a/components/core/tools/scripts/lib_install/lib_version_checks/check-build-tool-versions.sh +++ /dev/null @@ -1,40 +0,0 @@ -#!/usr/bin/env bash - -# Exit on any error, use of undefined variables, or failure within a pipeline -set -euo pipefail - -script_dir="$( cd "$( dirname "${BASH_SOURCE[0]}" )" &> /dev/null && pwd )" - -failures=0 - -run_check() { - local script_name="$1" - local label="$2" - local script_output - - if script_output=$("${script_dir}/${script_name}" 2>&1); then - echo "[OK] ${label}" - else - echo "[FAIL] ${label}" - echo "${script_output}" - failures=$((failures + 1)) - fi -} - -# Run all checks -# TODO: https://github.com/y-scope/clp/issues/795 -run_check check-cmake-version.sh "CMake version" - -# TODO: https://github.com/y-scope/clp/issues/872 -run_check check-go-task-version.sh "Go Task version" - -run_check check-uv-version.sh "uv version" - -# Print summary -echo -if [ "${failures}" -gt 0 ]; then - echo "${failures} check(s) failed." - exit 1 -else - echo "All checks passed." -fi diff --git a/components/core/tools/scripts/lib_install/macos/install-all.sh b/components/core/tools/scripts/lib_install/macos/install-all.sh index 54004938de..c3c6378670 100755 --- a/components/core/tools/scripts/lib_install/macos/install-all.sh +++ b/components/core/tools/scripts/lib_install/macos/install-all.sh @@ -26,21 +26,5 @@ if ! command -v pkg-config >/dev/null 2>&1; then brew install pkg-config fi -# Install `cmake` -# ystdlib requires CMake v3.23; ANTLR and yaml-cpp do not yet support CMake v4+. -# See also: https://github.com/y-scope/clp/issues/795 -if ! command -v cmake >/dev/null 2>&1; then - pipx install "cmake>=3.23,<3.24" -fi - -# Install a version of `task` < 3.43 to avoid https://github.com/y-scope/clp/issues/872 -if ! command -v task >/dev/null 2>&1; then - pipx install "go-task-bin>=3.40,<3.43" -fi - -# Install uv -if ! command -v uv >/dev/null 2>&1; then - pipx install "uv>=0.8" -fi - -"${script_dir}/../lib_version_checks/check-build-tool-versions.sh" +# Install `cmake`, `go-task` and `uv` +"${script_dir}/../pipx_install/install-all.sh" diff --git a/components/core/tools/scripts/lib_install/manylinux_2_28/install-all.sh b/components/core/tools/scripts/lib_install/manylinux_2_28/install-all.sh index 8ea47b541e..0b710aedb2 100755 --- a/components/core/tools/scripts/lib_install/manylinux_2_28/install-all.sh +++ b/components/core/tools/scripts/lib_install/manylinux_2_28/install-all.sh @@ -7,5 +7,3 @@ script_dir="$( cd "$( dirname "${BASH_SOURCE[0]}" )" &> /dev/null && pwd )" "${script_dir}/install-prebuilt-packages.sh" "${script_dir}/install-packages-from-source.sh" - -"${script_dir}/../lib_version_checks/check-build-tool-versions.sh" diff --git a/components/core/tools/scripts/lib_install/manylinux_2_28/install-prebuilt-packages.sh b/components/core/tools/scripts/lib_install/manylinux_2_28/install-prebuilt-packages.sh index 68167048ce..a371b6f861 100755 --- a/components/core/tools/scripts/lib_install/manylinux_2_28/install-prebuilt-packages.sh +++ b/components/core/tools/scripts/lib_install/manylinux_2_28/install-prebuilt-packages.sh @@ -13,23 +13,5 @@ dnf install -y \ zlib-devel \ zlib-static -# Install `cmake` -# ystdlib requires CMake v3.23; ANTLR and yaml-cpp do not yet support CMake v4+. -# See also: https://github.com/y-scope/clp/issues/795 -pipx uninstall cmake || true -if ! command -v cmake >/dev/null 2>&1; then - pipx install "cmake>=3.23,<3.24" -fi - -# Install `task` -# NOTE: We lock `task` to a version < 3.43 to avoid https://github.com/y-scope/clp/issues/872 -pipx uninstall go-task-bin || true -if ! command -v task >/dev/null 2>&1; then - pipx install "go-task-bin>=3.40,<3.43" -fi - -# Install `uv` -pipx uninstall uv || true -if ! command -v uv >/dev/null 2>&1; then - pipx install "uv>=0.8" -fi +# Install `cmake`, `go-task` and `uv` +"${script_dir}/../pipx_install/install-all.sh" diff --git a/components/core/tools/scripts/lib_install/musllinux_1_2/install-all.sh b/components/core/tools/scripts/lib_install/musllinux_1_2/install-all.sh index 8ea47b541e..0b710aedb2 100755 --- a/components/core/tools/scripts/lib_install/musllinux_1_2/install-all.sh +++ b/components/core/tools/scripts/lib_install/musllinux_1_2/install-all.sh @@ -7,5 +7,3 @@ script_dir="$( cd "$( dirname "${BASH_SOURCE[0]}" )" &> /dev/null && pwd )" "${script_dir}/install-prebuilt-packages.sh" "${script_dir}/install-packages-from-source.sh" - -"${script_dir}/../lib_version_checks/check-build-tool-versions.sh" diff --git a/components/core/tools/scripts/lib_install/musllinux_1_2/install-prebuilt-packages.sh b/components/core/tools/scripts/lib_install/musllinux_1_2/install-prebuilt-packages.sh index a8c285e6ee..da6e784a6e 100755 --- a/components/core/tools/scripts/lib_install/musllinux_1_2/install-prebuilt-packages.sh +++ b/components/core/tools/scripts/lib_install/musllinux_1_2/install-prebuilt-packages.sh @@ -14,23 +14,5 @@ apk update && apk add --no-cache \ zlib-dev \ zlib-static -# Install `cmake` -# ystdlib requires CMake v3.23; ANTLR and yaml-cpp do not yet support CMake v4+. -# See also: https://github.com/y-scope/clp/issues/795 -pipx uninstall cmake || true -if ! command -v cmake >/dev/null 2>&1; then - pipx install "cmake>=3.23,<3.24" -fi - -# Install `task` -# NOTE: We lock `task` to a version < 3.43 to avoid https://github.com/y-scope/clp/issues/872 -pipx uninstall go-task-bin || true -if ! command -v task >/dev/null 2>&1; then - pipx install "go-task-bin>=3.40,<3.43" -fi - -# Install `uv` -pipx uninstall uv || true -if ! command -v uv >/dev/null 2>&1; then - pipx install "uv>=0.8" -fi +# Install `cmake`, `go-task` and `uv` +"${script_dir}/../pipx_install/install-all.sh" diff --git a/components/core/tools/scripts/lib_install/pipx_install/install-all.sh b/components/core/tools/scripts/lib_install/pipx_install/install-all.sh new file mode 100755 index 0000000000..0a5ff04caa --- /dev/null +++ b/components/core/tools/scripts/lib_install/pipx_install/install-all.sh @@ -0,0 +1,10 @@ +#!/usr/bin/env bash + +# Exit on any error, use of undefined variables, or failure within a pipeline +set -euo pipefail + +script_dir="$( cd "$( dirname "${BASH_SOURCE[0]}" )" &> /dev/null && pwd )" + +"${script_dir}/install-cmake.sh" +"${script_dir}/install-go-task.sh" +"${script_dir}/install-uv.sh" diff --git a/components/core/tools/scripts/lib_install/pipx_install/install-cmake.sh b/components/core/tools/scripts/lib_install/pipx_install/install-cmake.sh new file mode 100755 index 0000000000..266d839db7 --- /dev/null +++ b/components/core/tools/scripts/lib_install/pipx_install/install-cmake.sh @@ -0,0 +1,24 @@ +#!/usr/bin/env bash + +# Exit on any error, use of undefined variables, or failure within a pipeline +set -euo pipefail + +script_dir="$( cd "$( dirname "${BASH_SOURCE[0]}" )" &> /dev/null && pwd )" + +if ! command -v pipx >/dev/null 2>&1; then + echo "Error: pipx is not available." + exit 1 +fi + +if command -v cmake >/dev/null 2>&1; then + version_check_script="${script_dir}/../lib_version_checks/check-cmake-version.sh" + if ! script_output=$("${version_check_script}" 2>&1); then + echo "${script_output}" + echo "Please manually uninstall the current cmake and then re-run the install script." + fi +else + # ystdlib requires CMake v3.23; ANTLR and yaml-cpp do not yet support CMake v4+. + # See also: https://github.com/y-scope/clp/issues/795 + pipx install --force "cmake>=3.23,<3.24" + pipx ensurepath +fi diff --git a/components/core/tools/scripts/lib_install/pipx_install/install-go-task.sh b/components/core/tools/scripts/lib_install/pipx_install/install-go-task.sh new file mode 100755 index 0000000000..12e07af9e0 --- /dev/null +++ b/components/core/tools/scripts/lib_install/pipx_install/install-go-task.sh @@ -0,0 +1,23 @@ +#!/usr/bin/env bash + +# Exit on any error, use of undefined variables, or failure within a pipeline +set -euo pipefail + +script_dir="$( cd "$( dirname "${BASH_SOURCE[0]}" )" &> /dev/null && pwd )" + +if ! command -v pipx >/dev/null 2>&1; then + echo "Error: pipx is not available." + exit 1 +fi + +if command -v task >/dev/null 2>&1; then + version_check_script="${script_dir}/../lib_version_checks/check-go-task-version.sh" + if ! script_output=$("${version_check_script}" 2>&1); then + echo "${script_output}" + echo "Please manually uninstall the current go-task and then re-run the install script." + fi +else + # We lock `task` to a version < 3.43 to avoid https://github.com/y-scope/clp/issues/872 + pipx install --force "go-task-bin>=3.40,<3.43" + pipx ensurepath +fi diff --git a/components/core/tools/scripts/lib_install/pipx_install/install-uv.sh b/components/core/tools/scripts/lib_install/pipx_install/install-uv.sh new file mode 100755 index 0000000000..37c62acba1 --- /dev/null +++ b/components/core/tools/scripts/lib_install/pipx_install/install-uv.sh @@ -0,0 +1,23 @@ +#!/usr/bin/env bash + +# Exit on any error, use of undefined variables, or failure within a pipeline +set -euo pipefail + +script_dir="$( cd "$( dirname "${BASH_SOURCE[0]}" )" &> /dev/null && pwd )" + +if ! command -v pipx >/dev/null 2>&1; then + echo "Error: pipx is not available." + exit 1 +fi + +if command -v uv >/dev/null 2>&1; then + version_check_script="${script_dir}/../lib_version_checks/check-uv-version.sh" + if ! script_output=$("${version_check_script}" 2>&1); then + echo "${script_output}" + echo "Please manually uninstall the current uv and then re-run the install script." + fi +else + pipx install --force "uv>=0.8" + pipx ensurepath +fi + diff --git a/components/core/tools/scripts/lib_install/ubuntu-jammy/install-all.sh b/components/core/tools/scripts/lib_install/ubuntu-jammy/install-all.sh index 8ea47b541e..0b710aedb2 100755 --- a/components/core/tools/scripts/lib_install/ubuntu-jammy/install-all.sh +++ b/components/core/tools/scripts/lib_install/ubuntu-jammy/install-all.sh @@ -7,5 +7,3 @@ script_dir="$( cd "$( dirname "${BASH_SOURCE[0]}" )" &> /dev/null && pwd )" "${script_dir}/install-prebuilt-packages.sh" "${script_dir}/install-packages-from-source.sh" - -"${script_dir}/../lib_version_checks/check-build-tool-versions.sh" diff --git a/components/core/tools/scripts/lib_install/ubuntu-jammy/install-prebuilt-packages.sh b/components/core/tools/scripts/lib_install/ubuntu-jammy/install-prebuilt-packages.sh index 1cb3e6a3a0..8d18596535 100755 --- a/components/core/tools/scripts/lib_install/ubuntu-jammy/install-prebuilt-packages.sh +++ b/components/core/tools/scripts/lib_install/ubuntu-jammy/install-prebuilt-packages.sh @@ -3,6 +3,8 @@ # Exit on any error, use of undefined variables, or failure within a pipeline set -euo pipefail +script_dir="$( cd "$( dirname "${BASH_SOURCE[0]}" )" &> /dev/null && pwd )" + apt-get update DEBIAN_FRONTEND=noninteractive apt-get install --no-install-recommends -y \ ca-certificates \ @@ -25,22 +27,5 @@ DEBIAN_FRONTEND=noninteractive apt-get install --no-install-recommends -y \ software-properties-common \ unzip -pipx ensurepath - -# Install `cmake` -# ystdlib requires CMake v3.23; ANTLR and yaml-cpp do not yet support CMake v4+. -# See also: https://github.com/y-scope/clp/issues/795 -if ! command -v cmake >/dev/null 2>&1; then - pipx install "cmake>=3.23,<3.24" -fi - -# Install `task` -# NOTE: We lock `task` to a version < 3.43 to avoid https://github.com/y-scope/clp/issues/872 -if ! command -v task >/dev/null 2>&1; then - pipx install "go-task-bin>=3.40,<3.43" -fi - -# Install `uv` -if ! command -v uv >/dev/null 2>&1; then - pipx install "uv>=0.8" -fi +# Install `cmake`, `go-task` and `uv` +"${script_dir}/../pipx_install/install-all.sh" From 0730260dae6cb74233349d7fdf2cfa6ed9992602 Mon Sep 17 00:00:00 2001 From: Bingran Hu Date: Mon, 1 Sep 2025 17:43:55 +0800 Subject: [PATCH 45/69] Add missing vars --- .../lib_install/centos-stream-9/install-prebuilt-packages.sh | 2 ++ .../lib_install/manylinux_2_28/install-prebuilt-packages.sh | 2 ++ .../lib_install/musllinux_1_2/install-prebuilt-packages.sh | 2 ++ 3 files changed, 6 insertions(+) diff --git a/components/core/tools/scripts/lib_install/centos-stream-9/install-prebuilt-packages.sh b/components/core/tools/scripts/lib_install/centos-stream-9/install-prebuilt-packages.sh index 03f7bc8b2c..d348e52fd6 100755 --- a/components/core/tools/scripts/lib_install/centos-stream-9/install-prebuilt-packages.sh +++ b/components/core/tools/scripts/lib_install/centos-stream-9/install-prebuilt-packages.sh @@ -3,6 +3,8 @@ # Exit on any error, use of undefined variables, or failure within a pipeline set -euo pipefail +script_dir="$( cd "$( dirname "${BASH_SOURCE[0]}" )" &> /dev/null && pwd )" + dnf install -y \ diffutils \ gcc-c++ \ diff --git a/components/core/tools/scripts/lib_install/manylinux_2_28/install-prebuilt-packages.sh b/components/core/tools/scripts/lib_install/manylinux_2_28/install-prebuilt-packages.sh index a371b6f861..00b53fd955 100755 --- a/components/core/tools/scripts/lib_install/manylinux_2_28/install-prebuilt-packages.sh +++ b/components/core/tools/scripts/lib_install/manylinux_2_28/install-prebuilt-packages.sh @@ -3,6 +3,8 @@ # Exit on any error, use of undefined variables, or failure within a pipeline set -euo pipefail +script_dir="$( cd "$( dirname "${BASH_SOURCE[0]}" )" &> /dev/null && pwd )" + dnf install -y \ gcc-c++ \ java-11-openjdk \ diff --git a/components/core/tools/scripts/lib_install/musllinux_1_2/install-prebuilt-packages.sh b/components/core/tools/scripts/lib_install/musllinux_1_2/install-prebuilt-packages.sh index da6e784a6e..5c1e8d932a 100755 --- a/components/core/tools/scripts/lib_install/musllinux_1_2/install-prebuilt-packages.sh +++ b/components/core/tools/scripts/lib_install/musllinux_1_2/install-prebuilt-packages.sh @@ -3,6 +3,8 @@ # Exit on any error, use of undefined variables, or failure within a pipeline set -euo pipefail +script_dir="$( cd "$( dirname "${BASH_SOURCE[0]}" )" &> /dev/null && pwd )" + apk update && apk add --no-cache \ bzip2-dev \ bzip2-static \ From f4a2c537eb672782ee32bced10bd20f385bcc649 Mon Sep 17 00:00:00 2001 From: Bingran Hu Date: Mon, 1 Sep 2025 17:44:56 +0800 Subject: [PATCH 46/69] Add missing exit flags --- .../core/tools/scripts/lib_install/pipx_install/install-cmake.sh | 1 + .../tools/scripts/lib_install/pipx_install/install-go-task.sh | 1 + .../core/tools/scripts/lib_install/pipx_install/install-uv.sh | 1 + 3 files changed, 3 insertions(+) diff --git a/components/core/tools/scripts/lib_install/pipx_install/install-cmake.sh b/components/core/tools/scripts/lib_install/pipx_install/install-cmake.sh index 266d839db7..8880d49563 100755 --- a/components/core/tools/scripts/lib_install/pipx_install/install-cmake.sh +++ b/components/core/tools/scripts/lib_install/pipx_install/install-cmake.sh @@ -15,6 +15,7 @@ if command -v cmake >/dev/null 2>&1; then if ! script_output=$("${version_check_script}" 2>&1); then echo "${script_output}" echo "Please manually uninstall the current cmake and then re-run the install script." + exit 1 fi else # ystdlib requires CMake v3.23; ANTLR and yaml-cpp do not yet support CMake v4+. diff --git a/components/core/tools/scripts/lib_install/pipx_install/install-go-task.sh b/components/core/tools/scripts/lib_install/pipx_install/install-go-task.sh index 12e07af9e0..9eef7fee49 100755 --- a/components/core/tools/scripts/lib_install/pipx_install/install-go-task.sh +++ b/components/core/tools/scripts/lib_install/pipx_install/install-go-task.sh @@ -15,6 +15,7 @@ if command -v task >/dev/null 2>&1; then if ! script_output=$("${version_check_script}" 2>&1); then echo "${script_output}" echo "Please manually uninstall the current go-task and then re-run the install script." + exit 1 fi else # We lock `task` to a version < 3.43 to avoid https://github.com/y-scope/clp/issues/872 diff --git a/components/core/tools/scripts/lib_install/pipx_install/install-uv.sh b/components/core/tools/scripts/lib_install/pipx_install/install-uv.sh index 37c62acba1..552052533e 100755 --- a/components/core/tools/scripts/lib_install/pipx_install/install-uv.sh +++ b/components/core/tools/scripts/lib_install/pipx_install/install-uv.sh @@ -15,6 +15,7 @@ if command -v uv >/dev/null 2>&1; then if ! script_output=$("${version_check_script}" 2>&1); then echo "${script_output}" echo "Please manually uninstall the current uv and then re-run the install script." + exit 1 fi else pipx install --force "uv>=0.8" From b686138ac2c779c3a9cf20bebc65aa7e4bf21889 Mon Sep 17 00:00:00 2001 From: Bingran Hu Date: Mon, 1 Sep 2025 10:31:15 -0400 Subject: [PATCH 47/69] Add missing line --- taskfiles/deps/main.yaml | 1 + 1 file changed, 1 insertion(+) diff --git a/taskfiles/deps/main.yaml b/taskfiles/deps/main.yaml index ee1254a26c..ee687a5590 100644 --- a/taskfiles/deps/main.yaml +++ b/taskfiles/deps/main.yaml @@ -352,6 +352,7 @@ tasks: TARBALL_URL: "https://github.com/microsoft/GSL/archive/refs/tags/v4.0.0.tar.gz" mongocxx: + internal: true vars: VERSION: "r4.1.1" run: "once" From fc783cdbe458c75a9534d1c15a5a247bbd0a1906 Mon Sep 17 00:00:00 2001 From: Bingran Hu Date: Wed, 3 Sep 2025 02:54:27 -0400 Subject: [PATCH 48/69] rewrite set euo pipefail --- .../tools/scripts/lib_install/centos-stream-9/install-all.sh | 5 +++-- .../lib_install/centos-stream-9/install-prebuilt-packages.sh | 5 +++-- .../lib_install/lib_version_checks/check-cmake-version.sh | 5 +++-- .../lib_install/lib_version_checks/check-go-task-version.sh | 5 +++-- .../lib_install/lib_version_checks/check-uv-version.sh | 5 +++-- .../core/tools/scripts/lib_install/macos/install-all.sh | 5 +++-- .../tools/scripts/lib_install/manylinux_2_28/install-all.sh | 5 +++-- .../lib_install/manylinux_2_28/install-prebuilt-packages.sh | 5 +++-- .../tools/scripts/lib_install/musllinux_1_2/install-all.sh | 5 +++-- .../lib_install/musllinux_1_2/install-prebuilt-packages.sh | 5 +++-- .../tools/scripts/lib_install/pipx_install/install-all.sh | 5 +++-- .../tools/scripts/lib_install/pipx_install/install-cmake.sh | 5 +++-- .../scripts/lib_install/pipx_install/install-go-task.sh | 5 +++-- .../tools/scripts/lib_install/pipx_install/install-uv.sh | 5 +++-- .../tools/scripts/lib_install/ubuntu-jammy/install-all.sh | 5 +++-- .../lib_install/ubuntu-jammy/install-prebuilt-packages.sh | 5 +++-- 16 files changed, 48 insertions(+), 32 deletions(-) diff --git a/components/core/tools/scripts/lib_install/centos-stream-9/install-all.sh b/components/core/tools/scripts/lib_install/centos-stream-9/install-all.sh index 0b710aedb2..d2ca30c297 100755 --- a/components/core/tools/scripts/lib_install/centos-stream-9/install-all.sh +++ b/components/core/tools/scripts/lib_install/centos-stream-9/install-all.sh @@ -1,7 +1,8 @@ #!/usr/bin/env bash -# Exit on any error, use of undefined variables, or failure within a pipeline -set -euo pipefail +set -o errexit +set -o nounset +set -o pipefail script_dir="$( cd "$( dirname "${BASH_SOURCE[0]}" )" &> /dev/null && pwd )" diff --git a/components/core/tools/scripts/lib_install/centos-stream-9/install-prebuilt-packages.sh b/components/core/tools/scripts/lib_install/centos-stream-9/install-prebuilt-packages.sh index d348e52fd6..0664d262f2 100755 --- a/components/core/tools/scripts/lib_install/centos-stream-9/install-prebuilt-packages.sh +++ b/components/core/tools/scripts/lib_install/centos-stream-9/install-prebuilt-packages.sh @@ -1,7 +1,8 @@ #!/usr/bin/env bash -# Exit on any error, use of undefined variables, or failure within a pipeline -set -euo pipefail +set -o errexit +set -o nounset +set -o pipefail script_dir="$( cd "$( dirname "${BASH_SOURCE[0]}" )" &> /dev/null && pwd )" diff --git a/components/core/tools/scripts/lib_install/lib_version_checks/check-cmake-version.sh b/components/core/tools/scripts/lib_install/lib_version_checks/check-cmake-version.sh index b3524589cd..0997ce8b58 100755 --- a/components/core/tools/scripts/lib_install/lib_version_checks/check-cmake-version.sh +++ b/components/core/tools/scripts/lib_install/lib_version_checks/check-cmake-version.sh @@ -1,7 +1,8 @@ #!/usr/bin/env bash -# Exit on any error, use of undefined variables, or failure within a pipeline -set -euo pipefail +set -o errexit +set -o nounset +set -o pipefail # Get the installed cmake version string cmake_version=$(cmake -E capabilities | jq --raw-output ".version.string") diff --git a/components/core/tools/scripts/lib_install/lib_version_checks/check-go-task-version.sh b/components/core/tools/scripts/lib_install/lib_version_checks/check-go-task-version.sh index c27e34d34c..e58d9cad36 100755 --- a/components/core/tools/scripts/lib_install/lib_version_checks/check-go-task-version.sh +++ b/components/core/tools/scripts/lib_install/lib_version_checks/check-go-task-version.sh @@ -1,7 +1,8 @@ #!/usr/bin/env bash -# Exit on any error, use of undefined variables, or failure within a pipeline -set -euo pipefail +set -o errexit +set -o nounset +set -o pipefail script_dir="$( cd "$( dirname "${BASH_SOURCE[0]}" )" &> /dev/null && pwd )" diff --git a/components/core/tools/scripts/lib_install/lib_version_checks/check-uv-version.sh b/components/core/tools/scripts/lib_install/lib_version_checks/check-uv-version.sh index 3b66c92039..f24892c1d4 100755 --- a/components/core/tools/scripts/lib_install/lib_version_checks/check-uv-version.sh +++ b/components/core/tools/scripts/lib_install/lib_version_checks/check-uv-version.sh @@ -1,7 +1,8 @@ #!/usr/bin/env bash -# Exit on any error, use of undefined variables, or failure within a pipeline -set -euo pipefail +set -o errexit +set -o nounset +set -o pipefail # Get the installed uv version string uv_version=$(uv self version --output-format json | jq --raw-output ".version") diff --git a/components/core/tools/scripts/lib_install/macos/install-all.sh b/components/core/tools/scripts/lib_install/macos/install-all.sh index c3c6378670..381d1190ea 100755 --- a/components/core/tools/scripts/lib_install/macos/install-all.sh +++ b/components/core/tools/scripts/lib_install/macos/install-all.sh @@ -1,7 +1,8 @@ #!/usr/bin/env bash -# Exit on any error, use of undefined variables, or failure within a pipeline -set -euo pipefail +set -o errexit +set -o nounset +set -o pipefail script_dir="$( cd "$( dirname "${BASH_SOURCE[0]}" )" &> /dev/null && pwd )" diff --git a/components/core/tools/scripts/lib_install/manylinux_2_28/install-all.sh b/components/core/tools/scripts/lib_install/manylinux_2_28/install-all.sh index 0b710aedb2..d2ca30c297 100755 --- a/components/core/tools/scripts/lib_install/manylinux_2_28/install-all.sh +++ b/components/core/tools/scripts/lib_install/manylinux_2_28/install-all.sh @@ -1,7 +1,8 @@ #!/usr/bin/env bash -# Exit on any error, use of undefined variables, or failure within a pipeline -set -euo pipefail +set -o errexit +set -o nounset +set -o pipefail script_dir="$( cd "$( dirname "${BASH_SOURCE[0]}" )" &> /dev/null && pwd )" diff --git a/components/core/tools/scripts/lib_install/manylinux_2_28/install-prebuilt-packages.sh b/components/core/tools/scripts/lib_install/manylinux_2_28/install-prebuilt-packages.sh index 00b53fd955..543c076d26 100755 --- a/components/core/tools/scripts/lib_install/manylinux_2_28/install-prebuilt-packages.sh +++ b/components/core/tools/scripts/lib_install/manylinux_2_28/install-prebuilt-packages.sh @@ -1,7 +1,8 @@ #!/usr/bin/env bash -# Exit on any error, use of undefined variables, or failure within a pipeline -set -euo pipefail +set -o errexit +set -o nounset +set -o pipefail script_dir="$( cd "$( dirname "${BASH_SOURCE[0]}" )" &> /dev/null && pwd )" diff --git a/components/core/tools/scripts/lib_install/musllinux_1_2/install-all.sh b/components/core/tools/scripts/lib_install/musllinux_1_2/install-all.sh index 0b710aedb2..d2ca30c297 100755 --- a/components/core/tools/scripts/lib_install/musllinux_1_2/install-all.sh +++ b/components/core/tools/scripts/lib_install/musllinux_1_2/install-all.sh @@ -1,7 +1,8 @@ #!/usr/bin/env bash -# Exit on any error, use of undefined variables, or failure within a pipeline -set -euo pipefail +set -o errexit +set -o nounset +set -o pipefail script_dir="$( cd "$( dirname "${BASH_SOURCE[0]}" )" &> /dev/null && pwd )" diff --git a/components/core/tools/scripts/lib_install/musllinux_1_2/install-prebuilt-packages.sh b/components/core/tools/scripts/lib_install/musllinux_1_2/install-prebuilt-packages.sh index 5c1e8d932a..45065b3aba 100755 --- a/components/core/tools/scripts/lib_install/musllinux_1_2/install-prebuilt-packages.sh +++ b/components/core/tools/scripts/lib_install/musllinux_1_2/install-prebuilt-packages.sh @@ -1,7 +1,8 @@ #!/usr/bin/env bash -# Exit on any error, use of undefined variables, or failure within a pipeline -set -euo pipefail +set -o errexit +set -o nounset +set -o pipefail script_dir="$( cd "$( dirname "${BASH_SOURCE[0]}" )" &> /dev/null && pwd )" diff --git a/components/core/tools/scripts/lib_install/pipx_install/install-all.sh b/components/core/tools/scripts/lib_install/pipx_install/install-all.sh index 0a5ff04caa..19348648d4 100755 --- a/components/core/tools/scripts/lib_install/pipx_install/install-all.sh +++ b/components/core/tools/scripts/lib_install/pipx_install/install-all.sh @@ -1,7 +1,8 @@ #!/usr/bin/env bash -# Exit on any error, use of undefined variables, or failure within a pipeline -set -euo pipefail +set -o errexit +set -o nounset +set -o pipefail script_dir="$( cd "$( dirname "${BASH_SOURCE[0]}" )" &> /dev/null && pwd )" diff --git a/components/core/tools/scripts/lib_install/pipx_install/install-cmake.sh b/components/core/tools/scripts/lib_install/pipx_install/install-cmake.sh index 8880d49563..2b10b44d91 100755 --- a/components/core/tools/scripts/lib_install/pipx_install/install-cmake.sh +++ b/components/core/tools/scripts/lib_install/pipx_install/install-cmake.sh @@ -1,7 +1,8 @@ #!/usr/bin/env bash -# Exit on any error, use of undefined variables, or failure within a pipeline -set -euo pipefail +set -o errexit +set -o nounset +set -o pipefail script_dir="$( cd "$( dirname "${BASH_SOURCE[0]}" )" &> /dev/null && pwd )" diff --git a/components/core/tools/scripts/lib_install/pipx_install/install-go-task.sh b/components/core/tools/scripts/lib_install/pipx_install/install-go-task.sh index d940ce3404..e73afeaf3a 100755 --- a/components/core/tools/scripts/lib_install/pipx_install/install-go-task.sh +++ b/components/core/tools/scripts/lib_install/pipx_install/install-go-task.sh @@ -1,7 +1,8 @@ #!/usr/bin/env bash -# Exit on any error, use of undefined variables, or failure within a pipeline -set -euo pipefail +set -o errexit +set -o nounset +set -o pipefail script_dir="$( cd "$( dirname "${BASH_SOURCE[0]}" )" &> /dev/null && pwd )" diff --git a/components/core/tools/scripts/lib_install/pipx_install/install-uv.sh b/components/core/tools/scripts/lib_install/pipx_install/install-uv.sh index 552052533e..9c89dd0443 100755 --- a/components/core/tools/scripts/lib_install/pipx_install/install-uv.sh +++ b/components/core/tools/scripts/lib_install/pipx_install/install-uv.sh @@ -1,7 +1,8 @@ #!/usr/bin/env bash -# Exit on any error, use of undefined variables, or failure within a pipeline -set -euo pipefail +set -o errexit +set -o nounset +set -o pipefail script_dir="$( cd "$( dirname "${BASH_SOURCE[0]}" )" &> /dev/null && pwd )" diff --git a/components/core/tools/scripts/lib_install/ubuntu-jammy/install-all.sh b/components/core/tools/scripts/lib_install/ubuntu-jammy/install-all.sh index 0b710aedb2..d2ca30c297 100755 --- a/components/core/tools/scripts/lib_install/ubuntu-jammy/install-all.sh +++ b/components/core/tools/scripts/lib_install/ubuntu-jammy/install-all.sh @@ -1,7 +1,8 @@ #!/usr/bin/env bash -# Exit on any error, use of undefined variables, or failure within a pipeline -set -euo pipefail +set -o errexit +set -o nounset +set -o pipefail script_dir="$( cd "$( dirname "${BASH_SOURCE[0]}" )" &> /dev/null && pwd )" diff --git a/components/core/tools/scripts/lib_install/ubuntu-jammy/install-prebuilt-packages.sh b/components/core/tools/scripts/lib_install/ubuntu-jammy/install-prebuilt-packages.sh index 8d18596535..a307693ca5 100755 --- a/components/core/tools/scripts/lib_install/ubuntu-jammy/install-prebuilt-packages.sh +++ b/components/core/tools/scripts/lib_install/ubuntu-jammy/install-prebuilt-packages.sh @@ -1,7 +1,8 @@ #!/usr/bin/env bash -# Exit on any error, use of undefined variables, or failure within a pipeline -set -euo pipefail +set -o errexit +set -o nounset +set -o pipefail script_dir="$( cd "$( dirname "${BASH_SOURCE[0]}" )" &> /dev/null && pwd )" From 8c94fbd15e3626137c7ad0b4670302b18b7da4b3 Mon Sep 17 00:00:00 2001 From: Bingran Hu Date: Thu, 4 Sep 2025 03:36:03 -0400 Subject: [PATCH 49/69] Update cmake version check --- .../lib_version_checks/check-cmake-version.sh | 10 +++++++--- 1 file changed, 7 insertions(+), 3 deletions(-) diff --git a/components/core/tools/scripts/lib_install/lib_version_checks/check-cmake-version.sh b/components/core/tools/scripts/lib_install/lib_version_checks/check-cmake-version.sh index 0997ce8b58..41bcd2102f 100755 --- a/components/core/tools/scripts/lib_install/lib_version_checks/check-cmake-version.sh +++ b/components/core/tools/scripts/lib_install/lib_version_checks/check-cmake-version.sh @@ -7,9 +7,13 @@ set -o pipefail # Get the installed cmake version string cmake_version=$(cmake -E capabilities | jq --raw-output ".version.string") cmake_major_version=$(cmake -E capabilities | jq --raw-output ".version.major") +cmake_minor_version=$(cmake -E capabilities | jq --raw-output ".version.minor") -# Check if version is 4.0 or higher -if [[ "${cmake_major_version}" -ge "4" ]]; then - echo "Error: CMake version ${cmake_version} is currently unsupported (>= 4.0)." +# ystdlib requires CMake v3.23; ANTLR and yaml-cpp do not yet support CMake v4+. +# See also: https://github.com/y-scope/clp/issues/795 +if [[ "${cmake_major_version}" -lt 3 ]] || \ + [[ "${cmake_major_version}" -eq 3 && "${cmake_minor_version}" -lt 23 ]] || \ + [[ "${cmake_major_version}" -ge 4 ]]; then + echo "Error: CMake version ${cmake_version} is unsupported (require 3.23 ≤ version < 4.0)." exit 1 fi From 0c5dce1590accefb57e6d4c48cc7b7c50c3c1a54 Mon Sep 17 00:00:00 2001 From: Kirk Rodrigues <2454684+kirkrodrigues@users.noreply.github.com> Date: Fri, 5 Sep 2025 13:20:32 -0400 Subject: [PATCH 50/69] docs: Refer to issues redirectly. --- docs/README.md | 4 ++-- docs/src/dev-docs/building-package.md | 4 ++-- docs/src/dev-docs/components-core/index.md | 8 ++++---- docs/src/dev-docs/contributing-linting.md | 2 ++ 4 files changed, 10 insertions(+), 8 deletions(-) diff --git a/docs/README.md b/docs/README.md index 4cc025787c..e24ee9243c 100644 --- a/docs/README.md +++ b/docs/README.md @@ -14,7 +14,7 @@ this project: * [Node.js] >= 16 to be able to [view the output](#viewing-the-output) * Python 3.10 or later * [Task] 3.44.0 - * We pin the version to 3.44.0 due to unresolved [issues][clp-ffi-js-issue-110]. + * We pin the version to 3.44.0 due to [y-scope/clp-ffi-js#110]. ## Build Commands @@ -41,8 +41,8 @@ task docs:serve The command above will install [http-server] and serve the built docs site; `http-server` will print the address it binds to (usually http://localhost:8080). -[clp-ffi-js-issue-110]: https://github.com/y-scope/clp-ffi-js/issues/110 [git-lfs]: https://git-lfs.com [http-server]: https://www.npmjs.com/package/http-server [Node.js]: https://nodejs.org/en/download/current [Task]: https://taskfile.dev/ +[y-scope/clp-ffi-js#110]: https://github.com/y-scope/clp-ffi-js/issues/110 diff --git a/docs/src/dev-docs/building-package.md b/docs/src/dev-docs/building-package.md index bded1c7622..b899dbd706 100644 --- a/docs/src/dev-docs/building-package.md +++ b/docs/src/dev-docs/building-package.md @@ -14,7 +14,7 @@ prebuilt version instead, check out the [releases](https://github.com/y-scope/cl * python3-dev * python3-venv (for the version of Python installed) * [Task] 3.44.0 - * We pin the version to 3.44.0 due to unresolved [issues][clp-ffi-js-issue-110]. + * We pin the version to 3.44.0 due to [y-scope/clp-ffi-js#110]. ## Setup @@ -79,5 +79,5 @@ task docker-images:package This will create a Docker image named `clp-package:dev`. -[clp-ffi-js-issue-110]: https://github.com/y-scope/clp-ffi-js/issues/110 [Task]: https://taskfile.dev/ +[y-scope/clp-ffi-js#110]: https://github.com/y-scope/clp-ffi-js/issues/110 diff --git a/docs/src/dev-docs/components-core/index.md b/docs/src/dev-docs/components-core/index.md index 6374561bde..a210243223 100644 --- a/docs/src/dev-docs/components-core/index.md +++ b/docs/src/dev-docs/components-core/index.md @@ -11,9 +11,9 @@ CLP core is the low-level component that performs compression, decompression, an * std::source_location * [CMake] >= 3.23.0 and < 4.0.0 * Minimum version 3.23.0 is required for [yscope-dev-utils]. - * We constrain the version to < 4.0.0 due to unresolved [issues][clp-issue-795]. + * We constrain the version to < 4.0.0 due to [y-scope/clp#795]. * [Task] 3.44.0 - * We pin the version to 3.44.0 due to unresolved [issues][clp-ffi-js-issue-110]. + * We pin the version to 3.44.0 due to [y-scope/clp-ffi-js#110]. To build, we require some source dependencies, packages from package managers, and libraries built from source. @@ -123,8 +123,8 @@ regex-utils ::: [CMake]: https://cmake.org/ -[clp-ffi-js-issue-110]: https://github.com/y-scope/clp-ffi-js/issues/110 -[clp-issue-795]: https://github.com/y-scope/clp/issues/795 [feature-req]: https://github.com/y-scope/clp/issues/new?assignees=&labels=enhancement&template=feature-request.yml [Task]: https://taskfile.dev/ +[y-scope/clp-ffi-js#110]: https://github.com/y-scope/clp-ffi-js/issues/110 +[y-scope/clp#795]: https://github.com/y-scope/clp/issues/795 [yscope-dev-utils]: https://github.com/y-scope/yscope-dev-utils diff --git a/docs/src/dev-docs/contributing-linting.md b/docs/src/dev-docs/contributing-linting.md index 0a646bc681..135d43ec58 100644 --- a/docs/src/dev-docs/contributing-linting.md +++ b/docs/src/dev-docs/contributing-linting.md @@ -16,6 +16,7 @@ To run the linting tools, besides commonly installed tools like `tar`, you'll ne * Python 3.9 or newer * python3-venv (for the version of Python installed) * [Task] 3.44.0 + * We pin the version to 3.44.0 due to [y-scope/clp-ffi-js#110]. * [uv] >= 0.8 ## Running the linters @@ -38,4 +39,5 @@ task lint:fix [feature-req]: https://github.com/y-scope/clp/issues/new?assignees=&labels=enhancement&projects=&template=feature-request.yml [Task]: https://taskfile.dev/ [uv]: https://docs.astral.sh/uv/ +[y-scope/clp-ffi-js#110]: https://github.com/y-scope/clp-ffi-js/issues/110 [yscope-dev-utils]: https://github.com/y-scope/yscope-dev-utils From d928f5e16f5dfee672badf146d9aeb51d0d9063a Mon Sep 17 00:00:00 2001 From: Kirk Rodrigues <2454684+kirkrodrigues@users.noreply.github.com> Date: Mon, 8 Sep 2025 05:46:33 -0400 Subject: [PATCH 51/69] Refactor workflow. --- .github/workflows/clp-core-build-macos.yaml | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/.github/workflows/clp-core-build-macos.yaml b/.github/workflows/clp-core-build-macos.yaml index 2372aa71ba..a51d9b268b 100644 --- a/.github/workflows/clp-core-build-macos.yaml +++ b/.github/workflows/clp-core-build-macos.yaml @@ -66,11 +66,12 @@ jobs: rm -f /usr/local/bin/pydoc3* rm -f /usr/local/bin/python3* - - name: "Remove preinstalled brew programs which will conflict with pipx's installs" + - name: "Remove preinstalled programs which will conflict with the pipx-installed packages" run: | if brew list --formula cmake >/dev/null 2>&1; then brew uninstall --force cmake fi + - name: "Install dependencies" run: "./components/core/tools/scripts/lib_install/macos/install-all.sh" From f8bde46033c0ecd5f39bec8a2e9ddddb7b835581 Mon Sep 17 00:00:00 2001 From: Kirk Rodrigues <2454684+kirkrodrigues@users.noreply.github.com> Date: Mon, 8 Sep 2025 05:51:31 -0400 Subject: [PATCH 52/69] Only uninstall preinstalled packages that are actually installed. --- .../clp-env-base-manylinux_2_28-aarch64/Dockerfile | 5 ++--- .../clp-env-base-manylinux_2_28-x86_64/Dockerfile | 5 ++--- .../clp-env-base-musllinux_1_2-aarch64/Dockerfile | 5 ++--- .../clp-env-base-musllinux_1_2-x86_64/Dockerfile | 5 ++--- 4 files changed, 8 insertions(+), 12 deletions(-) diff --git a/components/core/tools/docker-images/clp-env-base-manylinux_2_28-aarch64/Dockerfile b/components/core/tools/docker-images/clp-env-base-manylinux_2_28-aarch64/Dockerfile index 406410b00d..b47ab452d0 100644 --- a/components/core/tools/docker-images/clp-env-base-manylinux_2_28-aarch64/Dockerfile +++ b/components/core/tools/docker-images/clp-env-base-manylinux_2_28-aarch64/Dockerfile @@ -5,9 +5,8 @@ WORKDIR /root RUN mkdir -p ./tools/scripts/lib_install COPY ./tools/scripts/lib_install ./tools/scripts/lib_install -RUN pipx uninstall cmake || true -RUN pipx uninstall go-task-bin || true -RUN pipx uninstall uv || true +RUN pipx uninstall cmake +RUN pipx uninstall uv RUN ./tools/scripts/lib_install/manylinux_2_28/install-all.sh diff --git a/components/core/tools/docker-images/clp-env-base-manylinux_2_28-x86_64/Dockerfile b/components/core/tools/docker-images/clp-env-base-manylinux_2_28-x86_64/Dockerfile index a4383f0781..0a11a31f7a 100644 --- a/components/core/tools/docker-images/clp-env-base-manylinux_2_28-x86_64/Dockerfile +++ b/components/core/tools/docker-images/clp-env-base-manylinux_2_28-x86_64/Dockerfile @@ -5,9 +5,8 @@ WORKDIR /root RUN mkdir -p ./tools/scripts/lib_install COPY ./tools/scripts/lib_install ./tools/scripts/lib_install -RUN pipx uninstall cmake || true -RUN pipx uninstall go-task-bin || true -RUN pipx uninstall uv || true +RUN pipx uninstall cmake +RUN pipx uninstall uv RUN ./tools/scripts/lib_install/manylinux_2_28/install-all.sh diff --git a/components/core/tools/docker-images/clp-env-base-musllinux_1_2-aarch64/Dockerfile b/components/core/tools/docker-images/clp-env-base-musllinux_1_2-aarch64/Dockerfile index 5ef42a1d07..d6e1d5cd92 100644 --- a/components/core/tools/docker-images/clp-env-base-musllinux_1_2-aarch64/Dockerfile +++ b/components/core/tools/docker-images/clp-env-base-musllinux_1_2-aarch64/Dockerfile @@ -5,9 +5,8 @@ WORKDIR /root RUN mkdir -p ./tools/scripts/lib_install COPY ./tools/scripts/lib_install ./tools/scripts/lib_install -RUN pipx uninstall cmake || true -RUN pipx uninstall go-task-bin || true -RUN pipx uninstall uv || true +RUN pipx uninstall cmake +RUN pipx uninstall uv RUN ./tools/scripts/lib_install/musllinux_1_2/install-all.sh diff --git a/components/core/tools/docker-images/clp-env-base-musllinux_1_2-x86_64/Dockerfile b/components/core/tools/docker-images/clp-env-base-musllinux_1_2-x86_64/Dockerfile index 09bc5889d4..65e92cf197 100644 --- a/components/core/tools/docker-images/clp-env-base-musllinux_1_2-x86_64/Dockerfile +++ b/components/core/tools/docker-images/clp-env-base-musllinux_1_2-x86_64/Dockerfile @@ -5,9 +5,8 @@ WORKDIR /root RUN mkdir -p ./tools/scripts/lib_install COPY ./tools/scripts/lib_install ./tools/scripts/lib_install -RUN pipx uninstall cmake || true -RUN pipx uninstall go-task-bin || true -RUN pipx uninstall uv || true +RUN pipx uninstall cmake +RUN pipx uninstall uv RUN ./tools/scripts/lib_install/musllinux_1_2/install-all.sh From 132c3b51c1eb0849d187322370848b86a7a017cd Mon Sep 17 00:00:00 2001 From: Kirk Rodrigues <2454684+kirkrodrigues@users.noreply.github.com> Date: Mon, 8 Sep 2025 05:53:42 -0400 Subject: [PATCH 53/69] Refactor comment about installing pipx packages. --- .../lib_install/centos-stream-9/install-prebuilt-packages.sh | 2 +- components/core/tools/scripts/lib_install/macos/install-all.sh | 2 +- .../lib_install/manylinux_2_28/install-prebuilt-packages.sh | 2 +- .../lib_install/musllinux_1_2/install-prebuilt-packages.sh | 2 +- .../lib_install/ubuntu-jammy/install-prebuilt-packages.sh | 2 +- 5 files changed, 5 insertions(+), 5 deletions(-) diff --git a/components/core/tools/scripts/lib_install/centos-stream-9/install-prebuilt-packages.sh b/components/core/tools/scripts/lib_install/centos-stream-9/install-prebuilt-packages.sh index 0664d262f2..e4d9e91365 100755 --- a/components/core/tools/scripts/lib_install/centos-stream-9/install-prebuilt-packages.sh +++ b/components/core/tools/scripts/lib_install/centos-stream-9/install-prebuilt-packages.sh @@ -26,5 +26,5 @@ if ! command -v pipx >/dev/null 2>&1; then python3 -m pip install pipx fi -# Install `cmake`, `go-task` and `uv` +# Install remaining packages through pipx "${script_dir}/../pipx_install/install-all.sh" diff --git a/components/core/tools/scripts/lib_install/macos/install-all.sh b/components/core/tools/scripts/lib_install/macos/install-all.sh index 381d1190ea..ae448f0296 100755 --- a/components/core/tools/scripts/lib_install/macos/install-all.sh +++ b/components/core/tools/scripts/lib_install/macos/install-all.sh @@ -27,5 +27,5 @@ if ! command -v pkg-config >/dev/null 2>&1; then brew install pkg-config fi -# Install `cmake`, `go-task` and `uv` +# Install remaining packages through pipx "${script_dir}/../pipx_install/install-all.sh" diff --git a/components/core/tools/scripts/lib_install/manylinux_2_28/install-prebuilt-packages.sh b/components/core/tools/scripts/lib_install/manylinux_2_28/install-prebuilt-packages.sh index 543c076d26..ef3e48f3a8 100755 --- a/components/core/tools/scripts/lib_install/manylinux_2_28/install-prebuilt-packages.sh +++ b/components/core/tools/scripts/lib_install/manylinux_2_28/install-prebuilt-packages.sh @@ -16,5 +16,5 @@ dnf install -y \ zlib-devel \ zlib-static -# Install `cmake`, `go-task` and `uv` +# Install remaining packages through pipx "${script_dir}/../pipx_install/install-all.sh" diff --git a/components/core/tools/scripts/lib_install/musllinux_1_2/install-prebuilt-packages.sh b/components/core/tools/scripts/lib_install/musllinux_1_2/install-prebuilt-packages.sh index 45065b3aba..e7c1d37161 100755 --- a/components/core/tools/scripts/lib_install/musllinux_1_2/install-prebuilt-packages.sh +++ b/components/core/tools/scripts/lib_install/musllinux_1_2/install-prebuilt-packages.sh @@ -17,5 +17,5 @@ apk update && apk add --no-cache \ zlib-dev \ zlib-static -# Install `cmake`, `go-task` and `uv` +# Install remaining packages through pipx "${script_dir}/../pipx_install/install-all.sh" diff --git a/components/core/tools/scripts/lib_install/ubuntu-jammy/install-prebuilt-packages.sh b/components/core/tools/scripts/lib_install/ubuntu-jammy/install-prebuilt-packages.sh index 504274915b..a27b93f260 100755 --- a/components/core/tools/scripts/lib_install/ubuntu-jammy/install-prebuilt-packages.sh +++ b/components/core/tools/scripts/lib_install/ubuntu-jammy/install-prebuilt-packages.sh @@ -30,5 +30,5 @@ DEBIAN_FRONTEND=noninteractive apt-get install --no-install-recommends -y \ software-properties-common \ unzip -# Install `cmake`, `go-task` and `uv` +# Install remaining packages through pipx "${script_dir}/../pipx_install/install-all.sh" From e70f2e70abe867518cebe67a1682794c49888b6e Mon Sep 17 00:00:00 2001 From: Kirk Rodrigues <2454684+kirkrodrigues@users.noreply.github.com> Date: Mon, 8 Sep 2025 05:55:28 -0400 Subject: [PATCH 54/69] Refactor version checking scripts. --- .../lib_version_checks/check-cmake-version.sh | 11 +++++------ .../lib_version_checks/check-go-task-version.sh | 11 +++-------- .../lib_version_checks/check-uv-version.sh | 6 ++---- .../lib_version_checks/print-go-task-version.yaml | 2 +- 4 files changed, 11 insertions(+), 19 deletions(-) diff --git a/components/core/tools/scripts/lib_install/lib_version_checks/check-cmake-version.sh b/components/core/tools/scripts/lib_install/lib_version_checks/check-cmake-version.sh index 41bcd2102f..9b877d94be 100755 --- a/components/core/tools/scripts/lib_install/lib_version_checks/check-cmake-version.sh +++ b/components/core/tools/scripts/lib_install/lib_version_checks/check-cmake-version.sh @@ -4,16 +4,15 @@ set -o errexit set -o nounset set -o pipefail -# Get the installed cmake version string cmake_version=$(cmake -E capabilities | jq --raw-output ".version.string") cmake_major_version=$(cmake -E capabilities | jq --raw-output ".version.major") cmake_minor_version=$(cmake -E capabilities | jq --raw-output ".version.minor") -# ystdlib requires CMake v3.23; ANTLR and yaml-cpp do not yet support CMake v4+. -# See also: https://github.com/y-scope/clp/issues/795 -if [[ "${cmake_major_version}" -lt 3 ]] || \ - [[ "${cmake_major_version}" -eq 3 && "${cmake_minor_version}" -lt 23 ]] || \ - [[ "${cmake_major_version}" -ge 4 ]]; then +# ystdlib requires CMake v3.23; ANTLR and yaml-cpp do not yet support CMake v4+ +# (see https://github.com/y-scope/clp/issues/795). +if (( "${cmake_major_version}" < 3 )) || \ + (( "${cmake_major_version}" == 3 && "${cmake_minor_version}" < 23 )) || \ + (( "${cmake_major_version}" >= 4 )); then echo "Error: CMake version ${cmake_version} is unsupported (require 3.23 ≤ version < 4.0)." exit 1 fi diff --git a/components/core/tools/scripts/lib_install/lib_version_checks/check-go-task-version.sh b/components/core/tools/scripts/lib_install/lib_version_checks/check-go-task-version.sh index e58d9cad36..c03a8fdce7 100755 --- a/components/core/tools/scripts/lib_install/lib_version_checks/check-go-task-version.sh +++ b/components/core/tools/scripts/lib_install/lib_version_checks/check-go-task-version.sh @@ -6,15 +6,10 @@ set -o pipefail script_dir="$( cd "$( dirname "${BASH_SOURCE[0]}" )" &> /dev/null && pwd )" -# Get the installed go-task version string -task_version=$(task --silent --taskfile "${script_dir}/print-go-task-version.yaml" version) -IFS=. read -r task_major_version task_minor_version task_patch_version <<< "${task_version}" +task_version=$(task --silent --taskfile "${script_dir}/print-go-task-version.yaml") -# Check version constraints - # We lock `task` to version 3.44.0 to avoid https://github.com/y-scope/clp-ffi-js/issues/110 -if [ "${task_major_version}" -ne "3" ] || \ - [ "${task_minor_version}" -ne "44" ] || \ - [ "${task_patch_version}" -ne "0" ]; then +# We lock to version 3.44.0 to avoid https://github.com/y-scope/clp-ffi-js/issues/110 +if [[ "${task_version}" != "3.44.0" ]]; then echo "Error: Task version ${task_version} is currently unsupported (must be 3.44.0)." exit 1 fi diff --git a/components/core/tools/scripts/lib_install/lib_version_checks/check-uv-version.sh b/components/core/tools/scripts/lib_install/lib_version_checks/check-uv-version.sh index f24892c1d4..7df406fd58 100755 --- a/components/core/tools/scripts/lib_install/lib_version_checks/check-uv-version.sh +++ b/components/core/tools/scripts/lib_install/lib_version_checks/check-uv-version.sh @@ -4,12 +4,10 @@ set -o errexit set -o nounset set -o pipefail -# Get the installed uv version string uv_version=$(uv self version --output-format json | jq --raw-output ".version") IFS=. read -r uv_major_version uv_minor_version _ <<< "${uv_version}" -# Check version constraints -if [ "${uv_major_version}" -lt "1" ] && [ "${uv_minor_version}" -lt "8" ]; then - echo "Error: uv version ${uv_version} is currently unsupported (< 0.8)." +if (( "${uv_major_version}" == 0 && "${uv_minor_version}" < 8 )); then + echo "Error: uv version ${uv_version} is unsupported (require version ≥ 0.8)." exit 1 fi diff --git a/components/core/tools/scripts/lib_install/lib_version_checks/print-go-task-version.yaml b/components/core/tools/scripts/lib_install/lib_version_checks/print-go-task-version.yaml index 36bf9cbe2d..07961b280d 100644 --- a/components/core/tools/scripts/lib_install/lib_version_checks/print-go-task-version.yaml +++ b/components/core/tools/scripts/lib_install/lib_version_checks/print-go-task-version.yaml @@ -1,6 +1,6 @@ version: "3" tasks: - version: + default: cmd: >- echo {{ .TASK_VERSION | trimPrefix "v" }} From bd528a70975d014085f60ceb9868b0310af988f3 Mon Sep 17 00:00:00 2001 From: Kirk Rodrigues <2454684+kirkrodrigues@users.noreply.github.com> Date: Mon, 8 Sep 2025 05:56:03 -0400 Subject: [PATCH 55/69] Refactor installation scripts. --- .../lib_install/pipx_install/install-cmake.sh | 13 ++++++------- .../lib_install/pipx_install/install-go-task.sh | 9 ++++----- .../scripts/lib_install/pipx_install/install-uv.sh | 8 +++----- 3 files changed, 13 insertions(+), 17 deletions(-) diff --git a/components/core/tools/scripts/lib_install/pipx_install/install-cmake.sh b/components/core/tools/scripts/lib_install/pipx_install/install-cmake.sh index 2b10b44d91..3a4a598fd9 100755 --- a/components/core/tools/scripts/lib_install/pipx_install/install-cmake.sh +++ b/components/core/tools/scripts/lib_install/pipx_install/install-cmake.sh @@ -7,20 +7,19 @@ set -o pipefail script_dir="$( cd "$( dirname "${BASH_SOURCE[0]}" )" &> /dev/null && pwd )" if ! command -v pipx >/dev/null 2>&1; then - echo "Error: pipx is not available." + echo "Error: pipx not found." exit 1 fi if command -v cmake >/dev/null 2>&1; then version_check_script="${script_dir}/../lib_version_checks/check-cmake-version.sh" - if ! script_output=$("${version_check_script}" 2>&1); then - echo "${script_output}" - echo "Please manually uninstall the current cmake and then re-run the install script." + if ! "${version_check_script}"; then + echo "Please uninstall cmake and then re-run the install script." exit 1 fi else - # ystdlib requires CMake v3.23; ANTLR and yaml-cpp do not yet support CMake v4+. - # See also: https://github.com/y-scope/clp/issues/795 - pipx install --force "cmake>=3.23,<3.24" + # ystdlib requires CMake v3.23; ANTLR and yaml-cpp do not yet support CMake v4+ + # (see https://github.com/y-scope/clp/issues/795). + pipx install --force "cmake>=3.23,<4" pipx ensurepath fi diff --git a/components/core/tools/scripts/lib_install/pipx_install/install-go-task.sh b/components/core/tools/scripts/lib_install/pipx_install/install-go-task.sh index e73afeaf3a..97942a6862 100755 --- a/components/core/tools/scripts/lib_install/pipx_install/install-go-task.sh +++ b/components/core/tools/scripts/lib_install/pipx_install/install-go-task.sh @@ -7,19 +7,18 @@ set -o pipefail script_dir="$( cd "$( dirname "${BASH_SOURCE[0]}" )" &> /dev/null && pwd )" if ! command -v pipx >/dev/null 2>&1; then - echo "Error: pipx is not available." + echo "Error: pipx not found." exit 1 fi if command -v task >/dev/null 2>&1; then version_check_script="${script_dir}/../lib_version_checks/check-go-task-version.sh" - if ! script_output=$("${version_check_script}" 2>&1); then - echo "${script_output}" - echo "Please manually uninstall the current go-task and then re-run the install script." + if ! "${version_check_script}"; then + echo "Please uninstall go-task and then re-run the install script." exit 1 fi else - # We lock `task` to version 3.44.0 to avoid https://github.com/y-scope/clp-ffi-js/issues/110 + # We lock to version 3.44.0 to avoid https://github.com/y-scope/clp-ffi-js/issues/110 pipx install --force "go-task-bin==3.44.0" pipx ensurepath fi diff --git a/components/core/tools/scripts/lib_install/pipx_install/install-uv.sh b/components/core/tools/scripts/lib_install/pipx_install/install-uv.sh index 9c89dd0443..08c2097986 100755 --- a/components/core/tools/scripts/lib_install/pipx_install/install-uv.sh +++ b/components/core/tools/scripts/lib_install/pipx_install/install-uv.sh @@ -7,19 +7,17 @@ set -o pipefail script_dir="$( cd "$( dirname "${BASH_SOURCE[0]}" )" &> /dev/null && pwd )" if ! command -v pipx >/dev/null 2>&1; then - echo "Error: pipx is not available." + echo "Error: pipx not found." exit 1 fi if command -v uv >/dev/null 2>&1; then version_check_script="${script_dir}/../lib_version_checks/check-uv-version.sh" - if ! script_output=$("${version_check_script}" 2>&1); then - echo "${script_output}" - echo "Please manually uninstall the current uv and then re-run the install script." + if ! "${version_check_script}"; then + echo "Please uninstall uv and then re-run the install script." exit 1 fi else pipx install --force "uv>=0.8" pipx ensurepath fi - From cf30eeaca429e9e114110d0c4dfb539f9d2f4e94 Mon Sep 17 00:00:00 2001 From: Kirk Rodrigues <2454684+kirkrodrigues@users.noreply.github.com> Date: Mon, 8 Sep 2025 06:03:23 -0400 Subject: [PATCH 56/69] Add components/core/tools to yaml-linting. --- taskfiles/lint.yaml | 1 + 1 file changed, 1 insertion(+) diff --git a/taskfiles/lint.yaml b/taskfiles/lint.yaml index 2cf23949ff..758f1b15ee 100644 --- a/taskfiles/lint.yaml +++ b/taskfiles/lint.yaml @@ -100,6 +100,7 @@ tasks: .yamllint.yml \ components/core/.clang-format \ components/core/config \ + components/core/tools/ \ components/package-template/src/etc \ docs \ taskfile.yaml \ From d26a13f97d2fc7340a3deee45a180387a7268b92 Mon Sep 17 00:00:00 2001 From: Kirk Rodrigues <2454684+kirkrodrigues@users.noreply.github.com> Date: Mon, 8 Sep 2025 06:04:20 -0400 Subject: [PATCH 57/69] Apply shfmt. --- .../centos-stream-9/install-all.sh | 2 +- .../install-prebuilt-packages.sh | 2 +- .../lib_version_checks/check-cmake-version.sh | 10 ++--- .../check-go-task-version.sh | 6 +-- .../lib_version_checks/check-uv-version.sh | 8 ++-- .../scripts/lib_install/macos/install-all.sh | 22 +++++----- .../lib_install/manylinux_2_28/install-all.sh | 2 +- .../install-prebuilt-packages.sh | 2 +- .../lib_install/musllinux_1_2/install-all.sh | 2 +- .../install-prebuilt-packages.sh | 2 +- .../lib_install/pipx_install/install-all.sh | 2 +- .../lib_install/pipx_install/install-cmake.sh | 2 +- .../pipx_install/install-go-task.sh | 2 +- .../lib_install/pipx_install/install-uv.sh | 2 +- .../lib_install/ubuntu-jammy/install-all.sh | 2 +- .../ubuntu-jammy/install-prebuilt-packages.sh | 44 +++++++++---------- 16 files changed, 56 insertions(+), 56 deletions(-) diff --git a/components/core/tools/scripts/lib_install/centos-stream-9/install-all.sh b/components/core/tools/scripts/lib_install/centos-stream-9/install-all.sh index d2ca30c297..b24c2b96ba 100755 --- a/components/core/tools/scripts/lib_install/centos-stream-9/install-all.sh +++ b/components/core/tools/scripts/lib_install/centos-stream-9/install-all.sh @@ -4,7 +4,7 @@ set -o errexit set -o nounset set -o pipefail -script_dir="$( cd "$( dirname "${BASH_SOURCE[0]}" )" &> /dev/null && pwd )" +script_dir="$(cd "$(dirname "${BASH_SOURCE[0]}")" &>/dev/null && pwd)" "${script_dir}/install-prebuilt-packages.sh" "${script_dir}/install-packages-from-source.sh" diff --git a/components/core/tools/scripts/lib_install/centos-stream-9/install-prebuilt-packages.sh b/components/core/tools/scripts/lib_install/centos-stream-9/install-prebuilt-packages.sh index e4d9e91365..c39479f5f8 100755 --- a/components/core/tools/scripts/lib_install/centos-stream-9/install-prebuilt-packages.sh +++ b/components/core/tools/scripts/lib_install/centos-stream-9/install-prebuilt-packages.sh @@ -4,7 +4,7 @@ set -o errexit set -o nounset set -o pipefail -script_dir="$( cd "$( dirname "${BASH_SOURCE[0]}" )" &> /dev/null && pwd )" +script_dir="$(cd "$(dirname "${BASH_SOURCE[0]}")" &>/dev/null && pwd)" dnf install -y \ diffutils \ diff --git a/components/core/tools/scripts/lib_install/lib_version_checks/check-cmake-version.sh b/components/core/tools/scripts/lib_install/lib_version_checks/check-cmake-version.sh index 9b877d94be..bcc45a1d96 100755 --- a/components/core/tools/scripts/lib_install/lib_version_checks/check-cmake-version.sh +++ b/components/core/tools/scripts/lib_install/lib_version_checks/check-cmake-version.sh @@ -10,9 +10,9 @@ cmake_minor_version=$(cmake -E capabilities | jq --raw-output ".version.minor") # ystdlib requires CMake v3.23; ANTLR and yaml-cpp do not yet support CMake v4+ # (see https://github.com/y-scope/clp/issues/795). -if (( "${cmake_major_version}" < 3 )) || \ - (( "${cmake_major_version}" == 3 && "${cmake_minor_version}" < 23 )) || \ - (( "${cmake_major_version}" >= 4 )); then - echo "Error: CMake version ${cmake_version} is unsupported (require 3.23 ≤ version < 4.0)." - exit 1 +if (("${cmake_major_version}" < 3)) \ + || (("${cmake_major_version}" == 3 && "${cmake_minor_version}" < 23)) \ + || (("${cmake_major_version}" >= 4)); then + echo "Error: CMake version ${cmake_version} is unsupported (require 3.23 ≤ version < 4.0)." + exit 1 fi diff --git a/components/core/tools/scripts/lib_install/lib_version_checks/check-go-task-version.sh b/components/core/tools/scripts/lib_install/lib_version_checks/check-go-task-version.sh index c03a8fdce7..4604b66b2d 100755 --- a/components/core/tools/scripts/lib_install/lib_version_checks/check-go-task-version.sh +++ b/components/core/tools/scripts/lib_install/lib_version_checks/check-go-task-version.sh @@ -4,12 +4,12 @@ set -o errexit set -o nounset set -o pipefail -script_dir="$( cd "$( dirname "${BASH_SOURCE[0]}" )" &> /dev/null && pwd )" +script_dir="$(cd "$(dirname "${BASH_SOURCE[0]}")" &>/dev/null && pwd)" task_version=$(task --silent --taskfile "${script_dir}/print-go-task-version.yaml") # We lock to version 3.44.0 to avoid https://github.com/y-scope/clp-ffi-js/issues/110 if [[ "${task_version}" != "3.44.0" ]]; then - echo "Error: Task version ${task_version} is currently unsupported (must be 3.44.0)." - exit 1 + echo "Error: Task version ${task_version} is currently unsupported (must be 3.44.0)." + exit 1 fi diff --git a/components/core/tools/scripts/lib_install/lib_version_checks/check-uv-version.sh b/components/core/tools/scripts/lib_install/lib_version_checks/check-uv-version.sh index 7df406fd58..b6b022d4a7 100755 --- a/components/core/tools/scripts/lib_install/lib_version_checks/check-uv-version.sh +++ b/components/core/tools/scripts/lib_install/lib_version_checks/check-uv-version.sh @@ -5,9 +5,9 @@ set -o nounset set -o pipefail uv_version=$(uv self version --output-format json | jq --raw-output ".version") -IFS=. read -r uv_major_version uv_minor_version _ <<< "${uv_version}" +IFS=. read -r uv_major_version uv_minor_version _ <<<"${uv_version}" -if (( "${uv_major_version}" == 0 && "${uv_minor_version}" < 8 )); then - echo "Error: uv version ${uv_version} is unsupported (require version ≥ 0.8)." - exit 1 +if (("${uv_major_version}" == 0 && "${uv_minor_version}" < 8)); then + echo "Error: uv version ${uv_version} is unsupported (require version ≥ 0.8)." + exit 1 fi diff --git a/components/core/tools/scripts/lib_install/macos/install-all.sh b/components/core/tools/scripts/lib_install/macos/install-all.sh index ae448f0296..25b3fa836d 100755 --- a/components/core/tools/scripts/lib_install/macos/install-all.sh +++ b/components/core/tools/scripts/lib_install/macos/install-all.sh @@ -4,20 +4,20 @@ set -o errexit set -o nounset set -o pipefail -script_dir="$( cd "$( dirname "${BASH_SOURCE[0]}" )" &> /dev/null && pwd )" +script_dir="$(cd "$(dirname "${BASH_SOURCE[0]}")" &>/dev/null && pwd)" brew update brew install \ - coreutils \ - gcc \ - java11 \ - libarchive \ - llvm@16 \ - lz4 \ - mariadb-connector-c \ - pipx \ - xz \ - zstd + coreutils \ + gcc \ + java11 \ + libarchive \ + llvm@16 \ + lz4 \ + mariadb-connector-c \ + pipx \ + xz \ + zstd # Install pkg-config if it isn't already installed # NOTE: We might expect that pkg-config is installed through brew, so trying to install it again diff --git a/components/core/tools/scripts/lib_install/manylinux_2_28/install-all.sh b/components/core/tools/scripts/lib_install/manylinux_2_28/install-all.sh index d2ca30c297..b24c2b96ba 100755 --- a/components/core/tools/scripts/lib_install/manylinux_2_28/install-all.sh +++ b/components/core/tools/scripts/lib_install/manylinux_2_28/install-all.sh @@ -4,7 +4,7 @@ set -o errexit set -o nounset set -o pipefail -script_dir="$( cd "$( dirname "${BASH_SOURCE[0]}" )" &> /dev/null && pwd )" +script_dir="$(cd "$(dirname "${BASH_SOURCE[0]}")" &>/dev/null && pwd)" "${script_dir}/install-prebuilt-packages.sh" "${script_dir}/install-packages-from-source.sh" diff --git a/components/core/tools/scripts/lib_install/manylinux_2_28/install-prebuilt-packages.sh b/components/core/tools/scripts/lib_install/manylinux_2_28/install-prebuilt-packages.sh index ef3e48f3a8..fdf76a6fe7 100755 --- a/components/core/tools/scripts/lib_install/manylinux_2_28/install-prebuilt-packages.sh +++ b/components/core/tools/scripts/lib_install/manylinux_2_28/install-prebuilt-packages.sh @@ -4,7 +4,7 @@ set -o errexit set -o nounset set -o pipefail -script_dir="$( cd "$( dirname "${BASH_SOURCE[0]}" )" &> /dev/null && pwd )" +script_dir="$(cd "$(dirname "${BASH_SOURCE[0]}")" &>/dev/null && pwd)" dnf install -y \ gcc-c++ \ diff --git a/components/core/tools/scripts/lib_install/musllinux_1_2/install-all.sh b/components/core/tools/scripts/lib_install/musllinux_1_2/install-all.sh index d2ca30c297..b24c2b96ba 100755 --- a/components/core/tools/scripts/lib_install/musllinux_1_2/install-all.sh +++ b/components/core/tools/scripts/lib_install/musllinux_1_2/install-all.sh @@ -4,7 +4,7 @@ set -o errexit set -o nounset set -o pipefail -script_dir="$( cd "$( dirname "${BASH_SOURCE[0]}" )" &> /dev/null && pwd )" +script_dir="$(cd "$(dirname "${BASH_SOURCE[0]}")" &>/dev/null && pwd)" "${script_dir}/install-prebuilt-packages.sh" "${script_dir}/install-packages-from-source.sh" diff --git a/components/core/tools/scripts/lib_install/musllinux_1_2/install-prebuilt-packages.sh b/components/core/tools/scripts/lib_install/musllinux_1_2/install-prebuilt-packages.sh index e7c1d37161..ee7d8039d9 100755 --- a/components/core/tools/scripts/lib_install/musllinux_1_2/install-prebuilt-packages.sh +++ b/components/core/tools/scripts/lib_install/musllinux_1_2/install-prebuilt-packages.sh @@ -4,7 +4,7 @@ set -o errexit set -o nounset set -o pipefail -script_dir="$( cd "$( dirname "${BASH_SOURCE[0]}" )" &> /dev/null && pwd )" +script_dir="$(cd "$(dirname "${BASH_SOURCE[0]}")" &>/dev/null && pwd)" apk update && apk add --no-cache \ bzip2-dev \ diff --git a/components/core/tools/scripts/lib_install/pipx_install/install-all.sh b/components/core/tools/scripts/lib_install/pipx_install/install-all.sh index 19348648d4..226425faff 100755 --- a/components/core/tools/scripts/lib_install/pipx_install/install-all.sh +++ b/components/core/tools/scripts/lib_install/pipx_install/install-all.sh @@ -4,7 +4,7 @@ set -o errexit set -o nounset set -o pipefail -script_dir="$( cd "$( dirname "${BASH_SOURCE[0]}" )" &> /dev/null && pwd )" +script_dir="$(cd "$(dirname "${BASH_SOURCE[0]}")" &>/dev/null && pwd)" "${script_dir}/install-cmake.sh" "${script_dir}/install-go-task.sh" diff --git a/components/core/tools/scripts/lib_install/pipx_install/install-cmake.sh b/components/core/tools/scripts/lib_install/pipx_install/install-cmake.sh index 3a4a598fd9..937459156f 100755 --- a/components/core/tools/scripts/lib_install/pipx_install/install-cmake.sh +++ b/components/core/tools/scripts/lib_install/pipx_install/install-cmake.sh @@ -4,7 +4,7 @@ set -o errexit set -o nounset set -o pipefail -script_dir="$( cd "$( dirname "${BASH_SOURCE[0]}" )" &> /dev/null && pwd )" +script_dir="$(cd "$(dirname "${BASH_SOURCE[0]}")" &>/dev/null && pwd)" if ! command -v pipx >/dev/null 2>&1; then echo "Error: pipx not found." diff --git a/components/core/tools/scripts/lib_install/pipx_install/install-go-task.sh b/components/core/tools/scripts/lib_install/pipx_install/install-go-task.sh index 97942a6862..18cb470d30 100755 --- a/components/core/tools/scripts/lib_install/pipx_install/install-go-task.sh +++ b/components/core/tools/scripts/lib_install/pipx_install/install-go-task.sh @@ -4,7 +4,7 @@ set -o errexit set -o nounset set -o pipefail -script_dir="$( cd "$( dirname "${BASH_SOURCE[0]}" )" &> /dev/null && pwd )" +script_dir="$(cd "$(dirname "${BASH_SOURCE[0]}")" &>/dev/null && pwd)" if ! command -v pipx >/dev/null 2>&1; then echo "Error: pipx not found." diff --git a/components/core/tools/scripts/lib_install/pipx_install/install-uv.sh b/components/core/tools/scripts/lib_install/pipx_install/install-uv.sh index 08c2097986..db9157df12 100755 --- a/components/core/tools/scripts/lib_install/pipx_install/install-uv.sh +++ b/components/core/tools/scripts/lib_install/pipx_install/install-uv.sh @@ -4,7 +4,7 @@ set -o errexit set -o nounset set -o pipefail -script_dir="$( cd "$( dirname "${BASH_SOURCE[0]}" )" &> /dev/null && pwd )" +script_dir="$(cd "$(dirname "${BASH_SOURCE[0]}")" &>/dev/null && pwd)" if ! command -v pipx >/dev/null 2>&1; then echo "Error: pipx not found." diff --git a/components/core/tools/scripts/lib_install/ubuntu-jammy/install-all.sh b/components/core/tools/scripts/lib_install/ubuntu-jammy/install-all.sh index d2ca30c297..b24c2b96ba 100755 --- a/components/core/tools/scripts/lib_install/ubuntu-jammy/install-all.sh +++ b/components/core/tools/scripts/lib_install/ubuntu-jammy/install-all.sh @@ -4,7 +4,7 @@ set -o errexit set -o nounset set -o pipefail -script_dir="$( cd "$( dirname "${BASH_SOURCE[0]}" )" &> /dev/null && pwd )" +script_dir="$(cd "$(dirname "${BASH_SOURCE[0]}")" &>/dev/null && pwd)" "${script_dir}/install-prebuilt-packages.sh" "${script_dir}/install-packages-from-source.sh" diff --git a/components/core/tools/scripts/lib_install/ubuntu-jammy/install-prebuilt-packages.sh b/components/core/tools/scripts/lib_install/ubuntu-jammy/install-prebuilt-packages.sh index a27b93f260..d9dc7ae5cd 100755 --- a/components/core/tools/scripts/lib_install/ubuntu-jammy/install-prebuilt-packages.sh +++ b/components/core/tools/scripts/lib_install/ubuntu-jammy/install-prebuilt-packages.sh @@ -4,31 +4,31 @@ set -o errexit set -o nounset set -o pipefail -script_dir="$( cd "$( dirname "${BASH_SOURCE[0]}" )" &> /dev/null && pwd )" +script_dir="$(cd "$(dirname "${BASH_SOURCE[0]}")" &>/dev/null && pwd)" apt-get update DEBIAN_FRONTEND=noninteractive apt-get install --no-install-recommends -y \ - ca-certificates \ - checkinstall \ - curl \ - build-essential \ - git \ - jq \ - libcurl4 \ - libcurl4-openssl-dev \ - liblzma-dev \ - libmariadb-dev \ - libssl-dev \ - openjdk-11-jdk \ - pipx \ - pkg-config \ - python3 \ - python3-dev \ - python3-pip \ - python3-venv \ - rsync \ - software-properties-common \ - unzip + ca-certificates \ + checkinstall \ + curl \ + build-essential \ + git \ + jq \ + libcurl4 \ + libcurl4-openssl-dev \ + liblzma-dev \ + libmariadb-dev \ + libssl-dev \ + openjdk-11-jdk \ + pipx \ + pkg-config \ + python3 \ + python3-dev \ + python3-pip \ + python3-venv \ + rsync \ + software-properties-common \ + unzip # Install remaining packages through pipx "${script_dir}/../pipx_install/install-all.sh" From e890c0ec7a7812705e2d12cc4d710b418e2f87c1 Mon Sep 17 00:00:00 2001 From: Bingran Hu Date: Mon, 8 Sep 2025 23:52:29 +0800 Subject: [PATCH 58/69] Apply suggestions from code review Co-authored-by: kirkrodrigues <2454684+kirkrodrigues@users.noreply.github.com> --- .../docker-images/clp-env-base-centos-stream-9/Dockerfile | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/components/core/tools/docker-images/clp-env-base-centos-stream-9/Dockerfile b/components/core/tools/docker-images/clp-env-base-centos-stream-9/Dockerfile index 6c54690414..4375a5c26b 100644 --- a/components/core/tools/docker-images/clp-env-base-centos-stream-9/Dockerfile +++ b/components/core/tools/docker-images/clp-env-base-centos-stream-9/Dockerfile @@ -5,8 +5,10 @@ WORKDIR /root RUN mkdir -p ./tools/scripts/lib_install ADD ./tools/scripts/lib_install ./tools/scripts/lib_install +# Set pipx environment vars so that the pipx scripts install into the global locations without +# passing the `--global` flag. ENV PIPX_BIN_DIR=/usr/local/bin -ENV PIPX_HOME=/opt/_internal/pipx +ENV PIPX_HOME=/opt/pipx RUN ./tools/scripts/lib_install/centos-stream-9/install-all.sh From 81a0c1f8bd020117ec8b797635ee4511a1be23cd Mon Sep 17 00:00:00 2001 From: Bingran Hu Date: Mon, 8 Sep 2025 12:14:00 -0400 Subject: [PATCH 59/69] copy changes to ubuntu-jammy --- .../tools/docker-images/clp-env-base-ubuntu-jammy/Dockerfile | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/components/core/tools/docker-images/clp-env-base-ubuntu-jammy/Dockerfile b/components/core/tools/docker-images/clp-env-base-ubuntu-jammy/Dockerfile index 1bd42907df..993bb8ae82 100644 --- a/components/core/tools/docker-images/clp-env-base-ubuntu-jammy/Dockerfile +++ b/components/core/tools/docker-images/clp-env-base-ubuntu-jammy/Dockerfile @@ -5,8 +5,10 @@ WORKDIR /root RUN mkdir -p ./tools/scripts/lib_install ADD ./tools/scripts/lib_install ./tools/scripts/lib_install +# Set pipx environment vars so that the pipx scripts install into the global locations without +# passing the `--global` flag. ENV PIPX_BIN_DIR=/usr/local/bin -ENV PIPX_HOME=/opt/_internal/pipx +ENV PIPX_HOME=/opt/pipx RUN ./tools/scripts/lib_install/ubuntu-jammy/install-all.sh From 1ed15ec4c1ef408439aaf233395dcec10bdd072e Mon Sep 17 00:00:00 2001 From: Bingran Hu Date: Mon, 8 Sep 2025 12:59:41 -0400 Subject: [PATCH 60/69] refactor pipx-packages scripts --- .../install-prebuilt-packages.sh | 2 +- .../lib_version_checks/check-cmake-version.sh | 12 ------ .../scripts/lib_install/macos/install-all.sh | 2 +- .../install-prebuilt-packages.sh | 2 +- .../install-prebuilt-packages.sh | 2 +- .../install-all.sh | 0 .../pipx-packages/install-cmake.sh | 40 +++++++++++++++++++ .../pipx-packages/install-go-task.sh | 36 +++++++++++++++++ .../lib_install/pipx-packages/install-uv.sh | 33 +++++++++++++++ .../print-go-task-version.yaml | 0 .../lib_install/pipx_install/install-cmake.sh | 25 ------------ .../pipx_install/install-go-task.sh | 24 ----------- .../lib_install/pipx_install/install-uv.sh | 23 ----------- .../ubuntu-jammy/install-prebuilt-packages.sh | 2 +- 14 files changed, 114 insertions(+), 89 deletions(-) rename components/core/tools/scripts/lib_install/{pipx_install => pipx-packages}/install-all.sh (100%) create mode 100755 components/core/tools/scripts/lib_install/pipx-packages/install-cmake.sh create mode 100755 components/core/tools/scripts/lib_install/pipx-packages/install-go-task.sh create mode 100755 components/core/tools/scripts/lib_install/pipx-packages/install-uv.sh rename components/core/tools/scripts/lib_install/{lib_version_checks => pipx-packages}/print-go-task-version.yaml (100%) delete mode 100755 components/core/tools/scripts/lib_install/pipx_install/install-cmake.sh delete mode 100755 components/core/tools/scripts/lib_install/pipx_install/install-go-task.sh delete mode 100755 components/core/tools/scripts/lib_install/pipx_install/install-uv.sh diff --git a/components/core/tools/scripts/lib_install/centos-stream-9/install-prebuilt-packages.sh b/components/core/tools/scripts/lib_install/centos-stream-9/install-prebuilt-packages.sh index c39479f5f8..a8ee612930 100755 --- a/components/core/tools/scripts/lib_install/centos-stream-9/install-prebuilt-packages.sh +++ b/components/core/tools/scripts/lib_install/centos-stream-9/install-prebuilt-packages.sh @@ -27,4 +27,4 @@ if ! command -v pipx >/dev/null 2>&1; then fi # Install remaining packages through pipx -"${script_dir}/../pipx_install/install-all.sh" +"${script_dir}/../pipx-packages/install-all.sh" diff --git a/components/core/tools/scripts/lib_install/lib_version_checks/check-cmake-version.sh b/components/core/tools/scripts/lib_install/lib_version_checks/check-cmake-version.sh index bcc45a1d96..ca540b6bb7 100755 --- a/components/core/tools/scripts/lib_install/lib_version_checks/check-cmake-version.sh +++ b/components/core/tools/scripts/lib_install/lib_version_checks/check-cmake-version.sh @@ -4,15 +4,3 @@ set -o errexit set -o nounset set -o pipefail -cmake_version=$(cmake -E capabilities | jq --raw-output ".version.string") -cmake_major_version=$(cmake -E capabilities | jq --raw-output ".version.major") -cmake_minor_version=$(cmake -E capabilities | jq --raw-output ".version.minor") - -# ystdlib requires CMake v3.23; ANTLR and yaml-cpp do not yet support CMake v4+ -# (see https://github.com/y-scope/clp/issues/795). -if (("${cmake_major_version}" < 3)) \ - || (("${cmake_major_version}" == 3 && "${cmake_minor_version}" < 23)) \ - || (("${cmake_major_version}" >= 4)); then - echo "Error: CMake version ${cmake_version} is unsupported (require 3.23 ≤ version < 4.0)." - exit 1 -fi diff --git a/components/core/tools/scripts/lib_install/macos/install-all.sh b/components/core/tools/scripts/lib_install/macos/install-all.sh index 25b3fa836d..d2f4ea8f18 100755 --- a/components/core/tools/scripts/lib_install/macos/install-all.sh +++ b/components/core/tools/scripts/lib_install/macos/install-all.sh @@ -28,4 +28,4 @@ if ! command -v pkg-config >/dev/null 2>&1; then fi # Install remaining packages through pipx -"${script_dir}/../pipx_install/install-all.sh" +"${script_dir}/../pipx-packages/install-all.sh" diff --git a/components/core/tools/scripts/lib_install/manylinux_2_28/install-prebuilt-packages.sh b/components/core/tools/scripts/lib_install/manylinux_2_28/install-prebuilt-packages.sh index fdf76a6fe7..c89b31fbf7 100755 --- a/components/core/tools/scripts/lib_install/manylinux_2_28/install-prebuilt-packages.sh +++ b/components/core/tools/scripts/lib_install/manylinux_2_28/install-prebuilt-packages.sh @@ -17,4 +17,4 @@ dnf install -y \ zlib-static # Install remaining packages through pipx -"${script_dir}/../pipx_install/install-all.sh" +"${script_dir}/../pipx-packages/install-all.sh" diff --git a/components/core/tools/scripts/lib_install/musllinux_1_2/install-prebuilt-packages.sh b/components/core/tools/scripts/lib_install/musllinux_1_2/install-prebuilt-packages.sh index ee7d8039d9..158e17a198 100755 --- a/components/core/tools/scripts/lib_install/musllinux_1_2/install-prebuilt-packages.sh +++ b/components/core/tools/scripts/lib_install/musllinux_1_2/install-prebuilt-packages.sh @@ -18,4 +18,4 @@ apk update && apk add --no-cache \ zlib-static # Install remaining packages through pipx -"${script_dir}/../pipx_install/install-all.sh" +"${script_dir}/../pipx-packages/install-all.sh" diff --git a/components/core/tools/scripts/lib_install/pipx_install/install-all.sh b/components/core/tools/scripts/lib_install/pipx-packages/install-all.sh similarity index 100% rename from components/core/tools/scripts/lib_install/pipx_install/install-all.sh rename to components/core/tools/scripts/lib_install/pipx-packages/install-all.sh diff --git a/components/core/tools/scripts/lib_install/pipx-packages/install-cmake.sh b/components/core/tools/scripts/lib_install/pipx-packages/install-cmake.sh new file mode 100755 index 0000000000..6d9cf138e1 --- /dev/null +++ b/components/core/tools/scripts/lib_install/pipx-packages/install-cmake.sh @@ -0,0 +1,40 @@ +#!/usr/bin/env bash + +set -o errexit +set -o nounset +set -o pipefail + +if ! command -v pipx >/dev/null 2>&1; then + echo "Error: pipx not found." + exit 1 +fi + +package_exists=1 +if ! command -v cmake >/dev/null 2>&1; then + package_exists=0 + # ystdlib requires CMake v3.23; ANTLR and yaml-cpp do not yet support CMake v4+ + # (see https://github.com/y-scope/clp/issues/795). + pipx install --force "cmake>=3.23,<4" + pipx ensurepath +fi + +cmake_version=$(cmake -E capabilities | jq --raw-output ".version.string") +cmake_major_version=$(cmake -E capabilities | jq --raw-output ".version.major") +cmake_minor_version=$(cmake -E capabilities | jq --raw-output ".version.minor") + +# ystdlib requires CMake v3.23; ANTLR and yaml-cpp do not yet support CMake v4+ +# (see https://github.com/y-scope/clp/issues/795). +if (("${cmake_major_version}" < 3)) \ + || (("${cmake_major_version}" == 3 && "${cmake_minor_version}" < 23)) \ + || (("${cmake_major_version}" >= 4)); then + echo "Error: CMake version ${cmake_version} is unsupported (require 3.23 ≤ version < 4.0)." + + if package_exists; then + echo "Please uninstall CMake and then re-run the install script." + else + echo "Pipx failed to install the required version of CMake." + echo "To uninstall, run:" + echo " pipx uninstall cmake" + fi + exit 1 +fi diff --git a/components/core/tools/scripts/lib_install/pipx-packages/install-go-task.sh b/components/core/tools/scripts/lib_install/pipx-packages/install-go-task.sh new file mode 100755 index 0000000000..d9cd21d827 --- /dev/null +++ b/components/core/tools/scripts/lib_install/pipx-packages/install-go-task.sh @@ -0,0 +1,36 @@ +#!/usr/bin/env bash + +set -o errexit +set -o nounset +set -o pipefail + +script_dir="$(cd "$(dirname "${BASH_SOURCE[0]}")" &>/dev/null && pwd)" + +if ! command -v pipx >/dev/null 2>&1; then + echo "Error: pipx not found." + exit 1 +fi + +package_exists=1 +if ! command -v task >/dev/null 2>&1; then + package_exists=0 + # We lock to version 3.44.0 to avoid https://github.com/y-scope/clp-ffi-js/issues/110 + pipx install --force "go-task-bin==3.44.0" + pipx ensurepath +fi + +task_version=$(task --silent --taskfile "${script_dir}/print-go-task-version.yaml") + +# We lock to version 3.44.0 to avoid https://github.com/y-scope/clp-ffi-js/issues/110 +if [[ "${task_version}" != "3.44.0" ]]; then + echo "Error: Task version ${task_version} is currently unsupported (must be 3.44.0)." + + if package_exists; then + echo "Please uninstall Task and then re-run the install script." + else + echo "Pipx failed to install the required version of Task." + echo "To uninstall, run:" + echo " pipx uninstall go-task-bin" + fi + exit 1 +fi diff --git a/components/core/tools/scripts/lib_install/pipx-packages/install-uv.sh b/components/core/tools/scripts/lib_install/pipx-packages/install-uv.sh new file mode 100755 index 0000000000..3a89b9a95b --- /dev/null +++ b/components/core/tools/scripts/lib_install/pipx-packages/install-uv.sh @@ -0,0 +1,33 @@ +#!/usr/bin/env bash + +set -o errexit +set -o nounset +set -o pipefail + +if ! command -v pipx >/dev/null 2>&1; then + echo "Error: pipx not found." + exit 1 +fi + +package_exists=1 +if ! command -v uv >/dev/null 2>&1; then + package_exists=0 + pipx install --force "uv>=0.8" + pipx ensurepath +fi + +uv_version=$(uv self version --output-format json | jq --raw-output ".version") +IFS=. read -r uv_major_version uv_minor_version _ <<<"${uv_version}" + +if (("${uv_major_version}" == 0 && "${uv_minor_version}" < 8)); then + echo "Error: uv version ${uv_version} is unsupported (require version ≥ 0.8)." + + if package_exists; then + echo "Please uninstall uv and then re-run the install script." + else + echo "Pipx failed to install the required version of uv." + echo "To uninstall, run:" + echo " pipx uninstall uv" + fi + exit 1 +fi diff --git a/components/core/tools/scripts/lib_install/lib_version_checks/print-go-task-version.yaml b/components/core/tools/scripts/lib_install/pipx-packages/print-go-task-version.yaml similarity index 100% rename from components/core/tools/scripts/lib_install/lib_version_checks/print-go-task-version.yaml rename to components/core/tools/scripts/lib_install/pipx-packages/print-go-task-version.yaml diff --git a/components/core/tools/scripts/lib_install/pipx_install/install-cmake.sh b/components/core/tools/scripts/lib_install/pipx_install/install-cmake.sh deleted file mode 100755 index 937459156f..0000000000 --- a/components/core/tools/scripts/lib_install/pipx_install/install-cmake.sh +++ /dev/null @@ -1,25 +0,0 @@ -#!/usr/bin/env bash - -set -o errexit -set -o nounset -set -o pipefail - -script_dir="$(cd "$(dirname "${BASH_SOURCE[0]}")" &>/dev/null && pwd)" - -if ! command -v pipx >/dev/null 2>&1; then - echo "Error: pipx not found." - exit 1 -fi - -if command -v cmake >/dev/null 2>&1; then - version_check_script="${script_dir}/../lib_version_checks/check-cmake-version.sh" - if ! "${version_check_script}"; then - echo "Please uninstall cmake and then re-run the install script." - exit 1 - fi -else - # ystdlib requires CMake v3.23; ANTLR and yaml-cpp do not yet support CMake v4+ - # (see https://github.com/y-scope/clp/issues/795). - pipx install --force "cmake>=3.23,<4" - pipx ensurepath -fi diff --git a/components/core/tools/scripts/lib_install/pipx_install/install-go-task.sh b/components/core/tools/scripts/lib_install/pipx_install/install-go-task.sh deleted file mode 100755 index 18cb470d30..0000000000 --- a/components/core/tools/scripts/lib_install/pipx_install/install-go-task.sh +++ /dev/null @@ -1,24 +0,0 @@ -#!/usr/bin/env bash - -set -o errexit -set -o nounset -set -o pipefail - -script_dir="$(cd "$(dirname "${BASH_SOURCE[0]}")" &>/dev/null && pwd)" - -if ! command -v pipx >/dev/null 2>&1; then - echo "Error: pipx not found." - exit 1 -fi - -if command -v task >/dev/null 2>&1; then - version_check_script="${script_dir}/../lib_version_checks/check-go-task-version.sh" - if ! "${version_check_script}"; then - echo "Please uninstall go-task and then re-run the install script." - exit 1 - fi -else - # We lock to version 3.44.0 to avoid https://github.com/y-scope/clp-ffi-js/issues/110 - pipx install --force "go-task-bin==3.44.0" - pipx ensurepath -fi diff --git a/components/core/tools/scripts/lib_install/pipx_install/install-uv.sh b/components/core/tools/scripts/lib_install/pipx_install/install-uv.sh deleted file mode 100755 index db9157df12..0000000000 --- a/components/core/tools/scripts/lib_install/pipx_install/install-uv.sh +++ /dev/null @@ -1,23 +0,0 @@ -#!/usr/bin/env bash - -set -o errexit -set -o nounset -set -o pipefail - -script_dir="$(cd "$(dirname "${BASH_SOURCE[0]}")" &>/dev/null && pwd)" - -if ! command -v pipx >/dev/null 2>&1; then - echo "Error: pipx not found." - exit 1 -fi - -if command -v uv >/dev/null 2>&1; then - version_check_script="${script_dir}/../lib_version_checks/check-uv-version.sh" - if ! "${version_check_script}"; then - echo "Please uninstall uv and then re-run the install script." - exit 1 - fi -else - pipx install --force "uv>=0.8" - pipx ensurepath -fi diff --git a/components/core/tools/scripts/lib_install/ubuntu-jammy/install-prebuilt-packages.sh b/components/core/tools/scripts/lib_install/ubuntu-jammy/install-prebuilt-packages.sh index d9dc7ae5cd..63f679617a 100755 --- a/components/core/tools/scripts/lib_install/ubuntu-jammy/install-prebuilt-packages.sh +++ b/components/core/tools/scripts/lib_install/ubuntu-jammy/install-prebuilt-packages.sh @@ -31,4 +31,4 @@ DEBIAN_FRONTEND=noninteractive apt-get install --no-install-recommends -y \ unzip # Install remaining packages through pipx -"${script_dir}/../pipx_install/install-all.sh" +"${script_dir}/../pipx-packages/install-all.sh" From 56a9a545d76844b12f01c3530ec970b75ed6b646 Mon Sep 17 00:00:00 2001 From: Bingran Hu Date: Mon, 8 Sep 2025 13:03:06 -0400 Subject: [PATCH 61/69] fix original lib version checking script removal --- .../lib_version_checks/check-cmake-version.sh | 6 ------ .../lib_version_checks/check-go-task-version.sh | 15 --------------- .../lib_version_checks/check-uv-version.sh | 13 ------------- 3 files changed, 34 deletions(-) delete mode 100755 components/core/tools/scripts/lib_install/lib_version_checks/check-cmake-version.sh delete mode 100755 components/core/tools/scripts/lib_install/lib_version_checks/check-go-task-version.sh delete mode 100755 components/core/tools/scripts/lib_install/lib_version_checks/check-uv-version.sh diff --git a/components/core/tools/scripts/lib_install/lib_version_checks/check-cmake-version.sh b/components/core/tools/scripts/lib_install/lib_version_checks/check-cmake-version.sh deleted file mode 100755 index ca540b6bb7..0000000000 --- a/components/core/tools/scripts/lib_install/lib_version_checks/check-cmake-version.sh +++ /dev/null @@ -1,6 +0,0 @@ -#!/usr/bin/env bash - -set -o errexit -set -o nounset -set -o pipefail - diff --git a/components/core/tools/scripts/lib_install/lib_version_checks/check-go-task-version.sh b/components/core/tools/scripts/lib_install/lib_version_checks/check-go-task-version.sh deleted file mode 100755 index 4604b66b2d..0000000000 --- a/components/core/tools/scripts/lib_install/lib_version_checks/check-go-task-version.sh +++ /dev/null @@ -1,15 +0,0 @@ -#!/usr/bin/env bash - -set -o errexit -set -o nounset -set -o pipefail - -script_dir="$(cd "$(dirname "${BASH_SOURCE[0]}")" &>/dev/null && pwd)" - -task_version=$(task --silent --taskfile "${script_dir}/print-go-task-version.yaml") - -# We lock to version 3.44.0 to avoid https://github.com/y-scope/clp-ffi-js/issues/110 -if [[ "${task_version}" != "3.44.0" ]]; then - echo "Error: Task version ${task_version} is currently unsupported (must be 3.44.0)." - exit 1 -fi diff --git a/components/core/tools/scripts/lib_install/lib_version_checks/check-uv-version.sh b/components/core/tools/scripts/lib_install/lib_version_checks/check-uv-version.sh deleted file mode 100755 index b6b022d4a7..0000000000 --- a/components/core/tools/scripts/lib_install/lib_version_checks/check-uv-version.sh +++ /dev/null @@ -1,13 +0,0 @@ -#!/usr/bin/env bash - -set -o errexit -set -o nounset -set -o pipefail - -uv_version=$(uv self version --output-format json | jq --raw-output ".version") -IFS=. read -r uv_major_version uv_minor_version _ <<<"${uv_version}" - -if (("${uv_major_version}" == 0 && "${uv_minor_version}" < 8)); then - echo "Error: uv version ${uv_version} is unsupported (require version ≥ 0.8)." - exit 1 -fi From c395102c968586dc9e54265e7e59cd6c420de3ee Mon Sep 17 00:00:00 2001 From: Kirk Rodrigues <2454684+kirkrodrigues@users.noreply.github.com> Date: Wed, 10 Sep 2025 12:02:01 -0400 Subject: [PATCH 62/69] Rename package_exists to package_preinstalled; Use 0 for true and 1 for false; Fix condition check. --- .../scripts/lib_install/pipx-packages/install-cmake.sh | 6 +++--- .../scripts/lib_install/pipx-packages/install-go-task.sh | 6 +++--- .../tools/scripts/lib_install/pipx-packages/install-uv.sh | 6 +++--- 3 files changed, 9 insertions(+), 9 deletions(-) diff --git a/components/core/tools/scripts/lib_install/pipx-packages/install-cmake.sh b/components/core/tools/scripts/lib_install/pipx-packages/install-cmake.sh index 6d9cf138e1..4e52f43457 100755 --- a/components/core/tools/scripts/lib_install/pipx-packages/install-cmake.sh +++ b/components/core/tools/scripts/lib_install/pipx-packages/install-cmake.sh @@ -9,9 +9,9 @@ if ! command -v pipx >/dev/null 2>&1; then exit 1 fi -package_exists=1 +package_preinstalled=0 if ! command -v cmake >/dev/null 2>&1; then - package_exists=0 + package_preinstalled=1 # ystdlib requires CMake v3.23; ANTLR and yaml-cpp do not yet support CMake v4+ # (see https://github.com/y-scope/clp/issues/795). pipx install --force "cmake>=3.23,<4" @@ -29,7 +29,7 @@ if (("${cmake_major_version}" < 3)) \ || (("${cmake_major_version}" >= 4)); then echo "Error: CMake version ${cmake_version} is unsupported (require 3.23 ≤ version < 4.0)." - if package_exists; then + if ((0 == "${package_preinstalled}")); then echo "Please uninstall CMake and then re-run the install script." else echo "Pipx failed to install the required version of CMake." diff --git a/components/core/tools/scripts/lib_install/pipx-packages/install-go-task.sh b/components/core/tools/scripts/lib_install/pipx-packages/install-go-task.sh index d9cd21d827..3dcf347782 100755 --- a/components/core/tools/scripts/lib_install/pipx-packages/install-go-task.sh +++ b/components/core/tools/scripts/lib_install/pipx-packages/install-go-task.sh @@ -11,9 +11,9 @@ if ! command -v pipx >/dev/null 2>&1; then exit 1 fi -package_exists=1 +package_preinstalled=0 if ! command -v task >/dev/null 2>&1; then - package_exists=0 + package_preinstalled=1 # We lock to version 3.44.0 to avoid https://github.com/y-scope/clp-ffi-js/issues/110 pipx install --force "go-task-bin==3.44.0" pipx ensurepath @@ -25,7 +25,7 @@ task_version=$(task --silent --taskfile "${script_dir}/print-go-task-version.yam if [[ "${task_version}" != "3.44.0" ]]; then echo "Error: Task version ${task_version} is currently unsupported (must be 3.44.0)." - if package_exists; then + if ((0 == "${package_preinstalled}")); then echo "Please uninstall Task and then re-run the install script." else echo "Pipx failed to install the required version of Task." diff --git a/components/core/tools/scripts/lib_install/pipx-packages/install-uv.sh b/components/core/tools/scripts/lib_install/pipx-packages/install-uv.sh index 3a89b9a95b..584d0eb5d5 100755 --- a/components/core/tools/scripts/lib_install/pipx-packages/install-uv.sh +++ b/components/core/tools/scripts/lib_install/pipx-packages/install-uv.sh @@ -9,9 +9,9 @@ if ! command -v pipx >/dev/null 2>&1; then exit 1 fi -package_exists=1 +package_preinstalled=0 if ! command -v uv >/dev/null 2>&1; then - package_exists=0 + package_preinstalled=1 pipx install --force "uv>=0.8" pipx ensurepath fi @@ -22,7 +22,7 @@ IFS=. read -r uv_major_version uv_minor_version _ <<<"${uv_version}" if (("${uv_major_version}" == 0 && "${uv_minor_version}" < 8)); then echo "Error: uv version ${uv_version} is unsupported (require version ≥ 0.8)." - if package_exists; then + if ((0 == "${package_preinstalled}")); then echo "Please uninstall uv and then re-run the install script." else echo "Pipx failed to install the required version of uv." From 055a573c175f135426ac868db00298777c39fe4d Mon Sep 17 00:00:00 2001 From: Kirk Rodrigues <2454684+kirkrodrigues@users.noreply.github.com> Date: Wed, 10 Sep 2025 12:05:06 -0400 Subject: [PATCH 63/69] Minor refactorin. --- .../scripts/lib_install/pipx-packages/install-cmake.sh | 3 ++- .../scripts/lib_install/pipx-packages/install-go-task.sh | 6 +++--- .../tools/scripts/lib_install/pipx-packages/install-uv.sh | 3 ++- 3 files changed, 7 insertions(+), 5 deletions(-) diff --git a/components/core/tools/scripts/lib_install/pipx-packages/install-cmake.sh b/components/core/tools/scripts/lib_install/pipx-packages/install-cmake.sh index 4e52f43457..129a971399 100755 --- a/components/core/tools/scripts/lib_install/pipx-packages/install-cmake.sh +++ b/components/core/tools/scripts/lib_install/pipx-packages/install-cmake.sh @@ -32,9 +32,10 @@ if (("${cmake_major_version}" < 3)) \ if ((0 == "${package_preinstalled}")); then echo "Please uninstall CMake and then re-run the install script." else - echo "Pipx failed to install the required version of CMake." + echo "pipx failed to install the required version of CMake." echo "To uninstall, run:" echo " pipx uninstall cmake" fi + exit 1 fi diff --git a/components/core/tools/scripts/lib_install/pipx-packages/install-go-task.sh b/components/core/tools/scripts/lib_install/pipx-packages/install-go-task.sh index 3dcf347782..b5ec5ca290 100755 --- a/components/core/tools/scripts/lib_install/pipx-packages/install-go-task.sh +++ b/components/core/tools/scripts/lib_install/pipx-packages/install-go-task.sh @@ -4,8 +4,6 @@ set -o errexit set -o nounset set -o pipefail -script_dir="$(cd "$(dirname "${BASH_SOURCE[0]}")" &>/dev/null && pwd)" - if ! command -v pipx >/dev/null 2>&1; then echo "Error: pipx not found." exit 1 @@ -19,6 +17,7 @@ if ! command -v task >/dev/null 2>&1; then pipx ensurepath fi +script_dir="$(cd "$(dirname "${BASH_SOURCE[0]}")" &>/dev/null && pwd)" task_version=$(task --silent --taskfile "${script_dir}/print-go-task-version.yaml") # We lock to version 3.44.0 to avoid https://github.com/y-scope/clp-ffi-js/issues/110 @@ -28,9 +27,10 @@ if [[ "${task_version}" != "3.44.0" ]]; then if ((0 == "${package_preinstalled}")); then echo "Please uninstall Task and then re-run the install script." else - echo "Pipx failed to install the required version of Task." + echo "pipx failed to install the required version of Task." echo "To uninstall, run:" echo " pipx uninstall go-task-bin" fi + exit 1 fi diff --git a/components/core/tools/scripts/lib_install/pipx-packages/install-uv.sh b/components/core/tools/scripts/lib_install/pipx-packages/install-uv.sh index 584d0eb5d5..eb90b3185f 100755 --- a/components/core/tools/scripts/lib_install/pipx-packages/install-uv.sh +++ b/components/core/tools/scripts/lib_install/pipx-packages/install-uv.sh @@ -25,9 +25,10 @@ if (("${uv_major_version}" == 0 && "${uv_minor_version}" < 8)); then if ((0 == "${package_preinstalled}")); then echo "Please uninstall uv and then re-run the install script." else - echo "Pipx failed to install the required version of uv." + echo "pipx failed to install the required version of uv." echo "To uninstall, run:" echo " pipx uninstall uv" fi + exit 1 fi From 57df29e195ba09abeb7cde8e1b3ed6a830308b65 Mon Sep 17 00:00:00 2001 From: Kirk Rodrigues <2454684+kirkrodrigues@users.noreply.github.com> Date: Wed, 10 Sep 2025 12:10:31 -0400 Subject: [PATCH 64/69] Dedupliicate task version constraint. --- .../lib_install/pipx-packages/install-go-task.sh | 13 +++++++------ 1 file changed, 7 insertions(+), 6 deletions(-) diff --git a/components/core/tools/scripts/lib_install/pipx-packages/install-go-task.sh b/components/core/tools/scripts/lib_install/pipx-packages/install-go-task.sh index b5ec5ca290..5923a0a4cd 100755 --- a/components/core/tools/scripts/lib_install/pipx-packages/install-go-task.sh +++ b/components/core/tools/scripts/lib_install/pipx-packages/install-go-task.sh @@ -9,20 +9,21 @@ if ! command -v pipx >/dev/null 2>&1; then exit 1 fi +# We lock to version 3.44.0 to avoid https://github.com/y-scope/clp-ffi-js/issues/110 +readonly required_version="3.44.0" + package_preinstalled=0 if ! command -v task >/dev/null 2>&1; then package_preinstalled=1 - # We lock to version 3.44.0 to avoid https://github.com/y-scope/clp-ffi-js/issues/110 - pipx install --force "go-task-bin==3.44.0" + pipx install --force "go-task-bin==${required_version}" pipx ensurepath fi script_dir="$(cd "$(dirname "${BASH_SOURCE[0]}")" &>/dev/null && pwd)" task_version=$(task --silent --taskfile "${script_dir}/print-go-task-version.yaml") - -# We lock to version 3.44.0 to avoid https://github.com/y-scope/clp-ffi-js/issues/110 -if [[ "${task_version}" != "3.44.0" ]]; then - echo "Error: Task version ${task_version} is currently unsupported (must be 3.44.0)." +if [[ "${task_version}" != "${required_version}" ]]; then + echo "Error: Task version ${task_version} is currently unsupported (must be" \ + "${required_version})." if ((0 == "${package_preinstalled}")); then echo "Please uninstall Task and then re-run the install script." From 31b21c555ac0a0d8a907b6afb34745eaeaf967b6 Mon Sep 17 00:00:00 2001 From: Kirk Rodrigues <2454684+kirkrodrigues@users.noreply.github.com> Date: Wed, 10 Sep 2025 12:18:20 -0400 Subject: [PATCH 65/69] Dedupliicate uv version constraint. --- .../lib_install/pipx-packages/install-uv.sh | 16 +++++++++++----- 1 file changed, 11 insertions(+), 5 deletions(-) diff --git a/components/core/tools/scripts/lib_install/pipx-packages/install-uv.sh b/components/core/tools/scripts/lib_install/pipx-packages/install-uv.sh index eb90b3185f..a368dd7192 100755 --- a/components/core/tools/scripts/lib_install/pipx-packages/install-uv.sh +++ b/components/core/tools/scripts/lib_install/pipx-packages/install-uv.sh @@ -9,18 +9,24 @@ if ! command -v pipx >/dev/null 2>&1; then exit 1 fi +readonly required_version_major_min=0 +readonly required_version_minor_min=8 +readonly required_version_min="${required_version_major_min}.${required_version_minor_min}" + package_preinstalled=0 if ! command -v uv >/dev/null 2>&1; then package_preinstalled=1 - pipx install --force "uv>=0.8" + pipx install --force "uv>=${required_version_min}" pipx ensurepath fi -uv_version=$(uv self version --output-format json | jq --raw-output ".version") -IFS=. read -r uv_major_version uv_minor_version _ <<<"${uv_version}" +installed_version=$(uv self version --output-format json | jq --raw-output ".version") +IFS=. read -r installd_version_major installed_version_minor _ <<<"${installed_version}" -if (("${uv_major_version}" == 0 && "${uv_minor_version}" < 8)); then - echo "Error: uv version ${uv_version} is unsupported (require version ≥ 0.8)." +if (("${installd_version_major}" == "${required_version_major_min}" && \ + "${installed_version_minor}" < "${required_version_minor_min}")); then + echo "Error: uv version ${installed_version} is unsupported (require version" \ + "≥ ${required_version_min})." if ((0 == "${package_preinstalled}")); then echo "Please uninstall uv and then re-run the install script." From 9297f67c8270fe8dc3b886126317a2c3a82421b7 Mon Sep 17 00:00:00 2001 From: Kirk Rodrigues <2454684+kirkrodrigues@users.noreply.github.com> Date: Wed, 10 Sep 2025 12:26:04 -0400 Subject: [PATCH 66/69] Dedupliicate cmake version constraint. --- .../pipx-packages/install-cmake.sh | 24 ++++++++++++------- 1 file changed, 16 insertions(+), 8 deletions(-) diff --git a/components/core/tools/scripts/lib_install/pipx-packages/install-cmake.sh b/components/core/tools/scripts/lib_install/pipx-packages/install-cmake.sh index 129a971399..855a79b731 100755 --- a/components/core/tools/scripts/lib_install/pipx-packages/install-cmake.sh +++ b/components/core/tools/scripts/lib_install/pipx-packages/install-cmake.sh @@ -9,25 +9,33 @@ if ! command -v pipx >/dev/null 2>&1; then exit 1 fi +readonly required_version_major_min=3 +readonly required_version_minor_min=23 +readonly required_version_min="${required_version_major_min}.${required_version_minor_min}" +readonly required_version_major_max=3 +readonly required_version_major_max_plus_1=$((required_version_major_max + 1)) + package_preinstalled=0 if ! command -v cmake >/dev/null 2>&1; then package_preinstalled=1 # ystdlib requires CMake v3.23; ANTLR and yaml-cpp do not yet support CMake v4+ # (see https://github.com/y-scope/clp/issues/795). - pipx install --force "cmake>=3.23,<4" + pipx install --force "cmake>=${required_version_min},<${required_version_major_max_plus_1}" pipx ensurepath fi -cmake_version=$(cmake -E capabilities | jq --raw-output ".version.string") -cmake_major_version=$(cmake -E capabilities | jq --raw-output ".version.major") -cmake_minor_version=$(cmake -E capabilities | jq --raw-output ".version.minor") +installed_version=$(cmake -E capabilities | jq --raw-output ".version.string") +installed_version_major=$(cmake -E capabilities | jq --raw-output ".version.major") +installed_version_minor=$(cmake -E capabilities | jq --raw-output ".version.minor") # ystdlib requires CMake v3.23; ANTLR and yaml-cpp do not yet support CMake v4+ # (see https://github.com/y-scope/clp/issues/795). -if (("${cmake_major_version}" < 3)) \ - || (("${cmake_major_version}" == 3 && "${cmake_minor_version}" < 23)) \ - || (("${cmake_major_version}" >= 4)); then - echo "Error: CMake version ${cmake_version} is unsupported (require 3.23 ≤ version < 4.0)." +if (("${installed_version_major}" < "${required_version_major_min}")) \ + || (("${installed_version_major}" == "${required_version_major_min}" && \ + "${installed_version_minor}" < "${required_version_minor_min}")) \ + || (("${installed_version_major}" >= "${required_version_major_max_plus_1}")); then + echo "Error: CMake version ${installed_version} is unsupported (require" \ + "${required_version_min} ≤ version < ${required_version_major_max_plus_1})." if ((0 == "${package_preinstalled}")); then echo "Please uninstall CMake and then re-run the install script." From c1ab120e6828c45cd5459ec5c4b0744461b0c9bd Mon Sep 17 00:00:00 2001 From: Bingran Hu Date: Wed, 10 Sep 2025 13:20:16 -0400 Subject: [PATCH 67/69] Bump minimum required cmake ver of building clp-core to 3.23 --- components/core/CMakeLists.txt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/components/core/CMakeLists.txt b/components/core/CMakeLists.txt index 6761324359..ac8d84654c 100644 --- a/components/core/CMakeLists.txt +++ b/components/core/CMakeLists.txt @@ -1,4 +1,4 @@ -cmake_minimum_required(VERSION 3.21) +cmake_minimum_required(VERSION 3.23) # Toolchain setup must come before the first project() call in the entire CMake buildsystem. # If CLP is not the top-level project, the following setup has no effect. From a73e3899e41b987de8f577f7db777ceee995d6e7 Mon Sep 17 00:00:00 2001 From: Bingran Hu Date: Thu, 11 Sep 2025 01:39:23 +0800 Subject: [PATCH 68/69] Update components/core/CMakeLists.txt Co-authored-by: kirkrodrigues <2454684+kirkrodrigues@users.noreply.github.com> --- components/core/CMakeLists.txt | 1 + 1 file changed, 1 insertion(+) diff --git a/components/core/CMakeLists.txt b/components/core/CMakeLists.txt index ac8d84654c..d758a8ffa2 100644 --- a/components/core/CMakeLists.txt +++ b/components/core/CMakeLists.txt @@ -1,3 +1,4 @@ +# ystdlib requires CMake v3.23 or higher cmake_minimum_required(VERSION 3.23) # Toolchain setup must come before the first project() call in the entire CMake buildsystem. From 3bb30a6508c0a3f44bdb583ab49496dbce6859df Mon Sep 17 00:00:00 2001 From: Bingran Hu Date: Wed, 10 Sep 2025 14:00:37 -0400 Subject: [PATCH 69/69] Move script_dir to right before usage in updated shell scripts --- .../lib_install/centos-stream-9/install-prebuilt-packages.sh | 3 +-- components/core/tools/scripts/lib_install/macos/install-all.sh | 3 +-- .../lib_install/manylinux_2_28/install-prebuilt-packages.sh | 3 +-- .../lib_install/musllinux_1_2/install-prebuilt-packages.sh | 3 +-- .../lib_install/ubuntu-jammy/install-prebuilt-packages.sh | 3 +-- 5 files changed, 5 insertions(+), 10 deletions(-) diff --git a/components/core/tools/scripts/lib_install/centos-stream-9/install-prebuilt-packages.sh b/components/core/tools/scripts/lib_install/centos-stream-9/install-prebuilt-packages.sh index a8ee612930..fa56074a58 100755 --- a/components/core/tools/scripts/lib_install/centos-stream-9/install-prebuilt-packages.sh +++ b/components/core/tools/scripts/lib_install/centos-stream-9/install-prebuilt-packages.sh @@ -4,8 +4,6 @@ set -o errexit set -o nounset set -o pipefail -script_dir="$(cd "$(dirname "${BASH_SOURCE[0]}")" &>/dev/null && pwd)" - dnf install -y \ diffutils \ gcc-c++ \ @@ -27,4 +25,5 @@ if ! command -v pipx >/dev/null 2>&1; then fi # Install remaining packages through pipx +script_dir="$(cd "$(dirname "${BASH_SOURCE[0]}")" &>/dev/null && pwd)" "${script_dir}/../pipx-packages/install-all.sh" diff --git a/components/core/tools/scripts/lib_install/macos/install-all.sh b/components/core/tools/scripts/lib_install/macos/install-all.sh index d2f4ea8f18..aeb70a03be 100755 --- a/components/core/tools/scripts/lib_install/macos/install-all.sh +++ b/components/core/tools/scripts/lib_install/macos/install-all.sh @@ -4,8 +4,6 @@ set -o errexit set -o nounset set -o pipefail -script_dir="$(cd "$(dirname "${BASH_SOURCE[0]}")" &>/dev/null && pwd)" - brew update brew install \ coreutils \ @@ -28,4 +26,5 @@ if ! command -v pkg-config >/dev/null 2>&1; then fi # Install remaining packages through pipx +script_dir="$(cd "$(dirname "${BASH_SOURCE[0]}")" &>/dev/null && pwd)" "${script_dir}/../pipx-packages/install-all.sh" diff --git a/components/core/tools/scripts/lib_install/manylinux_2_28/install-prebuilt-packages.sh b/components/core/tools/scripts/lib_install/manylinux_2_28/install-prebuilt-packages.sh index c89b31fbf7..f73fadf19b 100755 --- a/components/core/tools/scripts/lib_install/manylinux_2_28/install-prebuilt-packages.sh +++ b/components/core/tools/scripts/lib_install/manylinux_2_28/install-prebuilt-packages.sh @@ -4,8 +4,6 @@ set -o errexit set -o nounset set -o pipefail -script_dir="$(cd "$(dirname "${BASH_SOURCE[0]}")" &>/dev/null && pwd)" - dnf install -y \ gcc-c++ \ java-11-openjdk \ @@ -17,4 +15,5 @@ dnf install -y \ zlib-static # Install remaining packages through pipx +script_dir="$(cd "$(dirname "${BASH_SOURCE[0]}")" &>/dev/null && pwd)" "${script_dir}/../pipx-packages/install-all.sh" diff --git a/components/core/tools/scripts/lib_install/musllinux_1_2/install-prebuilt-packages.sh b/components/core/tools/scripts/lib_install/musllinux_1_2/install-prebuilt-packages.sh index 158e17a198..79d9dbca47 100755 --- a/components/core/tools/scripts/lib_install/musllinux_1_2/install-prebuilt-packages.sh +++ b/components/core/tools/scripts/lib_install/musllinux_1_2/install-prebuilt-packages.sh @@ -4,8 +4,6 @@ set -o errexit set -o nounset set -o pipefail -script_dir="$(cd "$(dirname "${BASH_SOURCE[0]}")" &>/dev/null && pwd)" - apk update && apk add --no-cache \ bzip2-dev \ bzip2-static \ @@ -18,4 +16,5 @@ apk update && apk add --no-cache \ zlib-static # Install remaining packages through pipx +script_dir="$(cd "$(dirname "${BASH_SOURCE[0]}")" &>/dev/null && pwd)" "${script_dir}/../pipx-packages/install-all.sh" diff --git a/components/core/tools/scripts/lib_install/ubuntu-jammy/install-prebuilt-packages.sh b/components/core/tools/scripts/lib_install/ubuntu-jammy/install-prebuilt-packages.sh index 63f679617a..6e0e78b458 100755 --- a/components/core/tools/scripts/lib_install/ubuntu-jammy/install-prebuilt-packages.sh +++ b/components/core/tools/scripts/lib_install/ubuntu-jammy/install-prebuilt-packages.sh @@ -4,8 +4,6 @@ set -o errexit set -o nounset set -o pipefail -script_dir="$(cd "$(dirname "${BASH_SOURCE[0]}")" &>/dev/null && pwd)" - apt-get update DEBIAN_FRONTEND=noninteractive apt-get install --no-install-recommends -y \ ca-certificates \ @@ -31,4 +29,5 @@ DEBIAN_FRONTEND=noninteractive apt-get install --no-install-recommends -y \ unzip # Install remaining packages through pipx +script_dir="$(cd "$(dirname "${BASH_SOURCE[0]}")" &>/dev/null && pwd)" "${script_dir}/../pipx-packages/install-all.sh"