From 628ad41e47637117fad919b5ff688baebb064c4b Mon Sep 17 00:00:00 2001 From: Henry Borchers Date: Mon, 15 Dec 2025 16:05:05 -0600 Subject: [PATCH 1/3] ci: use ubuntu 24.04 as base for testing on ci --- ci/docker/linux/jenkins/Dockerfile | 32 ++++++++++++++--------- ci/docker/linux/jenkins/conan/profile.ini | 3 ++- ci/docker/linux/tox/Dockerfile | 22 +++++++--------- ci/docker/linux/tox/conan/profile.ini | 3 ++- 4 files changed, 33 insertions(+), 27 deletions(-) diff --git a/ci/docker/linux/jenkins/Dockerfile b/ci/docker/linux/jenkins/Dockerfile index 7d33d1d5..0e665344 100644 --- a/ci/docker/linux/jenkins/Dockerfile +++ b/ci/docker/linux/jenkins/Dockerfile @@ -14,9 +14,9 @@ ARG UV_CACHE_DIR=/.cache/uv FROM ghcr.io/astral-sh/uv:latest AS uv_builder -FROM ubuntu:22.04 AS base_image -RUN rm -f /etc/apt/apt.conf.d/docker-clean -RUN --mount=type=cache,target=/var/cache/apt \ +FROM ubuntu:24.04 AS base_image +RUN --mount=type=cache,target=/var/cache/apt,sharing=locked \ + --mount=type=cache,target=/var/lib/apt,sharing=locked \ DEBIAN_FRONTEND="noninteractive" apt-get update && \ apt-get install -yqq software-properties-common gpg-agent --no-install-recommends && \ apt-get -yqq install tzdata && \ @@ -32,14 +32,14 @@ RUN --mount=type=cache,target=/var/cache/apt \ python3-pip \ python3-venv \ ca-certificates \ - software-properties-common \ - jq \ - unzip && \ - rm -rf /var/lib/apt/lists/* + software-properties-common + #============================================================================== FROM base_image AS dr_memory_builder -RUN apt-get update && \ +RUN --mount=type=cache,target=/var/cache/apt,sharing=locked \ + --mount=type=cache,target=/var/lib/apt,sharing=locked \ + apt-get update && \ apt-get install -y --no-install-recommends wget WORKDIR /drmemory @@ -50,6 +50,10 @@ RUN wget --no-verbose https://github.com/DynamoRIO/drmemory/releases/download/re FROM base_image AS sonar_builder +RUN --mount=type=cache,target=/var/cache/apt,sharing=locked \ + --mount=type=cache,target=/var/lib/apt,sharing=locked \ + apt-get update && \ + apt-get install -yqq --no-install-recommends unzip RUN case $(uname -m) in \ x86_64) URL=https://sonarcloud.io/static/cpp/build-wrapper-linux-x86.zip ;; \ @@ -65,6 +69,11 @@ RUN case $(uname -m) in \ FROM base_image AS conan_builder +RUN --mount=type=cache,target=/var/cache/apt,sharing=locked \ + --mount=type=cache,target=/var/lib/apt,sharing=locked \ + DEBIAN_FRONTEND="noninteractive" apt-get update && \ + apt-get install -yqq --no-install-recommends jq + ARG CONAN_USER_HOME ARG PIP_EXTRA_INDEX_URL ARG PIP_INDEX_URL @@ -72,13 +81,13 @@ ARG UV_EXTRA_INDEX_URL ARG UV_INDEX_URL COPY --from=uv_builder /uv /uvx /bin/ -COPY ci/docker/linux/shared/ensure_compiler_support_by_conan.py /tmp/ensure_compiler_support_by_conan.py ARG UV_CACHE_DIR ARG PIP_DOWNLOAD_CACHE ARG CONAN_CENTER_PROXY_V2_URL ARG CONAN_HOME ENV UV_PROJECT=/tmp/uv ENV UV_FROZEN=1 +ENV UV_PROJECT_ENVIRONMENT=/tmp/uv/venv COPY ci/docker/linux/jenkins/conan/profile.ini ${CONAN_HOME}/profiles/default COPY ci/docker/shared/conan/remotes.json ${CONAN_HOME}/remotes.json RUN --mount=type=cache,target=${PIP_DOWNLOAD_CACHE} \ @@ -86,8 +95,8 @@ RUN --mount=type=cache,target=${PIP_DOWNLOAD_CACHE} \ --mount=type=bind,source=pyproject.toml,target=${UV_PROJECT}/pyproject.toml \ --mount=type=bind,source=uv.lock,target=${UV_PROJECT}/uv.lock \ --mount=type=bind,source=conanfile.py,target=/tmp/conanfile.py \ + --mount=type=tmpfs,target=${UV_PROJECT_ENVIRONMENT} \ uv run --only-group=conan conan profile detect --exist-ok && \ - python3 /tmp/ensure_compiler_support_by_conan.py $(uv run --only-group=conan conan config home)/settings.yml 11.4 && \ if [ "$(jq -r '.remotes[0].url' ${CONAN_HOME}/remotes.json )" != "${CONAN_CENTER_PROXY_V2_URL}" ]; then \ echo "Setting the conan center url for building cache: ${CONAN_CENTER_PROXY_V2_URL}" && \ cp ${CONAN_HOME}/remotes.json /tmp/remotes.json && \ @@ -95,7 +104,6 @@ RUN --mount=type=cache,target=${PIP_DOWNLOAD_CACHE} \ fi && \ uv run --only-group=conan conan install /tmp --build missing -pr:b=default && \ uv run --only-group=conan conan cache clean "*" -b --source --build --temp && \ - rm -rf venv && \ if [ "$(jq -r '.remotes[0].url' ${CONAN_HOME}/remotes.json )" != "${CONAN_CENTER_PROXY_V2_URL}" ]; then \ mv -f /tmp/remotes.json ${CONAN_HOME}/remotes.json; \ fi @@ -104,7 +112,7 @@ RUN --mount=type=cache,target=${PIP_DOWNLOAD_CACHE} \ # shared files are needed to run the Build C++ Tests stage FROM base_image - +COPY --from=uv_builder /uv /uvx /bin/ # ----------------------------------------------------------------------------- # Install sonar build-wrapper # ----------------------------------------------------------------------------- diff --git a/ci/docker/linux/jenkins/conan/profile.ini b/ci/docker/linux/jenkins/conan/profile.ini index fb6fb255..fa240a11 100644 --- a/ci/docker/linux/jenkins/conan/profile.ini +++ b/ci/docker/linux/jenkins/conan/profile.ini @@ -1,8 +1,9 @@ +{% set compiler, version, compiler_exe = detect_api.detect_default_compiler() %} [settings] os=Linux arch=x86_64 compiler=gcc -compiler.version=11.4 +compiler.version={{version.major}}.{{version.minor}} compiler.libcxx=libstdc++ build_type=Release [options] diff --git a/ci/docker/linux/tox/Dockerfile b/ci/docker/linux/tox/Dockerfile index a7667106..93786abe 100644 --- a/ci/docker/linux/tox/Dockerfile +++ b/ci/docker/linux/tox/Dockerfile @@ -13,18 +13,17 @@ ARG UV_EXTRA_INDEX_URL FROM ghcr.io/astral-sh/uv:latest AS uv_builder -FROM ubuntu:22.04 AS wheel_builder_base -COPY ci/docker/linux/tox/apt-packages.txt /tmp/apt-packages.txt +FROM ubuntu:24.04 AS wheel_builder_base SHELL ["/bin/bash", "-o", "pipefail", "-c"] -RUN rm -f /etc/apt/apt.conf.d/docker-clean -RUN --mount=type=cache,target=/var/cache/apt \ +RUN --mount=type=cache,target=/var/cache/apt,sharing=locked \ + --mount=type=cache,target=/var/lib/apt,sharing=locked \ + --mount=type=bind,source=ci/docker/linux/tox/apt-packages.txt,target=/tmp/apt-packages.txt \ apt-get update && \ apt-get install -yqq --no-install-recommends software-properties-common gpg-agent && \ add-apt-repository ppa:deadsnakes/ppa && \ DEBIAN_FRONTEND=noninteractive TZ=Etc/UTC apt-get -yqq install --no-install-recommends tzdata && \ apt-get update && \ - < /tmp/apt-packages.txt xargs apt-get install -yqq --no-install-recommends && \ - rm -rf /var/lib/apt/lists/* + < /tmp/apt-packages.txt xargs apt-get install -yqq --no-install-recommends ARG CONAN_USER_HOME RUN mkdir -p ${CONAN_USER_HOME} @@ -33,20 +32,17 @@ FROM wheel_builder_base AS conan_builder COPY --from=uv_builder /uv /uvx /bin/ ARG CONAN_USER_HOME ARG CONAN_HOME -COPY ci/docker/linux/shared/ensure_compiler_support_by_conan.py /tmp/ensure_compiler_support_by_conan.py COPY ci/docker/shared/conan/remotes.json ${CONAN_HOME}/remotes.json COPY conanfile.py /tmp/conanfile.py ARG CONAN_CENTER_PROXY_V2_URL -ARG PIP_EXTRA_INDEX_URL -ARG PIP_INDEX_URL ARG UV_EXTRA_INDEX_URL ARG UV_INDEX_URL -ARG PIP_DOWNLOAD_CACHE ARG UV_CACHE_DIR ENV UV_PROJECT=/tmp/uv ENV UV_FROZEN=1 -RUN --mount=type=cache,target=${PIP_DOWNLOAD_CACHE} \ - --mount=type=cache,target=${UV_CACHE_DIR} \ +ENV UV_PROJECT_ENVIRONMENT=/tmp/uv/venv +RUN --mount=type=cache,target=${UV_CACHE_DIR} \ + --mount=type=tmpfs,target=${UV_PROJECT_ENVIRONMENT} \ --mount=type=bind,source=pyproject.toml,target=${UV_PROJECT}/pyproject.toml \ --mount=type=bind,source=uv.lock,target=${UV_PROJECT}/uv.lock \ --mount=type=bind,source=conanfile.py,target=/tmp/conanfile.py \ @@ -68,7 +64,7 @@ RUN --mount=type=cache,target=${PIP_DOWNLOAD_CACHE} \ FROM wheel_builder_base - +COPY --from=uv_builder /uv /uvx /bin/ ARG CONAN_USER_HOME ARG CONAN_HOME COPY --from=conan_builder --chmod=777 ${CONAN_HOME} ${CONAN_HOME} diff --git a/ci/docker/linux/tox/conan/profile.ini b/ci/docker/linux/tox/conan/profile.ini index 98e1d86a..7ec431c8 100644 --- a/ci/docker/linux/tox/conan/profile.ini +++ b/ci/docker/linux/tox/conan/profile.ini @@ -1,10 +1,11 @@ +{% set compiler, version, compiler_exe = detect_api.detect_default_compiler() %} [settings] os=Linux os_build=Linux arch=x86_64 arch_build=x86_64 compiler=gcc -compiler.version=11.4 +compiler.version={{version.major}}.{{version.minor}} compiler.libcxx=libstdc++ build_type=Release [options] From 1958ff51592a7a2c5e69487accc28ae6fd6a566a Mon Sep 17 00:00:00 2001 From: Henry Borchers Date: Mon, 15 Dec 2025 16:33:23 -0600 Subject: [PATCH 2/3] feat: support conan.lock --- MANIFEST.in | 2 +- ci/docker/linux/jenkins/Dockerfile | 1 + ci/docker/linux/tox/Dockerfile | 1 + conan.lock | 9 +++++++++ scripts/resources/package_for_linux/Dockerfile | 1 + scripts/resources/windows/Dockerfile | 3 ++- 6 files changed, 15 insertions(+), 2 deletions(-) create mode 100644 conan.lock diff --git a/MANIFEST.in b/MANIFEST.in index b437dda7..5bc72a8f 100644 --- a/MANIFEST.in +++ b/MANIFEST.in @@ -9,7 +9,7 @@ include cmake/fixup.cmake # C++ Source files include CMakeLists.txt -include conanfile.py +include conanfile.py conan.lock recursive-include src/uiucprescon/imagevalidate CMakeLists.txt *.cpp *.h # Tox configuration setting diff --git a/ci/docker/linux/jenkins/Dockerfile b/ci/docker/linux/jenkins/Dockerfile index 0e665344..b03a6b58 100644 --- a/ci/docker/linux/jenkins/Dockerfile +++ b/ci/docker/linux/jenkins/Dockerfile @@ -95,6 +95,7 @@ RUN --mount=type=cache,target=${PIP_DOWNLOAD_CACHE} \ --mount=type=bind,source=pyproject.toml,target=${UV_PROJECT}/pyproject.toml \ --mount=type=bind,source=uv.lock,target=${UV_PROJECT}/uv.lock \ --mount=type=bind,source=conanfile.py,target=/tmp/conanfile.py \ + --mount=type=bind,source=conan.lock,target=${UV_PROJECT}/conan.lock \ --mount=type=tmpfs,target=${UV_PROJECT_ENVIRONMENT} \ uv run --only-group=conan conan profile detect --exist-ok && \ if [ "$(jq -r '.remotes[0].url' ${CONAN_HOME}/remotes.json )" != "${CONAN_CENTER_PROXY_V2_URL}" ]; then \ diff --git a/ci/docker/linux/tox/Dockerfile b/ci/docker/linux/tox/Dockerfile index 93786abe..3c8a9234 100644 --- a/ci/docker/linux/tox/Dockerfile +++ b/ci/docker/linux/tox/Dockerfile @@ -46,6 +46,7 @@ RUN --mount=type=cache,target=${UV_CACHE_DIR} \ --mount=type=bind,source=pyproject.toml,target=${UV_PROJECT}/pyproject.toml \ --mount=type=bind,source=uv.lock,target=${UV_PROJECT}/uv.lock \ --mount=type=bind,source=conanfile.py,target=/tmp/conanfile.py \ + --mount=type=bind,source=conan.lock,target=${UV_PROJECT}/conan.lock \ uv run --only-group=conan conan profile detect --exist-ok && \ cat $(uv run --only-group=conan conan config home)/settings.yml && \ uv run --only-group=conan conan config home && \ diff --git a/conan.lock b/conan.lock new file mode 100644 index 00000000..c3818db8 --- /dev/null +++ b/conan.lock @@ -0,0 +1,9 @@ +{ + "version": "0.5", + "requires": [ + "openjpeg/2.3.1#ada21840f723c5410056aa68f0b23c61%1765835638.220325" + ], + "build_requires": [], + "python_requires": [], + "config_requires": [] +} \ No newline at end of file diff --git a/scripts/resources/package_for_linux/Dockerfile b/scripts/resources/package_for_linux/Dockerfile index 635ebb19..267ce015 100644 --- a/scripts/resources/package_for_linux/Dockerfile +++ b/scripts/resources/package_for_linux/Dockerfile @@ -69,6 +69,7 @@ RUN --mount=type=cache,target=${PIP_DOWNLOAD_CACHE} \ --mount=type=bind,source=pyproject.toml,target=${UV_PROJECT}/pyproject.toml \ --mount=type=bind,source=uv.lock,target=${UV_PROJECT}/uv.lock \ --mount=type=bind,source=conanfile.py,target=/tmp/conanfile.py \ + --mount=type=bind,source=conan.lock,target=/tmp/conan.lock \ uv run --only-group conan --with ninja conan profile detect --exist-ok && \ if [ "$(jq -r '.remotes[0].url' ${CONAN_HOME}/remotes.json )" != "${CONAN_CENTER_PROXY_V2_URL}" ]; then \ echo "Setting the conan center url for building cache: ${CONAN_CENTER_PROXY_V2_URL}" && \ diff --git a/scripts/resources/windows/Dockerfile b/scripts/resources/windows/Dockerfile index a94d1e07..cade0586 100644 --- a/scripts/resources/windows/Dockerfile +++ b/scripts/resources/windows/Dockerfile @@ -71,9 +71,10 @@ ARG UV_INDEX_URL ARG UV_EXTRA_INDEX_URL ENV UV_PROJECT=c:/temp/uv COPY conanfile.py c:/temp/conanfile.py -COPY pyproject.toml uv.lock ${UV_PROJECT}/ +COPY pyproject.toml uv.lock conan.lock ${UV_PROJECT}/ COPY scripts/resources/windows/conan/profile.ini ${CONAN_HOME}/profiles/default COPY ci/docker/shared/conan/remotes.json ${CONAN_HOME}/remotes.json + ARG CONAN_CENTER_PROXY_V2_URL ENV UV_FROZEN=1 RUN uv run --only-group conan --with ninja conan profile detect --exist-ok ; ` From fc3e2230976f861becb274b6f367d2608fe706bd Mon Sep 17 00:00:00 2001 From: Henry Borchers Date: Tue, 16 Dec 2025 08:00:55 -0600 Subject: [PATCH 3/3] ci: fix GitHub Actions * use uv lock file to constrain versions used * uses latest macOS version for runner * cache key includes conan.lock for conan dependencies --- .github/workflows/tox_matrix.yml | 6 +++--- .github/workflows/wheels.yml | 6 +++--- 2 files changed, 6 insertions(+), 6 deletions(-) diff --git a/.github/workflows/tox_matrix.yml b/.github/workflows/tox_matrix.yml index ebb8f249..25cef898 100644 --- a/.github/workflows/tox_matrix.yml +++ b/.github/workflows/tox_matrix.yml @@ -68,7 +68,7 @@ jobs: uses: actions/cache@v3 with: path: "${{steps.conan-path-script.outputs.result}}/.conan2" - key: ${{ runner.os }}-conan-${{ hashFiles('conanfile.py') }} + key: ${{ runner.os }}-conan-${{ hashFiles('conanfile.py') }}-${{ hashFiles('conan.lock') }} - name: Prebuild Conan packages (Windows) if: | @@ -85,8 +85,8 @@ jobs: CONAN_USER_HOME: "${{steps.conan-path-script.outputs.result}}" if: "!contains(matrix.os, 'windows') && steps.cache-conan.outputs.cache-hit != 'true'" run: | - uvx --with cmake conan profile detect --exist-ok - uvx --with cmake conan install conanfile.py --build missing -of /tmp/ciwheelbuilder -pr:b=default + uv run --only-group=conan --frozen conan profile detect --exist-ok + uv run --only-group=conan --frozen conan install conanfile.py --build missing -of /tmp/ciwheelbuilder -pr:b=default - name: Run Tox (Windows) diff --git a/.github/workflows/wheels.yml b/.github/workflows/wheels.yml index 83f76e1b..1793d169 100644 --- a/.github/workflows/wheels.yml +++ b/.github/workflows/wheels.yml @@ -15,7 +15,7 @@ jobs: UV_EXTRA_INDEX_URL: ${{vars.PIP_EXTRA_INDEX_URL}} strategy: matrix: - os: [ubuntu-latest, windows-latest, macos-13] + os: [ubuntu-latest, windows-latest, macos-latest] steps: - uses: actions/checkout@v3 @@ -29,10 +29,10 @@ jobs: if: contains(matrix.os, 'windows') shell: cmd run: | - "C:\Program Files\Microsoft Visual Studio\2022\Enterprise\Common7\Tools\VsDevCmd.bat" -arch=amd64 && python -m cibuildwheel --output-dir wheelhouse + "C:\Program Files\Microsoft Visual Studio\2022\Enterprise\Common7\Tools\VsDevCmd.bat" -arch=amd64 && uv run --frozen --only-group=cibuildwheel cibuildwheel --output-dir wheelhouse env: CONAN_COMPILER_VERSION: ${{ matrix.compiler_version }} - name: Build wheels if: "!contains(matrix.os, 'windows')" - run: uv run --only-group=cibuildwheel -m cibuildwheel --output-dir wheelhouse + run: uv run --frozen --only-group=cibuildwheel -m cibuildwheel --output-dir wheelhouse # CIBW_SOME_OPTION: value