Merge pull request #163 from UIUCLibrary/dev

Dev

Author: henryborchers

.github/workflows/tox_matrix.yml

@@ -68,7 +68,7 @@ jobs:
         uses: actions/cache@v3
         with:
           path: "${{steps.conan-path-script.outputs.result}}/.conan2"
-          key: ${{ runner.os }}-conan-${{ hashFiles('conanfile.py') }}
+          key: ${{ runner.os }}-conan-${{ hashFiles('conanfile.py') }}-${{ hashFiles('conan.lock') }}
 
       - name: Prebuild Conan packages (Windows)
         if: |
@@ -85,8 +85,8 @@ jobs:
           CONAN_USER_HOME: "${{steps.conan-path-script.outputs.result}}"
         if: "!contains(matrix.os, 'windows') && steps.cache-conan.outputs.cache-hit != 'true'"
         run: |
-          uvx --with cmake conan profile detect --exist-ok
-          uvx --with cmake conan install conanfile.py --build missing -of /tmp/ciwheelbuilder -pr:b=default
+          uv run --only-group=conan --frozen conan profile detect --exist-ok
+          uv run --only-group=conan --frozen conan install conanfile.py --build missing -of /tmp/ciwheelbuilder -pr:b=default
           
 
       - name: Run Tox (Windows)

.github/workflows/wheels.yml

@@ -15,7 +15,7 @@ jobs:
       UV_EXTRA_INDEX_URL: ${{vars.PIP_EXTRA_INDEX_URL}}
     strategy:
       matrix:
-        os: [ubuntu-latest, windows-latest, macos-13]
+        os: [ubuntu-latest, windows-latest, macos-latest]
 
     steps:
       - uses: actions/checkout@v3
@@ -29,10 +29,10 @@ jobs:
         if: contains(matrix.os, 'windows')
         shell: cmd
         run: |
-          "C:\Program Files\Microsoft Visual Studio\2022\Enterprise\Common7\Tools\VsDevCmd.bat" -arch=amd64 && python -m cibuildwheel --output-dir wheelhouse
+          "C:\Program Files\Microsoft Visual Studio\2022\Enterprise\Common7\Tools\VsDevCmd.bat" -arch=amd64 && uv run --frozen --only-group=cibuildwheel cibuildwheel --output-dir wheelhouse
         env:
            CONAN_COMPILER_VERSION: ${{ matrix.compiler_version }}
       - name: Build wheels
         if: "!contains(matrix.os, 'windows')"
-        run: uv run --only-group=cibuildwheel -m cibuildwheel --output-dir wheelhouse
+        run: uv run --frozen --only-group=cibuildwheel -m cibuildwheel --output-dir wheelhouse
         #   CIBW_SOME_OPTION: value

MANIFEST.in

@@ -9,7 +9,7 @@ include cmake/fixup.cmake
 
 # C++ Source files
 include CMakeLists.txt
-include conanfile.py
+include conanfile.py conan.lock
 recursive-include src/uiucprescon/imagevalidate CMakeLists.txt *.cpp *.h
 
 # Tox configuration setting

ci/docker/linux/jenkins/Dockerfile

@@ -14,9 +14,9 @@ ARG UV_CACHE_DIR=/.cache/uv
 
 FROM ghcr.io/astral-sh/uv:latest AS uv_builder
 
-FROM ubuntu:22.04 AS base_image
-RUN rm -f /etc/apt/apt.conf.d/docker-clean
-RUN --mount=type=cache,target=/var/cache/apt \
+FROM ubuntu:24.04 AS base_image
+RUN --mount=type=cache,target=/var/cache/apt,sharing=locked \
+    --mount=type=cache,target=/var/lib/apt,sharing=locked \
     DEBIAN_FRONTEND="noninteractive" apt-get update &&  \
     apt-get install -yqq software-properties-common gpg-agent --no-install-recommends && \
     apt-get -yqq install tzdata && \
@@ -32,14 +32,14 @@ RUN --mount=type=cache,target=/var/cache/apt \
         python3-pip \
         python3-venv \
         ca-certificates \
-        software-properties-common \
-        jq \
-        unzip && \
-     rm -rf /var/lib/apt/lists/*
+        software-properties-common
+
 
 #==============================================================================
 FROM base_image AS dr_memory_builder
-RUN apt-get update &&  \
+RUN --mount=type=cache,target=/var/cache/apt,sharing=locked \
+    --mount=type=cache,target=/var/lib/apt,sharing=locked \
+    apt-get update &&  \
     apt-get install -y --no-install-recommends wget
 
 WORKDIR /drmemory
@@ -50,6 +50,10 @@ RUN wget --no-verbose https://github.com/DynamoRIO/drmemory/releases/download/re
 
 
 FROM base_image AS sonar_builder
+RUN --mount=type=cache,target=/var/cache/apt,sharing=locked \
+    --mount=type=cache,target=/var/lib/apt,sharing=locked \
+    apt-get update &&  \
+    apt-get install -yqq --no-install-recommends unzip
 
 RUN case $(uname -m) in \
     x86_64) URL=https://sonarcloud.io/static/cpp/build-wrapper-linux-x86.zip ;; \
@@ -65,37 +69,42 @@ RUN case $(uname -m) in \
 
 
 FROM base_image AS conan_builder
+RUN --mount=type=cache,target=/var/cache/apt,sharing=locked \
+    --mount=type=cache,target=/var/lib/apt,sharing=locked \
+    DEBIAN_FRONTEND="noninteractive" apt-get update &&  \
+    apt-get install -yqq --no-install-recommends jq
+
 ARG CONAN_USER_HOME
 ARG PIP_EXTRA_INDEX_URL
 ARG PIP_INDEX_URL
 ARG UV_EXTRA_INDEX_URL
 ARG UV_INDEX_URL
 
 COPY --from=uv_builder /uv /uvx /bin/
-COPY ci/docker/linux/shared/ensure_compiler_support_by_conan.py /tmp/ensure_compiler_support_by_conan.py
 ARG UV_CACHE_DIR
 ARG PIP_DOWNLOAD_CACHE
 ARG CONAN_CENTER_PROXY_V2_URL
 ARG CONAN_HOME
 ENV UV_PROJECT=/tmp/uv
 ENV UV_FROZEN=1
+ENV UV_PROJECT_ENVIRONMENT=/tmp/uv/venv
 COPY ci/docker/linux/jenkins/conan/profile.ini  ${CONAN_HOME}/profiles/default
 COPY ci/docker/shared/conan/remotes.json ${CONAN_HOME}/remotes.json
 RUN --mount=type=cache,target=${PIP_DOWNLOAD_CACHE} \
     --mount=type=cache,target=${UV_CACHE_DIR} \
     --mount=type=bind,source=pyproject.toml,target=${UV_PROJECT}/pyproject.toml \
     --mount=type=bind,source=uv.lock,target=${UV_PROJECT}/uv.lock \
     --mount=type=bind,source=conanfile.py,target=/tmp/conanfile.py \
+    --mount=type=bind,source=conan.lock,target=${UV_PROJECT}/conan.lock \
+    --mount=type=tmpfs,target=${UV_PROJECT_ENVIRONMENT} \
     uv run --only-group=conan conan profile detect --exist-ok &&  \
-    python3 /tmp/ensure_compiler_support_by_conan.py $(uv run --only-group=conan conan config home)/settings.yml  11.4 && \
     if [ "$(jq -r '.remotes[0].url' ${CONAN_HOME}/remotes.json )" != "${CONAN_CENTER_PROXY_V2_URL}" ]; then \
         echo "Setting the conan center url for building cache: ${CONAN_CENTER_PROXY_V2_URL}" && \
         cp ${CONAN_HOME}/remotes.json /tmp/remotes.json && \
         uv run --only-group=conan conan remote update conan-center --url ${CONAN_CENTER_PROXY_V2_URL}; \
     fi && \
     uv run --only-group=conan conan install /tmp --build missing -pr:b=default && \
     uv run --only-group=conan conan cache clean "*" -b --source --build --temp && \
-    rm -rf venv && \
     if [ "$(jq -r '.remotes[0].url' ${CONAN_HOME}/remotes.json )" != "${CONAN_CENTER_PROXY_V2_URL}" ]; then \
         mv -f /tmp/remotes.json ${CONAN_HOME}/remotes.json; \
     fi
@@ -104,7 +113,7 @@ RUN --mount=type=cache,target=${PIP_DOWNLOAD_CACHE} \
 # shared files are needed to run the Build C++ Tests stage
 
 FROM base_image
-
+COPY --from=uv_builder /uv /uvx /bin/
 # -----------------------------------------------------------------------------
 # Install sonar build-wrapper
 # -----------------------------------------------------------------------------

ci/docker/linux/jenkins/conan/profile.ini

@@ -1,8 +1,9 @@
+{% set compiler, version, compiler_exe = detect_api.detect_default_compiler() %}
 [settings]
 os=Linux
 arch=x86_64
 compiler=gcc
-compiler.version=11.4
+compiler.version={{version.major}}.{{version.minor}}
 compiler.libcxx=libstdc++
 build_type=Release
 [options]

ci/docker/linux/tox/Dockerfile

@@ -13,18 +13,17 @@ ARG UV_EXTRA_INDEX_URL
 
 FROM ghcr.io/astral-sh/uv:latest AS uv_builder
 
-FROM ubuntu:22.04 AS wheel_builder_base
-COPY ci/docker/linux/tox/apt-packages.txt /tmp/apt-packages.txt
+FROM ubuntu:24.04 AS wheel_builder_base
 SHELL ["/bin/bash", "-o", "pipefail", "-c"]
-RUN rm -f /etc/apt/apt.conf.d/docker-clean
-RUN --mount=type=cache,target=/var/cache/apt \
+RUN --mount=type=cache,target=/var/cache/apt,sharing=locked \
+    --mount=type=cache,target=/var/lib/apt,sharing=locked \
+    --mount=type=bind,source=ci/docker/linux/tox/apt-packages.txt,target=/tmp/apt-packages.txt \
     apt-get update && \
     apt-get install -yqq --no-install-recommends software-properties-common gpg-agent && \
     add-apt-repository ppa:deadsnakes/ppa && \
     DEBIAN_FRONTEND=noninteractive TZ=Etc/UTC apt-get -yqq install --no-install-recommends tzdata && \
     apt-get update && \
-     < /tmp/apt-packages.txt xargs apt-get install -yqq --no-install-recommends && \
-    rm -rf /var/lib/apt/lists/*
+     < /tmp/apt-packages.txt xargs apt-get install -yqq --no-install-recommends
 
 ARG CONAN_USER_HOME
 RUN mkdir -p ${CONAN_USER_HOME}
@@ -33,23 +32,21 @@ FROM wheel_builder_base AS conan_builder
 COPY --from=uv_builder /uv /uvx /bin/
 ARG CONAN_USER_HOME
 ARG CONAN_HOME
-COPY ci/docker/linux/shared/ensure_compiler_support_by_conan.py /tmp/ensure_compiler_support_by_conan.py
 COPY ci/docker/shared/conan/remotes.json ${CONAN_HOME}/remotes.json
 COPY conanfile.py /tmp/conanfile.py
 ARG CONAN_CENTER_PROXY_V2_URL
-ARG PIP_EXTRA_INDEX_URL
-ARG PIP_INDEX_URL
 ARG UV_EXTRA_INDEX_URL
 ARG UV_INDEX_URL
-ARG PIP_DOWNLOAD_CACHE
 ARG UV_CACHE_DIR
 ENV UV_PROJECT=/tmp/uv
 ENV UV_FROZEN=1
-RUN --mount=type=cache,target=${PIP_DOWNLOAD_CACHE} \
-    --mount=type=cache,target=${UV_CACHE_DIR} \
+ENV UV_PROJECT_ENVIRONMENT=/tmp/uv/venv
+RUN --mount=type=cache,target=${UV_CACHE_DIR} \
+    --mount=type=tmpfs,target=${UV_PROJECT_ENVIRONMENT} \
     --mount=type=bind,source=pyproject.toml,target=${UV_PROJECT}/pyproject.toml \
     --mount=type=bind,source=uv.lock,target=${UV_PROJECT}/uv.lock \
     --mount=type=bind,source=conanfile.py,target=/tmp/conanfile.py \
+    --mount=type=bind,source=conan.lock,target=${UV_PROJECT}/conan.lock \
     uv run --only-group=conan conan profile detect --exist-ok &&  \
     cat $(uv run --only-group=conan conan config home)/settings.yml && \
     uv run --only-group=conan conan config home && \
@@ -68,7 +65,7 @@ RUN --mount=type=cache,target=${PIP_DOWNLOAD_CACHE} \
 
 FROM wheel_builder_base
 
-
+COPY --from=uv_builder /uv /uvx /bin/
 ARG CONAN_USER_HOME
 ARG CONAN_HOME
 COPY --from=conan_builder --chmod=777 ${CONAN_HOME} ${CONAN_HOME}

ci/docker/linux/tox/conan/profile.ini

@@ -1,10 +1,11 @@
+{% set compiler, version, compiler_exe = detect_api.detect_default_compiler() %}
 [settings]
 os=Linux
 os_build=Linux
 arch=x86_64
 arch_build=x86_64
 compiler=gcc
-compiler.version=11.4
+compiler.version={{version.major}}.{{version.minor}}
 compiler.libcxx=libstdc++
 build_type=Release
 [options]

conan.lock

@@ -0,0 +1,9 @@
+{
+    "version": "0.5",
+    "requires": [
+        "openjpeg/2.3.1#ada21840f723c5410056aa68f0b23c61%1765835638.220325"
+    ],
+    "build_requires": [],
+    "python_requires": [],
+    "config_requires": []
+}

scripts/resources/package_for_linux/Dockerfile

@@ -69,6 +69,7 @@ RUN --mount=type=cache,target=${PIP_DOWNLOAD_CACHE} \
     --mount=type=bind,source=pyproject.toml,target=${UV_PROJECT}/pyproject.toml \
     --mount=type=bind,source=uv.lock,target=${UV_PROJECT}/uv.lock \
     --mount=type=bind,source=conanfile.py,target=/tmp/conanfile.py \
+    --mount=type=bind,source=conan.lock,target=/tmp/conan.lock \
     uv run --only-group conan --with ninja conan profile detect --exist-ok &&  \
     if [ "$(jq -r '.remotes[0].url' ${CONAN_HOME}/remotes.json )" != "${CONAN_CENTER_PROXY_V2_URL}" ]; then \
       echo "Setting the conan center url for building cache: ${CONAN_CENTER_PROXY_V2_URL}" && \

scripts/resources/windows/Dockerfile

@@ -71,9 +71,10 @@ ARG UV_INDEX_URL
 ARG UV_EXTRA_INDEX_URL
 ENV UV_PROJECT=c:/temp/uv
 COPY conanfile.py c:/temp/conanfile.py
-COPY pyproject.toml uv.lock ${UV_PROJECT}/
+COPY pyproject.toml uv.lock conan.lock ${UV_PROJECT}/
 COPY scripts/resources/windows/conan/profile.ini ${CONAN_HOME}/profiles/default
 COPY ci/docker/shared/conan/remotes.json ${CONAN_HOME}/remotes.json
+
 ARG CONAN_CENTER_PROXY_V2_URL
 ENV UV_FROZEN=1
 RUN uv run --only-group conan --with ninja conan profile detect --exist-ok ; `