tests: cache and verify hash value of sample images

henryborchers · henryborchers · commit ed63f733adc9 · 2025-12-18T12:46:54.000-06:00
diff --git a/.github/workflows/build_wheels.yml b/.github/workflows/build_wheels.yml
@@ -31,6 +31,15 @@ jobs:
 
     steps:
       - uses: actions/checkout@v4
+      - name: Cache sample files
+        id: cache_samples
+        uses: actions/cache@v4
+        with:
+          path: "${{ github.workspace }}/sampledata/"
+          key: sample-images
+      - name: download sample files
+        if: steps.cache_samples.outputs.cache-hit != 'true'
+        run: curl -L https://nexus.library.illinois.edu/repository/sample-data/images/sample_images.tar.gz --create-dirs -o "${{ github.workspace }}/sampledata/sample_images.tar.gz"
       - uses: actions/setup-python@v5
         with:
           python-version: '3.11'
@@ -51,6 +60,7 @@ jobs:
 #        uses: pypa/cibuildwheel@v2.11.2
         env:
            CONAN_COMPILER_VERSION: ${{ matrix.compiler_version }}
+           SAMPLE_IMAGES_ARCHIVE: "${{ github.workspace }}/sampledata/sample_images.tar.gz"
 
         #   CIBW_SOME_OPTION: value
       - uses: actions/upload-artifact@v4
diff --git a/.github/workflows/tox_matrix.yml b/.github/workflows/tox_matrix.yml
@@ -42,20 +42,28 @@ jobs:
       fail-fast: false
     name: Python ${{ matrix.python-version }} ${{ matrix.os }} build
     steps:
-
       - uses: actions/checkout@v3
+      - name: Cache sample files
+        id: cache_samples
+        uses: actions/cache@v4
+        with:
+          path: "${{ github.workspace }}/sampledata/"
+          key: sample-images
+      - name: download sample files
+        if: steps.cache_samples.outputs.cache-hit != 'true'
+        run: curl -L https://nexus.library.illinois.edu/repository/sample-data/images/sample_images.tar.gz --create-dirs -o "${{ github.workspace }}/sampledata/sample_images.tar.gz"
       - uses: actions/setup-python@v5
         with:
           python-version: ${{ matrix.python-version }}
           cache: 'pip' # caching pip dependencies
       - name: "install Python dependencies"
         run: |
           pip install uv
-      - uses: actions/cache@v3
+      - uses: actions/cache@v4
         id: cache
         with:
           path: "${{ matrix.conan_user_home }}/.conan2"
-          key: ${{ runner.os }}-${{ hashFiles('**/conanfile.py') }}
+          key: ${{ runner.os }}-${{ hashFiles('**/conanfile.py', 'conan.lock') }}
 
       - name: Build conan packages on Non-Windows Operating Systems
         if: ${{ !contains(matrix.os, 'windows') && steps.cache.outputs.cache-hit != 'true' }}
@@ -80,10 +88,11 @@ jobs:
           "C:\Program Files\Microsoft Visual Studio\2022\Enterprise\Common7\Tools\VsDevCmd.bat" -arch=amd64 && uv run --only-group tox --with tox-uv tox -e ${{ steps.tox-env.outputs.result }}
         env:
           CONAN_USER_HOME: ${{ matrix.conan_user_home }}
+          SAMPLE_IMAGES_ARCHIVE: "${{ github.workspace }}/sampledata/sample_images.tar.gz"
       - name: Run tox
         if: "!contains(matrix.os, 'windows')"
         run: cc --version &&  cc -dumpfullversion -dumpversion && uv run --only-group tox --with tox-uv tox -e ${{ steps.tox-env.outputs.result }} -vvv
         env:
           CONAN_COMPILER_LIBCXX: ${{ matrix.compiler_libcxx }}
           CONAN_USER_HOME: ${{ matrix.conan_user_home }}
-
+          SAMPLE_IMAGES_ARCHIVE: "${{ github.workspace }}/sampledata/sample_images.tar.gz"
diff --git a/ci/docker/linux/jenkins/Dockerfile b/ci/docker/linux/jenkins/Dockerfile
@@ -1,6 +1,8 @@
 ARG CONAN_USER_HOME=/conan
 ARG CONAN_HOME=${CONAN_USER_HOME}/.conan2
 
+ARG SAMPLE_IMAGES_URL=https://nexus.library.illinois.edu/repository/sample-data/images/sample_images.tar.gz
+
 ARG PYTHON_VERSION=latest
 
 ARG CONAN_CENTER_PROXY_V2_URL=https://center2.conan.io
@@ -33,6 +35,12 @@ RUN mkdir -p /.cache/pip &&  \
 #==============================================================================
 
 
+FROM base_image AS sample_files_downloader
+ARG SAMPLE_IMAGES_URL
+ENV EXPECTED_SHA256="0461f57db3806ca47d9063151eec4bc0720c66a83153dda04e230f838b64f063"
+RUN mkdir /sampledata && wget $SAMPLE_IMAGES_URL -P /sampledata && echo "$EXPECTED_SHA256 /sampledata/sample_images.tar.gz" | sha256sum --check
+
+
 FROM base_image AS sonar_builder
 ARG SONAR_USER_HOME
 
@@ -141,11 +149,14 @@ COPY --from=dr_memory_builder /opt/drmemory /opt/drmemory/
 RUN ln -s /opt/drmemory/bin64/drmemory /usr/local/bin/drmemory && \
     drmemory -version
 
+COPY --from=sample_files_downloader /sampledata /sampledata
+
 ARG CONAN_USER_HOME
 ARG CONAN_HOME
 COPY --from=conan_builder --chmod=777 ${CONAN_HOME} ${CONAN_HOME}
 ENV CONAN_USER_HOME=${CONAN_USER_HOME}\
-    CONAN_HOME=${CONAN_HOME}
+    CONAN_HOME=${CONAN_HOME}\
+    SAMPLE_IMAGES_ARCHIVE=/sampledata/sample_images.tar.gz
 
 # To help mark the image as a CI image so it can be cleaned up more easily
 LABEL purpose=ci
diff --git a/ci/docker/linux/tox/Dockerfile b/ci/docker/linux/tox/Dockerfile
@@ -11,6 +11,8 @@ ARG CONAN_CENTER_PROXY_V2_URL=https://center2.conan.io
 # If you want to use a diffrent remote for Conan, such as a proxy. Set the CONAN_CENTER_PROXY_V2_URL
 # Not this is only for building the image. The actual conan center proxy URL is set in the remotes.json file.
 
+ARG SAMPLE_IMAGES_URL=https://nexus.library.illinois.edu/repository/sample-data/images/sample_images.tar.gz
+
 FROM ghcr.io/astral-sh/uv:latest AS uv_builder
 
 FROM ubuntu:24.04 AS base
@@ -25,6 +27,18 @@ RUN --mount=type=cache,target=/var/cache/apt,sharing=locked \
      < /tmp/apt-packages.txt xargs apt-get install -y && \
     apt-get clean && \
     rm -rf /var/lib/apt/lists/*
+
+# ==============================================================================
+FROM base AS sample_files_downloader
+RUN --mount=type=cache,target=/var/cache/apt,sharing=locked \
+    apt-get update && \
+    apt-get install -y wget
+
+ARG SAMPLE_IMAGES_URL
+ENV EXPECTED_SHA256="0461f57db3806ca47d9063151eec4bc0720c66a83153dda04e230f838b64f063"
+
+RUN mkdir /sampledata && wget $SAMPLE_IMAGES_URL -P /sampledata && echo "$EXPECTED_SHA256 /sampledata/sample_images.tar.gz" | sha256sum --check
+
 # ==============================================================================
 
 FROM base AS conan_builder
@@ -74,9 +88,11 @@ FROM base
 COPY --from=uv_builder /uv /uvx /bin/
 ARG CONAN_USER_HOME
 ARG CONAN_HOME
+COPY --from=sample_files_downloader /sampledata /sampledata
 COPY --from=conan_builder --chmod=777 ${CONAN_HOME}/ ${CONAN_HOME}/
 ENV CONAN_USER_HOME=${CONAN_USER_HOME}\
-    CONAN_HOME=${CONAN_HOME}
+    CONAN_HOME=${CONAN_HOME}\
+    SAMPLE_IMAGES_ARCHIVE=/sampledata/sample_images.tar.gz
 
 # To help mark the image as a CI image so it can be cleaned up more easily
 LABEL purpose=ci
diff --git a/scripts/resources/windows/tox/Dockerfile b/scripts/resources/windows/tox/Dockerfile
@@ -15,6 +15,8 @@ ARG PIP_DOWNLOAD_CACHE=c:/users/containeradministrator/appdata/local/pip
 ARG CHOCOLATEY_SOURCE=https://chocolatey.org/api/v2
 ARG chocolateyVersion
 
+ARG SAMPLE_IMAGES_URL=https://nexus.library.illinois.edu/repository/sample-data/images/sample_images.tar.gz
+
 FROM ${FROM_IMAGE} AS certsgen
 RUN certutil -generateSSTFromWU roots.sst
 
@@ -61,6 +63,16 @@ RUN Set-ExecutionPolicy Bypass -Scope Process -Force; `
     Write-Host "Verifying installed packages - Done"
 
 
+# ==============================================================================
+FROM base_builder AS sample_files_downloader
+ARG SAMPLE_IMAGES_URL
+ENV EXPECTED_SHA256="0461f57db3806ca47d9063151eec4bc0720c66a83153dda04e230f838b64f063"
+ENV downloadPath="C:/sampledata/"
+RUN if (-not (Test-Path -Path ${env:downloadPath} -PathType Container)) {`
+        New-Item -Path ${env:downloadPath} -ItemType Directory -Force`
+    }; `
+    Invoke-WebRequest -Uri ${env:SAMPLE_IMAGES_URL} -OutFile "${env:downloadPath}\sample_images.tar.gz"
+
 # ==============================================================================
 FROM base_builder AS conan_builder
 ARG CONAN_HOME
@@ -105,11 +117,14 @@ RUN New-Item -type directory -path ${Env:PIP_DOWNLOAD_CACHE} -Force | Out-Null ;
     New-Item -type directory -path ${Env:UV_CACHE_DIR} -Force | Out-Null
 ARG CONAN_HOME
 COPY --from=conan_builder ${CONAN_HOME}/ ${CONAN_HOME}/
+COPY --from=sample_files_downloader c:/sampledata/ C:/sampledata/
+
 ARG CONAN_USER_HOME
 ENV CONAN_USER_HOME=${CONAN_USER_HOME}`
     CONAN_HOME=${CONAN_HOME}`
     PIP_DOWNLOAD_CACHE=${PIP_DOWNLOAD_CACHE}`
-    UV_CACHE_DIR=${UV_CACHE_DIR}
+    UV_CACHE_DIR=${UV_CACHE_DIR}`
+    SAMPLE_IMAGES_ARCHIVE=C:/sampledata/sample_images.tar.gz
 
 WORKDIR C:/src
 ENV DISTUTILS_USE_SDK=1
diff --git a/tests/CMakeLists.txt b/tests/CMakeLists.txt
@@ -1,15 +1,25 @@
 include(FetchContent)
 if (BUILD_TESTING AND Catch2_FOUND)
     include(Catch)
-    FetchContent_Declare(test_images
+    set(SAMPLE_IMAGES_ARCHIVE "" CACHE FILEPATH "Path to sample images archive. If not set, it will be downloaded from the internet.")
+    if(SAMPLE_IMAGES_ARCHIVE)
+        set(TEST_IMAGE_PATH "${CMAKE_CURRENT_BINARY_DIR}/sample_images/")
+        file(ARCHIVE_EXTRACT
+                INPUT "${SAMPLE_IMAGES_ARCHIVE}"
+                DESTINATION "${CMAKE_CURRENT_BINARY_DIR}/"
+        )
+    else ()
+        FetchContent_Declare(test_images
             URL https://nexus.library.illinois.edu/repository/sample-data/images/sample_images.tar.gz
+            URL_HASH SHA256=0461f57db3806ca47d9063151eec4bc0720c66a83153dda04e230f838b64f063
             SOURCE_DIR ${CMAKE_CURRENT_BINARY_DIR}/sample_images
-            )
-    FetchContent_GetProperties(test_images)
-    FetchContent_MakeAvailable(test_images)
-    find_path(TEST_IMAGE_PATH dummy.jp2
-            PATHS ${test_images_SOURCE_DIR})
-    set(TEST_IMAGE_PATH ${TEST_IMAGE_PATH}/)
+        )
+        FetchContent_GetProperties(test_images)
+        FetchContent_MakeAvailable(test_images)
+        find_path(TEST_IMAGE_PATH dummy.jp2
+                PATHS ${test_images_SOURCE_DIR})
+        set(TEST_IMAGE_PATH ${TEST_IMAGE_PATH}/)
+    endif ()
     set(PYTHON_TEST_FILES test_core.py)
     find_package(Python3 COMPONENTS Interpreter)
     if(pyexiv2bind_generate_python_bindings)
diff --git a/tests/conftest.py b/tests/conftest.py
@@ -1,24 +1,37 @@
 import os
 import shutil
 import tarfile
+import hashlib
 
 import pytest
 import urllib.request
 
+SAMPLE_IMAGES_SHA256 = "0461f57db3806ca47d9063151eec4bc0720c66a83153dda04e230f838b64f063"
+SAMPLE_IMAGES_URL = "https://nexus.library.illinois.edu/repository/sample-data/images/sample_images.tar.gz"
 
-def download_images(url, destination, download_path):
+# To save time from redownloading the sample_images.tar.gz file every time,
+# download it locally and set the environment variable SAMPLE_IMAGES_ARCHIVE
+# to the path of the downloaded file
 
-        print("Downloading {}".format(url))
-        urllib.request.urlretrieve(url,
-                                   filename=os.path.join(download_path, "sample_images.tar.gz"))
-        if not os.path.exists(os.path.join(download_path, "sample_images.tar.gz")):
+def download_images(url, download_path):
+
+        print(f"Downloading {url}")
+        output = os.path.join(download_path, "sample_images.tar.gz")
+        urllib.request.urlretrieve(url, filename=output)
+        if not os.path.exists(output):
             raise FileNotFoundError("sample images not download")
-        print("Extracting images")
-        with tarfile.open(os.path.join(download_path, "sample_images.tar.gz"), "r:gz") as archive_file:
-            for item in archive_file.getmembers():
-                print("Extracting {}".format(item.name))
-                archive_file.extract(item, path=destination)
-            pass
+        return output
+def extract_images(path, destination):
+    print("Extracting images")
+    with tarfile.open(path, "r:gz") as archive_file:
+        for item in archive_file.getmembers():
+            print("Extracting {}".format(item.name))
+            archive_file.extract(item, path=destination)
+
+def verify_hash(path, sha256_hash):
+    with open(path, "rb") as f:
+        file_hash = hashlib.sha256(f.read()).hexdigest()
+    assert file_hash == sha256_hash
 
 
 @pytest.fixture(scope="session")
@@ -28,16 +41,23 @@ def sample_images_readonly(tmpdir_factory):
     sample_images_path = os.path.join(test_path, "sample_images")
     download_path = tmpdir_factory.mktemp("downloaded_archives", numbered=False)
     if os.path.exists(sample_images_path):
-        print("{} already exits".format(sample_images_path))
+        print(f"{sample_images_path} already exits")
 
     else:
-        print("Downloading sample images")
-        download_images(url="https://nexus.library.illinois.edu/repository/sample-data/images/sample_images.tar.gz",
-                        destination=test_path,
-                        download_path=download_path)
-
+        archive = os.getenv('SAMPLE_IMAGES_ARCHIVE')
+        if not archive:
+            print("Downloading sample images")
+            archive = download_images(
+                url=SAMPLE_IMAGES_URL,
+                download_path=download_path
+            )
+        if not os.path.exists(archive):
+            raise FileNotFoundError(f"sample image archive not found. {archive} does not exist.")
+        verify_hash(archive, sha256_hash=SAMPLE_IMAGES_SHA256)
+        extract_images(path=archive, destination=test_path)
     yield sample_images_path
-    shutil.rmtree(test_path)
+    if os.path.exists(test_path):
+        shutil.rmtree(test_path)
 
 
 @pytest.fixture
diff --git a/tox.ini b/tox.ini
@@ -17,6 +17,7 @@ passenv =
     LIB
     LIBPATH
     DISTUTILS_USE_SDK
+    SAMPLE_IMAGES_ARCHIVE
 
 commands_pre =
     {env_bin_dir}{/}python {tox_root}/scripts/get_linked_exiv2_version.py
