tests: sample images files can run from a locally sourced file archive

Author: henryborchers

Jenkinsfile

@@ -536,6 +536,7 @@ pipeline {
                                                             -Wdev \
                                                             -DCMAKE_EXPORT_COMPILE_COMMANDS:BOOL=ON \
                                                             -DCMAKE_POSITION_INDEPENDENT_CODE:BOOL=true \
+                                                            -DSAMPLE_IMAGES_ARCHIVE=${SAMPLE_IMAGES_ARCHIVE} \
                                                             -DBUILD_TESTING:BOOL=true \
                                                             -DCMAKE_CXX_FLAGS="-fno-inline -fno-omit-frame-pointer -fprofile-arcs -ftest-coverage -Wall -Wextra" \
                                                             -DMEMORYCHECK_COMMAND=$(which drmemory) \

ci/docker/linux/jenkins/Dockerfile

@@ -2,6 +2,7 @@ ARG CONAN_USER_HOME=/conan
 
 ARG CONAN_HOME=${CONAN_USER_HOME}/.conan2
 
+
 ARG CONAN_CENTER_PROXY_V2_URL=https://center2.conan.io
 # If you want to use a diffrent remote for Conan, such as a proxy. Set the CONAN_CENTER_PROXY_V2_URL
 # Not this is only for building the image. The actual conan center proxy URL is set in the remotes.json file.
@@ -12,6 +13,8 @@ ARG PIP_DOWNLOAD_CACHE=/.cache/pip
 ARG UV_CACHE_DIR=/.cache/uv
 # UV Cache directory. Change this only if you need to override the default location.
 
+ARG SAMPLE_IMAGES_URL=https://nexus.library.illinois.edu/repository/sample-data/images/metadata_test_images.tar.gz
+
 FROM ghcr.io/astral-sh/uv:latest AS uv_builder
 
 FROM ubuntu:24.04 AS base_image
@@ -112,6 +115,21 @@ RUN --mount=type=cache,target=${PIP_DOWNLOAD_CACHE} \
 
 # shared files are needed to run the Build C++ Tests stage
 
+#==============================================================================
+
+
+FROM base_image AS sample_files_downloader
+RUN --mount=type=cache,target=/var/cache/apt,sharing=locked \
+    --mount=type=cache,target=/var/lib/apt,sharing=locked \
+    apt-get update &&  \
+    apt-get install -y --no-install-recommends wget
+
+ARG SAMPLE_IMAGES_URL
+ENV EXPECTED_SHA256="eee5fa3641628c7365e14de3f58da069cb332dc13a4b739dc168a1617109ebbf"
+RUN mkdir /sampledata && wget $SAMPLE_IMAGES_URL -P /sampledata && echo "$EXPECTED_SHA256 /sampledata/metadata_test_images.tar.gz" | sha256sum --check
+
+
+
 FROM base_image
 COPY --from=uv_builder /uv /uvx /bin/
 # -----------------------------------------------------------------------------
@@ -136,14 +154,15 @@ RUN ln -s /opt/drmemory/bin64/drmemory /usr/local/bin/drmemory && \
 
 ARG PIP_EXTRA_INDEX_URL
 
-ENV PIP_NO_CACHE_DIR=1
-
 ARG CONAN_USER_HOME
-ENV CONAN_USER_HOME=${CONAN_USER_HOME}
 ARG CONAN_HOME
-ENV CONAN_HOME=${CONAN_HOME}
 COPY --from=conan_builder --chmod=777 ${CONAN_HOME} ${CONAN_HOME}
-ENV TZ=UTC
+COPY --from=sample_files_downloader /sampledata /sampledata
+ENV TZ=UTC \
+    PIP_NO_CACHE_DIR=1 \
+    CONAN_USER_HOME=${CONAN_USER_HOME} \
+    SAMPLE_IMAGES_ARCHIVE=/sampledata/metadata_test_images.tar.gz \
+    CONAN_HOME=${CONAN_HOME}
 
 # To help mark the image as a CI image so it can be cleaned up more easily
 LABEL purpose=ci

tests/CMakeLists.txt

@@ -1,20 +1,27 @@
 if (BUILD_TESTING AND Catch2_FOUND)
     include(FetchContent)
     include(Catch)
+    set(SAMPLE_IMAGES_ARCHIVE "" CACHE FILEPATH "Path to sample images archive. If not set, it will be downloaded from the internet.")
 
     # Get sample images
-    if(NOT EXISTS ${PROJECT_BINARY_DIR}/metadata_test_images.tar.gz)
-        file(DOWNLOAD https://nexus.library.illinois.edu/repository/sample-data/images/metadata_test_images.tar.gz ${PROJECT_BINARY_DIR}/metadata_test_images.tar.gz SHOW_PROGRESS)
-    endif()
-    FetchContent_Declare(
-            test_images
-            URL ${PROJECT_BINARY_DIR}/metadata_test_images.tar.gz
-    )
-    FetchContent_MakeAvailable(test_images)
+    if(SAMPLE_IMAGES_ARCHIVE)
+        set(TEST_IMAGE_PATH "${CMAKE_CURRENT_BINARY_DIR}/metadata_test_images/")
+        file(ARCHIVE_EXTRACT
+                INPUT "${SAMPLE_IMAGES_ARCHIVE}"
+                DESTINATION "${CMAKE_CURRENT_BINARY_DIR}/"
+        )
+    else ()
+         FetchContent_Declare(
+                test_images
+                URL ${PROJECT_BINARY_DIR}/metadata_test_images.tar.gz
+        )
+        FetchContent_MakeAvailable(test_images)
+        set(TEST_IMAGE_PATH ${test_images_SOURCE_DIR})
+    endif ()
     add_executable(tester test-basic.cpp test_exceptions.cpp)
     target_include_directories(tester PRIVATE ${PROJECT_SOURCE_DIR}/uiucprescon/imagevalidate)
 
-    target_compile_definitions(tester PRIVATE TEST_IMAGE_PATH="${test_images_SOURCE_DIR}")
+    target_compile_definitions(tester PRIVATE TEST_IMAGE_PATH="${TEST_IMAGE_PATH}")
     target_link_libraries(tester PRIVATE Catch2::Catch2WithMain imagevalidate::glue)
     catch_discover_tests(tester
             TEST_PREFIX imagevalidate.
@@ -24,7 +31,7 @@ if (BUILD_TESTING AND Catch2_FOUND)
 
     add_executable(tester-internal tester-internal.cpp)
     target_compile_definitions(tester-internal
-            PRIVATE TEST_IMAGE_PATH="${test_images_SOURCE_DIR}")
+            PRIVATE TEST_IMAGE_PATH="${TEST_IMAGE_PATH}")
 
 
     target_link_libraries(tester-internal

tests/test_integration.py

@@ -1,3 +1,4 @@
+import hashlib
 import shutil
 
 import pytest
@@ -11,25 +12,20 @@
 
 # TEST_PATH = os.path.join(os.path.dirname(__file__), "henrytestmetadata")
 SAMPLE_IMAGES = "https://nexus.library.illinois.edu/repository/sample-data/images/metadata_test_images.tar.gz"
+SAMPLE_IMAGES_SHA256 = "eee5fa3641628c7365e14de3f58da069cb332dc13a4b739dc168a1617109ebbf"
 
+# To save time from redownloading the metadata_test_images.tar.gz file every time,
+# download it locally and set the environment variable SAMPLE_IMAGES_ARCHIVE
+# to the path of the downloaded file
 
-def download_images(url, destination):
-    with TemporaryDirectory() as download_path:
-        print("Downloading {}".format(url), flush=True)
-        urllib.request.urlretrieve(url,
-                                   filename=os.path.join(download_path,
-                                                         "sample_images.tar.gz"))
-        if not os.path.exists(
-                os.path.join(download_path, "sample_images.tar.gz")):
-            raise FileNotFoundError("sample images not download")
-        print("Extracting images")
-        with tarfile.open(os.path.join(download_path, "sample_images.tar.gz"),
-                          "r:gz") as archive_file:
-            for item in archive_file.getmembers():
-                print("Extracting {}".format(item.name))
-                archive_file.extract(item, path=destination)
-            pass
+def download_images(url, download_path):
 
+    print(f"Downloading {url}")
+    output = os.path.join(download_path, "sample_images.tar.gz")
+    urllib.request.urlretrieve(url, filename=output)
+    if not os.path.exists(output):
+        raise FileNotFoundError("sample images not download")
+    return output
 
 @pytest.fixture(scope="session")
 def sample_data():
@@ -40,14 +36,33 @@ def sample_data():
     if os.path.exists(sample_images_path):
         print("{} already exits".format(sample_images_path))
     else:
-        print("Downloading sample images")
-        if not os.path.exists(sample_images_path):
-            download_images(
-                url=SAMPLE_IMAGES,
-                destination=test_path)
+        archive = os.getenv('SAMPLE_IMAGES_ARCHIVE')
+        if not archive:
+            print("Downloading sample images")
+            if not os.path.exists(sample_images_path):
+                archive = download_images(
+                    url=SAMPLE_IMAGES,
+                    download_path=test_path
+                )
+        if not os.path.exists(archive):
+            raise FileNotFoundError(f"sample image archive not found. {archive} does not exist.")
+        verify_hash(archive, sha256_hash=SAMPLE_IMAGES_SHA256)
+        extract_images(path=archive, destination=test_path)
     yield sample_images_path
     shutil.rmtree(sample_images_path)
 
+def extract_images(path, destination):
+    print("Extracting images")
+    with tarfile.open(path, "r:gz") as archive_file:
+        for item in archive_file.getmembers():
+            print("Extracting {}".format(item.name))
+            archive_file.extract(item, path=destination)
+
+def verify_hash(path, sha256_hash):
+    with open(path, "rb") as f:
+        file_hash = hashlib.sha256(f.read()).hexdigest()
+    assert file_hash == sha256_hash
+
 
 @pytest.mark.integration
 @pytest.mark.parametrize("test_file,profile_name", [