tests: sample images files can run from a locally sourced file archive

Author: henryborchers

ci/docker/linux/jenkins/Dockerfile

@@ -2,6 +2,7 @@ ARG CONAN_USER_HOME=/conan
 
 ARG CONAN_HOME=${CONAN_USER_HOME}/.conan2
 
+
 ARG CONAN_CENTER_PROXY_V2_URL=https://center2.conan.io
 # If you want to use a diffrent remote for Conan, such as a proxy. Set the CONAN_CENTER_PROXY_V2_URL
 # Not this is only for building the image. The actual conan center proxy URL is set in the remotes.json file.
@@ -12,6 +13,8 @@ ARG PIP_DOWNLOAD_CACHE=/.cache/pip
 ARG UV_CACHE_DIR=/.cache/uv
 # UV Cache directory. Change this only if you need to override the default location.
 
+ARG SAMPLE_IMAGES_URL=https://nexus.library.illinois.edu/repository/sample-data/images/metadata_test_images.tar.gz
+
 FROM ghcr.io/astral-sh/uv:latest AS uv_builder
 
 FROM ubuntu:24.04 AS base_image
@@ -112,6 +115,21 @@ RUN --mount=type=cache,target=${PIP_DOWNLOAD_CACHE} \
 
 # shared files are needed to run the Build C++ Tests stage
 
+#==============================================================================
+
+
+FROM base_image AS sample_files_downloader
+RUN --mount=type=cache,target=/var/cache/apt,sharing=locked \
+    --mount=type=cache,target=/var/lib/apt,sharing=locked \
+    apt-get update &&  \
+    apt-get install -y --no-install-recommends wget
+
+ARG SAMPLE_IMAGES_URL
+ENV EXPECTED_SHA256="eee5fa3641628c7365e14de3f58da069cb332dc13a4b739dc168a1617109ebbf"
+RUN mkdir /sampledata && wget $SAMPLE_IMAGES_URL -P /sampledata && echo "$EXPECTED_SHA256 /sampledata/metadata_test_images.tar.gz" | sha256sum --check
+
+
+
 FROM base_image
 COPY --from=uv_builder /uv /uvx /bin/
 # -----------------------------------------------------------------------------
@@ -136,14 +154,15 @@ RUN ln -s /opt/drmemory/bin64/drmemory /usr/local/bin/drmemory && \
 
 ARG PIP_EXTRA_INDEX_URL
 
-ENV PIP_NO_CACHE_DIR=1
-
 ARG CONAN_USER_HOME
-ENV CONAN_USER_HOME=${CONAN_USER_HOME}
 ARG CONAN_HOME
-ENV CONAN_HOME=${CONAN_HOME}
 COPY --from=conan_builder --chmod=777 ${CONAN_HOME} ${CONAN_HOME}
-ENV TZ=UTC
+COPY --from=sample_files_downloader /sampledata /sampledata
+ENV TZ=UTC \
+    PIP_NO_CACHE_DIR=1 \
+    CONAN_USER_HOME=${CONAN_USER_HOME} \
+    SAMPLE_IMAGES_ARCHIVE=/sampledata/metadata_test_images.tar.gz \
+    CONAN_HOME=${CONAN_HOME}
 
 # To help mark the image as a CI image so it can be cleaned up more easily
 LABEL purpose=ci

tests/test_integration.py

@@ -1,3 +1,4 @@
+import hashlib
 import shutil
 
 import pytest
@@ -11,25 +12,20 @@
 
 # TEST_PATH = os.path.join(os.path.dirname(__file__), "henrytestmetadata")
 SAMPLE_IMAGES = "https://nexus.library.illinois.edu/repository/sample-data/images/metadata_test_images.tar.gz"
+SAMPLE_IMAGES_SHA256 = "eee5fa3641628c7365e14de3f58da069cb332dc13a4b739dc168a1617109ebbf"
 
+# To save time from redownloading the metadata_test_images.tar.gz file every time,
+# download it locally and set the environment variable SAMPLE_IMAGES_ARCHIVE
+# to the path of the downloaded file
 
-def download_images(url, destination):
-    with TemporaryDirectory() as download_path:
-        print("Downloading {}".format(url), flush=True)
-        urllib.request.urlretrieve(url,
-                                   filename=os.path.join(download_path,
-                                                         "sample_images.tar.gz"))
-        if not os.path.exists(
-                os.path.join(download_path, "sample_images.tar.gz")):
-            raise FileNotFoundError("sample images not download")
-        print("Extracting images")
-        with tarfile.open(os.path.join(download_path, "sample_images.tar.gz"),
-                          "r:gz") as archive_file:
-            for item in archive_file.getmembers():
-                print("Extracting {}".format(item.name))
-                archive_file.extract(item, path=destination)
-            pass
+def download_images(url, download_path):
 
+    print(f"Downloading {url}")
+    output = os.path.join(download_path, "sample_images.tar.gz")
+    urllib.request.urlretrieve(url, filename=output)
+    if not os.path.exists(output):
+        raise FileNotFoundError("sample images not download")
+    return output
 
 @pytest.fixture(scope="session")
 def sample_data():
@@ -40,14 +36,33 @@ def sample_data():
     if os.path.exists(sample_images_path):
         print("{} already exits".format(sample_images_path))
     else:
-        print("Downloading sample images")
-        if not os.path.exists(sample_images_path):
-            download_images(
-                url=SAMPLE_IMAGES,
-                destination=test_path)
+        archive = os.getenv('SAMPLE_IMAGES_ARCHIVE')
+        if not archive:
+            print("Downloading sample images")
+            if not os.path.exists(sample_images_path):
+                archive = download_images(
+                    url=SAMPLE_IMAGES,
+                    download_path=test_path
+                )
+        if not os.path.exists(archive):
+            raise FileNotFoundError(f"sample image archive not found. {archive} does not exist.")
+        verify_hash(archive, sha256_hash=SAMPLE_IMAGES_SHA256)
+        extract_images(path=archive, destination=test_path)
     yield sample_images_path
     shutil.rmtree(sample_images_path)
 
+def extract_images(path, destination):
+    print("Extracting images")
+    with tarfile.open(path, "r:gz") as archive_file:
+        for item in archive_file.getmembers():
+            print("Extracting {}".format(item.name))
+            archive_file.extract(item, path=destination)
+
+def verify_hash(path, sha256_hash):
+    with open(path, "rb") as f:
+        file_hash = hashlib.sha256(f.read()).hexdigest()
+    assert file_hash == sha256_hash
+
 
 @pytest.mark.integration
 @pytest.mark.parametrize("test_file,profile_name", [