Skip to content

Commit a4c63b4

Browse files
henryborchershenryborchers
authored
Merge pull request #247 from UIUCLibrary/dev
tests: cache and verify hash value of sample images
2 parents 5616737 + a0b49de commit a4c63b4

File tree

9 files changed

+141
-49
lines changed

9 files changed

+141
-49
lines changed

‎.github/workflows/build_wheels.yml‎

Lines changed: 10 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -31,6 +31,15 @@ jobs:
3131

3232
steps:
3333
- uses: actions/checkout@v4
34+
- name: Cache sample files
35+
id: cache_samples
36+
uses: actions/cache@v4
37+
with:
38+
path: "${{ github.workspace }}/sampledata/"
39+
key: sample-images
40+
- name: download sample files
41+
if: steps.cache_samples.outputs.cache-hit != 'true'
42+
run: curl -L https://nexus.library.illinois.edu/repository/sample-data/images/sample_images.tar.gz --create-dirs -o "${{ github.workspace }}/sampledata/sample_images.tar.gz"
3443
- uses: actions/setup-python@v5
3544
with:
3645
python-version: '3.11'
@@ -51,6 +60,7 @@ jobs:
5160
# uses: pypa/[email protected]
5261
env:
5362
CONAN_COMPILER_VERSION: ${{ matrix.compiler_version }}
63+
SAMPLE_IMAGES_ARCHIVE: "${{ github.workspace }}/sampledata/sample_images.tar.gz"
5464

5565
# CIBW_SOME_OPTION: value
5666
- uses: actions/upload-artifact@v4

‎.github/workflows/tox_matrix.yml‎

Lines changed: 13 additions & 4 deletions
Original file line numberDiff line numberDiff line change
@@ -42,20 +42,28 @@ jobs:
4242
fail-fast: false
4343
name: Python ${{ matrix.python-version }} ${{ matrix.os }} build
4444
steps:
45-
4645
- uses: actions/checkout@v3
46+
- name: Cache sample files
47+
id: cache_samples
48+
uses: actions/cache@v4
49+
with:
50+
path: "${{ github.workspace }}/sampledata/"
51+
key: sample-images
52+
- name: download sample files
53+
if: steps.cache_samples.outputs.cache-hit != 'true'
54+
run: curl -L https://nexus.library.illinois.edu/repository/sample-data/images/sample_images.tar.gz --create-dirs -o "${{ github.workspace }}/sampledata/sample_images.tar.gz"
4755
- uses: actions/setup-python@v5
4856
with:
4957
python-version: ${{ matrix.python-version }}
5058
cache: 'pip' # caching pip dependencies
5159
- name: "install Python dependencies"
5260
run: |
5361
pip install uv
54-
- uses: actions/cache@v3
62+
- uses: actions/cache@v4
5563
id: cache
5664
with:
5765
path: "${{ matrix.conan_user_home }}/.conan2"
58-
key: ${{ runner.os }}-${{ hashFiles('**/conanfile.py') }}
66+
key: ${{ runner.os }}-${{ hashFiles('**/conanfile.py', 'conan.lock') }}
5967

6068
- name: Build conan packages on Non-Windows Operating Systems
6169
if: ${{ !contains(matrix.os, 'windows') && steps.cache.outputs.cache-hit != 'true' }}
@@ -80,10 +88,11 @@ jobs:
8088
"C:\Program Files\Microsoft Visual Studio\2022\Enterprise\Common7\Tools\VsDevCmd.bat" -arch=amd64 && uv run --only-group tox --with tox-uv tox -e ${{ steps.tox-env.outputs.result }}
8189
env:
8290
CONAN_USER_HOME: ${{ matrix.conan_user_home }}
91+
SAMPLE_IMAGES_ARCHIVE: "${{ github.workspace }}/sampledata/sample_images.tar.gz"
8392
- name: Run tox
8493
if: "!contains(matrix.os, 'windows')"
8594
run: cc --version && cc -dumpfullversion -dumpversion && uv run --only-group tox --with tox-uv tox -e ${{ steps.tox-env.outputs.result }} -vvv
8695
env:
8796
CONAN_COMPILER_LIBCXX: ${{ matrix.compiler_libcxx }}
8897
CONAN_USER_HOME: ${{ matrix.conan_user_home }}
89-
98+
SAMPLE_IMAGES_ARCHIVE: "${{ github.workspace }}/sampledata/sample_images.tar.gz"

‎Jenkinsfile‎

Lines changed: 17 additions & 17 deletions
Original file line numberDiff line numberDiff line change
@@ -106,7 +106,7 @@ def test_cpp_code(buildPath){
106106
tee('logs/cmake-build.log'){
107107
sh(label: 'Building C++ Code',
108108
script: """conan install . -if ${buildPath}
109-
cmake -B ${buildPath} -Wdev -DCMAKE_TOOLCHAIN_FILE=build/conan_paths.cmake -DCMAKE_POSITION_INDEPENDENT_CODE:BOOL=true -DBUILD_TESTING:BOOL=true -DCMAKE_CXX_FLAGS="-fprofile-arcs -ftest-coverage -Wall -Wextra"
109+
cmake -B ${buildPath} -Wdev -DSAMPLE_IMAGES_ARCHIVE=\${SAMPLE_IMAGES_ARCHIVE} -DCMAKE_TOOLCHAIN_FILE=build/conan_paths.cmake -DCMAKE_POSITION_INDEPENDENT_CODE:BOOL=true -DBUILD_TESTING:BOOL=true -DCMAKE_CXX_FLAGS="-fprofile-arcs -ftest-coverage -Wall -Wextra"
110110
cmake --build ${buildPath} -j \$(grep -c ^processor /proc/cpuinfo)
111111
"""
112112
)
@@ -143,7 +143,7 @@ def windows_wheels(pythonVersions, testPackages, params, wheelStashes){
143143
script: "docker image rm --no-prune ${dockerImageName}",
144144
returnStatus: true
145145
)
146-
bat "${tool(name: 'Default', type: 'git')} clean -dfx"
146+
bat "${tool(name: 'Default', type: 'git')} clean -dffx"
147147
}
148148
}
149149
}
@@ -188,7 +188,7 @@ def windows_wheels(pythonVersions, testPackages, params, wheelStashes){
188188
}
189189
}
190190
} finally {
191-
bat "${tool(name: 'Default', type: 'git')} clean -dfx"
191+
bat "${tool(name: 'Default', type: 'git')} clean -dffx"
192192
}
193193
}
194194
}
@@ -236,7 +236,7 @@ def linux_wheels(pythonVersions, testPackages, params, wheelStashes){
236236
}
237237
}
238238
} finally {
239-
sh "${tool(name: 'Default', type: 'git')} clean -dfx"
239+
sh "${tool(name: 'Default', type: 'git')} clean -dffx"
240240
}
241241
}
242242
}
@@ -269,7 +269,7 @@ def linux_wheels(pythonVersions, testPackages, params, wheelStashes){
269269
}
270270
}
271271
} finally {
272-
sh "${tool(name: 'Default', type: 'git')} clean -dfx"
272+
sh "${tool(name: 'Default', type: 'git')} clean -dffx"
273273
}
274274
}
275275
}
@@ -327,7 +327,7 @@ def mac_wheels(pythonVersions, testPackages, params, wheelStashes){
327327
wheelStashes << "python${pythonVersion} mac ${arch} wheel"
328328
archiveArtifacts artifacts: 'dist/*.whl'
329329
} finally {
330-
sh "${tool(name: 'Default', type: 'git')} clean -dfx"
330+
sh "${tool(name: 'Default', type: 'git')} clean -dffx"
331331
}
332332
}
333333
}
@@ -353,7 +353,7 @@ def mac_wheels(pythonVersions, testPackages, params, wheelStashes){
353353
}
354354
}
355355
} finally {
356-
sh "${tool(name: 'Default', type: 'git')} clean -dfx"
356+
sh "${tool(name: 'Default', type: 'git')} clean -dffx"
357357
}
358358
}
359359
}
@@ -401,7 +401,7 @@ def mac_wheels(pythonVersions, testPackages, params, wheelStashes){
401401
archiveArtifacts artifacts: 'dist/*.whl'
402402
}
403403
} finally{
404-
sh "${tool(name: 'Default', type: 'git')} clean -dfx"
404+
sh "${tool(name: 'Default', type: 'git')} clean -dffx"
405405
}
406406
}
407407
}
@@ -434,7 +434,7 @@ def mac_wheels(pythonVersions, testPackages, params, wheelStashes){
434434
}
435435
archiveArtifacts artifacts: 'dist/*.whl'
436436
} finally {
437-
sh "${tool(name: 'Default', type: 'git')} clean -dfx"
437+
sh "${tool(name: 'Default', type: 'git')} clean -dffx"
438438
}
439439
}
440440
}
@@ -884,7 +884,7 @@ pipeline {
884884
}
885885
post{
886886
cleanup{
887-
sh "git clean -dfx"
887+
sh "git clean -dffx"
888888
}
889889
}
890890
}
@@ -920,7 +920,7 @@ pipeline {
920920
).trim().split('\n')
921921
}
922922
} finally{
923-
sh "${tool(name: 'Default', type: 'git')} clean -dfx"
923+
sh "${tool(name: 'Default', type: 'git')} clean -dffx"
924924
}
925925
}
926926

@@ -952,7 +952,7 @@ pipeline {
952952
}
953953
}
954954
} finally{
955-
sh "${tool(name: 'Default', type: 'git')} clean -dfx"
955+
sh "${tool(name: 'Default', type: 'git')} clean -dffx"
956956
}
957957
}
958958
} finally {
@@ -1002,7 +1002,7 @@ pipeline {
10021002
}
10031003
}
10041004
} finally{
1005-
bat "${tool(name: 'Default', type: 'git')} clean -dfx"
1005+
bat "${tool(name: 'Default', type: 'git')} clean -dffx"
10061006
}
10071007
}
10081008
parallel(
@@ -1051,7 +1051,7 @@ pipeline {
10511051
}
10521052
}
10531053
} finally {
1054-
bat "${tool(name: 'Default', type: 'git')} clean -dfx"
1054+
bat "${tool(name: 'Default', type: 'git')} clean -dffx"
10551055
}
10561056
} finally{
10571057
if (image){
@@ -1204,7 +1204,7 @@ pipeline {
12041204
}
12051205
}
12061206
}finally{
1207-
sh "${tool(name: 'Default', type: 'git')} clean -dfx"
1207+
sh "${tool(name: 'Default', type: 'git')} clean -dffx"
12081208
}
12091209
}
12101210
}
@@ -1276,7 +1276,7 @@ pipeline {
12761276
}
12771277
}
12781278
} finally {
1279-
bat "${tool(name: 'Default', type: 'git')} clean -dfx"
1279+
bat "${tool(name: 'Default', type: 'git')} clean -dffx"
12801280

12811281
}
12821282
}
@@ -1352,7 +1352,7 @@ pipeline {
13521352
}
13531353
}
13541354
} finally{
1355-
sh "${tool(name: 'Default', type: 'git')} clean -dfx"
1355+
sh "${tool(name: 'Default', type: 'git')} clean -dffx"
13561356
}
13571357
} finally{
13581358
if(image){

‎ci/docker/linux/jenkins/Dockerfile‎

Lines changed: 12 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -1,6 +1,8 @@
11
ARG CONAN_USER_HOME=/conan
22
ARG CONAN_HOME=${CONAN_USER_HOME}/.conan2
33

4+
ARG SAMPLE_IMAGES_URL=https://nexus.library.illinois.edu/repository/sample-data/images/sample_images.tar.gz
5+
46
ARG PYTHON_VERSION=latest
57

68
ARG CONAN_CENTER_PROXY_V2_URL=https://center2.conan.io
@@ -33,6 +35,12 @@ RUN mkdir -p /.cache/pip && \
3335
#==============================================================================
3436

3537

38+
FROM base_image AS sample_files_downloader
39+
ARG SAMPLE_IMAGES_URL
40+
ENV EXPECTED_SHA256="0461f57db3806ca47d9063151eec4bc0720c66a83153dda04e230f838b64f063"
41+
RUN mkdir /sampledata && wget $SAMPLE_IMAGES_URL -P /sampledata && echo "$EXPECTED_SHA256 /sampledata/sample_images.tar.gz" | sha256sum --check
42+
43+
3644
FROM base_image AS sonar_builder
3745
ARG SONAR_USER_HOME
3846

@@ -141,11 +149,14 @@ COPY --from=dr_memory_builder /opt/drmemory /opt/drmemory/
141149
RUN ln -s /opt/drmemory/bin64/drmemory /usr/local/bin/drmemory && \
142150
drmemory -version
143151

152+
COPY --from=sample_files_downloader /sampledata /sampledata
153+
144154
ARG CONAN_USER_HOME
145155
ARG CONAN_HOME
146156
COPY --from=conan_builder --chmod=777 ${CONAN_HOME} ${CONAN_HOME}
147157
ENV CONAN_USER_HOME=${CONAN_USER_HOME}\
148-
CONAN_HOME=${CONAN_HOME}
158+
CONAN_HOME=${CONAN_HOME}\
159+
SAMPLE_IMAGES_ARCHIVE=/sampledata/sample_images.tar.gz
149160

150161
# To help mark the image as a CI image so it can be cleaned up more easily
151162
LABEL purpose=ci

‎ci/docker/linux/tox/Dockerfile‎

Lines changed: 17 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -11,6 +11,8 @@ ARG CONAN_CENTER_PROXY_V2_URL=https://center2.conan.io
1111
# If you want to use a diffrent remote for Conan, such as a proxy. Set the CONAN_CENTER_PROXY_V2_URL
1212
# Not this is only for building the image. The actual conan center proxy URL is set in the remotes.json file.
1313

14+
ARG SAMPLE_IMAGES_URL=https://nexus.library.illinois.edu/repository/sample-data/images/sample_images.tar.gz
15+
1416
FROM ghcr.io/astral-sh/uv:latest AS uv_builder
1517

1618
FROM ubuntu:24.04 AS base
@@ -25,6 +27,18 @@ RUN --mount=type=cache,target=/var/cache/apt,sharing=locked \
2527
< /tmp/apt-packages.txt xargs apt-get install -y && \
2628
apt-get clean && \
2729
rm -rf /var/lib/apt/lists/*
30+
31+
# ==============================================================================
32+
FROM base AS sample_files_downloader
33+
RUN --mount=type=cache,target=/var/cache/apt,sharing=locked \
34+
apt-get update && \
35+
apt-get install -y wget
36+
37+
ARG SAMPLE_IMAGES_URL
38+
ENV EXPECTED_SHA256="0461f57db3806ca47d9063151eec4bc0720c66a83153dda04e230f838b64f063"
39+
40+
RUN mkdir /sampledata && wget $SAMPLE_IMAGES_URL -P /sampledata && echo "$EXPECTED_SHA256 /sampledata/sample_images.tar.gz" | sha256sum --check
41+
2842
# ==============================================================================
2943

3044
FROM base AS conan_builder
@@ -74,9 +88,11 @@ FROM base
7488
COPY --from=uv_builder /uv /uvx /bin/
7589
ARG CONAN_USER_HOME
7690
ARG CONAN_HOME
91+
COPY --from=sample_files_downloader /sampledata /sampledata
7792
COPY --from=conan_builder --chmod=777 ${CONAN_HOME}/ ${CONAN_HOME}/
7893
ENV CONAN_USER_HOME=${CONAN_USER_HOME}\
79-
CONAN_HOME=${CONAN_HOME}
94+
CONAN_HOME=${CONAN_HOME}\
95+
SAMPLE_IMAGES_ARCHIVE=/sampledata/sample_images.tar.gz
8096

8197
# To help mark the image as a CI image so it can be cleaned up more easily
8298
LABEL purpose=ci

‎scripts/resources/windows/tox/Dockerfile‎

Lines changed: 16 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -15,6 +15,8 @@ ARG PIP_DOWNLOAD_CACHE=c:/users/containeradministrator/appdata/local/pip
1515
ARG CHOCOLATEY_SOURCE=https://chocolatey.org/api/v2
1616
ARG chocolateyVersion
1717

18+
ARG SAMPLE_IMAGES_URL=https://nexus.library.illinois.edu/repository/sample-data/images/sample_images.tar.gz
19+
1820
FROM ${FROM_IMAGE} AS certsgen
1921
RUN certutil -generateSSTFromWU roots.sst
2022

@@ -61,6 +63,16 @@ RUN Set-ExecutionPolicy Bypass -Scope Process -Force; `
6163
Write-Host "Verifying installed packages - Done"
6264

6365

66+
# ==============================================================================
67+
FROM base_builder AS sample_files_downloader
68+
ARG SAMPLE_IMAGES_URL
69+
ENV EXPECTED_SHA256="0461f57db3806ca47d9063151eec4bc0720c66a83153dda04e230f838b64f063"
70+
ENV downloadPath="C:/sampledata/"
71+
RUN if (-not (Test-Path -Path ${env:downloadPath} -PathType Container)) {`
72+
New-Item -Path ${env:downloadPath} -ItemType Directory -Force`
73+
}; `
74+
Invoke-WebRequest -Uri ${env:SAMPLE_IMAGES_URL} -OutFile "${env:downloadPath}\sample_images.tar.gz"
75+
6476
# ==============================================================================
6577
FROM base_builder AS conan_builder
6678
ARG CONAN_HOME
@@ -105,11 +117,14 @@ RUN New-Item -type directory -path ${Env:PIP_DOWNLOAD_CACHE} -Force | Out-Null ;
105117
New-Item -type directory -path ${Env:UV_CACHE_DIR} -Force | Out-Null
106118
ARG CONAN_HOME
107119
COPY --from=conan_builder ${CONAN_HOME}/ ${CONAN_HOME}/
120+
COPY --from=sample_files_downloader c:/sampledata/ C:/sampledata/
121+
108122
ARG CONAN_USER_HOME
109123
ENV CONAN_USER_HOME=${CONAN_USER_HOME}`
110124
CONAN_HOME=${CONAN_HOME}`
111125
PIP_DOWNLOAD_CACHE=${PIP_DOWNLOAD_CACHE}`
112-
UV_CACHE_DIR=${UV_CACHE_DIR}
126+
UV_CACHE_DIR=${UV_CACHE_DIR}`
127+
SAMPLE_IMAGES_ARCHIVE=C:/sampledata/sample_images.tar.gz
113128

114129
WORKDIR C:/src
115130
ENV DISTUTILS_USE_SDK=1

0 commit comments

Comments
 (0)