Skip to content

Commit 3bceff3

Browse files
pzehnerpzehner
committed
Do not build images on demand
1 parent 6f6c60c commit 3bceff3

File tree

6 files changed

+58
-185
lines changed

6 files changed

+58
-185
lines changed

‎.github/workflows/__check_docker_files.yaml‎

Lines changed: 0 additions & 61 deletions
This file was deleted.

‎.github/workflows/build_test.yaml‎

Lines changed: 21 additions & 54 deletions
Original file line numberDiff line numberDiff line change
@@ -40,39 +40,10 @@ jobs:
4040
files: ./cmake/ ./CMakeLists.txt ./docs/ ./README.md ./common/ ./fft/ ./examples/ ./install_test/
4141
config: ./.typos.toml
4242

43-
# check if Docker images should be recreated
44-
check_docker_files:
45-
uses: ./.github/workflows/__check_docker_files.yaml
46-
47-
with:
48-
event_name: ${{ github.event_name }}
49-
50-
# build Docker images if necessary
51-
build_base:
52-
needs: check_docker_files
53-
54-
if: ${{ needs.check_docker_files.outputs.docker_files_have_changed == 'true' }}
55-
56-
uses: ./.github/workflows/__build_base.yaml
57-
permissions:
58-
contents: read
59-
packages: write
60-
61-
with:
62-
image_suffix: ${{ needs.check_docker_files.outputs.image_suffix }}
63-
image_tag: ${{ needs.check_docker_files.outputs.image_tag }}
64-
6543
# build project
6644
build:
6745
runs-on: ubuntu-latest
6846

69-
needs:
70-
- check_docker_files
71-
- build_base
72-
73-
# run this job even if build_base did not run
74-
if: ${{ ! cancelled() && (needs.build_base.result == 'success' || needs.build_base.result == 'skipped') }}
75-
7647
strategy:
7748
matrix:
7849
backend:
@@ -203,7 +174,7 @@ jobs:
203174

204175
- name: Configure
205176
run: |
206-
docker run -v ${{ github.workspace }}:/work ghcr.io/kokkos/kokkos-fft/base_${{ matrix.backend.image }}_${{ needs.check_docker_files.outputs.image_suffix }}:${{ needs.check_docker_files.outputs.image_tag }} \
177+
docker run -v ${{ github.workspace }}:/work ghcr.io/kokkos/kokkos-fft/base_${{ matrix.backend.image }}:latest \
207178
cmake -B build \
208179
-DCMAKE_BUILD_TYPE=${{ matrix.backend.cmake_build_type }} \
209180
-DCMAKE_C_COMPILER=${{ matrix.backend.compiler.c }} \
@@ -219,7 +190,7 @@ jobs:
219190
220191
- name: Build
221192
run: |
222-
docker run -v ${{ github.workspace }}:/work ghcr.io/kokkos/kokkos-fft/base_${{ matrix.backend.image }}_${{ needs.check_docker_files.outputs.image_suffix }}:${{ needs.check_docker_files.outputs.image_tag }} \
193+
docker run -v ${{ github.workspace }}:/work ghcr.io/kokkos/kokkos-fft/base_${{ matrix.backend.image }}:latest \
223194
cmake --build build -j $(( $(nproc) * 2 + 1 ))
224195
225196
- name: Prepare artifacts
@@ -236,12 +207,12 @@ jobs:
236207

237208
# consume the project in a dummy project
238209
# ideally, this part should be executed in a different job using almost
239-
# the same `matrix` (minus `target`) as the `build` job and having the
240-
# same `needs` and `if` as the `test` job; this would however make a
241-
# duplicated and error-prone `matrix`, and sharing a `matrix` between
242-
# jobs is currently not possible with GitHub Actions (neither directly,
243-
# nor using YAML anchors, other options were too cumbersome when writing
244-
# this); consequently, this part is just here for now
210+
# the same `matrix` (minus `target`) as the `build` job; this would
211+
# however make a duplicated and error-prone `matrix`, and sharing a
212+
# `matrix` between jobs is currently not possible with GitHub Actions
213+
# (neither directly, nor using YAML anchors, other options were too
214+
# cumbersome when writing this); consequently, this part is just here for
215+
# now
245216

246217
- name: Prepare as subdirectory
247218
run: |
@@ -252,7 +223,7 @@ jobs:
252223
253224
- name: Configure and build test project as subdirectory
254225
run: |
255-
docker run -v ${{ github.workspace }}:/work ghcr.io/kokkos/kokkos-fft/base_${{ matrix.backend.image }}_${{ needs.check_docker_files.outputs.image_suffix }}:${{ needs.check_docker_files.outputs.image_tag }} \
226+
docker run -v ${{ github.workspace }}:/work ghcr.io/kokkos/kokkos-fft/base_${{ matrix.backend.image }}:latest \
256227
cmake -B install_test/as_subdirectory/build \
257228
-DCMAKE_BUILD_TYPE=${{ matrix.backend.cmake_build_type }} \
258229
-DCMAKE_C_COMPILER=${{ matrix.backend.compiler.c }} \
@@ -262,12 +233,12 @@ jobs:
262233
${{ matrix.backend.cmake_flags.kokkos_fft }} \
263234
${{ matrix.target.cmake_flags }} \
264235
install_test/as_subdirectory
265-
docker run -v ${{ github.workspace }}:/work ghcr.io/kokkos/kokkos-fft/base_${{ matrix.backend.image }}_${{ needs.check_docker_files.outputs.image_suffix }}:${{ needs.check_docker_files.outputs.image_tag }} \
236+
docker run -v ${{ github.workspace }}:/work ghcr.io/kokkos/kokkos-fft/base_${{ matrix.backend.image }}:latest \
266237
cmake --build install_test/as_subdirectory/build
267238
268239
- name: Configure, build and install Kokkos as library
269240
run: |
270-
docker run -v ${{ github.workspace }}:/work ghcr.io/kokkos/kokkos-fft/base_${{ matrix.backend.image }}_${{ needs.check_docker_files.outputs.image_suffix }}:${{ needs.check_docker_files.outputs.image_tag }} \
241+
docker run -v ${{ github.workspace }}:/work ghcr.io/kokkos/kokkos-fft/base_${{ matrix.backend.image }}:latest \
271242
cmake -B build_kokkos \
272243
-DCMAKE_INSTALL_PREFIX=/work/install \
273244
-DCMAKE_BUILD_TYPE=${{ matrix.backend.cmake_build_type }} \
@@ -276,14 +247,14 @@ jobs:
276247
-DCMAKE_CXX_STANDARD=${{ matrix.backend.cmake_flags.cxx_standard }} \
277248
${{ matrix.backend.cmake_flags.kokkos }} \
278249
tpls/kokkos
279-
docker run -v ${{ github.workspace }}:/work ghcr.io/kokkos/kokkos-fft/base_${{ matrix.backend.image }}_${{ needs.check_docker_files.outputs.image_suffix }}:${{ needs.check_docker_files.outputs.image_tag }} \
250+
docker run -v ${{ github.workspace }}:/work ghcr.io/kokkos/kokkos-fft/base_${{ matrix.backend.image }}:latest \
280251
cmake --build build_kokkos -j $(( $(nproc) * 2 + 1 ))
281-
docker run -v ${{ github.workspace }}:/work ghcr.io/kokkos/kokkos-fft/base_${{ matrix.backend.image }}_${{ needs.check_docker_files.outputs.image_suffix }}:${{ needs.check_docker_files.outputs.image_tag }} \
252+
docker run -v ${{ github.workspace }}:/work ghcr.io/kokkos/kokkos-fft/base_${{ matrix.backend.image }}:latest \
282253
cmake --install build_kokkos
283254
284255
- name: Configure, build and install KokkosFFT as library
285256
run: |
286-
docker run -v ${{ github.workspace }}:/work ghcr.io/kokkos/kokkos-fft/base_${{ matrix.backend.image }}_${{ needs.check_docker_files.outputs.image_suffix }}:${{ needs.check_docker_files.outputs.image_tag }} \
257+
docker run -v ${{ github.workspace }}:/work ghcr.io/kokkos/kokkos-fft/base_${{ matrix.backend.image }}:latest \
287258
cmake -B build_kokkos_fft \
288259
-DCMAKE_INSTALL_PREFIX=/work/install\
289260
-DCMAKE_PREFIX_PATH=/work/install \
@@ -293,22 +264,22 @@ jobs:
293264
-DCMAKE_CXX_STANDARD=${{ matrix.backend.cmake_flags.cxx_standard }} \
294265
${{ matrix.backend.cmake_flags.kokkos_fft }} \
295266
${{ matrix.target.cmake_flags }}
296-
docker run -v ${{ github.workspace }}:/work ghcr.io/kokkos/kokkos-fft/base_${{ matrix.backend.image }}_${{ needs.check_docker_files.outputs.image_suffix }}:${{ needs.check_docker_files.outputs.image_tag }} \
267+
docker run -v ${{ github.workspace }}:/work ghcr.io/kokkos/kokkos-fft/base_${{ matrix.backend.image }}:latest \
297268
cmake --build build_kokkos_fft -j $(( $(nproc) * 2 + 1 ))
298-
docker run -v ${{ github.workspace }}:/work ghcr.io/kokkos/kokkos-fft/base_${{ matrix.backend.image }}_${{ needs.check_docker_files.outputs.image_suffix }}:${{ needs.check_docker_files.outputs.image_tag }} \
269+
docker run -v ${{ github.workspace }}:/work ghcr.io/kokkos/kokkos-fft/base_${{ matrix.backend.image }}:latest \
299270
cmake --install build_kokkos_fft
300271
301272
- name: Configure and build test project as library
302273
run: |
303-
docker run -v ${{ github.workspace }}:/work ghcr.io/kokkos/kokkos-fft/base_${{ matrix.backend.image }}_${{ needs.check_docker_files.outputs.image_suffix }}:${{ needs.check_docker_files.outputs.image_tag }} \
274+
docker run -v ${{ github.workspace }}:/work ghcr.io/kokkos/kokkos-fft/base_${{ matrix.backend.image }}:latest \
304275
cmake -B install_test/as_library/build \
305276
-DCMAKE_BUILD_TYPE=${{ matrix.backend.cmake_build_type }} \
306277
-DCMAKE_PREFIX_PATH=/work/install \
307278
-DCMAKE_C_COMPILER=${{ matrix.backend.compiler.c }} \
308279
-DCMAKE_CXX_COMPILER=${{ matrix.backend.compiler.cxx }} \
309280
-DCMAKE_CXX_STANDARD=${{ matrix.backend.cmake_flags.cxx_standard }} \
310281
install_test/as_library
311-
docker run -v ${{ github.workspace }}:/work ghcr.io/kokkos/kokkos-fft/base_${{ matrix.backend.image }}_${{ needs.check_docker_files.outputs.image_suffix }}:${{ needs.check_docker_files.outputs.image_tag }} \
282+
docker run -v ${{ github.workspace }}:/work ghcr.io/kokkos/kokkos-fft/base_${{ matrix.backend.image }}:latest \
312283
cmake --build install_test/as_library/build -j $(( $(nproc) * 2 + 1 ))
313284
314285
# test the project
@@ -317,12 +288,8 @@ jobs:
317288
continue-on-error: ${{ matrix.backend.unstable }}
318289

319290
needs:
320-
- check_docker_files
321291
- build
322292

323-
# run this job even if build_base did not run
324-
if: ${{ ! cancelled() && needs.build.result == 'success' }}
325-
326293
strategy:
327294
matrix:
328295
backend:
@@ -382,7 +349,7 @@ jobs:
382349
run \
383350
-m "singularity/3.8.3/gcc-11.2.0" \
384351
-e local \
385-
singularity pull oras://ghcr.io/kokkos/kokkos-fft/base_${{ matrix.backend.image }}_singularity_${{ needs.check_docker_files.outputs.image_suffix }}:${{ needs.check_docker_files.outputs.image_tag }}
352+
singularity pull oras://ghcr.io/kokkos/kokkos-fft/base_${{ matrix.backend.image }}_singularity:latest
386353
if: ${{ matrix.backend.use_singularity }}
387354

388355
- name: Run CUDA tests within Slurm job and Singularity image
@@ -391,12 +358,12 @@ jobs:
391358
-m "singularity/3.8.3/gcc-11.2.0" \
392359
-e gpu \
393360
-g a100 \
394-
singularity run --nv --bind $PWD/build:/work/build -H /work/build base_${{ matrix.backend.image }}_singularity_${{ needs.check_docker_files.outputs.image_suffix }}_${{ needs.check_docker_files.outputs.image_tag }}.sif \
361+
singularity run --nv --bind $PWD/build:/work/build -H /work/build base_${{ matrix.backend.image }}_singularity_latest.sif \
395362
ctest --output-on-failure
396363
if: ${{ matrix.backend.use_singularity }}
397364

398365
- name: Run OpenMP/Threads/Serial tests within Docker image
399366
run: |
400-
docker run -v $PWD/build:/work/build -w /work/build ghcr.io/kokkos/kokkos-fft/base_${{ matrix.backend.image }}_${{ needs.check_docker_files.outputs.image_suffix }}:${{ needs.check_docker_files.outputs.image_tag }} \
367+
docker run -v $PWD/build:/work/build -w /work/build ghcr.io/kokkos/kokkos-fft/base_${{ matrix.backend.image }}:latest \
401368
ctest --output-on-failure
402369
if: ${{ ! matrix.backend.use_singularity }}

‎.github/workflows/cleanup_base.yaml‎

Lines changed: 6 additions & 12 deletions
Original file line numberDiff line numberDiff line change
@@ -26,15 +26,9 @@ jobs:
2626
organization: kokkos
2727
type: container
2828
names: |
29-
kokkos-fft/base_nvcc_main
30-
kokkos-fft/base_nvcc_pr
31-
kokkos-fft/base_nvcc_singularity_main
32-
kokkos-fft/base_nvcc_singularity_pr
33-
kokkos-fft/base_rocm_main
34-
kokkos-fft/base_rocm_pr
35-
kokkos-fft/base_gcc_main
36-
kokkos-fft/base_gcc_pr
37-
kokkos-fft/base_clang_main
38-
kokkos-fft/base_clang_pr
39-
kokkos-fft/base_intel_main
40-
kokkos-fft/base_intel_pr
29+
kokkos-fft/base_nvcc
30+
kokkos-fft/base_nvcc_singularity
31+
kokkos-fft/base_rocm
32+
kokkos-fft/base_gcc
33+
kokkos-fft/base_clang
34+
kokkos-fft/base_intel

.github/workflows/__build_base.yaml renamed to .github/workflows/create_base.yaml

Lines changed: 20 additions & 20 deletions
Original file line numberDiff line numberDiff line change
@@ -4,25 +4,25 @@
44

55
# Generate base images for each compiler environment. Images are stored on
66
# Github registry as Docker format inconditionnaly and as Singularity format
7-
# only if requested. Docker images are used for building Kokkos FFT, while
7+
# only if requested. Docker images are used for building Kokkos FFT, while
88
# Singularity images are used for test execution on the righteous hardware.
9-
# This workflow can only be invoked through another workflows.
9+
# It runs once month, on any push on `main` if Docker files are changed, and
10+
# mannualy.
1011

11-
name: Build base images
12+
name: Create base images
1213

1314
on:
14-
workflow_call:
15-
inputs:
16-
image_suffix:
17-
description: "Suffix of the Docker and Singularity images"
18-
required: false
19-
default: main
20-
type: string
21-
image_tag:
22-
description: "Tag of the Docker and Singularity images"
23-
required: false
24-
default: latest
25-
type: string
15+
schedule:
16+
- cron: "0 1 2 * *" # every 2nd of the month at 1am UTC
17+
push:
18+
branches:
19+
- 'main'
20+
paths:
21+
- 'docker/**/Dockerfile'
22+
pull_request:
23+
paths:
24+
- 'docker/**/Dockerfile'
25+
workflow_dispatch:
2626

2727
env:
2828
# Force the use of BuildKit for Docker
@@ -79,19 +79,19 @@ jobs:
7979
- name: Build Docker image
8080
run: |
8181
docker build \
82-
-t ghcr.io/kokkos/kokkos-fft/base_${{ matrix.image.name }}_${{ inputs.image_suffix }}:${{ inputs.image_tag }} \
83-
--cache-from ghcr.io/kokkos/kokkos-fft/base_${{ matrix.image.name }}_main:latest \
82+
-t ghcr.io/kokkos/kokkos-fft/base_${{ matrix.image.name }}:latest \
83+
--cache-from ghcr.io/kokkos/kokkos-fft/base_${{ matrix.image.name }}:latest \
8484
--build-arg BUILDKIT_INLINE_CACHE=1 \
8585
--progress=plain \
8686
docker/${{ matrix.image.name }}
8787
8888
- name: Push Docker image
89-
run: docker push ghcr.io/kokkos/kokkos-fft/base_${{ matrix.image.name }}_${{ inputs.image_suffix }}:${{ inputs.image_tag }}
89+
run: docker push ghcr.io/kokkos/kokkos-fft/base_${{ matrix.image.name }}:latest
9090

9191
- name: Convert Docker image to Singularity
92-
run: singularity build base.sif docker://ghcr.io/kokkos/kokkos-fft/base_${{ matrix.image.name }}_${{ inputs.image_suffix }}:${{ inputs.image_tag }}
92+
run: singularity build base.sif docker://ghcr.io/kokkos/kokkos-fft/base_${{ matrix.image.name }}:latest
9393
if: ${{ matrix.image.use_singularity }}
9494

9595
- name: Push Singularity image
96-
run: singularity push base.sif oras://ghcr.io/kokkos/kokkos-fft/base_${{ matrix.image.name }}_singularity_${{ inputs.image_suffix }}:${{ inputs.image_tag }}
96+
run: singularity push base.sif oras://ghcr.io/kokkos/kokkos-fft/base_${{ matrix.image.name }}_singularity:latest
9797
if: ${{ matrix.image.use_singularity }}

‎.github/workflows/pre_build_base.yaml‎

Lines changed: 0 additions & 37 deletions
This file was deleted.

‎docker/README.md‎

Lines changed: 11 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -11,6 +11,16 @@ Each backend/compiler has a corresponding Dockerfile.
1111

1212
# Contributing
1313

14+
## Update the Dockerfiles
15+
16+
If you have to update a Dockerfile, please do it in a specific PR.
17+
This is cumbersome, but it guarantees that Docker images are not messed up in the registry.
18+
19+
By instance, let's say you want to add a new backend, which requires a new Dockerfile:
20+
21+
- Step 1: Create a branch for the new Dockerfile. Have it reviewed and merged. The new image will be added to the registry when the branch is merged.
22+
- Step 2: Create a branch for the new backend, which CI uses the newly added Dockerfile.
23+
1424
## CMake installation in Dockerfiles
1525

1626
As the project requires CMake v3.23 at least, and as some Dockerfiles are based on Ubuntu 20.04 images, CMake has to be installed manually.
@@ -24,4 +34,4 @@ When updating the Dockerfiles for a newer version of CMake (if needed), the proc
2434
2. Copy the key ID in the line "PGP sign by XXXXXXXX";
2535
3. Paste it in `https://keys.openpgp.org/` to retrieve the URL of the public key file;
2636
4. Copy the last part in the URL `https://keys.openpgp.org/vks/v1/by-fingerprint/YYYYYYYY`;
27-
5. Update the Dockrfiles with this value.
37+
5. Update the Dockerfiles with this value.

0 commit comments

Comments
 (0)