From 825688bbf2f590f749b19d325f13d4b1daea0419 Mon Sep 17 00:00:00 2001
From: Steve Brasier <steveb@stackhpc.com>
Date: Wed, 24 Jul 2024 10:58:44 +0100
Subject: [PATCH 01/17] determine cuda distro automatically

---
 ansible/roles/cuda/defaults/main.yml | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/ansible/roles/cuda/defaults/main.yml b/ansible/roles/cuda/defaults/main.yml
index 6b377a10b..c109ed90e 100644
--- a/ansible/roles/cuda/defaults/main.yml
+++ b/ansible/roles/cuda/defaults/main.yml
@@ -1,4 +1,4 @@
-cuda_distro: rhel8
+cuda_distro: "rhel{{ ansible_distribution_major_version }}"
 cuda_repo: "https://developer.download.nvidia.com/compute/cuda/repos/{{ cuda_distro }}/x86_64/cuda-{{ cuda_distro }}.repo"
 cuda_driver_stream: default
 cuda_packages:

From a115fe3d159fe3883c845ac3c353d803bd139569 Mon Sep 17 00:00:00 2001
From: Steve Brasier <steveb@stackhpc.com>
Date: Wed, 24 Jul 2024 10:03:11 +0100
Subject: [PATCH 02/17] fix typo in CUDA samples

---
 ansible/roles/cuda/defaults/main.yml | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/ansible/roles/cuda/defaults/main.yml b/ansible/roles/cuda/defaults/main.yml
index c109ed90e..15cdb95c8 100644
--- a/ansible/roles/cuda/defaults/main.yml
+++ b/ansible/roles/cuda/defaults/main.yml
@@ -5,7 +5,7 @@ cuda_packages:
   - cuda
   - nvidia-gds
 # _cuda_version_tuple: # discovered from installed package e.g. ('12', '1', '0')
-cuda_version_short: "{{ _cuda_version_tuple[0] }}.{{ cuda_version_tuple[1] }}"
+cuda_version_short: "{{ _cuda_version_tuple[0] }}.{{ _cuda_version_tuple[1] }}"
 cuda_samples_release_url: "https://github.com/NVIDIA/cuda-samples/archive/refs/tags/v{{ cuda_version_short }}.tar.gz"
 cuda_samples_path: "/home/{{ ansible_user }}/cuda_samples"
 cuda_samples_programs:

From 5e79a9c7b582d4b19cf33bc380d8d187021d4075 Mon Sep 17 00:00:00 2001
From: Steve Brasier <steveb@stackhpc.com>
Date: Wed, 24 Jul 2024 16:53:27 +0100
Subject: [PATCH 03/17] make facts available for cuda

---
 ansible/extras.yml | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/ansible/extras.yml b/ansible/extras.yml
index 445a0cc16..c32f51c32 100644
--- a/ansible/extras.yml
+++ b/ansible/extras.yml
@@ -21,7 +21,7 @@
 - name: Setup CUDA
   hosts: cuda
   become: yes
-  gather_facts: no
+  gather_facts: yes
   tags: cuda
   tasks:
     - import_role:

From f474ac8aa7dfe41b9fb1d67bc7604b35e0cc4bd1 Mon Sep 17 00:00:00 2001
From: Steve Brasier <steveb@stackhpc.com>
Date: Wed, 14 Aug 2024 15:16:54 +0000
Subject: [PATCH 04/17] add RL9 cuda build variant

---
 .github/workflows/fatimage.yml | 5 ++++-
 packer/openstack.pkr.hcl       | 6 ++++++
 2 files changed, 10 insertions(+), 1 deletion(-)

diff --git a/.github/workflows/fatimage.yml b/.github/workflows/fatimage.yml
index 31fcc789a..7d650e719 100644
--- a/.github/workflows/fatimage.yml
+++ b/.github/workflows/fatimage.yml
@@ -10,16 +10,19 @@ jobs:
     name: openstack-imagebuild
     runs-on: ubuntu-22.04
     strategy:
-      matrix:
+      matrix: # build RL8, RL9+OFED, RL9+CUDA versions
         os_version:
           - RL8
           - RL9
         build:
           - openstack.openhpc
           - openstack.openhpc-ofed
+          - openstack.opemhpc-cuda
         exclude:
           - os_version: RL8
             build: openstack.openhpc-ofed
+          - os_version: RL8
+            build: openstack.openhpc-cuda
           - os_version: RL9
             build: openstack.openhpc
     env:
diff --git a/packer/openstack.pkr.hcl b/packer/openstack.pkr.hcl
index bb6af1a38..ac98a7926 100644
--- a/packer/openstack.pkr.hcl
+++ b/packer/openstack.pkr.hcl
@@ -150,6 +150,7 @@ variable "groups" {
     # fat image builds:
     openhpc = ["control", "compute", "login"]
     openhpc-ofed = ["control", "compute", "login", "ofed"]
+    openhpc-cuda = ["control", "compute", "login", "ofed", "cuda"]
   }
 }
 
@@ -195,6 +196,11 @@ build {
     name = "openhpc-ofed"
   }
 
+  # CUDA fat image:
+  source "source.openstack.openhpc" {
+    name = "openhpc-cuda"
+  }
+
   # Extended site-specific image, built on fat image:
   source "source.openstack.openhpc" {
     name = "openhpc-extra"

From a985a1b2308a117926d415735623e3ca43b70dc5 Mon Sep 17 00:00:00 2001
From: Steve Brasier <steveb@stackhpc.com>
Date: Wed, 14 Aug 2024 15:21:53 +0000
Subject: [PATCH 05/17] fix typo in build definitions

---
 .github/workflows/fatimage.yml | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/.github/workflows/fatimage.yml b/.github/workflows/fatimage.yml
index 7d650e719..0e7a6e9a2 100644
--- a/.github/workflows/fatimage.yml
+++ b/.github/workflows/fatimage.yml
@@ -17,7 +17,7 @@ jobs:
         build:
           - openstack.openhpc
           - openstack.openhpc-ofed
-          - openstack.opemhpc-cuda
+          - openstack.openhpc-cuda
         exclude:
           - os_version: RL8
             build: openstack.openhpc-ofed

From 8b7edfdc795c88ca963c7389cb00c1ba3b10aa58 Mon Sep 17 00:00:00 2001
From: Steve Brasier <steveb@stackhpc.com>
Date: Wed, 14 Aug 2024 15:48:51 +0000
Subject: [PATCH 06/17] set packer build volume sizes depending on build
 variant

---
 environments/.stackhpc/ARCUS.pkrvars.hcl     |  3 ---
 environments/.stackhpc/LEAFCLOUD.pkrvars.hcl |  3 ---
 packer/openstack.pkr.hcl                     | 22 ++++++++++----------
 3 files changed, 11 insertions(+), 17 deletions(-)

diff --git a/environments/.stackhpc/ARCUS.pkrvars.hcl b/environments/.stackhpc/ARCUS.pkrvars.hcl
index c07717156..6fd80e7a6 100644
--- a/environments/.stackhpc/ARCUS.pkrvars.hcl
+++ b/environments/.stackhpc/ARCUS.pkrvars.hcl
@@ -1,7 +1,4 @@
 flavor = "vm.ska.cpu.general.small"
-use_blockstorage_volume = true
-volume_size = 15 # GB
-image_disk_format = "qcow2"
 networks = ["4b6b2722-ee5b-40ec-8e52-a6610e14cc51"] # portal-internal (DNS broken on ilab-60)
 ssh_keypair_name = "slurm-app-ci"
 ssh_private_key_file = "~/.ssh/id_rsa"
diff --git a/environments/.stackhpc/LEAFCLOUD.pkrvars.hcl b/environments/.stackhpc/LEAFCLOUD.pkrvars.hcl
index da2d96d38..5adf4199c 100644
--- a/environments/.stackhpc/LEAFCLOUD.pkrvars.hcl
+++ b/environments/.stackhpc/LEAFCLOUD.pkrvars.hcl
@@ -1,8 +1,5 @@
 flavor = "ec1.large"
-use_blockstorage_volume = true
-volume_size = 15 # GB
 volume_type = "unencrypted"
-image_disk_format = "qcow2"
 networks = ["909e49e8-6911-473a-bf88-0495ca63853c"] # slurmapp-ci
 ssh_keypair_name = "slurm-app-ci"
 ssh_private_key_file = "~/.ssh/id_rsa"
diff --git a/packer/openstack.pkr.hcl b/packer/openstack.pkr.hcl
index ac98a7926..16f91a5cc 100644
--- a/packer/openstack.pkr.hcl
+++ b/packer/openstack.pkr.hcl
@@ -118,24 +118,24 @@ variable "manifest_output_path" {
   default = "packer-manifest.json"
 }
 
-variable "use_blockstorage_volume" {
-  type = bool
-  default = false
-}
-
 variable "volume_type" {
   type = string
   default = null
 }
 
 variable "volume_size" {
-  type = number
-  default = null # When not specified use the size of the builder instance root disk
+  type = map(number)
+  default = {
+    # fat image builds, GB:
+    openhpc = 15
+    openhpc-ofed = 15
+    openhpc-cuda = 25
+  }
 }
 
 variable "image_disk_format" {
   type = string
-  default = null # When not specified use the image default
+  default = "qcow2"
 }
 
 variable "metadata" {
@@ -157,13 +157,13 @@ variable "groups" {
 source "openstack" "openhpc" {
   # Build VM:
   flavor = var.flavor
-  use_blockstorage_volume = var.use_blockstorage_volume
+  use_blockstorage_volume = true
   volume_type = var.volume_type
+  volume_size = var.volume_size
   metadata = var.metadata
   networks = var.networks
   floating_ip_network = var.floating_ip_network
   security_groups = var.security_groups
-  volume_size = var.volume_size
   
   # Input image:
   source_image = "${var.source_image[var.os_version]}"
@@ -179,7 +179,7 @@ source "openstack" "openhpc" {
   ssh_bastion_private_key_file = var.ssh_bastion_private_key_file
   
   # Output image:
-  image_disk_format = var.image_disk_format
+  image_disk_format = "qcow2"
   image_visibility = var.image_visibility
   image_name = "${source.name}-${var.os_version}-${local.timestamp}-${substr(local.git_commit, 0, 8)}"
 }

From 905113cbc13a7451b36de53efe616fb4d8fb4b6a Mon Sep 17 00:00:00 2001
From: Steve Brasier <steveb@stackhpc.com>
Date: Wed, 14 Aug 2024 16:20:55 +0000
Subject: [PATCH 07/17] fix volume size definition

---
 packer/openstack.pkr.hcl | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/packer/openstack.pkr.hcl b/packer/openstack.pkr.hcl
index 16f91a5cc..e8bbcf399 100644
--- a/packer/openstack.pkr.hcl
+++ b/packer/openstack.pkr.hcl
@@ -159,7 +159,7 @@ source "openstack" "openhpc" {
   flavor = var.flavor
   use_blockstorage_volume = true
   volume_type = var.volume_type
-  volume_size = var.volume_size
+  volume_size = var.volume_size[source.name]
   metadata = var.metadata
   networks = var.networks
   floating_ip_network = var.floating_ip_network

From 4a6a4fa0121094cf3f325455a932f1512a6fb9dc Mon Sep 17 00:00:00 2001
From: Steve Brasier <steveb@stackhpc.com>
Date: Thu, 15 Aug 2024 08:45:59 +0000
Subject: [PATCH 08/17] fix cuda verfsion to workaround issue with 12-6-0-1

---
 ansible/roles/cuda/defaults/main.yml | 3 ++-
 1 file changed, 2 insertions(+), 1 deletion(-)

diff --git a/ansible/roles/cuda/defaults/main.yml b/ansible/roles/cuda/defaults/main.yml
index 15cdb95c8..802e8e5ab 100644
--- a/ansible/roles/cuda/defaults/main.yml
+++ b/ansible/roles/cuda/defaults/main.yml
@@ -1,8 +1,9 @@
 cuda_distro: "rhel{{ ansible_distribution_major_version }}"
 cuda_repo: "https://developer.download.nvidia.com/compute/cuda/repos/{{ cuda_distro }}/x86_64/cuda-{{ cuda_distro }}.repo"
 cuda_driver_stream: default
+cuda_package_version: '12.5.1-1' # or 'latest'. Workaround https://forums.developer.nvidia.com/t/issues-with-cuda-12-6-0-1-x86-64-from-rhel8-repo/
 cuda_packages:
-  - cuda
+  - "cuda{{ ('-' + cuda_package_version) if cuda_package_version != 'latest' else '' }}"
   - nvidia-gds
 # _cuda_version_tuple: # discovered from installed package e.g. ('12', '1', '0')
 cuda_version_short: "{{ _cuda_version_tuple[0] }}.{{ _cuda_version_tuple[1] }}"

From 4ce4b5ac30f0ced2bc39fe173cd315e0fefceb6a Mon Sep 17 00:00:00 2001
From: Steve Brasier <steveb@stackhpc.com>
Date: Fri, 16 Aug 2024 11:08:46 +0000
Subject: [PATCH 09/17] don't fail all builds if one fails

---
 .github/workflows/fatimage.yml | 1 +
 1 file changed, 1 insertion(+)

diff --git a/.github/workflows/fatimage.yml b/.github/workflows/fatimage.yml
index 0e7a6e9a2..5d1e61a7c 100644
--- a/.github/workflows/fatimage.yml
+++ b/.github/workflows/fatimage.yml
@@ -10,6 +10,7 @@ jobs:
     name: openstack-imagebuild
     runs-on: ubuntu-22.04
     strategy:
+      fail-fast: false # allow other matrix jobs to continue even if one fails
       matrix: # build RL8, RL9+OFED, RL9+CUDA versions
         os_version:
           - RL8

From f517c6f75ff95ff4725862a07f4c29989c6eba6e Mon Sep 17 00:00:00 2001
From: Steve Brasier <steveb@stackhpc.com>
Date: Fri, 16 Aug 2024 14:01:30 +0000
Subject: [PATCH 10/17] bump CUDA builder disk size (build ran out of space)

---
 packer/openstack.pkr.hcl | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/packer/openstack.pkr.hcl b/packer/openstack.pkr.hcl
index e8bbcf399..d96c8859f 100644
--- a/packer/openstack.pkr.hcl
+++ b/packer/openstack.pkr.hcl
@@ -129,7 +129,7 @@ variable "volume_size" {
     # fat image builds, GB:
     openhpc = 15
     openhpc-ofed = 15
-    openhpc-cuda = 25
+    openhpc-cuda = 30
   }
 }
 

From 0abc46579da0db0aa56e546d0569b8f2180f5a0c Mon Sep 17 00:00:00 2001
From: bertiethorpe <bertie443@gmail.com>
Date: Wed, 4 Sep 2024 08:35:30 +0000
Subject: [PATCH 11/17] download cuda image to /mnt on gh runner

---
 .github/workflows/fatimage.yml | 14 +++++++-------
 1 file changed, 7 insertions(+), 7 deletions(-)

diff --git a/.github/workflows/fatimage.yml b/.github/workflows/fatimage.yml
index 5d1e61a7c..18072782f 100644
--- a/.github/workflows/fatimage.yml
+++ b/.github/workflows/fatimage.yml
@@ -85,7 +85,7 @@ jobs:
       - name: Download image
         run: |
           . venv/bin/activate
-          openstack image save --file ${{ steps.manifest.outputs.image-name }}.qcow2 ${{ steps.manifest.outputs.image-name }}
+          openstack image save --file /mnt/${{ steps.manifest.outputs.image-name }}.qcow2 ${{ steps.manifest.outputs.image-name }}
 
       - name: Set up QEMU
         uses: docker/setup-qemu-action@v3
@@ -96,32 +96,32 @@ jobs:
           sudo apt -y install libguestfs-tools
 
       - name: mkdir for mount
-        run: sudo mkdir -p './${{ steps.manifest.outputs.image-name }}'
+        run: sudo mkdir -p '/mnt/${{ steps.manifest.outputs.image-name }}'
 
       - name: mount qcow2 file
-        run: sudo guestmount -a ${{ steps.manifest.outputs.image-name }}.qcow2 -i --ro -o allow_other './${{ steps.manifest.outputs.image-name }}'
+        run: sudo guestmount -a ${{ steps.manifest.outputs.image-name }}.qcow2 -i --ro -o allow_other '/mnt/${{ steps.manifest.outputs.image-name }}'
 
       - name: Run Trivy vulnerability scanner
         uses: aquasecurity/trivy-action@0.17.0
         with:
           scan-type: fs
-          scan-ref: "./${{ steps.manifest.outputs.image-name }}"
+          scan-ref: "/mnt/${{ steps.manifest.outputs.image-name }}"
           scanners: "vuln"
           format: sarif
-          output: "${{ steps.manifest.outputs.image-name }}.sarif"
+          output: "/mnt/${{ steps.manifest.outputs.image-name }}.sarif"
           # turn off secret scanning to speed things up
 
       - name: Upload Trivy scan results to GitHub Security tab
         uses: github/codeql-action/upload-sarif@v3
         with:
-          sarif_file: "${{ steps.manifest.outputs.image-name }}.sarif"
+          sarif_file: "/mnt/${{ steps.manifest.outputs.image-name }}.sarif"
           category: "${{ matrix.os_version }}-${{ matrix.build }}"
 
       - name: Fail if scan has CRITICAL vulnerabilities
         uses: aquasecurity/trivy-action@0.16.1
         with:
           scan-type: fs
-          scan-ref: "./${{ steps.manifest.outputs.image-name }}"
+          scan-ref: "/mnt/${{ steps.manifest.outputs.image-name }}"
           scanners: "vuln"
           format: table
           exit-code: '1'

From 51d1991020185609ea57b9c71abf2b1e993e9fb2 Mon Sep 17 00:00:00 2001
From: bertiethorpe <bertie443@gmail.com>
Date: Wed, 4 Sep 2024 08:35:30 +0000
Subject: [PATCH 12/17] download cuda image to /mnt on gh runner

---
 .github/workflows/fatimage.yml | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/.github/workflows/fatimage.yml b/.github/workflows/fatimage.yml
index 18072782f..35d115715 100644
--- a/.github/workflows/fatimage.yml
+++ b/.github/workflows/fatimage.yml
@@ -85,7 +85,7 @@ jobs:
       - name: Download image
         run: |
           . venv/bin/activate
-          openstack image save --file /mnt/${{ steps.manifest.outputs.image-name }}.qcow2 ${{ steps.manifest.outputs.image-name }}
+          sudo openstack image save --file /mnt/${{ steps.manifest.outputs.image-name }}.qcow2 ${{ steps.manifest.outputs.image-name }}
 
       - name: Set up QEMU
         uses: docker/setup-qemu-action@v3

From aec6f50694537a322fa3879ee2feb8a6c460d8e7 Mon Sep 17 00:00:00 2001
From: bertiethorpe <bertie443@gmail.com>
Date: Wed, 4 Sep 2024 10:53:36 +0000
Subject: [PATCH 13/17] fix fatimage.yml mnt permissions

---
 .github/workflows/fatimage.yml | 16 +++++++++-------
 1 file changed, 9 insertions(+), 7 deletions(-)

diff --git a/.github/workflows/fatimage.yml b/.github/workflows/fatimage.yml
index 35d115715..7e2fc35b1 100644
--- a/.github/workflows/fatimage.yml
+++ b/.github/workflows/fatimage.yml
@@ -85,7 +85,9 @@ jobs:
       - name: Download image
         run: |
           . venv/bin/activate
-          sudo openstack image save --file /mnt/${{ steps.manifest.outputs.image-name }}.qcow2 ${{ steps.manifest.outputs.image-name }}
+          sudo mkdir /mnt/images
+          sudo chmod 777 /mnt/images
+          openstack image save --file /mnt/images/${{ steps.manifest.outputs.image-name }}.qcow2 ${{ steps.manifest.outputs.image-name }}
 
       - name: Set up QEMU
         uses: docker/setup-qemu-action@v3
@@ -96,32 +98,32 @@ jobs:
           sudo apt -y install libguestfs-tools
 
       - name: mkdir for mount
-        run: sudo mkdir -p '/mnt/${{ steps.manifest.outputs.image-name }}'
+        run: sudo mkdir -p './${{ steps.manifest.outputs.image-name }}'
 
       - name: mount qcow2 file
-        run: sudo guestmount -a ${{ steps.manifest.outputs.image-name }}.qcow2 -i --ro -o allow_other '/mnt/${{ steps.manifest.outputs.image-name }}'
+        run: sudo guestmount -a /mnt/images/${{ steps.manifest.outputs.image-name }}.qcow2 -i --ro -o allow_other './${{ steps.manifest.outputs.image-name }}'
 
       - name: Run Trivy vulnerability scanner
         uses: aquasecurity/trivy-action@0.17.0
         with:
           scan-type: fs
-          scan-ref: "/mnt/${{ steps.manifest.outputs.image-name }}"
+          scan-ref: "${{ steps.manifest.outputs.image-name }}"
           scanners: "vuln"
           format: sarif
-          output: "/mnt/${{ steps.manifest.outputs.image-name }}.sarif"
+          output: "${{ steps.manifest.outputs.image-name }}.sarif"
           # turn off secret scanning to speed things up
 
       - name: Upload Trivy scan results to GitHub Security tab
         uses: github/codeql-action/upload-sarif@v3
         with:
-          sarif_file: "/mnt/${{ steps.manifest.outputs.image-name }}.sarif"
+          sarif_file: "${{ steps.manifest.outputs.image-name }}.sarif"
           category: "${{ matrix.os_version }}-${{ matrix.build }}"
 
       - name: Fail if scan has CRITICAL vulnerabilities
         uses: aquasecurity/trivy-action@0.16.1
         with:
           scan-type: fs
-          scan-ref: "/mnt/${{ steps.manifest.outputs.image-name }}"
+          scan-ref: "${{ steps.manifest.outputs.image-name }}"
           scanners: "vuln"
           format: table
           exit-code: '1'

From bead06d6a2fda769b9bac419a7daa0e311683a71 Mon Sep 17 00:00:00 2001
From: bertiethorpe <84867280+bertiethorpe@users.noreply.github.com>
Date: Wed, 4 Sep 2024 14:03:13 +0100
Subject: [PATCH 14/17] Update main.yml

---
 ansible/roles/cuda/defaults/main.yml | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/ansible/roles/cuda/defaults/main.yml b/ansible/roles/cuda/defaults/main.yml
index 802e8e5ab..33a25d9b4 100644
--- a/ansible/roles/cuda/defaults/main.yml
+++ b/ansible/roles/cuda/defaults/main.yml
@@ -1,7 +1,7 @@
 cuda_distro: "rhel{{ ansible_distribution_major_version }}"
 cuda_repo: "https://developer.download.nvidia.com/compute/cuda/repos/{{ cuda_distro }}/x86_64/cuda-{{ cuda_distro }}.repo"
 cuda_driver_stream: default
-cuda_package_version: '12.5.1-1' # or 'latest'. Workaround https://forums.developer.nvidia.com/t/issues-with-cuda-12-6-0-1-x86-64-from-rhel8-repo/
+cuda_package_version: 'latest'
 cuda_packages:
   - "cuda{{ ('-' + cuda_package_version) if cuda_package_version != 'latest' else '' }}"
   - nvidia-gds

From 1687368f1b46c78308a3572b03edc899c384f83d Mon Sep 17 00:00:00 2001
From: Steve Brasier <steveb@stackhpc.com>
Date: Wed, 4 Sep 2024 15:07:33 +0000
Subject: [PATCH 15/17] switch to open nvidia drivers

---
 ansible/roles/cuda/tasks/main.yml | 13 ++-----------
 1 file changed, 2 insertions(+), 11 deletions(-)

diff --git a/ansible/roles/cuda/tasks/main.yml b/ansible/roles/cuda/tasks/main.yml
index b323cfc04..22f8e9e8e 100644
--- a/ansible/roles/cuda/tasks/main.yml
+++ b/ansible/roles/cuda/tasks/main.yml
@@ -24,22 +24,13 @@
   failed_when: false
   register: _cuda_driver_module_enabled
 
-- name: List nvidia driver dnf module stream versions
-  shell:
-    cmd: dnf module list nvidia-driver | grep -oP "\d+-dkms" | sort -V
-  # Output of interest from command is something like (some whitespace removed):
-  # "nvidia-driver 418-dkms   default [d], fm, ks    Nvidia driver for 418-dkms branch "
-  changed_when: false
-  register: _cuda_driver_module_streams
-  when: "'No matching Modules to list' in _cuda_driver_module_enabled.stderr"
-
 - name: Enable nvidia driver module
-  ansible.builtin.command: "dnf module enable -y nvidia-driver:{{ _cuda_driver_module_streams.stdout_lines | last }}"
+  ansible.builtin.command: "dnf module enable -y nvidia-driver:open-dkms"
   register: _cuda_driver_module_enable
   when: "'No matching Modules to list' in _cuda_driver_module_enabled.stderr"
   changed_when: "'Nothing to do' not in _cuda_driver_module_enable.stdout"
 
-- name: Install nvidia drivers # TODO: make removal possible?
+- name: Install nvidia drivers
   ansible.builtin.command: dnf module install -y nvidia-driver
   register: _cuda_driver_install
   when: "'No matching Modules to list' in _cuda_driver_module_enabled.stderr"

From 102e19a960fdfc257303fde80c6995d9ace7a0fe Mon Sep 17 00:00:00 2001
From: Steve Brasier <steveb@stackhpc.com>
Date: Thu, 5 Sep 2024 15:33:43 +0000
Subject: [PATCH 16/17] bump CI images

---
 environments/.stackhpc/terraform/main.tf | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)

diff --git a/environments/.stackhpc/terraform/main.tf b/environments/.stackhpc/terraform/main.tf
index ac588930c..96e04538b 100644
--- a/environments/.stackhpc/terraform/main.tf
+++ b/environments/.stackhpc/terraform/main.tf
@@ -30,8 +30,8 @@ variable "cluster_image" {
     type = map(string)
     default = {
         # https://github.com/stackhpc/ansible-slurm-appliance/pull/413
-        RL8: "openhpc-RL8-240813-1317-1b370a36"
-        RL9: "openhpc-ofed-RL9-240813-1317-1b370a36"
+        RL8: "openhpc-RL8-240904-1509-1687368f"
+        RL9: "openhpc-ofed-RL9-240904-1509-1687368f"
     }
 }
 

From 7f15afc4a659c4c43d828cf7fa77234b6e2f8fec Mon Sep 17 00:00:00 2001
From: Steve Brasier <steveb@stackhpc.com>
Date: Thu, 5 Sep 2024 17:14:24 +0000
Subject: [PATCH 17/17] make packer build volume-backed optional again

---
 packer/openstack.pkr.hcl | 7 ++++++-
 1 file changed, 6 insertions(+), 1 deletion(-)

diff --git a/packer/openstack.pkr.hcl b/packer/openstack.pkr.hcl
index d96c8859f..5f66c0320 100644
--- a/packer/openstack.pkr.hcl
+++ b/packer/openstack.pkr.hcl
@@ -118,6 +118,11 @@ variable "manifest_output_path" {
   default = "packer-manifest.json"
 }
 
+variable "use_blockstorage_volume" {
+  type = bool
+  default = true
+}
+
 variable "volume_type" {
   type = string
   default = null
@@ -157,7 +162,7 @@ variable "groups" {
 source "openstack" "openhpc" {
   # Build VM:
   flavor = var.flavor
-  use_blockstorage_volume = true
+  use_blockstorage_volume = var.use_blockstorage_volume
   volume_type = var.volume_type
   volume_size = var.volume_size[source.name]
   metadata = var.metadata