From c4b27956ded57e17ea72dbd2718a6ad9ea9269de Mon Sep 17 00:00:00 2001
From: Steve Brasier <steveb@stackhpc.com>
Date: Fri, 13 Dec 2024 10:34:12 +0000
Subject: [PATCH 01/16] move cuda tasks to install

---
 ansible/roles/cuda/tasks/{main.yml => install.yml} | 0
 1 file changed, 0 insertions(+), 0 deletions(-)
 rename ansible/roles/cuda/tasks/{main.yml => install.yml} (100%)

diff --git a/ansible/roles/cuda/tasks/main.yml b/ansible/roles/cuda/tasks/install.yml
similarity index 100%
rename from ansible/roles/cuda/tasks/main.yml
rename to ansible/roles/cuda/tasks/install.yml

From ad20d2022e99774768536615c1a371fe60849aa7 Mon Sep 17 00:00:00 2001
From: Steve Brasier <steveb@stackhpc.com>
Date: Fri, 13 Dec 2024 10:41:13 +0000
Subject: [PATCH 02/16] pin nvidia driver to working version and autodetect
 os/arch

---
 ansible/roles/cuda/defaults/main.yml |  5 ++---
 ansible/roles/cuda/tasks/install.yml | 14 +++++++++-----
 2 files changed, 11 insertions(+), 8 deletions(-)

diff --git a/ansible/roles/cuda/defaults/main.yml b/ansible/roles/cuda/defaults/main.yml
index 33a25d9b4..b56aac1ec 100644
--- a/ansible/roles/cuda/defaults/main.yml
+++ b/ansible/roles/cuda/defaults/main.yml
@@ -1,6 +1,5 @@
-cuda_distro: "rhel{{ ansible_distribution_major_version }}"
-cuda_repo: "https://developer.download.nvidia.com/compute/cuda/repos/{{ cuda_distro }}/x86_64/cuda-{{ cuda_distro }}.repo"
-cuda_driver_stream: default
+cuda_repo_url: "https://developer.download.nvidia.com/compute/cuda/repos/rhel{{ ansible_distribution_major_version }}/{{ ansible_architecture }}/cuda-rhel{{ ansible_distribution_major_version }}.repo"
+cuda_driver_stream: '560-open'
 cuda_package_version: 'latest'
 cuda_packages:
   - "cuda{{ ('-' + cuda_package_version) if cuda_package_version != 'latest' else '' }}"
diff --git a/ansible/roles/cuda/tasks/install.yml b/ansible/roles/cuda/tasks/install.yml
index 22f8e9e8e..9125d3635 100644
--- a/ansible/roles/cuda/tasks/install.yml
+++ b/ansible/roles/cuda/tasks/install.yml
@@ -1,7 +1,7 @@
 
 # Based on https://docs.nvidia.com/cuda/cuda-installation-guide-linux/index.html#redhat8-installation
 
-- name: Check for OFED
+- name: Check for OFED/DOCA
   command:
     cmd: dnf list --installed rdma-core
   register: _dnf_rdma_core
@@ -10,12 +10,12 @@
 - name: Assert OFED installed
   assert: 
     that: "'mlnx' in _dnf_rdma_core.stdout"
-    fail_msg: "Did not find 'mlnx' in installed rdma-core package, is OFED installed?"
+    fail_msg: "Did not find 'mlnx' in installed rdma-core package, is OFED/DOCA installed?"
 
 - name: Install cuda repo
   get_url:
-    dest: "/etc/yum.repos.d/cuda-{{ cuda_distro }}.repo"
-    url: "{{ cuda_repo }}"
+    dest: "/etc/yum.repos.d/cuda-rhel{{ ansible_distribution_major_version }}.repo"
+    url: "{{ cuda_repo_url }}"
 
 - name: Check if nvidia driver module is enabled
   shell:
@@ -25,7 +25,7 @@
   register: _cuda_driver_module_enabled
 
 - name: Enable nvidia driver module
-  ansible.builtin.command: "dnf module enable -y nvidia-driver:open-dkms"
+  ansible.builtin.command: "dnf module enable -y nvidia-driver:{{ cuda_driver_stream }}"
   register: _cuda_driver_module_enable
   when: "'No matching Modules to list' in _cuda_driver_module_enabled.stderr"
   changed_when: "'Nothing to do' not in _cuda_driver_module_enable.stdout"
@@ -36,6 +36,10 @@
   when: "'No matching Modules to list' in _cuda_driver_module_enabled.stderr"
   changed_when: "'Nothing to do' not in _cuda_driver_install.stdout"
 
+- name: Check kernel has not been modified
+  assert:
+    that: "'kernel' not in _cuda_driver_install.stdout"
+
 - name: Install cuda packages
   ansible.builtin.dnf:
     name: "{{ cuda_packages }}"

From 66a20560edc69432e65977cd67c3ab4e25fe4faa Mon Sep 17 00:00:00 2001
From: Steve Brasier <steveb@stackhpc.com>
Date: Fri, 13 Dec 2024 10:43:38 +0000
Subject: [PATCH 03/16] make install of cuda packages optional

---
 ansible/roles/cuda/tasks/install.yml | 2 ++
 1 file changed, 2 insertions(+)

diff --git a/ansible/roles/cuda/tasks/install.yml b/ansible/roles/cuda/tasks/install.yml
index 9125d3635..28c46184b 100644
--- a/ansible/roles/cuda/tasks/install.yml
+++ b/ansible/roles/cuda/tasks/install.yml
@@ -43,12 +43,14 @@
 - name: Install cuda packages
   ansible.builtin.dnf:
     name: "{{ cuda_packages }}"
+  when: cuda_package_version != 'none'
   register: cuda_package_install
 
 - name: Add cuda binaries to path
   lineinfile:
     path: /etc/profile.d/sh.local
     line: 'export PATH=$PATH:$(ls -1d /usr/local/cuda-* | sort -V | tail -1)/bin'
+  when: cuda_package_version != 'none'
 
 - name: Enable NVIDIA Persistence Daemon
   systemd:

From c40107d957121e281ac76b3218f1848f7d677b22 Mon Sep 17 00:00:00 2001
From: Steve Brasier <steveb@stackhpc.com>
Date: Fri, 13 Dec 2024 10:57:11 +0000
Subject: [PATCH 04/16] don't run cuda install tasks unless during build

---
 ansible/extras.yml | 1 +
 1 file changed, 1 insertion(+)

diff --git a/ansible/extras.yml b/ansible/extras.yml
index 107f85252..bff869780 100644
--- a/ansible/extras.yml
+++ b/ansible/extras.yml
@@ -26,6 +26,7 @@
   tasks:
     - import_role:
         name: cuda
+      tasks_from: "{{ 'runtime.yml' if appliances_mode == 'configure' else 'install.yml' }}"
 
 - name: Persist hostkeys across rebuilds
   # Must be after filesystems.yml (for storage)

From 63bf06ae81678183f283dadd22ddc42c33f0c93f Mon Sep 17 00:00:00 2001
From: Steve Brasier <steveb@stackhpc.com>
Date: Fri, 13 Dec 2024 11:00:23 +0000
Subject: [PATCH 05/16] move doca install before cuda

---
 ansible/fatimage.yml | 16 ++++++++--------
 1 file changed, 8 insertions(+), 8 deletions(-)

diff --git a/ansible/fatimage.yml b/ansible/fatimage.yml
index 439c50e70..ce0f56711 100644
--- a/ansible/fatimage.yml
+++ b/ansible/fatimage.yml
@@ -19,6 +19,14 @@
 
 - import_playbook: bootstrap.yml
 
+- hosts: doca
+  become: yes
+  gather_facts: yes
+  tasks:
+    - name: Install NVIDIA DOCA
+      import_role:
+        name: doca
+
 - name: Run post-bootstrap.yml hook
   vars:
     appliances_environment_root: "{{ lookup('env', 'APPLIANCES_ENVIRONMENT_ROOT') }}"
@@ -202,14 +210,6 @@
         name: cloudalchemy.grafana
         tasks_from: install.yml
 
-- hosts: doca
-  become: yes
-  gather_facts: yes
-  tasks:
-    - name: Install NVIDIA DOCA
-      import_role:
-        name: doca
-
 - name: Run post.yml hook
   vars:
     appliances_environment_root: "{{ lookup('env', 'APPLIANCES_ENVIRONMENT_ROOT') }}"

From fa260803aed6b59f589088728bc0bffb84181134 Mon Sep 17 00:00:00 2001
From: Steve Brasier <steveb@stackhpc.com>
Date: Fri, 13 Dec 2024 11:10:49 +0000
Subject: [PATCH 06/16] update cuda docs

---
 ansible/roles/cuda/README.md | 8 ++++----
 1 file changed, 4 insertions(+), 4 deletions(-)

diff --git a/ansible/roles/cuda/README.md b/ansible/roles/cuda/README.md
index 141e7b80d..d386a1bf1 100644
--- a/ansible/roles/cuda/README.md
+++ b/ansible/roles/cuda/README.md
@@ -1,6 +1,6 @@
 # cuda
 
-Install NVIDIA CUDA. The CUDA binaries are added to the PATH for all users, and the [NVIDIA persistence daemon](https://docs.nvidia.com/deploy/driver-persistence/index.html#persistence-daemon) is enabled.
+Install NVIDIA drivers and optionally CUDA packages. CUDA binaries are added to the `$PATH` for all users, and the [NVIDIA persistence daemon](https://docs.nvidia.com/deploy/driver-persistence/index.html#persistence-daemon) is enabled.
 
 ## Prerequisites
 
@@ -8,8 +8,8 @@ Requires OFED to be installed to provide required kernel-* packages.
 
 ## Role Variables
 
-- `cuda_distro`: Optional. Default `rhel8`.
-- `cuda_repo`: Optional. Default `https://developer.download.nvidia.com/compute/cuda/repos/{{ cuda_distro }}/x86_64/cuda-{{ cuda_distro }}.repo`
-- `cuda_driver_stream`: Optional. The default value `default` will, on first use of this role, enable the dkms-flavour `nvidia-driver` DNF module stream with the current highest version number. The `latest-dkms` stream is not enabled, and subsequent runs of the role will *not* change the enabled stream, even if a later version has become available. Changing this value once an `nvidia-driver` stream has been enabled raises an error. If an upgrade of the `nvidia-driver` module is required, the currently-enabled stream and all packages should be manually removed.
+- `cuda_repo_url`: Optional. URL of `.repo` file. Default is upstream for appropriate OS/architecture.
+- `cuda_driver_stream`: Optional. Version of `nvidia-driver` stream to enable. This controls whether the open or proprietary drivers are installed and the major version. Changing this once the drivers are installed does not change the version.
 - `cuda_packages`: Optional. Default: `['cuda', 'nvidia-gds']`.
+- `cuda_package_version`: Optional. Default `latest` which will install the latest packages if not installed but won't upgrade already-installed packages. Use `'none'` to skip installing CUDA.
 - `cuda_persistenced_state`: Optional. State of systemd `nvidia-persistenced` service. Values as [ansible.builtin.systemd:state](https://docs.ansible.com/ansible/latest/collections/ansible/builtin/systemd_module.html#parameter-state). Default `started`.

From 6a0fcb3e89dec0001bb5e770106e079a56fc617b Mon Sep 17 00:00:00 2001
From: Steve Brasier <steveb@stackhpc.com>
Date: Fri, 13 Dec 2024 11:13:17 +0000
Subject: [PATCH 07/16] add cuda to extra build test CI

---
 .github/workflows/{doca.yml => extra.yml} | 18 ++++++++++--------
 1 file changed, 10 insertions(+), 8 deletions(-)
 rename .github/workflows/{doca.yml => extra.yml} (92%)

diff --git a/.github/workflows/doca.yml b/.github/workflows/extra.yml
similarity index 92%
rename from .github/workflows/doca.yml
rename to .github/workflows/extra.yml
index cfd3bb982..ac83e9064 100644
--- a/.github/workflows/doca.yml
+++ b/.github/workflows/extra.yml
@@ -1,4 +1,4 @@
-name: Test DOCA extra build
+name: Test extra build
 on:
   workflow_dispatch:
   push:
@@ -7,16 +7,18 @@ on:
     paths:
       - 'environments/.stackhpc/terraform/cluster_image.auto.tfvars.json'
       - 'ansible/roles/doca/**'
-      - '.github/workflows/doca'
+      - 'ansible/roles/cuda/**'
+      - '.github/workflows/extra.yml'
   pull_request:
     paths:
       - 'environments/.stackhpc/terraform/cluster_image.auto.tfvars.json'
       - 'ansible/roles/doca/**'
-      - '.github/workflows/doca'
+      - 'ansible/roles/cuda/**'
+      - '.github/workflows/extra.yml'
 
 jobs:
   doca:
-    name: doca-build
+    name: extra-build
     concurrency:
       group: ${{ github.workflow }}-${{ github.ref }}-${{ matrix.build.image_name }} # to branch/PR + OS
       cancel-in-progress: true
@@ -25,12 +27,12 @@ jobs:
       fail-fast: false # allow other matrix jobs to continue even if one fails
       matrix: # build RL8, RL9
         build:
-          - image_name: openhpc-doca-RL8
+          - image_name: openhpc-extra-RL8
             source_image_name_key: RL8 # key into environments/.stackhpc/terraform/cluster_image.auto.tfvars.json
-            inventory_groups: doca
-          - image_name: openhpc-doca-RL9
+            inventory_groups: doca,cuda
+          - image_name: openhpc-extra-RL9
             source_image_name_key: RL9
-            inventory_groups: doca
+            inventory_groups: doca,cuda
     env:
       ANSIBLE_FORCE_COLOR: True
       OS_CLOUD: openstack

From dcf61da8957ac932b92940b4ee4bf38f54fbe8a2 Mon Sep 17 00:00:00 2001
From: Steve Brasier <steveb@stackhpc.com>
Date: Fri, 13 Dec 2024 11:13:36 +0000
Subject: [PATCH 08/16] add cuda runtime tasks

---
 ansible/roles/cuda/tasks/runtime.yml | 5 +++++
 1 file changed, 5 insertions(+)
 create mode 100644 ansible/roles/cuda/tasks/runtime.yml

diff --git a/ansible/roles/cuda/tasks/runtime.yml b/ansible/roles/cuda/tasks/runtime.yml
new file mode 100644
index 000000000..32a2da147
--- /dev/null
+++ b/ansible/roles/cuda/tasks/runtime.yml
@@ -0,0 +1,5 @@
+- name: Enable NVIDIA Persistence Daemon
+  systemd:
+    name: nvidia-persistenced
+    enabled: true
+    state: "{{ cuda_persistenced_state }}"

From be805ccab1b256184c10b2651e8cfc20f2f22b22 Mon Sep 17 00:00:00 2001
From: Steve Brasier <steveb@stackhpc.com>
Date: Fri, 13 Dec 2024 11:30:30 +0000
Subject: [PATCH 09/16] fix typo in extras playbook

---
 ansible/extras.yml | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/ansible/extras.yml b/ansible/extras.yml
index bff869780..53458cc60 100644
--- a/ansible/extras.yml
+++ b/ansible/extras.yml
@@ -26,7 +26,7 @@
   tasks:
     - import_role:
         name: cuda
-      tasks_from: "{{ 'runtime.yml' if appliances_mode == 'configure' else 'install.yml' }}"
+        tasks_from: "{{ 'runtime.yml' if appliances_mode == 'configure' else 'install.yml' }}"
 
 - name: Persist hostkeys across rebuilds
   # Must be after filesystems.yml (for storage)

From 1319e72d257e30a7916cec04e34f8a010542aa8e Mon Sep 17 00:00:00 2001
From: Steve Brasier <steveb@stackhpc.com>
Date: Fri, 13 Dec 2024 12:58:38 +0000
Subject: [PATCH 10/16] bump extra build size to 30GB for cuda

---
 .github/workflows/extra.yml | 3 +++
 1 file changed, 3 insertions(+)

diff --git a/.github/workflows/extra.yml b/.github/workflows/extra.yml
index ac83e9064..dece242ce 100644
--- a/.github/workflows/extra.yml
+++ b/.github/workflows/extra.yml
@@ -30,9 +30,11 @@ jobs:
           - image_name: openhpc-extra-RL8
             source_image_name_key: RL8 # key into environments/.stackhpc/terraform/cluster_image.auto.tfvars.json
             inventory_groups: doca,cuda
+            volume_size: 30 # needed for cuda
           - image_name: openhpc-extra-RL9
             source_image_name_key: RL9
             inventory_groups: doca,cuda
+            volume_size: 30 # needed for cuda
     env:
       ANSIBLE_FORCE_COLOR: True
       OS_CLOUD: openstack
@@ -97,6 +99,7 @@ jobs:
           -var "source_image_name=${{ fromJSON(env.FAT_IMAGES)['cluster_image'][matrix.build.source_image_name_key] }}" \
           -var "image_name=${{ matrix.build.image_name }}" \
           -var "inventory_groups=${{ matrix.build.inventory_groups }}" \
+          -var "volume_size=${{ matrix.build.volume_size }}" \
           openstack.pkr.hcl
         
       - name: Get created image names from manifest

From f7dc0d33ba41049321d56353aa00a025b52c458f Mon Sep 17 00:00:00 2001
From: Steve Brasier <steveb@stackhpc.com>
Date: Fri, 13 Dec 2024 13:10:16 +0000
Subject: [PATCH 11/16] pin both cuda package version

---
 ansible/roles/cuda/defaults/main.yml | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)

diff --git a/ansible/roles/cuda/defaults/main.yml b/ansible/roles/cuda/defaults/main.yml
index b56aac1ec..9c336cea2 100644
--- a/ansible/roles/cuda/defaults/main.yml
+++ b/ansible/roles/cuda/defaults/main.yml
@@ -1,6 +1,6 @@
 cuda_repo_url: "https://developer.download.nvidia.com/compute/cuda/repos/rhel{{ ansible_distribution_major_version }}/{{ ansible_architecture }}/cuda-rhel{{ ansible_distribution_major_version }}.repo"
-cuda_driver_stream: '560-open'
-cuda_package_version: 'latest'
+cuda_driver_stream: '560-open' # 565-open has problems with cuda packages
+cuda_package_version: '12.6.3-1'
 cuda_packages:
   - "cuda{{ ('-' + cuda_package_version) if cuda_package_version != 'latest' else '' }}"
   - nvidia-gds

From 4d6c44a1fa95044e0b9ee516d1c9fcc099b8e6fa Mon Sep 17 00:00:00 2001
From: Steve Brasier <steveb@stackhpc.com>
Date: Fri, 13 Dec 2024 13:16:46 +0000
Subject: [PATCH 12/16] make cuda idempotent/restartable

---
 ansible/roles/cuda/tasks/install.yml | 15 +++++++++++----
 ansible/roles/cuda/tasks/runtime.yml |  2 +-
 2 files changed, 12 insertions(+), 5 deletions(-)

diff --git a/ansible/roles/cuda/tasks/install.yml b/ansible/roles/cuda/tasks/install.yml
index 28c46184b..934df77d3 100644
--- a/ansible/roles/cuda/tasks/install.yml
+++ b/ansible/roles/cuda/tasks/install.yml
@@ -18,8 +18,7 @@
     url: "{{ cuda_repo_url }}"
 
 - name: Check if nvidia driver module is enabled
-  shell:
-    cmd: dnf module list --enabled nvidia-driver
+  ansible.builtin.command: dnf module list --enabled nvidia-driver
   changed_when: false
   failed_when: false
   register: _cuda_driver_module_enabled
@@ -30,15 +29,22 @@
   when: "'No matching Modules to list' in _cuda_driver_module_enabled.stderr"
   changed_when: "'Nothing to do' not in _cuda_driver_module_enable.stdout"
 
+- name: Check if nvidia driver module is installed
+  ansible.builtin.command: dnf module list --installed nvidia-driver
+  changed_when: false
+  failed_when: false
+  register: _cuda_driver_module_installed
+
 - name: Install nvidia drivers
   ansible.builtin.command: dnf module install -y nvidia-driver
   register: _cuda_driver_install
-  when: "'No matching Modules to list' in _cuda_driver_module_enabled.stderr"
+  when: "'No matching Modules to list' in _cuda_driver_module_installed.stderr"
   changed_when: "'Nothing to do' not in _cuda_driver_install.stdout"
 
 - name: Check kernel has not been modified
   assert:
-    that: "'kernel' not in _cuda_driver_install.stdout"
+    that: "'kernel ' not in _cuda_driver_install.stdout | default('')" # space ensures we don't flag e.g. kernel-devel-matched
+    fail_msg: "{{ _cuda_driver_install.stdout_lines | default([]) | select('search', 'kernel ') }}"
 
 - name: Install cuda packages
   ansible.builtin.dnf:
@@ -66,3 +72,4 @@
 - name: Wait for hosts to be reachable
   wait_for_connection:
     sleep: 15
+  when: cuda_package_install.changed
diff --git a/ansible/roles/cuda/tasks/runtime.yml b/ansible/roles/cuda/tasks/runtime.yml
index 32a2da147..c16a48c6f 100644
--- a/ansible/roles/cuda/tasks/runtime.yml
+++ b/ansible/roles/cuda/tasks/runtime.yml
@@ -1,4 +1,4 @@
-- name: Enable NVIDIA Persistence Daemon
+- name: Ensure NVIDIA Persistence Daemon state
   systemd:
     name: nvidia-persistenced
     enabled: true

From 4b82cba8c0c00621c343ff31cf831fbb959db212 Mon Sep 17 00:00:00 2001
From: Steve Brasier <steveb@stackhpc.com>
Date: Fri, 13 Dec 2024 15:17:29 +0000
Subject: [PATCH 13/16] allow using computed tasks_from for cuda role

---
 ansible/extras.yml | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/ansible/extras.yml b/ansible/extras.yml
index 53458cc60..0a74541a5 100644
--- a/ansible/extras.yml
+++ b/ansible/extras.yml
@@ -24,7 +24,7 @@
   gather_facts: yes
   tags: cuda
   tasks:
-    - import_role:
+    - include_role:
         name: cuda
         tasks_from: "{{ 'runtime.yml' if appliances_mode == 'configure' else 'install.yml' }}"
 

From 59e95def2aabdb50e8382f4355397477fd03ff81 Mon Sep 17 00:00:00 2001
From: Steve Brasier <steveb@stackhpc.com>
Date: Fri, 13 Dec 2024 16:59:16 +0000
Subject: [PATCH 14/16] fix showing image summary

---
 ansible/cleanup.yml | 3 +--
 1 file changed, 1 insertion(+), 2 deletions(-)

diff --git a/ansible/cleanup.yml b/ansible/cleanup.yml
index 3f059d157..670a99b29 100644
--- a/ansible/cleanup.yml
+++ b/ansible/cleanup.yml
@@ -66,5 +66,4 @@
       slurm-ohpc: "{{ ansible_facts.packages['slurm-ohpc'].0.version | default('-') }}"
 
 - name: Show image summary
-  debug:
-    var: image_info
+  command: cat /var/lib/image/image.json

From cae1ccfba97f556a6079046aee9f96ed88e62a6d Mon Sep 17 00:00:00 2001
From: Steve Brasier <steveb@stackhpc.com>
Date: Tue, 17 Dec 2024 17:12:26 +0000
Subject: [PATCH 15/16] rename nvidia driver version var

---
 ansible/roles/cuda/README.md         | 2 +-
 ansible/roles/cuda/defaults/main.yml | 2 +-
 ansible/roles/cuda/tasks/install.yml | 2 +-
 3 files changed, 3 insertions(+), 3 deletions(-)

diff --git a/ansible/roles/cuda/README.md b/ansible/roles/cuda/README.md
index d386a1bf1..be6439cd5 100644
--- a/ansible/roles/cuda/README.md
+++ b/ansible/roles/cuda/README.md
@@ -9,7 +9,7 @@ Requires OFED to be installed to provide required kernel-* packages.
 ## Role Variables
 
 - `cuda_repo_url`: Optional. URL of `.repo` file. Default is upstream for appropriate OS/architecture.
-- `cuda_driver_stream`: Optional. Version of `nvidia-driver` stream to enable. This controls whether the open or proprietary drivers are installed and the major version. Changing this once the drivers are installed does not change the version.
+- `cuda_nvidia_driver_stream`: Optional. Version of `nvidia-driver` stream to enable. This controls whether the open or proprietary drivers are installed and the major version. Changing this once the drivers are installed does not change the version.
 - `cuda_packages`: Optional. Default: `['cuda', 'nvidia-gds']`.
 - `cuda_package_version`: Optional. Default `latest` which will install the latest packages if not installed but won't upgrade already-installed packages. Use `'none'` to skip installing CUDA.
 - `cuda_persistenced_state`: Optional. State of systemd `nvidia-persistenced` service. Values as [ansible.builtin.systemd:state](https://docs.ansible.com/ansible/latest/collections/ansible/builtin/systemd_module.html#parameter-state). Default `started`.
diff --git a/ansible/roles/cuda/defaults/main.yml b/ansible/roles/cuda/defaults/main.yml
index 9c336cea2..05f1e093d 100644
--- a/ansible/roles/cuda/defaults/main.yml
+++ b/ansible/roles/cuda/defaults/main.yml
@@ -1,5 +1,5 @@
 cuda_repo_url: "https://developer.download.nvidia.com/compute/cuda/repos/rhel{{ ansible_distribution_major_version }}/{{ ansible_architecture }}/cuda-rhel{{ ansible_distribution_major_version }}.repo"
-cuda_driver_stream: '560-open' # 565-open has problems with cuda packages
+cuda_nvidia_driver_stream: '560-open' # 565-open has problems with cuda packages
 cuda_package_version: '12.6.3-1'
 cuda_packages:
   - "cuda{{ ('-' + cuda_package_version) if cuda_package_version != 'latest' else '' }}"
diff --git a/ansible/roles/cuda/tasks/install.yml b/ansible/roles/cuda/tasks/install.yml
index 934df77d3..51c92a0d3 100644
--- a/ansible/roles/cuda/tasks/install.yml
+++ b/ansible/roles/cuda/tasks/install.yml
@@ -24,7 +24,7 @@
   register: _cuda_driver_module_enabled
 
 - name: Enable nvidia driver module
-  ansible.builtin.command: "dnf module enable -y nvidia-driver:{{ cuda_driver_stream }}"
+  ansible.builtin.command: "dnf module enable -y nvidia-driver:{{ cuda_nvidia_driver_stream }}"
   register: _cuda_driver_module_enable
   when: "'No matching Modules to list' in _cuda_driver_module_enabled.stderr"
   changed_when: "'Nothing to do' not in _cuda_driver_module_enable.stdout"

From 1f6f2278716edc499c45f94e39ae0a740ec5aaf2 Mon Sep 17 00:00:00 2001
From: Steve Brasier <steveb@stackhpc.com>
Date: Thu, 19 Dec 2024 08:51:10 +0000
Subject: [PATCH 16/16] bump CI image

---
 .../.stackhpc/terraform/cluster_image.auto.tfvars.json        | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)

diff --git a/environments/.stackhpc/terraform/cluster_image.auto.tfvars.json b/environments/.stackhpc/terraform/cluster_image.auto.tfvars.json
index db25176e2..be2f156a3 100644
--- a/environments/.stackhpc/terraform/cluster_image.auto.tfvars.json
+++ b/environments/.stackhpc/terraform/cluster_image.auto.tfvars.json
@@ -1,6 +1,6 @@
 {
     "cluster_image": {
-        "RL8": "openhpc-RL8-241218-1011-5effb3fa",
-        "RL9": "openhpc-RL9-241218-1011-5effb3fa"
+        "RL8": "openhpc-RL8-241218-1705-09ac4268",
+        "RL9": "openhpc-RL9-241218-1705-09ac4268"
     }
 }