Merge branch 'main' into control-ip-addresses

Author: sjpb

README.md

@@ -25,8 +25,6 @@ The default configuration in this repository may be used to create a cluster to
 - Persistent state backed by an OpenStack volume.
 - NFS-based shared file system backed by another OpenStack volume.
 
-Note that the Open OnDemand portal and its remote apps are not usable with this default configuration.
-
 It requires an OpenStack cloud, and an Ansible "deploy host" with access to that cloud.
 
 Before starting ensure that:

ansible/roles/cuda/README.md

@@ -2,14 +2,10 @@
 
 Install NVIDIA drivers and optionally CUDA packages. CUDA binaries are added to the `$PATH` for all users, and the [NVIDIA persistence daemon](https://docs.nvidia.com/deploy/driver-persistence/index.html#persistence-daemon) is enabled.
 
-## Prerequisites
-
-Requires OFED to be installed to provide required kernel-* packages.
-
 ## Role Variables
 
 - `cuda_repo_url`: Optional. URL of `.repo` file. Default is upstream for appropriate OS/architecture.
 - `cuda_nvidia_driver_stream`: Optional. Version of `nvidia-driver` stream to enable. This controls whether the open or proprietary drivers are installed and the major version. Changing this once the drivers are installed does not change the version.
-- `cuda_packages`: Optional. Default: `['cuda', 'nvidia-gds', 'cmake', 'cuda-toolkit-12-9']`.
+- `cuda_packages`: Optional. Default provides CUDA Toolkit and GPUDirect Storage (GDS).
 - `cuda_package_version`: Optional. Default `latest` which will install the latest packages if not installed but won't upgrade already-installed packages. Use `'none'` to skip installing CUDA.
 - `cuda_persistenced_state`: Optional. State of systemd `nvidia-persistenced` service. Values as [ansible.builtin.systemd:state](https://docs.ansible.com/ansible/latest/collections/ansible/builtin/systemd_module.html#parameter-state). Default `started`.

ansible/roles/cuda/defaults/main.yml

@@ -1,12 +1,12 @@
 cuda_repo_url: "https://developer.download.nvidia.com/compute/cuda/repos/rhel{{ ansible_distribution_major_version }}/{{ ansible_architecture }}/cuda-rhel{{ ansible_distribution_major_version }}.repo"
 cuda_nvidia_driver_stream: '575-open'
-cuda_package_version: '12.9.0-1'
-cuda_version_short: '12.9'
+cuda_nvidia_driver_pkg: "nvidia-open-3:575.57.08-1.el{{ ansible_distribution_major_version }}"
+cuda_package_version: '12.9.1-1'
+cuda_version_short: "{{ (cuda_package_version | split('.'))[0:2] | join('.') }}" # major.minor
 cuda_packages:
-  - "cuda{{ ('-' + cuda_package_version) if cuda_package_version != 'latest' else '' }}"
+  - "cuda-toolkit-{{ cuda_package_version }}"
   - nvidia-gds
   - cmake
-  - cuda-toolkit-12-9
 cuda_samples_release_url: "https://github.com/NVIDIA/cuda-samples/archive/refs/tags/v{{ cuda_version_short }}.tar.gz"
 cuda_samples_path: "/var/lib/{{ ansible_user }}/cuda_samples"
 cuda_samples_programs:

ansible/roles/cuda/tasks/install.yml

@@ -1,16 +1,5 @@
 
-# Based on https://docs.nvidia.com/cuda/cuda-installation-guide-linux/index.html#redhat8-installation
-
-- name: Check for OFED/DOCA
-  command:
-    cmd: dnf list --installed rdma-core
-  register: _dnf_rdma_core
-  changed_when: false
-
-- name: Assert OFED installed
-  assert: 
-    that: "'mlnx' in _dnf_rdma_core.stdout"
-    fail_msg: "Did not find 'mlnx' in installed rdma-core package, is OFED/DOCA installed?"
+# Based on https://docs.nvidia.com/datacenter/tesla/driver-installation-guide/
 
 - name: Install cuda repo
   get_url:
@@ -29,23 +18,18 @@
   when: "'No matching Modules to list' in _cuda_driver_module_enabled.stderr"
   changed_when: "'Nothing to do' not in _cuda_driver_module_enable.stdout"
 
-- name: Check if nvidia driver module is installed
-  ansible.builtin.command: dnf module list --installed nvidia-driver
-  changed_when: false
-  failed_when: false
-  register: _cuda_driver_module_installed
-
 - name: Install nvidia drivers
-  ansible.builtin.command: dnf module install -y nvidia-driver
+  ansible.builtin.dnf:
+    name: "{{ cuda_nvidia_driver_pkg }}"
   register: _cuda_driver_install
-  when: "'No matching Modules to list' in _cuda_driver_module_installed.stderr"
-  changed_when: "'Nothing to do' not in _cuda_driver_install.stdout"
 
 - name: Check kernel has not been modified
   assert:
     that: "'kernel ' not in _cuda_driver_install.stdout | default('')" # space ensures we don't flag e.g. kernel-devel-matched
     fail_msg: "{{ _cuda_driver_install.stdout_lines | default([]) | select('search', 'kernel ') }}"
 
+# Based on https://docs.nvidia.com/cuda/cuda-installation-guide-linux/index.html
+
 - name: Install cuda packages
   ansible.builtin.dnf:
     name: "{{ cuda_packages }}"

ansible/roles/tuned/tasks/configure.yml

@@ -12,9 +12,17 @@
   register: _tuned_profile_current
   changed_when: false
 
-- name: Set tuned-adm profile
+- name: Set TuneD profile
   ansible.builtin.command:
     cmd: "tuned-adm profile {{ tuned_profile }}"
-  when: 
+  when:
+    - tuned_started | bool
+    - tuned_profile not in _tuned_profile_current.stdout
+
+- name: Verify TuneD profile
+  ansible.builtin.command:
+    cmd: tuned-adm verify
+  changed_when: false
+  when:
     - tuned_started | bool
     - tuned_profile not in _tuned_profile_current.stdout

ansible/roles/tuned/tasks/install.yml

@@ -2,4 +2,15 @@
 - name: Install tuneD
   ansible.builtin.dnf:
     name: tuned
-    state: present
+    state: present
+
+- name: Fix TuneD hpc-compute profile for hugepages
+  # See https://github.com/redhat-performance/tuned/issues/752
+  # This is done on install, not configure, so that it is available even
+  # for compute-init nodes
+  community.general.ini_file:
+    path: /usr/lib/tuned/hpc-compute/tuned.conf
+    section: sysctl
+    option: vm.min_free_kbytes
+    value: '>135168'
+    no_extra_spaces: true

environments/.stackhpc/inventory/extra_groups

@@ -29,7 +29,10 @@ cluster
 
 [tuned:children]
 # Install tuned into fat image
+# NB: builder has tuned_enabled and tuned_started false so does not configure it
 builder
+# Also test tuned during site playbook
+cluster
 
 [squid:children]
 # Install squid into fat image

environments/.stackhpc/inventory/group_vars/all/tuned.yml

@@ -0,0 +1,2 @@
+# Set profile which is not default (on VMs) for testing
+tuned_profile: hpc-compute

environments/.stackhpc/tofu/cluster_image.auto.tfvars.json

@@ -1,6 +1,6 @@
 {
     "cluster_image": {
-        "RL8": "openhpc-RL8-250514-1502-5a923b2c",
-        "RL9": "openhpc-RL9-250514-1502-5a923b2c"
+        "RL8": "openhpc-RL8-250610-1435-d0ef926e",
+        "RL9": "openhpc-RL9-250610-1435-d0ef926e"
     }
 }

environments/common/inventory/group_vars/all/nfs.yml

@@ -36,7 +36,7 @@ nfs_configurations_extra: [] # site-specific nfs shares
 
 nfs_configurations: >- # construct stackhpc.nfs variable
   {{
-    nfs_configuration_home_volume if (cluster_home_volume | default(true)) else []
+    (nfs_configuration_home_volume if (cluster_home_volume | default(true)) else [])
     +
     nfs_configuration_compute_nodes
     +