Merge branch 'main' into feat/verify-galaxy-installs-2

Author: sjpb

README.md

@@ -25,8 +25,6 @@ The default configuration in this repository may be used to create a cluster to
 - Persistent state backed by an OpenStack volume.
 - NFS-based shared file system backed by another OpenStack volume.
 
-Note that the Open OnDemand portal and its remote apps are not usable with this default configuration.
-
 It requires an OpenStack cloud, and an Ansible "deploy host" with access to that cloud.
 
 Before starting ensure that:

ansible/bootstrap.yml

@@ -143,7 +143,7 @@
       - appliances_mode == 'configure'
       - not (dnf_repos_allow_insecure_creds | default(false)) # useful for development
 
-- hosts: cacerts:!builder
+- hosts: cacerts
   tags: cacerts
   gather_facts: false
   tasks:

ansible/roles/cuda/README.md

@@ -2,14 +2,10 @@
 
 Install NVIDIA drivers and optionally CUDA packages. CUDA binaries are added to the `$PATH` for all users, and the [NVIDIA persistence daemon](https://docs.nvidia.com/deploy/driver-persistence/index.html#persistence-daemon) is enabled.
 
-## Prerequisites
-
-Requires OFED to be installed to provide required kernel-* packages.
-
 ## Role Variables
 
 - `cuda_repo_url`: Optional. URL of `.repo` file. Default is upstream for appropriate OS/architecture.
 - `cuda_nvidia_driver_stream`: Optional. Version of `nvidia-driver` stream to enable. This controls whether the open or proprietary drivers are installed and the major version. Changing this once the drivers are installed does not change the version.
-- `cuda_packages`: Optional. Default: `['cuda', 'nvidia-gds', 'cmake', 'cuda-toolkit-12-9']`.
+- `cuda_packages`: Optional. Default provides CUDA Toolkit and GPUDirect Storage (GDS).
 - `cuda_package_version`: Optional. Default `latest` which will install the latest packages if not installed but won't upgrade already-installed packages. Use `'none'` to skip installing CUDA.
 - `cuda_persistenced_state`: Optional. State of systemd `nvidia-persistenced` service. Values as [ansible.builtin.systemd:state](https://docs.ansible.com/ansible/latest/collections/ansible/builtin/systemd_module.html#parameter-state). Default `started`.

ansible/roles/cuda/defaults/main.yml

@@ -1,12 +1,12 @@
 cuda_repo_url: "https://developer.download.nvidia.com/compute/cuda/repos/rhel{{ ansible_distribution_major_version }}/{{ ansible_architecture }}/cuda-rhel{{ ansible_distribution_major_version }}.repo"
 cuda_nvidia_driver_stream: '575-open'
-cuda_package_version: '12.9.0-1'
-cuda_version_short: '12.9'
+cuda_nvidia_driver_pkg: "nvidia-open-3:575.57.08-1.el{{ ansible_distribution_major_version }}"
+cuda_package_version: '12.9.1-1'
+cuda_version_short: "{{ (cuda_package_version | split('.'))[0:2] | join('.') }}" # major.minor
 cuda_packages:
-  - "cuda{{ ('-' + cuda_package_version) if cuda_package_version != 'latest' else '' }}"
+  - "cuda-toolkit-{{ cuda_package_version }}"
   - nvidia-gds
   - cmake
-  - cuda-toolkit-12-9
 cuda_samples_release_url: "https://github.com/NVIDIA/cuda-samples/archive/refs/tags/v{{ cuda_version_short }}.tar.gz"
 cuda_samples_path: "/var/lib/{{ ansible_user }}/cuda_samples"
 cuda_samples_programs:

ansible/roles/cuda/tasks/install.yml

@@ -1,16 +1,5 @@
 
-# Based on https://docs.nvidia.com/cuda/cuda-installation-guide-linux/index.html#redhat8-installation
-
-- name: Check for OFED/DOCA
-  command:
-    cmd: dnf list --installed rdma-core
-  register: _dnf_rdma_core
-  changed_when: false
-
-- name: Assert OFED installed
-  assert: 
-    that: "'mlnx' in _dnf_rdma_core.stdout"
-    fail_msg: "Did not find 'mlnx' in installed rdma-core package, is OFED/DOCA installed?"
+# Based on https://docs.nvidia.com/datacenter/tesla/driver-installation-guide/
 
 - name: Install cuda repo
   get_url:
@@ -29,23 +18,18 @@
   when: "'No matching Modules to list' in _cuda_driver_module_enabled.stderr"
   changed_when: "'Nothing to do' not in _cuda_driver_module_enable.stdout"
 
-- name: Check if nvidia driver module is installed
-  ansible.builtin.command: dnf module list --installed nvidia-driver
-  changed_when: false
-  failed_when: false
-  register: _cuda_driver_module_installed
-
 - name: Install nvidia drivers
-  ansible.builtin.command: dnf module install -y nvidia-driver
+  ansible.builtin.dnf:
+    name: "{{ cuda_nvidia_driver_pkg }}"
   register: _cuda_driver_install
-  when: "'No matching Modules to list' in _cuda_driver_module_installed.stderr"
-  changed_when: "'Nothing to do' not in _cuda_driver_install.stdout"
 
 - name: Check kernel has not been modified
   assert:
     that: "'kernel ' not in _cuda_driver_install.stdout | default('')" # space ensures we don't flag e.g. kernel-devel-matched
     fail_msg: "{{ _cuda_driver_install.stdout_lines | default([]) | select('search', 'kernel ') }}"
 
+# Based on https://docs.nvidia.com/cuda/cuda-installation-guide-linux/index.html
+
 - name: Install cuda packages
   ansible.builtin.dnf:
     name: "{{ cuda_packages }}"

ansible/roles/tuned/tasks/configure.yml

@@ -12,9 +12,17 @@
   register: _tuned_profile_current
   changed_when: false
 
-- name: Set tuned-adm profile
+- name: Set TuneD profile
   ansible.builtin.command:
     cmd: "tuned-adm profile {{ tuned_profile }}"
-  when: 
+  when:
+    - tuned_started | bool
+    - tuned_profile not in _tuned_profile_current.stdout
+
+- name: Verify TuneD profile
+  ansible.builtin.command:
+    cmd: tuned-adm verify
+  changed_when: false
+  when:
     - tuned_started | bool
     - tuned_profile not in _tuned_profile_current.stdout

ansible/roles/tuned/tasks/install.yml

@@ -2,4 +2,15 @@
 - name: Install tuneD
   ansible.builtin.dnf:
     name: tuned
-    state: present
+    state: present
+
+- name: Fix TuneD hpc-compute profile for hugepages
+  # See https://github.com/redhat-performance/tuned/issues/752
+  # This is done on install, not configure, so that it is available even
+  # for compute-init nodes
+  community.general.ini_file:
+    path: /usr/lib/tuned/hpc-compute/tuned.conf
+    section: sysctl
+    option: vm.min_free_kbytes
+    value: '>135168'
+    no_extra_spaces: true

docs/production.md

@@ -134,23 +134,6 @@ and referenced from the `site` and `production` environments, e.g.:
 
 - Consider whether having (read-only) access to Grafana without login is OK. If not, remove `grafana_auth_anonymous` in `environments/$ENV/inventory/group_vars/all/grafana.yml`
 
-- Modify `environments/site/tofu/nodes.tf` to provide fixed IPs for at least
-  the control node, and (if not using FIPs) the login node(s):
-
-    ```
-    resource "openstack_networking_port_v2" "control" {
-        ...
-        fixed_ip {
-            subnet_id = data.openstack_networking_subnet_v2.cluster_subnet.id
-            ip_address = var.control_ip_address
-        }
-    }
-    ```
-    
-  Note the variable `control_ip_address` is new.
-
-  Using fixed IPs will require either using admin credentials or policy changes.
-
 - If floating IPs are required for login nodes, modify the OpenTofu configurations
   appropriately.
 

environments/.stackhpc/inventory/extra_groups

@@ -29,7 +29,10 @@ cluster
 
 [tuned:children]
 # Install tuned into fat image
+# NB: builder has tuned_enabled and tuned_started false so does not configure it
 builder
+# Also test tuned during site playbook
+cluster
 
 [squid:children]
 # Install squid into fat image

environments/.stackhpc/inventory/group_vars/all/tuned.yml

@@ -0,0 +1,2 @@
+# Set profile which is not default (on VMs) for testing
+tuned_profile: hpc-compute