Merge branch 'main' into fix/tuned-hpc-compute-hugemem

Author: sjpb

.github/workflows/release-image.yml

@@ -0,0 +1,37 @@
+name: Release images
+on:
+  workflow_dispatch:
+  release:
+    types:
+      - published # should work for both pre-releases and releases
+env:
+  IMAGE_PATH: environments/.stackhpc/tofu/cluster_image.auto.tfvars.json
+jobs:
+  ci-image-release:
+    name: ci-image-release
+    runs-on: ubuntu-22.04
+    concurrency: ${{ github.workflow }}-${{ github.ref }}
+    strategy:
+      fail-fast: false
+      matrix:
+        build:
+          - RL8
+          - RL9
+    steps:
+      - uses: actions/checkout@v2
+
+      - name: Write s3cmd configuration
+        run: echo "${{ secrets.ARCUS_S3_CFG }}" > ~/.s3cfg
+
+      - name: Install s3cmd
+        run: |
+          sudo apt-get update
+          sudo apt-get --yes install s3cmd
+      
+      - name: Retrieve image name
+        run: |
+          TARGET_IMAGE=$(jq --arg version "${{ matrix.build }}" -r '.cluster_image[$version]' "${{ env.IMAGE_PATH }}")
+          echo "TARGET_IMAGE=${TARGET_IMAGE}" >> "$GITHUB_ENV"
+      
+      - name: Copy image from pre-release to release bucket
+        run: s3cmd cp s3://openhpc-images-prerelease/${{ env.TARGET_IMAGE }} s3://openhpc-images

README.md

@@ -25,8 +25,6 @@ The default configuration in this repository may be used to create a cluster to
 - Persistent state backed by an OpenStack volume.
 - NFS-based shared file system backed by another OpenStack volume.
 
-Note that the Open OnDemand portal and its remote apps are not usable with this default configuration.
-
 It requires an OpenStack cloud, and an Ansible "deploy host" with access to that cloud.
 
 Before starting ensure that:

ansible/bootstrap.yml

@@ -143,7 +143,7 @@
       - appliances_mode == 'configure'
       - not (dnf_repos_allow_insecure_creds | default(false)) # useful for development
 
-- hosts: cacerts:!builder
+- hosts: cacerts
   tags: cacerts
   gather_facts: false
   tasks:

ansible/roles/cacerts/README.md

@@ -4,7 +4,7 @@ Configure CA certificates and trusts.
 
 ## Role variables
 
-- `ca-certificates`: Optional str. Path to directory containing certificates
+- `cacerts_cert_dir`: Optional str. Path to directory containing certificates
   in PEM or DER format. Any files here will be added to the list of CAs trusted
   by the system.
 

ansible/roles/cluster_infra/templates/outputs.tf.j2

@@ -32,12 +32,12 @@ output "cluster_nodes" {
         }
       }
     ],
-    {% for partition in openhpc_slurm_partitions %}
+    {% for nodegroup in openhpc_nodegroups %}
     [
-      for compute in openstack_compute_instance_v2.{{ partition.name }}: {
+      for compute in openstack_compute_instance_v2.{{ nodegroup.name }}: {
         name          = compute.name
         ip            = compute.network[0].fixed_ip_v4
-        groups        = ["compute", "{{ cluster_name }}_compute", "{{ cluster_name }}_{{ partition.name }}"],
+        groups        = ["compute", "{{ cluster_name }}_compute", "{{ cluster_name }}_{{ nodegroup.name }}"],
         facts  = {
           openstack_project_id = data.openstack_identity_auth_scope_v3.scope.project_id
         }

ansible/roles/cluster_infra/templates/resources.tf.j2

@@ -282,11 +282,11 @@ resource "openstack_networking_port_v2" "control_storage" {
 ###
 # Workers
 ###
-{% for partition in openhpc_slurm_partitions %}
+{% for nodegroup in openhpc_nodegroups %}
 # Primary network
-resource "openstack_networking_port_v2" "{{ partition.name }}" {
-  count = {{ partition.count }}
-  name = "{{ cluster_name }}-compute-{{ partition.name }}-${count.index}"
+resource "openstack_networking_port_v2" "{{ nodegroup.name }}" {
+  count = {{ nodegroup.count }}
+  name = "{{ cluster_name }}-compute-{{ nodegroup.name }}-${count.index}"
   network_id = "${data.openstack_networking_network_v2.cluster_network.id}"
   admin_state_up = "true"
 
@@ -305,9 +305,9 @@ resource "openstack_networking_port_v2" "{{ partition.name }}" {
 
 # Storage network
 {% if cluster_storage_network is defined %}
-resource "openstack_networking_port_v2" "{{ partition.name }}_storage" {
-  count          = {{ partition.count }}
-  name           = "{{ cluster_name }}-compute-{{ partition.name }}-storage-${count.index}"
+resource "openstack_networking_port_v2" "{{ nodegroup.name }}_storage" {
+  count          = {{ nodegroup.count }}
+  name           = "{{ cluster_name }}-compute-{{ nodegroup.name }}-storage-${count.index}"
   network_id     = data.openstack_networking_network_v2.cluster_storage.id
   admin_state_up = "true"
 
@@ -499,25 +499,25 @@ resource "openstack_compute_instance_v2" "control" {
   }
 }
 
-{% for partition in openhpc_slurm_partitions %}
-resource "openstack_compute_instance_v2" "{{ partition.name }}" {
-  count = {{ partition.count }}
+{% for nodegroup in openhpc_nodegroups %}
+resource "openstack_compute_instance_v2" "{{ nodegroup.name }}" {
+  count = {{ nodegroup.count }}
 
-  name      = "{{ cluster_name }}-compute-{{ partition.name }}-${count.index}"
+  name      = "{{ cluster_name }}-compute-{{ nodegroup.name }}-${count.index}"
   image_id  = "{{ cluster_image }}"
-  {% if 'flavor_name' in partition %}
-  flavor_name = "{{ partition.flavor_name }}"
+  {% if 'flavor_name' in nodegroup %}
+  flavor_name = "{{ nodegroup.flavor_name }}"
   {% else %}
-  flavor_id = "{{ partition.flavor }}"
+  flavor_id = "{{ nodegroup.flavor }}"
   {% endif %}
 
   network {
-    port = openstack_networking_port_v2.{{ partition.name }}[count.index].id
+    port = openstack_networking_port_v2.{{ nodegroup.name }}[count.index].id
   }
 
   {% if cluster_storage_network is defined %}
   network {
-    port = openstack_networking_port_v2.{{ partition.name }}_storage[count.index].id
+    port = openstack_networking_port_v2.{{ nodegroup.name }}_storage[count.index].id
   }
   {% endif %}
 

ansible/roles/cuda/README.md

@@ -10,6 +10,6 @@ Requires OFED to be installed to provide required kernel-* packages.
 
 - `cuda_repo_url`: Optional. URL of `.repo` file. Default is upstream for appropriate OS/architecture.
 - `cuda_nvidia_driver_stream`: Optional. Version of `nvidia-driver` stream to enable. This controls whether the open or proprietary drivers are installed and the major version. Changing this once the drivers are installed does not change the version.
-- `cuda_packages`: Optional. Default: `['cuda', 'nvidia-gds', 'cmake', 'cuda-toolkit-12-8']`.
+- `cuda_packages`: Optional. Default: `['cuda', 'nvidia-gds', 'cmake', 'cuda-toolkit-12-9']`.
 - `cuda_package_version`: Optional. Default `latest` which will install the latest packages if not installed but won't upgrade already-installed packages. Use `'none'` to skip installing CUDA.
 - `cuda_persistenced_state`: Optional. State of systemd `nvidia-persistenced` service. Values as [ansible.builtin.systemd:state](https://docs.ansible.com/ansible/latest/collections/ansible/builtin/systemd_module.html#parameter-state). Default `started`.

ansible/roles/cuda/defaults/main.yml

@@ -1,12 +1,12 @@
 cuda_repo_url: "https://developer.download.nvidia.com/compute/cuda/repos/rhel{{ ansible_distribution_major_version }}/{{ ansible_architecture }}/cuda-rhel{{ ansible_distribution_major_version }}.repo"
-cuda_nvidia_driver_stream: '570-open'
-cuda_package_version: '12.8.1-1'
-cuda_version_short: '12.8'
+cuda_nvidia_driver_stream: '575-open'
+cuda_package_version: '12.9.0-1'
+cuda_version_short: '12.9'
 cuda_packages:
   - "cuda{{ ('-' + cuda_package_version) if cuda_package_version != 'latest' else '' }}"
   - nvidia-gds
   - cmake
-  - cuda-toolkit-12-8
+  - cuda-toolkit-12-9
 cuda_samples_release_url: "https://github.com/NVIDIA/cuda-samples/archive/refs/tags/v{{ cuda_version_short }}.tar.gz"
 cuda_samples_path: "/var/lib/{{ ansible_user }}/cuda_samples"
 cuda_samples_programs:

ansible/roles/cuda/tasks/samples.yml

@@ -25,36 +25,3 @@
     cmd: . /etc/profile.d/sh.local && cmake .. && make -j {{ ansible_processor_vcpus }}
     chdir: "{{ cuda_samples_path }}/cuda-samples-{{ cuda_version_short }}/build"
     creates: "{{ cuda_samples_path }}/cuda-samples-{{ cuda_version_short }}/build/Samples/1_Utilities/deviceQuery/deviceQuery"
-
-- name: Run CUDA deviceQuery
-  command:
-    cmd: "{{ cuda_samples_path }}/cuda-samples-{{ cuda_version_short }}/build/Samples/1_Utilities/deviceQuery/deviceQuery"
-  register: _cuda_devicequery
-
-- name: Set fact for CUDA devices
-  set_fact:
-    cuda_devices: "{{ _cuda_devicequery.stdout | regex_findall('Device (\\d+):') }}"
-
-- name: Run CUDA bandwidth test
-  command:
-    cmd: "{{ cuda_samples_path }}/cuda-samples-{{ cuda_version_short }}/build/Samples/1_Utilities/bandwidthTest/bandwidthTest --device={{ item }}"
-  register: _cuda_bandwidthtest
-  loop: "{{ cuda_devices }}"
-  loop_control:
-    label: "Device {{ item }}" # e.g '0'
-
-- name: Summarise bandwidth test output
-  debug:
-    msg: |
-      {{ _parts[1].splitlines()[0] | trim }}
-      Bandwidths: (Gb/s)
-      Host to Device: {{ _parts[2].split()[-1] }}
-      Device to Host: {{ _parts[3].split()[-1] }}
-      Device to Device: {{ _parts[4].split()[-1] }}
-      {{ ': '.join(_parts[5].split('=') | map('trim')) }}
-      {{ _parts[6] }}
-  loop: "{{ _cuda_bandwidthtest.results }}"
-  vars:
-    _parts: "{{ item.stdout.split('\n\n') }}"
-  loop_control:
-    label: "Device {{ item.item }}" # e.g '0'

ansible/roles/lustre/README.md

@@ -7,8 +7,7 @@ Install and configure a Lustre client. This builds RPM packages from source.
 **NB:** Currently this only supports RockyLinux 9.
 
 ## Role Variables
-
-- `lustre_version`: Optional str. Version of lustre to build, default `2.15.6` which is the first version with EL9.5 support
+The following variables control configuration of Lustre clients.
 - `lustre_lnet_label`: Optional str. The "lnet label" part of the host's NID, e.g. `tcp0`. Only the `tcp` protocol type is currently supported. Default `tcp`.
 - `lustre_mgs_nid`: Required str. The NID(s) for the MGS, e.g. `192.168.227.11@tcp1` (separate mutiple MGS NIDs using `:`).
 - `lustre_mounts`: Required list. Define Lustre filesystems and mountpoints as a list of dicts with keys:
@@ -19,7 +18,11 @@ Install and configure a Lustre client. This builds RPM packages from source.
 - `lustre_mount_state`. Optional default mount state for all mounts, as for [ansible.posix.mount](https://docs.ansible.com/ansible/latest/collections/ansible/posix/mount_module.html#parameter-state). Default is `mounted`.
 - `lustre_mount_options`. Optional default mount options. Default values are systemd defaults from [Lustre client docs](http://wiki.lustre.org/Mounting_a_Lustre_File_System_on_Client_Nodes).
 
-The following variables control the package build and and install and should not generally be required:
+The following variables control the package build and and install:
+- `lustre_version`: Optional str. Version of lustre to build, default `2.15.6/lu-18085`
+  which is the first version with EL9.5 support, plus a fix for https://jira.whamcloud.com/browse/LU-18085.
+- `lustre_repo`: Optional str. URL for Lustre repo. Default is a StackHPC repo
+  incorporating the above fix.
 - `lustre_build_packages`: Optional list. Prerequisite packages required to build Lustre. See `defaults/main.yml`.
 - `lustre_build_dir`: Optional str. Path to build lustre at, default `/tmp/lustre-release`.
 - `lustre_configure_opts`: Optional list. Options to `./configure` command. Default builds client rpms supporting Mellanox OFED, without support for GSS keys.