Merge branch 'main' into update/openhpc-b64-mungekey

sjpb · sjpb · commit 35724c158758 · 2025-08-05T14:08:09.000Z
diff --git a/.github/workflows/fatimage.yml b/.github/workflows/fatimage.yml
@@ -31,7 +31,7 @@ jobs:
             source_image_name: Rocky-8-GenericCloud-Base-8.10-20240528.0.x86_64.raw
             inventory_groups: control,compute,login,update
           - image_name: openhpc-RL9
-            source_image_name: Rocky-9-GenericCloud-Base-9.5-20241118.0.x86_64.raw
+            source_image_name: Rocky-9-GenericCloud-Base-9.6-20250531.0.x86_64.qcow2
             inventory_groups: control,compute,login,update
     env:
       ANSIBLE_FORCE_COLOR: True
diff --git a/.github/workflows/stackhpc.yml b/.github/workflows/stackhpc.yml
@@ -9,8 +9,7 @@ on:
       - '**'
       - '!dev/**'
       - 'dev/setup-env.sh'
-      - '!docs/**'
-      - '!README.md'
+      - '!**.md'
       - '!.gitignore'
       - '!.github/workflows/'
       - '.github/workflows/stackhpc'
@@ -19,8 +18,7 @@ on:
       - '**'
       - '!dev/**'
       - 'dev/setup-env.sh'
-      - '!docs/**'
-      - '!README.md'
+      - '!**.md'
       - '!.gitignore'
       - '!.github/workflows/'
       - '.github/workflows/stackhpc'
diff --git a/ansible/fatimage.yml b/ansible/fatimage.yml
@@ -190,6 +190,11 @@
         tasks_from: install.yml
       when: "'alertmanager' in group_names"
 
+    - name: Download HPL source
+      include_role:
+        name: hpctests
+        tasks_from: source-hpl.yml
+
 - hosts: prometheus
   become: yes
   gather_facts: yes
@@ -262,7 +267,7 @@
         name: grafana-dashboards
 
 - name: Add support for NVIDIA GPU auto detection to Slurm
-  hosts: cuda
+  hosts: slurm_recompile
   become: yes
   tasks:
     - name: Recompile slurm
diff --git a/ansible/roles/dnf_repos/defaults/main.yml b/ansible/roles/dnf_repos/defaults/main.yml
@@ -47,7 +47,8 @@ dnf_repos_openhpc_repolist:
   file: OpenHPC
   base_url: "{{ dnf_repos_pulp_content_url }}/{{ appliances_pulp_repos.openhpc_updates[ansible_distribution_major_version] | appliances_repo_to_subpath }}"
 
-dnf_repos_repolist: "{{ dnf_repos_default_repolist + (dnf_repos_openhpc_repolist if (openhpc_install_type | default('ohpc')) == 'ohpc' else []) }}"
+dnf_repos_extra_repolist: []
+dnf_repos_repolist: "{{ dnf_repos_default_repolist + (dnf_repos_openhpc_repolist if (openhpc_install_type | default('ohpc')) == 'ohpc' else []) + dnf_repos_extra_repolist }}"
 
 dnf_repos_epel_baseurl: "{{ dnf_repos_pulp_content_url }}/{{ appliances_pulp_repos.epel[ansible_distribution_major_version] | appliances_repo_to_subpath }}"
 dnf_repos_epel_description: "epel"
diff --git a/ansible/roles/hpctests/tasks/build-hpl.yml b/ansible/roles/hpctests/tasks/build-hpl.yml
@@ -5,18 +5,21 @@
     path: "{{ hpctests_rootdir }}/hpl"
     state: directory
 
-- name: Download HPL sources
+- name: Unarchive HPL sources from /opt/hpl
   unarchive:
-    src: "http://www.netlib.org/benchmark/hpl/hpl-{{ hpctests_hpl_version }}.tar.gz"
-    remote_src: yes
+    src: "/opt/hpl/hpl-{{ hpctests_hpl_version }}.tar.gz"
     dest: "{{ hpctests_rootdir }}/hpl"
+    remote_src: yes
+    owner: "{{ hpctests_user }}"
+    group: "{{ hpctests_group }}"
+    mode: '0755'
     keep_newer: yes
 
-- name: Copy BLAS make file
-  command:
-    cmd: "cp setup/Make.Linux_PII_CBLAS Make.{{ hpctests_hpl_arch }}"
-    chdir: "{{ hpctests_hpl_srcdir }}"
-    creates: "{{ hpctests_hpl_srcdir }}/Make.{{ hpctests_hpl_arch }}"
+- name: Copy BLAS makefile
+  copy:
+    src: "{{ hpctests_hpl_srcdir }}/setup/Make.Linux_PII_CBLAS"
+    dest: "{{ hpctests_hpl_srcdir }}/Make.{{ hpctests_hpl_arch }}"
+    remote_src: yes
 
 - name: Modify make file
   replace:
diff --git a/ansible/roles/hpctests/tasks/source-hpl.yml b/ansible/roles/hpctests/tasks/source-hpl.yml
@@ -0,0 +1,17 @@
+---
+
+- name: Make directory
+  file:
+    path: "/opt/hpl"
+    state: directory
+    owner: root
+    group: root
+    mode: '0755'
+
+- name: Download HPL tarball
+  get_url:
+    url: "http://www.netlib.org/benchmark/hpl/hpl-{{ hpctests_hpl_version }}.tar.gz"
+    dest: "/opt/hpl/hpl-{{ hpctests_hpl_version }}.tar.gz"
+    owner: root
+    group: root
+    mode: '0644'
diff --git a/ansible/roles/lustre/README.md b/ansible/roles/lustre/README.md
@@ -19,10 +19,8 @@ The following variables control configuration of Lustre clients.
 - `lustre_mount_options`. Optional default mount options. Default values are systemd defaults from [Lustre client docs](http://wiki.lustre.org/Mounting_a_Lustre_File_System_on_Client_Nodes).
 
 The following variables control the package build and and install:
-- `lustre_version`: Optional str. Version of lustre to build, default `2.15.6/lu-18085`
-  which is the first version with EL9.5 support, plus a fix for https://jira.whamcloud.com/browse/LU-18085.
-- `lustre_repo`: Optional str. URL for Lustre repo. Default is a StackHPC repo
-  incorporating the above fix.
+- `lustre_version`: Optional str. Version of lustre to build, default `2.15.7`
+- `lustre_repo`: Optional str. URL for Lustre repo. Default is `git://git.whamcloud.com/fs/lustre-release`.git.
 - `lustre_build_packages`: Optional list. Prerequisite packages required to build Lustre. See `defaults/main.yml`.
 - `lustre_build_dir`: Optional str. Path to build lustre at, default `/tmp/lustre-release`.
 - `lustre_configure_opts`: Optional list. Options to `./configure` command. Default builds client rpms supporting Mellanox OFED, without support for GSS keys.
diff --git a/ansible/roles/lustre/defaults/main.yml b/ansible/roles/lustre/defaults/main.yml
@@ -1,13 +1,12 @@
-lustre_repo: https://github.com/stackhpc/lustre-release.git
-lustre_version: '2.15.6/lu-18085' # Fixes https://jira.whamcloud.com/browse/LU-18085
+lustre_version: '2.15.7'
 lustre_lnet_label: tcp
 #lustre_mgs_nid:
 lustre_mounts: []
 lustre_mount_state: mounted
 lustre_mount_options: 'defaults,_netdev,noauto,x-systemd.automount,x-systemd.requires=lnet.service,nosuid,nodev'
 
 # below variables are for build and should not generally require changes
-lustre_git_repo: "git://git.whamcloud.com/fs/lustre-release.git"
+lustre_repo: "git://git.whamcloud.com/fs/lustre-release.git"
 lustre_build_packages:
   - "kernel-devel-{{ ansible_kernel }}"
   - git
diff --git a/ansible/roles/ofed/README.md b/ansible/roles/ofed/README.md
@@ -1,5 +1,10 @@
 # ofed
 
+> [!IMPORTANT]
+> This role is deprecated - it is not regularly maintained and StackHPC CI
+> does not test that it works. Consider using [ansible/roles/doca](../doca/README.md)
+> instead.
+
 This role installs Mellanox OFED:
 - It checks that the running kernel is the latest installed one, and errors if not.
 - Installation uses the `mlnxofedinstall` command, with support for the running kernel
diff --git a/ansible/roles/proxy/defaults/main.yml b/ansible/roles/proxy/defaults/main.yml
@@ -1,6 +1,6 @@
 # proxy_http_proxy:
 proxy_https_proxy: "{{ proxy_http_proxy }}"
-proxy_no_proxy_defaults: "{{ ['localhost', '127.0.0.1'] + groups['all'] + hostvars.values() | map(attribute='ansible_host') }}"
+proxy_no_proxy_defaults: "{{ ['localhost', '127.0.0.1', '169.254.169.254'] + groups['all'] + hostvars.values() | map(attribute='ansible_host') }}"
 proxy_no_proxy_extras: []
 proxy_no_proxy: "{{ (proxy_no_proxy_defaults + proxy_no_proxy_extras) | unique | sort | join(',') }}"
 proxy_dnf: true
diff --git a/docs/experimental/isolated-clusters.md b/docs/experimental/isolated-clusters.md
@@ -12,14 +12,7 @@ created by cookiecutter for a new environment.
 The full list of features and whether they are functional on such an "isolated"
 network is shown in the table below. Note that:
 
-1. The `hpl` test from the `ansible/adhoc/hpctests.yml` playbook is not
-   functional and must be skipped using:
-
-    ```shell
-    ansible-playbook ansible/adhoc/hpctests.yml --skip-tags hpl-solo
-    ```
-
-2. Using [EESSI](https://www.eessi.io/docs/) necessarily requires outbound
+-  Using [EESSI](https://www.eessi.io/docs/) necessarily requires outbound
    network access for the CernVM File System. However this can be provided
    via an authenticated proxy. While the proxy configuration on the cluster node
    is readable by all users, this proxy could be limited via acls to only provide
@@ -51,7 +44,7 @@ See above for definition of "Default" features. In the "Isolated?" column:
 | freeipa_client        | - | Y - image build required |
 | gateway               | n/a | n/a - build only | 
 | grafana               | Y | Y | 
-| hpctests              | Y | Y  - except hpl-solo, see above | 
+| hpctests              | Y | Y | 
 | k3s_agent             | - | ? | 
 | k3s_server            | - | ? | 
 | k9s                   | - | ? | 
diff --git a/docs/openondemand.md b/docs/openondemand.md
@@ -31,7 +31,13 @@ The above functionality is configured by running the `ansible/portal.yml` playbo
 See the [ansible/roles/openondemand/README.md](../ansible/roles/openondemand/README.md) for more details on the variables described below.
 
 The following variables have been given default values to allow Open OnDemand to work in a newly created environment without additional configuration, but generally should be overridden in `environments/site/inventory/group_vars/all/` with site-specific values:
-- `openondemand_servername` - this must be defined for both `openondemand` and `grafana` hosts (when Grafana is enabled). Default is `ansible_host` (i.e. the IP address) of the first host in the `openondemand` group.
+- `openondemand_servername` - this must be defined for both `openondemand` and
+  `grafana` hosts (when Grafana is enabled). The default is `ansible_host` (i.e.
+  the IP address) of the first host in the `openondemand` group. For production
+  environments this should probably be a DNS name.
+- `openondemand_ssl_cert` and `openondemand_ssl_cert_key` - by default a
+  self-signed certificate is generated, which should probably be replaced for
+  production environments.
 - `openondemand_auth` and any corresponding options. Defaults to `basic_pam`.
 - `openondemand_desktop_partition` and `openondemand_jupyter_partition` if the corresponding inventory groups are defined. Defaults to the first compute group defined in the `compute` OpenTofu variable in `environments/$ENV/tofu`.
 
diff --git a/docs/production.md b/docs/production.md
@@ -24,6 +24,10 @@ production-ready deployments.
     inventory = ../common/inventory,../site/inventory,inventory
     ```
 
+  In general only the `site` environment will need an `inventory/groups` file -
+  this is templated out by cookiecutter and should be modified as required to
+  enable features for all environments at the site.
+
 - To avoid divergence of configuration all possible overrides for group/role
 vars should be placed in `environments/site/inventory/group_vars/all/*.yml`
 unless the value really is environment-specific (e.g. DNS names for
@@ -127,7 +131,17 @@ and referenced from the `site` and `production` environments, e.g.:
   set the "attach" options and run `tofu apply` again - this should show there
   are no changes planned.
 
-- Configure Open OnDemand - see [specific documentation](openondemand.md).
+- Consider whether Prometheus storage configuration is required. By default:
+  - A 200GB state volume is provisioned (but see above)
+  - The common environment [sets](../environments/common/inventory/group_vars/all/prometheus.yml)
+    a maximum retention of 100 GB and 31 days
+  These may or may not be appropriate depending on the number of nodes, the
+  scrape interval, and other uses of the state volume (primarily the `slurmctld`
+  state and the `slurmdbd` database). See [docs/monitoring-and-logging](./monitoring-and-logging.md)
+  for more options.
+
+- Configure Open OnDemand - see [specific documentation](openondemand.md) which
+  notes specific variables required.
 
 - Remove the `demo_user` user from `environments/$ENV/inventory/group_vars/all/basic_users.yml`
 
diff --git a/docs/upgrades.md b/docs/upgrades.md
@@ -50,17 +50,23 @@ All other commands should be run on the Ansible deploy host.
    site-specific configuration. In general changes to existing functionality will aim to be
    backward compatible. Alteration of site-specific configuration will usually only be
    necessary to use new functionality or where functionality has been upstreamed as above.
+   Note that the `environments/common/layouts/everything` file contains all possible
+   groups which can be used to enable features; diff this against your e.g.
+   `environments/site/inventory/groups` file to see new features which you may
+   wish to enable in the latter file.
 
    Make changes as necessary.
 
 1. Identify image(s) from the relevant [Slurm appliance release](https://github.com/stackhpc/ansible-slurm-appliance/releases), and download
-   using the link on the release plus the image name, e.g. for an image `openhpc-ofed-RL8-240906-1042-32568dbb`:
+   using the link on the release plus the image name, e.g. for an image `openhpc-RL9-250708-1547-1494192e`:
 
-        wget https://object.arcus.openstack.hpc.cam.ac.uk/swift/v1/AUTH_3a06571936a0424bb40bc5c672c4ccb1/openhpc-images/openhpc-ofed-RL8-240906-1042-32568dbb
+        wget https://object.arcus.openstack.hpc.cam.ac.uk/swift/v1/AUTH_3a06571936a0424bb40bc5c672c4ccb1/openhpc-images/openhpc-RL9-250708-1547-1494192e
 
     Note that some releases may not include new images. In this case use the image from the latest previous release with new images.
 
-1. If required, build an "extra" image with local modifications, see [docs/image-build.md](./image-build.md).
+1. If an "extra" image build with local modifications is required, update the
+   Packer build configuration to use the above new image and run a build. See
+   [docs/image-build.md](./image-build.md).
 
 1. Modify your site-specific environment to use this image, e.g. via `cluster_image_id` in `environments/$SITE_ENV/tofu/variables.tf`.
 
diff --git a/environments/.stackhpc/tofu/cluster_image.auto.tfvars.json b/environments/.stackhpc/tofu/cluster_image.auto.tfvars.json
@@ -1,6 +1,6 @@
 {
     "cluster_image": {
-        "RL8": "openhpc-RL8-250708-1502-1494192e",
-        "RL9": "openhpc-RL9-250708-1547-1494192e"
+        "RL8": "openhpc-RL8-250805-1048-59517120",
+        "RL9": "openhpc-RL9-250805-1050-59517120"
     }
 }
diff --git a/environments/common/inventory/group_vars/all/timestamps.yml b/environments/common/inventory/group_vars/all/timestamps.yml
@@ -9,6 +9,9 @@ appliances_pulp_repos:
     '9.5':
       path: rocky/9.5/AppStream/x86_64/os
       timestamp: 20250514T014704
+    '9.6':
+      path: rocky/9.6/AppStream/x86_64/os
+      timestamp: 20250726T040613
   baseos:
     '8.10':
       path: rocky/8.10/BaseOS/x86_64/os
@@ -19,13 +22,16 @@ appliances_pulp_repos:
     '9.5':
       path: rocky/9.5/BaseOS/x86_64/os
       timestamp: 20250513T031844
+    '9.6':
+      path: rocky/9.6/BaseOS/x86_64/os
+      timestamp: 20250726T052250
   ceph:
     '8':
       path: centos/8-stream/storage/x86_64/ceph-quincy
       timestamp: 20231104T015751
     '9':
       path: centos/9-stream/storage/x86_64/ceph-reef
-      timestamp: 20250514T025809
+      timestamp: 20250617T023108
   crb:
     '8.10':
       path: rocky/8.10/PowerTools/x86_64/os
@@ -36,13 +42,16 @@ appliances_pulp_repos:
     '9.5':
       path: rocky/9.5/CRB/x86_64/os
       timestamp: 20250514T014704
+    '9.6':
+      path: rocky/9.6/CRB/x86_64/os
+      timestamp: 20250726T040613
   epel:
     '8':
       path: epel/8/Everything/x86_64
       timestamp: 20250615T234151
     '9':
       path: epel/9/Everything/x86_64
-      timestamp: 20250615T000221
+      timestamp: 20250729T235750
   extras:
     '8.10':
       path: rocky/8.10/extras/x86_64/os
@@ -53,13 +62,16 @@ appliances_pulp_repos:
     '9.5':
       path: rocky/9.5/extras/x86_64/os
       timestamp: 20250506T032818
+    '9.6':
+      path: rocky/9.6/extras/x86_64/os
+      timestamp: 20250726T040613
   grafana:
     '8':
       path: grafana/oss/rpm
       timestamp: 20250615T005738
     '9':
       path: grafana/oss/rpm
-      timestamp: 20250615T005738
+      timestamp: 20250730T011314
   openhpc_base:
     '8':
       path: OpenHPC/2/EL_8
diff --git a/environments/common/inventory/groups b/environments/common/inventory/groups
@@ -118,6 +118,9 @@ freeipa_client
 [cuda]
 # Hosts to install NVIDIA CUDA on - see ansible/roles/cuda/README.md
 
+[slurm_recompile]
+# Hosts to recompile Slurm for - allows supporting Slurm autodetection method 'nvml'
+
 [vgpu]
 # Hosts where vGPU/MIG should be configured - see docs/mig.md
 
diff --git a/environments/common/layouts/everything b/environments/common/layouts/everything
@@ -65,6 +65,10 @@ cluster
 [cuda]
 # Hosts to install NVIDIA CUDA on - see ansible/roles/cuda/README.md
 
+[slurm_recompile:children]
+# Hosts to recompile Slurm for - allows supporting Slurm autodetection method 'nvml'
+cuda
+
 [eessi:children]
 # Hosts on which EESSI stack should be configured
 openhpc
