Merge branch 'main' into refactor/toplevel-playbooks

Author: sjpb

.github/workflows/extra.yml

@@ -121,7 +121,7 @@ jobs:
       - name: Make image usable for further builds
         run: |
           . venv/bin/activate
-          openstack image unset --property signature_verified "${{ steps.manifest.outputs.image-id }}"
+          openstack image unset --property signature_verified "${{ steps.manifest.outputs.image-id }}" || true
 
       - name: Delete image for automatically-run workflows
         run: |

.github/workflows/fatimage.yml

@@ -23,10 +23,10 @@ jobs:
       matrix: # build RL8, RL9
         build:
           - image_name: openhpc-RL8
-            source_image_name: Rocky-8-GenericCloud-Base-8.10-20240528.0.x86_64.qcow2
+            source_image_name: Rocky-8-GenericCloud-Base-8.10-20240528.0.x86_64.raw
             inventory_groups: control,compute,login,update
           - image_name: openhpc-RL9
-            source_image_name: Rocky-9-GenericCloud-Base-9.5-20241118.0.x86_64.qcow2
+            source_image_name: Rocky-9-GenericCloud-Base-9.5-20241118.0.x86_64.raw
             inventory_groups: control,compute,login,update
     env:
       ANSIBLE_FORCE_COLOR: True
@@ -102,7 +102,7 @@ jobs:
       - name: Make image usable for further builds
         run: |
           . venv/bin/activate
-          openstack image unset --property signature_verified "${{ steps.manifest.outputs.image-id }}"
+          openstack image unset --property signature_verified "${{ steps.manifest.outputs.image-id }}" || true
 
       - name: Upload manifest artifact
         uses: actions/upload-artifact@v4

.github/workflows/nightly-cleanup.yml

@@ -59,10 +59,34 @@ jobs:
       - name: Delete CI clusters
         run: |
           . venv/bin/activate
-          if [[ -z "${ci_clusters}" ]]; then
+          if [[ -z ${ci_clusters} ]]; then
             echo "No clusters to delete."
             exit 0
           fi
-          echo "Deleting clusters: ${ci_clusters}"
-          ./dev/delete-cluster.py ${ci_clusters} --force
+
+          for cluster_prefix in ${ci_clusters}
+          do
+            echo "Processing cluster: $cluster_prefix"
+
+            # Get all servers with the matching name for control node
+            CONTROL_SERVERS=$(openstack server list --name ${cluster_prefix}-control --format json)
+
+            # Get unique server names to avoid duplicate cleanup
+            UNIQUE_NAMES=$(echo "$CONTROL_SERVERS" | jq -r '.[].Name' | sort | uniq)
+            for name in $UNIQUE_NAMES; do
+              echo "Deleting cluster with control node: $name"
+
+              # Get the first matching server ID by name
+              server=$(echo "$CONTROL_SERVERS" | jq -r '.[] | select(.Name=="'"$name"'") | .ID' | head -n1)
+
+              # Make sure server still exists (wasn't deleted earlier)
+              if ! openstack server show "$server" &>/dev/null; then
+                echo "Server $server no longer exists, skipping $name."
+                continue
+              fi
+
+              echo "Deleting cluster $cluster_prefix (server $server)..."
+              ./dev/delete-cluster.py $cluster_prefix --force
+            done
+          done
         shell: bash

.github/workflows/s3-image-sync.yml

@@ -26,6 +26,7 @@ jobs:
 
       - name: Install s3cmd
         run: |
+          sudo apt-get update
           sudo apt-get --yes install s3cmd
       
       - name: Cleanup S3 bucket
@@ -75,20 +76,38 @@ jobs:
           echo "${{ secrets['ARCUS_S3_CFG'] }}" > ~/.s3cfg
         shell: bash
 
-      - name: Install s3cmd
+      - name: Install s3cmd and qemu-utils
         run: |
-          sudo apt-get --yes install s3cmd
+          sudo apt-get update
+          sudo apt-get --yes install s3cmd qemu-utils
 
       - name: Retrieve image name
         run: |
           TARGET_IMAGE=$(jq --arg version "${{ matrix.build }}" -r '.cluster_image[$version]' "${{ env.IMAGE_PATH }}")
           echo "TARGET_IMAGE=${TARGET_IMAGE}" >> "$GITHUB_ENV"
         shell: bash
 
+      - name: Clear up some space on runner
+        run: |
+          df -h
+          sudo rm -rf /usr/share/dotnet
+          sudo rm -rf /opt/ghc
+          sudo rm -rf "/usr/local/share/boost"
+          sudo rm -rf "$AGENT_TOOLSDIRECTORY"
+          sudo apt-get clean
+          df -h
+
       - name: Download image to runner
         run: |
           . venv/bin/activate
-          openstack image save --file ${{ env.TARGET_IMAGE }} ${{ env.TARGET_IMAGE }}
+          openstack image save --file "${{ env.TARGET_IMAGE }}.raw" "${{ env.TARGET_IMAGE }}"
+          df -h
+        shell: bash
+
+      - name: Convert image to QCOW2
+        run: |
+          . venv/bin/activate
+          qemu-img convert -f raw -O qcow2 -c "${{ env.TARGET_IMAGE }}.raw" "${{ env.TARGET_IMAGE }}"
         shell: bash
 
       - name: Upload Image to S3

ansible/roles/hpctests/defaults/main.yml

@@ -1,5 +1,6 @@
 ---
 hpctests_user: "{{ ansible_user }}"
+hpctests_group: "{{ ansible_user }}"
 hpctests_rootdir: "/home/{{ hpctests_user }}/hpctests"
 hpctests_pre_cmd: ''
 hpctests_pingmatrix_modules: [gnu12 openmpi4]

ansible/roles/hpctests/tasks/setup.yml

@@ -26,7 +26,7 @@
     path: "{{ hpctests_rootdir }}"
     state: directory
     owner: "{{ hpctests_user }}"
-    group: "{{ hpctests_user }}"
+    group: "{{ hpctests_group }}"
 
 - name: Set fact for UCX_NET_DEVICES
   set_fact:

docs/experimental/slurm-controlled-rebuild.md

@@ -103,7 +103,7 @@ The configuration of this is complex and involves:
    be in:
     - `environments/site/tofu/variables.tf`: `cluster_image_id` for the default
       cluster image.
-    - `enviroments/$ENV/tofu/main.tf`: parameter `image_id` in node groups
+    - `environments/$ENV/tofu/main.tf`: parameter `image_id` in node groups
       defined in the `compute` or `login` variables, to override the default
       image for specific node groups.
 

docs/operations.md

@@ -44,7 +44,7 @@ See [Reconfiguring Slurm](#Reconfiguring-Slurm) to apply changes.
 
 # Modifying Slurm Partition-specific Configuration
 
-Modify the `openhpc_slurm_partitions` mapping usually in `enviroments/$SITE_ENV/inventory/group_vars/all/openhpc.yml` as described for [stackhpc.openhpc:slurmconf](https://github.com/stackhpc/ansible-role-openhpc#slurmconf) (note the relevant version of this role is defined in the `requirements.yml`)
+Modify the `openhpc_slurm_partitions` mapping usually in `environments/$SITE_ENV/inventory/group_vars/all/openhpc.yml` as described for [stackhpc.openhpc:slurmconf](https://github.com/stackhpc/ansible-role-openhpc#slurmconf) (note the relevant version of this role is defined in the `requirements.yml`)
 
 Note an Ansible inventory group for the partition is required. This is generally auto-defined by a template in the OpenTofu configuration.
 

environments/common/inventory/group_vars/all/defaults.yml

@@ -42,7 +42,7 @@ appliances_local_users_default:
         home: /var/lib/{{ appliances_local_users_ansible_user_name }}
         move_home: true
         local: true
-    
+
     - user: "{{ appliances_local_users_podman }}"
       enable: "{{ 'podman' in group_names }}"
 
@@ -53,7 +53,7 @@ appliances_local_users_default:
         shell: /sbin/nologin
         uid: 202
         system: true
-    
+
     - group:
         name: prometheus
         gid: 976
@@ -64,7 +64,7 @@ appliances_local_users_default:
         shell: /usr/sbin/nologin
         system: true
       enable: "{{ 'prometheus' in group_names }}"
-    
+
     - group:
         name: grafana
         gid: 979
@@ -79,7 +79,7 @@ appliances_local_users_default:
 
 # Overide this to add extra users whilst keeping the defaults.
 appliances_local_users_extra: [] # see format of appliances_local_users_default above
-appliances_local_users: "{{ appliances_local_users_default + appliances_local_users_extra }}"
+appliances_local_users: "{{ (appliances_local_users_default + appliances_local_users_extra) | select | list }}"
 
 ################## bootstrap: extra package installs ######################################
 
@@ -94,7 +94,7 @@ appliances_extra_packages_default:
  - postfix
  - git
  - "{{ 'python36' if ansible_distribution_version == '8.9' else 'python312' }}"
- 
+
 appliances_extra_packages_other: []
-  
-appliances_extra_packages: "{{ appliances_extra_packages_default + appliances_extra_packages_other }}"
+
+appliances_extra_packages: "{{ (appliances_extra_packages_default + appliances_extra_packages_other) | select | list }}"

environments/common/inventory/group_vars/all/openhpc.yml

@@ -28,7 +28,7 @@ openhpc_packages_default:
   - apptainer
   - podman-compose
 openhpc_packages_extra: []
-openhpc_packages: "{{ openhpc_packages_default + openhpc_packages_extra }}"
+openhpc_packages: "{{ (openhpc_packages_default + openhpc_packages_extra) | select | list }}"
 openhpc_munge_key: "{{ vault_openhpc_mungekey | b64decode }}"
 openhpc_login_only_nodes: login
 openhpc_config_default: