Merge remote-tracking branch 'origin/main' into HEAD

Author: jovial

.github/workflows/fatimage.yml

@@ -10,6 +10,11 @@ on:
             - LEAFCLOUD
             - SMS
             - ARCUS
+        cleanup_on_failure:
+          description: Cleanup Packer resources on failure
+          type: boolean
+          required: true
+          default: true
 
 jobs:
   openstack:
@@ -78,7 +83,7 @@ jobs:
           packer init .
 
           PACKER_LOG=1 packer build \
-          -on-error=${{ vars.PACKER_ON_ERROR }} \
+          -on-error=${{ github.event.inputs.cleanup_on_failure && 'cleanup' || 'abort' }} \
           -var-file=$PKR_VAR_environment_root/${{ env.CI_CLOUD }}.pkrvars.hcl \
           -var "source_image_name=${{ matrix.build.source_image_name }}" \
           -var "image_name=${{ matrix.build.image_name }}" \

.github/workflows/nightly-cleanup.yml

@@ -59,10 +59,34 @@ jobs:
       - name: Delete CI clusters
         run: |
           . venv/bin/activate
-          if [[ -z "${ci_clusters}" ]]; then
+          if [[ -z ${ci_clusters} ]]; then
             echo "No clusters to delete."
             exit 0
           fi
-          echo "Deleting clusters: ${ci_clusters}"
-          ./dev/delete-cluster.py ${ci_clusters} --force
+
+          for cluster_prefix in ${ci_clusters}
+          do
+            echo "Processing cluster: $cluster_prefix"
+
+            # Get all servers with the matching name for control node
+            CONTROL_SERVERS=$(openstack server list --name ${cluster_prefix}-control --format json)
+
+            # Get unique server names to avoid duplicate cleanup
+            UNIQUE_NAMES=$(echo "$CONTROL_SERVERS" | jq -r '.[].Name' | sort | uniq)
+            for name in $UNIQUE_NAMES; do
+              echo "Deleting cluster with control node: $name"
+
+              # Get the first matching server ID by name
+              server=$(echo "$CONTROL_SERVERS" | jq -r '.[] | select(.Name=="'"$name"'") | .ID' | head -n1)
+
+              # Make sure server still exists (wasn't deleted earlier)
+              if ! openstack server show "$server" &>/dev/null; then
+                echo "Server $server no longer exists, skipping $name."
+                continue
+              fi
+
+              echo "Deleting cluster $cluster_prefix (server $server)..."
+              ./dev/delete-cluster.py $cluster_prefix --force
+            done
+          done
         shell: bash

.github/workflows/release-image.yml

@@ -0,0 +1,37 @@
+name: Release images
+on:
+  workflow_dispatch:
+  release:
+    types:
+      - published # should work for both pre-releases and releases
+env:
+  IMAGE_PATH: environments/.stackhpc/tofu/cluster_image.auto.tfvars.json
+jobs:
+  ci-image-release:
+    name: ci-image-release
+    runs-on: ubuntu-22.04
+    concurrency: ${{ github.workflow }}-${{ github.ref }}
+    strategy:
+      fail-fast: false
+      matrix:
+        build:
+          - RL8
+          - RL9
+    steps:
+      - uses: actions/checkout@v2
+
+      - name: Write s3cmd configuration
+        run: echo "${{ secrets.ARCUS_S3_CFG }}" > ~/.s3cfg
+
+      - name: Install s3cmd
+        run: |
+          sudo apt-get update
+          sudo apt-get --yes install s3cmd
+      
+      - name: Retrieve image name
+        run: |
+          TARGET_IMAGE=$(jq --arg version "${{ matrix.build }}" -r '.cluster_image[$version]' "${{ env.IMAGE_PATH }}")
+          echo "TARGET_IMAGE=${TARGET_IMAGE}" >> "$GITHUB_ENV"
+      
+      - name: Copy image from pre-release to release bucket
+        run: s3cmd cp s3://openhpc-images-prerelease/${{ env.TARGET_IMAGE }} s3://openhpc-images

.github/workflows/stackhpc.yml

@@ -91,9 +91,9 @@ jobs:
         run: dev/setup-env.sh
 
       - name: Install OpenTofu
-        uses: opentofu/setup-opentofu@v1
+        uses: opentofu/setup-opentofu@v1.0.5
         with:
-          tofu_version: 1.6.2
+          tofu_version: 1.9.0
 
       - name: Initialise tofu
         run: tofu init
@@ -230,6 +230,16 @@ jobs:
         env:
           DEMO_USER_PASSWORD: ${{ secrets.TEST_USER_PASSWORD }}
 
+      - name: Delete possible volume snapshot from slurm upgrade
+        run: |
+          . venv/bin/activate
+          . environments/.stackhpc/activate  
+          if [ -n "$SNAPSHOT" ]
+          then
+              echo Deleting $SNAPSHOT
+              openstack volume snapshot delete $SNAPSHOT
+          fi
+
       - name: Delete infrastructure
         run: |
           . venv/bin/activate

ansible/disable-repos.yml

@@ -5,3 +5,4 @@
       ansible.builtin.include_role:
         name: dnf_repos
         tasks_from: disable_repos.yml
+      when: not dnf_repos_enabled | default(false) | bool

ansible/roles/cluster_infra/templates/outputs.tf.j2

@@ -32,12 +32,12 @@ output "cluster_nodes" {
         }
       }
     ],
-    {% for partition in openhpc_slurm_partitions %}
+    {% for nodegroup in openhpc_nodegroups %}
     [
-      for compute in openstack_compute_instance_v2.{{ partition.name }}: {
+      for compute in openstack_compute_instance_v2.{{ nodegroup.name }}: {
         name          = compute.name
         ip            = compute.network[0].fixed_ip_v4
-        groups        = ["compute", "{{ cluster_name }}_compute", "{{ cluster_name }}_{{ partition.name }}"],
+        groups        = ["compute", "{{ cluster_name }}_compute", "{{ cluster_name }}_{{ nodegroup.name }}"],
         facts  = {
           openstack_project_id = data.openstack_identity_auth_scope_v3.scope.project_id
         }

ansible/roles/cluster_infra/templates/resources.tf.j2

@@ -282,11 +282,11 @@ resource "openstack_networking_port_v2" "control_storage" {
 ###
 # Workers
 ###
-{% for partition in openhpc_slurm_partitions %}
+{% for nodegroup in openhpc_nodegroups %}
 # Primary network
-resource "openstack_networking_port_v2" "{{ partition.name }}" {
-  count = {{ partition.count }}
-  name = "{{ cluster_name }}-compute-{{ partition.name }}-${count.index}"
+resource "openstack_networking_port_v2" "{{ nodegroup.name }}" {
+  count = {{ nodegroup.count }}
+  name = "{{ cluster_name }}-compute-{{ nodegroup.name }}-${count.index}"
   network_id = "${data.openstack_networking_network_v2.cluster_network.id}"
   admin_state_up = "true"
 
@@ -305,9 +305,9 @@ resource "openstack_networking_port_v2" "{{ partition.name }}" {
 
 # Storage network
 {% if cluster_storage_network is defined %}
-resource "openstack_networking_port_v2" "{{ partition.name }}_storage" {
-  count          = {{ partition.count }}
-  name           = "{{ cluster_name }}-compute-{{ partition.name }}-storage-${count.index}"
+resource "openstack_networking_port_v2" "{{ nodegroup.name }}_storage" {
+  count          = {{ nodegroup.count }}
+  name           = "{{ cluster_name }}-compute-{{ nodegroup.name }}-storage-${count.index}"
   network_id     = data.openstack_networking_network_v2.cluster_storage.id
   admin_state_up = "true"
 
@@ -499,25 +499,25 @@ resource "openstack_compute_instance_v2" "control" {
   }
 }
 
-{% for partition in openhpc_slurm_partitions %}
-resource "openstack_compute_instance_v2" "{{ partition.name }}" {
-  count = {{ partition.count }}
+{% for nodegroup in openhpc_nodegroups %}
+resource "openstack_compute_instance_v2" "{{ nodegroup.name }}" {
+  count = {{ nodegroup.count }}
 
-  name      = "{{ cluster_name }}-compute-{{ partition.name }}-${count.index}"
+  name      = "{{ cluster_name }}-compute-{{ nodegroup.name }}-${count.index}"
   image_id  = "{{ cluster_image }}"
-  {% if 'flavor_name' in partition %}
-  flavor_name = "{{ partition.flavor_name }}"
+  {% if 'flavor_name' in nodegroup %}
+  flavor_name = "{{ nodegroup.flavor_name }}"
   {% else %}
-  flavor_id = "{{ partition.flavor }}"
+  flavor_id = "{{ nodegroup.flavor }}"
   {% endif %}
 
   network {
-    port = openstack_networking_port_v2.{{ partition.name }}[count.index].id
+    port = openstack_networking_port_v2.{{ nodegroup.name }}[count.index].id
   }
 
   {% if cluster_storage_network is defined %}
   network {
-    port = openstack_networking_port_v2.{{ partition.name }}_storage[count.index].id
+    port = openstack_networking_port_v2.{{ nodegroup.name }}_storage[count.index].id
   }
   {% endif %}
 

ansible/roles/dnf_repos/defaults/main.yml

@@ -8,11 +8,13 @@ dnf_repos_filenames:
     appstream: 'Rocky-AppStream'
     crb: 'Rocky-PowerTools'
     extras: 'Rocky-Extras'
+    grafana: 'grafana'
   '9':
     baseos: 'rocky'
     appstream: 'rocky'
     crb: 'rocky'
     extras: 'rocky-extras'
+    grafana: 'grafana'
 
 dnf_repos_version_filenames: "{{ dnf_repos_filenames[ansible_distribution_major_version] }}"
 
@@ -33,6 +35,9 @@ dnf_repos_default_repolist:
 - file: ceph
   name: Ceph
   base_url: "{{ dnf_repos_pulp_content_url }}/{{ appliances_pulp_repos.ceph[ansible_distribution_major_version] | appliances_repo_to_subpath }}"
+- file: "{{ dnf_repos_version_filenames.grafana }}"
+  name: grafana
+  base_url: "{{ dnf_repos_pulp_content_url }}/{{ appliances_pulp_repos.grafana[ansible_distribution_major_version] | appliances_repo_to_subpath }}"
 
 dnf_repos_openhpc_repolist:
 - name: OpenHPC

ansible/roles/hpctests/defaults/main.yml

@@ -1,5 +1,6 @@
 ---
 hpctests_user: "{{ ansible_user }}"
+hpctests_group: "{{ hpctests_user }}"
 hpctests_rootdir: "/home/{{ hpctests_user }}/hpctests"
 hpctests_pre_cmd: ''
 hpctests_pingmatrix_modules: [gnu12 openmpi4]

ansible/roles/hpctests/tasks/setup.yml

@@ -26,7 +26,7 @@
     path: "{{ hpctests_rootdir }}"
     state: directory
     owner: "{{ hpctests_user }}"
-    group: "{{ hpctests_user }}"
+    group: "{{ hpctests_group }}"
 
 - name: Set fact for UCX_NET_DEVICES
   set_fact: