Merge branch 'main' into bandwidth-test

Author: claudia-lola

.ansible-lint.yml

@@ -6,8 +6,6 @@ skip_list:
   - jinja[spacing]
   - galaxy[no-changelog]
   - meta-runtime[unsupported-version]
-
-warn_list:
   - name[missing]
   - name[play]
   - var-naming

.github/workflows/fatimage.yml

@@ -36,10 +36,10 @@ jobs:
         build:
           - image_name: openhpc-RL8
             source_image_name: Rocky-8-GenericCloud-Base-8.10-20240528.0.x86_64.raw
-            inventory_groups: control,compute,login,update
+            inventory_groups: fatimage
           - image_name: openhpc-RL9
             source_image_name: Rocky-9-GenericCloud-Base-9.6-20250531.0.x86_64.qcow2
-            inventory_groups: control,compute,login,update
+            inventory_groups: fatimage
     env:
       ANSIBLE_FORCE_COLOR: True
       OS_CLOUD: openstack
@@ -118,6 +118,11 @@ jobs:
           . venv/bin/activate
           openstack image unset --property signature_verified "${{ steps.manifest.outputs.image-id }}" || true
 
+      - name: Set image properties
+        run: |
+          . venv/bin/activate
+          . dev/image-set-properties.sh "${{ steps.manifest.outputs.image-id }}"
+
       - name: Upload manifest artifact
         uses: actions/upload-artifact@v4
         with:

.github/workflows/main.yml

@@ -143,7 +143,6 @@ jobs:
     name: Trivy scan image for vulnerabilities
     needs: files_changed
     if: |
-      github.event_name == 'pull_request' &&
       needs.files_changed.outputs.trivyscan == 'true'
     uses: ./.github/workflows/trivyscan.yml
     secrets: inherit

.github/workflows/nightly-cleanup.yml

@@ -46,53 +46,20 @@ jobs:
           echo "${{ secrets[format('{0}_CLOUDS_YAML', env.CI_CLOUD)] }}" > ~/.config/openstack/clouds.yaml
         shell: bash
 
-      - name: Find CI clusters
+      - name: Delete all CI clusters
         run: |
           . venv/bin/activate
-          CI_CLUSTERS=$(openstack server list | grep --only-matching 'slurmci-RL.-[0-9]\+'  | sort | uniq || true)
-          echo "DEBUG: Raw CI clusters: $CI_CLUSTERS"
-      
-          if [[ -z "$CI_CLUSTERS" ]]; then
-            echo "No matching CI clusters found."
-          else
-            # Flatten multiline value so can be passed as env var
-            CI_CLUSTERS_FORMATTED=$(echo "$CI_CLUSTERS" | tr '\n' ' ' | sed 's/ $//')
-            echo "DEBUG: Formatted CI clusters: $CI_CLUSTERS_FORMATTED"
-            echo "ci_clusters=$CI_CLUSTERS_FORMATTED" >> "$GITHUB_ENV"
-          fi
+          ./dev/delete-cluster.py slurmci-RL --force
         shell: bash
-      
-      - name: Delete CI clusters
+
+      - name: Delete all CI extra build VMs and volumes
         run: |
           . venv/bin/activate
-          if [[ -z ${ci_clusters} ]]; then
-            echo "No clusters to delete."
-            exit 0
-          fi
-
-          for cluster_prefix in ${ci_clusters}
-          do
-            echo "Processing cluster: $cluster_prefix"
-
-            # Get all servers with the matching name for control node
-            CONTROL_SERVERS=$(openstack server list --name "${cluster_prefix}-control" --format json)
-
-            # Get unique server names to avoid duplicate cleanup
-            UNIQUE_NAMES=$(echo "$CONTROL_SERVERS" | jq -r '.[].Name' | sort | uniq)
-            for name in $UNIQUE_NAMES; do
-              echo "Deleting cluster with control node: $name"
-
-              # Get the first matching server ID by name
-              server=$(echo "$CONTROL_SERVERS" | jq -r '.[] | select(.Name=="'"$name"'") | .ID' | head -n1)
-
-              # Make sure server still exists (wasn't deleted earlier)
-              if ! openstack server show "$server" &>/dev/null; then
-                echo "Server $server no longer exists, skipping $name."
-                continue
-              fi
+          ./dev/delete-cluster.py openhpc-extra-RL --force
+        shell: bash
 
-              echo "Deleting cluster $cluster_prefix (server $server)..."
-              ./dev/delete-cluster.py "$cluster_prefix" --force
-            done
-          done
+      - name: Delete all fatimage build VMs and volumes
+        run: |
+          . venv/bin/activate
+          ./dev/delete-cluster.py openhpc-RL --force
         shell: bash

.github/workflows/s3-image-sync.yml

@@ -168,6 +168,11 @@ jobs:
           . venv/bin/activate
           bash .github/bin/get-s3-image.sh ${{ env.TARGET_IMAGE }} ${{ env.S3_BUCKET }}
 
+      - name: Set Glance image properties correctly for Slurm images
+        run: |
+          . venv/bin/activate
+          . dev/image-set-properties.sh "${{ env.TARGET_IMAGE }}"
+
       - name: Cleanup OpenStack Image (on error or cancellation)
         if: cancelled() || failure()
         run: |

.github/workflows/stackhpc.yml

@@ -107,7 +107,12 @@ jobs:
           . venv/bin/activate
           . environments/.stackhpc/activate
           cd "$STACKHPC_TF_DIR"
-          tofu apply -auto-approve -var-file="${{ env.CI_CLOUD }}.tfvars"
+          max_retries=3
+          delay=30
+          for i in $(seq 1 $max_retries); do
+            tofu apply -auto-approve -var-file="${{ env.CI_CLOUD }}.tfvars" && break
+            [ "$i" -lt "$max_retries" ] && sleep $delay || exit 1
+          done
 
       - name: Delete infrastructure if provisioning failed
         run: |

.github/workflows/trivyscan.yml

@@ -102,7 +102,7 @@ jobs:
         run: sudo guestmount -a /mnt/images/${{ steps.manifest.outputs.image-name }}.qcow2 -i --ro -o allow_other './${{ steps.manifest.outputs.image-name }}'
 
       - name: Run Trivy vulnerability scanner
-        uses: aquasecurity/trivy-action@0.24.0
+        uses: aquasecurity/trivy-action@v0.33.1
         with:
           scan-type: fs
           scan-ref: "${{ steps.manifest.outputs.image-name }}"
@@ -116,13 +116,13 @@ jobs:
           TRIVY_DB_REPOSITORY: ghcr.io/azimuth-cloud/trivy-db:2
 
       - name: Upload Trivy scan results to GitHub Security tab
-        uses: github/codeql-action/upload-sarif@v3
+        uses: github/codeql-action/upload-sarif@v4
         with:
           sarif_file: "${{ steps.manifest.outputs.image-name }}.sarif"
           category: "${{ matrix.build }}"
 
       - name: Fail if scan has CRITICAL vulnerabilities
-        uses: aquasecurity/trivy-action@0.24.0
+        uses: aquasecurity/trivy-action@v0.33.1
         with:
           scan-type: fs
           scan-ref: "${{ steps.manifest.outputs.image-name }}"
@@ -132,6 +132,8 @@ jobs:
           severity: 'CRITICAL'
           ignore-unfixed: true
           timeout: 15m
+          # On a subsequent call to the action we know trivy is already installed so can skip this
+          skip-setup-trivy: true
         env:
           GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}
           TRIVY_DB_REPOSITORY: ghcr.io/azimuth-cloud/trivy-db:2

ansible/adhoc/sync-pulp.yml

@@ -5,7 +5,5 @@
         name: pulp_site
         tasks_from: sync.yml
       vars:
-        pulp_site_target_arch: "x86_64"
-        pulp_site_target_distribution: "rocky"
         # default distribution to *latest* specified for baseos repo:
         pulp_site_target_distribution_version: "{{ dnf_repos_repos['baseos'].keys() | map('float') | sort | last }}"

ansible/fatimage.yml

@@ -117,7 +117,7 @@
     - name: Install OpenHPC
       ansible.builtin.import_role:
         name: stackhpc.openhpc
-        tasks_from: install.yml
+        tasks_from: install-ohpc.yml
       when: "'openhpc' in group_names"
 
     # - import_playbook: portal.yml
@@ -206,6 +206,7 @@
       ansible.builtin.include_role:
         name: hpctests
         tasks_from: source-hpl.yml
+      when: "'hpctests' in group_names"
 
 - hosts: prometheus
   become: true

ansible/roles/cuda/defaults/main.yml

@@ -2,13 +2,17 @@
 # yamllint disable-line rule:line-length
 cuda_repo_url: "https://developer.download.nvidia.com/compute/cuda/repos/rhel{{ ansible_distribution_major_version }}/{{ ansible_architecture }}/cuda-rhel{{ ansible_distribution_major_version }}.repo"
 cuda_nvidia_driver_stream: '580-open'
-cuda_nvidia_driver_pkg: "nvidia-open-3:580.82.07-1.el{{ ansible_distribution_major_version }}"
+cuda_nvidia_driver_version: '580.82.07-1'
+cuda_nvidia_driver_pkg: "nvidia-open-3:{{ cuda_nvidia_driver_version }}.el{{ ansible_distribution_major_version }}"
 cuda_package_version: '13.0.1-1'
 cuda_version_short: "{{ (cuda_package_version | split('.'))[0:2] | join('.') }}" # major.minor
-cuda_packages:
+cuda_packages_default:
   - "cuda-toolkit-{{ cuda_package_version }}"
   - nvidia-gds
   - cmake
+cuda_packages_fabricmanager:
+  - "nvidia-fabricmanager-{{ cuda_nvidia_driver_version }}"
+cuda_packages: "{{ cuda_packages_default + ( cuda_packages_fabricmanager if cuda_install_nvidiafabricmanger | bool else [] ) }}"
 cuda_samples_release_url: "https://github.com/NVIDIA/cuda-samples/archive/refs/tags/v{{ cuda_version_short }}.tar.gz"
 cuda_samples_path: "/var/lib/{{ ansible_user }}/cuda_samples"
 cuda_samples_programs: