Merge branch 'main' into feature/k3s-ansible-init

Author: wtripp180901

.github/workflows/fatimage.yml

@@ -1,39 +1,51 @@
 
 name: Build fat image
-'on':
+on:
   workflow_dispatch:
-concurrency:
-  group: ${{ github.ref }}-{{ matrix.os_version }}-{{ matrix.build }} # to branch/PR + OS + build
-  cancel-in-progress: true
+    inputs:
+      ci_cloud:
+        description: 'Select the CI_CLOUD'
+        required: true
+        type: choice
+        options:
+          - LEAFCLOUD
+          - SMS
+          - ARCUS
 jobs:
   openstack:
     name: openstack-imagebuild
+    concurrency:
+      group: ${{ github.workflow }}-${{ github.ref }}-${{ matrix.os_version }}-${{ matrix.build }} # to branch/PR + OS + build
+      cancel-in-progress: true
     runs-on: ubuntu-22.04
     strategy:
-      matrix:
+      fail-fast: false # allow other matrix jobs to continue even if one fails
+      matrix: # build RL8+OFED, RL9+OFED, RL9+OFED+CUDA versions
         os_version:
           - RL8
           - RL9
         build:
-          - openstack.openhpc
           - openstack.openhpc-ofed
+          - openstack.openhpc-cuda
         exclude:
           - os_version: RL8
-            build: openstack.openhpc-ofed
-          - os_version: RL9
-            build: openstack.openhpc
+            build: openstack.openhpc-cuda
     env:
       ANSIBLE_FORCE_COLOR: True
       OS_CLOUD: openstack
-      CI_CLOUD: ${{ vars.CI_CLOUD }}
+      CI_CLOUD: ${{ github.event.inputs.ci_cloud }}
     steps:
       - uses: actions/checkout@v2
 
+      - name: Record settings for CI cloud
+        run: |
+          echo CI_CLOUD: ${{ env.CI_CLOUD }}
+
       - name: Setup ssh
         run: |
           set -x
           mkdir ~/.ssh
-          echo "${{ secrets[format('{0}_SSH_KEY', vars.CI_CLOUD)] }}" > ~/.ssh/id_rsa
+          echo "${{ secrets[format('{0}_SSH_KEY', env.CI_CLOUD)] }}" > ~/.ssh/id_rsa
           chmod 0600 ~/.ssh/id_rsa
         shell: bash
 
@@ -47,7 +59,7 @@ jobs:
       - name: Write clouds.yaml
         run: |
           mkdir -p ~/.config/openstack/
-          echo "${{ secrets[format('{0}_CLOUDS_YAML', vars.CI_CLOUD)] }}" > ~/.config/openstack/clouds.yaml
+          echo "${{ secrets[format('{0}_CLOUDS_YAML', env.CI_CLOUD)] }}" > ~/.config/openstack/clouds.yaml
         shell: bash
 
       - name: Setup environment
@@ -62,7 +74,7 @@ jobs:
           . environments/.stackhpc/activate
           cd packer/
           packer init .
-          PACKER_LOG=1 packer build -on-error=${{ vars.PACKER_ON_ERROR }} -only=${{ matrix.build }} -var-file=$PKR_VAR_environment_root/${{ vars.CI_CLOUD }}.pkrvars.hcl openstack.pkr.hcl
+          PACKER_LOG=1 packer build -on-error=${{ vars.PACKER_ON_ERROR }} -only=${{ matrix.build }} -var-file=$PKR_VAR_environment_root/${{ env.CI_CLOUD }}.pkrvars.hcl openstack.pkr.hcl
         env:
           PKR_VAR_os_version: ${{ matrix.os_version }}
 
@@ -81,7 +93,9 @@ jobs:
       - name: Download image
         run: |
           . venv/bin/activate
-          openstack image save --file ${{ steps.manifest.outputs.image-name }}.qcow2 ${{ steps.manifest.outputs.image-name }}
+          sudo mkdir /mnt/images
+          sudo chmod 777 /mnt/images
+          openstack image save --file /mnt/images/${{ steps.manifest.outputs.image-name }}.qcow2 ${{ steps.manifest.outputs.image-name }}
 
       - name: Set up QEMU
         uses: docker/setup-qemu-action@v3
@@ -95,13 +109,13 @@ jobs:
         run: sudo mkdir -p './${{ steps.manifest.outputs.image-name }}'
 
       - name: mount qcow2 file
-        run: sudo guestmount -a ${{ steps.manifest.outputs.image-name }}.qcow2 -i --ro -o allow_other './${{ steps.manifest.outputs.image-name }}'
+        run: sudo guestmount -a /mnt/images/${{ steps.manifest.outputs.image-name }}.qcow2 -i --ro -o allow_other './${{ steps.manifest.outputs.image-name }}'
 
       - name: Run Trivy vulnerability scanner
         uses: aquasecurity/[email protected]
         with:
           scan-type: fs
-          scan-ref: "./${{ steps.manifest.outputs.image-name }}"
+          scan-ref: "${{ steps.manifest.outputs.image-name }}"
           scanners: "vuln"
           format: sarif
           output: "${{ steps.manifest.outputs.image-name }}.sarif"
@@ -117,7 +131,7 @@ jobs:
         uses: aquasecurity/[email protected]
         with:
           scan-type: fs
-          scan-ref: "./${{ steps.manifest.outputs.image-name }}"
+          scan-ref: "${{ steps.manifest.outputs.image-name }}"
           scanners: "vuln"
           format: table
           exit-code: '1'

.github/workflows/stackhpc.yml

@@ -2,75 +2,92 @@
 name: Test deployment and reimage on OpenStack
 on:
   workflow_dispatch:
-    inputs:
-      use_RL8:
-        required: true
-        description: Include RL8 tests
-        type: boolean
-        default: false
   push:
     branches:
       - main
+    paths:
+      - '**'
+      - '!dev/**'
+      - 'dev/setup-env.sh'
+      - '!docs/**'
+      - '!README.md'
+      - '!.gitignore'
   pull_request:
+    paths:
+      - '**'
+      - '!dev/**'
+      - 'dev/setup-env.sh'
+      - '!docs/**'
+      - '!README.md'
+      - '!.gitignore'
 jobs:
   openstack:
     name: openstack-ci
-    concurrency: ${{ github.ref }}-{{ matrix.os_version }} # to branch/PR + OS
+    concurrency:
+      group: ${{ github.workflow }}-${{ github.ref }}-${{ matrix.os_version }} # to branch/PR + OS
+      cancel-in-progress: true
     runs-on: ubuntu-22.04
     strategy:
+      fail-fast: false # allow other matrix jobs to continue even if one fails
       matrix:
-        os_version: [RL8, RL9]
-        rl8_selected:
-          - ${{ inputs.use_RL8 == true }} # only potentially true for workflow_dispatch
-        rl8_branch:
-          - ${{ startsWith(github.head_ref, 'rl8') == true }} # only potentially for pull_request, always false on merge
-        rl8_label:
-          - ${{ contains(github.event.pull_request.labels.*.name, 'RL8') }} # NB: needs a new commit if added after PR created
-        exclude:
-          - os_version: RL8
-            rl8_selected: false
-            rl8_branch: false
-            rl8_label: false
+        os_version:
+          - RL8
+          - RL9
     env:
       ANSIBLE_FORCE_COLOR: True
       OS_CLOUD: openstack
       TF_VAR_cluster_name: slurmci-${{ matrix.os_version }}-${{ github.run_number }}
-      CI_CLOUD: ${{ vars.CI_CLOUD }}
+      CI_CLOUD: ${{ vars.CI_CLOUD }} # default from repo settings
+      TF_VAR_os_version: ${{ matrix.os_version }}
     steps:
       - uses: actions/checkout@v2
 
+      - name: Override CI_CLOUD if PR label is present
+        if: ${{ github.event_name == 'pull_request' }}
+        run: |
+          # Iterate over the labels
+          labels=$(echo '${{ toJSON(github.event.pull_request.labels) }}' | jq -r '.[].name')
+          echo $labels
+          for label in $labels; do
+             if [[ $label == CI_CLOUD=* ]]; then
+              # Extract the value after 'CI_CLOUD='
+              CI_CLOUD_OVERRIDE=${label#CI_CLOUD=}
+              echo "CI_CLOUD=${CI_CLOUD_OVERRIDE}" >> $GITHUB_ENV
+            fi
+          done
+
       - name: Record settings for CI cloud
         run: |
-          echo CI_CLOUD: ${{ vars.CI_CLOUD }}
+          echo CI_CLOUD: ${{ env.CI_CLOUD }}
 
       - name: Setup ssh
         run: |
           set -x
           mkdir ~/.ssh
-          echo "${{ secrets[format('{0}_SSH_KEY', vars.CI_CLOUD)] }}" > ~/.ssh/id_rsa
+          echo "${{ secrets[format('{0}_SSH_KEY', env.CI_CLOUD)] }}" > ~/.ssh/id_rsa
           chmod 0600 ~/.ssh/id_rsa
         shell: bash
-        
+
       - name: Add bastion's ssh key to known_hosts
         run: cat environments/.stackhpc/bastion_fingerprints >> ~/.ssh/known_hosts
         shell: bash
-      
+
       - name: Install ansible etc
         run: dev/setup-env.sh
 
       - name: Install OpenTofu
         uses: opentofu/setup-opentofu@v1
         with:
           tofu_version: 1.6.2
-      
+
       - name: Initialise terraform
         run: terraform init
         working-directory: ${{ github.workspace }}/environments/.stackhpc/terraform
-        
+
       - name: Write clouds.yaml
         run: |
           mkdir -p ~/.config/openstack/
-          echo "${{ secrets[format('{0}_CLOUDS_YAML', vars.CI_CLOUD)] }}" > ~/.config/openstack/clouds.yaml
+          echo "${{ secrets[format('{0}_CLOUDS_YAML', env.CI_CLOUD)] }}" > ~/.config/openstack/clouds.yaml
         shell: bash
 
       - name: Setup environment-specific inventory/terraform inputs
@@ -88,19 +105,15 @@ jobs:
           . venv/bin/activate
           . environments/.stackhpc/activate
           cd $APPLIANCES_ENVIRONMENT_ROOT/terraform
-          terraform apply -auto-approve -var-file="${{ vars.CI_CLOUD }}.tfvars"
-        env:
-          TF_VAR_os_version: ${{ matrix.os_version }}
+          terraform apply -auto-approve -var-file="${{ env.CI_CLOUD }}.tfvars"
 
       - name: Delete infrastructure if provisioning failed
         run: |
           . venv/bin/activate
           . environments/.stackhpc/activate
           cd $APPLIANCES_ENVIRONMENT_ROOT/terraform
-          terraform destroy -auto-approve -var-file="${{ vars.CI_CLOUD }}.tfvars"
+          terraform destroy -auto-approve -var-file="${{ env.CI_CLOUD }}.tfvars"
         if: failure() && steps.provision_servers.outcome == 'failure'
-        env:
-          TF_VAR_os_version: ${{ matrix.os_version }}
 
       - name: Configure cluster
         run: |
@@ -126,14 +139,14 @@ jobs:
         run: |
           . venv/bin/activate
           . environments/.stackhpc/activate
-          
+
           # load ansible variables into shell:
           ansible-playbook ansible/ci/output_vars.yml \
             -e output_vars_hosts=openondemand \
             -e output_vars_path=$APPLIANCES_ENVIRONMENT_ROOT/vars.txt \
             -e output_vars_items=bastion_ip,bastion_user,openondemand_servername
           source $APPLIANCES_ENVIRONMENT_ROOT/vars.txt
-          
+
           # setup ssh proxying:
           sudo apt-get --yes install proxychains
           echo proxychains installed
@@ -170,7 +183,7 @@ jobs:
       #     ansible login -v -a "sudo scontrol reboot ASAP nextstate=RESUME reason='rebuild image:${{ steps.packer_build.outputs.NEW_COMPUTE_IMAGE_ID }}' ${TF_VAR_cluster_name}-compute-[0-3]"
       #     ansible compute -m wait_for_connection -a 'delay=60 timeout=600' # delay allows node to go down
       #     ansible-playbook -v ansible/ci/check_slurm.yml
-      
+
       - name: Test reimage of login and control nodes (via rebuild adhoc)
         run: |
           . venv/bin/activate
@@ -179,7 +192,7 @@ jobs:
           ansible all -m wait_for_connection -a 'delay=60 timeout=600' # delay allows node to go down
           ansible-playbook -v ansible/site.yml
           ansible-playbook -v ansible/ci/check_slurm.yml
-      
+
       - name: Check sacct state survived reimage
         run: |
           . venv/bin/activate
@@ -197,10 +210,8 @@ jobs:
           . venv/bin/activate
           . environments/.stackhpc/activate
           cd $APPLIANCES_ENVIRONMENT_ROOT/terraform
-          terraform destroy -auto-approve -var-file="${{ vars.CI_CLOUD }}.tfvars"
+          terraform destroy -auto-approve -var-file="${{ env.CI_CLOUD }}.tfvars"
         if: ${{ success() || cancelled() }}
-        env:
-          TF_VAR_os_version: ${{ matrix.os_version }}
 
       # - name: Delete images
       #   run: |

ansible/bootstrap.yml

@@ -227,24 +227,25 @@
     - update
   tasks:
     - name: Check for pending reboot from package updates
-      stat: 
-        path: /var/run/reboot-required
+      command:
+        cmd: dnf needs-restarting -r
       register: update_reboot_required
-    - debug:
-        msg: "setstatus:{{ (sestatus.reboot_required | default(false)) }} packages: {{ (update_reboot_required.stat.exists | bool) }}"
-    - name: Reboot if required from SELinux state change or package upgrades
+      failed_when: "update_reboot_required.rc not in [0, 1]"
+      changed_when: false
+    - name: Reboot to cover SELinux state change or package upgrades
       reboot:
         post_reboot_delay: 30
-      when: (sestatus['reboot_required'] | default(false)) or (update_reboot_required.stat.exists | bool)
+      when: (sestatus['reboot_required'] | default(false)) or (update_reboot_required.rc == 1)
     - name: Wait for hosts to be reachable
       wait_for_connection:
         sleep: 15
-    - name: update facts
+    - name: Clear facts
+      meta: clear_facts
+    - name: Update facts
       setup:
-      when: (sestatus.changed | default(false)) or (sestatus.reboot_required | default(false))
 
 - hosts: ofed
-  gather_facts: no
+  gather_facts: yes
   become: yes
   tags: ofed
   tasks:

ansible/extras.yml

@@ -21,7 +21,7 @@
 - name: Setup CUDA
   hosts: cuda
   become: yes
-  gather_facts: no
+  gather_facts: yes
   tags: cuda
   tasks:
     - import_role:

ansible/roles/cuda/defaults/main.yml

@@ -1,11 +1,12 @@
-cuda_distro: rhel8
+cuda_distro: "rhel{{ ansible_distribution_major_version }}"
 cuda_repo: "https://developer.download.nvidia.com/compute/cuda/repos/{{ cuda_distro }}/x86_64/cuda-{{ cuda_distro }}.repo"
 cuda_driver_stream: default
+cuda_package_version: 'latest'
 cuda_packages:
-  - cuda
+  - "cuda{{ ('-' + cuda_package_version) if cuda_package_version != 'latest' else '' }}"
   - nvidia-gds
 # _cuda_version_tuple: # discovered from installed package e.g. ('12', '1', '0')
-cuda_version_short: "{{ _cuda_version_tuple[0] }}.{{ cuda_version_tuple[1] }}"
+cuda_version_short: "{{ _cuda_version_tuple[0] }}.{{ _cuda_version_tuple[1] }}"
 cuda_samples_release_url: "https://github.com/NVIDIA/cuda-samples/archive/refs/tags/v{{ cuda_version_short }}.tar.gz"
 cuda_samples_path: "/home/{{ ansible_user }}/cuda_samples"
 cuda_samples_programs:

ansible/roles/cuda/tasks/main.yml

@@ -24,22 +24,13 @@
   failed_when: false
   register: _cuda_driver_module_enabled
 
-- name: List nvidia driver dnf module stream versions
-  shell:
-    cmd: dnf module list nvidia-driver | grep -oP "\d+-dkms" | sort -V
-  # Output of interest from command is something like (some whitespace removed):
-  # "nvidia-driver 418-dkms   default [d], fm, ks    Nvidia driver for 418-dkms branch "
-  changed_when: false
-  register: _cuda_driver_module_streams
-  when: "'No matching Modules to list' in _cuda_driver_module_enabled.stderr"
-
 - name: Enable nvidia driver module
-  ansible.builtin.command: "dnf module enable -y nvidia-driver:{{ _cuda_driver_module_streams.stdout_lines | last }}"
+  ansible.builtin.command: "dnf module enable -y nvidia-driver:open-dkms"
   register: _cuda_driver_module_enable
   when: "'No matching Modules to list' in _cuda_driver_module_enabled.stderr"
   changed_when: "'Nothing to do' not in _cuda_driver_module_enable.stdout"
 
-- name: Install nvidia drivers # TODO: make removal possible?
+- name: Install nvidia drivers
   ansible.builtin.command: dnf module install -y nvidia-driver
   register: _cuda_driver_install
   when: "'No matching Modules to list' in _cuda_driver_module_enabled.stderr"