Nightly Slurm CI Rocky update workflow (#440)

* Update openstack.pkr.hcl

* new image build workflow

* dynamically set packer vars from fatimage workflow

* remove openstack. prefix from image name

* echo image name

* make image_name var in packer config

* new changes

* fix merge changes

* temp workflow changes

* test nightly build

* change back fatimage workflow

* rename images built

* add update to builder group

* add update to fatimage build groups

* fatimage.yml fix

* move output image_name declaration into build blocks

* delete outdated nightly image

* test new fatimage build

* debug dnf remove cockpit

* --amend

* add cuda build back in

* cuda nightly build

* test cuda nightly builds

* test new fatimage build on SMS

* test image upploads across clouds

* test image uploads in separate workflow

* finish nightly build workflow

* fix image delete logic

* use azimuth-cloud trivy db mirror

* use GITHUB_TOKEN env

* test new fatimage build

* add final nightlybuilds workflow

* move trivy scan to separate workflow

* bump image and test new trivy scan

* fix artifact creation

* bump image and test trivy scan

* only run trivy scan on image bumps

* bump image to test trivy scan run condition

* bump cuda image

* bump image

* extend timeout for trivy scanning cuda image

* Run workflow on PR to main

* address PR comments

* fix source_image_name packer parse

* bump image

* additional PR comments

* bump image

Author: bertiethorpe

.github/workflows/fatimage.yml

@@ -1,16 +1,16 @@
-
 name: Build fat image
 on:
   workflow_dispatch:
-    inputs:
-      ci_cloud:
-        description: 'Select the CI_CLOUD'
-        required: true
-        type: choice
-        options:
-          - LEAFCLOUD
-          - SMS
-          - ARCUS
+      inputs:
+        ci_cloud:
+          description: 'Select the CI_CLOUD'
+          required: true
+          type: choice
+          options:
+            - LEAFCLOUD
+            - SMS
+            - ARCUS
+
 jobs:
   openstack:
     name: openstack-imagebuild
@@ -25,7 +25,7 @@ jobs:
           - RL8
           - RL9
         build:
-          - openstack.openhpc-ofed
+          - openstack.openhpc
           - openstack.openhpc-cuda
         exclude:
           - os_version: RL8
@@ -34,6 +34,18 @@ jobs:
       ANSIBLE_FORCE_COLOR: True
       OS_CLOUD: openstack
       CI_CLOUD: ${{ github.event.inputs.ci_cloud }}
+      SOURCE_IMAGES_MAP: |
+        {
+          "RL8": {
+            "openstack.openhpc": "rocky-latest-RL8",
+            "openstack.openhpc-cuda": "rocky-latest-cuda-RL8"
+          },
+          "RL9": {
+            "openstack.openhpc": "rocky-latest-RL9",
+            "openstack.openhpc-cuda": "rocky-latest-cuda-RL9"
+          }
+        }
+
     steps:
       - uses: actions/checkout@v2
 
@@ -52,10 +64,10 @@ jobs:
       - name: Add bastion's ssh key to known_hosts
         run: cat environments/.stackhpc/bastion_fingerprints >> ~/.ssh/known_hosts
         shell: bash
-      
+
       - name: Install ansible etc
         run: dev/setup-env.sh
-      
+
       - name: Write clouds.yaml
         run: |
           mkdir -p ~/.config/openstack/
@@ -66,17 +78,25 @@ jobs:
         run: |
           . venv/bin/activate
           . environments/.stackhpc/activate
-        
+
       - name: Build fat image with packer
         id: packer_build
         run: |
+          set -x
           . venv/bin/activate
           . environments/.stackhpc/activate
           cd packer/
           packer init .
-          PACKER_LOG=1 packer build -on-error=${{ vars.PACKER_ON_ERROR }} -only=${{ matrix.build }} -var-file=$PKR_VAR_environment_root/${{ env.CI_CLOUD }}.pkrvars.hcl openstack.pkr.hcl
+
+          PACKER_LOG=1 packer build \
+          -on-error=${{ vars.PACKER_ON_ERROR }} \
+          -only=${{ matrix.build }} \
+          -var-file=$PKR_VAR_environment_root/${{ env.CI_CLOUD }}.pkrvars.hcl \
+          -var "source_image_name=${{ env.SOURCE_IMAGE }}" \
+          openstack.pkr.hcl
         env:
           PKR_VAR_os_version: ${{ matrix.os_version }}
+          SOURCE_IMAGE: ${{ fromJSON(env.SOURCE_IMAGES_MAP)[matrix.os_version][matrix.build] }}
 
       - name: Get created image names from manifest
         id: manifest
@@ -87,53 +107,14 @@ jobs:
             sleep 5
           done
           IMAGE_NAME=$(openstack image show -f value -c name $IMAGE_ID)
-          echo "image-name=${IMAGE_NAME}" >> "$GITHUB_OUTPUT"
-          echo "image-id=$IMAGE_ID" >> "$GITHUB_OUTPUT"
-
-      - name: Download image
-        run: |
-          . venv/bin/activate
-          sudo mkdir /mnt/images
-          sudo chmod 777 /mnt/images
-          openstack image save --file /mnt/images/${{ steps.manifest.outputs.image-name }}.qcow2 ${{ steps.manifest.outputs.image-name }}
-
-      - name: Set up QEMU
-        uses: docker/setup-qemu-action@v3
-
-      - name: install libguestfs
-        run: |
-          sudo apt -y update
-          sudo apt -y install libguestfs-tools
-
-      - name: mkdir for mount
-        run: sudo mkdir -p './${{ steps.manifest.outputs.image-name }}'
-
-      - name: mount qcow2 file
-        run: sudo guestmount -a /mnt/images/${{ steps.manifest.outputs.image-name }}.qcow2 -i --ro -o allow_other './${{ steps.manifest.outputs.image-name }}'
-
-      - name: Run Trivy vulnerability scanner
-        uses: aquasecurity/[email protected]
-        with:
-          scan-type: fs
-          scan-ref: "${{ steps.manifest.outputs.image-name }}"
-          scanners: "vuln"
-          format: sarif
-          output: "${{ steps.manifest.outputs.image-name }}.sarif"
-          # turn off secret scanning to speed things up
-
-      - name: Upload Trivy scan results to GitHub Security tab
-        uses: github/codeql-action/upload-sarif@v3
-        with:
-          sarif_file: "${{ steps.manifest.outputs.image-name }}.sarif"
-          category: "${{ matrix.os_version }}-${{ matrix.build }}"
+          echo $IMAGE_ID > image-id.txt
+          echo $IMAGE_NAME > image-name.txt
 
-      - name: Fail if scan has CRITICAL vulnerabilities
-        uses: aquasecurity/[email protected]
+      - name: Upload manifest artifact
+        uses: actions/upload-artifact@v4
         with:
-          scan-type: fs
-          scan-ref: "${{ steps.manifest.outputs.image-name }}"
-          scanners: "vuln"
-          format: table
-          exit-code: '1'
-          severity: 'CRITICAL'
-          ignore-unfixed: true
+          name: image-details-${{ matrix.build }}-${{ matrix.os_version }}
+          path: |
+            ./image-id.txt
+            ./image-name.txt
+          overwrite: true