Ansible playbook to configure sshd for Conch CA certs.

.github/bin/create-merge-branch.sh

@@ -0,0 +1,81 @@
+#!/usr/bin/env bash
+
+#####
+# This script creates a branch that merges the latest release
+#####
+
+set -ex
+
+# Only allow running on main
+CURRENT_BRANCH="$(git branch --show-current)"
+if [ "$CURRENT_BRANCH" != "main" ]; then
+  echo "[ERROR] This script can only be run on the main branch" >&2
+  exit 1
+fi
+
+if [ -n "$(git status --short)" ]; then
+  echo "[ERROR] This script cannot run with uncommitted changes" >&2
+  exit 1
+fi
+
+UPSTREAM_REPO="${UPSTREAM_REPO:-"stackhpc/ansible-slurm-appliance"}"
+echo "[INFO] Using upstream repo - $UPSTREAM_REPO"
+
+# Fetch the tag for the latest release from the upstream repository
+RELEASE_TAG="$(curl -fsSL "https://api.github.com/repos/${UPSTREAM_REPO}/releases/latest" | jq -r '.tag_name')"
+echo "[INFO] Found latest release tag - $RELEASE_TAG"
+
+# Add the repository as an upstream
+echo "[INFO] Adding upstream remote..."
+git remote add upstream "https://github.com/${UPSTREAM_REPO}.git"
+git remote show upstream
+
+echo "[INFO] Fetching remote tags..."
+git remote update
+
+# Use a branch that is named for the release
+BRANCH_NAME="upgrade/$RELEASE_TAG"
+
+# Check if the branch already exists on the origin
+# If it does, there is nothing more to do as the branch can be rebased from the MR
+if git show-branch "remotes/origin/$BRANCH_NAME" >/dev/null 2>&1; then
+  echo "[INFO] Merge branch already created for $RELEASE_TAG"
+  exit
+fi
+
+echo "[INFO] Merging release tag - $RELEASE_TAG"
+git merge --strategy recursive -X theirs --no-commit $RELEASE_TAG
+
+# Check if the merge resulted in any changes being staged
+if [ -n "$(git status --short)" ]; then
+  echo "[INFO] Merge resulted in the following changes"
+  git status
+
+  # NOTE(scott): The GitHub create-pull-request action does
+  # the commiting for us, so we only need to make branches
+  # and commits if running outside of GitHub actions.
+  if [ ! $GITHUB_ACTIONS ]; then
+    echo "[INFO] Checking out temporary branch '$BRANCH_NAME'..."
+    git checkout -b "$BRANCH_NAME"
+
+    echo "[INFO] Committing changes"
+    git commit -m "Upgrade ansible-slurm-applaince to $RELEASE_TAG"
+
+    echo "[INFO] Pushing changes to origin"
+    git push --set-upstream origin "$BRANCH_NAME"
+
+    # Go back to the main branch at the end
+    echo "[INFO] Reverting back to main"
+    git checkout main
+
+    echo "[INFO] Removing temporary branch"
+    git branch -d "$BRANCH_NAME"
+  fi
+
+  # Write a file containing the branch name and tag
+  # for automatic PR or MR creation that follows
+  echo "BRANCH_NAME=\"$BRANCH_NAME\"" > .mergeenv
+  echo "RELEASE_TAG=\"$RELEASE_TAG\"" >> .mergeenv
+else
+  echo "[INFO] Merge resulted in no changes"
+fi

.github/bin/get-s3-image.sh

@@ -0,0 +1,26 @@
+#!/bin/bash
+
+#####
+# This script looks for an image in OpenStack and if not found, downloads from
+# S3 bucket, and then uploads to OpenStack
+#####
+
+set -ex
+
+image_name=$1
+bucket_name=$2
+echo "Checking if image $image_name exists in OpenStack"
+image_exists=$(openstack image list --name "$image_name" -f value -c Name)
+
+if [ -n "$image_exists" ]; then
+    echo "Image $image_name already exists in OpenStack."
+else
+    echo "Image $image_name not found in OpenStack. Getting it from S3."
+
+    wget https://object.arcus.openstack.hpc.cam.ac.uk/swift/v1/AUTH_3a06571936a0424bb40bc5c672c4ccb1/$bucket_name/$image_name --progress=dot:giga
+
+    echo "Uploading image $image_name to OpenStack..."
+    openstack image create --file $image_name --disk-format qcow2 $image_name --progress
+
+    echo "Image $image_name has been uploaded to OpenStack."
+fi

.github/workflows/fatimage.yml

@@ -1,79 +1,120 @@
-
 name: Build fat image
-'on':
+on:
   workflow_dispatch:
-    inputs:
-      use_RL8:
-        required: true
-        description: Include RL8 image build
-        type: boolean
-        default: false
+      inputs:
+        ci_cloud:
+          description: 'Select the CI_CLOUD'
+          required: true
+          type: choice
+          options:
+            - LEAFCLOUD
+            - SMS
+            - ARCUS
+
 jobs:
   openstack:
     name: openstack-imagebuild
-    runs-on: ubuntu-20.04
-    concurrency: ${{ github.ref }}-{{ matrix.os_version }} # to branch/PR + OS
+    concurrency:
+      group: ${{ github.workflow }}-${{ github.ref }}-${{ matrix.os_version }}-${{ matrix.build }} # to branch/PR + OS + build
+      cancel-in-progress: true
+    runs-on: ubuntu-22.04
     strategy:
-      matrix:
-        os_version: [RL8, RL9]
-        rl8_selected:
-          - ${{ inputs.use_RL8 == true }} # only potentially true for workflow_dispatch
+      fail-fast: false # allow other matrix jobs to continue even if one fails
+      matrix: # build RL8+OFED, RL9+OFED, RL9+OFED+CUDA versions
+        os_version:
+          - RL8
+          - RL9
+        build:
+          - openstack.openhpc
+          - openstack.openhpc-cuda
         exclude:
           - os_version: RL8
-            rl8_selected: false
+            build: openstack.openhpc-cuda
     env:
       ANSIBLE_FORCE_COLOR: True
       OS_CLOUD: openstack
-      CI_CLOUD: ${{ vars.CI_CLOUD }}
+      CI_CLOUD: ${{ github.event.inputs.ci_cloud }}
+      SOURCE_IMAGES_MAP: |
+        {
+          "RL8": {
+            "openstack.openhpc": "rocky-latest-RL8",
+            "openstack.openhpc-cuda": "rocky-latest-cuda-RL8"
+          },
+          "RL9": {
+            "openstack.openhpc": "rocky-latest-RL9",
+            "openstack.openhpc-cuda": "rocky-latest-cuda-RL9"
+          }
+        }
+
     steps:
       - uses: actions/checkout@v2
 
+      - name: Record settings for CI cloud
+        run: |
+          echo CI_CLOUD: ${{ env.CI_CLOUD }}
+
       - name: Setup ssh
         run: |
           set -x
           mkdir ~/.ssh
-          echo "${{ secrets[format('{0}_SSH_KEY', vars.CI_CLOUD)] }}" > ~/.ssh/id_rsa
+          echo "${{ secrets[format('{0}_SSH_KEY', env.CI_CLOUD)] }}" > ~/.ssh/id_rsa
           chmod 0600 ~/.ssh/id_rsa
         shell: bash
 
       - name: Add bastion's ssh key to known_hosts
         run: cat environments/.stackhpc/bastion_fingerprints >> ~/.ssh/known_hosts
         shell: bash
-      
+
       - name: Install ansible etc
         run: dev/setup-env.sh
-      
+
       - name: Write clouds.yaml
         run: |
           mkdir -p ~/.config/openstack/
-          echo "${{ secrets[format('{0}_CLOUDS_YAML', vars.CI_CLOUD)] }}" > ~/.config/openstack/clouds.yaml
+          echo "${{ secrets[format('{0}_CLOUDS_YAML', env.CI_CLOUD)] }}" > ~/.config/openstack/clouds.yaml
         shell: bash
 
       - name: Setup environment
         run: |
           . venv/bin/activate
           . environments/.stackhpc/activate
-        
+
       - name: Build fat image with packer
         id: packer_build
         run: |
+          set -x
           . venv/bin/activate
           . environments/.stackhpc/activate
           cd packer/
           packer init .
-          PACKER_LOG=1 packer build -on-error=${{ vars.PACKER_ON_ERROR }} -var-file=$PKR_VAR_environment_root/${{ vars.CI_CLOUD }}.pkrvars.hcl openstack.pkr.hcl
+
+          PACKER_LOG=1 packer build \
+          -on-error=${{ vars.PACKER_ON_ERROR }} \
+          -only=${{ matrix.build }} \
+          -var-file=$PKR_VAR_environment_root/${{ env.CI_CLOUD }}.pkrvars.hcl \
+          -var "source_image_name=${{ env.SOURCE_IMAGE }}" \
+          openstack.pkr.hcl
         env:
           PKR_VAR_os_version: ${{ matrix.os_version }}
+          SOURCE_IMAGE: ${{ fromJSON(env.SOURCE_IMAGES_MAP)[matrix.os_version][matrix.build] }}
 
       - name: Get created image names from manifest
         id: manifest
         run: |
           . venv/bin/activate
-          for IMAGE_ID in $(jq --raw-output '.builds[].artifact_id' packer/packer-manifest.json)
-          do
-            while ! openstack image show -f value -c name $IMAGE_ID; do
-              sleep 5
-            done
-            IMAGE_NAME=$(openstack image show -f value -c name $IMAGE_ID)
-            echo $IMAGE_NAME
+          IMAGE_ID=$(jq --raw-output '.builds[-1].artifact_id' packer/packer-manifest.json)
+          while ! openstack image show -f value -c name $IMAGE_ID; do
+            sleep 5
           done
+          IMAGE_NAME=$(openstack image show -f value -c name $IMAGE_ID)
+          echo $IMAGE_ID > image-id.txt
+          echo $IMAGE_NAME > image-name.txt
+
+      - name: Upload manifest artifact
+        uses: actions/upload-artifact@v4
+        with:
+          name: image-details-${{ matrix.build }}-${{ matrix.os_version }}
+          path: |
+            ./image-id.txt
+            ./image-name.txt
+          overwrite: true