Skip to content

Commit c9663ac

Browse files
sjpbsjpb
committed
Merge branch 'main' into docs/setup
2 parents 39f99a6 + 85c95f0 commit c9663ac

File tree

35 files changed

+635
-119
lines changed

35 files changed

+635
-119
lines changed

‎.github/bin/create-merge-branch.sh‎

Lines changed: 81 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,81 @@
1+
#!/usr/bin/env bash
2+
3+
#####
4+
# This script creates a branch that merges the latest release
5+
#####
6+
7+
set -ex
8+
9+
# Only allow running on main
10+
CURRENT_BRANCH="$(git branch --show-current)"
11+
if [ "$CURRENT_BRANCH" != "main" ]; then
12+
echo "[ERROR] This script can only be run on the main branch" >&2
13+
exit 1
14+
fi
15+
16+
if [ -n "$(git status --short)" ]; then
17+
echo "[ERROR] This script cannot run with uncommitted changes" >&2
18+
exit 1
19+
fi
20+
21+
UPSTREAM_REPO="${UPSTREAM_REPO:-"stackhpc/ansible-slurm-appliance"}"
22+
echo "[INFO] Using upstream repo - $UPSTREAM_REPO"
23+
24+
# Fetch the tag for the latest release from the upstream repository
25+
RELEASE_TAG="$(curl -fsSL "https://api.github.com/repos/${UPSTREAM_REPO}/releases/latest" | jq -r '.tag_name')"
26+
echo "[INFO] Found latest release tag - $RELEASE_TAG"
27+
28+
# Add the repository as an upstream
29+
echo "[INFO] Adding upstream remote..."
30+
git remote add upstream "https://github.com/${UPSTREAM_REPO}.git"
31+
git remote show upstream
32+
33+
echo "[INFO] Fetching remote tags..."
34+
git remote update
35+
36+
# Use a branch that is named for the release
37+
BRANCH_NAME="upgrade/$RELEASE_TAG"
38+
39+
# Check if the branch already exists on the origin
40+
# If it does, there is nothing more to do as the branch can be rebased from the MR
41+
if git show-branch "remotes/origin/$BRANCH_NAME" >/dev/null 2>&1; then
42+
echo "[INFO] Merge branch already created for $RELEASE_TAG"
43+
exit
44+
fi
45+
46+
echo "[INFO] Merging release tag - $RELEASE_TAG"
47+
git merge --strategy recursive -X theirs --no-commit $RELEASE_TAG
48+
49+
# Check if the merge resulted in any changes being staged
50+
if [ -n "$(git status --short)" ]; then
51+
echo "[INFO] Merge resulted in the following changes"
52+
git status
53+
54+
# NOTE(scott): The GitHub create-pull-request action does
55+
# the commiting for us, so we only need to make branches
56+
# and commits if running outside of GitHub actions.
57+
if [ ! $GITHUB_ACTIONS ]; then
58+
echo "[INFO] Checking out temporary branch '$BRANCH_NAME'..."
59+
git checkout -b "$BRANCH_NAME"
60+
61+
echo "[INFO] Committing changes"
62+
git commit -m "Upgrade ansible-slurm-applaince to $RELEASE_TAG"
63+
64+
echo "[INFO] Pushing changes to origin"
65+
git push --set-upstream origin "$BRANCH_NAME"
66+
67+
# Go back to the main branch at the end
68+
echo "[INFO] Reverting back to main"
69+
git checkout main
70+
71+
echo "[INFO] Removing temporary branch"
72+
git branch -d "$BRANCH_NAME"
73+
fi
74+
75+
# Write a file containing the branch name and tag
76+
# for automatic PR or MR creation that follows
77+
echo "BRANCH_NAME=\"$BRANCH_NAME\"" > .mergeenv
78+
echo "RELEASE_TAG=\"$RELEASE_TAG\"" >> .mergeenv
79+
else
80+
echo "[INFO] Merge resulted in no changes"
81+
fi

‎.github/bin/get-s3-image.sh‎

Lines changed: 26 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,26 @@
1+
#!/bin/bash
2+
3+
#####
4+
# This script looks for an image in OpenStack and if not found, downloads from
5+
# S3 bucket, and then uploads to OpenStack
6+
#####
7+
8+
set -ex
9+
10+
image_name=$1
11+
bucket_name=$2
12+
echo "Checking if image $image_name exists in OpenStack"
13+
image_exists=$(openstack image list --name "$image_name" -f value -c Name)
14+
15+
if [ -n "$image_exists" ]; then
16+
echo "Image $image_name already exists in OpenStack."
17+
else
18+
echo "Image $image_name not found in OpenStack. Getting it from S3."
19+
20+
wget https://object.arcus.openstack.hpc.cam.ac.uk/swift/v1/AUTH_3a06571936a0424bb40bc5c672c4ccb1/$bucket_name/$image_name --progress=dot:giga
21+
22+
echo "Uploading image $image_name to OpenStack..."
23+
openstack image create --file $image_name --disk-format qcow2 $image_name --progress
24+
25+
echo "Image $image_name has been uploaded to OpenStack."
26+
fi

‎.github/workflows/fatimage.yml‎

Lines changed: 83 additions & 25 deletions
Original file line numberDiff line numberDiff line change
@@ -1,40 +1,51 @@
11

22
name: Build fat image
3-
'on':
3+
on:
44
workflow_dispatch:
55
inputs:
6-
use_RL8:
6+
ci_cloud:
7+
description: 'Select the CI_CLOUD'
78
required: true
8-
description: Include RL8 image build
9-
type: boolean
10-
default: false
11-
concurrency:
12-
group: ${{ github.ref }}-{{ matrix.os_version }} # to branch/PR + OS
13-
cancel-in-progress: true
9+
type: choice
10+
options:
11+
- LEAFCLOUD
12+
- SMS
13+
- ARCUS
1414
jobs:
1515
openstack:
1616
name: openstack-imagebuild
17-
runs-on: ubuntu-20.04
17+
concurrency:
18+
group: ${{ github.workflow }}-${{ github.ref }}-${{ matrix.os_version }}-${{ matrix.build }} # to branch/PR + OS + build
19+
cancel-in-progress: true
20+
runs-on: ubuntu-22.04
1821
strategy:
19-
matrix:
20-
os_version: [RL8, RL9]
21-
rl8_selected:
22-
- ${{ inputs.use_RL8 == true }} # only potentially true for workflow_dispatch
22+
fail-fast: false # allow other matrix jobs to continue even if one fails
23+
matrix: # build RL8+OFED, RL9+OFED, RL9+OFED+CUDA versions
24+
os_version:
25+
- RL8
26+
- RL9
27+
build:
28+
- openstack.openhpc-ofed
29+
- openstack.openhpc-cuda
2330
exclude:
2431
- os_version: RL8
25-
rl8_selected: false
32+
build: openstack.openhpc-cuda
2633
env:
2734
ANSIBLE_FORCE_COLOR: True
2835
OS_CLOUD: openstack
29-
CI_CLOUD: ${{ vars.CI_CLOUD }}
36+
CI_CLOUD: ${{ github.event.inputs.ci_cloud }}
3037
steps:
3138
- uses: actions/checkout@v2
3239

40+
- name: Record settings for CI cloud
41+
run: |
42+
echo CI_CLOUD: ${{ env.CI_CLOUD }}
43+
3344
- name: Setup ssh
3445
run: |
3546
set -x
3647
mkdir ~/.ssh
37-
echo "${{ secrets[format('{0}_SSH_KEY', vars.CI_CLOUD)] }}" > ~/.ssh/id_rsa
48+
echo "${{ secrets[format('{0}_SSH_KEY', env.CI_CLOUD)] }}" > ~/.ssh/id_rsa
3849
chmod 0600 ~/.ssh/id_rsa
3950
shell: bash
4051

@@ -48,7 +59,7 @@ jobs:
4859
- name: Write clouds.yaml
4960
run: |
5061
mkdir -p ~/.config/openstack/
51-
echo "${{ secrets[format('{0}_CLOUDS_YAML', vars.CI_CLOUD)] }}" > ~/.config/openstack/clouds.yaml
62+
echo "${{ secrets[format('{0}_CLOUDS_YAML', env.CI_CLOUD)] }}" > ~/.config/openstack/clouds.yaml
5263
shell: bash
5364

5465
- name: Setup environment
@@ -63,19 +74,66 @@ jobs:
6374
. environments/.stackhpc/activate
6475
cd packer/
6576
packer init .
66-
PACKER_LOG=1 packer build -on-error=${{ vars.PACKER_ON_ERROR }} -except=openstack.openhpc-extra -var-file=$PKR_VAR_environment_root/${{ vars.CI_CLOUD }}.pkrvars.hcl openstack.pkr.hcl
77+
PACKER_LOG=1 packer build -on-error=${{ vars.PACKER_ON_ERROR }} -only=${{ matrix.build }} -var-file=$PKR_VAR_environment_root/${{ env.CI_CLOUD }}.pkrvars.hcl openstack.pkr.hcl
6778
env:
6879
PKR_VAR_os_version: ${{ matrix.os_version }}
6980

7081
- name: Get created image names from manifest
7182
id: manifest
7283
run: |
7384
. venv/bin/activate
74-
for IMAGE_ID in $(jq --raw-output '.builds[].artifact_id' packer/packer-manifest.json)
75-
do
76-
while ! openstack image show -f value -c name $IMAGE_ID; do
77-
sleep 5
78-
done
79-
IMAGE_NAME=$(openstack image show -f value -c name $IMAGE_ID)
80-
echo $IMAGE_NAME
85+
IMAGE_ID=$(jq --raw-output '.builds[-1].artifact_id' packer/packer-manifest.json)
86+
while ! openstack image show -f value -c name $IMAGE_ID; do
87+
sleep 5
8188
done
89+
IMAGE_NAME=$(openstack image show -f value -c name $IMAGE_ID)
90+
echo "image-name=${IMAGE_NAME}" >> "$GITHUB_OUTPUT"
91+
echo "image-id=$IMAGE_ID" >> "$GITHUB_OUTPUT"
92+
93+
- name: Download image
94+
run: |
95+
. venv/bin/activate
96+
sudo mkdir /mnt/images
97+
sudo chmod 777 /mnt/images
98+
openstack image save --file /mnt/images/${{ steps.manifest.outputs.image-name }}.qcow2 ${{ steps.manifest.outputs.image-name }}
99+
100+
- name: Set up QEMU
101+
uses: docker/setup-qemu-action@v3
102+
103+
- name: install libguestfs
104+
run: |
105+
sudo apt -y update
106+
sudo apt -y install libguestfs-tools
107+
108+
- name: mkdir for mount
109+
run: sudo mkdir -p './${{ steps.manifest.outputs.image-name }}'
110+
111+
- name: mount qcow2 file
112+
run: sudo guestmount -a /mnt/images/${{ steps.manifest.outputs.image-name }}.qcow2 -i --ro -o allow_other './${{ steps.manifest.outputs.image-name }}'
113+
114+
- name: Run Trivy vulnerability scanner
115+
uses: aquasecurity/[email protected]
116+
with:
117+
scan-type: fs
118+
scan-ref: "${{ steps.manifest.outputs.image-name }}"
119+
scanners: "vuln"
120+
format: sarif
121+
output: "${{ steps.manifest.outputs.image-name }}.sarif"
122+
# turn off secret scanning to speed things up
123+
124+
- name: Upload Trivy scan results to GitHub Security tab
125+
uses: github/codeql-action/upload-sarif@v3
126+
with:
127+
sarif_file: "${{ steps.manifest.outputs.image-name }}.sarif"
128+
category: "${{ matrix.os_version }}-${{ matrix.build }}"
129+
130+
- name: Fail if scan has CRITICAL vulnerabilities
131+
uses: aquasecurity/[email protected]
132+
with:
133+
scan-type: fs
134+
scan-ref: "${{ steps.manifest.outputs.image-name }}"
135+
scanners: "vuln"
136+
format: table
137+
exit-code: '1'
138+
severity: 'CRITICAL'
139+
ignore-unfixed: true

0 commit comments

Comments
 (0)