Skip to content

Commit 1b4ae57

Browse files
bertiethorpebertiethorpe
committed
test s3 image sync
1 parent 85c95f0 commit 1b4ae57

File tree

1 file changed

+120
-105
lines changed

1 file changed

+120
-105
lines changed

‎.github/workflows/fatimage.yml‎

Lines changed: 120 additions & 105 deletions
Original file line numberDiff line numberDiff line change
@@ -1,139 +1,154 @@
1-
2-
name: Build fat image
1+
name: Upload CI-tested images to Arcus S3 and sync clouds
32
on:
43
workflow_dispatch:
5-
inputs:
6-
ci_cloud:
7-
description: 'Select the CI_CLOUD'
8-
required: true
9-
type: choice
10-
options:
11-
- LEAFCLOUD
12-
- SMS
13-
- ARCUS
4+
push:
5+
branches:
6+
- main
7+
paths:
8+
- 'environments/.stackhpc/terraform/cluster_image.auto.tfvars.json'
9+
1410
jobs:
15-
openstack:
16-
name: openstack-imagebuild
17-
concurrency:
18-
group: ${{ github.workflow }}-${{ github.ref }}-${{ matrix.os_version }}-${{ matrix.build }} # to branch/PR + OS + build
19-
cancel-in-progress: true
11+
image_upload:
2012
runs-on: ubuntu-22.04
13+
concurrency: ${{ github.workflow }}-${{ github.ref }}-${{ matrix.build }}
2114
strategy:
22-
fail-fast: false # allow other matrix jobs to continue even if one fails
23-
matrix: # build RL8+OFED, RL9+OFED, RL9+OFED+CUDA versions
24-
os_version:
15+
fail-fast: false
16+
matrix:
17+
build:
2518
- RL8
2619
- RL9
27-
build:
28-
- openstack.openhpc-ofed
29-
- openstack.openhpc-cuda
30-
exclude:
31-
- os_version: RL8
32-
build: openstack.openhpc-cuda
20+
- RL9-cuda
3321
env:
3422
ANSIBLE_FORCE_COLOR: True
3523
OS_CLOUD: openstack
36-
CI_CLOUD: ${{ github.event.inputs.ci_cloud }}
24+
CI_CLOUD: ${{ vars.CI_CLOUD }}
25+
IMAGE_PATH: environments/.stackhpc/terraform/cluster_image.auto.tfvars.json
3726
steps:
3827
- uses: actions/checkout@v2
3928

40-
- name: Record settings for CI cloud
29+
- name: Record which cloud CI is running on
4130
run: |
4231
echo CI_CLOUD: ${{ env.CI_CLOUD }}
4332
44-
- name: Setup ssh
33+
- name: setup environment
4534
run: |
46-
set -x
47-
mkdir ~/.ssh
48-
echo "${{ secrets[format('{0}_SSH_KEY', env.CI_CLOUD)] }}" > ~/.ssh/id_rsa
49-
chmod 0600 ~/.ssh/id_rsa
35+
python3 -m venv venv
36+
. venv/bin/activate
37+
pip install -U pip
38+
pip install $(grep -o 'python-openstackclient[><=0-9\.]*' requirements.txt)
5039
shell: bash
5140

52-
- name: Add bastion's ssh key to known_hosts
53-
run: cat environments/.stackhpc/bastion_fingerprints >> ~/.ssh/known_hosts
54-
shell: bash
55-
56-
- name: Install ansible etc
57-
run: dev/setup-env.sh
58-
5941
- name: Write clouds.yaml
6042
run: |
6143
mkdir -p ~/.config/openstack/
6244
echo "${{ secrets[format('{0}_CLOUDS_YAML', env.CI_CLOUD)] }}" > ~/.config/openstack/clouds.yaml
6345
shell: bash
6446

65-
- name: Setup environment
47+
- name: Write s3cmd configuration
6648
run: |
67-
. venv/bin/activate
68-
. environments/.stackhpc/activate
69-
70-
- name: Build fat image with packer
71-
id: packer_build
49+
echo "${{ secrets['ARCUS_S3CFG'] }}" > ~/.s3cfg
50+
shell: bash
51+
52+
- name: Install s3cmd
7253
run: |
73-
. venv/bin/activate
74-
. environments/.stackhpc/activate
75-
cd packer/
76-
packer init .
77-
PACKER_LOG=1 packer build -on-error=${{ vars.PACKER_ON_ERROR }} -only=${{ matrix.build }} -var-file=$PKR_VAR_environment_root/${{ env.CI_CLOUD }}.pkrvars.hcl openstack.pkr.hcl
78-
env:
79-
PKR_VAR_os_version: ${{ matrix.os_version }}
80-
81-
- name: Get created image names from manifest
82-
id: manifest
54+
sudo apt-get --yes install s3cmd
55+
56+
- name: Check for image in Arcus S3 bucket
57+
id: s3_ls
58+
run: |
59+
60+
TARGET_IMAGE=$(jq --arg version "${{ matrix.build }}" -r '.cluster_image[$version]' "${{ env.IMAGE_PATH }}")
61+
echo "TARGET_IMAGE=${TARGET_IMAGE}" >> "$GITHUB_ENV"
62+
echo "target-image-${{ matrix.build }}=${TARGET_IMAGE}" >> "$GITHUB_OUTPUT"
63+
64+
S3_IMAGES=$(s3cmd ls s3://openhpc-images)
65+
66+
if echo "$S3_IMAGES" | grep -q "$TARGET_IMAGE"; then
67+
echo "Image ${TARGET_IMAGE} is already present in S3."
68+
echo "IMAGE_EXISTS=true" >> $GITHUB_ENV
69+
else
70+
echo "Image ${TARGET_IMAGE} is not present in S3."
71+
echo "IMAGE_EXISTS=false" >> $GITHUB_ENV
72+
fi
73+
shell: bash
74+
75+
- name: Download image to runner
76+
if: env.IMAGE_EXISTS == 'false'
8377
run: |
8478
. venv/bin/activate
85-
IMAGE_ID=$(jq --raw-output '.builds[-1].artifact_id' packer/packer-manifest.json)
86-
while ! openstack image show -f value -c name $IMAGE_ID; do
87-
sleep 5
88-
done
89-
IMAGE_NAME=$(openstack image show -f value -c name $IMAGE_ID)
90-
echo "image-name=${IMAGE_NAME}" >> "$GITHUB_OUTPUT"
91-
echo "image-id=$IMAGE_ID" >> "$GITHUB_OUTPUT"
92-
93-
- name: Download image
79+
openstack image save --file ${{ env.TARGET_IMAGE }} ${{ env.TARGET_IMAGE }}
80+
shell: bash
81+
82+
- name: Conditionally Upload Image to S3
83+
if: env.IMAGE_EXISTS == 'false'
9484
run: |
85+
echo "Uploading Image: ${{ env.TARGET_IMAGE }} to S3..."
86+
s3cmd put ${{ env.TARGET_IMAGE }}.qcow2 s3://openhpc-images
87+
shell: bash
88+
89+
image_sync:
90+
needs: image_upload
91+
runs-on: ubuntu-22.04
92+
concurrency: ${{ github.workflow }}-${{ github.ref }}-${{ matrix.cloud }}-${{ matrix.build }}
93+
strategy:
94+
fail-fast: false
95+
matrix:
96+
cloud:
97+
- LEAFCLOUD
98+
- SMS
99+
- ARCUS
100+
build:
101+
- RL8
102+
- RL9
103+
- RL9-cuda
104+
exclude:
105+
- cloud: LEAFCLOUD
106+
107+
env:
108+
ANSIBLE_FORCE_COLOR: True
109+
OS_CLOUD: openstack
110+
CI_CLOUD: ${{ matrix.cloud }}
111+
IMAGE_PATH: environments/.stackhpc/terraform/cluster_image.auto.tfvars.json
112+
steps:
113+
- uses: actions/checkout@v2
114+
115+
- name: Record which cloud CI is running on
116+
run: |
117+
echo CI_CLOUD: ${{ env.CI_CLOUD }}
118+
119+
- name: setup environment
120+
run: |
121+
python3 -m venv venv
95122
. venv/bin/activate
96-
sudo mkdir /mnt/images
97-
sudo chmod 777 /mnt/images
98-
openstack image save --file /mnt/images/${{ steps.manifest.outputs.image-name }}.qcow2 ${{ steps.manifest.outputs.image-name }}
123+
pip install -U pip
124+
pip install $(grep -o 'python-openstackclient[><=0-9\.]*' requirements.txt)
125+
shell: bash
99126

100-
- name: Set up QEMU
101-
uses: docker/setup-qemu-action@v3
127+
- name: Write clouds.yaml
128+
run: |
129+
mkdir -p ~/.config/openstack/
130+
echo "${{ secrets[format('{0}_CLOUDS_YAML', env.CI_CLOUD)] }}" > ~/.config/openstack/clouds.yaml
131+
shell: bash
102132

103-
- name: install libguestfs
133+
- name: Retrieve image name
104134
run: |
105-
sudo apt -y update
106-
sudo apt -y install libguestfs-tools
107-
108-
- name: mkdir for mount
109-
run: sudo mkdir -p './${{ steps.manifest.outputs.image-name }}'
110-
111-
- name: mount qcow2 file
112-
run: sudo guestmount -a /mnt/images/${{ steps.manifest.outputs.image-name }}.qcow2 -i --ro -o allow_other './${{ steps.manifest.outputs.image-name }}'
113-
114-
- name: Run Trivy vulnerability scanner
115-
uses: aquasecurity/[email protected]
116-
with:
117-
scan-type: fs
118-
scan-ref: "${{ steps.manifest.outputs.image-name }}"
119-
scanners: "vuln"
120-
format: sarif
121-
output: "${{ steps.manifest.outputs.image-name }}.sarif"
122-
# turn off secret scanning to speed things up
123-
124-
- name: Upload Trivy scan results to GitHub Security tab
125-
uses: github/codeql-action/upload-sarif@v3
126-
with:
127-
sarif_file: "${{ steps.manifest.outputs.image-name }}.sarif"
128-
category: "${{ matrix.os_version }}-${{ matrix.build }}"
129-
130-
- name: Fail if scan has CRITICAL vulnerabilities
131-
uses: aquasecurity/[email protected]
132-
with:
133-
scan-type: fs
134-
scan-ref: "${{ steps.manifest.outputs.image-name }}"
135-
scanners: "vuln"
136-
format: table
137-
exit-code: '1'
138-
severity: 'CRITICAL'
139-
ignore-unfixed: true
135+
TARGET_IMAGE=$(jq --arg version "${{ matrix.build }}" -r '.cluster_image[$version]' "${{ env.IMAGE_PATH }}")
136+
echo "TARGET_IMAGE=${TARGET_IMAGE}" >> "$GITHUB_ENV"
137+
138+
- name: Upload latest image if missing
139+
run: |
140+
. venv/bin/activate
141+
bash .github/bin/get-s3-image.sh ${{ env.TARGET_IMAGE }} openhpc-images
142+
143+
- name: Cleanup OpenStack Image (on error or cancellation)
144+
if: cancelled()
145+
run: |
146+
. venv/bin/activate
147+
image_hanging=$(openstack image list --name ${{ env.TARGET_IMAGE }} -f value -c ID -c Status | grep -v ' active$' | awk '{print $1}')
148+
if [ -n "$image_hanging" ]; then
149+
echo "Cleaning up OpenStack image with ID: $image_hanging"
150+
openstack image delete $image_hanging
151+
else
152+
echo "No image ID found, skipping cleanup."
153+
fi
154+
shell: bash

0 commit comments

Comments
 (0)