Skip to content

Commit 1f7c41c

Browse files
bertiethorpebertiethorpe
committed
test s3 image sync
1 parent a7fbdaf commit 1f7c41c

File tree

1 file changed

+119
-85
lines changed

1 file changed

+119
-85
lines changed

‎.github/workflows/fatimage.yml‎

Lines changed: 119 additions & 85 deletions
Original file line numberDiff line numberDiff line change
@@ -1,120 +1,154 @@
1-
name: Build fat image
1+
name: Upload CI-tested images to Arcus S3 and sync clouds
22
on:
33
workflow_dispatch:
4-
inputs:
5-
ci_cloud:
6-
description: 'Select the CI_CLOUD'
7-
required: true
8-
type: choice
9-
options:
10-
- LEAFCLOUD
11-
- SMS
12-
- ARCUS
4+
push:
5+
branches:
6+
- main
7+
paths:
8+
- 'environments/.stackhpc/terraform/cluster_image.auto.tfvars.json'
139

1410
jobs:
15-
openstack:
16-
name: openstack-imagebuild
17-
concurrency:
18-
group: ${{ github.workflow }}-${{ github.ref }}-${{ matrix.os_version }}-${{ matrix.build }} # to branch/PR + OS + build
19-
cancel-in-progress: true
11+
image_upload:
2012
runs-on: ubuntu-22.04
13+
concurrency: ${{ github.workflow }}-${{ github.ref }}-${{ matrix.build }}
2114
strategy:
22-
fail-fast: false # allow other matrix jobs to continue even if one fails
23-
matrix: # build RL8+OFED, RL9+OFED, RL9+OFED+CUDA versions
24-
os_version:
15+
fail-fast: false
16+
matrix:
17+
build:
2518
- RL8
2619
- RL9
27-
build:
28-
- openstack.openhpc
29-
- openstack.openhpc-cuda
30-
exclude:
31-
- os_version: RL8
32-
build: openstack.openhpc-cuda
20+
- RL9-cuda
3321
env:
3422
ANSIBLE_FORCE_COLOR: True
3523
OS_CLOUD: openstack
36-
CI_CLOUD: ${{ github.event.inputs.ci_cloud }}
37-
SOURCE_IMAGES_MAP: |
38-
{
39-
"RL8": {
40-
"openstack.openhpc": "rocky-latest-RL8",
41-
"openstack.openhpc-cuda": "rocky-latest-cuda-RL8"
42-
},
43-
"RL9": {
44-
"openstack.openhpc": "rocky-latest-RL9",
45-
"openstack.openhpc-cuda": "rocky-latest-cuda-RL9"
46-
}
47-
}
48-
24+
CI_CLOUD: ${{ vars.CI_CLOUD }}
25+
IMAGE_PATH: environments/.stackhpc/terraform/cluster_image.auto.tfvars.json
4926
steps:
5027
- uses: actions/checkout@v2
5128

52-
- name: Record settings for CI cloud
29+
- name: Record which cloud CI is running on
5330
run: |
5431
echo CI_CLOUD: ${{ env.CI_CLOUD }}
5532
56-
- name: Setup ssh
33+
- name: setup environment
34+
run: |
35+
python3 -m venv venv
36+
. venv/bin/activate
37+
pip install -U pip
38+
pip install $(grep -o 'python-openstackclient[><=0-9\.]*' requirements.txt)
39+
shell: bash
40+
41+
- name: Write clouds.yaml
42+
run: |
43+
mkdir -p ~/.config/openstack/
44+
echo "${{ secrets[format('{0}_CLOUDS_YAML', env.CI_CLOUD)] }}" > ~/.config/openstack/clouds.yaml
45+
shell: bash
46+
47+
- name: Write s3cmd configuration
48+
run: |
49+
echo "${{ secrets['ARCUS_S3CFG'] }}" > ~/.s3cfg
50+
shell: bash
51+
52+
- name: Install s3cmd
53+
run: |
54+
sudo apt-get --yes install s3cmd
55+
56+
- name: Check for image in Arcus S3 bucket
57+
id: s3_ls
58+
run: |
59+
60+
TARGET_IMAGE=$(jq --arg version "${{ matrix.build }}" -r '.cluster_image[$version]' "${{ env.IMAGE_PATH }}")
61+
echo "TARGET_IMAGE=${TARGET_IMAGE}" >> "$GITHUB_ENV"
62+
echo "target-image-${{ matrix.build }}=${TARGET_IMAGE}" >> "$GITHUB_OUTPUT"
63+
64+
S3_IMAGES=$(s3cmd ls s3://openhpc-images)
65+
66+
if echo "$S3_IMAGES" | grep -q "$TARGET_IMAGE"; then
67+
echo "Image ${TARGET_IMAGE} is already present in S3."
68+
echo "IMAGE_EXISTS=true" >> $GITHUB_ENV
69+
else
70+
echo "Image ${TARGET_IMAGE} is not present in S3."
71+
echo "IMAGE_EXISTS=false" >> $GITHUB_ENV
72+
fi
73+
shell: bash
74+
75+
- name: Download image to runner
76+
if: env.IMAGE_EXISTS == 'false'
5777
run: |
58-
set -x
59-
mkdir ~/.ssh
60-
echo "${{ secrets[format('{0}_SSH_KEY', env.CI_CLOUD)] }}" > ~/.ssh/id_rsa
61-
chmod 0600 ~/.ssh/id_rsa
78+
. venv/bin/activate
79+
openstack image save --file ${{ env.TARGET_IMAGE }} ${{ env.TARGET_IMAGE }}
6280
shell: bash
6381

64-
- name: Add bastion's ssh key to known_hosts
65-
run: cat environments/.stackhpc/bastion_fingerprints >> ~/.ssh/known_hosts
82+
- name: Conditionally Upload Image to S3
83+
if: env.IMAGE_EXISTS == 'false'
84+
run: |
85+
echo "Uploading Image: ${{ env.TARGET_IMAGE }} to S3..."
86+
s3cmd put ${{ env.TARGET_IMAGE }}.qcow2 s3://openhpc-images
6687
shell: bash
6788

68-
- name: Install ansible etc
69-
run: dev/setup-env.sh
89+
image_sync:
90+
needs: image_upload
91+
runs-on: ubuntu-22.04
92+
concurrency: ${{ github.workflow }}-${{ github.ref }}-${{ matrix.cloud }}-${{ matrix.build }}
93+
strategy:
94+
fail-fast: false
95+
matrix:
96+
cloud:
97+
- LEAFCLOUD
98+
- SMS
99+
- ARCUS
100+
build:
101+
- RL8
102+
- RL9
103+
- RL9-cuda
104+
exclude:
105+
- cloud: LEAFCLOUD
106+
107+
env:
108+
ANSIBLE_FORCE_COLOR: True
109+
OS_CLOUD: openstack
110+
CI_CLOUD: ${{ matrix.cloud }}
111+
IMAGE_PATH: environments/.stackhpc/terraform/cluster_image.auto.tfvars.json
112+
steps:
113+
- uses: actions/checkout@v2
114+
115+
- name: Record which cloud CI is running on
116+
run: |
117+
echo CI_CLOUD: ${{ env.CI_CLOUD }}
118+
119+
- name: setup environment
120+
run: |
121+
python3 -m venv venv
122+
. venv/bin/activate
123+
pip install -U pip
124+
pip install $(grep -o 'python-openstackclient[><=0-9\.]*' requirements.txt)
125+
shell: bash
70126

71127
- name: Write clouds.yaml
72128
run: |
73129
mkdir -p ~/.config/openstack/
74130
echo "${{ secrets[format('{0}_CLOUDS_YAML', env.CI_CLOUD)] }}" > ~/.config/openstack/clouds.yaml
75131
shell: bash
76132

77-
- name: Setup environment
133+
- name: Retrieve image name
78134
run: |
79-
. venv/bin/activate
80-
. environments/.stackhpc/activate
135+
TARGET_IMAGE=$(jq --arg version "${{ matrix.build }}" -r '.cluster_image[$version]' "${{ env.IMAGE_PATH }}")
136+
echo "TARGET_IMAGE=${TARGET_IMAGE}" >> "$GITHUB_ENV"
81137
82-
- name: Build fat image with packer
83-
id: packer_build
138+
- name: Upload latest image if missing
84139
run: |
85-
set -x
86140
. venv/bin/activate
87-
. environments/.stackhpc/activate
88-
cd packer/
89-
packer init .
90-
91-
PACKER_LOG=1 packer build \
92-
-on-error=${{ vars.PACKER_ON_ERROR }} \
93-
-only=${{ matrix.build }} \
94-
-var-file=$PKR_VAR_environment_root/${{ env.CI_CLOUD }}.pkrvars.hcl \
95-
-var "source_image_name=${{ env.SOURCE_IMAGE }}" \
96-
openstack.pkr.hcl
97-
env:
98-
PKR_VAR_os_version: ${{ matrix.os_version }}
99-
SOURCE_IMAGE: ${{ fromJSON(env.SOURCE_IMAGES_MAP)[matrix.os_version][matrix.build] }}
100-
101-
- name: Get created image names from manifest
102-
id: manifest
141+
bash .github/bin/get-s3-image.sh ${{ env.TARGET_IMAGE }} openhpc-images
142+
143+
- name: Cleanup OpenStack Image (on error or cancellation)
144+
if: cancelled()
103145
run: |
104146
. venv/bin/activate
105-
IMAGE_ID=$(jq --raw-output '.builds[-1].artifact_id' packer/packer-manifest.json)
106-
while ! openstack image show -f value -c name $IMAGE_ID; do
107-
sleep 5
108-
done
109-
IMAGE_NAME=$(openstack image show -f value -c name $IMAGE_ID)
110-
echo $IMAGE_ID > image-id.txt
111-
echo $IMAGE_NAME > image-name.txt
112-
113-
- name: Upload manifest artifact
114-
uses: actions/upload-artifact@v4
115-
with:
116-
name: image-details-${{ matrix.build }}-${{ matrix.os_version }}
117-
path: |
118-
./image-id.txt
119-
./image-name.txt
120-
overwrite: true
147+
image_hanging=$(openstack image list --name ${{ env.TARGET_IMAGE }} -f value -c ID -c Status | grep -v ' active$' | awk '{print $1}')
148+
if [ -n "$image_hanging" ]; then
149+
echo "Cleaning up OpenStack image with ID: $image_hanging"
150+
openstack image delete $image_hanging
151+
else
152+
echo "No image ID found, skipping cleanup."
153+
fi
154+
shell: bash

0 commit comments

Comments
 (0)