Skip to content

Commit a09fbb9

Browse files
markgoddardmarkgoddard
committed
Add a reusable GitHub workflow to deploy a multinode test cluster
1 parent 6027269 commit a09fbb9

File tree

2 files changed

+383
-2
lines changed

2 files changed

+383
-2
lines changed

‎.github/workflows/multinode.yml‎

Lines changed: 364 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,364 @@
1+
---
2+
# This reusable workflow deploys a multi-node test cluster on a cloud using
3+
# Terraform, then deploys OpenStack via Kayobe. Tempest is then used to test
4+
# the cloud.
5+
6+
name: Multinode
7+
8+
on:
9+
workflow_call:
10+
inputs:
11+
multinode_name:
12+
description: Multinode cluster name
13+
type: string
14+
required: true
15+
multinode_controller_count:
16+
description: Controller count
17+
type: number
18+
default: 3
19+
multinode_compute_count:
20+
description: Compute count
21+
type: number
22+
default: 2
23+
multinode_storage_count:
24+
description: Storage count
25+
type: number
26+
default: 3
27+
os_distribution:
28+
description: Host OS distribution
29+
type: string
30+
default: rocky
31+
os_release:
32+
description: Host OS release
33+
type: string
34+
default: '9'
35+
ssh_username:
36+
description: User for terraform to access the all-in-one VM
37+
type: string
38+
default: cloud-user
39+
neutron_plugin:
40+
description: Neutron ML2 plugin
41+
type: string
42+
default: ovn
43+
stackhpc_kayobe_config_version:
44+
description: stackhpc-kayobe-config version
45+
type: string
46+
stackhpc_kayobe_config_previous_version:
47+
description: stackhpc-kayobe-config previous version
48+
type: string
49+
terraform_kayobe_multinode_version:
50+
description: terraform-kayobe-multinode version
51+
type: string
52+
default: main
53+
upgrade:
54+
description: Whether to perform an upgrade
55+
type: boolean
56+
default: false
57+
break_on:
58+
description: When to break execution for manual interaction
59+
type: string
60+
default: never
61+
break_duration:
62+
description: How long to break execution for (minutes)
63+
type: number
64+
default: 60
65+
ssh_key:
66+
description: SSH public key to authorise on Ansible control host
67+
type: string
68+
secrets:
69+
KAYOBE_VAULT_PASSWORD_CI_MULTINODE:
70+
required: true
71+
CLOUDS_YAML:
72+
required: true
73+
OS_APPLICATION_CREDENTIAL_ID:
74+
required: true
75+
OS_APPLICATION_CREDENTIAL_SECRET:
76+
required: true
77+
78+
jobs:
79+
multinode:
80+
name: Multinode
81+
runs-on: arc-skc-aio-runner
82+
environment: Leafcloud
83+
permissions: {}
84+
env:
85+
ANSIBLE_FORCE_COLOR: True
86+
KAYOBE_ENVIRONMENT: ci-multinode
87+
KAYOBE_VAULT_PASSWORD: ${{ secrets.KAYOBE_VAULT_PASSWORD_CI_MULTINODE }}
88+
steps:
89+
- name: Fail if previous version is not defined
90+
run: |
91+
echo "StackHPC Kayobe Configuration previous version must be defined for upgrades"
92+
exit 1
93+
if: ${{ inputs.upgrade && inputs.stackhpc_kayobe_config_previous_version == '' }}
94+
95+
- name: Install Package
96+
uses: ConorMacBride/install-package@main
97+
with:
98+
apt: git unzip nodejs python3-pip python3-venv rsync
99+
100+
# If testing upgrade, checkout previous release, otherwise checkout current branch
101+
- name: Checkout ${{ inputs.upgrade && 'previous release' || 'current' }} config
102+
uses: actions/checkout@v4
103+
with:
104+
repository: stackhpc/stackhpc-kayobe-config
105+
ref: ${{ inputs.upgrade && inputs.stackhpc_kayobe_config_previous_version || inputs.stackhpc_kayobe_config_version }}
106+
107+
- name: Checkout terraform-kayobe-multinode
108+
uses: actions/checkout@v4
109+
with:
110+
repository: stackhpc/terraform-kayobe-multinode
111+
ref: ${{ inputs.terraform_kayobe_multinode_version }}
112+
path: terraform-kayobe-multinode
113+
114+
- name: Make sure dockerd is running and test Docker
115+
run: |
116+
docker ps
117+
118+
- name: Output image tag
119+
id: image_tag
120+
run: |
121+
echo image_tag=$(grep stackhpc_${{ inputs.os_distribution }}_$(sed s/-/_/ <(echo "${{ inputs.os_release }}"))_overcloud_host_image_version: etc/kayobe/pulp-host-image-versions.yml | awk '{print $2}') >> $GITHUB_OUTPUT
122+
123+
# Use the image override if set, otherwise use overcloud-os_distribution-os_release-tag
124+
- name: Output image name
125+
id: image_name
126+
run: |
127+
if [ -z "${{ inputs.multinode_image_override }}" ]; then
128+
echo image_name=overcloud-${{ inputs.os_distribution }}-${{ inputs.os_release }}-${{ steps.image_tag.outputs.image_tag }} >> $GITHUB_OUTPUT
129+
else
130+
echo image_name=${{ inputs.multinode_image_override }} >> $GITHUB_OUTPUT
131+
fi
132+
133+
- name: Install terraform
134+
uses: hashicorp/setup-terraform@v2
135+
with:
136+
terraform_wrapper: false
137+
138+
- name: Setup Ansible
139+
run: |
140+
python3 -m venv venv &&
141+
source venv/bin/activate &&
142+
pip install -U pip &&
143+
pip install ansible &&
144+
mkdir -p ansible/{collections,roles} &&
145+
ansible-galaxy role install -r ansible/requirements.yml -p ansible/roles &&
146+
ansible-galaxy collection install -r ansible/requirements.yml -p ansible/collections
147+
working-directory: ${{ github.workspace }}/terraform-kayobe-multinode
148+
149+
- name: Generate SSH keypair
150+
run: ssh-keygen -f id_rsa -N ''
151+
working-directory: ${{ github.workspace }}/terraform-kayobe-multinode
152+
153+
# NOTE: In Ansible 2.10 and lower the synchronize module used in the
154+
# ansible/fetch-logs.yml playbook does not respect SSH connection
155+
# variables. This may result in Permission Denied issues if using an SSH
156+
# key that is not in ~/.ssh.
157+
- name: Copy SSH keypair to .ssh/
158+
run: |
159+
install -d ~/.ssh -m 700 &&
160+
cp id_rsa* ~/.ssh/
161+
working-directory: ${{ github.workspace }}/terraform-kayobe-multinode
162+
163+
- name: Generate clouds.yaml
164+
run: |
165+
cat << EOF > clouds.yaml
166+
${{ secrets.CLOUDS_YAML }}
167+
EOF
168+
working-directory: ${{ github.workspace }}/terraform-kayobe-multinode
169+
170+
- name: Generate terraform.tfvars
171+
run: |
172+
cat << EOF > terraform.tfvars
173+
174+
prefix = "${{ env.MULTINODE_NAME }}"
175+
176+
ansible_control_vm_flavor = "${{ env.MULTINODE_ANSIBLE_CONTROL_VM_FLAVOR }}"
177+
ansible_control_vm_name = "ansible-control"
178+
ansible_control_disk_size = 100
179+
180+
seed_vm_flavor = "${{ env.MULTINODE_SEED_VM_FLAVOR }}"
181+
seed_disk_size = 100
182+
183+
multinode_flavor = "${{ env.MULTINODE_FLAVOR }}"
184+
multinode_image = "${{ env.MULTINODE_IMAGE }}"
185+
multinode_keypair = "${{ env.MULTINODE_NAME }}"
186+
multinode_vm_network = "${{ env.MULTINODE_NETWORK }}"
187+
multinode_vm_subnet = "${{ env.MULTINODE_SUBNET }}"
188+
compute_count = "${{ env.MULTINODE_COMPUTE_COUNT }}"
189+
controller_count = "${{ env.MULTINODE_CONTROLLER_COUNT }}"
190+
compute_disk_size = 100
191+
controller_disk_size = 100
192+
193+
ssh_public_key = "id_rsa.pub"
194+
ssh_user = "${{ env.SSH_USERNAME }}"
195+
196+
storage_count = "${{ env.MULTINODE_STORAGE_COUNT }}"
197+
storage_flavor = "${{ env.MULTINODE_STORAGE_FLAVOR }}"
198+
storage_disk_size = 100
199+
200+
deploy_wazuh = false
201+
infra_vm_flavor = "${{ env.MULTINODE_INFRA_VM_FLAVOR }}"
202+
infra_vm_disk_size = 100
203+
EOF
204+
205+
if [[ "${{ inputs.ssh_key }}" != "" ]]; then
206+
cat << EOF >> terraform.tfvars
207+
add_ansible_control_fip = true
208+
ansible_control_fip_pool = "${{ env.MULTINODE_FIP_POOL }}"
209+
EOF
210+
fi
211+
working-directory: ${{ github.workspace }}/terraform-kayobe-multinode
212+
env:
213+
MULTINODE_NAME: "${{ inputs.multinode_name }}"
214+
MULTINODE_ANSIBLE_CONTROL_VM_FLAVOR: ${{ vars.multinode_ansible_control_vm_flavor }} # en1.xsmall
215+
MULTINODE_SEED_VM_FLAVOR: ${{ vars.multinode_seed_vm_flavor }} # en1.xsmall
216+
MULTINODE_INFRA_VM_FLAVOR: ${{ vars.multinode_infra_vm_flavor }} # en1.xsmall
217+
MULTINODE_FLAVOR: ${{ vars.multinode_flavor }} # en1.large
218+
MULTINODE_STORAGE_FLAVOR: ${{ vars.multinode_storage_flavor }} # en1.medium
219+
MULTINODE_COMPUTE_COUNT: "${{ inputs.multinode_compute_count }}"
220+
MULTINODE_CONTROLLER_COUNT: "${{ inputs.multinode_controller_count }}"
221+
MULTINODE_STORAGE_COUNT: "${{ inputs.multinode_storage_count }}"
222+
MULTINODE_IMAGE: ${{ steps.image_name.outputs.image_name }}
223+
MULTINODE_NETWORK: ${{ vars.multinode_network }}
224+
MULTINODE_SUBNET: ${{ vars.multinode_subnet }}
225+
MULTINODE_FIP_POOL: ${{ vars.multinode_fip_pool }}
226+
SSH_USERNAME: "${{ inputs.ssh_username }}"
227+
228+
- name: Initialise terraform
229+
run: terraform init
230+
working-directory: ${{ github.workspace }}/terraform-kayobe-multinode
231+
232+
- name: Validate terraform
233+
id: tf_validate
234+
run: terraform validate
235+
working-directory: ${{ github.workspace }}/terraform-kayobe-multinode
236+
237+
- name: Configure Ansible
238+
run: |
239+
echo '${{ env.KAYOBE_VAULT_PASSWORD }}' > vault-pw
240+
241+
cat << EOF >> ansible/vars/defaults.yml
242+
kayobe_config_version: ${{ inputs.upgrade && inputs.stackhpc_kayobe_config_previous_version || inputs.stackhpc_kayobe_config_version }}
243+
ssh_key_path: ${{ github.workspace }}/terraform-kayobe-multinode/id_rsa
244+
vxlan_vni: 53299
245+
vault_password_path: ${{ github.workspace }}/terraform-kayobe-multinode/vault-pw
246+
kayobe_config_custom:
247+
- path: zz-multinode.yml
248+
block: |
249+
os_distribution: ${{ env.OS_DISTRIBUTION }}
250+
os_release: "${{ env.OS_RELEASE }}"
251+
kolla_enable_ovn: ${{ env.ENABLE_OVN }}
252+
EOF
253+
254+
if [[ "${{ env.SSH_KEY }}" != "" ]]; then
255+
cat << EOF >> ansible/vars/defaults.yml
256+
extra_ssh_public_keys:
257+
- "${{ env.SSH_KEY }}"
258+
EOF
259+
fi
260+
working-directory: ${{ github.workspace }}/terraform-kayobe-multinode
261+
env:
262+
ENABLE_OVN: ${{ inputs.neutron_plugin == 'ovn' }}
263+
OS_DISTRIBUTION: ${{ inputs.os_distribution }}
264+
OS_RELEASE: ${{ inputs.os_release }}
265+
SSH_KEY: ${{ inputs.ssh_key }}
266+
267+
- name: Terraform Plan
268+
run: terraform plan -input=false
269+
working-directory: ${{ github.workspace }}/terraform-kayobe-multinode
270+
env:
271+
OS_CLOUD: ${{ vars.OS_CLOUD }}
272+
OS_APPLICATION_CREDENTIAL_ID: ${{ secrets.OS_APPLICATION_CREDENTIAL_ID }}
273+
OS_APPLICATION_CREDENTIAL_SECRET: ${{ secrets.OS_APPLICATION_CREDENTIAL_SECRET }}
274+
275+
- name: Terraform Apply
276+
id: tf_apply
277+
run: |
278+
for attempt in $(seq 3); do
279+
if terraform apply -auto-approve -input=false; then
280+
echo "Created infrastructure on attempt $attempt"
281+
exit 0
282+
fi
283+
echo "Failed to create infrastructure on attempt $attempt"
284+
sleep 60
285+
done
286+
echo "Failed to create infrastructure after $attempt attempts"
287+
exit 1
288+
working-directory: ${{ github.workspace }}/terraform-kayobe-multinode
289+
env:
290+
OS_CLOUD: ${{ vars.OS_CLOUD }}
291+
OS_APPLICATION_CREDENTIAL_ID: ${{ secrets.OS_APPLICATION_CREDENTIAL_ID }}
292+
OS_APPLICATION_CREDENTIAL_SECRET: ${{ secrets.OS_APPLICATION_CREDENTIAL_SECRET }}
293+
294+
- name: Configure Ansible control host
295+
id: config_ach
296+
run: |
297+
source venv/bin/activate &&
298+
ansible-playbook -v -i ansible/inventory.yml ansible/configure-hosts.yml
299+
working-directory: ${{ github.workspace }}/terraform-kayobe-multinode
300+
301+
- name: Deploy OpenStack
302+
run: |
303+
source venv/bin/activate &&
304+
ansible-playbook -v -i ansible/inventory.yml ansible/deploy-openstack.yml
305+
working-directory: ${{ github.workspace }}/terraform-kayobe-multinode
306+
307+
- name: Upgrade Ansible control host
308+
run: |
309+
source venv/bin/activate &&
310+
ansible-playbook -v -i ansible/inventory.yml ansible/deploy-openstack-config.yml -e upgrade=true -e kayobe_config_version=${{ inputs.stackhpc_kayobe_config_version }}
311+
working-directory: ${{ github.workspace }}/terraform-kayobe-multinode
312+
if: inputs.upgrade
313+
314+
- name: Upgrade OpenStack
315+
run: |
316+
source venv/bin/activate &&
317+
ansible-playbook -v -i ansible/inventory.yml ansible/deploy-openstack.yml -e multinode_command=upgrade_overcloud
318+
working-directory: ${{ github.workspace }}/terraform-kayobe-multinode
319+
if: inputs.upgrade
320+
321+
- name: Run Tempest tests
322+
run: |
323+
source venv/bin/activate &&
324+
ansible-playbook -v -i ansible/inventory.yml ansible/deploy-openstack.yml -e multinode_command=run_tempest
325+
working-directory: ${{ github.workspace }}/terraform-kayobe-multinode
326+
if: inputs.upgrade
327+
328+
- name: Download deployment logs
329+
run: |
330+
mkdir -p ${{ github.workspace }}/logs &&
331+
source venv/bin/activate &&
332+
ansible-playbook -v -i ansible/inventory.yml ansible/fetch-logs.yml -e fetch_logs_dest=${{ github.workspace }}/logs
333+
working-directory: ${{ github.workspace }}/terraform-kayobe-multinode
334+
if: ${{ always() && steps.config_ach.outcome == 'success' }}
335+
336+
# NOTE: The tmux log rename is due to GitHub Actions not accepting files with a colon as artifacts.
337+
- name: Fix up deployment log filename
338+
run: |
339+
if [[ -f ${{ github.workspace }}/logs/tmux.kayobe:0.log ]]; then
340+
mv ${{ github.workspace }}/logs/tmux.kayobe:0.log ${{ github.workspace }}/logs/tmux.kayobe.log
341+
fi
342+
working-directory: ${{ github.workspace }}/terraform-kayobe-multinode
343+
if: ${{ always() && steps.config_ach.outcome == 'success' }}
344+
345+
- name: Upload test result artifacts
346+
uses: actions/upload-artifact@v4
347+
with:
348+
name: test-results-multinode-${{ inputs.os_distribution }}-${{ inputs.os_release }}-${{ inputs.neutron_plugin }}${{ inputs.upgrade && '-upgrade' || '' }}
349+
path: |
350+
${{ github.workspace }}/logs/
351+
if: ${{ always() && steps.config_ach.outcome == 'success' }}
352+
353+
- name: Break on failure
354+
run: sleep ${{ inputs.break_duration }}m
355+
if: ${{ failure() && steps.config_ach.outcome == 'success' && inputs.break_on == 'failure' }}
356+
357+
- name: Destroy
358+
run: terraform destroy -auto-approve -input=false
359+
working-directory: ${{ github.workspace }}/terraform-kayobe-multinode
360+
env:
361+
OS_CLOUD: ${{ vars.OS_CLOUD }}
362+
OS_APPLICATION_CREDENTIAL_ID: ${{ secrets.OS_APPLICATION_CREDENTIAL_ID }}
363+
OS_APPLICATION_CREDENTIAL_SECRET: ${{ secrets.OS_APPLICATION_CREDENTIAL_SECRET }}
364+
if: always() && steps.tf_validate.outcome == 'success'

‎README.md‎

Lines changed: 19 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -1,2 +1,19 @@
1-
# stackhpc-openstack-gh-workflows
2-
Reusable GitHub workflows and actions for StackHPC OpenStack
1+
# StackHPC OpenStack GitHub Workflows
2+
3+
Reusable GitHub workflows and actions for StackHPC OpenStack.
4+
5+
## Workflows
6+
7+
The following reusable workflows are provided in the `.github/workflows/`
8+
directory.
9+
10+
## `multinode.yml`
11+
12+
The `multinode.yml` workflow can be used to create a multinode test cluster and
13+
run tests and/or operations against it.
14+
15+
Features:
16+
17+
* Inject an SSH key to access the cluster
18+
* Break (pause) the workflow on failure
19+
* Upgrade from one OpenStack release to another

0 commit comments

Comments
 (0)