Skip to content

Commit de532a8

Browse files
markgoddardmarkgoddard
authored
Merge pull request #1 from stackhpc/multinode-workflow
Add multinode.yml workflow
2 parents 398c3fe + fdc5350 commit de532a8

File tree

3 files changed

+400
-2
lines changed

3 files changed

+400
-2
lines changed

‎.github/CODEOWNERS‎

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1 @@
1+
* @stackhpc/kayobe

‎.github/workflows/multinode.yml‎

Lines changed: 380 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,380 @@
1+
---
2+
# This reusable workflow deploys a multi-node test cluster on a cloud using
3+
# Terraform, then deploys OpenStack via Kayobe. Tempest is then used to test
4+
# the cloud.
5+
6+
name: Multinode
7+
on:
8+
workflow_call:
9+
inputs:
10+
multinode_name:
11+
description: Multinode cluster name
12+
type: string
13+
required: true
14+
multinode_controller_count:
15+
description: Controller count
16+
type: number
17+
default: 3
18+
multinode_compute_count:
19+
description: Compute count
20+
type: number
21+
default: 2
22+
multinode_storage_count:
23+
description: Storage count
24+
type: number
25+
default: 3
26+
os_distribution:
27+
description: Host OS distribution
28+
type: string
29+
default: rocky
30+
os_release:
31+
description: Host OS release
32+
type: string
33+
default: '9'
34+
ssh_username:
35+
description: User for terraform to access the Multinode hosts
36+
type: string
37+
default: cloud-user
38+
neutron_plugin:
39+
description: Neutron ML2 plugin
40+
type: string
41+
default: ovn
42+
stackhpc_kayobe_config_version:
43+
description: stackhpc-kayobe-config version
44+
type: string
45+
required: true
46+
stackhpc_kayobe_config_previous_version:
47+
description: stackhpc-kayobe-config previous version
48+
type: string
49+
terraform_kayobe_multinode_version:
50+
description: terraform-kayobe-multinode version
51+
type: string
52+
default: main
53+
upgrade:
54+
description: Whether to perform an upgrade
55+
type: boolean
56+
default: false
57+
break_on:
58+
# Supported values: 'always', 'never', 'failure', 'success'
59+
description: When to break execution for manual interaction
60+
type: string
61+
default: never
62+
break_duration:
63+
description: How long to break execution for (minutes)
64+
type: number
65+
default: 60
66+
ssh_key:
67+
description: SSH public key to authorise on Ansible control host
68+
type: string
69+
secrets:
70+
KAYOBE_VAULT_PASSWORD_CI_MULTINODE:
71+
required: true
72+
CLOUDS_YAML:
73+
required: true
74+
OS_APPLICATION_CREDENTIAL_ID:
75+
required: true
76+
OS_APPLICATION_CREDENTIAL_SECRET:
77+
required: true
78+
79+
jobs:
80+
multinode:
81+
name: Multinode
82+
runs-on: arc-skc-aio-runner
83+
environment: Leafcloud
84+
permissions: {}
85+
env:
86+
ANSIBLE_FORCE_COLOR: True
87+
KAYOBE_ENVIRONMENT: ci-multinode
88+
KAYOBE_VAULT_PASSWORD: ${{ secrets.KAYOBE_VAULT_PASSWORD_CI_MULTINODE }}
89+
steps:
90+
- name: Fail if previous version is not defined
91+
run: |
92+
echo "StackHPC Kayobe Configuration previous version must be defined for upgrades"
93+
exit 1
94+
if: ${{ inputs.upgrade && inputs.stackhpc_kayobe_config_previous_version == '' }}
95+
96+
- name: Install Package
97+
uses: ConorMacBride/install-package@main
98+
with:
99+
apt: git unzip nodejs python3-pip python3-venv rsync
100+
101+
# If testing upgrade, checkout previous release, otherwise checkout current branch
102+
- name: Checkout ${{ inputs.upgrade && 'previous release' || 'current' }} config
103+
uses: actions/checkout@v4
104+
with:
105+
repository: stackhpc/stackhpc-kayobe-config
106+
ref: ${{ inputs.upgrade && inputs.stackhpc_kayobe_config_previous_version || inputs.stackhpc_kayobe_config_version }}
107+
108+
- name: Checkout terraform-kayobe-multinode
109+
uses: actions/checkout@v4
110+
with:
111+
repository: stackhpc/terraform-kayobe-multinode
112+
ref: ${{ inputs.terraform_kayobe_multinode_version }}
113+
path: terraform-kayobe-multinode
114+
115+
- name: Make sure dockerd is running and test Docker
116+
run: |
117+
docker ps
118+
119+
- name: Output image tag
120+
id: image_tag
121+
run: |
122+
echo image_tag=$(grep stackhpc_${{ inputs.os_distribution }}_$(sed s/-/_/ <(echo "${{ inputs.os_release }}"))_overcloud_host_image_version: etc/kayobe/pulp-host-image-versions.yml | awk '{print $2}') >> $GITHUB_OUTPUT
123+
124+
# Use the image override if set, otherwise use overcloud-os_distribution-os_release-tag
125+
- name: Output image name
126+
id: image_name
127+
run: |
128+
if [ -z "${{ inputs.multinode_image_override }}" ]; then
129+
echo image_name=overcloud-${{ inputs.os_distribution }}-${{ inputs.os_release }}-${{ steps.image_tag.outputs.image_tag }} >> $GITHUB_OUTPUT
130+
else
131+
echo image_name=${{ inputs.multinode_image_override }} >> $GITHUB_OUTPUT
132+
fi
133+
134+
- name: Install terraform
135+
uses: hashicorp/setup-terraform@v2
136+
with:
137+
terraform_wrapper: false
138+
139+
- name: Setup Ansible
140+
run: |
141+
python3 -m venv venv &&
142+
source venv/bin/activate &&
143+
pip install -U pip &&
144+
pip install ansible &&
145+
mkdir -p ansible/{collections,roles} &&
146+
ansible-galaxy role install -r ansible/requirements.yml -p ansible/roles &&
147+
ansible-galaxy collection install -r ansible/requirements.yml -p ansible/collections
148+
working-directory: ${{ github.workspace }}/terraform-kayobe-multinode
149+
150+
- name: Generate a VXLAN VNI
151+
id: vxlan_vni
152+
run: |
153+
# There is an undocumented restriction limiting us to a max VNI of
154+
# 100,000.
155+
max_vni=100000
156+
timestamp=$(date +%s)
157+
vni=$(((timestamp % max_vni) + 1))
158+
echo vxlan_vni=$vni >> $GITHUB_OUTPUT
159+
160+
- name: Generate SSH keypair
161+
run: ssh-keygen -f id_rsa -N ''
162+
working-directory: ${{ github.workspace }}/terraform-kayobe-multinode
163+
164+
# NOTE: In Ansible 2.10 and lower the synchronize module used in the
165+
# ansible/fetch-logs.yml playbook does not respect SSH connection
166+
# variables. This may result in Permission Denied issues if using an SSH
167+
# key that is not in ~/.ssh.
168+
- name: Copy SSH keypair to .ssh/
169+
run: |
170+
install -d ~/.ssh -m 700 &&
171+
cp id_rsa* ~/.ssh/
172+
working-directory: ${{ github.workspace }}/terraform-kayobe-multinode
173+
174+
- name: Generate clouds.yaml
175+
run: |
176+
cat << EOF > clouds.yaml
177+
${{ secrets.CLOUDS_YAML }}
178+
EOF
179+
working-directory: ${{ github.workspace }}/terraform-kayobe-multinode
180+
181+
- name: Generate terraform.tfvars
182+
run: |
183+
cat << EOF > terraform.tfvars
184+
185+
prefix = "${{ env.MULTINODE_NAME }}"
186+
187+
ansible_control_vm_flavor = "${{ env.MULTINODE_ANSIBLE_CONTROL_VM_FLAVOR }}"
188+
ansible_control_vm_name = "ansible-control"
189+
ansible_control_disk_size = 100
190+
191+
seed_vm_flavor = "${{ env.MULTINODE_SEED_VM_FLAVOR }}"
192+
seed_disk_size = 100
193+
194+
multinode_flavor = "${{ env.MULTINODE_FLAVOR }}"
195+
multinode_image = "${{ env.MULTINODE_IMAGE }}"
196+
multinode_keypair = "${{ env.MULTINODE_NAME }}"
197+
multinode_vm_network = "${{ env.MULTINODE_NETWORK }}"
198+
multinode_vm_subnet = "${{ env.MULTINODE_SUBNET }}"
199+
compute_count = "${{ env.MULTINODE_COMPUTE_COUNT }}"
200+
controller_count = "${{ env.MULTINODE_CONTROLLER_COUNT }}"
201+
compute_disk_size = 100
202+
controller_disk_size = 100
203+
204+
ssh_public_key = "id_rsa.pub"
205+
ssh_user = "${{ env.SSH_USERNAME }}"
206+
207+
storage_count = "${{ env.MULTINODE_STORAGE_COUNT }}"
208+
storage_flavor = "${{ env.MULTINODE_STORAGE_FLAVOR }}"
209+
storage_disk_size = 100
210+
211+
deploy_wazuh = false
212+
infra_vm_flavor = "${{ env.MULTINODE_INFRA_VM_FLAVOR }}"
213+
infra_vm_disk_size = 100
214+
EOF
215+
216+
if [[ "${{ inputs.ssh_key }}" != "" ]]; then
217+
cat << EOF >> terraform.tfvars
218+
add_ansible_control_fip = true
219+
ansible_control_fip_pool = "${{ env.MULTINODE_FIP_POOL }}"
220+
EOF
221+
fi
222+
working-directory: ${{ github.workspace }}/terraform-kayobe-multinode
223+
env:
224+
MULTINODE_NAME: "${{ inputs.multinode_name }}"
225+
MULTINODE_ANSIBLE_CONTROL_VM_FLAVOR: ${{ vars.multinode_ansible_control_vm_flavor }} # en1.xsmall
226+
MULTINODE_SEED_VM_FLAVOR: ${{ vars.multinode_seed_vm_flavor }} # en1.xsmall
227+
MULTINODE_INFRA_VM_FLAVOR: ${{ vars.multinode_infra_vm_flavor }} # en1.xsmall
228+
MULTINODE_FLAVOR: ${{ vars.multinode_flavor }} # en1.large
229+
MULTINODE_STORAGE_FLAVOR: ${{ vars.multinode_storage_flavor }} # en1.medium
230+
MULTINODE_COMPUTE_COUNT: "${{ inputs.multinode_compute_count }}"
231+
MULTINODE_CONTROLLER_COUNT: "${{ inputs.multinode_controller_count }}"
232+
MULTINODE_STORAGE_COUNT: "${{ inputs.multinode_storage_count }}"
233+
MULTINODE_IMAGE: ${{ steps.image_name.outputs.image_name }}
234+
MULTINODE_NETWORK: ${{ vars.multinode_network }}
235+
MULTINODE_SUBNET: ${{ vars.multinode_subnet }}
236+
MULTINODE_FIP_POOL: ${{ vars.multinode_fip_pool }}
237+
SSH_USERNAME: "${{ inputs.ssh_username }}"
238+
239+
- name: Initialise terraform
240+
run: terraform init
241+
working-directory: ${{ github.workspace }}/terraform-kayobe-multinode
242+
243+
- name: Validate terraform
244+
id: tf_validate
245+
run: terraform validate
246+
working-directory: ${{ github.workspace }}/terraform-kayobe-multinode
247+
248+
- name: Configure Ansible
249+
run: |
250+
echo '${{ env.KAYOBE_VAULT_PASSWORD }}' > vault-pw
251+
252+
cat << EOF >> ansible/vars/defaults.yml
253+
kayobe_config_version: ${{ inputs.upgrade && inputs.stackhpc_kayobe_config_previous_version || inputs.stackhpc_kayobe_config_version }}
254+
ssh_key_path: ${{ github.workspace }}/terraform-kayobe-multinode/id_rsa
255+
vxlan_vni: ${{ steps.vxlan_vni.outputs.vxlan_vni }}
256+
vault_password_path: ${{ github.workspace }}/terraform-kayobe-multinode/vault-pw
257+
kayobe_config_custom:
258+
- path: zz-multinode.yml
259+
block: |
260+
os_distribution: ${{ env.OS_DISTRIBUTION }}
261+
os_release: "${{ env.OS_RELEASE }}"
262+
kolla_enable_ovn: ${{ env.ENABLE_OVN }}
263+
EOF
264+
265+
if [[ "${{ env.SSH_KEY }}" != "" ]]; then
266+
cat << EOF >> ansible/vars/defaults.yml
267+
extra_ssh_public_keys:
268+
- "${{ env.SSH_KEY }}"
269+
EOF
270+
fi
271+
working-directory: ${{ github.workspace }}/terraform-kayobe-multinode
272+
env:
273+
ENABLE_OVN: ${{ inputs.neutron_plugin == 'ovn' }}
274+
OS_DISTRIBUTION: ${{ inputs.os_distribution }}
275+
OS_RELEASE: ${{ inputs.os_release }}
276+
SSH_KEY: ${{ inputs.ssh_key }}
277+
278+
- name: Terraform Plan
279+
run: terraform plan -input=false
280+
working-directory: ${{ github.workspace }}/terraform-kayobe-multinode
281+
env:
282+
OS_CLOUD: ${{ vars.OS_CLOUD }}
283+
OS_APPLICATION_CREDENTIAL_ID: ${{ secrets.OS_APPLICATION_CREDENTIAL_ID }}
284+
OS_APPLICATION_CREDENTIAL_SECRET: ${{ secrets.OS_APPLICATION_CREDENTIAL_SECRET }}
285+
286+
- name: Terraform Apply
287+
run: |
288+
for attempt in $(seq 3); do
289+
if terraform apply -auto-approve -input=false; then
290+
echo "Created infrastructure on attempt $attempt"
291+
exit 0
292+
fi
293+
echo "Failed to create infrastructure on attempt $attempt"
294+
sleep 60
295+
done
296+
echo "Failed to create infrastructure after $attempt attempts"
297+
exit 1
298+
working-directory: ${{ github.workspace }}/terraform-kayobe-multinode
299+
env:
300+
OS_CLOUD: ${{ vars.OS_CLOUD }}
301+
OS_APPLICATION_CREDENTIAL_ID: ${{ secrets.OS_APPLICATION_CREDENTIAL_ID }}
302+
OS_APPLICATION_CREDENTIAL_SECRET: ${{ secrets.OS_APPLICATION_CREDENTIAL_SECRET }}
303+
304+
- name: Configure Ansible control host
305+
id: config_ach
306+
run: |
307+
source venv/bin/activate &&
308+
ansible-playbook -v -i ansible/inventory.yml ansible/configure-hosts.yml
309+
working-directory: ${{ github.workspace }}/terraform-kayobe-multinode
310+
311+
- name: Deploy OpenStack
312+
run: |
313+
source venv/bin/activate &&
314+
ansible-playbook -v -i ansible/inventory.yml ansible/deploy-openstack.yml
315+
working-directory: ${{ github.workspace }}/terraform-kayobe-multinode
316+
317+
- name: Upgrade Ansible control host
318+
run: |
319+
source venv/bin/activate &&
320+
ansible-playbook -v -i ansible/inventory.yml ansible/deploy-openstack-config.yml -e upgrade=true -e kayobe_config_version=${{ inputs.stackhpc_kayobe_config_version }}
321+
working-directory: ${{ github.workspace }}/terraform-kayobe-multinode
322+
if: inputs.upgrade
323+
324+
- name: Upgrade OpenStack
325+
run: |
326+
source venv/bin/activate &&
327+
ansible-playbook -v -i ansible/inventory.yml ansible/deploy-openstack.yml -e multinode_command=upgrade_overcloud
328+
working-directory: ${{ github.workspace }}/terraform-kayobe-multinode
329+
if: inputs.upgrade
330+
331+
- name: Run Tempest tests
332+
run: |
333+
source venv/bin/activate &&
334+
ansible-playbook -v -i ansible/inventory.yml ansible/deploy-openstack.yml -e multinode_command=run_tempest
335+
working-directory: ${{ github.workspace }}/terraform-kayobe-multinode
336+
if: inputs.upgrade
337+
338+
- name: Download deployment logs
339+
run: |
340+
mkdir -p ${{ github.workspace }}/logs &&
341+
source venv/bin/activate &&
342+
ansible-playbook -v -i ansible/inventory.yml ansible/fetch-logs.yml -e fetch_logs_dest=${{ github.workspace }}/logs
343+
working-directory: ${{ github.workspace }}/terraform-kayobe-multinode
344+
if: ${{ always() && steps.config_ach.outcome == 'success' }}
345+
346+
# NOTE: The tmux log rename is due to GitHub Actions not accepting files with a colon as artifacts.
347+
- name: Fix up deployment log filename
348+
run: |
349+
if [[ -f ${{ github.workspace }}/logs/tmux.kayobe:0.log ]]; then
350+
mv ${{ github.workspace }}/logs/tmux.kayobe:0.log ${{ github.workspace }}/logs/tmux.kayobe.log
351+
fi
352+
working-directory: ${{ github.workspace }}/terraform-kayobe-multinode
353+
if: ${{ always() && steps.config_ach.outcome == 'success' }}
354+
355+
- name: Upload test result artifacts
356+
uses: actions/upload-artifact@v4
357+
with:
358+
name: test-results-multinode-${{ inputs.os_distribution }}-${{ inputs.os_release }}-${{ inputs.neutron_plugin }}${{ inputs.upgrade && '-upgrade' || '' }}
359+
path: |
360+
${{ github.workspace }}/logs/
361+
if: ${{ always() && steps.config_ach.outcome == 'success' }}
362+
363+
- name: Break on failure
364+
run: |
365+
sleep ${{ env.break_duration }}m
366+
if: ${{ failure() && steps.config_ach.outcome == 'success' && contains(fromJSON('["failure", "always"]'), env.break_on) }}
367+
368+
- name: Break on success
369+
run: |
370+
sleep ${{ env.break_duration }}m
371+
if: ${{ steps.config_ach.outcome == 'success' && contains(fromJSON('["success", "always"]'), env.break_on) }}
372+
373+
- name: Destroy
374+
run: terraform destroy -auto-approve -input=false
375+
working-directory: ${{ github.workspace }}/terraform-kayobe-multinode
376+
env:
377+
OS_CLOUD: ${{ vars.OS_CLOUD }}
378+
OS_APPLICATION_CREDENTIAL_ID: ${{ secrets.OS_APPLICATION_CREDENTIAL_ID }}
379+
OS_APPLICATION_CREDENTIAL_SECRET: ${{ secrets.OS_APPLICATION_CREDENTIAL_SECRET }}
380+
if: always() && steps.tf_validate.outcome == 'success'

0 commit comments

Comments
 (0)