Skip to content

Commit 300a504

Browse files
MaxBed4dMaxBed4d
committed
Split workflows into two tests
1 parent 550396f commit 300a504

File tree

2 files changed

+253
-226
lines changed

2 files changed

+253
-226
lines changed

‎.github/workflows/AUFN-Test.yml‎

Lines changed: 231 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,231 @@
1+
---
2+
# This reusable workflow deploys a VM on a cloud using Terraform, then deploys
3+
# OpenStack in the VM via Kayobe. Tempest is then used to test the cloud.
4+
5+
name: AUFN Test
6+
7+
on:
8+
workflow_call:
9+
inputs:
10+
au_from_seed:
11+
description: Run 'A Universe From Seed'?
12+
type: boolean
13+
default: false
14+
aufn-runner-id:
15+
description: ID of the runner from the parent workflow
16+
type: string
17+
required: true
18+
os_image:
19+
description: Host OS image
20+
type: string
21+
required: true
22+
secrets:
23+
BASTION_TEST_PASSWORD:
24+
required: true
25+
CLOUDS_YAML:
26+
required: true
27+
OS_APPLICATION_CREDENTIAL_ID:
28+
required: true
29+
OS_APPLICATION_CREDENTIAL_SECRET:
30+
required: true
31+
32+
jobs:
33+
deployment-lab-testing:
34+
name: Test Lab VMs Setup
35+
environment: Test
36+
runs-on: ${{ inputs.aufn-runner-id }}
37+
steps:
38+
39+
- name: Check connection to Lab VMs
40+
run: |
41+
while IFS= read -r line; do
42+
ip=$(echo "$line" | awk '{print $2}')
43+
name=$(echo "$line" | awk '{print $3}')
44+
password=$(echo "$line" | awk '{print $5}')
45+
46+
echo "::add-mask::$password"
47+
48+
echo "Connecting to $name at $ip via bastion..."
49+
50+
sshpass -p "$password" ssh -o StrictHostKeyChecking=no \
51+
"${LAB_IMAGE_USER}@${ip}" \
52+
'echo "Connected to $(hostname)"'
53+
done < ssh_list.txt
54+
shell: bash
55+
env:
56+
LAB_IMAGE_USER: ${{ inputs.os_image == 'Ubuntu' && 'ubuntu' || inputs.os_image == 'Rocky9' && 'rocky' }}
57+
58+
- name: Validate lab VMs setup
59+
run: |
60+
index=0
61+
failed_indexes=()
62+
63+
while IFS= read -r line; do
64+
ip=$(echo "$line" | awk '{print $2}')
65+
name=$(echo "$line" | awk '{print $3}')
66+
password=$(echo "$line" | awk '{print $5}')
67+
taint="false"
68+
69+
echo "::add-mask::$password"
70+
echo "Connecting to $name at $ip..."
71+
72+
# Run the compound remote commands
73+
sshpass -p "$password" ssh -o StrictHostKeyChecking=no \
74+
"${LAB_IMAGE_USER}@${ip}" <<'EOF'
75+
76+
echo Checking 'virsh list --all'..."
77+
output=$(sudo virsh list --all)
78+
echo "$output"
79+
80+
if ! echo "$output" | grep -q 'seed.*running'; then echo "'seed' not running"; taint="true"; fi
81+
if ! echo "$output" | grep -q 'compute0.*shut off'; then echo "'compute0' not shut off"; taint="true"; fi
82+
if ! echo "$output" | grep -q 'controller0.*shut off'; then echo "'controller0' not shut off"; taint="true"; fi
83+
84+
echo "Checking 'bifrost_deploy' container..."
85+
container_output=$(ssh [email protected] 'sudo docker ps')
86+
echo "$container_output"
87+
if ! echo "$container_output" | grep -q bifrost_deploy; then echo "Container bifrost_deploy not found running"; taint="true"; fi
88+
89+
echo "Checking openssh package source..."
90+
pkg_output=$(ssh [email protected] 'sudo dnf info openssh')
91+
echo "$pkg_output"
92+
if ! echo "$pkg_output" | grep -q 'Repository *: *@System'; then echo "Package openssh not from @System"; taint="true"; fi
93+
94+
echo "Checking a-seed-from-nothing.out log result..."
95+
if ! tail -n 10 a-seed-from-nothing.out | grep -q 'PLAY RECAP.*failed=0'; then
96+
echo "Ansible PLAY RECAP failed != 0"
97+
taint="true"
98+
fi
99+
100+
echo "All checks passed on $HOSTNAME"
101+
EOF
102+
if [ "$taint" == "true" ]; then failed_indexes+=($index); fi
103+
index=$((index + 1))
104+
105+
done < ssh_list.txt
106+
echo "FAILED_VM_INDEXES=${failed_indexes[*]}" >> $GITHUB_ENV
107+
shell: bash
108+
env:
109+
LAB_IMAGE_USER: ${{ inputs.os_image == 'Ubuntu' && 'ubuntu' || inputs.os_image == 'Rocky9' && 'rocky' }}
110+
111+
- name: Taint failed lab VMs (if any)
112+
run: |
113+
if [ -z "${FAILED_VM_INDEXES}" ]; then
114+
echo "No failed VMs detected"
115+
exit 0
116+
fi
117+
118+
for idx in $FAILED_VM_INDEXES; do
119+
echo "Tainting openstack_compute_instance_v2.lab[$idx]"
120+
terraform taint "openstack_compute_instance_v2.lab[$idx]"
121+
done
122+
123+
echo "Re-running Terraform apply to fix failed VMs"
124+
terraform apply -auto-approve
125+
env:
126+
FAILED_VM_INDEXES: ${{ env.FAILED_VM_INDEXES }}
127+
OS_CLOUD: ${{ vars.OS_CLOUD }}
128+
shell: bash
129+
130+
- name: Get Terraform outputs
131+
id: tf_outputs_after_taint
132+
run: |
133+
terraform output -json
134+
135+
- name: Write Terraform outputs
136+
run: |
137+
cat << EOF > tf-outputs.yml
138+
${{ steps.tf_outputs_after_taint.outputs.stdout }}
139+
EOF
140+
141+
- name: Write out Lab VMs info
142+
run: |
143+
terraform output -raw labs > ssh_list.txt
144+
145+
- name: Re-test failed lab VMs after redeploy
146+
run: |
147+
set -euo pipefail
148+
149+
mapfile -t ssh_lines < ssh_list.txt
150+
151+
for idx in $FAILED_VM_INDEXES; do
152+
line="${ssh_lines[$idx]}"
153+
ip=$(echo "$line" | awk '{print $2}')
154+
name=$(echo "$line" | awk '{print $3}')
155+
password=$(echo "$line" | awk '{print $5}')
156+
157+
echo "::add-mask::$password"
158+
echo "Re-testing $name at $ip (index $idx)..."
159+
160+
sshpass -p "$password" ssh -o StrictHostKeyChecking=no \
161+
"${LAB_IMAGE_USER}@${ip}" <<'EOF' || {
162+
echo "Post-deploy check failed on $name. Destroying all infrastructure..."
163+
terraform destroy -auto-approve
164+
exit 1
165+
}
166+
167+
echo "Re-checking virsh VMs..."
168+
output=$(sudo virsh list --all)
169+
echo "$output"
170+
if ! echo "$output" | grep -q 'seed.*running'; then echo "'seed' not running"; exit 1; fi
171+
if ! echo "$output" | grep -q 'compute0.*shut off'; then echo "'compute0' not shut off"; exit 1; fi
172+
if ! echo "$output" | grep -q 'controller0.*shut off'; then echo "'controller0' not shut off"; exit 1; fi
173+
174+
echo "Checking bifrost container..."
175+
if ! ssh [email protected] 'sudo docker ps' | grep -q bifrost_deploy; then
176+
echo "bifrost_deploy container not running"; exit 1;
177+
fi
178+
179+
echo "Checking openssh package source..."
180+
if ! ssh [email protected] 'sudo dnf info openssh' | grep -q 'Repository *: *@System'; then
181+
echo "openssh not from @System"; exit 1;
182+
fi
183+
184+
echo "Checking a-seed-from-nothing.out for Ansible success..."
185+
if ! tail -n 20 a-seed-from-nothing.out | grep -q 'PLAY RECAP.*failed=0'; then
186+
echo "Ansible PLAY RECAP shows failures"; exit 1;
187+
fi
188+
189+
echo "All post-redeploy checks passed on $HOSTNAME"
190+
EOF
191+
192+
done
193+
shell: bash
194+
env:
195+
LAB_IMAGE_USER: ${{ inputs.os_image == 'Ubuntu' && 'ubuntu' || inputs.os_image == 'Rocky9' && 'rocky' }}
196+
FAILED_VM_INDEXES: ${{ env.FAILED_VM_INDEXES }}
197+
198+
- name: Run a-universe-from-seed.sh if true
199+
if: inputs.au_from_seed == true
200+
run: |
201+
mapfile -t ssh_lines < ssh_list.txt
202+
203+
for i in "${!ssh_lines[@]}"; do
204+
line="${ssh_lines[$i]}"
205+
ip=$(echo "$line" | awk '{print $2}')
206+
name=$(echo "$line" | awk '{print $3}')
207+
password=$(echo "$line" | awk '{print $5}')
208+
209+
echo "::add-mask::$password"
210+
echo "Launching a-universe-from-seed.sh on $name at $ip in tmux..."
211+
212+
sshpass -p "$password" ssh -o StrictHostKeyChecking=no \
213+
"${LAB_IMAGE_USER}@${ip}" \
214+
"tmux new-session -d -s aus-run './a-universe-from-seed.sh'"
215+
done
216+
shell: bash
217+
env:
218+
LAB_IMAGE_USER: ${{ inputs.os_image == 'Ubuntu' && 'ubuntu' || inputs.os_image == 'Rocky9' && 'rocky' }}
219+
220+
# - name: Run test workflow
221+
# if: inputs.deployment_type == 'Test'
222+
# uses: ./.github/workflows/AUFN-test.yml
223+
224+
# - name: Upload Terraform outputs
225+
# if: ${{ inputs.deployment_type == 'Deployment' || inputs.debug_mode == true }}
226+
# uses: actions/upload-artifact@v4
227+
# with:
228+
# name: ${{ inputs.deployment_type }}-terraform-artifacts
229+
230+
231+

0 commit comments

Comments
 (0)