Skip to content

Commit 550396f

Browse files
MaxBed4dMaxBed4d
committed
AUFN CI unfinished draft
1 parent 9771e0d commit 550396f

File tree

3 files changed

+393
-2
lines changed

3 files changed

+393
-2
lines changed

‎.github/workflows/deploy-aufn.yml‎

Lines changed: 391 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,391 @@
1+
---
2+
#
3+
4+
name: AUFN KIDDIN' ME?!
5+
6+
on:
7+
workflow_dispatch:
8+
inputs:
9+
deployment_type:
10+
description: Type of deployment
11+
type: choice
12+
options:
13+
- Test
14+
- Deployment
15+
default: Test
16+
lab_vm_count:
17+
description: Total number of Lab VMs to deploy
18+
type: number
19+
required: true
20+
default: 2
21+
reg_pwd: # When using in workflow use ::add-mask::$ to mask the password
22+
description: Password for registry access
23+
type: string
24+
default: "" # NOTE: This needs to be set at runtime via secrets
25+
os_image:
26+
description: Host OS image
27+
type: choice
28+
options:
29+
- Ubuntu
30+
- Rocky9
31+
default: 'Rocky9'
32+
aufn_branch:
33+
description: Which branch of AUFN to use
34+
type: string
35+
default: smslab/2023.1
36+
au_from_seed:
37+
description: Run 'A Universe From Seed'?
38+
type: boolean
39+
default: false
40+
debug_mode:
41+
description: Keep Test up to debug?
42+
type: boolean
43+
default: false
44+
secrets:
45+
BASTION_TEST_PASSWORD:
46+
required: true
47+
CLOUDS_YAML:
48+
required: true
49+
OS_APPLICATION_CREDENTIAL_ID:
50+
required: true
51+
OS_APPLICATION_CREDENTIAL_SECRET:
52+
required: true
53+
54+
jobs:
55+
set-up-vars:
56+
name: Set up variables
57+
environment: ${{ inputs.deployment_type }}
58+
runs-on: Ubuntu-latest
59+
60+
steps:
61+
- name: Install Package
62+
uses: ConorMacBride/install-package@main
63+
with:
64+
apt: git unzip nodejs python3-pip python3-venv openssh-server openssh-client jq
65+
66+
- name: Install sshpass
67+
run: sudo apt-get update && sudo apt-get install -y sshpass
68+
69+
- name: Start the SSH service
70+
run: |
71+
sudo /etc/init.d/ssh start
72+
73+
# - name: Check if 'Deployment' Lab is already deployed
74+
# uses: softwareforgood/check-artifact-v4-existence@v0
75+
# with:
76+
# name: ${{ inputs.deployment_type }}-terraform-artifacts
77+
#
78+
# or use a ping command to check if the bastion is up
79+
#
80+
81+
- name: Checkout
82+
uses: actions/checkout@v4
83+
84+
- name: Install terraform
85+
uses: hashicorp/setup-terraform@v2
86+
87+
- name: Initialise terraform
88+
run: terraform init
89+
90+
- name: Generate clouds.yaml
91+
run: |
92+
cat << EOF > clouds.yaml
93+
${{ secrets.CLOUDS_YAML }}
94+
EOF
95+
96+
- name: Generate terraform.tfvars
97+
run: |
98+
cat << EOF > terraform.tfvars
99+
lab_count = ${{ inputs.lab_vm_count }}
100+
lab_net_ipv4 = "${{ vars.LAB_NETWORK }}"
101+
image_id = "${{ env.LAB_IMAGE_ID }}"
102+
image_name = "${{ env.LAB_IMAGE_NAME }}"
103+
lab_flavor = "aufn.v1.large"
104+
registry_flavor = "general.v1.medium"
105+
boot_labs_from_volume = true
106+
image_user = "${{ env.LAB_IMAGE_USER }}"
107+
allocate_floating_ips = false
108+
create_bastion = "${{ env.LAB_CREATE_BASTION }}"
109+
EOF
110+
111+
# Conditionally append bastion_floating_ip
112+
if [ "${{ inputs.deployment_type }}" = "Deployment" ]; then
113+
echo 'bastion_floating_ip = "185.45.78.149"' >> terraform.tfvars
114+
fi
115+
env:
116+
LAB_IMAGE_ID: ${{ inputs.os_image == 'Rocky9' && vars.LAB_OS_IMAGE_ROCKY || inputs.os_image == 'Ubuntu' && vars.LAB_OS_IMAGE_UBUNTU }}
117+
LAB_IMAGE_NAME: ${{ inputs.os_image == 'Ubuntu' && 'Ubuntu-22.04' || inputs.os_image }}
118+
LAB_IMAGE_USER: ${{ inputs.os_image == 'Ubuntu' && 'ubuntu' || inputs.os_image == 'Rocky9' && 'rocky' }}
119+
LAB_CREATE_BASTION: ${{ inputs.deployment_type == 'Deployment' && 'true' || 'false' }}
120+
121+
- name: Terraform Plan
122+
run: terraform plan
123+
env:
124+
OS_CLOUD: ${{ vars.OS_CLOUD }}
125+
126+
127+
- name: Terraform Apply
128+
id: tf_apply
129+
run: |
130+
for attempt in $(seq 5); do
131+
if terraform apply -auto-approve; then
132+
echo "Created infrastructure on attempt $attempt"
133+
exit 0
134+
fi
135+
echo "Failed to create infrastructure on attempt $attempt"
136+
sleep 10
137+
138+
# Need to add a check to see which part failed and then
139+
# taint and retry once more before declating failure
140+
141+
terraform destroy -auto-approve
142+
sleep 60
143+
done
144+
echo "Failed to create infrastructure after $attempt attempts"
145+
exit 1
146+
env:
147+
OS_CLOUD: ${{ vars.OS_CLOUD }}
148+
149+
- name: Get Terraform outputs
150+
id: tf_outputs
151+
run: |
152+
terraform output -json
153+
154+
- name: Write Terraform outputs
155+
run: |
156+
cat << EOF > tf-outputs.yml
157+
${{ steps.tf_outputs.outputs.stdout }}
158+
EOF
159+
160+
- name: Write out Lab VMs info
161+
run: |
162+
terraform output -raw labs > ssh_list.txt
163+
164+
- name: Update bastion password authentication and set login password
165+
run: |
166+
echo "::add-mask::${{ env.reg_pwd_var }}"
167+
168+
ssh [email protected] -i default.pem <<EOF
169+
echo '${{ env.reg_pwd_var }}' | sudo passwd --stdin rocky
170+
sudo sed -i 's/PasswordAuthentication no/PasswordAuthentication yes/' /etc/ssh/sshd_config.d/50-cloud-init.conf
171+
sudo systemctl restart sshd
172+
EOF
173+
shell: bash
174+
env:
175+
reg_pwd_var: ${{ inputs.reg_pwd == '' && 'secrets.BASTION_TEST_PASSWORD' || inputs.reg_pwd }}
176+
177+
- name: Check connection to Lab VMs
178+
run: |
179+
bastion_ip=185.45.78.149
180+
bastion_key="default.pem"
181+
182+
while IFS= read -r line; do
183+
ip=$(echo "$line" | awk '{print $2}')
184+
name=$(echo "$line" | awk '{print $3}')
185+
password=$(echo "$line" | awk '{print $5}')
186+
187+
echo "::add-mask::$password"
188+
189+
echo "Connecting to $name at $ip via bastion..."
190+
191+
sshpass -p "$password" ssh -o StrictHostKeyChecking=no \
192+
-o ProxyJump="${LAB_IMAGE_USER}@${bastion_ip}" \
193+
-o IdentityFile=$bastion_key \
194+
"${LAB_IMAGE_USER}@${ip}" \
195+
'echo "Connected to $(hostname)"'
196+
done < ssh_list.txt
197+
shell: bash
198+
env:
199+
LAB_IMAGE_USER: ${{ inputs.os_image == 'Ubuntu' && 'ubuntu' || inputs.os_image == 'Rocky9' && 'rocky' }}
200+
201+
- name: Validate lab VMs setup
202+
run: |
203+
bastion_ip=185.45.78.149
204+
bastion_key="default.pem"
205+
index=0
206+
failed_indexes=()
207+
208+
while IFS= read -r line; do
209+
ip=$(echo "$line" | awk '{print $2}')
210+
name=$(echo "$line" | awk '{print $3}')
211+
password=$(echo "$line" | awk '{print $5}')
212+
taint="false"
213+
214+
echo "::add-mask::$password"
215+
echo "Connecting to $name at $ip..."
216+
217+
# Run the compound remote commands
218+
sshpass -p "$password" ssh -o StrictHostKeyChecking=no \
219+
-o ProxyJump="${LAB_IMAGE_USER}@${bastion_ip}" \
220+
-o IdentityFile=$bastion_key \
221+
"${LAB_IMAGE_USER}@${ip}" <<'EOF'
222+
223+
echo Checking 'virsh list --all'..."
224+
output=$(sudo virsh list --all)
225+
echo "$output"
226+
227+
if ! echo "$output" | grep -q 'seed.*running'; then echo "'seed' not running"; taint="true"; fi
228+
if ! echo "$output" | grep -q 'compute0.*shut off'; then echo "'compute0' not shut off"; taint="true"; fi
229+
if ! echo "$output" | grep -q 'controller0.*shut off'; then echo "'controller0' not shut off"; taint="true"; fi
230+
231+
echo "Checking 'bifrost_deploy' container..."
232+
container_output=$(ssh [email protected] 'sudo docker ps')
233+
echo "$container_output"
234+
if ! echo "$container_output" | grep -q bifrost_deploy; then echo "Container bifrost_deploy not found running"; taint="true"; fi
235+
236+
echo "Checking openssh package source..."
237+
pkg_output=$(ssh [email protected] 'sudo dnf info openssh')
238+
echo "$pkg_output"
239+
if ! echo "$pkg_output" | grep -q 'Repository *: *@System'; then echo "Package openssh not from @System"; taint="true"; fi
240+
241+
echo "Checking a-seed-from-nothing.out log result..."
242+
if ! tail -n 10 a-seed-from-nothing.out | grep -q 'PLAY RECAP.*failed=0'; then
243+
echo "Ansible PLAY RECAP failed != 0"
244+
taint="true"
245+
fi
246+
247+
echo "All checks passed on $HOSTNAME"
248+
EOF
249+
if [ "$taint" == "true" ]; then failed_indexes+=($index); fi
250+
index=$((index + 1))
251+
252+
done < ssh_list.txt
253+
echo "FAILED_VM_INDEXES=${failed_indexes[*]}" >> $GITHUB_ENV
254+
shell: bash
255+
env:
256+
LAB_IMAGE_USER: ${{ inputs.os_image == 'Ubuntu' && 'ubuntu' || inputs.os_image == 'Rocky9' && 'rocky' }}
257+
258+
- name: Taint failed lab VMs (if any)
259+
run: |
260+
if [ -z "${FAILED_VM_INDEXES}" ]; then
261+
echo "No failed VMs detected"
262+
exit 0
263+
fi
264+
265+
for idx in $FAILED_VM_INDEXES; do
266+
echo "Tainting openstack_compute_instance_v2.lab[$idx]"
267+
terraform taint "openstack_compute_instance_v2.lab[$idx]"
268+
done
269+
270+
echo "Re-running Terraform apply to fix failed VMs"
271+
terraform apply -auto-approve
272+
env:
273+
FAILED_VM_INDEXES: ${{ env.FAILED_VM_INDEXES }}
274+
shell: bash
275+
276+
- name: Get Terraform outputs
277+
id: tf_outputs_after_taint
278+
run: |
279+
terraform output -json
280+
281+
- name: Write Terraform outputs
282+
run: |
283+
cat << EOF > tf-outputs.yml
284+
${{ steps.tf_outputs_after_taint.outputs.stdout }}
285+
EOF
286+
287+
- name: Write out Lab VMs info
288+
run: |
289+
terraform output -raw labs > ssh_list.txt
290+
291+
- name: Re-test failed lab VMs after redeploy
292+
run: |
293+
set -euo pipefail
294+
295+
bastion_ip=185.45.78.149
296+
bastion_key="default.pem"
297+
mapfile -t ssh_lines < ssh_list.txt
298+
299+
for idx in $FAILED_VM_INDEXES; do
300+
line="${ssh_lines[$idx]}"
301+
ip=$(echo "$line" | awk '{print $2}')
302+
name=$(echo "$line" | awk '{print $3}')
303+
password=$(echo "$line" | awk '{print $5}')
304+
305+
echo "::add-mask::$password"
306+
echo "Re-testing $name at $ip (index $idx)..."
307+
308+
sshpass -p "$password" ssh -o StrictHostKeyChecking=no \
309+
-o ProxyJump="${LAB_IMAGE_USER}@${bastion_ip}" \
310+
-o IdentityFile=$bastion_key \
311+
"${LAB_IMAGE_USER}@${ip}" <<'EOF' || {
312+
echo "Post-deploy check failed on $name. Destroying all infrastructure..."
313+
terraform destroy -auto-approve
314+
exit 1
315+
}
316+
317+
echo "Re-checking virsh VMs..."
318+
output=$(sudo virsh list --all)
319+
echo "$output"
320+
if ! echo "$output" | grep -q 'seed.*running'; then echo "'seed' not running"; exit 1; fi
321+
if ! echo "$output" | grep -q 'compute0.*shut off'; then echo "'compute0' not shut off"; exit 1; fi
322+
if ! echo "$output" | grep -q 'controller0.*shut off'; then echo "'controller0' not shut off"; exit 1; fi
323+
324+
echo "Checking bifrost container..."
325+
if ! ssh [email protected] 'sudo docker ps' | grep -q bifrost_deploy; then
326+
echo "bifrost_deploy container not running"; exit 1;
327+
fi
328+
329+
echo "Checking openssh package source..."
330+
if ! ssh [email protected] 'sudo dnf info openssh' | grep -q 'Repository *: *@System'; then
331+
echo "openssh not from @System"; exit 1;
332+
fi
333+
334+
echo "Checking a-seed-from-nothing.out for Ansible success..."
335+
if ! tail -n 20 a-seed-from-nothing.out | grep -q 'PLAY RECAP.*failed=0'; then
336+
echo "Ansible PLAY RECAP shows failures"; exit 1;
337+
fi
338+
339+
echo "All post-redeploy checks passed on $HOSTNAME"
340+
EOF
341+
342+
done
343+
shell: bash
344+
env:
345+
LAB_IMAGE_USER: ${{ inputs.os_image == 'Ubuntu' && 'ubuntu' || inputs.os_image == 'Rocky9' && 'rocky' }}
346+
FAILED_VM_INDEXES: ${{ env.FAILED_VM_INDEXES }}
347+
348+
- name: Run a-universe-from-seed.sh if true
349+
if: inputs.au_from_seed == true
350+
run: |
351+
bastion_ip=185.45.78.149
352+
bastion_key="default.pem"
353+
354+
mapfile -t ssh_lines < ssh_list.txt
355+
356+
for i in "${!ssh_lines[@]}"; do
357+
line="${ssh_lines[$i]}"
358+
ip=$(echo "$line" | awk '{print $2}')
359+
name=$(echo "$line" | awk '{print $3}')
360+
password=$(echo "$line" | awk '{print $5}')
361+
362+
echo "::add-mask::$password"
363+
echo "Launching a-universe-from-seed.sh on $name at $ip in tmux..."
364+
365+
sshpass -p "$password" ssh -o StrictHostKeyChecking=no \
366+
-o ProxyJump="${LAB_IMAGE_USER}@${bastion_ip}" \
367+
-o IdentityFile=$bastion_key \
368+
"${LAB_IMAGE_USER}@${ip}" \
369+
"tmux new-session -d -s aus-run './a-universe-from-seed.sh'"
370+
done
371+
shell: bash
372+
env:
373+
LAB_IMAGE_USER: ${{ inputs.os_image == 'Ubuntu' && 'ubuntu' || inputs.os_image == 'Rocky9' && 'rocky' }}
374+
375+
# - name: Run test workflow
376+
# if: inputs.deployment_type == 'Test'
377+
# uses: ./.github/workflows/AUFN-test.yml
378+
379+
# - name: Upload Terraform outputs
380+
# if: ${{ inputs.deployment_type == 'Deployment' || inputs.debug_mode == true }}
381+
# uses: actions/upload-artifact@v4
382+
# with:
383+
# name: ${{ inputs.deployment_type }}-terraform-artifacts
384+
385+
- name: Destroy
386+
run: terraform destroy -auto-approve
387+
env:
388+
OS_CLOUD: ${{ vars.OS_CLOUD }}
389+
OS_APPLICATION_CREDENTIAL_ID: ${{ secrets.OS_APPLICATION_CREDENTIAL_ID }}
390+
OS_APPLICATION_CREDENTIAL_SECRET: ${{ secrets.OS_APPLICATION_CREDENTIAL_SECRET }}
391+
if: always()

0 commit comments

Comments
 (0)