Skip to content

Commit f8e17fe

Browse files
drosenfebshephar
drosenfe
authored and
bshephar
committed
Create bmh compute replace validator
Add a validator that verifies a faulty bmh compute node may be replaced
1 parent 052cac6 commit f8e17fe

File tree

2 files changed

+202
-0
lines changed

2 files changed

+202
-0
lines changed

‎roles/validations/defaults/main.yml‎

Lines changed: 6 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -51,3 +51,9 @@ cifmw_validations_xml_status_file_dir: "{{ cifmw_validations_basedir }}/tests/va
5151
cifmw_validations_edpm_scale_down_hostname: compute-2.ctlplane.example.com
5252
cifmw_validations_edpm_scale_down_nodename: edpm-compute-2
5353
cifmw_validations_timeout: 100
54+
55+
# variables needed for bmh compute replacement
56+
cifmw_validations_bmh_replace_leaf_label: leaf0-1
57+
cifmw_validations_bmh_spare_leaf_label: leaf0-0
58+
cifmw_validations_bmh_spare_nodename: edpm-compute-0-0
59+
cifmw_validations_bmh_spare_hostname: edpm-compute-0-0.ctlplane.openstack.lab

‎roles/validations/tasks/edpm/bmh_compute_replace.yml‎

Lines changed: 196 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,196 @@
1+
# This job tests the functionality of the openstack-operator to replace a
2+
# bmh compute node.
3+
#
4+
# This job was created to satisfy:
5+
# https://issues.redhat.com/browse/OSPRH-15061
6+
7+
- name: Get name of nodeset containing bmh node to be used as spare node
8+
cifmw.general.ci_script:
9+
output_dir: "{{ cifmw_validations_basedir }}/artifacts"
10+
script: >-
11+
oc -n {{ cifmw_validations_namespace }} get bmh {{ cifmw_validations_bmh_spare_leaf_label }} -o jsonpath='{.spec.consumerRef.name}'
12+
register: bmh_nodeset_name
13+
14+
- name: Verify bmh node being used as spare is provisioned before scale down
15+
cifmw.general.ci_script:
16+
output_dir: "{{ cifmw_validations_basedir }}/artifacts"
17+
script: >-
18+
oc -n {{ cifmw_validations_namespace }} get bmh {{ cifmw_validations_bmh_spare_leaf_label }} -o jsonpath='{.status.provisioning.state}'
19+
register: bmh_nodes_before_scale_down
20+
failed_when: bmh_nodes_before_scale_down.stdout != "provisioned"
21+
22+
- name: Get compute service list
23+
cifmw.general.ci_script:
24+
output_dir: "{{ cifmw_validations_basedir }}/artifacts"
25+
script: >-
26+
oc -n {{ cifmw_validations_namespace }} rsh openstackclient openstack compute service list
27+
register: compute_service_list_out
28+
until: '"{{ cifmw_validations_bmh_spare_hostname }}" in compute_service_list_out.stdout'
29+
30+
- name: Disable nova-compute for node being removed
31+
cifmw.general.ci_script:
32+
output_dir: "{{ cifmw_validations_basedir }}/artifacts"
33+
script: >-
34+
oc -n {{ cifmw_validations_namespace }} rsh openstackclient openstack compute service set {{ cifmw_validations_bmh_spare_hostname }} nova-compute --disable
35+
register: compute_service_set_out
36+
until: '"Failed" not in compute_service_set_out.stdout'
37+
38+
- name: Get ovn controller id of host to be removed
39+
cifmw.general.ci_script:
40+
output_dir: "{{ cifmw_validations_basedir }}/artifacts"
41+
script: >-
42+
oc -n {{ cifmw_validations_namespace }} rsh openstackclient openstack network agent list --host {{ cifmw_validations_bmh_spare_hostname }} | grep "OVN Controller agent" | awk '{print $2}'
43+
register: remove_ovn_id
44+
45+
- name: Delete network agent for compute being removed
46+
cifmw.general.ci_script:
47+
output_dir: "{{ cifmw_validations_basedir }}/artifacts"
48+
script: >-
49+
oc -n {{ cifmw_validations_namespace }} rsh openstackclient openstack network agent delete {{ remove_ovn_id.stdout }}
50+
51+
- name: Get compute service id of host to be removed
52+
cifmw.general.ci_script:
53+
output_dir: "{{ cifmw_validations_basedir }}/artifacts"
54+
script: >-
55+
oc -n {{ cifmw_validations_namespace }} rsh openstackclient openstack compute service list --host {{ cifmw_validations_bmh_spare_hostname }} -f value -c ID
56+
register: remove_compute_service_id
57+
58+
- name: Delete compute service for node being removed
59+
cifmw.general.ci_script:
60+
output_dir: "{{ cifmw_validations_basedir }}/artifacts"
61+
script: >-
62+
oc -n {{ cifmw_validations_namespace }} rsh openstackclient openstack compute service delete {{ remove_compute_service_id.stdout }}
63+
64+
- name: Patch nodeset to remove node
65+
environment:
66+
KUBECONFIG: "{{ cifmw_openshift_kubeconfig }}"
67+
PATH: "{{ cifmw_path }}"
68+
cifmw.general.ci_script:
69+
output_dir: "{{ cifmw_validations_basedir }}/artifacts"
70+
script: >-
71+
oc patch -n {{ cifmw_validations_namespace }} osdpns/"{{ bmh_nodeset_name.stdout | trim}}" --type=json --patch '[{ "op": "remove", "path": "/spec/nodes/{{ cifmw_validations_bmh_spare_nodename }}" }]'
72+
73+
- name: Wait for nodeset to be SetupReady again
74+
environment:
75+
KUBECONFIG: "{{ cifmw_openshift_kubeconfig }}"
76+
PATH: "{{ cifmw_path }}"
77+
cifmw.general.ci_script:
78+
output_dir: "{{ cifmw_validations_basedir }}/artifacts"
79+
script: >-
80+
oc wait osdpns "{{ bmh_nodeset_name.stdout | trim }}"
81+
--namespace={{ cifmw_validations_namespace }}
82+
--for=condition=SetupReady
83+
--timeout={{ cifmw_validations_timeout }}m
84+
85+
- name: Patch spare bmh node to change its label to match label of node being replaced
86+
environment:
87+
KUBECONFIG: "{{ cifmw_openshift_kubeconfig }}"
88+
PATH: "{{ cifmw_path }}"
89+
cifmw.general.ci_script:
90+
output_dir: "{{ cifmw_validations_basedir }}/artifacts"
91+
script: >-
92+
oc patch -n {{ cifmw_validations_namespace }} bmh/"{{ cifmw_validations_bmh_spare_leaf_label }}" --type=json --patch '[{ "op": "replace", "path": "/metadata/labels/nodeName", "value": "{{ cifmw_validations_bmh_replace_leaf_label }}" }]'
93+
94+
- name: Verify bmh node being used as spare is available after changing label
95+
cifmw.general.ci_script:
96+
output_dir: "{{ cifmw_validations_basedir }}/artifacts"
97+
script: >-
98+
oc -n {{ cifmw_validations_namespace }} get bmh {{ cifmw_validations_bmh_spare_leaf_label }} -o jsonpath='{.status.provisioning.state}'
99+
register: bmh_nodes_after_scale_down
100+
until: bmh_nodes_after_scale_down.stdout == "available"
101+
retries: 20
102+
delay: 20
103+
104+
- name: Create openstackdataplanedeployment to deploy the scaledown
105+
environment:
106+
KUBECONFIG: "{{ cifmw_openshift_kubeconfig }}"
107+
PATH: "{{ cifmw_path }}"
108+
cifmw.general.ci_script:
109+
output_dir: "{{ cifmw_validations_basedir }}/artifacts"
110+
script: |
111+
oc apply -f - <<EOF
112+
apiVersion: dataplane.openstack.org/v1beta1
113+
kind: OpenStackDataPlaneDeployment
114+
metadata:
115+
name: edpm-scaledown
116+
namespace: {{ cifmw_validations_namespace }}
117+
spec:
118+
nodeSets:
119+
- "{{ bmh_nodeset_name.stdout | trim }}"
120+
servicesOverride:
121+
- ssh-known-hosts
122+
EOF
123+
124+
- name: Wait for deployment to be complete
125+
environment:
126+
KUBECONFIG: "{{ cifmw_openshift_kubeconfig }}"
127+
PATH: "{{ cifmw_path }}"
128+
cifmw.general.ci_script:
129+
output_dir: "{{ cifmw_validations_basedir }}/artifacts"
130+
script: >-
131+
oc wait openstackdataplanedeployment edpm-scaledown
132+
--namespace={{ cifmw_validations_namespace }}
133+
--for=condition=ready
134+
--timeout={{ cifmw_validations_timeout }}m
135+
136+
- name: Delete faulty baremetal node
137+
cifmw.general.ci_script:
138+
output_dir: "{{ cifmw_validations_basedir }}/artifacts"
139+
script: >-
140+
oc -n {{ cifmw_validations_namespace }} delete bmh {{ cifmw_validations_bmh_replace_leaf_label }}
141+
142+
- name: Wait for nodeset to be SetupReady
143+
environment:
144+
KUBECONFIG: "{{ cifmw_openshift_kubeconfig }}"
145+
PATH: "{{ cifmw_path }}"
146+
cifmw.general.ci_script:
147+
output_dir: "{{ cifmw_validations_basedir }}/artifacts"
148+
script: >-
149+
oc wait osdpns "{{ bmh_nodeset_name.stdout | trim }}"
150+
--namespace={{ cifmw_validations_namespace }}
151+
--for=condition=SetupReady
152+
--timeout={{ cifmw_validations_timeout }}m
153+
154+
- name: Create openstackdataplanedeployment to deploy the compute replacement
155+
environment:
156+
KUBECONFIG: "{{ cifmw_openshift_kubeconfig }}"
157+
PATH: "{{ cifmw_path }}"
158+
cifmw.general.ci_script:
159+
output_dir: "{{ cifmw_validations_basedir }}/artifacts"
160+
script: |
161+
oc apply -f - <<EOF
162+
apiVersion: dataplane.openstack.org/v1beta1
163+
kind: OpenStackDataPlaneDeployment
164+
metadata:
165+
name: edpm-compute-replacement
166+
namespace: {{ cifmw_validations_namespace }}
167+
spec:
168+
nodeSets:
169+
- "{{ bmh_nodeset_name.stdout | trim }}"
170+
servicesOverride:
171+
- ssh-known-hosts
172+
EOF
173+
174+
- name: Wait for deployment to be complete
175+
environment:
176+
KUBECONFIG: "{{ cifmw_openshift_kubeconfig }}"
177+
PATH: "{{ cifmw_path }}"
178+
cifmw.general.ci_script:
179+
output_dir: "{{ cifmw_validations_basedir }}/artifacts"
180+
script: >-
181+
oc wait openstackdataplanedeployment edpm-compute-replacement
182+
--namespace={{ cifmw_validations_namespace }}
183+
--for=condition=ready
184+
--timeout={{ cifmw_validations_timeout }}m
185+
186+
- name: Wait for nodeset to be Ready
187+
environment:
188+
KUBECONFIG: "{{ cifmw_openshift_kubeconfig }}"
189+
PATH: "{{ cifmw_path }}"
190+
cifmw.general.ci_script:
191+
output_dir: "{{ cifmw_validations_basedir }}/artifacts"
192+
script: >-
193+
oc wait osdpns "{{ bmh_nodeset_name.stdout | trim }}"
194+
--namespace={{ cifmw_validations_namespace }}
195+
--for=condition=Ready
196+
--timeout={{ cifmw_validations_timeout }}m

0 commit comments

Comments
 (0)