Skip to content

Commit b92a9f2

Browse files
markgoddardAlex-Welsh
markgoddard
authored and
Alex-Welsh
committed
CI: Separate monitoring vs host test execution in SOT
There are currently two types of tests in StackHPC OpenStack Tests: service-level tests such as those for monitoring, and host-level tests such as those for Docker and SELinux. We should execute the service-level tests from a single remote host, while the host-level tests should be executed on each of the overcloud hosts. This means that we now get several results files, so it becomes important to provide a summary of failures.
1 parent 6066bf5 commit b92a9f2

File tree

3 files changed

+250
-24
lines changed

3 files changed

+250
-24
lines changed

‎.github/workflows/stackhpc-all-in-one.yml‎

Lines changed: 13 additions & 9 deletions
Original file line numberDiff line numberDiff line change
@@ -496,16 +496,20 @@ jobs:
496496
sct-results/
497497
if: ${{ !cancelled() && (steps.tempest.outcome == 'success' || steps.stackhpc-cloud-tests.outcome == 'success' || steps.diagnostics.outcome == 'success') }}
498498

499-
- name: Fail if any Tempest tests failed
499+
- name: Fail if any tests failed
500500
run: |
501-
test $(wc -l < tempest-artifacts/failed-tests) -lt 1
502-
503-
- name: Fail if any StackHPC Cloud tests failed
504-
run: |
505-
echo "Some StackHPC Cloud tests failed."
506-
echo "See HTML results artifact (sct-results) for details."
507-
exit 1
508-
if: steps.stackhpc-cloud-tests.outcome == 'failure'
501+
rc=0
502+
if [[ $(wc -l < tempest-artifacts/failed-tests) -ne 0 ]]; then
503+
echo "Some Tempest tests failed."
504+
echo "See HTML results artifact (tempest-artifacts) for details."
505+
rc=1
506+
fi
507+
if [[ $(wc -l < sct-results/failed-tests) -ne 0 ]]; then
508+
echo "Some StackHPC OpenStack tests failed."
509+
echo "See HTML results artifact (sct-results) for details."
510+
rc=1
511+
fi
512+
exit $rc
509513
510514
- name: Destroy
511515
run: terraform destroy -auto-approve

‎etc/kayobe/ansible/stackhpc-cloud-tests.yml‎

Lines changed: 66 additions & 15 deletions
Original file line numberDiff line numberDiff line change
@@ -1,6 +1,6 @@
11
---
22
- name: Run StackHPC Cloud tests
3-
hosts: tempest_runner
3+
hosts: tempest_runner:overcloud
44
tags:
55
- stackhpc-cloud-tests
66
vars:
@@ -12,6 +12,11 @@
1212
tasks:
1313
- name: Stackhpc Cloud tests
1414
block:
15+
- name: Assert that there is only one host in the tempest_runner group
16+
assert:
17+
that: groups.get('tempest_runner', []) | length == 1
18+
fail_msg: The tempest_runner group should contain exactly one host
19+
1520
- name: Create a temporary directory for tests repo
1621
ansible.builtin.tempfile:
1722
state: directory
@@ -60,19 +65,19 @@
6065
file: "{{ kayobe_env_config_path }}/kolla/passwords.yml"
6166
name: kolla_passwords
6267

63-
- name: Run StackHPC Cloud tests
68+
# Monitoring tests should run once, executed on the host in the
69+
# tempest_runner group.
70+
- name: Run StackHPC Cloud monitoring tests
6471
ansible.builtin.command:
6572
cmd: >
6673
{{ sct_venv }}/bin/py.test
67-
--html={{ results_tmpdir.path }}/stackhpc-cloud-tests.html
74+
--html={{ results_tmpdir.path }}/monitoring.html
6875
--self-contained-html
69-
--pyargs stackhpc_cloud_tests
76+
--pyargs stackhpc_cloud_tests.monitoring
7077
--timeout {{ sct_timeout }}
7178
-rfEx
7279
-vv
7380
environment:
74-
DOCKER_VERSION_MIN: "{{ sct_docker_version_min }}"
75-
DOCKER_VERSION_MAX: "{{ sct_docker_version_max }}"
7681
GRAFANA_URL: "{{ sct_grafana_url }}"
7782
GRAFANA_USERNAME: "{{ sct_grafana_username }}"
7883
GRAFANA_PASSWORD: "{{ sct_grafana_password }}"
@@ -85,14 +90,9 @@
8590
PROMETHEUS_URL: "{{ sct_prometheus_url }}"
8691
PROMETHEUS_USERNAME: "{{ sct_prometheus_username }}"
8792
PROMETHEUS_PASSWORD: "{{ sct_prometheus_password }}"
88-
SELINUX_STATE: "{{ sct_selinux_state }}"
8993
vars:
9094
kolla_external_scheme: "{{ 'https' if kolla_enable_tls_external | bool else 'http' }}"
9195
kolla_internal_scheme: "{{ 'https' if kolla_enable_tls_internal | bool else 'http' }}"
92-
# Inclusive min
93-
sct_docker_version_min: "24.0.0"
94-
# Exclusive max
95-
sct_docker_version_max: "27.0.0"
9696
sct_grafana_url: "{{ kolla_external_scheme }}://{{ kolla_external_fqdn }}:3000"
9797
sct_grafana_username: "grafana_local_admin"
9898
sct_grafana_password: "{{ kolla_passwords.grafana_admin_password }}"
@@ -105,13 +105,64 @@
105105
sct_prometheus_url: "{{ kolla_internal_scheme }}://{{ kolla_internal_fqdn }}:9091"
106106
sct_prometheus_username: admin
107107
sct_prometheus_password: "{{ kolla_passwords.prometheus_password }}"
108+
failed_when: monitoring_results.rc not in [0, 1]
109+
register: monitoring_results
110+
when: "'tempest_runner' in group_names"
111+
112+
# Host tests should run on every host in the overcloud group.
113+
# TODO: Use TestInfra's native Ansible or SSH connection plugins for
114+
# remote test execution? That would place all results in a single file
115+
# and allow us to execute all tests from a single host.
116+
# https://testinfra.readthedocs.io/en/latest/backends.html#connection-backends
117+
- name: Run StackHPC Cloud host tests
118+
ansible.builtin.command:
119+
cmd: >
120+
{{ sct_venv }}/bin/py.test
121+
--html={{ results_tmpdir.path }}/host-{{ inventory_hostname }}.html
122+
--self-contained-html
123+
--pyargs stackhpc_cloud_tests.host
124+
--timeout {{ sct_timeout }}
125+
-vv
126+
environment:
127+
DOCKER_VERSION_MIN: "{{ sct_docker_version_min }}"
128+
DOCKER_VERSION_MAX: "{{ sct_docker_version_max }}"
129+
SELINUX_STATE: "{{ sct_selinux_state }}"
130+
vars:
131+
# Inclusive min
132+
sct_docker_version_min: "24.0.0"
133+
# Exclusive max
134+
sct_docker_version_max: "27.0.0"
108135
sct_selinux_state: "{{ selinux_state }}"
136+
failed_when: host_results.rc not in [0, 1]
137+
register: host_results
138+
when: "'overcloud' in group_names"
109139
always:
110-
- name: Fetch results
111-
ansible.builtin.fetch:
112-
src: "{{ results_tmpdir.path }}/stackhpc-cloud-tests.html"
140+
- name: Synchronize results
141+
ansible.posix.synchronize:
142+
src: "{{ results_tmpdir.path }}/"
113143
dest: "{{ results_path_local }}/"
114-
flat: true
144+
mode: pull
145+
archive: no
146+
recursive: true
147+
# For jump host
148+
use_ssh_args: true
149+
- name: Write a file containing failed test runs
150+
ansible.builtin.copy:
151+
content: |-
152+
{% for host in ansible_play_hosts_all %}
153+
{% if host not in ansible_play_hosts %}
154+
{{ host }}: Host failure
155+
{% endif %}
156+
{% if hostvars[host].monitoring_results.rc | default(0) != 0 %}
157+
monitoring.html
158+
{% endif %}
159+
{% if hostvars[host].host_results.rc | default(0) != 0 %}
160+
host-{{ host }}.html
161+
{% endif %}
162+
{% endfor %}
163+
dest: "{{ results_path_local }}/failed-tests"
164+
delegate_to: localhost
165+
run_once: true
115166

116167
- name: Clean up temporary directory
117168
ansible.builtin.file:

‎etc/kayobe/ansible/stackhpc-openstack-tests.yml‎

Lines changed: 171 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,171 @@
1+
---
2+
# This playbook executes tests from the StackHPC OpenStack Tests repository.
3+
# https://github.com/stackhpc/stackhpc-openstack-tests
4+
5+
- name: Run StackHPC OpenStack tests
6+
hosts: tempest_runner:overcloud
7+
tags:
8+
- stackhpc-openstack-tests
9+
vars:
10+
sot_venv: "{{ virtualenv_path }}/sot-venv"
11+
sot_repo: "https://github.com/stackhpc/stackhpc-openstack-tests"
12+
sot_version: "v0.0.1"
13+
sot_timeout: 30
14+
results_path_local: "{{ lookup('env', 'HOME') }}/sot-results"
15+
tasks:
16+
- name: Assert that there is only one host in the tempest_runner group
17+
assert:
18+
that: groups.get('tempest_runner', []) | length == 1
19+
fail_msg: The tempest_runner group should contain exactly one host
20+
21+
- block:
22+
- name: Create a temporary directory for tests repo
23+
ansible.builtin.tempfile:
24+
state: directory
25+
suffix: sot-repo
26+
register: repo_tmpdir
27+
28+
- name: Create a temporary directory for results
29+
ansible.builtin.tempfile:
30+
state: directory
31+
suffix: sot-results
32+
register: results_tmpdir
33+
34+
- name: Clone the StackHPC OpenStack tests repository
35+
ansible.builtin.git:
36+
repo: "{{ sot_repo }}"
37+
version: "{{ sot_version }}"
38+
dest: "{{ repo_tmpdir.path }}"
39+
depth: 1
40+
single_branch: true
41+
42+
- name: Ensure the latest versions of pip and setuptools are installed # noqa package-latest
43+
ansible.builtin.pip:
44+
name: "{{ item.name }}"
45+
state: latest
46+
virtualenv: "{{ sot_venv }}"
47+
virtualenv_command: "python3 -m venv"
48+
with_items:
49+
- { name: pip }
50+
- { name: setuptools }
51+
52+
- name: Ensure required Python packages are installed
53+
ansible.builtin.pip:
54+
name:
55+
- "{{ repo_tmpdir.path }}"
56+
- "-r{{ repo_tmpdir.path }}/requirements.txt"
57+
- pytest-html
58+
- pytest-timeout
59+
virtualenv: "{{ sot_venv }}"
60+
61+
- name: Include Kolla Ansible passwords
62+
ansible.builtin.include_vars:
63+
file: "{{ kayobe_env_config_path }}/kolla/passwords.yml"
64+
name: kolla_passwords
65+
66+
# Monitoring tests should run once, executed on the host in the
67+
# tempest_runner group.
68+
- name: Run StackHPC OpenStack monitoring tests
69+
ansible.builtin.command:
70+
cmd: >
71+
{{ sot_venv }}/bin/py.test
72+
--html={{ results_tmpdir.path }}/monitoring.html
73+
--self-contained-html
74+
--pyargs stackhpc_openstack_tests.monitoring
75+
--timeout {{ sot_timeout }}
76+
-vv
77+
environment:
78+
GRAFANA_URL: "{{ sot_grafana_url }}"
79+
GRAFANA_USERNAME: "{{ sot_grafana_username }}"
80+
GRAFANA_PASSWORD: "{{ sot_grafana_password }}"
81+
OPENSEARCH_HOSTS: "{{ sot_opensearch_hosts }}"
82+
OPENSEARCH_PORT: "{{ sot_opensearch_port }}"
83+
OPENSEARCH_TLS: "{{ sot_opensearch_tls }}"
84+
OPENSEARCH_DASHBOARDS_URL: "{{ sot_opensearch_dashboards_url }}"
85+
OPENSEARCH_DASHBOARDS_USERNAME: "{{ sot_opensearch_dashboards_username }}"
86+
OPENSEARCH_DASHBOARDS_PASSWORD: "{{ sot_opensearch_dashboards_password }}"
87+
PROMETHEUS_URL: "{{ sot_prometheus_url }}"
88+
PROMETHEUS_USERNAME: "{{ sot_prometheus_username }}"
89+
PROMETHEUS_PASSWORD: "{{ sot_prometheus_password }}"
90+
vars:
91+
kolla_external_scheme: "{{ 'https' if kolla_enable_tls_external | bool else 'http' }}"
92+
kolla_internal_scheme: "{{ 'https' if kolla_enable_tls_internal | bool else 'http' }}"
93+
sot_grafana_url: "{{ kolla_external_scheme }}://{{ kolla_external_fqdn }}:3000"
94+
sot_grafana_username: "grafana_local_admin"
95+
sot_grafana_password: "{{ kolla_passwords.grafana_admin_password }}"
96+
sot_opensearch_hosts: "{{ kolla_internal_fqdn }}"
97+
sot_opensearch_port: 9200
98+
sot_opensearch_tls: false
99+
sot_opensearch_dashboards_url: "{{ kolla_external_scheme }}://{{ kolla_external_fqdn }}:5601"
100+
sot_opensearch_dashboards_username: "opensearch"
101+
sot_opensearch_dashboards_password: "{{ kolla_passwords.opensearch_dashboards_password }}"
102+
sot_prometheus_url: "{{ kolla_internal_scheme }}://{{ kolla_internal_fqdn }}:9091"
103+
sot_prometheus_username: "admin"
104+
sot_prometheus_password: "{{ kolla_passwords.prometheus_password }}"
105+
failed_when: monitoring_results.rc not in [0, 1]
106+
register: monitoring_results
107+
when: "'tempest_runner' in group_names"
108+
109+
# Host tests should run on every host in the overcloud group.
110+
# TODO: Use TestInfra's native Ansible or SSH connection plugins for
111+
# remote test execution? That would place all results in a single file
112+
# and allow us to execute all tests from a single host.
113+
# https://testinfra.readthedocs.io/en/latest/backends.html#connection-backends
114+
- name: Run StackHPC OpenStack host tests
115+
ansible.builtin.command:
116+
cmd: >
117+
{{ sot_venv }}/bin/py.test
118+
--html={{ results_tmpdir.path }}/host-{{ inventory_hostname }}.html
119+
--self-contained-html
120+
--pyargs stackhpc_openstack_tests.host
121+
--timeout {{ sot_timeout }}
122+
-vv
123+
environment:
124+
DOCKER_VERSION_MIN: "{{ sot_docker_version_min }}"
125+
DOCKER_VERSION_MAX: "{{ sot_docker_version_max }}"
126+
SELINUX_STATE: "{{ sot_selinux_state }}"
127+
vars:
128+
# Inclusive min
129+
sot_docker_version_min: "24.0.0"
130+
# Exclusive max
131+
sot_docker_version_max: "27.0.0"
132+
sot_selinux_state: "{{ selinux_state }}"
133+
failed_when: host_results.rc not in [0, 1]
134+
register: host_results
135+
when: "'overcloud' in group_names"
136+
always:
137+
- name: Synchronize results
138+
ansible.posix.synchronize:
139+
src: "{{ results_tmpdir.path }}/"
140+
dest: "{{ results_path_local }}/"
141+
mode: pull
142+
archive: no
143+
recursive: true
144+
# For jump host
145+
use_ssh_args: true
146+
147+
- name: Write a file containing failed test runs
148+
ansible.builtin.copy:
149+
content: |-
150+
{% for host in ansible_play_hosts_all %}
151+
{% if host not in ansible_play_hosts %}
152+
{{ host }}: Host failure
153+
{% endif %}
154+
{% if hostvars[host].monitoring_results.rc | default(0) != 0 %}
155+
monitoring.html
156+
{% endif %}
157+
{% if hostvars[host].host_results.rc | default(0) != 0 %}
158+
host-{{ host }}.html
159+
{% endif %}
160+
{% endfor %}
161+
dest: "{{ results_path_local }}/failed-tests"
162+
delegate_to: localhost
163+
run_once: true
164+
165+
- name: Clean up temporary directory
166+
ansible.builtin.file:
167+
path: "{{ item }}"
168+
state: absent
169+
loop:
170+
- "{{ repo_tmpdir.path }}"
171+
- "{{ results_tmpdir.path }}"

0 commit comments

Comments
 (0)