[ci] Try to combine the FVT and tempest (#256)

* [ci] Try to combine the FVT and tempest

This approach runs the FVTs before tempest and then collects the results after running test-operator
This means that the tempest tests will trigger a failure if they fail, and the FVTs will trigger a
failure only after the tempest tests are run, which means that the results are reported as expected.

This will work for github-check pipeline and the component pipeline.
However, this approach may not be usable with the unified jobs.

This change adds a tempest config to the logging, graphing,
metrics-verification tests.

Author: elfiesmelfie

.zuul.yaml

@@ -3,12 +3,14 @@
     name: functional-autoscaling-tests-osp18
     parent: telemetry-operator-multinode-autoscaling
     description: |
-      Run the autoscaling functional test on osp18+patched version of aodh and heat.
+      Run the autoscaling functional tests and tempest tests on osp18+patched
+      versions of aodh and heat.
     vars:
       patch_openstackversions: true
       cifmw_extras:
         - "@{{ ansible_user_dir }}/{{ zuul.projects['github.com/openstack-k8s-operators/ci-framework'].src_dir }}/scenarios/centos-9/multinode-ci.yml"
         - "@{{ ansible_user_dir }}/{{ zuul.projects['github.com/openstack-k8s-operators/telemetry-operator'].src_dir }}/ci/vars-autoscaling.yml"
+        - "@{{ ansible_user_dir }}/{{ zuul.projects['github.com/openstack-k8s-operators/telemetry-operator'].src_dir }}/ci/vars-autoscaling-tempest.yml"
         - "@{{ ansible_user_dir }}/{{ zuul.projects['github.com/infrawatch/feature-verification-tests'].src_dir }}/ci/vars-functional-test.yml"
         - "@{{ ansible_user_dir }}/{{ zuul.projects['github.com/infrawatch/feature-verification-tests'].src_dir }}/ci/vars-use-master-containers.yml"
     roles:
@@ -58,11 +60,14 @@
     name: functional-logging-tests-osp18
     parent: telemetry-operator-multinode-logging
     description: |
-      Run the logging functional tests on osp18
+      Run the logging functional tests and tempest smoketests on osp18
     vars:
       cifmw_extras:
         - "@{{ ansible_user_dir }}/{{ zuul.projects['github.com/openstack-k8s-operators/ci-framework'].src_dir  }}/scenarios/centos-9/multinode-ci.yml"
         - "@{{ ansible_user_dir }}/{{ zuul.projects['github.com/openstack-k8s-operators/telemetry-operator'].src_dir }}/ci/vars-logging.yml"
+          # Use the tempest config we have for autoscaling tests
+        - "@{{ ansible_user_dir }}/{{ zuul.projects['github.com/openstack-k8s-operators/telemetry-operator'].src_dir }}/ci/vars-autoscaling-tempest.yml"
+          # There's an override in here to modify the enabled tests to include only the smoke tests
         - "@{{ ansible_user_dir }}/{{ zuul.projects['github.com/infrawatch/feature-verification-tests'].src_dir }}/ci/vars-logging-test.yml"
     roles:
       - zuul: github.com/openstack-k8s-operators/ci-framework
@@ -73,12 +78,15 @@
     name: functional-metric-verification-tests-osp18
     parent: telemetry-operator-multinode-autoscaling
     description: |
-      Run the metric verification functional test on osp18.
+      Run the metric verification functional test and tempest smoketests on osp18.
     vars:
       patch_observabilityclient: true
       cifmw_extras:
         - "@{{ ansible_user_dir }}/{{ zuul.projects['github.com/openstack-k8s-operators/ci-framework'].src_dir  }}/scenarios/centos-9/multinode-ci.yml"
         - "@{{ ansible_user_dir }}/{{ zuul.projects['github.com/openstack-k8s-operators/telemetry-operator'].src_dir }}/ci/vars-autoscaling.yml"
+          # Use the tempest config we have for autoscaling tests
+        - "@{{ ansible_user_dir }}/{{ zuul.projects['github.com/openstack-k8s-operators/telemetry-operator'].src_dir }}/ci/vars-autoscaling-tempest.yml"
+          # tempest enabled tests list is overwritten in the metrics-verification-tests, so needs to be added after the autoscaling-tempest, so that we only run the smoke-tests
         - "@{{ ansible_user_dir }}/{{ zuul.projects['github.com/infrawatch/feature-verification-tests'].src_dir }}/ci/vars-metric-verification-test.yml"
         - "@{{ ansible_user_dir }}/{{ zuul.projects['github.com/openstack-k8s-operators/telemetry-operator'].src_dir }}/ci/vars-power-monitoring.yml"
         - "@{{ ansible_user_dir }}/{{ zuul.projects['github.com/infrawatch/feature-verification-tests'].src_dir }}/ci/vars-use-master-containers.yml"
@@ -100,13 +108,15 @@
     name: functional-graphing-tests-osp18
     parent: telemetry-operator-multinode-autoscaling
     description: |
-      Run the UI Graphing test on osp18.
+      Run the UI Graphing test and tempest smoketests on osp18.
     vars:
-      cifmw_run_tests: false
       cifmw_extras:
         - "@{{ ansible_user_dir }}/{{ zuul.projects['github.com/openstack-k8s-operators/ci-framework'].src_dir  }}/scenarios/centos-9/multinode-ci.yml"
         - "@{{ ansible_user_dir }}/{{ zuul.projects['github.com/openstack-k8s-operators/telemetry-operator'].src_dir }}/ci/vars-autoscaling.yml"
         - "@{{ ansible_user_dir }}/{{ zuul.projects['github.com/openstack-k8s-operators/telemetry-operator'].src_dir }}/ci/vars-power-monitoring.yml"
+          # Use the tempest config we have for autoscaling tests
+        - "@{{ ansible_user_dir }}/{{ zuul.projects['github.com/openstack-k8s-operators/telemetry-operator'].src_dir }}/ci/vars-autoscaling-tempest.yml"
+          # Currently, this runs tempest smoketests after the dashboards have been disabled again
         - "@{{ ansible_user_dir }}/{{ zuul.projects['github.com/infrawatch/feature-verification-tests'].src_dir }}/ci/vars-graphing-test.yml"
     roles:
       - zuul: github.com/openstack-k8s-operators/ci-framework

ci/vars-functional-test.yml

@@ -5,10 +5,16 @@
 # tests.
 # skip os-net-setup
 cifmw_os_net_setup_config: []
-post_deploy_00_run_functional_test:
+pre_tests_00_run_functional_test:
   source: "{{ ansible_user_dir }}/{{ zuul.projects['github.com/infrawatch/feature-verification-tests'].src_dir }}/ci/run_autoscaling_osp18.yml"
   type: playbook
   config_file: "{{ ansible_user_dir }}/{{ zuul.projects['github.com/infrawatch/feature-verification-tests'].src_dir }}/ci/ansible.cfg"
-post_deploy_99_collect_results:
+
+cifmw_run_tests: true
+# redefine this to clear the skipped test
+# this should be done in the parent job IF the openstack versions are updated
+cifmw_test_operator_tempest_exclude_list: ""
+
+post_tests_99_collect_results:
   source: "{{ ansible_user_dir }}/{{ zuul.projects['github.com/infrawatch/feature-verification-tests'].src_dir }}/ci/report_result.yml"
   type: playbook

ci/vars-graphing-test.yml

@@ -2,7 +2,12 @@
 post_deploy_00_run_graphing_test:
   source: "{{ ansible_user_dir }}/{{ zuul.projects['github.com/infrawatch/feature-verification-tests'].src_dir }}/ci/run_graphing_test.yml"
   config_file: "{{ ansible_user_dir }}/{{ zuul.projects['github.com/infrawatch/feature-verification-tests'].src_dir }}/ci/ansible.cfg"
-  type: playbook 
-post_deploy_99_collect_results:
+  type: playbook
+
+cifmw_run_tests: true
+# run only smoke tests
+cifmw_test_operator_tempest_include_list: |
+  ^tempest.*\[.*\bsmoke\b.*\]
+post_tests_99_collect_results:
   source: "{{ ansible_user_dir }}/{{ zuul.projects['github.com/infrawatch/feature-verification-tests'].src_dir }}/ci/report_result.yml"
   type: playbook

ci/vars-logging-test.yml

@@ -4,6 +4,12 @@ post_deploy_00_fvt_logging:
   source: "{{ ansible_user_dir }}/{{ zuul.projects['github.com/infrawatch/feature-verification-tests'].src_dir }}/ci/logging_tests_all.yml"
   config_file: "{{ ansible_user_dir }}/{{ zuul.projects['github.com/infrawatch/feature-verification-tests'].src_dir }}/ci/ansible.cfg"
   type: playbook
-post_deploy_99_collect_results:
+
+cifmw_run_tests: true
+# enable only the smoketests
+cifmw_test_operator_tempest_include_list: |
+  ^tempest.*\[.*\bsmoke\b.*\]
+
+post_tests_99_collect_results:
   source: "{{ ansible_user_dir }}/{{ zuul.projects['github.com/infrawatch/feature-verification-tests'].src_dir }}/ci/report_result.yml"
   type: playbook

ci/vars-metric-verification-test.yml

@@ -3,6 +3,9 @@ post_deploy_00_fvt_verify_metrics:
   source: "{{ ansible_user_dir }}/{{ zuul.projects['github.com/infrawatch/feature-verification-tests'].src_dir }}/ci/run_verify_metrics_osp18.yml"
   type: playbook
   config_file: "{{ ansible_user_dir }}/{{ zuul.projects['github.com/infrawatch/feature-verification-tests'].src_dir }}/ci/ansible.cfg"
-post_deploy_99_collect_results:
+cifmw_run_tests: true
+cifmw_test_operator_tempest_include_list: |
+  ^tempest.*\[.*\bsmoke\b.*\]
+post_tests_99_collect_results:
   source: "{{ ansible_user_dir }}/{{ zuul.projects['github.com/infrawatch/feature-verification-tests'].src_dir }}/ci/report_result.yml"
   type: playbook