[ci] Update report_results to use XML files (#246)

* [ci] Update report_result to look in xml files for failures

* Remove the custom_logger failure counter

This change means we can just use "TEST" to mark the testcases,
and we don't have to add in fake or placeholder IDs

Jira: https://issues.redhat.com/browse/OSPRH-15511

* [zuul] Run jobs when there's a change to ci/report_results.yml

Author: elfiesmelfie

.zuul.yaml

@@ -131,6 +131,7 @@
               - ci/vars-use-master-containers.yml
               - ci/use-master-containers.yml
               - ci/patch-openstack-version.yaml
+              - ci/report_result.yml
         - functional-logging-tests-osp18:
             irrelevant-files: *irrelevant_files
             files:
@@ -140,6 +141,7 @@
               - ci/logging_tests_all.yml
               - ci/logging_tests_computes.yml
               - ci/logging_tests_controller.yml
+              - ci/report_result.yml
         - functional-graphing-tests-osp18:
             voting:
               false
@@ -149,10 +151,12 @@
               - .zuul.yaml
               - ci/vars-graphing-test.yml
               - ci/run_graphing_test.yml
+              - ci/report_result.yml
         - functional-metric-verification-tests-osp18:
             irrelevant-files:  *irrelevant_files
             files:
               - roles/telemetry_verify_metrics/.*
               - .zuul.yaml
               - ci/vars-metric-verification-test.yml
               - ci/run_verify_metrics_osp18.yml
+              - ci/report_result.yml

ci/report_result.yml

@@ -19,10 +19,10 @@
   vars_files:
     - vars/common.yml
   tasks:
-    - name: "Find the XML file"
+    - name: "Find the XML file(s)"
       ansible.builtin.shell:
         cmd: |
-          find ~/.ansible.log/ -name *.xml
+          find {{ custom_xml_output_dir }} -name *.xml
       register: xml_file_list
       ignore_errors: true
 
@@ -54,28 +54,36 @@
         src: "{{ ansible_env.HOME }}/summary_results.log"
         dest: "{{ logs_dir }}/summary_result.log"
 
-- name: "Check the results files for failed tasks"
+- name: Check the XML file for failed tasks
   hosts: controller
   gather_facts: true
   vars_files:
     - vars/common.yml
   tasks:
-    - name: Grep the number of failed tasks
-      ansible.builtin.shell:
-        cmd: |
-          grep "failed" {{ logs_dir }}/test_run_result.out
-      register: tasks_failed
-      # The RC from grep is 1 when there is no string matched (i.e. when there are no failures)
-      ignore_errors: yes
+    - name: "Get the matching lines in the results files"
+      ansible.builtin.command:
+       cmd: |
+         grep -r "<testsuites .*>$" {{ custom_xml_output_dir }}/
+      register: verbose_matches
+      ignore_errors: true
+
+    - name: "Get the number of failed testcases"
+      ansible.builtin.set_fact:
+        tasks_failed: "{{ verbose_matches.stdout | regex_replace('.*failures=\"([0-9]*)\".*>', '\\1') | split('\n') | map('int') | sum }}"
 
       # The RC from grep is 0 if the string is matched
+    - name: Fail when there's a testcase failure
+      ansible.builtin.fail:
+        msg: There were {{ tasks_failed }} failed testcases.
+      when: tasks_failed | int > 0 and verbose_matches.rc == 0
+
     - name: Determine success or failure based on the number of failed tasks
       ansible.builtin.fail:
         msg: "The log file(s) contain failed task."
-      when: tasks_failed.stdout_lines | length > 0 and tasks_failed.rc == 0
+      when: tasks_failed | int > 0 and verbose_matches.rc == 0
 
       # The RC from grep is 2 if an error occurred.
     - name: Fail if the file was not found (or other grep error)
       ansible.builtin.fail:
-        msg: "{{ tasks_failed.stderr if tasks_failed.stderr | length > 0 else 'There was an error with grep.' }}"
-      when: tasks_failed.rc > 1
+        msg: "{{ verbose_matches.stderr if verbose_matches.stderr | length > 0 else 'There was an error with grep.' }}"
+      when: verbose_matches.rc > 1

ci/vars/common.yml

@@ -1,3 +1,4 @@
 ---
 fvt_dir: "{{ ansible_env.HOME }}/{{ zuul.projects['github.com/infrawatch/feature-verification-tests'].src_dir }}"
 logs_dir: "{{ ansible_env.HOME }}/ci-framework-data/logs/feature-verification-tests"
+custom_xml_output_dir: "~/.ansible.log/"