diff --git a/callback_plugins/custom_junit.py b/callback_plugins/custom_junit.py deleted file mode 100644 index 70bf5ebb7..000000000 --- a/callback_plugins/custom_junit.py +++ /dev/null @@ -1,108 +0,0 @@ -from ansible.plugins.callback.junit import CallbackModule as JunitCallbackModule -from ansible.plugins.callback.junit import HostData - -import os -import time -import re -from ansible.utils._junit_xml import TestCase, TestError, TestFailure, TestSuite, TestSuites - - -class CallbackModule(JunitCallbackModule): - """ - Custom callback that overrides the default JUnit callback - """ - CALLBACK_NAME = 'custom_junit' - - def __init__(self): - super(CallbackModule, self).__init__() - - # Custom environment variable handling - # Update this to parse these values from the config file, as well as the env. - self._output_dir = os.path.expanduser("~/.ansible.log") - self._test_case_prefix = os.getenv('JUNIT_TEST_CASE_PREFIX', 'TEST') - self._fail_on_ignore = 'true' # this is needed because we use "ignore_errors" on the playbooks so that all the tests are run - self._include_setup_tasks_in_report = os.getenv('JUNIT_INCLUDE_SETUP_TASKS_IN_REPORT', 'False').lower() - self._hide_task_arguments = os.getenv('JUNIT_HIDE_TASK_ARGUMENTS', 'True').lower() - self._task_class = False - - print("The output_dir is: %s" % self._output_dir) - # Ensure the output directory exists - if not os.path.exists(self._output_dir): - print("Creating output dir: %s" % (self._output_dir)) - os.makedirs(self._output_dir) - - def _finish_task(self, status, result): - """ record the results of a task for a single host """ - task_uuid = result._task._uuid - if hasattr(result, '_host'): - host_uuid = result._host._uuid - host_name = result._host.name - else: - host_uuid = 'include' - host_name = 'include' - - task_data = self._task_data[task_uuid] - - if self._fail_on_change == 'true' and status == 'ok' and result._result.get('changed', False): - status = 'failed' - - # ignore failure if expected and toggle result if asked for - if status == 'failed' and 'EXPECTED FAILURE' in task_data.name: - status = 'ok' - elif 'TOGGLE RESULT' in task_data.name: - if status == 'failed': - status = 'ok' - elif status == 'ok': - status = 'failed' - - if self._test_case_prefix in task_data.name: - task_data.add_host(HostData(host_uuid, host_name, status, result)) - - # Debugging - if task_data.name.startswith(self._test_case_prefix): - print(f"This task ({task_data.name}) starts with the test_prefix({self._test_case_prefix})") - if self._test_case_prefix in task_data.name: - print(f"This task ({task_data.name}) should be reported because it contains test_prefix({self._test_case_prefix})") - if status == 'failed': - print(f"This task ({task_data.name}) failed, but may not be reported") - - def mutate_task_name(self, task_name): - # Debugging - if not self._test_case_prefix in task_name: - print("task_name (%s) does not contain prefix (%s)" % (task_name, self._test_case_prefix)) - new_name = task_name - new_name = new_name.split("\n")[0] # only use the first line, so we can include IDs and additional description - # this covers when a task is included, but the including task is the one that is the test - new_name = new_name.split(":")[-1] # only provide the last part of the name when the role name is included - - if len(self._test_case_prefix) > 0: - # this one may not be needed... - new_name = new_name.split(self._test_case_prefix)[-1] # remove the test prefix and everything before it - - new_name = new_name.lower() - new_name = re.sub(r'\W', ' ', new_name) # replace all non-alphanumeric characters (except _) with a space - new_name = re.sub(r'(^\W*|\W*$)', '', new_name) # trim any trailing or leading non-alphanumeric characters - new_name = re.sub(r' +', '_', new_name) # replace any number of spaces with _ - - return new_name - - def _build_test_case(self, task_data, host_data): - """ - This is used in generate_report. The task_data and host data will get passed. - """ - # Use the original task name to define the final name - - print("%s\t(task_name, pre-_build_test_case)" % task_data.name) - tc = super()._build_test_case(task_data, host_data) - print("%s\t(tc.name, post-_build_test_case)" % tc.name) - tc.name = self.mutate_task_name(tc.name) - - print("%s\t(tc.name, post-mutate_task_name)" % tc.name) - - # These can be able to omit with a config option - # These two control whether testcases contain the system_out and - # system_err elements that show STDOUT and STDERR - tc.system_out = None - tc.system_err = None - tc.classname = "openstack-observability" - return tc diff --git a/ci/report_result.yml b/ci/report_result.yml deleted file mode 100644 index 736bec6e6..000000000 --- a/ci/report_result.yml +++ /dev/null @@ -1,94 +0,0 @@ ---- -- name: "Create the output files" - hosts: - - controller - vars_files: - - vars/common.yml - tasks: - - name: Create log dir - ansible.builtin.file: - path: "{{ logs_dir }}" - state: directory - mode: "0755" - -- name: "Run the log file collection" - hosts: - - controller - - compute - gather_facts: true - vars_files: - - vars/common.yml - tasks: - - name: "Find the XML file(s)" - ansible.builtin.shell: - cmd: | - find {{ custom_xml_output_dir }} -name *.xml - register: xml_file_list - ignore_errors: true - - - name: "Show the XML files" - ansible.builtin.debug: - var: xml_file_list.stdout_lines - ignore_errors: true - - - name: "Collect the XML files, renaming them for the host that they are collected from" - delegate_to: controller - ansible.builtin.copy: - remote_src: true - src: "{{ item }}" - dest: "{{ logs_dir }}/{{ item | basename }} " - with_items: "{{ xml_file_list.stdout_lines }}" - when: xml_file_list.stdout_lines | length != 0 - - - name: "Collect the custom_logger results" - delegate_to: controller - ansible.builtin.copy: - remote_src: true - src: "{{ ansible_env.HOME }}/test_run_result.out" - dest: "{{ logs_dir }}/test_run_result.out" - - - name: "Collect the results summary" - delegate_to: controller - ansible.builtin.copy: - remote_src: true - src: "{{ ansible_env.HOME }}/summary_results.log" - dest: "{{ logs_dir }}/summary_result.log" - -- name: Check the XML file for failed tasks - hosts: controller - gather_facts: true - vars_files: - - vars/common.yml - pre_tasks: - - name: Stop play early if we do not want to check the failures - when: - - not (check_failures | default(true) | bool) - ansible.builtin.meta: end_play - tasks: - - name: "Get the matching lines in the results files" - ansible.builtin.command: - cmd: | - grep -r "$" {{ custom_xml_output_dir }}/ - register: verbose_matches - ignore_errors: true - - - name: "Get the number of failed testcases" - ansible.builtin.set_fact: - tasks_failed: "{{ verbose_matches.stdout | regex_replace('.*failures=\"([0-9]*)\".*>', '\\1') | split('\n') | map('int') | sum }}" - - # The RC from grep is 0 if the string is matched - - name: Fail when there's a testcase failure - ansible.builtin.fail: - msg: There were {{ tasks_failed }} failed testcases. - when: tasks_failed | int > 0 and verbose_matches.rc == 0 - - - name: Determine success or failure based on the number of failed tasks - ansible.builtin.fail: - msg: "The log file(s) contain failed task." - when: tasks_failed | int > 0 and verbose_matches.rc == 0 - - # The RC from grep is 2 if an error occurred. - - name: Fail if the file was not found (or other grep error) - ansible.builtin.fail: - msg: "{{ verbose_matches.stderr if verbose_matches.stderr | length > 0 else 'There was an error with grep.' }}" - when: verbose_matches.rc > 1 diff --git a/ci/vars/common.yml b/ci/vars/common.yml deleted file mode 100644 index 6035e85a5..000000000 --- a/ci/vars/common.yml +++ /dev/null @@ -1,4 +0,0 @@ ---- -fvt_dir: "{{ ansible_env.HOME }}/{{ zuul.projects['github.com/infrawatch/feature-verification-tests'].src_dir }}" -logs_dir: "{{ ansible_env.HOME }}/ci-framework-data/logs/feature-verification-tests" -custom_xml_output_dir: "~/.ansible.log/" diff --git a/roles/common/tasks/container_test.yml b/roles/common/tasks/container_test.yml index 1854e2007..295da93db 100644 --- a/roles/common/tasks/container_test.yml +++ b/roles/common/tasks/container_test.yml @@ -25,7 +25,6 @@ - name: | TEST Verify {{ container_name }} container status - {{ common_container_test_id }} ansible.builtin.assert: that: - "container_status.stdout | length != 0" diff --git a/roles/common/tasks/cr_tests.yml b/roles/common/tasks/cr_tests.yml index 03d054238..3424e0285 100644 --- a/roles/common/tasks/cr_tests.yml +++ b/roles/common/tasks/cr_tests.yml @@ -1,7 +1,6 @@ --- - name: | TEST Verify that the {{ item.kind }} {{ item.name }} CR exists - {{ common_cr_test_id }} ansible.builtin.command: cmd: | oc get {{ item.kind }} {{ item.name }} @@ -10,7 +9,8 @@ failed_when: - result.rc != 0 -- name: Verify that a CR is ready {{ common_cr_ready_test_id }} +- name: | + TEST Verify that a CR is ready ansible.builtin.command: cmd: | oc get {{ item.kind }} {{ item.name }} -o jsonpath='{.status.conditions[?(@.type=="{{ item.condition_type }}")].status}{"\n"}' diff --git a/roles/common/tasks/crd_tests.yml b/roles/common/tasks/crd_tests.yml index afbe89306..a91ecf923 100644 --- a/roles/common/tasks/crd_tests.yml +++ b/roles/common/tasks/crd_tests.yml @@ -8,9 +8,8 @@ - name: | TEST Verify CRD {{ item }} is found - {{ common_crd_test_id }} ansible.builtin.assert: that: - "'NotFound' not in output.stderr" success_msg: "CRD {{ item }} is found." - fail_msg: "CRD {{ item }} not found. Error: {{ output.stderr }}" \ No newline at end of file + fail_msg: "CRD {{ item }} not found. Error: {{ output.stderr }}" diff --git a/roles/common/tasks/endpoint_tests.yml b/roles/common/tasks/endpoint_tests.yml index b04b937ae..94712b217 100644 --- a/roles/common/tasks/endpoint_tests.yml +++ b/roles/common/tasks/endpoint_tests.yml @@ -8,7 +8,6 @@ - name: | TEST Check that {{ item[0] }} {{ item[2] }} endpoint exists - {{ common_endpoint_test_id }} ansible.builtin.assert: that: - output.stdout != "" diff --git a/roles/common/tasks/file_tests.yml b/roles/common/tasks/file_tests.yml index c56d86341..d06a5f1fe 100644 --- a/roles/common/tasks/file_tests.yml +++ b/roles/common/tasks/file_tests.yml @@ -6,7 +6,6 @@ - name: | TEST Verify the file {{ item }} exists - {{ common_file_test_id }} ansible.builtin.assert: that: - fstats.stat.exists diff --git a/roles/common/tasks/manifest_tests.yml b/roles/common/tasks/manifest_tests.yml index 48c4ab869..8cd913086 100644 --- a/roles/common/tasks/manifest_tests.yml +++ b/roles/common/tasks/manifest_tests.yml @@ -15,7 +15,6 @@ - name: | TEST Get {{ item }} packagemanifest - {{ common_manifest_test_id }} ansible.builtin.shell: cmd: | oc get packagemanifests | grep "{{ pack_name.stdout }}" | wc -l diff --git a/roles/common/tasks/pod_tests.yml b/roles/common/tasks/pod_tests.yml index e688c9fcd..3f92900ae 100644 --- a/roles/common/tasks/pod_tests.yml +++ b/roles/common/tasks/pod_tests.yml @@ -13,7 +13,6 @@ - name: | TEST Check {{ item }} pod is {{ common_pod_status_str }} in {{ common_pod_nspace }} namespace - {{ common_pod_test_id }} ansible.builtin.command: cmd: | oc get pod -n "{{ common_pod_nspace }}" "{{ podinstance.stdout }}" diff --git a/roles/common/tasks/project_tests.yml b/roles/common/tasks/project_tests.yml index 487961a7b..ccf53fd0b 100644 --- a/roles/common/tasks/project_tests.yml +++ b/roles/common/tasks/project_tests.yml @@ -8,9 +8,8 @@ - name: | TEST Verify {{ item }} project exists - {{ common_project_test_id }} ansible.builtin.assert: that: - "'NotFound' not in output.stderr" success_msg: "project {{ item }} is found." - fail_msg: "project {{ item }} not found. Error: {{ output.stderr }}" \ No newline at end of file + fail_msg: "project {{ item }} not found. Error: {{ output.stderr }}" diff --git a/roles/common/tasks/service_tests.yml b/roles/common/tasks/service_tests.yml index 1a1fcf7e3..3c3884104 100644 --- a/roles/common/tasks/service_tests.yml +++ b/roles/common/tasks/service_tests.yml @@ -8,9 +8,8 @@ - name: | TEST Verify {{ item }} service is running in {{ common_service_nspace }} namespace - {{ common_service_test_id }} ansible.builtin.assert: that: - "'NotFound' not in output.stderr" success_msg: "service {{ item }} is running." - fail_msg: "service {{ item }} not running. Error: {{ output.stderr }}" \ No newline at end of file + fail_msg: "service {{ item }} not running. Error: {{ output.stderr }}" diff --git a/roles/common/tasks/subscription_tests.yml b/roles/common/tasks/subscription_tests.yml index 4b27530cd..8584198c3 100644 --- a/roles/common/tasks/subscription_tests.yml +++ b/roles/common/tasks/subscription_tests.yml @@ -8,9 +8,8 @@ - name: | TEST Check that {{ item }} subscription exists in {{ common_subscription_nspace }} namespace - {{ common_subscription_test_id }} ansible.builtin.assert: that: - "'NotFound' not in output.stderr" success_msg: "subscription {{ item }} is found." - fail_msg: "subscription {{ item }} not found. Error: {{ output.stderr }}" \ No newline at end of file + fail_msg: "subscription {{ item }} not found. Error: {{ output.stderr }}"