enable result retrieval for Sonobuoy (#937)

resolves #887

Signed-off-by: Matthias Büchse <[email protected]>

Author: mbuechse

Tests/kaas/requirements.in

@@ -3,4 +3,3 @@ pytest-kind
 kubernetes
 kubernetes_asyncio
 jinja2
-junitparser

Tests/kaas/requirements.txt

@@ -48,8 +48,6 @@ idna==3.10
     #   yarl
 jinja2==3.1.6
     # via -r requirements.in
-junitparser==3.2.0
-    # via -r requirements.in
 kubernetes==32.0.1
     # via -r requirements.in
 kubernetes-asyncio==32.3.2

Tests/kaas/sonobuoy_handler/sonobuoy_handler.py

@@ -7,7 +7,7 @@
 import shutil
 import subprocess
 
-from junitparser import JUnitXml
+import yaml
 
 logger = logging.getLogger(__name__)
 
@@ -17,9 +17,15 @@ def _find_sonobuoy():
     result = shutil.which('sonobuoy')
     if result:
         return result
+    logger.debug('sonobuoy not in PATH, trying $HOME/.local/bin')
     result = os.path.join(os.path.expanduser('~'), '.local', 'bin', 'sonobuoy')
     if os.path.exists(result):
         return result
+    logger.debug('sonobuoy executable not found; expect errors')
+
+
+def _fmt_result(counter):
+    return ', '.join(f"{counter.get(key, 0)} {key}" for key in ('passed', 'failed', 'skipped'))
 
 
 class SonobuoyHandler:
@@ -68,15 +74,14 @@ def _sonobuoy_status_result(self):
         json_data = json.loads(process.stdout)
         counter = Counter()
         for entry in json_data["plugins"]:
-            logger.debug(f"plugin:{entry['plugin']}:{entry['result-status']}")
-            for result, count in entry["result-counts"].items():
-                counter[result] += count
+            logger.debug(f"plugin {entry['plugin']}: {_fmt_result(entry['result-counts'])}")
+            for key, value in entry["result-counts"].items():
+                counter[key] += value
         return counter
 
     def _eval_result(self, counter):
         """evaluate test results and return return code"""
-        result_str = ', '.join(f"{counter[key]} {key}" for key in ('passed', 'failed', 'skipped'))
-        result_message = f"sonobuoy reports {result_str}"
+        result_message = f"sonobuoy reports {_fmt_result(counter)}"
         if counter['failed']:
             logger.error(result_message)
             return 3
@@ -90,37 +95,33 @@ def _preflight_check(self):
         if not self.sonobuoy:
             raise RuntimeError("sonobuoy executable not found; is it in PATH?")
 
-    def _sonobuoy_retrieve_result(self):
+    def _sonobuoy_retrieve_result(self, plugin='e2e'):
         """
-        This method invokes sonobuoy to store the results in a subdirectory of
-        the working directory. The Junit results file contained in it is then
-        analyzed in order to interpret the relevant information it containes
+        Invoke sonobuoy to retrieve results and to store them in a subdirectory of
+        the working directory. Analyze the results yaml file for given `plugin` and
+        log each failure as ERROR. Return summary dict like `_sonobuoy_status_result`.
         """
         logger.debug(f"retrieving results to {self.result_dir_name}")
         result_dir = os.path.join(self.working_directory, self.result_dir_name)
-        if os.path.exists(result_dir):
-            raise Exception("result directory already existing")
-        os.mkdir(result_dir)
-
-        # XXX use self._invoke_sonobuoy
-        os.system(
-            # ~ f"sonobuoy retrieve {result_dir} -x --filename='{result_dir}' --kubeconfig='{self.kubeconfig_path}'"
-            f"sonobuoy retrieve {result_dir} --kubeconfig='{self.kubeconfig_path}'"
-        )
-        logger.debug(
-            f"parsing JUnit result from {result_dir + '/plugins/e2e/results/global/junit_01.xml'} "
-        )
-        xml = JUnitXml.fromfile(result_dir + "/plugins/e2e/results/global/junit_01.xml")
+        os.makedirs(result_dir, exist_ok=True)
+
+        self._invoke_sonobuoy("retrieve", "-x", result_dir)
+        yaml_path = os.path.join(result_dir, 'plugins', plugin, 'sonobuoy_results.yaml')
+        logger.debug(f"parsing results from {yaml_path}")
+        with open(yaml_path, "r") as fileobj:
+            result_obj = yaml.load(fileobj.read(), yaml.SafeLoader)
         counter = Counter()
-        for suite in xml:
-            for case in suite:
-                if case.is_passed is True:  # XXX why `is True`???
-                    counter['passed'] += 1
-                elif case.is_skipped is True:
-                    counter['skipped'] += 1
-                else:
-                    counter['failed'] += 1
-                    logger.error(f"{case.name}")
+        for item1 in result_obj.get('items', ()):
+            # file ...
+            for item2 in item1.get('items', ()):
+                # suite ...
+                for item in item2.get('items', ()):
+                    # testcase ... or so
+                    status = item.get('status', 'skipped')
+                    counter[status] += 1
+                    if status == 'failed':
+                        logger.error(f"FAILED: {item['name']}")  # <-- this is why this method exists!
+        logger.info(f"{plugin} results: {_fmt_result(counter)}")
         return counter
 
     def run(self):
@@ -133,11 +134,12 @@ def run(self):
             self._sonobuoy_run()
             return_code = self._eval_result(self._sonobuoy_status_result())
             print(self.check_name + ": " + ("PASS", "FAIL")[min(1, return_code)])
+            try:
+                self._sonobuoy_retrieve_result()
+            except Exception:
+                # swallow exception for the time being
+                logger.debug('problem retrieving results', exc_info=True)
             return return_code
-
-            # ERROR: currently disabled due to: "error retrieving results: unexpected EOF"
-            #  might be related to following bug: https://github.com/vmware-tanzu/sonobuoy/issues/1633
-            # self._sonobuoy_retrieve_result(self)
         except BaseException:
             logger.exception("something went wrong")
             return 112