adapt testcase evaluation and reporting

Signed-off-by: Matthias Büchse <[email protected]>

Author: mbuechse

Tests/scs-compliance-check.py

@@ -282,16 +282,19 @@ def run_suite(suite: TestSuite, runner: CheckRunner):
 def print_report(subject: str, suite: TestSuite, targets: dict, results: dict, verbose=False):
     print(f"{subject} {suite.name}:")
     for tname, target_spec in targets.items():
-        failed, missing, passed = suite.select(tname, target_spec).eval_buckets(results)
-        verdict = 'FAIL' if failed else 'TENTATIVE pass' if missing else 'PASS'
+        by_value = suite.select(tname, target_spec).eval_buckets(results)
+        missing, failed, aborted, passed = by_value[None], by_value[-1], by_value[0], by_value[1]
+        verdict = 'FAIL' if failed or aborted else 'TENTATIVE pass' if missing else 'PASS'
         summary_parts = [f"{len(passed)} passed"]
         if failed:
             summary_parts.append(f"{len(failed)} failed")
+        if aborted:
+            summary_parts.append(f"{len(aborted)} aborted")
         if missing:
             summary_parts.append(f"{len(missing)} missing")
         verdict += f" ({', '.join(summary_parts)})"
         print(f"- {tname}: {verdict}")
-        reportcateg = [(failed, 'FAILED'), (missing, 'MISSING')]
+        reportcateg = [(failed, 'FAILED'), (aborted, 'ABORTED'), (missing, 'MISSING')]
         if verbose:
             reportcateg.append((passed, 'PASSED'))
         for offenders, category in reportcateg:

Tests/scs_cert_lib.py

@@ -23,12 +23,10 @@
     'testcases': ('lifetime', 'id', 'description', 'tags'),
     'include': ('ref', 'parameters'),
 }
-# The canonical result values are -1, 0, and 1, for FAIL, MISS (or DNF), and PASS, respectively;
-# these concrete numbers are important because we do rely on their ordering. Note that MISS/DNF should
-# not be reported because it is tantamount to a result being absent. (Therefore the NIL_RESULT default
-# value below.)
-TESTCASE_VERDICTS = {'PASS': 1, 'FAIL': -1}
-NIL_RESULT = {'result': 0}
+# The canonical result values are -1, 0, and 1, for FAIL, ABORT, and PASS, respectively;
+# -- in addition, None is used to encode a missing value, but must not be included in a formal report! --
+# these concrete numbers are important because we do rely on their ordering.
+TESTCASE_VERDICTS = {'PASS': 1, 'ABORT': 0, 'FAIL': -1}
 
 
 def _check_keywords(ctx, d, keywords=KEYWORDS):
@@ -208,18 +206,26 @@ def select(self, name, selectors):
         suite.include_testcases([tc for tc in self.testcases if test_selectors(selectors, tc['tags'])])
         return suite
 
-    def eval_buckets(self, results) -> tuple[list[dict], list[dict], list[dict]]:
-        """returns lists of (failed, missing, passed) test cases"""
+    def eval_buckets(self, results) -> dict:
+        """
+        returns buckets of test cases by means of a mapping
+
+        None: list of missing testcases
+        -1: list of failed testcases
+        0: list of aborted testcases
+        1: list of passed testcases
+        """
         by_value = defaultdict(list)
         for testcase in self.testcases:
-            value = results.get(testcase['id'], NIL_RESULT).get('result', 0)
+            value = results.get(testcase['id'], {}).get('result')
             by_value[value].append(testcase)
-        return by_value[-1], by_value[0], by_value[1]
+        return by_value
 
     def evaluate(self, results) -> int:
         """returns overall result"""
         return min([
-            results.get(testcase['id'], NIL_RESULT).get('result', 0)
+            # here, we treat None (MISSING) as 0 (ABORT)
+            results.get(testcase['id'], {}).get('result') or 0
             for testcase in self.testcases
         ], default=0)