twister: stats: fix suite statistics

suite stats were not correct, a mixup between skipped and filtered
suites was leading to inconsistent numbers. This is now fixed.

Signed-off-by: Anas Nashif <[email protected]>

Author: nashif

scripts/pylib/twister/twisterlib/runner.py

@@ -104,6 +104,7 @@ def __init__(self, total=0):
         # updated by report_out() in pipeline
         self._error = Value('i', 0)
         self._failed = Value('i', 0)
+        self._skipped = Value('i', 0)
 
         # initialized to number of test instances
         self._total = Value('i', total)
@@ -145,30 +146,27 @@ def _find_number_length(n):
 
     def summary(self):
         selected_cases = self.cases - self.filtered_cases
-        completed_configs = self.done - self.filtered_static
+        completed_configs = self.done - self.filtered_static - self.filtered_runtime
 
         # Find alignment length for aesthetic printing
         suites_n_length = self._find_number_length(self.total if self.total > self.done else self.done)
-        processed_suites_n_length = self._find_number_length(self.done)
         completed_suites_n_length = self._find_number_length(completed_configs)
-        skipped_suites_n_length = self._find_number_length(self.filtered_configs)
+        filtered_suites_n_length = self._find_number_length(self.filtered_configs)
         total_cases_n_length = self._find_number_length(self.cases)
         selected_cases_n_length = self._find_number_length(selected_cases)
 
         print("--------------------------------------------------")
         print(f"{'Total test suites: ':<23}{self.total:>{suites_n_length}}") # actually test instances
         print(f"{'Processed test suites: ':<23}{self.done:>{suites_n_length}}")
-        print(f"├─ {'Filtered test suites (static): ':<37}{self.filtered_static:>{processed_suites_n_length}}")
-        print(f"└─ {'Selected test suites: ':<37}{completed_configs:>{processed_suites_n_length}}")
-        print(f"   ├─ {'Skipped test suites: ':<37}{self.filtered_runtime:>{completed_suites_n_length}}")
+        print(f"└─{'Filtered test suites: ':<21}{self.filtered_configs}")
+        print(f"  ├─ {'Filtered test suites (static): ':<37}{self.filtered_static:>{filtered_suites_n_length}}")
+        print(f"  └─ {'Filtered test suites (at runtime): ':<37}{self.filtered_runtime:>{filtered_suites_n_length}}")
+        print(f"└─ {'Selected test suites: ':<37}{completed_configs:>{completed_suites_n_length}}")
+        print(f"   ├─ {'Skipped test suites: ':<37}{self.skipped:>{completed_suites_n_length}}")
         print(f"   ├─ {'Passed test suites: ':<37}{self.passed:>{completed_suites_n_length}}")
         print(f"   ├─ {'Built only test suites: ':<37}{self.notrun:>{completed_suites_n_length}}")
         print(f"   ├─ {'Failed test suites: ':<37}{self.failed:>{completed_suites_n_length}}")
         print(f"   └─ {'Errors in test suites: ':<37}{self.error:>{completed_suites_n_length}}")
-        print(f"")
-        print(f"{'Filtered test suites: ':<21}{self.filtered_configs}")
-        print(f"├─ {'Filtered test suites (static): ':<37}{self.filtered_static:>{skipped_suites_n_length}}")
-        print(f"└─ {'Filtered test suites (at runtime): ':<37}{self.filtered_runtime:>{skipped_suites_n_length}}")
         print("----------------------      ----------------------")
         print(f"{'Total test cases: ':<18}{self.cases}")
         print(f"├─ {'Filtered test cases: ':<21}{self.filtered_cases:>{total_cases_n_length}}")
@@ -341,6 +339,20 @@ def started_cases_increment(self, value=1):
         with self._started_cases.get_lock():
             self._started_cases.value += value
 
+    @property
+    def skipped(self):
+        with self._skipped.get_lock():
+            return self._skipped.value
+
+    @skipped.setter
+    def skipped(self, value):
+        with self._skipped.get_lock():
+            self._skipped.value = value
+
+    def skipped_increment(self, value=1):
+        with self._skipped.get_lock():
+            self._skipped.value += value
+
     @property
     def error(self):
         with self._error.get_lock():
@@ -892,6 +904,7 @@ def process(self, pipeline, done, message, lock, results):
                         logger.debug("filtering %s" % self.instance.name)
                         self.instance.status = TwisterStatus.FILTER
                         self.instance.reason = "runtime filter"
+                        results.filtered_runtime_increment()
                         self.instance.add_missing_case_status(TwisterStatus.FILTER)
                         next_op = 'report'
                     else:
@@ -951,7 +964,7 @@ def process(self, pipeline, done, message, lock, results):
                     # Count skipped cases during build, for example
                     # due to ram/rom overflow.
                     if  self.instance.status == TwisterStatus.SKIP:
-                        results.filtered_runtime_increment()
+                        results.skipped_increment()
                         self.instance.add_missing_case_status(TwisterStatus.SKIP, self.instance.reason)
 
                     if ret.get('returncode', 1) > 0:
@@ -1381,7 +1394,7 @@ def report_out(self, results):
             if not self.options.verbose:
                 self.log_info_file(self.options.inline_logs)
         elif instance.status == TwisterStatus.SKIP:
-            results.filtered_configs_increment()
+            results.skipped_increment()
         elif instance.status == TwisterStatus.FILTER:
             results.filtered_configs_increment()
         elif instance.status == TwisterStatus.PASS:

scripts/tests/twister/test_runner.py

@@ -209,17 +209,15 @@ def test_executioncounter(capfd):
         '--------------------------------------------------\n'
         'Total test suites:     12\n'
         'Processed test suites:  9\n'
-        '├─ Filtered test suites (static):       2\n'
-        '└─ Completed test suites:               7\n'
-        '   ├─ Filtered test suites (at runtime):   1\n'
+        '└─Filtered test suites: 3\n'
+        '  ├─ Filtered test suites (static):       2\n'
+        '  └─ Filtered test suites (at runtime):   1\n'
+        '└─ Selected test suites:                6\n'
+        '   ├─ Skipped test suites:                 0\n'
         '   ├─ Passed test suites:                  6\n'
         '   ├─ Built only test suites:              0\n'
         '   ├─ Failed test suites:                  1\n'
         '   └─ Errors in test suites:               2\n'
-        '\n'
-        'Filtered test suites: 3\n'
-        '├─ Filtered test suites (static):       2\n'
-        '└─ Filtered test suites (at runtime):   1\n'
         '----------------------      ----------------------\n'
         'Total test cases: 25\n'
         '├─ Filtered test cases:  0\n'
@@ -914,7 +912,7 @@ def mock_getsize(filename, *args, **kwargs):
         {'op': 'report', 'test': mock.ANY},
         TwisterStatus.FILTER,
         'runtime filter',
-        0,
+        1,
         (TwisterStatus.FILTER,)
     ),
     (
@@ -1091,7 +1089,7 @@ def mock_getsize(filename, *args, **kwargs):
         {'op': 'gather_metrics', 'test': mock.ANY},
         mock.ANY,
         mock.ANY,
-        1,
+        0,
         (TwisterStatus.SKIP, mock.ANY)
     ),
     (