twister: fixes to statistics and counting, reduce platform scan verbosity

scripts/pylib/twister/twisterlib/reports.py

@@ -596,7 +596,7 @@ def summary(self, results, ignore_unrecognized_sections, duration):
             f" {f'{TwisterStatus.get_color(TwisterStatus.NOTRUN)}{results.notrun}{Fore.RESET}' if results.notrun else f'{results.notrun}'} built (not run),"
             f" {f'{TwisterStatus.get_color(TwisterStatus.FAIL)}{results.failed}{Fore.RESET}' if results.failed else f'{results.failed}'} failed,"
             f" {f'{TwisterStatus.get_color(TwisterStatus.ERROR)}{results.error}{Fore.RESET}' if results.error else f'{results.error}'} errored,"
-            f" with {f'{Fore.YELLOW}{self.plan.warnings}{Fore.RESET}' if self.plan.warnings else 'no'} warnings"
+            f" with {f'{Fore.YELLOW}{self.plan.warnings + results.warnings}{Fore.RESET}' if (self.plan.warnings + results.warnings) else 'no'} warnings"
             f" in {duration:.2f} seconds."
         )
 
@@ -615,12 +615,11 @@ def summary(self, results, ignore_unrecognized_sections, duration):
                 f'{", " + str(results.none_cases) + " without a status" if results.none_cases else ""}'
                 f' on {len(self.filtered_platforms)} out of total {total_platforms} platforms ({platform_rate:02.2f}%).'
             )
-            if results.skipped_cases or results.filtered_cases or results.notrun_cases:
+            if results.skipped_cases or results.notrun_cases:
                 logger.info(
-                    f'{results.skipped_cases + results.filtered_cases} selected test cases not executed:' \
+                    f'{results.skipped_cases + results.notrun_cases} selected test cases not executed:' \
                     f'{" " + str(results.skipped_cases) + " skipped" if results.skipped_cases else ""}' \
-                    f'{(", " if results.skipped_cases else " ") + str(results.filtered_cases) + " filtered" if results.filtered_cases else ""}' \
-                    f'{(", " if results.skipped_cases or results.filtered_cases else " ") + str(results.notrun_cases) + " not run (built only)" if results.notrun_cases else ""}' \
+                    f'{(", " if results.skipped_cases else " ") + str(results.notrun_cases) + " not run (built only)" if results.notrun_cases else ""}' \
                     f'.'
                 )
 

scripts/pylib/twister/twisterlib/runner.py

@@ -129,6 +129,7 @@ def __init__(self, total=0):
         self._none_cases = Value('i', 0)
         self._started_cases = Value('i', 0)
 
+        self._warnings = Value('i', 0)
 
         self.lock = Lock()
 
@@ -143,7 +144,7 @@ def _find_number_length(n):
         return length
 
     def summary(self):
-        executed_cases = self.cases - self.skipped_cases - self.filtered_cases
+        selected_cases = self.cases - self.filtered_cases
         completed_configs = self.done - self.skipped_filter
 
         # Find alignment length for aesthetic printing
@@ -152,7 +153,7 @@ def summary(self):
         completed_suites_n_length = self._find_number_length(completed_configs)
         skipped_suites_n_length = self._find_number_length(self.skipped_configs)
         total_cases_n_length = self._find_number_length(self.cases)
-        executed_cases_n_length = self._find_number_length(executed_cases)
+        selected_cases_n_length = self._find_number_length(selected_cases)
 
         print("--------------------------------------------------")
         print(f"{'Total test suites: ':<23}{self.total:>{suites_n_length}}") # actually test instances
@@ -171,21 +172,35 @@ def summary(self):
         print("----------------------      ----------------------")
         print(f"{'Total test cases: ':<18}{self.cases}")
         print(f"├─ {'Filtered test cases: ':<21}{self.filtered_cases:>{total_cases_n_length}}")
-        print(f"├─ {'Skipped test cases: ':<21}{self.skipped_cases:>{total_cases_n_length}}")
-        print(f"└─ {'Executed test cases: ':<21}{executed_cases:>{total_cases_n_length}}")
-        print(f"   ├─ {'Passed test cases: ':<25}{self.passed_cases:>{executed_cases_n_length}}")
-        print(f"   ├─ {'Built only test cases: ':<25}{self.notrun_cases:>{executed_cases_n_length}}")
-        print(f"   ├─ {'Blocked test cases: ':<25}{self.blocked_cases:>{executed_cases_n_length}}")
-        print(f"   ├─ {'Failed test cases: ':<25}{self.failed_cases:>{executed_cases_n_length}}")
-        print(f"   {'├' if self.none_cases or self.started_cases else '└'}─ {'Errors in test cases: ':<25}{self.error_cases:>{executed_cases_n_length}}")
+        print(f"└─ {'Selected test cases: ':<21}{selected_cases:>{total_cases_n_length}}")
+        print(f"   ├─ {'Passed test cases: ':<25}{self.passed_cases:>{selected_cases_n_length}}")
+        print(f"   ├─ {'Skipped test cases: ':<25}{self.skipped_cases:>{total_cases_n_length}}")
+        print(f"   ├─ {'Built only test cases: ':<25}{self.notrun_cases:>{selected_cases_n_length}}")
+        print(f"   ├─ {'Blocked test cases: ':<25}{self.blocked_cases:>{selected_cases_n_length}}")
+        print(f"   ├─ {'Failed test cases: ':<25}{self.failed_cases:>{selected_cases_n_length}}")
+        print(f"   {'├' if self.none_cases or self.started_cases else '└'}─ {'Errors in test cases: ':<25}{self.error_cases:>{selected_cases_n_length}}")
         if self.none_cases or self.started_cases:
             print(f"   ├──── The following test case statuses should not appear in a proper execution ───")
         if self.none_cases:
-            print(f"   {'├' if self.started_cases else '└'}─ {'Statusless test cases: ':<25}{self.none_cases:>{executed_cases_n_length}}")
+            print(f"   {'├' if self.started_cases else '└'}─ {'Statusless test cases: ':<25}{self.none_cases:>{selected_cases_n_length}}")
         if self.started_cases:
-            print(f"   └─ {'Test cases only started: ':<25}{self.started_cases:>{executed_cases_n_length}}")
+            print(f"   └─ {'Test cases only started: ':<25}{self.started_cases:>{selected_cases_n_length}}")
         print("--------------------------------------------------")
 
+    @property
+    def warnings(self):
+        with self._warnings.get_lock():
+            return self._warnings.value
+
+    @warnings.setter
+    def warnings(self, value):
+        with self._warnings.get_lock():
+            self._warnings.value = value
+
+    def warnings_increment(self, value=1):
+        with self._warnings.get_lock():
+            self._warnings.value += value
+
     @property
     def cases(self):
         with self._cases.get_lock():
@@ -877,8 +892,7 @@ def process(self, pipeline, done, message, lock, results):
                         logger.debug("filtering %s" % self.instance.name)
                         self.instance.status = TwisterStatus.FILTER
                         self.instance.reason = "runtime filter"
-                        results.skipped_runtime_increment()
-                        self.instance.add_missing_case_status(TwisterStatus.SKIP)
+                        self.instance.add_missing_case_status(TwisterStatus.FILTER)
                         next_op = 'report'
                     else:
                         next_op = 'cmake'
@@ -911,7 +925,7 @@ def process(self, pipeline, done, message, lock, results):
                         self.instance.status = TwisterStatus.FILTER
                         self.instance.reason = "runtime filter"
                         results.skipped_runtime_increment()
-                        self.instance.add_missing_case_status(TwisterStatus.SKIP)
+                        self.instance.add_missing_case_status(TwisterStatus.FILTER)
                         next_op = 'report'
                     else:
                         next_op = 'build'
@@ -1324,15 +1338,18 @@ def _add_instance_testcases_to_status_counts(instance, results, decrement=False)
                 # but having those statuses in this part of processing is an error.
                 case TwisterStatus.NONE:
                     results.none_cases_increment(increment_value)
-                    logger.error(f'A None status detected in instance {instance.name},'
+                    logger.warning(f'A None status detected in instance {instance.name},'
                                  f' test case {tc.name}.')
+                    results.warnings_increment(1)
                 case TwisterStatus.STARTED:
                     results.started_cases_increment(increment_value)
-                    logger.error(f'A started status detected in instance {instance.name},'
+                    logger.warning(f'A started status detected in instance {instance.name},'
                                  f' test case {tc.name}.')
+                    results.warnings_increment(1)
                 case _:
-                    logger.error(f'An unknown status "{tc.status}" detected in instance {instance.name},'
+                    logger.warning(f'An unknown status "{tc.status}" detected in instance {instance.name},'
                                  f' test case {tc.name}.')
+                    results.warnings_increment(1)
 
 
     def report_out(self, results):
@@ -1369,15 +1386,8 @@ def report_out(self, results):
             results.skipped_configs_increment()
         elif instance.status == TwisterStatus.PASS:
             results.passed_increment()
-            for case in instance.testcases:
-                # test cases skipped at the test case level
-                if case.status == TwisterStatus.SKIP:
-                    results.skipped_cases_increment()
         elif instance.status == TwisterStatus.NOTRUN:
             results.notrun_increment()
-            for case in instance.testcases:
-                if case.status == TwisterStatus.SKIP:
-                    results.skipped_cases_increment()
         else:
             logger.debug(f"Unknown status = {instance.status}")
             status = Fore.YELLOW + "UNKNOWN" + Fore.RESET
@@ -1696,7 +1706,7 @@ def update_counting_before_pipeline(self):
                 self.results.error_increment()
 
     def show_brief(self):
-        logger.info("%d test scenarios (%d test instances) selected, "
+        logger.info("%d test scenarios (%d configurations) selected, "
                     "%d configurations filtered (%d by static filter, %d at runtime)." %
                     (len(self.suites), len(self.instances),
                     self.results.skipped_configs,

scripts/pylib/twister/twisterlib/testplan.py

@@ -439,7 +439,6 @@ def init_and_add_platforms(data, board, target, qualifier, aliases):
                 raise Exception(f"Duplicate platform identifier {platform.name} found")
             if not platform.twister:
                 return
-            logger.debug(f"Adding platform {platform.name} with aliases {platform.aliases}")
             self.platforms.append(platform)
 
         for board in known_boards.values():

scripts/tests/twister/test_runner.py

@@ -223,9 +223,9 @@ def test_executioncounter(capfd):
         '----------------------      ----------------------\n'
         'Total test cases: 25\n'
         '├─ Filtered test cases:  0\n'
-        '├─ Skipped test cases:   6\n'
-        '└─ Executed test cases: 19\n'
+        '└─ Selected test cases: 25\n'
         '   ├─ Passed test cases:        0\n'
+        '   ├─ Skipped test cases:       6\n'
         '   ├─ Built only test cases:    0\n'
         '   ├─ Blocked test cases:       0\n'
         '   ├─ Failed test cases:        0\n'
@@ -914,8 +914,8 @@ def mock_getsize(filename, *args, **kwargs):
         {'op': 'report', 'test': mock.ANY},
         TwisterStatus.FILTER,
         'runtime filter',
-        1,
-        (TwisterStatus.SKIP,)
+        0,
+        (TwisterStatus.FILTER,)
     ),
     (
         {'op': 'filter'},
@@ -1025,7 +1025,7 @@ def mock_getsize(filename, *args, **kwargs):
         TwisterStatus.FILTER,
         'runtime filter',
         1,
-        (TwisterStatus.SKIP,)
+        (TwisterStatus.FILTER,) # this is a tuple
     ),
     (
         {'op': 'cmake'},
@@ -2627,7 +2627,7 @@ def test_twisterrunner_show_brief(caplog):
 
     tr.show_brief()
 
-    log = '2 test scenarios (5 test instances) selected,' \
+    log = '2 test scenarios (5 configurations) selected,' \
           ' 4 configurations filtered (3 by static filter, 1 at runtime).'
 
     assert log in caplog.text

scripts/tests/twister_blackbox/test_platform.py

@@ -37,7 +37,7 @@ class TestPlatform:
                 'failed_configurations': 0,
                 'errored_configurations': 0,
                 'executed_test_cases': 8,
-                'skipped_test_cases': 5,
+                'skipped_test_cases': 2,
                 'platform_count': 3,
                 'executed_on_platform': 4,
                 'only_built': 2
@@ -58,7 +58,7 @@ class TestPlatform:
                 'failed_configurations': 0,
                 'errored_configurations': 0,
                 'executed_test_cases': 0,
-                'skipped_test_cases': 3,
+                'skipped_test_cases': 0,
                 'platform_count': 3,
                 'executed_on_platform': 0,
                 'only_built': 0
@@ -264,7 +264,7 @@ def test_emulation_only(self, capfd, out_path, test_path, test_platforms, expect
             self.loader.exec_module(self.twister_module)
 
         select_regex = r'^INFO    - (?P<test_scenarios>[0-9]+) test scenarios' \
-                       r' \((?P<test_instances>[0-9]+) test instances\) selected,' \
+                       r' \((?P<test_instances>[0-9]+) configurations\) selected,' \
                        r' (?P<skipped_configurations>[0-9]+) configurations filtered' \
                        r' \((?P<skipped_by_static_filter>[0-9]+) by static filter,' \
                        r' (?P<skipped_at_runtime>[0-9]+) at runtime\)\.$'

scripts/tests/twister_blackbox/test_runner.py

@@ -260,7 +260,7 @@ def test_runtest_only(self, capfd, out_path, test_path, test_platforms, expected
 
 
         select_regex = r'^INFO    - (?P<test_scenarios>[0-9]+) test scenarios' \
-                       r' \((?P<test_instances>[0-9]+) test instances\) selected,' \
+                       r' \((?P<test_instances>[0-9]+) configurations\) selected,' \
                        r' (?P<skipped_configurations>[0-9]+) configurations filtered' \
                        r' \((?P<skipped_by_static_filter>[0-9]+) by static filter,' \
                        r' (?P<skipped_at_runtime>[0-9]+) at runtime\)\.$'
@@ -627,7 +627,7 @@ def test_only_failed(self, capfd, out_path, test_path, test_platforms, expected)
             self.loader.exec_module(self.twister_module)
 
         select_regex = r'^INFO    - (?P<test_scenarios>[0-9]+) test scenarios' \
-                       r' \((?P<test_instances>[0-9]+) test instances\) selected,' \
+                       r' \((?P<test_instances>[0-9]+) configurations\) selected,' \
                        r' (?P<skipped_configurations>[0-9]+) configurations filtered' \
                        r' \((?P<skipped_by_static_filter>[0-9]+) by static filter,' \
                        r' (?P<skipped_at_runtime>[0-9]+) at runtime\)\.$'