scripts: twister: Fix NOTRUN in test_only

When using the --build-only into --test-only
Twister setup, NOTRUN statuses were not properly rerun.

Now they are properly run again if runnable.

Signed-off-by: Lukasz Mrugala <[email protected]>

Author: LukaszMrugala

scripts/pylib/twister/twisterlib/reports.py

@@ -354,6 +354,12 @@ def json_report(self, filename, version="NA", platform=None, filters=None):
             elif instance.status == TwisterStatus.SKIP:
                 suite["status"] = TwisterStatus.SKIP
                 suite["reason"] = instance.reason
+            elif instance.status == TwisterStatus.NOTRUN:
+                suite["status"] = TwisterStatus.NOTRUN
+                suite["reason"] = instance.reason
+            else:
+                suite["status"] = TwisterStatus.NONE
+                suite["reason"] = 'Unknown Instance status.'
 
             if instance.status != TwisterStatus.NONE:
                 suite["execution_time"] =  f"{float(handler_time):.2f}"

scripts/pylib/twister/twisterlib/testplan.py

@@ -659,6 +659,9 @@ def load_from_file(self, file, filter_platform=[]):
                         self.hwm
                     )
 
+                    if self.options.test_only and not instance.run:
+                        continue
+
                     instance.metrics['handler_time'] = ts.get('execution_time', 0)
                     instance.metrics['used_ram'] = ts.get("used_ram", 0)
                     instance.metrics['used_rom']  = ts.get("used_rom",0)
@@ -676,9 +679,9 @@ def load_from_file(self, file, filter_platform=[]):
                             instance.status = TwisterStatus.NONE
                             instance.reason = None
                             instance.retries += 1
-                    # test marked as passed (built only) but can run when
-                    # --test-only is used. Reset status to capture new results.
-                    elif status == TwisterStatus.PASS and instance.run and self.options.test_only:
+                    # test marked as built only can run when --test-only is used.
+                    # Reset status to capture new results.
+                    elif status == TwisterStatus.NOTRUN and instance.run and self.options.test_only:
                         instance.status = TwisterStatus.NONE
                         instance.reason = None
                     else:

scripts/tests/twister/test_testplan.py

@@ -766,6 +766,7 @@ def test_testplan_load(
     testplan.apply_filters = mock.Mock()
 
     with mock.patch('twisterlib.testinstance.TestInstance.create_overlay', mock.Mock()), \
+         mock.patch('twisterlib.testinstance.TestInstance.check_runnable', return_value=True), \
          pytest.raises(exception) if exception else nullcontext():
         testplan.load()
 
@@ -1600,7 +1601,7 @@ def get_platform(name):
             'testcases': {
                 'TS1.tc1': {
                     'status': TwisterStatus.PASS,
-                    'reason': None,
+                    'reason': 'passed',
                     'duration': 60.0,
                     'output': ''
                 }

scripts/tests/twister_blackbox/test_footprint.py

@@ -16,6 +16,7 @@
 
 # pylint: disable=no-name-in-module
 from conftest import ZEPHYR_BASE, TEST_DATA, testsuite_filename_mock, clear_log_in_test
+from twisterlib.statuses import TwisterStatus
 from twisterlib.testplan import TestPlan
 
 
@@ -76,9 +77,10 @@ def test_compare_report(self, caplog, out_path, old_ram_multiplier, expect_delta
         with open(os.path.join(out_path, 'twister.json')) as f:
             j = json.load(f)
             for ts in j['testsuites']:
-                if 'reason' not in ts:
+                if TwisterStatus(ts.get('status')) == TwisterStatus.NOTRUN:
                     # We assume positive RAM usage.
                     ts[self.RAM_KEY] *= old_ram_multiplier
+
         with open(os.path.join(out_path, 'twister.json'), 'w') as f:
             f.write(json.dumps(j, indent=4))
 
@@ -137,7 +139,7 @@ def test_footprint_from_buildlog(self, out_path):
         with open(os.path.join(out_path, 'twister.json')) as f:
             j = json.load(f)
             for ts in j['testsuites']:
-                if 'reason' not in ts:
+                if TwisterStatus(ts.get('status')) == TwisterStatus.NOTRUN:
                     assert self.RAM_KEY in ts
                     old_values += [ts[self.RAM_KEY]]
 
@@ -162,7 +164,7 @@ def test_footprint_from_buildlog(self, out_path):
         with open(os.path.join(out_path, 'twister.json')) as f:
             j = json.load(f)
             for ts in j['testsuites']:
-                if 'reason' not in ts:
+                if TwisterStatus(ts.get('status')) == TwisterStatus.NOTRUN:
                     assert self.RAM_KEY in ts
                     new_values += [ts[self.RAM_KEY]]
 
@@ -202,7 +204,7 @@ def test_footprint_threshold(self, caplog, out_path, old_ram_multiplier,
         with open(os.path.join(out_path, 'twister.json')) as f:
             j = json.load(f)
             for ts in j['testsuites']:
-                if 'reason' not in ts:
+                if TwisterStatus(ts.get('status')) == TwisterStatus.NOTRUN:
                     # We assume positive RAM usage.
                     ts[self.RAM_KEY] *= old_ram_multiplier
         with open(os.path.join(out_path, 'twister.json'), 'w') as f:
@@ -271,7 +273,7 @@ def test_show_footprint(self, caplog, out_path, flags, old_ram_multiplier, expec
         with open(os.path.join(out_path, 'twister.json')) as f:
             j = json.load(f)
             for ts in j['testsuites']:
-                if 'reason' not in ts:
+                if TwisterStatus(ts.get('status')) == TwisterStatus.NOTRUN:
                     # We assume positive RAM usage.
                     ts[self.RAM_KEY] *= old_ram_multiplier
         with open(os.path.join(out_path, 'twister.json'), 'w') as f:
@@ -344,7 +346,7 @@ def test_last_metrics(self, caplog, out_path, old_ram_multiplier, expect_delta_l
         with open(os.path.join(out_path, 'twister.json')) as f:
             j = json.load(f)
             for ts in j['testsuites']:
-                if 'reason' not in ts:
+                if TwisterStatus(ts.get('status')) == TwisterStatus.NOTRUN:
                     # We assume positive RAM usage.
                     ts[self.RAM_KEY] *= old_ram_multiplier
         with open(os.path.join(out_path, 'twister.json'), 'w') as f:
@@ -441,7 +443,7 @@ def test_all_deltas(self, caplog, out_path, old_ram_multiplier, expect_delta_log
         with open(os.path.join(out_path, 'twister.json')) as f:
             j = json.load(f)
             for ts in j['testsuites']:
-                if 'reason' not in ts:
+                if TwisterStatus(ts.get('status')) == TwisterStatus.NOTRUN:
                     # We assume positive RAM usage.
                     ts[self.RAM_KEY] *= old_ram_multiplier
         with open(os.path.join(out_path, 'twister.json'), 'w') as f:

scripts/tests/twister_blackbox/test_runner.py

@@ -46,19 +46,19 @@ class TestRunner:
             ['qemu_x86/atom', 'qemu_x86_64/atom', 'intel_adl_crb/alder_lake'],
             {
                 'selected_test_scenarios': 3,
-                'selected_test_instances': 6,
+                'selected_test_instances': 4,
                 'skipped_configurations': 0,
                 'skipped_by_static_filter': 0,
                 'skipped_at_runtime': 0,
                 'passed_configurations': 4,
-                'built_configurations': 2,
+                'built_configurations': 0,
                 'failed_configurations': 0,
                 'errored_configurations': 0,
                 'executed_test_cases': 8,
                 'skipped_test_cases': 0,
                 'platform_count': 0,
                 'executed_on_platform': 4,
-                'only_built': 2
+                'only_built': 0
             }
         )
     ]