twister: refactor twister_main to simplify the system tests

Refactored twister_main.py module to simplify the code and system tests.
Removed the need to patch `sys.argv` in blackbox tests.

Signed-off-by: Lukasz Fundakowski <[email protected]>

Author: fundakol

scripts/pylib/twister/twisterlib/twister_main.py

@@ -9,11 +9,17 @@
 import shutil
 import sys
 import time
+from collections.abc import Sequence
 
 import colorama
 from colorama import Fore
 from twisterlib.coverage import run_coverage
-from twisterlib.environment import TwisterEnv
+from twisterlib.environment import (
+    TwisterEnv,
+    add_parse_arguments,
+    parse_arguments,
+    python_version_guard,
+)
 from twisterlib.hardwaremap import HardwareMap
 from twisterlib.log_helper import close_logging, setup_logging
 from twisterlib.package import Artifacts
@@ -27,7 +33,7 @@ def init_color(colorama_strip):
     colorama.init(strip=colorama_strip)
 
 
-def twister(options: argparse.Namespace, default_options: argparse.Namespace):
+def twister(options: argparse.Namespace, default_options: argparse.Namespace) -> int:
     start_time = time.time()
 
     # Configure color output
@@ -230,9 +236,17 @@ def twister(options: argparse.Namespace, default_options: argparse.Namespace):
     return 0
 
 
-def main(options: argparse.Namespace, default_options: argparse.Namespace):
+def main(argv: Sequence[str] | None = None) -> int:
+    """Main function to run twister."""
     try:
-        return_code = twister(options, default_options)
+        python_version_guard()
+
+        parser = add_parse_arguments()
+        options = parse_arguments(parser, argv)
+        default_options = parse_arguments(parser, [], on_init=False)
+        return twister(options, default_options)
     finally:
         close_logging()
-    return return_code
+        if (os.name != "nt") and os.isatty(1):
+            # (OS is not Windows) and (stdout is interactive)
+            os.system("stty sane <&1")

scripts/tests/twister_blackbox/test_device.py

@@ -6,71 +6,42 @@
 Blackbox tests for twister's command line functions related to test filtering.
 """
 
-import importlib
 from unittest import mock
 import os
 import pytest
 import sys
 import re
 
 # pylint: disable=no-name-in-module
-from conftest import ZEPHYR_BASE, TEST_DATA, suite_filename_mock
+from conftest import TEST_DATA, suite_filename_mock
 from twisterlib.testplan import TestPlan
+from twisterlib.twister_main import main as twister_main
 
 
 class TestDevice:
-    TESTDATA_1 = [
-        (
-            1234,
-        ),
-        (
-            4321,
-        ),
-        (
-            1324,
-        )
-    ]
-
-    @classmethod
-    def setup_class(cls):
-        apath = os.path.join(ZEPHYR_BASE, 'scripts', 'twister')
-        cls.loader = importlib.machinery.SourceFileLoader('__main__', apath)
-        cls.spec = importlib.util.spec_from_loader(cls.loader.name, cls.loader)
-        cls.twister_module = importlib.util.module_from_spec(cls.spec)
-
-    @classmethod
-    def teardown_class(cls):
-        pass
 
     @pytest.mark.parametrize(
         'seed',
-        TESTDATA_1,
-        ids=[
-            'seed 1234',
-            'seed 4321',
-            'seed 1324'
-        ],
+        [1234, 4321, 1324],
     )
-
     @mock.patch.object(TestPlan, 'TESTSUITE_FILENAME', suite_filename_mock)
     def test_seed(self, capfd, out_path, seed):
         test_platforms = ['native_sim']
         path = os.path.join(TEST_DATA, 'tests', 'seed_native_sim')
-        args = ['--no-detailed-test-id', '--outdir', out_path, '-i', '-T', path, '-vv',] + \
-               ['--seed', f'{seed[0]}'] + \
-               [val for pair in zip(
-                   ['-p'] * len(test_platforms), test_platforms
-               ) for val in pair]
 
-        with mock.patch.object(sys, 'argv', [sys.argv[0]] + args), \
-                pytest.raises(SystemExit) as sys_exit:
-            self.loader.exec_module(self.twister_module)
+        args = [
+            '--no-detailed-test-id', '--outdir', out_path, '-i', '-T', path, '-vv',
+            '--seed', f'{seed}',
+            *[val for pair in zip(['-p'] * len(test_platforms), test_platforms) for val in pair]
+        ]
+
+        return_value = twister_main(args)
 
         out, err = capfd.readouterr()
         sys.stdout.write(out)
         sys.stderr.write(err)
 
-        assert str(sys_exit.value) == '1'
+        assert return_value == 1
 
-        expected_line = r'seed_native_sim.dummy\s+FAILED rc=1 \(native (\d+\.\d+)s/seed: {} <host>\)'.format(seed[0])
-        assert re.search(expected_line, err)
+        expected_line = r'seed_native_sim.dummy\s+FAILED rc=1 \(native (\d+\.\d+)s/seed: {} <host>\)'.format(seed)
+        assert re.search(expected_line, err), f'Regex not found: r"{expected_line}"'

scripts/tests/twister_blackbox/test_disable.py

@@ -6,16 +6,16 @@
 Blackbox tests for twister's command line functions related to disable features.
 """
 
-import importlib
 import pytest
 from unittest import mock
 import os
 import sys
 import re
 
 # pylint: disable=no-name-in-module
-from conftest import ZEPHYR_BASE, TEST_DATA, suite_filename_mock
+from conftest import TEST_DATA, suite_filename_mock
 from twisterlib.testplan import TestPlan
+from twisterlib.twister_main import main as twister_main
 
 
 @mock.patch.object(TestPlan, 'TESTSUITE_FILENAME', suite_filename_mock)
@@ -41,30 +41,17 @@ class TestDisable:
             os.path.join(TEST_DATA, 'tests', 'always_warning'),
             ['qemu_x86'],
             '--disable-warnings-as-errors',
-            '0'
+            0
         ),
         (
             os.path.join(TEST_DATA, 'tests', 'always_warning'),
             ['qemu_x86'],
             '-v',
-            '1'
+            1
         ),
     ]
 
 
-    @classmethod
-    def setup_class(cls):
-        apath = os.path.join(ZEPHYR_BASE, 'scripts', 'twister')
-        cls.loader = importlib.machinery.SourceFileLoader('__main__', apath)
-        cls.spec = importlib.util.spec_from_loader(cls.loader.name, cls.loader)
-        cls.twister_module = importlib.util.module_from_spec(cls.spec)
-
-
-    @classmethod
-    def teardown_class(cls):
-        pass
-
-
     @pytest.mark.parametrize(
         'test_path, test_platforms, flag, expected, expected_none',
         TESTDATA_1,
@@ -82,15 +69,14 @@ def test_disable_suite_name_check(self, capfd, out_path, test_path, test_platfor
                    ['-p'] * len(test_platforms), test_platforms
                ) for val in pair]
 
-        with mock.patch.object(sys, 'argv', [sys.argv[0]] + args), \
-            pytest.raises(SystemExit) as sys_exit:
-            self.loader.exec_module(self.twister_module)
+
+        return_value = twister_main(args)
 
         out, err = capfd.readouterr()
         sys.stdout.write(out)
         sys.stderr.write(err)
 
-        assert str(sys_exit.value) == '0'
+        assert return_value == 0
         if expected_none:
             assert re.search(expected[0], err) is None, f"Not expected string in log: {expected[0]}"
             assert re.search(expected[1], err) is None, f"Not expected: {expected[1]}"
@@ -117,13 +103,12 @@ def test_disable_warnings_as_errors(self, capfd, out_path, test_path, test_platf
                    ['-p'] * len(test_platforms), test_platforms
                ) for val in pair]
 
-        with mock.patch.object(sys, 'argv', [sys.argv[0]] + args), \
-            pytest.raises(SystemExit) as sys_exit:
-            self.loader.exec_module(self.twister_module)
+
+        return_value = twister_main(args)
 
         out, err = capfd.readouterr()
         sys.stdout.write(out)
         sys.stderr.write(err)
 
-        assert str(sys_exit.value) == expected_exit_code, \
-            f"Twister return not expected ({expected_exit_code}) exit code: ({sys_exit.value})"
+        assert return_value == expected_exit_code, \
+            f"Twister return not expected ({expected_exit_code}) exit code: ({return_value})"

scripts/tests/twister_blackbox/test_testlist.py

@@ -6,29 +6,17 @@
 Blackbox tests for twister's command line functions related to saving and loading a testlist.
 """
 
-import importlib
 from unittest import mock
 import os
-import pytest
-import sys
 import json
 
 # pylint: disable=no-name-in-module
-from conftest import ZEPHYR_BASE, TEST_DATA, suite_filename_mock, clear_log_in_test
+from conftest import TEST_DATA, suite_filename_mock, clear_log_in_test
 from twisterlib.testplan import TestPlan
+from twisterlib.twister_main import main as twister_main
 
 
 class TestTestlist:
-    @classmethod
-    def setup_class(cls):
-        apath = os.path.join(ZEPHYR_BASE, 'scripts', 'twister')
-        cls.loader = importlib.machinery.SourceFileLoader('__main__', apath)
-        cls.spec = importlib.util.spec_from_loader(cls.loader.name, cls.loader)
-        cls.twister_module = importlib.util.module_from_spec(cls.spec)
-
-    @classmethod
-    def teardown_class(cls):
-        pass
 
     @mock.patch.object(TestPlan, 'TESTSUITE_FILENAME', suite_filename_mock)
     def test_save_tests(self, out_path):
@@ -42,11 +30,7 @@ def test_save_tests(self, out_path):
                ) for val in pair]
 
         # Save agnostics tests
-        with mock.patch.object(sys, 'argv', [sys.argv[0]] + args), \
-                pytest.raises(SystemExit) as sys_exit:
-            self.loader.exec_module(self.twister_module)
-
-        assert str(sys_exit.value) == '0'
+        assert twister_main(args) == 0
 
         clear_log_in_test()
 
@@ -58,11 +42,7 @@ def test_save_tests(self, out_path):
                    ['-p'] * len(test_platforms), test_platforms
                ) for val in pair]
 
-        with mock.patch.object(sys, 'argv', [sys.argv[0]] + args), \
-                pytest.raises(SystemExit) as sys_exit:
-            self.loader.exec_module(self.twister_module)
-
-        assert str(sys_exit.value) == '0'
+        assert twister_main(args) == 0
 
         with open(os.path.join(out_path, 'testplan.json')) as f:
             j = json.load(f)

scripts/tests/twister_blackbox/test_testplan.py

@@ -6,22 +6,21 @@
 Blackbox tests for twister's command line functions - those requiring testplan.json
 """
 
-import importlib
 from unittest import mock
 import os
 import pytest
-import sys
 import json
 
 # pylint: disable=no-name-in-module
-from conftest import ZEPHYR_BASE, TEST_DATA, suite_filename_mock
+from conftest import TEST_DATA, suite_filename_mock
 from twisterlib.testplan import TestPlan
 from twisterlib.error import TwisterRuntimeError
+from twisterlib.twister_main import main as twister_main
 
 
 class TestTestPlan:
     TESTDATA_1 = [
-        ('dummy.agnostic.group2.a2_tests.assert1', SystemExit, 4),
+        ('dummy.agnostic.group2.a2_tests.assert1', None, 4),
         (
             os.path.join('scripts', 'tests', 'twister_blackbox', 'test_data', 'tests',
                          'dummy', 'agnostic', 'group1', 'subgroup1',
@@ -39,17 +38,6 @@ class TestTestPlan:
         (False, 7),
     ]
 
-    @classmethod
-    def setup_class(cls):
-        apath = os.path.join(ZEPHYR_BASE, 'scripts', 'twister')
-        cls.loader = importlib.machinery.SourceFileLoader('__main__', apath)
-        cls.spec = importlib.util.spec_from_loader(cls.loader.name, cls.loader)
-        cls.twister_module = importlib.util.module_from_spec(cls.spec)
-
-    @classmethod
-    def teardown_class(cls):
-        pass
-
     @pytest.mark.parametrize(
         'test, expected_exception, expected_subtest_count',
         TESTDATA_1,
@@ -65,24 +53,21 @@ def test_subtest(self, out_path, test, expected_exception, expected_subtest_coun
                    ['-p'] * len(test_platforms), test_platforms
                ) for val in pair]
 
-        with mock.patch.object(sys, 'argv', [sys.argv[0]] + args), \
-                pytest.raises(expected_exception) as exc:
-            self.loader.exec_module(self.twister_module)
-
-        if expected_exception != SystemExit:
-            assert True
-            return
-
-        with open(os.path.join(out_path, 'testplan.json')) as f:
-            j = json.load(f)
-        filtered_j = [
-            (ts['platform'], ts['name'], tc['identifier']) \
-                for ts in j['testsuites'] \
-                for tc in ts['testcases'] if 'reason' not in tc
-        ]
-
-        assert str(exc.value) == '0'
-        assert len(filtered_j) == expected_subtest_count
+        if expected_exception:
+            with pytest.raises(expected_exception):
+                twister_main(args)
+        else:
+            return_value = twister_main(args)
+            with open(os.path.join(out_path, 'testplan.json')) as f:
+                j = json.load(f)
+            filtered_j = [
+                (ts['platform'], ts['name'], tc['identifier']) \
+                    for ts in j['testsuites'] \
+                    for tc in ts['testcases'] if 'reason' not in tc
+            ]
+
+            assert return_value == 0
+            assert len(filtered_j) == expected_subtest_count
 
     @pytest.mark.parametrize(
         'filter, expected_count',
@@ -98,11 +83,8 @@ def test_filter(self, out_path, filter, expected_count):
                    ['-p'] * len(test_platforms), test_platforms
                ) for val in pair]
 
-        with mock.patch.object(sys, 'argv', [sys.argv[0]] + args), \
-                pytest.raises(SystemExit) as exc:
-            self.loader.exec_module(self.twister_module)
+        assert twister_main(args)  == 0
 
-        assert str(exc.value) == '0'
         import pprint
         with open(os.path.join(out_path, 'testplan.json')) as f:
             j = json.load(f)
@@ -132,11 +114,7 @@ def test_integration(self, out_path, integration, expected_count):
                    ['-p'] * len(test_platforms), test_platforms
                ) for val in pair]
 
-        with mock.patch.object(sys, 'argv', [sys.argv[0]] + args), \
-                pytest.raises(SystemExit) as exc:
-            self.loader.exec_module(self.twister_module)
-
-        assert str(exc.value) == '0'
+        assert twister_main(args) == 0
 
         with open(os.path.join(out_path, 'testplan.json')) as f:
             j = json.load(f)