twister: quarantine: Added pytests

Implemented scenarios to test the quarantine in twister
Signed-off-by: Grzegorz Chwierut <[email protected]>

Author: gchwier

scripts/pylib/twister/twisterlib/quarantine.py

@@ -80,6 +80,9 @@ def load_data_from_yaml(cls, filename: str | Path) -> QuarantineData:
         with open(filename, 'r', encoding='UTF-8') as yaml_fd:
             qlist_raw_data: list[dict] = safe_load(yaml_fd)
         try:
+            if not qlist_raw_data:
+                # in case of loading empty quarantine file
+                return cls()
             return cls(qlist_raw_data)
 
         except Exception as e:

scripts/pylib/twister/twisterlib/testplan.py

@@ -134,6 +134,9 @@ def discover(self):
             logger.error("No quarantine list given to be verified")
             raise TwisterRuntimeError("No quarantine list given to be verified")
         if ql:
+            for quarantine_file in ql:
+                # validate quarantine yaml file against the provided schema
+                scl.yaml_load_verify(quarantine_file, self.quarantine_schema)
             self.quarantine = Quarantine(ql)
 
     def load(self):

scripts/tests/twister/test_data/quarantines/basic.yaml

@@ -0,0 +1,6 @@
+- scenarios:
+  - test_a.check_1
+  platforms:
+  - demo_board_1
+  - demo_board_3
+  comment: "a1 on board_1 and board_3"

scripts/tests/twister/test_data/quarantines/empty.yaml

@@ -0,0 +1 @@
+# empty quarantine file

scripts/tests/twister/test_data/quarantines/platform.yaml

@@ -0,0 +1,4 @@
+
+- platforms:
+  - demo_board_3
+  comment: "all on board_3"

scripts/tests/twister/test_data/quarantines/with_regexp.yaml

@@ -0,0 +1,10 @@
+
+- scenarios:
+  - test_(a|c).check_2
+  architectures:
+  - x86.*
+  comment: "a2 and c2 on x86"
+
+- scenarios:
+  - test_d.*
+  comment: "all test_d"

scripts/tests/twister/test_testplan_class.py

@@ -17,6 +17,7 @@
 from twisterlib.testinstance import TestInstance
 from twisterlib.testsuite import TestSuite
 from twisterlib.platform import Platform
+from twisterlib.quarantine import Quarantine
 
 
 def test_testplan_add_testsuites(class_testplan):
@@ -256,3 +257,81 @@ def test_add_instances(test_data, class_env, all_testsuites_dict, platforms_list
 		   [platform.name + '/' + s for s in list(all_testsuites_dict.keys())]
     assert all(isinstance(n, TestInstance) for n in list(plan.instances.values()))
     assert list(plan.instances.values()) == instance_list
+
+
+QUARANTINE_BASIC = {
+    'demo_board_1/zephyr/scripts/tests/twister/test_data/testsuites/tests/test_a/test_a.check_1' : 'a1 on board_1 and board_3',
+    'demo_board_3/zephyr/scripts/tests/twister/test_data/testsuites/tests/test_a/test_a.check_1' : 'a1 on board_1 and board_3'
+}
+
+QUARANTINE_WITH_REGEXP = {
+    'demo_board_2/zephyr/scripts/tests/twister/test_data/testsuites/tests/test_a/test_a.check_2' : 'a2 and c2 on x86',
+    'demo_board_1/zephyr/scripts/tests/twister/test_data/testsuites/tests/test_d/test_d.check_1' : 'all test_d',
+    'demo_board_3/zephyr/scripts/tests/twister/test_data/testsuites/tests/test_d/test_d.check_1' : 'all test_d',
+    'demo_board_2/zephyr/scripts/tests/twister/test_data/testsuites/tests/test_d/test_d.check_1' : 'all test_d',
+    'demo_board_2/zephyr/scripts/tests/twister/test_data/testsuites/tests/test_c/test_c.check_2' : 'a2 and c2 on x86'
+}
+
+QUARANTINE_PLATFORM = {
+    'demo_board_3/zephyr/scripts/tests/twister/test_data/testsuites/tests/test_a/test_a.check_1' : 'all on board_3',
+    'demo_board_3/zephyr/scripts/tests/twister/test_data/testsuites/tests/test_a/test_a.check_2' : 'all on board_3',
+    'demo_board_3/zephyr/scripts/tests/twister/test_data/testsuites/tests/test_d/test_d.check_1' : 'all on board_3',
+    'demo_board_3/zephyr/scripts/tests/twister/test_data/testsuites/tests/test_b/test_b.check_1' : 'all on board_3',
+    'demo_board_3/zephyr/scripts/tests/twister/test_data/testsuites/tests/test_b/test_b.check_2' : 'all on board_3',
+    'demo_board_3/zephyr/scripts/tests/twister/test_data/testsuites/tests/test_c/test_c.check_1' : 'all on board_3',
+    'demo_board_3/zephyr/scripts/tests/twister/test_data/testsuites/tests/test_c/test_c.check_2' : 'all on board_3'
+}
+
+QUARANTINE_MULTIFILES = {
+    **QUARANTINE_BASIC,
+    **QUARANTINE_WITH_REGEXP
+}
+
+@pytest.mark.parametrize(
+    ("quarantine_files, quarantine_verify, expected_val"),
+    [
+        (['basic.yaml'], False, QUARANTINE_BASIC),
+        (['with_regexp.yaml'], False, QUARANTINE_WITH_REGEXP),
+        (['with_regexp.yaml'], True, QUARANTINE_WITH_REGEXP),
+        (['platform.yaml'], False, QUARANTINE_PLATFORM),
+        (['basic.yaml', 'with_regexp.yaml'], False, QUARANTINE_MULTIFILES),
+        (['empty.yaml'], False, {})
+    ],
+    ids=[
+        'basic',
+        'with_regexp',
+        'quarantine_verify',
+        'platform',
+        'multifiles',
+        'empty'
+    ])
+def test_quarantine(class_testplan, platforms_list, test_data,
+                    quarantine_files, quarantine_verify, expected_val):
+    """ Testing quarantine feature in Twister
+    """
+    class_testplan.options.all = True
+    class_testplan.platforms = platforms_list
+    class_testplan.platform_names = [p.name for p in platforms_list]
+    class_testplan.TESTSUITE_FILENAME = 'test_data.yaml'
+    class_testplan.add_testsuites()
+
+    quarantine_list = [
+        os.path.join(test_data, 'quarantines', quarantine_file) for quarantine_file in quarantine_files
+    ]
+    class_testplan.quarantine = Quarantine(quarantine_list)
+    class_testplan.options.quarantine_verify = quarantine_verify
+    class_testplan.apply_filters()
+
+    for testname, instance in class_testplan.instances.items():
+        if quarantine_verify:
+            if testname in expected_val:
+                assert not instance.status
+            else:
+                assert instance.status == 'filtered'
+                assert instance.reason == "Not under quarantine"
+        else:
+            if testname in expected_val:
+                assert instance.status == 'filtered'
+                assert instance.reason == expected_val[testname]
+            else:
+                assert not instance.status