Merge pull request #477 from MarekVCodasip/test-exclusion

Add a way to exclude tests by specifying an exclusion file

Author: timsifive

debug/Makefile

@@ -23,7 +23,8 @@ run.%:
 		--isolate \
 		--print-failures \
 		--sim_cmd $(RISCV)/bin/$(RISCV_SIM) \
-		--server_cmd $(RISCV)/bin/openocd
+		--server_cmd $(RISCV)/bin/openocd \
+		$(if $(EXCLUDE_TESTS),--exclude-tests $(EXCLUDE_TESTS))
 
 # Target to check all the multicore options.
 multi-tests: spike32-2 spike32-2-hwthread

debug/excluded.yaml.example

@@ -0,0 +1,17 @@
+# This is an example file showing the exclusion list format.
+# It can be passed to gdb_server.py using the --exclude-tests argument.
+
+
+# Tests excluded for all targets
+all:
+  - SimpleF18Test
+
+# Tests excluded for the "spike32" target only
+spike32:
+  - MemTest32
+  - PrivRw
+  - Sv32Test
+
+# Tests excluded for the "spike64" target only
+spike64:
+  - UserInterrupt

debug/requirements.txt

@@ -1 +1,2 @@
 pexpect>=4.0.0
+pyyaml>=3.12

debug/testlib.py

@@ -11,6 +11,7 @@
 import traceback
 
 import pexpect
+import yaml
 
 print_log_names = False
 real_stdout = sys.stdout
@@ -944,6 +945,32 @@ def __enter__(self):
     def __exit__(self, _type, _value, _traceback):
         self.gdb.pop_state()
 
+
+def load_excluded_tests(excluded_tests_file, target_name):
+    result = []
+    if excluded_tests_file is None or len(excluded_tests_file) == 0:
+        return result
+
+    target_excludes = {}
+    with open(excluded_tests_file) as file:
+        raw_data = yaml.safe_load(file)
+        for (target, test_list) in raw_data.items():
+            if not isinstance(test_list, list):
+                raise ValueError(f"Target {target!r} does not contain a test list", excluded_tests_file, test_list)
+            if not all(isinstance(s, str) for s in test_list):
+                raise ValueError(f"Not every element in the target test list {target!r} is a string",
+                                 excluded_tests_file, test_list)
+
+        target_excludes.update(raw_data)
+
+    if target_name in target_excludes:
+        result += target_excludes[target_name]
+    if "all" in target_excludes:
+        result += target_excludes["all"]
+
+    return result
+
+
 def run_all_tests(module, target, parsed):
     todo = []
     for name in dir(module):
@@ -982,6 +1009,9 @@ def run_all_tests(module, target, parsed):
             todo.insert(0, ("ExamineTarget", ExamineTarget, None))
             examine_added = True
 
+    excluded_tests = load_excluded_tests(parsed.exclude_tests, target.name)
+    target.skip_tests += excluded_tests
+
     results, count = run_tests(parsed, target, todo)
 
     header(f"ran {count} tests in {time.time() - overall_start:.0f}s", dash=':')
@@ -1068,6 +1098,8 @@ def add_test_run_options(parser):
     parser.add_argument("--misaval",
             help="Don't run ExamineTarget, just assume the misa value which is "
             "specified.")
+    parser.add_argument("--exclude-tests",
+            help="Specify yaml file listing tests to exclude")
 
 def header(title, dash='-', length=78):
     if title: