Move platform test excludes from mx to runner

Author: msimacek

graalpython/com.oracle.graal.python.test/src/runner.py

@@ -59,7 +59,7 @@
 import unittest.loader
 from abc import abstractmethod
 from collections import defaultdict
-from dataclasses import dataclass
+from dataclasses import dataclass, field
 from functools import lru_cache
 from pathlib import Path
 
@@ -679,21 +679,34 @@ class Config:
     new_worker_per_file: bool = False
     serial_tests: frozenset[str] = frozenset()
     partial_splits_individual_tests: frozenset[str] = frozenset()
+    excludes: dict[str, frozenset[str]] = field(default_factory=dict)
 
-    def is_serial_test(self, test_file: Path):
+    def test_file_name(self, test_file):
         resolved = test_file.resolve().relative_to(self.configdir)
-        name = str(resolved).removesuffix('.py')
-        return name in self.serial_tests
+        return str(resolved).removesuffix('.py')
+
+    def is_serial_test(self, test_file: Path):
+        return self.test_file_name(test_file) in self.serial_tests
 
     def is_partial_splits_individual_tests(self, test_file: Path):
-        resolved = test_file.resolve().relative_to(self.configdir)
-        name = str(resolved).removesuffix('.py')
-        return name in self.partial_splits_individual_tests
+        return self.test_file_name(test_file) in self.partial_splits_individual_tests
 
     def get_tag_file(self, test_file: Path):
         if self.tags_dir:
             return self.tags_dir / (test_file.name.removesuffix('.py') + '.txt')
 
+    def is_excluded(self, test_file: Path):
+        exclude_keys = [sys.platform]
+        if IS_GRAALPY:
+            # noinspection PyUnresolvedReferences
+            exclude_keys.append('native_image' if __graalpython__.is_native else 'jvm')
+        test_file_name = self.test_file_name(test_file)
+        for key in exclude_keys:
+            if excludes := self.excludes.get(key):
+                if test_file_name in excludes:
+                    return True
+        return False
+
 
 @lru_cache
 def config_for_dir(path: Path) -> Config:
@@ -712,6 +725,10 @@ def config_for_file(test_file: Path) -> Config:
     return config_for_dir(path)
 
 
+def test_file_name_set(test_files: list[str]):
+    return frozenset({test_file.removesuffix('.py') for test_file in test_files})
+
+
 @lru_cache
 def parse_config(config_path, path):
     with open(config_path, 'rb') as f:
@@ -725,10 +742,11 @@ def parse_config(config_path, path):
             tags_dir=tags_dir,
             run_top_level_functions=config_dict.get('run_top_level_functions', Config.run_top_level_functions),
             new_worker_per_file=config_dict.get('new_worker_per_file', Config.new_worker_per_file),
-            serial_tests=frozenset(config_dict.get('serial_tests', Config.serial_tests)),
-            partial_splits_individual_tests=frozenset(
+            serial_tests=test_file_name_set(config_dict.get('serial_tests', Config.serial_tests)),
+            partial_splits_individual_tests=test_file_name_set(
                 config_dict.get('partial_splits_individual_tests', Config.partial_splits_individual_tests)
             ),
+            excludes={key: test_file_name_set(excludes) for key, excludes in config_dict.get('excludes', {}).items()},
         )
 
 
@@ -827,16 +845,21 @@ def path_for_comparison(p: Path):
 
 
 def collect(all_specifiers: list[TestSpecifier], *, use_tags=False, ignore=None, partial=None,
-            continue_on_errors=False) -> list[TestSuite]:
+            continue_on_errors=False, no_excludes=False) -> list[TestSuite]:
     to_run = []
     all_specifiers = expand_specifier_paths(all_specifiers)
+    specifiers_by_file = group_specifiers_by_file(all_specifiers)
     if ignore:
         ignore = [path_for_comparison(i) for i in ignore]
-        all_specifiers = [
-            s for s in all_specifiers
-            if not any(path_for_comparison(s.test_file).is_relative_to(i) for i in ignore)
-        ]
-    specifiers_by_file = group_specifiers_by_file(all_specifiers)
+        for test_file in set(specifiers_by_file):
+            if any(path_for_comparison(test_file).is_relative_to(i) for i in ignore):
+                del specifiers_by_file[test_file]
+    if not no_excludes:
+        for test_file in set(specifiers_by_file):
+            config = config_for_file(test_file)
+            if config.is_excluded(test_file):
+                log(f"Test file {test_file} is excluded on this platform/configuration, use --no-excludes to overrride")
+                del specifiers_by_file[test_file]
     if partial:
         selected, total = partial
         to_split = []
@@ -910,7 +933,7 @@ def result_factory(suite):
         def result_factory(suite):
             return SimpleResult(suite, data)
 
-    for test_suite in collect(tests):
+    for test_suite in collect(tests, no_excludes=True):
         result = result_factory(test_suite)
         result.failfast = args.failfast
         test_suite.run(result)
@@ -939,6 +962,8 @@ def main():
                         help="Create a new worker process for each test file (when -n is specified). Default for tagged unit tests")
     parser.add_argument('--ignore', type=Path, action='append', default=[],
                         help="Ignore path during collection (multi-allowed)")
+    parser.add_argument('--no-excludes', action='store_true',
+                        help="Don't apply configuration exclusions")
     parser.add_argument('-f', '--failfast', action='store_true',
                         help="Exit immediately after the first failure")
     parser.add_argument('--all', action='store_true',
@@ -1017,6 +1042,7 @@ def main():
         ignore=args.ignore,
         partial=partial,
         continue_on_errors=args.continue_on_collection_errors,
+        no_excludes=args.no_excludes,
     )
     if args.collect_only:
         for test_suite in tests:

graalpython/com.oracle.graal.python.test/src/tests/conftest.toml

@@ -4,3 +4,54 @@ partial_splits_individual_tests = [
     'standalone/test_standalone',
     'standalone/test_jbang_integration',
 ]
+[tests.excludes]
+# Windows support is still experimental, so we exclude some unittests
+# on Windows for now. If you add unittests and cannot get them to work
+# on Windows, yet, add their files here.
+win32 = [
+    "test_code.py", # forward slash in path problem
+    "test_csv.py",
+    "test_imports.py", # import posix
+    "test_locale.py",
+    "test_math.py",
+    "test_memoryview.py",
+    "test_mmap.py", # sys.getwindowsversion
+    "test_multiprocessing.py", # import _winapi
+    "test_multiprocessing_graalpy.py", # import _winapi
+    "test_patched_pip.py",
+    "test_pathlib.py",
+    "test_pdb.py", # Tends to hit GR-41935
+    "test_posix.py", # import posix
+    "test_pyio.py",
+    "test_signal.py",
+    "test_struct.py",
+    "test_structseq.py", # import posix
+    "test_subprocess.py",
+    "test_thread.py", # sys.getwindowsversion
+    "test_traceback.py",
+    "test_zipimport.py", # sys.getwindowsversion
+    "test_ssl_java_integration.py",
+    "cpyext/test_abstract.py",
+    "cpyext/test_functions.py",
+    "cpyext/test_long.py",
+    "cpyext/test_member.py",
+    "cpyext/test_memoryview.py",
+    "cpyext/test_misc.py",
+    "cpyext/test_mmap.py",
+    "cpyext/test_slice.py",
+    "cpyext/test_shutdown.py",
+    "cpyext/test_thread.py",
+    "cpyext/test_unicode.py",
+    "cpyext/test_wiki.py",
+    "cpyext/test_tp_slots.py", # Temporarily disabled due to GR-54345
+]
+native_image = [
+    "test_interop.py",
+    "test_register_interop_behavior.py",
+    "test_jarray.py",
+    "test_ssl_java_integration.py",
+]
+jvm = [
+    # These test would work on JVM too, but they are prohibitively slow due to a large amount of subprocesses
+    "test_patched_pip.py",
+]

mx.graalpython/mx_graalpython.py

@@ -757,17 +757,6 @@ def update_unittest_tags(args):
         mx.warn("Potential regressions:\n" + '\n'.join(x[1] for x in diff))
 
 
-AOT_INCOMPATIBLE_TESTS = ["test_interop.py", "test_register_interop_behavior.py", "test_jarray.py", "test_ssl_java_integration.py"]
-# These test would work on JVM too, but they are prohibitively slow due to a large amount of subprocesses
-AOT_ONLY_TESTS = ["test_patched_pip.py", "test_multiprocessing_spawn.py"]
-
-GINSTALL_GATE_PACKAGES = {
-    "numpy": "numpy",
-    "scipy": "scipy",
-    "scikit_learn": "sklearn",
-}
-
-
 class GraalPythonTags(object):
     junit = 'python-junit'
     junit_maven = 'python-junit-maven'
@@ -1162,7 +1151,7 @@ def is_included(path):
     return testfiles
 
 
-def run_python_unittests(python_binary, args=None, paths=None, aot_compatible=False, exclude=None, env=None,
+def run_python_unittests(python_binary, args=None, paths=None, exclude=None, env=None,
                          use_pytest=False, cwd=None, lock=None, out=None, err=None, nonZeroIsFatal=True, timeout=None,
                          report=False, parallel=1, runner_args=None):
     if lock:
@@ -1175,7 +1164,6 @@ def run_python_unittests(python_binary, args=None, paths=None, aot_compatible=Fa
         "--python.EnableDebuggingBuiltins",
         *args,
     ]
-    exclude = exclude or []
     if env is None:
         env = os.environ.copy()
     env['PYTHONHASHSEED'] = '0'
@@ -1184,12 +1172,6 @@ def run_python_unittests(python_binary, args=None, paths=None, aot_compatible=Fa
     if mx.primary_suite() != SUITE:
         env.setdefault("GRAALPYTEST_ALLOW_NO_JAVA_ASSERTIONS", "true")
 
-    # list of excluded tests
-    if aot_compatible:
-        exclude += AOT_INCOMPATIBLE_TESTS
-    else:
-        exclude += AOT_ONLY_TESTS
-
     # just to be able to verify, print C ext mode (also works for CPython)
     mx.run(
         [
@@ -1209,8 +1191,9 @@ def run_python_unittests(python_binary, args=None, paths=None, aot_compatible=Fa
     if runner_args:
         args += runner_args
 
-    for file in exclude:
-        args += ['--ignore', file]
+    if exclude:
+        for file in exclude:
+            args += ['--ignore', file]
 
     if is_collecting_coverage() and mx_gate.get_jacoco_agent_args():
         # jacoco only dumps the data on exit, and when we run all our unittests
@@ -1386,49 +1369,6 @@ def get_wrapper_urls(wrapper_properties_file, keys):
 def graalpython_gate_runner(args, tasks):
     report = lambda: (not is_collecting_coverage()) and task
     nonZeroIsFatal = not is_collecting_coverage()
-    if WIN32:
-        # Windows support is still experimental, so we exclude some unittests
-        # on Windows for now. If you add unittests and cannot get them to work
-        # on Windows, yet, add their files here.
-        excluded_tests = [
-            "test_code.py", # forward slash in path problem
-            "test_csv.py",
-            "test_imports.py", # import posix
-            "test_locale.py",
-            "test_math.py",
-            "test_memoryview.py",
-            "test_mmap.py", # sys.getwindowsversion
-            "test_multiprocessing.py", # import _winapi
-            "test_multiprocessing_graalpy.py", # import _winapi
-            "test_patched_pip.py",
-            "test_pathlib.py",
-            "test_pdb.py", # Tends to hit GR-41935
-            "test_posix.py", # import posix
-            "test_pyio.py",
-            "test_signal.py",
-            "test_struct.py",
-            "test_structseq.py", # import posix
-            "test_subprocess.py",
-            "test_thread.py", # sys.getwindowsversion
-            "test_traceback.py",
-            "test_zipimport.py", # sys.getwindowsversion
-            "test_ssl_java_integration.py",
-            "cpyext/test_abstract.py",
-            "cpyext/test_functions.py",
-            "cpyext/test_long.py",
-            "cpyext/test_member.py",
-            "cpyext/test_memoryview.py",
-            "cpyext/test_misc.py",
-            "cpyext/test_mmap.py",
-            "cpyext/test_slice.py",
-            "cpyext/test_shutdown.py",
-            "cpyext/test_thread.py",
-            "cpyext/test_unicode.py",
-            "cpyext/test_wiki.py",
-            "cpyext/test_tp_slots.py",  # Temporarily disabled due to GR-54345
-        ]
-    else:
-        excluded_tests = []
 
     # JUnit tests
     with Task('GraalPython JUnit', tasks, tags=[GraalPythonTags.junit, GraalPythonTags.windows]) as task:
@@ -1480,7 +1420,6 @@ def graalpython_gate_runner(args, tasks):
         if task:
             run_python_unittests(
                 graalpy_standalone_jvm(),
-                exclude=excluded_tests,
                 nonZeroIsFatal=nonZeroIsFatal,
                 report=report()
             )
@@ -1493,11 +1432,11 @@ def graalpython_gate_runner(args, tasks):
 
     with Task('GraalPython sandboxed tests', tasks, tags=[GraalPythonTags.unittest_sandboxed]) as task:
         if task:
-            run_python_unittests(graalpy_standalone_jvm_enterprise(), args=SANDBOXED_OPTIONS, exclude=excluded_tests, report=report())
+            run_python_unittests(graalpy_standalone_jvm_enterprise(), args=SANDBOXED_OPTIONS, report=report())
 
     with Task('GraalPython multi-context unittests', tasks, tags=[GraalPythonTags.unittest_multi]) as task:
         if task:
-            run_python_unittests(graalpy_standalone_jvm(), args=["-multi-context"], exclude=excluded_tests, nonZeroIsFatal=nonZeroIsFatal, report=report())
+            run_python_unittests(graalpy_standalone_jvm(), args=["-multi-context"], nonZeroIsFatal=nonZeroIsFatal, report=report())
 
     with Task('GraalPython Jython emulation tests', tasks, tags=[GraalPythonTags.unittest_jython]) as task:
         if task:
@@ -1577,11 +1516,11 @@ def graalpython_gate_runner(args, tasks):
     # Unittests on SVM
     with Task('GraalPython tests on SVM', tasks, tags=[GraalPythonTags.svmunit, GraalPythonTags.windows]) as task:
         if task:
-            run_python_unittests(graalpy_standalone_native(), exclude=excluded_tests, aot_compatible=True, report=report())
+            run_python_unittests(graalpy_standalone_native(), report=report())
 
     with Task('GraalPython sandboxed tests on SVM', tasks, tags=[GraalPythonTags.svmunit_sandboxed]) as task:
         if task:
-            run_python_unittests(graalpy_standalone_native_enterprise(), args=SANDBOXED_OPTIONS, aot_compatible=True, report=report())
+            run_python_unittests(graalpy_standalone_native_enterprise(), args=SANDBOXED_OPTIONS, report=report())
 
     with Task('GraalPython license header update', tasks, tags=[GraalPythonTags.license]) as task:
         if task:
@@ -2601,13 +2540,13 @@ def python_coverage(args):
             ]
             env['GRAAL_PYTHON_ARGS'] = " ".join(extra_args)
             # deselect some tagged unittests that hang with coverage enabled
-            exclude = [
+            tagged_exclude = [
                 "test_multiprocessing_spawn",
                 "test_multiprocessing_main_handling",
                 "test_multiprocessing_graalpy",
             ]
             if kwds.pop("tagged", False):
-                run_tagged_unittests(executable, env=env, nonZeroIsFatal=False, parallel=1, exclude=exclude)
+                run_tagged_unittests(executable, env=env, nonZeroIsFatal=False, parallel=1, exclude=tagged_exclude)
             elif kwds.pop("hpy", False):
                 run_hpy_unittests(executable, env=env, nonZeroIsFatal=False, timeout=5*60*60) # hpy unittests are really slow under coverage
             else: