Split some test files per individual test

Author: msimacek

graalpython/com.oracle.graal.python.test/src/runner.py

@@ -449,7 +449,7 @@ def partition_tests_into_processes(self, suites: list['TestSuite']) -> list[list
     def run_tests(self, tests: list['TestSuite']):
         serial_suites, parallel_suites = partition_list(
             tests,
-            lambda suite: suite.test_file.name.removesuffix('.py') in suite.config.serial_tests,
+            lambda suite: suite.config.is_serial_test(suite.test_file),
         )
         parallel_partitions = self.partition_tests_into_processes(parallel_suites)
         serial_partitions = self.partition_tests_into_processes(serial_suites)
@@ -616,11 +616,23 @@ def filter_tree(test_file: Path, test_suite: unittest.TestSuite, specifiers: lis
 
 @dataclass
 class Config:
+    configdir: Path = Path('.').resolve()
     rootdir: Path = Path('.').resolve()
     tags_dir: Path | None = None
     run_top_level_functions: bool = False
     new_worker_per_file: bool = False
     serial_tests: frozenset[str] = frozenset()
+    partial_splits_individual_tests: frozenset[str] = frozenset()
+
+    def is_serial_test(self, test_file: Path):
+        resolved = test_file.resolve().relative_to(self.configdir)
+        name = str(resolved).removesuffix('.py')
+        return name in self.serial_tests
+
+    def is_partial_splits_individual_tests(self, test_file: Path):
+        resolved = test_file.resolve().relative_to(self.configdir)
+        name = str(resolved).removesuffix('.py')
+        return name in self.partial_splits_individual_tests
 
 
 @lru_cache
@@ -644,16 +656,19 @@ def config_for_file(test_file: Path) -> Config:
 def parse_config(config_path, path):
     with open(config_path, 'rb') as f:
         config_dict = tomllib.load(f)['tests']
-        rootdir = config_path.parent.parent.resolve()
         tags_dir = None
         if config_tags_dir := config_dict.get('tags_dir'):
             tags_dir = (path / config_tags_dir).resolve()
         return Config(
-            rootdir=rootdir,
+            configdir=config_path.parent.resolve(),
+            rootdir=config_path.parent.parent.resolve(),
             tags_dir=tags_dir,
             run_top_level_functions=config_dict.get('run_top_level_functions', Config.run_top_level_functions),
             new_worker_per_file=config_dict.get('new_worker_per_file', Config.new_worker_per_file),
             serial_tests=frozenset(config_dict.get('serial_tests', Config.serial_tests)),
+            partial_splits_individual_tests=frozenset(
+                config_dict.get('partial_splits_individual_tests', Config.partial_splits_individual_tests)
+            ),
         )
 
 
@@ -717,7 +732,7 @@ def expand_specifier_paths(specifiers: list[TestSpecifier]) -> list[TestSpecifie
     return expanded_specifiers
 
 
-def collect_module(test_file: Path, specifiers: list[TestSpecifier], use_tags=False) -> TestSuite | None:
+def collect_module(test_file: Path, specifiers: list[TestSpecifier], use_tags=False, partial=None) -> TestSuite | None:
     config = config_for_file(test_file)
     saved_path = sys.path[:]
     sys.path.insert(0, str(config.rootdir))
@@ -735,6 +750,9 @@ def collect_module(test_file: Path, specifiers: list[TestSpecifier], use_tags=Fa
             log(f"Test file {test_file} skipped: {e}")
             return
         collected_tests, untagged_tests = filter_tree(test_file, test_suite, specifiers, tags)
+        if partial and config.is_partial_splits_individual_tests(test_file):
+            selected, total = partial
+            collected_tests = collected_tests[selected::total]
         if collected_tests:
             return TestSuite(config, test_file, sys.path[:], test_suite, collected_tests, untagged_tests)
     finally:
@@ -755,13 +773,24 @@ def collect(all_specifiers: list[TestSpecifier], *, use_tags=False, ignore=None,
             s for s in all_specifiers
             if not any(path_for_comparison(s.test_file).is_relative_to(i) for i in ignore)
         ]
+    specifiers_by_file = group_specifiers_by_file(all_specifiers)
     if partial:
         selected, total = partial
-        all_specifiers = all_specifiers[selected::total]
-    for test_file, specifiers in group_specifiers_by_file(all_specifiers).items():
+        to_split = []
+        partial_files = set()
+        # Always keep files that are split per-test
+        for test_file in specifiers_by_file:
+            config = config_for_file(test_file)
+            if config.is_partial_splits_individual_tests(test_file):
+                partial_files.add(test_file)
+            else:
+                to_split.append(test_file)
+        partial_files |= set(to_split[selected::total])
+        specifiers_by_file = {f: s for f, s in specifiers_by_file.items() if f in partial_files}
+    for test_file, specifiers in specifiers_by_file.items():
         if not test_file.exists():
             sys.exit(f"File does not exist: {test_file}")
-        collected = collect_module(test_file, specifiers, use_tags=use_tags)
+        collected = collect_module(test_file, specifiers, use_tags=use_tags, partial=partial)
         if collected:
             to_run.append(collected)
     return to_run

graalpython/com.oracle.graal.python.test/src/tests/conftest.toml

@@ -1,2 +1,6 @@
 [tests]
 run_top_level_functions = true
+partial_splits_individual_tests = [
+    'standalone/test_standalone',
+    'standalone/test_jbang_integration',
+]

graalpython/com.oracle.graal.python.test/src/tests/standalone/test_jbang_integration.py

@@ -47,7 +47,6 @@
 import unittest
 
 from tests.standalone import util
-from tests.util import skip_deselected_test_functions
 
 is_enabled = 'ENABLE_JBANG_INTEGRATION_UNITTESTS' in os.environ and os.environ['ENABLE_JBANG_INTEGRATION_UNITTESTS'] == "true"
 MAVEN_REPO_LOCAL_URL = os.environ.get('org.graalvm.maven.downloader.repository')
@@ -415,6 +414,3 @@ def test_two_resource_dirs(self):
         out, result = run_cmd(command, cwd=work_dir)
         self.assertTrue(result == 1, f"Execution failed with code {result}\n    command: {command}\n    stdout: {out}")
         self.assertTrue("only one //PYTHON_RESOURCES_DIRECTORY comment is allowed" in out, f"Expected text:\nonly one //PYTHON_RESOURCES_DIRECTORY comment is allowed")
-
-
-skip_deselected_test_functions(globals())

graalpython/com.oracle.graal.python.test/src/tests/standalone/test_standalone.py

@@ -42,7 +42,6 @@
 import unittest
 
 from tests.standalone import util
-from tests.util import skip_deselected_test_functions
 
 is_enabled = 'ENABLE_STANDALONE_UNITTESTS' in os.environ and os.environ['ENABLE_STANDALONE_UNITTESTS'] == "true"
 
@@ -141,5 +140,3 @@ def test_native_executable_module():
         cmd = [target_file]
         out, return_code = util.run_cmd(cmd, env)
         util.check_ouput("hello standalone world", out)
-
-skip_deselected_test_functions(globals())

graalpython/com.oracle.graal.python.test/src/tests/util.py

@@ -36,9 +36,7 @@
 # LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
 # OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
 # SOFTWARE.
-import os
 import sys
-import unittest
 
 
 def storage_to_native(s):
@@ -59,42 +57,3 @@ def assert_raises(err, fn, *args, err_check=None, **kwargs):
             else:
                 assert err_check(e)
     assert raised
-
-
-# XXX we should split the tests in a better way
-def skip_deselected_test_functions(globals):
-    """
-    If TAGGED_UNITTEST_PARTIAL is set, this skips tests as appropriate.
-    """
-    envvar = os.environ.get("TAGGED_UNITTEST_PARTIAL", "")
-    if "/" not in envvar:
-        return
-    import types
-    batch, total = (int(x) for x in envvar.split("/"))
-    test_functions = []
-    for g in globals.values():
-        if isinstance(g, types.FunctionType) and g.__name__.startswith("test_"):
-            test_functions.append((globals, g))
-        elif isinstance(g, type) and issubclass(g, unittest.TestCase):
-            for f in (getattr(g, n) for n in dir(g)):
-                if isinstance(f, types.FunctionType) and f.__name__.startswith("test_"):
-                    test_functions.append((g, f))
-    for idx, (owner, test_func) in enumerate(test_functions):
-        if idx % total != batch - 1:
-            n = test_func.__name__
-            if isinstance(owner, type):
-                removeAttr(owner, n)
-            else:
-                del owner[n]
-
-
-def removeAttr(cls, attr):
-    if (cls is object):
-        return
-    if hasattr(cls, attr):
-        try:
-            delattr(cls, attr)
-        except AttributeError as e:
-            pass
-    for b in cls.__bases__:
-        removeAttr(b, attr)

graalpython/lib-python/3/test/conftest.toml

@@ -28,3 +28,7 @@ serial_tests = [
     'test_int',
     'test_zipfile',
 ]
+partial_splits_individual_tests = [
+    'test_multiprocessing_spawn',
+    'test_multiprocessing_graalpy',
+]