[GR-29352] Split tagged unittests into multiple jobs

PullRequest: graalpython/1633

Author: msimacek

ci.jsonnet

@@ -1 +1 @@
-{ "overlay": "cfa81790263e6faec8f9d4730631f1587b01947c" }
+{ "overlay": "6ddaf9b9d7fd029f2bd27f49e2dc63ee18ea3335" }

graalpython/com.oracle.graal.python.test/src/tests/test_tagged_unittests.py

@@ -37,19 +37,19 @@
 # OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
 # SOFTWARE.
 
-import glob
-import os
-import subprocess
 import sys
+
+import glob
 import test
 
+import os
+import subprocess
 
 if os.environ.get("ENABLE_CPYTHON_TAGGED_UNITTESTS") == "true" or __name__ == "__main__":
     TAGS_DIR = os.path.join(os.path.dirname(__file__), "unittest_tags")
 else:
     TAGS_DIR = "null"
 
-
 RUNNER = os.path.join(os.path.dirname(__file__), "run_cpython_test.py")
 
 
@@ -61,7 +61,7 @@ def working_selectors(tagfile):
         return None
 
 
-def working_tests():
+def collect_working_tests():
     working_tests = []
     glob_pattern = os.path.join(TAGS_DIR, "*.txt")
     for arg in sys.argv:
@@ -72,39 +72,60 @@ def working_tests():
     for tagfile in glob.glob(glob_pattern):
         test = os.path.splitext(os.path.basename(tagfile))[0]
         working_tests.append((test, working_selectors(tagfile)))
-    return working_tests
-
-
-class TestAllWorkingTests():
-    pass
-
-WORKING_TESTS = working_tests()
-for idx, working_test in enumerate(WORKING_TESTS):
-    def make_test_func(working_test):
-        def fun(self):
-            cmd = [sys.executable]
-            if "--inspect" in sys.argv:
-                cmd.append("--inspect")
-            if "-debug-java" in sys.argv:
-                cmd.append("-debug-java")
-            cmd += ["-S", RUNNER]
-            for testpattern in working_test[1]:
-                cmd.extend(["-k", testpattern])
-            testmod = working_test[0].rpartition(".")[2]
-            print("Running test:", working_test[0])
-            testfile = os.path.join(os.path.dirname(test.__file__), "%s.py" % testmod)
-            if not os.path.isfile(testfile):
-                testfile = os.path.join(os.path.dirname(test.__file__), "%s/__init__.py" % testmod)
-            cmd.append(testfile)
-            subprocess.check_call(cmd)
-            print(working_test[0], "was finished.")
+    return sorted(working_tests)
+
+
+def make_test_function(working_test):
+    testmod = working_test[0].rpartition(".")[2]
+
+    def test_tagged():
+        cmd = [sys.executable]
+        if "--inspect" in sys.argv:
+            cmd.append("--inspect")
+        if "-debug-java" in sys.argv:
+            cmd.append("-debug-java")
+        cmd += ["-S", RUNNER]
+        for testpattern in working_test[1]:
+            cmd.extend(["-k", testpattern])
+        print("Running test:", working_test[0])
+        testfile = os.path.join(os.path.dirname(test.__file__), "%s.py" % testmod)
+        if not os.path.isfile(testfile):
+            testfile = os.path.join(os.path.dirname(test.__file__), "%s/__init__.py" % testmod)
+        cmd.append(testfile)
+        subprocess.check_call(cmd)
+        print(working_test[0], "was finished.")
 
-        fun.__name__ = "%s[%d/%d]" % (working_test[0], idx + 1, len(WORKING_TESTS))
-        return fun
+    if testmod.startswith('test_'):
+        test_tagged.__name__ = testmod
+    else:
+        test_tagged.__name__ = 'test_' + testmod
+    return test_tagged
+
+
+def make_tests_class():
+    import unittest
+
+    global TestTaggedUnittests
+
+    class TestTaggedUnittests(unittest.TestCase):
+        pass
+
+    partial = os.environ.get('TAGGED_UNITTEST_PARTIAL')
+    if partial:
+        selected_str, total_str = partial.split('/', 1)
+        selected = int(selected_str) - 1
+        total = int(total_str)
+    else:
+        selected = 0
+        total = 1
+    assert selected < total
 
-    test_f = make_test_func(working_test)
-    setattr(TestAllWorkingTests, test_f.__name__, test_f)
-    del test_f
+    working_tests = collect_working_tests()[selected::total]
+    for idx, working_test in enumerate(working_tests):
+        fn = make_test_function(working_test)
+        fn.__name__ = "%s[%d/%d]" % (fn.__name__, idx + 1, len(working_tests))
+        fn.__qualname__ = "%s.%s" % (TestTaggedUnittests.__name__, fn.__name__)
+        setattr(TestTaggedUnittests, fn.__name__, staticmethod(fn))
 
 
 # This function has a unittest in test_tagger
@@ -130,11 +151,8 @@ def parse_unittest_output(output):
             return
 
 
-if __name__ == "__main__":
-    # find working tests
-    import re
-
-    executable = sys.executable.split(" ") # HACK: our sys.executable on Java is a cmdline
+def main():
+    executable = sys.executable.split(" ")  # HACK: our sys.executable on Java is a cmdline
     kwargs = {"stdout": subprocess.PIPE, "stderr": subprocess.PIPE, "text": True, "check": False}
 
     glob_pattern = os.path.join(os.path.dirname(test.__file__), "test_*.py")
@@ -195,7 +213,7 @@ def parse_unittest_output(output):
                 # shouldn't be tried).
                 continue
 
-            print("[%d/%d, Try %d] Testing %s" %(idx + 1, len(testfiles), repeat + 1, testmod))
+            print("[%d/%d, Try %d] Testing %s" % (idx + 1, len(testfiles), repeat + 1, testmod))
             for selector in test_selectors:
                 cmd += ["-k", selector]
             cmd.append(testfile)
@@ -232,8 +250,9 @@ def parse_unittest_output(output):
                     continue
                 print(f"Suite succeeded with {len(passing_tests)} tests")
                 break
-            elif p.returncode == -9:                
-                print(f"\nTimeout (return code -9)\nyou can try to increase the current timeout {tout}s by using --timeout=NNN")
+            elif p.returncode == -9:
+                print(
+                    f"\nTimeout (return code -9)\nyou can try to increase the current timeout {tout}s by using --timeout=NNN")
                 break
             else:
                 print(f"Suite failed, retrying with {len(passing_tests)} tests")
@@ -246,3 +265,9 @@ def parse_unittest_output(output):
                 os.unlink(tagfile)
             except Exception:
                 pass
+
+
+if __name__ == '__main__':
+    main()
+else:
+    make_tests_class()