parallelize hpy tests

Author: timfel

graalpython/lib-graalpython/modules/hpy/test/conftest.py

@@ -20,12 +20,16 @@
 # LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
 # OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
 # SOFTWARE.
+import os
 import sys
 import pytest
 from .support import ExtensionCompiler, DefaultExtensionTemplate,\
     PythonSubprocessRunner, HPyDebugCapture
 from hpy.debug.leakdetector import LeakDetector
 
+SELECTED_ABI_MODE = os.environ.get("TEST_HPY_ABI", None)
+if SELECTED_ABI_MODE:
+    SELECTED_ABI_MODE = [SELECTED_ABI_MODE]
 IS_VALGRIND_RUN = False
 GRAALPYTHON_NATIVE = sys.implementation.name == 'graalpython' and __graalpython__.platform_id == 'native'
 def pytest_addoption(parser):
@@ -64,7 +68,7 @@ def hpy_devel(request):
     from hpy.devel import HPyDevel
     return HPyDevel()
 
-@pytest.fixture(params=['cpython', 'universal', 'debug', 'nfi'] if GRAALPYTHON_NATIVE else ['cpython', 'universal', 'debug'])
+@pytest.fixture(params=SELECTED_ABI_MODE or (['cpython', 'universal', 'debug', 'nfi'] if GRAALPYTHON_NATIVE else ['cpython', 'universal', 'debug']))
 def hpy_abi(request):
     abi = request.param
     if abi == 'debug':

mx.graalpython/mx_graalpython.py

@@ -674,7 +674,9 @@ def is_included(path):
     return testfiles
 
 
-def run_python_unittests(python_binary, args=None, paths=None, aot_compatible=False, exclude=None, env=None, use_pytest=False, cwd=None):
+def run_python_unittests(python_binary, args=None, paths=None, aot_compatible=False, exclude=None, env=None, use_pytest=False, cwd=None, lock=None):
+    if lock:
+        lock.acquire()
     # ensure that the test distribution is up-to-date
     mx.command_function("build")(["--dep", "com.oracle.graal.python.test"])
 
@@ -737,6 +739,8 @@ def graalvm_vm_arg(java_arg):
     else:
         args += testfiles
         mx.logv(" ".join([python_binary] + args))
+        if lock:
+            lock.release()
         return mx.run([python_binary] + args, nonZeroIsFatal=True, env=env, cwd=cwd)
 
 
@@ -766,8 +770,31 @@ def run_hpy_unittests(python_binary, args=None):
         mx.log("LLVM Toolchain (vanilla): {!s}".format(env["LLVM_TOOLCHAIN_VANILLA"]))
         mx.log("Ensure 'setuptools' is installed")
         mx.run([python_binary] + args + ["-m", "ginstall", "install", "--user", "pytest"], nonZeroIsFatal=True, env=env)
-
-        return run_python_unittests(python_binary, args=args, paths=[_hpy_test_root()], env=env, use_pytest=True)
+        # parallelize
+        import threading
+        threads = []
+        lock = threading.RLock()
+
+        class RaisingThread(threading.Thread):
+            def run(self):
+                self.exc = None
+                try:
+                    super().run()
+                except Exception as e:
+                    self.exc = e
+
+        for abi in ['cpython', 'universal', 'debug', 'nfi']:
+            env["TEST_HPY_ABI"] = abi
+            threads.append(RaisingThread(target=run_python_unittests, args=(python_binary, ), kwargs={
+                "args": args, "paths": [_hpy_test_root()], "env": env.copy(), "use_pytest": True, "lock": lock,
+            }))
+            threads[-1].start()
+        retval = 0
+        for t in threads:
+            t.join()
+        for t in threads:
+            if t.exc:
+                raise t.exc
 
 
 def run_tagged_unittests(python_binary, env=None, cwd=None):