[GR-43510] cythonize in proc

PullRequest: graalpython/2623

Author: cosminbasca

graalpython/com.oracle.graal.python.test/src/tests/test_tagged_unittests.py

@@ -42,6 +42,7 @@
 import subprocess
 import sys
 import test
+from itertools import zip_longest
 
 if os.environ.get("ENABLE_CPYTHON_TAGGED_UNITTESTS") == "true" or __name__ == "__main__":
     TAGS_DIR = os.path.join(os.path.dirname(__file__), "unittest_tags")
@@ -52,6 +53,12 @@
 RUNNER = os.path.join(os.path.dirname(__file__), "run_cpython_test.py")
 LINE = "=" * 80
 
+
+def grouper(iterable, n):
+    args = [iter(iterable)] * n
+    return zip_longest(*args, fillvalue=None)
+
+
 def working_selectors(tagfile):
     if os.path.exists(tagfile):
         with open(tagfile) as f:
@@ -77,29 +84,30 @@ def collect_working_tests():
 def make_test_function(working_test):
     testmod = working_test[0].rpartition(".")[2]
 
-    def test_tagged():
-        cmd = [sys.executable]
-        if "--inspect" in sys.argv:
-            cmd.append("--inspect")
-        if "-debug-java" in sys.argv:
-            cmd.append("-debug-java")
-        cmd += [RUNNER]
-        if "-vv" in sys.argv:
-            cmd += ['-v']
-        for testpattern in working_test[1]:
-            cmd.extend(["-k", testpattern])
-        print("Running test:", working_test[0])
-        testfile = os.path.join(os.path.dirname(test.__file__), "%s.py" % testmod)
-        if not os.path.isfile(testfile):
-            testfile = os.path.join(os.path.dirname(test.__file__), "%s/__init__.py" % testmod)
-        cmd.append(testfile)
-        if os.environ.get("ENABLE_THREADED_GRAALPYTEST") == "true":
-            run_serialize_out(cmd)
-        else:
-            rcode = run_with_timeout(cmd).returncode
-            if rcode:
-                raise CalledProcessError(rcode, cmd)
-            print(working_test[0], "was finished.")
+    def test_tagged(max_patterns=100):
+        for working_test_group in grouper(working_test[1], max_patterns):
+            cmd = [sys.executable]
+            if "--inspect" in sys.argv:
+                cmd.append("--inspect")
+            if "-debug-java" in sys.argv:
+                cmd.append("-debug-java")
+            cmd += [RUNNER]
+            if "-vv" in sys.argv:
+                cmd += ['-v']
+            for testpattern in working_test_group:
+                cmd.extend(["-k", testpattern])
+            print("Running test:", working_test[0])
+            testfile = os.path.join(os.path.dirname(test.__file__), "%s.py" % testmod)
+            if not os.path.isfile(testfile):
+                testfile = os.path.join(os.path.dirname(test.__file__), "%s/__init__.py" % testmod)
+            cmd.append(testfile)
+            if os.environ.get("ENABLE_THREADED_GRAALPYTEST") == "true":
+                run_serialize_out(cmd)
+            else:
+                rcode = run_with_timeout(cmd).returncode
+                if rcode:
+                    raise CalledProcessError(rcode, cmd)
+                print(working_test[0], "was finished.")
 
     def run_serialize_out(cmd):
         result = run_with_timeout(cmd, stdout=subprocess.PIPE, stderr=subprocess.STDOUT)

graalpython/lib-graalpython/modules/ginstall.py

@@ -1,4 +1,4 @@
-# Copyright (c) 2019, 2022, Oracle and/or its affiliates. All rights reserved.
+# Copyright (c) 2019, 2023, Oracle and/or its affiliates. All rights reserved.
 # DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER.
 #
 # The Universal Permissive License (UPL), Version 1.0
@@ -400,7 +400,7 @@ def make_site_cfg(root):
             append_env_var(numpy_build_env, 'CFLAGS', '-Wno-error=implicit-function-declaration')
             info(f"have lapack or blas ... CLFAGS={numpy_build_env['CFLAGS']}")
 
-        install_from_pypi("numpy==1.23.1", build_cmd=["build_ext", "--disable-optimization"], env=numpy_build_env,
+        install_from_pypi("numpy==1.23.5", build_cmd=["build_ext", "--disable-optimization"], env=numpy_build_env,
                           pre_install_hook=make_site_cfg, **kwargs)
 
         # print numpy configuration
@@ -455,7 +455,7 @@ def pandas(**kwargs):
         dateutil(**kwargs)
         numpy(**kwargs)
 
-        install_from_pypi("pandas==1.4.3", **kwargs)
+        install_from_pypi("pandas==1.5.2", **kwargs)
 
     @pip_package()
     def scipy(**kwargs):
@@ -503,7 +503,7 @@ def scikit_learn(**kwargs):
         joblib(**kwargs)
         threadpoolctl(**kwargs)
 
-        install_from_pypi("scikit-learn==1.0.2", env=scikit_learn_build_env, **kwargs)
+        install_from_pypi("scikit-learn==1.1.3", env=scikit_learn_build_env, **kwargs)
 
     @pip_package()
     def cycler(**kwargs):

graalpython/lib-graalpython/patches/Cython/sdist/Cython-0.29.32.patch

@@ -1,3 +1,16 @@
+diff --git a/Cython/Utility/Coroutine.c b/Cython/Utility/Coroutine.c
+index 1ad27df..0965cb2 100644
+--- a/Cython/Utility/Coroutine.c
++++ b/Cython/Utility/Coroutine.c
+@@ -1299,7 +1299,7 @@ static void __Pyx_Coroutine_del(PyObject *self) {
+         _Py_NewReference(self);
+         __Pyx_SET_REFCNT(self, refcnt);
+     }
+-#if CYTHON_COMPILING_IN_CPYTHON
++#if CYTHON_COMPILING_IN_CPYTHON && !GRAALVM_PYTHON
+     assert(PyType_IS_GC(Py_TYPE(self)) &&
+            _Py_AS_GC(self)->gc.gc_refs != _PyGC_REFS_UNTRACKED);
+ 
 diff --git a/Cython/Utility/ModuleSetupCode.c b/Cython/Utility/ModuleSetupCode.c
 index f8bf885..9c2534d 100644
 --- a/Cython/Utility/ModuleSetupCode.c