[GR-54334] Support pandas 2

PullRequest: graalpython/3353

Author: msimacek

graalpython/com.oracle.graal.python.test/src/tests/cpyext/test_misc.py

@@ -165,6 +165,17 @@ class TestMisc(CPyExtTestCase):
         cmpfunc=unhandled_error_compare
     )
 
+    test_PyImport_GetModule = CPyExtFunction(
+        lambda args: sys.modules.get(args[0]),
+        lambda: (
+            ("os",),
+        ),
+        resultspec="O",
+        argspec="O",
+        arguments=["PyObject* name"],
+        cmpfunc=unhandled_error_compare
+    )
+
     test_PyTruffle_Intrinsic_Pmovmskb = CPyExtFunction(
         lambda args: True,
         lambda: (

graalpython/com.oracle.graal.python/src/com/oracle/graal/python/builtins/modules/PosixModuleBuiltins.java

@@ -2853,11 +2853,14 @@ static int getCpuCount() {
     abstract static class SysconfNode extends PythonUnaryBuiltinNode {
 
         public static final TruffleString T_SC_CLK_TCK = tsLiteral("SC_CLK_TCK");
+        public static final TruffleString T_SC_NPROCESSORS_ONLN = tsLiteral("SC_NPROCESSORS_ONLN");
         public static final int SC_CLK_TCK = 2;
+        public static final int SC_NPROCESSORS_ONLN = 84;
         public static final EconomicMapStorage SYSCONF_NAMES = EconomicMapStorage.create();
         static {
             // TODO populate from constants
             SYSCONF_NAMES.putUncachedWithJavaEq(T_SC_CLK_TCK, SC_CLK_TCK);
+            SYSCONF_NAMES.putUncachedWithJavaEq(T_SC_NPROCESSORS_ONLN, SC_NPROCESSORS_ONLN);
         }
 
         @Specialization
@@ -2885,6 +2888,8 @@ static int sysconf(VirtualFrame frame, Object arg,
             if (id == SC_CLK_TCK) {
                 return 100; // it's 100 on most default kernel configs. TODO: use real value through
                             // NFI
+            } else if (id == SC_NPROCESSORS_ONLN) {
+                return CpuCountNode.getCpuCount();
             }
             throw constructAndRaiseNode.get(inliningTarget).raiseOSError(frame, OSErrorEnum.EINVAL);
         }

graalpython/com.oracle.graal.python/src/com/oracle/graal/python/builtins/modules/cext/PythonCextImportBuiltins.java

@@ -52,15 +52,26 @@
 import static com.oracle.graal.python.nodes.BuiltinNames.T_GLOBALS;
 import static com.oracle.graal.python.nodes.BuiltinNames.T_LOCALS;
 import static com.oracle.graal.python.nodes.BuiltinNames.T___IMPORT__;
+import static com.oracle.graal.python.nodes.SpecialAttributeNames.T___INITIALIZING__;
+import static com.oracle.graal.python.nodes.SpecialAttributeNames.T___SPEC__;
+import static com.oracle.graal.python.util.PythonUtils.tsLiteral;
 
 import com.oracle.graal.python.builtins.modules.cext.PythonCextBuiltins.CApi5BuiltinNode;
 import com.oracle.graal.python.builtins.modules.cext.PythonCextBuiltins.CApiBuiltin;
 import com.oracle.graal.python.builtins.modules.cext.PythonCextBuiltins.CApiNullaryBuiltinNode;
 import com.oracle.graal.python.builtins.modules.cext.PythonCextBuiltins.CApiUnaryBuiltinNode;
+import com.oracle.graal.python.builtins.objects.PNone;
+import com.oracle.graal.python.builtins.objects.dict.PDict;
 import com.oracle.graal.python.builtins.objects.function.PKeyword;
+import com.oracle.graal.python.lib.PyObjectCallMethodObjArgs;
 import com.oracle.graal.python.lib.PyObjectGetAttr;
+import com.oracle.graal.python.lib.PyObjectGetItem;
+import com.oracle.graal.python.lib.PyObjectIsTrueNode;
 import com.oracle.graal.python.nodes.call.CallNode;
 import com.oracle.graal.python.nodes.statement.AbstractImportNode;
+import com.oracle.graal.python.runtime.PythonContext;
+import com.oracle.graal.python.runtime.exception.PException;
+import com.oracle.truffle.api.CompilerDirectives.TruffleBoundary;
 import com.oracle.truffle.api.dsl.Bind;
 import com.oracle.truffle.api.dsl.Cached;
 import com.oracle.truffle.api.dsl.Specialization;
@@ -103,4 +114,47 @@ Object importModuleLevelObject(TruffleString name, Object globals, Object locals
             });
         }
     }
+
+    @CApiBuiltin(ret = PyObjectTransfer, args = {PyObject}, call = Direct)
+    abstract static class PyImport_GetModule extends CApiUnaryBuiltinNode {
+
+        private static final TruffleString T__LOCK_UNLOCK_MODULE = tsLiteral("_lock_unlock_module");
+
+        @Specialization
+        Object getModule(Object name,
+                        @Bind("this") Node inliningTarget,
+                        @Cached PyObjectGetItem getItem,
+                        @Cached PyObjectGetAttr getAttr,
+                        @Cached PyObjectIsTrueNode isTrueNode) {
+            PythonContext context = PythonContext.get(inliningTarget);
+            PDict modules = context.getSysModules();
+            Object m;
+            try {
+                m = getItem.execute(null, inliningTarget, modules, name);
+            } catch (PException e) {
+                return context.getNativeNull();
+            }
+            if (m != PNone.NONE) {
+                boolean initializing = false;
+                try {
+                    Object spec = getAttr.execute(null, inliningTarget, m, T___SPEC__);
+                    Object initializingObj = getAttr.execute(null, inliningTarget, spec, T___INITIALIZING__);
+                    if (isTrueNode.execute(null, inliningTarget, initializingObj)) {
+                        initializing = true;
+                    }
+                } catch (PException e) {
+                    // ignore
+                }
+                if (initializing) {
+                    waitForInitialization(name, context);
+                }
+            }
+            return m;
+        }
+
+        @TruffleBoundary
+        private static void waitForInitialization(Object name, PythonContext context) {
+            PyObjectCallMethodObjArgs.executeUncached(context.getImportlib(), T__LOCK_UNLOCK_MODULE, name);
+        }
+    }
 }

graalpython/com.oracle.graal.python/src/com/oracle/graal/python/builtins/objects/cext/capi/CApiFunction.java

@@ -770,7 +770,6 @@ public final class CApiFunction {
     @CApiBuiltin(name = "PyImport_GetImporter", ret = PyObject, args = {PyObject}, call = NotImplemented)
     @CApiBuiltin(name = "PyImport_GetMagicNumber", ret = Long, args = {}, call = NotImplemented)
     @CApiBuiltin(name = "PyImport_GetMagicTag", ret = ConstCharPtrAsTruffleString, args = {}, call = NotImplemented)
-    @CApiBuiltin(name = "PyImport_GetModule", ret = PyObject, args = {PyObject}, call = NotImplemented)
     @CApiBuiltin(name = "PyImport_ImportFrozenModule", ret = Int, args = {ConstCharPtrAsTruffleString}, call = NotImplemented)
     @CApiBuiltin(name = "PyImport_ImportFrozenModuleObject", ret = Int, args = {PyObject}, call = NotImplemented)
     @CApiBuiltin(name = "PyImport_ReloadModule", ret = PyObject, args = {PyObject}, call = NotImplemented)

graalpython/com.oracle.graal.python/src/com/oracle/graal/python/nodes/SpecialAttributeNames.java

@@ -114,6 +114,8 @@ public abstract class SpecialAttributeNames {
 
     public static final TruffleString T___CACHED__ = tsLiteral("__cached__");
 
+    public static final TruffleString T___INITIALIZING__ = tsLiteral("__initializing__");
+
     public static final String J___TEXT_SIGNATURE__ = "__text_signature__";
     public static final TruffleString T___TEXT_SIGNATURE__ = tsLiteral(J___TEXT_SIGNATURE__);
 

graalpython/lib-graalpython/patches/Cython/metadata.toml

@@ -2,11 +2,9 @@
 version = '== 3.0.10'
 patch = 'Cython-3.0.10.patch'
 
-# Prefer pre-3 version for now
 [[rules]]
 version = '== 0.29.37'
 patch = 'Cython-0.29.37.patch'
-install-priority = 2
 
 [[rules]]
 version = '== 0.29.32'

graalpython/lib-graalpython/patches/blosc2/blosc2.patch

@@ -0,0 +1,14 @@
+diff --git a/setup.py b/setup.py
+index 3a55a00..0c6f89a 100644
+--- a/setup.py
++++ b/setup.py
+@@ -11,7 +11,8 @@ from skbuild import setup
+ # Blosc version
+ VERSION = open("VERSION").read().strip()
+ # Create the version.py file
+-open("blosc2/version.py", "w").write(f'__version__ = "{VERSION}"\n')
++with open("blosc2/version.py", "w") as f:
++    f.write(f'__version__ = "{VERSION}"\n')
+ 
+ 
+ def exclude_pkgconfig(cmake_manifest):

graalpython/lib-graalpython/patches/blosc2/metadata.toml

@@ -0,0 +1,2 @@
+[[rules]]
+patch = 'blosc2.patch'

graalpython/lib-graalpython/patches/pandas/metadata.toml

@@ -1,3 +1,13 @@
+[[rules]]
+version = '== 2.2.2'
+patch = 'pandas-2.2.2.patch'
+dist-type = 'sdist'
+
+[[rules]]
+version = '== 2.0.3'
+patch = 'pandas-2.0.3.patch'
+dist-type = 'sdist'
+
 [[rules]]
 version = '== 1.5.2'
 patch = 'pandas-1.5.2.patch'

graalpython/lib-graalpython/patches/pandas/pandas-2.0.3.patch

@@ -0,0 +1,72 @@
+diff --git a/pandas/_libs/src/klib/khash_python.h b/pandas/_libs/src/klib/khash_python.h
+index 56afea0..14025b7 100644
+--- a/pandas/_libs/src/klib/khash_python.h
++++ b/pandas/_libs/src/klib/khash_python.h
+@@ -178,28 +178,30 @@ int PANDAS_INLINE floatobject_cmp(PyFloatObject* a, PyFloatObject* b){
+ // PyObject_RichCompareBool for complexobjects has a different behavior
+ // needs to be replaced
+ int PANDAS_INLINE complexobject_cmp(PyComplexObject* a, PyComplexObject* b){
++    Py_complex a_cval = PyComplex_AsCComplex((PyObject*)a);
++    Py_complex b_cval = PyComplex_AsCComplex((PyObject*)b);
+     return (
+-                Py_IS_NAN(a->cval.real) &&
+-                Py_IS_NAN(b->cval.real) &&
+-                Py_IS_NAN(a->cval.imag) &&
+-                Py_IS_NAN(b->cval.imag)
++                Py_IS_NAN(a_cval.real) &&
++                Py_IS_NAN(b_cval.real) &&
++                Py_IS_NAN(a_cval.imag) &&
++                Py_IS_NAN(b_cval.imag)
+            )
+            ||
+            (
+-                Py_IS_NAN(a->cval.real) &&
+-                Py_IS_NAN(b->cval.real) &&
+-                a->cval.imag == b->cval.imag
++                Py_IS_NAN(a_cval.real) &&
++                Py_IS_NAN(b_cval.real) &&
++                a_cval.imag == b_cval.imag
+            )
+            ||
+            (
+-                a->cval.real == b->cval.real &&
+-                Py_IS_NAN(a->cval.imag) &&
+-                Py_IS_NAN(b->cval.imag)
++                a_cval.real == b_cval.real &&
++                Py_IS_NAN(a_cval.imag) &&
++                Py_IS_NAN(b_cval.imag)
+            )
+            ||
+            (
+-                a->cval.real == b->cval.real &&
+-                a->cval.imag == b->cval.imag
++                a_cval.real == b_cval.real &&
++                a_cval.imag == b_cval.imag
+            );
+ }
+ 
+@@ -276,8 +278,9 @@ Py_hash_t PANDAS_INLINE floatobject_hash(PyFloatObject* key) {
+ 
+ // replaces _Py_HashDouble with _Pandas_HashDouble
+ Py_hash_t PANDAS_INLINE complexobject_hash(PyComplexObject* key) {
+-    Py_uhash_t realhash = (Py_uhash_t)_Pandas_HashDouble(key->cval.real);
+-    Py_uhash_t imaghash = (Py_uhash_t)_Pandas_HashDouble(key->cval.imag);
++    Py_complex cval = PyComplex_AsCComplex((PyObject*)key);
++    Py_uhash_t realhash = (Py_uhash_t)_Pandas_HashDouble(cval.real);
++    Py_uhash_t imaghash = (Py_uhash_t)_Pandas_HashDouble(cval.imag);
+     if (realhash == (Py_uhash_t)-1 || imaghash == (Py_uhash_t)-1) {
+         return -1;
+     }
+diff --git a/setup.py b/setup.py
+index b6dfcc5..2446bcc 100755
+--- a/setup.py
++++ b/setup.py
+@@ -421,7 +421,7 @@ def maybe_cythonize(extensions, *args, **kwargs):
+     parser.add_argument("--parallel", "-j", type=int, default=1)
+     parsed, _ = parser.parse_known_args()
+ 
+-    kwargs["nthreads"] = parsed.parallel
++    kwargs["nthreads"] = 1 # parsed.parallel
+     build_ext.render_templates(_pxifiles)
+     return cythonize(extensions, *args, **kwargs)
+