[GR-54287] Support numpy 2.0

PullRequest: graalpython/3349

Author: msimacek

graalpython/com.oracle.graal.python.cext/include/methodobject.h

@@ -1,4 +1,4 @@
-/* Copyright (c) 2018, 2023, Oracle and/or its affiliates.
+/* Copyright (c) 2018, 2024, Oracle and/or its affiliates.
  * Copyright (C) 1996-2020 Python Software Foundation
  *
  * Licensed under the PYTHON SOFTWARE FOUNDATION LICENSE VERSION 2
@@ -125,6 +125,10 @@ PyAPI_FUNC(PyObject *) PyCMethod_New(PyMethodDef *, PyObject *,
 #define METH_METHOD 0x0200
 #endif
 
+// GraalPy public API functions
+PyAPI_FUNC(const char*) GraalPyCFunction_GetDoc(PyObject *func);
+PyAPI_FUNC(void) GraalPyCFunction_SetDoc(PyObject *func, const char *doc);
+
 
 #ifndef Py_LIMITED_API
 #  define Py_CPYTHON_METHODOBJECT_H

graalpython/com.oracle.graal.python.cext/src/methodobject.c

@@ -102,3 +102,17 @@ void _PyCFunction_SetModule(PyObject *func, PyObject *mod) {
 void _PyCFunction_SetMethodDef(PyObject *func, PyMethodDef *def) {
     set_PyCFunctionObject_m_ml(func, def);
 }
+
+// GraalPy additions
+const char *
+GraalPyCFunction_GetDoc(PyObject *func) {
+    return PyCFunctionObject_m_ml(func)->ml_doc;
+}
+
+void
+GraalPyCFunction_SetDoc(PyObject *func, const char *doc) {
+    PyCFunctionObject_m_ml(func)->ml_doc = doc;
+    if (points_to_py_handle_space(func)) {
+        GraalPyTruffleCFunction_SetDoc(func, doc);
+    }
+}

graalpython/com.oracle.graal.python.test/src/tests/cpyext/test_object_dict.py

@@ -45,12 +45,13 @@
 
 NativeTypeWithDict = CPyExtType(
     name='NativeTypeWithDict',
-    cmembers='PyObject* dict;',
+    cmembers='int some_field; PyObject* dict;',
     tp_dictoffset='offsetof(NativeTypeWithDictObject, dict)',
     tp_getset='{"__dict__", PyObject_GenericGetDict, PyObject_GenericSetDict}',
 )
 
 NativeHeapTypeWithoutDict = CPyExtHeapType(
+    cmembers='int some_field;',
     name='NativeTypeWithManagedDict',
 )
 
@@ -75,7 +76,7 @@
 
 
 # TODO it would be great if we could test creating heap types with Py_TPFLAGS_MANAGED_DICT, because pybind11 does that,
-# but there's no way to do that without abusing abusing implementations details
+# but there's no way to do that without abusing implementations details
 
 
 class NativeSubtypeWithDict(NativeTypeWithDict):
@@ -107,3 +108,15 @@ def test_dict(self):
         self.assert_has_working_dict(NativeHeapTypeWithDict())
         self.assert_has_working_dict(NativeSubtypeWithDict())
         self.assert_has_working_dict(NativeSubtypeWithAddedDict())
+
+    def test_multiple_inheritance(self):
+        class Base1(NativeTypeWithoutDict):
+            pass
+
+        class Base2(NativeTypeWithoutDict):
+            pass
+
+        class Derived(Base1, Base2):
+            pass
+
+        self.assert_has_working_dict(Derived())

graalpython/com.oracle.graal.python/src/com/oracle/graal/python/builtins/modules/cext/PythonCextFuncBuiltins.java

@@ -1,5 +1,5 @@
 /*
- * Copyright (c) 2022, 2023, Oracle and/or its affiliates. All rights reserved.
+ * Copyright (c) 2022, 2024, Oracle and/or its affiliates. All rights reserved.
  * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER.
  *
  * The Universal Permissive License (UPL), Version 1.0
@@ -41,16 +41,27 @@
 package com.oracle.graal.python.builtins.modules.cext;
 
 import static com.oracle.graal.python.builtins.modules.cext.PythonCextBuiltins.CApiCallPath.Direct;
+import static com.oracle.graal.python.builtins.modules.cext.PythonCextBuiltins.CApiCallPath.Ignored;
+import static com.oracle.graal.python.builtins.objects.cext.capi.transitions.ArgDescriptor.ConstCharPtrAsTruffleString;
 import static com.oracle.graal.python.builtins.objects.cext.capi.transitions.ArgDescriptor.PyObject;
 import static com.oracle.graal.python.builtins.objects.cext.capi.transitions.ArgDescriptor.PyObjectTransfer;
+import static com.oracle.graal.python.nodes.SpecialAttributeNames.T___DOC__;
 
 import com.oracle.graal.python.builtins.PythonBuiltinClassType;
+import com.oracle.graal.python.builtins.modules.cext.PythonCextBuiltins.CApiBinaryBuiltinNode;
 import com.oracle.graal.python.builtins.modules.cext.PythonCextBuiltins.CApiBuiltin;
 import com.oracle.graal.python.builtins.modules.cext.PythonCextBuiltins.CApiUnaryBuiltinNode;
+import com.oracle.graal.python.builtins.objects.PNone;
+import com.oracle.graal.python.builtins.objects.cext.capi.transitions.ArgDescriptor;
+import com.oracle.graal.python.builtins.objects.function.PBuiltinFunction;
+import com.oracle.graal.python.builtins.objects.method.PBuiltinMethod;
 import com.oracle.graal.python.builtins.objects.method.PDecoratedMethod;
 import com.oracle.graal.python.runtime.object.PythonObjectFactory;
+import com.oracle.truffle.api.CompilerDirectives;
+import com.oracle.truffle.api.CompilerDirectives.TruffleBoundary;
 import com.oracle.truffle.api.dsl.Cached;
 import com.oracle.truffle.api.dsl.Specialization;
+import com.oracle.truffle.api.strings.TruffleString;
 
 public final class PythonCextFuncBuiltins {
 
@@ -73,4 +84,22 @@ static Object staticmethod(Object callable,
             return factory.createClassmethodFromCallableObj(callable);
         }
     }
+
+    @CApiBuiltin(ret = ArgDescriptor.Void, args = {PyObject, ConstCharPtrAsTruffleString}, call = Ignored)
+    abstract static class PyTruffleCFunction_SetDoc extends CApiBinaryBuiltinNode {
+        @Specialization
+        @TruffleBoundary
+        static Object set(Object functionObj, TruffleString doc) {
+            PBuiltinFunction function;
+            if (functionObj instanceof PBuiltinFunction builtinFunction) {
+                function = builtinFunction;
+            } else if (functionObj instanceof PBuiltinMethod builtinMethod) {
+                function = builtinMethod.getBuiltinFunction();
+            } else {
+                throw CompilerDirectives.shouldNotReachHere("Unexpected object passed to GraalPyCFunction_SetDoc");
+            }
+            function.setAttribute(T___DOC__, doc != null ? doc : PNone.NONE);
+            return PNone.NO_VALUE;
+        }
+    }
 }

graalpython/com.oracle.graal.python/src/com/oracle/graal/python/builtins/objects/type/TypeNodes.java

@@ -2356,18 +2356,24 @@ private static void addNativeSlots(TypeNewContext ctx, PythonManagedClass python
             long dictOffset = GetDictOffsetNode.executeUncached(base);
             long weakListOffset = GetWeakListOffsetNode.executeUncached(base);
             long itemSize = GetItemSizeNode.executeUncached(base);
-            if (ctx.addDict) {
-                if (itemSize != 0) {
-                    dictOffset = -SIZEOF_PY_OBJECT_PTR;
-                } else {
-                    dictOffset = slotOffset;
-                }
+            if (ctx.addDict && itemSize != 0) {
+                dictOffset = -SIZEOF_PY_OBJECT_PTR;
                 slotOffset += SIZEOF_PY_OBJECT_PTR;
             }
             if (ctx.addWeak) {
                 weakListOffset = slotOffset;
                 slotOffset += SIZEOF_PY_OBJECT_PTR;
             }
+            if (ctx.addDict && itemSize == 0) {
+                long flags = GetTypeFlagsNode.executeUncached(pythonClass) | MANAGED_DICT;
+                SetTypeFlagsNode.executeUncached(pythonClass, flags);
+                /*
+                 * Negative offsets are computed from the end of the structure. Our managed dict is
+                 * right before the start of the object. CPython has 3 fields there, so our formula
+                 * differs.
+                 */
+                dictOffset = -slotOffset - SIZEOF_PY_OBJECT_PTR;
+            }
             SetDictOffsetNode.executeUncached(pythonClass, dictOffset);
             SetBasicSizeNode.executeUncached(pythonClass, slotOffset);
             SetItemSizeNode.executeUncached(pythonClass, itemSize);

graalpython/lib-graalpython/modules/autopatch_capi.py

@@ -156,7 +156,7 @@ def consume_whitespace_forward(idx):
                     r'\W(ob_refcnt)\W': (replace_field_access, 'Py_REFCNT(%receiver)'),
                     r'\W(ob_item)\W': (replace_field_access, 'PySequence_Fast_ITEMS((PyObject*)%receiver)'),
                     r'^\s*()(std::)?free\((const_cast<char \*>)?\(?\w+->m_ml->ml_doc\)?\);': (simple_replace, '//'),
-                    r'\W(m_ml\s*->\s*ml_doc)\W': (replace_field_access, 'PyObject_GetDoc((PyObject*)(%receiver))', 'PyObject_SetDoc((PyObject*)(%receiver), %value)'),
+                    r'\W(m_ml\s*->\s*ml_doc)\W': (replace_field_access, 'GraalPyCFunction_GetDoc((PyObject*)(%receiver))', 'GraalPyCFunction_SetDoc((PyObject*)(%receiver), %value)'),
                     # Py_CLEAR/Py_VISIT on a function's module is skipped for us, Java GC takes care of it
                     r'(Py_(?:CLEAR|VISIT)\((?:(?:\(?\([a-zA-Z0-9_]|[a-zA-Z0-9_])(?:[a-zA-Z0-9_]|\)|\*\)|->)*)->m_module\);)': (simple_replace, ''),
                     r'\W(m_ml)\W': (replace_field_access, '_PyCFunction_GetMethodDef((PyObject*)(%receiver))', '_PyCFunction_SetMethodDef((PyObject*)(%receiver), %value)'),

graalpython/lib-graalpython/patches/Cython/Cython-3.0.10.patch

@@ -222,9 +222,18 @@ index 3ea60f5..18d1842 100644
  #endif
      else if (PyMethod_Check(method)) {
 diff --git a/Cython/Utility/ModuleSetupCode.c b/Cython/Utility/ModuleSetupCode.c
-index d8f60a4..8ede926 100644
+index d8f60a4..cd61759 100644
 --- a/Cython/Utility/ModuleSetupCode.c
 +++ b/Cython/Utility/ModuleSetupCode.c
+@@ -113,7 +113,7 @@
+   #undef CYTHON_USE_DICT_VERSIONS
+   #define CYTHON_USE_DICT_VERSIONS 0
+   #undef CYTHON_USE_EXC_INFO_STACK
+-  #define CYTHON_USE_EXC_INFO_STACK 0
++  #define CYTHON_USE_EXC_INFO_STACK 1
+   #ifndef CYTHON_UPDATE_DESCRIPTOR_DOC
+     #define CYTHON_UPDATE_DESCRIPTOR_DOC 0
+   #endif
 @@ -995,7 +995,7 @@ static CYTHON_INLINE int __Pyx__IsSameCFunction(PyObject *func, void *cfunc) {
    #define __Pyx_PyFrame_SetLineNumber(frame, lineno)
  #else

graalpython/lib-graalpython/patches/numpy/metadata.toml

@@ -1,3 +1,9 @@
+[[rules]]
+version = '>= 2.0.0rc1, < 2.1'
+patch = 'numpy-2.0.0.patch'
+dist-type = 'sdist'
+ignore-rule-on-llvm = true
+
 [[rules]]
 version = '== 1.26.4'
 patch = 'numpy-1.26.4.patch'

graalpython/lib-graalpython/patches/numpy/numpy-1.26.4.patch

@@ -1,15 +1,5 @@
 diff --git a/numpy/core/include/numpy/ndarrayobject.h b/numpy/core/include/numpy/ndarrayobject.h
 index d4b73eb..af90a68 100644
---- a/vendored-meson/meson/mesonbuild/utils/universal.py
-+++ b/vendored-meson/meson/mesonbuild/utils/universal.py
-@@ -727,6 +727,7 @@ def windows_detect_native_arch() -> str:
-     """
-     if sys.platform != 'win32':
-         return ''
-+    return 'amd64'  # Workaround for GraalPy bug on Windows with kernel32.GetCurrentProcess()
-     try:
-         import ctypes
-         process_arch = ctypes.c_ushort()
 --- a/numpy/core/include/numpy/ndarrayobject.h
 +++ b/numpy/core/include/numpy/ndarrayobject.h
 @@ -225,7 +225,7 @@ NPY_TITLE_KEY_check(PyObject *key, PyObject *value)
@@ -71,46 +61,46 @@ index 2890406..353a657 100644
  #include <feature_detection_misc.h>
 
 diff --git a/numpy/core/src/npymath/ieee754.c.src b/numpy/core/src/npymath/ieee754.c.src
-index 8fccc9a..e7f79e1 100644
+index 8fccc9a..3bb9cf0 100644
 --- a/numpy/core/src/npymath/ieee754.c.src
 +++ b/numpy/core/src/npymath/ieee754.c.src
-@@ -362,6 +362,17 @@ int npy_get_floatstatus_barrier(char* param)
+@@ -362,6 +362,11 @@ int npy_get_floatstatus_barrier(char* param)
       * By using a volatile, the compiler cannot reorder this call
       */
      if (param != NULL) {
 +        // GraalPy change: the pointer needs to be dereferenced to establish
 +        // a data dependency to to ensure the compiler won't reorder the call
-+#define HANDLE_BASE 0x8000000000000000ULL
-+#define points_to_py_handle_space(PTR) ((((uintptr_t) (PTR)) & HANDLE_BASE) != 0)
-+#define pointer_to_stub(PTR) ((PyObject *)(((uintptr_t) (PTR)) & ~HANDLE_BASE))
 +        if (points_to_py_handle_space(param)) {
-+            param = pointer_to_stub(param);
++            param = (char*)pointer_to_stub(param);
 +        }
-+#undef points_to_py_handle_space
-+#undef pointer_to_stub
-+#undef HANDLE_BASE
          volatile char NPY_UNUSED(c) = *(char*)param;
      }
 
 diff --git a/numpy/core/src/npymath/ieee754.cpp b/numpy/core/src/npymath/ieee754.cpp
-index 1c59bf3..131985e 100644
+index 1c59bf3..519fabc 100644
 --- a/numpy/core/src/npymath/ieee754.cpp
 +++ b/numpy/core/src/npymath/ieee754.cpp
-@@ -428,6 +428,17 @@ npy_get_floatstatus_barrier(char *param)
+@@ -428,6 +428,11 @@ npy_get_floatstatus_barrier(char *param)
       * By using a volatile, the compiler cannot reorder this call
       */
      if (param != NULL) {
 +        // GraalPy change: the pointer needs to be dereferenced to establish
 +        // a data dependency to to ensure the compiler won't reorder the call
-+#define HANDLE_BASE 0x8000000000000000ULL
-+#define points_to_py_handle_space(PTR) ((((uintptr_t) (PTR)) & HANDLE_BASE) != 0)
-+#define pointer_to_stub(PTR) ((PyObject *)(((uintptr_t) (PTR)) & ~HANDLE_BASE))
 +        if (points_to_py_handle_space(param)) {
-+            param = pointer_to_stub(param);
++            param = (char*)pointer_to_stub(param);
 +        }
-+#undef points_to_py_handle_space
-+#undef pointer_to_stub
-+#undef HANDLE_BASE
          volatile char NPY_UNUSED(c) = *(char *)param;
      }
 
+diff --git a/vendored-meson/meson/mesonbuild/utils/universal.py b/vendored-meson/meson/mesonbuild/utils/universal.py
+index 1694912..a555fe3 100644
+--- a/vendored-meson/meson/mesonbuild/utils/universal.py
++++ b/vendored-meson/meson/mesonbuild/utils/universal.py
+@@ -727,6 +727,7 @@ def windows_detect_native_arch() -> str:
+     """
+     if sys.platform != 'win32':
+         return ''
++    return 'amd64'  # Workaround for GraalPy bug on Windows with kernel32.GetCurrentProcess()
+     try:
+         import ctypes
+         process_arch = ctypes.c_ushort()