[GR-53302] Fixes for pybind11, part #1

PullRequest: graalpython/3287

Author: msimacek

graalpython/com.oracle.graal.python.cext/src/capi.c

@@ -249,10 +249,27 @@ PyAPI_FUNC(void) mmap_init_bufferprotocol(PyObject* mmap_type) {
 	    (getbufferproc)mmap_getbuffer,
 	    (releasebufferproc)NULL,
 	};
-	set_PyTypeObject_tp_as_buffer(mmap_type, &mmap_as_buffer);
+	GraalPy_set_PyTypeObject_tp_as_buffer(mmap_type, &mmap_as_buffer);
 	((PyTypeObject*) mmap_type)->tp_as_buffer = &mmap_as_buffer;
 }
 
+static int cdata_getbuffer(PyObject* type, Py_buffer* view, int flags) {
+    return GraalPyTruffleCData_NewGetBuffer(type, view, flags);
+}
+
+static void cdata_releasebuffer(PyObject* obj, Py_buffer* view) {
+    return GraalPyTruffleCData_ReleaseBuffer(obj, view);
+}
+
+PyAPI_FUNC(void) PyTruffleCData_InitBufferProtocol(PyObject* type) {
+    static PyBufferProcs cdata_as_buffer = {
+        cdata_getbuffer,
+        cdata_releasebuffer,
+    };
+    GraalPy_set_PyTypeObject_tp_as_buffer(type, &cdata_as_buffer);
+    ((PyTypeObject*) type)->tp_as_buffer = &cdata_as_buffer;
+}
+
 struct _longobject* _Py_FalseStructReference;
 struct _longobject* _Py_TrueStructReference;
 PyObject* _Py_EllipsisObjectReference;

graalpython/com.oracle.graal.python.cext/src/capsule.c

@@ -1,4 +1,4 @@
-/* Copyright (c) 2021, 2023, Oracle and/or its affiliates.
+/* Copyright (c) 2021, 2024, Oracle and/or its affiliates.
  * Copyright (C) 1996-2021 Python Software Foundation
  *
  * Licensed under the PYTHON SOFTWARE FOUNDATION LICENSE VERSION 2
@@ -14,3 +14,7 @@ PyAPI_FUNC(PyTypeObject*) getPyCapsuleTypeReference() {
 	return &PyCapsule_Type;
 }
 
+PyAPI_FUNC(void) PyTruffleCapsule_CallDestructor(PyObject* capsule, PyCapsule_Destructor destructor) {
+    destructor(capsule);
+}
+

graalpython/com.oracle.graal.python.test/src/tests/cpyext/__init__.py

@@ -42,6 +42,7 @@
 import os
 import shutil
 import sys
+import time
 import unittest
 from copy import deepcopy
 from io import StringIO
@@ -54,6 +55,7 @@
 
 IS_MANAGED_LAUNCHER = not GRAALPYTHON or __graalpython__.is_managed_launcher()
 
+
 def assert_raises(err, fn, *args, **kwargs):
     raised = False
     try:

graalpython/com.oracle.graal.python.test/src/tests/cpyext/test_capsule.py

@@ -36,14 +36,16 @@
 # LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
 # OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
 # SOFTWARE.
+import gc
+import os
+import time
 
-from . import CPyExtTestCase, CPyExtFunction, unhandled_error_compare
+from . import CPyExtTestCase, CPyExtFunction, unhandled_error_compare, CPyExtType
 
 __dir__ = __file__.rpartition("/")[0]
 
 
 class TestPyCapsule(CPyExtTestCase):
-
     test_PyCapsule_CheckExact = CPyExtFunction(
         lambda args: True,
         lambda: (
@@ -112,3 +114,33 @@ class TestPyCapsule(CPyExtTestCase):
         callfunction="wrap_PyCapsule_SetContext",
         cmpfunc=unhandled_error_compare
     )
+
+    if os.environ.get('GRAALPYTEST_RUN_GC_TESTS'):
+        def test_capsule_destructor(self):
+            Tester = CPyExtType(
+                "CapsuleDestructorTester",
+                code="""
+                static void capsule_destructor(PyObject* capsule) {
+                    PyObject* contents = (PyObject*) PyCapsule_GetPointer(capsule, "capsule");
+                    assert(PyDict_Check(contents));
+                    PyDict_SetItemString(contents, "destructor_was_here", Py_NewRef(Py_True));
+                    Py_DECREF(contents);
+                }
+                
+                static PyObject* create_capsule(PyObject* unused, PyObject* contents) {
+                    return PyCapsule_New(Py_NewRef(contents), "capsule", capsule_destructor);
+                }
+                """,
+                tp_methods='{"create_capsule", (PyCFunction)create_capsule, METH_O | METH_STATIC, NULL}',
+            )
+            d = {}
+            capsule = Tester.create_capsule(d)
+            assert capsule
+            assert not d
+            del capsule
+            start = time.time()
+            while "destructor_was_here" not in d:
+                if time.time() - start > 60:
+                    raise AssertionError("Capsule destructor didn't execute within timeout")
+                gc.collect()
+                time.sleep(0.01)

graalpython/com.oracle.graal.python.test/src/tests/cpyext/test_ceval.py

@@ -37,16 +37,12 @@
 # OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
 # SOFTWARE.
 
-from . import CPyExtTestCase, CPyExtFunction, unhandled_error_compare
+from . import CPyExtTestCase, CPyExtFunction, unhandled_error_compare, CPyExtType
 
 __dir__ = __file__.rpartition("/")[0]
 
-class TestCeval(CPyExtTestCase):
-
-    def compile_module(self, name):
-        type(self).mro()[1].__dict__["test_%s" % name].create_module(name)
-        super(TestCeval, self).compile_module(name)
 
+class TestCeval(CPyExtTestCase):
     test_Py_EnterLeaveRecursiveCall = CPyExtFunction(
         # We don't know the exact limit on CPython since it uses the counter in
         # the interpreter as well.
@@ -81,3 +77,15 @@ def compile_module(self, name):
         arguments=["int n", "char* where"],
         cmpfunc=unhandled_error_compare
     )
+
+    def test_PyEval_GetGlobals(self):
+        Tester = CPyExtType(
+            "GetGlobalsTester",
+            code="""
+            static PyObject* get_globals(PyObject* unused) {
+                return Py_NewRef(PyEval_GetGlobals());
+            }
+            """,
+            tp_methods='{"get_globals", (PyCFunction)get_globals, METH_NOARGS | METH_STATIC, NULL}',
+        )
+        assert Tester.get_globals() is globals()

graalpython/com.oracle.graal.python.test/src/tests/cpyext/test_ctypes.py

@@ -0,0 +1,131 @@
+# Copyright (c) 2024, Oracle and/or its affiliates. All rights reserved.
+# DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER.
+#
+# The Universal Permissive License (UPL), Version 1.0
+#
+# Subject to the condition set forth below, permission is hereby granted to any
+# person obtaining a copy of this software, associated documentation and/or
+# data (collectively the "Software"), free of charge and under any and all
+# copyright rights in the Software, and any and all patent rights owned or
+# freely licensable by each licensor hereunder covering either (i) the
+# unmodified Software as contributed to or provided by such licensor, or (ii)
+# the Larger Works (as defined below), to deal in both
+#
+# (a) the Software, and
+#
+# (b) any piece of software and/or hardware listed in the lrgrwrks.txt file if
+# one is included with the Software each a "Larger Work" to which the Software
+# is contributed by such licensors),
+#
+# without restriction, including without limitation the rights to copy, create
+# derivative works of, display, perform, and distribute the Software and make,
+# use, sell, offer for sale, import, export, have made, and have sold the
+# Software and the Larger Work(s), and to sublicense the foregoing rights on
+# either these or other terms.
+#
+# This license is subject to the following condition:
+#
+# The above copyright notice and either this complete permission notice or at a
+# minimum a reference to the UPL must be included in all copies or substantial
+# portions of the Software.
+#
+# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+# IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+# FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
+# AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+# LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+# OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
+# SOFTWARE.
+import ctypes
+import struct
+import sys
+
+from tests.cpyext import CPyExtTestCase, CPyExtType
+
+BufferTester = CPyExtType(
+    'BufferTester',
+    cmembers='Py_buffer buffer;',
+    code='''
+    static PyObject* buffer_tester_new(PyTypeObject* type, PyObject* args, PyObject* kwargs) {
+        PyObject* obj;
+        if (PyArg_ParseTuple(args, "O", &obj) < 0)
+            return NULL;
+        PyObject* self = type->tp_alloc(type, 0);
+        if (!self)
+            return NULL;
+        if (PyObject_GetBuffer(obj, &((BufferTesterObject*)self)->buffer, PyBUF_CONTIG_RO) < 0) {
+            Py_DECREF(self);
+            return NULL;
+        }
+        return self;
+    }
+    
+    static PyObject* buffer_tester_enter(PyObject* self) {
+        return Py_NewRef(self);
+    }
+    
+    static PyObject* buffer_tester_exit(BufferTesterObject* self, PyObject* args) {
+        PyBuffer_Release(&self->buffer);
+        Py_RETURN_NONE;
+    }
+    
+    static PyObject* buffer_tester_obj(BufferTesterObject* self) {
+        return Py_NewRef(self->buffer.obj);
+    }
+    
+    static PyObject* buffer_tester_bytes(BufferTesterObject* self) {
+        return PyBytes_FromStringAndSize(self->buffer.buf, self->buffer.len);
+    }
+    
+    static PyObject* buffer_tester_itemsize(BufferTesterObject* self) {
+        return PyLong_FromSsize_t(self->buffer.itemsize);
+    }
+    
+    static PyObject* buffer_tester_format(BufferTesterObject* self) {
+        return PyBytes_FromString(self->buffer.format);
+    }
+    
+    static PyObject* buffer_tester_shape(BufferTesterObject* self) {
+        PyObject* tuple = PyTuple_New(self->buffer.ndim);
+        if (!tuple)
+            return NULL;
+        for (int i = 0; i < self->buffer.ndim; i++) {
+            PyObject* value = PyLong_FromSsize_t(self->buffer.shape[i]);
+            if (!value)
+                return NULL;
+            PyTuple_SET_ITEM(tuple, i, value);
+        }
+        return tuple;
+    }
+    ''',
+    tp_new='buffer_tester_new',
+    tp_methods='''
+    {"__enter__", (PyCFunction)buffer_tester_enter, METH_NOARGS, ""},
+    {"__exit__", (PyCFunction)buffer_tester_exit, METH_VARARGS, ""}
+    ''',
+    tp_getset='''
+    {"obj", (getter)buffer_tester_obj, NULL, NULL, NULL},
+    {"bytes", (getter)buffer_tester_bytes, NULL, NULL, NULL},
+    {"itemsize", (getter)buffer_tester_itemsize, NULL, NULL, NULL},
+    {"format", (getter)buffer_tester_format, NULL, NULL, NULL},
+    {"shape", (getter)buffer_tester_shape, NULL, NULL, NULL}
+    ''',
+)
+
+
+class TestCDataBuffer(CPyExtTestCase):
+    def test_buffer(self):
+        if sys.implementation.name == 'graalpy' and __graalpython__.get_platform_id() == 'managed':
+            # TODO we don't support converting ctypes arrays to native memory in managed
+            return
+        int_format = struct.Struct(">i")
+        inner_type = ctypes.c_int.__ctype_be__ * 2
+        outer_type = inner_type * 2
+        array = outer_type(inner_type(1, 2), inner_type(3, 4))
+        with BufferTester(array) as buffer:
+            assert buffer.obj is array
+            assert buffer.bytes == b''.join(int_format.pack(n) for n in [1, 2, 3, 4])
+            assert buffer.itemsize == int_format.size
+            assert buffer.format.startswith(b'>')
+            assert struct.Struct(buffer.format).size == int_format.size
+            assert buffer.shape == (2, 2)

graalpython/com.oracle.graal.python.test/src/tests/cpyext/test_pythonrun.py

@@ -37,54 +37,42 @@
 # OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
 # SOFTWARE.
 
-import sys
-
-from . import CPyExtTestCase, CPyExtFunction, unhandled_error_compare
+from . import CPyExtTestCase, CPyExtFunction, unhandled_error_compare, CPyExtType
 
 __dir__ = __file__.rpartition("/")[0]
 
 
-# python_run_test_result = None
-
-def _reference_run_string(args):
-    if not isinstance(args[2], dict):
-        if sys.version_info.minor >= 6:
-            raise SystemError
-        else:
-            raise TypeError
-    if not isinstance(args[3], dict):
-        raise TypeError
-    return None
-
-
-def _run_string_compare(x, y):
-    res = unhandled_error_compare(x, y)
-    if (isinstance(x, Exception)):
-        return res
+class TestPythonRun(CPyExtTestCase):
 
-    global python_run_test_result
-    pr = python_run_test_result
-    res = res and pr == 42
-    python_run_test_result = None
-    if not res:
-        assert False, "python_run_test_result is %s" % pr
-    return res
+    def test_PyRun_String(self):
+        Tester = CPyExtType(
+            "PyRunStringTester",
+            code="""
+            static PyObject* call_PyRun_String(PyObject* unused, PyObject* args) {
+                int eval;
+                char *string;
+                PyObject *globals, *locals;
+                if (PyArg_ParseTuple(args, "spOO", &string, &eval, &globals, &locals) < 0)
+                    return NULL;
+                int start = eval ? Py_eval_input : Py_file_input;
+                return PyRun_String(string, start, globals, locals);
+            }
+            """,
+            tp_methods='{"call_PyRun_String", (PyCFunction)call_PyRun_String, METH_VARARGS | METH_STATIC}'
+        )
+        g = {}
+        l = {}
+        assert Tester.call_PyRun_String("a = 1", False, g, l) is None
+        assert l.get('a') == 1
+        assert 'a' not in g
+        g = {}
+        l = {}
 
+        assert Tester.call_PyRun_String("global a\na = 1", False, g, l) is None
+        assert g.get('a') == 1
+        assert 'a' not in l
 
-class TestPythonRun(CPyExtTestCase):
-
-    test_PyRun_StringFlags = CPyExtFunction(
-        _reference_run_string,
-        lambda: (
-            ("globals().update({'python_run_test_result':42})", 256, globals(), locals(), 0),
-            ("globals().update({'python_run_test_result':42})", 256, 'globals()', locals(), 0),
-            ("globals().update({'python_run_test_result':42})", 256, globals(), 'locals()', 0),
-        ),
-        resultspec="O",
-        argspec='siOOk',
-        arguments=["char* source", "int type", "PyObject* globals", "PyObject* locals", "PyCompilerFlags* ignored"],
-        cmpfunc=unhandled_error_compare
-    )
+        assert Tester.call_PyRun_String("1 + a", True, {}, {'a': 2}) == 3
 
     test_Py_CompileString = CPyExtFunction(
         lambda args: compile(

graalpython/com.oracle.graal.python.test/src/tests/test_codecs.py

@@ -875,5 +875,11 @@ def test_encode(self):
         self.assertRaises(UnicodeEncodeError, codec.encode, '\xffff')
 
 
+class UTF32Test(unittest.TestCase):
+    def test_utf32_surrogate_error(self):
+        with self.assertRaisesRegex(UnicodeDecodeError, "codec can't decode bytes in position 4-7"):
+            b'a\x00\x00\x00\x00\xd8\x00\x00z\x00\x00\x00'.decode('utf-32')
+
+
 if __name__ == '__main__':
     unittest.main()

graalpython/com.oracle.graal.python/src/com/oracle/graal/python/builtins/modules/MMapModuleBuiltins.java

@@ -1,5 +1,5 @@
 /*
- * Copyright (c) 2019, 2023, Oracle and/or its affiliates. All rights reserved.
+ * Copyright (c) 2019, 2024, Oracle and/or its affiliates. All rights reserved.
  * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER.
  *
  * The Universal Permissive License (UPL), Version 1.0
@@ -135,7 +135,7 @@ public MMapModuleBuiltins() {
     public void postInitialize(Python3Core core) {
         super.postInitialize(core);
         core.getContext().registerCApiHook(() -> {
-            CExtNodes.PCallCapiFunction.getUncached().call(NativeCAPISymbol.FUN_MMAP_INIT_BUFFERPROTOCOL, PythonToNativeNode.executeUncached(PythonBuiltinClassType.PMMap));
+            CExtNodes.PCallCapiFunction.callUncached(NativeCAPISymbol.FUN_MMAP_INIT_BUFFERPROTOCOL, PythonToNativeNode.executeUncached(PythonBuiltinClassType.PMMap));
         });
     }