[GR-45350] Fixes for onnxruntime

PullRequest: graalpython/2726

Author: msimacek

graalpython/com.oracle.graal.python.cext/include/genobject.h

@@ -1,4 +1,4 @@
-/* Copyright (c) 2020, 2022, Oracle and/or its affiliates.
+/* Copyright (c) 2020, 2023, Oracle and/or its affiliates.
  * Copyright (C) 1996-2017 Python Software Foundation
  *
  * Licensed under the PYTHON SOFTWARE FOUNDATION LICENSE VERSION 2

graalpython/com.oracle.graal.python.cext/setup.py

@@ -1,4 +1,4 @@
-# Copyright (c) 2019, 2022, Oracle and/or its affiliates. All rights reserved.
+# Copyright (c) 2019, 2023, Oracle and/or its affiliates. All rights reserved.
 # DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER.
 #
 # The Universal Permissive License (UPL), Version 1.0
@@ -77,6 +77,7 @@ def setup(*args, **kwargs):
     # wrap the distutil setup. since we're running in the same process, running
     # a full clean will fail the next build, since distutils thinks it already
     # created the "build" directory
+    shutil.rmtree("build", ignore_errors=True)
     os.makedirs("build", exist_ok=False)
     return distutils_setup(*args, **kwargs)
 

graalpython/com.oracle.graal.python.cext/src/abstract.c

@@ -1,5 +1,5 @@
 /*
- * Copyright (c) 2018, 2022, Oracle and/or its affiliates. All rights reserved.
+ * Copyright (c) 2018, 2023, Oracle and/or its affiliates. All rights reserved.
  * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER.
  *
  * The Universal Permissive License (UPL), Version 1.0
@@ -56,13 +56,6 @@ static PyObject* null_error(void) {
     return NULL;
 }
 
-int PyIter_Check(PyObject *obj) {
-    PyTypeObject *tp = Py_TYPE(obj);
-    return (tp->tp_iternext != NULL &&
-            tp->tp_iternext != &_PyObject_NextNotImplemented &&
-            ((PyObject *)tp->tp_iternext) != Py_NotImplemented);
-}
-
 static PyObject * do_unaryop(PyObject *v, UnaryOp unaryop) {
     return GraalPyTruffleNumber_UnaryOp(v, (int) unaryop);
 }
@@ -480,16 +473,6 @@ PyObject ** _PySequence_Fast_ITEMS(PyObject *o) {
     return PyList_Check(o) ? PyListObject_ob_item(o) : PyTupleObject_ob_item(o);
 }
 
-int _PyIter_Check(PyObject* obj) {
-	iternextfunc func = PyTypeObject_tp_iternext(Py_TYPE(obj));
-	return func != NULL && func != &_PyObject_NextNotImplemented;
-
-}
-int _PyIndex_Check(PyObject* obj) {
-	PyNumberMethods* methods = PyTypeObject_tp_as_number(Py_TYPE(obj));
-	return methods != NULL && PyNumberMethods_nb_index(methods) != NULL;
-
-}
 PyObject* _PySequence_ITEM(PyObject* obj, Py_ssize_t index) {
 	PySequenceMethods* methods = PyTypeObject_tp_as_sequence(Py_TYPE(obj));
 	return PySequenceMethods_sq_item(methods)(obj, index);

graalpython/com.oracle.graal.python.cext/src/bytesobject.c

@@ -1,5 +1,5 @@
 /*
- * Copyright (c) 2018, 2022, Oracle and/or its affiliates. All rights reserved.
+ * Copyright (c) 2018, 2023, Oracle and/or its affiliates. All rights reserved.
  * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER.
  *
  * The Universal Permissive License (UPL), Version 1.0
@@ -514,3 +514,17 @@ _PyBytesWriter_WriteBytes(_PyBytesWriter *writer, void *ptr,
 
     return str;
 }
+
+PyObject* bytes_subtype_new(PyTypeObject *type, int8_t* contents, Py_ssize_t n) {
+    PyObject* bytes = type->tp_alloc(type, n);
+    if (bytes != NULL) {
+        char* dst = ((PyBytesObject*)bytes)->ob_sval;
+        memcpy(dst, contents, n);
+        dst[n] = '\0';
+    }
+    return bytes;
+}
+
+void* PyTruffle_NativeBytesItems(PyBytesObject* bytes) {
+    return polyglot_from_i8_array((int8_t*)bytes->ob_sval, bytes->ob_base.ob_size);
+}

graalpython/com.oracle.graal.python.cext/src/capi.c

@@ -1633,6 +1633,10 @@ PyAPI_FUNC(int) PyIndex_Check(PyObject* a) {
 PyAPI_FUNC(PyObject*) PyInstanceMethod_New(PyObject* a) {
     return GraalPyInstanceMethod_New(a);
 }
+#undef PyIter_Check
+PyAPI_FUNC(int) PyIter_Check(PyObject* a) {
+    return GraalPyIter_Check(a);
+}
 #undef PyIter_Next
 PyAPI_FUNC(PyObject*) PyIter_Next(PyObject* a) {
     return GraalPyIter_Next(a);

graalpython/com.oracle.graal.python.cext/src/capi.h

@@ -166,6 +166,7 @@ typedef struct {
     BUILTIN(PyImport_ImportModuleNoBlock, PyObject*, const char*) \
     BUILTIN(PyIndex_Check, int, PyObject*) \
     BUILTIN(PyInstanceMethod_New, PyObject*, PyObject*) \
+    BUILTIN(PyIter_Check, int, PyObject*) \
     BUILTIN(PyIter_Next, PyObject*, PyObject*) \
     BUILTIN(PyList_Append, int, PyObject*, PyObject*) \
     BUILTIN(PyList_AsTuple, PyObject*, PyObject*) \

graalpython/com.oracle.graal.python.jni/src/capi_forwards.h

@@ -2689,9 +2689,8 @@ PyAPI_FUNC(PyInterpreterState*) PyInterpreterState_Next(PyInterpreterState* a) {
 PyAPI_FUNC(PyThreadState*) PyInterpreterState_ThreadHead(PyInterpreterState* a) {
     unimplemented("PyInterpreterState_ThreadHead"); exit(-1);
 }
-int (*__target__PyIter_Check)(PyObject*) = NULL;
 PyAPI_FUNC(int) PyIter_Check(PyObject* a) {
-    int result = (int) __target__PyIter_Check(a);
+    int result = (int) GraalPyIter_Check(a);
     return result;
 }
 PyAPI_FUNC(PyObject*) PyIter_Next(PyObject* a) {
@@ -6229,7 +6228,6 @@ void initializeCAPIForwards(void* (*getAPI)(const char*)) {
     __target__PyInterpreterState_GetID = getAPI("PyInterpreterState_GetID");
     __target__PyInterpreterState_GetIDFromThreadState = getAPI("PyInterpreterState_GetIDFromThreadState");
     __target__PyInterpreterState_Main = getAPI("PyInterpreterState_Main");
-    __target__PyIter_Check = getAPI("PyIter_Check");
     __target__PyLong_FromString = getAPI("PyLong_FromString");
     __target__PyMapping_GetItemString = getAPI("PyMapping_GetItemString");
     __target__PyMemoryView_FromBuffer = getAPI("PyMemoryView_FromBuffer");

graalpython/com.oracle.graal.python.test/src/tests/cpyext/__init__.py

@@ -575,7 +575,7 @@ def CPyExtType(name, code, **kwargs):
         PyVarObject_HEAD_INIT(NULL, 0)
         "{name}.{name}",
         sizeof({name}Object),       /* tp_basicsize */
-        0,                          /* tp_itemsize */
+        {tp_itemsize},              /* tp_itemsize */
         {tp_dealloc},               /* tp_dealloc */
         {tp_vectorcall_offset},
         {tp_getattr},
@@ -644,7 +644,7 @@ def CPyExtType(name, code, **kwargs):
     kwargs["code"] = code
     kwargs.setdefault("ready_code", "")
     kwargs.setdefault("post_ready_code", "")
-    kwargs.setdefault("tp_methods", "{NULL, NULL, 0, NULL}")
+    kwargs.setdefault("tp_itemsize", "0")
     kwargs.setdefault("tp_new", "PyType_GenericNew")
     kwargs.setdefault("tp_alloc", "PyType_GenericAlloc")
     kwargs.setdefault("tp_free", "PyObject_Del")

graalpython/com.oracle.graal.python.test/src/tests/cpyext/test_bytes.py

@@ -72,6 +72,19 @@ class CIter:
     def __iter__(self):
         return iter([1, 2, 3])
 
+
+BytesSubclass = CPyExtType(
+    "BytesSubclass",
+    '',
+    struct_base='PyBytesObject bytes',
+    tp_itemsize='sizeof(char)',
+    tp_base='&PyBytes_Type',
+    tp_new='0',
+    tp_alloc='0',
+    tp_free='0',
+)
+
+
 class TestPyBytes(CPyExtTestCase):
 
     def compile_module(self, name):
@@ -115,7 +128,11 @@ def compile_module(self, name):
     # PyBytes_AsString
     test_PyBytes_AsString = CPyExtFunction(
         lambda b: b[0].decode(),
-        lambda: ((b"hello",), (b"world",)),
+        lambda: (
+            (b"hello",),
+            (b"world",),
+            (BytesSubclass(b"hello"),),
+        ),
         resultspec="s",
         argspec="O",
         arguments=["PyObject* arg"],
@@ -124,7 +141,11 @@ def compile_module(self, name):
     # PyBytes_AsStringAndSize
     test_PyBytes_AsStringAndSize = CPyExtFunctionOutVars(
         _as_string_and_size,
-        lambda: ((b"hello",), (b"world",)),
+        lambda: (
+            (b"hello",),
+            (b"world",),
+            (BytesSubclass(b"hello"),),
+        ),
         resultspec="isn",
         argspec="O",
         arguments=["PyObject* arg"],
@@ -160,7 +181,12 @@ def compile_module(self, name):
     # PyBytes_Size
     test_PyBytes_Size = CPyExtFunction(
         lambda b: len(b[0]),
-        lambda: ((b"hello",), (b"hello world",), (b"",)),
+        lambda: (
+            (b"hello",),
+            (b"hello world",),
+            (b"",),
+            (BytesSubclass(b"hello"),),
+        ),
         resultspec="n",
         argspec="O",
         arguments=["PyObject* arg"],
@@ -169,7 +195,12 @@ def compile_module(self, name):
     # PyBytes_GET_SIZE
     test_PyBytes_GET_SIZE = CPyExtFunction(
         lambda b: len(b[0]),
-        lambda: ((b"hello",), (b"hello world",), (b"",)),
+        lambda: (
+            (b"hello",),
+            (b"hello world",),
+            (b"",),
+            (BytesSubclass(b"hello"),),
+        ),
         resultspec="n",
         argspec="O",
         arguments=["PyObject* arg"],
@@ -252,6 +283,7 @@ def compile_module(self, name):
             (b"hello",),
             ("hello",),
             ("hellö".encode(),),
+            (BytesSubclass(b"hello"),),
         ),
         resultspec="i",
         argspec='O',
@@ -260,7 +292,7 @@ def compile_module(self, name):
     )
 
     test_PyBytes_CheckExact = CPyExtFunction(
-        lambda args: isinstance(args[0], bytes),
+        lambda args: type(args[0]) == bytes,
         lambda: (
             (b"hello",),
             (bytes(),),
@@ -269,6 +301,7 @@ def compile_module(self, name):
             (1,),
             (dict(),),
             (tuple(),),
+            (BytesSubclass(b"hello"),),
         ),
         resultspec="i",
         argspec='O',
@@ -283,34 +316,15 @@ def compile_module(self, name):
             (b"hello",),
             ("hellö".encode("utf-8"),),
             (b"hello world",),
-            (b"",)
+            (b"",),
+            (BytesSubclass(b"hello"),),
         ),
         resultspec="s",
         argspec="O",
         arguments=["PyObject* arg"],
         cmpfunc=unhandled_error_compare
     )
 
-    test_PyBytes_Mutation = CPyExtFunction(
-        lambda args: args[1],
-        lambda: (
-            (b"hello", b"hallo"),
-        ),
-        code="""PyObject* mutate_bytes(PyObject* bytesObj, PyObject* expected) {
-            char* content = PyBytes_AS_STRING(bytesObj);
-            char* copy = (char*) malloc(PyBytes_Size(bytesObj)+1);
-            content[1] = 'a';
-            memcpy(copy, content, PyBytes_Size(bytesObj)+1);
-            return PyBytes_FromString(copy);
-        }
-        """,
-        resultspec="O",
-        argspec="OO",
-        arguments=["PyObject* bytesObj", "PyObject* expected"],
-        callfunction="mutate_bytes",
-        cmpfunc=unhandled_error_compare
-    )
-
     test_PyBytes_Resize = CPyExtFunction(
         lambda args: args[0][:args[1]],
         lambda: (
@@ -509,3 +523,36 @@ def test_create_from_buffer_exception(self):
         )
         self.assertRaises(ValueError, bytes, TestType())
         self.assertRaises(ValueError, bytearray, TestType())
+
+
+class TestNativeSubclass(unittest.TestCase):
+    def test_builtins(self):
+        b = BytesSubclass(b"hello")
+        assert type(b) == BytesSubclass
+        assert b
+        assert not BytesSubclass(b'')
+        assert len(b) == 5
+        assert b[1] == ord('e')
+        assert b[1:] == b"ello"
+        assert b == b"hello"
+        assert b < b"xxxxx"
+        assert b > b"aaaaa"
+        assert ord('e') in b
+        assert b + b" world" == b"hello world"
+        assert b * 2 == b"hellohello"
+        assert list(b) == [ord('h'), ord('e'), ord('l'), ord('l'), ord('o')]
+        assert repr(b) == "b'hello'"
+        assert b.index(ord('l')) == 2
+        assert b.count(ord('l')) == 2
+        assert b.decode('ascii') == 'hello'
+        assert BytesSubclass(b'hello ').strip() == b'hello'
+        assert BytesSubclass(b',').join([b'a', BytesSubclass(b'b')]) == b'a,b'
+        assert hash(b) == hash(b'hello')
+        assert BytesSubclass(b'(%s)') % b'a' == b'(a)'
+        assert b.startswith(b'h')
+        assert b.endswith(b'o')
+        assert b.hex() == b'hello'.hex()
+        assert b.islower()
+        assert b.replace(b'e', b'a') == b'hallo'
+        assert b.upper() == b'HELLO'
+        assert BytesSubclass(b'a,b').split(BytesSubclass(b',')) == [b'a', b'b']