[GR-12552] [GR-13072] [GR-17067] [GR-17531] Preparation for SciPy support

PullRequest: graalpython/580

Author: fangerer

graalpython/com.oracle.graal.python.cext/src/capi.c

@@ -153,6 +153,7 @@ declare_type(PyMethod_Type, method, PyMethodObject);
 declare_type(PyCode_Type, code, PyCodeObject);
 declare_type(PyFrame_Type, frame, PyFrameObject);
 declare_type(PyTraceBack_Type, traceback, PyTracebackObject);
+declare_type(_PyWeakref_RefType, ReferenceType, PyWeakReference);
 // Below types use the same object structure as others, and thus
 // POLYGLOT_DECLARE_TYPE should not be called again
 initialize_type(PySuper_Type, super, _object);
@@ -162,6 +163,8 @@ initialize_type(PyBool_Type, bool, _longobject);
 initialize_type(_PyNotImplemented_Type, NotImplementedType, _object);
 initialize_type(PyDictProxy_Type, mappingproxy, _object);
 initialize_type(PyEllipsis_Type, ellipsis, _object);
+initialize_type(_PyWeakref_ProxyType, ProxyType, PyWeakReference);
+initialize_type(_PyWeakref_CallableProxyType, CallableProxyType, PyWeakReference);
 
 POLYGLOT_DECLARE_TYPE(PyThreadState);
 

graalpython/com.oracle.graal.python.cext/src/object.c

@@ -364,14 +364,39 @@ int PyObject_Print(PyObject* object, FILE* fd, int flags) {
     return 0;
 }
 
-UPCALL_ID(PyObject_GetAttr);
-PyObject* PyObject_GetAttrString(PyObject* obj, const char* attr) {
-    return UPCALL_CEXT_O(_jls_PyObject_GetAttr, native_to_java(obj), polyglot_from_string(attr, SRC_CS));
+// taken from CPython "Objects/object.c"
+PyObject * PyObject_GetAttrString(PyObject *v, const char *name) {
+    PyObject *w, *res;
+
+    if (Py_TYPE(v)->tp_getattr != NULL) {
+        return (*Py_TYPE(v)->tp_getattr)(v, (char*)name);
+    }
+    w = PyUnicode_FromString(name);
+    if (w == NULL) {
+        return NULL;
+    }
+    res = PyObject_GetAttr(v, w);
+    return res;
 }
 
-UPCALL_ID(PyObject_SetAttr);
-int PyObject_SetAttrString(PyObject* obj, const char* attr, PyObject* value) {
-    return UPCALL_CEXT_I(_jls_PyObject_SetAttr, native_to_java(obj), polyglot_from_string(attr, SRC_CS), native_to_java(value));
+
+// taken from CPython "Objects/object.c"
+int PyObject_SetAttrString(PyObject *v, const char *name, PyObject *w) {
+    PyObject *s;
+    int res;
+    PyTypeObject *type = Py_TYPE(v);
+
+    if (type->tp_setattr != NULL) {
+        return (*type->tp_setattr)(v, (char*)name, w);
+    }
+    // TODO(fa): CPython interns strings; verify if that makes sense for us as well
+    // s = PyUnicode_InternFromString(name);
+    s = PyUnicode_FromString(name);
+    if (s == NULL) {
+        return -1;
+    }
+    res = PyObject_SetAttr(v, s, w);
+    return res;
 }
 
 UPCALL_ID(PyObject_HasAttr);
@@ -383,20 +408,94 @@ int PyObject_HasAttrString(PyObject* obj, const char* attr) {
     return UPCALL_CEXT_I(_jls_PyObject_HasAttr, native_to_java(obj), polyglot_from_string(attr, SRC_CS));
 }
 
-PyObject* PyObject_GetAttr(PyObject* obj, PyObject* attr) {
-    return UPCALL_CEXT_O(_jls_PyObject_GetAttr, native_to_java(obj), native_to_java(attr));
+/* Note: We must implement this in native because it might happen that this function is used on an
+   uninitialized type which means that a managed attribute lookup won't work. */
+// taken from CPython "Objects/object.c"
+PyObject * PyObject_GetAttr(PyObject *v, PyObject *name) {
+    PyTypeObject *tp = Py_TYPE(v);
+
+    if (!PyUnicode_Check(name)) {
+        PyErr_Format(PyExc_TypeError,
+                     "attribute name must be string, not '%.200s'",
+                     name->ob_type->tp_name);
+        return NULL;
+    }
+    if (tp->tp_getattro != NULL) {
+        return (*tp->tp_getattro)(v, name);
+    }
+    if (tp->tp_getattr != NULL) {
+        const char *name_str = PyUnicode_AsUTF8(name);
+        if (name_str == NULL) {
+            return NULL;
+        }
+        return (*tp->tp_getattr)(v, (char *)name_str);
+    }
+    PyErr_Format(PyExc_AttributeError,
+                 "'%.50s' object has no attribute '%U'",
+                 tp->tp_name, name);
+    return NULL;
 }
 
+UPCALL_ID(PyObject_GenericGetAttr);
 PyObject* PyObject_GenericGetAttr(PyObject* obj, PyObject* attr) {
-    return PyObject_GetAttr(obj, attr);
+    PyTypeObject *tp = Py_TYPE(obj);
+    if (tp->tp_dict == NULL && PyType_Ready(tp) < 0) {
+    	return NULL;
+    }
+	return UPCALL_CEXT_O(_jls_PyObject_GenericGetAttr, native_to_java(obj), native_to_java(attr));
 }
 
-int PyObject_SetAttr(PyObject* obj, PyObject* attr, PyObject* value) {
-    return PyObject_SetAttrString(obj, as_char_pointer(attr), value);
+/* Note: We must implement this in native because it might happen that this function is used on an
+   unitialized type which means that a managed attribute lookup won't work. */
+// taken from CPython "Objects/object.c"
+int PyObject_SetAttr(PyObject *v, PyObject *name, PyObject *value) {
+    PyTypeObject *tp = Py_TYPE(v);
+    int err;
+
+    if (!PyUnicode_Check(name)) {
+        PyErr_Format(PyExc_TypeError,
+                     "attribute name must be string, not '%.200s'",
+                     name->ob_type->tp_name);
+        return -1;
+    }
+
+    // TODO(fa): CPython interns strings; verify if that makes sense for us as well
+    // PyUnicode_InternInPlace(&name);
+    if (tp->tp_setattro != NULL) {
+        err = (*tp->tp_setattro)(v, name, value);
+        return err;
+    }
+    if (tp->tp_setattr != NULL) {
+        const char *name_str = PyUnicode_AsUTF8(name);
+        if (name_str == NULL)
+            return -1;
+        err = (*tp->tp_setattr)(v, (char *)name_str, value);
+        return err;
+    }
+    if (tp->tp_getattr == NULL && tp->tp_getattro == NULL)
+        PyErr_Format(PyExc_TypeError,
+                     "'%.100s' object has no attributes "
+                     "(%s .%U)",
+                     tp->tp_name,
+                     value==NULL ? "del" : "assign to",
+                     name);
+    else
+        PyErr_Format(PyExc_TypeError,
+                     "'%.100s' object has only read-only attributes "
+                     "(%s .%U)",
+                     tp->tp_name,
+                     value==NULL ? "del" : "assign to",
+                     name);
+    return -1;
 }
 
+UPCALL_ID(PyObject_GenericSetAttr);
 int PyObject_GenericSetAttr(PyObject* obj, PyObject* attr, PyObject* value) {
-    return PyObject_SetAttr(obj, attr, value);
+    PyTypeObject *tp = Py_TYPE(obj);
+    if (tp->tp_dict == NULL && PyType_Ready(tp) < 0) {
+        return -1;
+    }
+	return (int) UPCALL_CEXT_L(_jls_PyObject_GenericSetAttr, native_to_java(obj), native_to_java(attr), native_to_java(value));
 }
 
 Py_hash_t PyObject_Hash(PyObject* obj) {
@@ -471,7 +570,7 @@ PyObject* PyObject_Init(PyObject *op, PyTypeObject *tp) {
     return op;
 }
 
-// taken from CPython 3.6.4 "Objects/object.c"
+// taken from CPython "Objects/object.c"
 PyVarObject * PyObject_InitVar(PyVarObject *op, PyTypeObject *tp, Py_ssize_t size) {
     if (op == NULL) {
         return (PyVarObject *) PyErr_NoMemory();

graalpython/com.oracle.graal.python.cext/src/typeobject.c

@@ -76,19 +76,22 @@ static int add_subclass(PyTypeObject *base, PyTypeObject *type) {
 /* Special C landing functions that convert some arguments to primitives. */
 
 static PyObject* wrap_allocfunc(allocfunc f, PyTypeObject* klass, PyObject* n) {
-	return f(klass, PyLong_AsSsize_t(n));
+	return native_to_java(f(klass, PyLong_AsSsize_t(n)));
 }
 
 /* Wrapper around a native function to be called by Python code. */
 static PyObject* wrap_getattrfunc(getattrfunc f, PyObject* obj, PyObject* unicode) {
 	// we really need to provide 'char *' since this often runs non-Sulong code
-	return f(obj, as_char_pointer(unicode));
+	return native_to_java(f(obj, as_char_pointer(unicode)));
 }
 
 /* Wrapper around the native function to be called by Python code. */
 static PyObject* wrap_setattrfunc(setattrfunc f, PyObject* obj, PyObject* unicode, PyObject* value) {
 	// we really need to provide 'char *' since this often runs non-Sulong code
-	return f(obj, as_char_pointer(unicode), value);
+    if (f(obj, as_char_pointer(unicode), value) < 0) {
+        return NULL;
+    }
+    return Py_None;
 }
 
 static PyObject* wrap_setattrofunc(setattrofunc f, PyObject* obj, PyObject* key, PyObject* item) {
@@ -108,7 +111,7 @@ static PyObject* wrap_descrgetfunc(descrgetfunc f, PyObject* self, PyObject* obj
 }
 
 static PyObject* wrap_richcmpfunc(richcmpfunc f, PyObject* a, PyObject* b, PyObject* n) {
-	return f(a, b, (int)PyLong_AsLong(n));
+	return native_to_java(f(a, b, (int)PyLong_AsLong(n)));
 }
 
 #undef RICHCMP_WRAPPER
@@ -182,9 +185,12 @@ static PyObject* wrap_reverse_binop(binaryfunc f, PyObject* a, PyObject* b) {
     return f(b, a);
 }
 
-static void
-inherit_special(PyTypeObject *type, PyTypeObject *base)
-{
+UPCALL_ID(PyTruffle_Type_Modified);
+void PyType_Modified(PyTypeObject* type) {
+	UPCALL_CEXT_VOID(_jls_PyTruffle_Type_Modified, native_type_to_java(type), polyglot_from_string(type->tp_name, SRC_CS), native_to_java(type->tp_mro));
+}
+
+static void inherit_special(PyTypeObject *type, PyTypeObject *base) {
 
     /* Copying basicsize is connected to the GC flags */
     if (!(type->tp_flags & Py_TPFLAGS_HAVE_GC) &&
@@ -245,6 +251,17 @@ inherit_special(PyTypeObject *type, PyTypeObject *base)
         type->tp_flags |= Py_TPFLAGS_DICT_SUBCLASS;
 }
 
+static void inherit_slots(PyTypeObject *type, PyTypeObject *base) {
+    if (type->tp_getattr == NULL && type->tp_getattro == NULL) {
+        type->tp_getattr = base->tp_getattr;
+        type->tp_getattro = base->tp_getattro;
+    }
+    if (type->tp_setattr == NULL && type->tp_setattro == NULL) {
+        type->tp_setattr = base->tp_setattr;
+        type->tp_setattro = base->tp_setattro;
+    }
+}
+
 // TODO support member flags other than READONLY
 UPCALL_ID(AddMember);
 static void add_member(PyTypeObject* cls, PyObject* type_dict, PyObject* mname, int mtype, Py_ssize_t moffset, int mflags, char* mdoc) {
@@ -295,6 +312,9 @@ int PyType_Ready(PyTypeObject* cls) {
 		add_method_or_slot(cls, dict, (__name__), (__meth__), (__clanding__), (__flags__), (__doc__)); \
 	}
 
+    Py_ssize_t n;
+    Py_ssize_t i;
+
     // https://docs.python.org/3/c-api/typeobj.html#Py_TPFLAGS_READY
     if ((cls->tp_flags & Py_TPFLAGS_READY) || (cls->tp_flags & Py_TPFLAGS_READYING)) {
         return 0;
@@ -408,6 +428,17 @@ int PyType_Ready(PyTypeObject* cls) {
     if (cls->tp_base != NULL)
         inherit_special(cls, cls->tp_base);
 
+    /* Initialize tp_dict properly */
+    bases = cls->tp_mro;
+    assert(bases != NULL);
+    assert(PyTuple_Check(bases));
+    n = PyTuple_GET_SIZE(bases);
+    for (i = 1; i < n; i++) {
+        PyObject *b = PyTuple_GET_ITEM(bases, i);
+        if (PyType_Check(b))
+            inherit_slots(cls, (PyTypeObject *)b);
+    }
+
     ADD_IF_MISSING(cls->tp_alloc, PyType_GenericAlloc);
     ADD_IF_MISSING(cls->tp_new, PyType_GenericNew);
 
@@ -456,6 +487,7 @@ int PyType_Ready(PyTypeObject* cls) {
         ADD_SLOT("__sub__", numbers->nb_subtract, -2);
         ADD_SLOT_CONV("__rsub__", wrap_reverse_binop, numbers->nb_subtract, -2);
         ADD_SLOT("__mul__", numbers->nb_multiply, -2);
+        ADD_SLOT_CONV("__rmul__", wrap_reverse_binop, numbers->nb_multiply, -2);
         ADD_SLOT("__mod__", numbers->nb_remainder, -2);
         ADD_SLOT_CONV("__rmod__", wrap_reverse_binop, numbers->nb_remainder, -2);
         ADD_SLOT("__divmod__", numbers->nb_divmod, -2);
@@ -540,8 +572,7 @@ int PyType_Ready(PyTypeObject* cls) {
 
     /* Link into each base class's list of subclasses */
     bases = cls->tp_bases;
-    Py_ssize_t n = PyTuple_GET_SIZE(bases);
-    Py_ssize_t i;
+    n = PyTuple_GET_SIZE(bases);
     for (i = 0; i < n; i++) {
         PyObject* base_class_object = PyTuple_GetItem(bases, i);
         PyTypeObject* b = (PyTypeObject*) base_class_object;
@@ -554,6 +585,9 @@ int PyType_Ready(PyTypeObject* cls) {
     cls->tp_flags = cls->tp_flags & ~Py_TPFLAGS_READYING;
     cls->tp_flags = cls->tp_flags | Py_TPFLAGS_READY;
 
+    // it may be that the type was used uninitialized
+	UPCALL_CEXT_VOID(_jls_PyTruffle_Type_Modified, cls, polyglot_from_string(cls->tp_name, SRC_CS), Py_NoValue);
+
     return 0;
 
 #undef ADD_IF_MISSING
@@ -562,11 +596,6 @@ int PyType_Ready(PyTypeObject* cls) {
 #undef ADD_METHOD_OR_SLOT
 }
 
-UPCALL_ID(PyTruffle_Type_Modified);
-void PyType_Modified(PyTypeObject* type) {
-	UPCALL_CEXT_VOID(_jls_PyTruffle_Type_Modified, native_type_to_java(type), polyglot_from_string(type->tp_name, SRC_CS), native_to_java(type->tp_mro));
-}
-
 MUST_INLINE static int valid_identifier(PyObject *s) {
     if (!PyUnicode_Check(s)) {
         PyErr_Format(PyExc_TypeError,

graalpython/com.oracle.graal.python.cext/src/weakrefobject.c

@@ -0,0 +1,45 @@
+/*
+ * Copyright (c) 2019, Oracle and/or its affiliates. All rights reserved.
+ * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER.
+ *
+ * The Universal Permissive License (UPL), Version 1.0
+ *
+ * Subject to the condition set forth below, permission is hereby granted to any
+ * person obtaining a copy of this software, associated documentation and/or
+ * data (collectively the "Software"), free of charge and under any and all
+ * copyright rights in the Software, and any and all patent rights owned or
+ * freely licensable by each licensor hereunder covering either (i) the
+ * unmodified Software as contributed to or provided by such licensor, or (ii)
+ * the Larger Works (as defined below), to deal in both
+ *
+ * (a) the Software, and
+ *
+ * (b) any piece of software and/or hardware listed in the lrgrwrks.txt file if
+ * one is included with the Software each a "Larger Work" to which the Software
+ * is contributed by such licensors),
+ *
+ * without restriction, including without limitation the rights to copy, create
+ * derivative works of, display, perform, and distribute the Software and make,
+ * use, sell, offer for sale, import, export, have made, and have sold the
+ * Software and the Larger Work(s), and to sublicense the foregoing rights on
+ * either these or other terms.
+ *
+ * This license is subject to the following condition:
+ *
+ * The above copyright notice and either this complete permission notice or at a
+ * minimum a reference to the UPL must be included in all copies or substantial
+ * portions of the Software.
+ *
+ * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+ * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+ * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
+ * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+ * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+ * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
+ * SOFTWARE.
+ */
+#include "capi.h"
+
+PyTypeObject _PyWeakref_RefType = PY_TRUFFLE_TYPE("weakref", &PyType_Type, Py_TPFLAGS_DEFAULT | Py_TPFLAGS_HAVE_GC | Py_TPFLAGS_BASETYPE, sizeof(PyWeakReference));
+PyTypeObject _PyWeakref_ProxyType = PY_TRUFFLE_TYPE("weakproxy", &PyType_Type, Py_TPFLAGS_DEFAULT | Py_TPFLAGS_HAVE_GC, sizeof(PyWeakReference));
+PyTypeObject _PyWeakref_CallableProxyType = PY_TRUFFLE_TYPE("weakcallableproxy", &PyType_Type, Py_TPFLAGS_DEFAULT | Py_TPFLAGS_HAVE_GC, sizeof(PyWeakReference));

graalpython/com.oracle.graal.python.test/src/tests/package1/__init__.py

@@ -40,4 +40,5 @@
 """PACKAGE DOC"""
 
 from .lib import *
+from .lib1 import *
 from . import exported