Merge remote-tracking branch 'origin/master' into tim/drop-pypy

Author: timfel

CONTRIBUTING.md

@@ -0,0 +1,44 @@
+**Thank you for thinking about contributing something to this Python
+implementation on Graal!**
+
+## First things first
+
+You will need to sign the [Oracle Contributor
+Agreement](http://www.graalvm.org/community/contributors/) for us to be able to
+merge your work.
+
+Please also take some time to review our [code of
+conduct](http://www.graalvm.org/community/conduct/) for contributors.
+
+## How to contribute?
+
+If you want to make a contribution, it's best to open an issue first about the
+area you would like to offer your help with. This implementation is in an early
+state and it will make more sense to work on some areas than others, so before
+you sink work into a pull request that we won't be able to accept, please talk
+to us :)
+
+To get started with development on Python, first make sure you can build at
+least the _truffle_ project in the [Graal](https://github.com/oracle/graal)
+repository with our build tool [mx](https://github.com/graalvm/mx), as well as
+[Sulong](https://github.com/graalvm/sulong).
+
+Once you can build those projects, clone this repository and run
+
+    $ mx build
+    $ mx python -c "print(42)"
+
+If this prints "42", then everything went fine and you just built and ran
+Python. Note that you don't need GraalVM for this, but then you'll only run
+interpreted, which won't give very good performance (but that might be fine for
+development).
+
+When you make your changes, you can test them with `mx python`. Additionally,
+there are various "gates" that we use on our CI system to check any code that
+goes in. You can run all of these gates using `mx python-gate` or just some by
+using `mx python-gate --tags [TAG]`. Two interesting tags to run that cover most
+things are:
+
+- python-unittest
+- python-graalvm
+- python-license

graalpython/com.oracle.graal.python.cext/modules/_cpython_sre.c

@@ -359,6 +359,7 @@ getstring(PyObject* string, Py_ssize_t* p_length,
     return view->buf;
 }
 
+
 LOCAL(PyObject*)
 state_init(SRE_STATE* state, PatternObject* pattern, PyObject* string,
            Py_ssize_t start, Py_ssize_t end)
@@ -1052,6 +1053,19 @@ _sre_SRE_Pattern_split_impl(PatternObject *self, PyObject *string,
 
 }
 
+__attribute__((always_inline))
+inline static void *_memchr(const void *s, int c, Py_ssize_t n) {
+    unsigned char *p = (unsigned char*)s;
+    while(n--) {
+        if(*p != (unsigned char)c) {
+            p++;
+        } else {
+            return p;
+        }
+    }
+    return 0;
+}
+
 static PyObject*
 pattern_subx(PatternObject* self, PyObject* ptemplate, PyObject* string,
              Py_ssize_t count, Py_ssize_t subn)
@@ -1083,7 +1097,7 @@ pattern_subx(PatternObject* self, PyObject* ptemplate, PyObject* string,
         b = charsize;
         if (ptr) {
             if (charsize == 1)
-                literal = memchr(ptr, '\\', n) == NULL;
+                literal = _memchr(ptr, '\\', n) == NULL;
             else
                 literal = PyUnicode_FindChar(ptemplate, '\\', 0, n, 1) == -1;
         } else {

graalpython/com.oracle.graal.python.cext/modules/_memoryview.c

@@ -6,7 +6,7 @@
 
 /* Memoryview object implementation */
 
-#include "Python.h"
+#include "../src/capi.h"
 #include <limits.h>
 /* #include "internal/mem.h" */
 /* #include "internal/pystate.h" */
@@ -57,6 +57,8 @@
      releasebufferprocs must NOT decrement view.obj.
 */
 
+POLYGLOT_DECLARE_TYPE(PyMemoryViewObject);
+
 PyTypeObject PyNativeMemoryView_Type;
 
 #define CHECK_MBUF_RELEASED(mbuf) \
@@ -646,7 +648,7 @@ memory_alloc(int ndim)
     mv->weakreflist = NULL;
 
     _PyObject_GC_TRACK(mv);
-    return mv;
+    return polyglot_from_PyMemoryViewObject(mv);
 }
 
 /*
@@ -966,7 +968,7 @@ memory_new(PyTypeObject *subtype, PyObject *args, PyObject *kwds)
         return NULL;
     }
 
-    return PyMemoryView_FromObject(obj);
+    return polyglot_from_PyMemoryViewObject((PyMemoryViewObject *)PyMemoryView_FromObject(obj));
 }
 
 
@@ -1413,11 +1415,18 @@ memory_cast(PyMemoryViewObject *self, PyObject *args, PyObject *kwds)
 /*                               getbuffer                                */
 /**************************************************************************/
 
+static PyMemoryViewObject* PyTruffle_MemoryView_GetDelegate(PyObject* managed_self) {
+    return (PyMemoryViewObject*) PyObject_GetAttrString(managed_self, "__c_memoryview");
+}
+
 static int
 memory_getbuf(PyMemoryViewObject *self, Py_buffer *view, int flags)
 {
-    Py_buffer *base = &self->view;
-    int baseflags = self->flags;
+    Py_buffer *base = NULL;
+    int baseflags = 0;
+
+    base = &self->view;
+    baseflags = self->flags;
 
     CHECK_RELEASED_INT(self);
 
@@ -3125,6 +3134,28 @@ PyTypeObject PyNativeMemoryView_Type = {
     memory_new,                               /* tp_new */
 };
 
+
+static PyMemoryViewObject* _get_managed_self(PyMemoryViewObject *original_self) {
+    PyMemoryViewObject *self = NULL;
+
+    // this code may be reached over different path; self may therefore be the managed wrapper or the native object
+    if (PyMemoryView_Check(original_self)) {
+        // we need to get the native delegate of the managed memoryview object
+        self = PyTruffle_MemoryView_GetDelegate((PyObject*) original_self);
+    }
+    assert(self != NULL);
+    assert(Py_TYPE(self) == &PyNativeMemoryView_Type);
+    return self;
+}
+
+static int memory_managed_getbuf(PyMemoryViewObject *self, Py_buffer *view, int flags) {
+    return memory_getbuf(_get_managed_self(self), view, flags);
+}
+
+static void memory_managed_releasebuf(PyMemoryViewObject *self, Py_buffer *view) {
+    memory_releasebuf(_get_managed_self(self), view);
+}
+
 static struct PyModuleDef _memoryviewmodule = {
     PyModuleDef_HEAD_INIT,
     "_memoryview",
@@ -3158,5 +3189,9 @@ PyInit__memoryview(void)
     Py_INCREF((PyObject*)&_PyManagedBuffer_Type);
     PyModule_AddObject(m, "managedbuffer", (PyObject*) &_PyManagedBuffer_Type);
 
+
+    // register buffer procs
+    polyglot_invoke(PY_TRUFFLE_CEXT, "PyTruffle_SetBufferProcs", native_to_java((PyObject*)&PyMemoryView_Type), (getbufferproc)memory_managed_getbuf, (releasebufferproc)memory_managed_releasebuf);
+
     return m;
 }

graalpython/com.oracle.graal.python.cext/src/bytesobject.c

@@ -267,3 +267,7 @@ int bytes_copy2mem(char* target, char* source, size_t nbytes) {
     }
     return 0;
 }
+
+PyObject *_PyBytes_Join(PyObject *sep, PyObject *x) {
+    return UPCALL_CEXT_O("PyBytes_Join", native_to_java(sep), native_to_java(x));
+}

graalpython/com.oracle.graal.python.cext/src/codeobject.c

@@ -42,15 +42,16 @@
 
 PyTypeObject PyCode_Type = PY_TRUFFLE_TYPE("code", &PyType_Type, Py_TPFLAGS_DEFAULT, sizeof(PyTypeObject));
 
-PyCodeObject* PyCode_New(int argcount, int kwonlyargcount, int nlocals,
-                         int stacksize, int flags, PyObject *code,
-                         PyObject *consts, PyObject *names, PyObject *varnames,
-                         PyObject *freevars, PyObject *cellvars,
+PyCodeObject* PyCode_New(int argcount, int kwonlyargcount,
+                         int nlocals, int stacksize, int flags,
+                         PyObject *code, PyObject *consts, PyObject *names,
+                         PyObject *varnames, PyObject *freevars, PyObject *cellvars,
                          PyObject *filename, PyObject *name, int firstlineno,
                          PyObject *lnotab) {
-    return UPCALL_CEXT_O("PyCode_New", argcount, kwonlyargcount, nlocals,
-                         stacksize, flags, native_to_java(code),
-                         native_to_java(consts), native_to_java(names), native_to_java(varnames),
+    return UPCALL_CEXT_O("PyCode_New", argcount, kwonlyargcount,
+                         nlocals, stacksize, flags,
+                         native_to_java(code), native_to_java(consts), native_to_java(names),
+                         native_to_java(varnames), native_to_java(freevars), native_to_java(cellvars),
                          native_to_java(filename), native_to_java(name), firstlineno,
                          native_to_java(lnotab));
 }

graalpython/com.oracle.graal.python.cext/src/typeobject.c

@@ -233,10 +233,10 @@ int PyType_Ready(PyTypeObject* cls) {
     // remember the managed wrapper
     ((PyObject*)cls)->ob_refcnt = truffle_handle_for_managed(javacls);
     if (cls->tp_dict != NULL) {
-        // TODO: (tfel) is this always safe?
-        PyDict_Update(javacls->tp_dict, cls->tp_dict);
+        javacls->tp_dict = native_to_java(cls->tp_dict);
+    } else {
+        cls->tp_dict = javacls->tp_dict;
     }
-    cls->tp_dict = javacls->tp_dict;
 
     PyMethodDef* methods = cls->tp_methods;
     if (methods) {

graalpython/com.oracle.graal.python.test/src/tests/cpyext/__init__.py

@@ -448,8 +448,15 @@ def CPyExtType(name, code, **kwargs):
         {nb_inplace_floor_divide},
         {nb_inplace_true_divide},
         {nb_index},
+    """ + ("""
         {nb_matrix_multiply},
         {nb_inplace_matrix_multiply},
+    """ if sys.version_info.minor >= 6 else "") + """
+    }};
+    
+    static struct PyMethodDef {name}_methods[] = {{
+        {tp_methods},
+        {{NULL, NULL, 0, NULL}}
     }};
 
     typedef struct {{
@@ -459,13 +466,13 @@ def CPyExtType(name, code, **kwargs):
     static PyTypeObject {name}Type = {{
         PyVarObject_HEAD_INIT(NULL, 0)
         "{name}.{name}",
-        sizeof({name}Object),      /* tp_basicsize */
-        0,                         /* tp_itemsize */
-        0,                         /* tp_dealloc */
+        sizeof({name}Object),       /* tp_basicsize */
+        0,                          /* tp_itemsize */
+        0,                          /* tp_dealloc */
         {tp_print},
         {tp_getattr},
         {tp_setattr},
-        0,                         /* tp_reserved */
+        0,                          /* tp_reserved */
         {tp_repr},
         &{name}_number_methods,
         {tp_as_sequence},
@@ -478,6 +485,24 @@ def CPyExtType(name, code, **kwargs):
         {tp_as_buffer},
         Py_TPFLAGS_DEFAULT,
         "",
+        {tp_traverse},              /* tp_traverse */
+        {tp_clear},                 /* tp_clear */
+        {tp_richcompare},           /* tp_richcompare */
+        0,                          /* tp_weaklistoffset */
+        {tp_iter},                  /* tp_iter */
+        {tp_iternext},              /* tp_iternext */
+        {name}_methods,             /* tp_methods */
+        NULL,                       /* tp_members */
+        0,                          /* tp_getset */
+        0,                          /* tp_base */
+        {tp_dict},                  /* tp_dict */
+        0,                          /* tp_descr_get */
+        0,                          /* tp_descr_set */
+        0,                          /* tp_dictoffset */
+        {tp_init},                  /* tp_init */
+        PyType_GenericAlloc,        /* tp_alloc */
+        PyType_GenericNew,          /* tp_new */
+        PyObject_Del,               /* tp_free */
     }};
 
     static PyModuleDef {name}module = {{
@@ -493,10 +518,10 @@ def CPyExtType(name, code, **kwargs):
     {{
         PyObject* m;
 
-        {name}Type.tp_new = PyType_GenericNew;
         {ready_code}
         if (PyType_Ready(&{name}Type) < 0)
             return NULL;
+        {post_ready_code}
 
         m = PyModule_Create(&{name}module);
         if (m == NULL)
@@ -511,6 +536,8 @@ def CPyExtType(name, code, **kwargs):
     kwargs["name"] = name
     kwargs["code"] = code
     kwargs.setdefault("ready_code", "")
+    kwargs.setdefault("post_ready_code", "")
+    kwargs.setdefault("tp_methods", "{NULL, NULL, 0, NULL}")
     c_source = UnseenFormatter().format(template, **kwargs)
 
     source_file = "%s/%s.c" % (__dir__, name)

graalpython/com.oracle.graal.python.test/src/tests/cpyext/test_codeobject.py

@@ -37,9 +37,12 @@
 # OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
 # SOFTWARE.
 
-import sys
 import types
-from . import CPyExtTestCase, CPyExtFunction, CPyExtFunctionOutVars, unhandled_error_compare, GRAALPYTHON
+
+import sys
+
+from . import CPyExtTestCase, CPyExtFunction
+
 __dir__ = __file__.rpartition("/")[0]
 
 
@@ -63,19 +66,23 @@ def compile_module(self, name):
         lambda args: args,
         lambda: (
             (
-                1,2,3,4,0,b"",tuple(),tuple(),tuple(),
-                tuple(),tuple(),"filename","name",1,b"",
+                1, 2,
+                3, 4, 0,
+                b"", tuple(), tuple(),
+                tuple(), tuple(), tuple(),
+                "filename", "name", 1,
+                b"",
             ),
         ),
         resultspec="O",
         argspec="iiiiiOOOOOOOOiO",
         arguments=[
-            "int argcount", "int kwonlyargcount", "int nlocals",
-            "int stacksize", "int flags", "PyObject* code",
-            "PyObject* consts", "PyObject* names", "PyObject* varnames",
-            "PyObject* freevars", "PyObject* cellvars",
+            "int argcount", "int kwonlyargcount",
+            "int nlocals", "int stacksize", "int flags",
+            "PyObject* code", "PyObject* consts", "PyObject* names",
+            "PyObject* varnames", "PyObject* freevars", "PyObject* cellvars",
             "PyObject* filename", "PyObject* name", "int firstlineno",
-            "PyObject* lnotab"
+            "PyObject* lnotab",
         ],
-        cmpfunc=lambda cr,pr: isinstance(cr, types.CodeType),
+        cmpfunc=lambda cr, pr: isinstance(cr, types.CodeType),
     )

graalpython/com.oracle.graal.python.test/src/tests/cpyext/test_dict.py

@@ -56,7 +56,11 @@ def _reference_set_item(args):
         d[args[1]] = args[2]
         return 0
     except:
-        raise SystemError
+
+        if sys.version_info.minor >= 6:
+            raise SystemError
+        else:
+            return -1
 
 
 def _reference_del_item(args):
@@ -105,6 +109,8 @@ class SubDict(dict):
 
 
 ExampleDict = {}
+
+
 def fresh_dict():
     global ExampleDict
     ExampleDict = {}
@@ -120,10 +126,16 @@ def compile_module(self, name):
     # PyDict_SetItem
     test_PyDict_SetItem = CPyExtFunction(
         _reference_set_item,
-        lambda: (({}, "a", "hello"), ({'a': "hello"}, "b", "world")),
+        lambda: (
+            ({}, "a", "hello")
+            , ({'a': "hello"}, "b", "world")
+            # mappingproxy
+            , (type(type.__dict__)({'a': "hello"}), "b", "world")
+            ),
         resultspec="i",
         argspec='OOO',
         arguments=("PyObject* dict", "PyObject* key", "PyObject* val"),
+        cmpfunc=unhandled_error_compare
     )
 
     # PyDict_GetItem
@@ -316,5 +328,5 @@ def compile_module(self, name):
             }
         }''',
         callfunction="wrap_PyDict_Update",
-        cmpfunc=lambda cr,pr: (cr == pr or (isinstance(cr, BaseException) and type(cr) == type(pr))) and (ExampleDict.get("a") == 1 or len(ExampleDict) == 0)
+        cmpfunc=lambda cr, pr: (cr == pr or (isinstance(cr, BaseException) and type(cr) == type(pr))) and (ExampleDict.get("a") == 1 or len(ExampleDict) == 0)
     )