Support buffer protocol for Python bytes.

Author: fangerer

graalpython/com.oracle.graal.python.cext/src/abstract.c

@@ -263,6 +263,7 @@ PyObject * PyMapping_GetItemString(PyObject *o, const char *key) {
 	return to_sulong(result);
 }
 
+// taken from CPython "Objects/abstract.c"
 int PyObject_GetBuffer(PyObject *obj, Py_buffer *view, int flags) {
     PyBufferProcs *pb = obj->ob_type->tp_as_buffer;
 
@@ -275,6 +276,19 @@ int PyObject_GetBuffer(PyObject *obj, Py_buffer *view, int flags) {
     return (*pb->bf_getbuffer)(obj, view, flags);
 }
 
+// taken from CPython "Objects/abstract.c"
+void PyBuffer_Release(Py_buffer *view) {
+    PyObject *obj = view->obj;
+    PyBufferProcs *pb;
+    if (obj == NULL)
+        return;
+    pb = Py_TYPE(obj)->tp_as_buffer;
+    if (pb && pb->bf_releasebuffer)
+        pb->bf_releasebuffer(obj, view);
+    view->obj = NULL;
+    Py_DECREF(obj);
+}
+
 // taken from CPython "Objects/abstract.c"
 /* we do this in native code since we need to fill in the values in a given 'Py_buffer' struct */
 int PyBuffer_FillInfo(Py_buffer *view, PyObject *obj, void *buf, Py_ssize_t len, int readonly, int flags) {

graalpython/com.oracle.graal.python.cext/src/capi.c

@@ -223,6 +223,12 @@ PyTypeObject* PyObjectHandle_ForJavaType(void* jobj) {
 	return jobj;
 }
 
+/** to be used from Java code only; creates the deref handle for a sequence wrapper */
+void* NativeHandle_ForArray(void* jobj, ssize_t element_size) {
+    // TODO do polyglot typecast depending on element_size
+    return truffle_deref_handle_for_managed(jobj);
+}
+
 const char* PyTruffle_StringToCstr(void* jlString) {
     return truffle_string_to_cstr(jlString);
 }

graalpython/com.oracle.graal.python/src/com/oracle/graal/python/builtins/modules/TruffleCextBuiltins.java

@@ -60,6 +60,7 @@
 import com.oracle.graal.python.builtins.objects.bytes.BytesBuiltins;
 import com.oracle.graal.python.builtins.objects.bytes.PBytes;
 import com.oracle.graal.python.builtins.objects.cext.CExtNodes;
+import com.oracle.graal.python.builtins.objects.cext.PythonClassNativeWrapper;
 import com.oracle.graal.python.builtins.objects.cext.PythonObjectNativeWrapper;
 import com.oracle.graal.python.builtins.objects.cext.UnicodeObjectNodes.UnicodeAsWideCharNode;
 import com.oracle.graal.python.builtins.objects.complex.PComplex;
@@ -68,7 +69,6 @@
 import com.oracle.graal.python.builtins.objects.function.Arity;
 import com.oracle.graal.python.builtins.objects.function.PArguments;
 import com.oracle.graal.python.builtins.objects.ints.PInt;
-import com.oracle.graal.python.builtins.objects.object.PythonObject;
 import com.oracle.graal.python.builtins.objects.str.PString;
 import com.oracle.graal.python.builtins.objects.traceback.PTraceback;
 import com.oracle.graal.python.builtins.objects.tuple.PTuple;
@@ -1057,9 +1057,10 @@ PythonObjectNativeWrapper doPythonObject(PythonObjectNativeWrapper nativeWrapper
     abstract static class PyTruffle_SetBufferProcs extends NativeBuiltin {
 
         @Specialization
-        Object doPythonObject(PythonObject obj, Object getBufferProc, Object releaseBufferProc) {
-            obj.setAttribute(SpecialAttributeNames.__GET_BUFFER__, getBufferProc);
-            obj.setAttribute(SpecialAttributeNames.__RELEASE_BUFFER__, releaseBufferProc);
+        Object doPythonObject(PythonClass obj, Object getBufferProc, Object releaseBufferProc) {
+            PythonClassNativeWrapper nativeWrapper = obj.getNativeWrapper();
+            nativeWrapper.setGetBufferProc(getBufferProc);
+            nativeWrapper.setReleaseBufferProc(releaseBufferProc);
             return PNone.NO_VALUE;
         }
     }

graalpython/com.oracle.graal.python/src/com/oracle/graal/python/builtins/objects/cext/CExtNodes.java

@@ -123,6 +123,7 @@ Object runNativeObject(PythonAbstractObject object) {
 
         @Fallback
         Object run(Object obj) {
+            assert obj != null : "Java 'null' cannot be a Sulong value";
             return obj;
         }
 

graalpython/com.oracle.graal.python/src/com/oracle/graal/python/builtins/objects/cext/NativeCAPISymbols.java

@@ -44,6 +44,7 @@ public abstract class NativeCAPISymbols {
     public static final String FUN_PY_TRUFFLE_STRING_TO_CSTR = "PyTruffle_StringToCstr";
     public static final String FUN_PY_OBJECT_HANDLE_FOR_JAVA_OBJECT = "PyObjectHandle_ForJavaObject";
     public static final String FUN_PY_OBJECT_HANDLE_FOR_JAVA_TYPE = "PyObjectHandle_ForJavaType";
+    public static final String FUN_NATIVE_HANDLE_FOR_ARRAY = "NativeHandle_ForArray";
     public static final String FUN_PY_NONE_HANDLE = "PyNoneHandle";
     public static final String FUN_WHCAR_SIZE = "PyTruffle_Wchar_Size";
 

graalpython/com.oracle.graal.python/src/com/oracle/graal/python/builtins/objects/cext/PyBufferProcsWrapper.java

@@ -0,0 +1,64 @@
+/*
+ * Copyright (c) 2018, Oracle and/or its affiliates.
+ *
+ * The Universal Permissive License (UPL), Version 1.0
+ *
+ * Subject to the condition set forth below, permission is hereby granted to any
+ * person obtaining a copy of this software, associated documentation and/or data
+ * (collectively the "Software"), free of charge and under any and all copyright
+ * rights in the Software, and any and all patent rights owned or freely
+ * licensable by each licensor hereunder covering either (i) the unmodified
+ * Software as contributed to or provided by such licensor, or (ii) the Larger
+ * Works (as defined below), to deal in both
+ *
+ * (a) the Software, and
+ * (b) any piece of software and/or hardware listed in the lrgrwrks.txt file if
+ *     one is included with the Software (each a "Larger Work" to which the
+ *     Software is contributed by such licensors),
+ *
+ * without restriction, including without limitation the rights to copy, create
+ * derivative works of, display, perform, and distribute the Software and make,
+ * use, sell, offer for sale, import, export, have made, and have sold the
+ * Software and the Larger Work(s), and to sublicense the foregoing rights on
+ * either these or other terms.
+ *
+ * This license is subject to the following condition:
+ *
+ * The above copyright notice and either this complete permission notice or at a
+ * minimum a reference to the UPL must be included in all copies or substantial
+ * portions of the Software.
+ *
+ * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+ * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+ * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
+ * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+ * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+ * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
+ * SOFTWARE.
+ */
+package com.oracle.graal.python.builtins.objects.cext;
+
+import com.oracle.graal.python.builtins.objects.type.PythonClass;
+import com.oracle.truffle.api.interop.ForeignAccess;
+import com.oracle.truffle.api.interop.TruffleObject;
+
+public class PyBufferProcsWrapper implements TruffleObject {
+
+    private final PythonClass delegate;
+
+    public PyBufferProcsWrapper(PythonClass delegate) {
+        this.delegate = delegate;
+    }
+
+    public PythonClass getDelegate() {
+        return delegate;
+    }
+
+    static boolean isInstance(TruffleObject o) {
+        return o instanceof PyBufferProcsWrapper;
+    }
+
+    public ForeignAccess getForeignAccess() {
+        return PyBufferProcsWrapperMRForeign.ACCESS;
+    }
+}

graalpython/com.oracle.graal.python/src/com/oracle/graal/python/builtins/objects/cext/PyBufferProcsWrapperMR.java

@@ -0,0 +1,85 @@
+/*
+ * Copyright (c) 2018, Oracle and/or its affiliates.
+ *
+ * The Universal Permissive License (UPL), Version 1.0
+ *
+ * Subject to the condition set forth below, permission is hereby granted to any
+ * person obtaining a copy of this software, associated documentation and/or data
+ * (collectively the "Software"), free of charge and under any and all copyright
+ * rights in the Software, and any and all patent rights owned or freely
+ * licensable by each licensor hereunder covering either (i) the unmodified
+ * Software as contributed to or provided by such licensor, or (ii) the Larger
+ * Works (as defined below), to deal in both
+ *
+ * (a) the Software, and
+ * (b) any piece of software and/or hardware listed in the lrgrwrks.txt file if
+ *     one is included with the Software (each a "Larger Work" to which the
+ *     Software is contributed by such licensors),
+ *
+ * without restriction, including without limitation the rights to copy, create
+ * derivative works of, display, perform, and distribute the Software and make,
+ * use, sell, offer for sale, import, export, have made, and have sold the
+ * Software and the Larger Work(s), and to sublicense the foregoing rights on
+ * either these or other terms.
+ *
+ * This license is subject to the following condition:
+ *
+ * The above copyright notice and either this complete permission notice or at a
+ * minimum a reference to the UPL must be included in all copies or substantial
+ * portions of the Software.
+ *
+ * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+ * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+ * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
+ * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+ * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+ * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
+ * SOFTWARE.
+ */
+package com.oracle.graal.python.builtins.objects.cext;
+
+import com.oracle.graal.python.builtins.objects.PNone;
+import com.oracle.graal.python.builtins.objects.cext.CExtNodes.ToSulongNode;
+import com.oracle.truffle.api.CompilerDirectives;
+import com.oracle.truffle.api.interop.MessageResolution;
+import com.oracle.truffle.api.interop.Resolve;
+import com.oracle.truffle.api.interop.UnknownIdentifierException;
+import com.oracle.truffle.api.nodes.Node;
+
+@MessageResolution(receiverType = PyBufferProcsWrapper.class)
+public class PyBufferProcsWrapperMR {
+
+    @Resolve(message = "READ")
+    abstract static class ReadNode extends Node {
+        @Child private ToSulongNode toSulongNode;
+
+        public Object access(PyBufferProcsWrapper object, String key) {
+            // translate key to attribute name
+            PythonClassNativeWrapper nativeWrapper = object.getDelegate().getNativeWrapper();
+            // TODO handle case if nativeWrapper does not exist yet
+            Object result;
+            switch (key) {
+                case "bf_getbuffer":
+                    result = nativeWrapper.getGetBufferProc();
+                    break;
+                case "bf_releasebuffer":
+                    // TODO
+                    result = nativeWrapper.getReleaseBufferProc();
+                    break;
+                default:
+                    // TODO extend list
+                    throw UnknownIdentifierException.raise(key);
+            }
+            // use NO_VALUE for NULL
+            return getToSulongNode().execute(result == null ? PNone.NO_VALUE : result);
+        }
+
+        private ToSulongNode getToSulongNode() {
+            if (toSulongNode == null) {
+                CompilerDirectives.transferToInterpreterAndInvalidate();
+                toSulongNode = insert(ToSulongNode.create());
+            }
+            return toSulongNode;
+        }
+    }
+}

graalpython/com.oracle.graal.python/src/com/oracle/graal/python/builtins/objects/cext/PySequenceArrayWrapper.java

@@ -44,7 +44,7 @@
 /**
  * Wraps a sequence object (like a list) such that it behaves like a bare C array.
  */
-public class PySequenceArrayWrapper implements TruffleObject {
+public class PySequenceArrayWrapper extends PythonNativeWrapper {
 
     private final Object delegate;
 

graalpython/com.oracle.graal.python/src/com/oracle/graal/python/builtins/objects/cext/PySequenceArrayWrapperMR.java

@@ -48,15 +48,21 @@
 import com.oracle.graal.python.builtins.objects.tuple.PTuple;
 import com.oracle.graal.python.builtins.objects.tuple.TupleBuiltins;
 import com.oracle.graal.python.builtins.objects.tuple.TupleBuiltinsFactory;
+import com.oracle.graal.python.builtins.objects.type.PythonClass;
 import com.oracle.graal.python.nodes.SpecialMethodNames;
 import com.oracle.graal.python.nodes.truffle.PythonTypes;
 import com.oracle.truffle.api.CompilerDirectives;
+import com.oracle.truffle.api.CompilerDirectives.CompilationFinal;
 import com.oracle.truffle.api.dsl.Cached;
 import com.oracle.truffle.api.dsl.ImportStatic;
 import com.oracle.truffle.api.dsl.Specialization;
 import com.oracle.truffle.api.dsl.TypeSystemReference;
+import com.oracle.truffle.api.interop.ForeignAccess;
+import com.oracle.truffle.api.interop.Message;
 import com.oracle.truffle.api.interop.MessageResolution;
 import com.oracle.truffle.api.interop.Resolve;
+import com.oracle.truffle.api.interop.TruffleObject;
+import com.oracle.truffle.api.interop.UnsupportedMessageException;
 import com.oracle.truffle.api.nodes.Node;
 
 @MessageResolution(receiverType = PySequenceArrayWrapper.class)
@@ -186,4 +192,82 @@ public static WriteArrayItemNode create() {
             return WriteArrayItemNodeGen.create();
         }
     }
+
+    @Resolve(message = "TO_NATIVE")
+    abstract static class ToNativeNode extends Node {
+        @Child private ToNativeArrayNode toPyObjectNode = ToNativeArrayNode.create();
+
+        Object access(PySequenceArrayWrapper obj) {
+            if (!obj.isNative()) {
+                // TODO
+                Object ptr = toPyObjectNode.execute(obj);
+                obj.setNativePointer(ptr);
+            }
+            return obj;
+        }
+    }
+
+    static class ToNativeArrayNode extends TransformToNativeNode {
+        @CompilationFinal private TruffleObject PyObjectHandle_FromJavaObject;
+        @Child private PCallNativeNode callNativeBinary;
+
+        public Object execute(PySequenceArrayWrapper object) {
+            // TODO correct element size
+            return ensureIsPointer(callBinaryIntoCapi(getNativeHandleForArray(), object, 8L));
+        }
+
+        private TruffleObject getNativeHandleForArray() {
+            if (PyObjectHandle_FromJavaObject == null) {
+                CompilerDirectives.transferToInterpreterAndInvalidate();
+                PyObjectHandle_FromJavaObject = (TruffleObject) getContext().getEnv().importSymbol(NativeCAPISymbols.FUN_NATIVE_HANDLE_FOR_ARRAY);
+            }
+            return PyObjectHandle_FromJavaObject;
+        }
+
+        protected boolean isNonNative(PythonClass klass) {
+            return !(klass instanceof PythonNativeClass);
+        }
+
+        private Object callBinaryIntoCapi(TruffleObject fun, Object arg0, Object arg1) {
+            if (callNativeBinary == null) {
+                CompilerDirectives.transferToInterpreterAndInvalidate();
+                callNativeBinary = insert(PCallNativeNode.create(1));
+            }
+            return callNativeBinary.execute(fun, new Object[]{arg0, arg1});
+        }
+
+        public static ToNativeArrayNode create() {
+            return new ToNativeArrayNode();
+        }
+    }
+
+    @Resolve(message = "IS_POINTER")
+    abstract static class IsPointerNode extends Node {
+        Object access(PySequenceArrayWrapper obj) {
+            return obj.isNative();
+        }
+    }
+
+    @Resolve(message = "AS_POINTER")
+    abstract static class AsPointerNode extends Node {
+        @Child private Node asPointerNode;
+
+        long access(PySequenceArrayWrapper obj) {
+            // the native pointer object must either be a TruffleObject or a primitive
+            Object nativePointer = obj.getNativePointer();
+            if (nativePointer instanceof TruffleObject) {
+                if (asPointerNode == null) {
+                    CompilerDirectives.transferToInterpreterAndInvalidate();
+                    asPointerNode = insert(Message.AS_POINTER.createNode());
+                }
+                try {
+                    return ForeignAccess.sendAsPointer(asPointerNode, (TruffleObject) nativePointer);
+                } catch (UnsupportedMessageException e) {
+                    throw e.raise();
+                }
+            }
+            return (long) nativePointer;
+        }
+    }
+
 }

graalpython/com.oracle.graal.python/src/com/oracle/graal/python/builtins/objects/cext/PythonClassNativeWrapper.java

@@ -47,6 +47,8 @@
  */
 public class PythonClassNativeWrapper extends PythonObjectNativeWrapper {
     private final CStringWrapper nameWrapper;
+    private Object getBufferProc;
+    private Object releaseBufferProc;
 
     public PythonClassNativeWrapper(PythonClass object) {
         super(object);
@@ -57,6 +59,22 @@ public CStringWrapper getNameWrapper() {
         return nameWrapper;
     }
 
+    public Object getGetBufferProc() {
+        return getBufferProc;
+    }
+
+    public void setGetBufferProc(Object getBufferProc) {
+        this.getBufferProc = getBufferProc;
+    }
+
+    public Object getReleaseBufferProc() {
+        return releaseBufferProc;
+    }
+
+    public void setReleaseBufferProc(Object releaseBufferProc) {
+        this.releaseBufferProc = releaseBufferProc;
+    }
+
     public static PythonClassNativeWrapper wrap(PythonClass obj) {
         // important: native wrappers are cached
         PythonClassNativeWrapper nativeWrapper = obj.getNativeWrapper();