Merge branch 'master' into feature/GR-12099

Author: ppisl

ci.jsonnet

@@ -70,7 +70,7 @@
       "git": ">=1.8.3",
       "mercurial": ">=3.2.4",
       "gcc": "==4.9.1",
-      "llvm": ">=4.0",
+      "llvm": "==4.0.1",
       "python": "==3.4.1",
       "libffi": ">=3.2.1",
       "bzip2": ">=1.0.6",

doc/INTEROP.md

@@ -145,9 +145,12 @@ Returns true for None only.
 
 ###### HAS_SIZE
 According to the Truffle interop contract answering `true` to `HAS_SIZE` implies
-that indexed element access is available. Thus, we answer `true` here only for
-(sub-)instances of `tuple`, `list`, `array.array`, `bytearray`, `bytes`, `str`,
-and `range`.
+that indexed element access is available. However, we cannot fully guarantee
+this. We may answer `true` here when the object has both a `__len__` field and a
+`__getitem__` field. If the object's length is reported >0, we also try to read
+the item `0` and if that fails, we answer `false`. If the object reports it's
+empty, we cannot know if a read with an index will actually work, but we'll
+report `true`.
 
 ###### GET_SIZE
 Calls `__len__`. Just because `GET_SIZE` returns something positive does not
@@ -159,27 +162,35 @@ knowing what the `__getitem__` method does with an integer argument. Use
 Returns true for those values that can be unboxed using the `UNBOX` message.
 
 ###### KEY_INFO
-This will lookup the key using `READ`, assume it is readable and writable, and
-check `IS_EXECUTABLE`.
+This will lookup the key using the Python MRO. It will check if it's readable
+and writable, and also check if it has side-effects based on wether it is an
+inherited descriptor (i.e., an object with `__get__`, `__set__`, and/or
+`__delete__`). If the owner of the key is mutable (the owner being the class the
+key is inherited from or the object itself) then `REMOVABLE` and `MODIFABLE` are
+true. If the object itself is mutable, `INSERTABLE` will also be true. Finally,
+if the attribute is a function or it is *not* a descriptor and has a `__call__`,
+we declare it `INOCABLE`. We don't do this for descriptors, because we would
+have to run the `__get__` method and this message should not have side-effects.
 
 ###### HAS_KEYS
-Returns true for any boxed Python object, so small integers, booleans, or floats
-usually don't return true.
+Always returns true.
 
 ###### KEYS
-This returns the direct attributes of the receiver object, which would usually
-be available through `__getattribute__`. 
-
-The `KEYS` message requires the returned object to have only `java.lang.String`
-items. If the object responds to `keys`, `values`, `items`, and `__getitem__`,
-we assume it is Mapping, and we present the result of the `keys` method in
-combination with the attributes if, and only if, all keys are strings. This is
-roughly parallel to how `READ` and `WRITE` would be handled for string keys.
+This returns the all attributes of the receiver object that would usually be
+available through `__getattribute__`, i.e., both inherited and direct
+attributes.
+
+If the object responds to `keys`, `values`, `items`, and `__getitem__`, we
+assume it is Mapping, and we present the String result of the `keys` method in
+combination with the attributes, prefixed with `[` if, and only if, the request
+asked for _internal_ keys also. The `KEYS` message requires the returned object
+to have only `java.lang.String` items, so inlo String keys are added to the
+result set. The `[` prefix ensures that in our handling of `READ` and `WRITE`
+messages we also treat them as mapping entries, not attributes.
 
 It's still possible that none of the keys can be `READ`: the `READ` message uses
 Python semantics for lookup, which means that an inherited descriptor with a
-`__get__` method may still come before the object's keys and do anything
-(including raising an `AttributeError`).
+`__get__` method or the `__getitem__` method may still intercept actual access.
 
 ###### IS_POINTER
 Returns true if the object is a Python function defined in a Python C extension

graalpython/com.oracle.graal.python.cext/include/truffle.h

@@ -54,6 +54,9 @@ void *truffle_assign_managed(void *dst, void *managed);
 void *truffle_deref_handle_for_managed(void *managed);
 bool truffle_cannot_be_handle(void *nativeHandle);
 
+// wrapping functions
+void *truffle_decorate_function(void *function, void *wrapper);
+
 /*
  * All function below here are deprecated and will be removed in a future release.
  * Use the equivalent functions from <polyglot.h> instead.

graalpython/com.oracle.graal.python.cext/src/capi.c

@@ -461,9 +461,8 @@ PyObject* WriteULongMember(PyObject* object, Py_ssize_t offset, PyObject* value)
     return value;
 }
 
-UPCALL_ID(__bool__);
 PyObject* WriteBoolMember(PyObject* object, Py_ssize_t offset, PyObject* value) {
-    WriteMember(object, offset, UPCALL_O(native_to_java(value), _jls___bool__) == Py_True ? (char)1 : (char)0, char);
+    WriteMember(object, offset, UPCALL_O(native_to_java(value), polyglot_from_string("__bool__", SRC_CS)) == Py_True ? (char)1 : (char)0, char);
     return value;
 }
 

graalpython/com.oracle.graal.python.cext/src/capi.h

@@ -75,9 +75,13 @@ typedef struct {
 PyAPI_DATA(PyTypeObject) PyBuffer_Type;
 PyAPI_DATA(PyTypeObject) _PyExc_BaseException;
 
+typedef void (*init_upcall)();
+
 extern void *PY_TRUFFLE_CEXT;
 extern void *PY_BUILTIN;
 extern void *Py_NoValue;
+extern init_upcall upcalls[];
+extern unsigned init_upcall_n;
 
 /* upcall functions for calling into Python */
 extern PyObject*(*PY_TRUFFLE_LANDING)(void *rcv, void* name, ...);
@@ -89,13 +93,6 @@ extern uint64_t (*PY_TRUFFLE_CEXT_LANDING_L)(void* name, ...);
 extern double (*PY_TRUFFLE_CEXT_LANDING_D)(void* name, ...);
 extern void* (*PY_TRUFFLE_CEXT_LANDING_PTR)(void* name, ...);
 
-#define UPCALL_ID(name)                                                 \
-    static void* _jls_ ## name;                                         \
-    __attribute__((constructor))                                        \
-    static void init_jls_ ## name(void) {                               \
-        _jls_ ## name = polyglot_from_string(#name, SRC_CS);            \
-    }
-
 /* Call function with return type 'PyObject *'; does polyglot cast and error handling */
 #define UPCALL_O(__recv__, __name__, ...) PY_TRUFFLE_LANDING((__recv__), __name__, ##__VA_ARGS__)
 
@@ -138,6 +135,13 @@ extern void* (*PY_TRUFFLE_CEXT_LANDING_PTR)(void* name, ...);
 /* Call function of 'python_cext' module with return type 'double'; no polyglot cast but error handling */
 #define UPCALL_CEXT_D(__name__, ...) (PY_TRUFFLE_CEXT_LANDING_D(__name__, ##__VA_ARGS__))
 
+#define UPCALL_ID(name)                                                 \
+    static void* _jls_ ## name;                                         \
+    __attribute__((constructor))                                        \
+    static void init_upcall_ ## name(void) {                               \
+       _jls_ ## name = polyglot_get_member(PY_TRUFFLE_CEXT, polyglot_from_string(#name, SRC_CS)); \
+    }
+
 #define as_char_pointer(obj) ((const char*)UPCALL_CEXT_PTR(polyglot_from_string("to_char_pointer", "ascii"), native_to_java(obj)))
 #define as_long(obj) ((long)polyglot_as_i64(polyglot_invoke(PY_TRUFFLE_CEXT, "to_long", to_java(obj))))
 #define as_long_long(obj) ((long long)polyglot_as_i64(polyglot_invoke(PY_TRUFFLE_CEXT, "PyLong_AsPrimitive", to_java(obj), 1, sizeof(long long), polyglot_from_string("long long", "utf-8"))))

graalpython/com.oracle.graal.python.cext/src/dictobject.c

@@ -114,9 +114,8 @@ int PyDict_DelItemString(PyObject *d, const char *key) {
     return UPCALL_CEXT_I(_jls_PyDict_DelItem, native_to_java(d), polyglot_from_string(key, SRC_CS));
 }
 
-UPCALL_ID(update);
 int PyDict_Update(PyObject *a, PyObject *b) {
-    PyObject* result = UPCALL_O(native_to_java(a), _jls_update, native_to_java(b));
+    PyObject* result = UPCALL_O(native_to_java(a), polyglot_from_string("update", SRC_CS), native_to_java(b));
     if (PyErr_Occurred()) {
         return -1;
     } else {

graalpython/com.oracle.graal.python.cext/src/errors.c

@@ -45,11 +45,11 @@ void _PyErr_BadInternalCall(const char *filename, int lineno) {
     UPCALL_CEXT_VOID(_jls__PyErr_BadInternalCall, polyglot_from_string(filename, SRC_CS), lineno, native_to_java(NULL));
 }
 
+UPCALL_ID(PyErr_CreateAndSetException);
 #undef PyErr_BadInternalCall
-UPCALL_ID(PyTruffle_Err_Format);
 void PyErr_BadInternalCall(void) {
     assert(0 && "bad argument to internal function");
-    UPCALL_CEXT_VOID(_jls_PyTruffle_Err_Format, native_to_java(PyExc_SystemError), polyglot_from_string("bad argument to internal function", SRC_CS));
+    UPCALL_CEXT_VOID(_jls_PyErr_CreateAndSetException, native_to_java(PyExc_SystemError), polyglot_from_string("bad argument to internal function", SRC_CS));
 }
 #define PyErr_BadInternalCall() _PyErr_BadInternalCall(__FILE__, __LINE__)
 
@@ -64,7 +64,6 @@ void PyErr_SetString(PyObject *exception, const char *string) {
     PyErr_SetObject(exception, value);
 }
 
-UPCALL_ID(PyErr_CreateAndSetException);
 void PyErr_SetObject(PyObject *exception, PyObject *value) {
     UPCALL_CEXT_VOID(_jls_PyErr_CreateAndSetException, native_to_java(exception), native_to_java(value));
 }

graalpython/com.oracle.graal.python.cext/src/exceptions.c

@@ -121,9 +121,8 @@ void initialize_exceptions() {
 }
 
 
-UPCALL_ID(with_traceback);
 int PyException_SetTraceback(PyObject *self, PyObject *tb) {
-    PyObject* result = UPCALL_O(native_to_java(self), _jls_with_traceback, native_to_java(tb));
+    PyObject* result = UPCALL_O(native_to_java(self), polyglot_from_string("with_traceback", SRC_CS), native_to_java(tb));
     if (result == NULL) {
         return -1;
     } else {

graalpython/com.oracle.graal.python.cext/src/functionobject.c

@@ -42,7 +42,6 @@
 
 PyTypeObject PyFunction_Type = PY_TRUFFLE_TYPE("function", &PyType_Type, Py_TPFLAGS_DEFAULT | Py_TPFLAGS_HAVE_GC, sizeof(PyFunctionObject));
 
-UPCALL_ID(classmethod);
 PyObject* PyClassMethod_New(PyObject* method) {
-    return UPCALL_O(PY_BUILTIN, _jls_classmethod, native_to_java(method));
+    return UPCALL_O(PY_BUILTIN, polyglot_from_string("classmethod", SRC_CS), native_to_java(method));
 }

graalpython/com.oracle.graal.python.cext/src/import.c

@@ -49,10 +49,9 @@ PyObject* PyImport_Import(PyObject *name) {
     return UPCALL_CEXT_O(_jls_PyImport_ImportModule, native_to_java(name));
 }
 
-UPCALL_ID(__import__);
 PyObject* PyImport_ImportModuleLevelObject(PyObject* name, PyObject* globals, PyObject* locals,
                                            PyObject* fromlist, int level) {
-    return UPCALL_O(PY_BUILTIN, _jls___import__, native_to_java(name), native_to_java(globals),
+    return UPCALL_O(PY_BUILTIN, polyglot_from_string("__import__", SRC_CS), native_to_java(name), native_to_java(globals),
                     native_to_java(locals), native_to_java(fromlist), level);
 }