[GR-20187] [GR-20189] [GR-21725] [GR-22572] Add support for HPy.

PullRequest: graalpython/760

Author: fangerer

.gitignore

@@ -80,6 +80,7 @@ Python3.g4.stamp
 /.*.csv*
 /graal_dumps
 *.jfr
+.DS_Store
 
 graalpython/com.oracle.graal.python/src/com/oracle/graal/python/parser/antlr/.antlr/
 graalpython/lib-python/3/test/data

THIRD_PARTY_LICENSE.txt

@@ -3151,3 +3151,31 @@ DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
 THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
 (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
 OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
+
+================================================================================
+
+HPy: a better API for Python
+
+License
+=======
+MIT License
+
+Copyright (c) 2019 pyhandle
+
+Permission is hereby granted, free of charge, to any person obtaining a copy
+of this software and associated documentation files (the "Software"), to deal
+in the Software without restriction, including without limitation the rights
+to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+copies of the Software, and to permit persons to whom the Software is
+furnished to do so, subject to the following conditions:
+
+The above copyright notice and this permission notice shall be included in all
+copies or substantial portions of the Software.
+
+THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
+AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
+SOFTWARE.

ci.jsonnet

@@ -1 +1 @@
-{ "overlay": "2feca4419570e36151cea7cda2a5bd5d57fd27e3" }
+{ "overlay": "7cf5c656b389584db3339485ccd8b9b4b40bf4d9" }

graalpython/com.oracle.graal.python.cext/hpy/hpy.c

@@ -0,0 +1,202 @@
+/*
+ * Copyright (c) 2019, 2020, Oracle and/or its affiliates. All rights reserved.
+ * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER.
+ *
+ * The Universal Permissive License (UPL), Version 1.0
+ *
+ * Subject to the condition set forth below, permission is hereby granted to any
+ * person obtaining a copy of this software, associated documentation and/or
+ * data (collectively the "Software"), free of charge and under any and all
+ * copyright rights in the Software, and any and all patent rights owned or
+ * freely licensable by each licensor hereunder covering either (i) the
+ * unmodified Software as contributed to or provided by such licensor, or (ii)
+ * the Larger Works (as defined below), to deal in both
+ *
+ * (a) the Software, and
+ *
+ * (b) any piece of software and/or hardware listed in the lrgrwrks.txt file if
+ * one is included with the Software each a "Larger Work" to which the Software
+ * is contributed by such licensors),
+ *
+ * without restriction, including without limitation the rights to copy, create
+ * derivative works of, display, perform, and distribute the Software and make,
+ * use, sell, offer for sale, import, export, have made, and have sold the
+ * Software and the Larger Work(s), and to sublicense the foregoing rights on
+ * either these or other terms.
+ *
+ * This license is subject to the following condition:
+ *
+ * The above copyright notice and either this complete permission notice or at a
+ * minimum a reference to the UPL must be included in all copies or substantial
+ * portions of the Software.
+ *
+ * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+ * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+ * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
+ * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+ * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+ * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
+ * SOFTWARE.
+ */
+#include "hpy.h"
+#include <polyglot.h>
+#include <truffle.h>
+
+#include <wchar.h>
+
+#define SRC_CS "utf-8"
+
+POLYGLOT_DECLARE_TYPE(HPy);
+POLYGLOT_DECLARE_STRUCT(_HPyContext_s);
+POLYGLOT_DECLARE_TYPE(HPyMethodDef);
+POLYGLOT_DECLARE_TYPE(HPyModuleDef);
+POLYGLOT_DECLARE_TYPE(wchar_t);
+
+int graal_hpy_init(void *initObject) {
+	// register the native type of HPy
+	polyglot_invoke(initObject, "setHPyContextNativeType", polyglot__HPyContext_s_typeid());
+	polyglot_invoke(initObject, "setHPyNativeType", polyglot_HPy_typeid());
+	polyglot_invoke(initObject, "setHPyArrayNativeType", polyglot_array_typeid(polyglot_HPy_typeid(), 0));
+
+	// register null handle
+	polyglot_invoke(initObject, "setHPyNullHandle", HPy_NULL);
+
+	// register size of wchar_t
+	polyglot_invoke(initObject, "setWcharSize", sizeof(wchar_t));
+
+	return 0;
+}
+
+
+void* graal_hpy_from_HPy_array(void *arr, uint64_t len) {
+	return polyglot_from_HPy_array(arr, len);
+}
+
+void* graal_hpy_from_i8_array(void *arr, uint64_t len) {
+	return polyglot_from_i8_array(arr, len);
+}
+
+wchar_t* graal_hpy_from_wchar_array(wchar_t *arr, uint64_t len) {
+    if (len == -1) {
+        len = (uint64_t) wcslen(arr);
+    }
+	return polyglot_from_wchar_t_array(arr, len);
+}
+
+void* graal_hpy_from_HPyMethodDef(void *ptr) {
+	return polyglot_from_HPyMethodDef(ptr);
+}
+
+void* graal_hpy_from_HPyModuleDef(void *ptr) {
+	return polyglot_from_HPyModuleDef(ptr);
+}
+
+void* graal_hpy_get_m_name(HPyModuleDef *moduleDef) {
+	return polyglot_from_string(moduleDef->m_name, SRC_CS);
+}
+
+void* graal_hpy_get_m_doc(HPyModuleDef *moduleDef) {
+	const char *m_doc = moduleDef->m_doc;
+	if (m_doc) {
+		return polyglot_from_string(m_doc, SRC_CS);
+	}
+	return polyglot_from_string("", SRC_CS);
+}
+
+void* graal_hpy_get_ml_name(HPyMethodDef *methodDef) {
+	return polyglot_from_string(methodDef->ml_name, SRC_CS);
+}
+
+void* graal_hpy_get_ml_doc(HPyMethodDef *methodDef) {
+	return polyglot_from_string(methodDef->ml_doc, SRC_CS);
+}
+
+void* graal_hpy_get_m_methods(HPyModuleDef *moduleDef) {
+	uint64_t len=0;
+	while ((moduleDef->m_methods[len]).ml_name != NULL) {
+		len++;
+	}
+	return polyglot_from_HPyMethodDef_array(moduleDef->m_methods, len);
+}
+
+void* graal_hpy_from_string(const char *ptr) {
+	return polyglot_from_string(ptr, SRC_CS);
+}
+
+/*
+ * Casts a 'wchar_t*' array to an 'int8_t*' array and also associates the proper length.
+ * The length is determined using 'wcslen' if 'len == -1'.
+ */
+int8_t* graal_hpy_i8_from_wchar_array(wchar_t *arr, uint64_t len) {
+    if (len == -1) {
+        len = (uint64_t) (wcslen(arr) * sizeof(wchar_t));
+    }
+	return polyglot_from_i8_array((int8_t *) arr, len);
+}
+
+typedef void* VoidPtr;
+POLYGLOT_DECLARE_TYPE(VoidPtr);
+
+typedef union {
+	void *ptr;
+	float f;
+	double d;
+	int64_t i64;
+	int32_t i32;
+	int16_t i16;
+	int8_t i8;
+	uint64_t u64;
+	uint32_t u32;
+	uint16_t u16;
+	uint8_t u8;
+	/* TODO(fa) this should be a Py_complex; is there any equivalent ? */
+	uint64_t c;
+} OutVar;
+
+POLYGLOT_DECLARE_TYPE(OutVar);
+typedef struct { OutVar *content; } OutVarPtr;
+POLYGLOT_DECLARE_TYPE(OutVarPtr);
+OutVarPtr* graal_hpy_allocate_outvar() {
+	return polyglot_from_OutVarPtr(truffle_managed_malloc(sizeof(OutVarPtr)));
+}
+
+/*
+ * Transforms a Java handle array to native.
+ * TODO(fa): This currently uses a workaround because Sulong does not fully
+ * support passing structs via interop. Therefore, we pretend to have 'void *'
+ * array and convert to handle using 'HPy_FromVoidP'.
+ */
+HPy* graal_hpy_array_to_native(VoidPtr *source, uint64_t len) {
+	uint64_t i;
+	HPy *dest = (HPy *)malloc(len*sizeof(HPy));
+	for (i=0; i < len; i++) {
+		dest[i] = HPy_FromVoidP(source[i]);
+	}
+	return dest;
+}
+
+void get_next_vaarg(va_list *p_va, OutVarPtr *p_outvar) {
+	p_outvar->content = (OutVar *)va_arg(*p_va, void *);
+}
+
+void* graal_hpy_context_to_native(void* cobj) {
+    return truffle_deref_handle_for_managed(cobj);
+}
+
+#define PRIMITIVE_ARRAY_TO_NATIVE(__jtype__, __ctype__, __polyglot_type__, __element_cast__) \
+    void* graal_hpy_##__jtype__##_array_to_native(const void* jarray, int64_t len) { \
+        int64_t i; \
+        int64_t size = len + 1; \
+        __ctype__* carr = (__ctype__*) malloc(size * sizeof(__ctype__)); \
+        carr[len] = (__ctype__)0; \
+        for (i=0; i < len; i++) { \
+            carr[i] = __element_cast__(polyglot_get_array_element(jarray, i)); \
+        } \
+        return polyglot_from_##__polyglot_type__##_array(carr, len); \
+    } \
+
+PRIMITIVE_ARRAY_TO_NATIVE(byte, int8_t, i8, polyglot_as_i8);
+PRIMITIVE_ARRAY_TO_NATIVE(int, int32_t, i32, polyglot_as_i32);
+PRIMITIVE_ARRAY_TO_NATIVE(long, int64_t, i64, polyglot_as_i64);
+PRIMITIVE_ARRAY_TO_NATIVE(double, double, double, polyglot_as_double);
+PRIMITIVE_ARRAY_TO_NATIVE(pointer, VoidPtr, VoidPtr, (VoidPtr));

graalpython/com.oracle.graal.python.cext/include/common/autogen_impl.h

@@ -0,0 +1,143 @@
+/* MIT License
+ *  
+ * Copyright (c) 2020, Oracle and/or its affiliates. 
+ * Copyright (c) 2019 pyhandle
+ *  
+ * Permission is hereby granted, free of charge, to any person obtaining a copy
+ * of this software and associated documentation files (the "Software"), to deal
+ * in the Software without restriction, including without limitation the rights
+ * to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+ * copies of the Software, and to permit persons to whom the Software is
+ * furnished to do so, subject to the following conditions:
+ *  
+ * The above copyright notice and this permission notice shall be included in all
+ * copies or substantial portions of the Software.
+ *  
+ * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+ * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+ * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
+ * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+ * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+ * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
+ * SOFTWARE.
+ */
+
+/*
+   DO NOT EDIT THIS FILE!
+
+   This file is automatically generated by tools/autogen.py from tools/public_api.h.
+   Run this to regenerate:
+       make autogen
+
+*/
+
+HPyAPI_STORAGE HPy _HPy_IMPL_NAME(Long_FromLong)(HPyContext ctx, long value)
+{
+    return _py2h(PyLong_FromLong(value));
+}
+
+HPyAPI_STORAGE HPy _HPy_IMPL_NAME(Long_FromLongLong)(HPyContext ctx, long long v)
+{
+    return _py2h(PyLong_FromLongLong(v));
+}
+
+HPyAPI_STORAGE HPy _HPy_IMPL_NAME(Long_FromUnsignedLongLong)(HPyContext ctx, unsigned long long v)
+{
+    return _py2h(PyLong_FromUnsignedLongLong(v));
+}
+
+HPyAPI_STORAGE long _HPy_IMPL_NAME(Long_AsLong)(HPyContext ctx, HPy h)
+{
+    return PyLong_AsLong(_h2py(h));
+}
+
+HPyAPI_STORAGE HPy _HPy_IMPL_NAME(Float_FromDouble)(HPyContext ctx, double v)
+{
+    return _py2h(PyFloat_FromDouble(v));
+}
+
+HPyAPI_STORAGE HPy _HPy_IMPL_NAME(Number_Add)(HPyContext ctx, HPy h1, HPy h2)
+{
+    return _py2h(PyNumber_Add(_h2py(h1), _h2py(h2)));
+}
+
+HPyAPI_STORAGE void _HPy_IMPL_NAME(Err_SetString)(HPyContext ctx, HPy h_type, const char *message)
+{
+    return PyErr_SetString(_h2py(h_type), message);
+}
+
+HPyAPI_STORAGE int _HPy_IMPL_NAME(Object_IsTrue)(HPyContext ctx, HPy h)
+{
+    return PyObject_IsTrue(_h2py(h));
+}
+
+HPyAPI_STORAGE int _HPy_IMPL_NAME(Bytes_Check)(HPyContext ctx, HPy h)
+{
+    return PyBytes_Check(_h2py(h));
+}
+
+HPyAPI_STORAGE HPy_ssize_t _HPy_IMPL_NAME(Bytes_Size)(HPyContext ctx, HPy h)
+{
+    return PyBytes_Size(_h2py(h));
+}
+
+HPyAPI_STORAGE HPy_ssize_t _HPy_IMPL_NAME(Bytes_GET_SIZE)(HPyContext ctx, HPy h)
+{
+    return PyBytes_GET_SIZE(_h2py(h));
+}
+
+HPyAPI_STORAGE char *_HPy_IMPL_NAME(Bytes_AsString)(HPyContext ctx, HPy h)
+{
+    return PyBytes_AsString(_h2py(h));
+}
+
+HPyAPI_STORAGE char *_HPy_IMPL_NAME(Bytes_AS_STRING)(HPyContext ctx, HPy h)
+{
+    return PyBytes_AS_STRING(_h2py(h));
+}
+
+HPyAPI_STORAGE HPy _HPy_IMPL_NAME(Unicode_FromString)(HPyContext ctx, const char *utf8)
+{
+    return _py2h(PyUnicode_FromString(utf8));
+}
+
+HPyAPI_STORAGE int _HPy_IMPL_NAME(Unicode_Check)(HPyContext ctx, HPy h)
+{
+    return PyUnicode_Check(_h2py(h));
+}
+
+HPyAPI_STORAGE HPy _HPy_IMPL_NAME(Unicode_AsUTF8String)(HPyContext ctx, HPy h)
+{
+    return _py2h(PyUnicode_AsUTF8String(_h2py(h)));
+}
+
+HPyAPI_STORAGE HPy _HPy_IMPL_NAME(Unicode_FromWideChar)(HPyContext ctx, const wchar_t *w, HPy_ssize_t size)
+{
+    return _py2h(PyUnicode_FromWideChar(w, size));
+}
+
+HPyAPI_STORAGE HPy _HPy_IMPL_NAME(List_New)(HPyContext ctx, HPy_ssize_t len)
+{
+    return _py2h(PyList_New(len));
+}
+
+HPyAPI_STORAGE int _HPy_IMPL_NAME(List_Append)(HPyContext ctx, HPy h_list, HPy h_item)
+{
+    return PyList_Append(_h2py(h_list), _h2py(h_item));
+}
+
+HPyAPI_STORAGE HPy _HPy_IMPL_NAME(Dict_New)(HPyContext ctx)
+{
+    return _py2h(PyDict_New());
+}
+
+HPyAPI_STORAGE int _HPy_IMPL_NAME(Dict_SetItem)(HPyContext ctx, HPy h_dict, HPy h_key, HPy h_val)
+{
+    return PyDict_SetItem(_h2py(h_dict), _h2py(h_key), _h2py(h_val));
+}
+
+HPyAPI_STORAGE HPy _HPy_IMPL_NAME(Dict_GetItem)(HPyContext ctx, HPy h_dict, HPy h_key)
+{
+    return _py2h(PyDict_GetItem(_h2py(h_dict), _h2py(h_key)));
+}
+