[GR-34607] Fixes for lxml

PullRequest: graalpython/2070

Author: msimacek

CHANGELOG.md

@@ -3,12 +3,15 @@
 This changelog summarizes major changes between GraalVM versions of the Python
 language runtime. The main focus is on user-observable behavior of the engine.
 
+## Version 22.1.0
+* String conversion (`__str__`) now calls `toString` for Java objects and `toDisplayString` interop message for foreign objects.
+* Improved compatibility with PyPI package `lxml`
+
 ## Version 22.0.0
 * Added support for `pyexpat` module.
 * Added partial support for `PYTHONHASHSEED` environment variable (also available via `HashSeed` context option), currently only affecting hashing in `pyexpat` module.
 * Implement `_csv` module.
 * Improved compatibility with PyPI packages `wheel` and `click`
-* String conversion (`__str__`) now calls `toString` for Java objects and `toDisplayString` interop message for foreign objects.
 
 ## Version 21.3.0
 
@@ -49,7 +52,7 @@ language runtime. The main focus is on user-observable behavior of the engine.
 * Implement name mangling for private attributes
 * Correctly raise an AttributeError when a class defines slots, but not dict
 * Fix infinite continuation prompt in REPL when pasting snippets
-* Add jarray module for compatiblity with Jython
+* Add jarray module for compatibility with Jython
 * Fix multiple memory leaks and crashes when running NumPy in a shared engine
 * Improved support for Pandas
 * Initial support for Matplotlib

docs/user/OsInterface.md

@@ -44,7 +44,7 @@ The java backend is the default when GraalVM Python is run via the `Context` API
 ### Limitations of the emulated backend
 
 GraalVM Python can log info about known incompatibility of functions executed at runtime, which includes the OS interface related functions.
-To turn on this logging, use `--log.python.compatibility=FINE` or other desired logging level.
+To turn on this logging, use `--log.python.compatibility.level=FINE` or other desired logging level.
 
 Known limitations of the emulated layer are:
 

graalpython/com.oracle.graal.python.cext/include/Python.h

@@ -55,6 +55,8 @@
  * supporting this in many cases, it still involves overhead. */
 #define CYTHON_USE_TYPE_SLOTS 0
 #define CYTHON_USE_PYTYPE_LOOKUP 0
+#define CYTHON_UNPACK_METHODS 0
+#define CYTHON_FAST_PYCALL 0
 #define CYTHON_FAST_PYCCALL 0
 #define CYTHON_USE_DICT_VERSIONS 0
 #define CYTHON_AVOID_BORROWED_REFS 1

graalpython/com.oracle.graal.python.cext/src/codeobject.c

@@ -42,6 +42,11 @@
 
 PyTypeObject PyCode_Type = PY_TRUFFLE_TYPE("code", &PyType_Type, Py_TPFLAGS_DEFAULT, sizeof(PyTypeObject));
 
+UPCALL_ID(PyCode_NewEmpty)
+PyCodeObject* PyCode_NewEmpty(const char *filename, const char *funcname, int firstlineno) {
+    return (PyCodeObject*)UPCALL_CEXT_O(_jls_PyCode_NewEmpty, polyglot_from_string(filename, SRC_CS), polyglot_from_string(funcname, SRC_CS), firstlineno);
+}
+
 UPCALL_ID(PyCode_New);
 PyCodeObject* PyCode_New(int argcount, int kwonlyargcount,
                          int nlocals, int stacksize, int flags,

graalpython/com.oracle.graal.python.cext/src/object.c

@@ -337,29 +337,66 @@ int PyObject_AsFileDescriptor(PyObject* obj) {
     return UPCALL_CEXT_I(_jls_PyObject_AsFileDescriptor, native_to_java(obj));
 }
 
-UPCALL_ID(PyTruffle_GetBuiltin);
-int PyObject_Print(PyObject* object, FILE* fd, int flags) {
-    void *openFunc, *args, *kwargs;
-    void *printfunc, *printargs, *printkwargs;
-    void *file;
-
-    openFunc = UPCALL_CEXT_O(_jls_PyTruffle_GetBuiltin, polyglot_from_string("open", SRC_CS));
-    args = PyTuple_New(1);
-    int f = fileno(fd);
-    PyTuple_SetItem(args, 0, PyLong_FromLong(f));
-    kwargs = PyDict_New();
-    int buffering = 1;
-    PyDict_SetItemString(kwargs, "buffering", PyLong_FromLong(buffering));
-    PyDict_SetItemString(kwargs, "mode", polyglot_from_string("w", SRC_CS));
-    file = PyObject_Call(openFunc, args, kwargs);
-
-    printfunc = UPCALL_CEXT_O(_jls_PyTruffle_GetBuiltin, polyglot_from_string("print", SRC_CS));
-    printargs = PyTuple_New(1);
-    PyTuple_SetItem(printargs, 0, object);
-    printkwargs = PyDict_New();
-    PyDict_SetItemString(printkwargs, "file", file);
-    PyObject_Call(printfunc, printargs, printkwargs);
-    return 0;
+// Taken from CPython
+int PyObject_Print(PyObject *op, FILE *fp, int flags)
+{
+    int ret = 0;
+    clearerr(fp); /* Clear any previous error condition */
+    if (op == NULL) {
+        Py_BEGIN_ALLOW_THREADS
+        fprintf(fp, "<nil>");
+        Py_END_ALLOW_THREADS
+    }
+    else {
+        if (op->ob_refcnt <= 0) {
+            /* XXX(twouters) cast refcount to long until %zd is
+               universally available */
+            Py_BEGIN_ALLOW_THREADS
+            fprintf(fp, "<refcnt %ld at %p>",
+                (long)op->ob_refcnt, (void *)op);
+            Py_END_ALLOW_THREADS
+        }
+        else {
+            PyObject *s;
+            if (flags & Py_PRINT_RAW)
+                s = PyObject_Str(op);
+            else
+                s = PyObject_Repr(op);
+            if (s == NULL)
+                ret = -1;
+            else if (PyBytes_Check(s)) {
+                fwrite(PyBytes_AS_STRING(s), 1,
+                       PyBytes_GET_SIZE(s), fp);
+            }
+            else if (PyUnicode_Check(s)) {
+                PyObject *t;
+                t = PyUnicode_AsEncodedString(s, "utf-8", "backslashreplace");
+                if (t == NULL) {
+                    ret = -1;
+                }
+                else {
+                    fwrite(PyBytes_AS_STRING(t), 1,
+                           PyBytes_GET_SIZE(t), fp);
+                    Py_DECREF(t);
+                }
+            }
+            else {
+                PyErr_Format(PyExc_TypeError,
+                             "str() or repr() returned '%.100s'",
+                             s->ob_type->tp_name);
+                ret = -1;
+            }
+            Py_XDECREF(s);
+        }
+    }
+    if (ret == 0) {
+        if (ferror(fp)) {
+            PyErr_SetFromErrno(PyExc_OSError);
+            clearerr(fp);
+            ret = -1;
+        }
+    }
+    return ret;
 }
 
 // taken from CPython "Objects/object.c"

graalpython/com.oracle.graal.python.cext/src/typeobject.c

@@ -343,6 +343,10 @@ static void inherit_slots(PyTypeObject *type, PyTypeObject *base) {
         }
         /* COPYSLOT(tp_call); */
     }
+    {
+        COPYSLOT(tp_iter);
+        COPYSLOT(tp_iternext);
+    }
 
     if ((type->tp_flags & Py_TPFLAGS_HAVE_FINALIZE) &&
         (base->tp_flags & Py_TPFLAGS_HAVE_FINALIZE)) {

graalpython/com.oracle.graal.python.cext/src/unicodeobject.c

@@ -408,7 +408,7 @@ Py_UNICODE* PyUnicode_AsUnicode(PyObject *unicode) {
 
 UPCALL_ID(PyTruffle_Unicode_AsWideChar);
 Py_UNICODE* PyUnicode_AsUnicodeAndSize(PyObject *unicode, Py_ssize_t *size) {
-    PyObject* bytes = UPCALL_CEXT_O(_jls_PyTruffle_Unicode_AsWideChar, native_to_java(unicode), Py_UNICODE_SIZE, native_to_java(Py_None), ERROR_MARKER);
+    PyObject* bytes = UPCALL_CEXT_O(_jls_PyTruffle_Unicode_AsWideChar, native_to_java(unicode), Py_UNICODE_SIZE, ERROR_MARKER);
     if (bytes != NULL) {
         // exclude null terminator at the end
         *size = PyBytes_Size(bytes) / Py_UNICODE_SIZE;
@@ -670,3 +670,8 @@ UPCALL_ID(PyUnicode_Contains)
 int PyUnicode_Contains(PyObject *str, PyObject *substr) {
     return UPCALL_CEXT_I(_jls_PyUnicode_Contains, native_to_java(str), native_to_java(substr));
 }
+
+UPCALL_ID(PyUnicode_Split)
+PyObject* PyUnicode_Split(PyObject *s, PyObject *sep, Py_ssize_t maxsplit) {
+    return UPCALL_CEXT_O(_jls_PyUnicode_Split, native_to_java(s), native_to_java(sep), maxsplit);
+}

graalpython/com.oracle.graal.python.test/src/tests/cpyext/test_codeobject.py

@@ -62,6 +62,17 @@ def compile_module(self, name):
 
     testmod = type(sys)("foo")
 
+    test_PyCode_NewEmpty = CPyExtFunction(
+        lambda args: args,
+        lambda: (
+            ("file.c", "myfunc", 54),
+        ),
+        resultspec="O",
+        argspec="ssi",
+        arguments=["char* filename", "char* funcname", "int firstlineno"],
+        cmpfunc=lambda cr, pr: pr[0] == cr.co_filename and pr[1] == cr.co_name and pr[2] == cr.co_firstlineno,
+    )
+
     test_PyCode_New = CPyExtFunction(
         lambda args: args,
         lambda: (

graalpython/com.oracle.graal.python.test/src/tests/cpyext/test_memoryview.py

@@ -74,6 +74,26 @@
     '''
 
 
+def decode_ucs(mv, kind):
+    b = bytes(mv)
+    if kind == 1:
+        return b.decode('latin-1')
+    elif kind == 2:
+        return b.decode('utf-16')
+    else:
+        assert kind == 4
+        return b.decode('utf-32')
+
+
+def compare_unicode_bytes_with_kind(x, y):
+    if isinstance(x, BaseException) and isinstance(y, BaseException):
+        return type(x) == type(y)
+    elif isinstance(x, BaseException) or isinstance(y, BaseException):
+        return False
+    else:
+        return decode_ucs(*x) == decode_ucs(*y)
+
+
 class TestPyMemoryView(CPyExtTestCase):
     def compile_module(self, name):
         type(self).mro()[1].__dict__["test_%s" % name].create_module(name)
@@ -167,6 +187,54 @@ def compile_module(self, name):
         cmpfunc=unhandled_error_compare_with_message,
     )
 
+    # Test creating memoryview on top of a pointer that is in fact a managed bytes object wrapper
+    test_memoryview_frommemory_bytes_wrapper = CPyExtFunction(
+        lambda args: memoryview(args[0])[args[1]],
+        lambda: (
+            (b"a", 0),
+            (b"abcdefgh", 5),
+            (b"abcdefgh", 8),
+        ),
+        code='''
+            static PyObject* test_frommemory_bytes(PyObject* input, PyObject *key) {
+                PyObject *mv = PyMemoryView_FromMemory(PyBytes_AS_STRING(input), PyBytes_GET_SIZE(input), PyBUF_READ);
+                if (!mv)
+                    return NULL;
+                PyObject *item = PyObject_GetItem(mv, key);
+                Py_DECREF(mv);
+                return item;
+            }
+        ''',
+        resultspec='O',
+        argspec='OO',
+        arguments=["PyObject* input", "PyObject* key"],
+        callfunction="test_frommemory_bytes",
+        cmpfunc=unhandled_error_compare,
+    )
+
+    # Test creating memoryview on top of a pointer that is in fact a managed unicode object wrapper
+    test_memoryview_frommemory_unicode_wrapper = CPyExtFunction(
+        lambda args: (memoryview(args[0].encode('utf-32')), 4),
+        lambda: (
+            ("asdf",),
+            ("ニャー",),
+            # ("😂",),  # TODO doesn't work because PyUnicode_GET_LENGTH returns 2
+        ),
+        code='''
+            static PyObject* test_frommemory_unicode(PyObject* input) {
+                PyObject *mv = PyMemoryView_FromMemory(PyUnicode_DATA(input), PyUnicode_GET_LENGTH(input) * PyUnicode_KIND(input), PyBUF_READ);
+                if (!mv)
+                    return NULL;
+                return Py_BuildValue("Oi", mv, PyUnicode_KIND(input));
+            }
+        ''',
+        resultspec='O',
+        argspec='O',
+        arguments=["PyObject* input"],
+        callfunction="test_frommemory_unicode",
+        cmpfunc=compare_unicode_bytes_with_kind,
+    )
+
     test_memoryview_tolist = CPyExtFunction(
         lambda args: args[0],
         lambda: [

graalpython/com.oracle.graal.python.test/src/tests/cpyext/test_method.py

@@ -36,8 +36,9 @@
 # LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
 # OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
 # SOFTWARE.
+import types
 
-from . import CPyExtType
+from . import CPyExtType, CPyExtTestCase, unhandled_error_compare, CPyExtFunction
 
 __dir__ = __file__.rpartition("/")[0]
 
@@ -101,3 +102,20 @@ def test_methods(self):
         assert obj.meth_static_noargs() == (None, None)
         assert obj.meth_static_varargs(1, 2, 3) == (None, (1, 2, 3))
         assert obj.meth_static_varargs_keywords(1, 2, 3, a=1, b=2) == (None, (1, 2, 3), {'a': 1, 'b': 2})
+
+
+class TestPyMethod(CPyExtTestCase):
+    def compile_module(self, name):
+        type(self).mro()[1].__dict__["test_%s" % name].create_module(name)
+        super().compile_module(name)
+
+    test_PyMethod_New = CPyExtFunction(
+        lambda args: types.MethodType(*args),
+        lambda: (
+            (list.append, 6),
+        ),
+        resultspec="O",
+        argspec='OO',
+        arguments=["PyObject* func", "PyObject* self"],
+        cmpfunc=unhandled_error_compare
+    )