[GR-26315] Correctly support PyUnicode_New.

PullRequest: graalpython/1295

Author: fangerer

graalpython/com.oracle.graal.python.cext/src/unicodeobject.c

@@ -45,6 +45,8 @@ PyTypeObject PyUnicode_Type = PY_TRUFFLE_TYPE("str", &PyType_Type, Py_TPFLAGS_DE
 /* The empty Unicode object is shared to improve performance. */
 static PyObject *unicode_empty = NULL;
 
+#define MAX_UNICODE 0x10ffff
+
 #define _Py_RETURN_UNICODE_EMPTY()                      \
     do {                                                \
         _Py_INCREF_UNICODE_EMPTY();                     \
@@ -489,8 +491,30 @@ PyObject* PyUnicode_Join(PyObject *separator, PyObject *seq) {
     return UPCALL_CEXT_O(_jls_PyUnicode_Join, native_to_java(separator), native_to_java(seq));
 }
 
+typedef PyObject* (*unicode_new_fun_t)(void* data, int elementSize, int is_ascii);
+UPCALL_TYPED_ID(PyUnicode_New, unicode_new_fun_t);
 PyObject* PyUnicode_New(Py_ssize_t size, Py_UCS4 maxchar) {
-    return to_sulong(polyglot_from_string("", "ascii"));
+	enum PyUnicode_Kind kind;
+	int is_ascii = 0;
+    if (maxchar < 128) {
+        kind = PyUnicode_1BYTE_KIND;
+        is_ascii = 1;
+    } else if (maxchar < 256) {
+        kind = PyUnicode_1BYTE_KIND;
+    } else if (maxchar < 65536) {
+        kind = PyUnicode_2BYTE_KIND;
+    } else {
+        if (maxchar > MAX_UNICODE) {
+            PyErr_SetString(PyExc_SystemError,
+                            "invalid maximum character passed to PyUnicode_New");
+            return NULL;
+        }
+        kind = PyUnicode_4BYTE_KIND;
+    }
+
+    size_t n = size * kind;
+    int8_t* ptr = (int8_t*) malloc(n);
+    return _jls_PyUnicode_New(polyglot_from_i8_array((int8_t*)ptr, n), kind, is_ascii);
 }
 
 UPCALL_ID(PyUnicode_Compare);

graalpython/com.oracle.graal.python/src/com/oracle/graal/python/builtins/modules/PythonCextBuiltins.java

@@ -187,6 +187,7 @@
 import com.oracle.graal.python.builtins.objects.object.PythonObject;
 import com.oracle.graal.python.builtins.objects.object.PythonObjectLibrary;
 import com.oracle.graal.python.builtins.objects.set.PBaseSet;
+import com.oracle.graal.python.builtins.objects.str.NativeCharSequence;
 import com.oracle.graal.python.builtins.objects.str.PString;
 import com.oracle.graal.python.builtins.objects.traceback.GetTracebackNode;
 import com.oracle.graal.python.builtins.objects.traceback.LazyTraceback;
@@ -714,6 +715,17 @@ Object run(PBaseException exception, Object object,
         }
     }
 
+    // directly called without landing function
+    @Builtin(name = "PyUnicode_New", minNumOfPositionalArgs = 3)
+    @GenerateNodeFactory
+    abstract static class PyUnicodeNewNode extends PythonBuiltinNode {
+        @Specialization
+        Object doGeneric(Object ptr, int elementSize, int isAscii,
+                        @Cached CExtNodes.ToNewRefNode toNewRefNode) {
+            return toNewRefNode.execute(factory().createString(new NativeCharSequence(ptr, elementSize, isAscii != 0)));
+        }
+    }
+
     @Builtin(name = "PyUnicode_FromString", minNumOfPositionalArgs = 1)
     @GenerateNodeFactory
     abstract static class PyUnicodeFromStringNode extends PythonBuiltinNode {

graalpython/com.oracle.graal.python/src/com/oracle/graal/python/builtins/objects/cext/CExtNodes.java

@@ -1451,7 +1451,7 @@ public abstract static class FromCharPointerNode extends Node {
         @Specialization
         PString execute(Object charPtr,
                         @Cached PythonObjectFactory factory) {
-            return factory.createString(new NativeCharSequence(charPtr));
+            return factory.createString(new NativeCharSequence(charPtr, 1, false));
         }
     }
 

graalpython/com.oracle.graal.python/src/com/oracle/graal/python/builtins/objects/cext/PyUnicodeWrappers.java

@@ -53,11 +53,15 @@
 import java.nio.charset.CharsetEncoder;
 import java.nio.charset.StandardCharsets;
 
+import com.oracle.graal.python.builtins.objects.cext.CExtNodes.SizeofWCharNode;
 import com.oracle.graal.python.builtins.objects.cext.DynamicObjectNativeWrapper.PAsPointerNode;
 import com.oracle.graal.python.builtins.objects.cext.DynamicObjectNativeWrapper.ToPyObjectNode;
 import com.oracle.graal.python.builtins.objects.cext.UnicodeObjectNodes.UnicodeAsWideCharNode;
+import com.oracle.graal.python.builtins.objects.str.NativeCharSequence;
 import com.oracle.graal.python.builtins.objects.str.PString;
 import com.oracle.graal.python.builtins.objects.str.StringNodes.StringLenNode;
+import com.oracle.graal.python.builtins.objects.str.StringNodes.StringMaterializeNode;
+import com.oracle.truffle.api.CompilerDirectives;
 import com.oracle.truffle.api.CompilerDirectives.CompilationFinal;
 import com.oracle.truffle.api.CompilerDirectives.TruffleBoundary;
 import com.oracle.truffle.api.dsl.Cached;
@@ -67,6 +71,7 @@
 import com.oracle.truffle.api.library.CachedLibrary;
 import com.oracle.truffle.api.library.ExportLibrary;
 import com.oracle.truffle.api.library.ExportMessage;
+import com.oracle.truffle.api.profiles.ConditionProfile;
 import com.oracle.truffle.llvm.spi.NativeTypeLibrary;
 
 public abstract class PyUnicodeWrappers {
@@ -163,6 +168,12 @@ Object readMember(String member,
             if (isMemberReadable(member)) {
                 int elementSize = (int) sizeofWcharNode.execute();
                 PString s = getPString(lib);
+                CharSequence content = s.getCharSequence();
+
+                if (content instanceof NativeCharSequence) {
+                    // in this case, we can just return the pointer
+                    return ((NativeCharSequence) content).getPtr();
+                }
                 return new PySequenceArrayWrapper(asWideCharNode.execute(s, elementSize, stringLenNode.execute(s)), elementSize);
             }
             throw UnknownIdentifierException.create(member);
@@ -209,25 +220,46 @@ boolean isMemberReadable(String member) {
         @ExportMessage
         Object readMember(String member,
                         @CachedLibrary("this") PythonNativeWrapperLibrary lib,
+                        @Cached ConditionProfile storageProfile,
+                        @Cached StringMaterializeNode materializeNode,
                         @Cached CExtNodes.SizeofWCharNode sizeofWcharNode) throws UnknownIdentifierException {
             // padding(24), ready(1), ascii(1), compact(1), kind(3), interned(2)
             int value = 0b000000000000000000000000_1_0_0_000_00;
-            if (onlyAscii(getPString(lib).getValue())) {
+            PString delegate = getPString(lib);
+            if (onlyAscii(delegate, storageProfile, materializeNode)) {
                 value |= 0b1_0_000_00;
             }
-            value |= ((int) sizeofWcharNode.execute() << 2) & 0b11100;
+            value |= (getKind(delegate, storageProfile, sizeofWcharNode) << 2) & 0b11100;
             if (isMemberReadable(member)) {
                 // it's a bit field; so we need to return the whole 32-bit word
                 return value;
             }
             throw UnknownIdentifierException.create(member);
         }
 
-        private boolean onlyAscii(String value) {
+        private boolean onlyAscii(PString value, ConditionProfile storageProfile, StringMaterializeNode stringMaterializeNode) {
+            CharSequence storage = value.getCharSequence();
+
+            // important: avoid materialization of native sequences
+            if (storageProfile.profile(storage instanceof NativeCharSequence)) {
+                return ((NativeCharSequence) storage).isAsciiOnly();
+            }
+
             if (asciiEncoder == null) {
+                CompilerDirectives.transferToInterpreterAndInvalidate();
                 asciiEncoder = newAsciiEncoder();
             }
-            return doCheck(value, asciiEncoder);
+            return doCheck(stringMaterializeNode.execute(value), asciiEncoder);
+        }
+
+        private static int getKind(PString value, ConditionProfile storageProfile, SizeofWCharNode sizeofWcharNode) {
+            CharSequence storage = value.getCharSequence();
+
+            // important: avoid materialization of native sequences
+            if (storageProfile.profile(storage instanceof NativeCharSequence)) {
+                return ((NativeCharSequence) storage).getElementSize();
+            }
+            return (int) sizeofWcharNode.execute();
         }
 
         @TruffleBoundary

graalpython/com.oracle.graal.python/src/com/oracle/graal/python/builtins/objects/cext/UnicodeObjectNodes.java

@@ -1,5 +1,5 @@
 /*
- * Copyright (c) 2018, 2019, Oracle and/or its affiliates. All rights reserved.
+ * Copyright (c) 2018, 2020, Oracle and/or its affiliates. All rights reserved.
  * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER.
  *
  * The Universal Permissive License (UPL), Version 1.0
@@ -49,6 +49,7 @@
 import com.oracle.graal.python.builtins.objects.cext.UnicodeObjectNodesFactory.UnicodeAsWideCharNodeGen.LittleEndianNodeGen;
 import com.oracle.graal.python.builtins.objects.cext.UnicodeObjectNodesFactory.UnicodeAsWideCharNodeGen.NativeOrderNodeGen;
 import com.oracle.graal.python.builtins.objects.str.PString;
+import com.oracle.graal.python.builtins.objects.str.StringNodes.StringMaterializeNode;
 import com.oracle.graal.python.runtime.object.PythonObjectFactory;
 import com.oracle.truffle.api.CompilerDirectives.TruffleBoundary;
 import com.oracle.truffle.api.dsl.Cached;
@@ -129,8 +130,9 @@ public static UnicodeAsWideCharNode getUncachedBigEndian() {
 
         @Specialization
         PBytes doUnicode(PString s, long elementSize, long elements,
+                        @Cached StringMaterializeNode materializeNode,
                         @Shared("factory") @Cached PythonObjectFactory factory) {
-            return doUnicode(s.getValue(), elementSize, elements, factory);
+            return doUnicode(materializeNode.execute(s), elementSize, elements, factory);
         }
 
         @Specialization

graalpython/com.oracle.graal.python/src/com/oracle/graal/python/builtins/objects/str/NativeCharSequence.java

@@ -48,11 +48,29 @@
 
 public final class NativeCharSequence implements PCharSequence {
 
+    /**
+     * Pointer to the native buffer (most like a {@code char*} containing ASCII characters but could
+     * also be an arbitrary {@code void*} for {@code Py_UCS1}, {@code Py_UCS2}, or {@code Py_UCS4}
+     * characters)
+     */
     private final Object ptr;
+
+    /**
+     * The size of a single character in bytes (valid values are {@code 1, 2, 4}).
+     */
+    private final int elementSize;
+
+    /**
+     * Specifies if the native buffer contains only ASCII characters.
+     */
+    private final boolean asciiOnly;
+
     private String materialized;
 
-    public NativeCharSequence(Object ptr) {
+    public NativeCharSequence(Object ptr, int elementSize, boolean asciiOnly) {
         this.ptr = ptr;
+        this.elementSize = elementSize;
+        this.asciiOnly = asciiOnly;
     }
 
     @Override
@@ -90,10 +108,18 @@ public String materialize(PCallCapiFunction node) {
         return materialized;
     }
 
-    Object getPtr() {
+    public Object getPtr() {
         return ptr;
     }
 
+    public int getElementSize() {
+        return elementSize;
+    }
+
+    public boolean isAsciiOnly() {
+        return asciiOnly;
+    }
+
     @Override
     public String toString() {
         CompilerAsserts.neverPartOfCompilation();