Use native char sequence for PyUnicode_New

Author: fangerer

graalpython/com.oracle.graal.python.cext/src/unicodeobject.c

@@ -45,6 +45,8 @@ PyTypeObject PyUnicode_Type = PY_TRUFFLE_TYPE("str", &PyType_Type, Py_TPFLAGS_DE
 /* The empty Unicode object is shared to improve performance. */
 static PyObject *unicode_empty = NULL;
 
+#define MAX_UNICODE 0x10ffff
+
 #define _Py_RETURN_UNICODE_EMPTY()                      \
     do {                                                \
         _Py_INCREF_UNICODE_EMPTY();                     \
@@ -489,8 +491,30 @@ PyObject* PyUnicode_Join(PyObject *separator, PyObject *seq) {
     return UPCALL_CEXT_O(_jls_PyUnicode_Join, native_to_java(separator), native_to_java(seq));
 }
 
+typedef PyObject* (*unicode_new_fun_t)(void* data, int elementSize, int is_ascii);
+UPCALL_TYPED_ID(PyUnicode_New, unicode_new_fun_t);
 PyObject* PyUnicode_New(Py_ssize_t size, Py_UCS4 maxchar) {
-    return to_sulong(polyglot_from_string("", "ascii"));
+	enum PyUnicode_Kind kind;
+	int is_ascii = 0;
+    if (maxchar < 128) {
+        kind = PyUnicode_1BYTE_KIND;
+        is_ascii = 1;
+    } else if (maxchar < 256) {
+        kind = PyUnicode_1BYTE_KIND;
+    } else if (maxchar < 65536) {
+        kind = PyUnicode_2BYTE_KIND;
+    } else {
+        if (maxchar > MAX_UNICODE) {
+            PyErr_SetString(PyExc_SystemError,
+                            "invalid maximum character passed to PyUnicode_New");
+            return NULL;
+        }
+        kind = PyUnicode_4BYTE_KIND;
+    }
+
+    size_t n = size * kind;
+    int8_t* ptr = (int8_t*) malloc(n);
+    return _jls_PyUnicode_New(polyglot_from_i8_array((int8_t*)ptr, n), kind, is_ascii);
 }
 
 UPCALL_ID(PyUnicode_Compare);

graalpython/com.oracle.graal.python/src/com/oracle/graal/python/builtins/modules/PythonCextBuiltins.java

@@ -187,6 +187,7 @@
 import com.oracle.graal.python.builtins.objects.object.PythonObject;
 import com.oracle.graal.python.builtins.objects.object.PythonObjectLibrary;
 import com.oracle.graal.python.builtins.objects.set.PBaseSet;
+import com.oracle.graal.python.builtins.objects.str.NativeCharSequence;
 import com.oracle.graal.python.builtins.objects.str.PString;
 import com.oracle.graal.python.builtins.objects.traceback.GetTracebackNode;
 import com.oracle.graal.python.builtins.objects.traceback.LazyTraceback;
@@ -714,6 +715,17 @@ Object run(PBaseException exception, Object object,
         }
     }
 
+    // directly called without landing function
+    @Builtin(name = "PyUnicode_New", minNumOfPositionalArgs = 3)
+    @GenerateNodeFactory
+    abstract static class PyUnicodeNewNode extends PythonBuiltinNode {
+        @Specialization
+        Object doGeneric(Object ptr, int elementSize, int isAscii,
+                        @Cached CExtNodes.ToNewRefNode toNewRefNode) {
+            return toNewRefNode.execute(factory().createString(new NativeCharSequence(ptr, elementSize, isAscii != 0)));
+        }
+    }
+
     @Builtin(name = "PyUnicode_FromString", minNumOfPositionalArgs = 1)
     @GenerateNodeFactory
     abstract static class PyUnicodeFromStringNode extends PythonBuiltinNode {

graalpython/com.oracle.graal.python/src/com/oracle/graal/python/builtins/objects/cext/CExtNodes.java

@@ -1451,7 +1451,7 @@ public abstract static class FromCharPointerNode extends Node {
         @Specialization
         PString execute(Object charPtr,
                         @Cached PythonObjectFactory factory) {
-            return factory.createString(new NativeCharSequence(charPtr));
+            return factory.createString(new NativeCharSequence(charPtr, 1, false));
         }
     }
 

graalpython/com.oracle.graal.python/src/com/oracle/graal/python/builtins/objects/str/NativeCharSequence.java

@@ -48,11 +48,29 @@
 
 public final class NativeCharSequence implements PCharSequence {
 
+    /**
+     * Pointer to the native buffer (most like a {@code char*} containing ASCII characters but could
+     * also be an arbitrary {@code void*} for {@code Py_UCS1}, {@code Py_UCS2}, or {@code Py_UCS4}
+     * characters)
+     */
     private final Object ptr;
+
+    /**
+     * The size of a single character in bytes (valid values are {@code 1, 2, 4}).
+     */
+    private final int elementSize;
+
+    /**
+     * Specifies if the native buffer contains only ASCII characters.
+     */
+    private final boolean asciiOnly;
+
     private String materialized;
 
-    public NativeCharSequence(Object ptr) {
+    public NativeCharSequence(Object ptr, int elementSize, boolean asciiOnly) {
         this.ptr = ptr;
+        this.elementSize = elementSize;
+        this.asciiOnly = asciiOnly;
     }
 
     @Override
@@ -90,10 +108,18 @@ public String materialize(PCallCapiFunction node) {
         return materialized;
     }
 
-    Object getPtr() {
+    public Object getPtr() {
         return ptr;
     }
 
+    public int getElementSize() {
+        return elementSize;
+    }
+
+    public boolean isAsciiOnly() {
+        return asciiOnly;
+    }
+
     @Override
     public String toString() {
         CompilerAsserts.neverPartOfCompilation();