[GR-23568] [GR-26784] [GR-27479] Support running Pandas tests.

PullRequest: graalpython/1374

Author: fangerer

graalpython/com.oracle.graal.python.annotations/src/com/oracle/graal/python/annotations/ArgumentClinic.java

@@ -135,5 +135,10 @@ enum ClinicConversion {
          * Corresponds to CPython's {@code Py_buffer} converter.
          */
         Buffer,
+        /**
+         * Corresponds to CPython's {@code double} converter. Supports {@link #defaultValue()}, and
+         * {@link #useDefaultForNone()}.
+         */
+        Double,
     }
 }

graalpython/com.oracle.graal.python.cext/src/_warnings.c

@@ -72,15 +72,15 @@ int PyErr_WarnEx(PyObject *category, const char *text, Py_ssize_t stack_level) {
 NO_INLINE int PyErr_WarnFormat(PyObject *category, Py_ssize_t stack_level, const char *format, ...) {
     va_list args;
     va_start(args, format);
-    PyObject* result = PyTruffle_Unicode_FromFormat(format, args);
+    PyObject* result = PyUnicode_FromFormatV(format, args);
     va_end(args);
     return warn_unicode(category, result, stack_level, Py_None);
 }
 
 int PyErr_ResourceWarning(PyObject *source, Py_ssize_t stack_level, const char *format, ...) {
     va_list args;
     va_start(args, format);
-    PyObject* result = PyTruffle_Unicode_FromFormat(format, args);
+    PyObject* result = PyUnicode_FromFormatV(format, args);
     va_end(args);
     return warn_unicode(PyExc_ResourceWarning, result, stack_level, source);
 }

graalpython/com.oracle.graal.python.cext/src/capi.h

@@ -382,9 +382,6 @@ extern PyObject* wrapped_null;
 
 /* internal functions to avoid unnecessary managed <-> native conversions */
 
-/* STR */
-__attribute__((always_inline)) PyObject* PyTruffle_Unicode_FromFormat(const char *fmt, va_list va);
-
 /* BYTES, BYTEARRAY */
 int bytes_buffer_getbuffer(PyBytesObject *self, Py_buffer *view, int flags);
 int bytearray_getbuffer(PyByteArrayObject *obj, Py_buffer *view, int flags);

graalpython/com.oracle.graal.python.cext/src/errors.c

@@ -163,7 +163,7 @@ NO_INLINE
 PyObject* PyErr_Format(PyObject* exception, const char* fmt, ...) {
     va_list args;
     va_start(args, fmt);
-    PyObject* formatted_msg = PyTruffle_Unicode_FromFormat(fmt, args);
+    PyObject* formatted_msg = PyUnicode_FromFormatV(fmt, args);
     va_end(args);
     UPCALL_CEXT_VOID(_jls_PyErr_CreateAndSetException, native_to_java(exception), native_to_java(formatted_msg));
     return NULL;

graalpython/com.oracle.graal.python.cext/src/modsupport.c

@@ -96,10 +96,24 @@ static void init_upcall_PyTruffle_Arg_ParseTupleAndKeyword(void) {
     }
 
 /* just a renaming to avoid name clash with Java types */
-typedef char       char_t;
-typedef float      float_t;
-typedef double     double_t;
-
+typedef void*              void_ptr_t;
+typedef char               char_t;
+typedef float              float_t;
+typedef double             double_t;
+typedef int                int_t;
+typedef unsigned int       uint_t;
+typedef long               long_t;
+typedef unsigned long      ulong_t;
+typedef long long          longlong_t;
+typedef unsigned long long ulonglong_t;
+
+REGISTER_BASIC_TYPE(void_ptr_t);
+REGISTER_BASIC_TYPE(int_t);
+REGISTER_BASIC_TYPE(uint_t);
+REGISTER_BASIC_TYPE(long_t);
+REGISTER_BASIC_TYPE(ulong_t);
+REGISTER_BASIC_TYPE(longlong_t);
+REGISTER_BASIC_TYPE(ulonglong_t);
 REGISTER_BASIC_TYPE(int64_t);
 REGISTER_BASIC_TYPE(int32_t);
 REGISTER_BASIC_TYPE(int16_t);
@@ -114,6 +128,7 @@ REGISTER_BASIC_TYPE(PyObject);
 REGISTER_BASIC_TYPE(float_t);
 REGISTER_BASIC_TYPE(double_t);
 REGISTER_BASIC_TYPE(Py_ssize_t);
+REGISTER_BASIC_TYPE(size_t);
 
 /* For pointers, make them look like an array of size 1 such that it is
    possible to dereference the pointer by accessing element 0. */

graalpython/com.oracle.graal.python.cext/src/unicodeobject.c

@@ -230,97 +230,17 @@ PyObject * PyUnicode_FromStringAndSize(const char *u, Py_ssize_t size) {
     return to_sulong(polyglot_from_string_n(u, size, SRC_CS));
 }
 
-#define AS_I64(__arg__) (polyglot_fits_in_i64((__arg__)) ? polyglot_as_i64((__arg__)) : (int64_t)(__arg__))
-
-MUST_INLINE PyObject* PyTruffle_Unicode_FromFormat(const char *fmt, va_list va) {
-    size_t fmt_size = strlen(fmt) + 1;
-    char* fmtcpy = strdup(fmt);
-    char* c = fmtcpy;
-
-    int remaining_space = 2047;
-    char* buffer = (char*)calloc(sizeof(char), remaining_space + 1);
-    char* full_buffer = buffer;
-
-    void *variable = NULL;
-    char *allocated = NULL; // points to the same as variable, if it has to be free'd
-
-    while (c[0]) {
-        if (c[0] == '%') {
-            // we've reached the next directive, write until here
-            c[0] = '\0';
-            int bytes_written;
-            if (variable != NULL) {
-                bytes_written = snprintf(buffer, remaining_space, fmtcpy, AS_I64(variable));
-                if (allocated != NULL) {
-                    free(allocated);
-                    allocated = NULL;
-                }
-                variable = NULL;
-            } else {
-                bytes_written = vsnprintf(buffer, remaining_space, fmtcpy, va);
-            }
-            remaining_space -= bytes_written;
-            buffer += bytes_written;
-            fmtcpy = c;
-            c[0] = '%';
-
-            // now decide if we need to do something special with this directive
-            PyObject* (*converter)(PyObject*) = NULL;
-            switch (c[1]) {
-            case 'A':
-                // The conversion cases, these all use a function to convert the
-                // PyObject* to a char* and they fall through
-                converter = PyObject_ASCII;
-            case 'U':
-                if (converter == NULL) converter = PyObject_Str;
-            case 'S':
-                if (converter == NULL) converter = PyObject_Str;
-            case 'R':
-                if (converter == NULL) converter = PyObject_Repr;
-                c[1] = 's';
-                allocated = variable = as_char_pointer(converter(va_arg(va, PyObject*)));
-                break;
-            case 'd':
-            	break;
-            case '%':
-                // literal %
-                break;
-            case 'c':
-                // This case should just treat it's argument as an integer
-                c[1] = 'd';
-            default:
-                variable = va_arg(va, PyObject*);
-            }
-            // skip over next char, we checked it
-            c += 1;
-        }
-        c += 1;
-    }
-
-    // write the remaining buffer
-    if (variable != NULL) {
-        snprintf(buffer, remaining_space, fmtcpy, AS_I64(variable));
-        if (allocated) {
-            free(allocated);
-        }
-    } else {
-        vsnprintf(buffer, remaining_space, fmtcpy, va);
-    }
-
-    PyObject* result = PyUnicode_FromString(full_buffer);
-    free(full_buffer);
-    return result;
-}
-
+typedef PyObject* (*unicode_fromformat_fun_t)(void* jstr, va_list va);
+UPCALL_TYPED_ID(PyTruffle_Unicode_FromFormat, unicode_fromformat_fun_t);
 PyObject* PyUnicode_FromFormatV(const char* format, va_list va) {
-    return PyTruffle_Unicode_FromFormat(format, va);
+    return _jls_PyTruffle_Unicode_FromFormat(polyglot_from_string(format, "ascii"), va);
 }
 
 NO_INLINE
 PyObject* PyUnicode_FromFormat(const char* format, ...) {
     va_list args;
     va_start(args, format);
-    PyObject* result = PyTruffle_Unicode_FromFormat(format, args);
+    PyObject* result = _jls_PyTruffle_Unicode_FromFormat(polyglot_from_string(format, "ascii"), args);
     va_end(args);
     return result;
 }

graalpython/com.oracle.graal.python.processor/src/com/oracle/graal/python/processor/ConverterFactory.java

@@ -96,6 +96,7 @@ public enum Param {
     private static ConverterFactory BuiltinIndex;
     private static ConverterFactory BuiltinSliceIndex;
     private static ConverterFactory BuiltinNone;
+    private static ConverterFactory BuiltinDouble;
 
     public final String fullClassName;
     public final String className;
@@ -121,6 +122,8 @@ public static ConverterFactory getBuiltin(ArgumentClinic annotation) {
                 return annotation.defaultValue().isEmpty() ? BuiltinString : BuiltinStringWithDefaultValue;
             case Int:
                 return BuiltinInt;
+            case Double:
+                return BuiltinDouble;
             case CodePoint:
                 return BuiltinCodePoint;
             case Buffer:
@@ -197,6 +200,7 @@ public static void initBuiltins(Elements elementUtils) throws ProcessingError {
         BuiltinString = forBuiltin(elementUtils, "JavaStringConverterNode");
         BuiltinStringWithDefaultValue = forBuiltin(elementUtils, "JavaStringConverterWithDefaultValueNode");
         BuiltinInt = forBuiltin(elementUtils, "JavaIntConversionNode");
+        BuiltinDouble = forBuiltin(elementUtils, "JavaDoubleConversionNode");
         BuiltinCodePoint = forBuiltin(elementUtils, "CodePointConversionNode");
         BuiltinBuffer = forBuiltin(elementUtils, "BufferConversionNode");
         BuiltinIndex = forBuiltin(elementUtils, "IndexConversionNode");

graalpython/com.oracle.graal.python.test/src/tests/cpyext/test_err.py

@@ -198,6 +198,20 @@ def compile_module(self, name):
         cmpfunc=unhandled_error_compare
     )
 
+    test_PyErr_Format_dS = CPyExtFunctionVoid(
+        _reference_format,
+        lambda: (
+            (ValueError, "hello %d times %S", 10, "world"),
+            (ValueError, "hello %c times %R", 95, "world"),
+        ),
+        resultspec="O",
+        argspec='OsiO',
+        arguments=["PyObject* v", "char* msg", "int arg0", "PyObject* arg1"],
+        resultval="NULL",
+        callfunction="PyErr_Format",
+        cmpfunc=unhandled_error_compare
+    )
+
     test_PyErr_PrintEx = CPyExtFunction(
         lambda args: None,
         lambda: (

graalpython/com.oracle.graal.python.test/src/tests/cpyext/test_unicode.py

@@ -75,6 +75,15 @@ def _reference_unicode_escape(args):
     return _codecs.unicode_escape_encode(args[0])[0]
 
 
+def _reference_fromformat(args):
+    fmt = args[0]
+    fmt_args = args[1:]
+    # replace specifiers
+    for s in ["%ld", "%li", "%lu", "%lld", "%lli", "%llu"]:
+        fmt = fmt.replace(s, "%d")
+    return fmt % fmt_args
+
+
 class CustomString(str):
     pass
 
@@ -128,7 +137,7 @@ def compile_module(self, name):
     )
 
     test_PyUnicode_FromFormat0 = CPyExtFunction(
-        lambda args: args[0] % tuple(args[1:]),
+        _reference_fromformat,
         lambda: (
             ("hello, world!",),
         ),
@@ -143,19 +152,17 @@ def compile_module(self, name):
         cmpfunc=unhandled_error_compare
     )
 
-    test_PyUnicode_FromFormat3 = CPyExtFunction(
-        lambda args: args[0] % tuple(args[1:]),
+    test_PyUnicode_FromFormat4 = CPyExtFunction(
+        _reference_fromformat,
         lambda: (
-            ("word0: %s; word1: %s; int: %d", "hello", "world", 1234),
+            ("word0: %s; word1: %s; int: %d; long long: %lld", "hello", "world", 1234, 1234),
+            ("word0: %s; word1: %s; int: %d; long long: %lld", "hello", "world", 1234, (1<<44)+123),
         ),
-        code="""PyObject* wrap_PyUnicode_FromFormat3(char* fmt, char* arg0, char* arg1, int n) {
-            return PyUnicode_FromFormat(fmt, arg0, arg1, n);
-        }
-        """,
+        code="typedef long long longlong_t;",
         resultspec="O",
-        argspec='sssi',
-        arguments=["char* fmt", "char* arg0", "char* arg1", "int n"],
-        callfunction="wrap_PyUnicode_FromFormat3",
+        argspec='sssiL',
+        arguments=["char* fmt", "char* arg0", "char* arg1", "int n", "longlong_t l"],
+        callfunction="PyUnicode_FromFormat",
         cmpfunc=unhandled_error_compare
     )
 

graalpython/com.oracle.graal.python.test/src/tests/unittest_tags/test_webbrowser.txt

@@ -19,7 +19,6 @@
 *graalpython.lib-python.3.test.test_webbrowser.ImportTest.test_environment_preferred
 *graalpython.lib-python.3.test.test_webbrowser.ImportTest.test_get
 *graalpython.lib-python.3.test.test_webbrowser.ImportTest.test_register
-*graalpython.lib-python.3.test.test_webbrowser.ImportTest.test_synthesize
 *graalpython.lib-python.3.test.test_webbrowser.MozillaCommandTest.test_open
 *graalpython.lib-python.3.test.test_webbrowser.MozillaCommandTest.test_open_new
 *graalpython.lib-python.3.test.test_webbrowser.MozillaCommandTest.test_open_new_tab