[GR-23268] Fixes for test_unicode, part 4

PullRequest: graalpython/1285

Author: msimacek

graalpython/com.oracle.graal.python.test/src/tests/test_codecs.py

@@ -1,4 +1,4 @@
-# Copyright (c) 2019, Oracle and/or its affiliates.
+# Copyright (c) 2019, 2020, Oracle and/or its affiliates.
 # Copyright (C) 1996-2017 Python Software Foundation
 #
 # Licensed under the PYTHON SOFTWARE FOUNDATION LICENSE VERSION 2
@@ -151,8 +151,7 @@ def test_escape_encode(self):
         for b in range(127, 256):
             check(chr(b), ('\\x%02x' % b).encode())
         check('\u20ac', br'\u20ac')
-        # TODO Truffle: not working yet
-        # check('\U0001d120', br'\U0001d120')
+        check('\U0001d120', br'\U0001d120')
 
     def test_escape_decode(self):
         decode = codecs.unicode_escape_decode

graalpython/com.oracle.graal.python.test/src/tests/unittest_tags/test_codeccallbacks.txt

@@ -1,5 +1,5 @@
+*graalpython.lib-python.3.test.test_codeccallbacks.CodecCallbackTest.test_backslashescape
 *graalpython.lib-python.3.test.test_codeccallbacks.CodecCallbackTest.test_badregistercall
 *graalpython.lib-python.3.test.test_codeccallbacks.CodecCallbackTest.test_bug828737
 *graalpython.lib-python.3.test.test_codeccallbacks.CodecCallbackTest.test_translatehelper
-*graalpython.lib-python.3.test.test_codeccallbacks.CodecCallbackTest.test_unencodablereplacement
 *graalpython.lib-python.3.test.test_codeccallbacks.CodecCallbackTest.test_unknownhandler

graalpython/com.oracle.graal.python.test/src/tests/unittest_tags/test_unicode.txt

@@ -22,6 +22,7 @@
 *graalpython.lib-python.3.test.test_unicode.UnicodeTest.test_codecs
 *graalpython.lib-python.3.test.test_unicode.UnicodeTest.test_codecs_idna
 *graalpython.lib-python.3.test.test_unicode.UnicodeTest.test_codecs_utf7
+*graalpython.lib-python.3.test.test_unicode.UnicodeTest.test_codecs_utf8
 *graalpython.lib-python.3.test.test_unicode.UnicodeTest.test_compare
 *graalpython.lib-python.3.test.test_unicode.UnicodeTest.test_comparison
 *graalpython.lib-python.3.test.test_unicode.UnicodeTest.test_concatenation
@@ -108,6 +109,7 @@
 *graalpython.lib-python.3.test.test_unicode.UnicodeTest.test_surrogates
 *graalpython.lib-python.3.test.test_unicode.UnicodeTest.test_swapcase
 *graalpython.lib-python.3.test.test_unicode.UnicodeTest.test_title
+*graalpython.lib-python.3.test.test_unicode.UnicodeTest.test_ucs4
 *graalpython.lib-python.3.test.test_unicode.UnicodeTest.test_unexpected_end_of_data
 *graalpython.lib-python.3.test.test_unicode.UnicodeTest.test_unicode_repr
 *graalpython.lib-python.3.test.test_unicode.UnicodeTest.test_upper

graalpython/com.oracle.graal.python/src/com/oracle/graal/python/builtins/modules/CodecsModuleBuiltins.java

@@ -297,18 +297,24 @@ public static int unicodeEscape(int codePoint, int startIndex, byte[] buffer) {
                 buffer[i++] = 'r';
             } else {
                 /* Map non-printable US ASCII and 8-bit characters to '\xHH' */
-                buffer[i++] = '\\';
-                buffer[i++] = 'x';
-                buffer[i++] = hexdigits[(codePoint >> 4) & 0x000F];
-                buffer[i++] = hexdigits[codePoint & 0x000F];
+                byteEscape(codePoint, i, buffer);
+                i += 4;
             }
         } else {
             i = unicodeNonAsciiEscape(codePoint, i, buffer);
         }
         return i;
     }
 
-    private static int unicodeNonAsciiEscape(int codePoint, int startIndex, byte[] buffer) {
+    public static void byteEscape(int codePoint, int startIndex, byte[] buffer) {
+        int i = startIndex;
+        buffer[i++] = '\\';
+        buffer[i++] = 'x';
+        buffer[i++] = hexdigits[(codePoint >> 4) & 0x000F];
+        buffer[i] = hexdigits[codePoint & 0x000F];
+    }
+
+    public static int unicodeNonAsciiEscape(int codePoint, int startIndex, byte[] buffer) {
         int i = startIndex;
         if (codePoint < 0x100) {
             buffer[i++] = (byte) codePoint;

graalpython/com.oracle.graal.python/src/com/oracle/graal/python/builtins/objects/bytes/BytesUtils.java

@@ -0,0 +1,66 @@
+/*
+ * Copyright (c) 2020, Oracle and/or its affiliates. All rights reserved.
+ * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER.
+ *
+ * The Universal Permissive License (UPL), Version 1.0
+ *
+ * Subject to the condition set forth below, permission is hereby granted to any
+ * person obtaining a copy of this software, associated documentation and/or
+ * data (collectively the "Software"), free of charge and under any and all
+ * copyright rights in the Software, and any and all patent rights owned or
+ * freely licensable by each licensor hereunder covering either (i) the
+ * unmodified Software as contributed to or provided by such licensor, or (ii)
+ * the Larger Works (as defined below), to deal in both
+ *
+ * (a) the Software, and
+ *
+ * (b) any piece of software and/or hardware listed in the lrgrwrks.txt file if
+ * one is included with the Software each a "Larger Work" to which the Software
+ * is contributed by such licensors),
+ *
+ * without restriction, including without limitation the rights to copy, create
+ * derivative works of, display, perform, and distribute the Software and make,
+ * use, sell, offer for sale, import, export, have made, and have sold the
+ * Software and the Larger Work(s), and to sublicense the foregoing rights on
+ * either these or other terms.
+ *
+ * This license is subject to the following condition:
+ *
+ * The above copyright notice and either this complete permission notice or at a
+ * minimum a reference to the UPL must be included in all copies or substantial
+ * portions of the Software.
+ *
+ * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+ * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+ * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
+ * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+ * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+ * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
+ * SOFTWARE.
+ */
+package com.oracle.graal.python.charset;
+
+import java.nio.charset.Charset;
+import java.nio.charset.CharsetDecoder;
+import java.nio.charset.CharsetEncoder;
+
+public class PythonRawUnicodeEscapeCharset extends Charset {
+    public PythonRawUnicodeEscapeCharset() {
+        super("x-python-raw-unicode-escape", new String[0]);
+    }
+
+    @Override
+    public boolean contains(Charset charset) {
+        return false;
+    }
+
+    @Override
+    public CharsetDecoder newDecoder() {
+        return new PythonRawUnicodeEscapeCharsetDecoder(this);
+    }
+
+    @Override
+    public CharsetEncoder newEncoder() {
+        return new PythonRawUnicodeEscapeCharsetEncoder(this);
+    }
+}

graalpython/com.oracle.graal.python/src/com/oracle/graal/python/charset/PythonRawUnicodeEscapeCharset.java

@@ -0,0 +1,106 @@
+/*
+ * Copyright (c) 2020, Oracle and/or its affiliates. All rights reserved.
+ * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER.
+ *
+ * The Universal Permissive License (UPL), Version 1.0
+ *
+ * Subject to the condition set forth below, permission is hereby granted to any
+ * person obtaining a copy of this software, associated documentation and/or
+ * data (collectively the "Software"), free of charge and under any and all
+ * copyright rights in the Software, and any and all patent rights owned or
+ * freely licensable by each licensor hereunder covering either (i) the
+ * unmodified Software as contributed to or provided by such licensor, or (ii)
+ * the Larger Works (as defined below), to deal in both
+ *
+ * (a) the Software, and
+ *
+ * (b) any piece of software and/or hardware listed in the lrgrwrks.txt file if
+ * one is included with the Software each a "Larger Work" to which the Software
+ * is contributed by such licensors),
+ *
+ * without restriction, including without limitation the rights to copy, create
+ * derivative works of, display, perform, and distribute the Software and make,
+ * use, sell, offer for sale, import, export, have made, and have sold the
+ * Software and the Larger Work(s), and to sublicense the foregoing rights on
+ * either these or other terms.
+ *
+ * This license is subject to the following condition:
+ *
+ * The above copyright notice and either this complete permission notice or at a
+ * minimum a reference to the UPL must be included in all copies or substantial
+ * portions of the Software.
+ *
+ * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+ * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+ * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
+ * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+ * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+ * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
+ * SOFTWARE.
+ */
+package com.oracle.graal.python.charset;
+
+import java.nio.ByteBuffer;
+import java.nio.CharBuffer;
+import java.nio.charset.Charset;
+import java.nio.charset.CharsetDecoder;
+import java.nio.charset.CoderResult;
+
+public class PythonRawUnicodeEscapeCharsetDecoder extends CharsetDecoder {
+    private boolean seenBackslash = false;
+
+    protected PythonRawUnicodeEscapeCharsetDecoder(Charset cs) {
+        super(cs, 1, 1);
+    }
+
+    @Override
+    protected CoderResult decodeLoop(ByteBuffer source, CharBuffer target) {
+        while (true) {
+            if (!source.hasRemaining()) {
+                return CoderResult.UNDERFLOW;
+            }
+            if (!target.hasRemaining()) {
+                return CoderResult.OVERFLOW;
+            }
+            int initialPosition = source.position();
+            byte b = source.get();
+            if (seenBackslash) {
+                // Report error from the backslash included
+                initialPosition--;
+                if (b == (byte) 'u' || b == (byte) 'U') {
+                    CoderResult result = PythonUnicodeEscapeCharsetDecoder.decodeHexUnicodeEscape(source, target, b, initialPosition);
+                    if (result != null) {
+                        return result;
+                    }
+                    seenBackslash = false;
+                } else {
+                    target.put('\\');
+                    seenBackslash = false;
+                }
+            } else if (b == (byte) '\\') {
+                seenBackslash = true;
+            } else {
+                // Bytes that are not an escape sequence are latin-1, which maps to unicode
+                // codepoints directly
+                target.put((char) (b & 0xFF));
+            }
+        }
+    }
+
+    @Override
+    protected CoderResult implFlush(CharBuffer target) {
+        if (seenBackslash) {
+            if (!target.hasRemaining()) {
+                return CoderResult.OVERFLOW;
+            }
+            target.put('\\');
+            seenBackslash = false;
+        }
+        return CoderResult.UNDERFLOW;
+    }
+
+    @Override
+    protected void implReset() {
+        seenBackslash = false;
+    }
+}

graalpython/com.oracle.graal.python/src/com/oracle/graal/python/charset/PythonRawUnicodeEscapeCharsetDecoder.java

@@ -0,0 +1,97 @@
+/*
+ * Copyright (c) 2020, Oracle and/or its affiliates. All rights reserved.
+ * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER.
+ *
+ * The Universal Permissive License (UPL), Version 1.0
+ *
+ * Subject to the condition set forth below, permission is hereby granted to any
+ * person obtaining a copy of this software, associated documentation and/or
+ * data (collectively the "Software"), free of charge and under any and all
+ * copyright rights in the Software, and any and all patent rights owned or
+ * freely licensable by each licensor hereunder covering either (i) the
+ * unmodified Software as contributed to or provided by such licensor, or (ii)
+ * the Larger Works (as defined below), to deal in both
+ *
+ * (a) the Software, and
+ *
+ * (b) any piece of software and/or hardware listed in the lrgrwrks.txt file if
+ * one is included with the Software each a "Larger Work" to which the Software
+ * is contributed by such licensors),
+ *
+ * without restriction, including without limitation the rights to copy, create
+ * derivative works of, display, perform, and distribute the Software and make,
+ * use, sell, offer for sale, import, export, have made, and have sold the
+ * Software and the Larger Work(s), and to sublicense the foregoing rights on
+ * either these or other terms.
+ *
+ * This license is subject to the following condition:
+ *
+ * The above copyright notice and either this complete permission notice or at a
+ * minimum a reference to the UPL must be included in all copies or substantial
+ * portions of the Software.
+ *
+ * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+ * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+ * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
+ * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+ * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+ * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
+ * SOFTWARE.
+ */
+package com.oracle.graal.python.charset;
+
+import java.nio.ByteBuffer;
+import java.nio.CharBuffer;
+import java.nio.charset.Charset;
+import java.nio.charset.CharsetEncoder;
+import java.nio.charset.CoderResult;
+
+public class PythonRawUnicodeEscapeCharsetEncoder extends CharsetEncoder {
+    protected PythonRawUnicodeEscapeCharsetEncoder(Charset cs) {
+        super(cs, 2, 10, new byte[]{(byte) '?'});
+    }
+
+    @Override
+    protected CoderResult encodeLoop(CharBuffer source, ByteBuffer target) {
+        while (true) {
+            if (!source.hasRemaining()) {
+                return CoderResult.UNDERFLOW;
+            }
+            if (!target.hasRemaining()) {
+                return CoderResult.OVERFLOW;
+            }
+            int initialPosition = source.position();
+            char ch = source.get();
+            int codePoint = ch;
+            if (Character.isHighSurrogate(ch)) {
+                if (!source.hasRemaining()) {
+                    source.position(initialPosition);
+                    return CoderResult.UNDERFLOW;
+                }
+                char low = source.get();
+                if (Character.isLowSurrogate(low)) {
+                    codePoint = Character.toCodePoint(ch, low);
+                } else {
+                    // Unpaired surrogate, this shouldn't happen in any sanely constructed Java
+                    // String
+                    source.position(source.position() - 2);
+                    return CoderResult.malformedForLength(2);
+                }
+            }
+            if (codePoint <= 0xFF) {
+                // ASCII
+                target.put((byte) codePoint);
+            } else {
+                String hexString = String.format((codePoint <= 0xFFFF ? "\\u%04x" : "\\U%08x"), codePoint);
+                for (int i = 0; i < hexString.length(); i++) {
+                    if (!target.hasRemaining()) {
+                        source.position(initialPosition);
+                        target.position(target.position() - i);
+                        return CoderResult.OVERFLOW;
+                    }
+                    target.put((byte) hexString.charAt(i));
+                }
+            }
+        }
+    }
+}