decode bytes to string through the parser

Author: timfel

graalpython/com.oracle.graal.python.test/src/tests/test_codecs.py

@@ -4,6 +4,12 @@
 # Licensed under the PYTHON SOFTWARE FOUNDATION LICENSE VERSION 2
 
 
+def coding_checker(self, coder):
+    def check(input, expect):
+        self.assertEqual(coder(input), (expect, len(input)))
+    return check
+
+
 def assert_raises(err, fn, *args, **kwargs):
     raised = False
     try:
@@ -113,3 +119,85 @@ def test_constructorx(self):
 
     def test_encodex(self):
         self.assertEqual(codecs.encode('abc', codecname), list('abc'))
+
+
+class UnicodeEscapeTest(unittest.TestCase):
+    def test_empty(self):
+        self.assertEqual(codecs.unicode_escape_encode(""), (b"", 0))
+        self.assertEqual(codecs.unicode_escape_decode(b""), ("", 0))
+
+    def test_raw_encode(self):
+        encode = codecs.unicode_escape_encode
+        for b in range(32, 127):
+            if b != b'\\'[0]:
+                self.assertEqual(encode(chr(b)), (bytes([b]), 1))
+
+    def test_escape_encode(self):
+        encode = codecs.unicode_escape_encode
+        check = coding_checker(self, encode)
+        check('\t', br'\t')
+        check('\n', br'\n')
+        check('\r', br'\r')
+        check('\\', br'\\')
+        for b in range(32):
+            if chr(b) not in '\t\n\r':
+                check(chr(b), ('\\x%02x' % b).encode())
+        for b in range(127, 256):
+            check(chr(b), ('\\x%02x' % b).encode())
+        check('\u20ac', br'\u20ac')
+        check('\U0001d120', br'\U0001d120')
+
+    def test_escape_decode(self):
+        decode = codecs.unicode_escape_decode
+        check = coding_checker(self, decode)
+        check(b"[\\\n]", "[]")
+        check(br'[\"]', '["]')
+        check(br"[\']", "[']")
+        check(br"[\\]", r"[\]")
+        check(br"[\a]", "[\x07]")
+        check(br"[\b]", "[\x08]")
+        check(br"[\t]", "[\x09]")
+        check(br"[\n]", "[\x0a]")
+        check(br"[\v]", "[\x0b]")
+        check(br"[\f]", "[\x0c]")
+        check(br"[\r]", "[\x0d]")
+        check(br"[\7]", "[\x07]")
+        check(br"[\78]", "[\x078]")
+        check(br"[\41]", "[!]")
+        check(br"[\418]", "[!8]")
+        check(br"[\101]", "[A]")
+        check(br"[\1010]", "[A0]")
+        check(br"[\x41]", "[A]")
+        check(br"[\x410]", "[A0]")
+        check(br"\u20ac", "\u20ac")
+        check(br"\U0001d120", "\U0001d120")
+        for i in range(97, 123):
+            b = bytes([i])
+            if b not in b'abfnrtuvx':
+                with self.assertWarns(DeprecationWarning):
+                    check(b"\\" + b, "\\" + chr(i))
+            if b.upper() not in b'UN':
+                with self.assertWarns(DeprecationWarning):
+                    check(b"\\" + b.upper(), "\\" + chr(i-32))
+        with self.assertWarns(DeprecationWarning):
+            check(br"\8", "\\8")
+        with self.assertWarns(DeprecationWarning):
+            check(br"\9", "\\9")
+        with self.assertWarns(DeprecationWarning):
+            check(b"\\\xfa", "\\\xfa")
+
+    def test_decode_errors(self):
+        decode = codecs.unicode_escape_decode
+        for c, d in (b'x', 2), (b'u', 4), (b'U', 4):
+            for i in range(d):
+                self.assertRaises(UnicodeDecodeError, decode,
+                                  b"\\" + c + b"0"*i)
+                self.assertRaises(UnicodeDecodeError, decode,
+                                  b"[\\" + c + b"0"*i + b"]")
+                data = b"[\\" + c + b"0"*i + b"]\\" + c + b"0"*i
+                self.assertEqual(decode(data, "ignore"), ("[]", len(data)))
+                self.assertEqual(decode(data, "replace"),
+                                 ("[\ufffd]\ufffd", len(data)))
+        self.assertRaises(UnicodeDecodeError, decode, br"\U00110000")
+        self.assertEqual(decode(br"\U00110000", "ignore"), ("", 10))
+        self.assertEqual(decode(br"\U00110000", "replace"), ("\ufffd", 10))

graalpython/com.oracle.graal.python/src/com/oracle/graal/python/builtins/modules/CodecsModuleBuiltins.java

@@ -58,12 +58,14 @@
 import com.oracle.graal.python.builtins.CoreFunctions;
 import com.oracle.graal.python.builtins.PythonBuiltins;
 import com.oracle.graal.python.builtins.objects.PNone;
+import com.oracle.graal.python.builtins.objects.bytes.BytesNodes;
 import com.oracle.graal.python.builtins.objects.bytes.PBytes;
 import com.oracle.graal.python.builtins.objects.bytes.PIBytesLike;
 import com.oracle.graal.python.builtins.objects.common.SequenceStorageNodes;
 import com.oracle.graal.python.builtins.objects.tuple.PTuple;
 import com.oracle.graal.python.nodes.function.PythonBuiltinBaseNode;
 import com.oracle.graal.python.nodes.function.PythonBuiltinNode;
+import com.oracle.graal.python.runtime.PythonCore;
 import com.oracle.truffle.api.CompilerDirectives;
 import com.oracle.truffle.api.CompilerDirectives.TruffleBoundary;
 import com.oracle.truffle.api.dsl.Cached;
@@ -251,6 +253,26 @@ protected static CodingErrorAction convertCodingErrorAction(String errors) {
         }
     }
 
+    @Builtin(name = "unicode_escape_decode", fixedNumOfPositionalArgs = 1, keywordArguments = {"errors"})
+    @GenerateNodeFactory
+    abstract static class UnicodeEscapeDecode extends PythonBuiltinNode {
+        @Specialization(guards = "isBytes(bytes)")
+        Object encode(Object bytes, @SuppressWarnings("unused") PNone errors,
+                        @Cached("create()") BytesNodes.ToBytesNode toBytes) {
+            // this is basically just parsing as a String
+            PythonCore core = getCore();
+            byte[] byteArray = toBytes.execute(bytes);
+            String string = strFromBytes(byteArray);
+            String unescapedString = core.getParser().unescapeJavaString(string);
+            return factory().createTuple(new Object[]{unescapedString, byteArray.length});
+        }
+
+        @TruffleBoundary
+        private static String strFromBytes(byte[] execute) {
+            return new String(execute);
+        }
+    }
+
     // _codecs.encode(obj, encoding='utf-8', errors='strict')
     @Builtin(name = "__truffle_encode", fixedNumOfPositionalArgs = 1, keywordArguments = {"encoding", "errors"})
     @GenerateNodeFactory

graalpython/com.oracle.graal.python/src/com/oracle/graal/python/parser/PythonParserImpl.java

@@ -114,6 +114,12 @@ public boolean isIdentifier(PythonCore core, String snippet) {
         return input.NAME() != null;
     }
 
+    @Override
+    @TruffleBoundary
+    public String unescapeJavaString(String str) {
+        return PythonTreeTranslator.unescapeJavaString(str);
+    }
+
     private static PException handleParserError(ParserErrorCallback errors, Source source, Exception e) {
         SourceSection section = PythonErrorStrategy.getPosition(source, e);
         throw errors.raiseInvalidSyntax(source, section);

graalpython/com.oracle.graal.python/src/com/oracle/graal/python/parser/PythonTreeTranslator.java

@@ -658,7 +658,7 @@ private PNode parseString(String[] strings) {
         }
     }
 
-    private static String unescapeJavaString(String st) {
+    public static String unescapeJavaString(String st) {
         if (st.indexOf("\\") == -1) {
             return st;
         }

graalpython/com.oracle.graal.python/src/com/oracle/graal/python/runtime/PythonParser.java

@@ -93,6 +93,11 @@ default RuntimeException raiseInvalidSyntax(Source source, SourceSection section
      */
     boolean isIdentifier(PythonCore core, String snippet);
 
+    /**
+     * Unescape Python escapes from a Java string
+     */
+    public abstract String unescapeJavaString(String str);
+
     /**
      * Runtime exception used to indicate incomplete source code during parsing.
      */

graalpython/lib-graalpython/_codecs.py

@@ -252,11 +252,6 @@ def unicode_escape_encode(string, errors=None):
     return __truffle_encode(repr(string)[1:-1], "latin-1", errors)
 
 
-@__builtin__
-def unicode_escape_decode(string, errors=None):
-    raise NotImplementedError("unicode_escape_decode")
-
-
 @__builtin__
 def unicode_internal_encode(obj, errors=None):
     raise NotImplementedError("unicode_internal_encode")