[GR-18284] Escaping letters in string literals using name in Unicode database.

ppisl · ppisl · commit afafe5a5997f · 2020-06-10T16:21:56.000Z
PullRequest: graalpython/1012
diff --git a/CHANGELOG.md b/CHANGELOG.md
@@ -3,6 +3,10 @@
 This changelog summarizes major changes between GraalVM versions of the Python
 language runtime. The main focus is on user-observable behavior of the engine.
 
+## Version 20.2
+
+* Escaping Unicode characters using the character names in strings like "\N{GREEK CAPITAL LETTER DELTA}".
+
 ## Version 20.1.0
 
 * Update language support target and standard library to 3.8.2
diff --git a/THIRD_PARTY_LICENSE.txt b/THIRD_PARTY_LICENSE.txt
diff --git a/graalpython/com.oracle.graal.python.test/src/com/oracle/graal/python/test/parser/StringUtilsTests.java b/graalpython/com.oracle.graal.python.test/src/com/oracle/graal/python/test/parser/StringUtilsTests.java
@@ -0,0 +1,103 @@
+/*
+ * Copyright (c) 2020, Oracle and/or its affiliates. All rights reserved.
+ * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER.
+ *
+ * The Universal Permissive License (UPL), Version 1.0
+ *
+ * Subject to the condition set forth below, permission is hereby granted to any
+ * person obtaining a copy of this software, associated documentation and/or
+ * data (collectively the "Software"), free of charge and under any and all
+ * copyright rights in the Software, and any and all patent rights owned or
+ * freely licensable by each licensor hereunder covering either (i) the
+ * unmodified Software as contributed to or provided by such licensor, or (ii)
+ * the Larger Works (as defined below), to deal in both
+ *
+ * (a) the Software, and
+ *
+ * (b) any piece of software and/or hardware listed in the lrgrwrks.txt file if
+ * one is included with the Software each a "Larger Work" to which the Software
+ * is contributed by such licensors),
+ *
+ * without restriction, including without limitation the rights to copy, create
+ * derivative works of, display, perform, and distribute the Software and make,
+ * use, sell, offer for sale, import, export, have made, and have sold the
+ * Software and the Larger Work(s), and to sublicense the foregoing rights on
+ * either these or other terms.
+ *
+ * This license is subject to the following condition:
+ *
+ * The above copyright notice and either this complete permission notice or at a
+ * minimum a reference to the UPL must be included in all copies or substantial
+ * portions of the Software.
+ *
+ * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+ * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+ * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
+ * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+ * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+ * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
+ * SOFTWARE.
+ */
+package com.oracle.graal.python.test.parser;
+
+import com.oracle.graal.python.parser.sst.StringUtils;
+import org.junit.Assert;
+import org.junit.Test;
+
+public class StringUtilsTests extends ParserTestBase {
+    @Test
+    public void unicodeCharNameBasic() throws Exception {
+        Assert.assertEquals("Δ", StringUtils.unescapeJavaString("\\N{GREEK CAPITAL LETTER DELTA}"));
+        Assert.assertEquals("A", StringUtils.unescapeJavaString("\\N{LATIN CAPITAL LETTER A}"));
+        Assert.assertEquals("A", StringUtils.unescapeJavaString("\\N{LATIN CAPITAL LETTER a}"));
+        Assert.assertEquals("A", StringUtils.unescapeJavaString("\\N{LATIN CAPITAL LETTEr a}"));
+        Assert.assertEquals("A", StringUtils.unescapeJavaString("\\N{latin capital letter a}"));
+        Assert.assertEquals("AHOJ", StringUtils.unescapeJavaString("A\\N{LATIN CAPITAL LETTER H}OJ"));
+        Assert.assertEquals("AHOJ", StringUtils.unescapeJavaString("\\N{LATIN CAPITAL LETTER A}\\N{LATIN CAPITAL LETTER H}\\N{LATIN CAPITAL LETTER O}\\N{LATIN CAPITAL LETTER J}"));
+        checkUnknownChar("ahoj");
+    }
+
+    @Test
+    public void blockHangulSyllables() throws Exception {
+        Assert.assertEquals("가", StringUtils.unescapeJavaString("\\N{HANGUL SYLLABLE GA}"));
+        Assert.assertEquals("돐", StringUtils.unescapeJavaString("\\N{HANGUL SYLLABLE DOLS}"));
+        Assert.assertEquals("똜", StringUtils.unescapeJavaString("\\N{HANGUL SYLLABLE DDOLS}"));
+    }
+
+    @Test
+    public void blockCjkUnifiedIdeograph() throws Exception {
+        Assert.assertEquals("㐀", StringUtils.unescapeJavaString("\\N{CJK Unified Ideograph-3400}"));
+        Assert.assertEquals("𫝜", StringUtils.unescapeJavaString("\\N{CJK Unified Ideograph-2B75C}"));
+        Assert.assertEquals("丳", StringUtils.unescapeJavaString("\\N{CJK Unified Ideograph-4E33}"));
+    }
+
+    @Test
+    public void blockCjkUnifiedIdeographUnknownCharacters() throws Exception {
+        checkUnknownChar("CJK Unified Ideograph-33FF");
+        checkUnknownChar("CJK Unified Ideograph-4DC0");
+        checkUnknownChar("CJK Unified Ideograph-4DFF");
+        checkUnknownChar("CJK Unified Ideograph-33FF");
+        checkUnknownChar("CJK Unified Ideograph-2A6E0");
+    }
+
+    @Test
+    public void malformedError() throws Exception {
+        checkSyntaxErrorMessage("'\\N {LATIN CAPITAL LETTER A}'", "SyntaxError: (unicode error) 'unicodeescape' codec can't decode bytes in position 0-1: malformed \\N character escape");
+        checkSyntaxErrorMessage("'\\N LATIN CAPITAL LETTER A}'", "SyntaxError: (unicode error) 'unicodeescape' codec can't decode bytes in position 0-1: malformed \\N character escape");
+        checkSyntaxErrorMessage("'\\N{LATIN CAPITAL LETTER A'", "SyntaxError: (unicode error) 'unicodeescape' codec can't decode bytes in position 0-24: malformed \\N character escape");
+        checkSyntaxErrorMessage("'\\N{LATIN CAPITAL LETTER A \\N{LATIN CAPITAL LETTER B}'",
+                        "SyntaxError: (unicode error) 'unicodeescape' codec can't decode bytes in position 0-51: unknown Unicode character name");
+    }
+
+    @Test
+    public void doNotTrimNames() throws Exception {
+        checkUnknownChar("LATIN CAPITAL LETTER A ");
+        checkUnknownChar(" LATIN CAPITAL LETTER A");
+        checkUnknownChar(" LATIN CAPITAL LETTER A ");
+    }
+
+    private void checkUnknownChar(String charName) throws Exception {
+        String code = "'\\N{" + charName + "}'";
+        checkSyntaxErrorMessage(code, "SyntaxError: (unicode error) 'unicodeescape' codec can't decode bytes in position 0-" + (charName.length() + 3) + ": unknown Unicode character name");
+    }
+}
diff --git a/graalpython/com.oracle.graal.python.test/src/tests/unittest_tags/test_normalization.txt b/graalpython/com.oracle.graal.python.test/src/tests/unittest_tags/test_normalization.txt
@@ -0,0 +1 @@
+*NormalizationTest.test_bug_834676
diff --git a/graalpython/com.oracle.graal.python.test/src/tests/unittest_tags/test_ucn.txt b/graalpython/com.oracle.graal.python.test/src/tests/unittest_tags/test_ucn.txt
@@ -0,0 +1,4 @@
+*graalpython.lib-python.3.test.test_ucn.UnicodeNamesTest.test_cjk_unified_ideographs
+*graalpython.lib-python.3.test.test_ucn.UnicodeNamesTest.test_general
+*graalpython.lib-python.3.test.test_ucn.UnicodeNamesTest.test_misc_symbols
+*graalpython.lib-python.3.test.test_ucn.UnicodeNamesTest.test_strict_error_handling
diff --git a/graalpython/com.oracle.graal.python/src/com/oracle/graal/python/builtins/modules/BuiltinFunctions.java b/graalpython/com.oracle.graal.python/src/com/oracle/graal/python/builtins/modules/BuiltinFunctions.java
@@ -1417,6 +1417,11 @@ public int ord(String chr) {
             return chr.charAt(0);
         }
 
+        @Specialization
+        public int ord(PString chr) {
+            return ord(chr.getValue());
+        }
+
         @Specialization
         public long ord(VirtualFrame frame, PIBytesLike chr,
                         @Cached CastToJavaLongExactNode castNode,
diff --git a/graalpython/com.oracle.graal.python/src/com/oracle/graal/python/builtins/modules/UnicodeDataModuleBuiltins.java b/graalpython/com.oracle.graal.python/src/com/oracle/graal/python/builtins/modules/UnicodeDataModuleBuiltins.java
@@ -158,7 +158,7 @@ public static String getUnicodeVersion() {
     @Override
     public void initialize(PythonCore core) {
         super.initialize(core);
-        builtinConstants.put("version", getUnicodeVersion());
+        builtinConstants.put("unidata_version", getUnicodeVersion());
         PythonBuiltinClass objectType = core.lookupType(PythonBuiltinClassType.PythonObject);
         PythonObject ucd_3_2_0 = core.factory().createPythonObject(objectType);
         ucd_3_2_0.setAttribute("unidata_version", "3.2.0");
@@ -198,4 +198,36 @@ public String normalize(String form, PString unistr,
         }
 
     }
+
+    // unicodedata.is_normalized(form, unistr)
+    @Builtin(name = "is_normalized", minNumOfPositionalArgs = 2)
+    @GenerateNodeFactory
+    public abstract static class IsNormalizedNode extends PythonBuiltinNode {
+        @TruffleBoundary
+        protected Normalizer.Form getForm(String form) {
+            try {
+                return Normalizer.Form.valueOf(form);
+            } catch (IllegalArgumentException e) {
+                return null;
+            }
+        }
+
+        @Specialization(guards = {"form.equals(cachedForm)"}, limit = "4")
+        @TruffleBoundary
+        public boolean isNormalized(@SuppressWarnings("unused") String form, String unistr,
+                        @SuppressWarnings("unused") @Cached("form") String cachedForm,
+                        @Cached("getForm(cachedForm)") Normalizer.Form cachedNormForm) {
+            if (cachedNormForm == null) {
+                throw raise(ValueError, ErrorMessages.INVALID_NORMALIZATION_FORM);
+            }
+            return Normalizer.isNormalized(unistr, cachedNormForm);
+        }
+
+        @Specialization(guards = {"form.equals(cachedForm)"}, limit = "4")
+        public boolean normalize(String form, PString unistr,
+                        @Cached("form") String cachedForm,
+                        @Cached("getForm(cachedForm)") Normalizer.Form cachedNormForm) {
+            return isNormalized(form, unistr.getValue(), cachedForm, cachedNormForm);
+        }
+    }
 }
diff --git a/graalpython/com.oracle.graal.python/src/com/oracle/graal/python/parser/sst/FactorySSTVisitor.java b/graalpython/com.oracle.graal.python/src/com/oracle/graal/python/parser/sst/FactorySSTVisitor.java
@@ -1158,7 +1158,7 @@ public PNode visit(StarSSTNode node) {
 
     @Override
     public PNode visit(StringLiteralSSTNode node) {
-        PNode result = StringUtils.parseString(node.values, nodeFactory, errors);
+        PNode result = StringUtils.parseString(source, node, nodeFactory, errors);
         result.assignSourceSection(createSourceSection(node.startOffset, node.endOffset));
         return result;
     }
diff --git a/graalpython/com.oracle.graal.python/src/com/oracle/graal/python/parser/sst/StringUtils.java b/graalpython/com.oracle.graal.python/src/com/oracle/graal/python/parser/sst/StringUtils.java
@@ -41,7 +41,9 @@
 
 package com.oracle.graal.python.parser.sst;
 
-import static com.oracle.graal.python.runtime.exception.PythonErrorType.SyntaxError;
+import com.ibm.icu.lang.UCharacter;
+import com.oracle.graal.python.PythonLanguage;
+import com.oracle.graal.python.builtins.PythonBuiltinClassType;
 
 import java.util.ArrayList;
 import java.util.List;
@@ -53,9 +55,13 @@
 import com.oracle.graal.python.nodes.expression.ExpressionNode;
 import com.oracle.graal.python.nodes.literal.FormatStringLiteralNode;
 import com.oracle.graal.python.nodes.literal.StringLiteralNode;
+import com.oracle.graal.python.nodes.object.IsBuiltinClassProfile;
 import com.oracle.graal.python.nodes.statement.StatementNode;
 import com.oracle.graal.python.runtime.PythonParser;
+import com.oracle.graal.python.runtime.exception.PException;
 import com.oracle.graal.python.util.PythonUtils;
+import com.oracle.truffle.api.CompilerDirectives;
+import com.oracle.truffle.api.source.Source;
 
 public class StringUtils {
 
@@ -108,12 +114,12 @@ byte[] build() {
         }
     }
 
-    public static PNode parseString(String[] strings, NodeFactory nodeFactory, PythonParser.ParserErrorCallback errors) {
+    public static PNode parseString(Source source, StringLiteralSSTNode node, NodeFactory nodeFactory, PythonParser.ParserErrorCallback errors) {
         StringBuilder sb = null;
         BytesBuilder bb = null;
         boolean isFormatString = false;
         List<FormatStringLiteralNode.StringPart> formatStrings = null;
-        for (String text : strings) {
+        for (String text : node.values) {
             boolean isRaw = false;
             boolean isBytes = false;
             boolean isFormat = false;
@@ -146,7 +152,7 @@ public static PNode parseString(String[] strings, NodeFactory nodeFactory, Pytho
             text = text.substring(strStartIndex, strEndIndex);
             if (isBytes) {
                 if (sb != null || isFormatString) {
-                    throw errors.raise(SyntaxError, CANNOT_MIX_MESSAGE);
+                    throw errors.raiseInvalidSyntax(source, source.createSection(node.startOffset, node.endOffset - node.startOffset), CANNOT_MIX_MESSAGE);
                 }
                 if (bb == null) {
                     bb = new BytesBuilder();
@@ -158,10 +164,17 @@ public static PNode parseString(String[] strings, NodeFactory nodeFactory, Pytho
                 }
             } else {
                 if (bb != null) {
-                    throw errors.raise(SyntaxError, CANNOT_MIX_MESSAGE);
+                    throw errors.raiseInvalidSyntax(source, source.createSection(node.startOffset, node.endOffset - node.startOffset), CANNOT_MIX_MESSAGE);
                 }
                 if (!isRaw) {
-                    text = unescapeJavaString(text);
+                    try {
+                        text = unescapeJavaString(text);
+                    } catch (PException e) {
+                        e.expect(PythonBuiltinClassType.UnicodeDecodeError, IsBuiltinClassProfile.getUncached());
+                        String message = e.getMessage();
+                        message = "(unicode error)" + message.substring(PythonBuiltinClassType.UnicodeDecodeError.getName().length() + 1);
+                        throw errors.raiseInvalidSyntax(source, source.createSection(node.startOffset, node.endOffset - node.startOffset), message);
+                    }
                 }
                 if (isFormat) {
                     isFormatString = true;
@@ -293,6 +306,10 @@ public static String unescapeJavaString(String st) {
                         sb.append(Character.toChars(hexCode));
                         i += 3;
                         continue;
+                    case 'N':
+                        // a character from Unicode Data Database
+                        i = doCharacterName(st, sb, i + 2);
+                        continue;
                     default:
                         sb.append(ch);
                         sb.append(nextChar);
@@ -305,4 +322,55 @@ public static String unescapeJavaString(String st) {
         }
         return sb.toString();
     }
+
+    private static final String UNICODE_ERROR = "'unicodeescape' codec can't decode bytes in position %d-%d:";
+    private static final String MALFORMED_ERROR = " malformed \\N character escape";
+    private static final String UNKNOWN_UNICODE_ERROR = " unknown Unicode character name";
+
+    /**
+     * Replace '/N{Unicode Character Name}' with the code point of the character.
+     * 
+     * @param text a text that contains /N{...} escape sequence
+     * @param sb string builder where the result code point will be written
+     * @param offset this is offset of the open brace
+     * @return offset of the close brace
+     */
+    @CompilerDirectives.TruffleBoundary
+    private static int doCharacterName(String text, StringBuilder sb, int offset) {
+        char ch = text.charAt(offset);
+        if (ch != '{') {
+            throw PythonLanguage.getCore().raise(PythonBuiltinClassType.UnicodeDecodeError, UNICODE_ERROR + MALFORMED_ERROR, offset - 2, offset - 1);
+        }
+        int closeIndex = text.indexOf("}", offset + 1);
+        if (closeIndex == -1) {
+            throw PythonLanguage.getCore().raise(PythonBuiltinClassType.UnicodeDecodeError, UNICODE_ERROR + MALFORMED_ERROR, offset - 2, text.length() - 1);
+        }
+        String charName = text.substring(offset + 1, closeIndex).toUpperCase();
+        // When JDK 1.8 will not be supported, we can replace with Character.codePointOf(String
+        // name) in the
+        int cp = getCodePoint(charName);
+        if (cp >= 0) {
+            sb.append(Character.toChars(cp));
+        } else {
+            throw PythonLanguage.getCore().raise(PythonBuiltinClassType.UnicodeDecodeError, UNICODE_ERROR + UNKNOWN_UNICODE_ERROR, offset - 2, closeIndex);
+        }
+        return closeIndex;
+    }
+
+    @CompilerDirectives.TruffleBoundary
+    public static int getCodePoint(String charName) {
+        int possibleChar = UCharacter.getCharFromName(charName);
+        if (possibleChar > -1) {
+            return possibleChar;
+        }
+        possibleChar = UCharacter.getCharFromExtendedName(charName);
+        if (possibleChar > -1) {
+            return possibleChar;
+        }
+        possibleChar = UCharacter.getCharFromNameAlias(charName);
+        if (possibleChar > -1) {
+            return possibleChar;
+        }
+        return -1;
+    }
 }
diff --git a/mx.graalpython/suite.py b/mx.graalpython/suite.py
@@ -113,7 +113,17 @@
             ],
             "packedResource": True,
             "sha1": "bf7badf7e248e0ecf465d33c2f5aeec774209227",
-        }
+        },
+        "ICU4J" : {
+            "moduleName" : "com.ibm.icu",
+            "sha1" : "72c7519b6d91f7a1f993bd44a99fe95d67211b27",
+            "sourceSha1" : "57d00b7135ca8fa82311d6b9fd345309e4c46f0b",
+            "maven" : {
+              "groupId" : "com.ibm.icu",
+              "artifactId" : "icu4j",
+              "version" : "66.1",
+            },
+        },
     },
 
     # --------------------------------------------------------------------------------------------------------------
@@ -196,6 +206,7 @@
                 "truffle:ANTLR4",
                 "sulong:SULONG_API",
                 "XZ-1.8",
+                "ICU4J",
             ],
             "buildDependencies": ["com.oracle.graal.python.parser.antlr"],
             "jacoco": "include",

Original file line number	Diff line number	Diff line change
`@@ -1158,7 +1158,7 @@ public PNode visit(StarSSTNode node) {`
`1158`	`1158`
`1159`	`1159`	`@Override`
`1160`	`1160`	`public PNode visit(StringLiteralSSTNode node) {`
`1161`		`- PNode result = StringUtils.parseString(node.values, nodeFactory, errors);`
	`1161`	`+ PNode result = StringUtils.parseString(source, node, nodeFactory, errors);`
`1162`	`1162`	`result.assignSourceSection(createSourceSection(node.startOffset, node.endOffset));`
`1163`	`1163`	`return result;`
`1164`	`1164`	`}`