[GR-23268] UTF7 support

PullRequest: graalpython/1251

Author: msimacek

graalpython/com.oracle.graal.python.test/src/tests/unittest_tags/test_unicode.txt

@@ -14,6 +14,7 @@
 *graalpython.lib-python.3.test.test_unicode.UnicodeTest.test_bytes_comparison
 *graalpython.lib-python.3.test.test_unicode.UnicodeTest.test_case_operation_overflow
 *graalpython.lib-python.3.test.test_unicode.UnicodeTest.test_center
+*graalpython.lib-python.3.test.test_unicode.UnicodeTest.test_codecs
 *graalpython.lib-python.3.test.test_unicode.UnicodeTest.test_codecs_idna
 *graalpython.lib-python.3.test.test_unicode.UnicodeTest.test_comparison
 *graalpython.lib-python.3.test.test_unicode.UnicodeTest.test_concatenation

graalpython/com.oracle.graal.python/src/com/oracle/graal/python/resources/reflect-config.json

@@ -42,5 +42,13 @@
 {
   "name":"com.ibm.icu.text.BreakIteratorFactory",
   "methods":[{"name":"<init>","parameterTypes":[] }]
+},
+{
+  "name":"com.ibm.icu.charset.CharsetUTF7",
+  "methods":[{"name":"<init>","parameterTypes":["java.lang.String", "java.lang.String", "java.lang.String[]"]}]
+},
+{
+  "name":"com.ibm.icu.charset.CharsetHZ",
+  "methods":[{"name":"<init>","parameterTypes":["java.lang.String", "java.lang.String", "java.lang.String[]"]}]
 }
 ]

graalpython/com.oracle.graal.python/src/com/oracle/graal/python/util/CharsetMapping.java

@@ -47,6 +47,7 @@
 import java.util.Locale;
 import java.util.Map;
 
+import com.ibm.icu.charset.CharsetICU;
 import com.oracle.truffle.api.CompilerDirectives.TruffleBoundary;
 
 /**
@@ -78,10 +79,20 @@ public static String normalize(String encoding) {
     private static Charset getJavaCharset(String name) {
         Charset charset = JAVA_CHARSETS.get(name);
         if (charset == null) {
-            try {
-                charset = Charset.forName(name);
-            } catch (UnsupportedCharsetException e) {
-                // Let it stay null
+            // Important: When adding additional ICU4J charset, the implementation class needs to be
+            // added to reflect-config.json
+            if (name.equals("UTF-7") || name.equals("HZ")) {
+                try {
+                    charset = CharsetICU.forNameICU(name);
+                } catch (UnsupportedCharsetException e) {
+                    // Let it stay null
+                }
+            } else {
+                try {
+                    charset = Charset.forName(name);
+                } catch (UnsupportedCharsetException e) {
+                    // Let it stay null
+                }
             }
             JAVA_CHARSETS.put(name, charset);
         }
@@ -161,7 +172,7 @@ private static void addAlias(String alias, String pythonName) {
         addMapping("gb2312", "GB2312");
         addMapping("gbk", "GBK");
         addMapping("hp_roman8", null);
-        addMapping("hz", "GB2312");
+        addMapping("hz", "HZ");
         addMapping("iso2022_jp_1", null);
         addMapping("iso2022_jp_2004", null);
         addMapping("iso2022_jp_2", "ISO-2022-JP-2");
@@ -213,7 +224,7 @@ private static void addAlias(String alias, String pythonName) {
         addMapping("utf_32_be", "UTF-32BE");
         addMapping("utf_32_le", "UTF-32LE");
         addMapping("utf_32", "UTF-32");
-        addMapping("utf_7", null);
+        addMapping("utf_7", "UTF-7");
         addMapping("utf_8", "UTF-8");
 
         // Generated from encodings.aliases.aliases, removed non-language encodings like base64

mx.graalpython/mx_graalpython.py

@@ -1438,10 +1438,25 @@ def _register_bench_suites(namespace):
         mx_benchmark.add_bm_suite(java_bench_suite)
 
 
+class CharsetFilteringPariticpant:
+    """
+    Remove charset providers from the resulting JAR distribution. Done to avoid libraries (icu4j-charset)
+    adding their charsets implicitly to native image. We need to add them explicitly in a controlled way.
+    """
+    def __opened__(self, archive, src_archive, services):
+        self.__services = services
+
+    def __closing__(self):
+        self.__services.pop('java.nio.charset.spi.CharsetProvider', None)
+
+
 def mx_post_parse_cmd_line(namespace):
     # all projects are now available at this time
     _register_vms(namespace)
     _register_bench_suites(namespace)
+    for dist in mx.suite('graalpython').dists:
+        if hasattr(dist, 'set_archiveparticipant'):
+            dist.set_archiveparticipant(CharsetFilteringPariticpant())
 
 
 def python_coverage(args):

mx.graalpython/suite.py

@@ -124,6 +124,16 @@
               "version" : "66.1",
             },
         },
+        "ICU4J-CHARSET-66.1" : {
+            "moduleName" : "com.ibm.icu.charset",
+            "sha1" : "292f8736709f5c69afd9275faf92fac11a342b3e",
+            "sourceSha1" : "e981b54cb052d22f5512626dac6a29c79b1460d6",
+            "maven" : {
+              "groupId" : "com.ibm.icu",
+              "artifactId" : "icu4j-charset",
+              "version" : "66.1",
+            },
+        },
     },
 
     # --------------------------------------------------------------------------------------------------------------
@@ -229,6 +239,7 @@
                 "sulong:SULONG_API",
                 "XZ-1.8",
                 "ICU4J-66.1",
+                "ICU4J-CHARSET-66.1",
             ],
             "buildDependencies": ["com.oracle.graal.python.parser.antlr"],
             "jacoco": "include",