[GR-33232] Intrinsify libgraalpython/_codecs.

PullRequest: graalpython/1936

Author: tomasstupka

graalpython/com.oracle.graal.python.test/src/tests/test_codecs.py

@@ -268,3 +268,16 @@ def test_errors(self):
         self.assertRaises(ValueError, decode, br"[\x0]")
         self.assertEqual(decode(br"[\x0]\x0", "ignore"), (b"[]", 8))
         self.assertEqual(decode(br"[\x0]\x0", "replace"), (b"[?]?", 8))
+
+class LookupTest(unittest.TestCase):
+    def test_lookup(self):
+        self.assertEqual(codecs.lookup('UTF-8').name, "utf-8")
+
+    def test_lookup_error(self):
+        def errhandler():
+            pass
+        self.assertRaises(TypeError, codecs.register_error, 1)
+        self.assertRaises(TypeError, codecs.register_error, 'a', 1)
+        self.assertRaises(LookupError, codecs.lookup_error, 'a')
+        codecs.register_error('testhandler', errhandler)
+        self.assertEqual(codecs.lookup_error('testhandler'), errhandler)

graalpython/com.oracle.graal.python.test/src/tests/test_tagged_unittests.py

@@ -108,7 +108,7 @@ def run_serialize_out(cmd):
         if append_out:
             message += LINE + "\n"
             message += "output:\n"
-            message += result.stdout.decode('ascii')
+            message += result.stdout.decode('utf-8')
             message += LINE + "\n"
         print(message)
         if result.returncode:

graalpython/com.oracle.graal.python.test/src/tests/unittest_tags/test_locale.txt

@@ -14,10 +14,6 @@
 *graalpython.lib-python.3.test.test_locale.TestCDelocalizeTest.test_delocalize
 *graalpython.lib-python.3.test.test_locale.TestCNumberFormatting.test_grouping
 *graalpython.lib-python.3.test.test_locale.TestCNumberFormatting.test_grouping_and_padding
-*graalpython.lib-python.3.test.test_locale.TestEnUSCollation.test_strcoll
-*graalpython.lib-python.3.test.test_locale.TestEnUSCollation.test_strcoll_with_diacritic
-*graalpython.lib-python.3.test.test_locale.TestEnUSCollation.test_strxfrm
-*graalpython.lib-python.3.test.test_locale.TestEnUSCollation.test_strxfrm_with_diacritic
 *graalpython.lib-python.3.test.test_locale.TestEnUSDelocalize.test_atof
 *graalpython.lib-python.3.test.test_locale.TestEnUSDelocalize.test_atoi
 *graalpython.lib-python.3.test.test_locale.TestEnUSDelocalize.test_delocalize

graalpython/com.oracle.graal.python/src/com/oracle/graal/python/builtins/Python3Core.java

@@ -243,8 +243,10 @@
 import com.oracle.graal.python.builtins.objects.type.TypeBuiltins;
 import com.oracle.graal.python.builtins.objects.type.TypeNodes.GetNameNode;
 import com.oracle.graal.python.builtins.objects.zipimporter.ZipImporterBuiltins;
+import com.oracle.graal.python.lib.PyObjectCallMethodObjArgs;
 import com.oracle.graal.python.nodes.BuiltinNames;
 import com.oracle.graal.python.nodes.call.GenericInvokeNode;
+import com.oracle.graal.python.nodes.statement.AbstractImportNode;
 import com.oracle.graal.python.runtime.PythonCodeSerializer;
 import com.oracle.graal.python.runtime.PythonContext;
 import com.oracle.graal.python.runtime.PythonOptions;
@@ -301,7 +303,9 @@ private static String[] initializeCoreFiles() {
                         "base_exception",
                         PythonCextBuiltins.PYTHON_CEXT,
                         "_collections",
-                        "_codecs",
+                        // TODO: see the encodings initialization before _codecs_truffle.py is
+                        // loaded in initializePython3Core;
+                        // once _codecs_truffle.py is gone, it should not be necessary
                         "_codecs_truffle",
                         "bytes",
                         "bytearray",
@@ -654,11 +658,26 @@ private void initializeJavaCore() {
     private void initializePython3Core(String coreHome) {
         loadFile(BuiltinNames.BUILTINS, coreHome);
         for (String s : coreFiles) {
+            // TODO: once _codecs_truffle.py is gone, this should not be necessary
+            if (s.equals("_codecs_truffle")) {
+                importEncoding();
+            }
             loadFile(s, coreHome);
         }
         initialized = true;
     }
 
+    private void importEncoding() {
+        PythonModule sys = lookupBuiltinModule("sys");
+        Object sysPath = sys.getAttribute("path");
+        PyObjectCallMethodObjArgs.getUncached().execute(null, sysPath, "insert", 0, getContext().getStdlibHome());
+        try {
+            AbstractImportNode.importModule("encodings");
+        } finally {
+            PyObjectCallMethodObjArgs.getUncached().execute(null, sysPath, "pop");
+        }
+    }
+
     /**
      * Run post-initialization code that needs a fully working Python environment. This will be run
      * eagerly when the context is initialized on the JVM or a new context is created on SVM, but is

graalpython/com.oracle.graal.python/src/com/oracle/graal/python/builtins/modules/CodecsModuleBuiltins.java

@@ -140,6 +140,7 @@ public abstract class BuiltinNames {
     public static final String SET = "set";
     public static final String BUILTINS = "builtins";
     public static final String __GRAALPYTHON__ = "__graalpython__";
+    public static final String _CODECS = "_codecs";
     public static final String GETSET_DESCRIPTOR = "getset_descriptor";
     public static final String MEMBER_DESCRIPTOR = "member_descriptor";
     public static final String SIMPLE_QUEUE = "SimpleQueue";

graalpython/com.oracle.graal.python/src/com/oracle/graal/python/nodes/BuiltinNames.java

@@ -57,6 +57,7 @@ public abstract class ErrorMessages {
     public static final String ARG_MUST_BE_S_NOT_P = "%s argument must be a %s, not %p";
     public static final String S_BRACKETS_ARG_MUST_BE_S_NOT_P = "%s() argument must be a %s, not %p";
     public static final String S_BRACKETS_ARG_MUST_BE_READ_WRITE_BYTES_LIKE_NOT_P = "%s() argument must be read-write bytes-like object, not %p";
+    public static final String ARG_MUST_BE_CALLABLE = "argument must be callable";
     public static final String ARG_MUST_BE_STRING_OR_BYTELIKE_OR_NUMBER = "%s argument must be a string, a bytes-like object or a number, not %p";
     public static final String ARG_MUST_BE_STRING_OR_NUMBER = "%s argument must be a string or a number, not '%p'";
     public static final String ARG_MUST_NOT_BE_ZERO = "%s arg %d must not be zero";
@@ -187,6 +188,7 @@ public abstract class ErrorMessages {
     public static final String MUST_BE_SET_TO_S_NOT_P = "%s must be set to a %s, not a '%p'";
     public static final String CLASSPATH_ARG_MUST_BE_STRING = "classpath argument %d must be string, not %p";
     public static final String CODE_OBJ_NO_FREE_VARIABLES = "code object passed to %s may not contain free variables";
+    public static final String CODEC_SEARCH_MUST_RETURN_4 = "codec search functions must return 4-tuples";
     public static final String COMPILE_MUST_BE = "compile() mode must be 'exec', 'eval' or 'single'";
     public static final String COMPLEX_ARG_IS_MALFORMED_STR = "complex() arg is a malformed string";
     public static final String COMPLEX_CANT_TAKE_ARG = "complex() can't take second arg if first is a string";
@@ -277,6 +279,7 @@ public abstract class ErrorMessages {
     public static final String GOT_MULTIPLE_VALUES_FOR_ARG = "%s() got multiple values for keyword argument '%s'";
     public static final String GOT_SOME_POS_ONLY_ARGS_PASSED_AS_KEYWORD = "%s() got some positional-only arguments passed as keyword arguments: '%s'";
     public static final String GOT_UNEXPECTED_KEYWORD_ARG = "%s() got an unexpected keyword argument '%s'";
+    public static final String HANDLER_MUST_BE_CALLABLE = "handler must be callable";
     public static final String HAS_NO_ATTR = "%s has no attribute %s";
     public static final String HASH_SHOULD_RETURN_INTEGER = "__hash__ method should return an integer";
     public static final String HEX_VALUE_TOO_LARGE_AS_FLOAT = "hexadecimal value too large to represent as a float";
@@ -422,6 +425,7 @@ public abstract class ErrorMessages {
     public static final String MUST_BE_STRINGS_NOT_P = "%s must be strings, not %p";
     public static final String MUST_BE_TUPLE_OF_CLASSES_NOT_P = "%s.%s must be tuple of classes, not '%p'";
     public static final String MUST_RETURN_2TUPLE = "%p.__divmod__() must return a 2-tuple, not %p";
+    public static final String S_MUST_RETURN_TUPLE = "%s must return a tuple (object, integer)";
     public static final String MUST_S_ITER_RETURN_2TUPLE = "%s iterator must return 2-tuples";
     public static final String S_MUST_RETURN_S_NOT_P = "%s must return a %s, not %p";
     public static final String S_MUST_RETURN_S_OR_S = "%s must return a %s or %s";

graalpython/com.oracle.graal.python/src/com/oracle/graal/python/nodes/ErrorMessages.java

@@ -57,6 +57,7 @@
 import java.util.concurrent.atomic.AtomicLong;
 import java.util.concurrent.locks.ReentrantLock;
 import java.util.logging.Level;
+import com.oracle.graal.python.builtins.objects.tuple.PTuple;
 
 import org.graalvm.nativeimage.ImageInfo;
 import org.graalvm.options.OptionKey;
@@ -495,6 +496,22 @@ public Thread getOwner() {
     private final ChildContextData childContextData;
     private final SharedMultiprocessingData sharedMultiprocessingData;
 
+    private final List<Object> codecSearchPath = new ArrayList<>();
+    private final Map<String, PTuple> codecSearchCache = new HashMap<>();
+    private final Map<String, Object> codecErrorRegistry = new HashMap<>();
+
+    public List<Object> getCodecSearchPath() {
+        return codecSearchPath;
+    }
+
+    public Map<String, PTuple> getCodecSearchCache() {
+        return codecSearchCache;
+    }
+
+    public Map<String, Object> getCodecErrorRegistry() {
+        return codecErrorRegistry;
+    }
+
     public static final class ChildContextData {
         private int exitCode = 0;
         private boolean signaled;