[GR-21522] Implement _pickle EE.

PullRequest: graalpython/1538

Author: cosminbasca

graalpython/com.oracle.graal.python.test/src/tests/test_struct.py

@@ -36,8 +36,9 @@
 # LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
 # OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
 # SOFTWARE.
-import sys
 import struct
+import sys
+
 
 def assert_raises(err, fn, *args, **kwargs):
     raised = False
@@ -116,7 +117,7 @@ def test_pack_unpack():
         for align in alignment:
             _fmt = align + fmt
             result = struct.pack(_fmt, *vals)
-            assert vals == struct.unpack(_fmt, result)
+            assert vals == struct.unpack(_fmt, result), "{} != struct.unpack({}, {})".format(vals, _fmt, result)
             assert struct.calcsize(_fmt) == len(result), "calcsize('{}')={} != len({})={}".format(
                 _fmt, struct.calcsize(_fmt), result, len(result))
 
@@ -169,6 +170,7 @@ def test_pack_large_long():
         assert struct.pack(fmt, 18446744073709551615) == b'\xff\xff\xff\xff\xff\xff\xff\xff'
         assert struct.unpack(fmt, b'\xff\xff\xff\xff\xff\xff\xff\xff') == (18446744073709551615,)
 
+
 def test_pack_into():
     test_string = b'Reykjavik rocks, eow!'
     writable_buf = bytearray(b' '*100)

graalpython/com.oracle.graal.python/src/com/oracle/graal/python/builtins/PythonBuiltinClassType.java

@@ -162,6 +162,11 @@ public enum PythonBuiltinClassType implements TruffleObject {
     LsprofProfiler("Profiler", "_lsprof"),
     PStruct("Struct", "_struct"),
     PStructUnpackIterator("unpack_iterator", "_struct"),
+    Pickler("Pickler", "_pickle"),
+    PicklerMemoProxy("PicklerMemoProxy", "_pickle"),
+    UnpicklerMemoProxy("UnpicklerMemoProxy", "_pickle"),
+    Unpickler("Unpickler", "_pickle"),
+    PickleBuffer("PickleBuffer", "_pickle"),
 
     // bz2
     BZ2Compressor("BZ2Compressor", "_bz2"),
@@ -264,6 +269,9 @@ public enum PythonBuiltinClassType implements TruffleObject {
     ZLibError("error", "zlib", Flags.EXCEPTION),
     LZMAError("LZMAError", "_lzma", Flags.EXCEPTION),
     StructError("StructError", "_struct", Flags.EXCEPTION),
+    PickleError("PickleError", "_pickle", Flags.EXCEPTION),
+    PicklingError("PicklingError", "_pickle", Flags.EXCEPTION),
+    UnpicklingError("UnpicklingError", "_pickle", Flags.EXCEPTION),
     SocketGAIError("gaierror", "_socket", Flags.EXCEPTION),
     SocketHError("herror", "_socket", Flags.EXCEPTION),
     SocketTimeout("timeout", "_socket", Flags.EXCEPTION),
@@ -569,6 +577,9 @@ public final Shape getInstanceShape(PythonLanguage lang) {
         StructError.base = Exception;
         BinasciiError.base = ValueError;
         BinasciiIncomplete.base = Exception;
+        PickleError.base = Exception;
+        PicklingError.base = PickleError;
+        UnpicklingError.base = PickleError;
 
         // warnings
         Warning.base = Exception;

graalpython/com.oracle.graal.python/src/com/oracle/graal/python/builtins/modules/BuiltinConstructors.java

@@ -1029,6 +1029,14 @@ public abstract static class IntNode extends PythonTernaryBuiltinNode {
         @Child private LookupAndCallUnaryNode callReprNode;
         @Child private WarnNode warnNode;
 
+        public final Object executeWith(VirtualFrame frame, Object number) {
+            return execute(frame, PythonBuiltinClassType.PInt, number, 10);
+        }
+
+        public final Object executeWith(VirtualFrame frame, Object number, Object base) {
+            return execute(frame, PythonBuiltinClassType.PInt, number, base);
+        }
+
         @TruffleBoundary
         private static Object stringToIntInternal(String num, int base) {
             try {

graalpython/com.oracle.graal.python/src/com/oracle/graal/python/builtins/modules/CodecsModuleBuiltins.java

@@ -112,6 +112,30 @@ public class CodecsModuleBuiltins extends PythonBuiltins {
     public static final String SURROGATEESCAPE = "surrogateescape";
     public static final String SURROGATEPASS = "surrogatepass";
 
+    static CodingErrorAction convertCodingErrorAction(String errors) {
+        CodingErrorAction errorAction;
+        switch (errors) {
+            // TODO: see [GR-10256] to implement the correct handling mechanics
+            case IGNORE:
+                errorAction = CodingErrorAction.IGNORE;
+                break;
+            case REPLACE:
+            case NAMEREPLACE:
+                errorAction = CodingErrorAction.REPLACE;
+                break;
+            case STRICT:
+            case BACKSLASHREPLACE:
+            case SURROGATEPASS:
+            case SURROGATEESCAPE:
+            case XMLCHARREFREPLACE:
+            default:
+                // Everything else will be handled by our Handle nodes
+                errorAction = CodingErrorAction.REPORT;
+                break;
+        }
+        return errorAction;
+    }
+
     @Override
     protected List<? extends NodeFactory<? extends PythonBuiltinBaseNode>> getNodeFactories() {
         return CodecsModuleBuiltinsFactory.getFactories();
@@ -432,36 +456,13 @@ public static HandleDecodingErrorNode create() {
         }
     }
 
-    protected static CodingErrorAction convertCodingErrorAction(String errors) {
-        CodingErrorAction errorAction;
-        switch (errors) {
-            // TODO: see [GR-10256] to implement the correct handling mechanics
-            case IGNORE:
-                errorAction = CodingErrorAction.IGNORE;
-                break;
-            case REPLACE:
-            case NAMEREPLACE:
-                errorAction = CodingErrorAction.REPLACE;
-                break;
-            case STRICT:
-            case BACKSLASHREPLACE:
-            case SURROGATEPASS:
-            case SURROGATEESCAPE:
-            case XMLCHARREFREPLACE:
-            default:
-                // Everything else will be handled by our Handle nodes
-                errorAction = CodingErrorAction.REPORT;
-                break;
-        }
-        return errorAction;
-    }
-
     // _codecs.encode(obj, encoding='utf-8', errors='strict')
     @Builtin(name = "__truffle_encode__", minNumOfPositionalArgs = 1, parameterNames = {"obj", "encoding", "errors"})
     @ArgumentClinic(name = "encoding", conversion = ArgumentClinic.ClinicConversion.String, defaultValue = "\"utf-8\"", useDefaultForNone = true)
     @ArgumentClinic(name = "errors", conversion = ArgumentClinic.ClinicConversion.String, defaultValue = "\"strict\"", useDefaultForNone = true)
     @GenerateNodeFactory
     public abstract static class CodecsEncodeNode extends PythonTernaryClinicBuiltinNode {
+        public abstract Object execute(Object str, Object encoding, Object errors);
 
         @Override
         protected ArgumentClinicProvider getArgumentClinic() {
@@ -504,7 +505,7 @@ Object encode(Object str, @SuppressWarnings("unused") Object encoding, @Suppress
     @ArgumentClinic(name = "errors", conversion = ArgumentClinic.ClinicConversion.String, defaultValue = "\"strict\"", useDefaultForNone = true)
     @ArgumentClinic(name = "final", conversion = ArgumentClinic.ClinicConversion.Boolean, defaultValue = "false", useDefaultForNone = true)
     @GenerateNodeFactory
-    abstract static class CodecsDecodeNode extends PythonQuaternaryClinicBuiltinNode {
+    public abstract static class CodecsDecodeNode extends PythonQuaternaryClinicBuiltinNode {
 
         @Override
         protected ArgumentClinicProvider getArgumentClinic() {
@@ -544,7 +545,7 @@ Object decode(Object bytes, @SuppressWarnings("unused") Object encoding, @Suppre
     @ArgumentClinic(name = "data", conversion = ArgumentClinic.ClinicConversion.Buffer)
     @ArgumentClinic(name = "errors", conversion = ArgumentClinic.ClinicConversion.String, defaultValue = "\"strict\"", useDefaultForNone = true)
     @GenerateNodeFactory
-    abstract static class CodecsEscapeDecodeNode extends PythonBinaryClinicBuiltinNode {
+    public abstract static class CodecsEscapeDecodeNode extends PythonBinaryClinicBuiltinNode {
         enum Errors {
             ERR_STRICT,
             ERR_IGNORE,
@@ -695,8 +696,7 @@ private static boolean isHexDigit(char digit) {
     @Builtin(name = "escape_encode", minNumOfPositionalArgs = 1, parameterNames = {"data", "errors"})
     @ArgumentClinic(name = "errors", conversion = ArgumentClinic.ClinicConversion.String, defaultValue = "\"strict\"", useDefaultForNone = true)
     @GenerateNodeFactory
-    abstract static class CodecsEscapeEncodeNode extends PythonBinaryClinicBuiltinNode {
-
+    public abstract static class CodecsEscapeEncodeNode extends PythonBinaryClinicBuiltinNode {
         @Override
         protected ArgumentClinicProvider getArgumentClinic() {
             return CodecsModuleBuiltinsClinicProviders.CodecsEscapeEncodeNodeClinicProviderGen.INSTANCE;

graalpython/com.oracle.graal.python/src/com/oracle/graal/python/builtins/objects/bytes/BytesUtils.java

@@ -33,11 +33,11 @@
 import java.nio.charset.StandardCharsets;
 import java.util.Arrays;
 
+import com.oracle.graal.python.builtins.Python3Core;
 import com.oracle.graal.python.builtins.PythonBuiltinClassType;
 import com.oracle.graal.python.builtins.objects.object.PythonObjectLibrary;
 import com.oracle.graal.python.nodes.ErrorMessages;
 import com.oracle.graal.python.nodes.PRaiseNode;
-import com.oracle.graal.python.builtins.Python3Core;
 import com.oracle.graal.python.runtime.PythonParser.ParserErrorCallback;
 import com.oracle.graal.python.util.PythonUtils;
 import com.oracle.truffle.api.CompilerAsserts;

graalpython/com.oracle.graal.python/src/com/oracle/graal/python/builtins/objects/floats/FloatUtils.java

@@ -48,7 +48,9 @@
 /**
  * Contains helper methods for parsing float numbers in float() and complex() constructors.
  */
-public class FloatUtils {
+public final class FloatUtils {
+    private FloatUtils() {
+    }
 
     public static int skipAsciiWhitespace(String str, int start, int len) {
         int offset = start;

graalpython/com.oracle.graal.python/src/com/oracle/graal/python/builtins/objects/ints/IntNodes.java

@@ -0,0 +1,160 @@
+/*
+ * Copyright (c) 2021, Oracle and/or its affiliates. All rights reserved.
+ * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER.
+ *
+ * The Universal Permissive License (UPL), Version 1.0
+ *
+ * Subject to the condition set forth below, permission is hereby granted to any
+ * person obtaining a copy of this software, associated documentation and/or
+ * data (collectively the "Software"), free of charge and under any and all
+ * copyright rights in the Software, and any and all patent rights owned or
+ * freely licensable by each licensor hereunder covering either (i) the
+ * unmodified Software as contributed to or provided by such licensor, or (ii)
+ * the Larger Works (as defined below), to deal in both
+ *
+ * (a) the Software, and
+ *
+ * (b) any piece of software and/or hardware listed in the lrgrwrks.txt file if
+ * one is included with the Software each a "Larger Work" to which the Software
+ * is contributed by such licensors),
+ *
+ * without restriction, including without limitation the rights to copy, create
+ * derivative works of, display, perform, and distribute the Software and make,
+ * use, sell, offer for sale, import, export, have made, and have sold the
+ * Software and the Larger Work(s), and to sublicense the foregoing rights on
+ * either these or other terms.
+ *
+ * This license is subject to the following condition:
+ *
+ * The above copyright notice and either this complete permission notice or at a
+ * minimum a reference to the UPL must be included in all copies or substantial
+ * portions of the Software.
+ *
+ * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+ * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+ * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
+ * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+ * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+ * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
+ * SOFTWARE.
+ */
+package com.oracle.graal.python.builtins.objects.ints;
+
+import static com.oracle.graal.python.nodes.ErrorMessages.TOO_LARGE_TO_CONVERT;
+
+import com.oracle.graal.python.builtins.PythonBuiltinClassType;
+import com.oracle.graal.python.nodes.PRaiseNode;
+import com.oracle.graal.python.runtime.object.PythonObjectFactory;
+import com.oracle.graal.python.util.NumericSupport;
+import com.oracle.graal.python.util.OverflowException;
+import com.oracle.truffle.api.dsl.Cached;
+import com.oracle.truffle.api.dsl.Specialization;
+import com.oracle.truffle.api.nodes.Node;
+
+/**
+ * Namespace containing equivalent nodes of {@code _Pylong_XXX} private function from
+ * {@code longobject.c}
+ */
+public final class IntNodes {
+    private IntNodes() {
+    }
+
+    /**
+     * Equivalent of CPython's {@code _PyLong_Sign}. Return 0 if v is 0, -1 if v < 0, +1 if v > 0.
+     */
+    public abstract static class PyLongSign extends Node {
+        public abstract int execute(Object value);
+
+        @Specialization
+        static int doInt(int value) {
+            return Integer.compare(value, 0);
+        }
+
+        @Specialization
+        static int doLong(long value) {
+            return Long.compare(value, 0);
+        }
+
+        @Specialization
+        static int doPInt(PInt value) {
+            return value.compareTo(0);
+        }
+    }
+
+    /**
+     * Equivalent to CPython's {@code _PyLong_NumBits}. Return the number of bits needed to
+     * represent the absolute value of a long.
+     */
+    public abstract static class PyLongNumBits extends Node {
+        public abstract int execute(Object value);
+
+        @Specialization
+        static int doInt(int value) {
+            return Integer.SIZE - Integer.numberOfLeadingZeros(Math.abs(value));
+        }
+
+        @Specialization
+        static int doLong(long value) {
+            return Long.SIZE - Long.numberOfLeadingZeros(Math.abs(value));
+        }
+
+        @Specialization
+        static int doPInt(PInt value) {
+            return value.bitLength();
+        }
+    }
+
+    /**
+     * Equivalent to CPython's {@code _PyLong_AsByteArray}. Convert the least-significant 8*n bits
+     * of long v to a base-256 integer, stored in array bytes.
+     */
+    public abstract static class PyLongAsByteArray extends Node {
+        public abstract byte[] execute(Object value, int size, boolean bigEndian);
+
+        @Specialization
+        static byte[] doLong(long value, int size, boolean bigEndian) {
+            final byte[] bytes = new byte[size];
+            NumericSupport support = bigEndian ? NumericSupport.bigEndian() : NumericSupport.littleEndian();
+            support.putLong(bytes, 0, value);
+            return bytes;
+        }
+
+        @Specialization
+        static byte[] doPInt(PInt value, int size, boolean bigEndian,
+                        @Cached PRaiseNode raiseNode) {
+            final byte[] bytes = new byte[size];
+            NumericSupport support = bigEndian ? NumericSupport.bigEndian() : NumericSupport.littleEndian();
+            try {
+                support.putBigInteger(bytes, 0, value.getValue(), size);
+            } catch (OverflowException oe) {
+                throw raiseNode.raise(PythonBuiltinClassType.OverflowError, TOO_LARGE_TO_CONVERT, "int");
+            }
+            return bytes;
+        }
+    }
+
+    /**
+     * Equivalent to CPython's {@code _PyLong_FromByteArray}. View the n unsigned bytes as a binary
+     * integer in base 256, and return a Python int with the same numeric value.
+     */
+    public abstract static class PyLongFromByteArray extends Node {
+        public abstract Object execute(byte[] data, boolean bigEndian);
+
+        protected boolean fitsInLong(byte[] data) {
+            return data.length <= Long.BYTES;
+        }
+
+        @Specialization(guards = "fitsInLong(data)")
+        static Object doLong(byte[] data, boolean bigEndian) {
+            NumericSupport support = bigEndian ? NumericSupport.bigEndian() : NumericSupport.littleEndian();
+            return support.getLong(data, 0);
+        }
+
+        @Specialization(guards = "!fitsInLong(data)")
+        static Object doPInt(byte[] data, boolean bigEndian,
+                        @Cached PythonObjectFactory factory) {
+            NumericSupport support = bigEndian ? NumericSupport.bigEndian() : NumericSupport.littleEndian();
+            return factory.createInt(support.getBigInteger(data, 0));
+        }
+    }
+}