[GR-12457] Implementation of int.from_bytes has to be corrected.

Author: ppisl

graalpython/com.oracle.graal.python.test/src/tests/test_int.py

@@ -37,6 +37,10 @@
 # OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
 # SOFTWARE.
 
+import unittest
+
+import array
+
 class _NamedIntConstant(int):
     def __new__(cls, value, name):
         self = super(_NamedIntConstant, cls).__new__(cls, value)
@@ -315,3 +319,241 @@ def __int__(self):
 
 def test_create_int_from_string():
   assert int("5c7920a80f5261a2e5322163c79b71a25a41f414", 16) == 527928385865769069253929759180846776123316630548
+
+
+class FromBytesTests(unittest.TestCase):
+    
+    def check(self, tests, byteorder, signed=False):
+        for test, expected in tests.items():
+            try:
+                self.assertEqual( int.from_bytes(test, byteorder, signed=signed), expected)
+            except Exception as err:
+                raise AssertionError(
+                    "failed to convert {0} with byteorder={1!r} and signed={2}"
+                    .format(test, byteorder, signed)) from err
+
+    def test_SignedBigEndian(self):
+        # Convert signed big-endian byte arrays to integers.
+        tests1 = {
+            b'': 0,
+            b'\x00': 0,
+            b'\x00\x00': 0,
+            b'\x01': 1,
+            b'\x00\x01': 1,
+            b'\xff': -1,
+            b'\xff\xff': -1,
+            b'\x81': -127,
+            b'\x80': -128,
+            b'\xff\x7f': -129,
+            b'\x7f': 127,
+            b'\x00\x81': 129,
+            b'\xff\x01': -255,
+            b'\xff\x00': -256,
+            b'\x00\xff': 255,
+            b'\x01\x00': 256,
+            b'\x7f\xff': 32767,
+            b'\x80\x00': -32768,
+            b'\x00\xff\xff': 65535,
+            b'\xff\x00\x00': -65536,
+            b'\x80\x00\x00': -8388608
+        }
+        self.check(tests1, 'big', signed=True)
+
+    def test_SignedLittleEndian(self):
+        # Convert signed little-endian byte arrays to integers.
+        tests2 = {
+            b'': 0,
+            b'\x00': 0,
+            b'\x00\x00': 0,
+            b'\x01': 1,
+            b'\x00\x01': 256,
+            b'\xff': -1,
+            b'\xff\xff': -1,
+            b'\x81': -127,
+            b'\x80': -128,
+            b'\x7f\xff': -129,
+            b'\x7f': 127,
+            b'\x81\x00': 129,
+            b'\x01\xff': -255,
+            b'\x00\xff': -256,
+            b'\xff\x00': 255,
+            b'\x00\x01': 256,
+            b'\xff\x7f': 32767,
+            b'\x00\x80': -32768,
+            b'\xff\xff\x00': 65535,
+            b'\x00\x00\xff': -65536,
+            b'\x00\x00\x80': -8388608
+        }
+        self.check(tests2, 'little', signed=True)
+
+    def test_UnsignedBigEndian(self):
+        # Convert unsigned big-endian byte arrays to integers.
+        tests3 = {
+            b'': 0,
+            b'\x00': 0,
+            b'\x01': 1,
+            b'\x7f': 127,
+            b'\x80': 128,
+            b'\xff': 255,
+            b'\x01\x00': 256,
+            b'\x7f\xff': 32767,
+            b'\x80\x00': 32768,
+            b'\xff\xff': 65535,
+            b'\x01\x00\x00': 65536,
+        }
+        self.check(tests3, 'big', signed=False)
+
+    def test_UnsignedLittleEndian(self):
+        # Convert integers to unsigned little-endian byte arrays.
+        tests4 = {
+            b'': 0,
+            b'\x00': 0,
+            b'\x01': 1,
+            b'\x7f': 127,
+            b'\x80': 128,
+            b'\xff': 255,
+            b'\x00\x01': 256,
+            b'\xff\x7f': 32767,
+            b'\x00\x80': 32768,
+            b'\xff\xff': 65535,
+            b'\x00\x00\x01': 65536,
+        }
+        self.check(tests4, 'little', signed=False)
+
+    def test_IntObject(self):
+        myint = MyInt
+        self.assertIs(type(myint.from_bytes(b'\x00', 'big')), MyInt)
+        self.assertEqual(myint.from_bytes(b'\x01', 'big'), 1)
+        self.assertIs(
+            type(myint.from_bytes(b'\x00', 'big', signed=False)), myint)
+        self.assertEqual(myint.from_bytes(b'\x01', 'big', signed=False), 1)
+        self.assertIs(type(myint.from_bytes(b'\x00', 'little')), myint)
+        self.assertEqual(myint.from_bytes(b'\x01', 'little'), 1)
+        self.assertIs(type(myint.from_bytes(
+            b'\x00', 'little', signed=False)), myint)
+        self.assertEqual(myint.from_bytes(b'\x01', 'little', signed=False), 1)
+
+    def test_from_list(self):
+        self.assertEqual(
+            int.from_bytes([255, 0, 0], 'big', signed=True), -65536)
+        self.assertEqual(
+            MyInt.from_bytes([255, 0, 0], 'big', signed=True), -65536)
+        self.assertIs(type(MyInt.from_bytes(
+            [255, 0, 0], 'little', signed=False)), MyInt)
+
+        class LyingList(list):
+            def __iter__(self):
+                return iter([10, 20, 30, 40])
+        
+        self.assertEqual(
+            int.from_bytes(LyingList([255, 1, 1]), 'big'), 169090600)
+         
+    def test_from_tuple(self):
+        self.assertEqual(
+            int.from_bytes((255, 0, 0), 'big', signed=True), -65536)
+        self.assertEqual(
+            MyInt.from_bytes((255, 0, 0), 'big', signed=True), -65536)
+        self.assertIs(type(MyInt.from_bytes(
+            (255, 0, 0), 'little', signed=False)), MyInt)
+
+        class LyingTuple(tuple):
+            def __iter__(self):
+                return iter((15, 25, 35, 45))
+
+        self.assertEqual(
+            int.from_bytes(LyingTuple((255, 1, 1)), 'big'), 253305645)
+        
+    def test_from_bytearray(self):
+        self.assertEqual(int.from_bytes(
+            bytearray(b'\xff\x00\x00'), 'big', signed=True), -65536)
+        self.assertEqual(int.from_bytes(
+            bytearray(b'\xff\x00\x00'), 'big', signed=True), -65536)
+
+    def test_from_array(self):
+        self.assertEqual(int.from_bytes(
+            array.array('b', b'\xff\x00\x00'), 'big', signed=True), -65536)
+
+    '''
+    TODO This test is commented out until GR-12448 is not fixed. 
+    def test_from_memoryview(self):
+        self.assertEqual(int.from_bytes(
+            memoryview(b'\xff\x00\x00'), 'big', signed=True), -65536)
+    '''
+
+    def test_wrong_input(self):
+        self.assertRaises(ValueError, int.from_bytes, [256], 'big')
+        self.assertRaises(ValueError, int.from_bytes, (256,), 'big')
+        self.assertRaises(ValueError, int.from_bytes, [0], 'big\x00')
+        self.assertRaises(ValueError, int.from_bytes, [0], 'little\x00')
+        self.assertRaises(TypeError, int.from_bytes, 0, 'big')
+        self.assertRaises(TypeError, int.from_bytes, 0, 'big', True)
+
+        #TODO uncoment these tests, when GR-12453 is fixed
+        #self.assertRaises(TypeError, int.from_bytes, "", 'big')
+        #self.assertRaises(TypeError, int.from_bytes, "\x00", 'big')
+        #self.assertRaises(TypeError, MyInt.from_bytes, "", 'big')
+        #self.assertRaises(TypeError, MyInt.from_bytes, "\x00", 'big')
+        self.assertRaises(TypeError, MyInt.from_bytes, 0, 'big')
+        self.assertRaises(TypeError, int.from_bytes, 0, 'big', True)
+
+    def test_int_subclass(self):
+        class myint2(int):
+            def __new__(cls, value):
+                return int.__new__(cls, value + 1)
+
+        i = myint2.from_bytes(b'\x01', 'big')
+        self.assertIs(type(i), myint2)
+        self.assertEqual(i, 2)
+
+        class myint3(int):
+            def __init__(self, value):
+                self.foo = 'bar'
+
+        i = myint3.from_bytes(b'\x01', 'big')
+        self.assertIs(type(i), myint3)
+        self.assertEqual(i, 1)
+        self.assertEqual(getattr(i, 'foo', 'none'), 'bar')
+
+    def test_range(self):
+        self.assertEqual(int.from_bytes(range(5), 'big'), 16909060)
+        self.assertEqual(int.from_bytes(range(5), 'little'), 17230332160)
+        self.assertEqual(int.from_bytes(range(200,225), 'big'), 1260368276743602661175172759269383066378083427695751132536800)
+        r = range(10)
+        self.assertEqual(int.from_bytes(r[:], 'big'), 18591708106338011145)
+        self.assertEqual(int.from_bytes(r[1:3], 'big'), 258)
+        self.assertEqual(int.from_bytes(r[3:1], 'big'), 0)
+        self.assertEqual(int.from_bytes(r[3:-1], 'big'), 3315799033608)
+
+    def test_map(self):
+        def myconvert(text): 
+            return int(text)
+        self.assertEqual(int.from_bytes(map(myconvert, ["100","10","1"]), 'big'), 6556161)
+
+    def test_from_byteslike_object(self):
+        class mybyteslike():
+            def __bytes__(self):
+                return bytes([10,20])
+
+        self.assertEqual(int.from_bytes(mybyteslike(), 'big'), 2580)
+
+    def test_from_wrong_byteslike_object(self):
+        class mybyteslike1():
+            def __bytes__(self):
+                return range(3)
+
+        self.assertRaises(TypeError, int.from_bytes, mybyteslike1(), 'big') 
+
+        class mybyteslike2():
+            def __bytes__(self):
+                return array.array('b', [2, 2, 3])
+
+        self.assertRaises(TypeError, int.from_bytes, mybyteslike2(), 'big') 
+
+    def test_from_list_with_byteslike(self):
+        class StrangeList(list):
+            def __bytes__(self):
+                return bytes([3])
+            def __iter__(self):
+                return iter([10])
+
+        self.assertEqual(int.from_bytes(StrangeList([4,5]), 'big'), 3)

graalpython/com.oracle.graal.python/src/com/oracle/graal/python/builtins/Python3Core.java

@@ -190,6 +190,7 @@ private static final String[] initializeCoreFiles() {
                         "_warnings",
                         "posix",
                         "_io",
+                        "int",
                         "_frozen_importlib",
                         "classes",
                         "_weakref",

graalpython/com.oracle.graal.python/src/com/oracle/graal/python/builtins/objects/bytes/BytesNodes.java

@@ -51,6 +51,7 @@
 import com.oracle.graal.python.builtins.objects.common.SequenceStorageNodes.NormalizeIndexNode;
 import com.oracle.graal.python.builtins.objects.list.PList;
 import com.oracle.graal.python.builtins.objects.memoryview.PMemoryView;
+import com.oracle.graal.python.builtins.objects.tuple.PTuple;
 import com.oracle.graal.python.nodes.PGuards;
 import com.oracle.graal.python.nodes.PNodeWithContext;
 import com.oracle.graal.python.nodes.SpecialMethodNames;
@@ -66,6 +67,7 @@
 import com.oracle.truffle.api.dsl.Fallback;
 import com.oracle.truffle.api.dsl.ImportStatic;
 import com.oracle.truffle.api.dsl.Specialization;
+import com.oracle.truffle.api.nodes.Node;
 import com.oracle.truffle.api.profiles.ValueProfile;
 
 public abstract class BytesNodes {
@@ -281,18 +283,20 @@ public static FindNode create() {
         }
     }
 
-    public abstract static class FromListNode extends PNodeWithContext {
+    public abstract static class FromSequenceStorageNode extends PNodeWithContext {
 
-        @Child private SequenceStorageNodes.GetItemNode getItemNode;
-        @Child private SequenceStorageNodes.CastToByteNode castToByteNode;
-        @Child private SequenceStorageNodes.LenNode lenNode;
+        @Node.Child private SequenceStorageNodes.GetItemNode getItemNode;
+        @Node.Child private SequenceStorageNodes.CastToByteNode castToByteNode;
+        @Node.Child private SequenceStorageNodes.LenNode lenNode;
 
-        public byte[] execute(PList list) {
-            SequenceStorage listStore = list.getSequenceStorage();
-            int len = getLenNode().execute(listStore);
+        public abstract byte[] execute(SequenceStorage storage);
+
+        @Specialization
+        public byte[] doIt(SequenceStorage storage) {
+            int len = getLenNode().execute(storage);
             byte[] bytes = new byte[len];
             for (int i = 0; i < len; i++) {
-                Object item = getGetItemNode().execute(listStore, i);
+                Object item = getGetItemNode().execute(storage, i);
                 bytes[i] = getCastToByteNode().execute(item);
             }
             return bytes;
@@ -321,6 +325,87 @@ private SequenceStorageNodes.LenNode getLenNode() {
             }
             return lenNode;
         }
+
+        public static FromSequenceStorageNode createNode() {
+            return BytesNodesFactory.FromSequenceStorageNodeGen.create();
+        }
+    }
+
+    public abstract static class FromListNode extends FromSequenceStorageNode {
+
+        public abstract byte[] execute(PList iterator);
+
+        @Specialization
+        public byte[] doIt(PList list) {
+            return doIt(list.getSequenceStorage());
+        }
+
+        public static FromListNode create() {
+            return BytesNodesFactory.FromListNodeGen.create();
+        }
+    }
+
+    public abstract static class FromTupleNode extends FromSequenceStorageNode {
+
+        public abstract byte[] execute(PTuple tuple);
+
+        @Specialization
+        public byte[] doIt(PTuple tuple) {
+            return doIt(tuple.getSequenceStorage());
+        }
+
+        public static FromTupleNode create() {
+            return BytesNodesFactory.FromTupleNodeGen.create();
+        }
+    }
+
+    public abstract static class FromIteratorNode extends PNodeWithContext {
+
+        public abstract byte[] execute(Object iterator);
+
+        @Specialization
+        public byte[] execute(Object iterObject,
+                        @Cached("create()") SequenceStorageNodes.CastToByteNode castToByteNode,
+                        @Cached("create()") GetNextNode getNextNode,
+                        @Cached("create()") IsBuiltinClassProfile errorProfile) {
+            ArrayList<byte[]> parts = new ArrayList<>();
+            int currentLenght = 16;
+            int currentOffset = 0;
+            byte[] currentArray = new byte[currentLenght];
+            parts.add(currentArray);
+            int size = 0;
+            while (true) {
+                if (currentOffset >= currentArray.length) {
+                    // we need to create new array
+                    if (currentLenght < 65536) {
+                        currentLenght = currentLenght * 2;
+                    }
+                    currentArray = new byte[currentLenght];
+                    parts.add(currentArray);
+                    currentOffset = 0;
+                }
+                try {
+                    currentArray[currentOffset] = castToByteNode.execute(getNextNode.execute(iterObject));
+                    currentOffset++;
+                    size++;
+                } catch (PException e) {
+                    e.expectStopIteration(errorProfile);
+                    // convert parts into result array
+                    byte[] bytes = new byte[size];
+
+                    int offset = 0;
+                    for (byte[] part : parts) {
+                        System.arraycopy(part, 0, bytes, offset, Math.min(part.length, size - offset));
+                        offset += part.length;
+                    }
+                    return bytes;
+                }
+            }
+        }
+
+        public static FromIteratorNode create() {
+            return BytesNodesFactory.FromIteratorNodeGen.create();
+        }
     }
 
     public static class CmpNode extends PNodeWithContext {