Fix: undetected overfow in int.to_bytes

Author: fangerer

graalpython/com.oracle.graal.python.test/src/tests/test_int.py

@@ -639,7 +639,7 @@ def checkPIntSpec(self, tests, byteorder, signed=False):
                     .format(test, byteorder, signed)) from err
 
 
-    def test_SignedBitEndian(self):
+    def test_SignedBigEndian(self):
         # Convert integers to signed big-endian byte arrays.
         tests1 = {
             0: b'\x00',
@@ -699,7 +699,7 @@ def test_UnsignedBigEndian(self):
             32767: b'\x7f\xff',
             32768: b'\x80\x00',
             65535: b'\xff\xff',
-            65536: b'\x01\x00\x00'
+            65536: b'\x01\x00\x00',
         }
         self.check(tests3, 'big', signed=False)
         self.checkPIntSpec(tests3, 'big', signed=False)
@@ -736,6 +736,8 @@ def test_WrongInput(self):
         self.assertRaises(OverflowError, (-1).to_bytes, 2, 'big', signed=False)
         self.assertRaises(OverflowError, (-1).to_bytes, 2, 'little', signed=False)
         self.assertRaises(OverflowError, (1).to_bytes, 0, 'big')
+        self.assertRaises(OverflowError, (4294967296).to_bytes, 4, 'big')
+        self.assertRaises(OverflowError, (4294967296).to_bytes, 4, 'little')
 
     def test_WrongTypes(self):
         class MyTest():

graalpython/com.oracle.graal.python/src/com/oracle/graal/python/builtins/objects/ints/IntBuiltins.java

@@ -2131,10 +2131,11 @@ public static byte[] fromLong(long self, int byteCount, boolean isBigEndian, boo
                 number >>= 8;
                 index += delta;
             }
-            if (overflowProfile.profile((number != 0 && bytes.length == 1 && bytes[0] != self) || (signed && bytes.length == 1 && bytes[0] != self) || (byteCount == 0 && self != 0))) {
 
+            if (overflowProfile.profile(!signed && number != 0 || (signed && bytes.length == 1 && bytes[0] != self) || (byteCount == 0 && self != 0))) {
                 throw raise.raise(PythonErrorType.OverflowError, MESSAGE_INT_TO_BIG);
             }
+
             if (signed) {
                 while (0 <= index && index <= (byteCount - 1)) {
                     bytes[index] = signByte;