bug: BytesUtils.compare() Arithmetic Overflow (#420)

fixed overflow in `compare()` and added test case

Author: adraffy

contracts/test/TestBytesUtils.sol

@@ -134,6 +134,10 @@ contract TestBytesUtils {
             longChar.compare(otherLongChar) < 0 == true,
             "Compare long char with difference at start"
         );
+        require(
+            abi.encodePacked(type(int256).min).compare(abi.encodePacked(type(int256).max)) > 0,
+            "Compare maximum difference"
+        );
     }
 
     function testSubstring() public pure {

contracts/utils/BytesUtils.sol

@@ -123,15 +123,18 @@ library BytesUtils {
                 b := mload(otherptr)
             }
             if (a != b) {
-                // Mask out irrelevant bytes and check again
-                uint256 mask;
-                if (shortest - idx >= 32) {
-                    mask = type(uint256).max;
-                } else {
-                    mask = ~(2 ** (8 * (idx + 32 - shortest)) - 1);
+                uint256 rest = shortest - idx;
+                if (rest < 32) {
+                    // shift out the irrelevant bits
+                    rest = (32 - rest) << 3; // bits to drop
+                    a >>= rest;
+                    b >>= rest;
+                }
+                if (a < b) {
+                    return -1;
+                } else if (a > b) {
+                    return 1;
                 }
-                int256 diff = int256(a & mask) - int256(b & mask);
-                if (diff != 0) return diff;
             }
             selfptr += 32;
             otherptr += 32;