Follow-up to hashing BigInts; expand testing (JuliaLang#58714)

fixes JuliaLang#58711
closes JuliaLang#58712

Author: adienes

base/gmp.jl

@@ -853,12 +853,16 @@ if Limb === UInt64 === UInt
     using .Base: HASH_SECRET, hash_bytes, hash_finalizer
 
     function hash_integer(n::BigInt, h::UInt)
+        iszero(n) && return hash_integer(0, h)
         GC.@preserve n begin
             s = n.size
             h ⊻= (s < 0)
+
+            us = abs(s)
+            leading_zero_bytes = div(leading_zeros(unsafe_load(n.d, us)), 8)
             hash_bytes(
                 Ptr{UInt8}(n.d),
-                8 * abs(s),
+                8 * us - leading_zero_bytes,
                 h,
                 HASH_SECRET
             )
@@ -895,16 +899,14 @@ if Limb === UInt64 === UInt
             h = hash_integer(pow, h)
 
             h ⊻= (sz < 0)
+            leading_zero_bytes = div(leading_zeros(unsafe_load(x.d, asz)), 8)
             trailing_zero_bytes = div(pow, 8)
-            GC.@preserve x begin
-                h = hash_bytes(
-                    Ptr{UInt8}(x.d) + 8 * trailing_zero_bytes,
-                    8 * (asz - trailing_zero_bytes),
-                    h,
-                    HASH_SECRET
-                )
-            end
-            return h
+            return hash_bytes(
+                Ptr{UInt8}(x.d) + trailing_zero_bytes,
+                8 * asz - (leading_zero_bytes + trailing_zero_bytes),
+                h,
+                HASH_SECRET
+            )
         end
     end
 end

base/hashing.jl

@@ -78,21 +78,30 @@ function _hash_integer(
     seed ⊻= (x < 0)
     u = abs(x)
 
-    # always left-pad to multiple of 8 bytes
-    buflen = UInt(cld(top_set_bit(u), 64) * 8)
+    # always left-pad to full byte
+    buflen = UInt(max(cld(top_set_bit(u), 8), 1))
     seed = seed ⊻ (hash_mix(seed ⊻ secret[1], secret[2]) ⊻ buflen)
 
     a = zero(UInt64)
     b = zero(UInt64)
 
     if buflen ≤ 16
-        a = (UInt64(u % UInt32) << 32) |
-            UInt64((u >>> ((buflen - 4) * 8)) % UInt32)
+        if buflen ≥ 4
+            a = (UInt64(u % UInt32) << 32) |
+                UInt64((u >>> ((buflen - 4) * 8)) % UInt32)
 
-        delta = (buflen & 24) >>> (buflen >>> 3)
+            delta = (buflen & 24) >>> (buflen >>> 3)
 
-        b = (UInt64((u >>> (8 * delta)) % UInt32) << 32) |
-            UInt64((u >>> (8 * (buflen - 4 - delta))) % UInt32)
+            b = (UInt64((u >>> (8 * delta)) % UInt32) << 32) |
+                UInt64((u >>> (8 * (buflen - 4 - delta))) % UInt32)
+        else # buflen > 0
+            b0 = u % UInt8
+            b1 = (u >>> (8 * div(buflen, 2))) % UInt8
+            b2 = (u >>> (8 * (buflen - 1))) % UInt8
+            a = (UInt64(b0) << 56) |
+                (UInt64(b1) << 32) |
+                UInt64(b2)
+        end
     else
         a = (u >>> 8(buflen - 16)) % UInt
         b = (u >>> 8(buflen - 8)) % UInt
@@ -112,9 +121,9 @@ function _hash_integer(
                 seed = hash_mix(l0 ⊻ secret[1], l1 ⊻ seed)
                 see1 = hash_mix(l2 ⊻ secret[2], l3 ⊻ see1)
                 see2 = hash_mix(l4 ⊻ secret[3], l5 ⊻ see2)
+                i -= 48
             end
             seed = seed ⊻ see1 ⊻ see2
-            i -= 48
         end
         if i > 16
             l0 = u % UInt; u >>>= 64

test/gmp.jl

@@ -811,14 +811,24 @@ end
 
 @testset "hashing" begin
     for i in 1:10:100
-        for shift in 0:3
-            bint = big(11)^i << shift
-            bfloat = float(bint)
-            @test (hash(bint) == hash(bfloat)) == (bint == bfloat)
-            @test hash(bint, Base.HASH_SEED) ==
-                @invoke(hash(bint::Real, Base.HASH_SEED))
-            @test Base.hash_integer(bint, Base.HASH_SEED) ==
-                @invoke(Base.hash_integer(bint::Integer, Base.HASH_SEED))
+        for shift in vcat(0:8, 9:8:81)
+            for sgn in (1, -1)
+                bint = sgn * (big(11)^i << shift)
+                bfloat = float(bint)
+                @test (hash(bint) == hash(bfloat)) == (bint == bfloat)
+                @test hash(bint, Base.HASH_SEED) ==
+                    @invoke(hash(bint::Real, Base.HASH_SEED))
+                @test Base.hash_integer(bint, Base.HASH_SEED) ==
+                    @invoke(Base.hash_integer(bint::Integer, Base.HASH_SEED))
+            end
         end
     end
+
+    bint = big(0)
+    bfloat = float(bint)
+    @test (hash(bint) == hash(bfloat)) == (bint == bfloat)
+    @test hash(bint, Base.HASH_SEED) ==
+        @invoke(hash(bint::Real, Base.HASH_SEED))
+    @test Base.hash_integer(bint, Base.HASH_SEED) ==
+        @invoke(Base.hash_integer(bint::Integer, Base.HASH_SEED))
 end