Integer can be hashed rapidly as well (JuliaLang#58440)

as noted in
JuliaLang#58388 (comment) ,
using `hash_bytes` for `BigInt` limbs in isolation will break some
hashing invariants. accordingly this PR updates also the `hash_integer`
fallback to use rapidhash


some surgery was needed to make `BigFloat` and `Rational` and such still
match, but I think (hope?) I got it all

below are some benchmarks. the `y` axis is nanoseconds and the `x` axis
is an input of size `1234^x`, so `length = 10` means input
`hash(big(1234^10))`

hashing `BigFloat` got a small bit slower, but this already allocates
and already seems less common than hashing `BigInt` (whose hashing
remains non-allocating)



![image](https://github.com/user-attachments/assets/df6de62d-5c8e-4779-b5fc-67143f868572)

![image](https://github.com/user-attachments/assets/5c7201b2-78a4-4e80-bee6-989fd29cfc86)

Author: adienes

base/gmp.jl

@@ -843,24 +843,25 @@ Base.deepcopy_internal(x::BigInt, stackdict::IdDict) = get!(() -> MPZ.set(x), st
 
 ## streamlined hashing for BigInt, by avoiding allocation from shifts ##
 
+Base._hash_shl!(x::BigInt, n) = MPZ.mul_2exp!(x, n)
+
 if Limb === UInt64 === UInt
     # On 64 bit systems we can define
     # an optimized version for BigInt of hash_integer (used e.g. for Rational{BigInt}),
     # and of hash
 
-    using .Base: hash_finalizer
+    using .Base: HASH_SECRET, hash_bytes, hash_finalizer
 
     function hash_integer(n::BigInt, h::UInt)
         GC.@preserve n begin
             s = n.size
-            s == 0 && return hash_integer(0, h)
-            p = convert(Ptr{UInt64}, n.d)
-            b = unsafe_load(p)
-            h ⊻= hash_finalizer(ifelse(s < 0, -b, b) ⊻ h)
-            for k = 2:abs(s)
-                h ⊻= hash_finalizer(unsafe_load(p, k) ⊻ h)
-            end
-            return h
+            h ⊻= (s < 0)
+            hash_bytes(
+                Ptr{UInt8}(n.d),
+                8 * abs(s),
+                h,
+                HASH_SECRET
+            )
         end
     end
 
@@ -892,21 +893,16 @@ if Limb === UInt64 === UInt
                 return hash(ldexp(flipsign(Float64(limb), sz), pow), h)
             end
             h = hash_integer(pow, h)
-            h ⊻= hash_finalizer(flipsign(limb, sz) ⊻ h)
-            for idx = idx+1:asz
-                if shift == 0
-                    limb = unsafe_load(ptr, idx)
-                else
-                    limb1 = limb2
-                    if idx == asz
-                        limb = limb1 >> shift
-                        limb == 0 && break # don't hash leading zeros
-                    else
-                        limb2 = unsafe_load(ptr, idx+1)
-                        limb = limb2 << upshift | limb1 >> shift
-                    end
-                end
-                h ⊻= hash_finalizer(limb ⊻ h)
+
+            h ⊻= (sz < 0)
+            trailing_zero_bytes = div(pow, 8)
+            GC.@preserve x begin
+                h = hash_bytes(
+                    Ptr{UInt8}(x.d) + 8 * trailing_zero_bytes,
+                    8 * (asz - trailing_zero_bytes),
+                    h,
+                    HASH_SECRET
+                )
             end
             return h
         end

base/hashing.jl

@@ -69,17 +69,72 @@ hash(x::UInt64, h::UInt) = hash_uint64(hash_mix_linear(x, h))
 hash(x::Int64, h::UInt) = hash(bitcast(UInt64, x), h)
 hash(x::Union{Bool, Int8, UInt8, Int16, UInt16, Int32, UInt32}, h::UInt) = hash(Int64(x), h)
 
-function hash_integer(n::Integer, h::UInt)
-    h ⊻= hash_uint((n % UInt) ⊻ h)
-    n = abs(n)
-    n >>>= sizeof(UInt) << 3
-    while n != 0
-        h ⊻= hash_uint((n % UInt) ⊻ h)
-        n >>>= sizeof(UInt) << 3
+hash_integer(x::Integer, h::UInt) = _hash_integer(x, UInt64(h)) % UInt
+function _hash_integer(
+        x::Integer,
+        seed::UInt64 = HASH_SEED,
+        secret::NTuple{3, UInt64} = HASH_SECRET
+    )
+    seed ⊻= (x < 0)
+    u = abs(x)
+
+    # always left-pad to multiple of 8 bytes
+    buflen = UInt(cld(top_set_bit(u), 64) * 8)
+    seed = seed ⊻ (hash_mix(seed ⊻ secret[1], secret[2]) ⊻ buflen)
+
+    a = zero(UInt64)
+    b = zero(UInt64)
+
+    if buflen ≤ 16
+        a = (UInt64(u % UInt32) << 32) |
+            UInt64((u >>> ((buflen - 4) * 8)) % UInt32)
+
+        delta = (buflen & 24) >>> (buflen >>> 3)
+
+        b = (UInt64((u >>> (8 * delta)) % UInt32) << 32) |
+            UInt64((u >>> (8 * (buflen - 4 - delta))) % UInt32)
+    else
+        a = (u >>> 8(buflen - 16)) % UInt
+        b = (u >>> 8(buflen - 8)) % UInt
+
+        i = buflen
+        if i > 48
+            see1 = seed
+            see2 = seed
+            while i ≥ 48
+                l0 = u % UInt; u >>>= 64
+                l1 = u % UInt; u >>>= 64
+                l2 = u % UInt; u >>>= 64
+                l3 = u % UInt; u >>>= 64
+                l4 = u % UInt; u >>>= 64
+                l5 = u % UInt; u >>>= 64
+
+                seed = hash_mix(l0 ⊻ secret[1], l1 ⊻ seed)
+                see1 = hash_mix(l2 ⊻ secret[2], l3 ⊻ see1)
+                see2 = hash_mix(l4 ⊻ secret[3], l5 ⊻ see2)
+            end
+            seed = seed ⊻ see1 ⊻ see2
+            i -= 48
+        end
+        if i > 16
+            l0 = u % UInt; u >>>= 64
+            l1 = u % UInt; u >>>= 64
+            seed = hash_mix(l0 ⊻ secret[3], l1 ⊻ seed ⊻ secret[2])
+            if i > 32
+                l2 = u % UInt; u >>>= 64
+                l3 = u % UInt; u >>>= 64
+                seed = hash_mix(l2 ⊻ secret[3], l3 ⊻ seed)
+            end
+        end
     end
-    return h
+
+    a = a ⊻ secret[2]
+    b = b ⊻ seed
+    b, a = mul_parts(a, b)
+    return hash_mix(a ⊻ secret[1] ⊻ buflen, b ⊻ secret[2])
 end
 
+
 ## efficient value-based hashing of floats ##
 
 const hx_NaN = hash(reinterpret(UInt64, NaN))
@@ -117,6 +172,7 @@ function hash(x::Float16, h::UInt)
 end
 
 ## generic hashing for rational values ##
+_hash_shl!(x, n) = (x << n)
 function hash(x::Real, h::UInt)
     # decompose x as num*2^pow/den
     num, pow, den = decompose(x)
@@ -132,6 +188,7 @@ function hash(x::Real, h::UInt)
         den = -den
     end
     num_z = trailing_zeros(num)
+
     num >>= num_z
     den_z = trailing_zeros(den)
     den >>= den_z
@@ -156,7 +213,10 @@ function hash(x::Real, h::UInt)
     end
     # handle generic rational values
     h = hash_integer(pow, h)
-    h = hash_integer(num, h)
+
+    # trimming only whole bytes of trailing zeros simplifies greatly
+    # some specializations for memory-backed bitintegers
+    h = hash_integer((pow > 0) ? _hash_shl!(num, pow % 8) : num, h)
     return h
 end
 
@@ -209,7 +269,7 @@ end
     else
         pos = 1
         i = buflen
-        while i ≥ 48
+        if i > 48
             see1 = seed
             see2 = seed
             while i ≥ 48

base/irrationals.jl

@@ -182,7 +182,7 @@ isinteger(::AbstractIrrational) = false
 iszero(::AbstractIrrational) = false
 isone(::AbstractIrrational) = false
 
-hash(x::Irrational, h::UInt) = 3*objectid(x) - h
+hash(x::Irrational, h::UInt) = 3h - objectid(x)
 
 widen(::Type{T}) where {T<:Irrational} = T
 

base/rational.jl

@@ -620,7 +620,7 @@ function hash(x::Rational{<:BitInteger64}, h::UInt)
         end
     end
     h = hash_integer(pow, h)
-    h = hash_integer(num, h)
+    h = hash_integer((pow > 0) ? (num << (pow % 64)) : num, h)
     return h
 end
 

test/gmp.jl

@@ -811,8 +811,14 @@ end
 
 @testset "hashing" begin
     for i in 1:10:100
-        bint = big(11)^i
-        bfloat = big(11.0)^i
-        @test (hash(bint) == hash(bfloat)) == (bint == bfloat)
+        for shift in 0:3
+            bint = big(11)^i << shift
+            bfloat = float(bint)
+            @test (hash(bint) == hash(bfloat)) == (bint == bfloat)
+            @test hash(bint, Base.HASH_SEED) ==
+                @invoke(hash(bint::Real, Base.HASH_SEED))
+            @test Base.hash_integer(bint, Base.HASH_SEED) ==
+                @invoke(Base.hash_integer(bint::Integer, Base.HASH_SEED))
+        end
     end
 end