Random: make a new "strong" RNG using SHA

The most convenient way to define `seed!` for new RNGs is via an
another RNG, with `seed!(rng::AbstractRNG, seeder::AbstractRNG)`.
But RNGs want to also support more usual seeds.

In order to allow them to only define the method above, a new
`SeedHasher` RNG is implemented, whose purpose is to convert
an initial given seed into a stream of random numbers.
Given that it's not always "safe" to seed an RNG from
another RNG, `SeedHasher` uses a strong cryptographic
hash (SHA2) to produces random streams.

The generic `seed!(rng::AbstractRNG, seed)` method now takes
care of forwarding the call to `seed!(rng, SeedHasher(seed))`.

Author: rfourquet

stdlib/Random/src/RNGs.jl

@@ -285,10 +285,81 @@ end
 
 ### seeding
 
+"""
+    Random.SeedHasher(seed=nothing)
+
+Create a `Random.SeedHasher` RNG object, which generates random bytes with the help
+of a cryptographic hash function (SHA2), via calls to [`Random.hash_seed`](@ref).
+
+Given two seeds `s1` and `s2`, the random streams generated by
+`SeedHasher(s1)` and `SeedHasher(s2)` should be distinct if and only if
+`s1` and `s2` are distinct.
+
+This RNG is used by default in `Random.seed!(::AbstractRNG, seed::Any)`, such that
+RNGs usually need only to implement `seed!(rng, ::AbstractRNG)`.
+
+This is an internal type, subject to change.
+"""
+mutable struct SeedHasher <: AbstractRNG
+    bytes::Vector{UInt8}
+    idx::Int
+    cnt::Int64
+
+    SeedHasher(seed=nothing) = seed!(new(), seed)
+end
+
+seed!(rng::SeedHasher, seeder::AbstractRNG) = seed!(rng, rand(seeder, UInt64, 4))
+seed!(rng::SeedHasher, ::Nothing) = seed!(rng, RandomDevice())
+
+function seed!(rng::SeedHasher, seed)
+    # typically, no more than 256 bits will be needed, so use
+    # SHA2_256 because it's faster
+    ctx = SHA2_256_CTX()
+    hash_seed(seed, ctx)
+    rng.bytes = SHA.digest!(ctx)::Vector{UInt8}
+    rng.idx = 0
+    rng.cnt = 0
+    rng
+end
+
+@noinline function rehash!(rng::SeedHasher)
+    # more random bytes are necessary, from now on use SHA2_512 to generate
+    # more bytes at once
+    ctx = SHA2_512_CTX()
+    SHA.update!(ctx, rng.bytes)
+    # also hash the counter, just for the extremely unlikely case where the hash of
+    # rng.bytes is equal to rng.bytes (i.e. rng.bytes is a "fixed point"), or more generally
+    # if there is a small cycle
+    SHA.update!(ctx, reinterpret(NTuple{8, UInt8}, rng.cnt += 1))
+    rng.bytes = SHA.digest!(ctx)
+    rng.idx = 0
+    rng
+end
+
+function rand(rng::SeedHasher, ::SamplerType{UInt8})
+    rng.idx < length(rng.bytes) || rehash!(rng)
+    rng.bytes[rng.idx += 1]
+end
+
+for TT = Base.BitInteger_types
+    TT === UInt8 && continue
+    @eval function rand(rng::SeedHasher, ::SamplerType{$TT})
+        xx = zero($TT)
+        for ii = 0:sizeof($TT)-1
+            xx |= (rand(rng, UInt8) % $TT) << (8 * ii)
+        end
+        xx
+    end
+end
+
+rand(rng::SeedHasher, ::SamplerType{Bool}) = rand(rng, UInt8) % Bool
+
+rng_native_52(::SeedHasher) = UInt64
+
+
 #### hash_seed()
 
-function hash_seed(seed::Integer)
-    ctx = SHA.SHA2_256_CTX()
+function hash_seed(seed::Integer, ctx::SHA_CTX)
     neg = signbit(seed)
     if neg
         seed = ~seed
@@ -302,21 +373,18 @@ function hash_seed(seed::Integer)
     end
     # make sure the hash of negative numbers is different from the hash of positive numbers
     neg && SHA.update!(ctx, (0x01,))
-    SHA.digest!(ctx)
+    nothing
 end
 
-function hash_seed(seed::Union{AbstractArray{UInt32}, AbstractArray{UInt64}})
-    ctx = SHA.SHA2_256_CTX()
+function hash_seed(seed::Union{AbstractArray{UInt32}, AbstractArray{UInt64}}, ctx::SHA_CTX)
     for xx in seed
         SHA.update!(ctx, reinterpret(NTuple{8, UInt8}, UInt64(xx)))
     end
     # discriminate from hash_seed(::Integer)
     SHA.update!(ctx, (0x10,))
-    SHA.digest!(ctx)
 end
 
-function hash_seed(str::AbstractString)
-    ctx = SHA.SHA2_256_CTX()
+function hash_seed(str::AbstractString, ctx::SHA_CTX)
     # convert to String such that `codeunits(str)` below is consistent between equal
     # strings of different types
     str = String(str)
@@ -331,25 +399,29 @@ function hash_seed(str::AbstractString)
         SHA.update!(ctx, (pad % UInt8,))
     end
     SHA.update!(ctx, (0x05,))
-    SHA.digest!(ctx)
 end
 
 
 """
-    hash_seed(seed)::AbstractVector{UInt8}
+    Random.hash_seed(seed, ctx::SHA_CTX)::AbstractVector{UInt8}
+
+Update `ctx` via `SHA.update!` with the content of `seed`.
+This function is used by the [`SeedHasher`](@ref) RNG to produce
+random bytes.
 
-Return a cryptographic hash of `seed` of size 256 bits (32 bytes).
 `seed` can currently be of type
 `Union{Integer, AbstractString, AbstractArray{UInt32}, AbstractArray{UInt64}}`,
 but modules can extend this function for types they own.
 
-`hash_seed` is "injective" : if `n != m`, then `hash_seed(n) != `hash_seed(m)`.
-Moreover, if `n == m`, then `hash_seed(n) == hash_seed(m)`.
-
-This is an internal function subject to change.
+`hash_seed` is "injective" : for two equivalent context objects `cn` and `cm`,
+if `n != m`, then `cn` and `cm` will be distinct after calling
+`hash_seed(n, cn); hash_seed(m, cm)`.
+Moreover, if `n == m`, then `cn` and `cm` remain equivalent after calling
+`hash_seed(n, cn); hash_seed(m, cm)`.
 """
 hash_seed
 
+
 #### seed!()
 
 function initstate!(r::MersenneTwister, data::StridedVector, seed)
@@ -372,7 +444,7 @@ end
 # seeds, while having them printed reasonably tersely.
 seed!(r::MersenneTwister, seeder::AbstractRNG) = seed!(r, rand(seeder, UInt128))
 seed!(r::MersenneTwister, ::Nothing) = seed!(r, RandomDevice())
-seed!(r::MersenneTwister, seed) = initstate!(r, hash_seed(seed), seed)
+seed!(r::MersenneTwister, seed) = initstate!(r, rand(SeedHasher(seed), UInt32, 8), seed)
 
 
 ### Global RNG

stdlib/Random/src/Random.jl

@@ -13,7 +13,7 @@ include("DSFMT.jl")
 using .DSFMT
 using Base.GMP.MPZ
 using Base.GMP: Limb
-import SHA
+using SHA: SHA, SHA2_256_CTX, SHA2_512_CTX, SHA_CTX
 
 using Base: BitInteger, BitInteger_types, BitUnsigned, require_one_based_indexing
 import Base: copymutable, copy, copy!, ==, hash, convert,
@@ -457,11 +457,14 @@ julia> rand(Xoshiro(), Bool) # not reproducible either
 true
 ```
 """
-function seed!(rng::AbstractRNG, seed=nothing)
+function seed!(rng::AbstractRNG, seed::Any=nothing)
     if seed === nothing
         seed!(rng, RandomDevice())
-    else
+    elseif seed isa AbstractRNG
+        # avoid getting into an infinite recursive call from the other branches
         throw(MethodError(seed!, (rng, seed)))
+    else
+        seed!(rng, SeedHasher(seed))
     end
 end
 

stdlib/Random/src/Xoshiro.jl

@@ -246,11 +246,6 @@ hash(x::Union{TaskLocalRNG, Xoshiro}, h::UInt) = hash(getstate(x), h + 0x49a62c2
 seed!(rng::Union{TaskLocalRNG, Xoshiro}, seeder::AbstractRNG) =
     initstate!(rng, rand(seeder, NTuple{4, UInt64}))
 
-seed!(rng::Union{TaskLocalRNG, Xoshiro}, ::Nothing) = @invoke seed!(rng::AbstractRNG, nothing::Any)
-
-seed!(rng::Union{TaskLocalRNG, Xoshiro}, seed) =
-    initstate!(rng, reinterpret(UInt64, hash_seed(seed)))
-
 
 @inline function rand(x::Union{TaskLocalRNG, Xoshiro}, ::SamplerType{UInt64})
     s0, s1, s2, s3 = getstate(x)

stdlib/Random/test/runtests.jl

@@ -11,8 +11,9 @@ using Random
 using Random.DSFMT
 
 using Random: default_rng, Sampler, SamplerRangeFast, SamplerRangeInt, SamplerRangeNDL, MT_CACHE_F, MT_CACHE_I
-using Random: jump_128, jump_192, jump_128!, jump_192!
+using Random: jump_128, jump_192, jump_128!, jump_192!, SeedHasher
 
+import SHA
 import Future # randjump
 
 function test_uniform(xs::AbstractArray{T}) where {T<:AbstractFloat}
@@ -297,7 +298,7 @@ for f in (:<, :<=, :>, :>=, :(==), :(!=))
 end
 
 # test all rand APIs
-for rng in ([], [MersenneTwister(0)], [RandomDevice()], [Xoshiro()])
+for rng in ([], [MersenneTwister(0)], [RandomDevice()], [Xoshiro(0)], [SeedHasher(0)])
     realrng = rng == [] ? default_rng() : only(rng)
     ftypes = [Float16, Float32, Float64, FakeFloat64, BigFloat]
     cftypes = [ComplexF16, ComplexF32, ComplexF64, ftypes...]
@@ -453,7 +454,8 @@ function hist(X, n)
 end
 
 @testset "uniform distribution of floats" begin
-    for rng in [MersenneTwister(), RandomDevice(), Xoshiro()],
+    seed = rand(UInt128)
+    for rng in [MersenneTwister(seed), RandomDevice(), Xoshiro(seed), SeedHasher(seed)],
         T in [Float16, Float32, Float64, BigFloat],
         prec in (T == BigFloat ? [3, 53, 64, 100, 256, 1000] : [256])
 
@@ -480,7 +482,8 @@ end
         # but also for 3 linear combinations of positions (for the array version)
         lcs = unique!.([rand(1:n, 2), rand(1:n, 3), rand(1:n, 5)])
         aslcs = zeros(Int, 3)
-        for rng = (MersenneTwister(), RandomDevice(), Xoshiro())
+        seed = rand(UInt128)
+        for rng = (MersenneTwister(seed), RandomDevice(), Xoshiro(seed), SeedHasher(seed))
             for scalar = [false, true]
                 fill!(a, 0)
                 fill!(as, 0)
@@ -662,7 +665,7 @@ end
 @testset "Random.seed!(rng, ...) returns rng" begin
     # issue #21248
     seed = rand(UInt)
-    for m = ([MersenneTwister(seed)], [Xoshiro(seed)], [])
+    for m = ([MersenneTwister(seed)], [Xoshiro(seed)], [SeedHasher(seed)], [])
         m2 = m == [] ? default_rng() : m[1]
         @test Random.seed!(m...) === m2
         @test Random.seed!(m..., rand(UInt)) === m2
@@ -708,7 +711,7 @@ end
 # this shouldn't crash (#22403)
 @test_throws MethodError rand!(Union{UInt,Int}[1, 2, 3])
 
-@testset "$RNG() & Random.seed!(rng::$RNG) initializes randomly" for RNG in (MersenneTwister, RandomDevice, Xoshiro)
+@testset "$RNG() & Random.seed!(rng::$RNG) initializes randomly" for RNG in (MersenneTwister, RandomDevice, Xoshiro, SeedHasher)
     m = RNG()
     a = rand(m, Int)
     m = RNG()
@@ -729,7 +732,7 @@ end
     @test rand(m, Int) ∉ (a, b, c, d)
 end
 
-@testset "$RNG(seed) & Random.seed!(m::$RNG, seed) produce the same stream" for RNG=(MersenneTwister, Xoshiro)
+@testset "$RNG(seed) & Random.seed!(m::$RNG, seed) produce the same stream" for RNG=(MersenneTwister, Xoshiro, SeedHasher)
     seeds = Any[0, 1, 2, 10000, 10001, rand(UInt32, 8), randstring(), randstring(), rand(UInt128, 3)...]
     if RNG == Xoshiro
         push!(seeds, rand(UInt64, rand(1:4)))
@@ -769,7 +772,10 @@ struct RandomStruct23964 end
     @test_throws MethodError rand(RandomStruct23964())
 end
 
-@testset "rand(::$(typeof(RNG)), ::UnitRange{$T}" for RNG ∈ (MersenneTwister(rand(UInt128)), RandomDevice(), Xoshiro()),
+@testset "rand(::$(typeof(RNG)), ::UnitRange{$T}" for RNG ∈ (MersenneTwister(rand(UInt128)),
+                                                             RandomDevice(),
+                                                             Xoshiro(rand(UInt128)),
+                                                             SeedHasher(rand(UInt128))),
                                                         T ∈ (Bool, Int8, Int16, Int32, UInt32, Int64, Int128, UInt128)
     if T === Bool
         @test rand(RNG, false:true) ∈ (false, true)
@@ -912,8 +918,11 @@ end
     @test rand(rng) == rand(GLOBAL_RNG)
 end
 
-@testset "RNGs broadcast as scalars: T" for T in (MersenneTwister, RandomDevice)
-    @test length.(rand.(T(), 1:3)) == 1:3
+@testset "RNGs broadcast as scalars: $(typeof(RNG))" for RNG in (MersenneTwister(0),
+                                                                 RandomDevice(),
+                                                                 Xoshiro(0),
+                                                                 SeedHasher(0))
+    @test length.(rand.(RNG, 1:3)) == 1:3
 end
 
 @testset "generated scalar integers do not overlap" begin
@@ -1211,7 +1220,14 @@ end
     end
 end
 
+
 @testset "seed! and hash_seed" begin
+    function hash_seed(seed)
+        ctx = SHA.SHA2_256_CTX()
+        Random.hash_seed(seed, ctx)
+        bytes2hex(SHA.digest!(ctx))
+    end
+
     # Test that:
     # 1) if n == m, then hash_seed(n) == hash_seed(m)
     # 2) if n != m, then hash_seed(n) != hash_seed(m)
@@ -1224,12 +1240,12 @@ end
         T <: Signed && push!(seeds, T(0), T(1), T(2), T(-1), T(-2))
     end
 
-    vseeds = Dict{Vector{UInt8}, BigInt}()
+    vseeds = Dict{String, BigInt}()
     for seed = seeds
         bigseed = big(seed)
-        vseed = Random.hash_seed(bigseed)
+        vseed = hash_seed(bigseed)
         # test property 1) above
-        @test Random.hash_seed(seed) == vseed
+        @test hash_seed(seed) == vseed
         # test property 2) above
         @test bigseed == get!(vseeds, vseed, bigseed)
         # test that the property 1) is actually inherited by `seed!`
@@ -1241,16 +1257,16 @@ end
     end
 
     seed32 = rand(UInt32, rand(1:9))
-    hash32 = Random.hash_seed(seed32)
-    @test Random.hash_seed(map(UInt64, seed32)) == hash32
+    hash32 = hash_seed(seed32)
+    @test hash_seed(map(UInt64, seed32)) == hash32
     @test hash32 ∉ keys(vseeds)
 
     seed_str = randstring()
     seed_gstr = GenericString(seed_str)
-    @test Random.hash_seed(seed_str) == Random.hash_seed(seed_gstr)
-    string_seeds = Set{Vector{UInt8}}()
+    @test hash_seed(seed_str) == hash_seed(seed_gstr)
+    string_seeds = Set{String}()
     for ch = 'A':'z'
-        vseed = Random.hash_seed(string(ch))
+        vseed = hash_seed(string(ch))
         @test vseed ∉ keys(vseeds)
         @test vseed ∉ string_seeds
         push!(string_seeds, vseed)