Random: allow seeding from an RNG (#51533)

rfourquet · web-flow · commit 6adb4cf355d5 · 2025-05-06T15:22:49.000+02:00
This is the combination of two changes:

1) Allow seeding an RNG from another RNG, e.g. `seed!(Xoshiro(),
RandomDevice())`, because it's convenient, in particular for RNGs which
need a variable number of random bytes (think "rejection sampling").

2) In turn, it's convenient to only having to define this one method for
a given RNG type, and delegate handling of other kind of seeds (integer,
strings, ...) to another object: `SeedHasher(seed)` is a new RNG which
produces random streams from such seeds, with the help of SHA2; this
essentially gives an RNG API to what we are already using to transform
integer seeds for `Xoshiro` and `MersenneTwister` into vectors suitable
for their initialization.

While `SeedHasher` shouldn't be considered as a CSPRNG, it should have
very good statistical properties; if we go with this PR, I will first
run PractRand (looks good so far) and BigCrush tests before merging.

The API is not changed in this PR, `SeedHasher` remains internal; the goal
is to experiment more with it before commiting to an API, possibly in
v1.13.
diff --git a/stdlib/Random/src/RNGs.jl b/stdlib/Random/src/RNGs.jl
@@ -285,10 +285,81 @@ end
 
 ### seeding
 
+"""
+    Random.SeedHasher(seed=nothing)
+
+Create a `Random.SeedHasher` RNG object, which generates random bytes with the help
+of a cryptographic hash function (SHA2), via calls to [`Random.hash_seed`](@ref).
+
+Given two seeds `s1` and `s2`, the random streams generated by
+`SeedHasher(s1)` and `SeedHasher(s2)` should be distinct if and only if
+`s1` and `s2` are distinct.
+
+This RNG is used by default in `Random.seed!(::AbstractRNG, seed::Any)`, such that
+RNGs usually need only to implement `seed!(rng, ::AbstractRNG)`.
+
+This is an internal type, subject to change.
+"""
+mutable struct SeedHasher <: AbstractRNG
+    bytes::Vector{UInt8}
+    idx::Int
+    cnt::Int64
+
+    SeedHasher(seed=nothing) = seed!(new(), seed)
+end
+
+seed!(rng::SeedHasher, seeder::AbstractRNG) = seed!(rng, rand(seeder, UInt64, 4))
+seed!(rng::SeedHasher, ::Nothing) = seed!(rng, RandomDevice())
+
+function seed!(rng::SeedHasher, seed)
+    # typically, no more than 256 bits will be needed, so use
+    # SHA2_256 because it's faster
+    ctx = SHA2_256_CTX()
+    hash_seed(seed, ctx)
+    rng.bytes = SHA.digest!(ctx)::Vector{UInt8}
+    rng.idx = 0
+    rng.cnt = 0
+    rng
+end
+
+@noinline function rehash!(rng::SeedHasher)
+    # more random bytes are necessary, from now on use SHA2_512 to generate
+    # more bytes at once
+    ctx = SHA2_512_CTX()
+    SHA.update!(ctx, rng.bytes)
+    # also hash the counter, just for the extremely unlikely case where the hash of
+    # rng.bytes is equal to rng.bytes (i.e. rng.bytes is a "fixed point"), or more generally
+    # if there is a small cycle
+    SHA.update!(ctx, reinterpret(NTuple{8, UInt8}, rng.cnt += 1))
+    rng.bytes = SHA.digest!(ctx)
+    rng.idx = 0
+    rng
+end
+
+function rand(rng::SeedHasher, ::SamplerType{UInt8})
+    rng.idx < length(rng.bytes) || rehash!(rng)
+    rng.bytes[rng.idx += 1]
+end
+
+for TT = Base.BitInteger_types
+    TT === UInt8 && continue
+    @eval function rand(rng::SeedHasher, ::SamplerType{$TT})
+        xx = zero($TT)
+        for ii = 0:sizeof($TT)-1
+            xx |= (rand(rng, UInt8) % $TT) << (8 * ii)
+        end
+        xx
+    end
+end
+
+rand(rng::SeedHasher, ::SamplerType{Bool}) = rand(rng, UInt8) % Bool
+
+rng_native_52(::SeedHasher) = UInt64
+
+
 #### hash_seed()
 
-function hash_seed(seed::Integer)
-    ctx = SHA.SHA2_256_CTX()
+function hash_seed(seed::Integer, ctx::SHA_CTX)
     neg = signbit(seed)
     if neg
         seed = ~seed
@@ -302,21 +373,18 @@ function hash_seed(seed::Integer)
     end
     # make sure the hash of negative numbers is different from the hash of positive numbers
     neg && SHA.update!(ctx, (0x01,))
-    SHA.digest!(ctx)
+    nothing
 end
 
-function hash_seed(seed::Union{AbstractArray{UInt32}, AbstractArray{UInt64}})
-    ctx = SHA.SHA2_256_CTX()
+function hash_seed(seed::Union{AbstractArray{UInt32}, AbstractArray{UInt64}}, ctx::SHA_CTX)
     for xx in seed
         SHA.update!(ctx, reinterpret(NTuple{8, UInt8}, UInt64(xx)))
     end
     # discriminate from hash_seed(::Integer)
     SHA.update!(ctx, (0x10,))
-    SHA.digest!(ctx)
 end
 
-function hash_seed(str::AbstractString)
-    ctx = SHA.SHA2_256_CTX()
+function hash_seed(str::AbstractString, ctx::SHA_CTX)
     # convert to String such that `codeunits(str)` below is consistent between equal
     # strings of different types
     str = String(str)
@@ -331,25 +399,29 @@ function hash_seed(str::AbstractString)
         SHA.update!(ctx, (pad % UInt8,))
     end
     SHA.update!(ctx, (0x05,))
-    SHA.digest!(ctx)
 end
 
 
 """
-    hash_seed(seed)::AbstractVector{UInt8}
+    Random.hash_seed(seed, ctx::SHA_CTX)::AbstractVector{UInt8}
+
+Update `ctx` via `SHA.update!` with the content of `seed`.
+This function is used by the [`SeedHasher`](@ref) RNG to produce
+random bytes.
 
-Return a cryptographic hash of `seed` of size 256 bits (32 bytes).
 `seed` can currently be of type
 `Union{Integer, AbstractString, AbstractArray{UInt32}, AbstractArray{UInt64}}`,
 but modules can extend this function for types they own.
 
-`hash_seed` is "injective" : if `n != m`, then `hash_seed(n) != `hash_seed(m)`.
-Moreover, if `n == m`, then `hash_seed(n) == hash_seed(m)`.
-
-This is an internal function subject to change.
+`hash_seed` is "injective" : for two equivalent context objects `cn` and `cm`,
+if `n != m`, then `cn` and `cm` will be distinct after calling
+`hash_seed(n, cn); hash_seed(m, cm)`.
+Moreover, if `n == m`, then `cn` and `cm` remain equivalent after calling
+`hash_seed(n, cn); hash_seed(m, cm)`.
 """
 hash_seed
 
+
 #### seed!()
 
 function initstate!(r::MersenneTwister, data::StridedVector, seed)
@@ -365,13 +437,14 @@ function initstate!(r::MersenneTwister, data::StridedVector, seed)
 end
 
 # When a seed is not provided, we generate one via `RandomDevice()` rather
-# than calling directly `initstate!` with `rand(RandomDevice(), UInt32, whatever)` because the
+# than calling directly `initstate!` with `rand(RandomDevice(), UInt32, 8)` because the
 # seed is printed in `show(::MersenneTwister)`, so we need one; the cost of `hash_seed` is a
-# small overhead compared to `initstate!`, so this simple solution is fine.
-# A random seed with 128 bits is a good compromise for almost surely always getting distinct
+# small overhead compared to `initstate!`.
+# A random seed with 128 bits is a good compromise for almost surely getting distinct
 # seeds, while having them printed reasonably tersely.
-seed!(r::MersenneTwister, ::Nothing) = seed!(r, rand(RandomDevice(), UInt128))
-seed!(r::MersenneTwister, seed) = initstate!(r, hash_seed(seed), seed)
+seed!(r::MersenneTwister, seeder::AbstractRNG) = seed!(r, rand(seeder, UInt128))
+seed!(r::MersenneTwister, ::Nothing) = seed!(r, RandomDevice())
+seed!(r::MersenneTwister, seed) = initstate!(r, rand(SeedHasher(seed), UInt32, 8), seed)
 
 
 ### Global RNG
diff --git a/stdlib/Random/src/Random.jl b/stdlib/Random/src/Random.jl
@@ -13,7 +13,7 @@ include("DSFMT.jl")
 using .DSFMT
 using Base.GMP.MPZ
 using Base.GMP: Limb
-import SHA
+using SHA: SHA, SHA2_256_CTX, SHA2_512_CTX, SHA_CTX
 
 using Base: BitInteger, BitInteger_types, BitUnsigned, require_one_based_indexing
 import Base: copymutable, copy, copy!, ==, hash, convert,
@@ -414,8 +414,10 @@ sequence of numbers if and only if a `seed` is provided. Some RNGs
 don't accept a seed, like `RandomDevice`.
 After the call to `seed!`, `rng` is equivalent to a newly created
 object initialized with the same seed.
+
 The types of accepted seeds depend on the type of `rng`, but in general,
-integer seeds should work.
+integer seeds should work. Providing `nothing` as the seed should be
+equivalent to not providing one.
 
 If `rng` is not specified, it defaults to seeding the state of the
 shared task-local generator.
@@ -455,11 +457,15 @@ julia> rand(Xoshiro(), Bool) # not reproducible either
 true
 ```
 """
-seed!(rng::AbstractRNG) = seed!(rng, nothing)
-#=
-We have this generic definition instead of the alternative option
-`seed!(rng::AbstractRNG, ::Nothing) = seed!(rng)`
-because it would lead too easily to ambiguities, e.g. when we define `seed!(::Xoshiro, seed)`.
-=#
+function seed!(rng::AbstractRNG, seed::Any=nothing)
+    if seed === nothing
+        seed!(rng, RandomDevice())
+    elseif seed isa AbstractRNG
+        # avoid getting into an infinite recursive call from the other branches
+        throw(MethodError(seed!, (rng, seed)))
+    else
+        seed!(rng, SeedHasher(seed))
+    end
+end
 
 end # module
diff --git a/stdlib/Random/src/Xoshiro.jl b/stdlib/Random/src/Xoshiro.jl
@@ -243,13 +243,8 @@ copy!(dst::Union{TaskLocalRNG, Xoshiro}, src::Union{TaskLocalRNG, Xoshiro}) = se
 # use a magic (random) number to scramble `h` so that `hash(x)` is distinct from `hash(getstate(x))`
 hash(x::Union{TaskLocalRNG, Xoshiro}, h::UInt) = hash(getstate(x), h + 0x49a62c2dda6fa9be % UInt)
 
-function seed!(rng::Union{TaskLocalRNG, Xoshiro}, ::Nothing)
-    # as we get good randomness from RandomDevice, we can skip hashing
-    initstate!(rng, rand(RandomDevice(), NTuple{4, UInt64}))
-end
-
-seed!(rng::Union{TaskLocalRNG, Xoshiro}, seed) =
-    initstate!(rng, reinterpret(UInt64, hash_seed(seed)))
+seed!(rng::Union{TaskLocalRNG, Xoshiro}, seeder::AbstractRNG) =
+    initstate!(rng, rand(seeder, NTuple{4, UInt64}))
 
 
 @inline function rand(x::Union{TaskLocalRNG, Xoshiro}, ::SamplerType{UInt64})
diff --git a/stdlib/Random/test/runtests.jl b/stdlib/Random/test/runtests.jl
@@ -11,8 +11,9 @@ using Random
 using Random.DSFMT
 
 using Random: default_rng, Sampler, SamplerRangeFast, SamplerRangeInt, SamplerRangeNDL, MT_CACHE_F, MT_CACHE_I
-using Random: jump_128, jump_192, jump_128!, jump_192!
+using Random: jump_128, jump_192, jump_128!, jump_192!, SeedHasher
 
+import SHA
 import Future # randjump
 
 function test_uniform(xs::AbstractArray{T}) where {T<:AbstractFloat}
@@ -297,7 +298,7 @@ for f in (:<, :<=, :>, :>=, :(==), :(!=))
 end
 
 # test all rand APIs
-for rng in ([], [MersenneTwister(0)], [RandomDevice()], [Xoshiro()])
+for rng in ([], [MersenneTwister(0)], [RandomDevice()], [Xoshiro(0)], [SeedHasher(0)])
     realrng = rng == [] ? default_rng() : only(rng)
     ftypes = [Float16, Float32, Float64, FakeFloat64, BigFloat]
     cftypes = [ComplexF16, ComplexF32, ComplexF64, ftypes...]
@@ -453,7 +454,8 @@ function hist(X, n)
 end
 
 @testset "uniform distribution of floats" begin
-    for rng in [MersenneTwister(), RandomDevice(), Xoshiro()],
+    seed = rand(UInt128)
+    for rng in [MersenneTwister(seed), RandomDevice(), Xoshiro(seed), SeedHasher(seed)],
         T in [Float16, Float32, Float64, BigFloat],
         prec in (T == BigFloat ? [3, 53, 64, 100, 256, 1000] : [256])
 
@@ -480,7 +482,8 @@ end
         # but also for 3 linear combinations of positions (for the array version)
         lcs = unique!.([rand(1:n, 2), rand(1:n, 3), rand(1:n, 5)])
         aslcs = zeros(Int, 3)
-        for rng = (MersenneTwister(), RandomDevice(), Xoshiro())
+        seed = rand(UInt128)
+        for rng = (MersenneTwister(seed), RandomDevice(), Xoshiro(seed), SeedHasher(seed))
             for scalar = [false, true]
                 fill!(a, 0)
                 fill!(as, 0)
@@ -662,7 +665,7 @@ end
 @testset "Random.seed!(rng, ...) returns rng" begin
     # issue #21248
     seed = rand(UInt)
-    for m = ([MersenneTwister(seed)], [Xoshiro(seed)], [])
+    for m = ([MersenneTwister(seed)], [Xoshiro(seed)], [SeedHasher(seed)], [])
         m2 = m == [] ? default_rng() : m[1]
         @test Random.seed!(m...) === m2
         @test Random.seed!(m..., rand(UInt)) === m2
@@ -675,6 +678,7 @@ end
         @test Random.seed!(m..., typemax(UInt)) === m2
         @test Random.seed!(m..., typemax(UInt128)) === m2
         @test Random.seed!(m..., "a random seed") === m2
+        @test Random.seed!(m..., Random.default_rng()) === m2
     end
 end
 
@@ -707,7 +711,7 @@ end
 # this shouldn't crash (#22403)
 @test_throws MethodError rand!(Union{UInt,Int}[1, 2, 3])
 
-@testset "$RNG() & Random.seed!(rng::$RNG) initializes randomly" for RNG in (MersenneTwister, RandomDevice, Xoshiro)
+@testset "$RNG() & Random.seed!(rng::$RNG) initializes randomly" for RNG in (MersenneTwister, RandomDevice, Xoshiro, SeedHasher)
     m = RNG()
     a = rand(m, Int)
     m = RNG()
@@ -728,7 +732,7 @@ end
     @test rand(m, Int) ∉ (a, b, c, d)
 end
 
-@testset "$RNG(seed) & Random.seed!(m::$RNG, seed) produce the same stream" for RNG=(MersenneTwister,Xoshiro)
+@testset "$RNG(seed) & Random.seed!(m::$RNG, seed) produce the same stream" for RNG=(MersenneTwister, Xoshiro, SeedHasher)
     seeds = Any[0, 1, 2, 10000, 10001, rand(UInt32, 8), randstring(), randstring(), rand(UInt128, 3)...]
     if RNG == Xoshiro
         push!(seeds, rand(UInt64, rand(1:4)))
@@ -739,6 +743,11 @@ end
         Random.seed!(m, seed)
         @test a == [rand(m) for _=1:100]
     end
+    # rng as a seed
+    m = RNG(Xoshiro(0))
+    a = [rand(m) for _=1:100]
+    Random.seed!(m, Xoshiro(0))
+    @test a == [rand(m) for _=1:100]
 end
 
 @testset "Random.seed!(seed) sets Random.GLOBAL_SEED" begin
@@ -763,7 +772,10 @@ struct RandomStruct23964 end
     @test_throws MethodError rand(RandomStruct23964())
 end
 
-@testset "rand(::$(typeof(RNG)), ::UnitRange{$T}" for RNG ∈ (MersenneTwister(rand(UInt128)), RandomDevice(), Xoshiro()),
+@testset "rand(::$(typeof(RNG)), ::UnitRange{$T}" for RNG ∈ (MersenneTwister(rand(UInt128)),
+                                                             RandomDevice(),
+                                                             Xoshiro(rand(UInt128)),
+                                                             SeedHasher(rand(UInt128))),
                                                         T ∈ (Bool, Int8, Int16, Int32, UInt32, Int64, Int128, UInt128)
     if T === Bool
         @test rand(RNG, false:true) ∈ (false, true)
@@ -906,8 +918,11 @@ end
     @test rand(rng) == rand(GLOBAL_RNG)
 end
 
-@testset "RNGs broadcast as scalars: T" for T in (MersenneTwister, RandomDevice)
-    @test length.(rand.(T(), 1:3)) == 1:3
+@testset "RNGs broadcast as scalars: $(typeof(RNG))" for RNG in (MersenneTwister(0),
+                                                                 RandomDevice(),
+                                                                 Xoshiro(0),
+                                                                 SeedHasher(0))
+    @test length.(rand.(RNG, 1:3)) == 1:3
 end
 
 @testset "generated scalar integers do not overlap" begin
@@ -1205,7 +1220,14 @@ end
     end
 end
 
+
 @testset "seed! and hash_seed" begin
+    function hash_seed(seed)
+        ctx = SHA.SHA2_256_CTX()
+        Random.hash_seed(seed, ctx)
+        bytes2hex(SHA.digest!(ctx))
+    end
+
     # Test that:
     # 1) if n == m, then hash_seed(n) == hash_seed(m)
     # 2) if n != m, then hash_seed(n) != hash_seed(m)
@@ -1218,12 +1240,12 @@ end
         T <: Signed && push!(seeds, T(0), T(1), T(2), T(-1), T(-2))
     end
 
-    vseeds = Dict{Vector{UInt8}, BigInt}()
+    vseeds = Dict{String, BigInt}()
     for seed = seeds
         bigseed = big(seed)
-        vseed = Random.hash_seed(bigseed)
+        vseed = hash_seed(bigseed)
         # test property 1) above
-        @test Random.hash_seed(seed) == vseed
+        @test hash_seed(seed) == vseed
         # test property 2) above
         @test bigseed == get!(vseeds, vseed, bigseed)
         # test that the property 1) is actually inherited by `seed!`
@@ -1235,16 +1257,16 @@ end
     end
 
     seed32 = rand(UInt32, rand(1:9))
-    hash32 = Random.hash_seed(seed32)
-    @test Random.hash_seed(map(UInt64, seed32)) == hash32
+    hash32 = hash_seed(seed32)
+    @test hash_seed(map(UInt64, seed32)) == hash32
     @test hash32 ∉ keys(vseeds)
 
     seed_str = randstring()
     seed_gstr = GenericString(seed_str)
-    @test Random.hash_seed(seed_str) == Random.hash_seed(seed_gstr)
-    string_seeds = Set{Vector{UInt8}}()
+    @test hash_seed(seed_str) == hash_seed(seed_gstr)
+    string_seeds = Set{String}()
     for ch = 'A':'z'
-        vseed = Random.hash_seed(string(ch))
+        vseed = hash_seed(string(ch))
         @test vseed ∉ keys(vseeds)
         @test vseed ∉ string_seeds
         push!(string_seeds, vseed)
