diff --git a/docs/src/api/inits.md b/docs/src/api/inits.md index 1807a847..eb9ef1b4 100644 --- a/docs/src/api/inits.md +++ b/docs/src/api/inits.md @@ -24,4 +24,9 @@ chaotic_init low_connectivity double_cycle + selfloop_cycle + selfloop_feedback_cycle + selfloop_delayline_backward + selfloop_forward_connection + forward_connection ``` diff --git a/src/ReservoirComputing.jl b/src/ReservoirComputing.jl index f9e23ddd..3a032246 100644 --- a/src/ReservoirComputing.jl +++ b/src/ReservoirComputing.jl @@ -39,7 +39,9 @@ export StandardRidge export scaled_rand, weighted_init, informed_init, minimal_init, chebyshev_mapping, logistic_mapping, modified_lm export rand_sparse, delay_line, delay_line_backward, cycle_jumps, - simple_cycle, pseudo_svd, chaotic_init, low_connectivity, double_cycle + simple_cycle, pseudo_svd, chaotic_init, low_connectivity, double_cycle, + selfloop_cycle, selfloop_feedback_cycle, selfloop_delayline_backward, + selfloop_forward_connection, forward_connection export RNN, MRNN, GRU, GRUParams, FullyGated, Minimal export train export ESN, HybridESN, KnowledgeModel, DeepESN diff --git a/src/esn/esn_inits.jl b/src/esn/esn_inits.jl index aae0a2b0..67af54b9 100644 --- a/src/esn/esn_inits.jl +++ b/src/esn/esn_inits.jl @@ -886,10 +886,10 @@ function simple_cycle(rng::AbstractRNG, ::Type{T}, dims::Integer...; reservoir_matrix = DeviceAgnostic.zeros(rng, T, dims...) for idx in first(axes(reservoir_matrix, 1)):(last(axes(reservoir_matrix, 1)) - 1) - reservoir_matrix[idx + 1, idx] = weight + reservoir_matrix[idx + 1, idx] = T(weight) end - reservoir_matrix[1, dims[1]] = weight + reservoir_matrix[1, dims[1]] = T(weight) return return_init_as(Val(return_sparse), reservoir_matrix) end @@ -1281,12 +1281,384 @@ function double_cycle(rng::AbstractRNG, ::Type{T}, dims::Integer...; return return_init_as(Val(return_sparse), reservoir_matrix) end +@doc raw""" + selfloop_cycle([rng], [T], dims...; + cycle_weight=0.1, selfloop_weight=0.1, + return_sparse=false) + +Creates a simple cycle reservoir with the addition of self loops [^elsarraj2019]. + +This architecture is referred to as TP1 in the original paper. + +# Equations + +```math +W_{i,j} = +\begin{cases} + ll, & \text{if } i = j \\ + r, & \text{if } j = i - 1 \text{ for } i = 2 \dots N \\ + r, & \text{if } i = 1, j = N \\ + 0, & \text{otherwise} +\end{cases} +``` + +# Arguments + + - `rng`: Random number generator. Default is `Utils.default_rng()` + from WeightInitializers. + - `T`: Type of the elements in the reservoir matrix. Default is `Float32`. + - `dims`: Dimensions of the reservoir matrix. + +# Keyword arguments + + - `cycle_weight`: Weight of the cycle connections in the reservoir matrix. + Default is 0.1. + - `selfloop_weight`: Weight of the self loops in the reservoir matrix. + Default is 0.1. + - `return_sparse`: flag for returning a `sparse` matrix. + Default is `false`. + +# Examples + +```jldoctest +julia> reservoir_matrix = selfloop_cycle(5, 5) +5×5 Matrix{Float32}: + 0.1 0.0 0.0 0.0 0.1 + 0.1 0.1 0.0 0.0 0.0 + 0.0 0.1 0.1 0.0 0.0 + 0.0 0.0 0.1 0.1 0.0 + 0.0 0.0 0.0 0.1 0.1 + +julia> reservoir_matrix = selfloop_cycle(5, 5; weight=0.2, selfloop_weight=0.5) +5×5 Matrix{Float32}: + 0.5 0.0 0.0 0.0 0.2 + 0.2 0.5 0.0 0.0 0.0 + 0.0 0.2 0.5 0.0 0.0 + 0.0 0.0 0.2 0.5 0.0 + 0.0 0.0 0.0 0.2 0.5 +``` + +[^elsarraj2019]: Elsarraj, Duaa, et al. + "Demystifying echo state network with deterministic simple topologies." + International Journal of Computational Science and Engineering 19.3 (2019): 407-417. +""" +function selfloop_cycle(rng::AbstractRNG, ::Type{T}, dims::Integer...; + cycle_weight=T(0.1f0), selfloop_weight=T(0.1f0), + return_sparse::Bool=false) where {T <: Number} + throw_sparse_error(return_sparse) + reservoir_matrix = simple_cycle(rng, T, dims...; + weight=cycle_weight, return_sparse=false) + reservoir_matrix += T(selfloop_weight) .* I(dims[1]) + return return_init_as(Val(return_sparse), reservoir_matrix) +end + +@doc raw""" + selfloop_feedback_cycle([rng], [T], dims...; + cycle_weight=0.1, selfloop_weight=0.1, + return_sparse=false) + +Creates a cycle reservoir with feedback connections on even neurons and +self loops on odd neurons [^elsarraj2019]. + +This architecture is referred to as TP2 in the original paper. + +# Equations + +```math +W_{i,j} = +\begin{cases} + r, & \text{if } j = i - 1 \text{ for } i = 2 \dots N \\ + r, & \text{if } i = 1, j = N \\ + ll, & \text{if } i = j \text{ and } i \text{ is odd} \\ + r, & \text{if } j = i + 1 \text{ and } i \text{ is even}, i \neq N \\ + 0, & \text{otherwise} +\end{cases} +``` + +# Arguments + + - `rng`: Random number generator. Default is `Utils.default_rng()` + from WeightInitializers. + - `T`: Type of the elements in the reservoir matrix. Default is `Float32`. + - `dims`: Dimensions of the reservoir matrix. + +# Keyword arguments + + - `cycle_weight`: Weight of the cycle connections in the reservoir matrix. + Default is 0.1. + - `selfloop_weight`: Weight of the self loops in the reservoir matrix. + Default is 0.1. + - `return_sparse`: flag for returning a `sparse` matrix. + Default is `false`. + +# Examples + +```jldoctest +julia> reservoir_matrix = selfloop_feedback_cycle(5, 5) +5×5 Matrix{Float32}: + 0.1 0.1 0.0 0.0 0.1 + 0.1 0.0 0.0 0.0 0.0 + 0.0 0.1 0.1 0.1 0.0 + 0.0 0.0 0.1 0.0 0.0 + 0.0 0.0 0.0 0.1 0.1 + +julia> reservoir_matrix = selfloop_feedback_cycle(5, 5; self_loop_weight=0.5) +5×5 Matrix{Float32}: + 0.5 0.1 0.0 0.0 0.1 + 0.1 0.0 0.0 0.0 0.0 + 0.0 0.1 0.5 0.1 0.0 + 0.0 0.0 0.1 0.0 0.0 + 0.0 0.0 0.0 0.1 0.5 +``` + +[^elsarraj2019]: Elsarraj, Duaa, et al. + "Demystifying echo state network with deterministic simple topologies." + International Journal of Computational Science and Engineering 19.3 (2019): 407-417. +""" +function selfloop_feedback_cycle(rng::AbstractRNG, ::Type{T}, dims::Integer...; + cycle_weight=T(0.1f0), selfloop_weight=T(0.1f0), + return_sparse::Bool=false) where {T <: Number} + throw_sparse_error(return_sparse) + reservoir_matrix = simple_cycle(rng, T, dims...; + weight=T(cycle_weight), return_sparse=false) + for idx in axes(reservoir_matrix, 1) + if isodd(idx) + reservoir_matrix[idx, idx] = T(selfloop_weight) + end + end + for idx in (first(axes(reservoir_matrix, 1)) + 1):last(axes(reservoir_matrix, 1)) + if iseven(idx) + reservoir_matrix[idx - 1, idx] = T(cycle_weight) + end + end + return return_init_as(Val(return_sparse), reservoir_matrix) +end + +@doc raw""" + selfloop_delayline_backward([rng], [T], dims...; + weight=0.1, selfloop_weight=0.1, + return_sparse=false) + +Creates a reservoir based on a delay line with the addition of self loops and +backward connections shifted by one [^elsarraj2019]. + +This architecture is referred to as TP3 in the original paper. + +# Equations + +```math +W_{i,j} = +\begin{cases} + ll, & \text{if } i = j \text{ for } i = 1 \dots N \\ + r, & \text{if } j = i - 1 \text{ for } i = 2 \dots N \\ + r, & \text{if } j = i - 2 \text{ for } i = 3 \dots N \\ + 0, & \text{otherwise} +\end{cases} +``` + +# Arguments + + - `rng`: Random number generator. Default is `Utils.default_rng()` + from WeightInitializers. + - `T`: Type of the elements in the reservoir matrix. Default is `Float32`. + - `dims`: Dimensions of the reservoir matrix. + +# Keyword arguments + + - `weight`: Weight of the cycle connections in the reservoir matrix. + Default is 0.1. + - `selfloop_weight`: Weight of the self loops in the reservoir matrix. + Default is 0.1. + - `return_sparse`: flag for returning a `sparse` matrix. + Default is `false`. + +# Examples + +```jldoctest +julia> reservoir_matrix = selfloop_delayline_backward(5, 5) +5×5 Matrix{Float32}: + 0.1 0.0 0.1 0.0 0.0 + 0.1 0.1 0.0 0.1 0.0 + 0.0 0.1 0.1 0.0 0.1 + 0.0 0.0 0.1 0.1 0.0 + 0.0 0.0 0.0 0.1 0.1 + +julia> reservoir_matrix = selfloop_delayline_backward(5, 5; weight=0.3) +5×5 Matrix{Float32}: + 0.1 0.0 0.3 0.0 0.0 + 0.3 0.1 0.0 0.3 0.0 + 0.0 0.3 0.1 0.0 0.3 + 0.0 0.0 0.3 0.1 0.0 + 0.0 0.0 0.0 0.3 0.1 +``` + +[^elsarraj2019]: Elsarraj, Duaa, et al. + "Demystifying echo state network with deterministic simple topologies." + International Journal of Computational Science and Engineering 19.3 (2019): 407-417. +""" +function selfloop_delayline_backward(rng::AbstractRNG, ::Type{T}, dims::Integer...; + weight=T(0.1f0), selfloop_weight=T(0.1f0), + return_sparse::Bool=false) where {T <: Number} + throw_sparse_error(return_sparse) + reservoir_matrix = DeviceAgnostic.zeros(rng, T, dims...) + reservoir_matrix += T(selfloop_weight) .* I(dims[1]) + for idx in first(axes(reservoir_matrix, 1)):(last(axes(reservoir_matrix, 1)) - 1) + reservoir_matrix[idx + 1, idx] = T(weight) + end + for idx in (first(axes(reservoir_matrix, 1))):(last(axes(reservoir_matrix, 1)) - 2) + reservoir_matrix[idx, idx + 2] = T(weight) + end + return return_init_as(Val(return_sparse), reservoir_matrix) +end + +@doc raw""" + selfloop_forward_connection([rng], [T], dims...; + weight=0.1, selfloop_weight=0.1, + return_sparse=false) + +Creates a reservoir based on a forward connection of weights between even nodes +with the addition of self loops [^elsarraj2019]. + +This architecture is referred to as TP4 in the original paper. + +# Equations + +```math +W_{i,j} = +\begin{cases} + ll, & \text{if } i = j \text{ for } i = 1 \dots N \\ + r, & \text{if } j = i - 2 \text{ for } i = 3 \dots N \\ + 0, & \text{otherwise} +\end{cases} +``` + +# Arguments + + - `rng`: Random number generator. Default is `Utils.default_rng()` + from WeightInitializers. + - `T`: Type of the elements in the reservoir matrix. Default is `Float32`. + - `dims`: Dimensions of the reservoir matrix. + +# Keyword arguments + + - `weight`: Weight of the cycle connections in the reservoir matrix. + Default is 0.1. + - `selfloop_weight`: Weight of the self loops in the reservoir matrix. + Default is 0.1. + - `return_sparse`: flag for returning a `sparse` matrix. + Default is `false`. + +# Examples + +```jldoctest +julia> reservoir_matrix = selfloop_forward_connection(5, 5) +5×5 Matrix{Float32}: + 0.1 0.0 0.0 0.0 0.0 + 0.0 0.1 0.0 0.0 0.0 + 0.1 0.0 0.1 0.0 0.0 + 0.0 0.1 0.0 0.1 0.0 + 0.0 0.0 0.1 0.0 0.1 + +julia> reservoir_matrix = selfloop_forward_connection(5, 5; weight=0.5) +5×5 Matrix{Float32}: + 0.1 0.0 0.0 0.0 0.0 + 0.0 0.1 0.0 0.0 0.0 + 0.5 0.0 0.1 0.0 0.0 + 0.0 0.5 0.0 0.1 0.0 + 0.0 0.0 0.5 0.0 0.1 +``` + +[^elsarraj2019]: Elsarraj, Duaa, et al. + "Demystifying echo state network with deterministic simple topologies." + International Journal of Computational Science and Engineering 19.3 (2019): 407-417. +""" +function selfloop_forward_connection(rng::AbstractRNG, ::Type{T}, dims::Integer...; + weight=T(0.1f0), selfloop_weight=T(0.1f0), + return_sparse::Bool=false) where {T <: Number} + throw_sparse_error(return_sparse) + reservoir_matrix = DeviceAgnostic.zeros(rng, T, dims...) + reservoir_matrix += T(selfloop_weight) .* I(dims[1]) + for idx in first(axes(reservoir_matrix, 1)):(last(axes(reservoir_matrix, 1)) - 2) + reservoir_matrix[idx + 2, idx] = T(weight) + end + return return_init_as(Val(return_sparse), reservoir_matrix) +end + +@doc raw""" + forward_connection([rng], [T], dims...; + weight=0.1, selfloop_weight=0.1, + return_sparse=false) + +Creates a reservoir based on a forward connection of weights [^elsarraj2019]. + +This architecture is referred to as TP5 in the original paper. + +# Equations + +```math +W_{i,j} = +\begin{cases} + r, & \text{if } j = i - 2 \text{ for } i = 3 \dots N \\ + 0, & \text{otherwise} +\end{cases} +``` + +# Arguments + + - `rng`: Random number generator. Default is `Utils.default_rng()` + from WeightInitializers. + - `T`: Type of the elements in the reservoir matrix. Default is `Float32`. + - `dims`: Dimensions of the reservoir matrix. + +# Keyword arguments + + - `weight`: Weight of the cycle connections in the reservoir matrix. + Default is 0.1. + - `return_sparse`: flag for returning a `sparse` matrix. + Default is `false`. + +# Examples + +```jldoctest +julia> reservoir_matrix = forward_connection(5, 5) +5×5 Matrix{Float32}: + 0.0 0.0 0.0 0.0 0.0 + 0.0 0.0 0.0 0.0 0.0 + 0.1 0.0 0.0 0.0 0.0 + 0.0 0.1 0.0 0.0 0.0 + 0.0 0.0 0.1 0.0 0.0 + +julia> reservoir_matrix = forward_connection(5, 5; weight=0.5) +5×5 Matrix{Float32}: + 0.0 0.0 0.0 0.0 0.0 + 0.0 0.0 0.0 0.0 0.0 + 0.5 0.0 0.0 0.0 0.0 + 0.0 0.5 0.0 0.0 0.0 + 0.0 0.0 0.5 0.0 0.0 +``` + +[^elsarraj2019]: Elsarraj, Duaa, et al. + "Demystifying echo state network with deterministic simple topologies." + International Journal of Computational Science and Engineering 19.3 (2019): 407-417. +""" +function forward_connection(rng::AbstractRNG, ::Type{T}, dims::Integer...; + weight=T(0.1f0), return_sparse::Bool=false) where {T <: Number} + throw_sparse_error(return_sparse) + reservoir_matrix = DeviceAgnostic.zeros(rng, T, dims...) + for idx in first(axes(reservoir_matrix, 1)):(last(axes(reservoir_matrix, 1)) - 2) + reservoir_matrix[idx + 2, idx] = T(weight) + end + return return_init_as(Val(return_sparse), reservoir_matrix) +end + ### fallbacks #fallbacks for initializers #eventually to remove once migrated to WeightInitializers.jl for initializer in (:rand_sparse, :delay_line, :delay_line_backward, :cycle_jumps, :simple_cycle, :pseudo_svd, :chaotic_init, :scaled_rand, :weighted_init, :informed_init, :minimal_init, :chebyshev_mapping, - :logistic_mapping, :modified_lm, :low_connectivity, :double_cycle) + :logistic_mapping, :modified_lm, :low_connectivity, :double_cycle, :selfloop_cycle, + :selfloop_feedback_cycle, :selfloop_delayline_backward, :selfloop_forward_connection, + :forward_connection) @eval begin function ($initializer)(dims::Integer...; kwargs...) return $initializer(Utils.default_rng(), Float32, dims...; kwargs...) diff --git a/test/esn/test_inits.jl b/test/esn/test_inits.jl index 29f7897a..9f475715 100644 --- a/test/esn/test_inits.jl +++ b/test/esn/test_inits.jl @@ -27,7 +27,12 @@ reservoir_inits = [ pseudo_svd, chaotic_init, low_connectivity, - double_cycle + double_cycle, + selfloop_cycle, + selfloop_feedback_cycle, + selfloop_delayline_backward, + selfloop_forward_connection, + forward_connection ] input_inits = [ scaled_rand,