From b1ba13131c47e6f76cff08c124c2d9c612e8f481 Mon Sep 17 00:00:00 2001 From: MartinuzziFrancesco Date: Mon, 10 Mar 2025 10:59:44 +0100 Subject: [PATCH 1/6] added cycle reservoir with self loops --- docs/src/api/inits.md | 1 + src/ReservoirComputing.jl | 3 +- src/esn/esn_inits.jl | 63 +++++++++++++++++++++++++++++++++++++-- test/esn/test_inits.jl | 3 +- 4 files changed, 65 insertions(+), 5 deletions(-) diff --git a/docs/src/api/inits.md b/docs/src/api/inits.md index 1807a847..e3da5fc5 100644 --- a/docs/src/api/inits.md +++ b/docs/src/api/inits.md @@ -24,4 +24,5 @@ chaotic_init low_connectivity double_cycle + self_loop_cycle ``` diff --git a/src/ReservoirComputing.jl b/src/ReservoirComputing.jl index f9e23ddd..64162ade 100644 --- a/src/ReservoirComputing.jl +++ b/src/ReservoirComputing.jl @@ -39,7 +39,8 @@ export StandardRidge export scaled_rand, weighted_init, informed_init, minimal_init, chebyshev_mapping, logistic_mapping, modified_lm export rand_sparse, delay_line, delay_line_backward, cycle_jumps, - simple_cycle, pseudo_svd, chaotic_init, low_connectivity, double_cycle + simple_cycle, pseudo_svd, chaotic_init, low_connectivity, double_cycle, + self_loop_cycle export RNN, MRNN, GRU, GRUParams, FullyGated, Minimal export train export ESN, HybridESN, KnowledgeModel, DeepESN diff --git a/src/esn/esn_inits.jl b/src/esn/esn_inits.jl index aae0a2b0..f8e8b068 100644 --- a/src/esn/esn_inits.jl +++ b/src/esn/esn_inits.jl @@ -886,10 +886,10 @@ function simple_cycle(rng::AbstractRNG, ::Type{T}, dims::Integer...; reservoir_matrix = DeviceAgnostic.zeros(rng, T, dims...) for idx in first(axes(reservoir_matrix, 1)):(last(axes(reservoir_matrix, 1)) - 1) - reservoir_matrix[idx + 1, idx] = weight + reservoir_matrix[idx + 1, idx] = T(weight) end - reservoir_matrix[1, dims[1]] = weight + reservoir_matrix[1, dims[1]] = T(weight) return return_init_as(Val(return_sparse), reservoir_matrix) end @@ -1281,12 +1281,69 @@ function double_cycle(rng::AbstractRNG, ::Type{T}, dims::Integer...; return return_init_as(Val(return_sparse), reservoir_matrix) end +""" + self_loop_cycle([rng], [T], dims...; + cycle_weight=0.1, self_loop_weight=0.1, + return_sparse=false) + +Creates a simple cycle reservoir with the addition of self loops [^elsarraj2019]. + +# Arguments + + - `rng`: Random number generator. Default is `Utils.default_rng()` + from WeightInitializers. + - `T`: Type of the elements in the reservoir matrix. Default is `Float32`. + - `dims`: Dimensions of the reservoir matrix. + +# Keyword arguments + + - `cycle_weight`: Weight of the upper cycle connections in the reservoir matrix. + Default is 0.1. + - `self_loop_weight`: Weight of the self loops in the reservoir matrix. + Default is 0.1. + - `return_sparse`: flag for returning a `sparse` matrix. + Default is `false`. + +# Examples + +```jldoctest +julia> reservoir_matrix = self_loop_cycle(5, 5) +5×5 Matrix{Float32}: + 0.1 0.0 0.0 0.0 0.1 + 0.1 0.1 0.0 0.0 0.0 + 0.0 0.1 0.1 0.0 0.0 + 0.0 0.0 0.1 0.1 0.0 + 0.0 0.0 0.0 0.1 0.1 + +julia> reservoir_matrix = self_loop_cycle(5, 5; weight=0.2, self_loop_weight=0.5) +5×5 Matrix{Float32}: + 0.5 0.0 0.0 0.0 0.2 + 0.2 0.5 0.0 0.0 0.0 + 0.0 0.2 0.5 0.0 0.0 + 0.0 0.0 0.2 0.5 0.0 + 0.0 0.0 0.0 0.2 0.5 +``` + +[^elsarraj2019]: Elsarraj, Duaa, et al. + "Demystifying echo state network with deterministic simple topologies." + International Journal of Computational Science and Engineering 19.3 (2019): 407-417. +""" +function self_loop_cycle(rng::AbstractRNG, ::Type{T}, dims::Integer...; + weight=T(0.1f0), self_loop_weight=T(0.1f0), + return_sparse::Bool=false) where {T <: Number} + throw_sparse_error(return_sparse) + reservoir_matrix = simple_cycle(rng, T, dims...; + weight=weight, return_sparse=false) + reservoir_matrix += T(self_loop_weight) .* I(dims[1]) + return return_init_as(Val(return_sparse), reservoir_matrix) +end + ### fallbacks #fallbacks for initializers #eventually to remove once migrated to WeightInitializers.jl for initializer in (:rand_sparse, :delay_line, :delay_line_backward, :cycle_jumps, :simple_cycle, :pseudo_svd, :chaotic_init, :scaled_rand, :weighted_init, :informed_init, :minimal_init, :chebyshev_mapping, - :logistic_mapping, :modified_lm, :low_connectivity, :double_cycle) + :logistic_mapping, :modified_lm, :low_connectivity, :double_cycle, :self_loop_cycle) @eval begin function ($initializer)(dims::Integer...; kwargs...) return $initializer(Utils.default_rng(), Float32, dims...; kwargs...) diff --git a/test/esn/test_inits.jl b/test/esn/test_inits.jl index 29f7897a..5a53475f 100644 --- a/test/esn/test_inits.jl +++ b/test/esn/test_inits.jl @@ -27,7 +27,8 @@ reservoir_inits = [ pseudo_svd, chaotic_init, low_connectivity, - double_cycle + double_cycle, + self_loop_cycle ] input_inits = [ scaled_rand, From 97b460e1bb08171b5c4c0ec1c837da79e203d0b4 Mon Sep 17 00:00:00 2001 From: MartinuzziFrancesco Date: Tue, 11 Mar 2025 10:52:46 +0100 Subject: [PATCH 2/6] added selfloop_feedback_cycle --- docs/src/api/inits.md | 1 + src/ReservoirComputing.jl | 2 +- src/esn/esn_inits.jl | 88 +++++++++++++++++++++++++++++++++++---- test/esn/test_inits.jl | 3 +- 4 files changed, 84 insertions(+), 10 deletions(-) diff --git a/docs/src/api/inits.md b/docs/src/api/inits.md index e3da5fc5..68b92728 100644 --- a/docs/src/api/inits.md +++ b/docs/src/api/inits.md @@ -25,4 +25,5 @@ low_connectivity double_cycle self_loop_cycle + selfloop_feedback_cycle ``` diff --git a/src/ReservoirComputing.jl b/src/ReservoirComputing.jl index 64162ade..0682f431 100644 --- a/src/ReservoirComputing.jl +++ b/src/ReservoirComputing.jl @@ -40,7 +40,7 @@ export scaled_rand, weighted_init, informed_init, minimal_init, chebyshev_mappin logistic_mapping, modified_lm export rand_sparse, delay_line, delay_line_backward, cycle_jumps, simple_cycle, pseudo_svd, chaotic_init, low_connectivity, double_cycle, - self_loop_cycle + self_loop_cycle, selfloop_feedback_cycle export RNN, MRNN, GRU, GRUParams, FullyGated, Minimal export train export ESN, HybridESN, KnowledgeModel, DeepESN diff --git a/src/esn/esn_inits.jl b/src/esn/esn_inits.jl index f8e8b068..a41844f1 100644 --- a/src/esn/esn_inits.jl +++ b/src/esn/esn_inits.jl @@ -1283,11 +1283,13 @@ end """ self_loop_cycle([rng], [T], dims...; - cycle_weight=0.1, self_loop_weight=0.1, + cycle_weight=0.1, selfloop_weight=0.1, return_sparse=false) Creates a simple cycle reservoir with the addition of self loops [^elsarraj2019]. +This architecture is referred to as TP1 in the original paper. + # Arguments - `rng`: Random number generator. Default is `Utils.default_rng()` @@ -1297,9 +1299,9 @@ Creates a simple cycle reservoir with the addition of self loops [^elsarraj2019] # Keyword arguments - - `cycle_weight`: Weight of the upper cycle connections in the reservoir matrix. + - `cycle_weight`: Weight of the cycle connections in the reservoir matrix. Default is 0.1. - - `self_loop_weight`: Weight of the self loops in the reservoir matrix. + - `selfloop_weight`: Weight of the self loops in the reservoir matrix. Default is 0.1. - `return_sparse`: flag for returning a `sparse` matrix. Default is `false`. @@ -1315,7 +1317,7 @@ julia> reservoir_matrix = self_loop_cycle(5, 5) 0.0 0.0 0.1 0.1 0.0 0.0 0.0 0.0 0.1 0.1 -julia> reservoir_matrix = self_loop_cycle(5, 5; weight=0.2, self_loop_weight=0.5) +julia> reservoir_matrix = self_loop_cycle(5, 5; weight=0.2, selfloop_weight=0.5) 5×5 Matrix{Float32}: 0.5 0.0 0.0 0.0 0.2 0.2 0.5 0.0 0.0 0.0 @@ -1329,12 +1331,81 @@ julia> reservoir_matrix = self_loop_cycle(5, 5; weight=0.2, self_loop_weight=0.5 International Journal of Computational Science and Engineering 19.3 (2019): 407-417. """ function self_loop_cycle(rng::AbstractRNG, ::Type{T}, dims::Integer...; - weight=T(0.1f0), self_loop_weight=T(0.1f0), + cycle_weight=T(0.1f0), selfloop_weight=T(0.1f0), return_sparse::Bool=false) where {T <: Number} throw_sparse_error(return_sparse) reservoir_matrix = simple_cycle(rng, T, dims...; - weight=weight, return_sparse=false) - reservoir_matrix += T(self_loop_weight) .* I(dims[1]) + weight=cycle_weight, return_sparse=false) + reservoir_matrix += T(selfloop_weight) .* I(dims[1]) + return return_init_as(Val(return_sparse), reservoir_matrix) +end + +""" + selfloop_feedback_cycle([rng], [T], dims...; + cycle_weight=0.1, selfloop_weight=0.1, + return_sparse=false) + +Creates a cycle reservoir with feedback connections on even neurons and +self loops on odd neurons [^elsarraj2019]. + +This architecture is referred to as TP2 in the original paper. + +# Arguments + + - `rng`: Random number generator. Default is `Utils.default_rng()` + from WeightInitializers. + - `T`: Type of the elements in the reservoir matrix. Default is `Float32`. + - `dims`: Dimensions of the reservoir matrix. + +# Keyword arguments + + - `cycle_weight`: Weight of the cycle connections in the reservoir matrix. + Default is 0.1. + - `selfloop_weight`: Weight of the self loops in the reservoir matrix. + Default is 0.1. + - `return_sparse`: flag for returning a `sparse` matrix. + Default is `false`. + +# Examples + +```jldoctest +julia> reservoir_matrix = selfloop_feedback_cycle(5, 5) +5×5 Matrix{Float32}: + 0.1 0.1 0.0 0.0 0.1 + 0.1 0.0 0.0 0.0 0.0 + 0.0 0.1 0.1 0.1 0.0 + 0.0 0.0 0.1 0.0 0.0 + 0.0 0.0 0.0 0.1 0.1 + +julia> reservoir_matrix = selfloop_feedback_cycle(5, 5; self_loop_weight=0.5) +5×5 Matrix{Float32}: + 0.5 0.1 0.0 0.0 0.1 + 0.1 0.0 0.0 0.0 0.0 + 0.0 0.1 0.5 0.1 0.0 + 0.0 0.0 0.1 0.0 0.0 + 0.0 0.0 0.0 0.1 0.5 +``` + +[^elsarraj2019]: Elsarraj, Duaa, et al. + "Demystifying echo state network with deterministic simple topologies." + International Journal of Computational Science and Engineering 19.3 (2019): 407-417. +""" +function selfloop_feedback_cycle(rng::AbstractRNG, ::Type{T}, dims::Integer...; + cycle_weight=T(0.1f0), selfloop_weight=T(0.1f0), + return_sparse::Bool=false) where {T <: Number} + throw_sparse_error(return_sparse) + reservoir_matrix = simple_cycle(rng, T, dims...; + weight=T(cycle_weight), return_sparse=false) + for idx in axes(reservoir_matrix, 1) + if isodd(idx) + reservoir_matrix[idx, idx] = T(selfloop_weight) + end + end + for idx in (first(axes(reservoir_matrix, 1)) + 1):last(axes(reservoir_matrix, 1)) + if iseven(idx) + reservoir_matrix[idx - 1, idx] = T(cycle_weight) + end + end return return_init_as(Val(return_sparse), reservoir_matrix) end @@ -1343,7 +1414,8 @@ end for initializer in (:rand_sparse, :delay_line, :delay_line_backward, :cycle_jumps, :simple_cycle, :pseudo_svd, :chaotic_init, :scaled_rand, :weighted_init, :informed_init, :minimal_init, :chebyshev_mapping, - :logistic_mapping, :modified_lm, :low_connectivity, :double_cycle, :self_loop_cycle) + :logistic_mapping, :modified_lm, :low_connectivity, :double_cycle, :self_loop_cycle, + :selfloop_feedback_cycle) @eval begin function ($initializer)(dims::Integer...; kwargs...) return $initializer(Utils.default_rng(), Float32, dims...; kwargs...) diff --git a/test/esn/test_inits.jl b/test/esn/test_inits.jl index 5a53475f..faac741c 100644 --- a/test/esn/test_inits.jl +++ b/test/esn/test_inits.jl @@ -28,7 +28,8 @@ reservoir_inits = [ chaotic_init, low_connectivity, double_cycle, - self_loop_cycle + self_loop_cycle, + selfloop_feedback_cycle ] input_inits = [ scaled_rand, From 67b4ce29c7eea1a2021b5cfcd36ddb19812a42dd Mon Sep 17 00:00:00 2001 From: MartinuzziFrancesco Date: Wed, 12 Mar 2025 10:30:57 +0100 Subject: [PATCH 3/6] added tp3 --- docs/src/api/inits.md | 1 + src/ReservoirComputing.jl | 2 +- src/esn/esn_inits.jl | 67 ++++++++++++++++++++++++++++++++++++++- test/esn/test_inits.jl | 3 +- 4 files changed, 70 insertions(+), 3 deletions(-) diff --git a/docs/src/api/inits.md b/docs/src/api/inits.md index 68b92728..6f78bdf8 100644 --- a/docs/src/api/inits.md +++ b/docs/src/api/inits.md @@ -26,4 +26,5 @@ double_cycle self_loop_cycle selfloop_feedback_cycle + selfloop_delayline_backward ``` diff --git a/src/ReservoirComputing.jl b/src/ReservoirComputing.jl index 0682f431..e219416e 100644 --- a/src/ReservoirComputing.jl +++ b/src/ReservoirComputing.jl @@ -40,7 +40,7 @@ export scaled_rand, weighted_init, informed_init, minimal_init, chebyshev_mappin logistic_mapping, modified_lm export rand_sparse, delay_line, delay_line_backward, cycle_jumps, simple_cycle, pseudo_svd, chaotic_init, low_connectivity, double_cycle, - self_loop_cycle, selfloop_feedback_cycle + self_loop_cycle, selfloop_feedback_cycle, selfloop_delayline_backward export RNN, MRNN, GRU, GRUParams, FullyGated, Minimal export train export ESN, HybridESN, KnowledgeModel, DeepESN diff --git a/src/esn/esn_inits.jl b/src/esn/esn_inits.jl index a41844f1..d769d73f 100644 --- a/src/esn/esn_inits.jl +++ b/src/esn/esn_inits.jl @@ -1409,13 +1409,78 @@ function selfloop_feedback_cycle(rng::AbstractRNG, ::Type{T}, dims::Integer...; return return_init_as(Val(return_sparse), reservoir_matrix) end +""" + selfloop_delayline_backward([rng], [T], dims...; + cycle_weight=0.1, selfloop_weight=0.1, + return_sparse=false) + +Creates a reservoir based on a delay line with the addition of self loops and +backward connections shifted by one [^elsarraj2019]. + +This architecture is referred to as TP3 in the original paper. + +# Arguments + + - `rng`: Random number generator. Default is `Utils.default_rng()` + from WeightInitializers. + - `T`: Type of the elements in the reservoir matrix. Default is `Float32`. + - `dims`: Dimensions of the reservoir matrix. + +# Keyword arguments + + - `weight`: Weight of the cycle connections in the reservoir matrix. + Default is 0.1. + - `selfloop_weight`: Weight of the self loops in the reservoir matrix. + Default is 0.1. + - `return_sparse`: flag for returning a `sparse` matrix. + Default is `false`. + +# Examples + +```jldoctest +julia> reservoir_matrix = selfloop_delayline_backward(5, 5) +5×5 Matrix{Float32}: + 0.1 0.0 0.1 0.0 0.0 + 0.1 0.1 0.0 0.1 0.0 + 0.0 0.1 0.1 0.0 0.1 + 0.0 0.0 0.1 0.1 0.0 + 0.0 0.0 0.0 0.1 0.1 + +julia> reservoir_matrix = selfloop_delayline_backward(5, 5; weight=0.3) +5×5 Matrix{Float32}: + 0.1 0.0 0.3 0.0 0.0 + 0.3 0.1 0.0 0.3 0.0 + 0.0 0.3 0.1 0.0 0.3 + 0.0 0.0 0.3 0.1 0.0 + 0.0 0.0 0.0 0.3 0.1 +``` + +[^elsarraj2019]: Elsarraj, Duaa, et al. + "Demystifying echo state network with deterministic simple topologies." + International Journal of Computational Science and Engineering 19.3 (2019): 407-417. +""" +function selfloop_delayline_backward(rng::AbstractRNG, ::Type{T}, dims::Integer...; + weight=T(0.1f0), selfloop_weight=T(0.1f0), + return_sparse::Bool=false) where {T <: Number} + throw_sparse_error(return_sparse) + reservoir_matrix = DeviceAgnostic.zeros(rng, T, dims...) + reservoir_matrix += T(selfloop_weight) .* I(dims[1]) + for idx in first(axes(reservoir_matrix, 1)):(last(axes(reservoir_matrix, 1)) - 1) + reservoir_matrix[idx + 1, idx] = T(weight) + end + for idx in (first(axes(reservoir_matrix, 1))):(last(axes(reservoir_matrix, 1)) - 2) + reservoir_matrix[idx, idx + 2] = T(weight) + end + return return_init_as(Val(return_sparse), reservoir_matrix) +end + ### fallbacks #fallbacks for initializers #eventually to remove once migrated to WeightInitializers.jl for initializer in (:rand_sparse, :delay_line, :delay_line_backward, :cycle_jumps, :simple_cycle, :pseudo_svd, :chaotic_init, :scaled_rand, :weighted_init, :informed_init, :minimal_init, :chebyshev_mapping, :logistic_mapping, :modified_lm, :low_connectivity, :double_cycle, :self_loop_cycle, - :selfloop_feedback_cycle) + :selfloop_feedback_cycle, :selfloop_delayline_backward) @eval begin function ($initializer)(dims::Integer...; kwargs...) return $initializer(Utils.default_rng(), Float32, dims...; kwargs...) diff --git a/test/esn/test_inits.jl b/test/esn/test_inits.jl index faac741c..92f84e73 100644 --- a/test/esn/test_inits.jl +++ b/test/esn/test_inits.jl @@ -29,7 +29,8 @@ reservoir_inits = [ low_connectivity, double_cycle, self_loop_cycle, - selfloop_feedback_cycle + selfloop_feedback_cycle, + selfloop_delayline_backward ] input_inits = [ scaled_rand, From ed3838a7eb65d2bd209260e49fc6324361d8ea92 Mon Sep 17 00:00:00 2001 From: MartinuzziFrancesco Date: Thu, 13 Mar 2025 09:02:14 +0100 Subject: [PATCH 4/6] added tp4 --- docs/src/api/inits.md | 1 + src/ReservoirComputing.jl | 3 +- src/esn/esn_inits.jl | 67 +++++++++++++++++++++++++++++++++++++-- test/esn/test_inits.jl | 3 +- 4 files changed, 70 insertions(+), 4 deletions(-) diff --git a/docs/src/api/inits.md b/docs/src/api/inits.md index 6f78bdf8..406ab344 100644 --- a/docs/src/api/inits.md +++ b/docs/src/api/inits.md @@ -27,4 +27,5 @@ self_loop_cycle selfloop_feedback_cycle selfloop_delayline_backward + selfloop_forward_connection ``` diff --git a/src/ReservoirComputing.jl b/src/ReservoirComputing.jl index e219416e..d99a437b 100644 --- a/src/ReservoirComputing.jl +++ b/src/ReservoirComputing.jl @@ -40,7 +40,8 @@ export scaled_rand, weighted_init, informed_init, minimal_init, chebyshev_mappin logistic_mapping, modified_lm export rand_sparse, delay_line, delay_line_backward, cycle_jumps, simple_cycle, pseudo_svd, chaotic_init, low_connectivity, double_cycle, - self_loop_cycle, selfloop_feedback_cycle, selfloop_delayline_backward + self_loop_cycle, selfloop_feedback_cycle, selfloop_delayline_backward, + selfloop_forward_connection export RNN, MRNN, GRU, GRUParams, FullyGated, Minimal export train export ESN, HybridESN, KnowledgeModel, DeepESN diff --git a/src/esn/esn_inits.jl b/src/esn/esn_inits.jl index d769d73f..ff5b7ea6 100644 --- a/src/esn/esn_inits.jl +++ b/src/esn/esn_inits.jl @@ -1411,7 +1411,7 @@ end """ selfloop_delayline_backward([rng], [T], dims...; - cycle_weight=0.1, selfloop_weight=0.1, + weight=0.1, selfloop_weight=0.1, return_sparse=false) Creates a reservoir based on a delay line with the addition of self loops and @@ -1474,13 +1474,76 @@ function selfloop_delayline_backward(rng::AbstractRNG, ::Type{T}, dims::Integer. return return_init_as(Val(return_sparse), reservoir_matrix) end +""" + selfloop_forward_connection([rng], [T], dims...; + weight=0.1, selfloop_weight=0.1, + return_sparse=false) + +Creates a reservoir based on a forward connection of weights between even nodes +with the addition of self loops [^elsarraj2019]. + +This architecture is referred to as TP4 in the original paper. + +# Arguments + + - `rng`: Random number generator. Default is `Utils.default_rng()` + from WeightInitializers. + - `T`: Type of the elements in the reservoir matrix. Default is `Float32`. + - `dims`: Dimensions of the reservoir matrix. + +# Keyword arguments + + - `weight`: Weight of the cycle connections in the reservoir matrix. + Default is 0.1. + - `selfloop_weight`: Weight of the self loops in the reservoir matrix. + Default is 0.1. + - `return_sparse`: flag for returning a `sparse` matrix. + Default is `false`. + +# Examples + +```jldoctest +julia> reservoir_matrix = selfloop_forward_connection(5, 5) +5×5 Matrix{Float32}: + 0.1 0.0 0.0 0.0 0.0 + 0.0 0.1 0.0 0.0 0.0 + 0.1 0.0 0.1 0.0 0.0 + 0.0 0.1 0.0 0.1 0.0 + 0.0 0.0 0.1 0.0 0.1 + +julia> reservoir_matrix = selfloop_forward_connection(5, 5; weight=0.5) +5×5 Matrix{Float32}: + 0.1 0.0 0.0 0.0 0.0 + 0.0 0.1 0.0 0.0 0.0 + 0.5 0.0 0.1 0.0 0.0 + 0.0 0.5 0.0 0.1 0.0 + 0.0 0.0 0.5 0.0 0.1 + +``` + +[^elsarraj2019]: Elsarraj, Duaa, et al. + "Demystifying echo state network with deterministic simple topologies." + International Journal of Computational Science and Engineering 19.3 (2019): 407-417. +""" +function selfloop_forward_connection(rng::AbstractRNG, ::Type{T}, dims::Integer...; + weight=T(0.1f0), selfloop_weight=T(0.1f0), + return_sparse::Bool=false) where {T <: Number} + throw_sparse_error(return_sparse) + reservoir_matrix = DeviceAgnostic.zeros(rng, T, dims...) + reservoir_matrix += T(selfloop_weight) .* I(dims[1]) + for idx in first(axes(reservoir_matrix, 1)):(last(axes(reservoir_matrix, 1)) - 2) + reservoir_matrix[idx + 2, idx] = T(weight) + end + return return_init_as(Val(return_sparse), reservoir_matrix) +end + ### fallbacks #fallbacks for initializers #eventually to remove once migrated to WeightInitializers.jl for initializer in (:rand_sparse, :delay_line, :delay_line_backward, :cycle_jumps, :simple_cycle, :pseudo_svd, :chaotic_init, :scaled_rand, :weighted_init, :informed_init, :minimal_init, :chebyshev_mapping, :logistic_mapping, :modified_lm, :low_connectivity, :double_cycle, :self_loop_cycle, - :selfloop_feedback_cycle, :selfloop_delayline_backward) + :selfloop_feedback_cycle, :selfloop_delayline_backward, :selfloop_forward_connection) @eval begin function ($initializer)(dims::Integer...; kwargs...) return $initializer(Utils.default_rng(), Float32, dims...; kwargs...) diff --git a/test/esn/test_inits.jl b/test/esn/test_inits.jl index 92f84e73..930f57df 100644 --- a/test/esn/test_inits.jl +++ b/test/esn/test_inits.jl @@ -30,7 +30,8 @@ reservoir_inits = [ double_cycle, self_loop_cycle, selfloop_feedback_cycle, - selfloop_delayline_backward + selfloop_delayline_backward, + selfloop_forward_connection ] input_inits = [ scaled_rand, From a1a8e4f577c9de65b4d39e6f4fce7bec5f660061 Mon Sep 17 00:00:00 2001 From: MartinuzziFrancesco Date: Thu, 13 Mar 2025 09:08:01 +0100 Subject: [PATCH 5/6] added tp5 --- docs/src/api/inits.md | 1 + src/ReservoirComputing.jl | 2 +- src/esn/esn_inits.jl | 61 +++++++++++++++++++++++++++++++++++++-- test/esn/test_inits.jl | 3 +- 4 files changed, 63 insertions(+), 4 deletions(-) diff --git a/docs/src/api/inits.md b/docs/src/api/inits.md index 406ab344..e4f930df 100644 --- a/docs/src/api/inits.md +++ b/docs/src/api/inits.md @@ -28,4 +28,5 @@ selfloop_feedback_cycle selfloop_delayline_backward selfloop_forward_connection + forward_connection ``` diff --git a/src/ReservoirComputing.jl b/src/ReservoirComputing.jl index d99a437b..ac3ae95f 100644 --- a/src/ReservoirComputing.jl +++ b/src/ReservoirComputing.jl @@ -41,7 +41,7 @@ export scaled_rand, weighted_init, informed_init, minimal_init, chebyshev_mappin export rand_sparse, delay_line, delay_line_backward, cycle_jumps, simple_cycle, pseudo_svd, chaotic_init, low_connectivity, double_cycle, self_loop_cycle, selfloop_feedback_cycle, selfloop_delayline_backward, - selfloop_forward_connection + selfloop_forward_connection, forward_connection export RNN, MRNN, GRU, GRUParams, FullyGated, Minimal export train export ESN, HybridESN, KnowledgeModel, DeepESN diff --git a/src/esn/esn_inits.jl b/src/esn/esn_inits.jl index ff5b7ea6..598c95e2 100644 --- a/src/esn/esn_inits.jl +++ b/src/esn/esn_inits.jl @@ -1518,7 +1518,6 @@ julia> reservoir_matrix = selfloop_forward_connection(5, 5; weight=0.5) 0.5 0.0 0.1 0.0 0.0 0.0 0.5 0.0 0.1 0.0 0.0 0.0 0.5 0.0 0.1 - ``` [^elsarraj2019]: Elsarraj, Duaa, et al. @@ -1537,13 +1536,71 @@ function selfloop_forward_connection(rng::AbstractRNG, ::Type{T}, dims::Integer. return return_init_as(Val(return_sparse), reservoir_matrix) end +""" + forward_connection([rng], [T], dims...; + weight=0.1, selfloop_weight=0.1, + return_sparse=false) + +Creates a reservoir based on a forward connection of weights [^elsarraj2019]. + +This architecture is referred to as TP5 in the original paper. + +# Arguments + + - `rng`: Random number generator. Default is `Utils.default_rng()` + from WeightInitializers. + - `T`: Type of the elements in the reservoir matrix. Default is `Float32`. + - `dims`: Dimensions of the reservoir matrix. + +# Keyword arguments + + - `weight`: Weight of the cycle connections in the reservoir matrix. + Default is 0.1. + - `return_sparse`: flag for returning a `sparse` matrix. + Default is `false`. + +# Examples + +```jldoctest +julia> reservoir_matrix = forward_connection(5, 5) +5×5 Matrix{Float32}: + 0.0 0.0 0.0 0.0 0.0 + 0.0 0.0 0.0 0.0 0.0 + 0.1 0.0 0.0 0.0 0.0 + 0.0 0.1 0.0 0.0 0.0 + 0.0 0.0 0.1 0.0 0.0 + +julia> reservoir_matrix = forward_connection(5, 5; weight=0.5) +5×5 Matrix{Float32}: + 0.0 0.0 0.0 0.0 0.0 + 0.0 0.0 0.0 0.0 0.0 + 0.5 0.0 0.0 0.0 0.0 + 0.0 0.5 0.0 0.0 0.0 + 0.0 0.0 0.5 0.0 0.0 +``` + +[^elsarraj2019]: Elsarraj, Duaa, et al. + "Demystifying echo state network with deterministic simple topologies." + International Journal of Computational Science and Engineering 19.3 (2019): 407-417. +""" +function forward_connection(rng::AbstractRNG, ::Type{T}, dims::Integer...; + weight=T(0.1f0), return_sparse::Bool=false) where {T <: Number} + throw_sparse_error(return_sparse) + reservoir_matrix = DeviceAgnostic.zeros(rng, T, dims...) + for idx in first(axes(reservoir_matrix, 1)):(last(axes(reservoir_matrix, 1)) - 2) + reservoir_matrix[idx + 2, idx] = T(weight) + end + return return_init_as(Val(return_sparse), reservoir_matrix) +end + ### fallbacks #fallbacks for initializers #eventually to remove once migrated to WeightInitializers.jl for initializer in (:rand_sparse, :delay_line, :delay_line_backward, :cycle_jumps, :simple_cycle, :pseudo_svd, :chaotic_init, :scaled_rand, :weighted_init, :informed_init, :minimal_init, :chebyshev_mapping, :logistic_mapping, :modified_lm, :low_connectivity, :double_cycle, :self_loop_cycle, - :selfloop_feedback_cycle, :selfloop_delayline_backward, :selfloop_forward_connection) + :selfloop_feedback_cycle, :selfloop_delayline_backward, :selfloop_forward_connection, + :forward_connection) @eval begin function ($initializer)(dims::Integer...; kwargs...) return $initializer(Utils.default_rng(), Float32, dims...; kwargs...) diff --git a/test/esn/test_inits.jl b/test/esn/test_inits.jl index 930f57df..7aa35bf9 100644 --- a/test/esn/test_inits.jl +++ b/test/esn/test_inits.jl @@ -31,7 +31,8 @@ reservoir_inits = [ self_loop_cycle, selfloop_feedback_cycle, selfloop_delayline_backward, - selfloop_forward_connection + selfloop_forward_connection, + forward_connection ] input_inits = [ scaled_rand, From 82c633f641f677bce901cf9ebf1f79f5c33537a2 Mon Sep 17 00:00:00 2001 From: MartinuzziFrancesco Date: Thu, 13 Mar 2025 09:51:20 +0100 Subject: [PATCH 6/6] uniformed naming, added equations --- docs/src/api/inits.md | 2 +- src/ReservoirComputing.jl | 2 +- src/esn/esn_inits.jl | 78 ++++++++++++++++++++++++++++++++++----- test/esn/test_inits.jl | 2 +- 4 files changed, 71 insertions(+), 13 deletions(-) diff --git a/docs/src/api/inits.md b/docs/src/api/inits.md index e4f930df..eb9ef1b4 100644 --- a/docs/src/api/inits.md +++ b/docs/src/api/inits.md @@ -24,7 +24,7 @@ chaotic_init low_connectivity double_cycle - self_loop_cycle + selfloop_cycle selfloop_feedback_cycle selfloop_delayline_backward selfloop_forward_connection diff --git a/src/ReservoirComputing.jl b/src/ReservoirComputing.jl index ac3ae95f..3a032246 100644 --- a/src/ReservoirComputing.jl +++ b/src/ReservoirComputing.jl @@ -40,7 +40,7 @@ export scaled_rand, weighted_init, informed_init, minimal_init, chebyshev_mappin logistic_mapping, modified_lm export rand_sparse, delay_line, delay_line_backward, cycle_jumps, simple_cycle, pseudo_svd, chaotic_init, low_connectivity, double_cycle, - self_loop_cycle, selfloop_feedback_cycle, selfloop_delayline_backward, + selfloop_cycle, selfloop_feedback_cycle, selfloop_delayline_backward, selfloop_forward_connection, forward_connection export RNN, MRNN, GRU, GRUParams, FullyGated, Minimal export train diff --git a/src/esn/esn_inits.jl b/src/esn/esn_inits.jl index 598c95e2..67af54b9 100644 --- a/src/esn/esn_inits.jl +++ b/src/esn/esn_inits.jl @@ -1281,8 +1281,8 @@ function double_cycle(rng::AbstractRNG, ::Type{T}, dims::Integer...; return return_init_as(Val(return_sparse), reservoir_matrix) end -""" - self_loop_cycle([rng], [T], dims...; +@doc raw""" + selfloop_cycle([rng], [T], dims...; cycle_weight=0.1, selfloop_weight=0.1, return_sparse=false) @@ -1290,6 +1290,18 @@ Creates a simple cycle reservoir with the addition of self loops [^elsarraj2019] This architecture is referred to as TP1 in the original paper. +# Equations + +```math +W_{i,j} = +\begin{cases} + ll, & \text{if } i = j \\ + r, & \text{if } j = i - 1 \text{ for } i = 2 \dots N \\ + r, & \text{if } i = 1, j = N \\ + 0, & \text{otherwise} +\end{cases} +``` + # Arguments - `rng`: Random number generator. Default is `Utils.default_rng()` @@ -1309,7 +1321,7 @@ This architecture is referred to as TP1 in the original paper. # Examples ```jldoctest -julia> reservoir_matrix = self_loop_cycle(5, 5) +julia> reservoir_matrix = selfloop_cycle(5, 5) 5×5 Matrix{Float32}: 0.1 0.0 0.0 0.0 0.1 0.1 0.1 0.0 0.0 0.0 @@ -1317,7 +1329,7 @@ julia> reservoir_matrix = self_loop_cycle(5, 5) 0.0 0.0 0.1 0.1 0.0 0.0 0.0 0.0 0.1 0.1 -julia> reservoir_matrix = self_loop_cycle(5, 5; weight=0.2, selfloop_weight=0.5) +julia> reservoir_matrix = selfloop_cycle(5, 5; weight=0.2, selfloop_weight=0.5) 5×5 Matrix{Float32}: 0.5 0.0 0.0 0.0 0.2 0.2 0.5 0.0 0.0 0.0 @@ -1330,7 +1342,7 @@ julia> reservoir_matrix = self_loop_cycle(5, 5; weight=0.2, selfloop_weight=0.5) "Demystifying echo state network with deterministic simple topologies." International Journal of Computational Science and Engineering 19.3 (2019): 407-417. """ -function self_loop_cycle(rng::AbstractRNG, ::Type{T}, dims::Integer...; +function selfloop_cycle(rng::AbstractRNG, ::Type{T}, dims::Integer...; cycle_weight=T(0.1f0), selfloop_weight=T(0.1f0), return_sparse::Bool=false) where {T <: Number} throw_sparse_error(return_sparse) @@ -1340,7 +1352,7 @@ function self_loop_cycle(rng::AbstractRNG, ::Type{T}, dims::Integer...; return return_init_as(Val(return_sparse), reservoir_matrix) end -""" +@doc raw""" selfloop_feedback_cycle([rng], [T], dims...; cycle_weight=0.1, selfloop_weight=0.1, return_sparse=false) @@ -1350,6 +1362,19 @@ self loops on odd neurons [^elsarraj2019]. This architecture is referred to as TP2 in the original paper. +# Equations + +```math +W_{i,j} = +\begin{cases} + r, & \text{if } j = i - 1 \text{ for } i = 2 \dots N \\ + r, & \text{if } i = 1, j = N \\ + ll, & \text{if } i = j \text{ and } i \text{ is odd} \\ + r, & \text{if } j = i + 1 \text{ and } i \text{ is even}, i \neq N \\ + 0, & \text{otherwise} +\end{cases} +``` + # Arguments - `rng`: Random number generator. Default is `Utils.default_rng()` @@ -1409,7 +1434,7 @@ function selfloop_feedback_cycle(rng::AbstractRNG, ::Type{T}, dims::Integer...; return return_init_as(Val(return_sparse), reservoir_matrix) end -""" +@doc raw""" selfloop_delayline_backward([rng], [T], dims...; weight=0.1, selfloop_weight=0.1, return_sparse=false) @@ -1419,6 +1444,18 @@ backward connections shifted by one [^elsarraj2019]. This architecture is referred to as TP3 in the original paper. +# Equations + +```math +W_{i,j} = +\begin{cases} + ll, & \text{if } i = j \text{ for } i = 1 \dots N \\ + r, & \text{if } j = i - 1 \text{ for } i = 2 \dots N \\ + r, & \text{if } j = i - 2 \text{ for } i = 3 \dots N \\ + 0, & \text{otherwise} +\end{cases} +``` + # Arguments - `rng`: Random number generator. Default is `Utils.default_rng()` @@ -1474,7 +1511,7 @@ function selfloop_delayline_backward(rng::AbstractRNG, ::Type{T}, dims::Integer. return return_init_as(Val(return_sparse), reservoir_matrix) end -""" +@doc raw""" selfloop_forward_connection([rng], [T], dims...; weight=0.1, selfloop_weight=0.1, return_sparse=false) @@ -1484,6 +1521,17 @@ with the addition of self loops [^elsarraj2019]. This architecture is referred to as TP4 in the original paper. +# Equations + +```math +W_{i,j} = +\begin{cases} + ll, & \text{if } i = j \text{ for } i = 1 \dots N \\ + r, & \text{if } j = i - 2 \text{ for } i = 3 \dots N \\ + 0, & \text{otherwise} +\end{cases} +``` + # Arguments - `rng`: Random number generator. Default is `Utils.default_rng()` @@ -1536,7 +1584,7 @@ function selfloop_forward_connection(rng::AbstractRNG, ::Type{T}, dims::Integer. return return_init_as(Val(return_sparse), reservoir_matrix) end -""" +@doc raw""" forward_connection([rng], [T], dims...; weight=0.1, selfloop_weight=0.1, return_sparse=false) @@ -1545,6 +1593,16 @@ Creates a reservoir based on a forward connection of weights [^elsarraj2019]. This architecture is referred to as TP5 in the original paper. +# Equations + +```math +W_{i,j} = +\begin{cases} + r, & \text{if } j = i - 2 \text{ for } i = 3 \dots N \\ + 0, & \text{otherwise} +\end{cases} +``` + # Arguments - `rng`: Random number generator. Default is `Utils.default_rng()` @@ -1598,7 +1656,7 @@ end for initializer in (:rand_sparse, :delay_line, :delay_line_backward, :cycle_jumps, :simple_cycle, :pseudo_svd, :chaotic_init, :scaled_rand, :weighted_init, :informed_init, :minimal_init, :chebyshev_mapping, - :logistic_mapping, :modified_lm, :low_connectivity, :double_cycle, :self_loop_cycle, + :logistic_mapping, :modified_lm, :low_connectivity, :double_cycle, :selfloop_cycle, :selfloop_feedback_cycle, :selfloop_delayline_backward, :selfloop_forward_connection, :forward_connection) @eval begin diff --git a/test/esn/test_inits.jl b/test/esn/test_inits.jl index 7aa35bf9..9f475715 100644 --- a/test/esn/test_inits.jl +++ b/test/esn/test_inits.jl @@ -28,7 +28,7 @@ reservoir_inits = [ chaotic_init, low_connectivity, double_cycle, - self_loop_cycle, + selfloop_cycle, selfloop_feedback_cycle, selfloop_delayline_backward, selfloop_forward_connection,