docs: added relevant docstrings for new init kwargs

Author: MartinuzziFrancesco

Project.toml

@@ -1,7 +1,7 @@
 name = "ReservoirComputing"
 uuid = "7c2d2b1e-3dd4-11ea-355a-8f6a8116e294"
 authors = ["Francesco Martinuzzi"]
-version = "0.10.13"
+version = "0.11.0"
 
 [deps]
 Adapt = "79e6a3ab-5dfb-504d-930d-738a2a938a0e"

docs/src/esn_tutorials/change_layers.md

@@ -1,6 +1,10 @@
 # Using different layers
 
-A great deal of efforts in the ESNs field are devoted to finding an ideal construction for the reservoir matrices. ReservoirComputing.jl offers multiple implementation of reservoir and input matrices initializations found in the literature. The API is standardized, and follows by [WeightInitializers.jl](https://github.com/LuxDL/Lux.jl/tree/main/lib/WeightInitializers):
+A great deal of efforts in the ESNs field are devoted to finding an ideal construction
+for the reservoir matrices. ReservoirComputing.jl offers multiple implementation of
+reservoir and input matrices initializations found in the literature.
+The API is standardized, and follows 
+[WeightInitializers.jl](https://github.com/LuxDL/Lux.jl/tree/main/lib/WeightInitializers):
 
 ```julia
 weights = init(rng, dims...)
@@ -22,9 +26,13 @@ Custom layers only need to follow these APIs to be compatible with ReservoirComp
 
 ## Example of minimally complex ESN
 
-Using [^rodan2012] and [^rodan2010] as references this section will provide an example on how to change both the input layer and the reservoir for ESNs.
+Using [^rodan2012] and [^rodan2010] as references this section will provide an
+example on how to change both the input layer and the reservoir for ESNs.
 
-The task for this example will be the one step ahead prediction of the Henon map. To obtain the data one can leverage the package [PredefinedDynamicalSystems.jl](https://juliadynamics.github.io/PredefinedDynamicalSystems.jl/dev/). The data is scaled to be between -1 and 1.
+The task for this example will be the one step ahead prediction of the Henon map.
+To obtain the data one can leverage the package
+[PredefinedDynamicalSystems.jl](https://juliadynamics.github.io/PredefinedDynamicalSystems.jl/dev/).
+The data is scaled to be between -1 and 1.
 
 ```@example minesn
 using PredefinedDynamicalSystems
@@ -43,14 +51,16 @@ testing_input = data[:, (shift + train_len):(shift + train_len + predict_len - 1
 testing_target = data[:, (shift + train_len + 1):(shift + train_len + predict_len)]
 ```
 
-Now it is possible to define the input layers and reservoirs we want to compare and run the comparison in a simple for loop. The accuracy will be tested using the mean squared deviation msd from StatsBase.
+Now it is possible to define the input layers and reservoirs we want to compare and run
+the comparison in a simple for loop. The accuracy will be tested using the mean squared
+deviation msd from StatsBase.
 
 ```@example minesn
 using ReservoirComputing, StatsBase
 
 res_size = 300
-input_layer = [minimal_init(; weight=0.85, sampling_type=:irrational),
-    minimal_init(; weight=0.95, sampling_type=:irrational)]
+input_layer = [minimal_init(; weight=0.85, sampling_type=:irrational_sample!),
+    minimal_init(; weight=0.95, sampling_type=:irrational_sample!)]
 reservoirs = [simple_cycle(; weight=0.7),
     cycle_jumps(; cycle_weight=0.7, jump_weight=0.2, jump_size=5)]
 
@@ -64,9 +74,14 @@ for i in 1:length(reservoirs)
 end
 ```
 
-As it is possible to see, changing layers in ESN models is straightforward. Be sure to check the API documentation for a full list of reservoir and layers.
+As it is possible to see, changing layers in ESN models is straightforward.
+Be sure to check the API documentation for a full list of reservoir and layers.
 
 ## Bibliography
 
-[^rodan2012]: Rodan, Ali, and Peter Tiňo. “Simple deterministically constructed cycle reservoirs with regular jumps.” Neural computation 24.7 (2012): 1822-1852.
-[^rodan2010]: Rodan, Ali, and Peter Tiňo. “Minimum complexity echo state network.” IEEE transactions on neural networks 22.1 (2010): 131-144.
+[^rodan2012]: Rodan, Ali, and Peter Tiňo.
+    “Simple deterministically constructed cycle reservoirs with regular jumps.”
+    Neural computation 24.7 (2012): 1822-1852.
+[^rodan2010]: Rodan, Ali, and Peter Tiňo.
+    “Minimum complexity echo state network.”
+    IEEE transactions on neural networks 22.1 (2010): 131-144.

src/ReservoirComputing.jl

@@ -43,7 +43,8 @@ export rand_sparse, delay_line, delay_line_backward, cycle_jumps,
        simple_cycle, pseudo_svd, chaotic_init, low_connectivity, double_cycle,
        selfloop_cycle, selfloop_feedback_cycle, selfloop_delayline_backward,
        selfloop_forward_connection, forward_connection
-export scale_radius!, delay_line!, backward_connection!, simple_cycle!, add_jumps!
+export scale_radius!, delay_line!, backward_connection!, simple_cycle!, add_jumps!,
+       self_loop!
 export RNN, MRNN, GRU, GRUParams, FullyGated, Minimal
 export train
 export ESN, HybridESN, KnowledgeModel, DeepESN

src/esn/esn_inits.jl

@@ -243,7 +243,7 @@ julia> res_input = minimal_init(8, 3; p=0.8)# higher p -> more positive signs
 """
 function minimal_init(rng::AbstractRNG, ::Type{T}, dims::Integer...;
         weight::Number=T(0.1), sampling_type::Symbol=:bernoulli_sample!, kwargs...) where {T <:
-                                                                                    Number}
+                                                                                           Number}
     res_size, in_size = dims
     input_matrix = DeviceAgnostic.zeros(rng, T, res_size, in_size)
     input_matrix .+= T(weight)
@@ -577,7 +577,8 @@ end
 
 """
     delay_line([rng], [T], dims...;
-        weight=0.1, return_sparse=false)
+        weight=0.1, return_sparse=false,
+        kwargs...)
 
 Create and return a delay line reservoir matrix [^rodan2010].
 
@@ -608,7 +609,6 @@ Create and return a delay line reservoir matrix [^rodan2010].
   - `start`: Which place after the decimal point the counting starts for the `irrational`
     sign counting. Default is 1.
 
-
 # Examples
 
 ```jldoctest
@@ -648,7 +648,8 @@ end
 
 """
     delay_line_backward([rng], [T], dims...;
-        weight=0.1, fb_weight=0.2, return_sparse=false)
+        weight=0.1, fb_weight=0.2, return_sparse=false,
+        delay_kwargs=(), fb_kwargs=())
 
 Create a delay line backward reservoir with the specified by `dims` and weights.
 Creates a matrix with backward connections as described in [^rodan2010].
@@ -669,6 +670,19 @@ Creates a matrix with backward connections as described in [^rodan2010].
     in the reservoir. Default is 0.2
   - `return_sparse`: flag for returning a `sparse` matrix.
     Default is `false`.
+  - `delay_kwargs` and `fb_kwargs`: named tuples that control the kwargs for the 
+    delay line weight and feedback weights respectively. The kwargs are as follows:
+    + `sampling_type`: Sampling that decides the distribution of `weight` negative numbers.
+      If set to `:no_sample` the sign is unchanged. If set to `:bernoulli_sample!` then each
+      `weight` can be positive with a probability set by `positive_prob`. If set to
+      `:irrational_sample!` the `weight` is negative if the decimal number of the
+      irrational number chosen is odd. Default is `:no_sample`.
+    + `positive_prob`: probability of the `weight` being positive when `sampling_type` is
+      set to `:bernoulli_sample!`. Default is 0.5
+    + `irrational`: Irrational number whose decimals decide the sign of `weight`.
+      Default is `pi`.
+    + `start`: Which place after the decimal point the counting starts for the `irrational`
+      sign counting. Default is 1.
 
 # Examples
 
@@ -707,7 +721,8 @@ end
 
 """
     cycle_jumps([rng], [T], dims...; 
-        cycle_weight=0.1, jump_weight=0.1, jump_size=3, return_sparse=false)
+        cycle_weight=0.1, jump_weight=0.1, jump_size=3, return_sparse=false,
+        cycle_kwargs=(), jump_kwargs=())
 
 Create a cycle jumps reservoir [^Rodan2012].
 
@@ -729,6 +744,19 @@ Create a cycle jumps reservoir [^Rodan2012].
     Default is 3.
   - `return_sparse`: flag for returning a `sparse` matrix.
     Default is `false`.
+  - `cycle_kwargs` and `jump_kwargs`: named tuples that control the kwargs for the 
+    cycle and jump weights respectively. The kwargs are as follows:
+    + `sampling_type`: Sampling that decides the distribution of `weight` negative numbers.
+      If set to `:no_sample` the sign is unchanged. If set to `:bernoulli_sample!` then each
+      `weight` can be positive with a probability set by `positive_prob`. If set to
+      `:irrational_sample!` the `weight` is negative if the decimal number of the
+      irrational number chosen is odd. Default is `:no_sample`.
+    + `positive_prob`: probability of the `weight` being positive when `sampling_type` is
+      set to `:bernoulli_sample!`. Default is 0.5
+    + `irrational`: Irrational number whose decimals decide the sign of `weight`.
+      Default is `pi`.
+    + `start`: Which place after the decimal point the counting starts for the `irrational`
+      sign counting. Default is 1.
 
 # Examples
 
@@ -769,7 +797,8 @@ end
 
 """
     simple_cycle([rng], [T], dims...; 
-        weight=0.1, return_sparse=false)
+        weight=0.1, return_sparse=false,
+        kwargs...)
 
 Create a simple cycle reservoir [^rodan2010].
 
@@ -786,6 +815,17 @@ Create a simple cycle reservoir [^rodan2010].
     Default is 0.1.
   - `return_sparse`: flag for returning a `sparse` matrix.
     Default is `false`.
+  - `sampling_type`: Sampling that decides the distribution of `weight` negative numbers.
+    If set to `:no_sample` the sign is unchanged. If set to `:bernoulli_sample!` then each
+    `weight` can be positive with a probability set by `positive_prob`. If set to
+    `:irrational_sample!` the `weight` is negative if the decimal number of the
+    irrational number chosen is odd. Default is `:no_sample`.
+  - `positive_prob`: probability of the `weight` being positive when `sampling_type` is
+    set to `:bernoulli_sample!`. Default is 0.5
+  - `irrational`: Irrational number whose decimals decide the sign of `weight`.
+    Default is `pi`.
+  - `start`: Which place after the decimal point the counting starts for the `irrational`
+    sign counting. Default is 1.
 
 # Examples
 
@@ -1210,7 +1250,7 @@ end
 @doc raw"""
     selfloop_cycle([rng], [T], dims...; 
         cycle_weight=0.1, selfloop_weight=0.1,
-        return_sparse=false)
+        return_sparse=false, kwargs...)
 
 Creates a simple cycle reservoir with the addition of self loops [^elsarraj2019].
 
@@ -1243,6 +1283,17 @@ W_{i,j} =
     Default is 0.1.
   - `return_sparse`: flag for returning a `sparse` matrix.
     Default is `false`.
+  - `sampling_type`: Sampling that decides the distribution of `weight` negative numbers.
+    If set to `:no_sample` the sign is unchanged. If set to `:bernoulli_sample!` then each
+    `weight` can be positive with a probability set by `positive_prob`. If set to
+    `:irrational_sample!` the `weight` is negative if the decimal number of the
+    irrational number chosen is odd. Default is `:no_sample`.
+  - `positive_prob`: probability of the `weight` being positive when `sampling_type` is
+    set to `:bernoulli_sample!`. Default is 0.5
+  - `irrational`: Irrational number whose decimals decide the sign of `weight`.
+    Default is `pi`.
+  - `start`: Which place after the decimal point the counting starts for the `irrational`
+    sign counting. Default is 1.
 
 # Examples
 
@@ -1363,7 +1414,8 @@ end
 @doc raw"""
     selfloop_delayline_backward([rng], [T], dims...; 
         weight=0.1, selfloop_weight=0.1,
-        return_sparse=false)
+        return_sparse=false, fb_kwargs=(), selfloop_kwargs=(),
+        delay_kwargs=())
 
 Creates a reservoir based on a delay line with the addition of self loops and
 backward connections shifted by one [^elsarraj2019].
@@ -1397,6 +1449,19 @@ W_{i,j} =
     Default is 0.1.
   - `return_sparse`: flag for returning a `sparse` matrix.
     Default is `false`.
+  - `delay_kwargs`, `selfloop_kwargs`, and `fb_kwargs`: named tuples that control the kwargs
+    for the weights generation. The kwargs are as follows:
+    + `sampling_type`: Sampling that decides the distribution of `weight` negative numbers.
+      If set to `:no_sample` the sign is unchanged. If set to `:bernoulli_sample!` then each
+      `weight` can be positive with a probability set by `positive_prob`. If set to
+      `:irrational_sample!` the `weight` is negative if the decimal number of the
+      irrational number chosen is odd. Default is `:no_sample`.
+    + `positive_prob`: probability of the `weight` being positive when `sampling_type` is
+      set to `:bernoulli_sample!`. Default is 0.5
+    + `irrational`: Irrational number whose decimals decide the sign of `weight`.
+      Default is `pi`.
+    + `start`: Which place after the decimal point the counting starts for the `irrational`
+      sign counting. Default is 1.
 
 # Examples
 
@@ -1438,7 +1503,8 @@ end
 @doc raw"""
     selfloop_forward_connection([rng], [T], dims...; 
         weight=0.1, selfloop_weight=0.1,
-        return_sparse=false)
+        return_sparse=false, selfloop_kwargs=(),
+        delay_kwargs=())
 
 Creates a reservoir based on a forward connection of weights between even nodes
 with the addition of self loops [^elsarraj2019].
@@ -1471,6 +1537,19 @@ W_{i,j} =
     Default is 0.1.
   - `return_sparse`: flag for returning a `sparse` matrix.
     Default is `false`.
+  - `delay_kwargs` and `selfloop_kwargs`: named tuples that control the kwargs for the 
+    delay line weight and self loop weights respectively. The kwargs are as follows:
+    + `sampling_type`: Sampling that decides the distribution of `weight` negative numbers.
+      If set to `:no_sample` the sign is unchanged. If set to `:bernoulli_sample!` then each
+      `weight` can be positive with a probability set by `positive_prob`. If set to
+      `:irrational_sample!` the `weight` is negative if the decimal number of the
+      irrational number chosen is odd. Default is `:no_sample`.
+    + `positive_prob`: probability of the `weight` being positive when `sampling_type` is
+      set to `:bernoulli_sample!`. Default is 0.5
+    + `irrational`: Irrational number whose decimals decide the sign of `weight`.
+      Default is `pi`.
+    + `start`: Which place after the decimal point the counting starts for the `irrational`
+      sign counting. Default is 1.
 
 # Examples
 

src/esn/inits_components.jl

@@ -43,7 +43,8 @@ function no_sample(rng::AbstractRNG, vecormat::AbstractVecOrMat)
     return vecormat
 end
 
-function bernoulli_sample!(rng::AbstractRNG, vecormat::AbstractVecOrMat; positive_prob::Number=0.5)
+function bernoulli_sample!(
+        rng::AbstractRNG, vecormat::AbstractVecOrMat; positive_prob::Number=0.5)
     for idx in eachindex(vecormat)
         if rand(rng) > positive_prob
             vecormat[idx] = -vecormat[idx]
@@ -328,7 +329,7 @@ Adds jumps to a given `reservoir_matrix` with chosen `weight` and determined `ju
   - `start`: Which place after the decimal point the counting starts for the `irrational`
     sign counting. Default is 1.
 
-# Examples 
+# Examples
 
 ```jldoctest
 julia> matrix = zeros(Float32, 5, 5)
@@ -423,7 +424,6 @@ julia> self_loop!(matrix, 1.0)
   0.0  0.0   0.0   0.0   1.0
 ```
 """
-
 function self_loop!(rng::AbstractRNG, reservoir_matrix::AbstractMatrix,
         weight::Number; kwargs...)
     weights = fill(weight, size(reservoir_matrix, 1))