whitespace

Author: DhairyaLGandhi

src/layers/basic.jl

@@ -1,16 +1,23 @@
 """
     Chain(layers...)
+
 Chain multiple layers / functions together, so that they are called in sequence
 on a given input.
+
 `Chain` also supports indexing and slicing, e.g. `m[2]` or `m[1:end-1]`.
 `m[1:3](x)` will calculate the output of the first three layers.
+
 # Examples
 ```jldoctest
 julia> m = Chain(x -> x^2, x -> x+1);
+
 julia> m(5) == 26
 true
+
 julia> m = Chain(Dense(10, 5), Dense(5, 2));
+
 julia> x = rand(10);
+
 julia> m(x) == m[2](m[1](x))
 true
 ```
@@ -63,30 +70,40 @@ extraChain(::Tuple{}, x) = ()
 """
     Dense(in, out, σ=identity; bias=true, init=glorot_uniform)
     Dense(W::AbstractMatrix, [bias, σ])
+
 Create a traditional `Dense` layer, whose forward pass is given by:
+
     y = σ.(W * x .+ bias)
+
 The input `x` should be a vector of length `in`, or batch of vectors represented
 as an `in × N` matrix, or any array with `size(x,1) == in`.
 The out `y` will be a vector  of length `out`, or a batch with
 `size(y) == (out, size(x)[2:end]...)`
+
 Keyword `bias=false` will switch off trainable bias for the layer.
 The initialisation of the weight matrix is `W = init(out, in)`, calling the function
 given to keyword `init`, with default [`glorot_uniform`](@doc Flux.glorot_uniform).
 The weight matrix and/or the bias vector (of length `out`) may also be provided explicitly.
+
 # Examples
 ```jldoctest
 julia> d = Dense(5, 2)
 Dense(5, 2)
+
 julia> d(rand(Float32, 5, 64)) |> size
 (2, 64)
+
 julia> d(rand(Float32, 5, 1, 1, 64)) |> size  # treated as three batch dimensions
 (2, 1, 1, 64)
+
 julia> d1 = Dense(ones(2, 5), false, tanh)  # using provided weight matrix
 Dense(5, 2, tanh; bias=false)
+
 julia> d1(ones(5))
 2-element Array{Float64,1}:
  0.9999092042625951
  0.9999092042625951
+
 julia> Flux.params(d1)  # no trainable bias
 Params([[1.0 1.0 … 1.0 1.0; 1.0 1.0 … 1.0 1.0]])
 ```
@@ -142,10 +159,14 @@ end
 """
     Diagonal(α, β)
     Diagonal(size::Integer...)
+
 Create an element-wise linear layer, which performs
+
     y = α .* x .+ β
+
 The learnable arrays are initialised `α = ones(Float32, size)` and
 `β = zeros(Float32, size)`.
+
 Used by [`LayerNorm`](@ref).
 """
 struct Diagonal{T}
@@ -179,9 +200,11 @@ end
 
 """
     Maxout(over)
+
 The [Maxout](https://arxiv.org/abs/1302.4389) layer has a number of
 internal layers which all receive the same input. It returns the elementwise
 maximum of the internal layers' outputs.
+
 Maxout over linear dense layers satisfies the univeral approximation theorem.
 """
 struct Maxout{FS<:Tuple}
@@ -190,15 +213,20 @@ end
 
 """
     Maxout(f, n_alts)
+
 Construct a Maxout layer over `n_alts` instances of the layer given by `f`.
 The function takes no arguments and should return some callable layer.
 Conventionally, this is a linear dense layer.
+
 # Examples
+
 This constructs a `Maxout` layer over 4 internal dense linear layers, each
 identical in structure (784 inputs, 128 outputs):
 ```jldoctest
 julia> insize = 784;
+
 julia> outsize = 128;
+
 julia> Maxout(()->Dense(insize, outsize), 4);
 ```
 """
@@ -215,19 +243,25 @@ end
 
 """
     SkipConnection(layer, connection)
+
 Create a skip connection which consists of a layer or `Chain` of consecutive
 layers and a shortcut connection linking the block's input to the output
 through a user-supplied 2-argument callable. The first argument to the callable
 will be propagated through the given `layer` while the second is the unchanged,
 "skipped" input.
+
 The simplest "ResNet"-type connection is just `SkipConnection(layer, +)`.
 Here is a more complicated example:
 ```jldoctest
 julia> m = Conv((3,3), 4 => 7, pad=(1,1));
+
 julia> x = ones(Float32, 5, 5, 4, 10);
+
 julia> size(m(x)) == (5, 5, 7, 10)
 true
+
 julia> sm = SkipConnection(m, (mx, x) -> cat(mx, x, dims=3));
+
 julia> size(sm(x)) == (5, 5, 11, 10)
 true
 ```
@@ -250,32 +284,45 @@ end
 """
     Bilinear(in1, in2, out, σ=identity; bias=true, init=glorot_uniform)
     Bilinear(W::AbstractArray, [bias, σ])
+
 Creates a Bilinear layer, which operates on two inputs at the same time.
 Its output, given vectors `x` & `y`, is another vector `z` with,
 for all `i ∈ 1:out`:
+
     z[i] = σ(x' * W[i,:,:] * y + bias[i])
+
 If `x` and `y` are matrices, then each column of the output `z = B(x, y)` is of this form,
 with `B` a Bilinear layer.
+
 If `y` is not given, it is taken to be equal to `x`, i.e. `B(x) == B(x, x)`
+
 The two inputs may also be provided as a tuple, `B((x, y)) == B(x, y)`,
 which is accepted as the input to a `Chain`.
+
 The initialisation works as for [`Dense`](@ref) layer, with `W = init(out, in1, in2)`.
 By default the bias vector is `zeros(Float32, out)`, option `bias=false` will switch off
 trainable bias. Either of these may be provided explicitly.
+
 # Examples
 ```jldoctest
 julia> x, y = randn(Float32, 5, 32), randn(Float32, 5, 32);
+
 julia> B = Flux.Bilinear(5, 5, 7);
+
 julia> B(x) |> size  # interactions based on one input
 (7, 32)
+
 julia> B(x,y) == B((x,y))  # two inputs, may be given as a tuple
 true
+
 julia> sc = SkipConnection(
                 Chain(Dense(5, 20, tanh), Dense(20, 9, tanh)),
                 Flux.Bilinear(9, 5, 3, bias=false),
             );  # used as the recombinator, with skip as the second input
+
 julia> sc(x) |> size
 (3, 32)
+
 julia> Flux.Bilinear(rand(4,8,16), false, tanh)  # first dim of weight is the output
 Bilinear(8, 16, 4, tanh, bias=false)
 ```
@@ -329,17 +376,22 @@ end
 
 """
     Parallel(connection, layers...)
+
 Create a 'Parallel' layer that passes an input array to each path in
 `layers`, reducing the output with `connection`.
+
 Called with one input `x`, this is equivalent to `reduce(connection, [l(x) for l in layers])`.
 If called with multiple inputs, they are `zip`ped with the layers, thus `Parallel(+, f, g)(x, y) = f(x) + g(y)`.
+
 # Examples
 ```jldoctest
 julia> model = Chain(Dense(3, 5),
                      Parallel(vcat, Dense(5, 4), Chain(Dense(5, 7), Dense(7, 4))),
                      Dense(8, 17));
+
 julia> size(model(rand(3)))
 (17,)
+
 julia> model = Parallel(+, Dense(10, 2), Dense(5, 2))
 Parallel(+, Dense(10, 2), Dense(5, 2))
 julia> size(model(rand(10), rand(5)))
@@ -366,4 +418,4 @@ function Base.show(io::IO, m::Parallel)
   print(io, "Parallel(", m.connection, ", ")
   join(io, m.layers, ", ")
   print(io, ")")
-end
+end