Merge pull request #13 from ChevronETC/activations

adding hyperbolic tangent and tests

Author: deckerla

src/JetPack.jl

@@ -39,6 +39,7 @@ include("jop_roughness.jl")
 include("jop_shift.jl")
 include("jop_sigmoid.jl")
 include("jop_taper.jl")
+include("jop_tanh.jl")
 include("jop_translation.jl")
 
 end

src/jop_tanh.jl

@@ -0,0 +1,17 @@
+"""
+    F = JopTanh(spc)
+
+where `F` is the hyperbolic tangent operator with domain and range given by `spc::JetSpace`.
+we use 'tanh'(c*x) = (\exp{c*x}-\exp{-c*x})/(\exp{c*x}+\exp{-c*x}),
+the derivative of which is 1-tanh(c*x)^2.
+We expect the domain and range to be real.
+"""
+JopTanh(spc::JetSpace{T}, c = 1) where {T} = JopNl(dom = spc, rng = spc, f! = JopTanh_f!, df! = JopTanh_df!, df′! = JopTanh_df′!, s = (c=T(c),))
+export JopTanh
+
+JopTanh_f!(d::AbstractArray, m::AbstractArray; c) = d .= tanh.(c .* m)
+
+JopTanh_df!(δd::AbstractArray, δm::AbstractArray; mₒ, c) = δd .= c .* (1 .- tanh.(c .* mₒ).^2) .* δm
+
+JopTanh_df′!(δd::AbstractArray, δm::AbstractArray; mₒ, c) = δd .= conj.(c .* (1 .- tanh.(c .* mₒ).^2)) .* δm
+

test/jop_tanh.jl

@@ -0,0 +1,40 @@
+using LinearAlgebra, Jets, JetPack, Test
+
+n1,n2 = 33,44
+
+@testset "JopTanh, correctness T=$(T)" for T in (Float64,Float32,Complex{Float64},Complex{Float32})
+    F = JopTanh(JetSpace(T,n1,n2))
+    x1 = rand(domain(F)) 
+    @test F*x1 ≈ tanh.(x1)
+end
+
+@testset "JopTanh, linearity test, T=$(T)" for T in (Float64,Float32,Complex{Float64},Complex{Float32})
+    F = JopTanh(JetSpace(T,n1,n2))
+    J = jacobian(F, rand(domain(F)) )
+    lhs,rhs = linearity_test(J)
+    @test lhs ≈ rhs
+end
+
+@testset "JopTanh, dot product test, T=$(T)" for T in (Float64,Float32,Complex{Float64},Complex{Float32})
+    F = JopTanh(JetSpace(T,n1,n2))
+    J  = jacobian!(F, rand(domain(F)) )
+    lhs, rhs = dot_product_test(J, -1 .+ 2 .* rand(domain(J)), -1 .+ 2 .* rand(range(J)))
+    @test lhs ≈ rhs
+end
+
+# note the key here is to increase the size of the nonlinear vector
+@testset "JopTanh, linearization test, T=$(T)" for T in (Float64,Float32,Complex{Float64},Complex{Float32})
+    F = JopTanh(JetSpace(T,n1,n2))
+    m0 = -0.5 .* rand(domain(F))
+    μ  = sqrt.([1/1,1/2,1/4,1/8,1/16,1/32,1/64,1/128,1/256,1/512,1/1024,1/2048])
+    δm = rand(domain(F))
+    observed, expected = linearization_test(F, m0, μ = μ, δm = δm)
+    δ = minimum(abs, observed - expected)
+    @test δ < 0.1
+end
+
+@testset "JopTanh, odd funcion T=$(T)" for T in (Float64,Float32,Complex{Float64},Complex{Float32})
+    F = JopTanh(JetSpace(T,n1,n2))
+    x1 = 2 .* (-1 .+ 2 .* rand(domain(F)))
+    @test F * (-1 .* x1) ≈ -(F * x1)
+end

test/runtests.jl

@@ -36,6 +36,7 @@ for filename in (
         "jop_shift.jl",
         "jop_sigmoid.jl",
         "jop_taper.jl",
+        "jop_tanh.jl",
         "jop_translation.jl")
     include(filename)
 end