Merge pull request #108 from pshashk/logcosh

Add `logcosh` activation

Author: staticfloat

src/activation.jl

@@ -1,5 +1,5 @@
 export σ, sigmoid, relu, leakyrelu, elu, gelu, swish, selu, softplus, softsign, logσ,
-       logsigmoid
+       logsigmoid, logcosh
 
 """
     σ(x) = 1 / (1 + exp(-x))
@@ -118,8 +118,16 @@ See [Deep Sparse Rectifier Neural Networks](http://proceedings.mlr.press/v15/glo
 """
 softplus(x::Real) = ifelse(x > 0, x + log1p(exp(-x)), log1p(exp(x)))
 
+
+"""
+    logcosh(x)
+
+Return `log(cosh(x))` which is computed in a numerically stable way.
+"""
+logcosh(x::T) where T = x + softplus(-2x) - log(convert(T, 2))
+
 # Provide an informative error message if activation functions are called with an array
-for f in (:σ, :σ_stable, :logσ, :relu, :leakyrelu, :elu, :gelu, :swish, :selu, :softsign, :softplus)
+for f in (:σ, :σ_stable, :logσ, :relu, :leakyrelu, :elu, :gelu, :swish, :selu, :softsign, :softplus, :logcosh)
   @eval $(f)(x::AbstractArray, args...) =
     error("Use broadcasting (`", $(string(f)), ".(x)`) to apply activation functions to arrays.")
 end

test/activation.jl

@@ -1,6 +1,6 @@
 using NNlib, Test
 
-ACTIVATION_FUNCTIONS = [σ, relu, leakyrelu, elu, gelu, swish, selu, softplus, softsign];
+ACTIVATION_FUNCTIONS = [σ, relu, leakyrelu, elu, gelu, swish, selu, softplus, softsign, logcosh];
 
 function test_value_float_precision_preserving(a)
     @testset "$(a): " begin
@@ -36,6 +36,7 @@ end
     @test softplus(-1e8) ≈ 0.0
     @test softsign(0.0) == 0.0
     @test selu(0.0) == 0.0
+    @test logcosh(0.0) == log(cosh(0.0))
 
     @test σ(1.0) == 1.0 / (1.0 + exp(-1.0))
     @test relu(1.0) == 1.0
@@ -46,6 +47,7 @@ end
     @test softplus(1.0) ≈ log(exp(1.0) + 1.0)
     @test softsign(1.0) == 0.5
     @test selu(1.0) == 1.0507009873554804934193349852946
+    @test logcosh(1.0) ≈ log(cosh(1.0))
 
     @test σ(-1.0) == 1.0 / (1.0 + exp(1.0))
     @test relu(-1.0) == 0.0
@@ -56,6 +58,7 @@ end
     @test softplus(-1.0) ≈ log(exp(-1.0) + 1.0)
     @test softsign(-1.0) == -0.5
     @test selu(-1.0) == 1.0507009873554804934193349852946 * 1.6732632423543772848170429916717 * (exp(-1.0) - 1.0)
+    @test log(cosh(-1.0)) ≈ log(cosh(-1.0))
 
     @testset "Float inference" begin
         test_value_float_precision_preserving.(ACTIVATION_FUNCTIONS)
@@ -132,4 +135,6 @@ end
             @eval @test logsigmoid.($T[-100_000, 100_000.]) ≈ $T[-100_000, 0.]
         end
     end
+
+    @test logcosh(1_000.0) + log(2) == 1_000.0
 end