Merge pull request #59 from pevnak/pull-request/ea057b05

softplus

Author: MikeInnes

src/activation.jl

@@ -116,4 +116,4 @@ softsign(x) = x / (one(x) + abs(x))
 
 See [Deep Sparse Rectifier Neural Networks](http://proceedings.mlr.press/v15/glorot11a/glorot11a.pdf).
 """
-softplus(x) = log1p(exp(x))
+softplus(x) = ifelse(x > 0, x + log1p(exp(-x)), log1p(exp(x)))

test/activation.jl

@@ -45,6 +45,8 @@ end
   @test gelu(0.0) == 0.0
   @test swish(0.0) == 0.0
   @test softplus(0.0) ≈ log(2.0)
+  @test softplus(1e8) ≈ 1e8
+  @test softplus(-1e8) ≈ 0.0
   @test softsign(0.0) == 0.0
   @test selu(0.0) == 0.0