Merge branch 'master' into onlyreal

Author: devmotion

.travis.yml

@@ -4,7 +4,6 @@ os:
   - linux
   - osx
 julia:
-  - 0.7
   - 1.0
   - 1.1
   - nightly
@@ -15,6 +14,10 @@ notifications:
   email: false
 git:
   depth: 99999999
+env:
+  # Disable test fuzzing for the moment, as we're a little too slow for Travis
+  - NNLIB_TEST_FUZZING=false
+
 # Submit to Codecov
 after_success:
    - if [[ $TRAVIS_JULIA_VERSION = 1.1 ]] && [[ $TRAVIS_OS_NAME = linux ]]; then

Manifest.toml

@@ -3,6 +3,12 @@
 [[Base64]]
 uuid = "2a0f44e3-6c83-55bd-87e4-b1978d98bd5f"
 
+[[Crayons]]
+deps = ["Test"]
+git-tree-sha1 = "f621b8ef51fd2004c7cf157ea47f027fdeac5523"
+uuid = "a8cc5b0e-0ffa-5ad4-8c14-923d3ee1735f"
+version = "4.0.0"
+
 [[Distributed]]
 deps = ["Random", "Serialization", "Sockets"]
 uuid = "8ba89e20-285c-5b6f-9357-94700520ee1b"
@@ -25,6 +31,10 @@ uuid = "56ddb016-857b-54e1-b83d-db4d58db5568"
 deps = ["Base64"]
 uuid = "d6f4376e-aef5-505a-96c1-9c027394607a"
 
+[[Printf]]
+deps = ["Unicode"]
+uuid = "de0858da-6303-5e67-8744-51eddeeeb8d7"
+
 [[Random]]
 deps = ["Serialization"]
 uuid = "9a3f8284-a2c9-5f02-9a11-845980a1fd5c"
@@ -41,6 +51,23 @@ uuid = "9e88b42a-f829-5b0c-bbe9-9e923198166b"
 [[Sockets]]
 uuid = "6462fe0b-24de-5631-8697-dd941f90decc"
 
+[[SparseArrays]]
+deps = ["LinearAlgebra", "Random"]
+uuid = "2f01184e-e22b-5df5-ae63-d93ebab69eaf"
+
+[[Statistics]]
+deps = ["LinearAlgebra", "SparseArrays"]
+uuid = "10745b16-79ce-11e8-11f9-7d13ad32a3b2"
+
 [[Test]]
 deps = ["Distributed", "InteractiveUtils", "Logging", "Random"]
 uuid = "8dfed614-e22c-5e08-85e1-65c5234f0b40"
+
+[[TimerOutputs]]
+deps = ["Crayons", "Printf", "Test", "Unicode"]
+git-tree-sha1 = "b80671c06f8f8bae08c55d67b5ce292c5ae2660c"
+uuid = "a759f4b9-e2f1-59dc-863e-4aeb61b1ea8f"
+version = "0.5.0"
+
+[[Unicode]]
+uuid = "4ec0a83e-493e-50e2-b9ac-8f72acf5a8f5"

Project.toml

@@ -1,10 +1,13 @@
 name = "NNlib"
 uuid = "872c559c-99b0-510c-b3b7-b6c96a88d5cd"
+version = "0.6.0"
 
 [deps]
 Libdl = "8f399da3-3557-5675-b5ff-fb832c97cbdb"
 LinearAlgebra = "37e2e46d-f89d-539d-b4ee-838fcccc9c8e"
 Requires = "ae029012-a4dd-5104-9daa-d747884805df"
+Statistics = "10745b16-79ce-11e8-11f9-7d13ad32a3b2"
+TimerOutputs = "a759f4b9-e2f1-59dc-863e-4aeb61b1ea8f"
 
 [extras]
 Test = "8dfed614-e22c-5e08-85e1-65c5234f0b40"

REQUIRE

@@ -1,2 +1,3 @@
-julia 0.7-
+julia 1.0
 Requires
+TimerOutputs

src/NNlib.jl

@@ -1,16 +1,27 @@
 module NNlib
+using Requires, TimerOutputs
 
-using Requires, Libdl
+const to = TimerOutput()
 
-export σ, sigmoid, relu, leakyrelu, elu, gelu, swish, selu, softplus, softsign, logσ, logsigmoid,
-  softmax, logsoftmax, maxpool, meanpool
-
-include("numeric.jl")
+# Include APIs
+include("dim_helpers.jl")
 include("activation.jl")
 include("softmax.jl")
-include("logsoftmax.jl")
-include("linalg.jl")
+include("gemm.jl")
 include("conv.jl")
-include("cubroadcast.jl")
+include("pooling.jl")
+
+## Include implementations
+include("impl/padding_edges.jl")
+
+# Direct implementations of convolutional and depthwise-convolutional algorithms
+include("impl/conv_direct.jl")
+include("impl/depthwiseconv_direct.jl")
+# im2col implementations of convolutional and depthwise-convolutional algorithms
+include("impl/conv_im2col.jl")
+include("impl/depthwiseconv_im2col.jl")
+
+# Direct implementations of pooling
+include("impl/pooling_direct.jl")
 
-end # module
+end # module NNlib

src/activation.jl

@@ -1,43 +1,40 @@
+export σ, sigmoid, relu, leakyrelu, elu, gelu, swish, selu, softplus, softsign, logσ,
+       logsigmoid
+
 """
     σ(x) = 1 / (1 + exp(-x))
 
 Classic [sigmoid](https://en.wikipedia.org/wiki/Sigmoid_function) activation
 function.
 """
 σ(x::Real) = one(x) / (one(x) + exp(-x))
-
 const sigmoid = σ
 
 # ForwardDiff numerical stability hack
 σ_stable(x::Real) = ifelse(x < -80, zero(x), one(x) / (one(x) + exp(-x)))
-
 σ(x::Float32) = σ_stable(x)
-
 @init @require ForwardDiff="f6369f11-7733-5829-9624-2563aa707210" begin
   σ(x::ForwardDiff.Dual{T,Float32}) where T = σ_stable(x)
 end
 
+
 """
     logσ(x)
 
 Return `log(σ(x))` which is computed in a numerically stable way.
 
-    julia> logσ(0.)
+    julia> logσ(0)
     -0.6931471805599453
-    julia> logσ.([-100, -10, 100.])
+    julia> logσ.([-100, -10, 100])
     3-element Array{Float64,1}:
      -100.0
-      -10.0
-       -0.0
+      -10.000045398899218
+       -3.720075976020836e-44
 """
-function logσ(x::Real)
-  max_v = max(zero(x), -x)
-  z = exp(-max_v) + exp(-x-max_v)
-  -(max_v + log(z))
-end
-
+logσ(x::Real) = -softplus(-x)
 const logsigmoid = logσ
 
+
 """
     relu(x) = max(0, x)
 
@@ -56,6 +53,7 @@ You can also specify the coefficient explicitly, e.g. `leakyrelu(x, 0.01)`.
 """
 leakyrelu(x::Real, a = oftype(x/1, 0.01)) = max(a*x, x/1)
 
+
 """
     elu(x, α = 1) =
       x > 0 ? x : α * (exp(x) - 1)
@@ -66,6 +64,7 @@ You can also specify the coefficient explicitly, e.g. `elu(x, 1)`.
 """
 elu(x, α = one(x)) = ifelse(x ≥ 0, x/1, α * (exp(x) - one(x)))
 
+
 """
     gelu(x) = 0.5x*(1 + tanh(√(2/π)*(x + 0.044715x^3)))
 
@@ -103,6 +102,7 @@ function selu(x::Real)
   λ * ifelse(x > 0, x/1, α * (exp(x) - 1))
 end
 
+
 """
     softsign(x) = x / (1 + |x|)