Update NNPACK interface to conform to the new NNlib

Author: Avik Pal

.travis.yml

@@ -9,6 +9,9 @@ notifications:
   email: false
 git:
   depth: 99999999
+env:
+  # Disable test fuzzing for the moment, as we're a little too slow for Travis
+  - NNLIB_TEST_FUZZING=false
 
 # Submit to Codecov  
 after_success:

Manifest.toml

@@ -1,32 +1,30 @@
+# This file is machine-generated - editing it directly is not advised
+
 [[Base64]]
 uuid = "2a0f44e3-6c83-55bd-87e4-b1978d98bd5f"
 
 [[BinaryProvider]]
 deps = ["Libdl", "Pkg", "SHA", "Test"]
-git-tree-sha1 = "9930c1a6cd49d9fcd7218df6be417e6ae4f1468a"
+git-tree-sha1 = "055eb2690182ebc31087859c3dd8598371d3ef9e"
 uuid = "b99e7846-7c00-51b0-8f62-c81ae34c0232"
-version = "0.5.2"
+version = "0.5.3"
 
-[[Compat]]
-deps = ["Base64", "Dates", "DelimitedFiles", "Distributed", "InteractiveUtils", "LibGit2", "Libdl", "LinearAlgebra", "Markdown", "Mmap", "Pkg", "Printf", "REPL", "Random", "Serialization", "SharedArrays", "Sockets", "SparseArrays", "Statistics", "Test", "UUIDs", "Unicode"]
-git-tree-sha1 = "2d9e14d19bad3f9ad5cc5e4cffabc3cfa59de825"
-uuid = "34da2185-b29b-5c13-b0c7-acf172513d20"
-version = "1.3.0"
+[[Crayons]]
+deps = ["Test"]
+git-tree-sha1 = "f621b8ef51fd2004c7cf157ea47f027fdeac5523"
+uuid = "a8cc5b0e-0ffa-5ad4-8c14-923d3ee1735f"
+version = "4.0.0"
 
 [[Dates]]
 deps = ["Printf"]
 uuid = "ade2ca70-3891-5945-98fb-dc099432e06a"
 
-[[DelimitedFiles]]
-deps = ["Mmap"]
-uuid = "8bb1440f-4735-579b-a4ab-409b98df4dab"
-
 [[Distributed]]
-deps = ["LinearAlgebra", "Random", "Serialization", "Sockets"]
+deps = ["Random", "Serialization", "Sockets"]
 uuid = "8ba89e20-285c-5b6f-9357-94700520ee1b"
 
 [[InteractiveUtils]]
-deps = ["LinearAlgebra", "Markdown"]
+deps = ["Markdown"]
 uuid = "b77e0a4c-d291-57a0-90e8-8db25a27a240"
 
 [[LibGit2]]
@@ -42,19 +40,10 @@ uuid = "37e2e46d-f89d-539d-b4ee-838fcccc9c8e"
 [[Logging]]
 uuid = "56ddb016-857b-54e1-b83d-db4d58db5568"
 
-[[MacroTools]]
-deps = ["Compat"]
-git-tree-sha1 = "c443e1c8d58a4e9f61b708ad0a88286c7042145b"
-uuid = "1914dd2f-81c6-5fcd-8719-6d5c9610ff09"
-version = "0.4.4"
-
 [[Markdown]]
 deps = ["Base64"]
 uuid = "d6f4376e-aef5-505a-96c1-9c027394607a"
 
-[[Mmap]]
-uuid = "a63ad114-7e13-5084-954f-fe012c677804"
-
 [[Pkg]]
 deps = ["Dates", "LibGit2", "Markdown", "Printf", "REPL", "Random", "SHA", "UUIDs"]
 uuid = "44cfe95a-1eb2-52ea-b672-e2afdf69b78f"
@@ -83,10 +72,6 @@ uuid = "ea8e919c-243c-51af-8825-aaa63cd721ce"
 [[Serialization]]
 uuid = "9e88b42a-f829-5b0c-bbe9-9e923198166b"
 
-[[SharedArrays]]
-deps = ["Distributed", "Mmap", "Random", "Serialization"]
-uuid = "1a1011a3-84de-559e-8e89-a11a2f7dc383"
-
 [[Sockets]]
 uuid = "6462fe0b-24de-5631-8697-dd941f90decc"
 
@@ -102,8 +87,14 @@ uuid = "10745b16-79ce-11e8-11f9-7d13ad32a3b2"
 deps = ["Distributed", "InteractiveUtils", "Logging", "Random"]
 uuid = "8dfed614-e22c-5e08-85e1-65c5234f0b40"
 
+[[TimerOutputs]]
+deps = ["Crayons", "Printf", "Test", "Unicode"]
+git-tree-sha1 = "b80671c06f8f8bae08c55d67b5ce292c5ae2660c"
+uuid = "a759f4b9-e2f1-59dc-863e-4aeb61b1ea8f"
+version = "0.5.0"
+
 [[UUIDs]]
-deps = ["Random"]
+deps = ["Random", "SHA"]
 uuid = "cf7118a7-6976-5b1a-9a39-7adc72f591a4"
 
 [[Unicode]]

Project.toml

@@ -1,10 +1,12 @@
 name = "NNlib"
 uuid = "872c559c-99b0-510c-b3b7-b6c96a88d5cd"
+version = "0.6.0"
 
 [deps]
 BinaryProvider = "b99e7846-7c00-51b0-8f62-c81ae34c0232"
 Libdl = "8f399da3-3557-5675-b5ff-fb832c97cbdb"
 LinearAlgebra = "37e2e46d-f89d-539d-b4ee-838fcccc9c8e"
-MacroTools = "1914dd2f-81c6-5fcd-8719-6d5c9610ff09"
 Requires = "ae029012-a4dd-5104-9daa-d747884805df"
+Statistics = "10745b16-79ce-11e8-11f9-7d13ad32a3b2"
 Test = "8dfed614-e22c-5e08-85e1-65c5234f0b40"
+TimerOutputs = "a759f4b9-e2f1-59dc-863e-4aeb61b1ea8f"

REQUIRE

@@ -1,4 +1,4 @@
-julia 0.7-
+julia 1.0
 Requires
 MacroTools
 BinaryProvider

src/NNlib.jl

@@ -1,27 +1,31 @@
 module NNlib
+using Requires, TimerOutputs
 
-using Requires, Libdl
-
-export σ, sigmoid, relu, leakyrelu, elu, gelu, swish, selu, softplus, softsign, logσ, logsigmoid,
-  softmax, logsoftmax, maxpool, meanpool
-
-include("numeric.jl")
+# Include APIs
+include("dim_helpers.jl")
 include("activation.jl")
 include("softmax.jl")
-include("logsoftmax.jl")
-include("linalg.jl")
+include("gemm.jl")
 include("conv.jl")
-include("cubroadcast.jl")
+include("pooling.jl")
 
-try
-    global ENABLE_NNPACK = parse(UInt64, ENV["ENABLE_NNPACK"])
-catch
-    global ENABLE_NNPACK = 1
-end
+## Include implementations
+include("impl/padding_edges.jl")
+
+# Direct implementations of convolutional and depthwise-convolutional algorithms
+include("impl/conv_direct.jl")
+include("impl/depthwiseconv_direct.jl")
+# im2col implementations of convolutional and depthwise-convolutional algorithms
+include("impl/conv_im2col.jl")
+include("impl/depthwiseconv_im2col.jl")
+
+# Direct implementations of pooling
+include("impl/pooling_direct.jl")
+
+to = TimerOutput()
 
-if Sys.islinux() && ENABLE_NNPACK == 1
+if Sys.islinux()
     include("nnpack/NNPACK.jl")
-    include("backends.jl")
 end
 
-end # module
+end # module NNlib

src/activation.jl

@@ -1,43 +1,40 @@
+export σ, sigmoid, relu, leakyrelu, elu, gelu, swish, selu, softplus, softsign, logσ,
+       logsigmoid
+
 """
     σ(x) = 1 / (1 + exp(-x))
 
 Classic [sigmoid](https://en.wikipedia.org/wiki/Sigmoid_function) activation
 function.
 """
 σ(x) = one(x) / (one(x) + exp(-x))
-
 const sigmoid = σ
 
 # ForwardDiff numerical stability hack
 σ_stable(x) = ifelse(x < -80, zero(x), one(x) / (one(x) + exp(-x)))
-
 σ(x::Float32) = σ_stable(x)
-
 @init @require ForwardDiff="f6369f11-7733-5829-9624-2563aa707210" begin
   σ(x::ForwardDiff.Dual{T,Float32}) where T = σ_stable(x)
 end
 
+
 """
     logσ(x)
 
 Return `log(σ(x))` which is computed in a numerically stable way.
 
-    julia> logσ(0.)
+    julia> logσ(0)
     -0.6931471805599453
-    julia> logσ.([-100, -10, 100.])
+    julia> logσ.([-100, -10, 100])
     3-element Array{Float64,1}:
-     -100.0
-      -10.0
-       -0.0
-"""
-function logσ(x)
-  max_v = max(zero(x), -x)
-  z = exp(-max_v) + exp(-x-max_v)
-  -(max_v + log(z))
-end
-
+     -100.0                  
+      -10.000045398899218    
+       -3.720075976020836e-44
+"""
+logσ(x) = -softplus(-x)
 const logsigmoid = logσ
 
+
 """
     relu(x) = max(0, x)
 
@@ -56,6 +53,7 @@ You can also specify the coefficient explicitly, e.g. `leakyrelu(x, 0.01)`.
 """
 leakyrelu(x, a = oftype(x/1, 0.01)) = max(a*x, x/1)
 
+
 """
     elu(x, α = 1) =
       x > 0 ? x : α * (exp(x) - 1)
@@ -66,6 +64,7 @@ You can also specify the coefficient explicitly, e.g. `elu(x, 1)`.
 """
 elu(x, α = one(x)) = ifelse(x ≥ 0, x/1, α * (exp(x) - one(x)))
 
+
 """
     gelu(x) = 0.5x*(1 + tanh(√(2/π)*(x + 0.044715x^3)))
 
@@ -103,6 +102,7 @@ function selu(x)
   λ * ifelse(x > 0, x/1, α * (exp(x) - 1))
 end
 
+
 """
     softsign(x) = x / (1 + |x|)
 
@@ -116,4 +116,4 @@ softsign(x) = x / (one(x) + abs(x))
 
 See [Deep Sparse Rectifier Neural Networks](http://proceedings.mlr.press/v15/glorot11a/glorot11a.pdf).
 """
-softplus(x) = log1p(exp(x))
+softplus(x) = ifelse(x > 0, x + log1p(exp(-x)), log1p(exp(x)))