Merge branch 'master' into conv

Author: MikeInnes

appveyor.yml

@@ -1,16 +1,17 @@
 environment:
   matrix:
-  - JULIA_URL: "https://julialang-s3.julialang.org/bin/winnt/x86/0.6/julia-0.6-latest-win32.exe"
-  - JULIA_URL: "https://julialang-s3.julialang.org/bin/winnt/x64/0.6/julia-0.6-latest-win64.exe"
-  # - JULIA_URL: "https://julialangnightlies-s3.julialang.org/bin/winnt/x86/julia-latest-win32.exe"
-  # - JULIA_URL: "https://julialangnightlies-s3.julialang.org/bin/winnt/x64/julia-latest-win64.exe"
-
-## uncomment the following lines to allow failures on nightly julia
-## (tests will run but not make your overall status red)
-#matrix:
-#  allow_failures:
-#  - JULIA_URL: "https://julialangnightlies-s3.julialang.org/bin/winnt/x86/julia-latest-win32.exe"
-#  - JULIA_URL: "https://julialangnightlies-s3.julialang.org/bin/winnt/x64/julia-latest-win64.exe"
+  - julia_version: 1
+  - julia_version: nightly
+
+platform:
+  - x86 # 32-bit
+  - x64 # 64-bit
+
+# # Uncomment the following lines to allow failures on nightly julia
+# # (tests will run but not make your overall status red)
+# matrix:
+#   allow_failures:
+#   - julia_version: nightly
 
 branches:
   only:
@@ -24,24 +25,18 @@ notifications:
     on_build_status_changed: false
 
 install:
-  - ps: "[System.Net.ServicePointManager]::SecurityProtocol = [System.Net.SecurityProtocolType]::Tls12"
-# If there's a newer build queued for the same PR, cancel this one
-  - ps: if ($env:APPVEYOR_PULL_REQUEST_NUMBER -and $env:APPVEYOR_BUILD_NUMBER -ne ((Invoke-RestMethod `
-        https://ci.appveyor.com/api/projects/$env:APPVEYOR_ACCOUNT_NAME/$env:APPVEYOR_PROJECT_SLUG/history?recordsNumber=50).builds | `
-        Where-Object pullRequestId -eq $env:APPVEYOR_PULL_REQUEST_NUMBER)[0].buildNumber) { `
-        throw "There are newer queued builds for this pull request, failing early." }
-# Download most recent Julia Windows binary
-  - ps: (new-object net.webclient).DownloadFile(
-        $env:JULIA_URL,
-        "C:\projects\julia-binary.exe")
-# Run installer silently, output to C:\projects\julia
-  - C:\projects\julia-binary.exe /S /D=C:\projects\julia
+  - ps: iex ((new-object net.webclient).DownloadString("https://raw.githubusercontent.com/JuliaCI/Appveyor.jl/version-1/bin/install.ps1"))
 
 build_script:
-# Need to convert from shallow to complete for Pkg.clone to work
-  - IF EXIST .git\shallow (git fetch --unshallow)
-  - C:\projects\julia\bin\julia -e "versioninfo();
-      Pkg.clone(pwd(), \"NNlib\"); Pkg.build(\"NNlib\")"
+  - echo "%JL_BUILD_SCRIPT%"
+  - C:\julia\bin\julia -e "%JL_BUILD_SCRIPT%"
 
 test_script:
-  - C:\projects\julia\bin\julia -e "Pkg.test(\"NNlib\")"
+  - echo "%JL_TEST_SCRIPT%"
+  - C:\julia\bin\julia -e "%JL_TEST_SCRIPT%"
+
+# # Uncomment to support code coverage upload. Should only be enabled for packages
+# # which would have coverage gaps without running on Windows
+# on_success:
+#   - echo "%JL_CODECOV_SCRIPT%"
+#   - C:\julia\bin\julia -e "%JL_CODECOV_SCRIPT%"

src/conv.jl

@@ -30,62 +30,73 @@ function conv(x::A, w::B; pad = 0, stride = 1, dilation = 1) where {A<:AbstractA
         x, w, pad = pad_, stride = stride_, dilation = dilation)
 end
 
-∇conv_data(dy::A, x::B, w::C; pad = 0, stride = 1, dilation = 1) where {A<:AbstractArray, B<:AbstractArray, C<:AbstractArray} =
-  ∇conv_data!(zero(x), dy, x, w; pad = pad, stride = stride, dilation = dilation)
+function crosscor(x::A, w::A; pad = 0, stride = 1, dilation = 1) where A<:AbstractArray
+  pad_, stride_ = padtuple(x, pad), padtuple(x, stride)
+  crosscor!(similar(x, cdims(size(x), dilation_dims(w, dilation), pad_, stride_)),
+        x, w, pad = pad_, stride = stride_, dilation = dilation)
+end
 
-∇conv_filter(dy::A, x::B, w::C; pad = 0, stride = 1, dilation = 1) where {A<:AbstractArray, B<:AbstractArray, C<:AbstractArray} =
-  ∇conv_filter!(zero(w), dy, x, w; pad = pad, stride = stride, dilation = dilation)
+∇conv_data(dy::A, x::B, w::C; pad = 0, stride = 1, dilation = 1, flipkernel = 0) where {A<:AbstractArray, B<:AbstractArray, C<:AbstractArray} =
+  ∇conv_data!(zero(x), dy, x, w; pad = pad, stride = stride, dilation = dilation, flipkernel=flipkernel)
+
+∇conv_filter(dy::A, x::B, w::C; pad = 0, stride = 1, dilation = 1, flipkernel=0) where {A<:AbstractArray, B<:AbstractArray, C<:AbstractArray} =
+  ∇conv_filter!(zero(w), dy, x, w; pad = pad, stride = stride, dilation = dilation, flipkernel=flipkernel)
 
 # N-D dispatch
 
 function conv!(y::AbstractArray{T,3}, x::AbstractArray{T,3}, w::AbstractArray{T,3};
-               pad = 0, stride = 1, dilation = 1) where T
+               pad = 0, stride = 1, dilation = 1, flipkernel =0) where T
     args = map(x -> reshape(x, size(x,1),1,size(x,2),size(x,3)), (y, x, w))
-    conv!(args..., pad = (pad...,0), stride = (stride...,1), dilation = (dilation...,1))
+    conv!(args..., pad = (pad...,0), stride = (stride...,1), dilation = (dilation...,1), flipkernel=flipkernel)
     return y
 end
 
+function crosscor!(y::AbstractArray, x::AbstractArray, w::AbstractArray;
+               pad = 0, stride = 1, dilation = 1)
+    conv!(y, x, w, pad=pad, stride=stride, dilation=dilation, flipkernel=1)
+end
+
 function ∇conv_filter!(dw::AbstractArray{T,3}, dy::AbstractArray{T,3},
                        x::AbstractArray{T,3}, w::AbstractArray{T,3};
-                       pad = 0, stride = 1, dilation = 1) where T
+                       pad = 0, stride = 1, dilation = 1, flipkernel=0) where T
     args = map(x -> reshape(x, size(x,1),1,size(x,2),size(x,3)), (dw, dy, x, w))
-    ∇conv_filter!(args..., pad = (pad...,0), stride = (stride...,1), dilation = (dilation...,1))
+    ∇conv_filter!(args..., pad = (pad...,0), stride = (stride...,1), dilation = (dilation...,1), flipkernel=flipkernel)
     return dw
 end
 
 function ∇conv_data!(dx::AbstractArray{T,3}, dy::AbstractArray{T,3},
                      x::AbstractArray{T,3}, w::AbstractArray{T,3};
-                     pad = 0, stride = 1, dilation = 1) where T
+                     pad = 0, stride = 1, dilation = 1, flipkernel = 0) where T
     args = map(x -> reshape(x, size(x,1),1,size(x,2),size(x,3)), (dx, dy, x, w))
-    ∇conv_data!(args..., pad = (pad...,0), stride = (stride...,1), dilation = (dilation..., 1))
+    ∇conv_data!(args..., pad = (pad...,0), stride = (stride...,1), dilation = (dilation..., 1), flipkernel = flipkernel)
     return dx
 end
 
 conv!(y::AbstractArray{T,4}, x::AbstractArray{T,4}, w::AbstractArray{T,4};
-      pad = 0, stride = 1, dilation = 1) where T =
-  conv2d!(y, x, w, padding = pad, stride = stride, dilation = dilation)
+      pad = 0, stride = 1, dilation = 1, flipkernel=0) where T =
+  conv2d!(y, x, w, padding = pad, stride = stride, dilation = dilation, mode=flipkernel)
 
 ∇conv_filter!(dw::AbstractArray{T,4}, dy::AbstractArray{T,4}, x::AbstractArray{T,4}, w::AbstractArray{T,4};
-              pad = 0, stride = 1, dilation = 1) where T =
-  conv2d_grad_w!(dw, x, w, dy, padding = pad, stride = stride, dilation = dilation)
+              pad = 0, stride = 1, dilation = 1, flipkernel=0) where T =
+  conv2d_grad_w!(dw, x, w, dy, padding = pad, stride = stride, dilation = dilation, mode=flipkernel)
 
 ∇conv_data!(dx::AbstractArray{T,4}, dy::AbstractArray{T,4}, x::AbstractArray{T,4}, w::AbstractArray{T,4};
-            pad = 0, stride = 1, dilation = 1) where T =
-  conv2d_grad_x!(dx, x, w, dy, padding = pad, stride = stride, dilation = dilation)
+            pad = 0, stride = 1, dilation = 1, flipkernel=0) where T =
+  conv2d_grad_x!(dx, x, w, dy, padding = pad, stride = stride, dilation = dilation, mode=flipkernel)
 
 conv!(y::AbstractArray{T,5}, x::AbstractArray{T,5}, w::AbstractArray{T,5};
-      pad = 0, stride = 1, dilation = 1) where T =
-  conv3d!(y, x, w, padding = pad, stride = stride, dilation = dilation)
+      pad = 0, stride = 1, dilation = 1, flipkernel=0) where T =
+  conv3d!(y, x, w, padding = pad, stride = stride, dilation = dilation, mode=flipkernel)
 
 ∇conv_filter!(dw::AbstractArray{T,5}, dy::AbstractArray{T,5}, x::AbstractArray{T,5}, w::AbstractArray{T,5};
-              pad = 0, stride = 1, dilation = 1) where T =
-  conv3d_grad_w!(dw, x, w, dy, padding = pad, stride = stride, dilation = dilation)
+              pad = 0, stride = 1, dilation = 1, flipkernel=0) where T =
+  conv3d_grad_w!(dw, x, w, dy, padding = pad, stride = stride, dilation = dilation, mode=flipkernel)
 
 ∇conv_data!(dx::AbstractArray{T,5}, dy::AbstractArray{T,5}, x::AbstractArray{T,5}, w::AbstractArray{T,5};
-            pad = 0, stride = 1, dilation = 1) where T =
-  conv3d_grad_x!(dx, x, w, dy, padding = pad, stride = stride, dilation = dilation)
+            pad = 0, stride = 1, dilation = 1, flipkernel=0) where T =
+  conv3d_grad_x!(dx, x, w, dy, padding = pad, stride = stride, dilation = dilation, mode=flipkernel)
 
-# Depthwise Conv
+  # Depthwise Conv
 
 function dcdims(x::NTuple{4,Int}, w::NTuple{4,Int}, pad, stride)
   ((x[1] + 2 * pad[1] - w[1])÷stride[1] + 1,(x[2] + 2 * pad[2] - w[2])÷stride[2] + 1,w[3]*w[4],x[4])
@@ -96,23 +107,32 @@ function depthwiseconv(x::A, w::B; pad = 0, stride = 1) where {A<:AbstractArray,
   depthwiseconv!(similar(x, dcdims(size(x), size(w), pad_, stride_)), x, w, pad = pad_, stride = stride_)
 end
 
+function depthwisecrosscor(x::A, w::A; pad = 0, stride = 1) where A<:AbstractArray
+  pad_, stride_ = padtuple(x, pad), padtuple(x, stride)
+  depthwisecrosscor!(similar(x, dcdims(size(x), size(w), pad_, stride_)), x, w, pad = pad_, stride = stride_)
+end
+
 depthwiseconv!(y::AbstractArray{T,4}, x::AbstractArray{T,4}, w::AbstractArray{T,4};
+      pad = 0, stride = 1, flipkernel=0) where T =
+  depthwiseconv2d!(y, x, w, padding = pad, stride = stride, mode= flipkernel)
+
+depthwisecrosscor!(y::AbstractArray{T,4}, x::AbstractArray{T,4}, w::AbstractArray{T,4};
       pad = 0, stride = 1) where T =
-  depthwiseconv2d!(y, x, w, padding = pad, stride = stride)
+  depthwiseconv!(y, x, w, pad = pad, stride = stride, flipkernel=1)
 
-∇depthwiseconv_data(dy::A, x::B, w::C; pad = 0, stride = 1) where {A<:AbstractArray, B<:AbstractArray, C<:AbstractArray} =
-  ∇depthwiseconv_data!(zero(x), dy, x, w; pad = pad, stride = stride)
+∇depthwiseconv_data(dy::A, x::B, w::C; pad = 0, stride = 1, flipkernel=0) where {A<:AbstractArray, B<:AbstractArray, C<:AbstractArray} =
+  ∇depthwiseconv_data!(zero(x), dy, x, w; pad = pad, stride = stride, flipkernel=flipkernel)
 
-∇depthwiseconv_filter(dy::A, x::B, w::C; pad = 0, stride = 1) where {A<:AbstractArray, B<:AbstractArray, C<:AbstractArray} =
-  ∇depthwiseconv_filter!(zero(w), dy, x, w; pad = pad, stride = stride)
+∇depthwiseconv_filter(dy::A, x::B, w::C; pad = 0, stride = 1, flipkernel=0) where {A<:AbstractArray, B<:AbstractArray, C<:AbstractArray} =
+  ∇depthwiseconv_filter!(zero(w), dy, x, w; pad = pad, stride = stride, flipkernel=flipkernel)
 
 ∇depthwiseconv_filter!(dw::AbstractArray{T,4}, dy::AbstractArray{T,4}, x::AbstractArray{T,4}, w::AbstractArray{T,4};
-              pad = 0, stride = 1) where T =
-  depthwiseconv2d_grad_w!(dw, x, w, dy, padding = pad, stride = stride)
+              pad = 0, stride = 1, flipkernel=0) where T =
+  depthwiseconv2d_grad_w!(dw, x, w, dy, padding = pad, stride = stride, mode=flipkernel)
 
 ∇depthwiseconv_data!(dx::AbstractArray{T,4}, dy::AbstractArray{T,4}, x::AbstractArray{T,4}, w::AbstractArray{T,4};
-            pad = 0, stride = 1) where T =
-  depthwiseconv2d_grad_x!(dx, x, w, dy, padding = pad, stride = stride)
+            pad = 0, stride = 1, flipkernel=0) where T =
+  depthwiseconv2d_grad_x!(dx, x, w, dy, padding = pad, stride = stride, mode=flipkernel)
 
 # Pooling
 

src/impl/conv.jl

@@ -2,9 +2,9 @@
 function psize(p, x)
   nd = ndims(x)-2
   if isa(p,Number)
-    fill(Int(p),nd)
+    ntuple(_->Int(p), nd)
   elseif length(p)==nd
-    collect(Int,p)
+    tuple(p...)
   else
     throw(DimensionMismatch("psize: $p $nd"))
   end

src/linalg.jl

@@ -18,10 +18,10 @@ for (gemm, elty) in ((:dgemm_,:Float64), (:sgemm_,:Float32))
             if transA=='N'; lda=M; else; lda=K; end
             if transB=='N'; ldb=K; else; ldb=N; end
             ldc = M;
-            ccall((@blasfunc(dgemm_), libblas), Nothing,
+            ccall((@blasfunc($(gemm)), libblas), Nothing,
                   (Ref{UInt8}, Ref{UInt8}, Ref{BlasInt}, Ref{BlasInt},
-                   Ref{BlasInt}, Ref{Float64}, Ptr{Float64}, Ref{BlasInt},
-                   Ptr{Float64}, Ref{BlasInt}, Ref{Float64}, Ptr{Float64},
+                   Ref{BlasInt}, Ref{$elty}, Ptr{$elty}, Ref{BlasInt},
+                   Ptr{$elty}, Ref{BlasInt}, Ref{$elty}, Ptr{$elty},
                    Ref{BlasInt}),
                   transA, transB, M, N, K,
                   alpha, A, lda, B, ldb, beta, C, ldc)

test/conv.jl

@@ -16,10 +16,20 @@ using NNlib: conv, ∇conv_filter, ∇conv_data, ∇maxpool, maxpool, depthwisec
         49 99 149;
         59 109 159.]
 
+    @test dropdims(conv(Float32.(x), Float32.(w)), dims=(3,4)) == Float32.([
+        29 79 129;
+        39 89 139;
+        49 99 149;
+        59 109 159.])
+
     @test dropdims(conv(x, w; stride=2), dims = (3,4)) == [
         29 129;
         49 149.]
 
+    @test dropdims(conv(Float32.(x), Float32.(w); stride=2), dims = (3,4)) == Float32.([
+        29 129;
+        49 149.])    
+
     @test dropdims(conv(x, w; pad=1), dims = (3,4)) == [
         1.0   9.0   29.0   49.0   48.0;
         4.0  29.0   79.0  129.0  115.0;
@@ -29,6 +39,15 @@ using NNlib: conv, ∇conv_filter, ∇conv_data, ∇maxpool, maxpool, depthwisec
         10.0  40.0   70.0  100.0   80.0
     ]
 
+    @test dropdims(conv(Float32.(x), Float32.(w); pad=1), dims = (3,4)) == Float32.([
+        1.0   9.0   29.0   49.0   48.0;
+        4.0  29.0   79.0  129.0  115.0;
+        7.0  39.0   89.0  139.0  122.0;
+        10.0  49.0   99.0  149.0  129.0;
+        13.0  59.0  109.0  159.0  136.0;
+        10.0  40.0   70.0  100.0   80.0
+    ])
+
     @test dropdims(conv(x, w; dilation=2), dims = (3,4)) == [
         48 98;
         58 108;
@@ -157,10 +176,16 @@ end
         1150.0  1330.0  1510.0]
     @test dropdims(conv(x, w), dims = (4,5)) == res
 
+    @test dropdims(conv(Float32.(x), Float32.(w)), dims = (4,5)) == Float32.(res)
+
     @test dropdims(conv(x, w; stride=2), dims = (3,4,5)) == [
         322.0 682.0;
         394.0 754.0]
 
+    @test dropdims(conv(Float32.(x), Float32.(w); stride=2), dims = (3,4,5)) == Float32.([
+        322.0 682.0;
+        394.0 754.0])
+
     res = zeros(6,5,4)
     res[:, :, 1] = [
         1.0   9.0   29.0   49.0   48.0;
@@ -192,12 +217,19 @@ end
         270.0   660.0   730.0   800.0  480.0]
     @test dropdims(conv(x, w; pad=1), dims = (4,5)) == res
 
+    @test dropdims(conv(Float32.(x), Float32.(w); pad=1), dims = (4,5)) == Float32.(res)
+
     @test dropdims(conv(x, w; dilation=2), dims = (3,4,5)) == [
         608 788;
         644 824;
         680 860.
     ]
 
+    @test dropdims(conv(Float32.(x), Float32.(w); dilation=2), dims = (3,4,5)) == Float32.([
+        608 788;
+        644 824;
+        680 860.
+    ])
 	# NaN tests for dilation forward pass
 
 	ys = []