Merge branch 'master' into depthwiseconv

Author: Avik Pal

src/impl/conv.jl

@@ -10,9 +10,9 @@ function psize(p, x)
   end
 end
 
-function im2col_2d!{T}(img::AbstractArray{T,3}, col::AbstractArray{T,2}, width::Int, height::Int, channels::Int,
+function im2col_2d!(img::AbstractArray{T,3}, col::AbstractArray{T,2}, width::Int, height::Int, channels::Int,
   kernel_w::Int, kernel_h::Int, pad_w::Int, pad_h::Int, stride_w::Int, stride_h::Int,
-  dil_w::Int, dil_h::Int, mode::Int)
+  dil_w::Int, dil_h::Int, mode::Int) where T
 
   height_col = div(height + 2pad_h - (kernel_h - 1) * dil_h - 1, stride_h) + 1
   width_col = div(width + 2pad_w - (kernel_w - 1) * dil_w - 1, stride_w) + 1
@@ -42,9 +42,9 @@ function im2col_2d!{T}(img::AbstractArray{T,3}, col::AbstractArray{T,2}, width::
   end
 end
 
-function col2im_2d!{T}(col::AbstractArray{T,2}, img::AbstractArray{T,3}, width::Int, height::Int,
+function col2im_2d!(col::AbstractArray{T,2}, img::AbstractArray{T,3}, width::Int, height::Int,
   channels::Int, kernel_w::Int, kernel_h::Int, pad_w::Int, pad_h::Int, stride_w::Int,
-  stride_h::Int, dil_h::Int, dil_w::Int, mode::Int)
+  stride_h::Int, dil_w::Int, dil_h::Int, mode::Int) where T
 
   height_col = div(height + 2pad_h - (kernel_h - 1) * dil_h - 1, stride_h) + 1
   width_col = div(width + 2pad_w - (kernel_w - 1) * dil_w - 1, stride_w) + 1
@@ -71,9 +71,9 @@ function col2im_2d!{T}(col::AbstractArray{T,2}, img::AbstractArray{T,3}, width::
   end
 end
 
-function im2col_3d!{T}(img::AbstractArray{T,4}, col::AbstractArray{T,2}, width::Int, height::Int, depth::Int,
+function im2col_3d!(img::AbstractArray{T,4}, col::AbstractArray{T,2}, width::Int, height::Int, depth::Int,
   channels::Int, kernel_w::Int, kernel_h::Int, kernel_d::Int, pad_w::Int, pad_h::Int, pad_d::Int,
-  stride_w::Int, stride_h::Int, stride_d::Int, dil_w::Int, dil_h::Int, dil_d::Int, mode::Int)
+  stride_w::Int, stride_h::Int, stride_d::Int, dil_w::Int, dil_h::Int, dil_d::Int, mode::Int) where T
 
   height_col = div(height + 2pad_h - (kernel_h - 1) * dil_h - 1, stride_h) + 1
   width_col = div(width + 2pad_w - (kernel_w - 1) * dil_w - 1, stride_w) + 1
@@ -107,10 +107,10 @@ function im2col_3d!{T}(img::AbstractArray{T,4}, col::AbstractArray{T,2}, width::
   end
 end
 
-function col2im_3d!{T}(col::AbstractArray{T,2}, img::AbstractArray{T,4}, width::Int, height::Int,
+function col2im_3d!(col::AbstractArray{T,2}, img::AbstractArray{T,4}, width::Int, height::Int,
   depth::Int, channels::Int, kernel_w::Int, kernel_h::Int, kernel_d::Int,
   pad_w::Int, pad_h::Int, pad_d::Int, stride_w::Int, stride_h::Int, stride_d::Int,
-  dil_w::Int, dil_h::Int, dil_d::Int, mode::Int)
+  dil_w::Int, dil_h::Int, dil_d::Int, mode::Int) where T
 
   height_col = div(height + 2pad_h - (kernel_h - 1) * dil_h - 1, stride_h) + 1
   width_col = div(width + 2pad_w - (kernel_w - 1) * dil_w - 1, stride_w) + 1
@@ -159,15 +159,14 @@ function dilation_dims(w, dilation = 1)
   end
 end
 
-function im2col_dims(w,y,dilation=1)
+function im2col_dims(w,y)
     N = ndims(y)
-    dil = dilation_dims(w, dilation)
     r,c = 1,1
     for i=1:N-2
         r *= size(y,i)
-        c *= dil[i]
+        c *= size(w,i)
     end
-    c *= dil[N-1]
+    c *= size(w,N-1)
     return (r, c)
 end
 
@@ -183,7 +182,7 @@ function im2col_dims(w::NTuple{4, Int}, y)
 end
 
 function depthwiseconv2d!(y::AbstractArray{T,4}, x::AbstractArray{T,4}, w::AbstractArray{T,4};
-                  padding = 0, stride = 1, mode = 1, alpha = T(1)) where {T<:Real}
+                  padding = 0, stride = 1, mode = 1, alpha = T(1)) where T
     Wx,Hx,Cx,Nx = size(x)
     Ww,Hw,Cm,Cw = size(w) # Cm = Channel Multiplier
     @assert Cx == Cw DimensionMismatch()
@@ -206,7 +205,7 @@ function depthwiseconv2d!(y::AbstractArray{T,4}, x::AbstractArray{T,4}, w::Abstr
 end
 
 function depthwiseconv2d_grad_w!(dw::AbstractArray{T,4}, x::AbstractArray{T,4}, w::AbstractArray{T,4}, dy::AbstractArray{T,4};
-        padding=0, stride=1, mode=0, alpha=1) where {T<:Real}
+        padding=0, stride=1, mode=0, alpha=1) where T
     Wx,Hx,Cx,Nx = size(x)
     Ww,Hw,Cm,Cw = size(w) # Cm = Channel Multiplier
     @assert Cx == Cw DimensionMismatch()
@@ -232,8 +231,8 @@ function depthwiseconv2d_grad_w!(dw::AbstractArray{T,4}, x::AbstractArray{T,4},
     return dw
 end
 
-function depthwiseconv2d_grad_x!{T}(dx::AbstractArray{T,4}, x::AbstractArray{T,4}, w::AbstractArray{T,4}, dy::AbstractArray{T,4};
-                   padding=0, stride=1, mode=0, alpha=1)
+function depthwiseconv2d_grad_x!(dx::AbstractArray{T,4}, x::AbstractArray{T,4}, w::AbstractArray{T,4}, dy::AbstractArray{T,4};
+                   padding=0, stride=1, mode=0, alpha=1) where T
     Wx,Hx,Cx,Nx = size(x)
     Ww,Hw,Cm,Cw = size(w) # Cm = Channel Multiplier
     @assert Cx == Cw DimensionMismatch()
@@ -257,14 +256,14 @@ function depthwiseconv2d_grad_x!{T}(dx::AbstractArray{T,4}, x::AbstractArray{T,4
     return dx
 end
 
-function conv2d!{T}(y::AbstractArray{T,4}, x::AbstractArray{T,4}, w::AbstractArray{T,4};
-                  padding=0, stride=1, dilation=1, mode=0, alpha=T(1))
+function conv2d!(y::AbstractArray{T,4}, x::AbstractArray{T,4}, w::AbstractArray{T,4};
+               padding=0, stride=1, dilation=1, mode=0, alpha=T(1)) where T
     if mode != 0 && mode != 1; throw(ArgumentError("conv2d only supports mode=0 or 1.")); end
     Wx,Hx,Cx,Nx = size(x)
-    Ww,Hw,C1,C2 = dilation_dims(w, dilation)
+    Ww,Hw,C1,C2 = size(w)
     if Cx!=C1; throw(DimensionMismatch()); end
     Wy,Hy,Cy,Ny = size(y)
-    x2dims = im2col_dims(w,y,dilation)
+    x2dims = im2col_dims(w,y)
     x2 = similar(x, x2dims)
     (p1,p2) = psize(padding,x)
     (s1,s2) = psize(stride,x)
@@ -279,15 +278,15 @@ function conv2d!{T}(y::AbstractArray{T,4}, x::AbstractArray{T,4}, w::AbstractArr
     return y
 end
 
-function conv2d_grad_w!{T}(dw::AbstractArray{T,4}, x::AbstractArray{T,4}, w::AbstractArray{T,4}, dy::AbstractArray{T,4};
-                   padding=0, stride=1, dilation=1, mode=0, alpha=1)
+function conv2d_grad_w!(dw::AbstractArray{T,4}, x::AbstractArray{T,4}, w::AbstractArray{T,4}, dy::AbstractArray{T,4};
+                   padding=0, stride=1, dilation=1, mode=0, alpha=1) where T
     # dw = x'*dy
     Wx,Hx,Cx,Nx = size(x)
-    Ww,Hw,C1,C2 = dilation_dims(w, dilation)
+    Ww,Hw,C1,C2 = size(w)
     Wy,Hy,Cy,Ny = size(dy)
     # if mode != 0 && mode != 1; throw(ArgumentError("conv2d only supports mode=0 or 1.")); end
     # @assert Cx==C1 && Cy==C2 && Ny==Nx
-    x2dims = im2col_dims(w,dy,dilation)
+    x2dims = im2col_dims(w,dy)
     x2 = similar(x, x2dims)
     # op(A) is an m-by-k matrix, op(B) is a k-by-n matrix, C is an m-by-n matrix.
     Y,M,N,K = Wy*Hy*Cy,Ww*Hw*Cx,Cy,Wy*Hy
@@ -304,15 +303,15 @@ function conv2d_grad_w!{T}(dw::AbstractArray{T,4}, x::AbstractArray{T,4}, w::Abs
     return dw
 end
 
-function conv2d_grad_x!{T}(dx::AbstractArray{T,4}, x::AbstractArray{T,4}, w::AbstractArray{T,4}, dy::AbstractArray{T,4};
-                   padding=0, stride=1, dilation=1, mode=0, alpha=1)
+function conv2d_grad_x!(dx::AbstractArray{T,4}, x::AbstractArray{T,4}, w::AbstractArray{T,4}, dy::AbstractArray{T,4};
+                   padding=0, stride=1, dilation=1, mode=0, alpha=1) where T
     # dx = dy*w'
     Wx,Hx,Cx,Nx = size(x)
-    Ww,Hw,C1,C2 = dilation_dims(w, dilation)
+    Ww,Hw,C1,C2 = size(w)
     Wy,Hy,Cy,Ny = size(dy)
     # if mode != 0 && mode != 1; throw(ArgumentError("conv2d only supports mode=0 or 1.")); end
     @assert Cx==C1 && Cy==C2 && Ny==Nx
-    x2dims = im2col_dims(w,dy,dilation)
+    x2dims = im2col_dims(w,dy)
     x2 = similar(x, x2dims)
     # op(A) is an m-by-k matrix, op(B) is a k-by-n matrix, C is an m-by-n matrix.
     Y,M,N,K = Wy*Hy*Cy,Wy*Hy,Ww*Hw*Cx,Cy
@@ -367,15 +366,15 @@ function col2im2d!(w::AbstractArray{T,4}, x::AbstractArray{T,4}, x2::AbstractArr
     return x
 end
 
-function conv3d!{T}(y::AbstractArray{T,5}, x::AbstractArray{T,5}, w::AbstractArray{T,5};
-                  padding=0, stride=1, dilation = 1, mode=0, alpha=T(1))
+function conv3d!(y::AbstractArray{T,5}, x::AbstractArray{T,5}, w::AbstractArray{T,5};
+               padding=0, stride=1, dilation = 1, mode=0, alpha=T(1)) where T
     if mode != 0 && mode != 1; throw(ArgumentError("conv3d only supports mode=0 or 1.")); end
     Wx,Hx,Dx,Cx,Nx = size(x)
-    Ww,Hw,Dw,C1,C2 = dilation_dims(w, dilation)
+    Ww,Hw,Dw,C1,C2 = size(w)
     if Cx!=C1; throw(DimensionMismatch()); end
     Wy,Hy,Dy,Cy,Ny = size(y)
     # @assert Cy==C2 && Ny==Nx
-    x2dims = im2col_dims(w,y,dilation)
+    x2dims = im2col_dims(w,y)
     x2 = similar(x, x2dims)
     (p1,p2,p3) = psize(padding,x)
     (s1,s2,s3) = psize(stride,x)
@@ -391,15 +390,15 @@ function conv3d!{T}(y::AbstractArray{T,5}, x::AbstractArray{T,5}, w::AbstractArr
     return y
 end
 
-function conv3d_grad_w!{T}(dw::AbstractArray{T,5}, x::AbstractArray{T,5}, w::AbstractArray{T,5}, dy::AbstractArray{T,5};
-                   padding=0, stride=1, dilation = 1, mode=0, alpha=1)
+function conv3d_grad_w!(dw::AbstractArray{T,5}, x::AbstractArray{T,5}, w::AbstractArray{T,5}, dy::AbstractArray{T,5};
+                   padding=0, stride=1, dilation = 1, mode=0, alpha=1) where T
     # dw = x'*dy
     Wx,Hx,Dx,Cx,Nx = size(x)
-    Ww,Hw,Dw,C1,C2 = dilation_dims(w, dilation)
+    Ww,Hw,Dw,C1,C2 = size(w)
     Wy,Hy,Dy,Cy,Ny = size(dy)
     # if mode != 0 && mode != 1; throw(ArgumentError("conv2d only supports mode=0 or 1.")); end
     # @assert Cx==C1 && Cy==C2 && Ny==Nx
-    x2dims = im2col_dims(w,dy,dilation)
+    x2dims = im2col_dims(w,dy)
     x2 = similar(x, x2dims)
     # op(A) is an m-by-k matrix, op(B) is a k-by-n matrix, C is an m-by-n matrix.
     Y,M,N,K = Wy*Hy*Dy*Cy,Ww*Hw*Dw*Cx,Cy,Wy*Hy*Dy
@@ -416,15 +415,15 @@ function conv3d_grad_w!{T}(dw::AbstractArray{T,5}, x::AbstractArray{T,5}, w::Abs
     return dw
 end
 
-function conv3d_grad_x!{T}(dx::AbstractArray{T,5}, x::AbstractArray{T,5}, w::AbstractArray{T,5}, dy::AbstractArray{T,5};
-                   padding=0, stride=1, dilation = 1, mode=0, alpha=1)
+function conv3d_grad_x!(dx::AbstractArray{T,5}, x::AbstractArray{T,5}, w::AbstractArray{T,5}, dy::AbstractArray{T,5};
+                   padding=0, stride=1, dilation = 1, mode=0, alpha=1) where T
     # dx = dy*w'
     Wx,Hx,Dx,Cx,Nx = size(x)
-    Ww,Hw,Dw,C1,C2 = dilation_dims(w, dilation)
+    Ww,Hw,Dw,C1,C2 = size(w)
     Wy,Hy,Dy,Cy,Ny = size(dy)
     # if mode != 0 && mode != 1; throw(ArgumentError("conv2d only supports mode=0 or 1.")); end
     @assert Cx==C1 && Cy==C2 && Ny==Nx
-    x2dims = im2col_dims(w,dy,dilation)
+    x2dims = im2col_dims(w,dy)
     x2 = similar(x, x2dims)
     # op(A) is an m-by-k matrix, op(B) is a k-by-n matrix, C is an m-by-n matrix.
     Y,M,N,K = Wy*Hy*Dy*Cy,Wy*Hy*Dy,Ww*Hw*Dw*Cx,Cy

src/impl/pool.jl

@@ -1,7 +1,7 @@
-function max_pooling2d_fwd!{T}(x::AbstractArray{T,4}, y::AbstractArray{T,4},
-                               width::Int, height::Int, channels::Int, num::Int, pooled_width::Int,
-                               pooled_height::Int, kernel_w::Int, kernel_h::Int, pad_w::Int, pad_h::Int,
-                               stride_w::Int, stride_h::Int)
+function max_pooling2d_fwd!(x::AbstractArray{T,4}, y::AbstractArray{T,4},
+                            width::Int, height::Int, channels::Int, num::Int, pooled_width::Int,
+                            pooled_height::Int, kernel_w::Int, kernel_h::Int, pad_w::Int, pad_h::Int,
+                            stride_w::Int, stride_h::Int) where T
   for n = 1:num, c = 1:channels, ph = 1:pooled_height, pw = 1:pooled_width
     hstart = (ph - 1)*stride_h - pad_h
     wstart = (pw - 1)*stride_w - pad_w
@@ -27,10 +27,10 @@ function maxpool2d!(y::AbstractArray{T,4}, x::AbstractArray{T,4};
     return y
 end
 
-function max_pooling2d_bwd!{T}(x::AbstractArray{T,4}, y::AbstractArray{T,4},
+function max_pooling2d_bwd!(x::AbstractArray{T,4}, y::AbstractArray{T,4},
   grad_output::AbstractArray{T,4}, grad_input::AbstractArray{T,4}, width::Int, height::Int,
   channels::Int, num::Int, pooled_width::Int, pooled_height::Int, kernel_w::Int,
-  kernel_h::Int, pad_w::Int, pad_h::Int, stride_w::Int, stride_h::Int)
+  kernel_h::Int, pad_w::Int, pad_h::Int, stride_w::Int, stride_h::Int) where T
 
   grad_input[:, :, :, :] = 0
   #pragma omp parallel for
@@ -64,10 +64,10 @@ function maxpool2d_grad!(dx::AbstractArray{T,4}, dy::AbstractArray{T,4}, y::Abst
 end
 
 
-function mean_pooling2d_fwd!{T}(x::AbstractArray{T,4}, y::AbstractArray{T,4},
+function mean_pooling2d_fwd!(x::AbstractArray{T,4}, y::AbstractArray{T,4},
   width::Int, height::Int, channels::Int, num::Int, pooled_width::Int,
   pooled_height::Int, kernel_w::Int, kernel_h::Int,pad_w::Int, pad_h::Int,
-  stride_w::Int, stride_h::Int)
+  stride_w::Int, stride_h::Int) where T
   kernel_size = kernel_w * kernel_h
   for n = 1:num, c = 1:channels, ph = 1:pooled_height, pw = 1:pooled_width
     hstart = (ph - 1) * stride_h - pad_h
@@ -94,10 +94,10 @@ function meanpool2d!(y::AbstractArray{T,4}, x::AbstractArray{T,4};
     return y
 end
 
-function mean_pooling2d_bwd!{T}(x::AbstractArray{T,4}, y::AbstractArray{T,4},
+function mean_pooling2d_bwd!(x::AbstractArray{T,4}, y::AbstractArray{T,4},
   width::Int, height::Int, channels::Int, num::Int, pooled_width::Int,
   pooled_height::Int, kernel_w::Int, kernel_h::Int, pad_w::Int, pad_h::Int,
-  stride_w::Int, stride_h::Int)
+  stride_w::Int, stride_h::Int) where T
 
   x[:, :, :, :] = 0
   kernel_size = kernel_w * kernel_h
@@ -128,10 +128,10 @@ function meanpool2d_grad!(dx::AbstractArray{T,4}, dy::AbstractArray{T,4}, y::Abs
     return dx
 end
 
-function max_pooling3d_fwd!{T}(x::AbstractArray{T,5}, y::AbstractArray{T,5},
+function max_pooling3d_fwd!(x::AbstractArray{T,5}, y::AbstractArray{T,5},
   width::Int, height::Int, depth::Int, channels::Int, num::Int, pooled_width::Int,
   pooled_height::Int, pooled_depth::Int, kernel_w::Int, kernel_h::Int, kernel_d::Int,
-  pad_w::Int, pad_h::Int, pad_d::Int, stride_w::Int, stride_h::Int, stride_d::Int)
+  pad_w::Int, pad_h::Int, pad_d::Int, stride_w::Int, stride_h::Int, stride_d::Int) where T
   for n = 1:num, c = 1:channels, pd = 1:pooled_depth, ph = 1:pooled_height, pw = 1:pooled_width
     dstart = (pd - 1)* stride_d - pad_d
     hstart = (ph - 1)* stride_h - pad_h
@@ -162,11 +162,11 @@ function maxpool3d!(y::AbstractArray{T,5}, x::AbstractArray{T,5};
     return y
 end
 
-function max_pooling3d_bwd!{T}(x::AbstractArray{T,5}, y::AbstractArray{T,5},
+function max_pooling3d_bwd!(x::AbstractArray{T,5}, y::AbstractArray{T,5},
   grad_output::AbstractArray{T,5}, grad_input::AbstractArray{T,5}, width::Int, height::Int, depth::Int,
   channels::Int, num::Int, pooled_width::Int, pooled_height::Int, pooled_depth::Int,
   kernel_w::Int, kernel_h::Int, kernel_d::Int, pad_w::Int, pad_h::Int, pad_d::Int,
-  stride_w::Int, stride_h::Int, stride_d::Int)
+  stride_w::Int, stride_h::Int, stride_d::Int) where T
 
   grad_input[:, :, :, :, :] = 0
 
@@ -206,10 +206,10 @@ function maxpool3d_grad!(dx::AbstractArray{T,5}, dy::AbstractArray{T,5}, y::Abst
     return dx
 end
 
-function mean_pooling3d_fwd!{T}(x::AbstractArray{T,5}, y::AbstractArray{T,5},
+function mean_pooling3d_fwd!(x::AbstractArray{T,5}, y::AbstractArray{T,5},
   width::Int, height::Int, depth::Int, channels::Int, num::Int, pooled_width::Int,
   pooled_height::Int, pooled_depth::Int, kernel_w::Int, kernel_h::Int, kernel_d::Int,
-  pad_w::Int, pad_h::Int, pad_d::Int, stride_w::Int, stride_h::Int, stride_d::Int)
+  pad_w::Int, pad_h::Int, pad_d::Int, stride_w::Int, stride_h::Int, stride_d::Int) where T
 
   kernel_size = kernel_w * kernel_h * kernel_d
   #pragma omp parallel for
@@ -243,10 +243,10 @@ function meanpool3d!(y::AbstractArray{T,5}, x::AbstractArray{T,5};
     return y
 end
 
-function mean_pooling3d_bwd!{T}(grad_input::AbstractArray{T,5}, grad_output::AbstractArray{T,5},
+function mean_pooling3d_bwd!(grad_input::AbstractArray{T,5}, grad_output::AbstractArray{T,5},
   width::Int, height::Int, depth::Int, channels::Int, num::Int, pooled_width::Int,
   pooled_height::Int, pooled_depth::Int, kernel_w::Int, kernel_h::Int, kernel_d::Int,
-  pad_w::Int, pad_h::Int, pad_d::Int, stride_w::Int, stride_h::Int, stride_d::Int)
+  pad_w::Int, pad_h::Int, pad_d::Int, stride_w::Int, stride_h::Int, stride_d::Int) where T
 
   kernel_size = kernel_w * kernel_h * kernel_d
   fill!(grad_input, 0.0)

src/softmax.jl

@@ -1,6 +1,6 @@
 using Base.Threads
 
-function softmax!(out::AbstractVecOrMat, xs::AbstractVecOrMat)
+function softmax!(out::AbstractVecOrMat{T}, xs::AbstractVecOrMat{T}) where T<:AbstractFloat
   @threads for j = 1:size(xs, 2)
     @inbounds begin
       # out[end, :] .= maximum(xs, 1)