Merge branch 'master' of https://github.com/chriselrod/LoopVectorization.jl

Author: chriselrod

Project.toml

@@ -15,7 +15,7 @@ VectorizationBase = "3d5dd08c-fd9d-11e8-17fa-ed2836048c2f"
 [compat]
 DocStringExtensions = "0.8"
 OffsetArrays = "1"
-SIMDPirates = "0.8.4"
+SIMDPirates = "0.8.6"
 SLEEFPirates = "0.5"
 UnPack = "0,1"
 VectorizationBase = "0.12.6"

src/determinestrategy.jl

@@ -186,7 +186,7 @@ function unroll_no_reductions(ls, order, unrolled, vectorized, Wshift, size_T)
     # @show compute_rt, load_rt
     # roundpow2(min(4, round(Int, (compute_rt + load_rt + 1) / compute_rt)))
     rt = max(compute_rt, load_rt)
-    rt == 0.0 && return 4
+    iszero(rt) && return 4
     max(1, roundpow2( min( 4, round(Int, 16 / rt) ) ))
 end
 function determine_unroll_factor(
@@ -286,9 +286,10 @@ function solve_unroll(X, R, u₁L, u₂L, u₁step, u₂step)
     discriminant < 0 && return -1,-1,Inf
     u₁float = max(float(u₁step), (sqrt(discriminant) + b) / (-2a)) # must be at least 1
     u₂float = (RR - u₁float*R₂)/(u₁float*R₁)
-    if !(isfinite(u₂float) && isfinite(u₁float))
-        return 4, 4, unroll_cost(X, 4, 4, u₁L, u₂L)
-        # return itertilesize(X, u₁L, u₂L)
+    if !(isfinite(u₂float) & isfinite(u₁float)) # brute force
+        u₁low = u₂low = 1
+        u₁high = u₂high = REGISTER_COUNT == 32 ? 10 : 6#8
+        return solve_unroll_iter(X, R, u₁L, u₂L, u₁low:u₁step:u₁high, u₂low:u₂step:u₂high)
     end
     u₁low = floor(Int, u₁float)
     u₂low = max(u₂step, floor(Int, u₂float)) # must be at least 1
@@ -564,6 +565,13 @@ function load_elimination_cost_factor!(
         false
     end
 end
+function loadintostore(ls::LoopSet, op::Operation)
+    # isload(op) || return false # leads to bad behavior more than it helps
+    # for opp ∈ operations(ls)
+    #     isstore(opp) && opp.ref == op.ref && return true
+    # end
+    false
+end
 function add_constant_offset_load_elmination_cost!(
     X, R, choose_to_inline, ls::LoopSet, op::Operation, iters, unrollsyms::UnrollSymbols, u₁reduces::Bool, u₂reduces::Bool, Wshift::Int, size_T::Int, opisininnerloop::Bool
 )
@@ -575,6 +583,9 @@ function add_constant_offset_load_elmination_cost!(
         rt, lat, rp = cost(ls, op, vectorized, Wshift, size_T)
         rt *= iters
         rp = opisininnerloop ? rp : zero(rp)
+        # if loadintostore(ls, op) # For now, let's just avoid unrolling in this way...
+        #     rt = Inf
+        # end
         # u_uid is getting eliminated
         # we treat this as the unrolled loop getting eliminated is split into 2 parts:
         # 1 a non-cost-reduced part, with factor udependent_reduction
@@ -700,7 +711,8 @@ function evaluate_cost_tile(
             prefetch_good_idea = true
         end
         # @show isunrolled₁, isunrolled₂, op rt, lat, rp
-        rp = opisininnerloop ? rp : zero(rp) # we only care about register pressure within the inner most loop
+        rp = (opisininnerloop && !(loadintostore(ls, op))) ? rp : zero(rp) # we only care about register pressure within the inner most loop
+        # rp = opisininnerloop ? rp : zero(rp) # we only care about register pressure within the inner most loop
         rt *= iters[id]
         if u₁reduces & u₂reduces
             cost_vec[4] += rt

src/mapreduce.jl

@@ -79,3 +79,28 @@ Vectorized version of `reduce`. Reduces the array `A` using the operator `op`.
 """
 @inline vreduce(op, arg) = vmapreduce(identity, op, arg)
 
+for (op, init) in zip((:+, :max, :min), (:zero, :identity, :identity))
+    @eval function vreduce(::typeof($op), arg; dims = nothing)
+        isnothing(dims) && return _vreduce($op, arg)
+        @assert length(dims) == 1
+        out = $init(arg[ntuple(d -> d == dims ? (1:1) : (1:size(arg, d)), ndims(arg))...])
+        Rpre = CartesianIndices(axes(arg)[1:dims-1])
+        Rpost = CartesianIndices(axes(arg)[dims+1:end])
+        _vreduce_dims!(out, $op, Rpre, 1:size(arg, dims), Rpost, arg)
+    end
+
+    @eval function _vreduce_dims!(out, ::typeof($op), Rpre, is, Rpost, arg)
+        @avx for Ipost in Rpost, i in is, Ipre in Rpre
+            out[Ipre, 1, Ipost] = $op(out[Ipre, 1, Ipost], arg[Ipre, i, Ipost])
+        end
+        return out
+    end
+
+    @eval function _vreduce(::typeof($op), arg)
+        s = $init(arg[1])
+        @avx for i in 1:length(arg)
+            s = $op(s, arg[i])
+        end
+        return s
+    end
+end

test/mapreduce.jl

@@ -13,10 +13,10 @@
         if T <: Integer
             R = T(1):T(100)
             x7 = rand(R, 7); y7 = rand(R, 7);
-            x = rand(R, 127); y = rand(R, 127);
+            x = rand(R, 127, 7, 7); y = rand(R, 127, 7, 7);
         else
             x7 = rand(T, 7); y7 = rand(T, 7);
-            x = rand(T, 127); y = rand(T, 127);
+            x = rand(T, 127, 7, 7); y = rand(T, 127, 7, 7);
             if VERSION ≥ v"1.4"
                 @test vmapreduce(hypot, +, x, y) ≈ mapreduce(hypot, +, x, y)
                 @test vmapreduce(^, (a,b) -> a + b, x7, y7) ≈ mapreduce(^, +, x7, y7)
@@ -38,6 +38,12 @@
         @test vmapreduce(log, +, x) ≈ sum(log, x)
         @test vmapreduce(abs2, +, x) ≈ sum(abs2, x)
         @test maximum(x) == vreduce(max, x) == maximum_avx(x)
+
+        for d in 1:ndims(x)
+            @test vreduce(max, x; dims = d) ≈ maximum(x; dims = d)
+            @test vreduce(min, x; dims = d) ≈ minimum(x; dims = d)
+            @test vreduce(+, x; dims = d) ≈ sum(x; dims = d)
+        end
     end
 
 end