Few minor tweaks.

Author: chriselrod

Project.toml

@@ -1,7 +1,7 @@
 name = "LoopVectorization"
 uuid = "bdcacae8-1622-11e9-2a5c-532679323890"
 authors = ["Chris Elrod <[email protected]>"]
-version = "0.8.5"
+version = "0.8.6"
 
 [deps]
 DocStringExtensions = "ffbed154-4ef7-542d-bbb7-c09d3a79fcae"

src/determinestrategy.jl

@@ -299,7 +299,7 @@ function solve_unroll_iter(X, R, u₁L, u₂L, u₁range, u₂range)
         for u₂temp ∈ u₂range
             RR ≥ u₁temp*u₂temp*R₁ + u₁temp*R₂ + u₂temp*R₅ || continue
             tempcost = unroll_cost(X, u₁temp, u₂temp, u₁L, u₂L)
-            if tempcost < bestcost
+            if tempcost ≤ bestcost
                 bestcost = tempcost
                 u₁best, u₂best = u₁temp, u₂temp
             end
@@ -329,6 +329,11 @@ function solve_unroll(X, R, u₁L, u₂L, u₁step, u₂step)
     u₂low = max(u₂step, floor(Int, u₂float)) # must be at least 1
     u₁high = solve_unroll_constT(R, u₂low) + u₁step
     u₂high = solve_unroll_constU(R, u₁low) + u₂step
+    maxunroll = REGISTER_COUNT == 32 ? 10 : 6
+    u₁low = min(u₁low, maxunroll)
+    u₂low = min(u₂low, maxunroll)
+    u₁high = min(u₁high, maxunroll)
+    u₂high = min(u₂high, maxunroll)
     solve_unroll_iter(X, R, u₁L, u₂L, u₁low:u₁step:u₁high, u₂low:u₂step:u₂high)
 end
 
@@ -658,6 +663,17 @@ function add_constant_offset_load_elmination_cost!(
     end
 end
 
+function update_costs!(costs, cost, u₁reduces, u₂reduces)
+    if u₁reduces & u₂reduces
+        costs[4] += cost
+    elseif u₂reduces # cost decreased by unrolling u₂loop
+        costs[2] += cost
+    elseif u₁reduces # cost decreased by unrolling u₁loop
+        costs[3] += cost
+    else # no cost decrease; cost must be repeated
+        costs[1] += cost
+    end
+end
 
 # Just tile outer two loops?
 # But optimal order within tile must still be determined
@@ -674,7 +690,7 @@ function evaluate_cost_tile(
     ops = operations(ls)
     nops = length(ops)
     included_vars = fill!(resize!(ls.included_vars, nops), false)
-    reduced_by_unrolling = fill(false, 2, nops)
+    reduced_by_unrolling = fill(false, 2, 2, nops)
     descendentsininnerloop = fill!(resize!(ls.place_after_loop, nops), false)
     innerloop = last(order)
     iters = fill(-99.9, nops)
@@ -718,13 +734,19 @@ function evaluate_cost_tile(
             rd = reduceddependencies(op)
             hasintersection(rd, @view(nested_loop_syms[1:end-length(rd)])) && return 0,0,Inf,false
             included_vars[id] = true
-            depends_on_u₁ = isu₁unrolled(op)
-            depends_on_u₂ = isu₂unrolled(op)
+            if isconstant(op)
+                depends_on_u₁, depends_on_u₂ = isunrolled_sym(op, u₁loopsym, u₂loopsym)
+                reduced_by_unrolling[1,1,id] = !depends_on_u₁
+                reduced_by_unrolling[2,1,id] = !depends_on_u₂
+            else
+                depends_on_u₁ = isu₁unrolled(op)
+                depends_on_u₂ = isu₂unrolled(op)
+                reduced_by_unrolling[1,1,id] = (u₁reached) & !depends_on_u₁
+                reduced_by_unrolling[2,1,id] = (u₂reached) & !depends_on_u₂
+            end
             # cost is reduced by unrolling u₁ if it is interior to u₁loop (true if either u₁reached, or if depends on u₂ [or u₁]) and doesn't depend on u₁
-            # reduced_by_unrolling[1,id] = (u₁reached | depends_on_u₂) & !depends_on_u₁
-            # reduced_by_unrolling[2,id] = (u₂reached | depends_on_u₁) & !depends_on_u₂
-            reduced_by_unrolling[1,id] = (u₁reached) & !depends_on_u₁
-            reduced_by_unrolling[2,id] = (u₂reached) & !depends_on_u₂
+            reduced_by_unrolling[1,2,id] = (u₁reached | depends_on_u₂) & !depends_on_u₁
+            reduced_by_unrolling[2,2,id] = (u₂reached | depends_on_u₁) & !depends_on_u₂
             iters[id] = iter
             innerloop ∈ loopdependencies(op) && set_upstream_family!(descendentsininnerloop, op, true)
         end
@@ -733,13 +755,15 @@ function evaluate_cost_tile(
         iters[id] == -99.9 && continue
         opisininnerloop = descendentsininnerloop[id]
 
-        u₁reduces, u₂reduces = reduced_by_unrolling[1,id], reduced_by_unrolling[2,id]
+        u₁reducesrt, u₂reducesrt = reduced_by_unrolling[1,1,id], reduced_by_unrolling[2,1,id]
+        u₁reducesrp, u₂reducesrp = reduced_by_unrolling[1,2,id], reduced_by_unrolling[2,2,id]
         if isload(op)
-            if add_constant_offset_load_elmination_cost!(cost_vec, reg_pressure, choose_to_inline, ls, op, iters[id], unrollsyms, u₁reduces, u₂reduces, Wshift, size_T, opisininnerloop)
+            if add_constant_offset_load_elmination_cost!(cost_vec, reg_pressure, choose_to_inline, ls, op, iters[id], unrollsyms, u₁reducesrp, u₂reducesrp, Wshift, size_T, opisininnerloop)
                 continue
             elseif load_elimination_cost_factor!(cost_vec, reg_pressure, choose_to_inline, ls, op, iters[id], unrollsyms, Wshift, size_T)
                 continue
-            end                
+            end
+        elseif isconstant(op)
         end
         rt, lat, rp = cost(ls, op, vectorized, Wshift, size_T)
         if isload(op) && !iszero(prefetchisagoodidea(ls, op, UnrollArgs(4, unrollsyms, 4, 0)))
@@ -750,19 +774,8 @@ function evaluate_cost_tile(
         # rp = opisininnerloop ? rp : zero(rp) # we only care about register pressure within the inner most loop
         rto = rt
         rt *= iters[id]
-        if u₁reduces & u₂reduces
-            cost_vec[4] += rt
-            reg_pressure[4] += rp
-        elseif u₂reduces # cost decreased by unrolling u₂loop
-            cost_vec[2] += rt
-            reg_pressure[2] += rp
-        elseif u₁reduces # cost decreased by unrolling u₁loop
-            cost_vec[3] += rt
-            reg_pressure[3] += rp
-        else # no cost decrease; cost must be repeated
-            cost_vec[1] += rt
-            reg_pressure[1] += rp
-        end
+        update_costs!(cost_vec, rt, u₁reducesrt, u₂reducesrt)
+        update_costs!(reg_pressure, rp, u₁reducesrp, u₂reducesrp)
     end
     # @inbounds ((cost_vec[4] > 0) || ((cost_vec[2] > 0) & (cost_vec[3] > 0))) || return 0,0,Inf,false
     costpenalty = (sum(reg_pressure) > REGISTER_COUNT) ? 2 : 1

src/loopstartstopmanager.jl

@@ -201,7 +201,19 @@ function defpointermax(ls::LoopSet, ar::ArrayReferenceMeta, n::Int, sub::Int, is
     Expr(:(=), maxsym(vptr(ar), sub), pointermax(ls, ar, n, sub, isvectorized))
 end
 
-function startloop(ls::LoopSet, us::UnrollSpecification, n::Int)
+function maxunroll(us::UnrollSpecification, n)
+    @unpack u₁loopnum, u₂loopnum, u₁, u₂ = us
+    if n == u₁loopnum# && u₁ > 1
+        u₁
+    elseif n == u₂loopnum# && u₂ > 1
+        u₂
+    else
+        1
+    end
+end
+    
+
+function startloop(ls::LoopSet, us::UnrollSpecification, n::Int, submax = maxunroll(us, n))
     @unpack u₁loopnum, u₂loopnum, vectorizedloopnum, u₁, u₂ = us
     lssm = ls.lssm[]
     termind = lssm.terminators[n]
@@ -217,17 +229,6 @@ function startloop(ls::LoopSet, us::UnrollSpecification, n::Int)
         push!(loopstart.args, startloop(getloop(ls, loopsym), loopsym))
     else
         isvectorized = n == vectorizedloopnum
-        submax = if n == u₁loopnum# && u₁ > 1
-            # push!(loopstart.args, defpointermax(ls, ptrdefs[termind], n, u₁ - 1, isvectorized))
-            u₁
-        elseif n == u₂loopnum# && u₂ > 1
-            u₂
-            # push!(loopstart.args, defpointermax(ls, ptrdefs[termind], n, u₂ - 1, isvectorized))
-            # elseif isvectorized
-            # push!(loopstart.args, defpointermax(ls, ptrdefs[termind], n, 1, isvectorized))
-        else
-            1
-        end
         for sub ∈ 0:submax
             push!(loopstart.args, defpointermax(ls, ptrdefs[termind], n, sub, isvectorized))
         end

src/lower_load.jl

@@ -99,9 +99,7 @@ function prefetchisagoodidea(ls::LoopSet, op::Operation, td::UnrollArgs)
     0
 end
 function add_prefetches!(q::Expr, ls::LoopSet, op::Operation, td::UnrollArgs, prefetchind::Int, umin::Int)
-
     @unpack u₁, u₁loopsym, u₂loopsym, vectorized, u₂max = td
-    
     dontskip = (64 ÷ VectorizationBase.REGISTER_SIZE) - 1
     ptr = vptr(op)
     innermostloopsym = first(names(ls))
@@ -117,8 +115,8 @@ function add_prefetches!(q::Expr, ls::LoopSet, op::Operation, td::UnrollArgs, pr
     offsets[prefetchind] = inner_offset
     ptr = vptr(op)
     gptr = Symbol(ptr, "##GESPEDPREFETCH##")
-    for i ∈ eachindex(gespinds.args)
-        gespinds.args[i] = Expr(:call, lv(:extract_data), gespinds.args[i])
+    for i ∈ eachindex(gespinds.args) 
+       gespinds.args[i] = Expr(:call, lv(:extract_data), gespinds.args[i])
     end    
     push!(q.args, Expr(:(=), gptr, Expr(:call, lv(:gesp), ptr, gespinds)))
 

src/lowering.jl

@@ -185,24 +185,48 @@ end
 #         Expr(:block, loopiteratesatleastonce(loop), q)
 #     end
 # end
-# function lower_unroll_for_throughput(ls::LoopSet, us::UnrollSpecification, n::Int, loop::Loop, loopsym::Symbol)
-#     sl = startloop(loop, false, loopsym)
-#     UF = 4
-#     tcc = terminatecondition(loop, us, n, loopsym, false, 1)
-#     tcu = terminatecondition(loop, us, n, loopsym, false, UF)
-#     body = lower_block(ls, us, n, false, 1)
-#     # loopisstatic = isstaticloop(loop)
-#     unrolledlabel = gensym(:unrolled)
-#     cleanuplabel = gensym(:cleanup)
-#     gotounrolled = Expr(:macrocall, Symbol("@goto"), LineNumberNode(@__LINE__, Symbol(@__FILE__)), unrolledlabel)
-#     gotocleanup = Expr(:macrocall, Symbol("@goto"), LineNumberNode(@__LINE__, Symbol(@__FILE__)), cleanuplabel)
-#     branch = Expr(:if, tcu, gotounrolled, gotocleanup)
-#     unrolled = Expr(:block, Expr(:macrocall, Symbol("@label"), LineNumberNode(@__LINE__, Symbol(@__FILE__)), unrolledlabel))
-#     foreach(_ -> push!(unrolled.args, body), 1:UF)
-#     push!(unrolled.args, Expr(:if, tcu, gotounrolled, Expr(:if, tcc, gotocleanup)))
-#     cleanup = Expr(:block, Expr(:macrocall, Symbol("@label"), LineNumberNode(@__LINE__, Symbol(@__FILE__)), cleanuplabel), body, Expr(:if, tcc, gotocleanup))
-#     Expr(:let, sl, Expr(:block, branch, unrolled, cleanup))
-# end
+function lower_unroll_for_throughput(ls::LoopSet, us::UnrollSpecification, loop::Loop, loopsym::Symbol)
+    UF = 4
+    sl = startloop(ls, us, 1, UF)
+    tcc = terminatecondition(ls, us, 1, false, 1)
+    tcu = terminatecondition(ls, us, 1, false, UF)
+    body = lower_block(ls, us, 1, false, 1)
+    loopisstatic = isstaticloop(loop)
+    tcu = loopisstatic ? tcu : expect(tcu)
+    termcondu = gensym(:maybetermu)
+    unrolledbody = Expr(:block)
+    foreach(_ -> push!(unrolledbody.args, body), 1:UF)
+
+    # q = Expr(
+    #     :block,
+    #     Expr(:while, tcu, unrolledbody),
+    #     Expr(:while, tcc, body)
+    # )
+    # return Expr(:let, sl, q)
+    
+    push!(unrolledbody.args, Expr(:(=), termcondu, tcu))
+
+    unrolledloop = Expr(
+        :block,
+        Expr(:while, termcondu, unrolledbody),
+        Expr(:while, tcc, body)
+    )
+
+    termcond = gensym(:maybeterm)
+    singleloop = Expr(
+        :block,
+        Expr(:(=), termcond, true),
+        Expr(:while, termcond, Expr(:block, body, Expr(:(=), termcond, tcc)))
+    )
+    
+    q = Expr(
+        :block,
+        assume(tcc),
+        Expr(:(=), termcondu, tcu),
+        Expr(:if, termcondu, unrolledloop, singleloop)
+    )
+    Expr(:let, sl, q)
+end
 
 function assume(ex)
     Expr(:call, Expr(:(.), Expr(:(.), :LoopVectorization, QuoteNode(:SIMDPirates)), QuoteNode(:assume)), ex)
@@ -228,25 +252,29 @@ function lower_no_unroll(ls::LoopSet, us::UnrollSpecification, n::Int, inclmask:
     nisvectorized = isvectorized(us, n)
     loopsym = names(ls)[n]
     loop = getloop(ls, loopsym)
-    # if !nisvectorized && !inclmask && isone(n) && !ls.loadelimination[] && (us.u₁ > 1) && (usorig.u₁ == us.u₁) && (usorig.u₂ == us.u₂) && length(loop) > 7
-    #     # return lower_unroll_for_throughput(ls, us, n, loop, loopsym)
-    #     return lower_llvm_unroll(ls, us, n, loop)
+    # if !nisvectorized && !inclmask && isone(n) && !iszero(ls.lssm[].terminators[1]) && !ls.loadelimination[] && (us.u₁ > 1) && (usorig.u₁ == us.u₁) && (usorig.u₂ == us.u₂) && length(loop) > 15
+    #     return lower_unroll_for_throughput(ls, us, loop, loopsym)
+    #     # return lower_llvm_unroll(ls, us, n, loop)
     # end
     # sl = startloop(loop, nisvectorized, loopsym)
     sl = startloop(ls, us, n)
     tc = terminatecondition(ls, us, n, inclmask, 1)
     body = lower_block(ls, us, n, inclmask, 1)
     isstatic = isstaticloop(loop)
+
+    if !isstatic && (usorig.u₁ == us.u₁) && (usorig.u₂ == us.u₂) && !inclmask
+        tc = expect(tc)
+    end
     q = if nisvectorized
             # Expr(:block, loopiteratesatleastonce(loop, true), Expr(:while, expect(tc), body))
-        Expr(:block, Expr(:while, isstatic ? tc : expect(tc), body))
+        Expr(:block, Expr(:while, tc, body))
     elseif isstatic && length(loop) ≤ 8
         bodyq = Expr(:block)
         foreach(_ -> push!(bodyq.args, body), 1:length(loop))
         bodyq
     else
         termcond = gensym(:maybeterm)
-        push!(body.args, Expr(:(=), termcond, isstatic ? tc : expect(tc)))
+        push!(body.args, Expr(:(=), termcond, tc))
         Expr(:block, Expr(:(=), termcond, true), Expr(:while, termcond, body))
         # Expr(:block, Expr(:while, expect(tc), body))
         # Expr(:block, assume(tc), Expr(:while, tc, body))

test/gemm.jl

@@ -578,6 +578,19 @@
     # end)
     # lsmul2x2q = LoopVectorization.LoopSet(mul2x2q)
 
+    lsAtmulBt8 = :(for m ∈ 1:8, n ∈ 1:8
+    ΔCₘₙ = zero(eltype(C))
+    for k ∈ 1:8
+    ΔCₘₙ += A[k,m] * B[n,k]
+    end
+    C[m,n] = ΔCₘₙ
+    end) |> LoopVectorization.LoopSet;
+    if LoopVectorization.VectorizationBase.REGISTER_COUNT == 32
+        @test LoopVectorization.choose_order(lsAtmulBt8) == ([:n, :m, :k], :m, :n, :m, 1, 8)
+    else
+        # @test LoopVectorization.choose_order(lsAtmulBt8) == ([:n, :m, :k], :m, :n, :m, 2, 4)
+    end
+    
     struct SizedMatrix{M,N,T} <: DenseMatrix{T}
         data::Matrix{T}
     end

test/gemv.jl

@@ -3,7 +3,7 @@ using Test
 # T = Float32
 @testset "GEMV" begin
     # Unum, Tnum = LoopVectorization.VectorizationBase.REGISTER_COUNT == 16 ? (3, 4) : (4, 6)
-    Unum, Tnum = LoopVectorization.VectorizationBase.REGISTER_COUNT == 16 ? (1, 6) : (1, 10)
+    Unum, Tnum = LoopVectorization.VectorizationBase.REGISTER_COUNT == 16 ? (2, 6) : (2, 10)
     gemvq = :(for i ∈ eachindex(y)
               yᵢ = 0.0
               for j ∈ eachindex(x)

test/miscellaneous.jl

@@ -4,8 +4,7 @@ using Test
 
 @testset "Miscellaneous" begin
 
-    # Unum, Tnum = LoopVectorization.VectorizationBase.REGISTER_COUNT == 16 ? (3, 4) : (4, 4)
-    Unum, Tnum = LoopVectorization.REGISTER_COUNT == 16 ? (1, 6) : (1, 10)
+    Unum, Tnum = LoopVectorization.REGISTER_COUNT == 16 ? (2, 6) : (2, 10)
     dot3q = :(for m ∈ 1:M, n ∈ 1:N
               s += x[m] * A[m,n] * y[n]
               end);