Tile remained with nested ifs.

Author: chriselrod

Project.toml

@@ -1,7 +1,7 @@
 name = "LoopVectorization"
 uuid = "bdcacae8-1622-11e9-2a5c-532679323890"
 authors = ["Chris Elrod <[email protected]>"]
-version = "0.1.0"
+version = "0.1.1"
 
 [deps]
 LinearAlgebra = "37e2e46d-f89d-539d-b4ee-838fcccc9c8e"

src/LoopVectorization.jl

@@ -1,7 +1,7 @@
 module LoopVectorization
 
 using VectorizationBase, SIMDPirates, SLEEFPirates, MacroTools, Parameters
-using VectorizationBase: REGISTER_SIZE, extract_data, num_vector_load_expr, mask
+using VectorizationBase: REGISTER_SIZE, REGISTER_COUNT, extract_data, num_vector_load_expr, mask
 using SIMDPirates: VECTOR_SYMBOLS, evadd, evmul, vrange, reduced_add, reduced_prod
 using Base.Broadcast: Broadcasted, DefaultArrayStyle
 using LinearAlgebra: Adjoint

src/determinestrategy.jl

@@ -167,7 +167,7 @@ function solve_tilesize(X, R)
     # X is vector of costs, and R is of register pressures
     # @show X
     # @show R 
-    RR = VectorizationBase.REGISTER_COUNT - R[3] - R[4]
+    RR = REGISTER_COUNT - R[3] - R[4]
     a = (R[1])^2*X[2] - (R[2])^2*R[1]*X[3]/RR
     b = 2*R[1]*R[2]*X[3]
     c = -RR*R[1]*X[3]
@@ -180,7 +180,7 @@ function solve_tilesize(X, R)
     Uhigh = Ulow + 1 #ceil(Int, Ufloat)
     Thigh = Tlow + 1 #ceil(Int, Tfloat)
 
-    RR = VectorizationBase.REGISTER_COUNT - R[3] - R[4]
+    RR = REGISTER_COUNT - R[3] - R[4]
     U, T = Ulow, Tlow
     tcost = tile_cost(X, Ulow, Tlow)
     # @show Ulow*Thigh*R[1] + Ulow*R[2]
@@ -208,10 +208,14 @@ function solve_tilesize(X, R)
     min(U,RR), min(T,RR), tcost
 end
 function solve_tilesize_constU(X, R, U)
-    floor(Int, (VectorizationBase.REGISTER_COUNT - R[3] - R[4] - U*R[2]) / (U * R[1]))
+    floor(Int, (REGISTER_COUNT - R[3] - R[4] - U*R[2]) / (U * R[1]))
 end
 function solve_tilesize_constT(X, R, T)
-    floor(Int, (VectorizationBase.REGISTER_COUNT - R[3] - R[4]) / (T * R[1] + R[2]))
+    floor(Int, (REGISTER_COUNT - R[3] - R[4]) / (T * R[1] + R[2]))
+end
+function solve_tilesize_constT(ls, T)
+    R = @view ls.reg_pres[:,1]
+    floor(Int, (REGISTER_COUNT - R[3] - R[4]) / (T * R[1] + R[2]))
 end
 # Tiling here is about alleviating register pressure for the UxT
 function solve_tilesize(X, R, Umax, Tmax)
@@ -233,7 +237,15 @@ function solve_tilesize(X, R, Umax, Tmax)
     end
     U, T, cost
 end
-
+function solve_tilesize(
+    ls::LoopSet, unrolled::Symbol, tiled::Symbol,
+    cost_vec::AbstractVector{Float64} = @view(ls.cost_vec[:,1]),
+    reg_pressure::AbstractVector{Int} = @view(ls.reg_pres[:,1])
+)
+    maxT = isstaticloop(ls, tiled) ? looprangehint(ls, tiled) : REGISTER_COUNT
+    maxU = isstaticloop(ls, unrolled) ? looprangehint(ls, unrolled) : REGISTER_COUNT
+    solve_tilesize(cost_vec, reg_pressure, maxT, maxU)
+end
 
 # Just tile outer two loops?
 # But optimal order within tile must still be determined
@@ -257,8 +269,8 @@ function evaluate_cost_tile(
     # cost_mat[3] / ( unrolled)
     # cost_mat[4]
     # @show order
-    cost_vec = zeros(Float64, 4)
-    reg_pressure = zeros(Int, 4)
+    cost_vec = cost_vec_buf(ls)
+    reg_pressure = reg_pres_buf(ls)
     # @inbounds reg_pressure[2] = 1
     # @inbounds reg_pressure[3] = 1
     for n ∈ 1:N
@@ -304,22 +316,7 @@ function evaluate_cost_tile(
             end
         end
     end
-    Tstatic = isstaticloop(ls, tiled)
-    Ustatic = isstaticloop(ls, unrolled)
-    # @show order, cost_vec, reg_pressure
-    if Tstatic
-        if Ustatic
-            solve_tilesize(cost_vec, reg_pressure, looprangehint(ls, tiled), looprangehint(ls, unrolled))
-        else
-            solve_tilesize(cost_vec, reg_pressure, looprangehint(ls, tiled), typemax(Int))
-        end
-    else
-        if Ustatic
-            solve_tilesize(cost_vec, reg_pressure, typemax(Int), looprangehint(ls, unrolled))
-        else
-            solve_tilesize(cost_vec, reg_pressure)#, typemax(Int), typemax(Int))
-        end
-    end
+    solve_tilesize(ls, unrolled, tiled, cost_vec, reg_pressure)
 end
 
 
@@ -385,7 +382,7 @@ function choose_unroll_order(ls::LoopSet, lowest_cost::Float64 = Inf)
 end
 function choose_tile(ls::LoopSet)
     lo = LoopOrders(ls)
-    best_order = copy(lo.syms)
+    best_order = copyto!(ls.loop_order.bestorder, lo.syms)
     new_order, state = iterate(lo) # right now, new_order === best_order
     U, T, lowest_cost = 0, 0, Inf
     while true
@@ -394,6 +391,7 @@ function choose_tile(ls::LoopSet)
             lowest_cost = cost_temp
             U, T = U_temp, T_temp
             copyto!(best_order, new_order)
+            save_tilecost!(ls)
         end
         iter = iterate(lo, state)
         iter === nothing && return best_order, U, T, lowest_cost

src/graphs.jl

@@ -53,11 +53,12 @@ end
 struct LoopOrder <: AbstractArray{Vector{Operation},5}
     oporder::Vector{Vector{Operation}}
     loopnames::Vector{Symbol}
+    bestorder::Vector{Symbol}
 end
 function LoopOrder(N::Int)
-    LoopOrder( [ Operation[] for i ∈ 1:24N ], Vector{Symbol}(undef, N) )
+    LoopOrder( [ Operation[] for i ∈ 1:24N ], Vector{Symbol}(undef, N), Vector{Symbol}(undef, N) )
 end
-LoopOrder() = LoopOrder(Vector{Operation}[],Symbol[])
+LoopOrder() = LoopOrder(Vector{Operation}[],Symbol[],Symbol[])
 Base.empty!(lo::LoopOrder) = foreach(empty!, lo.oporder)
 function Base.resize!(lo::LoopOrder, N::Int)
     Nold = length(lo.loopnames)
@@ -66,6 +67,7 @@ function Base.resize!(lo::LoopOrder, N::Int)
         lo.oporder[n] = Operation[]
     end
     resize!(lo.loopnames, N)
+    resize!(lo.bestorder, N)
     lo
 end
 Base.size(lo::LoopOrder) = (4,2,2,2,length(lo.loopnames))
@@ -85,10 +87,34 @@ struct LoopSet
     includedarrays::Vector{Tuple{Symbol,Int}}
     syms_aliasing_refs::Vector{Symbol} # O(N) search is faster at small sizes
     refs_aliasing_syms::Vector{ArrayReference}
+    cost_vec::Matrix{Float64}
+    reg_pres::Matrix{Int}
     # sym_to_ref_aliases::Dict{Symbol,ArrayReference}
     # ref_to_sym_aliases::Dict{ArrayReference,Symbol}
 end
 
+function cost_vec_buf(ls::LoopSet)
+    cv = @view(ls.cost_vec[:,2])
+    @inbounds for i ∈ 1:4
+        cv[i] = 0.0
+    end
+    cv
+end
+function reg_pres_buf(ls::LoopSet)
+    ps = @view(ls.reg_pres[:,2])
+    @inbounds for i ∈ 1:4
+        ps[i] = 0
+    end
+    ps
+end
+function save_tilecost!(ls::LoopSet)
+    @inbounds for i ∈ 1:4
+        ls.cost_vec[i,1] = ls.cost_vec[i,2]
+        ls.reg_pres[i,1] = ls.reg_pres[i,2]
+    end
+end
+
+
 # function op_to_ref(ls::LoopSet, op::Operation)
     # s = op.variable
     # id = findfirst(ls.syms_aliasing_regs)
@@ -113,9 +139,9 @@ function LoopSet()
         Expr(:block,),
         Tuple{Symbol,Int}[],
         Symbol[],
-        ArrayReference[]
-        # Dict{Symbol,ArrayReference}()
-        # Dict{ArrayReference,Symbol}()
+        ArrayReference[],
+        Matrix{Float64}(undef, 4, 2),
+        Matrix{Int}(undef, 4, 2)
     )
 end
 num_loops(ls::LoopSet) = length(ls.loops)
@@ -131,8 +157,7 @@ looprangesym(ls::LoopSet, s::Symbol) = ls.loops[s].rangesym
 getop(ls::LoopSet, s::Symbol) = ls.opdict[s]
 getop(ls::LoopSet, i::Int) = ls.operations[i + 1]
 
-function looprange(ls::LoopSet, s::Symbol, incr::Int = 1, mangledname::Symbol = s)
-    loop = ls.loops[s]
+function looprange(ls::LoopSet, s::Symbol, incr::Int = 1, mangledname::Symbol = s, loop = ls.loops[s])
     incr -= 1
     if iszero(incr)
         Expr(:call, :<, mangledname, loop.hintexact ? loop.rangehint : loop.rangesym)

src/lowering.jl

@@ -49,7 +49,6 @@ function lower_load_scalar!(
     q::Expr, op::Operation, W::Int, unrolled::Symbol, U::Int,
     suffix::Union{Nothing,Int}, mask::Union{Nothing,Symbol,Unsigned} = nothing
 )
-
     loopdeps = loopdependencies(op)
     @assert unrolled ∉ loopdeps
     var = op.variable
@@ -559,7 +558,6 @@ function lower_unrolled!(
     end
     Wt = W
     Ut = U
-    Urem = 0
     Urepeat = true
     while Urepeat
         if Uexprtype !== :skip
@@ -639,7 +637,8 @@ function lower_tiled(ls::LoopSet, U::Int, T::Int)
     unrolled = order[end-1]
     mangledtiled = tiledsym(tiled)
     W = VectorizationBase.pick_vector_width(ls, unrolled)
-    static_tile = isstaticloop(ls, tiled)
+    tiledloop = ls.loops[tiled]
+    static_tile = tiledloop.hintexact
     static_unroll = isstaticloop(ls, unrolled)
     unrolled_iter = looprangehint(ls, unrolled)
     unrolled_itersym = looprangesym(ls, unrolled)
@@ -649,32 +648,51 @@ function lower_tiled(ls::LoopSet, U::Int, T::Int)
     Trem = Tt = T
     nloops = num_loops(ls);
     # addtileonly = sum(length, @view(oporder(ls)[:,:,:,:,end])) > 0
-    Texprtype = (static_tile && tiled_iter < 2T) ? :block : :while
+    # Texprtype = (static_tile && tiled_iter < 2T) ? :block : :while
+    firstiter = true
+    mangledtiled = tiledsym(tiled)
+    local qifelse::Expr
     while Tt > 0
-        # 
         tiledloopbody = Expr(:block, )
-        # else
-            # Expr(:block, Expr(:(=), unrolled, 0))
-        # end
         lower_unrolled!(tiledloopbody, ls, U, Tt, W, static_unroll, unrolled_iter, unrolled_itersym)
         tiledloopbody = lower_nest(ls, nloops, U, Tt, tiledloopbody, 0, W, nothing, :block)
-        push!(q.args, Texprtype === :block ? tiledloopbody : Expr(Texprtype, looprange(ls, tiled, Tt, tiledsym(tiled)), tiledloopbody))
+        if firstiter
+            push!(q.args, (static_tile && tiled_iter < 2T) ? tiledloopbody : Expr(:while, looprange(ls, tiled, Tt, mangledtiled, tiledloop), tiledloopbody))
+        elseif static_tile
+            push!(q.args, tiledloopbody)
+        else # not static, not firstiter
+            comparison = Expr(:call, :(==), mangledtiled, Expr(:call, :-, tiledloop.rangesym, Tt))
+            qifelsenew = Expr(:elseif, comparison, tiledloopbody)
+            push!(qifelse.args, qifelsenew)
+            qifelse = qifelsenew
+        end
         if static_tile
-            Tt = if Tt == T
+            if Tt == T
                 # push!(tiledloopbody.args, Expr(:+=, mangledtiled, Tt))
                 Texprtype = :block
-                looprangehint(ls, tiled) % T
+                Tt = looprangehint(ls, tiled) % T
+                # Recalculate U
+                U = solve_tilesize_constT(ls, Tt)
             else
-                0 # terminate
+                Tt = 0 # terminate
             end
             nothing
         else
-            Ttold = Tt
-            Tt >>>= 1
-            # Tt == 0 || push!(tiledloopbody.args, Expr(:+=, mangledtiled, Ttold))
-            Texprtype = 2Tt == Ttold ? :if : :while
+            if firstiter
+                comparison = Expr(:call, :(==), mangledtiled, tiledloop.rangesym)
+                qifelse = Expr(:if, comparison, Expr(:block)) # do nothing
+                push!(q.args, qifelse)
+                Tt = 0
+            end
+            Tt += 1 # we start counting up by 1
+            if Tt == T # terminate on Tt = T
+                Tt = 0
+            else
+                U = solve_tilesize_constT(ls, Tt)
+            end
             nothing
         end
+        firstiter = false
     end
     q = gc_preserve(ls, q)
     reduce_expr!(q, ls, U)