Fixed before/after loop placement, hopefully improved performance through reducing calls/arguments passed to , and tweaked solve_tilesize.

Author: chriselrod

src/broadcast.jl

@@ -134,8 +134,9 @@ end
 function add_broadcast!(
     ls::LoopSet, destname::Symbol, bcname::Symbol, loopsyms::Vector{Symbol}, ::Type{T}, elementbytes::Int = 8
 ) where {T<:Union{Integer,Float32,Float64}}
-    pushpreamble!(ls, Expr(:(=), Symbol("##", destname), bcname))
-    add_constant!(ls, destname, elementbytes) # or replace elementbytes with sizeof(T) ? u
+    op = add_constant!(ls, destname, elementbytes) # or replace elementbytes with sizeof(T) ? u
+    pushpreamble!(ls, Expr(:(=), mangledvar(op), bcname))
+    op
 end
 function add_broadcast!(
     ls::LoopSet, destname::Symbol, bcname::Symbol, loopsyms::Vector{Symbol},

src/determinestrategy.jl

@@ -236,8 +236,8 @@ end
 function solve_tilesize(X, R, Umax, Tmax)
     first(R) == 0 && return -1,-1,Inf #solve_smalltilesize(X, R, Umax, Tmax)
     U, T, cost = solve_tilesize(X, R)
-    T -= T & 1
-    U = min(U, T)
+    # T -= T & 1
+    # U = min(U, T)
     U_too_large = U > Umax
     T_too_large = T > Tmax
     if U_too_large
@@ -259,20 +259,21 @@ function solve_tilesize(
     cost_vec::AbstractVector{Float64} = @view(ls.cost_vec[:,1]),
     reg_pressure::AbstractVector{Int} = @view(ls.reg_pres[:,1])
 )
-    maxT = 8
-    maxU = 8
+    maxT = 4#8
+    maxU = 4#8
     if isstaticloop(ls, tiled)
-        maxT = min(maxT, looprangehint(ls, tiled))
+        maxT = min(2maxT, looprangehint(ls, tiled))
     end
     if isstaticloop(ls, unrolled)
-        maxU = min(maxU, looprangehint(ls, unrolled))
+        maxU = min(2maxU, looprangehint(ls, unrolled))
     end
     solve_tilesize(cost_vec, reg_pressure, maxU, maxT)
 end
 
-function set_for_each_parent!(adal::Vector{T}, op::Operation, val::T) where {T}
-    @inbounds for opp ∈ parents(op)
-        adal[identifier(opp)] = val
+function set_upstream_family!(adal::Vector{T}, op::Operation, val::T) where {T}
+    adal[identifier(op)] = val
+    for opp ∈ parents(op)
+        set_upstream_family!(adal, opp, val)
     end
 end
 
@@ -333,7 +334,7 @@ function evaluate_cost_tile(
             unrolledtiled[1,id] = unrolled ∈ loopdependencies(op)
             unrolledtiled[2,id] = tiled ∈ loopdependencies(op)
             iters[id] = iter
-            innerloop ∈ loopdependencies(op) && set_for_each_parent!(descendentsininnerloop, op, true)
+            innerloop ∈ loopdependencies(op) && set_upstream_family!(descendentsininnerloop, op, true)
         end
     end
     for (id, op) ∈ enumerate(ops)

src/graphs.jl

@@ -299,10 +299,10 @@ end
 function add_loop!(ls::LoopSet, loop::Loop)
     ls.loops[loop.itersym] = loop
 end
-function add_vptr!(ls::LoopSet, indexed::Symbol, id::Int)
+function add_vptr!(ls::LoopSet, indexed::Symbol, id::Int, ptr::Symbol = Symbol("##vptr##_", indexed))
     if includesarray(ls, indexed) < 0
         push!(ls.includedarrays, (indexed, id))
-        pushpreamble!(ls, Expr(:(=), Symbol("##vptr##_", indexed), Expr(:call, lv(:stridedpointer), indexed)))
+        pushpreamble!(ls, Expr(:(=), ptr, Expr(:call, lv(:stridedpointer), indexed)))
     end
     nothing
 end
@@ -339,7 +339,7 @@ function add_load!(
         :getindex, memload, loopdependencies(ref, ls),
         NODEPENDENCY, NOPARENTS, ref
     )
-    add_vptr!(ls, ref.array, identifier(op))
+    add_vptr!(ls, ref.array, identifier(op), ref.ptr)
     pushop!(ls, op, var)
 end
 function add_load_ref!(ls::LoopSet, var::Symbol, ex::Expr, elementbytes::Int = 8)
@@ -390,8 +390,9 @@ function add_constant!(ls::LoopSet, var::Symbol, elementbytes::Int = 8)
 end
 function add_constant!(ls::LoopSet, var, elementbytes::Int = 8)
     sym = gensym(:temp)
-    pushpreamble!(ls, Expr(:(=), Symbol("##", sym), var))
-    pushop!(ls, Operation(length(operations(ls)), sym, elementbytes, LOOPCONSTANT, constant, NODEPENDENCY, Symbol[], NOPARENTS), sym)
+    op = Operation(length(operations(ls)), sym, elementbytes, LOOPCONSTANT, constant, NODEPENDENCY, Symbol[], NOPARENTS)
+    pushpreamble!(ls, Expr(:(=), mangledvar(op), var))
+    pushop!(ls, op, sym)
 end
 # This version has loop dependencies. var gets assigned to sym when lowering.
 function add_constant!(ls::LoopSet, var::Symbol, deps::Vector{Symbol}, sym::Symbol = gensym(:constant), elementbytes::Int = 8)
@@ -450,8 +451,9 @@ function add_parent!(
     parent = if var isa Symbol
         get!(ls.opdict, var) do
             # might add constant
-            pushpreamble!(ls, Expr(:(=), Symbol("##", var), var))
-            add_constant!(ls, var, elementbytes)
+            op = add_constant!(ls, var, elementbytes)
+            pushpreamble!(ls, Expr(:(=), mangledvar(op), var))
+            op
         end
     elseif var isa Expr #CSE candidate
         maybe_cse_load!(ls, var, elementbytes)
@@ -511,7 +513,7 @@ function add_store!(
 )
     parent = getop(ls, var)
     op = Operation( length(operations(ls)), ref.array, elementbytes, :setindex!, memstore, loopdependencies(ref), reduceddependencies(parent), [parent], ref )
-    add_vptr!(ls, ref.array, identifier(op))
+    add_vptr!(ls, ref.array, identifier(op), ref.ptr)
     pushop!(ls, op, ref.array)
 end
 function add_store_ref!(ls::LoopSet, var::Symbol, ex::Expr, elementbytes::Int = 8)
@@ -603,18 +605,6 @@ function Base.push!(ls::LoopSet, ex::Expr, elementbytes::Int = 8)
     end
 end
 
-function place_after_loop!(adal::Vector{Bool}, op::Operation)
-    pal = if isload(op) || length(reduceddependencies(op)) == 0
-        1
-    elseif length(reduceddependencies(op)) > 1
-        2
-    else
-        rd = first(reduceddependencies(op))
-        any(d -> d === rd, loopdependencies(op)) ? 1 : 2
-    end
-    pal == 1 && set_for_each_parent!(adal, op, false)
-    pal
-end
 
 function fillorder!(ls::LoopSet, order::Vector{Symbol}, loopistiled::Bool)
     lo = ls.loop_order
@@ -633,9 +623,9 @@ function fillorder!(ls::LoopSet, order::Vector{Symbol}, loopistiled::Bool)
     ops = operations(ls)
     nops = length(ops)
     included_vars = fill(false, nops)
-    all_descendents_after_loop = fill(true, nops)
-    positions = fill((-1,-1,-1,-1,-1), nops)#Vector{NTuple{5,Int}}(undef, nops)
+    place_after_loop = fill(true, nops)
     # to go inside out, we just have to include all those not-yet included depending on the current sym
+    empty!(lo)
     for _n ∈ 1:nloops
         n = 1 + nloops - _n
         ro[_n] = loopsym = order[n]
@@ -647,20 +637,11 @@ function fillorder!(ls::LoopSet, order::Vector{Symbol}, loopistiled::Bool)
             isunrolled = (unrolled ∈ loopdependencies(op)) + 1
             istiled = (loopistiled ? (tiled ∈ loopdependencies(op)) : false) + 1
             optype = Int(op.node_type) + 1
-            after_loop = place_after_loop!(all_descendents_after_loop, op)
-            positions[id] = (optype,isunrolled,istiled,after_loop,_n)
-        end
-    end
-    empty!(lo)
-    for id ∈ 1:nops
-        optype,isunrolled,istiled,after_loop,_n = positions[id]
-        optype == -1 && continue#@show ops[id]
-        if all_descendents_after_loop[id]
-            after_loop = 2
+            after_loop = place_after_loop[id] + 1
+            push!(lo[optype,isunrolled,istiled,after_loop,_n], ops[id])
+            set_upstream_family!(place_after_loop, op, false) # parents that have already been included are not moved, so no need to check included_vars to filter
         end
-        push!(lo[optype,isunrolled,istiled,after_loop,_n], ops[id])
     end
-    # 3, ro, order
 end
 
 

src/lowering.jl

@@ -1,13 +1,6 @@
 
-# function unitstride(op::Operation, sym::Symbol)
-    # (first(op.symbolic_metadata) === sym) && (first(op.numerical_metadata) == 1)
-# end
-
-function variable_name(op::Operation, suffix)
-    var = op.variable
-    suffix === nothing ? var : Symbol(var, :_, suffix)
-end
-    
+variable_name(op::Operation, ::Nothing) = mangledvar(op)
+variable_name(op::Operation, suffix) = Symbol(mangledvar(op), suffix, :_)
 
 function append_inds!(ret, indices, deps)
     start = (first(indices) === Symbol("##DISCONTIGUOUSSUBARRAY##")) + 1# && return append_inds!(ret, @view(indices[2:end]), deps)
@@ -86,15 +79,15 @@ end
 #     end
 # end
 function varassignname(var::Symbol, u::Int, isunrolled::Bool)
-    isunrolled ? Symbol("##", var, :_, u) : Symbol("##", var)
+    isunrolled ? Symbol(var, u) : var
 end
 # name_mo only gets called when vectorized
 function name_mo(var::Symbol, op::Operation, u::Int, W::Symbol, vecnotunrolled::Bool, unrolled::Symbol)
     if u < 0 # sentinel value meaning not unrolled
-        name = Symbol("##",var)
+        name = var
         mo = mem_offset(op)
     else
-        name = Symbol("##",var,:_,u)
+        name = Symbol(var, u)
         mo = vecnotunrolled ? mem_offset(op, u, unrolled) : mem_offset(op, W, u, unrolled)
     end
     name, mo
@@ -184,18 +177,18 @@ function lower_load!(
 end
 function reduce_range!(q::Expr, toreduct::Symbol, instr::Instruction, Uh::Int, Uh2::Int)
     for u ∈ 0:Uh-1
-        tru = Symbol("##",toreduct,:_,u)
-        push!(q.args, Expr(:(=), tru, Expr(instr, tru, Symbol("##",toreduct,:_,u + Uh))))
+        tru = Symbol(toreduct, u)
+        push!(q.args, Expr(:(=), tru, Expr(instr, tru, Symbol(toreduct, u + Uh))))
     end
     for u ∈ 2Uh:Uh2-1
-        tru = Symbol("##",toreduct,:_, u + 1 - 2Uh)
-        push!(q.args, Expr(:(=), tru, Expr(instr, tru, Symbol("##",toreduct,:_,u))))
+        tru = Symbol(toreduct, u + 1 - 2Uh)
+        push!(q.args, Expr(:(=), tru, Expr(instr, tru, Symbol(toreduct, u))))
     end
 end
 function reduce_range!(q::Expr, ls::LoopSet, Ulow::Int, Uhigh::Int)
     for or ∈ ls.outer_reductions
         op = ls.operations[or]
-        var = op.variable
+        var = mangledvar(op)
         temp = gensym(var)
         instr = op.instruction
         instr = get(REDUCTION_TRANSLATION, instr, instr)
@@ -221,10 +214,9 @@ function reduce_expr!(q::Expr, toreduct::Symbol, instr::Instruction, U::Int)
     nothing
 end
 
-function pvariable_name(op::Operation, suffix)
-    var = first(parents(op)).variable
-    suffix === nothing ? var : Symbol(var, :_, suffix)
-end
+pvariable_name(op::Operation, ::Nothing) = mangledvar(first(parents(op)))
+pvariable_name(op::Operation, suffix) = Symbol(pvariable_name(op, nothing), suffix, :_)
+
 function reduce_unroll!(q, op, U, unrolled)
     loopdeps = loopdependencies(op)
     isunrolled = unrolled ∈ loopdeps
@@ -332,13 +324,16 @@ function lower_compute_unrolled!(
 )
     lower_compute!(q, op, vectorized, W, unrolled, tiled, U, suffix, mask, true)
 end
+struct FalseCollection end
+Base.getindex(::FalseCollection, i...) = false
 function lower_compute!(
     q::Expr, op::Operation, vectorized::Symbol, W::Symbol, unrolled::Symbol, tiled::Symbol, U::Int,
     suffix::Union{Nothing,Int}, mask::Union{Nothing,Symbol,Unsigned} = nothing,
     opunrolled = unrolled ∈ loopdependencies(op)
 )
 
     var = op.variable
+    mvar = mangledvar(op)
     parents_op = parents(op)
     nparents = length(parents_op)
     if opunrolled
@@ -355,11 +350,12 @@ function lower_compute!(
     parentstiled = if suffix === nothing
         optiled = false
         tiledouterreduction = -1
-        fill(false, nparents)
+        FalseCollection()
     else
         tiledouterreduction = isouterreduction(op)
+        suffix_ = Symbol(suffix, :_)
         if tiledouterreduction == -1
-            var = Symbol(var, :_, suffix)
+            mvar = Symbol(mvar, suffix_)
         end
         optiled = true
         [tiled ∈ loopdependencies(opp) for opp ∈ parents_op]
@@ -384,25 +380,25 @@ function lower_compute!(
         instrcall = Expr(instr) # Expr(:call, instr)
         varsym = if tiledouterreduction > 0 # then suffix !== nothing
             modsuffix = ((u + suffix*U) & 3)
-            Symbol("##",var,:_, modsuffix)
+            Symbol(mvar, modsuffix)
         elseif opunrolled
-            Symbol("##",var,:_,u)
+            Symbol(mvar, u)
         else
-            Symbol("##",var)
+            mvar
         end
         for n ∈ 1:nparents
-            parent = parents_op[n].variable
+            parent = mangledvar(parents_op[n])
             if n == tiledouterreduction
-                parent = Symbol(parent,:_,modsuffix)
+                parent = Symbol(parent, modsuffix)
             else
                 if parentstiled[n]
-                    parent = Symbol(parent,:_,suffix)
+                    parent = Symbol(parent, suffix_)
                 end
                 if parentsunrolled[n]
-                    parent = Symbol(parent,:_,u)
+                    parent = Symbol(parent, u)
                 end
             end
-            push!(instrcall.args, Symbol("##", parent))
+            push!(instrcall.args, parent)
         end
         if maskreduct && u == Uiter # only mask last
             push!(q.args, Expr(:(=), varsym, Expr(:call, lv(:vifelse), mask, instrcall, varsym)))
@@ -429,21 +425,18 @@ function lower_constant!(
     q::Expr, op::Operation, vectorized::Symbol, W::Symbol, unrolled::Symbol, U::Int,
     suffix::Union{Nothing,Int}, mask::Any = nothing
 )
-    @unpack variable, instruction = op
-    if suffix !== nothing
-        variable = Symbol(variable, :_, suffix)
-    end
-    # store parent's reduction deps
-    # @show op.instruction, loopdependencies(op), reduceddependencies(op), unrolled, unrolled ∈ loopdependencies(op)
+    instruction = op.instruction
+    mvar = variable_name(op, suffix)
+
     constsym = instruction.instr
     if vectorized ∈ loopdependencies(op) || vectorized ∈ reduceddependencies(op)
         call = Expr(:call, lv(:vbroadcast), W, constsym)
         for u ∈ 0:U-1
-            push!(q.args, Expr(:(=), Symbol("##", variable, :_, u), call))
+            push!(q.args, Expr(:(=), Symbol(mvar, u), call))
         end
     else
         for u ∈ 0:U-1
-            push!(q.args, Expr(:(=), Symbol("##", variable, :_, u), constsym))
+            push!(q.args, Expr(:(=), Symbol(mvar, u), constsym))
         end
     end
     nothing
@@ -621,16 +614,13 @@ function initialize_outer_reductions!(
     q::Expr, op::Operation, Umin::Int, Umax::Int, W::Symbol, typeT::Symbol, unrolled::Symbol, suffix::Union{Symbol,Nothing} = nothing
 )
     # T = op.elementbytes == 8 ? :Float64 : :Float32
-    var = op.variable
     z = Expr(:call, REDUCTION_ZERO[op.instruction], typeT)
     if unrolled ∈ reduceddependencies(op)
         z = Expr(:call, lv(:vbroadcast), W, z)
     end
-    if suffix !== nothing
-        var = Symbol(var, :_, suffix)
-    end
+    mvar = variable_name(op, suffix)
     for u ∈ Umin:Umax-1
-        push!(q.args, Expr(:(=), Symbol("##", var, :_, u), z))
+        push!(q.args, Expr(:(=), Symbol(mvar, u), z))
     end
     nothing
 end
@@ -658,9 +648,10 @@ function reduce_expr!(q::Expr, ls::LoopSet, U::Int)
     for or ∈ ls.outer_reductions
         op = ls.operations[or]
         var = op.variable
+        mvar = mangledvar(op)
         instr = op.instruction
-        reduce_expr!(q, var, instr, U)
-        push!(q.args, Expr(:(=), var, Expr(:call, REDUCTION_SCALAR_COMBINE[instr], var, Symbol("##", var, :_0))))
+        reduce_expr!(q, mvar, instr, U)
+        push!(q.args, Expr(:(=), var, Expr(:call, REDUCTION_SCALAR_COMBINE[instr], var, Symbol(mvar, 0))))
     end
 end
 function gc_preserve(ls::LoopSet, q::Expr)