hoist some more constant loads

Author: chriselrod

src/codegen/loopstartstopmanager.jl

@@ -324,6 +324,7 @@ function cse_constant_offsets!(
         end
       end
       ops = operations(ls)
+      # @show ar
       while licmoffset # repeat until we run out
         # ind = indices[ii]
         # indices are all the same across operations, so we look to the first for checking compatibility...

src/codegen/lower_constant.jl

@@ -158,6 +158,17 @@ function lower_constant!(
 end
 
 isconstantop(op::Operation) = (instruction(op) == LOOPCONSTANT) || (isconstant(op) && length(loopdependencies(op)) == 0)
+function isinitializedconst(op::Operation)
+  if isconstant(op)
+    return true
+  elseif iscompute(op)
+    for opp ∈ parents(op)
+      isinitializedconst(opp) || return false
+    end
+    return true
+  end
+  false
+end
 function constantopname(op::Operation)
   instr = instruction(op)
   if instr === LOOPCONSTANT
@@ -181,7 +192,13 @@ end
 # @inline maybeconvert(::Type{T}, s::Number) where {T} = convert(T, s)
 # @inline maybeconvert(::Type{T}, s::T) where {T <: Number} = s
 # @inline maybeconvert(::Type, s) = s
-
+function sizeequivalent_symint_expr(intval::Int, signed::Bool)
+  if signed
+    Expr(:call, lv(:sizeequivalentint), ELTYPESYMBOL, intval)
+  else
+    Expr(:call, lv(:sizeequivalentint), ELTYPESYMBOL, intval % UInt)
+  end
+end
 
 function lower_licm_constants!(ls::LoopSet)
     ops = operations(ls)
@@ -197,10 +214,8 @@ function lower_licm_constants!(ls::LoopSet)
     for (id,(intval,intsz,signed)) ∈ ls.preamble_symint
         if intsz == 1
             setop!(ls, ops[id], intval % Bool)
-        elseif signed
-            setop!(ls, ops[id], Expr(:call, lv(:sizeequivalentint), ELTYPESYMBOL, intval))
         else
-            setop!(ls, ops[id], Expr(:call, lv(:sizeequivalentint), ELTYPESYMBOL, intval % UInt))
+            setop!(ls, ops[id], sizeequivalent_symint_expr(intval, signed))
         end
     end
     for (id,floatval) ∈ ls.preamble_symfloat

src/codegen/lower_memory_common.jl

@@ -1,4 +1,4 @@
-function parentind(ind::Symbol, op::Operation)
+function parentind(ind::Symbol, op::Union{Operation,ArrayReferenceMetaPosition})
   for (id,opp) ∈ enumerate(parents(op))
     name(opp) === ind && return id
   end

src/modeling/determinestrategy.jl

@@ -253,6 +253,7 @@ function unroll_no_reductions(ls, order, vloopsym)
   else
     max(1, min(4, round(Int, 1.75compute_rt / load_rt)))
   end
+  # @show load_rt, store_rt, compute_rt, compute_l, u
   # u = min(u, max(1, (reg_count(ls) ÷ max(1,round(Int,rp)))))
   # commented out here is to decide to align loops
   # if memory_rt > compute_rt && isone(u) && (length(order) > 1) && (last(order) === vloopsym) && length(getloop(ls, last(order))) > 8W

src/modeling/operations.jl

@@ -354,6 +354,7 @@ function Operation(id::Int, var::Symbol, elementbytes::Int, instr, optype::Opera
     Operation( id, var, elementbytes, instr, optype, mpref.loopdependencies, mpref.reduceddeps, mpref.parents, mpref.mref )
 end
 Base.:(==)(x::ArrayReferenceMetaPosition, y::ArrayReferenceMetaPosition) = x.mref == y.mref
+parents(op::ArrayReferenceMetaPosition) = op.parents
 # Avoid memory allocations by using this for ops that aren't references
 const NOTAREFERENCE = ArrayReferenceMeta(ArrayReference(Symbol(""), Symbol[]),Bool[],Symbol(""))
 const NOTAREFERENCEMP = ArrayReferenceMetaPosition(NOTAREFERENCE, NOPARENTS, Symbol[], Symbol[],Symbol(""))
@@ -364,7 +365,7 @@ loopdependencies(ref::ArrayReferenceMetaPosition) = ref.loopdependencies
 reduceddependencies(ref::ArrayReferenceMetaPosition) = ref.reduceddeps
 arrayref(ref::ArrayReference) = ref
 arrayref(ref::ArrayReferenceMeta) = ref.ref
-arrayref(ref::ArrayReferenceMetaPosition) = ref.ref.ref
+arrayref(ref::ArrayReferenceMetaPosition) = ref.mref.ref
 arrayref(op::Operation) = op.ref.ref
 getindices(ref) = arrayref(ref).indices
 getoffsets(ref) = arrayref(ref).offsets

src/parse/add_constants.jl

@@ -40,22 +40,83 @@ function add_constant!(ls::LoopSet, var::Number, elementbytes::Int = 8)
     pushpreamble!(ls, Expr(:(=), name(op), var))
     rop
 end
+function ensure_constant_lowered!(ls::LoopSet, op::Operation)
+  if iscompute(op)
+    call = callexpr(instruction(op))
+    for opp ∈ parents(op)
+      ensure_constant_lowered!(ls, opp)
+      push!(call.args, name(opp))
+    end
+    pushpreamble!(ls, Expr(:(=), name(op), call))
+  elseif isconstant(op) & !isconstantop(op)
+    opid = identifier(op)
+    for (id, sym) ∈ ls.preamble_symsym
+      if id == opid
+        pushpreamble!(ls, Expr(:(=), name(op), sym))
+        return
+      end
+    end
+    for (id,(intval,intsz,signed)) ∈ ls.preamble_symint
+      if id == opid
+        if intsz == 1
+          pushpreamble!(ls, Expr(:(=), name(op), intval % Bool))
+        elseif signed
+          pushpreamble!(ls, Expr(:(=), name(op), intval))
+        else
+          pushpreamble!(ls, Expr(:(=), name(op), intval % UInt))
+        end
+        return
+      end
+    end
+    for (id,floatval) ∈ ls.preamble_symfloat
+      if id == opid
+        pushpreamble!(ls, Expr(:(=), name(op), floatval))
+        return
+      end
+      
+    end
+    for (id,typ) ∈ ls.preamble_zeros
+      if id == opid
+        pushpreamble!(ls, Expr(:(=), name(op), staticexpr(0)))
+        return
+      end
+    end
+    for (id,f) ∈ ls.preamble_funcofeltypes
+      if id == opid
+        pushpreamble!(ls, Expr(:(=), name(op), Expr(:call, reduction_zero(f), Float64)))
+        return
+      end
+    end
+  end
+end
+function ensure_constant_lowered!(ls::LoopSet, mpref::ArrayReferenceMetaPosition, ind::Symbol)
+  length(loopdependencies(mpref)) == 0 && return
+  for (id,opp) ∈ enumerate(parents(mpref))
+    if name(opp) === ind
+      ensure_constant_lowered!(ls, opp)
+    end
+  end
+  return nothing
+end
 function add_constant!(ls::LoopSet, mpref::ArrayReferenceMetaPosition, elementbytes::Int)
-    op = Operation(length(operations(ls)), varname(mpref), elementbytes, LOOPCONSTANT, constant, NODEPENDENCY, Symbol[], NOPARENTS, mpref.mref)
-    add_vptr!(ls, op)
-    temp = gensym!(ls, "intermediateconstref")
-    vloadcall = Expr(:call, lv(:_vload), mpref.mref.ptr)
-    nindices = length(getindices(op))
-    # getoffsets(op) .+= 1
-    if nindices > 0
-        dummyloop = first(ls.loops)
-        push!(vloadcall.args, mem_offset(op, UnrollArgs(dummyloop, dummyloop, dummyloop, 0, 0, 0), fill(false,nindices), true, ls))
+  op = Operation(length(operations(ls)), varname(mpref), elementbytes, LOOPCONSTANT, constant, NODEPENDENCY, Symbol[], NOPARENTS, mpref.mref)
+  add_vptr!(ls, op)
+  temp = gensym!(ls, "intermediateconstref")
+  vloadcall = Expr(:call, lv(:_vload), mpref.mref.ptr)
+  nindices = length(getindices(op))
+  # getoffsets(op) .+= 1
+  if nindices > 0
+    dummyloop = first(ls.loops)
+    for ind ∈ getindicesonly(op)
+      ensure_constant_lowered!(ls, mpref, ind)
     end
-    push!(vloadcall.args, Expr(:call, lv(:False)), staticexpr(reg_size(ls)))
-    pushpreamble!(ls, Expr(:(=), temp, vloadcall))
-    pushpreamble!(ls, Expr(:(=), name(op), temp))
-    pushpreamble!(ls, op, temp)
-    pushop!(ls, op, temp)
+    push!(vloadcall.args, mem_offset(op, UnrollArgs(dummyloop, dummyloop, dummyloop, 0, 0, 0), fill(false,nindices), true, ls))
+  end
+  push!(vloadcall.args, Expr(:call, lv(:False)), staticexpr(reg_size(ls)))
+  pushpreamble!(ls, Expr(:(=), temp, vloadcall))
+  pushpreamble!(ls, Expr(:(=), name(op), temp))
+  pushpreamble!(ls, op, temp)
+  pushop!(ls, op, temp)
 end
 # This version has loop dependencies. var gets assigned to sym when lowering.
 # value is what will get assigned within the loop.

src/parse/add_loads.jl

@@ -46,12 +46,25 @@ function add_load!(
     mpref = array_reference_meta!(ls, array, rawindices, elementbytes, var)
     add_load!(ls, mpref, elementbytes)
 end
+function load_is_constant(ls::LoopSet, mpref::ArrayReferenceMetaPosition)
+  li = mpref.mref.loopedindex
+  inds = getindicesonly(mpref)
+  for i ∈ eachindex(li)
+    li[i] && return false
+    if (id = parentind(inds[i], mpref)) > 0
+      isinitializedconst(parents(mpref)[id]) || return false
+    end
+  end
+  true
+end
 function add_load!(
-    ls::LoopSet, mpref::ArrayReferenceMetaPosition, elementbytes::Int
+  ls::LoopSet, mpref::ArrayReferenceMetaPosition, elementbytes::Int
 )
-    iszero(length(mpref.loopdependencies)) && return add_constant!(ls, mpref, elementbytes)
-    op = Operation( ls, varname(mpref), elementbytes, :getindex, memload, mpref )
-    add_load!(ls, op, true)
+  if length(mpref.loopdependencies) == 0 || load_is_constant(ls, mpref)
+    return add_constant!(ls, mpref, elementbytes)
+  end
+  op = Operation( ls, varname(mpref), elementbytes, :getindex, memload, mpref )
+  add_load!(ls, op, true)
 end
 
 # for use with broadcasting