Special case some global constant symbols to avoid passing them as arguments.

Author: chriselrod

Project.toml

@@ -1,7 +1,7 @@
 name = "LoopVectorization"
 uuid = "bdcacae8-1622-11e9-2a5c-532679323890"
 authors = ["Chris Elrod <[email protected]>"]
-version = "0.12.13"
+version = "0.12.14"
 
 [deps]
 ArrayInterface = "4fba245c-0d91-5ea0-9b3e-6abc04ee57a9"

src/codegen/lower_compute.jl

@@ -512,7 +512,6 @@ function lower_compute!(
             loopval = first(loopdependencies(opp))
             add_loopvalue!(instrcall, loopval, ua, u₁)
         elseif name(opp) === name(op)
-          
             selfdep = n
             if ((isvectorized(opp) && !isvectorized(op))) ||
                 (parents_u₁syms[n] != u₁unrolledsym) || (parents_u₂syms[n] != u₂unrolledsym)
@@ -531,6 +530,8 @@ function lower_compute!(
             push!(q.args, Expr(:(=), reducedparentname, reduced_u₂))
             reduced_u₂ = reduce_parent!(q, ls, op, opp, reducedparentname)
             push!(instrcall.args, reduced_u₂)
+        elseif isconstant(opp) && instruction(opp).mod === GLOBALCONSTANT
+            push!(instrcall.args, GlobalRef(Base, instruction(opp).instr))
         else
             parent, uₚ = parent_op_name!(q, ls, parents_op, n, modsuffix, suffix_, parents_u₁syms, parents_u₂syms, u₁, u₂max, u₂unrolledsym, op, tiledouterreduction)
             parent = reduce_parent!(q, ls, op, opp, parent)

src/codegen/lower_constant.jl

@@ -102,9 +102,9 @@ function lower_constant!(
   @unpack u₁, u₁loopsym, u₂loopsym, vloopsym, u₂max, suffix = ua
   mvar, opu₁, opu₂ = variable_name_and_unrolled(op, u₁loopsym, u₂loopsym, vloopsym, suffix, ls)
   !opu₂ && suffix > 0 && return
-  instruction = op.instruction
-  constsym = instruction.instr
-  # constsym = Symbol(instruction.instr, '_', 1)
+  instr = instruction(op)
+  instr.mod === GLOBALCONSTANT && return
+  constsym = instr.instr
   reducedchildvectorized = vloopsym ∈ reducedchildren(op)
   if reducedchildvectorized || isvectorized(op) || vloopsym ∈ reduceddependencies(op) || should_broadcast_op(op)
     # call = Expr(:call, lv(:vbroadcast), W, Expr(:call, lv(:maybeconvert), typeT, constsym))
@@ -160,7 +160,7 @@ function lower_constant!(
   nothing
 end
 
-isconstantop(op::Operation) = (instruction(op) === LOOPCONSTANT) || (isconstant(op) && length(loopdependencies(op)) == 0)
+isconstantop(op::Operation) = (instruction(op) == LOOPCONSTANT) || (isconstant(op) && length(loopdependencies(op)) == 0)
 function constantopname(op::Operation)
   instr = instruction(op)
   if instr === LOOPCONSTANT

src/condense_loopset.jl

@@ -243,7 +243,7 @@ function loopset_return_value(ls::LoopSet, ::Val{extract}) where {extract}
 end
 const DROPPEDCONSTANT = Instruction(Symbol("##DROPPED#CONSTANT##"),Symbol("##DROPPED#CONSTANT##"))
 function skip_constant(instr::Instruction)
-    ((instr == LOOPCONSTANT) || (instr.mod === :numericconstant)) || (instr == DROPPEDCONSTANT)
+  (((instr == LOOPCONSTANT) || (instr.mod === :numericconstant)) || (instr == DROPPEDCONSTANT)) || instr.mod === GLOBALCONSTANT
 end
 
 function add_reassigned_syms!(q::Expr, ls::LoopSet)
@@ -456,6 +456,15 @@ function remove_outer_reducts!(roots::Vector{Bool}, ls::LoopSet)
     end
 end
 
+# function generate_call_split(ls::LoopSet, (inline,u₁,u₂)::Tuple{Bool,Int8,Int8}, thread::UInt, debug::Bool = false)
+#   ops = operations(ls)
+#   for op ∈ ops
+#     if (iscompute(op) && (instruction(op).instr === :ifelse)) && iszero(length(loopdependencies(first(parents(op)))))
+#       # we want to eliminate
+#     end
+#   end
+# end
+
 # Try to condense in type stable manner
 function generate_call(ls::LoopSet, (inline,u₁,u₂)::Tuple{Bool,Int8,Int8}, thread::UInt, debug::Bool = false)
     extra_args = Expr(:tuple)

src/modeling/costs.jl

@@ -36,8 +36,6 @@ Base.hash(instr::Instruction, h::UInt64) = hash(instr.instr, hash(instr.mod, h))
 # end
 Base.isequal(ins1::Instruction, ins2::Instruction) = (ins1.instr === ins2.instr) && (ins1.mod === ins2.mod)
 
-const LOOPCONSTANT = Instruction(:LoopVectorization, Symbol("LOOPCONSTANTINSTRUCTION"))
-
 """
     InstructionCost
 

src/modeling/graphs.jl

@@ -1344,32 +1344,34 @@ Returns `n`, where `n` is the constant declarations's index among parents(op), i
 Returns `-1` if not an outerreduction.
 """
 function isouterreduction(ls::LoopSet, op::Operation)
-    if isconstant(op) # equivalent to checking if length(loopdependencies(op)) == 0
-        op.instruction == LOOPCONSTANT && return 0
-        ops = operations(ls)
-        for or ∈ ls.outer_reductions
-            name(op) === name(ops[or]) && return 0
+  if isconstant(op) # equivalent to checking if length(loopdependencies(op)) == 0
+    instr = op.instruction
+    instr == LOOPCONSTANT && return 0
+    instr.mod === GLOBALCONSTANT && return -1
+    ops = operations(ls)
+    for or ∈ ls.outer_reductions
+      name(op) === name(ops[or]) && return 0
+    end
+    -1
+  elseif iscompute(op)
+    var = op.variable
+    for opid ∈ ls.outer_reductions
+      rop = operations(ls)[opid]
+      if rop === op
+        for (n,opp) ∈ enumerate(parents(op))
+          opp.variable === var && return n
         end
-        -1
-    elseif iscompute(op)
-        var = op.variable
-        for opid ∈ ls.outer_reductions
-            rop = operations(ls)[opid]
-            if rop === op
-                for (n,opp) ∈ enumerate(parents(op))
-                    opp.variable === var && return n
-                end
-            else
-                for (n,opp) ∈ enumerate(parents(op))
-                    opp === rop && return n
-                    search_tree(parents(opp), rop.variable) && return n
-                end
-            end
+      else
+        for (n,opp) ∈ enumerate(parents(op))
+          opp === rop && return n
+          search_tree(parents(opp), rop.variable) && return n
         end
-        -1
-    else
-        -1
+      end
     end
+    -1
+  else
+    -1
+  end
 end
 
 struct LoopError <: Exception

src/modeling/operations.jl

@@ -1,5 +1,8 @@
 const DISCONTIGUOUS = Symbol("##DISCONTIGUOUSSUBARRAY##")
 const CONSTANTZEROINDEX = Symbol("##CONSTANTZEROINDEX##")
+const LOOPCONSTANT = Instruction(:LoopVectorization, Symbol("LOOPCONSTANTINSTRUCTION"))
+const GLOBALCONSTANT = Symbol("##GLOBAL##CONSTANT##")
+
 
 
 """
@@ -250,8 +253,9 @@ const NOPARENTS = Operation[]
 function Base.show(io::IO, op::Operation)
     if isconstant(op)
         if op.instruction === LOOPCONSTANT
-
             print(io, Expr(:(=), op.variable, 0))
+        elseif op.instruction.instr === GLOBALCONSTANT
+            print(io, op.instruction.instr)
         else
             print(io, Expr(:(=), op.variable, op.instruction.instr))
         end

src/parse/add_constants.jl

@@ -1,8 +1,11 @@
+const CONSTANT_SYMBOLS = (:nothing, :Float64, :Float32, :Int8, :UInt8, :Int16, :UInt16, :Int32, :UInt32, :Int64, :UInt64)
 function add_constant!(ls::LoopSet, var::Symbol, elementbytes::Int)
-    op = Operation(length(operations(ls)), var, elementbytes, LOOPCONSTANT, constant, NODEPENDENCY, Symbol[], NOPARENTS)
-    rop = pushop!(ls, op, var)
-    rop === op && pushpreamble!(ls, op, var)
-    rop
+  globalconst = Base.sym_in(var, CONSTANT_SYMBOLS)
+  instr = globalconst ? Instruction(GLOBALCONSTANT, var) : LOOPCONSTANT
+  op = Operation(length(operations(ls)), var, elementbytes, instr, constant, NODEPENDENCY, Symbol[], NOPARENTS)
+  rop = pushop!(ls, op, var)
+  (!globalconst && (rop === op)) && pushpreamble!(ls, op, var)
+  rop
 end
 # function add_constant!(ls::LoopSet, var, elementbytes::Int = 8)
 #     sym = gensym(:loopconstant)
@@ -14,7 +17,7 @@ function add_constant!(ls::LoopSet, var::Number, elementbytes::Int = 8)
     ops = operations(ls)
     typ = var isa Integer ? HardInt : HardFloat
     rop = pushop!(ls, op)
-    rop !== op && return rop
+    rop === op || return rop
     if iszero(var)
         for (id,typ_) ∈ ls.preamble_zeros
             (instruction(ops[id]) == LOOPCONSTANT && typ == typ_) && return ops[id]