Change handling of vector reductions

Author: chriselrod

Project.toml

@@ -1,7 +1,7 @@
 name = "LoopVectorization"
 uuid = "bdcacae8-1622-11e9-2a5c-532679323890"
 authors = ["Chris Elrod <[email protected]>"]
-version = "0.12.35"
+version = "0.12.36"
 
 [deps]
 ArrayInterface = "4fba245c-0d91-5ea0-9b3e-6abc04ee57a9"

src/codegen/lower_compute.jl

@@ -363,18 +363,31 @@ function getu₁forreduct(ls::LoopSet, op::Operation, u₁::Int)
 end
 isidentityop(op::Operation) = iscompute(op) && (instruction(op).instr === :identity) && (length(parents(op)) == 1)
 function reduce_parent!(q::Expr, ls::LoopSet, op::Operation, opp::Operation, parent::Symbol)
+  # if instruction(op).instr === :log_fast
+  #   @show op opp isvectorized(op) isvectorized(opp) dependent_outer_reducts(ls, op)
+  # end
   isvectorized(op) && return parent
-  dependent_outer_reducts(ls, op) && return parent
+  # if dependent_outer_reducts(ls, op)
+    
+  #   return parent
+  # end
+  # @show op opp isvectorized(opp)
   if isvectorized(opp)
     oppt = opp
   elseif isidentityop(opp)
     oppt = only(parents(opp))
+    # @show oppt
     isvectorized(oppt) || return parent
   else
     return parent
   end
+  reduct_class = reduction_instruction_class(oppt.instruction)
+  if (instruction(op).instr === :mul_fast) & (reduct_class == ADDITIVE_IN_REDUCTIONS)
+    op.vectorized = true
+    return parent
+  end
   newp = gensym(parent)
-  push!(q.args, Expr(:(=), newp, Expr(:call, lv(reduction_to_scalar(oppt.instruction)), parent)))
+  push!(q.args, Expr(:(=), newp, Expr(:call, lv(reduction_to_scalar(reduct_class)), parent)))
   newp
 end
 function lower_compute!(
@@ -528,7 +541,7 @@ function lower_compute!(
           # elseif parents_u₂syms[n] & (!u₂unrolledsym)
             #&& (isouterreduction(ls, opp) != -1)
             # this checks if the parent is u₂ unrolled but this operation is not, in which case we need to reduce it.
-            reduced_u₂ = reduce_expr_u₂(mangledvar(opp), instruction(opp), u₂max, Symbol("__", u₁))#ureduct(ls))
+          reduced_u₂ = reduce_expr_u₂(mangledvar(opp), instruction(opp), u₂max, Symbol("__", u₁))#ureduct(ls))
             reducedparentname = gensym!(ls, "reducedop")
             push!(q.args, Expr(:(=), reducedparentname, reduced_u₂))
             reduced_u₂ = reduce_parent!(q, ls, op, opp, reducedparentname)

src/codegen/lower_store.jl

@@ -6,7 +6,7 @@ function storeinstr_preprend(op::Operation, vloopsym::Symbol)
     # vectorized is not a loopdep, but is a reduced dep
     opp = first(parents(op))
     # while vectorized ∉ loopdependencies(opp)
-    while !isvectorized(opp)
+    while ((!isvectorized(opp)) || (any(rdep -> rdep ∉ loopdependencies(opp), reduceddependencies(opp))))
         oppold = opp
         for oppp ∈ parents(opp)
             if vloopsym ∈ reduceddependencies(oppp)

src/codegen/operation_evaluation_order.jl

@@ -16,29 +16,28 @@
 
 # end
 
-function dependent_outer_reducts(ls::LoopSet, op)
-    for i ∈ ls.outer_reductions
-        search_tree(parents(operations(ls)[i]), name(op)) && return true
-    end
-    false
-end
+# function dependent_outer_reducts(ls::LoopSet, op)
+#     for i ∈ ls.outer_reductions
+#         search_tree(parents(operations(ls)[i]), name(op)) && return true
+#     end
+#     false
+# end
 
 function isnopidentity(ls::LoopSet, op::Operation, u₁loop::Symbol, u₂loop::Symbol, vectorized::Symbol, u₂max::Int)
-    parents_op = parents(op)
+  parents_op = parents(op)
     if iscompute(op) && instruction(op).instr === :identity && isone(length(parents_op)) && name(first(parents_op)) === name(op)
-        loopistiled = u₂max ≠ -1
+        # loopistiled = u₂max ≠ -1
         # parents_u₁syms, parents_u₂syms = parent_unroll_status(op, u₁loop, u₂loop, u₂max)
-        # @show  (u₁unrolledsym, first(parents_u₁syms)), (isu₂unrolled(op), parents_u₂syms[1])
-        # @show op parents(op) isu₁unrolled(op), isu₁unrolled(only(parents(op)))
         # if (u₁unrolledsym == first(parents_u₁syms)) && (isu₂unrolled(op) == parents_u₂syms[1])
         opp = only(parents_op)
+        # @show op opp isu₁unrolled(op), isu₁unrolled(opp), isu₂unrolled(op), isu₂unrolled(opp)
         if (isu₁unrolled(op) == isu₁unrolled(opp)) & (isu₂unrolled(op) == isu₂unrolled(opp))
             true
         else
-            if isvectorized(opp) & (!isvectorized(op))
-                op.instruction = reduction_to_scalar(instruction(opp))
-                op.mangledvariable = gensym(op.mangledvariable)
-            end
+            # if isvectorized(opp) & (!isvectorized(op))
+            #     op.instruction = reduction_to_scalar(instruction(opp))
+            #     op.mangledvariable = gensym(op.mangledvariable)
+            # end
             false
         end
     else

test/outer_reductions.jl

@@ -50,7 +50,7 @@ function test_awmean(::Type{T}) where {T}
       wxlv, wσlv, mswdlv = awmean_lv(x, σ)
       isfinite(wx)   && @test wx ≈ wxlv
       isfinite(wσ)   && @test wσ ≈ wσlv
-      isfinite(mswd) && @test mswd ≈ mswdlv atol=1e-16 rtol=sqrt(eps(typeof(mswdlv)))
+      isfinite(mswd) && @test mswd ≈ mswdlv atol=eps(typeof(mswdlv)) rtol=sqrt(eps(typeof(mswdlv)))
     end
 end