Switch lowering for twice-unrolled outer-reductions to use s_mod(u1 + U1*u2,4) rather than s_u2).

Author: chriselrod

src/lower_compute.jl

@@ -39,30 +39,27 @@ function lower_compute!(
             parentsunrolled[i] || continue
             parentsunrolled[i] = false
             parentop = parents_op[i]
+            i == tiledouterreduction && isconstant(parentop) && continue
             newparentop = Operation(
                 parentop.identifier, gensym(parentop.variable), parentop.elementbytes, parentop.instruction, parentop.node_type,
                 parentop.dependencies, parentop.reduced_deps, parentop.parents, parentop.ref, parentop.reduced_children
             )
             parentname = mangledvar(parentop)
             newparentname = mangledvar(newparentop)
             parents_op[i] = newparentop
-            if i == tiledouterreduction && isconstant(newparentop)
-                push!(q.args, Expr(:(=), Symbol(newparentname, suffix), Symbol(parentname, suffix)))
+            if parentstiled[i]
+                parentname = Symbol(parentname, suffix_)
+                newparentname = Symbol(newparentname, suffix_)
+            end
+            if isconstant(newparentop)
+                # @show i, parentstiled[i], newparentname, parentname
+                push!(q.args, Expr(:(=), newparentname, Symbol(parentname, 0)))
             else
-                if parentstiled[i]
-                    parentname = Symbol(parentname, suffix_)
-                    newparentname = Symbol(newparentname, suffix_)
-                end
-                if isconstant(newparentop)
-                    # @show i, parentstiled[i], newparentname, parentname
-                    push!(q.args, Expr(:(=), newparentname, Symbol(parentname, 0)))
-                else
-                    for u ∈ 0:U-1
-                        push!(q.args, Expr(:(=), Symbol(newparentname, u), Symbol(parentname, u)))
-                    end
-                    reduce_expr!(q, newparentname, Instruction(reduction_to_single_vector(instruction(newparentop))), U)
-                    push!(q.args, Expr(:(=), newparentname, Symbol(newparentname, 0)))
+                for u ∈ 0:U-1
+                    push!(q.args, Expr(:(=), Symbol(newparentname, u), Symbol(parentname, u)))
                 end
+                reduce_expr!(q, newparentname, Instruction(reduction_to_single_vector(instruction(newparentop))), U)
+                push!(q.args, Expr(:(=), newparentname, Symbol(newparentname, 0)))
             end
         end
     end
@@ -96,8 +93,8 @@ function lower_compute!(
     for u ∈ 0:Uiter
         instrcall = Expr(instr) # Expr(:call, instr)
         varsym = if tiledouterreduction > 0 # then suffix !== nothing
-            # modsuffix = ((u + suffix*U) & 3)
-            modsuffix = suffix # (suffix & 3)
+            modsuffix = ((u + suffix*U) & 3)
+            # modsuffix = suffix # (suffix & 3)
             Symbol(mvar, modsuffix)
         elseif unrollsym
             Symbol(mvar, u)

src/lower_store.jl

@@ -43,9 +43,9 @@ variable_name(op::Operation, ::Nothing) = mangledvar(op)
 variable_name(op::Operation, suffix) = Symbol(mangledvar(op), suffix, :_)
 # variable_name(op::Operation, suffix, u::Int) = (n = variable_name(op, suffix); u < 0 ? n : Symbol(n, u))
 function reduce_range!(q::Expr, toreduct::Symbol, instr::Instruction, Uh::Int, Uh2::Int)
-    for u ∈ 0:Uh-1
-        tru = Symbol(toreduct, u)
-        push!(q.args, Expr(:(=), tru, Expr(instr, tru, Symbol(toreduct, u + Uh))))
+    for u ∈ Uh:Uh2-1
+        tru = Symbol(toreduct, u - Uh)
+        push!(q.args, Expr(:(=), tru, Expr(instr, tru, Symbol(toreduct, u))))
     end
     for u ∈ 2Uh:Uh2-1
         tru = Symbol(toreduct, u - 2Uh)

src/lowering.jl

@@ -166,7 +166,7 @@ function lower_unrolled_dynamic(ls::LoopSet, us::UnrollSpecification, n::Int, in
     loopisstatic = isstaticloop(loop) & (!nisvectorized)
 
     remmask = inclmask | nisvectorized
-    Ureduct = (n == num_loops(ls)) ? calc_Ureduct(ls, us) : -1
+    Ureduct = (n == num_loops(ls) && (u₂ == -1)) ? calc_Ureduct(ls, us) : -1
     sl = startloop(loop, nisvectorized, ls.W, loopsym)
     tc = terminatecondition(loop, us, n, ls.W, loopsym, inclmask, UF)
     body = lower_block(ls, us, n, inclmask, UF)
@@ -373,7 +373,7 @@ function calc_Ureduct(ls::LoopSet, us::UnrollSpecification)
     elseif num_loops(ls) == u₁loopnum
         min(u₁, 4)
     else
-        u₂ == -1 ? u₁ : u₂
+        u₂ == -1 ? u₁ : 4#u₂
     end
 end
 function lower(ls::LoopSet, us::UnrollSpecification)