Try additionally unrolling innermost non-unrolled non-vectorized loops.

Author: chriselrod

Project.toml

@@ -15,7 +15,7 @@ VectorizationBase = "3d5dd08c-fd9d-11e8-17fa-ed2836048c2f"
 [compat]
 DocStringExtensions = "0.8"
 OffsetArrays = "1"
-SIMDPirates = "0.7.16"
+SIMDPirates = "0.7.20"
 SLEEFPirates = "0.4.4"
 UnPack = "0"
 VectorizationBase = "0.11.2"

src/determinestrategy.jl

@@ -860,6 +860,7 @@ function choose_tile(ls::LoopSet)
             new_order, state = iter
         end
     end
+    ls.loadelimination[] = shouldinline
     best_order, bestu₁, bestu₂, best_vec, u₁, u₂, lowest_cost, shouldinline
 end
 # Last in order is the inner most loop

src/graphs.jl

@@ -182,6 +182,7 @@ struct LoopSet
     included_vars::Vector{Bool}
     place_after_loop::Vector{Bool}
     unrollspecification::Base.RefValue{UnrollSpecification}
+    loadelimination::Base.RefValue{Bool}
     mod::Symbol
 end
 
@@ -280,7 +281,7 @@ function LoopSet(mod::Symbol)
         ArrayReferenceMeta[],
         Matrix{Float64}(undef, 4, 2),
         Matrix{Float64}(undef, 4, 2),
-        Bool[], Bool[], Ref{UnrollSpecification}(), mod
+        Bool[], Bool[], Ref{UnrollSpecification}(), Ref(false), mod
     )
 end
 

src/lowering.jl

@@ -145,14 +145,46 @@ function lower_block(
     push!(blockq.args, incrementloopcounter(us, n, loopsym, UF))
     blockq
 end
+
+function lower_llvm_unroll(ls::LoopSet, us::UnrollSpecification, n::Int, loop::Loop)
+    loopsym = names(ls)[n]
+    loop = getloop(ls, loopsym)
+    # loopsym = mangletiledsym(loopsym, us, n)
+    nisvectorized = false#isvectorized(us, n)
+    sl = startloop(loop, nisvectorized, loopsym)
+    # tc = terminatecondition(loop, us, n, loopsym, inclmask, 1)
+    looprange = if loop.startexact
+        if loop.stopexact
+            Expr(:(=), loopsym, Expr(:call, :(:), loop.starthint, loop.stophint))
+        else
+            Expr(:(=), loopsym, Expr(:call, :(:), loop.starthint, loop.stopsym))
+        end
+    elseif loop.stopexact
+        Expr(:(=), loopsym, Expr(:call, :(:), loop.startsym, loop.stophint))
+    else
+        Expr(:(=), loopsym, Expr(:call, :(:), loop.startsym, loop.stopsym))
+    end
+    body = lower_block(ls, us, n, false, 1)
+    push!(body.args, Expr(:loopinfo, (Symbol("llvm.loop.unroll.count"), 4)))
+    # q = Expr( :block, sl, Expr(:while, tc, body))
+    q = Expr(:for, looprange, body)
+    # if nisvectorized
+    #     tc = terminatecondition(loop, us, n, loopsym, true, 1)
+    #     body = lower_block(ls, us, n, true, 1)
+    #     push!(q.args, Expr(:if, tc, body))
+    # end
+    q
+end
 # tiledsym(s::Symbol) = Symbol("##outer##", s, "##outer##")
 # mangletiledsym(s::Symbol, us::UnrollSpecification, n::Int) = isunrolled2(us, n) ? tiledsym(s) : s
 function lower_no_unroll(ls::LoopSet, us::UnrollSpecification, n::Int, inclmask::Bool)
+    usorig = ls.unrollspecification[]
+    nisvectorized = isvectorized(us, n)
     loopsym = names(ls)[n]
     loop = getloop(ls, loopsym)
-    # loopsym = mangletiledsym(loopsym, us, n)
-    nisvectorized = isvectorized(us, n)
-
+    if VERSION ≥ v"1.4" && !nisvectorized && !inclmask && isone(n) && !ls.loadelimination[] && (us.u₁ > 1) && (usorig.u₁ == us.u₁) && (usorig.u₂ == us.u₂) && length(loop) > 7
+        return lower_llvm_unroll(ls, us, n, loop)
+    end
     sl = startloop(loop, nisvectorized, loopsym)
     tc = terminatecondition(loop, us, n, loopsym, inclmask, 1)
     body = lower_block(ls, us, n, inclmask, 1)
@@ -171,7 +203,6 @@ function lower_unrolled_dynamic(ls::LoopSet, us::UnrollSpecification, n::Int, in
     order = names(ls)
     loopsym = order[n]
     loop = getloop(ls, loopsym)
-    # loopsym = mangletiledsym(loopsym, us, n)
     vectorized = order[vectorizedloopnum]
     nisunrolled = isunrolled1(us, n)
     nisvectorized = isvectorized(us, n)

src/reconstruct_loopset.jl

@@ -461,7 +461,7 @@ Execute an `@avx` block. The block's code is represented via the arguments:
 - `vargs...` holds the encoded pointers of all the arrays (see `VectorizationBase`'s various pointer types).
 """
 @generated function _avx_!(::Val{UNROLL}, ::Type{OPS}, ::Type{ARF}, ::Type{AM}, ::Type{LPSYM}, lb::LB, vargs...) where {UNROLL, OPS, ARF, AM, LPSYM, LB}
-    # 1 + 1 # Irrelevant line you can comment out/in to force recompilation...
+    1 + 1 # Irrelevant line you can comment out/in to force recompilation...
     ls = _avx_loopset(OPS.parameters, ARF.parameters, AM.parameters, LPSYM.parameters, LB.parameters, vargs)
     # @show avx_body(ls, UNROLL)
     avx_body(ls, UNROLL)