Attempted to get LLVM to not calculate addresses using SIMD instructions + extracts for the convolution example, but failed Leaving that infrastructure there.

Author: chriselrod

src/determinestrategy.jl

@@ -453,16 +453,16 @@ function stride_penalty(ls::LoopSet, order::Vector{Symbol})
 end
 function isoptranslation(ls::LoopSet, op::Operation, unrollsyms::UnrollSymbols)
     @unpack u₁loopsym, u₂loopsym, vectorized = unrollsyms
-    (vectorized == u₁loopsym || vectorized == u₂loopsym) && return false, false
-    (isu₁unrolled(op) && isu₂unrolled(op)) || return false, false
-    istranslation = false
+    (vectorized == u₁loopsym || vectorized == u₂loopsym) && return 0, false
+    (isu₁unrolled(op) && isu₂unrolled(op)) || return 0, false
+    istranslation = 0
     inds = getindices(op); li = op.ref.loopedindex
     translationplus = false
     for i ∈ eachindex(li)
         if !li[i]
             opp = findparent(ls, inds[i + (first(inds) === Symbol("##DISCONTIGUOUSSUBARRAY##"))])
             if instruction(opp).instr ∈ (:+, :-) && u₁loopsym ∈ loopdependencies(opp) && u₂loopsym ∈ loopdependencies(opp)
-                istranslation = true
+                istranslation = i
                 translationplus = instruction(opp).instr === :+
             end
         end
@@ -527,7 +527,7 @@ function maxnegativeoffset(ls::LoopSet, op::Operation, unrollsyms::UnrollSymbols
 end
 function load_elimination_cost_factor(ls::LoopSet, op::Operation, unrollsyms::UnrollSymbols)
     @unpack u₁loopsym, u₂loopsym = unrollsyms
-    if first(isoptranslation(ls, op, unrollsyms))
+    if !iszero(first(isoptranslation(ls, op, unrollsyms)))
         for loop ∈ ls.loops
             # If another loop is short, assume that LLVM will unroll it, in which case
             # we want to be a little more conservative in terms of register pressure.

src/lower_load.jl

@@ -142,45 +142,150 @@ function lower_load_vectorized!(
     end
     nothing
 end
+function indisvectorized(ls::LoopSet, ind::Symbol, vectorized::Symbol)
+    for op ∈ operations(ls)
+        ((op.variable === ind) && isvectorized(op)) && return true
+    end
+    false
+end
+
+# function lower_load_for_optranslation!(
+#     q::Expr, op::Operation, ls::LoopSet, td::UnrollArgs{Int}, mask::Union{Nothing,Symbol,Unsigned}, translationind::Int
+# )
+#     @unpack u₁, u₁loopsym, u₂loopsym, vectorized, u₂max, suffix = td
+#     iszero(suffix) || return
+
+#     gespinds = mem_offset(op, UnrollArgs(td, 0), indices_calculated_by_pointer_offsets(ls, op.ref), false)
+#     ptr = vptr(op)
+#     gptr = Symbol(ptr, "##GESPED##")
+#     for i ∈ eachindex(gespinds.args)
+#         if i != translationind
+#             gespinds.args[i] = Expr(:call, lv(:extract_data), gespinds.args[i])
+#         end
+#     end    
+#     push!(q.args, Expr(:(=), gptr, Expr(:call, lv(:gesp), ptr, gespinds)))
+
+#     inds = Expr(:tuple)
+#     ginds = Expr(:tuple)
+#     indices = getindicesonly(op)
+
+#     for (i,ind) ∈ enumerate(indices)
+#         if i == translationind # ind cannot be the translation ind
+#             push!(inds.args, Expr(:call, lv(:Zero)))
+#             push!(ginds.args, Expr(:call, Expr(:curly, lv(:Static), 1)))
+#         elseif (ind === vectorized) || indisvectorized(ls, ind, vectorized)
+#             push!(inds.args, _MMind(Expr(:call, lv(:Zero))))
+#             push!(ginds.args, Expr(:call, lv(:Zero)))
+#         else
+#             push!(inds.args, Expr(:call, lv(:Zero)))
+#             push!(ginds.args, Expr(:call, lv(:Zero)))
+#         end
+#     end
+#     varbase = variable_name(op, 0)
+#     vloadexpr = Expr(:call, lv(:vload), gptr, inds)
+#     gespexpr  = Expr(:(=), gptr, Expr(:call, lv(:gesp), gptr, ginds))
+#     push!(q.args, Expr(:(=), Symbol(varbase, 0), vloadexpr))
+
+#     for u ∈ 1:u₁-1
+#         push!(q.args, gespexpr)
+#         push!(q.args, Expr(:(=), Symbol(varbase, u), vloadexpr))
+#     end
+#     # this takes care of u₂ == 0
+#     offset = u₁
+#     for u₂ ∈ 1:u₂max-1
+#         varold = varbase
+#         varbase = variable_name(op, u₂)
+#         for u ∈ 0:u₁-2
+#             push!(q.args, Expr(:(=), Symbol(varbase, u), Symbol(varold, u + 1)))
+#         end
+#         push!(q.args, gespexpr)
+#         push!(q.args, Expr(:(=), Symbol(varbase, u₁ - 1), vloadexpr))
+#         offset += 1
+#     end
+#     nothing
+# end
+
+
+function lower_load_for_optranslation!(
+    q::Expr, op::Operation, ls::LoopSet, td::UnrollArgs{Int}, mask::Union{Nothing,Symbol,Unsigned}, translationind::Int
+)
+    @unpack u₁, u₁loopsym, u₂loopsym, vectorized, u₂max, suffix = td
+    iszero(suffix) || return
+
+    gespinds = mem_offset(op, UnrollArgs(td, 0), indices_calculated_by_pointer_offsets(ls, op.ref), false)
+    ptr = vptr(op)
+    gptr = Symbol(ptr, "##GESPED##")
+    for i ∈ eachindex(gespinds.args)
+        if i != translationind
+            gespinds.args[i] = Expr(:call, lv(:extract_data), gespinds.args[i])
+        end
+    end    
+    push!(q.args, Expr(:(=), gptr, Expr(:call, lv(:gesp), ptr, gespinds)))
+
+    inds = Expr(:tuple)
+    indices = getindicesonly(op)
+
+    for (i,ind) ∈ enumerate(indices)
+        if i == translationind # ind cannot be the translation ind
+            push!(inds.args, Expr(:call, Expr(:curly, lv(:Static), 0)))
+        elseif (ind === vectorized) || indisvectorized(ls, ind, vectorized)
+            push!(inds.args, _MMind(Expr(:call, lv(:Zero))))
+        else
+            push!(inds.args, Expr(:call, lv(:Zero)))
+        end
+    end
+    varbase = variable_name(op, 0)
+    push!(q.args, Expr(:(=), Symbol(varbase, 0), Expr(:call, lv(:vload), gptr, copy(inds))))
+
+    for u ∈ 1:u₁-1
+        inds.args[translationind] = Expr(:call, Expr(:curly, lv(:Static), u))
+        push!(q.args, Expr(:(=), Symbol(varbase, u), Expr(:call, lv(:vload), gptr, copy(inds))))
+    end
+    # this takes care of u₂ == 0
+    offset = u₁
+    for u₂ ∈ 1:u₂max-1
+        varold = varbase
+        varbase = variable_name(op, u₂)
+        for u ∈ 0:u₁-2
+            push!(q.args, Expr(:(=), Symbol(varbase, u), Symbol(varold, u + 1)))
+        end
+        inds.args[translationind] = Expr(:call, Expr(:curly, lv(:Static), offset))
+        push!(q.args, Expr(:(=), Symbol(varbase, u₁ - 1), Expr(:call, lv(:vload), gptr, copy(inds))))
+        offset += 1
+    end
+    nothing
+end
 
 # TODO: this code should be rewritten to be more "orthogonal", so that we're just combining separate pieces.
 # Using sentinel values (eg, T = -1 for non tiling) in part to avoid recompilation.
 function lower_load!(
     q::Expr, op::Operation, ls::LoopSet, td::UnrollArgs, mask::Union{Nothing,Symbol,Unsigned} = nothing
 )
     @unpack u₁, u₁loopsym, u₂loopsym, vectorized, suffix = td
-    if !isnothing(suffix) && suffix > 0
+    if !isnothing(suffix) && ls.loadelimination[]
         istr, ispl = isoptranslation(ls, op, UnrollSymbols(u₁loopsym, u₂loopsym, vectorized))
-        if istr && ispl
-            varnew = variable_name(op, suffix)
-            varold = variable_name(op, suffix - 1)
-            for u ∈ 0:u₁-2
-                push!(q.args, Expr(:(=), Symbol(varnew, u), Symbol(varold, u + 1)))
-            end
-            umin = u₁ - 1
-        elseif u₂loopsym !== vectorized
-            mno, id = maxnegativeoffset(ls, op, u₂loopsym)
-            if -suffix < mno < 0
-                varnew = variable_name(op, suffix)
-                varold = variable_name(operations(ls)[id], suffix + mno)
-                opold = operations(ls)[id]
-                if isu₁unrolled(op)
-                    for u ∈ 0:u₁-1
-                        push!(q.args, Expr(:(=), Symbol(varnew, u), Symbol(varold, u)))
+        if !iszero(istr) & ispl
+            lower_load_for_optranslation!(q, op, ls, td, mask, istr)
+        elseif suffix > 0
+            if u₂loopsym !== vectorized
+                mno, id = maxnegativeoffset(ls, op, u₂loopsym)
+                if -suffix < mno < 0
+                    varnew = variable_name(op, suffix)
+                    varold = variable_name(operations(ls)[id], suffix + mno)
+                    opold = operations(ls)[id]
+                    if isu₁unrolled(op)
+                        for u ∈ 0:u₁-1
+                            push!(q.args, Expr(:(=), Symbol(varnew, u), Symbol(varold, u)))
+                        end
+                    else
+                        push!(q.args, Expr(:(=), varnew, varold))
                     end
-                else
-                    push!(q.args, Expr(:(=), varnew, varold))
+                    return
                 end
-                return
-            else
-                umin = 0
             end
-        else
-            umin = 0
         end
-    else
-        umin = 0
     end
+    umin = 0
     if isvectorized(op)
         lower_load_vectorized!(q, ls, op, td, mask, umin)
     else

src/lower_memory_common.jl

@@ -66,7 +66,7 @@ unrolled loads are calculated as offsets with respect to an initial gesp. This h
 Therefore, unrolled === true results in inds being ignored.
 _mm means to insert `mm`s.
 """
-function mem_offset(op::Operation, td::UnrollArgs, inds_calc_by_ptr_offset::Vector{Bool})
+function mem_offset(op::Operation, td::UnrollArgs, inds_calc_by_ptr_offset::Vector{Bool}, _mm::Bool = true)
     # @assert accesses_memory(op) "Computing memory offset only makes sense for operations that access memory."
     ret = Expr(:tuple)
     indices = getindicesonly(op)
@@ -81,9 +81,9 @@ function mem_offset(op::Operation, td::UnrollArgs, inds_calc_by_ptr_offset::Vect
         # else
         if loopedindex[n]
             if inds_calc_by_ptr_offset[n]
-                addoffset!(ret, offset, ind === vectorized)
+                addoffset!(ret, offset, _mm & (ind === vectorized))
             else
-                addoffset!(ret, ind, offset, ind === vectorized)
+                addoffset!(ret, ind, offset, _mm & (ind === vectorized))
             end
         else
             addoffset!(ret, symbolind(ind, op, td), offset)
@@ -92,6 +92,7 @@ function mem_offset(op::Operation, td::UnrollArgs, inds_calc_by_ptr_offset::Vect
     ret
 end
 
+
 function add_vectorized_offset!(ret::Expr, ind, offset, incr)
     if isone(incr)
         if iszero(offset)

src/lowering.jl

@@ -238,6 +238,10 @@ function lower_no_unroll(ls::LoopSet, us::UnrollSpecification, n::Int, inclmask:
     q = if nisvectorized
             # Expr(:block, loopiteratesatleastonce(loop, true), Expr(:while, expect(tc), body))
         Expr(:block, Expr(:while, expect(tc), body))
+    elseif isstaticloop(loop) && length(loop) ≤ 4
+        qt = Expr(:block)
+        foreach(_ -> push!(qt.args, body), 1:length(loop))
+        qt
     else
         # Expr(:block, sl, assume(tc), Expr(:while, tc, body))
         push!(body.args, Expr(:||, expect(tc), Expr(:break)))