LoopVectorization tests passed locally.

Author: chriselrod

Project.toml

@@ -15,10 +15,10 @@ VectorizationBase = "3d5dd08c-fd9d-11e8-17fa-ed2836048c2f"
 [compat]
 DocStringExtensions = "0.8"
 OffsetArrays = "1"
-SIMDPirates = "0.7.25"
-SLEEFPirates = "0.4.8"
+SIMDPirates = "0.8"
+SLEEFPirates = "0.5"
 UnPack = "0,1"
-VectorizationBase = "0.11.5"
+VectorizationBase = "0.12"
 julia = "1.1"
 
 [extras]

src/LoopVectorization.jl

@@ -7,7 +7,7 @@ using VectorizationBase: REGISTER_SIZE, REGISTER_COUNT, extract_data, num_vector
     Static, Zero, StaticUnitRange, StaticLowerUnitRange, StaticUpperUnitRange, unwrap, maybestaticrange,
     AbstractColumnMajorStridedPointer, AbstractRowMajorStridedPointer, AbstractSparseStridedPointer, AbstractStaticStridedPointer,
     PackedStridedPointer, SparseStridedPointer, RowMajorStridedPointer, StaticStridedPointer, StaticStridedStruct,
-    maybestaticfirst, maybestaticlast, scalar_less, scalar_greater, noalias!, gesp, gepbyte
+    maybestaticfirst, maybestaticlast, scalar_less, scalar_greater, noalias!, gesp, gepbyte, pointerforcomparison
 using SIMDPirates: VECTOR_SYMBOLS, evadd, evsub, evmul, evfdiv, vrange, 
     reduced_add, reduced_prod, reduce_to_add, reduced_max, reduced_min, vsum, vprod, vmaximum, vminimum,
     sizeequivalentfloat, sizeequivalentint, vadd!, vsub!, vmul!, vfdiv!, vfmadd!, vfnmadd!, vfmsub!, vfnmsub!,
@@ -24,7 +24,7 @@ using Base.FastMath: add_fast, sub_fast, mul_fast, div_fast
 
 const NativeTypes = Union{Bool, Base.HWReal}
 
-export LowDimArray, stridedpointer, vectorizable,
+export LowDimArray, stridedpointer,
     @avx, @_avx, *ˡ, _avx_!,
     vmap, vmap!, vmapnt, vmapnt!, vmapntt, vmapntt!,
     vfilter, vfilter!, vmapreduce, vreduce

src/add_constants.jl

@@ -36,7 +36,7 @@ function add_constant!(ls::LoopSet, mpref::ArrayReferenceMetaPosition, elementby
     temp = gensym(:intermediateconstref)
     vloadcall = Expr(:call, lv(:vload), mpref.mref.ptr)
     if length(getindices(op)) > 0
-        push!(vloadcall.args, mem_offset(op, UnrollArgs(0, Symbol(""), Symbol(""), Symbol(""), 0, nothing), false, false))
+        push!(vloadcall.args, mem_offset(op, UnrollArgs(0, Symbol(""), Symbol(""), Symbol(""), 0, nothing), Bool[]))
     end
     pushpreamble!(ls, Expr(:(=), temp, vloadcall))
     pushpreamble!(ls, op, temp)

src/add_loads.jl

@@ -10,7 +10,7 @@ function add_load!(ls::LoopSet, op::Operation, actualarray::Bool = true, broadca
         opp = ls.opdict[ls.syms_aliasing_refs[id]] # throw an error if not found.
         return isstore(opp) ? getop(ls, first(parents(opp))) : opp
     end    
-    add_vptr!(ls, op.ref.ref.array, vptr(op.ref), actualarray, broadcast)
+    add_vptr!(ls, op.ref.ref.array, vptr(op), actualarray, broadcast)
     pushop!(ls, op, name(op))
 end
 
@@ -35,7 +35,7 @@ function add_simple_load!(
 )
     loopdeps = Symbol[s for s ∈ ref.indices]
     mref = ArrayReferenceMeta(
-        ref, fill(true, length(loopdeps))
+        ref, fill(true, length(loopdeps) - isdiscontiguous(ref))
     )
     add_simple_load!(ls, var, mref, loopdeps, elementbytes, actualarray, broadcast)
 end
@@ -48,7 +48,7 @@ function add_simple_load!(
         :getindex, memload, loopdeps,
         NODEPENDENCY, NOPARENTS, mref
     )
-    add_vptr!(ls, op.ref.ref.array, vptr(op.ref), actualarray, broadcast)
+    add_vptr!(ls, op.ref.ref.array, vptr(op), actualarray, broadcast)
     pushop!(ls, op, var)
 end
 function add_load_ref!(ls::LoopSet, var::Symbol, ex::Expr, elementbytes::Int)

src/determinestrategy.jl

@@ -704,14 +704,6 @@ function evaluate_cost_tile(
     u₁, u₂, costpenalty * ucost + stride_penalty(ls, order) + outer_reduct_penalty + favoring_heuristics, choose_to_inline[]
 end
 
-function should_inline(ls::LoopSet, u₁::Int, u₂::Int)
-    # Extremely simplistic heuristic
-    prod(length, ls.loops) ≤ 1024^2 && return true
-    for op ∈ operations(ls)
-
-    end
-    false
-end
 
 struct LoopOrders
     syms::Vector{Symbol}
@@ -866,7 +858,7 @@ function choose_tile(ls::LoopSet)
         end
     end
     ls.loadelimination[] = shouldinline
-    best_order, bestu₁, bestu₂, best_vec, u₁, u₂, lowest_cost, shouldinline
+    best_order, bestu₁, bestu₂, best_vec, u₁, u₂, lowest_cost, false#shouldinline
 end
 # Last in order is the inner most loop
 function choose_order_cost(ls::LoopSet)

src/graphs.jl

@@ -94,42 +94,47 @@ subexpr(ex::Number, incr::Number) = ex - incr
 subexpr(ex, incr::Number) = addexpr(ex,  -incr)
 
 staticmulincr(ptr, incr) = Expr(:call, lv(:staticmul), Expr(:call, :eltype, ptr), incr)
-callpointer(sym) = Expr(:call, :pointer, sym)
+callpointerforcomparison(sym) = Expr(:call, lv(:pointerforcomparison), sym)
 function vec_looprange(loopmax, UF::Int, mangledname::Symbol, ptrcomp::Bool)
-    incr = if isone(UF)
-        Expr(:call, lv(:valsub), VECTORWIDTHSYMBOL, 1)
+    if ptrcomp
+        vec_looprange(loopmax, UF, callpointerforcomparison(mangledname), staticmulincr(mangledname, VECTORWIDTHSYMBOL))
     else
-        Expr(:call, lv(:valmulsub), VECTORWIDTHSYMBOL, UF, 1)
+        vec_looprange(loopmax, UF, mangledname, VECTORWIDTHSYMBOL)
     end
-    incr = ptrcomp ? staticmulincr(mangledname, incr) : incr
-    compexpr = subexpr(loopmax, incr)
-    if ptrcomp
-        Expr(:call, :<, callpointer(mangledname), compexpr)
+end
+function vec_looprange(loopmax, UF::Int, mangledname, W)
+    incr = if isone(UF)
+        Expr(:call, lv(:valsub), W, 1)
     else
-        Expr(:call, :<, mangledname, compexpr)
+        Expr(:call, lv(:valmulsub), W, UF, 1)
     end
+    compexpr = subexpr(loopmax, incr)
+    Expr(:call, :<, mangledname, compexpr)
 end
 
-function looprange(stopcon, incr::Int, mangledname::Symbol, ptrcomp::Bool)
-    incr = 1 - incr
+# function looprange(stopcon, incr::Int, mangledname::Symbol, ptrcomp::Bool, verbose)
+#     if ptrcomp
+#         looprange(stopcon, Expr(:call, lv(:vsub), staticmulincr(mangledname, incr), 1), callpointer(mangledname), verbose)
+#     else
+#         looprange(stopcon, incr - 1, mangledname)
+#     end
+# end
+# function looprange(stopcon, incr, mangledname, verbose)
+#     if verbose
+#         Expr(:call, :<, :(@show $mangledname), :(@show $(subexpr(stopcon, incr))))
+#     else
+#         Expr(:call, :<, mangledname, subexpr(stopcon, incr))
+#     end
+# end
+function looprange(stopcon, incr::Int, mangledname)
     if iszero(incr)
-        if ptrcomp
-            Expr(:call, :<, callpointer(mangledname), stopcon)
-        else
-            Expr(:call, :<, mangledname, stopcon)
-        end
-    elseif ptrcomp
-        Expr(:call, :<, callpointer(mangledname), addexpr(stopcon, staticmulincr(mangledname, incr)))
+        Expr(:call, :≤, mangledname, stopcon)
     else
-        if isone(incr)
-            Expr(:call, :≤, mangledname, stopcon)
-        else
-            Expr(:call, :<, mangledname, addexpr(stopcon, incr))
-        end
+        Expr(:call, :≤, mangledname, subexpr(stopcon, incr))
     end
 end
-function looprange(loop::Loop, incr::Int, mangledname::Symbol)
-    loop.stopexact ? looprange(loop.stophint, incr, mangledname, false) : looprange(loop.stopsym, incr, mangledname, false)
+function looprange(loop::Loop, incr::Int, mangledname)
+    loop.stopexact ? looprange(loop.stophint, incr, mangledname) : looprange(loop.stopsym, incr, mangledname)
 end
 function terminatecondition(
     loop::Loop, us::UnrollSpecification, n::Int, mangledname::Symbol, inclmask::Bool, UF::Int = unrollfactor(us, n)
@@ -168,6 +173,23 @@ function incrementloopcounter!(q, us::UnrollSpecification, n::Int, UF::Int = unr
         push!(q.args, UF)
     end
 end
+function looplengthexpr(loop::Loop)
+    if loop.stopexact
+        if loop.startexact
+            length(loop)
+        else
+            Expr(:call, lv(:vsub), loop.stophint + 1, loop.startsym)
+        end
+    elseif loop.startexact
+        if isone(loop.starthint)
+            loop.stopsym
+        else
+            Expr(:call, lv(:vsub), loop.stopsym, loop.starthint - 1)
+        end
+    else
+        Expr(:call, lv(:vsub), loop.stopsym, Expr(:call, lv(:staticm1), loop.startsym))
+    end
+end
 
 # load/compute/store × isunrolled × istiled × pre/post loop × Loop number
 struct LoopOrder <: AbstractArray{Vector{Operation},5}
@@ -199,6 +221,11 @@ Base.@propagate_inbounds Base.getindex(lo::LoopOrder, i...) = lo.oporder[LinearI
 
 @enum NumberType::Int8 HardInt HardFloat IntOrFloat INVALID
 
+struct LoopStartStopManager
+    terminators::Vector{Int}
+    incrementedptrs::Vector{Vector{ArrayReferenceMeta}}
+    uniquearrayrefs::Vector{ArrayReferenceMeta}
+end
 # Must make it easy to iterate
 # outer_reductions is a vector of indices (within operation vectors) of the reduction operation, eg the vmuladd op in a dot product
 # O(N) search is faster at small sizes
@@ -228,10 +255,12 @@ struct LoopSet
     place_after_loop::Vector{Bool}
     unrollspecification::Base.RefValue{UnrollSpecification}
     loadelimination::Base.RefValue{Bool}
+    lssm::Base.RefValue{LoopStartStopManager}
     mod::Symbol
 end
 
 
+
 function cost_vec_buf(ls::LoopSet)
     cv = @view(ls.cost_vec[:,2])
     @inbounds for i ∈ 1:4
@@ -312,7 +341,8 @@ function LoopSet(mod::Symbol)
         ArrayReferenceMeta[],
         Matrix{Float64}(undef, 4, 2),
         Matrix{Float64}(undef, 4, 2),
-        Bool[], Bool[], Ref{UnrollSpecification}(), Ref(false), mod
+        Bool[], Bool[], Ref{UnrollSpecification}(),
+        Ref(false), Ref{LoopStartStopManager}(), mod
     )
 end