To support CartesianIndexing, we must punt to the generated function even when there is only a single loop (because that single loop could be over a CartesianIndex).

Author: chriselrod

src/LoopVectorization.jl

@@ -5,6 +5,7 @@ using VectorizationBase: REGISTER_SIZE, REGISTER_COUNT, extract_data, num_vector
     mask, masktable, pick_vector_width_val, valmul, valrem, valmuladd, valadd, valsub, _MM,
     maybestaticlength, maybestaticsize, staticm1, subsetview, vzero, stridedpointer_for_broadcast,
     Static, StaticUnitRange, StaticLowerUnitRange, StaticUpperUnitRange, unwrap, maybestaticrange,
+    AbstractColumnMajorStridedPointer, AbstractRowMajorStridedPointer, AbstractSparseStridedPointer, AbstractStaticStridedPointer,
     PackedStridedPointer, SparseStridedPointer, RowMajorStridedPointer, StaticStridedPointer, StaticStridedStruct,
     maybestaticfirst, maybestaticlast
 using SIMDPirates: VECTOR_SYMBOLS, evadd, evsub, evmul, evfdiv, vrange, reduced_add, reduced_prod, reduce_to_add, reduce_to_prod,

src/condense_loopset.jl

@@ -392,11 +392,7 @@ function setup_call(ls::LoopSet, inline = Int8(2), U = zero(Int8), T = zero(Int8
     # Creating an anonymous function and calling it also achieves the outlining, while still
     # inlining the generated function into the loop preamble.
     if inline == Int8(2)
-        if num_loops(ls) == 1
-            iszero(U) ? lower(ls) : lower(ls, U, -one(U))
-        else
-            setup_call_inline(ls, U, T)
-        end
+        setup_call_inline(ls, U, T)
     else
         setup_call_noinline(ls, U, T)
     end

src/reconstruct_loopset.jl

@@ -105,27 +105,35 @@ function pushvarg′!(ls::LoopSet, ar::ArrayReferenceMeta, i)
     reverse!(ar.loopedindex); reverse!(getindices(ar)) # reverse the listed indices here, and transpose it to make it column major
     pushpreamble!(ls, Expr(:(=), vptr(ar), Expr(:call, lv(:transpose), extract_varg(i))))
 end
-function add_mref!(ls::LoopSet, ar::ArrayReferenceMeta, i::Int, ::Type{PackedStridedPointer{T, N}}) where {T, N}
+function add_mref!(
+    ls::LoopSet, ar::ArrayReferenceMeta, i::Int, ::Type{S}
+) where {T, N, S <: AbstractColumnMajorStridedPointer{T,N}}
     pushvarg!(ls, ar, i)
 end
-function add_mref!(ls::LoopSet, ar::ArrayReferenceMeta, i::Int, ::Type{RowMajorStridedPointer{T, N}}) where {T, N}
+function add_mref!(
+    ls::LoopSet, ar::ArrayReferenceMeta, i::Int, ::Type{S}
+) where {T, N, S <: AbstractRowMajorStridedPointer{T, N}}
     pushvarg′!(ls, ar, i)
 end
-function add_mref!(ls::LoopSet, ar::ArrayReferenceMeta, i::Int, ::Type{OffsetStridedPointer{T,N,P}}) where {T,N,P}
+function add_mref!(
+    ls::LoopSet, ar::ArrayReferenceMeta, i::Int, ::Type{OffsetStridedPointer{T,N,P}}
+) where {T,N,P}
     add_mref!(ls, ar, i, P)
 end
 
 function add_mref!(
     ls::LoopSet, ar::ArrayReferenceMeta, i::Int, ::Type{S}
-) where {T, X <: Tuple, S <: VectorizationBase.AbstractStaticStridedPointer{T,X}}
+) where {T, X <: Tuple, S <: AbstractStaticStridedPointer{T,X}}
     if last(X.parameters)::Int == 1
         pushvarg′!(ls, ar, i)
     else
         pushvarg!(ls, ar, i)
         first(X.parameters)::Int == 1 || pushfirst!(getindices(ar), Symbol("##DISCONTIGUOUSSUBARRAY##"))
     end
 end
-function add_mref!(ls::LoopSet, ar::ArrayReferenceMeta, i::Int, ::Type{SparseStridedPointer{T, N}}) where {T, N}
+function add_mref!(
+    ls::LoopSet, ar::ArrayReferenceMeta, i::Int, ::Type{S}
+) where {T, N, S <: AbstractSparseStridedPointer{T, N}}
     pushvarg!(ls, ar, i)
     pushfirst!(getindices(ar), Symbol("##DISCONTIGUOUSSUBARRAY##"))
 end
@@ -222,6 +230,7 @@ function calcnops(ls::LoopSet, os::OperationStruct)
     end
     offsets = ls.loopsymbol_offsets
     idxs = loopindex(ls, os.loopdeps, 0x04)  # FIXME DRY
+    iszero(length(idxs)) && return 1
     Δidxs = map(i->offsets[i+1]-offsets[i], idxs)
     nops = first(Δidxs)
     @assert all(isequal(nops), Δidxs)

test/ifelsemasks.jl

@@ -339,15 +339,15 @@ T = Float32
         fill!(c2, -999999999); maybewriteoravx!(c2, a, b)
         @test c1 ≈ c2
 
-        andorassignment!(c1, a, b)
-        andorassignmentavx!(c2, a, b)
+        andorassignment!(c1, a, b);
+        andorassignmentavx!(c2, a, b);
         @test c1 ≈ c2
-        fill!(c2, -999999999); andorassignment_avx!(c2, a, b)
+        fill!(c2, -999999999); andorassignment_avx!(c2, a, b);
         @test c1 ≈ c2
 
         if T <: Union{Float32,Float64}
             a .*= 100;
-        end
+        end;
         b1 = copy(a);
         b2 = copy(a);
         condstore!(b1)
@@ -384,7 +384,7 @@ T = Float32
     t = Bernoulli_logit(bit, a);
     @test t ≈ Bernoulli_logitavx(bit, a)
     @test t ≈ Bernoulli_logit_avx(bit, a)
-    a = rand(43)
+    a = rand(43);
     bit = a .> 0.5;
     t = Bernoulli_logit(bit, a);
     @test t ≈ Bernoulli_logitavx(bit, a)