a few improvements

Author: chriselrod

Project.toml

@@ -1,7 +1,7 @@
 name = "LoopVectorization"
 uuid = "bdcacae8-1622-11e9-2a5c-532679323890"
 authors = ["Chris Elrod <[email protected]>"]
-version = "0.12.129"
+version = "0.12.130"
 
 [deps]
 ArrayInterface = "4fba245c-0d91-5ea0-9b3e-6abc04ee57a9"

benchmark/looptests.jl

@@ -13,17 +13,17 @@ end
 Base.size(::SizedOffsetMatrix{<:Any,LR,UR,LC,UC}) where {LR,UR,LC,UC} =
   (UR - LR + 1, UC - LC + 1)
 Base.axes(::SizedOffsetMatrix{T,LR,UR,LC,UC}) where {T,LR,UR,LC,UC} =
-  (Static{LR}():Static{UR}(), Static{LC}():Static{UC}())
+  (StaticInt{LR}():StaticInt{UR}(), StaticInt{LC}():StaticInt{UC}())
 Base.parent(A::SizedOffsetMatrix) = A.data
 Base.unsafe_convert(::Type{Ptr{T}}, A::SizedOffsetMatrix{T}) where {T} = pointer(A.data)
-ArrayInterface.contiguous_axis(::Type{<:SizedOffsetMatrix}) = Static(1)
-ArrayInterface.contiguous_batch_size(::Type{<:SizedOffsetMatrix}) = Static(0)
-ArrayInterface.stride_rank(::Type{<:SizedOffsetMatrix}) = (Static(1), Static(2))
+ArrayInterface.contiguous_axis(::Type{<:SizedOffsetMatrix}) = StaticInt(1)
+ArrayInterface.contiguous_batch_size(::Type{<:SizedOffsetMatrix}) = StaticInt(0)
+ArrayInterface.stride_rank(::Type{<:SizedOffsetMatrix}) = (StaticInt(1), StaticInt(2))
 function ArrayInterface.strides(A::SizedOffsetMatrix{T,LR,UR,LC,UC}) where {T,LR,UR,LC,UC}
-  (Static{1}(), (Static{UR}() - Static{LR}() + Static{1}()))
+  (StaticInt{1}(), (StaticInt{UR}() - StaticInt{LR}() + StaticInt{1}()))
 end
 ArrayInterface.offsets(A::SizedOffsetMatrix{T,LR,UR,LC,UC}) where {T,LR,UR,LC,UC} =
-  (Static{LR}(), Static{LC}())
+  (StaticInt{LR}(), StaticInt{LC}())
 ArrayInterface.parent_type(::Type{<:SizedOffsetMatrix{T}}) where {T} = Matrix{T}
 Base.getindex(A::SizedOffsetMatrix, i, j) =
   LoopVectorization.vload(LoopVectorization.stridedpointer(A), (i, j))

src/LoopVectorization.jl

@@ -150,7 +150,6 @@ using CloseOpenIntervals: AbstractCloseOpen, CloseOpen#, SafeCloseOpen
 @inline static_step(x::CartesianIndices) =
   VectorizationBase.CartesianVIndex(map(static_step, x.indices))
 # end
-const Static = StaticInt
 
 export LowDimArray,
   stridedpointer,

src/broadcast.jl

@@ -191,11 +191,11 @@ function _strides_expr(@nospecialize(s), @nospecialize(x), R::Vector{Int}, D::Ve
   for n ∈ Nrange
     xₙ_type = x[n]
     # xₙ_type = typeof(x).parameters[n]
-    xₙ_static = xₙ_type <: Static
+    xₙ_static = xₙ_type <: StaticInt
     xₙ_value::Int = xₙ_static ? (xₙ_type.parameters[1])::Int : 0
     s_type = s[n]
     # s_type = typeof(s).parameters[n]
-    sₙ_static = s_type <: Static
+    sₙ_static = s_type <: StaticInt
     if sₙ_static
       sₙ_value = s_type.parameters[1]
       if s_type === One

src/codegen/loopstartstopmanager.jl

@@ -125,7 +125,7 @@ end
 #     end
 # end
 # set_first_stride(x) = x # cross fingers that this works
-# @inline onetozeroindexgephack(sptr::AbstractStridedPointer) = gesp(set_first_stride(sptr), (Static{-1}(),)) # go backwords
+# @inline onetozeroindexgephack(sptr::AbstractStridedPointer) = gesp(set_first_stride(sptr), (StaticInt{-1}(),)) # go backwords
 # @inline onetozeroindexgephack(sptr::AbstractStridedPointer{T,1}) where {T} = sptr
 # @inline onetozeroindexgephack(sptr::StridedPointer{T,1}) where {T} = sptr
 # @inline onetozeroindexgephack(x) = x
@@ -678,7 +678,7 @@ function pushgespind!(
             nostep = if fromgsp | (!index_by_index)
               GlobalRef(VectorizationBase, :NullStep)
             else
-              GlobalRef(ArrayInterface.Static, :Zero)
+              GlobalRef(LoopVectorization, :Zero)
             end
             ns = Expr(:call, nostep)
             if fromgsp

src/modeling/costs.jl

@@ -331,7 +331,7 @@ end
   ifelse(ier.f(r, y), r, y)
 end
 @inline (ier::IfElseReduced)(x::VecUnroll, y::VecUnroll) =
-  VecUnroll(fmap(ier, getfield(x, :data), getfield(y, :data)))
+  VecUnroll(VectorizationBase.fmap(ier, getfield(x, :data), getfield(y, :data)))
 @inline function (ier::IfElseReduced)(x::AbstractSIMD, y::NativeTypes)
   f = ier.f
   r = VectorizationBase.ifelse_reduce(f, x)
@@ -351,7 +351,7 @@ end
   VectorizationBase.fmap(ier, VectorizationBase.data(a), VectorizationBase.data(b)),
 )
 
-@inline (iec::IfElseCollapser)(a) = VectorizationBase.collapse(IfElseOp(iec.f), a)
+@inline (iec::IfElseCollapser)(a) = VectorizationBase.contract(IfElseOp(iec.f), a, StaticInt{1}())
 @inline (iec::IfElseCollapser)(a, ::StaticInt{C}) where {C} =
   VectorizationBase.contract(IfElseOp(iec.f), a, StaticInt{C}())
 
@@ -381,8 +381,8 @@ end
 @inline (ieo::IfElseOpMirror)(a, b) = ifelse(ieo.f(ieo.a, ieo.b), a, b)
 
 @inline _first_ifelse_reduce_mirror(f::F, a, b) where {F} =
-  getfield(VectorizationBase.ifelse_reduce_mirror(f, a, b), 1, false)
-@inline (ier::IfElseReducerMirror)(a) = _ifelse_reduce_mirror(ier.f, a, ier.a)
+  getfield(VectorizationBase.ifelse_reduce_mirror(f, a, b), 1)
+@inline (ier::IfElseReducerMirror)(a) = _first_ifelse_reduce_mirror(ier.f, a, ier.a)
 @inline function _ifelse_reduce_mirror(f::F, a, b, c, d) where {F}
   r, rm = VectorizationBase.ifelse_reduce_mirror(f, b, d)
   ifelse(f(c, rm), a, r)
@@ -416,13 +416,13 @@ end
 @inline (ier::IfElseReducedMirror)(x::AbstractSIMD{W}, y::AbstractSIMD{W}) where {W} =
   ifelse(ier.f(ier.a, ier.b), x, y)
 @inline function _reduce_mirror(f::F, x, y, a, b) where {F}
-  r, rm = IfElseReduceToMirror(f, a, b)(x, y)
+  r, _ = IfElseReducedMirror(f, a, b)(x, y)
   ifelse(f(r, y), r, y)
 end
 @inline (ier::IfElseReducedMirror)(x::AbstractSIMD, y::AbstractSIMD) =
   _reduce_mirror(ier.f, x, y, ier.a, ier.b)
 @inline (ier::IfElseReducedMirror)(x::VecUnroll, y::VecUnroll) = VecUnroll(
-  fmap(
+  VectorizationBase.fmap(
     _reduce_mirror,
     ier.f,
     getfield(x, :data),

src/modeling/graphs.jl

@@ -1202,7 +1202,6 @@ function indices_loop!(ls::LoopSet, r::Expr, itersym::Symbol)::Loop
           end
         end
         push!(ls.equalarraydims, (vptrs, mdims))
-        # push!(ls.equalarraydims, ids)
       end
     end
   end
@@ -1601,13 +1600,13 @@ function add_assignment!(ls::LoopSet, LHS, RHS, elementbytes::Int, position::Int
   end
 end
 
-function Base.push!(
+function push_op!(
   ls::LoopSet,
   ex::Expr,
   elementbytes::Int,
   position::Int,
   mpref::Union{Nothing,ArrayReferenceMetaPosition} = nothing,
-)
+)::Operation
   if ex.head === :call
     finex = first(ex.args)::Symbol
     if finex === :setindex!
@@ -1621,14 +1620,10 @@ function Base.push!(
         position,
       )
     else
-      error("Function $finex not recognized.")
+      throw(LoopError("Don't know how to handle expression.", finex))
     end
   elseif ex.head === :(=)
     add_assignment!(ls, ex.args[1], ex.args[2], elementbytes, position)
-  elseif ex.head === :block
-    add_block!(ls, ex, elementbytes, position)
-  elseif ex.head === :for
-    add_loop!(ls, ex, elementbytes)
   elseif ex.head === :&&
     add_andblock!(ls, ex, elementbytes, position)
   elseif ex.head === :||
@@ -1641,9 +1636,9 @@ function Base.push!(
     LHS = (localbody.args[1])::Symbol
     RHS_1 = localbody.args[2]
     if RHS_1 isa Symbol
-      return push!(ls, localbody, elementbytes, position, mpref)
+      return push_op!(ls, localbody, elementbytes, position, mpref)
     elseif Meta.isexpr(RHS_1, :(=), 2)
-      RHS = push!(ls, RHS_1, elementbytes, position, mpref)
+      RHS = push_op!(ls, RHS_1, elementbytes, position, mpref)
       if isstore(RHS)
         RHS
       else
@@ -1657,6 +1652,22 @@ function Base.push!(
   end
 end
 
+function Base.push!(
+  ls::LoopSet,
+  ex::Expr,
+  elementbytes::Int,
+  position::Int,
+  mpref::Union{Nothing,ArrayReferenceMetaPosition} = nothing,
+)
+  if ex.head === :block
+    add_block!(ls, ex, elementbytes, position)
+  elseif ex.head === :for
+    add_loop!(ls, ex, elementbytes)
+  else
+    push_op!(ls, ex, elementbytes, position, mpref)
+  end
+end
+
 function UnrollSpecification(
   ls::LoopSet,
   u₁loop::Symbol,

src/reconstruct_loopset.jl

@@ -99,7 +99,7 @@ function Loop(
   ::LoopSet,
   ::Expr,
   sym::Symbol,
-  ::Type{OptionallyStaticUnitRange{Static{L},Static{U}}},
+  ::Type{OptionallyStaticUnitRange{StaticInt{L},StaticInt{U}}},
 ) where {L,U}
   static_loop(sym, L, 1, U)
 end
@@ -116,7 +116,7 @@ function Loop(
   ::Expr,
   sym::Symbol,
   ::Type{CO},
-) where {L,U,CO<:AbstractCloseOpen{Static{L},Static{U}}}
+) where {L,U,CO<:AbstractCloseOpen{StaticInt{L},StaticInt{U}}}
   static_loop(sym, L, 1, U - 1)
 end
 
@@ -259,7 +259,7 @@ end
 
 
 extract_varg(i) = :(getfield(var"#vargs#", $i, false))
-# _extract(::Type{Static{N}}) where {N} = N
+# _extract(::Type{StaticInt{N}}) where {N} = N
 extract_gsp!(sptrs::Expr, name::Symbol) = (push!(sptrs.args, name); nothing)
 tupleranks(R::NTuple{8,Int}) = ntuple(n -> sum(R[n] .≥ R), Val{8}())
 function rank_to_sortperm((R, N)::Tuple{NTuple{8,Int},Int})::Vector{Int}

src/simdfunctionals/filter.jl

@@ -6,7 +6,7 @@ function vfilter!(f::F, x::Vector{T}, y::AbstractArray{T}) where {F,T<:NativeTyp
   Nrem = N & (W - 1)
   j = 0
   st = VectorizationBase.static_sizeof(T)
-  zero_index = MM(W, Static(0), st)
+  zero_index = MM(W, StaticInt(0), st)
   incr = W * VectorizationBase.static_sizeof(T)
   GC.@preserve x y begin
     # ptr_x = llvmptr(x); ptr_y = llvmptr(y)

src/simdfunctionals/map.jl

@@ -13,7 +13,7 @@ function setup_vmap!(
   ptrargs = map(VectorizationBase.zstridedpointer, args)
   V = pick_vector_width(T)
   W = unwrap(V)
-  zero_index = (MM{W}(Static(0)),)
+  zero_index = (MM{W}(StaticInt(0)),)
   uintptry = reinterpret(UInt, pointer(ptry))
   @assert iszero(uintptry & (sizeof(T) - 1)) "The destination vector (`dest`) must be aligned to `sizeof(eltype(dest)) == $(sizeof(T))` bytes."
   alignment = uintptry & (register_size() - 1)
@@ -233,18 +233,12 @@ function vmap_multithread!(
   end
   nothing
 end
-@generated function gc_preserve_vmap!(
-  f::F,
-  y::AbstractArray,
-  ::Val{NonTemporal},
-  ::Val{Threaded},
-  args::Vararg{AbstractArray,A},
-) where {F,A,NonTemporal,Threaded}
+function gc_preserve_vmap_quote(NonTemporal::Bool, Threaded::Bool, A::Int)
   m = Threaded ? :vmap_multithread! : :vmap_singlethread!
   call = Expr(:call, m, :f, :y, Expr(:call, Expr(:curly, :Val, NonTemporal)))
   q = Expr(:block, Expr(:meta, :inline))
   gcpres = Expr(:gc_preserve, call)
-  for a ∈ 1:A
+  for a ∈ 1:Int(A)::Int
     arg = Symbol(:arg_, a)
     parg = Symbol(:parg_, a)
     push!(q.args, Expr(:(=), arg, :(@inbounds args[$a])))#Expr(:ref, :args, a)))
@@ -255,6 +249,15 @@ end
   push!(q.args, gcpres, :y)
   q
 end
+@generated function gc_preserve_vmap!(
+  f::F,
+  y::AbstractArray,
+  ::Val{NonTemporal},
+  ::Val{Threaded},
+  args::Vararg{AbstractArray,A},
+) where {F,A,NonTemporal,Threaded}
+  gc_preserve_vmap_quote(NonTemporal,Threaded,A)
+end
 
 
 @inline _all_dense(t::Tuple{ArrayInterface.True}) = true