Merge branch 'master' of github.com:chriselrod/LoopVectorization.jl

Author: chriselrod

Project.toml

@@ -1,7 +1,7 @@
 name = "LoopVectorization"
 uuid = "bdcacae8-1622-11e9-2a5c-532679323890"
 authors = ["Chris Elrod <[email protected]>"]
-version = "0.12.38"
+version = "0.12.42"
 
 [deps]
 ArrayInterface = "4fba245c-0d91-5ea0-9b3e-6abc04ee57a9"
@@ -30,5 +30,5 @@ Static = "0.2"
 StrideArraysCore = "0.1.12"
 ThreadingUtilities = "0.4.2"
 UnPack = "1"
-VectorizationBase = "0.20.16"
+VectorizationBase = "0.20.17"
 julia = "1.5"

src/LoopVectorization.jl

@@ -4,7 +4,7 @@ using Static: StaticInt, gt
 using VectorizationBase, SLEEFPirates, UnPack, OffsetArrays
 using VectorizationBase: register_size, register_count, cache_linesize, cache_size, has_opmask_registers,
   mask, pick_vector_width, MM, AbstractMask, data, grouped_strided_pointer, AbstractSIMD,
-  maybestaticlength, maybestaticsize, vzero, maybestaticrange, offsetprecalc, lazymul,
+  vzero, offsetprecalc, lazymul,
   vadd_nw, vadd_nsw, vadd_nuw, vsub_nw, vsub_nsw, vsub_nuw, vmul_nw, vmul_nsw, vmul_nuw,
     maybestaticfirst, maybestaticlast, gep, gesp, NativeTypes, #llvmptr,
     vfmadd, vfmsub, vfnmadd, vfnmsub, vfmadd_fast, vfmsub_fast, vfnmadd_fast, vfnmsub_fast, vfmadd231, vfmsub231, vfnmadd231, vfnmsub231,
@@ -23,6 +23,7 @@ using VectorizationBase: register_size, register_count, cache_linesize, cache_si
     num_threads, num_cores,
     max_mask#,zero_mask
 
+using VectorizationBase: maybestaticsize # for compatibility
 
 using IfElse: ifelse
 

src/broadcast.jl

@@ -385,7 +385,7 @@ function add_broadcast_loops!(ls::LoopSet, loopsyms::Vector{Symbol}, destsym::Sy
         push!(axes_tuple.args, Nrange)
         pushpreamble!(ls, Expr(:(=), Nlower, Expr(:call, lv(:maybestaticfirst), Nrange)))
         pushpreamble!(ls, Expr(:(=), Nupper, Expr(:call, lv(:maybestaticlast), Nrange)))
-        pushpreamble!(ls, Expr(:(=), Nlen, Expr(:call, lv(:maybestaticlength), Nrange)))
+        pushpreamble!(ls, Expr(:(=), Nlen, Expr(:call, GlobalRef(ArrayInterface,:static_length), Nrange)))
     end
 end
 # size of dest determines loops
@@ -465,9 +465,8 @@ end
     bc::Broadcasted, ::Val{Mod}, ::Val{UNROLL}
 ) where {Mod,UNROLL}
   ElType = Base.Broadcast.combine_eltypes(bc.f, bc.args)
-  @show ElType
-    dest = similar(bc, ElType)
-    vmaterialize!(dest, bc, Val{Mod}(), Val{UNROLL}())
+  dest = similar(bc, ElType)
+  vmaterialize!(dest, bc, Val{Mod}(), Val{UNROLL}())
 end
 
 vmaterialize!(dest, bc, ::Val, ::Val, ::StaticInt, ::StaticInt, ::StaticInt) = Base.Broadcast.materialize!(dest, bc)

src/codegen/operation_evaluation_order.jl

@@ -142,7 +142,7 @@ function load_short_static_reduction_first!(ls::LoopSet, u₁loop::Symbol, u₂l
 end
 
 function fillorder!(ls::LoopSet, order::Vector{Symbol}, u₁loop::Symbol, u₂loop::Symbol, u₂max::Int, vectorized::Symbol)
-  load_short_static_reduction_first!(ls, u₁loop, u₂loop, vectorized)
+    load_short_static_reduction_first!(ls, u₁loop, u₂loop, vectorized)
     lo = ls.loop_order
     resize!(lo, length(ls.loopsymbols))
     ro = lo.loopnames # reverse order; will have same order as lo

src/codegen/split_loops.jl

@@ -89,39 +89,41 @@ function returned_ops(ls::LoopSet)
 end
 
 function lower_and_split_loops(ls::LoopSet, inline::Int)
-    split_candidates = returned_ops(ls)
-    length(split_candidates) > 1 || return lower(ls, inline)
-    order_fused, unrolled_fused, tiled_fused, vectorized_fused, U_fused, T_fused, cost_fused, shouldinline_fused = choose_order_cost(ls)
-    remaining_ops = Vector{Int}(undef, length(split_candidates) - 1); split_1 = Int[0];
-    # for (ind,i) ∈ enumerate(split_candidates)
-    for (ind,i) ∈ enumerate(split_candidates)
-        split_1[1] = i
-        ls_1 = split_loopset(ls, split_1)
-        order_1, unrolled_1, tiled_1, vectorized_1, U_1, T_1, cost_1, shouldinline_1 = choose_order_cost(ls_1)
-        remaining_ops[1:ind-1] .= @view(split_candidates[1:ind-1]); remaining_ops[ind:end] .= @view(split_candidates[ind+1:end])
-        ls_2 = split_loopset(ls, remaining_ops)
-        order_2, unrolled_2, tiled_2, vectorized_2, U_2, T_2, cost_2, shouldinline_2 = choose_order_cost(ls_2)
-        # U_1 = T_1 = U_2 = T_2 = 2
-        #@show cost_1 + cost_2 ≤ cost_fused, cost_1, cost_2, cost_fused
-        if cost_1 + cost_2 ≤ cost_fused
-            ls_2_lowered = if length(remaining_ops) > 1
-                inline = iszero(inline) ? (shouldinline_1 % Int) : inline
-                lower_and_split_loops(ls_2, inline)
-            else
-                doinline = inlinedecision(inline, shouldinline_1 | shouldinline_2)
-                lower(ls_2, order_2, unrolled_2, tiled_2, vectorized_2, U_2, T_2, doinline)
-            end
-            return Expr(
-                :block,
-                ls.preamble,
-                lower(ls_1, order_1, unrolled_1, tiled_1, vectorized_1, U_1, T_1, false),
-                ls_2_lowered,
-                nothing
-            )
-        end
+  split_candidates = returned_ops(ls)
+  length(split_candidates) > 1 || return lower(ls, inline)
+  order_fused, unrolled_fused, tiled_fused, vectorized_fused, U_fused, T_fused, cost_fused, shouldinline_fused = choose_order_cost(ls)
+  remaining_ops = Vector{Int}(undef, length(split_candidates) - 1); split_1 = Int[0];
+  # for (ind,i) ∈ enumerate(split_candidates)
+  for (ind,i) ∈ enumerate(split_candidates)
+    split_1[1] = i
+    ls_1 = split_loopset(ls, split_1)
+    order_1, unrolled_1, tiled_1, vectorized_1, U_1, T_1, cost_1, shouldinline_1 = choose_order_cost(ls_1)
+    remaining_ops[1:ind-1] .= @view(split_candidates[1:ind-1]); remaining_ops[ind:end] .= @view(split_candidates[ind+1:end])
+    ls_2 = split_loopset(ls, remaining_ops)
+    order_2, unrolled_2, tiled_2, vectorized_2, U_2, T_2, cost_2, shouldinline_2 = choose_order_cost(ls_2)
+    # U_1 = T_1 = U_2 = T_2 = 2
+    # return ls_1, ls_2
+    # @show cost_1 + cost_2 ≤ cost_fused, cost_1, cost_2, cost_fused
+    if cost_1 + cost_2 ≤ 0.9cost_fused
+      ls_2_lowered = if length(remaining_ops) > 1
+        inline = iszero(inline) ? (shouldinline_1 % Int) : inline
+        lower_and_split_loops(ls_2, inline)
+      else
+        doinline = inlinedecision(inline, shouldinline_1 | shouldinline_2)
+        lower(ls_2, order_2, unrolled_2, tiled_2, vectorized_2, U_2, T_2, doinline)
+      end
+      return Expr(
+        :block,
+        ls.preamble,
+        lower(ls_1, order_1, unrolled_1, tiled_1, vectorized_1, U_1, T_1, false),
+        ls_2_lowered,
+        nothing
+      )
     end
-    doinline = inlinedecision(inline, shouldinline_fused)
-    lower(ls, order_fused, unrolled_fused, tiled_fused, vectorized_fused, U_fused, T_fused, doinline)
+    length(split_candidates) == 2 && break
+  end
+  doinline = inlinedecision(inline, shouldinline_fused)
+  lower(ls, order_fused, unrolled_fused, tiled_fused, vectorized_fused, U_fused, T_fused, doinline)
 end
 
 

src/condense_loopset.jl

@@ -75,57 +75,57 @@ It supports array-references with up to 8 indexes, where the data for each conse
 of `index_types` (storing the enum `IndexType`), `indices` (the `id` for each index symbol), and `offsets` (currently unused).
 """
 struct ArrayRefStruct{array,ptr}
-    index_types::UInt64
-    indices::UInt64
-    offsets::UInt64
-    strides::UInt64
+  index_types::UInt128
+  indices::UInt128
+  offsets::UInt128
+  strides::UInt128
 end
 array_and_ptr(@nospecialize(ar::ArrayRefStruct{a,p})) where {a,p} = (a::Symbol,p::Symbol)
 # array(@nospecialize(ar::ArrayRefStruct{a,p})) where {a,p} = a::Symbol
 # ptr(@nospecialize(ar::ArrayRefStruct{a,p})) where {a,p}   = p::Symbol
 
 function findindoradd!(v::Vector{T}, s::T) where {T}
-    ind = findfirst(==(s), v)
-    ind === nothing || return ind
-    push!(v, s)
-    length(v)
+  ind = findfirst(==(s), v)
+  ind === nothing || return ind
+  push!(v, s)
+  length(v)
 end
 function ArrayRefStruct(ls::LoopSet, mref::ArrayReferenceMeta, arraysymbolinds::Vector{Symbol}, ids::Vector{Int})
-    index_types = zero(UInt64)
-    indices = zero(UInt64)
-    offsets = zero(UInt64)
-    strides = zero(UInt64)
-    @unpack loopedindex, ref = mref
-    indv = ref.indices
-    offv = ref.offsets
-    strv = ref.strides
-    # we can discard that the array was considered discontiguous, as it should be recovered from type information
-    start = 1 + (first(indv) === DISCONTIGUOUS)
-    for (n,ind) ∈ enumerate(@view(indv[start:end]))
-        index_types <<= 8
-        indices <<= 8
-        offsets <<= 8
-        offsets |= (offv[n] % UInt8)
-        strides <<= 8
-        strides |= (strv[n] % UInt8)
-        if loopedindex[n]
-            index_types |= LoopIndex
-            if strv[n] ≠ 0
-                indices |= getloopid(ls, ind)
-            end
-        else
-            parent = get(ls.opdict, ind, nothing)
-            @assert !(parent === nothing) "Index $ind not found in array."
-            # if parent === nothing
-            #     index_types |= SymbolicIndex
-            #     indices |= findindoradd!(arraysymbolinds, ind)
-            # else
-            index_types |= ComputedIndex
-            indices |= ids[identifier(parent)]
-            # end
-        end
+  index_types = zero(UInt128)
+  indices = zero(UInt128)
+  offsets = zero(UInt128)
+  strides = zero(UInt128)
+  @unpack loopedindex, ref = mref
+  indv = ref.indices
+  offv = ref.offsets
+  strv = ref.strides
+  # we can discard that the array was considered discontiguous, as it should be recovered from type information
+  start = 1 + (first(indv) === DISCONTIGUOUS)
+  for (n,ind) ∈ enumerate(@view(indv[start:end]))
+    index_types <<= 8
+    indices <<= 8
+    offsets <<= 8
+    offsets |= (offv[n] % UInt8)
+    strides <<= 8
+    strides |= (strv[n] % UInt8)
+    if loopedindex[n]
+      index_types |= LoopIndex
+      if strv[n] ≠ 0
+        indices |= getloopid(ls, ind)
+      end
+    else
+      parent = get(ls.opdict, ind, nothing)
+      @assert !(parent === nothing) "Index $ind not found in array."
+      # if parent === nothing
+      #     index_types |= SymbolicIndex
+      #     indices |= findindoradd!(arraysymbolinds, ind)
+      # else
+      index_types |= ComputedIndex
+      indices |= ids[identifier(parent)]
+      # end
     end
-    ArrayRefStruct{mref.ref.array,mref.ptr}( index_types, indices, offsets, strides )
+  end
+  ArrayRefStruct{mref.ref.array,mref.ptr}( index_types, indices, offsets, strides )
 end
 
 """
@@ -135,62 +135,56 @@ A condensed representation of an [`Operation`](@ref).
 """
 struct OperationStruct <: AbstractLoopOperation
     # instruction::Instruction
-    loopdeps::UInt64
-    reduceddeps::UInt64
-    childdeps::UInt64
-    parents::UInt64
-    node_type::OperationType
-    array::UInt8
-    symid::UInt8
+  loopdeps::UInt128
+  reduceddeps::UInt128
+  childdeps::UInt128
+  parents::UInt128
+  node_type::OperationType
+  array::UInt8
+  symid::UInt8
 end
 optype(os) = os.node_type
 
 function findmatchingarray(ls::LoopSet, mref::ArrayReferenceMeta)
-    id = 0x01
-    for r ∈ ls.refs_aliasing_syms
-        r == mref && return id
-        id += 0x01
-    end
-    0x00
+  id = 0x01
+  for r ∈ ls.refs_aliasing_syms
+    r == mref && return id
+    id += 0x01
+  end
+  0x00
 end
-# filled_4byte_chunks(u::UInt64) = 16 - (leading_zeros(u) >>> 2)
-filled_8byte_chunks(u::UInt64) = 8 - (leading_zeros(u) >>> 3)
-
-# num_loop_deps(os::OperationStruct) = filled_4byte_chunks(os.loopdeps)
-# num_reduced_deps(os::OperationStruct) = filled_4byte_chunks(os.reduceddeps)
-# num_child_deps(os::OperationStruct) = filled_4byte_chunks(os.childdeps)
-# num_parents(os::OperationStruct) = filled_4byte_chunks(os.parents)
+filled_8byte_chunks(u::T) where {T<:Unsigned} = sizeof(T) - (leading_zeros(u) >>> 3)
 
 function shifted_loopset(ls::LoopSet, loopsyms::Vector{Symbol})
-    ld = zero(UInt64) # leading_zeros(ld) >> 2 yields the number of loopdeps
-    for d ∈ loopsyms
-        ld <<= 4
-        ld |= getloopid(ls, d)::Int
-    end
-    ld
+  ld = zero(UInt128) # leading_zeros(ld) >> 2 yields the number of loopdeps
+  for d ∈ loopsyms
+    ld <<= 4
+    ld |= getloopid(ls, d)::Int
+  end
+  ld
 end
 loopdeps_uint(ls::LoopSet, op::Operation) = shifted_loopset(ls, loopdependencies(op))
 reduceddeps_uint(ls::LoopSet, op::Operation) = shifted_loopset(ls, reduceddependencies(op))
 childdeps_uint(ls::LoopSet, op::Operation) = shifted_loopset(ls, reducedchildren(op))
 function parents_uint(ls::LoopSet, op::Operation)
-    p = zero(UInt64)
-    for parent ∈ parents(op)
-        p <<= 8
-        p |= identifier(parent)
-    end
-    p
+  p = zero(UInt128)
+  for parent ∈ parents(op)
+    p <<= 8
+    p |= identifier(parent)
+  end
+  p
 end
 function recursively_set_parents_true!(x::Vector{Bool}, op::Operation)
-    x[identifier(op)] && return nothing # don't redescend
-    x[identifier(op)] = true
-    for opp ∈ parents(op)
-        recursively_set_parents_true!(x, opp)
-    end
-    return nothing
+  x[identifier(op)] && return nothing # don't redescend
+  x[identifier(op)] = true
+  for opp ∈ parents(op)
+    recursively_set_parents_true!(x, opp)
+  end
+  return nothing
 end
 function getroots(ls::LoopSet)::Vector{Bool}
-    rooted = Vector{Bool}(undef, length(operations(ls)))
-    getroots!(rooted, ls)
+  rooted = Vector{Bool}(undef, length(operations(ls)))
+  getroots!(rooted, ls)
 end
 function getroots!(rooted::Vector{Bool}, ls::LoopSet)
   fill!(rooted, false)