merge master

Author: chriselrod

Project.toml

@@ -1,7 +1,7 @@
 name = "LoopVectorization"
 uuid = "bdcacae8-1622-11e9-2a5c-532679323890"
 authors = ["Chris Elrod <[email protected]>"]
-version = "0.12.38"
+version = "0.12.39"
 
 [deps]
 ArrayInterface = "4fba245c-0d91-5ea0-9b3e-6abc04ee57a9"
@@ -30,5 +30,5 @@ Static = "0.2"
 StrideArraysCore = "0.1.12"
 ThreadingUtilities = "0.4.2"
 UnPack = "1"
-VectorizationBase = "0.20.16"
+VectorizationBase = "0.20.17"
 julia = "1.5"

src/LoopVectorization.jl

@@ -4,7 +4,7 @@ using Static: StaticInt, gt
 using VectorizationBase, SLEEFPirates, UnPack, OffsetArrays
 using VectorizationBase: register_size, register_count, cache_linesize, cache_size, has_opmask_registers,
   mask, pick_vector_width, MM, AbstractMask, data, grouped_strided_pointer, AbstractSIMD,
-  maybestaticlength, maybestaticsize, vzero, maybestaticrange, offsetprecalc, lazymul,
+  vzero, offsetprecalc, lazymul,
   vadd_nw, vadd_nsw, vadd_nuw, vsub_nw, vsub_nsw, vsub_nuw, vmul_nw, vmul_nsw, vmul_nuw,
     maybestaticfirst, maybestaticlast, gep, gesp, NativeTypes, #llvmptr,
     vfmadd, vfmsub, vfnmadd, vfnmsub, vfmadd_fast, vfmsub_fast, vfnmadd_fast, vfnmsub_fast, vfmadd231, vfmsub231, vfnmadd231, vfnmsub231,

src/broadcast.jl

@@ -385,7 +385,7 @@ function add_broadcast_loops!(ls::LoopSet, loopsyms::Vector{Symbol}, destsym::Sy
         push!(axes_tuple.args, Nrange)
         pushpreamble!(ls, Expr(:(=), Nlower, Expr(:call, lv(:maybestaticfirst), Nrange)))
         pushpreamble!(ls, Expr(:(=), Nupper, Expr(:call, lv(:maybestaticlast), Nrange)))
-        pushpreamble!(ls, Expr(:(=), Nlen, Expr(:call, lv(:maybestaticlength), Nrange)))
+        pushpreamble!(ls, Expr(:(=), Nlen, Expr(:call, GlobalRef(ArrayInterface,:static_length), Nrange)))
     end
 end
 # size of dest determines loops

src/codegen/split_loops.jl

@@ -89,38 +89,41 @@ function returned_ops(ls::LoopSet)
 end
 
 function lower_and_split_loops(ls::LoopSet, inline::Int)
-    split_candidates = returned_ops(ls)
-    length(split_candidates) > 1 || return lower(ls, inline)
-    order_fused, unrolled_fused, tiled_fused, vectorized_fused, U_fused, T_fused, cost_fused, shouldinline_fused = choose_order_cost(ls)
-    remaining_ops = Vector{Int}(undef, length(split_candidates) - 1); split_1 = Int[0];
-    # for (ind,i) ∈ enumerate(split_candidates)
-    for (ind,i) ∈ enumerate(split_candidates)
-        split_1[1] = i
-        ls_1 = split_loopset(ls, split_1)
-        order_1, unrolled_1, tiled_1, vectorized_1, U_1, T_1, cost_1, shouldinline_1 = choose_order_cost(ls_1)
-        remaining_ops[1:ind-1] .= @view(split_candidates[1:ind-1]); remaining_ops[ind:end] .= @view(split_candidates[ind+1:end])
-        ls_2 = split_loopset(ls, remaining_ops)
-        order_2, unrolled_2, tiled_2, vectorized_2, U_2, T_2, cost_2, shouldinline_2 = choose_order_cost(ls_2)
-        # U_1 = T_1 = U_2 = T_2 = 2
-        if cost_1 + cost_2 ≤ cost_fused
-            ls_2_lowered = if length(remaining_ops) > 1
-                inline = iszero(inline) ? (shouldinline_1 % Int) : inline
-                lower_and_split_loops(ls_2, inline)
-            else
-                doinline = inlinedecision(inline, shouldinline_1 | shouldinline_2)
-                lower(ls_2, order_2, unrolled_2, tiled_2, vectorized_2, U_2, T_2, doinline)
-            end
-            return Expr(
-                :block,
-                ls.preamble,
-                lower(ls_1, order_1, unrolled_1, tiled_1, vectorized_1, U_1, T_1, false),
-                ls_2_lowered,
-                nothing
-            )
-        end
+  split_candidates = returned_ops(ls)
+  length(split_candidates) > 1 || return lower(ls, inline)
+  order_fused, unrolled_fused, tiled_fused, vectorized_fused, U_fused, T_fused, cost_fused, shouldinline_fused = choose_order_cost(ls)
+  remaining_ops = Vector{Int}(undef, length(split_candidates) - 1); split_1 = Int[0];
+  # for (ind,i) ∈ enumerate(split_candidates)
+  for (ind,i) ∈ enumerate(split_candidates)
+    split_1[1] = i
+    ls_1 = split_loopset(ls, split_1)
+    order_1, unrolled_1, tiled_1, vectorized_1, U_1, T_1, cost_1, shouldinline_1 = choose_order_cost(ls_1)
+    remaining_ops[1:ind-1] .= @view(split_candidates[1:ind-1]); remaining_ops[ind:end] .= @view(split_candidates[ind+1:end])
+    ls_2 = split_loopset(ls, remaining_ops)
+    order_2, unrolled_2, tiled_2, vectorized_2, U_2, T_2, cost_2, shouldinline_2 = choose_order_cost(ls_2)
+    # U_1 = T_1 = U_2 = T_2 = 2
+    # return ls_1, ls_2
+    # @show cost_1 + cost_2 ≤ cost_fused, cost_1, cost_2, cost_fused
+    if cost_1 + cost_2 ≤ 0.9cost_fused
+      ls_2_lowered = if length(remaining_ops) > 1
+        inline = iszero(inline) ? (shouldinline_1 % Int) : inline
+        lower_and_split_loops(ls_2, inline)
+      else
+        doinline = inlinedecision(inline, shouldinline_1 | shouldinline_2)
+        lower(ls_2, order_2, unrolled_2, tiled_2, vectorized_2, U_2, T_2, doinline)
+      end
+      return Expr(
+        :block,
+        ls.preamble,
+        lower(ls_1, order_1, unrolled_1, tiled_1, vectorized_1, U_1, T_1, false),
+        ls_2_lowered,
+        nothing
+      )
     end
-    doinline = inlinedecision(inline, shouldinline_fused)
-    lower(ls, order_fused, unrolled_fused, tiled_fused, vectorized_fused, U_fused, T_fused, doinline)
+    length(split_candidates) == 2 && break
+  end
+  doinline = inlinedecision(inline, shouldinline_fused)
+  lower(ls, order_fused, unrolled_fused, tiled_fused, vectorized_fused, U_fused, T_fused, doinline)
 end
 
 

src/modeling/graphs.jl

@@ -756,21 +756,21 @@ function add_block!(ls::LoopSet, ex::Expr, elementbytes::Int, position::Int)
     end
 end
 function maybestatic!(expr::Expr)
-    if expr.head === :call
-        f = first(expr.args)
-        if f === :length
-            expr.args[1] = lv(:maybestaticlength)
-        elseif f === :size && length(expr.args) == 3
-            i = expr.args[3]
-            if i isa Integer
-                expr.args[1] = lv(:maybestaticsize)
-                expr.args[3] = Expr(:call, Expr(:curly, :Val, convert(Int, i)))
-            end
-        else
-            static_literals!(expr)
-        end
+  if expr.head === :call
+    f = first(expr.args)
+    if f === :length
+      expr.args[1] = GlobalRef(ArrayInterface,:static_length)
+    elseif f === :size && length(expr.args) == 3
+      i = expr.args[3]
+      if i isa Integer
+        expr.args[1] = GlobalRef(ArrayInterface,:size)
+        expr.args[3] = staticexpr(convert(Int,i)::Int)
+      end
+    else
+      static_literals!(expr)
     end
-    expr
+  end
+  expr
 end
 add_loop_bound!(ls::LoopSet, itersym::Symbol, bound::Union{Integer,Symbol}, upper::Bool, step::Bool)::MaybeKnown = MaybeKnown(bound, upper ? 1024 : 1)
 function add_loop_bound!(ls::LoopSet, itersym::Symbol, bound::Expr, upper::Bool, step::Bool)::MaybeKnown
@@ -797,7 +797,7 @@ function range_loop!(ls::LoopSet, itersym::Symbol, l::MaybeKnown, u::MaybeKnown,
     isone(s) || pushexpr!(range, s)
     pushexpr!(range, u)
     pushprepreamble!(ls, Expr(:(=), rangename, range))
-    pushprepreamble!(ls, Expr(:(=), lenname, Expr(:call, lv(:maybestaticlength), rangename)))
+    pushprepreamble!(ls, Expr(:(=), lenname, Expr(:call, GlobalRef(ArrayInterface,:static_length), rangename)))
     Loop(itersym, l, u, s, rangename, lenname)
 end
 function range_loop!(ls::LoopSet, r::Expr, itersym::Symbol)::Loop
@@ -853,7 +853,7 @@ end
 function misc_loop!(ls::LoopSet, r::Union{Expr,Symbol}, itersym::Symbol, staticstepone::Bool)::Loop
     rangename = gensym!(ls, "looprange" * string(itersym)); lenname = gensym!(ls, "looplen" * string(itersym));
     pushprepreamble!(ls, Expr(:(=), rangename, Expr(:call, lv(:canonicalize_range), :(@inbounds $(static_literals!(r))))))
-    pushprepreamble!(ls, Expr(:(=), lenname, Expr(:call, lv(:maybestaticlength), rangename)))
+    pushprepreamble!(ls, Expr(:(=), lenname, Expr(:call, GlobalRef(ArrayInterface,:static_length), rangename)))
     L = add_loop_bound!(ls, itersym, Expr(:call, lv(:maybestaticfirst), rangename), false, false)
     U = add_loop_bound!(ls, itersym, Expr(:call, lv(:maybestaticlast), rangename), true, false)
     if staticstepone

src/reconstruct_loopset.jl

@@ -30,7 +30,7 @@ function Loop(ls::LoopSet, ex::Expr, sym::Symbol, f, s, l, ub::Int)
     rangesym = gensym(ssym * "_loop");
     lensym = gensym(ssym * "_looplen")
     pushpreamble!(ls, Expr(:(=), rangesym, ex))
-    pushpreamble!(ls, Expr(:(=), lensym, Expr(:call, lv(:maybestaticlength), rangesym)))
+    pushpreamble!(ls, Expr(:(=), lensym, Expr(:call, GlobalRef(ArrayInterface,:static_length), rangesym)))
     F = if f === nothing
         start = gensym(ssym*"_loopstart")
         pushpreamble!(ls, Expr(:(=), start, Expr(:call, %, Expr(:call, lv(:first), rangesym), Int)))

test/gemv.jl

@@ -87,7 +87,7 @@ using Test
     δ = 0
     z = zero(eltype(G))
     @turbo for d1=1:d
-      G[d1,κ] = z
+      G[d1,κ - δ] = z
       for d2=1:d
         G[d1, κ - δ] += B[d2, d1] * B[d2, κ + δ]
       end