Merge branch 'master' of github.com:JuliaSIMD/LoopVectorization.jl

Author: chriselrod

Project.toml

@@ -1,7 +1,7 @@
 name = "LoopVectorization"
 uuid = "bdcacae8-1622-11e9-2a5c-532679323890"
 authors = ["Chris Elrod <[email protected]>"]
-version = "0.12.36"
+version = "0.12.39"
 
 [deps]
 ArrayInterface = "4fba245c-0d91-5ea0-9b3e-6abc04ee57a9"
@@ -30,5 +30,5 @@ Static = "0.2"
 StrideArraysCore = "0.1.12"
 ThreadingUtilities = "0.4.2"
 UnPack = "1"
-VectorizationBase = "0.20.16"
+VectorizationBase = "0.20.17"
 julia = "1.5"

src/broadcast.jl

@@ -464,9 +464,9 @@ end
 @inline function vmaterialize(
     bc::Broadcasted, ::Val{Mod}, ::Val{UNROLL}
 ) where {Mod,UNROLL}
-    ElType = Base.Broadcast.combine_eltypes(bc.f, bc.args)
-    dest = similar(bc, ElType)
-    vmaterialize!(dest, bc, Val{Mod}(), Val{UNROLL}())
+  ElType = Base.Broadcast.combine_eltypes(bc.f, bc.args)
+  dest = similar(bc, ElType)
+  vmaterialize!(dest, bc, Val{Mod}(), Val{UNROLL}())
 end
 
 vmaterialize!(dest, bc, ::Val, ::Val, ::StaticInt, ::StaticInt, ::StaticInt) = Base.Broadcast.materialize!(dest, bc)

src/codegen/split_loops.jl

@@ -89,39 +89,41 @@ function returned_ops(ls::LoopSet)
 end
 
 function lower_and_split_loops(ls::LoopSet, inline::Int)
-    split_candidates = returned_ops(ls)
-    length(split_candidates) > 1 || return lower(ls, inline)
-    order_fused, unrolled_fused, tiled_fused, vectorized_fused, U_fused, T_fused, cost_fused, shouldinline_fused = choose_order_cost(ls)
-    remaining_ops = Vector{Int}(undef, length(split_candidates) - 1); split_1 = Int[0];
-    # for (ind,i) ∈ enumerate(split_candidates)
-    for (ind,i) ∈ enumerate(split_candidates)
-        split_1[1] = i
-        ls_1 = split_loopset(ls, split_1)
-        order_1, unrolled_1, tiled_1, vectorized_1, U_1, T_1, cost_1, shouldinline_1 = choose_order_cost(ls_1)
-        remaining_ops[1:ind-1] .= @view(split_candidates[1:ind-1]); remaining_ops[ind:end] .= @view(split_candidates[ind+1:end])
-        ls_2 = split_loopset(ls, remaining_ops)
-        order_2, unrolled_2, tiled_2, vectorized_2, U_2, T_2, cost_2, shouldinline_2 = choose_order_cost(ls_2)
-        # U_1 = T_1 = U_2 = T_2 = 2
-        #@show cost_1 + cost_2 ≤ cost_fused, cost_1, cost_2, cost_fused
-        if cost_1 + cost_2 ≤ cost_fused
-            ls_2_lowered = if length(remaining_ops) > 1
-                inline = iszero(inline) ? (shouldinline_1 % Int) : inline
-                lower_and_split_loops(ls_2, inline)
-            else
-                doinline = inlinedecision(inline, shouldinline_1 | shouldinline_2)
-                lower(ls_2, order_2, unrolled_2, tiled_2, vectorized_2, U_2, T_2, doinline)
-            end
-            return Expr(
-                :block,
-                ls.preamble,
-                lower(ls_1, order_1, unrolled_1, tiled_1, vectorized_1, U_1, T_1, false),
-                ls_2_lowered,
-                nothing
-            )
-        end
+  split_candidates = returned_ops(ls)
+  length(split_candidates) > 1 || return lower(ls, inline)
+  order_fused, unrolled_fused, tiled_fused, vectorized_fused, U_fused, T_fused, cost_fused, shouldinline_fused = choose_order_cost(ls)
+  remaining_ops = Vector{Int}(undef, length(split_candidates) - 1); split_1 = Int[0];
+  # for (ind,i) ∈ enumerate(split_candidates)
+  for (ind,i) ∈ enumerate(split_candidates)
+    split_1[1] = i
+    ls_1 = split_loopset(ls, split_1)
+    order_1, unrolled_1, tiled_1, vectorized_1, U_1, T_1, cost_1, shouldinline_1 = choose_order_cost(ls_1)
+    remaining_ops[1:ind-1] .= @view(split_candidates[1:ind-1]); remaining_ops[ind:end] .= @view(split_candidates[ind+1:end])
+    ls_2 = split_loopset(ls, remaining_ops)
+    order_2, unrolled_2, tiled_2, vectorized_2, U_2, T_2, cost_2, shouldinline_2 = choose_order_cost(ls_2)
+    # U_1 = T_1 = U_2 = T_2 = 2
+    # return ls_1, ls_2
+    # @show cost_1 + cost_2 ≤ cost_fused, cost_1, cost_2, cost_fused
+    if cost_1 + cost_2 ≤ 0.9cost_fused
+      ls_2_lowered = if length(remaining_ops) > 1
+        inline = iszero(inline) ? (shouldinline_1 % Int) : inline
+        lower_and_split_loops(ls_2, inline)
+      else
+        doinline = inlinedecision(inline, shouldinline_1 | shouldinline_2)
+        lower(ls_2, order_2, unrolled_2, tiled_2, vectorized_2, U_2, T_2, doinline)
+      end
+      return Expr(
+        :block,
+        ls.preamble,
+        lower(ls_1, order_1, unrolled_1, tiled_1, vectorized_1, U_1, T_1, false),
+        ls_2_lowered,
+        nothing
+      )
     end
-    doinline = inlinedecision(inline, shouldinline_fused)
-    lower(ls, order_fused, unrolled_fused, tiled_fused, vectorized_fused, U_fused, T_fused, doinline)
+    length(split_candidates) == 2 && break
+  end
+  doinline = inlinedecision(inline, shouldinline_fused)
+  lower(ls, order_fused, unrolled_fused, tiled_fused, vectorized_fused, U_fused, T_fused, doinline)
 end
 
 

src/modeling/determinestrategy.jl

@@ -1,20 +1,4 @@
 
-
-# function indexappearences(op::Operation, s::Symbol)
-#     s ∉ loopdependencies(op) && return 0
-#     appearences = 0
-#     if isloopvalue(op)
-#         return s === first(loopdependencies(op)) ? 1 : 0
-#     elseif isload(op)
-#         return 100
-#     end
-#     newapp = 0
-#     for opp ∈ parents(op)
-#         newapp += indexappearences(opp, s)
-#     end
-#     factor = instruction(op).instr ∈ (:+, :vadd, :add_fast, :vadd_fast) ? 1 : 10
-#     newapp * factor
-# end
 function check_linear_parents(ls::LoopSet, op::Operation, s::Symbol)
   (s ∈ loopdependencies(op)) || return true
   if isload(op) # TODO: handle loading from ranges.
@@ -155,44 +139,44 @@ end
 function evaluate_cost_unroll(
     ls::LoopSet, order::Vector{Symbol}, vloopsym::Symbol, max_cost = typemax(Float64)
 )
-    included_vars = fill!(resize!(ls.included_vars, length(operations(ls))), false)
-    nested_loop_syms = Symbol[]#Set{Symbol}()
-    total_cost = 0.0
-    iter = 1.0
-    size_T = biggest_type_size(ls)
-    W, Wshift = lsvecwidthshift(ls, vloopsym, size_T)
-    # Need to check if fusion is possible
-    for itersym ∈ order
-        cacheunrolled!(ls, itersym, Symbol(""), vloopsym)
-        # Add to set of defined symbles
-        push!(nested_loop_syms, itersym)
-        looplength = length(ls, itersym)
-        liter = itersym === vloopsym ? num_iterations(looplength, W) : looplength
-        iter *= liter
-        # check which vars we can define at this level of loop nest
-        for (id,op) ∈ enumerate(operations(ls))
-            # won't define if already defined...
-            # id = identifier(op)
-            included_vars[id] && continue
-            # it must also be a subset of defined symbols
-            loopdependencies(op) ⊆ nested_loop_syms || continue
-          # hasintersection(reduceddependencies(op), nested_loop_syms) && return Inf
-            rd = reduceddependencies(op)
-            hasintersection(rd, @view(nested_loop_syms[1:end-length(rd)])) && return Inf
-            if isstore(op) #TODO: DRY (this is repeated in evaluate_cost_tile)
-                loadstoredeps = store_load_deps(op)
-                if !(loadstoredeps === nothing)
-                    any(s -> (s ∉ loadstoredeps), nested_loop_syms) && return Inf
-                end
-            end
-            included_vars[id] = true
-            # TODO: use actual unrolls here?
-            c = first(cost(ls, op, (Symbol(""),Symbol("")), vloopsym, Wshift, size_T))
-            total_cost += iter * c
-            total_cost > max_cost && return total_cost # abort if more expensive; we only want to know the cheapest
+  included_vars = fill!(resize!(ls.included_vars, length(operations(ls))), false)
+  nested_loop_syms = Symbol[]#Set{Symbol}()
+  total_cost = 0.0
+  iter = 1.0
+  size_T = biggest_type_size(ls)
+  W, Wshift = lsvecwidthshift(ls, vloopsym, size_T)
+  # Need to check if fusion is possible
+  for itersym ∈ order
+    cacheunrolled!(ls, itersym, Symbol(""), vloopsym)
+    # Add to set of defined symbles
+    push!(nested_loop_syms, itersym)
+    looplength = length(ls, itersym)
+    liter = itersym === vloopsym ? num_iterations(looplength, W) : looplength
+    iter *= liter
+    # check which vars we can define at this level of loop nest
+    for (id,op) ∈ enumerate(operations(ls))
+      # won't define if already defined...
+      # id = identifier(op)
+      included_vars[id] && continue
+      # it must also be a subset of defined symbols
+      loopdependencies(op) ⊆ nested_loop_syms || continue
+      # hasintersection(reduceddependencies(op), nested_loop_syms) && return Inf
+      rd = reduceddependencies(op)
+      hasintersection(rd, @view(nested_loop_syms[1:end-length(rd)])) && return Inf
+      if isstore(op) #TODO: DRY (this is repeated in evaluate_cost_tile)
+        loadstoredeps = store_load_deps(op)
+        if !(loadstoredeps === nothing)
+          any(s -> (s ∉ loadstoredeps), nested_loop_syms) && return Inf
         end
+      end
+      included_vars[id] = true
+      # TODO: use actual unrolls here?
+      c = first(cost(ls, op, (Symbol(""),Symbol("")), vloopsym, Wshift, size_T))
+      total_cost += iter * c
+      0.9total_cost > max_cost && return total_cost # abort if more expensive; we only want to know the cheapest
     end
-    0.999total_cost + stride_penalty(ls, order) # 0.999 to place finger on scale in its favor
+  end
+  0.9total_cost + stride_penalty(ls, order) # 0.999 to place finger on scale in its favor
 end
 
 # only covers vectorized ops; everything else considered lifted?
@@ -275,7 +259,7 @@ function unroll_no_reductions(ls, order, vloopsym)
   elseif iszero(load_rt)
     iszero(store_rt) ? 4 : max(1, min(4, round(Int, 2compute_rt / store_rt)))
   else
-    max(1, min(4, round(Int, 2compute_rt / load_rt)))
+    max(1, min(4, round(Int, 1.75compute_rt / load_rt)))
   end
   # u = min(u, max(1, (reg_count(ls) ÷ max(1,round(Int,rp)))))
   # commented out here is to decide to align loops
@@ -1062,7 +1046,8 @@ function evaluate_cost_tile!(
                 # println("constoffelim")
                 continue
             elseif load_elimination_cost_factor!(cost_vec, reg_pressure, choose_to_inline, ls, op, iters[id], unrollsyms, Wshift, size_T)
-                # println("loadelim")
+              # println("loadelim")
+              # A[i,j-1], A[i,j]
                 continue
             end
         #elseif isconstant(op)

src/modeling/graphs.jl

@@ -971,21 +971,20 @@ end
 # end
 # instruction(ls::LoopSet, f::Symbol) = instruction!(ls, f)
 function instruction!(ls::LoopSet, x::Expr)
-    # x isa Symbol && return x
-    if x.head === :$
-        _x = only(x.args)
-        _x isa Symbol && return instruction!(ls, _x)
-        @assert _x isa Expr
-        x = _x
-    end
+  # x isa Symbol && return x
+  if x.head === :$
+    _x = only(x.args)
+    _x isa Symbol && return instruction!(ls, _x)
+    @assert _x isa Expr
+    x = _x
+  end
+  if x.head ≢ :(->)
     instr = last(x.args).value
-    if instr ∉ keys(COST)
-        instr = gensym!(ls, "f")
-        pushpreamble!(ls, Expr(:(=), instr, x))
-        Instruction(Symbol(""), instr)
-    else
-        Instruction(:LoopVectorization, instr)
-    end
+    instr ∈ keys(COST) && return Instruction(:LoopVectorization, instr)
+  end
+  instr = gensym!(ls, "f")
+  pushpreamble!(ls, Expr(:(=), instr, x))
+  Instruction(Symbol(""), instr)
 end
 instruction!(ls::LoopSet, x::Symbol) = instruction(x)
 function instruction!(ls::LoopSet, f::F) where {F <: Function}

test/miscellaneous.jl

@@ -735,7 +735,7 @@ function findreducedparentfornonvecstoreavx!(U::AbstractMatrix{T}, E1::AbstractV
             U[i,j] = t
             _s += a * (1 - t^2)
         end
-        E1[j] = _s / n
+        E1[j] = (x -> x / n)(_s)
     end
     U,E1
 end

test/runtests.jl

@@ -14,7 +14,7 @@ const START_TIME = time()
 @time @testset "LoopVectorization.jl" begin
 
   @time if LOOPVECTORIZATION_TEST == "all" || LOOPVECTORIZATION_TEST == "part1"
-    @time Aqua.test_all(LoopVectorization)
+    @time Aqua.test_all(LoopVectorization, ambiguities = VERSION ≥ v"1.6")
     # @test isempty(detect_unbound_args(LoopVectorization))
 
     @time include("printmethods.jl")