Bump SIMDPirates requirement to fix #153.

Author: chriselrod

Project.toml

@@ -1,7 +1,7 @@
 name = "LoopVectorization"
 uuid = "bdcacae8-1622-11e9-2a5c-532679323890"
 authors = ["Chris Elrod <[email protected]>"]
-version = "0.8.25"
+version = "0.8.26"
 
 [deps]
 DocStringExtensions = "ffbed154-4ef7-542d-bbb7-c09d3a79fcae"
@@ -15,7 +15,7 @@ VectorizationBase = "3d5dd08c-fd9d-11e8-17fa-ed2836048c2f"
 [compat]
 DocStringExtensions = "0.8"
 OffsetArrays = "1"
-SIMDPirates = "0.8.24"
+SIMDPirates = "0.8.25"
 SLEEFPirates = "0.5.4"
 UnPack = "0,1"
 VectorizationBase = "0.12.31"

src/graphs.jl

@@ -346,7 +346,15 @@ function LoopSet(mod::Symbol)
     )
 end
 
-cacheunrolled!(ls::LoopSet, u₁loop, u₂loop, vectorized) = foreach(op -> setunrolled!(op, u₁loop, u₂loop, vectorized), operations(ls))
+function cacheunrolled!(ls::LoopSet, u₁loop, u₂loop, vectorized)
+    foreach(op -> setunrolled!(op, u₁loop, u₂loop, vectorized), operations(ls))
+    foreach(empty! ∘ children, operations(ls))
+    for op ∈ operations(ls)
+        for opp ∈ parents(op)
+            push!(children(opp), op)
+        end
+    end    
+end
 
 num_loops(ls::LoopSet) = length(ls.loops)
 function oporder(ls::LoopSet)

src/lower_compute.jl

@@ -255,6 +255,8 @@ function lower_compute!(
         end
         if instr.instr === :identity && isone(length(parents_op))
             push!(q.args, Expr(:(=), varsym, instrcall.args[2]))
+        elseif should_broadcast_op(op)
+            push!(q.args, Expr(:(=), varsym, Expr(:call, lv(:vbroadcast), VECTORWIDTHSYMBOL, instrcall)))
         else
             push!(q.args, Expr(:(=), varsym, instrcall))
         end

src/lower_constant.jl

@@ -1,10 +1,17 @@
 
+function should_broadcast_op(op::Operation)
+    isvectorized(op) && return false
+    for opc ∈ children(op)
+        (!isvectorized(op) || accesses_memory(op)) && return false
+    end
+    true
+end
+
 # @inline onefloat(::Type{T}) where {T} = one(sizeequivalentfloat(T))
 # @inline oneinteger(::Type{T}) where {T} = one(sizeequivalentint(T))
 @inline zerofloat(::Type{T}) where {T} = zero(sizeequivalentfloat(T))
 @inline zerointeger(::Type{T}) where {T} = zero(sizeequivalentint(T))
 
-
 function lower_zero!(
     q::Expr, op::Operation, ls::LoopSet, ua::UnrollArgs, zerotyp::NumberType = zerotype(ls, op)
 )
@@ -21,7 +28,8 @@ function lower_zero!(
         pushpreamble!(ls, Expr(:(=), newtypeT, Expr(:call, lv(:sizeequivalentfloat), ELTYPESYMBOL)))
         typeT = newtypeT
     end
-    if vectorized ∈ loopdependencies(op) || vectorized ∈ reducedchildren(op) || vectorized ∈ reduceddependencies(op)
+    # TODO: make should_broadcast_op handle everything.
+    if isvectorized(op) || vectorized ∈ reducedchildren(op) || vectorized ∈ reduceddependencies(op) || should_broadcast_op(op)
         call = Expr(:call, lv(:vzero), VECTORWIDTHSYMBOL, typeT)
     else
         call = Expr(:call, :zero, typeT)
@@ -57,7 +65,7 @@ function lower_constant!(
     instruction = op.instruction
     constsym = instruction.instr
     reducedchildvectorized = vectorized ∈ reducedchildren(op)
-    if reducedchildvectorized || vectorized ∈ loopdependencies(op) || vectorized ∈ reduceddependencies(op)
+    if reducedchildvectorized || isvectorized(op) || vectorized ∈ reduceddependencies(op) || should_broadcast_op(op)
         # call = Expr(:call, lv(:vbroadcast), W, Expr(:call, lv(:maybeconvert), typeT, constsym))
         call = if reducedchildvectorized && vectorized ∉ loopdependencies(op)
             instrclass = getparentsreductzero(ls, op)

src/lowering.jl

@@ -643,7 +643,7 @@ function define_eltype_vec_width!(q::Expr, ls::LoopSet, vectorized)
     push!(q.args, Expr(:(=), ELTYPESYMBOL, determine_eltype(ls)))
     push!(q.args, Expr(:(=), VECTORWIDTHSYMBOL, determine_width(ls, vectorized)))
 end
-function setup_preamble!(ls::LoopSet, us::UnrollSpecification)
+function setup_preamble!(ls::LoopSet, us::UnrollSpecification, Ureduct::Int)
     @unpack u₁loopnum, u₂loopnum, vectorizedloopnum, u₁, u₂ = us
     order = names(ls)
     u₁loopsym = order[u₁loopnum]
@@ -653,6 +653,7 @@ function setup_preamble!(ls::LoopSet, us::UnrollSpecification)
     iszero(length(ls.includedactualarrays)) || define_eltype_vec_width!(ls.preamble, ls, vectorized)
     lower_licm_constants!(ls)
     isone(num_loops(ls)) || pushpreamble!(ls, definemask(getloop(ls, vectorized)))#, u₁ > 1 && u₁loopnum == vectorizedloopnum))
+    initialize_outer_reductions!(ls, 0, Ureduct, vectorized)
     for op ∈ operations(ls)
         (iszero(length(loopdependencies(op))) && iscompute(op)) && lower_compute!(ls.preamble, op, ls, UnrollArgs(u₁, u₁loopsym, u₂loopsym, vectorized, u₂, nothing), nothing)
     end
@@ -684,9 +685,8 @@ function lower_unrollspec(ls::LoopSet)
     # @show u₁, u₂
     order = names(ls)
     vectorized = order[vectorizedloopnum]
-    setup_preamble!(ls, us)
     Ureduct = calc_Ureduct(ls, us)
-    initialize_outer_reductions!(ls, 0, Ureduct, vectorized)
+    setup_preamble!(ls, us, Ureduct)
     initgesps = add_loop_start_stop_manager!(ls)
     q = Expr(:let, initgesps, lower_unrolled_dynamic(ls, us, num_loops(ls), false))
     q = gc_preserve( ls, Expr(:block, q) )

src/operations.jl

@@ -155,6 +155,8 @@ mutable struct Operation <: AbstractLoopOperation
     reduced_deps::Vector{Symbol}
     "Operations whose result this operation depends on"
     parents::Vector{Operation}
+    "Operations who depend on this result"
+    children::Vector{Operation}
     "For `memload` or `memstore`, encodes the array location"
     ref::ArrayReferenceMeta
     "`gensymmed` name of result."
@@ -184,7 +186,7 @@ mutable struct Operation <: AbstractLoopOperation
             identifier, variable, elementbytes, instruction, node_type,
             convert(Vector{Symbol},dependencies),
             convert(Vector{Symbol},reduced_deps),
-            convert(Vector{Operation},parents),
+            convert(Vector{Operation},parents), Operation[],
             ref, Symbol("##", variable, :_),
             reduced_children
         )
@@ -260,7 +262,7 @@ accesses_memory(op::AbstractLoopOperation) = isload(op) | isstore(op)
 elsize(op::Operation) = op.elementbytes
 dependson(op::Operation, sym::Symbol) = sym ∈ op.dependencies
 parents(op::Operation) = op.parents
-# children(op::Operation) = op.children
+children(op::Operation) = op.children
 loopdependencies(op::Operation) = op.dependencies
 reduceddependencies(op::Operation) = op.reduced_deps
 reducedchildren(op::Operation) = op.reduced_children