Bug fixes

Author: chriselrod

Project.toml

@@ -26,7 +26,7 @@ OffsetArrays = "1.4.1, 1.5"
 Requires = "1"
 SLEEFPirates = "0.6.7"
 Static = "0.2"
-ThreadingUtilities = "0.2.3"
+ThreadingUtilities = "0.3"
 UnPack = "1"
 VectorizationBase = "0.19"
 julia = "1.5"

src/broadcast.jl

@@ -156,7 +156,7 @@ end
 struct LowDimArray{D,T,N,A<:DenseArray{T,N}} <: DenseArray{T,N}
     data::A
 end
-Base.@propagate_inbounds Base.getindex(A::LowDimArray, i...) = getindex(A.data, i...)
+Base.@propagate_inbounds Base.getindex(A::LowDimArray, i::Vararg{Union{Integer,CartesianIndex},K}) where {K} = getindex(A.data, i...)
 @inline Base.size(A::LowDimArray) = Base.size(A.data)
 @inline Base.size(A::LowDimArray, i) = Base.size(A.data, i)
 @inline Base.strides(A::LowDimArray) = strides(A.data)
@@ -229,7 +229,10 @@ function LowDimArray{D}(data::A) where {D,T,N,A <: AbstractArray{T,N}}
 end
 function extract_all_1_array!(ls::LoopSet, bcname::Symbol, N::Int, elementbytes::Int)
     refextract = gensym!(ls, bcname)
-    ref = Expr(:ref, bcname); foreach(_ -> push!(ref.args, :begin), 1:N)
+    ref = Expr(:ref, bcname);
+    for _ ∈ 1:N
+        push!(ref.args, :begin)
+    end
     pushprepreamble!(ls, Expr(:(=), refextract, ref))
     return add_constant!(ls, refextract, elementbytes) # or replace elementbytes with sizeof(T) ? u
 end
@@ -355,15 +358,15 @@ end
 # size of dest determines loops
 # function vmaterialize!(
 @generated function vmaterialize!(
-    dest::AbstractArray{T,N}, bc::BC,
-    ::Val{Mod}, ::Val{UNROLL}, ::StaticInt{RS}, ::StaticInt{RC}, ::StaticInt{CLS}
-) where {T <: NativeTypes, N, BC <: Union{Broadcasted,Product}, Mod, UNROLL, RS, RC, CLS}
+    dest::AbstractArray{T,N}, bc::BC, ::Val{Mod}, ::Val{UNROLL}
+) where {T <: NativeTypes, N, BC <: Union{Broadcasted,Product}, Mod, UNROLL}
     # 2+1
     # we have an N dimensional loop.
     # need to construct the LoopSet
     # @show typeof(dest)
     ls = LoopSet(Mod)
-    set_hw!(ls, RS, RC, CLS)
+    inline, u₁, u₂, W, rs, rc, cls, l1, l2, l3, threads = UNROLL
+    set_hw!(ls, rs, rc, cls, l1, l2, l3)
     loopsyms = [gensym!(ls, "n") for n ∈ 1:N]
     ls.isbroadcast[] = true
     add_broadcast_loops!(ls, loopsyms, :dest)
@@ -372,8 +375,9 @@ end
     add_simple_store!(ls, :dest, ArrayReference(:dest, loopsyms), elementbytes)
     resize!(ls.loop_order, num_loops(ls)) # num_loops may be greater than N, eg Product
     # return ls
-    inline, u₁, u₂, threads = UNROLL
-    q = lower(ls, u₁ % Int, u₂ % Int, inline % Int)
+    # @show u₁, u₂, inline
+    q = iszero(u₁) ? lower_and_split_loops(ls, inline % Int) : lower(ls, u₁ % Int, u₂ % Int, inline % Int)
+    # q = lower(ls, u₁ % Int, u₂ % Int, inline % Int)
     push!(q.args, :dest)
     # @show q
     # q
@@ -388,13 +392,13 @@ end
      # ls
 end
 @generated function vmaterialize!(
-    dest′::Union{Adjoint{T,A},Transpose{T,A}}, bc::BC,
-    ::Val{Mod}, ::Val{UNROLL}, ::StaticInt{RS}, ::StaticInt{RC}, ::StaticInt{CLS}
-) where {T <: NativeTypes, N, A <: AbstractArray{T,N}, BC <: Union{Broadcasted,Product}, Mod, UNROLL, RS, RC, CLS}
+    dest′::Union{Adjoint{T,A},Transpose{T,A}}, bc::BC, ::Val{Mod}, ::Val{UNROLL}
+) where {T <: NativeTypes, N, A <: AbstractArray{T,N}, BC <: Union{Broadcasted,Product}, Mod, UNROLL}
     # we have an N dimensional loop.
     # need to construct the LoopSet
     ls = LoopSet(Mod)
-    set_hw!(ls, RS, RC, CLS)
+    inline, u₁, u₂, W, rs, rc, cls, l1, l2, l3, threads = UNROLL
+    set_hw!(ls, rs, rc, cls, l1, l2, l3)
     loopsyms = [gensym!(ls, "n") for n ∈ 1:N]
     ls.isbroadcast[] = true
     pushprepreamble!(ls, Expr(:(=), :dest, Expr(:call, :parent, :dest′)))
@@ -403,8 +407,8 @@ end
     add_broadcast!(ls, :dest, :bc, loopsyms, BC, elementbytes)
     add_simple_store!(ls, :dest, ArrayReference(:dest, reverse(loopsyms)), elementbytes)
     resize!(ls.loop_order, num_loops(ls)) # num_loops may be greater than N, eg Product
-    inline, u₁, u₂, threads = UNROLL
-    q = lower(ls, u₁ % Int, u₂ % Int, inline % Int)
+    q = iszero(u₁) ? lower_and_split_loops(ls, inline % Int) : lower(ls, u₁ % Int, u₂ % Int, inline % Int)
+    # q = lower(ls, u₁ % Int, u₂ % Int, inline % Int)
     push!(q.args, :dest′)
     q = Expr(
         :block,
@@ -417,10 +421,9 @@ end
 end
 # these are marked `@inline` so the `@avx` itself can choose whether or not to inline.
 @generated function vmaterialize!(
-    dest::AbstractArray{T,N}, bc::Broadcasted{Base.Broadcast.DefaultArrayStyle{0},Nothing,typeof(identity),Tuple{T2}},
-    ::Val{Mod}, ::Val{UNROLL}, ::StaticInt{RS}, ::StaticInt{RC}, ::StaticInt{CLS}
-) where {T <: NativeTypes, N, T2 <: Number, Mod, UNROLL,RS,RC,CLS}
-    inline, u₁, u₂, threads = UNROLL
+    dest::AbstractArray{T,N}, bc::Broadcasted{Base.Broadcast.DefaultArrayStyle{0},Nothing,typeof(identity),Tuple{T2}}, ::Val{Mod}, ::Val{UNROLL}
+) where {T <: NativeTypes, N, T2 <: Number, Mod, UNROLL}
+    inline, u₁, u₂, W, rs, rc, cls, l1, l2, l3, threads = UNROLL
     quote
         $(Expr(:meta,:inline))
         arg = T(first(bc.args))
@@ -431,10 +434,9 @@ end
     end
 end
 @generated function vmaterialize!(
-    dest′::Union{Adjoint{T,A},Transpose{T,A}}, bc::Broadcasted{Base.Broadcast.DefaultArrayStyle{0},Nothing,typeof(identity),Tuple{T2}},
-    ::Val{Mod}, ::Val{UNROLL}, ::StaticInt{RS}, ::StaticInt{RC}, ::StaticInt{CLS}
-) where {T <: NativeTypes, N, A <: AbstractArray{T,N}, T2 <: Number, Mod, UNROLL,RS,RC,CLS}
-    inline, u₁, u₂, threads = UNROLL
+    dest′::Union{Adjoint{T,A},Transpose{T,A}}, bc::Broadcasted{Base.Broadcast.DefaultArrayStyle{0},Nothing,typeof(identity),Tuple{T2}}, ::Val{Mod}, ::Val{UNROLL}
+) where {T <: NativeTypes, N, A <: AbstractArray{T,N}, T2 <: Number, Mod, UNROLL}
+    inline, u₁, u₂, W, rs, rc, cls, l1, l2, l3, threads = UNROLL
     quote
         $(Expr(:meta,:inline))
         arg = T(first(bc.args))
@@ -447,10 +449,10 @@ end
 end
 
 @inline function vmaterialize(
-    bc::Broadcasted, ::Val{Mod}, ::Val{UNROLL}, ::StaticInt{RS}, ::StaticInt{RC}, ::StaticInt{CLS}
-) where {Mod,UNROLL,RS,RC,CLS}
+    bc::Broadcasted, ::Val{Mod}, ::Val{UNROLL}
+) where {Mod,UNROLL}
     ElType = Base.Broadcast.combine_eltypes(bc.f, bc.args)
-    vmaterialize!(similar(bc, ElType), bc, Val{Mod}(), StaticInt{UNROLL}(), StaticInt{RS}(), StaticInt{RC}(), StaticInt{CLS}())
+    vmaterialize!(similar(bc, ElType), bc, Val{Mod}(), Val{UNROLL}())
 end
 
 vmaterialize!(dest, bc, ::Val, ::Val, ::StaticInt, ::StaticInt, ::StaticInt) = Base.Broadcast.materialize!(dest, bc)

src/codegen/loopstartstopmanager.jl

@@ -114,14 +114,14 @@ function use_loop_induct_var!(ls::LoopSet, q::Expr, ar::ArrayReferenceMeta, alla
              # ((ls.align_loops[] > 0) && (first(names(ls)) == ind))
 
             # Not doing normal offset indexing
-            uliv[i] = -findfirst(isequal(ind), looporder)::Int
+            uliv[i] = -findfirst(Base.Fix2(===,ind), looporder)::Int
             # push!(gespinds.args, Expr(:call, lv(:Zero)))
             # push!(gespinds.args, staticexpr(1))
             push!(gespinds.args, staticexpr(convert(Int, strides[i])))
 
             push!(offsetprecalc_descript.args, 0) # not doing offset indexing, so push 0
         else
-            uliv[i] = findfirst(isequal(ind), looporder)::Int
+            uliv[i] = findfirst(Base.Fix2(===,ind), looporder)::Int
             loop = getloop(ls, ind)
             if isknown(first(loop))
                 push!(gespinds.args, staticexpr(gethint(first(loop))))

src/codegen/lower_compute.jl

@@ -303,8 +303,7 @@ function getu₁forreduct(ls::LoopSet, op::Operation, u₁::Int)
     # if it is, then op's `u₁` will be the current `u₁`
     # if it is not, then the opp is initialized once per full u₁
     while true
-        opname = name(op)
-        selfparentid = findfirst(opp -> name(opp) === opname, parents(op))
+        selfparentid = findfirst(Base.Fix2(===,name(op)) ∘ name, parents(op))
         selfparentid === nothing && return u₁
         op = parents(op)[selfparentid]
         isreduction(op) || break
@@ -383,7 +382,7 @@ function lower_compute!(
     if Base.libllvm_version < v"11.0.0" && (suffix ≠ -1) && isreduct# && (iszero(suffix) || (ls.unrollspecification[].u₂ - 1 == suffix))
     # if (length(reduceddependencies(op)) > 0) | (length(reducedchildren(op)) > 0)# && (iszero(suffix) || (ls.unrollspecification[].u₂ - 1 == suffix))
         # instrfid = findfirst(isequal(instr.instr), (:vfmadd, :vfnmadd, :vfmsub, :vfnmsub))
-        instrfid = findfirst(isequal(instr.instr), (:vfmadd_fast, :vfnmadd_fast, :vfmsub_fast, :vfnmsub_fast))
+        instrfid = findfirst(Base.Fix2(===,instr.instr), (:vfmadd_fast, :vfnmadd_fast, :vfmsub_fast, :vfnmsub_fast))
         # instrfid = findfirst(isequal(instr.instr), (:vfnmadd_fast, :vfmsub_fast, :vfnmsub_fast))
         # @show isreduct, instrfid, instr.instr sub_fmas(ls, op, ua)
         # want to instcombine when parent load's deps are superset

src/codegen/lower_load.jl

@@ -117,7 +117,6 @@ function lower_load_no_optranslation!(
     u = ifelse(opu₁, u₁, 1)
     mvar = Symbol(variable_name(op, Core.ifelse(isu₂unrolled(op), suffix,-1)), '_', u)
     falseexpr = Expr(:call, lv(:False)); rs = staticexpr(reg_size(ls))
-
     if all(op.ref.loopedindex)
         inds = unrolledindex(op, td, mask, inds_calc_by_ptr_offset)
         loadexpr = Expr(:call, lv(:_vload), vptr(op), inds)
@@ -207,39 +206,56 @@ function lower_load_for_optranslation!(
     step₂ = gethint(step(u₂loop))
 
     # abs of steps are equal
-    #
     equal_steps = (step₁ == step₂) ⊻ (posindicator ≠ 0x03)
-    _td = UnrollArgs(u₁loop, u₂loop, vloop, total_unroll, u₂max, Core.ifelse(equal_steps, 0, u₂max - 1))
+    # @show step₁, step₂, posindicator, equal_steps
+    # _td = UnrollArgs(u₁loop, u₂loop, vloop, total_unroll, u₂max, Core.ifelse(equal_steps, 0, u₂max - 1))
+    _td = UnrollArgs(u₁loop, u₂loop, vloop, u₁, u₂max, Core.ifelse(equal_steps, 0, u₂max - 1))
     gespinds = mem_offset(op, _td, inds_by_ptroff, false)
     ptr = vptr(op)
     gptr = Symbol(ptr, "##GESPED##")
     for i ∈ eachindex(gespinds.args)
         if i == translationind
             gespinds.args[i] = Expr(:call, lv(Core.ifelse(equal_steps, :firstunroll, :lastunroll)), gespinds.args[i])
-        else
-            gespinds.args[i] = Expr(:call, lv(:data), gespinds.args[i])
+        # else
+        #     gespinds.args[i] = Expr(:call, lv(:data), gespinds.args[i])
         end
     end
     push!(q.args, Expr(:(=), gptr, Expr(:call, lv(:gesp), ptr, gespinds)))
 
     fill!(inds_by_ptroff, true)
 
     @unpack ref, loopedindex = mref
-    indices = getindicesonly(ref)
-    old_translation_index = indices[translationind]
-    indices[translationind] = u₁loop.itersymbol
+    indices = copy(getindices(ref))
+    # old_translation_index = indices[translationind]
+    # indices[translationind] = u₁loop.itersymbol
+    # @show indices, translationind, vloop
     # getindicesonly returns a view of `getindices`
-    dummyref = ArrayReference(ref.array, getindices(ref), zero(getoffsets(ref)), getstrides(ref))
-    loopedindex[translationind] = true
-    dummymref = ArrayReferenceMeta(dummyref, loopedindex, gptr)
-
+    dummyref = ArrayReference(ref.array, indices, zero(getoffsets(ref)), getstrides(ref))
+    # loopedindex[translationind] = true
+    dummymref = ArrayReferenceMeta(dummyref, fill!(similar(loopedindex), true), gptr)
+    indonly = getindicesonly(dummyref)
+    for i ∈ eachindex(indonly)
+        if i == translationind
+            indonly[i] = u₁loop.itersymbol
+        elseif !loopedindex[i]
+            ind = indonly[i]
+            for indop ∈ operations(ls)
+                if isvectorized(indop) & (name(indop) === ind)
+                    indonly[i] = vloop.itersymbol
+                    break
+                end
+            end
+        end
+    end
+    # @show indices
     _td = UnrollArgs(u₁loop, u₂loop, vloop, total_unroll, u₂max, -1)
     op.ref = dummymref
+    # @show isu₁unrolled(op), isu₂unrolled(op)
     _lower_load!(q, ls, op, _td, mask)
     # set old values
     op.ref = mref
-    loopedindex[translationind] = false
-    indices[translationind] = old_translation_index
+    # loopedindex[translationind] = false
+    # indices[translationind] = old_translation_index
 
     shouldbroadcast = (!isvectorized(op)) && any(isvectorized, children(op))
 

src/codegen/lower_memory_common.jl

@@ -145,7 +145,7 @@ function mem_offset(op::Operation, td::UnrollArgs, inds_calc_by_ptr_offset::Vect
             # addoffset!(ret, newname, stride, offset, _mmi)
             _mmi = indvectorized && parent !== op && (!isvectorized(parent))
             @assert !_mmi "Please file an issue with an example of how you got this."
-            addoffset!(ret, newname, 1, offset, false)
+            addoffset!(ret, 0, newname, offset, false)
         end
     end
     ret

src/codegen/lowering.jl

@@ -184,7 +184,9 @@ function lower_no_unroll(ls::LoopSet, us::UnrollSpecification, n::Int, inclmask:
     # elseif nisvectorized
     if loopisstatic && (isone(length(loop) ÷ W) || (n ≤ 3 && length(loop) ≤ 8W && allinteriorunrolled(ls, us, n)))
         q = Expr(:block)
-        foreach(_ -> push!(q.args, body), 1:(length(loop) ÷ W))
+        for _ ∈ 1:(length(loop) ÷ W)
+            push!(q.args, body)
+        end
     elseif nisvectorized
             # Expr(:block, loopiteratesatleastonce(loop, true), Expr(:while, expect(tc), body))
         # q = Expr(:block, Expr(maxiters == 1 ? :if : :while, tc, body))
@@ -253,7 +255,9 @@ function lower_unrolled_dynamic(ls::LoopSet, us::UnrollSpecification, n::Int, in
         iters = length(loop) ÷ UFW
         if (iters ≤ 1) || (iters*UF ≤ 16 && allinteriorunrolled(ls, us, n))# Let's set a limit on total unrolling
             q = Expr(:block)
-            foreach(_ -> push!(q.args, body), 1:iters)
+            for _ ∈ 1:iters
+                push!(q.args, body)
+            end
         else
             q = Expr(:while, tc, body)
         end

src/codegen/split_loops.jl

@@ -69,7 +69,8 @@ function split_loopset(ls::LoopSet, ids)
     # it shouldn't.
     # Current behavior is incorrect when VECWIDTH chosen does actually differ between
     # split loops and the loops are statically sized, because code gen will then assume it is correct...
-    set_hw!(ls_new, reg_size(ls), reg_count(ls), cache_lnsze(ls))
+    l1, l2, l3 = cache_sze(ls)
+    set_hw!(ls_new, reg_size(ls), reg_count(ls), cache_lnsze(ls), l1, l2, l3)
     ls_new.vector_width[] = ls.vector_width[]
     fill_offset_memop_collection!(ls)
     ls_new

src/condense_loopset.jl

@@ -21,7 +21,7 @@ array(ar::ArrayRefStruct{a,p}) where {a,p} = a
 ptr(ar::ArrayRefStruct{a,p}) where {a,p}   = p
 
 function findindoradd!(v::Vector{T}, s::T) where {T}
-    ind = findfirst(sᵢ -> sᵢ == s, v)
+    ind = findfirst(==(s), v)
     ind === nothing || return ind
     push!(v, s)
     length(v)
@@ -144,7 +144,9 @@ end
 
 function loop_boundaries(ls::LoopSet)
     lbd = Expr(:tuple)
-    foreach(loop -> loop_boundary!(lbd, loop), ls.loops)
+    for loop ∈ ls.loops
+        loop_boundary!(lbd, loop)
+    end
     lbd
 end
 
@@ -225,7 +227,6 @@ end
 val(x) = Expr(:call, Expr(:curly, :Val, x))
 
 function add_grouped_strided_pointer!(extra_args::Expr, ls::LoopSet)
-    # foreach(ref -> push!(extra_args.args, vptr(ref)), ls.refs_aliasing_syms)
     gsp = Expr(:call, lv(:grouped_strided_pointer))
     tgarrays = Expr(:tuple)
     i = 0
@@ -247,7 +248,7 @@ function add_grouped_strided_pointer!(extra_args::Expr, ls::LoopSet)
     for (vptrs,dims) ∈ ls.equalarraydims
         t = Expr(:tuple)
         for (vp,d) ∈ zip(vptrs,dims)
-            _id = findfirst(r -> vptr(r) === vp, ls.refs_aliasing_syms)
+            _id = findfirst(Base.Fix2(===,vp) ∘ vptr, ls.refs_aliasing_syms)
             _id === nothing && continue
             push!(t.args, Expr(:tuple, _id, d))
         end
@@ -258,13 +259,13 @@ function add_grouped_strided_pointer!(extra_args::Expr, ls::LoopSet)
     nothing
 end
 
-first_cache() = ifelse(gt(num_cache_levels(), StaticInt{2}()), StaticInt{2}(), StaticInt{1}())
-function _first_cache_size(::StaticInt{FCS}) where {FCS}
-    L1inclusive = StaticInt{FCS}() - VectorizationBase.cache_size(One())
-    ifelse(eq(first_cache(), StaticInt(2)) & VectorizationBase.cache_inclusive(StaticInt(2)), L1inclusive, StaticInt{FCS}())
-end
-_first_cache_size(::Nothing) = StaticInt(262144)
-first_cache_size() = _first_cache_size(cache_size(first_cache()))
+# first_cache() = ifelse(gt(num_cache_levels(), StaticInt{2}()), StaticInt{2}(), StaticInt{1}())
+# function _first_cache_size(::StaticInt{FCS}) where {FCS}
+#     L1inclusive = StaticInt{FCS}() - VectorizationBase.cache_size(One())
+#     ifelse(eq(first_cache(), StaticInt(2)) & VectorizationBase.cache_inclusive(StaticInt(2)), L1inclusive, StaticInt{FCS}())
+# end
+# _first_cache_size(::Nothing) = StaticInt(262144)
+# first_cache_size() = _first_cache_size(cache_size(first_cache()))
 
 @generated function _avx_config_val(
     ::Val{inline}, ::Val{u₁}, ::Val{u₂}, ::Val{thread}, ::StaticInt{W},
@@ -294,7 +295,9 @@ function generate_call(ls::LoopSet, (inline,u₁,u₂)::Tuple{Bool,Int8,Int8}, t
     end
     arraysymbolinds = Symbol[]
     arrayref_descriptions = Expr(:tuple)
-    foreach(ref -> push!(arrayref_descriptions.args, ArrayRefStruct(ls, ref, arraysymbolinds, ids)), ls.refs_aliasing_syms)
+    for ref ∈ ls.refs_aliasing_syms
+        push!(arrayref_descriptions.args, ArrayRefStruct(ls, ref, arraysymbolinds, ids))
+    end
     argmeta = argmeta_and_consts_description(ls, arraysymbolinds)
     loop_bounds = loop_boundaries(ls)
     loop_syms = tuple_expr(QuoteNode, ls.loopsymbols)
@@ -307,8 +310,9 @@ function generate_call(ls::LoopSet, (inline,u₁,u₂)::Tuple{Bool,Int8,Int8}, t
     vargs_as_tuple || push!(q.args, lbarg)
     extra_args = vargs_as_tuple ? Expr(:tuple) : q
     add_grouped_strided_pointer!(extra_args, ls)
-    
-    foreach(is -> push!(extra_args.args, last(is)), ls.preamble_symsym)
+    for is ∈ ls.preamble_symsym
+        push!(extra_args.args, last(is))
+    end
     append!(extra_args.args, arraysymbolinds)
     add_reassigned_syms!(extra_args, ls)
     add_external_functions!(extra_args, ls)