Some more internal restructuring; loop indices are now internally symbols; ints are are CSE-ed as views (expressions are bound to symbols/converted to a more ssa-like form).

Author: chriselrod

Project.toml

@@ -13,11 +13,11 @@ VectorizationBase = "3d5dd08c-fd9d-11e8-17fa-ed2836048c2f"
 
 [compat]
 MacroTools = "0.5"
-Parameters = "0.12.0"
-SIMDPirates = "0.1.6"
-SLEEFPirates = "0.1.3"
-VectorizationBase = "0.1.9"
-julia = "1.3.0"
+Parameters = ">=0.12.0"
+SIMDPirates = ">=0.1.7"
+SLEEFPirates = ">=0.1.3"
+VectorizationBase = ">=0.1.10"
+julia = "1.3"
 
 [extras]
 Test = "8dfed614-e22c-5e08-85e1-65c5234f0b40"

src/LoopVectorization.jl

@@ -3,7 +3,7 @@ module LoopVectorization
 using VectorizationBase, SIMDPirates, SLEEFPirates, MacroTools, Parameters
 using VectorizationBase: REGISTER_SIZE, REGISTER_COUNT, extract_data, num_vector_load_expr,
     mask, masktable, pick_vector_width_val, valmul, valrem, valmuladd, valadd, valsub, _MM,
-    maybestaticlength, maybestaticsize, Static, staticm1
+    maybestaticlength, maybestaticsize, Static, staticm1, subsetview
 using SIMDPirates: VECTOR_SYMBOLS, evadd, evmul, vrange, reduced_add, reduced_prod, reduce_to_add, reduce_to_prod
 using Base.Broadcast: Broadcasted, DefaultArrayStyle
 using LinearAlgebra: Adjoint, Transpose

src/add_loads.jl

@@ -9,7 +9,7 @@ function add_load!(
     ls::LoopSet, var::Symbol, mpref::ArrayReferenceMetaPosition, elementbytes::Int = 8
 )
     length(mpref.loopdependencies) == 0 && return add_constant!(ls, var, mpref, elementbytes)
-    ref = mpref.mref.ref
+    ref = mpref.mref
     # try to CSE
     id = findfirst(r -> r == ref, ls.refs_aliasing_syms)
     if id === nothing
@@ -29,13 +29,6 @@ end
 function add_simple_load!(
     ls::LoopSet, var::Symbol, ref::ArrayReference, elementbytes::Int = 8
 )
-    # if ref.loaded[] == true
-        # op = getop(ls, var, elementbytes)
-        # @assert var === op.variable
-        # return op
-    # end
-    # loopset = keys(ls.loops)
-    # loopdeps = Symbol[s for s ∈ loopdependencies(ref) if (s isa Symbol && s ∈ loopset)]
     loopdeps = Symbol[s for s ∈ ref.indices]
     mref = ArrayReferenceMeta(
         ref, fill(true, length(loopdeps))

src/add_stores.jl

@@ -26,7 +26,7 @@ function add_store!(
     id = nops
     if pvar ∉ ls.syms_aliasing_refs
         push!(ls.syms_aliasing_refs, pvar)
-        push!(ls.refs_aliasing_syms, mpref.mref.ref)
+        push!(ls.refs_aliasing_syms, mpref.mref)
         # add_unique_store!(ls, mref, parents, ldref, reduceddeps, elementbytes)
     else
         # try to cse store

src/broadcast.jl

@@ -52,7 +52,7 @@ function add_broadcast!(
     pushpreamble!(ls, Expr(:(=), K, Expr(:call, :size, mB, 1)))
 
     k = gensym(:k)
-    ls.loops[k] = Loop(k, 0, K)
+    add_loop!(ls, Loop(k, 0, K), k)
     m = loopsyms[1];
     if ndims(B) == 1
         bloopsyms = Symbol[k]
@@ -100,7 +100,7 @@ function add_broadcast!(
     ls::LoopSet, destname::Symbol, bcname::Symbol, loopsyms::Vector{Symbol},
     ::Type{<:LowDimArray{D,T,N}}, elementbytes::Int = 8
 ) where {D,T,N}
-    fulldims = Union{Symbol,Int}[loopsyms[n] for n ∈ 1:N if D[n]]
+    fulldims = Symbol[loopsyms[n] for n ∈ 1:N if D[n]]
     ref = ArrayReference(bcname, fulldims)
     add_simple_load!(ls, destname, ref, elementbytes )::Operation
 end
@@ -109,13 +109,13 @@ function add_broadcast_adjoint_array!(
 ) where {T,N,A<:AbstractArray{T,N}}
     parent = gensym(:parent)
     pushpreamble!(ls, Expr(:(=), parent, Expr(:call, :parent, bcname)))
-    ref = ArrayReference(parent, Union{Symbol,Int}[loopsyms[N + 1 - n] for n ∈ 1:N])
+    ref = ArrayReference(parent, Symbol[loopsyms[N + 1 - n] for n ∈ 1:N])
     add_simple_load!( ls, destname, ref, elementbytes )::Operation    
 end
 function add_broadcast_adjoint_array!(
     ls::LoopSet, destname::Symbol, bcname::Symbol, loopsyms::Vector{Symbol}, ::Type{<:AbstractVector}, elementbytes::Int = 8
 )
-    ref = ArrayReference(bcname, Union{Symbol,Int}[loopsyms[2]])
+    ref = ArrayReference(bcname, Symbol[loopsyms[2]])
     add_simple_load!( ls, destname, ref, elementbytes )
 end
 function add_broadcast!(
@@ -147,7 +147,7 @@ function add_broadcast!(
     ls::LoopSet, destname::Symbol, bcname::Symbol, loopsyms::Vector{Symbol},
     ::Type{SubArray{T,N,A,S,B}}, elementbytes::Int = 8
 ) where {T,N,N2,A<:AbstractArray{T,N2},B,N3,S <: Tuple{Int,Vararg{Any,N3}}}
-    inds = Vector{Union{Int,Symbol}}(undef, N+1)
+    inds = Vector{Symbol}(undef, N+1)
     inds[1] = Symbol("##DISCONTIGUOUSSUBARRAY##")
     inds[2:end] .= @view(loopsyms[1:N])
     add_simple_load!(ls, destname, ArrayReference(bcname, inds), elementbytes)
@@ -192,7 +192,7 @@ end
     sizes = Expr(:tuple)
     for (n,itersym) ∈ enumerate(loopsyms)
         Nsym = gensym(:N)
-        ls.loops[itersym] = Loop(itersym, 0, Nsym)
+        add_loop!(ls, Loop(itersym, 0, Nsym), itersym)
         push!(sizes.args, Nsym)
     end
     pushpreamble!(ls, Expr(:(=), sizes, Expr(:call, :size, :dest)))
@@ -217,7 +217,7 @@ end
     sizes = Expr(:tuple)
     for (n,itersym) ∈ enumerate(loopsyms)
         Nsym = gensym(:N)
-        ls.loops[itersym] = Loop(itersym, 0, Nsym)
+        add_loop!(ls, Loop(itersym, 0, Nsym), itersym)
         push!(sizes.args, Nsym)
     end
     pushpreamble!(ls, Expr(:(=), sizes, Expr(:call, :size, :dest′)))

src/condense_loopset.jl

@@ -1,4 +1,65 @@
 
+@enum IndexType::UInt8 NotAnIndex=0 LoopIndex=1 ComputedIndex=2 SymbolicIndex=3 LiteralIndex=4
+
+struct ArrayRefStruct
+    index_types::UInt64
+    indices::UInt64
+end
+tup_to_vec(t::NTuple{W,T}) where {W,T} = ntuple(Val(W)) do w @inbounds Core.VecElement(t[w]) end
+vec_to_tup(v::Vec{W,T}) where {W,T} = ntuple(Val(W)) do w @inbounds (v[w]).value end
+vec_to_tup(v::SVec{W,T}) where {W,T} = ntuple(Val(W)) do w @inbounds (v[w]) end
+function ArrayRefStruct(ls::LoopSet, mref::ArrayReferenceMeta)
+    index_types = zero(UInt64)
+    indices = vbroadcast(SVec{8,UInt64}, zero(UInt64))
+    indv = mref.ref.indices
+    start = 1 + (first(indv) === Symbol("##DISCONTIGUOUSSUBARRAY##"))
+    for (n,ind) ∈ enumerate(@view(indv[start:end]))
+        index_types <<= 8
+        indices <<= 16
+        if ind isa Int
+            
+        elseif mref.loopindex[n]
+        else
+        end
+    end
+    ArrayRefStruct( index_types, vec_to_tup(indices) )
+end
+
+struct OperationStruct
+    instruction::Instruction
+    loopdeps::UInt64
+    reduceddeps::UInt64
+    parents::UInt64
+    array::UInt64
+end
+function findmatchingarray(ls::LoopSet, array::Symbol)
+    id = zero(UInt64)
+    for (as,_) ∈ ls.includedarrays
+        id += one(UInt64)
+        if as === arraysym
+            return id
+        end
+    end
+    zero(UInt64)
+end
+filled_4byte_chunks(u::UInt64) = leading_zeros(u) >> 2
+num_loop_deps(os::OperationStruct) = filled_4byte_chunks(os.loopdeps)
+num_reduced_deps(os::OperationStruct) = filled_4byte_chunks(os.reduced_deps)
+num_parents(os::OperationStruct) = filled_4byte_chunks(os.parents)
+
+function loodeps_uint(ls::LoopSet, op::Operation)
+    ld = zero(UInt64) # leading_zeros(ld) >> 2 yields the number of loopdeps
+    for d ∈ loopdependencies(op)
+        ld <<= 4
+        ld |= getloopid(ls, d)
+    end
+    ld
+end
+function OperationStruct(ls::LoopSet, op::Operation)
+    instr = instruction(op)
+    array = accesses_memory(op) ? findmatchingarray(ls, name(op.ref)) : zero(UInt64)
+    
+end
 ## turn a LoopSet into a type object which can be used to reconstruct the LoopSet.
 
 

src/costs.jl

@@ -8,6 +8,13 @@ Base.convert(::Type{Instruction}, instr::Symbol) = Instruction(instr)
 lower(instr::Instruction) = Expr(:(.), instr.mod, QuoteNode(instr.instr))
 Base.Expr(instr::Instruction, args...) = Expr(:call, lower(instr), args...)::Expr
 Base.hash(instr::Instruction, h::UInt64) = hash(instr.instr, hash(instr.mod, h))
+function Base.isless(instr1::Instruction, instr2::Instruction)
+    if instr1.mod === instr2.mod
+        isless(instr1.instr, instr2.instr)
+    else
+        isless(instr1.mod, instr2.mod)
+    end
+end
 
 const LOOPCONSTANT = Instruction(gensym())
 

src/determinestrategy.jl

@@ -287,8 +287,8 @@ function solve_tilesize(
 )
     maxT = 4#8
     maxU = 4#8
-    tiledloop = ls.loops[tiled]
-    unrolledloop = ls.loops[unrolled]
+    tiledloop = getloop(ls, tiled)
+    unrolledloop = getloop(ls, unrolled)
     if isstaticloop(tiledloop)
         maxT = min(4maxT, length(tiledloop))
     end
@@ -314,6 +314,27 @@ function set_upstream_family!(adal::Vector{T}, op::Operation, val::T) where {T}
     end
 end
 
+function stride_penalty(ls::LoopSet, op::Operation, order::Vector{Symbol})
+    num_loops = length(order)
+    contigsym = first(loopdependencies(op))
+    contigsym == Symbol("##DISCONTIGUOUSSUBARRAY##") && return 0
+    iter = 0
+    for i ∈ 0:num_loops - 1
+        loopsym = order[num_loops - i]
+        loopsym === contigsym && return iter
+        iter *= length(getloop(ls, loopsym))
+    end
+    iter
+end
+function stride_penalty(ls::LoopSet, order::Vector{Symbol})
+    stridepenalty = 0
+    for op ∈ operations(ls)
+        if accesses_memory(op)
+            stridepenalty += stride_penalty(ls, op, order)
+        end
+    end
+    stridepenalty * 1e-9
+end
 # Just tile outer two loops?
 # But optimal order within tile must still be determined
 # as well as size of the tiles.
@@ -402,7 +423,8 @@ function evaluate_cost_tile(
     end
     # @show order, vectorized cost_vec reg_pressure
     # @show solve_tilesize(ls, unrolled, tiled, cost_vec, reg_pressure)
-    solve_tilesize(ls, unrolled, tiled, cost_vec, reg_pressure)
+    U, T, tcost = solve_tilesize(ls, unrolled, tiled, cost_vec, reg_pressure)
+    U, T, tcost + stride_penalty(ls, order)
 end
 
 
@@ -411,7 +433,7 @@ struct LoopOrders
     buff::Vector{Symbol}
 end
 function LoopOrders(ls::LoopSet)
-    syms = [s for s ∈ keys(ls.loops)]
+    syms = copy(ls.loopsymbols)
     LoopOrders(syms, similar(syms))
 end
 function Base.iterate(lo::LoopOrders)

src/graphs.jl

@@ -143,7 +143,8 @@ Base.@propagate_inbounds Base.getindex(lo::LoopOrder, i...) = lo.oporder[LinearI
 # Must make it easy to iterate
 # outer_reductions is a vector of indixes (within operation vectors) of the reduction operation, eg the vmuladd op in a dot product
 struct LoopSet
-    loops::Dict{Symbol,Loop} # sym === loops[sym].itersymbol
+    loopsymbols::Vector{Symbol}
+    loops::Vector{Loop}
     opdict::Dict{Symbol,Operation}
     operations::Vector{Operation} # Split them to make it easier to iterate over just a subset
     outer_reductions::Vector{Int} # IDs of reduction operations that need to be reduced at end.
@@ -155,9 +156,9 @@ struct LoopSet
     preamble_symfloat::Vector{Tuple{Int,Float64}}
     preamble_zeros::Vector{Int}
     preamble_ones::Vector{Int}
-    includedarrays::Vector{Tuple{Symbol,Int}}
+    includedarrays::Vector{Symbol}
     syms_aliasing_refs::Vector{Symbol} # O(N) search is faster at small sizes
-    refs_aliasing_syms::Vector{ArrayReference}
+    refs_aliasing_syms::Vector{ArrayReferenceMeta}
     cost_vec::Matrix{Float64}
     reg_pres::Matrix{Int}
     included_vars::Vector{Bool}
@@ -211,15 +212,11 @@ function pushpreamble!(ls::LoopSet, op::Operation, RHS::Expr)
     nothing
 end
 
-function includesarray(ls::LoopSet, array::Symbol)
-    for (a,i) ∈ ls.includedarrays
-        a === array && return i
-    end
-    -1
-end
+includesarray(ls::LoopSet, array::Symbol) = array ∈ ls.includedarrays
+
 function LoopSet()
     LoopSet(
-        Dict{Symbol,Loop}(),
+        Symbol[], Loop[],
         Dict{Symbol,Operation}(),
         Operation[],
         Int[],
@@ -231,7 +228,7 @@ function LoopSet()
         Int[],Int[],
         Tuple{Symbol,Int}[],
         Symbol[],
-        ArrayReference[],
+        ArrayReferenceMeta[],
         Matrix{Float64}(undef, 4, 2),
         Matrix{Int}(undef, 4, 2),
         Bool[], Bool[], gensym(:W), gensym(:T)
@@ -243,11 +240,17 @@ function oporder(ls::LoopSet)
     reshape(ls.loop_order.oporder, (2,2,2,N))
 end
 names(ls::LoopSet) = ls.loop_order.loopnames
-Base.length(ls::LoopSet, s::Symbol) = length(ls.loops[s])
-isstaticloop(ls::LoopSet, s::Symbol) = isstaticloop(ls.loops[s])
-looprangehint(ls::LoopSet, s::Symbol) = length(ls.loops[s])
-looprangesym(ls::LoopSet, s::Symbol) = ls.loops[s].rangesym
-# itersyms(ls::LoopSet) = keys(ls.loops)
+function getloopid(ls::LoopSet, s::Symbol)::Int
+    for (loopnum,sym) ∈ enumerate(ls.loopsymbols)
+        s === sym && return loopnum
+    end
+end
+getloop(ls::LoopSet, s::Symbol) = ls.loops[getloopid(ls, s)]
+Base.length(ls::LoopSet, s::Symbol) = length(getloop(ls, s))
+
+isstaticloop(ls::LoopSet, s::Symbol) = isstaticloop(getloop(ls,s))
+looprangehint(ls::LoopSet, s::Symbol) = length(getloop(ls, s))
+looprangesym(ls::LoopSet, s::Symbol) = getloop(ls, s).rangesym
 function getop(ls::LoopSet, var::Symbol, elementbytes::Int = 8)
     get!(ls.opdict, var) do
         # might add constant
@@ -377,7 +380,7 @@ function register_single_loop!(ls::LoopSet, looprange::Expr)
     else
         throw("Unrecognized loop range type: $r.")
     end
-    ls.loops[itersym] = loop
+    add_loop!(ls, loop, itersym)
     nothing
 end
 function register_loop!(ls::LoopSet, looprange::Expr)
@@ -399,8 +402,10 @@ function add_loop!(ls::LoopSet, q::Expr, elementbytes::Int = 8)
         push!(ls, q, elementbytes)
     end
 end
-function add_loop!(ls::LoopSet, loop::Loop)
-    ls.loops[loop.itersym] = loop
+function add_loop!(ls::LoopSet, loop::Loop, itersym::Symbol = loop.itersym)
+    push!(ls.loopsymbols, itersym)
+    push!(ls.loops, loop)
+    nothing
 end
 
 function instruction(x)
@@ -422,7 +427,7 @@ function add_operation!(
         elseif f === :zero || f === :one
             c = gensym(f)
             # pushpreamble!(ls, Expr(:(=), c, RHS))
-            op = add_constant!(ls, c, [keys(ls.loops)...], LHS, f, elementbytes)
+            op = add_constant!(ls, c, copy(ls.loopsymbols), LHS, f, elementbytes)
             push!(f === :zero ? ls.preamble_zeros : ls.preamble_ones, identifier(op))
             op
         else
@@ -447,7 +452,7 @@ function add_operation!(
         elseif f === :zero || f === :one
             c = gensym(f)
             # pushpreamble!(ls, Expr(:(=), c, RHS))
-            op = add_constant!(ls, c, [keys(ls.loops)...], LHS_sym, f, elementbytes)
+            op = add_constant!(ls, c, copy(ls.loopsymbols), LHS_sym, f, elementbytes)
             push!(f === :zero ? ls.preamble_zeros : ls.preamble_ones, identifier(op))
             op
         else
@@ -475,7 +480,7 @@ function Base.push!(ls::LoopSet, ex::Expr, elementbytes::Int = 8)
             if RHS isa Expr
                 add_operation!(ls, LHS, RHS, elementbytes)
             else
-                deps = [keys(ls.loops)...]
+                deps = copy(ls.loopsymbols)
                 if RHS isa Number
                     fisone = false
                     fiszero = false