Some progress/updates.

Author: chriselrod

Project.toml

@@ -1,7 +1,7 @@
 name = "LoopVectorization"
 uuid = "bdcacae8-1622-11e9-2a5c-532679323890"
 authors = ["Chris Elrod <[email protected]>"]
-version = "0.12"
+version = "0.12.0"
 
 [deps]
 ArrayInterface = "4fba245c-0d91-5ea0-9b3e-6abc04ee57a9"
@@ -11,6 +11,7 @@ LinearAlgebra = "37e2e46d-f89d-539d-b4ee-838fcccc9c8e"
 OffsetArrays = "6fe1bfb0-de20-5000-8ca7-80f57d26f881"
 Requires = "ae029012-a4dd-5104-9daa-d747884805df"
 SLEEFPirates = "476501e8-09a2-5ece-8869-fb82de89a1fa"
+Static = "aedffcd0-7271-4cad-89d0-dc628f76c6d3"
 ThreadingUtilities = "8290d209-cae3-49c0-8002-c8c24d57dab5"
 UnPack = "3a884ed6-31ef-47d7-9d2a-63182c4928ed"
 VectorizationBase = "3d5dd08c-fd9d-11e8-17fa-ed2836048c2f"
@@ -22,6 +23,7 @@ IfElse = "0.1"
 OffsetArrays = "1.4.1, 1.5"
 Requires = "1"
 SLEEFPirates = "0.6.7"
+Static = "0.2"
 ThreadingUtilities = "0.2.3"
 UnPack = "1"
 VectorizationBase = "0.19"

src/LoopVectorization.jl

@@ -1,6 +1,6 @@
 module LoopVectorization
 
-
+using Static: StaticInt, gt
 using VectorizationBase, SLEEFPirates, UnPack, OffsetArrays
 using VectorizationBase: register_size, register_count, cache_linesize, has_opmask_registers,
     mask, pick_vector_width, MM, AbstractMask, data, grouped_strided_pointer,
@@ -18,7 +18,8 @@ using VectorizationBase: register_size, register_count, cache_linesize, has_opma
     contract_max, collapse_max,
     contract_min, collapse_min,
     contract_and, collapse_and,
-    contract_or,  collapse_or
+    contract_or,  collapse_or,
+    num_threads, num_cores
 
 
 using IfElse: ifelse
@@ -76,6 +77,7 @@ include("codegen/lower_load.jl")
 include("codegen/lower_store.jl")
 include("codegen/lowering.jl")
 include("codegen/split_loops.jl")
+include("codegen/lower_threads.jl")
 include("condense_loopset.jl")
 include("reconstruct_loopset.jl")
 include("constructors.jl")

src/codegen/loopstartstopmanager.jl

@@ -437,13 +437,12 @@ function offset_ptr(
     end
     Expr(:(=), vptr(ar), Expr(:call, lv(:gesp), vptr(ar), gespinds))
 end
-function incrementloopcounter(ls::LoopSet, us::UnrollSpecification, n::Int, UF::Int)
+function incrementloopcounter!(q::Expr, ls::LoopSet, us::UnrollSpecification, n::Int, UF::Int)
     @unpack u₁loopnum, u₂loopnum, vloopnum, u₁, u₂ = us
     lssm = ls.lssm[]
     ptrdefs = lssm.incrementedptrs[n]
     looporder = names(ls)
     loopsym = looporder[n]
-    q = Expr(:block)
     termind = lssm.terminators[n]
     loop = getloop(ls, n)
     if iszero(termind) # increment liv
@@ -453,7 +452,7 @@ function incrementloopcounter(ls::LoopSet, us::UnrollSpecification, n::Int, UF::
         offsetinds = indices_calculated_by_pointer_offsets(ls, ar)
         push!(q.args, offset_ptr(ar, us, loopsym, n, UF, offsetinds, loop))
     end
-    q
+    nothing
 end
 function terminatecondition(ls::LoopSet, us::UnrollSpecification, n::Int, inclmask::Bool, UF::Int)
     lssm = ls.lssm[]

src/codegen/lower_compute.jl

@@ -106,12 +106,12 @@ function ifelselastexpr(hasf::Bool, M::Int, vargtypes, K::Int, S::Int, maskearly
     for k ∈ 1:K
         lengths[k] = l = vecunrolllen(vargtypes[k])
         if hasf
-            if l == -1
-                push!(q.args, :($(vargs[k]) = getfield(vargs, $k, false)))
-            else
-                push!(q.args, :($(vargs[k]) = data(getfield(vargs, $k, false))))
+            gfvarg = Expr(:call, GlobalRef(Core, :getfield), :vargs, k, false)
+            if l ≠ -1 # VecUnroll
+                gfvarg = Expr(:call, GlobalRef(Core, :getfield), gfvarg, 1, false)
             end
-        elseif l != -1
+            push!(q.args, Expr(:(=), vargs[k], gfvarg))
+        elseif l ≠ -1
             varg = vargs[k]
             vargs[k] = dvarg = Symbol(:d, varg)
             push!(q.args, :($dvarg = data($varg)))

src/codegen/lower_threads.jl

@@ -3,16 +3,58 @@ struct AVX{UNROLL,OPS,ARF,AM,LPSYM,LB,V} <: Function end
 
 # This should call the same `_avx_!(Val{UNROLL}(), Val{OPS}(), Val{ARF}(), Val{AM}(), Val{LPSYM}(), _vargs)` as normal so that this
 # hopefully shouldn't add much to compile time.
+
 function (::AVX{UNROLL,OPS,ARF,AM,LPSYM,LB,V})(p::Ptr{UInt}) where {UNROLL,OPS,ARF,AM,LPSYM,LB,V}
     _vargs = ThreadingUtilities.load(p, Tuple{LB,V}, 1)
     ret = _avx_!(Val{UNROLL}(), Val{OPS}(), Val{ARF}(), Val{AM}(), Val{LPSYM}(), _vargs)
     ThreadingUtilities.store!(p, ret, 7)
     nothing
 end
 
+# function approx_cbrt(x)
+#     s = significand(x)
+#     e = exponent(x)
+    
+#     # 40 + 0.00020833333333333335*(x-64000)  -2.1701388888888896e-9*(x-64000)^2*0.5 + 5.6514033564814844e-14 * (x-64000)^3/6
+# end
+
+function choose_threads(::StaticInt{C}, x) where {C}
+    nt = ifelse(gt(num_threads(), num_cores()), num_cores(), num_threads())
+    fx = Base.uitofp(Float64, x)
+    min(Base.fptosi(Int, Base.ceil_llvm(5.0852672001495816e-11*C*Base.sqrt_llvm(fx))), nt)
+end
+
+function thread_single_loop_expr(ls::LoopSet, UNROLL, id)
+
+end
+function thread_multiple_loop_expr(ls::LoopSet, UNROLL, valid_thread_loop)
 
+end
 
-function _avx_threads!()
+function avx_threads_expr(ls::LoopSet, UNROLL)
+    order, u₁loop, u₂loop, vectorized, u₁, u₂, c, shouldinline = choose_order_cost(ls)
+    valid_thread_loop = fill(true, length(order))
+    for op ∈ operations(ls)
+        if isstore(op) && (length(reduceddependencies(op)) > 0)
+            for reduceddep ∈ reduceddependencies(op)
+                for (i,o) ∈ enumerate(order)
+                    if o === reduceddep
+                        valid_thread_loop[i] = false
+                    end
+                end
+            end
+        end
+    end
+    num_candiates = sum(valid_thread_loop)
+    # num_to_thread = min(num_candiates, 2)
+    # candidate_ids = 
+    if num_candiates == 0
+        avx_body(ls, UNROLL)
+    elseif num_candiates == 1
+        thread_single_loop_expr(ls, UNROLL, findfirst(isone, valid_thread_loop)::Int)
+    else
+        thread_multiple_loop_expr(ls, UNROLL, vald_thread_loop)
+    end
 
 end
 

src/codegen/lowering.jl

@@ -82,7 +82,7 @@ function lower_block(
     end
     loopsym = order[n]
     # if n > 1 || iszero(ls.align_loops[])
-    push!(blockq.args, incrementloopcounter(ls, us, n, UF))
+    incrementloopcounter!(blockq, ls, us, n, UF)
     # else
     #     loopsym = names(ls)[n]
     #     push!(blockq.args, Expr(:(=), loopsym, Expr(:call, lv(:vadd_fast), loopsym, Symbol("##ALIGNMENT#STEP##"))))

src/modeling/graphs.jl

@@ -607,7 +607,11 @@ Base.length(ls::LoopSet, s::Symbol) = length(getloop(ls, s))
 function init_loop_map!(ls::LoopSet)
     @unpack loopordermap = ls
     order = names(ls)
-    sortperm!(resize!(loopordermap, length(order)), order, by = x -> getloopid(ls,x))
+    orderids = Vector{Int}(undef, length(order))
+    for (i,o) ∈ enumerate(order)
+        orderids[i] = getloopid(ls,o)
+    end
+    sortperm!(resize!(loopordermap, length(order)), orderids)
     nothing
 end
 

src/reconstruct_loopset.jl

@@ -570,7 +570,7 @@ Execute an `@avx` block. The block's code is represented via the arguments:
 - `vargs...` holds the encoded pointers of all the arrays (see `VectorizationBase`'s various pointer types).
 """
 @generated function _avx_!(::Val{UNROLL}, ::Val{OPS}, ::Val{ARF}, ::Val{AM}, ::Val{LPSYM}, _vargs::Tuple{LB,V}) where {UNROLL, OPS, ARF, AM, LPSYM, LB, V}
-    1 + 1 # Irrelevant line you can comment out/in to force recompilation...
+    # 1 + 1 # Irrelevant line you can comment out/in to force recompilation...
     ls = _avx_loopset(OPS, ARF, AM, LPSYM, LB.parameters, V.parameters)
     # return @show avx_body(ls, UNROLL)
     # @show UNROLL, OPS, ARF, AM, LPSYM, LB