Bug fixes

Author: chriselrod

Project.toml

@@ -1,7 +1,7 @@
 name = "LoopVectorization"
 uuid = "bdcacae8-1622-11e9-2a5c-532679323890"
 authors = ["Chris Elrod <[email protected]>"]
-version = "0.12.5"
+version = "0.12.6"
 
 [deps]
 ArrayInterface = "4fba245c-0d91-5ea0-9b3e-6abc04ee57a9"
@@ -28,7 +28,7 @@ SLEEFPirates = "0.6.12"
 Static = "0.2"
 ThreadingUtilities = "0.4.1"
 UnPack = "1"
-VectorizationBase = "0.19.9"
+VectorizationBase = "0.19.14"
 julia = "1.5"
 
 [extras]

src/LoopVectorization.jl

@@ -11,15 +11,17 @@ using VectorizationBase: register_size, register_count, cache_linesize, cache_si
     vfma_fast, vmuladd_fast, vdiv_fast, vadd_fast, vsub_fast, vmul_fast,
     relu, stridedpointer, StridedPointer, StridedBitPointer, AbstractStridedPointer, _vload, _vstore!,
     reduced_add, reduced_prod, reduce_to_add, reduce_to_prod, reduced_max, reduced_min, reduce_to_max, reduce_to_min,
-    vsum, vprod, vmaximum, vminimum, unwrap, Unroll, VecUnroll,
+    reduced_all, reduced_any, reduce_to_all, reduce_to_any,
+    vsum, vprod, vmaximum, vminimum, vany, vall, unwrap, Unroll, VecUnroll,
     preserve_buffer, zero_vecunroll, vbroadcast_vecunroll, _vzero, _vbroadcast,
     contract_add, collapse_add,
     contract_mul, collapse_mul,
     contract_max, collapse_max,
     contract_min, collapse_min,
     contract_and, collapse_and,
     contract_or,  collapse_or,
-    num_threads, num_cores
+    num_threads, num_cores,
+    max_mask
 
 
 using IfElse: ifelse

src/codegen/loopstartstopmanager.jl

@@ -117,8 +117,8 @@ function use_loop_induct_var!(ls::LoopSet, q::Expr, ar::ArrayReferenceMeta, alla
             # Not doing normal offset indexing
             uliv[i] = -findfirst(Base.Fix2(===,ind), looporder)::Int
             # push!(gespinds.args, Expr(:call, lv(:Zero)))
-            # push!(gespinds.args, staticexpr(1))
-            push!(gespinds.args, staticexpr(convert(Int, strides[i])))
+            push!(gespinds.args, staticexpr(1))
+            # push!(gespinds.args, staticexpr(convert(Int, strides[i])))
 
             push!(offsetprecalc_descript.args, 0) # not doing offset indexing, so push 0
         else

src/codegen/lower_memory_common.jl

@@ -135,9 +135,13 @@ function mem_offset(op::Operation, td::UnrollArgs, inds_calc_by_ptr_offset::Vect
         indvectorized = _mm & (ind === vloopsym)
         offset = offsets[n] % Int
         stride = strides[n] % Int
+        ind_by_offset = inds_calc_by_ptr_offset[n] | (ind === CONSTANTZEROINDEX)
+        if !ind_by_offset
+            offset += (stride - 1)
+        end
         @unpack vstep = td
         if loopedindex[n]
-            addoffset!(ret, indvectorized, vstep, stride, ind, offset, inds_calc_by_ptr_offset[n] | (ind === CONSTANTZEROINDEX)) # 7 arg
+            addoffset!(ret, indvectorized, vstep, stride, ind, offset, ind_by_offset) # 7 arg
         else
             offset -= 1
             newname, parent = symbolind(ind, op, td)
@@ -279,6 +283,9 @@ function mem_offset_u(op::Operation, td::UnrollArgs, inds_calc_by_ptr_offset::Ve
             stride = convert(Int, strides[n])
             indvectorized = ind === vloopsym
             indvectorizedmm = _mm & indvectorized
+            if !ind_by_offset
+                offset += (stride - 1)
+            end
             if ind === u₁loopsym
                 addvectoroffset!(ret, indvectorizedmm, incr₁, u₁step, vstep, stride, ind, offset, ind_by_offset, indvectorized) # 9 arg
             elseif ind === u₂loopsym

src/codegen/lower_store.jl

@@ -114,11 +114,12 @@ function lower_store!(
     # isunrolled₂ = isu₂unrolled(op)    
     falseexpr = Expr(:call, lv(:False)); trueexpr = Expr(:call, lv(:True)); rs = staticexpr(reg_size(ls));
     opp = first(parents(op))
-    if (opp.instruction.instr === reductfunc) && isone(length(parents(opp)))
+    if ((opp.instruction.instr === reductfunc) || (opp.instruction.instr === :identity)) && isone(length(parents(opp)))
         opp = only(parents(opp))
     end
     # __u₂max = ls.unrollspecification[].u₂
     isu₁, isu₂ = isunrolled_sym(opp, u₁loopsym, u₂loopsym, vloopsym)#, __u₂max)
+    # @show isu₁, isu₂, opp, u₁loopsym, u₂loopsym, vloopsym
     u = isu₁ ? u₁ : 1
     mvar = Symbol(variable_name(opp, ifelse(isu₂, suffix, -1)), '_', u)
     if all(op.ref.loopedindex)
@@ -133,14 +134,21 @@ function lower_store!(
         push!(q.args, storeexpr)
     elseif u₁ > 1
         mvard = Symbol(mvar, "##data##")
-        push!(q.args, Expr(:(=), mvard, Expr(:call, lv(:data), mvar)))
+        isu₁ && push!(q.args, Expr(:(=), mvard, Expr(:call, lv(:data), mvar)))
         for u ∈ 1:u₁
             mvaru = :(getfield($mvard, $u, false))
             inds = mem_offset_u(op, ua, inds_calc_by_ptr_offset, true, u-1)
-            storeexpr = if reductfunc === Symbol("")
-                Expr(:call, lv(:_vstore!), vptr(op), mvaru, inds)
+            # @show isu₁unrolled(opp), opp
+            storeexpr = if isu₁
+                if reductfunc === Symbol("")
+                    Expr(:call, lv(:_vstore!), vptr(op), mvaru, inds)
+                else
+                    Expr(:call, lv(:_vstore!), lv(reductfunc), vptr(op), mvaru, inds)
+                end
+            elseif reductfunc === Symbol("")
+                Expr(:call, lv(:_vstore!), vptr(op), mvar, inds)
             else
-                Expr(:call, lv(:_vstore!), lv(reductfunc), vptr(op), mvaru, inds)
+                Expr(:call, lv(:_vstore!), lv(reductfunc), vptr(op), mvar, inds)
             end
             domask = mask && (isvectorized(op) & ((u == u₁) | (vloopsym !== u₁loopsym)))
             add_memory_mask!(storeexpr, op, ua, domask)# & ((u == u₁) | isvectorized(op)))

src/condense_loopset.jl

@@ -172,14 +172,17 @@ function argmeta_and_consts_description(ls::LoopSet, arraysymbolinds)
         tuple_expr(ls.preamble_funcofeltypes)
     )
 end
+@inline vdata(v::Vec) = getfield(v, :data)
+@inline vdata(v::VecUnroll) = getfield(v, :data)
+@inline vdata(x) = x
 
 function loopset_return_value(ls::LoopSet, ::Val{extract}) where {extract}
     @assert !iszero(length(ls.outer_reductions))
     if isone(length(ls.outer_reductions))
         op = getop(ls, ls.outer_reductions[1])
         if extract
             # if (isu₁unrolled(op) | isu₂unrolled(op))
-                Expr(:call, :data, Symbol(mangledvar(op), "##onevec##"))
+                Expr(:call, :vdata, Symbol(mangledvar(op), "##onevec##"))
             # else
                 # Expr(:call, :data, mangledvar(op))
             # end
@@ -192,7 +195,7 @@ function loopset_return_value(ls::LoopSet, ::Val{extract}) where {extract}
         for or ∈ ls.outer_reductions
             op = ops[or]
             if extract
-                push!(ret.args, Expr(:call, :data, Symbol(mangledvar(op), "##onevec##")))
+                push!(ret.args, Expr(:call, :vdata, Symbol(mangledvar(op), "##onevec##")))
             else
                 push!(ret.args, Symbol(mangledvar(ops[or]), "##onevec##"))
             end
@@ -421,6 +424,7 @@ make_crashy(q) = Expr(:macrocall, Symbol("@inbounds"), LineNumberNode(@__LINE__,
 @inline vecmemaybe(x::NativeTypes) = x
 @inline vecmemaybe(x::VectorizationBase._Vec) = Vec(x)
 @inline vecmemaybe(x::Tuple) = VectorizationBase.VecUnroll(x)
+@inline vecmemaybe(x::Mask) = x
 
 function gc_preserve(call::Expr, preserve::Vector{Symbol})
     q = Expr(:gc_preserve, call)

src/modeling/operations.jl

@@ -31,15 +31,16 @@ function ArrayReference(array, indices)
     ArrayReference(array, indices, zeros(Int8, ninds), ones(Int8, ninds))
 end
 function sameref(x::ArrayReference, y::ArrayReference)
-    (x.array === y.array) && (x.indices == y.indices)
+    (x.array === y.array) && (x.indices == y.indices) && (x.strides == y.strides)
 end
 function Base.isequal(x::ArrayReference, y::ArrayReference)
     sameref(x, y) || return false
     xoffs = x.offsets; yoffs = y.offsets
-    xmult = x.strides; ymult = y.strides
+    # xmult = x.strides; ymult = y.strides
     length(xoffs) == length(yoffs) || return false
     for n ∈ eachindex(xoffs)
-        ((xoffs[n] == yoffs[n]) & (xmult[n] == ymult[n])) || return false
+        # ((xoffs[n] == yoffs[n]) & (xmult[n] == ymult[n])) || return false
+        (xoffs[n] == yoffs[n]) || return false
     end
     true
 end

test/runtests.jl

@@ -60,6 +60,8 @@ const START_TIME = time()
     @time include("gemm.jl")
 
     @time include("threading.jl")
+
+    @time include("tullio.jl")
 end
 
 const ELAPSED_MINUTES = (time() - START_TIME)/60

test/tullio.jl

@@ -0,0 +1,74 @@
+using LoopVectorization, Test
+# Tests for Tullio
+
+@testset "Tullio Tests" begin
+    A = (1:10) .^ 2; K = rand(10);
+
+    function act!(ℛ::AbstractArray, A, 𝒶𝓍i = axes(A,1), 𝒶𝓍j = axes(ℛ,2))
+        @avx for j in 𝒶𝓍j
+            for i in 𝒶𝓍i
+                ℛ[i, j] = A[i] / j
+            end
+        end
+        ℛ
+    end
+    function act_noavx!(ℛ::AbstractArray, A, 𝒶𝓍i = axes(A,1), 𝒶𝓍j = axes(ℛ,2))
+        for j in 𝒶𝓍j
+            for i in 𝒶𝓍i
+                ℛ[i, j] = A[i] / j
+            end
+        end
+        ℛ
+    end
+    @test act!(rand(10,10), A) ≈ act_noavx!(rand(10,10), A)
+
+    D = similar(A, 10, 10) .= 999;
+
+    inds = [2,3,5,2];
+
+    function two!(ℛ::AbstractArray, inds, A, 𝒶𝓍j = axes(ℛ,2), 𝒶𝓍i = axes(inds,1))         
+        @avx for i = 𝒶𝓍i
+            for j = 𝒶𝓍j
+                ℛ[inds[i], j] = A[j]
+            end
+        end
+        ℛ
+    end
+    function two_noavx!(ℛ::AbstractArray, inds, A, 𝒶𝓍j = axes(ℛ,2), 𝒶𝓍i = axes(inds,1))         
+        for i = 𝒶𝓍i
+            for j = 𝒶𝓍j
+                ℛ[inds[i], j] = A[j]
+            end
+        end
+        ℛ
+    end
+    @test two!(copy(D), inds, A) == two!(copy(D), inds, A)
+
+    function three!(ℛ::AbstractArray, A, 𝒶𝓍i = axes(ℛ,1))
+        @avx for i = 𝒶𝓍i
+            ℛ[i] = A[2i + 1] + A[i]
+        end
+        ℛ
+    end
+    function three_noavx!(ℛ::AbstractArray, A, 𝒶𝓍i = axes(ℛ,1))
+        for i = 𝒶𝓍i
+            ℛ[i] = A[2i + 1] + A[i]
+        end
+        ℛ
+    end
+    @test three!(rand(4), A) == three_noavx!(rand(4), A)
+
+    function and(A, 𝒶𝓍i = axes(A,1))
+        𝒜𝒸𝒸 = true
+        @avx for i = 𝒶𝓍i
+            𝒜𝒸𝒸 = 𝒜𝒸𝒸 & (A[i] > 0)
+        end
+        𝒜𝒸𝒸
+    end
+    @test and(A)
+    A[3] = -1
+    @test !and(A)
+    
+end
+
+