Allow loop variables to be used in vectorized comparisons, and fix bug related to incorrectly eliminated stores.

Author: chriselrod

Project.toml

@@ -1,7 +1,7 @@
 name = "LoopVectorization"
 uuid = "bdcacae8-1622-11e9-2a5c-532679323890"
 authors = ["Chris Elrod <[email protected]>"]
-version = "0.6.26"
+version = "0.6.27"
 
 [deps]
 LinearAlgebra = "37e2e46d-f89d-539d-b4ee-838fcccc9c8e"
@@ -13,10 +13,10 @@ VectorizationBase = "3d5dd08c-fd9d-11e8-17fa-ed2836048c2f"
 
 [compat]
 OffsetArrays = "1"
-SIMDPirates = "0.7.6"
+SIMDPirates = "0.7.7"
 SLEEFPirates = "0.4"
 UnPack = "0"
-VectorizationBase = "0.9.6"
+VectorizationBase = "0.10"
 julia = "1.1"
 
 [extras]

benchmark/directcalljit.f90

@@ -1,22 +1,52 @@
 module jitmul
 
-include "/opt/intel/mkl/include/mkl_direct_call.fi"
-
 use ISO_C_BINDING
+use mkl_service
 implicit none
 
-contains
+include "/opt/intel/mkl/include/mkl_direct_call.fi"
+! include "/opt/intel/mkl/include/mkl_service.fi"
+! include "/opt/intel/mkl/include/mkl.fi"
 
+contains
+  subroutine set_num_threads(N) bind(C, name = "set_num_threads")
+    integer(C_int32_t) :: N
+    call mkl_set_num_threads(N)
+  end subroutine set_num_threads
+  
     ! subroutine dgemmjit(C,A,B,M,K,N,alpha,beta) bind(C, name = "dgemmjit")
+  subroutine sgemmjit(C,A,B,M,K,N,At,Bt) bind(C, name = "sgemmjit")
+    integer(C_int32_t),  intent(in)  :: M, K, N
+    integer(C_int8_t),   intent(in) :: At, Bt
+    real(C_float), parameter :: alpha = 1.0e0, beta = 0.0e0
+    ! real(C_float),                 intent(in)  :: alpha, beta
+    real(C_float), dimension(M,K), intent(in)  :: A
+    real(C_float), dimension(K,N), intent(in)  :: B
+    real(C_float), dimension(M,N), intent(out) :: C
+    character :: Atc, Btc
+    ! call mkl_set_threading_layer(MKL_THREADING_SEQUENTIAL)
+    if (At == 1_C_int8_t) then
+       Atc = 'T'
+    else
+       Atc = 'N'
+    end if
+    if (Bt == 1_C_int8_t) then
+       Btc = 'T'
+    else
+       Btc = 'N'
+    end if
+    call sgemm(Atc, Btc, M, N, K, alpha, A, M, B, K, beta, C, M)
+  end subroutine sgemmjit
   subroutine dgemmjit(C,A,B,M,K,N,At,Bt) bind(C, name = "dgemmjit")
     integer(C_int32_t),  intent(in)  :: M, K, N
     integer(C_int8_t),   intent(in) :: At, Bt
-    real(C_double), parameter :: alpha = 1.0D0, beta = 0.0D0
+    real(C_double), parameter :: alpha = 1.0d0, beta = 0.0d0
     ! real(C_double),                 intent(in)  :: alpha, beta
     real(C_double), dimension(M,K), intent(in)  :: A
     real(C_double), dimension(K,N), intent(in)  :: B
     real(C_double), dimension(M,N), intent(out) :: C
     character :: Atc, Btc
+    ! call mkl_set_threading_layer(MKL_THREADING_SEQUENTIAL)
     if (At == 1_C_int8_t) then
        Atc = 'T'
     else

src/LoopVectorization.jl

@@ -7,7 +7,7 @@ using VectorizationBase: REGISTER_SIZE, REGISTER_COUNT, extract_data, num_vector
     Static, StaticUnitRange, StaticLowerUnitRange, StaticUpperUnitRange, unwrap, maybestaticrange,
     AbstractColumnMajorStridedPointer, AbstractRowMajorStridedPointer, AbstractSparseStridedPointer, AbstractStaticStridedPointer,
     PackedStridedPointer, SparseStridedPointer, RowMajorStridedPointer, StaticStridedPointer, StaticStridedStruct,
-    maybestaticfirst, maybestaticlast
+    maybestaticfirst, maybestaticlast, scalar_less, scalar_greater
 using SIMDPirates: VECTOR_SYMBOLS, evadd, evsub, evmul, evfdiv, vrange, reduced_add, reduced_prod, reduce_to_add, reduce_to_prod,
     sizeequivalentfloat, sizeequivalentint, vadd!, vsub!, vmul!, vfdiv!, vfmadd!, vfnmadd!, vfmsub!, vfnmsub!,
     vfmadd231, vfmsub231, vfnmadd231, vfnmsub231, sizeequivalentfloat, sizeequivalentint, #prefetch,

src/add_stores.jl

@@ -35,16 +35,18 @@ function add_store!(
     pvar = name(parent)
     id = length(ls.operations)
     # try to cse store, by replacing the previous one
-    ref = mpref.mref.ref
+    mref = mpref.mref
     add_pvar = true
     for opp ∈ operations(ls)
         isstore(opp) || continue
-        if ref == opp.ref.ref
+        if mref == opp.ref
             id = opp.identifier
+            add_pvar = false
             break
         end
-        add_pvar &= (name(first(parents(opp))) != pvar)
+        # add_pvar &= (name(first(parents(opp))) != pvar)
     end
+    # @show add_pvar
     pushfirst!(vparents, parent)
     update_deps!(ldref, reduceddeps, parent)
     op = Operation( id, name(mpref), elementbytes, :setindex!, memstore, mpref )

src/graphs.jl

@@ -66,17 +66,17 @@ function vec_looprange(loop::Loop, W::Symbol, UF::Int, mangledname::Symbol)
         Expr(:call, lv(:valsub), W, 2)
     end
     if loop.stopexact # split for type stability
-        Expr(:call, :<, mangledname, Expr(:call, :-, loop.stophint, incr))
+        Expr(:call, lv(:scalar_less), mangledname, Expr(:call, :-, loop.stophint, incr))
     else
-        Expr(:call, :<, mangledname, Expr(:call, :-, loop.stopsym, incr))
+        Expr(:call, lv(:scalar_less), mangledname, Expr(:call, :-, loop.stopsym, incr))
     end
 end
 function looprange(loop::Loop, incr::Int, mangledname::Symbol)
     incr = 2 - incr
     if iszero(incr)
-        Expr(:call, :<, mangledname, loop.stopexact ? loop.stophint : loop.stopsym)
+        Expr(:call, lv(:scalar_less), mangledname, loop.stopexact ? loop.stophint : loop.stopsym)
     else
-        Expr(:call, :<, mangledname, loop.stopexact ? loop.stophint + incr : Expr(:call, :+, loop.stopsym, incr))
+        Expr(:call, lv(:scalar_less), mangledname, loop.stopexact ? loop.stophint + incr : Expr(:call, :+, loop.stopsym, incr))
     end
 end
 function terminatecondition(

src/lower_store.jl

@@ -1,5 +1,5 @@
 using VectorizationBase: vnoaliasstore!
-# const STOREOP = :vnoaliasstore!
+
 
 @inline vstoreadditivereduce!(args...) = vnoaliasstore!(args...)
 @inline vstoremultiplicativevereduce!(args...) = vnoaliasstore!(args...)
@@ -21,16 +21,18 @@ function storeinstr(op::Operation)
     if instruction(opp).instr === :identity
         opp = first(parents(opp))
     end
+    defaultstoreop = :vnoaliasstore!
+    # defaultstoreop = :vstore!
     instr = if iszero(length(reduceddependencies(opp)))
-        :vnoaliasstore!
+        defaultstoreop
     else
         instr_class = reduction_instruction_class(instruction(opp))
         if instr_class === ADDITIVE_IN_REDUCTIONS
             :vstoreadditivereduce!
         elseif instr_class === MULTIPLICATIVE_IN_REDUCTIONS
             :vstoremultiplicativevereduce!
         else #FIXME
-            :vnoaliasstore!
+            defaultstoreop
         end
     end
     lv(instr)

src/lowering.jl

@@ -191,11 +191,11 @@ function lower_unrolled_dynamic(ls::LoopSet, us::UnrollSpecification, n::Int, in
             else
                 Expr(:call, :-, loop.stopsym, Expr(:call, lv(:valmul), ls.W, UFt))
             end
-            Expr(:call, :>, loopsym, itercount)
+            Expr(:call, lv(:scalar_greater), loopsym, itercount)
         elseif loop.stopexact
-            Expr(:call, :>, loopsym, loop.stophint - UFt)
+            Expr(:call, lv(:scalar_greater), loopsym, loop.stophint - UFt)
         else
-            Expr(:call, :>, loopsym, Expr(:call, :-, loop.stopsym, UFt))
+            Expr(:call, lv(:scalar_greater), loopsym, Expr(:call, :-, loop.stopsym, UFt))
         end
         ust = nisunrolled ? UnrollSpecification(us, UFt, T) : UnrollSpecification(us, U, UFt)
         remblocknew = Expr(:elseif, comparison, lower_block(ls, ust, n, remmask, UFt))
@@ -244,15 +244,15 @@ function add_upper_outer_reductions(ls::LoopSet, loopq::Expr, Ulow::Int, Uhigh::
     push!(ifq.args, loopq)
     reduce_range!(ifq, ls, Ulow, Uhigh)
     comparison = if isstaticloop(unrolledloop)
-        Expr(:call, :<, length(unrolledloop), Expr(:call, lv(:valmul), ls.W, Uhigh))
+        Expr(:call, lv(:scalar_less), length(unrolledloop), Expr(:call, lv(:valmul), ls.W, Uhigh))
     elseif unrolledloop.starthint == 1
-        Expr(:call, :<, unrolledloop.stopsym, Expr(:call, lv(:valmul), ls.W, Uhigh))
+        Expr(:call, lv(:scalar_less), unrolledloop.stopsym, Expr(:call, lv(:valmul), ls.W, Uhigh))
     elseif unrolledloop.startexact
-        Expr(:call, :<, Expr(:call, :-, unrolledloop.stopsym, unrolledloop.starthint-1), Expr(:call, lv(:valmul), ls.W, Uhigh))
+        Expr(:call, lv(:scalar_less), Expr(:call, :-, unrolledloop.stopsym, unrolledloop.starthint-1), Expr(:call, lv(:valmul), ls.W, Uhigh))
     elseif unrolledloop.stopexact
-        Expr(:call, :<, Expr(:call, :-, unrolledloop.stophint+1, unrolledloop.sartsym), Expr(:call, lv(:valmul), ls.W, Uhigh))
+        Expr(:call, lv(:scalar_less), Expr(:call, :-, unrolledloop.stophint+1, unrolledloop.sartsym), Expr(:call, lv(:valmul), ls.W, Uhigh))
     else# both are given by symbols
-        Expr(:call, :<, Expr(:call, :-, unrolledloop.stopsym, Expr(:call,:-,unrolledloop.startsym)), Expr(:call, lv(:valmul), ls.W, Uhigh))
+        Expr(:call, lv(:scalar_less), Expr(:call, :-, unrolledloop.stopsym, Expr(:call,:-,unrolledloop.startsym)), Expr(:call, lv(:valmul), ls.W, Uhigh))
     end
     ncomparison = Expr(:call, :!, comparison)
     Expr(:if, ncomparison, ifq)
@@ -307,9 +307,9 @@ function determine_width(ls::LoopSet, vectorized::Symbol)
 end
 function init_remblock(unrolledloop::Loop, unrolled::Symbol = unrolledloop.itersymbol)
     condition = if unrolledloop.stopexact
-        Expr(:call, :(>), unrolled, unrolledloop.stophint)
+        Expr(:call, lv(:scalar_greater), unrolled, unrolledloop.stophint)
     else
-        Expr(:call, :(>), unrolled, unrolledloop.stopsym)
+        Expr(:call, lv(:scalar_greater), unrolled, unrolledloop.stopsym)
     end
     Expr(:if, condition, nothing)
 end