Pass arguments to _avx_! in registers. (#261)

* Pass arguments to _avx_! in registers.

* Use `new` to bypass inner constructors

* VectorizationBase 0.20

Author: chriselrod

Project.toml

@@ -1,7 +1,7 @@
 name = "LoopVectorization"
 uuid = "bdcacae8-1622-11e9-2a5c-532679323890"
 authors = ["Chris Elrod <[email protected]>"]
-version = "0.12.18"
+version = "0.12.19"
 
 [deps]
 ArrayInterface = "4fba245c-0d91-5ea0-9b3e-6abc04ee57a9"
@@ -20,17 +20,17 @@ VectorizationBase = "3d5dd08c-fd9d-11e8-17fa-ed2836048c2f"
 
 [compat]
 ArrayInterface = "3.1.9"
-CheapThreads = "0.1.3,0.2"
+CheapThreads = "0.2"
 DocStringExtensions = "0.8"
 IfElse = "0.1"
-OffsetArrays = "1.4.1, 1.5"
+OffsetArrays = "1.4.1"
 Requires = "1"
-SLEEFPirates = "0.6.14"
+SLEEFPirates = "0.6.18"
 Static = "0.2"
 StrideArraysCore = "0.1.5"
-ThreadingUtilities = "0.4.1"
+ThreadingUtilities = "0.4.2"
 UnPack = "1"
-VectorizationBase = "0.19.36"
+VectorizationBase = "0.20.1"
 julia = "1.5"
 
 [extras]

src/LoopVectorization.jl

@@ -3,10 +3,10 @@ module LoopVectorization
 using Static: StaticInt, gt
 using VectorizationBase, SLEEFPirates, UnPack, OffsetArrays
 using VectorizationBase: register_size, register_count, cache_linesize, cache_size, has_opmask_registers,
-    mask, pick_vector_width, MM, AbstractMask, data, grouped_strided_pointer,
-    maybestaticlength, maybestaticsize, staticm1, staticp1, staticmul, vzero,
-    maybestaticrange, offsetprecalc, lazymul,
-    maybestaticfirst, maybestaticlast, scalar_less, scalar_greaterequal, gep, gesp, NativeTypes, #llvmptr,
+  mask, pick_vector_width, MM, AbstractMask, data, grouped_strided_pointer,
+  maybestaticlength, maybestaticsize, vzero, maybestaticrange, offsetprecalc, lazymul,
+  vadd_nw, vadd_nsw, vadd_nuw, vsub_nw, vsub_nsw, vsub_nuw, vmul_nw, vmul_nsw, vmul_nuw,
+    maybestaticfirst, maybestaticlast, gep, gesp, NativeTypes, #llvmptr,
     vfmadd, vfmsub, vfnmadd, vfnmsub, vfmadd_fast, vfmsub_fast, vfnmadd_fast, vfnmsub_fast, vfmadd231, vfmsub231, vfnmadd231, vfnmsub231,
     vfma_fast, vmuladd_fast, vdiv_fast, vadd_fast, vsub_fast, vmul_fast,
     relu, stridedpointer, stridedpointer_preserve, StridedPointer, StridedBitPointer, AbstractStridedPointer, _vload, _vstore!,

src/codegen/loopstartstopmanager.jl

@@ -679,9 +679,9 @@ function pointermax_index(
             else
                 _ind = if isvectorized
                     if isone(sub)
-                        Expr(:call, lv(:vsub_fast), staticexpr(stophint), VECTORWIDTHSYMBOL)
+                        Expr(:call, lv(:vsub_nsw), staticexpr(stophint), VECTORWIDTHSYMBOL)
                     else
-                        Expr(:call, lv(:vsub_fast), staticexpr(stophint), mulexpr(VECTORWIDTHSYMBOL, sub))
+                        Expr(:call, lv(:vsub_nsw), staticexpr(stophint), mulexpr(VECTORWIDTHSYMBOL, sub))
                     end
                 else
                     staticexpr(stophint - sub)
@@ -717,9 +717,9 @@ function pointermax_index(ls::LoopSet, ar::ArrayReferenceMeta, n::Int, sub::Int,
             else
                 _ind = if isvectorized
                     if isone(sub)
-                        Expr(:call, lv(:vsub_fast), stopsym, VECTORWIDTHSYMBOL)
+                        Expr(:call, lv(:vsub_nsw), stopsym, VECTORWIDTHSYMBOL)
                     else
-                        Expr(:call, lv(:vsub_fast), stopsym, mulexpr(VECTORWIDTHSYMBOL, sub))
+                        Expr(:call, lv(:vsub_nsw), stopsym, mulexpr(VECTORWIDTHSYMBOL, sub))
                     end
                 else
                      subexpr(stopsym, sub)
@@ -817,25 +817,26 @@ function maxunroll(us::UnrollSpecification, n)
 end
 
 
-function startloop(ls::LoopSet, us::UnrollSpecification, n::Int, submax = maxunroll(us, n))
-    @unpack u₁loopnum, u₂loopnum, vloopnum, u₁, u₂ = us
-    lssm = ls.lssm
-    termind = lssm.terminators[n]
-    ptrdefs = lssm.incrementedptrs[n]
-    loopstart = Expr(:block)
-    firstloop = n == num_loops(ls)
-    for ar ∈ ptrdefs
-      ptr_offset = vptr_offset(ar)
-      push!(loopstart.args, Expr(:(=), ptr_offset, ptr_offset))
-    end
-    if iszero(termind)
-      loopsym = names(ls)[n]
-      push!(loopstart.args, startloop(getloop(ls, loopsym), loopsym))
-    else
-        isvectorized = n == vloopnum
-        append_pointer_maxes!(loopstart, ls, ptrdefs[termind], n, submax, isvectorized)
-    end
-    loopstart
+function startloop(ls::LoopSet, us::UnrollSpecification, n::Int, staticinit::Bool = false)
+  @unpack u₁loopnum, u₂loopnum, vloopnum, u₁, u₂ = us
+  lssm = ls.lssm
+  termind = lssm.terminators[n]
+  ptrdefs = lssm.incrementedptrs[n]
+  loopstart = Expr(:block)
+  firstloop = n == num_loops(ls)
+  for ar ∈ ptrdefs
+    ptr_offset = vptr_offset(ar)
+    push!(loopstart.args, Expr(:(=), ptr_offset, ptr_offset))
+  end
+  if iszero(termind)
+    loopsym = names(ls)[n]
+    push!(loopstart.args, startloop(getloop(ls, loopsym), loopsym, staticinit))
+  else
+    isvectorized = n == vloopnum
+    submax = maxunroll(us, n)
+    append_pointer_maxes!(loopstart, ls, ptrdefs[termind], n, submax, isvectorized)
+  end
+  loopstart
 end
 function offset_ptr(
     ar::ArrayReferenceMeta, us::UnrollSpecification, loopsym::Symbol, n::Int, UF::Int, offsetinds::Vector{Bool}, loop::Loop

src/codegen/lower_compute.jl

@@ -69,15 +69,15 @@ function _add_loopvalue!(ex::Expr, loopval::Symbol, vloop::Loop, u::Int)
         else
             mm = _MMind(loopval, step(vloop))
             if isone(u)
-                push!(ex.args, Expr(:call, lv(:vadd_fast), VECTORWIDTHSYMBOL, mm))
+                push!(ex.args, Expr(:call, lv(:vadd_nsw), VECTORWIDTHSYMBOL, mm))
             else
-                push!(ex.args, Expr(:call, lv(:vadd_fast), Expr(:call, lv(:vmul_fast), VECTORWIDTHSYMBOL, u), mm))
+                push!(ex.args, Expr(:call, lv(:vadd_nsw), Expr(:call, lv(:vmul_nsw), VECTORWIDTHSYMBOL, u), mm))
             end
         end
     elseif u == 0
         push!(ex.args, loopval)
     else
-        push!(ex.args, Expr(:call, lv(:vadd_fast), loopval, staticexpr(u)))
+        push!(ex.args, Expr(:call, lv(:vadd_nsw), loopval, staticexpr(u)))
     end
 end
 function add_loopvalue!(instrcall::Expr, loopval, ua::UnrollArgs, u₁::Int)