Switch to LLVMPtr

Author: chriselrod

src/LoopVectorization.jl

@@ -6,7 +6,7 @@ using VectorizationBase: register_size, register_count, cache_linesize, cache_si
     mask, pick_vector_width, MM, AbstractMask, data, grouped_strided_pointer,
     maybestaticlength, maybestaticsize, staticm1, staticp1, staticmul, vzero,
     maybestaticrange, offsetprecalc, lazymul,
-    maybestaticfirst, maybestaticlast, scalar_less, scalar_greaterequal, gep, gesp, pointerforcomparison, NativeTypes,
+    maybestaticfirst, maybestaticlast, scalar_less, scalar_greaterequal, gep, gesp, llvmptr, NativeTypes,
     vfmadd, vfmsub, vfnmadd, vfnmsub, vfmadd_fast, vfmsub_fast, vfnmadd_fast, vfnmsub_fast, vfmadd231, vfmsub231, vfnmadd231, vfnmsub231,
     vfma_fast, vmuladd_fast, vdiv_fast, vadd_fast, vsub_fast, vmul_fast,
     relu, stridedpointer, StridedPointer, StridedBitPointer, AbstractStridedPointer, _vload, _vstore!,

src/broadcast.jl

@@ -162,6 +162,7 @@ Base.@propagate_inbounds Base.getindex(A::LowDimArray, i::Vararg{Union{Integer,C
 @inline Base.strides(A::LowDimArray) = strides(A.data)
 @inline ArrayInterface.parent_type(::Type{LowDimArray{D,T,N,A}}) where {T,D,N,A} = A
 @inline ArrayInterface.strides(A::LowDimArray) = ArrayInterface.strides(A.data)
+@inline ArrayInterface.device(::LowDimArray) = ArrayInterface.CPUPointer()
 @generated function ArrayInterface.size(A::LowDimArray{D,T,N}) where {D,T,N}
     t = Expr(:tuple)
     for n ∈ 1:N
@@ -206,7 +207,7 @@ end
         Nnew += 1
     end
     typ = Expr(:curly, :StridedPointer, T, Nnew, Cnew, Bnew, Rtup)
-    ptr = Expr(:call, typ, :(pointer(p)), strd, offsets)
+    ptr = Expr(:call, typ, Expr(:call, lv(:llvmptr), :p), strd, offsets)
     Expr(:block, Expr(:meta,:inline), :(strd = p.strd), :(offs = p.offsets), ptr)
 end
 # @generated function VectorizationBase.stridedpointer(A::LowDimArray{D,T,N}) where {D,T,N}

src/codegen/loopstartstopmanager.jl

@@ -65,7 +65,7 @@ end
     newB = C > 0 ? (C == minrank ? B : 0) : B #TODO: confirm correctness
     quote
         $(Expr(:meta,:inline))
-        VectorizationBase.StridedPointer{$T,1,$newC,$newB,$(R[minrank],)}(pointer(sptr), (sptr.strd[$minrank],), (Zero(),))
+        VectorizationBase.StridedPointer{$T,1,$newC,$newB,$(R[minrank],)}($(lv(llvmptr))(sptr), (sptr.strd[$minrank],), (Zero(),))
     end
 end
 set_first_stride(x) = x # cross fingers that this works
@@ -310,7 +310,7 @@ function pointermax_index(ls::LoopSet, ar::ArrayReferenceMeta, n::Int, sub::Int,
 end
 function pointermax(ls::LoopSet, ar::ArrayReferenceMeta, n::Int, sub::Int, isvectorized::Bool, stopsym, incr::MaybeKnown)::Expr
     index = first(pointermax_index(ls, ar, n, sub, isvectorized, stopsym, incr))
-    Expr(:call, lv(:pointerforcomparison), vptr(ar), index)
+    Expr(:call, lv(:gesp), vptr(ar), index)
 end
 
 function defpointermax(ls::LoopSet, ar::ArrayReferenceMeta, n::Int, sub::Int, isvectorized::Bool)::Expr
@@ -347,16 +347,14 @@ function append_pointer_maxes!(
         # @show n, getloop(ls, n) ar
         index, ind = pointermax_index(ls, ar, n, submax, isvectorized, stopindicator, incr)
         vptr_ar = vptr(ar)
-        _pointercompbase = maxsym(vptr_ar, submax)
-        pointercompbase = gensym(_pointercompbase)
+        pointercompbase = maxsym(vptr_ar, submax)
         push!(loopstart.args, Expr(:(=), pointercompbase, Expr(:call, lv(:gesp), vptr_ar, index)))
-        push!(loopstart.args, Expr(:(=), _pointercompbase, Expr(:call, lv(:pointerforcomparison), pointercompbase)))
         dim = length(getindicesonly(ar))
         # OFFSETPRECALCDEF = true
         # if OFFSETPRECALCDEF
         strd = getstrides(ar)[dim]
         for sub ∈ 0:submax-1
-            ptrcmp = Expr(:call, lv(:pointerforcomparison), pointercompbase, offsetindex(dim, ind, (submax - sub)*strd, isvectorized, incr))
+            ptrcmp = Expr(:call, lv(:gesp), pointercompbase, offsetindex(dim, ind, (submax - sub)*strd, isvectorized, incr))
             push!(loopstart.args, Expr(:(=), maxsym(vptr_ar, sub), ptrcmp))
         end
         # else
@@ -466,9 +464,9 @@ function terminatecondition(ls::LoopSet, us::UnrollSpecification, n::Int, inclma
     ptr = vptr(termar)
     # @show UF, isvectorized(us, n)
     if inclmask && isvectorized(us, n)
-        Expr(:call, :<, callpointerforcomparison(ptr), maxsym(ptr, 0))
+        Expr(:call, :<, ptr, maxsym(ptr, 0))
     else
-        Expr(:call, :≤, callpointerforcomparison(ptr), maxsym(ptr, UF))
+        Expr(:call, :≤, ptr, maxsym(ptr, UF))
     end
 end
 

src/codegen/lower_threads.jl

@@ -7,7 +7,7 @@ function (::AVX{UNROLL,OPS,ARF,AM,LPSYM,LB,V})(p::Ptr{UInt}) where {UNROLL,OPS,A
     (_, _vargs) = ThreadingUtilities.load(p, Tuple{LB,V}, 2*sizeof(UInt))
     # Main.VARGS[Threads.threadid()] = first(_vargs)
     ret = _avx_!(Val{UNROLL}(), Val{OPS}(), Val{ARF}(), Val{AM}(), Val{LPSYM}(), _vargs)
-    ThreadingUtilities.store!(p, ret, 64)
+    ThreadingUtilities.store!(p, ret, Int(register_size()))
     nothing
 end
 @generated function Base.pointer(::AVX{UNROLL,OPS,ARF,AM,LPSYM,LB,V}) where {UNROLL,OPS,ARF,AM,LPSYM,LB,V}
@@ -28,16 +28,11 @@ function launch(
     ::Val{UNROLL}, ::Val{OPS}, ::Val{ARF}, ::Val{AM}, ::Val{LPSYM}, lb::LB, vargs::V, tid
 ) where {UNROLL,OPS,ARF,AM,LPSYM,LB,V}
     p = ThreadingUtilities.taskpointer(tid)
-    f = AVX{UNROLL,OPS,ARF,AM,LPSYM,LB,V}()
-    fptr = pointer(f)
+    launch!(p, pointer(AVX{UNROLL,OPS,ARF,AM,LPSYM,LB,V}()), (lb,vargs))
     while true
-        if ThreadingUtilities._atomic_cas_cmp!(p, ThreadingUtilities.SPIN, ThreadingUtilities.STUP)
-            launch!(p, fptr, (lb,vargs))
-            @assert ThreadingUtilities._atomic_cas_cmp!(p, ThreadingUtilities.STUP, ThreadingUtilities.TASK)
+        if ThreadingUtilities._atomic_cas_cmp!(p, ThreadingUtilities.SPIN, ThreadingUtilities.TASK)
             return
-        elseif ThreadingUtilities._atomic_cas_cmp!(p, ThreadingUtilities.WAIT, ThreadingUtilities.STUP)
-            launch!(p, fptr, (lb,vargs))
-            @assert ThreadingUtilities._atomic_cas_cmp!(p, ThreadingUtilities.STUP, ThreadingUtilities.LOCK)
+        elseif ThreadingUtilities._atomic_cas_cmp!(p, ThreadingUtilities.WAIT, ThreadingUtilities.LOCK)
             ThreadingUtilities.wake_thread!(tid % UInt)
             return
         end

src/codegen/lowering.jl

@@ -425,12 +425,10 @@ function pointerremcomparison(ls::LoopSet, termind::Int, UFt::Int, n::Int, nisve
     termar = lssm.incrementedptrs[n][termind]
     ptrdef = lssm.incrementedptrs[n][termind]
     ptr = vptr(termar)
-    ptrex = callpointerforcomparison(ptr)
     if remfirst
-        Expr(:call, :<, ptrex, pointermax(ls, ptrdef, n, 1 - UFt, nisvectorized, loop))
+        Expr(:call, :<, ptr, pointermax(ls, ptrdef, n, 1 - UFt, nisvectorized, loop))
     else
-        # Expr(:call, :≥, ptrex, pointermax(ls, ptrdef, n, UFt, nisvectorized, loop))
-        Expr(:call, :≥, ptrex, maxsym(ptr, UFt))
+        Expr(:call, :≥, ptr, maxsym(ptr, UFt))
     end
 end
 
@@ -657,7 +655,7 @@ function init_remblock(unrolledloop::Loop, lssm::LoopStartStopManager, n::Int)#u
     else
         termar = lssm.incrementedptrs[n][termind]
         ptr = vptr(termar)
-        condition = Expr(:call, :<, callpointerforcomparison(ptr), maxsym(ptr, 0))
+        condition = Expr(:call, :<, vptr(ptr), maxsym(ptr, 0))
     end
     Expr(:if, condition)
 end

src/modeling/graphs.jl

@@ -201,7 +201,6 @@ function addexpr(ex, incr::Integer)
 end
 
 staticmulincr(ptr, incr) = Expr(:call, lv(:staticmul), Expr(:call, :eltype, ptr), incr)
-callpointerforcomparison(sym) = Expr(:call, lv(:pointerforcomparison), sym)
 function vec_looprange(loop::Loop, UF::Int, mangledname)
     compexpr = Expr(:call, lv(:vsub_fast))
     pushexpr!(compexpr, last(loop))

src/simdfunctionals/filter.jl

@@ -8,17 +8,17 @@ function vfilter!(f::F, x::Vector{T}, y::AbstractArray{T}) where {F,T <: NativeT
     st = VectorizationBase.static_sizeof(T)
     zero_index = MM(W, Static(0), st)
     GC.@preserve x y begin
-        ptr_x = pointer(x)
-        ptr_y = pointer(y)
+        ptr_x = llvmptr(x)
+        ptr_y = llvmptr(y)
         for _ ∈ 1:Nrep
-            vy = vload(ptr_y, zero_index)
+            vy = VectorizationBase.__vload(ptr_y, zero_index, False(), register_size())
             mask = f(vy)
             VectorizationBase.compressstore!(gep(ptr_x, VectorizationBase.lazymul(st, j)), vy, mask)
             ptr_y = gep(ptr_y, register_size())
             j = vadd_fast(j, count_ones(mask))
         end
         rem_mask = VectorizationBase.mask(T, Nrem)
-        vy = vload(ptr_y, zero_index, rem_mask)
+        vy = VectorizationBase.__vload(ptr_y, zero_index, rem_mask, False(), register_size())
         mask = rem_mask & f(vy)
         VectorizationBase.compressstore!(gep(ptr_x, VectorizationBase.lazymul(st, j)), vy, mask)
         j = vadd_fast(j, count_ones(mask))

src/simdfunctionals/map.jl

@@ -107,7 +107,7 @@ end
 #     :(_vmap_thread_call!($(F.instance), p, $D, $A, Val{$NonTemporal}()))
 # end
 function (m::VmapClosure{NonTemporal,F,D,N,A})(p::Ptr{UInt}) where {NonTemporal,F,D,N,A}
-    (offset, dest) = ThreadingUtilities.load(p, D, 1)
+    (offset, dest) = ThreadingUtilities.load(p, D, 2*sizeof(UInt))
     (offset, args) = ThreadingUtilities.load(p, A, offset)
 
     (offset, start) = ThreadingUtilities.load(p, Int, offset)
@@ -132,7 +132,7 @@ end
     p, cfunc, ptry, ptrargs, start, stop
 )
     fptr = _get_fptr(cfunc)
-    offset = ThreadingUtilities.store!(p, fptr, 0)
+    offset = ThreadingUtilities.store!(p, fptr, sizeof(UInt))
     offset = ThreadingUtilities.store!(p, ptry, offset)
     offset = ThreadingUtilities.store!(p, ptrargs, offset)
     offset = ThreadingUtilities.store!(p, start, offset)

src/simdfunctionals/mapreduce.jl

@@ -19,9 +19,9 @@ function mapreduce_simple(f::F, op::OP, args::Vararg{AbstractArray,A}) where {F,
     N = length(first(args))
     iszero(N) && throw("Length of vector is 0!")
     st = ntuple(a -> VectorizationBase.static_sizeof(eltype(args[a])), Val(A))
-    a_0 = f(vload.(ptrargs)...); i = 1
+    a_0 = f(VectorizationBase.__vload.(ptrargs, False(), register_size())...); i = 1
     while i < N
-        a_0 = op(a_0, f(vload.(ptrargs, VectorizationBase.lazymul.(st, i))...)); i += 1
+        a_0 = op(a_0, f(VectorizationBase.vload.(ptrargs, VectorizationBase.lazymul.(st, i), False(), register_size())...)); i += 1
     end
     a_0
 end