Accidentally bumped version too high, and added support for transposed destinations.

Author: chriselrod

Project.toml

@@ -1,7 +1,7 @@
 name = "LoopVectorization"
 uuid = "bdcacae8-1622-11e9-2a5c-532679323890"
 authors = ["Chris Elrod <[email protected]>"]
-version = "0.1.5"
+version = "0.1.4"
 
 [deps]
 LinearAlgebra = "37e2e46d-f89d-539d-b4ee-838fcccc9c8e"

src/broadcast.jl

@@ -161,12 +161,11 @@ end
 @generated function vmaterialize!(
     dest::AbstractArray{T,N}, bc::BC
 ) where {T <: Union{Float32,Float64}, N, BC <: Broadcasted}
-# ) where {N, T, BC <: Broadcasted}
     # we have an N dimensional loop.
     # need to construct the LoopSet
     loopsyms = [gensym(:n) for n ∈ 1:N]
     ls = LoopSet()
-    sizes = Expr(:tuple,)
+    sizes = Expr(:tuple)
     for (n,itersym) ∈ enumerate(loopsyms)
         Nsym = gensym(:N)
         ls.loops[itersym] = Loop(itersym, Nsym)
@@ -183,6 +182,31 @@ end
     q
     # ls
 end
+@generated function vmaterialize!(
+    dest′::Union{Adjoint{T,A},Transpose{T,A}}, bc::BC
+) where {T <: Union{Float32,Float64}, N, A <: AbstractArray{T,N}, BC <: Broadcasted}
+    # we have an N dimensional loop.
+    # need to construct the LoopSet
+    loopsyms = [gensym(:n) for n ∈ 1:N]
+    ls = LoopSet()
+    pushpreamble!(ls, Expr(:(=), :dest, Expr(:call, :parent, :dest′)))
+    sizes = Expr(:tuple)
+    for (n,itersym) ∈ enumerate(loopsyms)
+        Nsym = gensym(:N)
+        ls.loops[itersym] = Loop(itersym, Nsym)
+        push!(sizes.args, Nsym)
+    end
+    pushpreamble!(ls, Expr(:(=), sizes, Expr(:call, :size, :dest′)))
+    elementbytes = sizeof(T)
+    add_broadcast!(ls, :dest, :bc, loopsyms, BC, elementbytes)
+    add_store!(ls, :dest, ArrayReference(:dest, reverse(loopsyms), Ref{Bool}(false)), elementbytes)
+    resize!(ls.loop_order, num_loops(ls)) # num_loops may be greater than N, eg Product
+    q = lower(ls)
+    push!(q.args, :dest′)
+    pushfirst!(q.args, Expr(:meta,:inline))
+    q
+    # ls
+end
 
 function vmaterialize(bc::Broadcasted)
     ElType = Base.Broadcast.combine_eltypes(bc.f, bc.args)

test/runtests.jl

@@ -367,6 +367,19 @@ end
         @. D3 = exp(Bt)
         @avx @. D4 = exp(Bt)
         @test D3 ≈ D4
+
+        D1 = similar(B); D2 = similar(B)
+        D1t = Transpose(D1)
+        D2t = Transpose(D2)
+        @. D1t = exp(Bt)
+        @avx @. D2t = exp(Bt)
+        @test D1t ≈ D2t
+
+        fill!(D1, -1e3)
+        fill!(D2, 9e9)
+        @. D1' = exp(Bt)
+        @avx @. D2' = exp(Bt)
+        @test D1 ≈ D2
     end
 end