Fix a statically sized codegen bug

Author: chriselrod

Project.toml

@@ -1,7 +1,7 @@
 name = "LoopVectorization"
 uuid = "bdcacae8-1622-11e9-2a5c-532679323890"
 authors = ["Chris Elrod <[email protected]>"]
-version = "0.12.158"
+version = "0.12.159"
 
 [weakdeps]
 ChainRulesCore = "d360d2e6-b24c-11e9-a2a3-2a2ae2dbcce4"

src/codegen/lower_compute.jl

@@ -323,7 +323,11 @@ end
     for k ∈ 1:K
       push!(q.args, :($gf(vargs, $k, false)))
     end
-    return Expr(:block, Expr(:meta, :inline), q)
+    if VERSION >= v"1.8"
+      return Expr(:block, Expr(:meta, :inline), :(@inline $q))
+    else
+      return Expr(:block, Expr(:meta, :inline), q)
+    end
   end
   if Sreduced
     M = N
@@ -346,6 +350,9 @@ end
         push!(call.args, Expr(:call, gf, syms[k], m, false))
       end
     end
+    if VERSION >= v"1.8"
+      call = :(@inline $call)
+    end
     # minimal change in behavior to fix case when !Sreduced by N -> Dlen; TODO: what should Dlen be here?
     if Sreduced ? (N == -1) : (Dlen == -1)
       push!(q.args, call)
@@ -516,6 +523,14 @@ function reduce_parent!(
   end
   newp
 end
+if VERSION >= v"1.8"
+  function inlinecall(x)
+    Meta.isexpr(x, :call) || return x
+    :(@inline $x)
+  end
+else
+  const inlinecall = identity
+end
 function lower_compute!(
   q::Expr,
   op::Operation,
@@ -769,7 +784,13 @@ function lower_compute!(
           Expr(
             :(=),
             varsym,
-            Expr(:call, lv(ifelsefunc), MASKSYMBOL, instrcall, selfopname)
+            Expr(
+              :call,
+              lv(ifelsefunc),
+              MASKSYMBOL,
+              inlinecall(instrcall),
+              selfopname
+            )
           )
         )
       elseif ((u₁ ≡ 1) | (selfdepreduce ≡ 0))
@@ -786,7 +807,7 @@ function lower_compute!(
               MASKSYMBOL,
               staticexpr(u₁),
               staticexpr(selfdepreduce),
-              instrcall,
+              inlinecall(instrcall),
               selfopname
             )
           )
@@ -799,7 +820,13 @@ function lower_compute!(
           Expr(
             :(=),
             varsym,
-            Expr(:call, lv(:ifelse), MASKSYMBOL, instrcall, selfopname)
+            Expr(
+              :call,
+              lv(:ifelse),
+              MASKSYMBOL,
+              inlinecall(instrcall),
+              selfopname
+            )
           )
         )
       end
@@ -832,11 +859,11 @@ function lower_compute!(
       Expr(
         :(=),
         varsym,
-        Expr(:call, lv(:vbroadcast), VECTORWIDTHSYMBOL, instrcall)
+        Expr(:call, lv(:vbroadcast), VECTORWIDTHSYMBOL, inlinecall(instrcall))
       )
     )
   else
-    push!(q.args, Expr(:(=), varsym, instrcall))
+    push!(q.args, Expr(:(=), varsym, inlinecall(instrcall)))
   end
   # end
 end

src/codegen/lowering.jl

@@ -311,7 +311,7 @@ function lower_unrolled_dynamic(
     loopisstatic = false
     loopisbounded = false
   end
-  Ureduct = (n == num_loops(ls) && (u₂ == -1)) ? ureduct(ls) : -1
+  Ureduct = ((n == num_loops(ls) && (u₂ == -1))) ? ureduct(ls) : -1
   # for now, require loopisstatic or !Ureduct-ing for reducing UF
   if loopisbounded & (loopisstatic | (Ureduct < 0))
     UFWnew = cld(looplength, cld(looplength, UFW))
@@ -390,9 +390,11 @@ function lower_unrolled_dynamic(
       initialize_outer_reductions!(q, ls, Ureduct)
       q
     elseif loopisstatic
+      blockhead = :block
       if length(loop) < UF * W
         Expr(:block)
       else
+        UFt -= Ureduct
         Expr(
           :block,
           add_upper_outer_reductions(ls, q, Ureduct, UF, loop, nisvectorized)

test/staticsize.jl

@@ -96,6 +96,13 @@ function sum_turbo(x)
   end
   s
 end
+function sum2_10turbo(x)
+  s = zero(eltype(x))
+  for i = 1:10, j = 1:2
+    s += x[j, i]
+  end
+  s
+end
 
 @testset "Statically Sized Arrays" begin
   @show @__LINE__
@@ -124,4 +131,7 @@ end
     @test maxabs(x) == maximum(abs, x)
     @test sum_turbo(x) ≈ sum(x)
   end
+  let A = rand(2, 10)
+    @test sum2_10turbo(A) ≈ sum(A)
+  end
 end