fix 3-level scan downsweep step

Author: keptsecret

examples_tests

@@ -305,15 +305,15 @@ struct scan<Config, BinOp, Exclusive, 3, device_capabilities>
         // level 1 scan
         const uint32_t lv1_smem_size = Config::__ItemsPerVirtualWorkgroup;
         const uint32_t lv1_num_invoc = Config::SubgroupSize*Config::ItemsPerInvocation_2;
-        subgroup2::exclusive_scan<params_lv1_t> exclusiveScan1;
+        subgroup2::inclusive_scan<params_lv1_t> inclusiveScan1;
         if (glsl::gl_SubgroupID() < lv1_num_invoc)
         {
             vector_lv1_t lv1_val;
             [unroll]
             for (uint32_t i = 0; i < Config::ItemsPerInvocation_1; i++)
                 scratchAccessor.template get<scalar_t, uint32_t>(Config::template sharedLoadIndex<1>(invocationIndex, i),lv1_val[i]);
             // lv1_val[0] = hlsl::mix(BinOp::identity, lv1_val[0], bool(invocationIndex));
-            lv1_val = exclusiveScan1(lv1_val);
+            lv1_val = inclusiveScan1(lv1_val);
             [unroll]
             for (uint32_t i = 0; i < Config::ItemsPerInvocation_1; i++)
                 scratchAccessor.template set<scalar_t, uint32_t>(Config::template sharedLoadIndex<1>(invocationIndex, i),lv1_val[i]);
@@ -333,7 +333,7 @@ struct scan<Config, BinOp, Exclusive, 3, device_capabilities>
             [unroll]
             for (uint32_t i = 0; i < Config::ItemsPerInvocation_2; i++)
                 scratchAccessor.template get<scalar_t, uint32_t>(lv1_smem_size+Config::template sharedLoadIndex<2>(invocationIndex, i),lv2_val[i]);
-            lv2_val[0] = hlsl::mix(BinOp::identity, lv2_val[0], bool(invocationIndex));
+            // lv2_val[0] = hlsl::mix(BinOp::identity, lv2_val[0], bool(invocationIndex));
             lv2_val = exclusiveScan2(lv2_val);
             [unroll]
             for (uint32_t i = 0; i < Config::ItemsPerInvocation_2; i++)
@@ -347,16 +347,20 @@ struct scan<Config, BinOp, Exclusive, 3, device_capabilities>
             vector_lv1_t lv1_val;
             [unroll]
             for (uint32_t i = 0; i < Config::ItemsPerInvocation_1; i++)
-                scratchAccessor.template get<scalar_t, uint32_t>(i*Config::SubgroupSize+invocationIndex,lv1_val[i]);
+                scratchAccessor.template get<scalar_t, uint32_t>(Config::template sharedLoadIndex<1>(invocationIndex, i), lv1_val[i]);
+
+            const scalar_t left_last_elem = hlsl::mix(BinOp::identity, glsl::subgroupShuffleUp<scalar_t>(lv1_val[Config::ItemsPerInvocation_1-1],1), bool(glsl::gl_SubgroupInvocationID()));
 
             scalar_t lv2_scan;
             const uint32_t bankedIndex = Config::template sharedStoreIndex<2>(glsl::gl_SubgroupID());
-            scratchAccessor.template set<scalar_t, uint32_t>(lv1_smem_size+bankedIndex, lv2_scan);
+            scratchAccessor.template get<scalar_t, uint32_t>(lv1_smem_size+bankedIndex, lv2_scan);
 
             [unroll]
-            for (uint32_t i = 0; i < Config::ItemsPerInvocation_1; i++)
-                scratchAccessor.template set<scalar_t, uint32_t>(Config::template sharedLoadIndex<1>(invocationIndex, i), binop(lv1_val[i],lv2_scan));
+            for (uint32_t i = Config::ItemsPerInvocation_1-1; i > 0; i--)
+                scratchAccessor.template set<scalar_t, uint32_t>(Config::template sharedLoadIndex<1>(invocationIndex, i), binop(lv1_val[i-1],lv2_scan));
+            scratchAccessor.template set<scalar_t, uint32_t>(Config::template sharedLoadIndex<1>(invocationIndex, 0), binop(left_last_elem,lv2_scan));
         }
+        scratchAccessor.workgroupExecutionAndMemoryBarrier();
 
         // combine with level 0
         [unroll]