padding to shared mem indexing to avoid bank conflict

Author: keptsecret

include/nbl/builtin/hlsl/workgroup2/arithmetic_config.hlsl

@@ -70,10 +70,11 @@ struct ArithmeticConfiguration
     NBL_CONSTEXPR_STATIC_INLINE uint32_t SharedScratchElementCount = conditional_value<LevelCount==1,uint16_t,
         0,
         conditional_value<LevelCount==3,uint16_t,
-            LevelInputCount_2,
+            LevelInputCount_2+(SubgroupSize*ItemsPerInvocation_1)-1,
             0
             >::value + LevelInputCount_1
         >::value;
+    NBL_CONSTEXPR_STATIC_INLINE uint16_t __padding = conditional_value<LevelCount==3,uint16_t,SubgroupSize-1,0>::value;
 
     static bool electLast()
     {
@@ -90,40 +91,42 @@ struct ArithmeticConfiguration
     // get a coalesced index to store for the next level in shared mem, e.g. level 0 -> level 1
     // specify the next level to store values for in template param
     // at level==LevelCount-1, it is guaranteed to have SubgroupSize elements
-    template<uint16_t level NBL_FUNC_REQUIRES(level>0 && level<LevelCount)
+    template<uint16_t level>// NBL_FUNC_REQUIRES(level>0 && level<LevelCount)
     static uint16_t sharedStoreIndex(const uint16_t virtualSubgroupID)
     {
-        uint16_t offsetBySubgroup;
+        uint16_t nextLevelInvocationCount;
         if (level == LevelCount-1)
-            offsetBySubgroup = SubgroupSize;
+            nextLevelInvocationCount = SubgroupSize;
         else
-            offsetBySubgroup = __SubgroupsPerVirtualWorkgroup;
+            nextLevelInvocationCount = __SubgroupsPerVirtualWorkgroup;
 
         if (level==2)
-            return LevelInputCount_1 + (virtualSubgroupID & (ItemsPerInvocation_2-uint16_t(1u))) * offsetBySubgroup + (virtualSubgroupID/ItemsPerInvocation_2);
+            return LevelInputCount_1 + ((SubgroupSize-uint16_t(1u))*ItemsPerInvocation_1) + (virtualSubgroupID & (ItemsPerInvocation_2-uint16_t(1u))) * nextLevelInvocationCount + (virtualSubgroupID/ItemsPerInvocation_2);
         else
-            return (virtualSubgroupID & (ItemsPerInvocation_1-uint16_t(1u))) * offsetBySubgroup + (virtualSubgroupID/ItemsPerInvocation_1);
+            return (virtualSubgroupID & (ItemsPerInvocation_1-uint16_t(1u))) * (nextLevelInvocationCount+__padding) + (virtualSubgroupID/ItemsPerInvocation_1) + virtualSubgroupID/(SubgroupSize*ItemsPerInvocation_1);
     }
 
-    template<uint16_t level NBL_FUNC_REQUIRES(level>0 && level<LevelCount)
+    template<uint16_t level>// NBL_FUNC_REQUIRES(level>0 && level<LevelCount)
     static uint16_t sharedStoreIndexFromVirtualIndex(const uint16_t subgroupID, const uint16_t workgroupInVirtualIndex)
     {
         const uint16_t virtualID = virtualSubgroupID(subgroupID, workgroupInVirtualIndex);
         return sharedStoreIndex<level>(virtualID);
     }
 
     // get the coalesced index in shared mem at the current level
-    template<uint16_t level NBL_FUNC_REQUIRES(level>0 && level<LevelCount)
+    template<uint16_t level>// NBL_FUNC_REQUIRES(level>0 && level<LevelCount)
     static uint16_t sharedLoadIndex(const uint16_t invocationIndex, const uint16_t component)
     {
-        uint16_t smem_offset = 0u;
-        if (level == 2)
-            smem_offset += LevelInputCount_1;
-
+        uint16_t levelInvocationCount;
         if (level == LevelCount-1)
-            return component * SubgroupSize + invocationIndex + smem_offset;
+            levelInvocationCount = SubgroupSize;
+        else
+            levelInvocationCount = __SubgroupsPerVirtualWorkgroup;
+
+        if (level==2)
+            return LevelInputCount_1 + ((SubgroupSize-uint16_t(1u))*ItemsPerInvocation_1) + component * levelInvocationCount + invocationIndex + invocationIndex/SubgroupSize;
         else
-            return component * __SubgroupsPerVirtualWorkgroup + invocationIndex;
+            return component * (levelInvocationCount+__padding) + invocationIndex + invocationIndex/SubgroupSize;
     }
 };
 

include/nbl/builtin/hlsl/workgroup2/shared_scan.hlsl

@@ -330,11 +330,6 @@ struct scan<Config, BinOp, Exclusive, 3, device_capabilities>
             [unroll]
             for (uint16_t i = 0; i < Config::ItemsPerInvocation_1; i++)
                 scratchAccessor.template set<scalar_t, uint16_t>(Config::template sharedLoadIndex<1>(invocationIndex, i),lv1_val[i]);
-            if (Config::electLast())
-            {
-                const uint16_t bankedIndex = Config::template sharedStoreIndex<2>(uint16_t(glsl::gl_SubgroupID()));
-                scratchAccessor.template set<scalar_t, uint16_t>(bankedIndex, lv1_val[Config::ItemsPerInvocation_1-1]);
-            }
         }
         scratchAccessor.workgroupExecutionAndMemoryBarrier();
 
@@ -345,7 +340,7 @@ struct scan<Config, BinOp, Exclusive, 3, device_capabilities>
             vector_lv2_t lv2_val;
             [unroll]
             for (uint16_t i = 0; i < Config::ItemsPerInvocation_2; i++)
-                scratchAccessor.template get<scalar_t, uint16_t>(Config::template sharedLoadIndex<2>(invocationIndex, i),lv2_val[i]);
+                scratchAccessor.template get<scalar_t, uint16_t>(Config::template sharedLoadIndex<1>(((invocationIndex*Config::ItemsPerInvocation_1)+i+1)*Config::SubgroupSize-1, Config::ItemsPerInvocation_1-1),lv2_val[i]);
             lv2_val = inclusiveScan2(lv2_val);
             [unroll]
             for (uint16_t i = 0; i < Config::ItemsPerInvocation_2; i++)