Changes in code.

Author: Dimus77

IGC/BiFModule/Implementation/group.cl

@@ -1798,26 +1798,27 @@ DEFN_ARITH_OPERATIONS(double)
 DEFN_ARITH_OPERATIONS(half)
 #endif // defined(cl_khr_fp16)
 
-#define DEFN_WORK_GROUP_REDUCE(type, op, X)                                                 \
-{                                                                                           \
-    GET_MEMPOOL_PTR(data, type, true, 0)                                                    \
-    uint lid = __spirv_BuiltInLocalInvocationIndex();                                       \
-    uint lsize = __spirv_WorkgroupSize();                                                   \
-    data[lid] = X;                                                                          \
-                                                                                            \
-    uint i = 1 << ( ((8 * sizeof(uint)) - __builtin_spirv_OpenCL_clz_i32(lsize - 1)) - 1);                \
-    while(i > 0)                                                                                          \
-    {                                                                                                     \
+#define DEFN_WORK_GROUP_REDUCE(type, op, identity, X)                                       \
+{                                                                                          \
+    GET_MEMPOOL_PTR(data, type, true, 0)                                                     \
+    uint lid = __spirv_BuiltInLocalInvocationIndex();                                        \
+    uint lsize = __spirv_WorkgroupSize();                                                 \
+    data[lid] = X;                                                                       \
+    __builtin_spirv_OpControlBarrier_i32_i32_i32(Execution, 0, AcquireRelease | WorkgroupMemory);         \
+    uint mask = 1 << ( ((8 * sizeof(uint)) - __builtin_spirv_OpenCL_clz_i32(lsize - 1)) - 1) ;  \
+    while( mask > 0 )                                                                    \
+    {                                                                                    \
+        uint c = lid ^ mask;                                                             \
+        type other = ( c < lsize ) ? data[ c ] : identity;                                  \
+        X = op( other, X );                                                                 \
+        __builtin_spirv_OpControlBarrier_i32_i32_i32(Workgroup, 0, AcquireRelease | WorkgroupMemory);     \
+        data[lid] = X;                                                              \
         __builtin_spirv_OpControlBarrier_i32_i32_i32(Workgroup, 0, AcquireRelease | WorkgroupMemory);     \
-        if ((lid < i) && (lid + i < lsize))                                                               \
-        {                                                                                                 \
-            X = op(X, data[lid + i]);                                                                     \
-            data[lid] = X;                                                                                \
-        }                                                                                                 \
-        i >>= 1;                                                                                          \
-    }                                                                                                     \
+        mask >>= 1;                                                                      \
+    }                                                                                    \
+    type ret = data[0];                                                                  \
     __builtin_spirv_OpControlBarrier_i32_i32_i32(Workgroup, 0, AcquireRelease | WorkgroupMemory);         \
-    return data[0];                                                                                       \
+    return ret;                                                                           \
 }
 
 
@@ -1987,7 +1988,7 @@ DEFN_ARITH_OPERATIONS(half)
 {                                                                                         \
     switch(Operation){                                                                     \
         case GroupOperationReduce:                                                         \
-            DEFN_WORK_GROUP_REDUCE(type, op, X)                                            \
+            DEFN_WORK_GROUP_REDUCE(type, op, identity, X)                                 \
             break;                                                                         \
         case GroupOperationInclusiveScan:                                                 \
             DEFN_WORK_GROUP_SCAN_INCL(type, op, identity, X)                             \