Added subgroup support.

Author: EJainDev

src/kernels/level1/xdot.opencl

@@ -14,8 +14,10 @@
 // literal). Comment-out this line for syntax-highlighting when developing.
 R"(
 
-#ifdef cl_khr_work_group_uniform_arithmetic
+#if defined(cl_khr_work_group_uniform_arithmetic)
 #pragma OPENCL EXTENSION cl_khr_work_group_uniform_arithmetic : enable
+#elif defined(cl_khr_subgroups)
+#pragma OPENCL EXTENSION cl_khr_subgroups : enable
 #endif
 
 // Parameters set by the tuner or by the database. Here they are given a basic default value in case
@@ -66,12 +68,22 @@ void Xdot(const int n,
       output[wgid] = result;
     }
   #else
-    for (int s=WGS1/2; s>0; s=s>>1) {
-      if (lid < s) {
-        Add(lm[lid], lm[lid], lm[lid + s]);
-      }
+    #if defined(cl_khr_subgroups) || defined(__opencl_c_subgroups)
+      lm[get_sub_group_local_id()] = sub_group_reduce_add(lm[lid]);
       barrier(CLK_LOCAL_MEM_FENCE);
-    }
+      for (int s = get_num_sub_groups() >> 1; s > 0; s >>= 1) {
+        if (lid < s) {
+          Add(lm[lid], lm[lid], lm[lid + s]);
+        }
+      }
+    #else
+      for (int s=WGS1/2; s>0; s=s>>1) {
+        if (lid < s) {
+          Add(lm[lid], lm[lid], lm[lid + s]);
+        }
+        barrier(CLK_LOCAL_MEM_FENCE);
+      }
+    #endif
 
     if (lid == 0) {
       output[wgid] = lm[0];
@@ -97,23 +109,33 @@ void XdotEpilogue(const __global real* restrict input,
   Add(lm[lid], input[lid], input[lid + WGS2]);
   barrier(CLK_LOCAL_MEM_FENCE);
 
-  // Performs reduction in local memory and stores the per work group result
+  // Performs reduction in local memory and stores final result
   #if defined(cl_khr_work_group_uniform_arithmetic) || defined(__opencl_c_work_group_collective_functions)
     real result = work_group_reduce_add(lm[lid])
 
     if (lid == 0) {
       dot[dot_offset] = result;
     }
   #else
-    for (int s=WGS1/2; s>0; s=s>>1) {
-      if (lid < s) {
-        Add(lm[lid], lm[lid], lm[lid + s]);
-      }
+    #if defined(cl_khr_subgroups) || defined(__opencl_c_subgroups)
+      lm[get_sub_group_local_id()] = sub_group_reduce_add(lm[lid]);
       barrier(CLK_LOCAL_MEM_FENCE);
-    }
+      for (int s = get_num_sub_groups() >> 1; s > 0; s >>= 1) {
+        if (lid < s) {
+          Add(lm[lid], lm[lid], lm[lid + s]);
+        }
+      }
+    #else
+      for (int s=WGS1/2; s>0; s=s>>1) {
+        if (lid < s) {
+          Add(lm[lid], lm[lid], lm[lid + s]);
+        }
+        barrier(CLK_LOCAL_MEM_FENCE);
+      }
+    #endif
 
     if (lid == 0) {
-      dot[dot_offset] = lm[0];
+      dot[dot_offset] = lm[lid];
     }
   #endif
 }