Add smem aggregations implementations

Author: kurapov-peter

omniscidb/QueryEngine/Compiler/genx.cpp

@@ -25,6 +25,9 @@ void sync_threadblock();
 int64_t get_thread_index();
 int64_t get_block_dim();
 
+int32_t agg_sum_int32_shared(GENERIC_ADDR_SPACE int32_t* agg, const int32_t val);
+int64_t agg_sum_shared(GENERIC_ADDR_SPACE int64_t* agg, const int64_t val);
+void agg_max_int32_shared(GENERIC_ADDR_SPACE int32_t* agg, const int32_t val);
 void agg_max_shared(GENERIC_ADDR_SPACE int64_t* agg, const int64_t val);
 int64_t agg_count_shared(GENERIC_ADDR_SPACE int64_t* agg, const int64_t val);
 uint32_t agg_count_int32_shared(GENERIC_ADDR_SPACE uint32_t* agg, const int32_t val);
@@ -94,6 +97,50 @@ void agg_max_double_skip_val_shared(GENERIC_ADDR_SPACE int64_t* agg,
   }
 }
 
+int32_t atomicSum32SkipVal(GENERIC_ADDR_SPACE int32_t* addr,
+                           const int32_t val,
+                           const int32_t skip_val) {
+  int32_t old = atomic_xchg_int_32(addr, 0);
+  int32_t old2 = agg_sum_int32_shared(addr, old == skip_val ? val : (val + old));
+  return old == skip_val ? old2 : (old2 + old);
+}
+
+int64_t atomicSum64SkipVal(GENERIC_ADDR_SPACE int64_t* addr,
+                           const int64_t val,
+                           const int64_t skip_val) {
+  int32_t old = atomic_xchg_int_64(addr, 0);
+  int32_t old2 = agg_sum_shared(addr, old == skip_val ? val : (val + old));
+  return old == skip_val ? old2 : (old2 + old);
+}
+
+int32_t agg_sum_int32_skip_val_shared(GENERIC_ADDR_SPACE int32_t* agg,
+                                      const int32_t val,
+                                      const int32_t skip_val) {
+  if (val != skip_val) {
+    const int32_t old = atomicSum32SkipVal(agg, val, skip_val);
+    return old;
+  }
+  return 0;
+}
+
+int64_t agg_sum_int64_skip_val_shared(GENERIC_ADDR_SPACE int64_t* agg,
+                                      const int64_t val,
+                                      const int64_t skip_val) {
+  if (val != skip_val) {
+    const int64_t old = atomicSum64SkipVal(agg, val, skip_val);
+    return old;
+  }
+  return 0;
+}
+
+void agg_max_int32_skip_val_shared(GENERIC_ADDR_SPACE int32_t* agg,
+                                   const int32_t val,
+                                   const int32_t skip_val) {
+  if (val != skip_val) {
+    agg_max_int32_shared(agg, val);
+  }
+}
+
 const GENERIC_ADDR_SPACE int64_t* init_shared_mem(
     const GENERIC_ADDR_SPACE int64_t* global_groups_buffer,
     const int32_t groups_buffer_size) {

omniscidb/QueryEngine/Compiler/genx.ll

@@ -309,6 +309,11 @@ define i32 @atomic_xchg_int_32(i32 addrspace(4)* %p, i32 %val) {
     ret i32 %old
 }
 
+define void @agg_max_int32_shared(i32 addrspace(4)* %agg, i32 noundef %val) {
+    %old = atomicrmw max i32 addrspace(4)* %agg, i32 %val monotonic
+    ret void
+}
+
 define void @agg_max_shared(i64 addrspace(4)* %agg, i64 noundef %val) {
     %old = atomicrmw max i64 addrspace(4)* %agg, i64 %val monotonic
     ret void
@@ -330,6 +335,7 @@ define void @agg_min_shared(i64 addrspace(4)* %agg, i64 noundef %val) {
 }
 
 declare i64 @llvm.smin.i64(i64, i64)
+declare i32 @llvm.smin.i32(i32, i32)
 
 define void @agg_min_skip_val_shared(i64 addrspace(4)* %agg, i64 noundef %val, i64 noundef %skip_val) {
     %no_skip = icmp ne i64 %val, %skip_val
@@ -348,3 +354,21 @@ define void @agg_min_skip_val_shared(i64 addrspace(4)* %agg, i64 noundef %val, i
 .skip:
     ret void
 }
+
+define void @agg_min_int32_skip_val_shared(i32 addrspace(4)* %agg, i32 noundef %val, i32 noundef %skip_val) {
+    %no_skip = icmp ne i32 %val, %skip_val
+    br i1 %no_skip, label %.noskip, label %.skip
+.noskip:
+    %orig = load atomic i32, i32 addrspace(4)* %agg unordered, align 8
+    br label %.loop
+.loop:
+    %loaded = phi i32 [ %orig, %.noskip ], [ %old, %.loop ]
+    %isnull = icmp eq i32 %loaded, %skip_val
+    %min = call i32 @llvm.smin.i32(i32 %loaded, i32 %val)
+    %st = select i1 %isnull, i32 %val, i32 %min
+    %old = call i32 @atomic_cas_int_32(i32 addrspace(4)* %agg, i32 %loaded, i32 %st)
+    %success = icmp eq i32 %old, %loaded
+    br i1 %success, label %.skip, label %.loop
+.skip:
+    ret void
+}