[SYCL] Impl of fallback group sort in device code

libdevice/atomic.hpp

@@ -7,78 +7,9 @@
 //===----------------------------------------------------------------------===//
 #pragma once
 
-#include <cstdint>
-
-#include "device.h"
-
-#ifdef __SPIR__
-
-#define SPIR_GLOBAL __attribute__((opencl_global))
-
-namespace __spv {
-struct Scope {
-
-  enum Flag : uint32_t {
-    CrossDevice = 0,
-    Device = 1,
-    Workgroup = 2,
-    Subgroup = 3,
-    Invocation = 4,
-  };
-
-  constexpr Scope(Flag flag) : flag_value(flag) {}
-
-  constexpr operator uint32_t() const { return flag_value; }
-
-  Flag flag_value;
-};
-
-struct MemorySemanticsMask {
-
-  enum Flag : uint32_t {
-    None = 0x0,
-    Acquire = 0x2,
-    Release = 0x4,
-    AcquireRelease = 0x8,
-    SequentiallyConsistent = 0x10,
-    UniformMemory = 0x40,
-    SubgroupMemory = 0x80,
-    WorkgroupMemory = 0x100,
-    CrossWorkgroupMemory = 0x200,
-    AtomicCounterMemory = 0x400,
-    ImageMemory = 0x800,
-  };
-
-  constexpr MemorySemanticsMask(Flag flag) : flag_value(flag) {}
-
-  constexpr operator uint32_t() const { return flag_value; }
-
-  Flag flag_value;
-};
-} // namespace __spv
-
-extern DEVICE_EXTERNAL int
-__spirv_AtomicCompareExchange(int SPIR_GLOBAL *, __spv::Scope::Flag,
-                              __spv::MemorySemanticsMask::Flag,
-                              __spv::MemorySemanticsMask::Flag, int, int);
-
-extern DEVICE_EXTERNAL int
-__spirv_AtomicCompareExchange(int *, __spv::Scope::Flag,
-                              __spv::MemorySemanticsMask::Flag,
-                              __spv::MemorySemanticsMask::Flag, int, int);
-
-extern DEVICE_EXTERNAL int __spirv_AtomicLoad(const int SPIR_GLOBAL *,
-                                              __spv::Scope::Flag,
-                                              __spv::MemorySemanticsMask::Flag);
-
-extern DEVICE_EXTERNAL void
-__spirv_AtomicStore(int SPIR_GLOBAL *, __spv::Scope::Flag,
-                    __spv::MemorySemanticsMask::Flag, int);
-
-extern DEVICE_EXTERNAL void
-__spirv_AtomicStore(int *, __spv::Scope::Flag, __spv::MemorySemanticsMask::Flag,
-                    int);
+#include "spirv_decls.hpp"
 
+#if defined(__SPIR__)
 /// Atomically set the value in *Ptr with Desired if and only if it is Expected
 /// Return the value which already was in *Ptr
 static inline int atomicCompareAndSet(SPIR_GLOBAL int *Ptr, int Desired,

libdevice/cmake/modules/SYCLLibdevice.cmake

@@ -102,15 +102,16 @@ function(add_fallback_devicelib fallback_filename)
   add_devicelib_obj(${fallback_filename} SRC ${FB_SRC} DEP ${FB_DEP} EXTRA_ARGS ${FB_EXTRA_ARGS})
 endfunction()
 
-set(crt_obj_deps wrapper.h device.h spirv_vars.h sycl-compiler)
+set(crt_obj_deps wrapper.h device.h atomic.hpp spirv_decls.hpp spirv_vars.h sycl-compiler)
 set(complex_obj_deps device_complex.h device.h sycl-compiler)
 set(cmath_obj_deps device_math.h device.h sycl-compiler)
 set(imf_obj_deps device_imf.hpp imf_half.hpp imf_bf16.hpp imf_rounding_op.hpp imf_impl_utils.hpp device.h sycl-compiler)
 set(itt_obj_deps device_itt.h spirv_vars.h device.h sycl-compiler)
 set(bfloat16_obj_deps sycl-headers sycl-compiler)
 if (NOT MSVC)
-  set(sanitizer_obj_deps device.h atomic.hpp spirv_vars.h include/sanitizer_device_utils.hpp include/spir_global_var.hpp sycl-compiler)
+  set(sanitizer_obj_deps device.h atomic.hpp spirv_decls.hpp  spirv_vars.h include/sanitizer_device_utils.hpp include/spir_global_var.hpp sycl-compiler)
 endif()
+set(gsort_obj_deps device.h spirv_decls.hpp spirv_vars.h group_helper.hpp sort_helper.hpp sycl-compiler)
 
 add_devicelib_obj(libsycl-itt-stubs SRC itt_stubs.cpp DEP ${itt_obj_deps})
 add_devicelib_obj(libsycl-itt-compiler-wrappers SRC itt_compiler_wrappers.cpp DEP ${itt_obj_deps})
@@ -139,6 +140,7 @@ add_fallback_devicelib(libsycl-fallback-cmath SRC fallback-cmath.cpp DEP ${cmath
 add_fallback_devicelib(libsycl-fallback-cmath-fp64 SRC fallback-cmath-fp64.cpp DEP ${cmath_obj_deps})
 add_fallback_devicelib(libsycl-fallback-bfloat16 SRC fallback-bfloat16.cpp DEP ${bfloat16_obj_deps})
 add_fallback_devicelib(libsycl-native-bfloat16 SRC bfloat16_wrapper.cpp DEP ${bfloat16_obj_deps})
+add_fallback_devicelib(libsycl-fallback-gsort SRC fallback-gsort.cpp DEP ${gsort_obj_deps})
 
 file(MAKE_DIRECTORY ${obj_binary_dir}/libdevice)
 set(imf_fallback_src_dir ${obj_binary_dir}/libdevice)

libdevice/fallback-gsort.cpp

@@ -0,0 +1,254 @@
+
+//==--- fallback_gsort_fp32.cpp - fallback implementation of group sort
+//-----==//
+//
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+//
+//===----------------------------------------------------------------------===//
+#include "device.h"
+#include "sort_helper.hpp"
+#include <cstdint>
+#include <functional>
+#if defined(__SPIR__)
+
+DEVICE_EXTERN_C_INLINE
+void __devicelib_default_work_group_joint_sort_ascending_p1i8_u32_p1i8(
+    int8_t *first, uint32_t n, uint8_t *scratch) {
+  merge_sort(first, n, scratch, std::less<int8_t>{});
+}
+
+DEVICE_EXTERN_C_INLINE
+void __devicelib_default_work_group_joint_sort_ascending_p1i8_u32_p3i8(
+    int8_t *first, uint32_t n, uint8_t *scratch) {
+  merge_sort(first, n, scratch, std::less<int8_t>{});
+}
+
+DEVICE_EXTERN_C_INLINE
+void __devicelib_default_work_group_joint_sort_ascending_p3i8_u32_p1i8(
+    int8_t *first, uint32_t n, uint8_t *scratch) {
+  merge_sort(first, n, scratch, std::less<int8_t>{});
+}
+
+DEVICE_EXTERN_C_INLINE
+void __devicelib_default_work_group_joint_sort_ascending_p3i8_u32_p3i8(
+    int8_t *first, uint32_t n, uint8_t *scratch) {
+  merge_sort(first, n, scratch, std::less<int8_t>{});
+}
+
+DEVICE_EXTERN_C_INLINE
+void __devicelib_default_work_group_joint_sort_descending_p1i8_u32_p1i8(
+    int8_t *first, uint32_t n, uint8_t *scratch) {
+  merge_sort(first, n, scratch, std::greater<int8_t>{});
+}
+
+DEVICE_EXTERN_C_INLINE
+void __devicelib_default_work_group_joint_sort_descending_p1i8_u32_p3i8(
+    int8_t *first, uint32_t n, uint8_t *scratch) {
+  merge_sort(first, n, scratch, std::greater<int8_t>{});
+}
+
+DEVICE_EXTERN_C_INLINE
+void __devicelib_default_work_group_joint_sort_descending_p3i8_u32_p1i8(
+    int8_t *first, uint32_t n, uint8_t *scratch) {
+  merge_sort(first, n, scratch, std::greater<int8_t>{});
+}
+
+DEVICE_EXTERN_C_INLINE
+void __devicelib_default_work_group_joint_sort_descending_p3i8_u32_p3i8(
+    int8_t *first, uint32_t n, uint8_t *scratch) {
+  merge_sort(first, n, scratch, std::greater<int8_t>{});
+}
+
+DEVICE_EXTERN_C_INLINE
+void __devicelib_default_work_group_joint_sort_ascending_p1i16_u32_p1i8(
+    int16_t *first, uint32_t n, uint8_t *scratch) {
+  merge_sort(first, n, scratch, std::less<int16_t>{});
+}
+
+DEVICE_EXTERN_C_INLINE
+void __devicelib_default_work_group_joint_sort_ascending_p1i16_u32_p3i8(
+    int16_t *first, uint32_t n, uint8_t *scratch) {
+  merge_sort(first, n, scratch, std::less<int16_t>{});
+}
+
+DEVICE_EXTERN_C_INLINE
+void __devicelib_default_work_group_joint_sort_ascending_p3i16_u32_p1i8(
+    int16_t *first, uint32_t n, uint8_t *scratch) {
+  merge_sort(first, n, scratch, std::less<int16_t>{});
+}
+
+DEVICE_EXTERN_C_INLINE
+void __devicelib_default_work_group_joint_sort_ascending_p3i16_u32_p3i8(
+    int16_t *first, uint32_t n, uint8_t *scratch) {
+  merge_sort(first, n, scratch, std::less<int16_t>{});
+}
+
+DEVICE_EXTERN_C_INLINE
+void __devicelib_default_work_group_joint_sort_descending_p1i16_u32_p1i8(
+    int16_t *first, uint32_t n, uint8_t *scratch) {
+  merge_sort(first, n, scratch, std::greater<int16_t>{});
+}
+
+DEVICE_EXTERN_C_INLINE
+void __devicelib_default_work_group_joint_sort_descending_p1i16_u32_p3i8(
+    int16_t *first, uint32_t n, uint8_t *scratch) {
+  merge_sort(first, n, scratch, std::greater<int16_t>{});
+}
+
+DEVICE_EXTERN_C_INLINE
+void __devicelib_default_work_group_joint_sort_descending_p3i16_u32_p1i8(
+    int16_t *first, uint32_t n, uint8_t *scratch) {
+  merge_sort(first, n, scratch, std::greater<int16_t>{});
+}
+
+DEVICE_EXTERN_C_INLINE
+void __devicelib_default_work_group_joint_sort_descending_p3i16_u32_p3i8(
+    int16_t *first, uint32_t n, uint8_t *scratch) {
+  merge_sort(first, n, scratch, std::greater<int16_t>{});
+}
+
+DEVICE_EXTERN_C_INLINE
+void __devicelib_default_work_group_joint_sort_ascending_p1i32_u32_p1i8(
+    int32_t *first, uint32_t n, uint8_t *scratch) {
+  merge_sort(first, n, scratch, std::less<int32_t>{});
+}
+
+DEVICE_EXTERN_C_INLINE
+void __devicelib_default_work_group_joint_sort_ascending_p1i32_u32_p3i8(
+    int32_t *first, uint32_t n, uint8_t *scratch) {
+  merge_sort(first, n, scratch, std::less<int32_t>{});
+}
+
+DEVICE_EXTERN_C_INLINE
+void __devicelib_default_work_group_joint_sort_ascending_p3i32_u32_p1i8(
+    int32_t *first, uint32_t n, uint8_t *scratch) {
+  merge_sort(first, n, scratch, std::less<int32_t>{});
+}
+
+DEVICE_EXTERN_C_INLINE
+void __devicelib_default_work_group_joint_sort_ascending_p3i32_u32_p3i8(
+    int32_t *first, uint32_t n, uint8_t *scratch) {
+  merge_sort(first, n, scratch, std::less<int32_t>{});
+}
+
+DEVICE_EXTERN_C_INLINE
+void __devicelib_default_work_group_joint_sort_descending_p1i32_u32_p1i8(
+    int32_t *first, uint32_t n, uint8_t *scratch) {
+  merge_sort(first, n, scratch, std::greater<int32_t>{});
+}
+
+DEVICE_EXTERN_C_INLINE
+void __devicelib_default_work_group_joint_sort_descending_p1i32_u32_p3i8(
+    int32_t *first, uint32_t n, uint8_t *scratch) {
+  merge_sort(first, n, scratch, std::greater<int32_t>{});
+}
+
+DEVICE_EXTERN_C_INLINE
+void __devicelib_default_work_group_joint_sort_descending_p3i32_u32_p1i8(
+    int32_t *first, uint32_t n, uint8_t *scratch) {
+  merge_sort(first, n, scratch, std::greater<int32_t>{});
+}
+
+DEVICE_EXTERN_C_INLINE
+void __devicelib_default_work_group_joint_sort_descending_p3i32_u32_p3i8(
+    int32_t *first, uint32_t n, uint8_t *scratch) {
+  merge_sort(first, n, scratch, std::greater<int32_t>{});
+}
+
+void __devicelib_default_work_group_joint_sort_ascending_p1i64_u32_p1i8(
+    int64_t *first, uint32_t n, uint8_t *scratch) {
+  merge_sort(first, n, scratch, std::less<int64_t>{});
+}
+
+DEVICE_EXTERN_C_INLINE
+void __devicelib_default_work_group_joint_sort_ascending_p1i64_u32_p3i8(
+    int64_t *first, uint32_t n, uint8_t *scratch) {
+  merge_sort(first, n, scratch, std::less<int64_t>{});
+}
+
+DEVICE_EXTERN_C_INLINE
+void __devicelib_default_work_group_joint_sort_ascending_p3i64_u32_p1i8(
+    int64_t *first, uint32_t n, uint8_t *scratch) {
+  merge_sort(first, n, scratch, std::less<int64_t>{});
+}
+
+DEVICE_EXTERN_C_INLINE
+void __devicelib_default_work_group_joint_sort_ascending_p3i64_u32_p3i8(
+    int64_t *first, uint32_t n, uint8_t *scratch) {
+  merge_sort(first, n, scratch, std::less<int64_t>{});
+}
+
+DEVICE_EXTERN_C_INLINE
+void __devicelib_default_work_group_joint_sort_descending_p1i64_u32_p1i8(
+    int64_t *first, uint32_t n, uint8_t *scratch) {
+  merge_sort(first, n, scratch, std::greater<int64_t>{});
+}
+
+DEVICE_EXTERN_C_INLINE
+void __devicelib_default_work_group_joint_sort_descending_p1i64_u32_p3i8(
+    int64_t *first, uint32_t n, uint8_t *scratch) {
+  merge_sort(first, n, scratch, std::greater<int64_t>{});
+}
+
+DEVICE_EXTERN_C_INLINE
+void __devicelib_default_work_group_joint_sort_descending_p3i64_u32_p1i8(
+    int64_t *first, uint32_t n, uint8_t *scratch) {
+  merge_sort(first, n, scratch, std::greater<int64_t>{});
+}
+
+DEVICE_EXTERN_C_INLINE
+void __devicelib_default_work_group_joint_sort_descending_p3i64_u32_p3i8(
+    int64_t *first, uint32_t n, uint8_t *scratch) {
+  merge_sort(first, n, scratch, std::greater<int64_t>{});
+}
+
+DEVICE_EXTERN_C_INLINE
+void __devicelib_default_work_group_joint_sort_ascending_p1f32_u32_p1i8(
+    float *first, uint32_t n, uint8_t *scratch) {
+  merge_sort(first, n, scratch, std::less<float>{});
+}
+
+DEVICE_EXTERN_C_INLINE
+void __devicelib_default_work_group_joint_sort_ascending_p1f32_u32_p3i8(
+    float *first, uint32_t n, uint8_t *scratch) {
+  merge_sort(first, n, scratch, std::less<float>{});
+}
+
+DEVICE_EXTERN_C_INLINE
+void __devicelib_default_work_group_joint_sort_ascending_p3f32_u32_p1i8(
+    float *first, uint32_t n, uint8_t *scratch) {
+  merge_sort(first, n, scratch, std::less<float>{});
+}
+
+DEVICE_EXTERN_C_INLINE
+void __devicelib_default_work_group_joint_sort_ascending_p3f32_u32_p3i8(
+    float *first, uint32_t n, uint8_t *scratch) {
+  merge_sort(first, n, scratch, std::less<float>{});
+}
+
+DEVICE_EXTERN_C_INLINE
+void __devicelib_default_work_group_joint_sort_descending_p1f32_u32_p1i8(
+    float *first, uint32_t n, uint8_t *scratch) {
+  merge_sort(first, n, scratch, std::greater<float>{});
+}
+
+DEVICE_EXTERN_C_INLINE
+void __devicelib_default_work_group_joint_sort_descending_p1f32_u32_p3i8(
+    float *first, uint32_t n, uint8_t *scratch) {
+  merge_sort(first, n, scratch, std::greater<float>{});
+}
+
+DEVICE_EXTERN_C_INLINE
+void __devicelib_default_work_group_joint_sort_descending_p3f32_u32_p1i8(
+    float *first, uint32_t n, uint8_t *scratch) {
+  merge_sort(first, n, scratch, std::greater<float>{});
+}
+
+DEVICE_EXTERN_C_INLINE
+void __devicelib_default_work_group_joint_sort_descending_p3f32_u32_p3i8(
+    float *first, uint32_t n, uint8_t *scratch) {
+  merge_sort(first, n, scratch, std::greater<float>{});
+}
+#endif

libdevice/group_helper.hpp

@@ -0,0 +1,33 @@
+//==------- group_helper.hpp - utils related to work-group operations-------==//
+//
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+//
+//==------------------------------------------------------------------------==//
+#pragma once
+#include "spirv_decls.hpp"
+#include "spirv_vars.h"
+#include <cstdint>
+#if defined(__SPIR__)
+
+static inline size_t __get_wg_local_range() {
+  return __spirv_BuiltInWorkgroupSize.x * __spirv_BuiltInWorkgroupSize.y *
+         __spirv_BuiltInWorkgroupSize.z;
+}
+
+static inline size_t __get_wg_local_linear_id() {
+  return (__spirv_BuiltInLocalInvocationId.x * __spirv_BuiltInWorkgroupSize.y *
+          __spirv_BuiltInWorkgroupSize.z) +
+         (__spirv_BuiltInLocalInvocationId.y * __spirv_BuiltInWorkgroupSize.z) +
+         __spirv_BuiltInLocalInvocationId.z;
+}
+
+static inline void group_barrier() {
+  __spirv_ControlBarrier(__spv::Scope::Workgroup, __spv::Scope::Workgroup,
+                         __spv::MemorySemanticsMask::SequentiallyConsistent |
+                             __spv::MemorySemanticsMask::SubgroupMemory |
+                             __spv::MemorySemanticsMask::WorkgroupMemory |
+                             __spv::MemorySemanticsMask::CrossWorkgroupMemory);
+}
+#endif