Merge branch 'sycl' into pre-ci-ur-2394

zhaomaosu · zhaomaosu · commit a2b314952064 · 2024-12-05T07:05:35.000-08:00
diff --git a/libdevice/sanitizer/asan_rtl.cpp b/libdevice/sanitizer/asan_rtl.cpp
@@ -80,7 +80,8 @@ static __SYCL_CONSTANT__ const char __generic_to[] =
 
 #define ASAN_DEBUG(X)                                                          \
   do {                                                                         \
-    auto launch_info = (__SYCL_GLOBAL__ const LaunchInfo *)__AsanLaunchInfo;   \
+    auto launch_info =                                                         \
+        (__SYCL_GLOBAL__ const AsanRuntimeData *)__AsanLaunchInfo;             \
     if (launch_info->Debug) {                                                  \
       X;                                                                       \
     }                                                                          \
@@ -125,7 +126,7 @@ inline void ConvertGenericPointer(uptr &addr, uint32_t &as) {
 }
 
 inline uptr MemToShadow_CPU(uptr addr) {
-  auto launch_info = (__SYCL_GLOBAL__ const LaunchInfo *)__AsanLaunchInfo;
+  auto launch_info = (__SYCL_GLOBAL__ const AsanRuntimeData *)__AsanLaunchInfo;
   return launch_info->GlobalShadowOffset + (addr >> ASAN_SHADOW_SCALE);
 }
 
@@ -134,7 +135,7 @@ inline uptr MemToShadow_DG2(uptr addr, uint32_t as) {
     ConvertGenericPointer(addr, as);
   }
 
-  auto launch_info = (__SYCL_GLOBAL__ const LaunchInfo *)__AsanLaunchInfo;
+  auto launch_info = (__SYCL_GLOBAL__ const AsanRuntimeData *)__AsanLaunchInfo;
   if (as == ADDRESS_SPACE_GLOBAL) { // global
     uptr shadow_ptr;
     if (addr & 0xFFFF000000000000ULL) { // Device USM
@@ -214,7 +215,7 @@ inline uptr MemToShadow_PVC(uptr addr, uint32_t as) {
     ConvertGenericPointer(addr, as);
   }
 
-  auto launch_info = (__SYCL_GLOBAL__ const LaunchInfo *)__AsanLaunchInfo;
+  auto launch_info = (__SYCL_GLOBAL__ const AsanRuntimeData *)__AsanLaunchInfo;
   if (as == ADDRESS_SPACE_GLOBAL) { // global
     uptr shadow_ptr;
     if (addr & 0xFF00000000000000) { // Device USM
@@ -301,7 +302,7 @@ inline uptr MemToShadow(uptr addr, uint32_t as) {
 #elif defined(__LIBDEVICE_DG2__)
   shadow_ptr = MemToShadow_DG2(addr, as);
 #else
-  auto launch_info = (__SYCL_GLOBAL__ const LaunchInfo *)__AsanLaunchInfo;
+  auto launch_info = (__SYCL_GLOBAL__ const AsanRuntimeData *)__AsanLaunchInfo;
   if (launch_info->DeviceTy == DeviceType::CPU) {
     shadow_ptr = MemToShadow_CPU(addr);
   } else if (launch_info->DeviceTy == DeviceType::GPU_PVC) {
@@ -379,12 +380,12 @@ void __asan_internal_report_save(ErrorType error_type) {
       __spirv_BuiltInWorkgroupId.y * __spirv_BuiltInNumWorkgroups.z +
       __spirv_BuiltInWorkgroupId.z;
 
-  auto &SanitizerReport = ((__SYCL_GLOBAL__ LaunchInfo *)__AsanLaunchInfo)
+  auto &SanitizerReport = ((__SYCL_GLOBAL__ AsanRuntimeData *)__AsanLaunchInfo)
                               ->Report[WG_LID % ASAN_MAX_NUM_REPORTS];
 
   if (atomicCompareAndSet(
-          &(((__SYCL_GLOBAL__ LaunchInfo *)__AsanLaunchInfo)->ReportFlag), 1,
-          0) == 0 &&
+          &(((__SYCL_GLOBAL__ AsanRuntimeData *)__AsanLaunchInfo)->ReportFlag),
+          1, 0) == 0 &&
       atomicCompareAndSet(&SanitizerReport.Flag, Desired, Expected) ==
           Expected) {
     SanitizerReport.ErrorTy = error_type;
@@ -415,13 +416,13 @@ void __asan_internal_report_save(
       __spirv_BuiltInWorkgroupId.y * __spirv_BuiltInNumWorkgroups.z +
       __spirv_BuiltInWorkgroupId.z;
 
-  auto &SanitizerReport = ((__SYCL_GLOBAL__ LaunchInfo *)__AsanLaunchInfo)
+  auto &SanitizerReport = ((__SYCL_GLOBAL__ AsanRuntimeData *)__AsanLaunchInfo)
                               ->Report[WG_LID % ASAN_MAX_NUM_REPORTS];
 
   if ((is_recover ||
        atomicCompareAndSet(
-           &(((__SYCL_GLOBAL__ LaunchInfo *)__AsanLaunchInfo)->ReportFlag), 1,
-           0) == 0) &&
+           &(((__SYCL_GLOBAL__ AsanRuntimeData *)__AsanLaunchInfo)->ReportFlag),
+           1, 0) == 0) &&
       atomicCompareAndSet(&SanitizerReport.Flag, Desired, Expected) ==
           Expected) {
 
@@ -853,7 +854,7 @@ __asan_set_shadow_dynamic_local(uptr ptr, uint32_t num_args) {
 
   ASAN_DEBUG(__spirv_ocl_printf(__mem_set_shadow_dynamic_local_begin));
 
-  auto *launch_info = (__SYCL_GLOBAL__ const LaunchInfo *)__AsanLaunchInfo;
+  auto *launch_info = (__SYCL_GLOBAL__ const AsanRuntimeData *)__AsanLaunchInfo;
   if (num_args != launch_info->NumLocalArgs) {
     __spirv_ocl_printf(__mem_report_arg_count_incorrect, num_args,
                        launch_info->NumLocalArgs);
@@ -887,7 +888,7 @@ __asan_unpoison_shadow_dynamic_local(uptr ptr, uint32_t num_args) {
 
   ASAN_DEBUG(__spirv_ocl_printf(__mem_unpoison_shadow_dynamic_local_begin));
 
-  auto *launch_info = (__SYCL_GLOBAL__ const LaunchInfo *)__AsanLaunchInfo;
+  auto *launch_info = (__SYCL_GLOBAL__ const AsanRuntimeData *)__AsanLaunchInfo;
   if (num_args != launch_info->NumLocalArgs) {
     __spirv_ocl_printf(__mem_report_arg_count_incorrect, num_args,
                        launch_info->NumLocalArgs);
@@ -926,7 +927,7 @@ DEVICE_EXTERN_C_NOINLINE void __asan_set_shadow_private(uptr begin, uptr size,
 
   ASAN_DEBUG(__spirv_ocl_printf(__mem_set_shadow_private_begin));
 
-  auto *launch_info = (__SYCL_GLOBAL__ const LaunchInfo *)__AsanLaunchInfo;
+  auto *launch_info = (__SYCL_GLOBAL__ const AsanRuntimeData *)__AsanLaunchInfo;
   if (launch_info->PrivateShadowOffset == 0)
     return;
 
diff --git a/sycl/doc/extensions/experimental/sycl_ext_oneapi_graph.asciidoc b/sycl/doc/extensions/experimental/sycl_ext_oneapi_graph.asciidoc
@@ -1977,14 +1977,20 @@ Removing this restriction is something we may look at for future revisions of
 The command submission functions defined in
 link:../experimental/sycl_ext_oneapi_enqueue_functions.asciidoc[sycl_ext_oneapi_enqueue_functions]
 can be used adding nodes to a graph when creating a graph from queue recording.
-New methods are also defined that enable submitting an executable graph, 
+New methods are also defined that enable submitting an executable graph,
 e.g. directly to a queue without returning an event.
 
 ==== sycl_ext_oneapi_free_function_kernels
 
-`sycl_ext_oneapi_free_function_kernels`, defined in 
+`sycl_ext_oneapi_free_function_kernels`, defined in
 link:../proposed/sycl_ext_oneapi_free_function_kernels.asciidoc[sycl_ext_oneapi_free_function_kernels]
-can be used with SYCL Graphs. 
+can be used with SYCL Graphs.
+
+==== sycl_ext_oneapi_work_group_memory
+
+Using the `work_group_memory` object defined in
+link:../experimental/sycl_ext_oneapi_work_group_memory.asciidoc[sycl_ext_oneapi_work_group_memory]
+inside graph kernel nodes is supported.
 
 ==== sycl_ext_oneapi_work_group_scratch_memory
 
diff --git a/sycl/include/sycl/detail/backend_traits_cuda.hpp b/sycl/include/sycl/detail/backend_traits_cuda.hpp
@@ -14,7 +14,6 @@
 
 #pragma once
 
-#include <sycl/context.hpp>
 #include <sycl/detail/backend_traits.hpp>
 #include <sycl/device.hpp>
 #include <sycl/event.hpp>
@@ -35,6 +34,9 @@ typedef unsigned int CUdeviceptr;
 
 namespace sycl {
 inline namespace _V1 {
+
+class context;
+
 namespace detail {
 
 // TODO the interops for context, device, event, platform and program
diff --git a/sycl/include/sycl/detail/backend_traits_hip.hpp b/sycl/include/sycl/detail/backend_traits_hip.hpp
@@ -14,7 +14,6 @@
 
 #pragma once
 
-#include <sycl/context.hpp>
 #include <sycl/detail/backend_traits.hpp>
 #include <sycl/device.hpp>
 #include <sycl/event.hpp>
@@ -29,6 +28,9 @@ typedef void *HIPdeviceptr;
 
 namespace sycl {
 inline namespace _V1 {
+
+class context;
+
 namespace detail {
 
 // TODO the interops for context, device, event, platform and program
diff --git a/sycl/include/sycl/detail/backend_traits_level_zero.hpp b/sycl/include/sycl/detail/backend_traits_level_zero.hpp
@@ -16,7 +16,6 @@
 #pragma once
 
 #include <sycl/backend_types.hpp>                           // for backend
-#include <sycl/context.hpp>                                 // for context
 #include <sycl/detail/backend_traits.hpp>                   // for BackendI...
 #include <sycl/device.hpp>                                  // for device
 #include <sycl/event.hpp>                                   // for event
@@ -45,6 +44,7 @@ typedef struct _ze_module_handle_t *ze_module_handle_t;
 namespace sycl {
 inline namespace _V1 {
 class queue;
+class context;
 
 namespace detail {
 
diff --git a/sycl/include/sycl/detail/backend_traits_opencl.hpp b/sycl/include/sycl/detail/backend_traits_opencl.hpp
@@ -16,7 +16,6 @@
 #pragma once
 
 #include <sycl/backend_types.hpp>         // for backend
-#include <sycl/context.hpp>               // for context
 #include <sycl/detail/backend_traits.hpp> // for BackendInput, BackendReturn
 #include <sycl/detail/cl.h>               // for _cl_event, cl_event, cl_de...
 #include <sycl/detail/ur.hpp>             // for assertion and ur handles
@@ -35,6 +34,7 @@ template <bundle_state State> class kernel_bundle;
 class queue;
 template <typename T, int Dimensions, typename AllocatorT, typename Enable>
 class buffer;
+class context;
 
 namespace detail {
 
diff --git a/sycl/include/sycl/handler.hpp b/sycl/include/sycl/handler.hpp
@@ -10,7 +10,6 @@
 
 #include <sycl/access/access.hpp>
 #include <sycl/accessor.hpp>
-#include <sycl/context.hpp>
 #include <sycl/detail/cg_types.hpp>
 #include <sycl/detail/cl.h>
 #include <sycl/detail/common.hpp>
@@ -170,6 +169,7 @@ class kernel_impl;
 class queue_impl;
 class stream_impl;
 class event_impl;
+class context_impl;
 template <typename DataT, int Dimensions, access::mode AccessMode,
           access::target AccessTarget, access::placeholder IsPlaceholder>
 class image_accessor;
diff --git a/sycl/include/sycl/image.hpp b/sycl/include/sycl/image.hpp
@@ -12,7 +12,6 @@
 #include <sycl/aliases.hpp>                           // for cl_float, cl_half
 #include <sycl/backend_types.hpp>                     // for backend, backe...
 #include <sycl/buffer.hpp>                            // for range
-#include <sycl/context.hpp>                           // for context
 #include <sycl/detail/aligned_allocator.hpp>          // for aligned_allocator
 #include <sycl/detail/backend_traits.hpp>             // for InteropFeature...
 #include <sycl/detail/common.hpp>                     // for convertToArrayOfN
@@ -43,6 +42,7 @@ inline namespace _V1 {
 
 // forward declarations
 class handler;
+class context;
 
 template <int D, typename A> class image;
 
diff --git a/sycl/include/sycl/kernel_bundle.hpp b/sycl/include/sycl/kernel_bundle.hpp
@@ -9,7 +9,6 @@
 #pragma once
 
 #include <sycl/backend_types.hpp>          // for backend, backend_return_t
-#include <sycl/context.hpp>                // for context
 #include <sycl/detail/export.hpp>          // for __SYCL_EXPORT
 #include <sycl/detail/kernel_desc.hpp>     // for get_spec_constant_symboli...
 #include <sycl/detail/owner_less_base.hpp> // for OwnerLessBase
@@ -46,6 +45,7 @@ template <backend Backend> class backend_traits;
 template <backend Backend, bundle_state State>
 auto get_native(const kernel_bundle<State> &Obj)
     -> backend_return_t<Backend, kernel_bundle<State>>;
+class context;
 
 namespace detail {
 class kernel_id_impl;
diff --git a/sycl/include/sycl/queue.hpp b/sycl/include/sycl/queue.hpp
@@ -14,7 +14,6 @@
 #include <sycl/async_handler.hpp>             // for async_handler
 #include <sycl/backend_types.hpp>             // for backend, backe...
 #include <sycl/buffer.hpp>                    // for buffer
-#include <sycl/context.hpp>                   // for context
 #include <sycl/detail/assert_happened.hpp>    // for AssertHappened
 #include <sycl/detail/cg_types.hpp>           // for check_fn_signa...
 #include <sycl/detail/common.hpp>             // for code_location
diff --git a/sycl/include/sycl/usm.hpp b/sycl/include/sycl/usm.hpp
@@ -8,7 +8,6 @@
 #pragma once
 
 #include <sycl/builtins.hpp>
-#include <sycl/context.hpp>
 #include <sycl/detail/common.hpp>
 #include <sycl/detail/export.hpp>
 #include <sycl/device.hpp>
@@ -21,6 +20,9 @@
 
 namespace sycl {
 inline namespace _V1 {
+
+class context;
+
 ///
 // Explicit USM
 ///
diff --git a/sycl/source/platform.cpp b/sycl/source/platform.cpp
@@ -11,6 +11,7 @@
 #include <detail/global_handler.hpp>
 #include <detail/platform_impl.hpp>
 #include <detail/ur.hpp>
+#include <sycl/context.hpp>
 #include <sycl/device.hpp>
 #include <sycl/device_selector.hpp>
 #include <sycl/image.hpp>
diff --git a/sycl/test-e2e/Graph/Explicit/work_group_memory.cpp b/sycl/test-e2e/Graph/Explicit/work_group_memory.cpp
@@ -0,0 +1,10 @@
+// RUN: %{build} -o %t.out
+// RUN: %{run} %t.out
+// Extra run to check for leaks in Level Zero using UR_L0_LEAKS_DEBUG
+// RUN: %if level_zero %{env SYCL_PI_LEVEL_ZERO_USE_IMMEDIATE_COMMANDLISTS=0 %{l0_leak_check} %{run} %t.out 2>&1 | FileCheck %s --implicit-check-not=LEAK %}
+// Extra run to check for immediate-command-list in Level Zero
+// RUN: %if level_zero %{env SYCL_PI_LEVEL_ZERO_USE_IMMEDIATE_COMMANDLISTS=1 %{l0_leak_check} %{run} %t.out 2>&1 | FileCheck %s --implicit-check-not=LEAK %}
+
+#define GRAPH_E2E_EXPLICIT
+
+#include "../Inputs/work_group_memory.cpp"
diff --git a/sycl/test-e2e/Graph/Explicit/work_group_memory_free_function.cpp b/sycl/test-e2e/Graph/Explicit/work_group_memory_free_function.cpp
@@ -0,0 +1,13 @@
+// RUN: %{build} -o %t.out
+// RUN: %{run} %t.out
+// Extra run to check for leaks in Level Zero using UR_L0_LEAKS_DEBUG
+// RUN: %if level_zero %{env SYCL_PI_LEVEL_ZERO_USE_IMMEDIATE_COMMANDLISTS=0 %{l0_leak_check} %{run} %t.out 2>&1 | FileCheck %s --implicit-check-not=LEAK %}
+// Extra run to check for immediate-command-list in Level Zero
+// RUN: %if level_zero %{env SYCL_PI_LEVEL_ZERO_USE_IMMEDIATE_COMMANDLISTS=1 %{l0_leak_check} %{run} %t.out 2>&1 | FileCheck %s --implicit-check-not=LEAK %}
+
+// XFAIL: cuda
+// XFAIL-TRACKER: https://github.com/intel/llvm/issues/16004
+
+#define GRAPH_E2E_EXPLICIT
+
+#include "../Inputs/work_group_memory_free_function.cpp"
diff --git a/sycl/test-e2e/Graph/Inputs/free_function_kernels.cpp b/sycl/test-e2e/Graph/Inputs/free_function_kernels.cpp
@@ -25,7 +25,7 @@ int main() {
   kernel_bundle Bundle = get_kernel_bundle<bundle_state::executable>(Ctxt);
   kernel_id Kernel_id = exp_ext::get_kernel_id<ff_0>();
   kernel Kernel = Bundle.get_kernel(Kernel_id);
-  auto KernelNode = Graph.add([&](handler &cgh) {
+  auto KernelNode = add_node(Graph, Queue, [&](handler &cgh) {
     cgh.set_arg(0, PtrA);
     cgh.single_task(Kernel);
   });
diff --git a/sycl/test-e2e/Graph/Inputs/work_group_memory.cpp b/sycl/test-e2e/Graph/Inputs/work_group_memory.cpp
@@ -0,0 +1,43 @@
+// Tests using sycl_ext_oneapi_work_group_memory in a graph node
+
+#include "../graph_common.hpp"
+#include <sycl/ext/oneapi/experimental/work_group_memory.hpp>
+
+int main() {
+  queue Queue;
+  exp_ext::command_graph Graph{Queue};
+
+  std::vector<int> HostData(Size);
+  std::iota(HostData.begin(), HostData.end(), 10);
+
+  int *Ptr = malloc_device<int>(Size, Queue);
+  Queue.copy(HostData.data(), Ptr, Size).wait();
+
+  const size_t LocalSize = 128;
+  auto node = add_node(Graph, Queue, [&](handler &CGH) {
+    exp_ext::work_group_memory<int[]> WGMem{LocalSize, CGH};
+
+    CGH.parallel_for(nd_range({Size}, {LocalSize}), [=](nd_item<1> Item) {
+      WGMem[Item.get_local_linear_id()] = Item.get_global_linear_id() * 2;
+      Ptr[Item.get_global_linear_id()] += WGMem[Item.get_local_linear_id()];
+    });
+  });
+
+  auto GraphExec = Graph.finalize();
+
+  for (unsigned N = 0; N < Iterations; N++) {
+    Queue.submit([&](handler &CGH) { CGH.ext_oneapi_graph(GraphExec); });
+  }
+  Queue.wait_and_throw();
+
+  Queue.copy(Ptr, HostData.data(), Size);
+  Queue.wait_and_throw();
+
+  for (size_t i = 0; i < Size; i++) {
+    int Ref = 10 + i + (Iterations * (i * 2));
+    assert(check_value(i, Ref, HostData[i], "Ptr"));
+  }
+
+  free(Ptr, Queue);
+  return 0;
+}
diff --git a/sycl/test-e2e/Graph/Inputs/work_group_memory_free_function.cpp b/sycl/test-e2e/Graph/Inputs/work_group_memory_free_function.cpp
diff --git a/sycl/test-e2e/Graph/RecordReplay/work_group_memory.cpp b/sycl/test-e2e/Graph/RecordReplay/work_group_memory.cpp
diff --git a/sycl/test-e2e/Graph/RecordReplay/work_group_memory_free_function.cpp b/sycl/test-e2e/Graph/RecordReplay/work_group_memory_free_function.cpp
