Merge branch 'sycl' into refactorcache

Author: sergey-semenov

sycl/CMakeLists.txt

@@ -83,7 +83,7 @@ if(MSVC)
 endif()
 
 include(FetchEmhash)
-include(FetchUnifiedRuntime)
+include(BuildUnifiedRuntime)
 
 # The change in SYCL_MAJOR_VERSION must be accompanied with the same update in
 # llvm/clang/lib/Driver/CMakeLists.txt.

sycl/cmake/modules/FetchUnifiedRuntime.cmake renamed to sycl/cmake/modules/BuildUnifiedRuntime.cmake

@@ -1,5 +1,4 @@
-# Either fetches UR from the appropriate repo or sets up variables based on user
-# preference.
+# Builds in-tree UR
 
 # TODO: taken from sycl/plugins/CMakeLists.txt - maybe we should handle this
 # within UR (although it is an obscure warning that the build system here
@@ -9,20 +8,6 @@ if ("${CMAKE_CXX_COMPILER_ID}" MATCHES "Clang|IntelLLVM" )
 endif()
 
 
-# Options to override the default behaviour of the FetchContent to include UR
-# source code.
-set(SYCL_UR_OVERRIDE_FETCH_CONTENT_REPO
-  "" CACHE STRING "Override the Unified Runtime FetchContent repository")
-set(SYCL_UR_OVERRIDE_FETCH_CONTENT_TAG
-  "" CACHE STRING "Override the Unified Runtime FetchContent tag")
-
-# Options to disable use of FetchContent to include Unified Runtime source code
-# to improve developer workflow.
-option(SYCL_UR_USE_FETCH_CONTENT
-  "Use FetchContent to acquire the Unified Runtime source code" ON)
-set(SYCL_UR_SOURCE_DIR
-  "" CACHE PATH "Path to root of Unified Runtime repository")
-
 option(SYCL_UR_BUILD_TESTS "Build tests for UR" OFF)
 set(UR_BUILD_TESTS "${SYCL_UR_BUILD_TESTS}" CACHE BOOL "" FORCE)
 # UR tests require the examples to be built
@@ -80,154 +65,26 @@ endif()
 set(UR_INTREE_SOURCE_DIR "${LLVM_SOURCE_DIR}/../unified-runtime")
 cmake_path(NORMAL_PATH UR_INTREE_SOURCE_DIR OUTPUT_VARIABLE UR_INTREE_SOURCE_DIR)
 
-if(IS_DIRECTORY "${UR_INTREE_SOURCE_DIR}")
-  set(UR_INTREE_BINARY_DIR ${LLVM_BINARY_DIR}/unified-runtime)
-  set(UNIFIED_RUNTIME_SOURCE_DIR
-    "${UR_INTREE_SOURCE_DIR}" CACHE PATH
-    "Path to Unified Runtime Headers" FORCE)
-  set(UMF_BUILD_EXAMPLES OFF CACHE INTERNAL "EXAMPLES")
-  # Due to the use of dependentloadflag and no installer for UMF and hwloc we need
-  # to link statically on windows
-  if(WIN32)
-    set(UMF_BUILD_SHARED_LIBRARY OFF CACHE INTERNAL "Build UMF shared library")
-    set(UMF_LINK_HWLOC_STATICALLY ON CACHE INTERNAL "static HWLOC")
-  endif()
-  add_subdirectory(${UNIFIED_RUNTIME_SOURCE_DIR} ${UR_INTREE_BINARY_DIR})
-elseif(SYCL_UR_USE_FETCH_CONTENT)
-  include(FetchContent)
-
-  # The fetch_adapter_source function can be used to perform a separate content
-  # fetch for a UR adapter (backend), this allows development of adapters to be decoupled
-  # from each other.
-  #
-  # A separate content fetch will not be performed if:
-  # * The adapter name is not present in the SYCL_ENABLE_BACKENDS variable.
-  # * The repo and tag provided match the values of the
-  #   UNIFIED_RUNTIME_REPO/UNIFIED_RUNTIME_TAG variables
-  #
-  # Args:
-  #   * name - Must be the directory name of the adapter
-  #   * repo - A valid Git URL of a Unified Runtime repo
-  #   * tag - A valid Git branch/tag/commit in the Unified Runtime repo
-  function(fetch_adapter_source name repo tag)
-    if(NOT ${name} IN_LIST SYCL_ENABLE_BACKENDS)
-      return()
-    endif()
-    if(repo STREQUAL UNIFIED_RUNTIME_REPO AND
-        tag STREQUAL UNIFIED_RUNTIME_TAG)
-      # If the adapter sources are taken from the main checkout, reset the
-      # adapter specific source path.
-      string(TOUPPER ${name} NAME)
-      set(UR_ADAPTER_${NAME}_SOURCE_DIR ""
-        CACHE PATH "Path to external '${name}' adapter source dir" FORCE)
-      return()
-    endif()
-    message(STATUS
-      "Will fetch Unified Runtime ${name} adapter from ${repo} at ${tag}")
-    set(fetch-name ur-${name})
-    FetchContent_Declare(${fetch-name}
-      GIT_REPOSITORY ${repo} GIT_TAG ${tag})
-    # We don't want to add this repo to the build, only fetch its source.
-    FetchContent_Populate(${fetch-name})
-    # Get the path to the source directory
-    string(TOUPPER ${name} NAME)
-    set(source_dir_var UR_ADAPTER_${NAME}_SOURCE_DIR)
-    FetchContent_GetProperties(${fetch-name} SOURCE_DIR UR_ADAPTER_${NAME}_SOURCE_DIR)
-    # Set the variable which informs UR where to get the adapter source from.
-    set(UR_ADAPTER_${NAME}_SOURCE_DIR
-      "${UR_ADAPTER_${NAME}_SOURCE_DIR}/source/adapters/${name}"
-      CACHE PATH "Path to external '${name}' adapter source dir" FORCE)
-  endfunction()
-
-  set(UNIFIED_RUNTIME_REPO "https://github.com/oneapi-src/unified-runtime.git")
-  include(${CMAKE_CURRENT_SOURCE_DIR}/cmake/modules/UnifiedRuntimeTag.cmake)
-
-  set(UMF_BUILD_EXAMPLES OFF CACHE INTERNAL "EXAMPLES")
-  # Due to the use of dependentloadflag and no installer for UMF and hwloc we need
-  # to link statically on windows
-  if(WIN32)
-    set(UMF_BUILD_SHARED_LIBRARY OFF CACHE INTERNAL "Build UMF shared library")
-    set(UMF_LINK_HWLOC_STATICALLY ON CACHE INTERNAL "static HWLOC")
-  endif()
-
-  fetch_adapter_source(level_zero
-    ${UNIFIED_RUNTIME_REPO}
-    ${UNIFIED_RUNTIME_TAG}
-  )
-
-  fetch_adapter_source(opencl
-    ${UNIFIED_RUNTIME_REPO}
-    ${UNIFIED_RUNTIME_TAG}
-  )
-
-  fetch_adapter_source(cuda
-    ${UNIFIED_RUNTIME_REPO}
-    ${UNIFIED_RUNTIME_TAG}
-  )
-
-  fetch_adapter_source(hip
-    ${UNIFIED_RUNTIME_REPO}
-    ${UNIFIED_RUNTIME_TAG}
-  )
-
-  fetch_adapter_source(native_cpu
-    ${UNIFIED_RUNTIME_REPO}
-    ${UNIFIED_RUNTIME_TAG}
-  )
-
-  if(SYCL_UR_OVERRIDE_FETCH_CONTENT_REPO)
-    set(UNIFIED_RUNTIME_REPO "${SYCL_UR_OVERRIDE_FETCH_CONTENT_REPO}")
-  endif()
-  if(SYCL_UR_OVERRIDE_FETCH_CONTENT_TAG)
-    set(UNIFIED_RUNTIME_TAG "${SYCL_UR_OVERRIDE_FETCH_CONTENT_TAG}")
-  endif()
-
-  message(STATUS "Will fetch Unified Runtime from ${UNIFIED_RUNTIME_REPO}")
-  FetchContent_Declare(unified-runtime
-    GIT_REPOSITORY    ${UNIFIED_RUNTIME_REPO}
-    GIT_TAG           ${UNIFIED_RUNTIME_TAG}
-  )
+if(NOT IS_DIRECTORY "${UR_INTREE_SOURCE_DIR}")
+  message(FATAL_ERROR "unified-runtime folder not found at repo root")
+endif()
 
-  FetchContent_GetProperties(unified-runtime)
-  FetchContent_MakeAvailable(unified-runtime)
-
-  set(UNIFIED_RUNTIME_SOURCE_DIR
-    "${unified-runtime_SOURCE_DIR}" CACHE PATH
-    "Path to Unified Runtime Headers" FORCE)
-elseif(SYCL_UR_SOURCE_DIR)
-  # SYCL_UR_USE_FETCH_CONTENT is OFF and SYCL_UR_SOURCE_DIR has been set,
-  # use the external Unified Runtime source directory.
-  set(UNIFIED_RUNTIME_SOURCE_DIR
-    "${SYCL_UR_SOURCE_DIR}" CACHE PATH
-    "Path to Unified Runtime Headers" FORCE)
-  add_subdirectory(
-    ${UNIFIED_RUNTIME_SOURCE_DIR}
-    ${CMAKE_CURRENT_BINARY_DIR}/unified-runtime)
-else()
-  # SYCL_UR_USE_FETCH_CONTENT is OFF and SYCL_UR_SOURCE_DIR has not been
-  # set, check if the fallback local directory exists.
-  if(NOT EXISTS ${CMAKE_CURRENT_SOURCE_DIR}/unified-runtime)
-    message(FATAL_ERROR
-      "SYCL_UR_USE_FETCH_CONTENT is disabled but no alternative Unified \
-      Runtime source directory has been provided, either:
-
-      * Set -DSYCL_UR_SOURCE_DIR=/path/to/unified-runtime
-      * Clone the UR repo in ${CMAKE_CURRENT_SOURCE_DIR}/unified-runtime")
-  endif()
-  # The fallback local directory for the Unified Runtime repository has been
-  # found, use it.
-  set(UNIFIED_RUNTIME_SOURCE_DIR
-    "${CMAKE_CURRENT_SOURCE_DIR}/unified-runtime" CACHE PATH
-    "Path to Unified Runtime Headers" FORCE)
-  add_subdirectory(${UNIFIED_RUNTIME_SOURCE_DIR})
+set(UR_INTREE_BINARY_DIR ${LLVM_BINARY_DIR}/unified-runtime)
+set(UNIFIED_RUNTIME_SOURCE_DIR
+  "${UR_INTREE_SOURCE_DIR}" CACHE PATH
+  "Path to Unified Runtime Headers" FORCE)
+set(UMF_BUILD_EXAMPLES OFF CACHE INTERNAL "EXAMPLES")
+# Due to the use of dependentloadflag and no installer for UMF and hwloc we need
+# to link statically on windows
+if(WIN32)
+  set(UMF_BUILD_SHARED_LIBRARY OFF CACHE INTERNAL "Build UMF shared library")
+  set(UMF_LINK_HWLOC_STATICALLY ON CACHE INTERNAL "static HWLOC")
 endif()
+add_subdirectory(${UNIFIED_RUNTIME_SOURCE_DIR} ${UR_INTREE_BINARY_DIR})
 
 # Restore original flags
 set(CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS_BAK}")
 
-message(STATUS
-  "Using Unified Runtime source directory: ${UNIFIED_RUNTIME_SOURCE_DIR}")
-
 set(UNIFIED_RUNTIME_INCLUDE_DIR "${UNIFIED_RUNTIME_SOURCE_DIR}/include")
 set(UNIFIED_RUNTIME_SRC_INCLUDE_DIR "${UNIFIED_RUNTIME_SOURCE_DIR}/source")
 set(UNIFIED_RUNTIME_COMMON_INCLUDE_DIR "${UNIFIED_RUNTIME_SOURCE_DIR}/source/common")

sycl/cmake/modules/UnifiedRuntimeTag.cmake

@@ -14,7 +14,6 @@ and a wide range of compute accelerators such as GPU and FPGA.
     * [Build DPC++ toolchain with support for HIP NVIDIA](#build-dpc-toolchain-with-support-for-hip-nvidia)
     * [Build DPC++ toolchain with support for ARM processors](#build-dpc-toolchain-with-support-for-arm-processors)
     * [Build DPC++ toolchain with additional features enabled that require runtime/JIT compilation](#build-dpc-toolchain-with-additional-features-enabled-that-require-runtimejit-compilation)
-    * [Build DPC++ toolchain with a custom Unified Runtime](#build-dpc-toolchain-with-a-custom-unified-runtime)
     * [Build DPC++ toolchain with device image compression support](#build-dpc-toolchain-with-device-image-compression-support)
     * [Build Doxygen documentation](#build-doxygen-documentation)
     * [Deployment](#deployment)
@@ -369,36 +368,6 @@ After CMake cache is generated, build the documentation with `doxygen-sycl`
 target. It will be put to `$DPCPP_HOME/llvm/build/tools/sycl/doc/html`
 directory.
 
-### Build DPC++ toolchain with a custom Unified Runtime
-
-DPC++ uses the [Unified Runtime](https://github.com/oneapi-src/unified-runtime)
-under the hood to provide implementations of various SYCL backends. By default
-the source code for the Unified Runtime will be acquired using CMake's
-[FetchCotent](https://cmake.org/cmake/help/latest/module/FetchContent.html). The
-specific repository URL and revision tag used can be found in the file
-`sycl/cmake/modules/FetchUnifiedRuntime.cmake` searching for the variables
-`UNIFIED_RUNTIME_REPO` and `UNIFIED_RUNTIME_TAG`.
-
-In order to enable developers, a number of CMake variables are available to
-control which revision of Unified Runtime should be used when building DPC++:
-
-* `SYCL_UR_OVERRIDE_FETCH_CONTENT_REPO` is a variable which can be used to
-  override the `UNIFIED_RUNTIME_REPO` variable used by `FetchContent` to attain
-  the Unified Runtime source code.
-* `SYCL_UR_OVERRIDE_FETCH_CONTENT_TAG` is a variable which can be used to
-  override the `UNIFIED_RUNTIME_TAG` variable used by `FetchContent` to attain
-  the Unified Runtime source code.
-* `SYCL_UR_USE_FETCH_CONTENT` is an option to control if CMake should use
-  `FetchContent` to pull in the Unified Runtime repository, it defaults to `ON`.
-  When set to `OFF`, `FetchContent` will not be used, instead:
-  * The path specified by variable `SYCL_UR_SOURCE_DIR` will be used with
-    `add_directory()`. This can be used to point at an adjacent directory
-    containing a clone of the Unified Runtime repository.
-  * The path `sycl/unified-runtime` will be used, if it
-    exists. This can be used as-if an in-tree build.
-* `SYCL_UR_SOURCE_DIR` is a variable used to specify the path to the Unified
-  Runtime repository when `SYCL_UR_USE_FETCH_CONTENT` is set of `OFF`.
-
 ### Build DPC++ libclc with a custom toolchain
 
 libclc is an implementation of the OpenCL required libraries, as described in

sycl/doc/GetStartedGuide.md

@@ -18,9 +18,6 @@
 namespace sycl {
 inline namespace _V1 {
 namespace detail {
-#ifdef XPTI_ENABLE_INSTRUMENTATION
-uint8_t GBufferStreamID;
-#endif
 void *buffer_impl::allocateMem(context_impl *Context, bool InitFromUserData,
                                void *HostPtr,
                                ur_event_handle_t &OutEventToWait) {

sycl/source/detail/buffer_impl.cpp

@@ -19,7 +19,6 @@
 #include <chrono>
 
 #ifdef XPTI_ENABLE_INSTRUMENTATION
-#include "xpti/xpti_trace_framework.hpp"
 #include <atomic>
 #include <detail/xpti_registry.hpp>
 #include <sstream>
@@ -28,10 +27,6 @@
 namespace sycl {
 inline namespace _V1 {
 namespace detail {
-#ifdef XPTI_ENABLE_INSTRUMENTATION
-extern xpti::trace_event_data_t *GSYCLGraphEvent;
-#endif
-
 // If we do not yet have a context, use the default one.
 void event_impl::initContextIfNeeded() {
   if (MContext || !MIsDefaultConstructed)
@@ -293,8 +288,7 @@ void event_impl::wait(bool *Success) {
   void *TelemetryEvent = nullptr;
   uint64_t IId = 0;
   std::string Name;
-  xpti::stream_id_t StreamID = xptiRegisterStream(SYCL_STREAM_NAME);
-  TelemetryEvent = instrumentationProlog(Name, StreamID, IId);
+  TelemetryEvent = instrumentationProlog(Name, GSYCLStreamID, IId);
 #endif
 
   auto EventHandle = getHandle();
@@ -306,7 +300,7 @@ void event_impl::wait(bool *Success) {
     detail::Scheduler::getInstance().waitForEvent(*this, Success);
 
 #ifdef XPTI_ENABLE_INSTRUMENTATION
-  instrumentationEpilog(TelemetryEvent, Name, StreamID, IId);
+  instrumentationEpilog(TelemetryEvent, Name, GSYCLStreamID, IId);
 #endif
 }
 

sycl/source/detail/event_impl.cpp

@@ -77,46 +77,28 @@ std::atomic_uint ObjectUsageCounter::MCounter{0};
 GlobalHandler::GlobalHandler() = default;
 GlobalHandler::~GlobalHandler() = default;
 
-void GlobalHandler::InitXPTI() {
-#ifdef XPTI_ENABLE_INSTRUMENTATION
-  // Let subscribers know a new stream is being initialized
-  getXPTIRegistry().initializeStream(SYCL_STREAM_NAME, GMajVer, GMinVer,
-                                     GVerStr);
-  xpti::payload_t SYCLPayload("SYCL Runtime Exceptions");
-  uint64_t SYCLInstanceNo;
-  GSYCLCallEvent = xptiMakeEvent("SYCL Try-catch Exceptions", &SYCLPayload,
-                                 xpti::trace_algorithm_event, xpti_at::active,
-                                 &SYCLInstanceNo);
-#endif
-}
-
 void GlobalHandler::TraceEventXPTI(const char *Message) {
   if (!Message)
     return;
 #ifdef XPTI_ENABLE_INSTRUMENTATION
-  static std::once_flag InitXPTIFlag;
+  // This section is used to emit XPTI trace events when exceptions occur
   if (xptiTraceEnabled()) {
-    std::call_once(InitXPTIFlag, [&]() { InitXPTI(); });
+    GlobalHandler::instance().getXPTIRegistry().initializeFrameworkOnce();
 
     // We have to handle the cases where: (1) we may have just the code location
     // set and not UID and (2) UID set
     detail::tls_code_loc_t Tls;
     auto CodeLocation = Tls.query();
 
-    // Creating a tracepoint will convert a CodeLocation to UID, if not set
-    xpti::framework::tracepoint_t TP(
+    // Creating a tracepoint using the stashed code location and notifying the
+    // subscriber with the diagnostic message
+    xpti::framework::tracepoint_scope_t TP(
         CodeLocation.fileName(), CodeLocation.functionName(),
         CodeLocation.lineNumber(), CodeLocation.columnNumber(), nullptr);
 
-    // The call to notify will have the signature of:
-    // (1) the stream defined in .stream()
-    // (2) The trace type equal to what is set by .trace_type()
-    // (3) Parent event set to NULL
-    // (4) Current event set to one created from CodeLocation and UID
-    // (5) An instance ID that records the number of times this code location
-    // has been seen (6) The message generated by the exception handler
-    TP.stream(SYCL_STREAM_NAME)
-        .trace_type(xpti::trace_point_type_t::diagnostics)
+    TP.stream(detail::GSYCLStreamID)
+        .traceType(xpti::trace_point_type_t::diagnostics)
+        .parentEvent(GSYCLCallEvent)
         .notify(static_cast<const void *>(Message));
   }
 

sycl/source/detail/global_handler.cpp

@@ -90,17 +90,12 @@ class GlobalHandler {
   void drainThreadPool();
   void prepareSchedulerToRelease(bool Blocking);
 
-  void InitXPTI();
   void TraceEventXPTI(const char *Message);
 
   // For testing purposes only
   void attachScheduler(Scheduler *Scheduler);
 
 private:
-#ifdef XPTI_ENABLE_INSTRUMENTATION
-  void *GSYCLCallEvent = nullptr;
-#endif
-
   bool OkToDefer = true;
 
   friend void shutdown_early(bool);