[PTI-LIB] Avoid conflict with unitrace for sycl runtime (#644)

* [PTI-LIB] Avoid conflict with unitrace or other subscriber for sycl runtime
- not subscribe for XPTI if someone else (e.g. unitrace) already subscribed
- removed xpti subscription from pti core library, it goes via pti_view
- added xpti_interop_test

---------

Signed-off-by: jfedorov <julia.fedorova@intel.com>
Co-authored-by: Copilot <175728472+Copilot@users.noreply.github.com>

Author: jfedorov

sdk/src/metrics_handler.h

@@ -1502,9 +1502,7 @@ class PtiMetricsCollectorHandler {
     if (!env_string.empty()) {
       spdlog::cfg::helpers::load_levels(env_string);
     }
-
-    // https://github.com/gabime/spdlog/wiki/3.-Custom-formatting
-    spdlog::set_pattern("[%H:%M][%^-%l-%$]%P:%t %s:%# %v");
+    utils::SetGlobalSpdLogPattern();
 
     // Initialize L0
     ze_result_t status = zeInit(ZE_INIT_FLAG_GPU_ONLY);

sdk/src/pti_lib_handler.h

@@ -49,6 +49,11 @@ inline std::string GetPathToWindowsLibraryDirectory() {
 }
 #endif
 
+// API implemented in PTI Core library
+// to pass from PTI Interface library status of detected "foreign" XPTI subscribers,
+// if such were detected at the library load time
+void PtiSetXPTIEnvironmentDetails(bool is_foreign_subscriber, bool is_likely_unitrace_subscriber);
+
 class PtiLibHandler {
  public:
   static auto& Instance() {
@@ -94,8 +99,21 @@ class PtiLibHandler {
   decltype(&ptiMetricsGetDevices) ptiMetricsGetDevices_ = nullptr;                        // NOLINT
   decltype(&ptiMetricsStartCollection) ptiMetricsStartCollection_ = nullptr;              // NOLINT
   decltype(&ptiMetricGetCalculatedData) ptiMetricGetCalculatedData_ = nullptr;            // NOLINT
+  decltype(&PtiSetXPTIEnvironmentDetails) PtiSetXPTIEnvironmentDetails_ = nullptr;        // NOLINT
 
  private:
+  inline void CommunicateForeignXPTISubscriber() {
+    // Passing information about XPTI subscriber to PTI Core library right after it is loaded.
+    // This should be done before any other call to PTI Core, as it might create Sycl collector
+    // and it should not enable it, in case if some foreign subscriber already subscribed for XPTI
+    if (PtiSetXPTIEnvironmentDetails_) {
+      auto [is_foreign_subscriber, is_likely_unitrace_subscriber] = IsForeignXPTISubscriber();
+      PtiSetXPTIEnvironmentDetails_(is_foreign_subscriber, is_likely_unitrace_subscriber);
+    } else {
+      SPDLOG_DEBUG("PtiSetXPTIEnvironmentDetails_ is not available in the loaded library.");
+    }
+  }
+
   PtiLibHandler() {
     try {
       spdlog::set_level(spdlog::level::off);
@@ -104,6 +122,7 @@ class PtiLibHandler {
       if (!env_string.empty()) {
         spdlog::cfg::helpers::load_levels(env_string);
       }
+      ::utils::SetGlobalSpdLogPattern();
 
       std::string pti_dir;
 #if defined(_WIN32)
@@ -148,7 +167,9 @@ class PtiLibHandler {
     PTI_VIEW_GET_SYMBOL(ptiMetricsGetDevices);
     PTI_VIEW_GET_SYMBOL(ptiMetricsStartCollection);
     PTI_VIEW_GET_SYMBOL(ptiMetricGetCalculatedData);
+    PTI_VIEW_GET_SYMBOL(PtiSetXPTIEnvironmentDetails);
 #undef PTI_VIEW_GET_SYMBOL
+    CommunicateForeignXPTISubscriber();
   }
   std::unique_ptr<LibraryLoader> pti_view_lib_ = nullptr;
 };

sdk/src/sycl/sycl_collector.h

@@ -150,12 +150,21 @@ inline bool InMemoryCoreApis(const char* function_name) {
 
 class SyclCollector {
  public:
+  // Variables to indicate the presence of a foreign subscriber to XPTI.
+  // To be set via call to ptiSetXPTIEnvironmentDetails before SyclCollector created
+  static inline bool foreign_subscriber_ = false;
+  static inline bool likely_unitrace_subscriber_ = false;
+
   inline static auto& Instance() {
     static SyclCollector sycl_collector{nullptr};
     return sycl_collector;
   }
 
   inline void EnableTracing() {
+    // Do not change the behaviour here depending on foreign_subscriber_!
+    // The current behavior ensures that in the absense of XPTI subscription (and Sycl records)
+    // PTI generates to-called Special Records
+    // About Special Records: see comments in unitrace.h
     enabled_ = true;
     xptiForceSetTraceEnabled(enabled_);
   }
@@ -361,8 +370,24 @@ class SyclCollector {
   }
 
  private:
+#define WARNING_FOREIGN_SUBSCRIBER                                                 \
+  "Another subscriber already subscribed to Sycl runtime events, "                 \
+  "so PTI will not subscribe to them. It will affect correctness of PTI profile: " \
+  "e.g. report zero XPU time for CPU callers of GPU kernels."
+
+#define WARNING_LIKELY_UNITRACE_SUBSCRIBER           \
+  " Likely the application running under Unitrace. " \
+  "To get correct PTI profile - run without Unitrace."
+
   explicit SyclCollector(OnSyclRuntimeViewCallback buffer_callback)
-      : acallback_(buffer_callback), xptiGetStashedKV(GetStashedFuncPtrFromSharedObject()) {}
+      : acallback_(buffer_callback), xptiGetStashedKV(GetStashedFuncPtrFromSharedObject()) {
+    static constexpr char warn_foreign_subscriber[] = WARNING_FOREIGN_SUBSCRIBER;
+    static constexpr char warn_likely_unitrace_subscriber[] = WARNING_LIKELY_UNITRACE_SUBSCRIBER;
+    if (foreign_subscriber_) {
+      const char* warn_add = likely_unitrace_subscriber_ ? warn_likely_unitrace_subscriber : "";
+      SPDLOG_WARN("{}{}", warn_foreign_subscriber, warn_add);
+    }
+  }
 
   int32_t trace_all_env_value = utils::IsSetEnv("PTI_VIEW_RUNTIME_API");
   inline static thread_local ZeKernelCommandExecutionRecord sycl_runtime_rec_;
@@ -571,18 +596,30 @@ XPTI_CALLBACK_API void xptiTraceInit(unsigned int major_version, unsigned int mi
 }
 
 XPTI_CALLBACK_API void xptiTraceFinish(const char* /*stream_name*/) {}
+
+// clang-format off
+extern "C" {
+  void
+#if (defined(_WIN32) || defined(_WIN64))
+  __declspec(dllexport)
+#else
+  __attribute__((visibility("default")))
+#endif
+  PtiSetXPTIEnvironmentDetails(bool is_foreign_subscriber,
+                               bool is_likely_unitrace_subscriber) {
+    SyclCollector::foreign_subscriber_ = is_foreign_subscriber;
+    SyclCollector::likely_unitrace_subscriber_ = is_likely_unitrace_subscriber;
+  }
+}
+// clang-format on
+
 #if (defined(_WIN32) || defined(_WIN64))
 
 #include <windows.h>
 
-BOOL WINAPI DllMain(HINSTANCE hinstDLL, DWORD fwdReason, LPVOID /*lpvReserved*/) {
+BOOL WINAPI DllMain(HINSTANCE /*hinstDLL*/, DWORD fwdReason, LPVOID /*lpvReserved*/) {
   switch (fwdReason) {
     case DLL_PROCESS_ATTACH: {
-      utils::SetEnv("XPTI_SUBSCRIBERS", utils::GetPathToSharedObject(hinstDLL).c_str());
-      utils::SetEnv("XPTI_FRAMEWORK_DISPATCHER",
-                    utils::GetPathToSharedObject(pti::strings::kXptiLibName).c_str());
-      utils::SetEnv("XPTI_TRACE_ENABLE", "1");
-      utils::SetEnv("UR_ENABLE_LAYERS", "UR_LAYER_TRACING");
       break;
     }
     case DLL_THREAD_ATTACH:
@@ -599,16 +636,6 @@ BOOL WINAPI DllMain(HINSTANCE hinstDLL, DWORD fwdReason, LPVOID /*lpvReserved*/)
 
 #else  // Linux (possibly macOS?)
 
-// Work-around for ensuring XPTI_SUBSCRIBERS and XPTI_FRAMEWORK_DISPATCHER are
-// set before xptiTraceInit() is called.
-__attribute__((constructor)) static void framework_init() {
-  utils::SetEnv("XPTI_SUBSCRIBERS", utils::GetPathToSharedObject(Truncate).c_str());
-  utils::SetEnv("XPTI_FRAMEWORK_DISPATCHER", utils::GetPathToSharedObject(xptiReset).c_str());
-  utils::SetEnv("XPTI_TRACE_ENABLE", "1");
-
-  utils::SetEnv("UR_ENABLE_LAYERS", "UR_LAYER_TRACING");
-}
-
 __attribute__((destructor)) static void framework_fini() { framework_finalized = true; }
 
 #endif

sdk/src/utils/utils.h

@@ -460,6 +460,17 @@ inline void* GetFunctionPtr(HMODULE lib_handle, const char* func_name) {
 #endif /* _WIN32 */
 }
 
+inline std::pair<bool, bool> IsSubscriberToXPTI() {
+  auto current_xpti_subscriber = utils::GetEnv("XPTI_SUBSCRIBERS");
+  // and may be it is unitrace
+  return {!current_xpti_subscriber.empty(),
+          current_xpti_subscriber.find("unitrace") != std::string::npos};
+}
+
+inline void SetGlobalSpdLogPattern() {
+  // https://github.com/gabime/spdlog/wiki/3.-Custom-formatting
+  spdlog::set_pattern("PTI:[%H:%M][%^-%l-%$]%P:%t %s:%# %v");
+}
 }  // namespace utils
 
 #endif  // PTI_UTILS_UTILS_H_

sdk/src/view_handler.h

@@ -278,8 +278,7 @@ struct PtiViewRecordHandler {
     if (!env_string.empty()) {
       spdlog::cfg::helpers::load_levels(env_string);
     }
-    // https://github.com/gabime/spdlog/wiki/3.-Custom-formatting
-    spdlog::set_pattern("[%H:%M][%^-%l-%$]%P:%t %s:%# %v");
+    utils::SetGlobalSpdLogPattern();
 
     if (!collector_) {
       CollectorOptions collector_options{};

sdk/src/xpti_adapter.cc

@@ -9,16 +9,34 @@
 
 class GlobalSyclInitializer {
  public:
+  // variables to store if XPTI is subscribed by some other subscriber
+  // and if this subscriber is likely unitrace
+  inline static bool foreign_subscriber_ = false;
+  inline static bool likely_unitrace_subscriber_ = false;
+
   inline static bool Initialize() {
-    utils::SetEnv("XPTI_SUBSCRIBERS", utils::GetPathToSharedObject(Initialize).c_str());
-    utils::SetEnv("XPTI_FRAMEWORK_DISPATCHER", pti::strings::kXptiLibName);
-    utils::SetEnv("XPTI_TRACE_ENABLE", "1");
+    // respecting another XPTI subscriber:
+    // if someone already subscribed (e.g. might be that the app runs under unitrace) -
+    // we do not subscribe for XPTI
+    std::tie(foreign_subscriber_, likely_unitrace_subscriber_) = utils::IsSubscriberToXPTI();
+    if (!foreign_subscriber_) {
+      utils::SetEnv("XPTI_SUBSCRIBERS", utils::GetPathToSharedObject(Initialize).c_str());
+      utils::SetEnv("XPTI_FRAMEWORK_DISPATCHER", pti::strings::kXptiLibName);
+      utils::SetEnv("XPTI_TRACE_ENABLE", "1");
+
+      utils::SetEnv("UR_ENABLE_LAYERS", "UR_LAYER_TRACING");
+    }
     return true;
   }
 
   inline static bool result_ = Initialize();
 };
 
+std::pair<bool, bool> IsForeignXPTISubscriber() {
+  return {GlobalSyclInitializer::foreign_subscriber_,
+          GlobalSyclInitializer::likely_unitrace_subscriber_};
+}
+
 void xptiTraceInit(unsigned int major_version, unsigned int minor_version, const char* version_str,
                    const char* stream_name) {
   if (!pti::PtiLibHandler::Instance().xptiTraceInit_) {

sdk/src/xpti_adapter.h

@@ -19,4 +19,6 @@ PTI_XPTI_HOOK_VISIBILITY void xptiTraceInit(unsigned int major_version, unsigned
 PTI_XPTI_HOOK_VISIBILITY void xptiTraceFinish(const char* stream_name);  // NOLINT
 }
 
+std::pair<bool, bool> IsForeignXPTISubscriber();
+
 #endif  // SRC_XPTI_ADAPTER_H_

sdk/test/CMakeLists.txt

@@ -257,23 +257,33 @@ if(HAVE_SYCL)
   add_executable(dpc_gemm_threaded_profiled perf_dpc_gemm_threaded.cc)
   add_executable(dpc_gemm_threaded_baseline perf_dpc_gemm_threaded.cc)
   add_executable(dpc_gemm_threaded_linkonly perf_dpc_gemm_threaded.cc)
+  add_executable(dpc_gemm_threaded_prof_print perf_dpc_gemm_threaded.cc)
 
   target_include_directories(
     dpc_gemm_threaded_profiled
     PUBLIC "${CMAKE_BINARY_DIR}" "${PROJECT_SOURCE_DIR}/include"
          "${PROJECT_SOURCE_DIR}/samples/samples_utilities"
          "${PROJECT_SOURCE_DIR}/src/utils")
 
+  target_include_directories(
+    dpc_gemm_threaded_prof_print
+    PUBLIC "${CMAKE_BINARY_DIR}" "${PROJECT_SOURCE_DIR}/include"
+         "${PROJECT_SOURCE_DIR}/samples/samples_utilities"
+         "${PROJECT_SOURCE_DIR}/src/utils")
+
   target_compile_options(dpc_gemm_threaded_profiled PRIVATE -fsycl -gline-tables-only)
   target_compile_options(dpc_gemm_threaded_baseline PRIVATE -fsycl -gline-tables-only)
   target_compile_options(dpc_gemm_threaded_linkonly PRIVATE -fsycl -gline-tables-only)
+  target_compile_options(dpc_gemm_threaded_prof_print PRIVATE -fsycl -gline-tables-only)
 
   target_compile_definitions(dpc_gemm_threaded_baseline PRIVATE NO_PTI)
   target_compile_definitions(dpc_gemm_threaded_linkonly PRIVATE NO_PTI)
+  target_compile_definitions(dpc_gemm_threaded_prof_print PRIVATE  RECORD_PARSE_AND_PRINT)
 
   target_link_options(dpc_gemm_threaded_profiled PRIVATE -fsycl -gline-tables-only)
   target_link_options(dpc_gemm_threaded_baseline PRIVATE -fsycl -gline-tables-only)
   target_link_options(dpc_gemm_threaded_linkonly PRIVATE -fsycl -gline-tables-only)
+  target_link_options(dpc_gemm_threaded_prof_print PRIVATE -fsycl -gline-tables-only)
 
   target_link_libraries(dpc_gemm_threaded_profiled PUBLIC spdlog::spdlog
     Threads::Threads Pti::pti_view)
@@ -283,6 +293,9 @@ if(HAVE_SYCL)
 
   target_link_libraries(dpc_gemm_threaded_baseline PUBLIC Threads::Threads)
 
+  target_link_libraries(dpc_gemm_threaded_prof_print PUBLIC spdlog::spdlog
+    Threads::Threads Pti::pti_view)
+
   gtest_discover_tests(
     dpcgemm_suite
     DISCOVERY_TIMEOUT 60
@@ -415,6 +428,14 @@ if(HAVE_SYCL)
   set_tests_properties(
     perf-overhead-view PROPERTIES LABELS "performance; functional")
 
+  # Add interoperability test
+  add_test(
+    NAME xpti_interop_test
+    COMMAND ${Python_EXECUTABLE} ${PROJECT_SOURCE_DIR}/test/xpti_interop_test.py "${PTI_TEST_BIN_DIR}"
+  )
+  set_tests_properties(
+    xpti_interop_test PROPERTIES LABELS "interop; functional")
+
 endif()
 
 if (HAVE_SYCL AND TARGET unified-runtime::loader)