[SYCL][UR] Unify logging and leak checking for L0 v1 and v2 (#19328)

This is needed so that we can enable V2 adapter by default on certain
platforms: #19333

The reason is that we need to load both adapters (legacy and v2) to
check the device version. However, loading v2 adapter causes L0 loader
to emit logs for all API calls (if ZE_DEBUG=1 is set). Since the legacy
adapter used different logic for printing API calls, this would result
in printing the same logs twice. This patch fixes that.

Author: igchor

sycl/test-e2e/USM/usm_pooling.cpp

@@ -106,8 +106,8 @@ int main(int argc, char *argv[]) {
 // CHECK-NOPOOL-NEXT:  zeDeviceGetMemoryAccessProperties
 // CHECK-NOPOOL-NEXT:  [[API]](
 // CHECK-NOPOOL-NEXT:  [[API]](
-// CHECK-NOPOOL-NEXT:  zeMemFree
-// CHECK-NOPOOL-NEXT:  zeMemFree
+// CHECK-NOPOOL:  zeMemFree
+// CHECK-NOPOOL:  zeMemFree
 // CHECK-NOPOOL-NEXT:  [[API]](
 // CHECK-NOPOOL-NEXT:  [[API]](
 // CHECK-NOPOOL-NEXT:  [[API]](
@@ -116,8 +116,8 @@ int main(int argc, char *argv[]) {
 // CHECK-12345-NEXT:  zeDeviceGetMemoryAccessProperties
 // CHECK-12345-NEXT:  [[API]](
 // CHECK-12345-NEXT:  [[API]](
-// CHECK-12345-NEXT:  zeMemFree
-// CHECK-12345-NEXT:  zeMemFree
+// CHECK-12345:  zeMemFree
+// CHECK-12345:  zeMemFree
 // CHECK-12345-NEXT:  [[API]](
 // CHECK-12345-NEXT:  [[API]](
 // CHECK-12345-NEXT:  [[API]](
@@ -126,12 +126,12 @@ int main(int argc, char *argv[]) {
 // CHECK-1245-NEXT:  zeDeviceGetMemoryAccessProperties
 // CHECK-1245-NEXT:  [[API]](
 // CHECK-1245-NEXT:  [[API]](
-// CHECK-1245-NEXT:  zeMemFree
+// CHECK-1245:  zeMemFree
 // CHECK-1245-NEXT:  [[API]](
 // CHECK-1245-NEXT:  [[API]](
 
 // CHECK-15: Test [[API:zeMemAllocHost|zeMemAllocDevice|zeMemAllocShared]]
 // CHECK-15-NEXT:  zeDeviceGetMemoryAccessProperties
 // CHECK-15-NEXT:  [[API]](
 // CHECK-15-NEXT:  [[API]](
-// CHECK-15-NEXT:  zeMemFree
+// CHECK-15:  zeMemFree

sycl/ur_win_proxy_loader/ur_win_proxy_loader.cpp

@@ -107,27 +107,6 @@ void *&getDllHandle() {
   return dllHandle;
 }
 
-static bool shouldLoadL0V2adapter() {
-  auto SyclEnv = std::getenv("SYCL_UR_USE_LEVEL_ZERO_V2");
-  auto UREnv = std::getenv("UR_LOADER_USE_LEVEL_ZERO_V2");
-
-  try {
-    if (SyclEnv && std::stoi(SyclEnv) == 1) {
-      return true;
-    }
-  } catch (...) {
-  }
-
-  try {
-    if (UREnv && std::stoi(UREnv) == 1) {
-      return true;
-    }
-  } catch (...) {
-  }
-
-  return false;
-}
-
 /// Load the adapter libraries
 void preloadLibraries() {
   // Suppress system errors.
@@ -163,8 +142,7 @@ void preloadLibraries() {
   getDllHandle() = loadAdapter(UR_LIBRARY_NAME(loader));
   loadAdapter(UR_LIBRARY_NAME(adapter_opencl));
   loadAdapter(UR_LIBRARY_NAME(adapter_level_zero));
-  if (shouldLoadL0V2adapter())
-    loadAdapter(UR_LIBRARY_NAME(adapter_level_zero_v2));
+  loadAdapter(UR_LIBRARY_NAME(adapter_level_zero_v2));
   loadAdapter(UR_LIBRARY_NAME(adapter_cuda));
   loadAdapter(UR_LIBRARY_NAME(adapter_hip));
   loadAdapter(UR_LIBRARY_NAME(adapter_native_cpu));

unified-runtime/source/adapters/level_zero/adapter.cpp

@@ -309,32 +309,23 @@ ur_adapter_handle_t_::ur_adapter_handle_t_()
 
   if (UrL0Debug & UR_L0_DEBUG_BASIC) {
     logger.setLegacySink(std::make_unique<ur_legacy_sink>());
-#ifdef UR_ADAPTER_LEVEL_ZERO_V2
     setEnvVar("ZEL_ENABLE_LOADER_LOGGING", "1");
     setEnvVar("ZEL_LOADER_LOGGING_LEVEL", "trace");
     setEnvVar("ZEL_LOADER_LOG_CONSOLE", "1");
     setEnvVar("ZE_ENABLE_VALIDATION_LAYER", "1");
-#endif
   };
 
   if (UrL0Debug & UR_L0_DEBUG_VALIDATION) {
     setEnvVar("ZE_ENABLE_VALIDATION_LAYER", "1");
     setEnvVar("ZE_ENABLE_PARAMETER_VALIDATION", "1");
   }
 
-  PlatformCache.Compute = [](Result<PlatformVec> &result) {
-    static std::once_flag ZeCallCountInitialized;
-    try {
-      std::call_once(ZeCallCountInitialized, []() {
-        if (UrL0LeaksDebug) {
-          ZeCallCount = new std::map<std::string, int>;
-        }
-      });
-    } catch (...) {
-      result = exceptionToResult(std::current_exception());
-      return;
-    }
+  if (UrL0LeaksDebug) {
+    setEnvVar("ZE_ENABLE_VALIDATION_LAYER", "1");
+    setEnvVar("ZEL_ENABLE_BASIC_LEAK_CHECKER", "1");
+  }
 
+  PlatformCache.Compute = [](Result<PlatformVec> &result) {
     uint32_t UserForcedSysManInit = 0;
     // Check if the user has disabled the default L0 Env initialization.
     const int UrSysManEnvInitEnabled = [&UserForcedSysManInit] {
@@ -426,7 +417,6 @@ ur_adapter_handle_t_::ur_adapter_handle_t_()
           useInitDrivers = true;
         }
 
-#ifdef UR_ADAPTER_LEVEL_ZERO_V2
         if ((loader_version.major == 1 && loader_version.minor < 21) ||
             (loader_version.major == 1 && loader_version.minor == 21 &&
              loader_version.patch < 2)) {
@@ -435,7 +425,6 @@ ur_adapter_handle_t_::ur_adapter_handle_t_()
               "WARNING: Level Zero Loader version is older than 1.21.2. "
               "Please update to the latest version for API logging support.\n");
         }
-#endif
       }
 
       if (useInitDrivers) {
@@ -552,97 +541,6 @@ void globalAdapterOnDemandCleanup() {
 }
 
 ur_result_t adapterStateTeardown() {
-  // Print the balance of various create/destroy native calls.
-  // The idea is to verify if the number of create(+) and destroy(-) calls are
-  // matched.
-  if (ZeCallCount && (UrL0LeaksDebug) != 0) {
-    bool LeakFound = false;
-    // clang-format off
-    //
-    // The format of this table is such that each row accounts for a
-    // specific type of objects, and all elements in the raw except the last
-    // one are allocating objects of that type, while the last element is known
-    // to deallocate objects of that type.
-    //
-    std::vector<std::vector<std::string>> CreateDestroySet = {
-      {"zeContextCreate",      "zeContextDestroy"},
-      {"zeCommandQueueCreate", "zeCommandQueueDestroy"},
-      {"zeModuleCreate",       "zeModuleDestroy"},
-      {"zeKernelCreate",       "zeKernelDestroy"},
-      {"zeEventPoolCreate",    "zeEventPoolDestroy"},
-      {"zeCommandListCreateImmediate", "zeCommandListCreate", "zeCommandListDestroy"},
-      {"zeEventCreate",        "zeEventDestroy"},
-      {"zeFenceCreate",        "zeFenceDestroy"},
-      {"zeImageCreate","zeImageViewCreateExt",        "zeImageDestroy"},
-      {"zeSamplerCreate",      "zeSamplerDestroy"},
-      {"zeMemAllocDevice", "zeMemAllocHost", "zeMemAllocShared", "zeMemFree"},
-    };
-
-    // A sample output aimed below is this:
-    // ------------------------------------------------------------------------
-    //                zeContextCreate = 1     \--->        zeContextDestroy = 1
-    //           zeCommandQueueCreate = 1     \--->   zeCommandQueueDestroy = 1
-    //                 zeModuleCreate = 1     \--->         zeModuleDestroy = 1
-    //                 zeKernelCreate = 1     \--->         zeKernelDestroy = 1
-    //              zeEventPoolCreate = 1     \--->      zeEventPoolDestroy = 1
-    //   zeCommandListCreateImmediate = 1     |
-    //            zeCommandListCreate = 1     \--->    zeCommandListDestroy = 1  ---> LEAK = 1
-    //                  zeEventCreate = 2     \--->          zeEventDestroy = 2
-    //                  zeFenceCreate = 1     \--->          zeFenceDestroy = 1
-    //                  zeImageCreate = 0     \--->          zeImageDestroy = 0
-    //                zeSamplerCreate = 0     \--->        zeSamplerDestroy = 0
-    //               zeMemAllocDevice = 0     |
-    //                 zeMemAllocHost = 1     |
-    //               zeMemAllocShared = 0     \--->               zeMemFree = 1
-    //
-    // clang-format on
-    // TODO: use logger to print this messages
-    std::cerr << "Check balance of create/destroy calls\n";
-    std::cerr << "----------------------------------------------------------\n";
-    std::stringstream ss;
-    for (const auto &Row : CreateDestroySet) {
-      int diff = 0;
-      for (auto I = Row.begin(); I != Row.end();) {
-        const char *ZeName = (*I).c_str();
-        const auto &ZeCount = (*ZeCallCount)[*I];
-
-        bool First = (I == Row.begin());
-        bool Last = (++I == Row.end());
-
-        if (Last) {
-          ss << " \\--->";
-          diff -= ZeCount;
-        } else {
-          diff += ZeCount;
-          if (!First) {
-            ss << " | ";
-            std::cerr << ss.str() << "\n";
-            ss.str("");
-            ss.clear();
-          }
-        }
-        ss << std::setw(30) << std::right << ZeName;
-        ss << " = ";
-        ss << std::setw(5) << std::left << ZeCount;
-      }
-
-      if (diff) {
-        LeakFound = true;
-        ss << " ---> LEAK = " << diff;
-      }
-
-      std::cerr << ss.str() << '\n';
-      ss.str("");
-      ss.clear();
-    }
-
-    ZeCallCount->clear();
-    delete ZeCallCount;
-    ZeCallCount = nullptr;
-    if (LeakFound)
-      return UR_RESULT_ERROR_INVALID_MEM_OBJECT;
-  }
-
   // Due to multiple DLLMain definitions with SYCL, register to cleanup the
   // Global Adapter after refcnt is 0
 #if defined(_WIN32)

unified-runtime/source/adapters/level_zero/common.cpp

@@ -86,8 +86,6 @@ bool setEnvVar(const char *name, const char *value) {
 
 ZeUSMImportExtension ZeUSMImport;
 
-std::map<std::string, int> *ZeCallCount = nullptr;
-
 void zeParseError(ze_result_t ZeError, const char *&ErrorString) {
   switch (ZeError) {
 #define ZE_ERRCASE(ERR)                                                        \
@@ -137,31 +135,10 @@ void zeParseError(ze_result_t ZeError, const char *&ErrorString) {
   } // switch
 }
 
-#ifdef UR_ADAPTER_LEVEL_ZERO_V2
 ze_result_t ZeCall::doCall(ze_result_t ZeResult, const char *, const char *,
                            bool) {
   return ZeResult;
 }
-#else
-ze_result_t ZeCall::doCall(ze_result_t ZeResult, const char *ZeName,
-                           const char *ZeArgs, bool TraceError) {
-  UR_LOG(DEBUG, "ZE ---> {}{}", ZeName, ZeArgs);
-
-  if (ZeResult == ZE_RESULT_SUCCESS) {
-    if (UrL0LeaksDebug) {
-      ++(*ZeCallCount)[ZeName];
-    }
-    return ZE_RESULT_SUCCESS;
-  }
-
-  if (TraceError) {
-    const char *ErrorString = "Unknown";
-    zeParseError(ZeResult, ErrorString);
-    UR_LOG(ERR, "Error ({}) in {}", ErrorString, ZeName);
-  }
-  return ZeResult;
-}
-#endif
 
 // Specializations for various L0 structures
 template <> ze_structure_type_t getZeStructureType<ze_event_pool_desc_t>() {

unified-runtime/source/adapters/level_zero/common.hpp

@@ -328,9 +328,6 @@ class ZeUSMImportExtension {
 // Helper wrapper for working with USM import extension in Level Zero.
 extern ZeUSMImportExtension ZeUSMImport;
 
-// This will count the calls to Level-Zero
-extern std::map<std::string, int> *ZeCallCount;
-
 // Some opencl extensions we know are supported by all Level Zero devices.
 constexpr char ZE_SUPPORTED_EXTENSIONS[] =
     "cl_khr_il_program cl_khr_subgroups cl_intel_subgroups "

unified-runtime/source/adapters/level_zero/v2/command_buffer.cpp

@@ -167,6 +167,8 @@ ur_result_t ur_exp_command_buffer_handle_t_::registerExecutionEventUnlocked(
 }
 
 ur_exp_command_buffer_handle_t_::~ur_exp_command_buffer_handle_t_() {
+  UR_CALL_NOCHECK(commandListManager.lock()->releaseSubmittedKernels());
+
   if (currentExecution) {
     currentExecution->release();
   }