[L0] Implement Support for zeInitDrivers

- As of v1.10 of the L0 spec, zeInit and zeDriverGet is the old init
  pathway and the desired init api is zeInitDrivers. This new api allows
for multi heterogenous drivers to coexist in a single L0 Process.

Signed-off-by: Neil R. Spruit <[email protected]>

Author: nrspruit

.github/workflows/build-fuzz-reusable.yml

@@ -41,7 +41,7 @@ jobs:
 
     - name: Build level zero with gcc
       run: |
-        git clone -b v1.17.6 --depth=1 https://github.com/oneapi-src/level-zero.git ${{github.workspace}}/level-zero
+        git clone -b v1.18.5 --depth=1 https://github.com/oneapi-src/level-zero.git ${{github.workspace}}/level-zero
         cd ${{github.workspace}}/level-zero
         cmake -B build -DCMAKE_BUILD_TYPE=${{matrix.build_type}} -DCMAKE_C_COMPILER=gcc -DCMAKE_CXX_COMPILER=g++
         cmake --build build -j $(nproc)

cmake/FetchLevelZero.cmake

@@ -40,7 +40,7 @@ if (NOT DEFINED LEVEL_ZERO_LIBRARY OR NOT DEFINED LEVEL_ZERO_INCLUDE_DIR)
         set(UR_LEVEL_ZERO_LOADER_REPO "https://github.com/oneapi-src/level-zero.git")
     endif()
     if (UR_LEVEL_ZERO_LOADER_TAG STREQUAL "")
-        set(UR_LEVEL_ZERO_LOADER_TAG v1.18.5)
+        set(UR_LEVEL_ZERO_LOADER_TAG ea296eb72e760b9c04b93e008063ce95cdfe6488)
     endif()
 
     # Disable due to a bug https://github.com/oneapi-src/level-zero/issues/104

source/adapters/level_zero/adapter.cpp

@@ -76,19 +76,90 @@ ur_result_t getZesDeviceHandle(zes_uuid_t coreDeviceUuid,
   return UR_RESULT_ERROR_INVALID_ARGUMENT;
 }
 
+/**
+ * @brief Initializes the platforms by querying Level Zero drivers and devices.
+ *
+ * This function initializes the platforms by querying the available Level Zero
+ * drivers and devices. It handles different behaviors based on the presence of
+ * drivers obtained through `zeDriverGet` and initialized drivers through
+ * `zeInitDrivers`.
+ *
+ * @param platforms A vector to store the initialized platform handles.
+ * @param ZesResult The result of a previous ZES (Level Zero System) operation.
+ * @return ur_result_t The result of the initialization process.
+ *
+ * The function performs the following steps:
+ * 1. Queries the number of Level Zero drivers using `zeDriverGet`.
+ * 2. If drivers are found, it retrieves their handles.
+ * 3. If no drivers are found in either `zeInitDrivers` or `zeDriverGet`,
+ *    it logs a message and returns success.
+ * 4. If `zeInitDrivers` is supported by the global adapter, it retrieves
+ *    their handles and properties.
+ * 5. It compares the drivers obtained from `zeDriverGet` and `zeInitDrivers`,
+ *    adding unique drivers to the list.
+ * 6. If `zeInitDrivers` is not supported, it uses the drivers obtained
+ *    from `zeDriverGet`.
+ * 7. For each driver, it queries the devices and checks if they are GPU
+ * devices.
+ * 8. If a GPU device is found, it initializes a platform for the driver and
+ *    adds it to the platforms vector.
+ * 9. If ZES operations are successful, it populates the ZES/ZE device mapping
+ *    for the devices into the platform.
+ * 10. The function handles exceptions and returns the appropriate result.
+ */
 ur_result_t initPlatforms(PlatformVec &platforms,
                           ze_result_t ZesResult) noexcept try {
+  std::vector<ze_driver_handle_t> ZeDrivers;
+  std::vector<ze_driver_handle_t> ZeDriverGetHandles;
+  std::vector<ze_driver_handle_t> ZeInitDriversHandles;
+  std::vector<ze_device_handle_t> ZeDevices;
   uint32_t ZeDriverCount = 0;
-  ZE2UR_CALL(zeDriverGet, (&ZeDriverCount, nullptr));
-  if (ZeDriverCount == 0) {
+  uint32_t ZeDriverGetCount = 0;
+
+  auto ZeDriverGetResult =
+      ZE_CALL_NOCHECK(zeDriverGet, (&ZeDriverGetCount, nullptr));
+  if (ZeDriverGetCount > 0 && ZeDriverGetResult == ZE_RESULT_SUCCESS) {
+    ZeDriverGetHandles.resize(ZeDriverGetCount);
+    ZE2UR_CALL(zeDriverGet, (&ZeDriverGetCount, ZeDriverGetHandles.data()));
+  }
+  if (ZeDriverGetCount == 0 && GlobalAdapter->ZeInitDriversCount == 0) {
+    logger::debug("\nNo Valid L0 Drivers found.\n");
     return UR_RESULT_SUCCESS;
   }
 
-  std::vector<ze_driver_handle_t> ZeDrivers;
-  std::vector<ze_device_handle_t> ZeDevices;
-  ZeDrivers.resize(ZeDriverCount);
-
-  ZE2UR_CALL(zeDriverGet, (&ZeDriverCount, ZeDrivers.data()));
+  if (GlobalAdapter->InitDriversSupported) {
+    ZE2UR_CALL(GlobalAdapter->initDriversFunctionPtr,
+               (&GlobalAdapter->ZeInitDriversCount, ZeInitDriversHandles.data(),
+                &GlobalAdapter->InitDriversDesc));
+    ZeInitDriversHandles.resize(GlobalAdapter->ZeInitDriversCount);
+    ZeDrivers.resize(GlobalAdapter->ZeInitDriversCount);
+    ZeDrivers.assign(ZeInitDriversHandles.begin(), ZeInitDriversHandles.end());
+    if (ZeDriverGetCount > 0 && GlobalAdapter->ZeInitDriversCount > 0) {
+      for (uint32_t X = 0; X < GlobalAdapter->ZeInitDriversCount; ++X) {
+        for (uint32_t Y = 0; Y < ZeDriverGetCount; ++Y) {
+          ZeStruct<ze_driver_properties_t> ZeDriverGetProperties;
+          ZeStruct<ze_driver_properties_t> ZeInitDriverProperties;
+          ZE2UR_CALL(zeDriverGetProperties,
+                     (ZeDriverGetHandles[Y], &ZeDriverGetProperties));
+          ZE2UR_CALL(zeDriverGetProperties,
+                     (ZeInitDriversHandles[X], &ZeInitDriverProperties));
+          // If zeDriverGet driver is different from zeInitDriver driver, add it
+          // to the list. This allows for older drivers to be used alongside
+          // newer drivers.
+          if (ZeDriverGetProperties.uuid.id != ZeInitDriverProperties.uuid.id) {
+            logger::debug("\nzeDriverHandle {} added to the zeInitDrivers list "
+                          "of possible handles.\n",
+                          ZeDriverGetHandles[Y]);
+            ZeDrivers.push_back(ZeDriverGetHandles[Y]);
+          }
+        }
+      }
+    }
+  } else {
+    ZeDriverCount = ZeDriverGetCount;
+    ZeDrivers.resize(ZeDriverCount);
+    ZeDrivers.assign(ZeDriverGetHandles.begin(), ZeDriverGetHandles.end());
+  }
   for (uint32_t I = 0; I < ZeDriverCount; ++I) {
     // Keep track of the first platform init for this Driver
     bool DriverPlatformInit = false;
@@ -214,6 +285,15 @@ ur_adapter_handle_t_::ur_adapter_handle_t_()
       return std::atoi(UrRet);
     }();
 
+    // Dynamically load the new L0 apis separately.
+    // This must be done to avoid attempting to use symbols that do
+    // not exist in older loader runtimes.
+#ifdef _WIN32
+    HMODULE processHandle = GetModuleHandle(NULL);
+#else
+    HMODULE processHandle = nullptr;
+#endif
+
     // initialize level zero only once.
     if (GlobalAdapter->ZeResult == std::nullopt) {
       // Setting these environment variables before running zeInit will enable
@@ -235,20 +315,52 @@ ur_adapter_handle_t_::ur_adapter_handle_t_()
       // called multiple times.  Declaring the return value as "static" ensures
       // it's only called once.
 
+      GlobalAdapter->initDriversFunctionPtr =
+          (ze_pfnInitDrivers_t)ur_loader::LibLoader::getFunctionPtr(
+              processHandle, "zeInitDrivers");
+
+      // Set ZES_ENABLE_SYSMAN by default if the user has not set it.
+      if (UrSysManEnvInitEnabled) {
+        setEnvVar("ZES_ENABLE_SYSMAN", "1");
+      }
+
       // Init with all flags set to enable for all driver types to be init in
       // the application.
       ze_init_flags_t L0InitFlags = ZE_INIT_FLAG_GPU_ONLY;
       if (UrL0InitAllDrivers) {
         L0InitFlags |= ZE_INIT_FLAG_VPU_ONLY;
       }
-
-      // Set ZES_ENABLE_SYSMAN by default if the user has not set it.
-      if (UrSysManEnvInitEnabled) {
-        setEnvVar("ZES_ENABLE_SYSMAN", "1");
-      }
       logger::debug("\nzeInit with flags value of {}\n",
                     static_cast<int>(L0InitFlags));
-      GlobalAdapter->ZeResult = ZE_CALL_NOCHECK(zeInit, (L0InitFlags));
+      GlobalAdapter->ZeInitResult = ZE_CALL_NOCHECK(zeInit, (L0InitFlags));
+      if (*GlobalAdapter->ZeInitResult != ZE_RESULT_SUCCESS) {
+        logger::debug("\nzeInit failed with {}\n",
+                      *GlobalAdapter->ZeInitResult);
+      }
+
+      if (GlobalAdapter->initDriversFunctionPtr) {
+        GlobalAdapter->InitDriversDesc.pNext = nullptr;
+        GlobalAdapter->InitDriversDesc.flags = ZE_INIT_DRIVER_TYPE_FLAG_GPU;
+        logger::debug("\nzeInitDrivers with flags value of {}\n",
+                      static_cast<int>(GlobalAdapter->InitDriversDesc.flags));
+        GlobalAdapter->ZeInitDriversResult =
+            ZE_CALL_NOCHECK(GlobalAdapter->initDriversFunctionPtr,
+                            (&GlobalAdapter->ZeInitDriversCount, nullptr,
+                             &GlobalAdapter->InitDriversDesc));
+        if (*GlobalAdapter->ZeInitDriversResult == ZE_RESULT_SUCCESS) {
+          GlobalAdapter->InitDriversSupported = true;
+        } else {
+          logger::debug("\nzeInitDrivers failed with {}\n",
+                        *GlobalAdapter->ZeInitDriversResult);
+        }
+      }
+
+      if (*GlobalAdapter->ZeInitResult == ZE_RESULT_SUCCESS ||
+          *GlobalAdapter->ZeInitDriversResult == ZE_RESULT_SUCCESS) {
+        GlobalAdapter->ZeResult = ZE_RESULT_SUCCESS;
+      } else {
+        GlobalAdapter->ZeResult = ZE_RESULT_ERROR_UNINITIALIZED;
+      }
     }
     assert(GlobalAdapter->ZeResult !=
            std::nullopt); // verify that level-zero is initialized
@@ -260,19 +372,11 @@ ur_adapter_handle_t_::ur_adapter_handle_t_()
       return;
     }
     if (*GlobalAdapter->ZeResult != ZE_RESULT_SUCCESS) {
-      logger::error("zeInit: Level Zero initialization failure\n");
+      logger::error("Level Zero initialization failure\n");
       result = ze2urResult(*GlobalAdapter->ZeResult);
 
       return;
     }
-    // Dynamically load the new L0 SysMan separate init and new EXP apis
-    // separately. This must be done to avoid attempting to use symbols that do
-    // not exist in older loader runtimes.
-#ifdef _WIN32
-    HMODULE processHandle = GetModuleHandle(NULL);
-#else
-    HMODULE processHandle = nullptr;
-#endif
 
     // Check if the user has enabled the default L0 SysMan initialization.
     const int UrSysmanZesinitEnable = [] {

source/adapters/level_zero/adapter.hpp

@@ -17,6 +17,7 @@
 #include <optional>
 #include <ur/ur.hpp>
 #include <ze_api.h>
+#include <ze_ddi.h>
 #include <zes_ddi.h>
 
 using PlatformVec = std::vector<std::unique_ptr<ur_platform_handle_t_>>;
@@ -31,7 +32,14 @@ struct ur_adapter_handle_t_ {
   zes_pfnDriverGetDeviceByUuidExp_t getDeviceByUUIdFunctionPtr = nullptr;
   zes_pfnDriverGet_t getSysManDriversFunctionPtr = nullptr;
   zes_pfnInit_t sysManInitFunctionPtr = nullptr;
+  ze_pfnInitDrivers_t initDriversFunctionPtr = nullptr;
+  ze_init_driver_type_desc_t InitDriversDesc = {
+      ZE_STRUCTURE_TYPE_INIT_DRIVER_TYPE_DESC, nullptr, 0};
+  uint32_t ZeInitDriversCount = 0;
+  bool InitDriversSupported = false;
 
+  std::optional<ze_result_t> ZeInitDriversResult;
+  std::optional<ze_result_t> ZeInitResult;
   std::optional<ze_result_t> ZeResult;
   std::optional<ze_result_t> ZesResult;
   ZeCache<Result<PlatformVec>> PlatformCache;