Implement ZE_ENABLE_PCI_ID_DEVICE_ORDER environment variable

Signed-off-by: Jitendra Sharma <[email protected]>

Author: jitendrasharma1989

level_zero/CMakeLists.txt

@@ -361,15 +361,18 @@ if(BUILD_WITH_L0)
                                    L0_SOURCES_WINDOWS
                                    L0_SRCS_COMPILER_INTERFACE
                                    L0_SRCS_DEBUGGER
+                                   L0_SRCS_DRIVER
                                    L0_SRCS_OCLOC_SHARED
     )
     if(WIN32)
       append_sources_from_properties(L0_RUNTIME_SOURCES
                                      L0_SRCS_DEBUGGER_WINDOWS
+                                     L0_SRCS_DRIVER_WINDOWS
       )
     else()
       append_sources_from_properties(L0_RUNTIME_SOURCES
                                      L0_SRCS_DEBUGGER_LINUX
+                                     L0_SRCS_DRIVER_LINUX
       )
     endif()
 

level_zero/core/source/driver/CMakeLists.txt

@@ -0,0 +1,12 @@
+#
+# Copyright (C) 2020 Intel Corporation
+#
+# SPDX-License-Identifier: MIT
+#
+
+set(L0_SRCS_DRIVER
+    ${CMAKE_CURRENT_SOURCE_DIR}/CMakeLists.txt
+)
+
+add_subdirectories()
+set_property(GLOBAL PROPERTY L0_SRCS_DRIVER ${L0_SRCS_DRIVER})

level_zero/core/source/driver/driver.cpp

@@ -41,6 +41,8 @@ void DriverImp::initialize(ze_result_t *result) {
         envReader.getSetting("ZET_ENABLE_PROGRAM_INSTRUMENTATION", false);
     envVariables.sysman =
         envReader.getSetting("ZES_ENABLE_SYSMAN", false);
+    envVariables.pciIdDeviceOrder =
+        envReader.getSetting("ZE_ENABLE_PCI_ID_DEVICE_ORDER", false);
 
     auto executionEnvironment = new NEO::ExecutionEnvironment();
     UNRECOVERABLE_IF(nullptr == executionEnvironment);

level_zero/core/source/driver/driver_handle_imp.cpp

@@ -229,6 +229,10 @@ ze_result_t DriverHandleImp::initialize(std::vector<std::unique_ptr<NEO::Device>
 
     uint32_t affinityMask = std::numeric_limits<uint32_t>::max();
 
+    if (enablePciIdDeviceOrder) {
+        sortNeoDevices(neoDevices);
+    }
+
     if (this->affinityMaskString.length() > 0) {
         affinityMask = parseAffinityMask(neoDevices);
     }
@@ -292,6 +296,7 @@ DriverHandle *DriverHandle::create(std::vector<std::unique_ptr<NEO::Device>> dev
     driverHandle->affinityMaskString = envVariables.affinityMask;
     driverHandle->enableProgramDebugging = envVariables.programDebugging;
     driverHandle->enableSysman = envVariables.sysman;
+    driverHandle->enablePciIdDeviceOrder = envVariables.pciIdDeviceOrder;
 
     ze_result_t res = driverHandle->initialize(std::move(devices));
     if (res != ZE_RESULT_SUCCESS) {

level_zero/core/source/driver/driver_handle_imp.h

@@ -82,6 +82,7 @@ struct DriverHandleImp : public DriverHandle {
                                                                      uintptr_t *gpuAddress) override;
     uint32_t parseAffinityMask(std::vector<std::unique_ptr<NEO::Device>> &neoDevices);
     void createHostPointerManager();
+    void sortNeoDevices(std::vector<std::unique_ptr<NEO::Device>> &neoDevices);
 
     std::unique_ptr<HostPointerManager> hostPointerManager;
     // Experimental functions
@@ -107,6 +108,7 @@ struct DriverHandleImp : public DriverHandle {
     // Environment Variables
     bool enableProgramDebugging = false;
     bool enableSysman = false;
+    bool enablePciIdDeviceOrder = false;
 };
 
 extern struct DriverHandleImp *GlobalDriver;

level_zero/core/source/driver/driver_imp.h

@@ -31,6 +31,7 @@ struct L0EnvVariables {
     bool metrics;
     bool pin;
     bool sysman;
+    bool pciIdDeviceOrder;
 };
 
 } // namespace L0

level_zero/core/source/driver/linux/CMakeLists.txt

@@ -0,0 +1,14 @@
+#
+# Copyright (C) 2020 Intel Corporation
+#
+# SPDX-License-Identifier: MIT
+#
+
+set(L0_SRCS_DRIVER_LINUX
+    ${CMAKE_CURRENT_SOURCE_DIR}/CMakeLists.txt
+    ${CMAKE_CURRENT_SOURCE_DIR}/driver_handle_imp_linux.cpp
+)
+
+if(UNIX)
+  set_property(GLOBAL PROPERTY L0_SRCS_DRIVER_LINUX ${L0_SRCS_DRIVER_LINUX})
+endif()

level_zero/core/source/driver/linux/driver_handle_imp_linux.cpp

@@ -0,0 +1,35 @@
+/*
+ * Copyright (C) 2020 Intel Corporation
+ *
+ * SPDX-License-Identifier: MIT
+ *
+ */
+
+#include "shared/source/os_interface/linux/drm_neo.h"
+#include "shared/source/os_interface/linux/os_interface.h"
+
+#include "level_zero/core/source/driver/driver_handle_imp.h"
+
+namespace L0 {
+
+bool comparePciIdBusNumber(std::unique_ptr<NEO::Device> &neoDevice1, std::unique_ptr<NEO::Device> &neoDevice2) {
+    // BDF sample format is : 00:02.0
+    auto bdfDevice1 = neoDevice1.get()->getRootDeviceEnvironment().osInterface.get()->get()->getDrm()->getPciPath();
+    uint32_t bus1 = 0;
+    std::stringstream ss;
+    ss << std::hex << bdfDevice1;
+    ss >> bus1;
+    ss.str("");
+    ss.clear();
+    auto bdfDevice2 = neoDevice2.get()->getRootDeviceEnvironment().osInterface.get()->get()->getDrm()->getPciPath();
+    uint32_t bus2 = 0;
+    ss << std::hex << bdfDevice2;
+    ss >> bus2;
+    return bus1 < bus2;
+}
+
+void DriverHandleImp::sortNeoDevices(std::vector<std::unique_ptr<NEO::Device>> &neoDevices) {
+    std::sort(neoDevices.begin(), neoDevices.end(), comparePciIdBusNumber);
+}
+
+} // namespace L0

level_zero/core/source/driver/windows/CMakeLists.txt

@@ -0,0 +1,14 @@
+#
+# Copyright (C) 2020 Intel Corporation
+#
+# SPDX-License-Identifier: MIT
+#
+
+set(L0_SRCS_DRIVER_WINDOWS
+    ${CMAKE_CURRENT_SOURCE_DIR}/CMakeLists.txt
+    ${CMAKE_CURRENT_SOURCE_DIR}/driver_handle_imp_windows.cpp
+)
+
+if(WIN32)
+  set_property(GLOBAL PROPERTY L0_SRCS_DRIVER_WINDOWS ${L0_SRCS_DRIVER_WINDOWS})
+endif()

level_zero/core/source/driver/windows/driver_handle_imp_windows.cpp

@@ -0,0 +1,15 @@
+/*
+ * Copyright (C) 2020 Intel Corporation
+ *
+ * SPDX-License-Identifier: MIT
+ *
+ */
+
+#include "level_zero/core/source/driver/driver_handle_imp.h"
+
+namespace L0 {
+
+void DriverHandleImp::sortNeoDevices(std::vector<std::unique_ptr<NEO::Device>> &neoDevices) {
+}
+
+} // namespace L0