Introduce new interface for reinitializing level zero device

jitendrasharma1989 · Compute-Runtime-Automation · commit cfad41f28aba · 2021-10-06T07:16:33.000+02:00
This change introduce
- new method deviceReinit, which could be used, to reinitialize
existing level zero device.

Related-To: LOCI-2612

Signed-off-by: Jitendra Sharma &lt;jitendra.sharma@intel.com&gt;
diff --git a/level_zero/core/source/device/device.h b/level_zero/core/source/device/device.h
@@ -109,6 +109,8 @@ struct Device : _ze_device_handle_t {
     inline ze_device_handle_t toHandle() { return this; }
 
     static Device *create(DriverHandle *driverHandle, NEO::Device *neoDevice, uint32_t currentDeviceMask, bool isSubDevice, ze_result_t *returnValue);
+    static Device *create(DriverHandle *driverHandle, NEO::Device *neoDevice, uint32_t currentDeviceMask, bool isSubDevice, ze_result_t *returnValue, L0::Device *deviceL0);
+    static Device *deviceReinit(DriverHandle *driverHandle, L0::Device *device, std::unique_ptr<NEO::Device> &neoDevice, ze_result_t *returnValue);
 
     virtual NEO::PreemptionMode getDevicePreemptionMode() const = 0;
     virtual const NEO::DeviceInfo &getDeviceInfo() const = 0;
diff --git a/level_zero/core/source/device/device_imp.cpp b/level_zero/core/source/device/device_imp.cpp
@@ -692,8 +692,28 @@ uint32_t DeviceImp::getMaxNumHwThreads() const { return maxNumHwThreads; }
 
 const NEO::HardwareInfo &DeviceImp::getHwInfo() const { return neoDevice->getHardwareInfo(); }
 
+// Use this method to reinitialize L0::Device *device, that was created during zeInit, with the help of Device::create
+Device *Device::deviceReinit(DriverHandle *driverHandle, L0::Device *device, std::unique_ptr<NEO::Device> &neoDevice, ze_result_t *returnValue) {
+    const auto rootDeviceIndex = neoDevice->getRootDeviceIndex();
+    auto rootDeviceEnvironment = neoDevice->getExecutionEnvironment()->rootDeviceEnvironments[rootDeviceIndex].get();
+    auto pNeoDevice = neoDevice.release();
+
+    auto subDevicesMask = static_cast<uint32_t>(rootDeviceEnvironment->deviceAffinityMask.getGenericSubDevicesMask().to_ulong());
+    return Device::create(driverHandle, pNeoDevice, subDevicesMask, false, returnValue, device);
+}
+
 Device *Device::create(DriverHandle *driverHandle, NEO::Device *neoDevice, uint32_t currentDeviceMask, bool isSubDevice, ze_result_t *returnValue) {
-    auto device = new DeviceImp;
+    return Device::create(driverHandle, neoDevice, currentDeviceMask, isSubDevice, returnValue, nullptr);
+}
+
+Device *Device::create(DriverHandle *driverHandle, NEO::Device *neoDevice, uint32_t currentDeviceMask, bool isSubDevice, ze_result_t *returnValue, L0::Device *deviceL0) {
+    L0::DeviceImp *device = nullptr;
+    if (deviceL0 == nullptr) {
+        device = new DeviceImp;
+    } else {
+        device = static_cast<L0::DeviceImp *>(deviceL0);
+    }
+
     UNRECOVERABLE_IF(device == nullptr);
 
     device->setDriverHandle(driverHandle);
@@ -731,7 +751,7 @@ Device *Device::create(DriverHandle *driverHandle, NEO::Device *neoDevice, uint3
         ze_device_handle_t subDevice = Device::create(driverHandle,
                                                       device->neoDevice->getSubDevice(i),
                                                       0,
-                                                      true, returnValue);
+                                                      true, returnValue, nullptr);
         if (subDevice == nullptr) {
             return nullptr;
         }
@@ -783,10 +803,8 @@ Device *Device::create(DriverHandle *driverHandle, NEO::Device *neoDevice, uint3
             ->notifyNewDevice(osInterface ? osInterface->getDriverModel()->getDeviceHandle() : 0);
     }
 
-    if (static_cast<DriverHandleImp *>(driverHandle)->enableSysman && !isSubDevice) {
-        device->setSysmanHandle(L0::SysmanDeviceHandleContext::init(device->toHandle()));
-    }
-
+    device->createSysmanHandle(isSubDevice);
+    device->resourcesReleased = false;
     return device;
 }
 
@@ -801,6 +819,7 @@ void DeviceImp::releaseResources() {
     for (uint32_t i = 0; i < this->numSubDevices; i++) {
         delete this->subDevices[i];
     }
+    this->subDevices.clear();
     this->numSubDevices = 0;
 
     if (this->pageFaultCommandList) {
diff --git a/level_zero/core/source/device/device_imp.h b/level_zero/core/source/device/device_imp.h
@@ -137,6 +137,7 @@ struct DeviceImp : public Device {
     NEO::SpinLock peerAllocationsMutex;
     std::map<NEO::SvmAllocationData *, MemAdviseFlags> memAdviseSharedAllocations;
     NEO::AllocationsList allocationsForReuse;
+    void createSysmanHandle(bool isSubDevice);
 
   protected:
     NEO::GraphicsAllocation *debugSurface = nullptr;
diff --git a/level_zero/core/source/driver/driver_handle_imp.cpp b/level_zero/core/source/driver/driver_handle_imp.cpp
@@ -139,6 +139,26 @@ DriverHandleImp::~DriverHandleImp() {
     }
 }
 
+void DriverHandleImp::updateRootDeviceBitFields(std::unique_ptr<NEO::Device> &neoDevice) {
+    const auto rootDeviceIndex = neoDevice->getRootDeviceIndex();
+    auto entry = this->deviceBitfields.find(rootDeviceIndex);
+    entry->second = neoDevice->getDeviceBitfield();
+}
+
+void DriverHandleImp::enableRootDeviceDebugger(std::unique_ptr<NEO::Device> &neoDevice) {
+    const auto rootDeviceIndex = neoDevice->getRootDeviceIndex();
+    auto rootDeviceEnvironment = neoDevice->getExecutionEnvironment()->rootDeviceEnvironments[rootDeviceIndex].get();
+
+    if (enableProgramDebugging) {
+        if (neoDevice->getDebugger() != nullptr) {
+            NEO::printDebugString(NEO::DebugManager.flags.PrintDebugMessages.get(), stderr,
+                                  "%s", "Source Level Debugger cannot be used with Environment Variable enabling program debugging.\n");
+            UNRECOVERABLE_IF(neoDevice->getDebugger() != nullptr && enableProgramDebugging);
+        }
+        rootDeviceEnvironment->debugger = DebuggerL0::create(neoDevice.get());
+    }
+}
+
 ze_result_t DriverHandleImp::initialize(std::vector<std::unique_ptr<NEO::Device>> neoDevices) {
     bool multiOsContextDriver = false;
     for (auto &neoDevice : neoDevices) {
@@ -157,14 +177,7 @@ ze_result_t DriverHandleImp::initialize(std::vector<std::unique_ptr<NEO::Device>
         const auto rootDeviceIndex = neoDevice->getRootDeviceIndex();
         auto rootDeviceEnvironment = neoDevice->getExecutionEnvironment()->rootDeviceEnvironments[rootDeviceIndex].get();
 
-        if (enableProgramDebugging) {
-            if (neoDevice->getDebugger() != nullptr) {
-                NEO::printDebugString(NEO::DebugManager.flags.PrintDebugMessages.get(), stderr,
-                                      "%s", "Source Level Debugger cannot be used with Environment Variable enabling program debugging.\n");
-                UNRECOVERABLE_IF(neoDevice->getDebugger() != nullptr && enableProgramDebugging);
-            }
-            rootDeviceEnvironment->debugger = DebuggerL0::create(neoDevice.get());
-        }
+        enableRootDeviceDebugger(neoDevice);
 
         this->rootDeviceIndices.insert(rootDeviceIndex);
         this->deviceBitfields.insert({rootDeviceIndex, neoDevice->getDeviceBitfield()});
diff --git a/level_zero/core/source/driver/driver_handle_imp.h b/level_zero/core/source/driver/driver_handle_imp.h
@@ -98,6 +98,8 @@ struct DriverHandleImp : public DriverHandle {
 
     std::set<uint32_t> rootDeviceIndices = {};
     std::map<uint32_t, NEO::DeviceBitfield> deviceBitfields;
+    void updateRootDeviceBitFields(std::unique_ptr<NEO::Device> &neoDevice);
+    void enableRootDeviceDebugger(std::unique_ptr<NEO::Device> &neoDevice);
 
     // Environment Variables
     bool enableProgramDebugging = false;
diff --git a/level_zero/core/test/unit_tests/sources/device/test_device.cpp b/level_zero/core/test/unit_tests/sources/device/test_device.cpp
@@ -41,6 +41,20 @@ extern HwHelper *hwHelperFactory[IGFX_MAX_CORE];
 namespace L0 {
 namespace ult {
 
+TEST(L0DeviceTest, GivenCreatedDeviceHandleWhenCallingdeviceReinitThenNewDeviceHandleIsNotCreated) {
+    ze_result_t returnValue = ZE_RESULT_SUCCESS;
+    std::unique_ptr<DriverHandleImp> driverHandle(new DriverHandleImp);
+    auto hwInfo = *NEO::defaultHwInfo;
+    auto neoDevice = std::unique_ptr<NEO::Device>(NEO::MockDevice::createWithNewExecutionEnvironment<NEO::MockDevice>(&hwInfo, 0));
+    auto device = Device::create(driverHandle.get(), neoDevice.release(), 1, false, &returnValue);
+    ASSERT_NE(nullptr, device);
+    static_cast<DeviceImp *>(device)->releaseResources();
+
+    auto newNeoDevice = std::unique_ptr<NEO::Device>(NEO::MockDevice::createWithNewExecutionEnvironment<NEO::MockDevice>(&hwInfo, 0));
+    EXPECT_EQ(device, Device::deviceReinit(device->getDriverHandle(), device, newNeoDevice, &returnValue));
+    delete device;
+}
+
 TEST(L0DeviceTest, GivenDualStorageSharedMemorySupportedWhenCreatingDeviceThenPageFaultCmdListImmediateWithInitializedCmdQIsCreated) {
     ze_result_t returnValue = ZE_RESULT_SUCCESS;
     DebugManagerStateRestore restorer;
diff --git a/level_zero/core/test/unit_tests/sources/driver/test_driver.cpp b/level_zero/core/test/unit_tests/sources/driver/test_driver.cpp
@@ -53,6 +53,16 @@ TEST(zeInit, whenCallingZeInitWithoutGpuOnlyFlagThenInitializeOnDriverIsNotCalle
     EXPECT_EQ(0u, driver.initCalledCount);
 }
 
+using DriverHandleImpTest = Test<DeviceFixture>;
+TEST_F(DriverHandleImpTest, givenDriverImpWhenCallingupdateRootDeviceBitFieldsThendeviceBitfieldsAreUpdatedInAccordanceWithNeoDevice) {
+    auto hwInfo = *NEO::defaultHwInfo;
+    auto newNeoDevice = std::unique_ptr<NEO::Device>(NEO::MockDevice::createWithNewExecutionEnvironment<NEO::MockDevice>(&hwInfo, 0));
+    driverHandle->updateRootDeviceBitFields(newNeoDevice);
+    const auto rootDeviceIndex = neoDevice->getRootDeviceIndex();
+    auto entry = driverHandle->deviceBitfields.find(rootDeviceIndex);
+    EXPECT_EQ(newNeoDevice->getDeviceBitfield(), entry->second);
+}
+
 using DriverVersionTest = Test<DeviceFixture>;
 
 TEST_F(DriverVersionTest, givenCallToGetExtensionPropertiesThenSupportedExtensionsAreReturned) {
diff --git a/level_zero/tools/source/sysman/sysman.cpp b/level_zero/tools/source/sysman/sysman.cpp
@@ -16,6 +16,15 @@
 
 namespace L0 {
 
+void DeviceImp::createSysmanHandle(bool isSubDevice) {
+    if (static_cast<DriverHandleImp *>(driverHandle)->enableSysman && !isSubDevice) {
+        if (this->getSysmanHandle() == nullptr) {
+            // Sysman handles are created only during zeInit time device creation. And destroyed during L0::device destroy.
+            this->setSysmanHandle(L0::SysmanDeviceHandleContext::init(this->toHandle()));
+        }
+    }
+}
+
 SysmanDevice *SysmanDeviceHandleContext::init(ze_device_handle_t coreDevice) {
     SysmanDeviceImp *sysmanDevice = new SysmanDeviceImp(coreDevice);
     DEBUG_BREAK_IF(!sysmanDevice);
diff --git a/level_zero/tools/test/unit_tests/sources/sysman/linux/test_sysman.cpp b/level_zero/tools/test/unit_tests/sources/sysman/linux/test_sysman.cpp
@@ -338,6 +338,25 @@ TEST_F(SysmanMultiDeviceFixture, GivenValidEffectiveUserIdCheckWhetherPermission
     }
 }
 
+TEST_F(SysmanMultiDeviceFixture, GivenSysmanEnvironmentVariableSetWhenCreateL0DeviceThenSysmanHandleCreateIsAttempted) {
+    driverHandle->enableSysman = true;
+    // In SetUp of SysmanMultiDeviceFixture, sysman handle for device is already created, so new sysman handle should not be created
+    static_cast<DeviceImp *>(device)->createSysmanHandle(true);
+    EXPECT_EQ(device->getSysmanHandle(), pSysmanDevice);
+
+    static_cast<DeviceImp *>(device)->createSysmanHandle(false);
+    EXPECT_EQ(device->getSysmanHandle(), pSysmanDevice);
+
+    // delete previously allocated sysman handle and then attempt to create sysman handle again
+    delete pSysmanDevice;
+    device->setSysmanHandle(nullptr);
+    static_cast<DeviceImp *>(device)->createSysmanHandle(true);
+    EXPECT_EQ(device->getSysmanHandle(), nullptr);
+
+    static_cast<DeviceImp *>(device)->createSysmanHandle(false);
+    EXPECT_EQ(device->getSysmanHandle(), nullptr);
+}
+
 class UnknownDriverModel : public DriverModel {
   public:
     UnknownDriverModel() : DriverModel(DriverModelType::UNKNOWN) {}
