Evict USM cpu allocation after migration

LukaszJobczyk · Compute-Runtime-Automation · commit b81a78d0e99a · 2020-09-18T13:01:45.000+02:00
Related-To: NEO-5007

Change-Id: I3c91af3ca22cb6233d530b252cc0c75d8fc2f8b5
Signed-off-by: Lukasz Jobczyk &lt;lukasz.jobczyk@intel.com&gt;
diff --git a/level_zero/core/source/memory/cpu_page_fault_memory_manager.cpp b/level_zero/core/source/memory/cpu_page_fault_memory_manager.cpp
@@ -36,5 +36,7 @@ void PageFaultManager::transferToGpu(void *ptr, void *device) {
                                                              allocData->cpuAllocation,
                                                              allocData->size, false);
     UNRECOVERABLE_IF(ret);
+
+    this->evictMemoryAfterImplCopy(allocData->cpuAllocation, deviceImp->getNEODevice());
 }
 } // namespace NEO
diff --git a/opencl/source/memory_manager/cpu_page_fault_manager_memory_sync.cpp b/opencl/source/memory_manager/cpu_page_fault_manager_memory_sync.cpp
@@ -24,5 +24,8 @@ void PageFaultManager::transferToGpu(void *ptr, void *cmdQ) {
     UNRECOVERABLE_IF(retVal);
     retVal = commandQueue->finish();
     UNRECOVERABLE_IF(retVal);
+
+    auto allocData = memoryData[ptr].unifiedMemoryManager->getSVMAlloc(ptr);
+    this->evictMemoryAfterImplCopy(allocData->cpuAllocation, &commandQueue->getDevice());
 }
 } // namespace NEO
diff --git a/opencl/test/unit_test/aub_tests/fixtures/aub_fixture.h b/opencl/test/unit_test/aub_tests/fixtures/aub_fixture.h
@@ -19,6 +19,7 @@
 #include "opencl/source/platform/platform.h"
 #include "opencl/test/unit_test/command_queue/command_queue_fixture.h"
 #include "opencl/test/unit_test/mocks/mock_cl_device.h"
+#include "opencl/test/unit_test/mocks/mock_memory_operations_handler.h"
 #include "opencl/test/unit_test/mocks/mock_platform.h"
 
 #include "gtest/gtest.h"
@@ -42,6 +43,7 @@ class AUBFixture : public CommandQueueHwFixture {
         executionEnvironment = platform()->peekExecutionEnvironment();
         executionEnvironment->prepareRootDeviceEnvironments(1u);
         executionEnvironment->rootDeviceEnvironments[0]->setHwInfo(&hwInfo);
+        executionEnvironment->rootDeviceEnvironments[0]->memoryOperationsInterface = std::make_unique<MockMemoryOperationsHandler>();
 
         device = std::make_unique<MockClDevice>(MockDevice::create<MockDevice>(executionEnvironment, rootDeviceIndex));
 
diff --git a/opencl/test/unit_test/memory_manager/cpu_page_fault_manager_memory_sync_tests.cpp b/opencl/test/unit_test/memory_manager/cpu_page_fault_manager_memory_sync_tests.cpp
@@ -10,6 +10,7 @@
 #include "shared/test/unit_test/test_macros/test_checks_shared.h"
 
 #include "opencl/source/command_queue/command_queue.h"
+#include "opencl/test/unit_test/mocks/mock_cl_device.h"
 #include "opencl/test/unit_test/mocks/mock_command_queue.h"
 #include "opencl/test/unit_test/mocks/mock_graphics_allocation.h"
 #include "opencl/test/unit_test/mocks/mock_memory_manager.h"
@@ -52,7 +53,9 @@ TEST_F(PageFaultManagerTest, givenUnifiedMemoryAllocWhenSynchronizeMemoryThenEnq
     auto svmAllocsManager = std::make_unique<SVMAllocsManager>(memoryManager.get());
     void *alloc = svmAllocsManager->createSVMAlloc(mockRootDeviceIndex, 256, {}, mockDeviceBitfield);
 
+    auto device = std::unique_ptr<MockClDevice>(new MockClDevice{MockDevice::createWithNewExecutionEnvironment<MockDevice>(nullptr)});
     auto cmdQ = std::make_unique<CommandQueueMock>();
+    cmdQ->device = device.get();
     pageFaultManager->insertAllocation(alloc, 256, svmAllocsManager.get(), cmdQ.get(), {});
 
     pageFaultManager->baseCpuTransfer(alloc, 10, cmdQ.get());
@@ -66,6 +69,7 @@ TEST_F(PageFaultManagerTest, givenUnifiedMemoryAllocWhenSynchronizeMemoryThenEnq
     EXPECT_EQ(cmdQ->finishCalled, 1);
 
     svmAllocsManager->freeSVMAlloc(alloc);
+    cmdQ->device = nullptr;
 }
 
 TEST_F(PageFaultManagerTest, givenUnifiedMemoryAllocWhenGpuTransferIsInvokedThenInsertMapOperation) {
@@ -83,12 +87,15 @@ TEST_F(PageFaultManagerTest, givenUnifiedMemoryAllocWhenGpuTransferIsInvokedThen
     auto memoryManager = std::make_unique<MockMemoryManager>(executionEnvironment);
     auto svmAllocsManager = std::make_unique<MockSVMAllocsManager>(memoryManager.get());
     void *alloc = svmAllocsManager->createSVMAlloc(mockRootDeviceIndex, 256, {}, mockDeviceBitfield);
+    auto device = std::unique_ptr<MockClDevice>(new MockClDevice{MockDevice::createWithNewExecutionEnvironment<MockDevice>(nullptr)});
     auto cmdQ = std::make_unique<CommandQueueMock>();
+    cmdQ->device = device.get();
     pageFaultManager->insertAllocation(alloc, 256, svmAllocsManager.get(), cmdQ.get(), {});
 
     EXPECT_EQ(svmAllocsManager->insertSvmMapOperationCalled, 0);
     pageFaultManager->baseGpuTransfer(alloc, cmdQ.get());
     EXPECT_EQ(svmAllocsManager->insertSvmMapOperationCalled, 1);
 
     svmAllocsManager->freeSVMAlloc(alloc);
+    cmdQ->device = nullptr;
 }
diff --git a/opencl/test/unit_test/test_files/igdrcl.config b/opencl/test/unit_test/test_files/igdrcl.config
@@ -85,6 +85,7 @@ DirectSubmissionEnableDebugBuffer = 0
 DirectSubmissionDiagnosticExecutionCount = 30
 DirectSubmissionDisableCacheFlush = 0
 DirectSubmissionDisableMonitorFence = 0
+USMEvictAfterMigration = 1
 EnableNullHardware = 0
 ForceLinearImages = 0
 ForceSLML3Config = 0
diff --git a/shared/source/debug_settings/debug_variables_base.inl b/shared/source/debug_settings/debug_variables_base.inl
@@ -137,6 +137,7 @@ DECLARE_DEBUG_VARIABLE(int32_t, DirectSubmissionDiagnosticExecutionCount, 30, "N
 DECLARE_DEBUG_VARIABLE(int32_t, DirectSubmissionOverrideBlitterSupport, -1, "Overrides default blitter support: -1: do not override, 0: disable engine support, 1: enable engine support with init start, 2: enable engine support without init start")
 DECLARE_DEBUG_VARIABLE(int32_t, DirectSubmissionOverrideRenderSupport, -1, "Overrides default render support: -1: do not override, 0: disable engine support, 1: enable engine support with init start, 2: enable engine support without init start")
 DECLARE_DEBUG_VARIABLE(int32_t, DirectSubmissionOverrideComputeSupport, -1, "Overrides default compute support: -1: do not override, 0: disable engine support, 1: enable engine support with init start, 2: enable engine support without init start")
+DECLARE_DEBUG_VARIABLE(bool, USMEvictAfterMigration, true, "Evict USM allocation after implicit migration to GPU")
 DECLARE_DEBUG_VARIABLE(bool, DirectSubmissionDisableCacheFlush, false, "Disable dispatching cache flush commands")
 DECLARE_DEBUG_VARIABLE(bool, DirectSubmissionDisableMonitorFence, false, "Disable dispatching monitor fence commands")
 
diff --git a/shared/source/page_fault_manager/cpu_page_fault_manager.h b/shared/source/page_fault_manager/cpu_page_fault_manager.h
@@ -16,6 +16,8 @@
 #include <unordered_map>
 
 namespace NEO {
+class GraphicsAllocation;
+class Device;
 class SVMAllocsManager;
 
 class PageFaultManager : public NonCopyableOrMovableClass {
@@ -46,6 +48,7 @@ class PageFaultManager : public NonCopyableOrMovableClass {
     virtual void allowCPUMemoryAccess(void *ptr, size_t size) = 0;
     virtual void protectCPUMemoryAccess(void *ptr, size_t size) = 0;
 
+    virtual void evictMemoryAfterImplCopy(GraphicsAllocation *allocation, Device *device) = 0;
     virtual void broadcastWaitSignal() = 0;
     MOCKABLE_VIRTUAL void waitForCopy();
 
diff --git a/shared/source/page_fault_manager/linux/cpu_page_fault_manager_linux.cpp b/shared/source/page_fault_manager/linux/cpu_page_fault_manager_linux.cpp
@@ -7,7 +7,10 @@
 
 #include "shared/source/page_fault_manager/linux/cpu_page_fault_manager_linux.h"
 
+#include "shared/source/debug_settings/debug_settings_manager.h"
+#include "shared/source/device/device.h"
 #include "shared/source/helpers/debug_helpers.h"
+#include "shared/source/memory_manager/memory_operations_handler.h"
 
 #include <dirent.h>
 #include <sys/mman.h>
@@ -40,6 +43,9 @@ PageFaultManagerLinux::PageFaultManagerLinux() {
 
     retVal = sigaction(SIGUSR1, &pageFaultManagerHandler, &previousUserSignalHandler);
     UNRECOVERABLE_IF(retVal != 0);
+
+    this->evictMemoryAfterCopy = DebugManager.flags.EnableDirectSubmission.get() &&
+                                 DebugManager.flags.USMEvictAfterMigration.get();
 }
 
 PageFaultManagerLinux::~PageFaultManagerLinux() {
@@ -113,4 +119,10 @@ void PageFaultManagerLinux::sendSignalToThread(int threadId) {
     syscall(SYS_tkill, threadId, SIGUSR1);
 }
 
+void PageFaultManagerLinux::evictMemoryAfterImplCopy(GraphicsAllocation *allocation, Device *device) {
+    if (evictMemoryAfterCopy) {
+        device->getRootDeviceEnvironment().memoryOperationsInterface->evict(device, *allocation);
+    }
+};
+
 } // namespace NEO
diff --git a/shared/source/page_fault_manager/linux/cpu_page_fault_manager_linux.h b/shared/source/page_fault_manager/linux/cpu_page_fault_manager_linux.h
@@ -24,6 +24,7 @@ class PageFaultManagerLinux : public PageFaultManager {
     void allowCPUMemoryAccess(void *ptr, size_t size) override;
     void protectCPUMemoryAccess(void *ptr, size_t size) override;
 
+    void evictMemoryAfterImplCopy(GraphicsAllocation *allocation, Device *device) override;
     void broadcastWaitSignal() override;
     MOCKABLE_VIRTUAL void sendSignalToThread(int threadId);
 
@@ -34,5 +35,7 @@ class PageFaultManagerLinux : public PageFaultManager {
 
     struct sigaction previousPageFaultHandler = {};
     struct sigaction previousUserSignalHandler = {};
+
+    bool evictMemoryAfterCopy = false;
 };
 } // namespace NEO
diff --git a/shared/source/page_fault_manager/windows/cpu_page_fault_manager_windows.cpp b/shared/source/page_fault_manager/windows/cpu_page_fault_manager_windows.cpp
@@ -51,6 +51,8 @@ void PageFaultManagerWindows::protectCPUMemoryAccess(void *ptr, size_t size) {
     UNRECOVERABLE_IF(!retVal);
 }
 
+void PageFaultManagerWindows::evictMemoryAfterImplCopy(GraphicsAllocation *allocation, Device *device) {}
+
 void PageFaultManagerWindows::broadcastWaitSignal() {}
 
 } // namespace NEO
diff --git a/shared/source/page_fault_manager/windows/cpu_page_fault_manager_windows.h b/shared/source/page_fault_manager/windows/cpu_page_fault_manager_windows.h
@@ -25,6 +25,7 @@ class PageFaultManagerWindows : public PageFaultManager {
     void allowCPUMemoryAccess(void *ptr, size_t size) override;
     void protectCPUMemoryAccess(void *ptr, size_t size) override;
 
+    void evictMemoryAfterImplCopy(GraphicsAllocation *allocation, Device *device) override;
     void broadcastWaitSignal() override;
 
     static std::function<LONG(struct _EXCEPTION_POINTERS *exceptionInfo)> pageFaultHandler;
diff --git a/shared/test/unit_test/page_fault_manager/linux/cpu_page_fault_manager_linux_tests.cpp b/shared/test/unit_test/page_fault_manager/linux/cpu_page_fault_manager_linux_tests.cpp
@@ -6,9 +6,14 @@
  */
 
 #include "shared/source/page_fault_manager/linux/cpu_page_fault_manager_linux.h"
+#include "shared/test/unit_test/helpers/debug_manager_state_restore.h"
+#include "shared/test/unit_test/mocks/mock_device.h"
 #include "shared/test/unit_test/page_fault_manager/cpu_page_fault_manager_tests_fixture.h"
 #include "shared/test/unit_test/page_fault_manager/mock_cpu_page_fault_manager.h"
 
+#include "opencl/test/unit_test/mocks/mock_graphics_allocation.h"
+#include "opencl/test/unit_test/mocks/mock_memory_operations_handler.h"
+
 #include "gtest/gtest.h"
 
 #include <csignal>
@@ -73,6 +78,78 @@ TEST_F(PageFaultManagerLinuxTest, whenPageFaultIsRaisedThenHandlerIsInvoked) {
     EXPECT_TRUE(pageFaultManager->handlerInvoked);
 }
 
+struct MockOperationsInterface : public MockMemoryOperationsHandler {
+    bool evictCalled = false;
+    MemoryOperationsStatus evict(Device *device, GraphicsAllocation &gfxAllocation) override {
+        this->evictCalled = true;
+        return MemoryOperationsStatus::UNSUPPORTED;
+    }
+};
+
+TEST_F(PageFaultManagerLinuxTest, givenDirectSubmissionAndUSMEvictWaEnabledWhenEvitMemoryAfterCopyThenMemoryOperationsHandlerEvictMethodIsCalled) {
+    DebugManagerStateRestore restorer;
+    DebugManager.flags.EnableDirectSubmission.set(true);
+    DebugManager.flags.USMEvictAfterMigration.set(true);
+
+    auto pageFaultManager = std::make_unique<MockPageFaultManagerLinux>();
+    std::unique_ptr<Device> device(MockDevice::createWithNewExecutionEnvironment<MockDevice>(nullptr));
+    device->getExecutionEnvironment()->rootDeviceEnvironments[0u]->memoryOperationsInterface = std::make_unique<MockOperationsInterface>();
+    auto operationInterface = static_cast<MockOperationsInterface *>(device->getExecutionEnvironment()->rootDeviceEnvironments[0u]->memoryOperationsInterface.get());
+    MockGraphicsAllocation allocation;
+
+    EXPECT_FALSE(operationInterface->evictCalled);
+    pageFaultManager->evictMemoryAfterImplCopy(&allocation, device.get());
+    EXPECT_TRUE(operationInterface->evictCalled);
+}
+
+TEST_F(PageFaultManagerLinuxTest, givenDirectSubmissionEnabledAndUSMEvictWaDisabledWhenEvitMemoryAfterCopyThenMemoryOperationsHandlerEvictMethodIsNotCalled) {
+    DebugManagerStateRestore restorer;
+    DebugManager.flags.EnableDirectSubmission.set(true);
+    DebugManager.flags.USMEvictAfterMigration.set(false);
+
+    auto pageFaultManager = std::make_unique<MockPageFaultManagerLinux>();
+    std::unique_ptr<Device> device(MockDevice::createWithNewExecutionEnvironment<MockDevice>(nullptr));
+    device->getExecutionEnvironment()->rootDeviceEnvironments[0u]->memoryOperationsInterface = std::make_unique<MockOperationsInterface>();
+    auto operationInterface = static_cast<MockOperationsInterface *>(device->getExecutionEnvironment()->rootDeviceEnvironments[0u]->memoryOperationsInterface.get());
+    MockGraphicsAllocation allocation;
+
+    EXPECT_FALSE(operationInterface->evictCalled);
+    pageFaultManager->evictMemoryAfterImplCopy(&allocation, device.get());
+    EXPECT_FALSE(operationInterface->evictCalled);
+}
+
+TEST_F(PageFaultManagerLinuxTest, givenDirectSubmissionAndUSMEvictWaDisabledWhenEvitMemoryAfterCopyThenMemoryOperationsHandlerEvictMethodIsNotCalled) {
+    DebugManagerStateRestore restorer;
+    DebugManager.flags.EnableDirectSubmission.set(false);
+    DebugManager.flags.USMEvictAfterMigration.set(false);
+
+    auto pageFaultManager = std::make_unique<MockPageFaultManagerLinux>();
+    std::unique_ptr<Device> device(MockDevice::createWithNewExecutionEnvironment<MockDevice>(nullptr));
+    device->getExecutionEnvironment()->rootDeviceEnvironments[0u]->memoryOperationsInterface = std::make_unique<MockOperationsInterface>();
+    auto operationInterface = static_cast<MockOperationsInterface *>(device->getExecutionEnvironment()->rootDeviceEnvironments[0u]->memoryOperationsInterface.get());
+    MockGraphicsAllocation allocation;
+
+    EXPECT_FALSE(operationInterface->evictCalled);
+    pageFaultManager->evictMemoryAfterImplCopy(&allocation, device.get());
+    EXPECT_FALSE(operationInterface->evictCalled);
+}
+
+TEST_F(PageFaultManagerLinuxTest, givenDirectSubmissionDisabledAndUSMEvictWaEnabledWhenEvitMemoryAfterCopyThenMemoryOperationsHandlerEvictMethodIsNotCalled) {
+    DebugManagerStateRestore restorer;
+    DebugManager.flags.EnableDirectSubmission.set(false);
+    DebugManager.flags.USMEvictAfterMigration.set(true);
+
+    auto pageFaultManager = std::make_unique<MockPageFaultManagerLinux>();
+    std::unique_ptr<Device> device(MockDevice::createWithNewExecutionEnvironment<MockDevice>(nullptr));
+    device->getExecutionEnvironment()->rootDeviceEnvironments[0u]->memoryOperationsInterface = std::make_unique<MockOperationsInterface>();
+    auto operationInterface = static_cast<MockOperationsInterface *>(device->getExecutionEnvironment()->rootDeviceEnvironments[0u]->memoryOperationsInterface.get());
+    MockGraphicsAllocation allocation;
+
+    EXPECT_FALSE(operationInterface->evictCalled);
+    pageFaultManager->evictMemoryAfterImplCopy(&allocation, device.get());
+    EXPECT_FALSE(operationInterface->evictCalled);
+}
+
 TEST_F(PageFaultManagerLinuxTest, givenProtectedMemoryWhenTryingToAccessThenPageFaultIsRaisedAndMemoryIsAccessibleAfterHandling) {
     auto pageFaultManager = std::make_unique<MockPageFaultManagerLinux>();
     pageFaultManager->allowCPUMemoryAccessOnPageFault = true;
diff --git a/shared/test/unit_test/page_fault_manager/mock_cpu_page_fault_manager.h b/shared/test/unit_test/page_fault_manager/mock_cpu_page_fault_manager.h
@@ -50,6 +50,7 @@ class MockPageFaultManager : public PageFaultManager {
         PageFaultManager::transferToGpu(ptr, cmdQ);
     }
     void broadcastWaitSignal() override {}
+    void evictMemoryAfterImplCopy(GraphicsAllocation *allocation, Device *device) override {}
 
     int allowMemoryAccessCalled = 0;
     int protectMemoryCalled = 0;
@@ -69,6 +70,7 @@ template <class T>
 class MockPageFaultManagerHandlerInvoke : public T {
   public:
     using T::allowCPUMemoryAccess;
+    using T::evictMemoryAfterImplCopy;
     using T::protectCPUMemoryAccess;
     using T::T;
 

Original file line number	Diff line number	Diff line change
`@@ -36,5 +36,7 @@ void PageFaultManager::transferToGpu(void ptr, void device) {`
`36`	`36`	`allocData->cpuAllocation,`
`37`	`37`	`allocData->size, false);`
`38`	`38`	`UNRECOVERABLE_IF(ret);`
	`39`	`+`
	`40`	`+ this->evictMemoryAfterImplCopy(allocData->cpuAllocation, deviceImp->getNEODevice());`
`39`	`41`	`}`
`40`	`42`	`} // namespace NEO`
Original file line number	Diff line number	Diff line change
`@@ -24,5 +24,8 @@ void PageFaultManager::transferToGpu(void ptr, void cmdQ) {`
`24`	`24`	`UNRECOVERABLE_IF(retVal);`
`25`	`25`	`retVal = commandQueue->finish();`
`26`	`26`	`UNRECOVERABLE_IF(retVal);`
	`27`	`+`
	`28`	`+ auto allocData = memoryData[ptr].unifiedMemoryManager->getSVMAlloc(ptr);`
	`29`	`+ this->evictMemoryAfterImplCopy(allocData->cpuAllocation, &commandQueue->getDevice());`
`27`	`30`	`}`
`28`	`31`	`} // namespace NEO`
Original file line number	Diff line number	Diff line change
`@@ -51,6 +51,8 @@ void PageFaultManagerWindows::protectCPUMemoryAccess(void *ptr, size_t size) {`
`51`	`51`	`UNRECOVERABLE_IF(!retVal);`
`52`	`52`	`}`
`53`	`53`
	`54`	`+void PageFaultManagerWindows::evictMemoryAfterImplCopy(GraphicsAllocation allocation, Device device) {}`
	`55`	`+`
`54`	`56`	`void PageFaultManagerWindows::broadcastWaitSignal() {}`
`55`	`57`
`56`	`58`	`} // namespace NEO`