Add debug flag for EOT WA

JablonskiMateusz · Compute-Runtime-Automation · commit cf3817e05884 · 2022-07-07T16:58:24.000+02:00
EOT WA requires allocating last 64KB of kernel heap and putting EOT
signature at the last 16 bytes of kernel heap

Related-To: NEO-7099
Signed-off-by: Mateusz Jablonski &lt;mateusz.jablonski@intel.com&gt;
diff --git a/opencl/test/unit_test/command_stream/command_stream_receiver_flush_task_2_tests.cpp b/opencl/test/unit_test/command_stream/command_stream_receiver_flush_task_2_tests.cpp
@@ -500,6 +500,25 @@ HWTEST_F(CommandStreamReceiverFlushTaskTests, WhenFlushingThenScratchAllocationI
     EXPECT_TRUE(commandStreamReceiver->isMadeNonResident(scratchAllocation));
 }
 
+HWTEST_F(CommandStreamReceiverFlushTaskTests, givenAdditionalAllocationForResidencyWhenFlushingThenHandleResidency) {
+    auto commandStreamReceiver = new MockCsrHw<FamilyType>(*pDevice->executionEnvironment, pDevice->getRootDeviceIndex(), pDevice->getDeviceBitfield());
+    pDevice->resetCommandStreamReceiver(commandStreamReceiver);
+
+    MockGraphicsAllocation allocation{};
+    commandStreamReceiver->addAdditionalAllocationForResidency(&allocation);
+
+    flushTask(*commandStreamReceiver);
+
+    auto tagAllocation = commandStreamReceiver->getTagAllocation();
+    ASSERT_NE(tagAllocation, nullptr);
+
+    EXPECT_TRUE(commandStreamReceiver->isMadeResident(tagAllocation));
+    EXPECT_TRUE(commandStreamReceiver->isMadeResident(&allocation));
+
+    EXPECT_TRUE(commandStreamReceiver->isMadeNonResident(tagAllocation));
+    EXPECT_TRUE(commandStreamReceiver->isMadeNonResident(&allocation));
+}
+
 HWTEST_F(CommandStreamReceiverFlushTaskTests, givenCommandStreamReceiverWhenFenceAllocationIsRequiredAndFlushTaskIsCalledThenFenceAllocationIsMadeResident) {
     RAIIHwHelperFactory<MockHwHelperWithFenceAllocation<FamilyType>> hwHelperBackup{pDevice->getHardwareInfo().platform.eRenderCoreFamily};
 
diff --git a/shared/source/command_stream/command_stream_receiver.h b/shared/source/command_stream/command_stream_receiver.h
@@ -337,6 +337,9 @@ class CommandStreamReceiver {
     DispatchMode getDispatchMode() const {
         return this->dispatchMode;
     }
+    void addAdditionalAllocationForResidency(GraphicsAllocation *graphicsAllocation) {
+        this->additionalAllocationsForResidency.push_back(graphicsAllocation);
+    }
 
     LogicalStateHelper *getLogicalStateHelper() const;
 
@@ -389,6 +392,7 @@ class CommandStreamReceiver {
     GraphicsAllocation *perDssBackedBuffer = nullptr;
     GraphicsAllocation *clearColorAllocation = nullptr;
     GraphicsAllocation *workPartitionAllocation = nullptr;
+    StackVec<GraphicsAllocation *, 1> additionalAllocationsForResidency;
 
     MultiGraphicsAllocation *tagsMultiAllocation = nullptr;
 
diff --git a/shared/source/command_stream/command_stream_receiver_hw_base.inl b/shared/source/command_stream/command_stream_receiver_hw_base.inl
@@ -515,6 +515,10 @@ CompletionStamp CommandStreamReceiverHw<GfxFamily>::flushTask(
 
     this->makeResident(*tagAllocation);
 
+    for (auto &additionalAllocationForResidency : additionalAllocationsForResidency) {
+        this->makeResident(*additionalAllocationForResidency);
+    }
+
     if (globalFenceAllocation) {
         makeResident(*globalFenceAllocation);
     }
@@ -772,6 +776,9 @@ inline bool CommandStreamReceiverHw<GfxFamily>::flushBatchedSubmissions() {
             for (auto &surface : resourcePackage) {
                 surfacesForSubmit.push_back(surface);
             }
+            for (auto &additionalAllocationForResidency : additionalAllocationsForResidency) {
+                surfacesForSubmit.push_back(additionalAllocationForResidency);
+            }
 
             // make sure we flush DC if needed
             if (epiloguePipeControlLocation && MemorySynchronizationCommands<GfxFamily>::getDcFlushEnable(true, hwInfo)) {
diff --git a/shared/source/debug_settings/debug_variables_base.inl b/shared/source/debug_settings/debug_variables_base.inl
@@ -74,6 +74,7 @@ DECLARE_DEBUG_VARIABLE(bool, PrintMemoryRegionSizes, false, "print memory bank t
 DECLARE_DEBUG_VARIABLE(bool, UpdateCrossThreadDataSize, false, "Turn on cross thread data size calculation for PATCH TOKEN binary")
 DECLARE_DEBUG_VARIABLE(bool, UseNewQueryTopoIoctl, true, "Use DRM_I915_QUERY_COMPUTE_SLICES")
 DECLARE_DEBUG_VARIABLE(bool, DisableGpuHangDetection, false, "Disable GPU hang detection")
+DECLARE_DEBUG_VARIABLE(bool, EnableEotWa, false, "Enable WA to program EOT instruction at the end of kernel heap")
 DECLARE_DEBUG_VARIABLE(std::string, ForceDeviceId, std::string("unk"), "DeviceId selected for testing")
 DECLARE_DEBUG_VARIABLE(std::string, FilterDeviceId, std::string("unk"), "Device id filter, adapter matching device id will be opened. Ignored when unk.")
 DECLARE_DEBUG_VARIABLE(std::string, FilterBdfPath, std::string("unk"), "Linux-only, BDF path filter, only matching paths will be opened. Ignored when unk.")
diff --git a/shared/source/device/device.cpp b/shared/source/device/device.cpp
@@ -59,6 +59,9 @@ Device::~Device() {
 
     syncBufferHandler.reset();
     commandStreamReceivers.clear();
+    if (kernelEotWaAllocation) {
+        executionEnvironment->memoryManager->freeGraphicsMemory(kernelEotWaAllocation);
+    }
     executionEnvironment->memoryManager->waitForDeletions();
 
     executionEnvironment->decRefInternal();
@@ -215,10 +218,22 @@ bool Device::createDeviceImpl() {
     if (getDebugger() && hwHelper.disableL3CacheForDebug(hwInfo)) {
         getGmmHelper()->forceAllResourcesUncached();
     }
+    if (DebugManager.flags.EnableEotWa.get()) {
+        AllocationProperties allocationProperties{rootDeviceIndex, MemoryConstants::pageSize64k, AllocationType::KERNEL_ISA, deviceBitfield};
+        auto memoryManager = executionEnvironment->memoryManager.get();
+        auto heapBase = memoryManager->getInternalHeapBaseAddress(rootDeviceIndex, memoryManager->isLocalMemoryUsedForIsa(rootDeviceIndex));
+        allocationProperties.gpuAddress = heapBase + 4 * MemoryConstants::gigaByte - MemoryConstants::pageSize64k;
+        kernelEotWaAllocation = executionEnvironment->memoryManager->allocateGraphicsMemoryWithProperties(allocationProperties);
+    }
 
     if (!createEngines()) {
         return false;
     }
+    if (kernelEotWaAllocation) {
+        auto memoryManager = executionEnvironment->memoryManager.get();
+        uint8_t eotMemoryPattern[]{0x31, 0x09, 0x0C, 0x80, 0x04, 0x00, 0x00, 0x00, 0x0C, 0x7F, 0x20, 0x30, 0x00, 0x00, 0x00, 0x00};
+        memoryManager->copyMemoryToAllocation(kernelEotWaAllocation, MemoryConstants::pageSize64k - sizeof(eotMemoryPattern) - MemoryConstants::pageSize, eotMemoryPattern, sizeof(eotMemoryPattern));
+    }
 
     getDefaultEngine().osContext->setDefaultContext(true);
 
@@ -373,6 +388,11 @@ bool Device::createEngine(uint32_t deviceCsrIndex, EngineTypeUsage engineTypeUsa
         addEngineToEngineGroup(engine);
     }
 
+    if (kernelEotWaAllocation) {
+        if (!EngineHelpers::isBcs(engineType)) {
+            commandStreamReceiver->addAdditionalAllocationForResidency(kernelEotWaAllocation);
+        }
+    }
     commandStreamReceivers.push_back(std::move(commandStreamReceiver));
 
     return true;
diff --git a/shared/source/device/device.h b/shared/source/device/device.h
@@ -202,6 +202,7 @@ class Device : public ReferenceTrackedObject<Device> {
 
     uintptr_t specializedDevice = reinterpret_cast<uintptr_t>(nullptr);
 
+    GraphicsAllocation *kernelEotWaAllocation = nullptr;
     GraphicsAllocation *rtMemoryBackedBuffer = nullptr;
     std::vector<GraphicsAllocation *> rtDispatchGlobals;
     struct {
diff --git a/shared/source/device/root_device.cpp b/shared/source/device/root_device.cpp
@@ -1,5 +1,5 @@
 /*
- * Copyright (C) 2019-2021 Intel Corporation
+ * Copyright (C) 2019-2022 Intel Corporation
  *
  * SPDX-License-Identifier: MIT
  *
@@ -67,6 +67,9 @@ void RootDevice::initializeRootCommandStreamReceiver() {
     rootCommandStreamReceiver->initializeTagAllocation();
     rootCommandStreamReceiver->createGlobalFenceAllocation();
     rootCommandStreamReceiver->createWorkPartitionAllocation(*this);
+    if (kernelEotWaAllocation) {
+        rootCommandStreamReceiver->addAdditionalAllocationForResidency(kernelEotWaAllocation);
+    }
     commandStreamReceivers.push_back(std::move(rootCommandStreamReceiver));
 
     EngineControl engine{commandStreamReceivers.back().get(), osContext};
diff --git a/shared/source/memory_manager/gfx_partition.cpp b/shared/source/memory_manager/gfx_partition.cpp
@@ -1,5 +1,5 @@
 /*
- * Copyright (C) 2019-2021 Intel Corporation
+ * Copyright (C) 2019-2022 Intel Corporation
  *
  * SPDX-License-Identifier: MIT
  *
@@ -237,7 +237,11 @@ bool GfxPartition::init(uint64_t gpuAddressSpace, size_t cpuAddressRangeSizeToRe
             heapInitExternalWithFrontWindow(HeapAssigner::mapExternalWindowIndex(heap), heapAllocate(heap, externalFrontWindowSize),
                                             externalFrontWindowSize);
         } else if (HeapAssigner::isInternalHeap(heap)) {
-            heapInitWithFrontWindow(heap, gfxBase, gfxHeap32Size, GfxPartition::internalFrontWindowPoolSize);
+            auto heapSize = gfxHeap32Size;
+            if (DebugManager.flags.EnableEotWa.get()) {
+                heapSize = 4 * MemoryConstants::gigaByte - MemoryConstants::pageSize64k;
+            }
+            heapInitWithFrontWindow(heap, gfxBase, heapSize, GfxPartition::internalFrontWindowPoolSize);
             heapInitFrontWindow(HeapAssigner::mapInternalWindowIndex(heap), gfxBase, GfxPartition::internalFrontWindowPoolSize);
         } else {
             heapInit(heap, gfxBase, gfxHeap32Size);
diff --git a/shared/source/memory_manager/memory_manager.cpp b/shared/source/memory_manager/memory_manager.cpp
@@ -434,8 +434,9 @@ bool MemoryManager::getAllocationData(AllocationData &allocationData, const Allo
     }
 
     allocationData.hostPtr = hostPtr;
-    if (properties.allocationType == AllocationType::KERNEL_ISA ||
-        properties.allocationType == AllocationType::KERNEL_ISA_INTERNAL) {
+    if ((properties.allocationType == AllocationType::KERNEL_ISA ||
+         properties.allocationType == AllocationType::KERNEL_ISA_INTERNAL) &&
+        properties.gpuAddress == 0) {
         allocationData.size = properties.size + hwHelper.getPaddingForISAAllocation();
     } else {
         allocationData.size = properties.size;
diff --git a/shared/source/os_interface/linux/drm_memory_manager.cpp b/shared/source/os_interface/linux/drm_memory_manager.cpp
@@ -617,8 +617,13 @@ DrmAllocation *DrmMemoryManager::allocate32BitGraphicsMemoryImpl(const Allocatio
     size_t alignedAllocationSize = alignUp(allocationData.size, MemoryConstants::pageSize);
     auto allocationSize = alignedAllocationSize;
     auto gfxPartition = getGfxPartition(allocationData.rootDeviceIndex);
-    auto gpuVA = gfxPartition->heapAllocate(allocatorToUse, allocationSize);
+    uint64_t gpuVA;
 
+    if (GraphicsAllocation::isIsaAllocationType(allocationData.type) && allocationData.gpuAddress != 0) {
+        gpuVA = allocationData.gpuAddress;
+    } else {
+        gpuVA = gfxPartition->heapAllocate(allocatorToUse, allocationSize);
+    }
     if (!gpuVA) {
         return nullptr;
     }
@@ -1403,9 +1408,15 @@ GraphicsAllocation *DrmMemoryManager::allocateGraphicsMemoryInDevicePool(const A
 
     auto sizeAllocated = sizeAligned;
     auto gfxPartition = getGfxPartition(allocationData.rootDeviceIndex);
-    auto gpuAddress = getGpuAddress(this->alignmentSelector, this->heapAssigner, *hwInfo,
-                                    allocationData.type, gfxPartition, sizeAllocated,
-                                    allocationData.hostPtr, allocationData.flags.resource48Bit, allocationData.flags.use32BitFrontWindow, *gmmHelper);
+    uint64_t gpuAddress;
+
+    if (GraphicsAllocation::isIsaAllocationType(allocationData.type) && allocationData.gpuAddress != 0) {
+        gpuAddress = allocationData.gpuAddress;
+    } else {
+        gpuAddress = getGpuAddress(this->alignmentSelector, this->heapAssigner, *hwInfo,
+                                   allocationData.type, gfxPartition, sizeAllocated,
+                                   allocationData.hostPtr, allocationData.flags.resource48Bit, allocationData.flags.use32BitFrontWindow, *gmmHelper);
+    }
     auto canonizedGpuAddress = gmmHelper->canonize(gpuAddress);
     auto allocation = std::make_unique<DrmAllocation>(allocationData.rootDeviceIndex, numHandles, allocationData.type, nullptr, nullptr, canonizedGpuAddress, sizeAligned, MemoryPool::LocalMemory);
     DrmAllocation *drmAllocation = static_cast<DrmAllocation *>(allocation.get());
diff --git a/shared/source/os_interface/windows/wddm_memory_manager.cpp b/shared/source/os_interface/windows/wddm_memory_manager.cpp
@@ -403,7 +403,12 @@ GraphicsAllocation *WddmMemoryManager::allocate32BitGraphicsMemoryImpl(const All
                   CacheSettingsHelper::getGmmUsageType(wddmAllocation->getAllocationType(), !!allocationData.flags.uncacheable, *hwInfo), false, {}, true);
     wddmAllocation->setDefaultGmm(gmm);
 
-    if (!createWddmAllocation(wddmAllocation.get(), nullptr)) {
+    void *requiredGpuAddress = nullptr;
+    if (GraphicsAllocation::isIsaAllocationType(allocationData.type) && allocationData.gpuAddress != 0) {
+        requiredGpuAddress = reinterpret_cast<void *>(allocationData.gpuAddress);
+    }
+
+    if (!createWddmAllocation(wddmAllocation.get(), requiredGpuAddress)) {
         delete gmm;
         freeSystemMemory(pSysMem);
         return nullptr;
@@ -1095,6 +1100,10 @@ GraphicsAllocation *WddmMemoryManager::allocateGraphicsMemoryInDevicePool(const
         }
     }
 
+    if (GraphicsAllocation::isIsaAllocationType(allocationData.type) && allocationData.gpuAddress != 0) {
+        requiredGpuVa = reinterpret_cast<void *>(allocationData.gpuAddress);
+    }
+
     if (!createWddmAllocation(wddmAllocation.get(), requiredGpuVa)) {
         for (auto handleId = 0u; handleId < allocationData.storageInfo.getNumBanks(); handleId++) {
             delete wddmAllocation->getGmm(handleId);
diff --git a/shared/test/common/libult/ult_command_stream_receiver.h b/shared/test/common/libult/ult_command_stream_receiver.h
@@ -65,6 +65,7 @@ class UltCommandStreamReceiver : public CommandStreamReceiverHw<GfxFamily>, publ
     using BaseClass::wasSubmittedToSingleSubdevice;
     using BaseClass::CommandStreamReceiver::activePartitions;
     using BaseClass::CommandStreamReceiver::activePartitionsConfig;
+    using BaseClass::CommandStreamReceiver::additionalAllocationsForResidency;
     using BaseClass::CommandStreamReceiver::baseWaitFunction;
     using BaseClass::CommandStreamReceiver::bindingTableBaseAddressRequired;
     using BaseClass::CommandStreamReceiver::canUse4GbHeaps;
diff --git a/shared/test/common/mock_gdi/mock_gdi.cpp b/shared/test/common/mock_gdi/mock_gdi.cpp
@@ -388,11 +388,13 @@ NTSTATUS __stdcall D3DKMTQueryResourceInfoFromNtHandle(IN OUT D3DKMT_QUERYRESOUR
     return STATUS_SUCCESS;
 }
 
+uint8_t lockedData[0x20000]{};
+
 NTSTATUS __stdcall D3DKMTLock2(IN OUT D3DKMT_LOCK2 *lock2) {
     if (lock2->hAllocation == 0 || lock2->hDevice == 0) {
         return STATUS_INVALID_PARAMETER;
     }
-    lock2->pData = (void *)65536;
+    lock2->pData = reinterpret_cast<void *>((reinterpret_cast<uintptr_t>(lockedData) + 0x10000) & (-0xFFFF));
     return STATUS_SUCCESS;
 }
 
diff --git a/shared/test/common/mocks/mock_device.h b/shared/test/common/mocks/mock_device.h
@@ -61,6 +61,7 @@ class MockDevice : public RootDevice {
     using Device::getGlobalMemorySize;
     using Device::initializeCaps;
     using Device::isDebuggerActive;
+    using Device::kernelEotWaAllocation;
     using Device::regularEngineGroups;
     using Device::rootCsrCreated;
     using Device::rtMemoryBackedBuffer;
diff --git a/shared/test/common/test_files/igdrcl.config b/shared/test/common/test_files/igdrcl.config
@@ -271,6 +271,7 @@ CFEMaximumNumberOfThreads = -1
 CFEOverDispatchControl = -1
 CFELargeGRFThreadAdjustDisable = -1
 SynchronizeWalkerInWparidMode = -1
+EnableEotWa = 0
 EnableWalkerPartition = -1
 OverrideNumComputeUnitsForScratch = -1
 ForceThreadGroupDispatchSize = -1
diff --git a/shared/test/unit_test/command_stream/command_stream_receiver_tests.cpp b/shared/test/unit_test/command_stream/command_stream_receiver_tests.cpp
@@ -98,6 +98,18 @@ HWTEST_F(CommandStreamReceiverTest, WhenCreatingCsrThenTimestampTypeIs32b) {
     EXPECT_EQ(expectedOffset, tag->getGlobalStartOffset());
 }
 
+HWTEST_F(CommandStreamReceiverTest, whenAddingAdditionalAllocationForResidencyThenItIsRegisteredInCsr) {
+    auto &csr = pDevice->getUltCommandStreamReceiver<FamilyType>();
+
+    EXPECT_TRUE(csr.additionalAllocationsForResidency.empty());
+
+    MockGraphicsAllocation allocation{};
+    csr.addAdditionalAllocationForResidency(&allocation);
+
+    EXPECT_EQ(1u, csr.additionalAllocationsForResidency.size());
+    EXPECT_EQ(&allocation, csr.additionalAllocationsForResidency[0]);
+}
+
 HWTEST_F(CommandStreamReceiverTest, WhenCreatingCsrThenFlagsAreSetCorrectly) {
     auto &csr = pDevice->getUltCommandStreamReceiver<FamilyType>();
     csr.initProgrammingFlags();
@@ -2074,4 +2086,4 @@ HWTEST_F(CommandStreamReceiverTest, givenMultipleActivePartitionsWhenWaitLogIsEn
                    << " " << tagValue << std::endl;
 
     EXPECT_STREQ(expectedOutput.str().c_str(), output.c_str());
-}
+}
diff --git a/shared/test/unit_test/device/neo_device_tests.cpp b/shared/test/unit_test/device/neo_device_tests.cpp
diff --git a/shared/test/unit_test/memory_manager/gfx_partition_tests.cpp b/shared/test/unit_test/memory_manager/gfx_partition_tests.cpp
diff --git a/shared/test/unit_test/os_interface/linux/drm_memory_manager_tests.cpp b/shared/test/unit_test/os_interface/linux/drm_memory_manager_tests.cpp
diff --git a/shared/test/unit_test/os_interface/windows/CMakeLists.txt b/shared/test/unit_test/os_interface/windows/CMakeLists.txt
diff --git a/shared/test/unit_test/os_interface/windows/wddm_memory_manager_with_localmem_tests.cpp b/shared/test/unit_test/os_interface/windows/wddm_memory_manager_with_localmem_tests.cpp
