fix: request instruction cache invalidation on module destroy

Invalidation is requested on both linux and windows,
on Csr's that used Isa allocation.

Related-To: NEO-10045
Signed-off-by: Fabian Zwolinski <[email protected]>

Author: fzwolinski

level_zero/core/source/cmdqueue/cmdqueue_hw.inl

@@ -135,6 +135,10 @@ ze_result_t CommandQueueHw<gfxCoreFamily>::executeCommandListsRegular(
         linearStreamSizeEstimate += NEO::MemorySynchronizationCommands<GfxFamily>::getSizeForBarrierWithPostSyncOperation(neoDevice->getRootDeviceEnvironment(), false);
     }
 
+    if (this->csr->isInstructionCacheFlushRequired()) {
+        linearStreamSizeEstimate += NEO::MemorySynchronizationCommands<GfxFamily>::getSizeForInstructionCacheFlush();
+    }
+
     this->csr->getResidencyAllocations().reserve(ctx.spaceForResidency);
 
     NEO::LinearStream child(nullptr);
@@ -226,6 +230,11 @@ ze_result_t CommandQueueHw<gfxCoreFamily>::executeCommandListsRegular(
     this->assignCsrTaskCountToFenceIfAvailable(hFence);
     this->dispatchTaskCountPostSyncRegular(ctx.isDispatchTaskCountPostSyncRequired, child);
 
+    if (this->csr->isInstructionCacheFlushRequired()) {
+        NEO::MemorySynchronizationCommands<GfxFamily>::addInstructionCacheFlush(child);
+        this->csr->setInstructionCacheFlushed();
+    }
+
     auto submitResult = this->prepareAndSubmitBatchBuffer(ctx, child);
 
     this->csr->setPreemptionMode(ctx.statePreemption);

level_zero/core/source/module/module_imp.cpp

@@ -7,6 +7,7 @@
 
 #include "level_zero/core/source/module/module_imp.h"
 
+#include "shared/source/command_stream/command_stream_receiver.h"
 #include "shared/source/compiler_interface/compiler_options.h"
 #include "shared/source/compiler_interface/compiler_options_extra.h"
 #include "shared/source/compiler_interface/compiler_warnings/compiler_warnings.h"
@@ -37,6 +38,7 @@
 #include "shared/source/memory_manager/memory_manager.h"
 #include "shared/source/memory_manager/memory_operations_handler.h"
 #include "shared/source/memory_manager/unified_memory_manager.h"
+#include "shared/source/os_interface/os_context.h"
 #include "shared/source/program/kernel_info.h"
 #include "shared/source/program/program_initialization.h"
 
@@ -1540,6 +1542,19 @@ ze_result_t ModuleImp::destroy() {
 
     auto tempHandle = debugModuleHandle;
     auto tempDevice = device;
+
+    auto rootDeviceIndex = getDevice()->getNEODevice()->getRootDeviceIndex();
+    auto &executionEnvironment = getDevice()->getNEODevice()->getRootDeviceEnvironment().executionEnvironment;
+
+    for (const auto &kernelImmData : this->kernelImmDatas) {
+        for (auto &engine : executionEnvironment.memoryManager->getRegisteredEngines(rootDeviceIndex)) {
+            auto contextId = engine.osContext->getContextId();
+            if (kernelImmData->getIsaGraphicsAllocation()->isUsedByOsContext(contextId)) {
+                engine.commandStreamReceiver->registerInstructionCacheFlush();
+            }
+        }
+    }
+
     delete this;
 
     if (tempDevice->getL0Debugger() && tempHandle != 0) {

level_zero/core/test/unit_tests/sources/cmdqueue/test_cmdqueue_enqueue_cmdlist_2.cpp

@@ -1,5 +1,5 @@
 /*
- * Copyright (C) 2022-2023 Intel Corporation
+ * Copyright (C) 2022-2024 Intel Corporation
  *
  * SPDX-License-Identifier: MIT
  *
@@ -837,5 +837,55 @@ HWTEST_F(CommandQueueExecuteCommandListsSimpleTest, GivenDirtyFlagForContextInBi
     commandQueue->destroy();
 }
 
+HWTEST_F(CommandQueueExecuteCommandListsSimpleTest, GivenRegisterInstructionCacheFlushWhenExecutingCmdListsThenInstructionCacheInvalidateIsSent) {
+    using PIPE_CONTROL = typename FamilyType::PIPE_CONTROL;
+    ze_command_queue_desc_t queueDesc = {};
+    ze_result_t returnValue;
+
+    neoDevice->getDefaultEngine().commandStreamReceiver->registerInstructionCacheFlush();
+
+    queueDesc.mode = ZE_COMMAND_QUEUE_MODE_SYNCHRONOUS;
+    auto commandQueue = whiteboxCast(CommandQueue::create(productFamily, device, neoDevice->getDefaultEngine().commandStreamReceiver, &queueDesc, false, false, false, returnValue));
+    ASSERT_NE(nullptr, commandQueue);
+
+    auto usedSpaceBefore = commandQueue->commandStream.getUsed();
+
+    ze_command_list_handle_t commandLists[] = {
+        CommandList::create(productFamily, device, NEO::EngineGroupType::renderCompute, 0u, returnValue, false)->toHandle()};
+    uint32_t numCommandLists = 1;
+    CommandList::fromHandle(commandLists[0])->close();
+    auto result = commandQueue->executeCommandLists(numCommandLists, commandLists, nullptr, true);
+
+    ASSERT_EQ(ZE_RESULT_SUCCESS, result);
+
+    auto usedSpaceAfter = commandQueue->commandStream.getUsed();
+    ASSERT_GT(usedSpaceAfter, usedSpaceBefore);
+
+    GenCmdList cmdList;
+    ASSERT_TRUE(FamilyType::Parse::parseCommandBuffer(
+        cmdList, ptrOffset(commandQueue->commandStream.getCpuBase(), 0), usedSpaceAfter));
+
+    auto pipeControls = findAll<PIPE_CONTROL *>(cmdList.begin(), cmdList.end());
+    ASSERT_NE(0u, pipeControls.size());
+
+    bool foundInstructionCacheInvalidate = false;
+    for (auto pipeControlIT : pipeControls) {
+        auto pipeControl = reinterpret_cast<PIPE_CONTROL *>(*pipeControlIT);
+        if (pipeControl->getInstructionCacheInvalidateEnable()) {
+            foundInstructionCacheInvalidate = true;
+            break;
+        }
+    }
+
+    EXPECT_TRUE(foundInstructionCacheInvalidate);
+
+    for (auto i = 0u; i < numCommandLists; i++) {
+        auto commandList = CommandList::fromHandle(commandLists[i]);
+        commandList->destroy();
+    }
+
+    commandQueue->destroy();
+}
+
 } // namespace ult
 } // namespace L0

level_zero/core/test/unit_tests/sources/module/test_module_2.cpp

@@ -1,5 +1,5 @@
 /*
- * Copyright (C) 2022-2023 Intel Corporation
+ * Copyright (C) 2022-2024 Intel Corporation
  *
  * SPDX-License-Identifier: MIT
  *
@@ -9,6 +9,7 @@
 #include "shared/source/helpers/aligned_memory.h"
 #include "shared/source/helpers/file_io.h"
 #include "shared/test/common/helpers/test_files.h"
+#include "shared/test/common/mocks/mock_command_stream_receiver.h"
 #include "shared/test/common/mocks/mock_device.h"
 #include "shared/test/common/mocks/mock_modules_zebin.h"
 #include "shared/test/common/test_macros/test.h"
@@ -17,7 +18,9 @@
 #include "level_zero/core/source/kernel/kernel.h"
 #include "level_zero/core/source/module/module_build_log.h"
 #include "level_zero/core/test/unit_tests/fixtures/device_fixture.h"
+#include "level_zero/core/test/unit_tests/mocks/mock_device.h"
 #include "level_zero/core/test/unit_tests/mocks/mock_module.h"
+
 namespace L0 {
 namespace ult {
 
@@ -75,6 +78,36 @@ TEST_F(ModuleTests, whenCreatingAutoGrfBuildOptionsThenOptionsParsedCorrectly) {
     EXPECT_TRUE(NEO::CompilerOptions::contains(internalBuildOptions, NEO::CompilerOptions::autoGrf));
 }
 
+TEST(ModuleDestroyTest, givenIsaAllocationWhenIsModuleDestroyedThenRequireInstructionCacheFlushInCsrThatUsedTheAllocation) {
+    const uint32_t rootDeviceIndex = 0u;
+    NEO::HardwareInfo hwInfo = *NEO::defaultHwInfo.get();
+    auto *neoMockDevice = NEO::MockDevice::createWithNewExecutionEnvironment<NEO::MockDevice>(&hwInfo, rootDeviceIndex);
+
+    MockCommandStreamReceiver *mockCommandStreamReceiver = new MockCommandStreamReceiver(*neoMockDevice->executionEnvironment, neoMockDevice->getRootDeviceIndex(), neoMockDevice->getDeviceBitfield());
+    mockCommandStreamReceiver->makeResidentParentCall = true;
+
+    neoMockDevice->resetCommandStreamReceiver(mockCommandStreamReceiver);
+
+    MockDeviceImp deviceImp(neoMockDevice, neoMockDevice->getExecutionEnvironment());
+
+    auto module = new MockModule{&deviceImp, nullptr, ModuleType::user};
+    module->translationUnit.reset(new MockModuleTranslationUnit{&deviceImp});
+
+    auto kernelInfo = new KernelInfo{};
+    kernelInfo->heapInfo.pKernelHeap = reinterpret_cast<const void *>(0xdeadbeef0000);
+    kernelInfo->heapInfo.kernelHeapSize = static_cast<uint32_t>(0x40);
+    module->translationUnit->programInfo.kernelInfos.push_back(kernelInfo);
+
+    module->initializeKernelImmutableDatas();
+    auto &kernelImmDatas = module->getKernelImmutableDataVector();
+    auto csr = deviceImp.getNEODevice()->getEngine(0).commandStreamReceiver;
+    csr->makeResident(*kernelImmDatas[0]->getIsaParentAllocation());
+
+    module->destroy();
+
+    EXPECT_TRUE(mockCommandStreamReceiver->requiresInstructionCacheFlush);
+}
+
 TEST(ModuleBuildLog, WhenCreatingModuleBuildLogThenNonNullPointerReturned) {
     auto moduleBuildLog = ModuleBuildLog::create();
     ASSERT_NE(nullptr, moduleBuildLog);

shared/source/command_stream/command_stream_receiver.h

@@ -273,6 +273,14 @@ class CommandStreamReceiver {
 
     void downloadAllocation(GraphicsAllocation &gfxAllocation);
 
+    bool isInstructionCacheFlushRequired() const {
+        return requiresInstructionCacheFlush;
+    }
+
+    void setInstructionCacheFlushed() {
+        requiresInstructionCacheFlush = false;
+    }
+
     void registerInstructionCacheFlush() {
         auto mutex = obtainUniqueOwnership();
         requiresInstructionCacheFlush = true;

shared/source/command_stream/command_stream_receiver_hw_base.inl

@@ -304,6 +304,10 @@ CompletionStamp CommandStreamReceiverHw<GfxFamily>::flushImmediateTask(
         flushData.estimatedSize += MemorySynchronizationCommands<GfxFamily>::getSizeForFullCacheFlush();
     }
 
+    if (requiresInstructionCacheFlush) {
+        flushData.estimatedSize += MemorySynchronizationCommands<GfxFamily>::getSizeForInstructionCacheFlush();
+    }
+
     auto &csrCommandStream = getCS(flushData.estimatedSize);
     flushData.csrStartOffset = csrCommandStream.getUsed();
 
@@ -312,6 +316,11 @@ CompletionStamp CommandStreamReceiverHw<GfxFamily>::flushImmediateTask(
         MemorySynchronizationCommands<GfxFamily>::addStateCacheFlush(csrCommandStream, device.getRootDeviceEnvironment());
     }
 
+    if (requiresInstructionCacheFlush) {
+        MemorySynchronizationCommands<GfxFamily>::addInstructionCacheFlush(csrCommandStream);
+        requiresInstructionCacheFlush = false;
+    }
+
     dispatchImmediateFlushPipelineSelectCommand(flushData, csrCommandStream);
     dispatchImmediateFlushFrontEndCommand(flushData, device, csrCommandStream);
     dispatchImmediateFlushStateComputeModeCommand(flushData, csrCommandStream);
@@ -540,9 +549,7 @@ CompletionStamp CommandStreamReceiverHw<GfxFamily>::flushTask(
     }
 
     if (requiresInstructionCacheFlush) {
-        PipeControlArgs args;
-        args.instructionCacheInvalidateEnable = true;
-        MemorySynchronizationCommands<GfxFamily>::addSingleBarrier(commandStreamCSR, args);
+        MemorySynchronizationCommands<GfxFamily>::addInstructionCacheFlush(commandStreamCSR);
         requiresInstructionCacheFlush = false;
     }
 
@@ -942,7 +949,7 @@ size_t CommandStreamReceiverHw<GfxFamily>::getRequiredCmdStreamSize(const Dispat
     }
 
     if (requiresInstructionCacheFlush) {
-        size += MemorySynchronizationCommands<GfxFamily>::getSizeForSingleBarrier(false);
+        size += MemorySynchronizationCommands<GfxFamily>::getSizeForInstructionCacheFlush();
     }
 
     if (debugManager.flags.ForcePipeControlPriorToWalker.get()) {

shared/source/helpers/gfx_core_helper.h

@@ -459,13 +459,15 @@ struct MemorySynchronizationCommands {
     static void addFullCacheFlush(LinearStream &commandStream, const RootDeviceEnvironment &rootDeviceEnvironment);
     static void setCacheFlushExtraProperties(PipeControlArgs &args);
     static void addStateCacheFlush(LinearStream &commandStream, const RootDeviceEnvironment &rootDeviceEnvironment);
+    static void addInstructionCacheFlush(LinearStream &commandStream);
 
     static size_t getSizeForBarrierWithPostSyncOperation(const RootDeviceEnvironment &rootDeviceEnvironment, bool tlbInvalidationRequired);
     static size_t getSizeForBarrierWa(const RootDeviceEnvironment &rootDeviceEnvironment);
     static size_t getSizeForSingleBarrier(bool tlbInvalidationRequired);
     static size_t getSizeForSingleAdditionalSynchronizationForDirectSubmission(const RootDeviceEnvironment &rootDeviceEnvironment);
     static size_t getSizeForSingleAdditionalSynchronization(const RootDeviceEnvironment &rootDeviceEnvironment);
     static size_t getSizeForAdditonalSynchronization(const RootDeviceEnvironment &rootDeviceEnvironment);
+    static size_t getSizeForInstructionCacheFlush();
     static size_t getSizeForFullCacheFlush();
 
     static bool isBarrierWaRequired(const RootDeviceEnvironment &rootDeviceEnvironment);

shared/source/helpers/gfx_core_helper_base.inl

@@ -528,6 +528,19 @@ void MemorySynchronizationCommands<GfxFamily>::addStateCacheFlush(LinearStream &
     *reinterpret_cast<PIPE_CONTROL *>(commandsBuffer) = cmd;
 }
 
+template <typename GfxFamily>
+size_t MemorySynchronizationCommands<GfxFamily>::getSizeForInstructionCacheFlush() {
+    return MemorySynchronizationCommands<GfxFamily>::getSizeForSingleBarrier(false);
+}
+
+template <typename GfxFamily>
+void MemorySynchronizationCommands<GfxFamily>::addInstructionCacheFlush(LinearStream &commandStream) {
+    PipeControlArgs args;
+    args.instructionCacheInvalidateEnable = true;
+
+    MemorySynchronizationCommands<GfxFamily>::addSingleBarrier(commandStream, args);
+}
+
 template <typename GfxFamily>
 const StackVec<size_t, 3> GfxCoreHelperHw<GfxFamily>::getDeviceSubGroupSizes() const {
     return {8, 16, 32};

shared/test/common/mocks/mock_command_stream_receiver.h

@@ -49,6 +49,7 @@ class MockCommandStreamReceiver : public CommandStreamReceiver {
     using CommandStreamReceiver::osContext;
     using CommandStreamReceiver::ownershipMutex;
     using CommandStreamReceiver::preemptionAllocation;
+    using CommandStreamReceiver::requiresInstructionCacheFlush;
     using CommandStreamReceiver::tagAddress;
     using CommandStreamReceiver::tagsMultiAllocation;
     using CommandStreamReceiver::taskCount;
@@ -183,6 +184,9 @@ class MockCommandStreamReceiver : public CommandStreamReceiver {
     GraphicsAllocation *getClearColorAllocation() override { return nullptr; }
     void makeResident(GraphicsAllocation &gfxAllocation) override {
         makeResidentCalledTimes++;
+        if (makeResidentParentCall) {
+            return CommandStreamReceiver::makeResident(gfxAllocation);
+        }
     }
 
     std::unique_lock<CommandStreamReceiver::MutexType> obtainHostPtrSurfaceCreationLock() override {
@@ -220,6 +224,7 @@ class MockCommandStreamReceiver : public CommandStreamReceiver {
     bool programHardwareContextCalled = false;
     bool createPreemptionAllocationReturn = true;
     bool createPreemptionAllocationParentCall = false;
+    bool makeResidentParentCall = false;
     bool programComputeBarrierCommandCalled = false;
     bool programStallingCommandsForBarrierCalled = false;
     std::optional<bool> isGpuHangDetectedReturnValue{};

shared/test/unit_test/command_stream/command_stream_receiver_tests.cpp

@@ -4729,6 +4729,25 @@ HWTEST_F(CommandStreamReceiverHwTest, GivenDirtyFlagForContextInBindlessHelperWh
     EXPECT_FALSE(bindlessHeapsHelperPtr->getStateDirtyForContext(commandStreamReceiver.getOsContext().getContextId()));
 }
 
+HWTEST_F(CommandStreamReceiverHwTest, givenRequiresInstructionCacheFlushWhenFlushImmediateThenInstructionCacheInvalidateEnableIsSent) {
+    using PIPE_CONTROL = typename FamilyType::PIPE_CONTROL;
+
+    auto &commandStreamReceiver = pDevice->getUltCommandStreamReceiver<FamilyType>();
+    commandStreamReceiver.registerInstructionCacheFlush();
+
+    this->requiredStreamProperties.stateComputeMode.setPropertiesAll(false, GrfConfig::defaultGrfNumber, ThreadArbitrationPolicy::AgeBased, NEO::PreemptionMode::ThreadGroup);
+
+    commandStreamReceiver.flushImmediateTask(commandStream, commandStream.getUsed(), immediateFlushTaskFlags, *pDevice);
+
+    HardwareParse hwParserCsr;
+    hwParserCsr.parseCommands<FamilyType>(commandStreamReceiver.commandStream, 0);
+    auto pcCmd = hwParserCsr.getCommand<PIPE_CONTROL>();
+    ASSERT_NE(nullptr, pcCmd);
+
+    EXPECT_TRUE(pcCmd->getInstructionCacheInvalidateEnable());
+    EXPECT_FALSE(commandStreamReceiver.requiresInstructionCacheFlush);
+}
+
 HWTEST_F(CommandStreamReceiverHwTest, GivenFlushIsBlockingWhenFlushTaskCalledThenExpectMonitorFenceFlagTrue) {
     auto &commandStreamReceiver = pDevice->getUltCommandStreamReceiver<FamilyType>();
     commandStreamReceiver.recordFlusheBatchBuffer = true;