fix: serialize printf kernel accesses using device-wise locks

lyu · Compute-Runtime-Automation · commit 37deaf1ae560 · 2023-11-27T10:42:51.000+01:00
Related-To: LOCI-4114

Signed-off-by: Lu, Wenbin &lt;wenbin.lu@intel.com&gt;
diff --git a/level_zero/core/source/cmdlist/cmdlist.cpp b/level_zero/core/source/cmdlist/cmdlist.cpp
@@ -22,6 +22,7 @@
 #include "level_zero/core/source/driver/driver_handle_imp.h"
 #include "level_zero/core/source/event/event.h"
 #include "level_zero/core/source/kernel/kernel.h"
+#include "level_zero/core/source/kernel/kernel_imp.h"
 
 namespace L0 {
 
@@ -38,11 +39,11 @@ CommandList::~CommandList() {
 }
 
 void CommandList::storePrintfKernel(Kernel *kernel) {
-    auto it = std::find(this->printfKernelContainer.begin(), this->printfKernelContainer.end(),
-                        kernel);
+    auto it = std::find_if(this->printfKernelContainer.begin(), this->printfKernelContainer.end(), [&kernel](const auto &kernelWeakPtr) { return kernelWeakPtr.lock().get() == kernel; });
 
     if (it == this->printfKernelContainer.end()) {
-        this->printfKernelContainer.push_back(kernel);
+        auto module = static_cast<const ModuleImp *>(&static_cast<KernelImp *>(kernel)->getParentModule());
+        this->printfKernelContainer.push_back(module->getPrintfKernelWeakPtr(kernel->toHandle()));
     }
 }
 
diff --git a/level_zero/core/source/cmdlist/cmdlist.h b/level_zero/core/source/cmdlist/cmdlist.h
@@ -231,7 +231,7 @@ struct CommandList : _ze_command_list_handle_t {
         return indirectAllocationsAllowed;
     }
 
-    std::vector<Kernel *> &getPrintfKernelContainer() {
+    std::vector<std::weak_ptr<Kernel>> &getPrintfKernelContainer() {
         return this->printfKernelContainer;
     }
 
@@ -360,7 +360,7 @@ struct CommandList : _ze_command_list_handle_t {
     std::map<const void *, NEO::GraphicsAllocation *> hostPtrMap;
     NEO::PrivateAllocsToReuseContainer ownedPrivateAllocations;
     std::vector<NEO::GraphicsAllocation *> patternAllocations;
-    std::vector<Kernel *> printfKernelContainer;
+    std::vector<std::weak_ptr<Kernel>> printfKernelContainer;
 
     NEO::CommandContainer commandContainer;
 
diff --git a/level_zero/core/source/cmdlist/cmdlist_hw.inl b/level_zero/core/source/cmdlist/cmdlist_hw.inl
@@ -434,7 +434,10 @@ ze_result_t CommandListCoreFamily<gfxCoreFamily>::appendLaunchKernelIndirect(ze_
     if (hEvent) {
         event = Event::fromHandle(hEvent);
         if (Kernel::fromHandle(kernelHandle)->getPrintfBufferAllocation() != nullptr) {
-            event->setKernelForPrintf(Kernel::fromHandle(kernelHandle));
+            Kernel *kernel = Kernel::fromHandle(kernelHandle);
+            auto module = static_cast<const ModuleImp *>(&static_cast<KernelImp *>(kernel)->getParentModule());
+            event->setKernelForPrintf(module->getPrintfKernelWeakPtr(kernelHandle));
+            event->setKernelWithPrintfDeviceMutex(kernel->getDevicePrintfKernelMutex());
         }
         launchParams.isHostSignalScopeEvent = event->isSignalScope(ZE_EVENT_SCOPE_FLAG_HOST);
     }
diff --git a/level_zero/core/source/cmdlist/cmdlist_hw_immediate.inl b/level_zero/core/source/cmdlist/cmdlist_hw_immediate.inl
@@ -387,7 +387,7 @@ inline ze_result_t CommandListCoreFamilyImmediate<gfxCoreFamily>::executeCommand
 
     this->cmdQImmediate->setTaskCount(completionStamp.taskCount);
 
-    if (this->isSyncModeQueue || this->printfKernelContainer.size() > 0u) {
+    if (this->isSyncModeQueue) {
         status = hostSynchronize(std::numeric_limits<uint64_t>::max(), completionStamp.taskCount, true);
     }
 
@@ -1238,9 +1238,11 @@ bool CommandListCoreFamilyImmediate<gfxCoreFamily>::isBarrierRequired() {
 
 template <GFXCORE_FAMILY gfxCoreFamily>
 void CommandListCoreFamilyImmediate<gfxCoreFamily>::printKernelsPrintfOutput(bool hangDetected) {
-    size_t size = this->printfKernelContainer.size();
-    for (size_t i = 0; i < size; i++) {
-        this->printfKernelContainer[i]->printPrintfOutput(hangDetected);
+    for (auto &kernelWeakPtr : this->printfKernelContainer) {
+        std::lock_guard<std::mutex> lock(static_cast<DeviceImp *>(this->device)->printfKernelMutex);
+        if (!kernelWeakPtr.expired()) {
+            kernelWeakPtr.lock()->printPrintfOutput(hangDetected);
+        }
     }
     this->printfKernelContainer.clear();
 }
diff --git a/level_zero/core/source/cmdlist/cmdlist_hw_skl_to_tgllp.inl b/level_zero/core/source/cmdlist/cmdlist_hw_skl_to_tgllp.inl
@@ -271,7 +271,9 @@ ze_result_t CommandListCoreFamily<gfxCoreFamily>::appendLaunchKernelWithParams(K
     }
 
     if (event != nullptr && kernel->getPrintfBufferAllocation() != nullptr) {
-        event->setKernelForPrintf(kernel);
+        auto module = static_cast<const ModuleImp *>(&static_cast<KernelImp *>(kernel)->getParentModule());
+        event->setKernelForPrintf(module->getPrintfKernelWeakPtr(kernel->toHandle()));
+        event->setKernelWithPrintfDeviceMutex(kernel->getDevicePrintfKernelMutex());
     }
 
     if (this->isInOrderExecutionEnabled() && !launchParams.isKernelSplitOperation) {
diff --git a/level_zero/core/source/cmdlist/cmdlist_hw_xehp_and_later.inl b/level_zero/core/source/cmdlist/cmdlist_hw_xehp_and_later.inl
@@ -23,7 +23,9 @@
 #include "shared/source/utilities/software_tags_manager.h"
 
 #include "level_zero/core/source/cmdlist/cmdlist_hw.h"
+#include "level_zero/core/source/device/device.h"
 #include "level_zero/core/source/driver/driver_handle_imp.h"
+#include "level_zero/core/source/event/event.h"
 #include "level_zero/core/source/gfx_core_helpers/l0_gfx_core_helper.h"
 #include "level_zero/core/source/kernel/kernel_imp.h"
 #include "level_zero/core/source/module/module.h"
@@ -180,7 +182,9 @@ ze_result_t CommandListCoreFamily<gfxCoreFamily>::appendLaunchKernelWithParams(K
     Event *eventForInOrderExec = event;
     if (event) {
         if (kernel->getPrintfBufferAllocation() != nullptr) {
-            event->setKernelForPrintf(kernel);
+            auto module = static_cast<const ModuleImp *>(&static_cast<KernelImp *>(kernel)->getParentModule());
+            event->setKernelForPrintf(module->getPrintfKernelWeakPtr(kernel->toHandle()));
+            event->setKernelWithPrintfDeviceMutex(kernel->getDevicePrintfKernelMutex());
         }
         isHostSignalScopeEvent = event->isSignalScope(ZE_EVENT_SCOPE_FLAG_HOST);
         if (compactL3FlushEvent(getDcFlushRequired(event->isSignalScope()))) {
diff --git a/level_zero/core/source/cmdqueue/cmdqueue.cpp b/level_zero/core/source/cmdqueue/cmdqueue.cpp
@@ -190,9 +190,11 @@ ze_result_t CommandQueueImp::synchronizeByPollingForTaskCount(uint64_t timeout)
 }
 
 void CommandQueueImp::printKernelsPrintfOutput(bool hangDetected) {
-    size_t size = this->printfKernelContainer.size();
-    for (size_t i = 0; i < size; i++) {
-        this->printfKernelContainer[i]->printPrintfOutput(hangDetected);
+    for (auto &kernelWeakPtr : this->printfKernelContainer) {
+        std::lock_guard<std::mutex> lock(static_cast<DeviceImp *>(this->getDevice())->printfKernelMutex);
+        if (!kernelWeakPtr.expired()) {
+            kernelWeakPtr.lock()->printPrintfOutput(hangDetected);
+        }
     }
     this->printfKernelContainer.clear();
 }
diff --git a/level_zero/core/source/cmdqueue/cmdqueue_imp.h b/level_zero/core/source/cmdqueue/cmdqueue_imp.h
@@ -146,7 +146,7 @@ struct CommandQueueImp : public CommandQueue {
     NEO::LinearStream firstCmdListStream{};
     NEO::HeapContainer heapContainer;
     ze_command_queue_desc_t desc;
-    std::vector<Kernel *> printfKernelContainer;
+    std::vector<std::weak_ptr<Kernel>> printfKernelContainer;
 
     Device *device = nullptr;
     NEO::CommandStreamReceiver *csr = nullptr;
diff --git a/level_zero/core/source/device/device_imp.h b/level_zero/core/source/device/device_imp.h
@@ -132,6 +132,8 @@ struct DeviceImp : public Device, NEO::NonCopyableOrMovableClass {
     ze_pci_speed_ext_t pciMaxSpeed = {-1, -1, -1};
     Device *rootDevice = nullptr;
 
+    std::mutex printfKernelMutex;
+
     BcsSplit bcsSplit;
 
     bool resourcesReleased = false;
diff --git a/level_zero/core/source/event/event.h b/level_zero/core/source/event/event.h
@@ -18,6 +18,7 @@
 #include <chrono>
 #include <limits>
 #include <memory>
+#include <mutex>
 #include <vector>
 
 struct _ze_event_handle_t {};
@@ -206,12 +207,24 @@ struct Event : _ze_event_handle_t {
     uint32_t getMaxKernelCount() const {
         return maxKernelCount;
     }
-    void setKernelForPrintf(Kernel *inputKernelPtr) {
-        kernelWithPrintf = inputKernelPtr;
+    void setKernelForPrintf(std::weak_ptr<Kernel> inputKernelWeakPtr) {
+        kernelWithPrintf = inputKernelWeakPtr;
     }
-    Kernel *getKernelForPrintf() {
+    std::weak_ptr<Kernel> getKernelForPrintf() {
         return kernelWithPrintf;
     }
+    void resetKernelForPrintf() {
+        kernelWithPrintf.reset();
+    }
+    void setKernelWithPrintfDeviceMutex(std::mutex *mutexPtr) {
+        kernelWithPrintfDeviceMutex = mutexPtr;
+    }
+    std::mutex *getKernelWithPrintfDeviceMutex() {
+        return kernelWithPrintfDeviceMutex;
+    }
+    void resetKernelWithPrintfDeviceMutex() {
+        kernelWithPrintfDeviceMutex = nullptr;
+    }
 
     bool isSignalScope() const {
         return !!signalScope;
@@ -280,7 +293,8 @@ struct Event : _ze_event_handle_t {
     void *hostAddress = nullptr;
     Device *device = nullptr;
     EventPool *eventPool = nullptr;
-    Kernel *kernelWithPrintf = nullptr;
+    std::weak_ptr<Kernel> kernelWithPrintf = std::weak_ptr<Kernel>{};
+    std::mutex *kernelWithPrintfDeviceMutex = nullptr;
     std::shared_ptr<InOrderExecInfo> inOrderExecInfo;
     CommandQueue *latestUsedCmdQueue = nullptr;
 
diff --git a/level_zero/core/source/event/event_impl.inl b/level_zero/core/source/event/event_impl.inl
@@ -15,6 +15,7 @@
 #include "shared/source/os_interface/os_context.h"
 #include "shared/source/os_interface/os_time.h"
 
+#include "level_zero/core/source/device/device.h"
 #include "level_zero/core/source/event/event_imp.h"
 #include "level_zero/core/source/gfx_core_helpers/l0_gfx_core_helper.h"
 #include "level_zero/core/source/kernel/kernel.h"
@@ -467,9 +468,13 @@ ze_result_t EventImp<TagSizeT>::hostSynchronize(uint64_t timeout) {
             ret = queryStatus();
         }
         if (ret == ZE_RESULT_SUCCESS) {
-            if (this->getKernelForPrintf() != nullptr) {
-                static_cast<Kernel *>(this->getKernelForPrintf())->printPrintfOutput(true);
-                this->setKernelForPrintf(nullptr);
+            if (this->getKernelWithPrintfDeviceMutex() != nullptr) {
+                std::lock_guard<std::mutex> lock(*this->getKernelWithPrintfDeviceMutex());
+                if (!this->getKernelForPrintf().expired()) {
+                    this->getKernelForPrintf().lock()->printPrintfOutput(true);
+                }
+                this->resetKernelForPrintf();
+                this->resetKernelWithPrintfDeviceMutex();
             }
             if (device->getNEODevice()->getRootDeviceEnvironment().assertHandler.get()) {
                 device->getNEODevice()->getRootDeviceEnvironment().assertHandler->printAssertAndAbort();
diff --git a/level_zero/core/source/kernel/kernel.h b/level_zero/core/source/kernel/kernel.h
@@ -151,6 +151,7 @@ struct Kernel : _ze_kernel_handle_t, virtual NEO::DispatchKernelEncoderI {
     virtual UnifiedMemoryControls getUnifiedMemoryControls() const = 0;
     virtual bool hasIndirectAllocationsAllowed() const = 0;
 
+    virtual std::mutex *getDevicePrintfKernelMutex() = 0;
     virtual NEO::GraphicsAllocation *getPrintfBufferAllocation() = 0;
     virtual void printPrintfOutput(bool hangDetected) = 0;
 
diff --git a/level_zero/core/source/kernel/kernel_imp.cpp b/level_zero/core/source/kernel/kernel_imp.cpp
@@ -1114,11 +1114,11 @@ void KernelImp::createPrintfBuffer() {
         if (pImplicitArgs) {
             pImplicitArgs->printfBufferPtr = printfBuffer->getGpuAddress();
         }
+        this->devicePrintfKernelMutex = &(static_cast<DeviceImp *>(this->module->getDevice())->printfKernelMutex);
     }
 }
 
 void KernelImp::printPrintfOutput(bool hangDetected) {
-    std::lock_guard<std::mutex> lock(this->printfLock);
     PrintfHandler::printOutput(kernelImmData, this->printfBuffer, module->getDevice(), hangDetected);
 }
 
diff --git a/level_zero/core/source/kernel/kernel_imp.h b/level_zero/core/source/kernel/kernel_imp.h
@@ -14,6 +14,8 @@
 #include "shared/source/unified_memory/unified_memory.h"
 
 #include "level_zero/core/source/kernel/kernel.h"
+#include "level_zero/core/source/module/module.h"
+#include "level_zero/core/source/module/module_imp.h"
 
 #include <memory>
 #include <mutex>
@@ -38,8 +40,12 @@ struct KernelImp : Kernel {
     ~KernelImp() override;
 
     ze_result_t destroy() override {
-        delete this;
-        return ZE_RESULT_SUCCESS;
+        if (this->devicePrintfKernelMutex == nullptr) {
+            delete this;
+            return ZE_RESULT_SUCCESS;
+        } else {
+            return static_cast<ModuleImp *>(this->module)->destroyPrintfKernel(this);
+        }
     }
 
     ze_result_t getBaseAddress(uint64_t *baseAddress) override;
@@ -99,6 +105,7 @@ struct KernelImp : Kernel {
     uint32_t getNumThreadsPerThreadGroup() const override { return numThreadsPerThreadGroup; }
     uint32_t getThreadExecutionMask() const override { return threadExecutionMask; }
 
+    std::mutex *getDevicePrintfKernelMutex() override { return this->devicePrintfKernelMutex; }
     NEO::GraphicsAllocation *getPrintfBufferAllocation() override { return this->printfBuffer; }
     void printPrintfOutput(bool hangDetected) override;
 
@@ -207,6 +214,7 @@ struct KernelImp : Kernel {
     std::vector<KernelImp::KernelArgHandler> kernelArgHandlers;
     std::vector<NEO::GraphicsAllocation *> residencyContainer;
 
+    std::mutex *devicePrintfKernelMutex = nullptr;
     NEO::GraphicsAllocation *printfBuffer = nullptr;
 
     uint32_t groupSize[3] = {0u, 0u, 0u};
@@ -248,7 +256,6 @@ struct KernelImp : Kernel {
     std::unique_ptr<NEO::ImplicitArgs> pImplicitArgs;
 
     std::unique_ptr<KernelExt> pExtension;
-    std::mutex printfLock;
 
     struct SuggestGroupSizeCacheEntry {
         Vec3<size_t> groupSize;
diff --git a/level_zero/core/source/module/module_imp.cpp b/level_zero/core/source/module/module_imp.cpp
@@ -41,6 +41,7 @@
 #include "shared/source/program/program_initialization.h"
 
 #include "level_zero/core/source/device/device.h"
+#include "level_zero/core/source/device/device_imp.h"
 #include "level_zero/core/source/driver/driver_handle.h"
 #include "level_zero/core/source/driver/driver_handle_imp.h"
 #include "level_zero/core/source/gfx_core_helpers/l0_gfx_core_helper.h"
@@ -507,6 +508,11 @@ ModuleImp::ModuleImp(Device *device, ModuleBuildLog *moduleBuildLog, ModuleType
 }
 
 ModuleImp::~ModuleImp() {
+    for (auto &kernel : this->printfKernelContainer) {
+        if (kernel.get() != nullptr) {
+            destroyPrintfKernel(kernel->toHandle());
+        }
+    }
     this->kernelImmDatas.clear();
     if (this->kernelsIsaParentRegion) {
         DEBUG_BREAK_IF(this->device->getNEODevice()->getMemoryManager() == nullptr);
@@ -955,6 +961,9 @@ ze_result_t ModuleImp::createKernel(const ze_kernel_desc_t *desc,
 
     if (res == ZE_RESULT_SUCCESS) {
         *kernelHandle = kernel->toHandle();
+        if (kernel->getPrintfBufferAllocation() != nullptr) {
+            this->printfKernelContainer.push_back(std::shared_ptr<Kernel>(kernel));
+        }
     } else {
         driverHandle->clearErrorDescription();
     }
@@ -973,6 +982,28 @@ ze_result_t ModuleImp::createKernel(const ze_kernel_desc_t *desc,
     return res;
 }
 
+std::weak_ptr<Kernel> ModuleImp::getPrintfKernelWeakPtr(ze_kernel_handle_t kernelHandle) const {
+    std::lock_guard<std::mutex> lock(static_cast<DeviceImp *>(device)->printfKernelMutex);
+    Kernel *kernel = Kernel::fromHandle(kernelHandle);
+    auto it = std::find_if(this->printfKernelContainer.begin(), this->printfKernelContainer.end(), [&kernel](const auto &kernelSharedPtr) { return kernelSharedPtr.get() == kernel; });
+    if (it == this->printfKernelContainer.end()) {
+        return std::weak_ptr<Kernel>{};
+    } else {
+        return std::weak_ptr<Kernel>{*it};
+    }
+}
+
+ze_result_t ModuleImp::destroyPrintfKernel(ze_kernel_handle_t kernelHandle) {
+    std::lock_guard<std::mutex> lock(static_cast<DeviceImp *>(device)->printfKernelMutex);
+    Kernel *kernel = Kernel::fromHandle(kernelHandle);
+    auto it = std::find_if(this->printfKernelContainer.begin(), this->printfKernelContainer.end(), [&kernel](const auto &kernelSharedPtr) { return kernelSharedPtr.get() == kernel; });
+    if (it == this->printfKernelContainer.end()) {
+        return ZE_RESULT_ERROR_INVALID_ARGUMENT;
+    }
+    it->reset();
+    return ZE_RESULT_SUCCESS;
+}
+
 ze_result_t ModuleImp::getNativeBinary(size_t *pSize, uint8_t *pModuleNativeBinary) {
     auto genBinary = this->translationUnit->packedDeviceBinary.get();
 
diff --git a/level_zero/core/source/module/module_imp.h b/level_zero/core/source/module/module_imp.h
@@ -11,6 +11,7 @@
 #include "shared/source/compiler_interface/linker.h"
 #include "shared/source/program/program_info.h"
 
+#include "level_zero/core/source/kernel/kernel.h"
 #include "level_zero/core/source/module/module.h"
 
 #include "igfxfmid.h"
@@ -155,6 +156,10 @@ struct ModuleImp : public Module {
         return this->translationUnit.get();
     }
 
+    std::vector<std::shared_ptr<Kernel>> &getPrintfKernelContainer() { return this->printfKernelContainer; }
+    std::weak_ptr<Kernel> getPrintfKernelWeakPtr(ze_kernel_handle_t kernelHandle) const;
+    ze_result_t destroyPrintfKernel(ze_kernel_handle_t kernelHandle);
+
   protected:
     MOCKABLE_VIRTUAL ze_result_t initializeTranslationUnit(const ze_module_desc_t *desc, NEO::Device *neoDevice);
     ze_result_t checkIfBuildShouldBeFailed(NEO::Device *neoDevice);
@@ -182,6 +187,7 @@ struct ModuleImp : public Module {
     ModuleBuildLog *moduleBuildLog = nullptr;
     NEO::GraphicsAllocation *exportedFunctionsSurface = nullptr;
     std::unique_ptr<NEO::GraphicsAllocation> kernelsIsaParentRegion;
+    std::vector<std::shared_ptr<Kernel>> printfKernelContainer;
     std::vector<std::unique_ptr<KernelImmutableData>> kernelImmDatas;
     NEO::Linker::RelocatedSymbolsMap symbols;
 
diff --git a/level_zero/core/test/unit_tests/sources/cmdlist/test_cmdlist_append_launch_kernel_1.cpp b/level_zero/core/test/unit_tests/sources/cmdlist/test_cmdlist_append_launch_kernel_1.cpp
diff --git a/level_zero/core/test/unit_tests/sources/cmdlist/test_cmdlist_append_launch_kernel_3.cpp b/level_zero/core/test/unit_tests/sources/cmdlist/test_cmdlist_append_launch_kernel_3.cpp
diff --git a/level_zero/core/test/unit_tests/sources/cmdqueue/test_cmdqueue_1.cpp b/level_zero/core/test/unit_tests/sources/cmdqueue/test_cmdqueue_1.cpp
diff --git a/level_zero/core/test/unit_tests/sources/event/test_event.cpp b/level_zero/core/test/unit_tests/sources/event/test_event.cpp
diff --git a/level_zero/core/test/unit_tests/sources/fence/test_fence.cpp b/level_zero/core/test/unit_tests/sources/fence/test_fence.cpp
diff --git a/level_zero/core/test/unit_tests/sources/kernel/test_kernel.cpp b/level_zero/core/test/unit_tests/sources/kernel/test_kernel.cpp

Original file line number	Diff line number	Diff line change
`@@ -22,6 +22,7 @@`
`22`	`22`	`#include "level_zero/core/source/driver/driver_handle_imp.h"`
`23`	`23`	`#include "level_zero/core/source/event/event.h"`
`24`	`24`	`#include "level_zero/core/source/kernel/kernel.h"`
	`25`	`+#include "level_zero/core/source/kernel/kernel_imp.h"`
`25`	`26`
`26`	`27`	`namespace L0 {`
`27`	`28`
`@@ -38,11 +39,11 @@ CommandList::~CommandList() {`
`38`	`39`	`}`
`39`	`40`
`40`	`41`	`void CommandList::storePrintfKernel(Kernel *kernel) {`
`41`		`- auto it = std::find(this->printfKernelContainer.begin(), this->printfKernelContainer.end(),`
`42`		`- kernel);`
	`42`	`+ auto it = std::find_if(this->printfKernelContainer.begin(), this->printfKernelContainer.end(), [&kernel](const auto &kernelWeakPtr) { return kernelWeakPtr.lock().get() == kernel; });`
`43`	`43`
`44`	`44`	`if (it == this->printfKernelContainer.end()) {`
`45`		`- this->printfKernelContainer.push_back(kernel);`
	`45`	`+ auto module = static_cast<const ModuleImp >(&static_cast<KernelImp >(kernel)->getParentModule());`
	`46`	`+ this->printfKernelContainer.push_back(module->getPrintfKernelWeakPtr(kernel->toHandle()));`
`46`	`47`	`}`
`47`	`48`	`}`
`48`	`49`
Original file line number	Diff line number	Diff line change
`@@ -231,7 +231,7 @@ struct CommandList : _ze_command_list_handle_t {`
`231`	`231`	`return indirectAllocationsAllowed;`
`232`	`232`	`}`
`233`	`233`
`234`		`- std::vector<Kernel *> &getPrintfKernelContainer() {`
	`234`	`+ std::vector<std::weak_ptr<Kernel>> &getPrintfKernelContainer() {`
`235`	`235`	`return this->printfKernelContainer;`
`236`	`236`	`}`
`237`	`237`
`@@ -360,7 +360,7 @@ struct CommandList : _ze_command_list_handle_t {`
`360`	`360`	`std::map<const void , NEO::GraphicsAllocation > hostPtrMap;`
`361`	`361`	`NEO::PrivateAllocsToReuseContainer ownedPrivateAllocations;`
`362`	`362`	`std::vector<NEO::GraphicsAllocation *> patternAllocations;`
`363`		`- std::vector<Kernel *> printfKernelContainer;`
	`363`	`+ std::vector<std::weak_ptr<Kernel>> printfKernelContainer;`
`364`	`364`
`365`	`365`	`NEO::CommandContainer commandContainer;`
`366`	`366`
Original file line number	Diff line number	Diff line change
`@@ -434,7 +434,10 @@ ze_result_t CommandListCoreFamily<gfxCoreFamily>::appendLaunchKernelIndirect(ze_`
`434`	`434`	`if (hEvent) {`
`435`	`435`	`event = Event::fromHandle(hEvent);`
`436`	`436`	`if (Kernel::fromHandle(kernelHandle)->getPrintfBufferAllocation() != nullptr) {`
`437`		`- event->setKernelForPrintf(Kernel::fromHandle(kernelHandle));`
	`437`	`+ Kernel *kernel = Kernel::fromHandle(kernelHandle);`
	`438`	`+ auto module = static_cast<const ModuleImp >(&static_cast<KernelImp >(kernel)->getParentModule());`
	`439`	`+ event->setKernelForPrintf(module->getPrintfKernelWeakPtr(kernelHandle));`
	`440`	`+ event->setKernelWithPrintfDeviceMutex(kernel->getDevicePrintfKernelMutex());`
`438`	`441`	`}`
`439`	`442`	`launchParams.isHostSignalScopeEvent = event->isSignalScope(ZE_EVENT_SCOPE_FLAG_HOST);`
`440`	`443`	`}`
Original file line number	Diff line number	Diff line change
`@@ -387,7 +387,7 @@ inline ze_result_t CommandListCoreFamilyImmediate<gfxCoreFamily>::executeCommand`
`387`	`387`
`388`	`388`	`this->cmdQImmediate->setTaskCount(completionStamp.taskCount);`
`389`	`389`
`390`		`- if (this->isSyncModeQueue \|\| this->printfKernelContainer.size() > 0u) {`
	`390`	`+ if (this->isSyncModeQueue) {`
`391`	`391`	`status = hostSynchronize(std::numeric_limits<uint64_t>::max(), completionStamp.taskCount, true);`
`392`	`392`	`}`
`393`	`393`
`@@ -1238,9 +1238,11 @@ bool CommandListCoreFamilyImmediate<gfxCoreFamily>::isBarrierRequired() {`
`1238`	`1238`
`1239`	`1239`	`template <GFXCORE_FAMILY gfxCoreFamily>`
`1240`	`1240`	`void CommandListCoreFamilyImmediate<gfxCoreFamily>::printKernelsPrintfOutput(bool hangDetected) {`
`1241`		`- size_t size = this->printfKernelContainer.size();`
`1242`		`- for (size_t i = 0; i < size; i++) {`
`1243`		`- this->printfKernelContainer[i]->printPrintfOutput(hangDetected);`
	`1241`	`+ for (auto &kernelWeakPtr : this->printfKernelContainer) {`
	`1242`	`+ std::lock_guard<std::mutex> lock(static_cast<DeviceImp *>(this->device)->printfKernelMutex);`
	`1243`	`+ if (!kernelWeakPtr.expired()) {`
	`1244`	`+ kernelWeakPtr.lock()->printPrintfOutput(hangDetected);`
	`1245`	`+ }`
`1244`	`1246`	`}`
`1245`	`1247`	`this->printfKernelContainer.clear();`
`1246`	`1248`	`}`
Original file line number	Diff line number	Diff line change
`@@ -271,7 +271,9 @@ ze_result_t CommandListCoreFamily<gfxCoreFamily>::appendLaunchKernelWithParams(K`
`271`	`271`	`}`
`272`	`272`
`273`	`273`	`if (event != nullptr && kernel->getPrintfBufferAllocation() != nullptr) {`
`274`		`- event->setKernelForPrintf(kernel);`
	`274`	`+ auto module = static_cast<const ModuleImp >(&static_cast<KernelImp >(kernel)->getParentModule());`
	`275`	`+ event->setKernelForPrintf(module->getPrintfKernelWeakPtr(kernel->toHandle()));`
	`276`	`+ event->setKernelWithPrintfDeviceMutex(kernel->getDevicePrintfKernelMutex());`
`275`	`277`	`}`
`276`	`278`
`277`	`279`	`if (this->isInOrderExecutionEnabled() && !launchParams.isKernelSplitOperation) {`
Original file line number	Diff line number	Diff line change
`@@ -190,9 +190,11 @@ ze_result_t CommandQueueImp::synchronizeByPollingForTaskCount(uint64_t timeout)`
`190`	`190`	`}`
`191`	`191`
`192`	`192`	`void CommandQueueImp::printKernelsPrintfOutput(bool hangDetected) {`
`193`		`- size_t size = this->printfKernelContainer.size();`
`194`		`- for (size_t i = 0; i < size; i++) {`
`195`		`- this->printfKernelContainer[i]->printPrintfOutput(hangDetected);`
	`193`	`+ for (auto &kernelWeakPtr : this->printfKernelContainer) {`
	`194`	`+ std::lock_guard<std::mutex> lock(static_cast<DeviceImp *>(this->getDevice())->printfKernelMutex);`
	`195`	`+ if (!kernelWeakPtr.expired()) {`
	`196`	`+ kernelWeakPtr.lock()->printPrintfOutput(hangDetected);`
	`197`	`+ }`
`196`	`198`	`}`
`197`	`199`	`this->printfKernelContainer.clear();`
`198`	`200`	`}`