refactor: pass InOrderExecInfo to encode method

Related-To: NEO-7966

Signed-off-by: Dunajski, Bartosz <[email protected]>

Author: BartoszDunajski

level_zero/core/source/cmdlist/cmdlist_hw.h

@@ -171,7 +171,7 @@ struct CommandListCoreFamily : public CommandListImp {
 
     ze_result_t appendSignalEvent(ze_event_handle_t hEvent) override;
     ze_result_t appendWaitOnEvents(uint32_t numEvents, ze_event_handle_t *phEvent, bool relaxedOrderingAllowed, bool trackDependencies, bool apiRequest) override;
-    void appendWaitOnInOrderDependency(std::shared_ptr<InOrderExecInfo> &inOrderExecInfo, uint64_t waitValue, uint32_t offset, bool relaxedOrderingAllowed, bool implicitDependency);
+    void appendWaitOnInOrderDependency(std::shared_ptr<NEO::InOrderExecInfo> &inOrderExecInfo, uint64_t waitValue, uint32_t offset, bool relaxedOrderingAllowed, bool implicitDependency);
     void appendSignalInOrderDependencyCounter(Event *signalEvent);
     void handleInOrderDependencyCounter(Event *signalEvent, bool nonWalkerInOrderCmdsChaining);
 
@@ -340,12 +340,12 @@ struct CommandListCoreFamily : public CommandListImp {
     bool isInOrderNonWalkerSignalingRequired(const Event *event) const;
     bool hasInOrderDependencies() const;
 
-    void addCmdForPatching(std::shared_ptr<InOrderExecInfo> *externalInOrderExecInfo, void *cmd1, void *cmd2, uint64_t counterValue, InOrderPatchCommandHelpers::PatchCmdType patchCmdType);
+    void addCmdForPatching(std::shared_ptr<NEO::InOrderExecInfo> *externalInOrderExecInfo, void *cmd1, void *cmd2, uint64_t counterValue, NEO::InOrderPatchCommandHelpers::PatchCmdType patchCmdType);
 
     bool inOrderAtomicSignallingEnabled() const override;
     uint64_t getInOrderIncrementValue() const;
 
-    InOrderPatchCommandsContainer<GfxFamily> inOrderPatchCmds;
+    NEO::InOrderPatchCommandsContainer<GfxFamily> inOrderPatchCmds;
 
     uint64_t latestHostWaitedInOrderSyncValue = 0;
     bool latestOperationRequiredNonWalkerInOrderCmdsChaining = false;

level_zero/core/source/cmdlist/cmdlist_hw.inl

@@ -20,6 +20,7 @@
 #include "shared/source/helpers/definitions/command_encoder_args.h"
 #include "shared/source/helpers/gfx_core_helper.h"
 #include "shared/source/helpers/hw_info.h"
+#include "shared/source/helpers/in_order_cmd_helpers.h"
 #include "shared/source/helpers/kernel_helpers.h"
 #include "shared/source/helpers/pipe_control_args.h"
 #include "shared/source/helpers/preamble.h"
@@ -46,7 +47,6 @@
 #include "level_zero/core/source/driver/driver_handle_imp.h"
 #include "level_zero/core/source/event/event.h"
 #include "level_zero/core/source/gfx_core_helpers/l0_gfx_core_helper.h"
-#include "level_zero/core/source/helpers/in_order_cmd_helpers.h"
 #include "level_zero/core/source/image/image.h"
 #include "level_zero/core/source/kernel/kernel.h"
 #include "level_zero/core/source/kernel/kernel_imp.h"
@@ -2408,7 +2408,7 @@ ze_result_t CommandListCoreFamily<gfxCoreFamily>::appendSignalEvent(ze_event_han
 }
 
 template <GFXCORE_FAMILY gfxCoreFamily>
-void CommandListCoreFamily<gfxCoreFamily>::appendWaitOnInOrderDependency(std::shared_ptr<InOrderExecInfo> &inOrderExecInfo, uint64_t waitValue, uint32_t offset, bool relaxedOrderingAllowed, bool implicitDependency) {
+void CommandListCoreFamily<gfxCoreFamily>::appendWaitOnInOrderDependency(std::shared_ptr<NEO::InOrderExecInfo> &inOrderExecInfo, uint64_t waitValue, uint32_t offset, bool relaxedOrderingAllowed, bool implicitDependency) {
     using COMPARE_OPERATION = typename GfxFamily::MI_SEMAPHORE_WAIT::COMPARE_OPERATION;
 
     UNRECOVERABLE_IF(waitValue > static_cast<uint64_t>(std::numeric_limits<uint32_t>::max()) && !isQwordInOrderCounter());
@@ -2435,7 +2435,7 @@ void CommandListCoreFamily<gfxCoreFamily>::appendWaitOnInOrderDependency(std::sh
                 auto lri2 = NEO::LriHelper<GfxFamily>::program(commandContainer.getCommandStream(), RegisterOffsets::csGprR0 + 4, getHighPart(waitValue), true);
 
                 if (inOrderExecInfo->isRegularCmdList()) {
-                    addCmdForPatching((implicitDependency ? nullptr : &inOrderExecInfo), lri1, lri2, waitValue, InOrderPatchCommandHelpers::PatchCmdType::Lri64b);
+                    addCmdForPatching((implicitDependency ? nullptr : &inOrderExecInfo), lri1, lri2, waitValue, NEO::InOrderPatchCommandHelpers::PatchCmdType::Lri64b);
                 }
             }
 
@@ -2445,7 +2445,7 @@ void CommandListCoreFamily<gfxCoreFamily>::appendWaitOnInOrderDependency(std::sh
                                                                     false, true, isQwordInOrderCounter(), indirectMode);
 
             if (inOrderExecInfo->isRegularCmdList() && !isQwordInOrderCounter()) {
-                addCmdForPatching((implicitDependency ? nullptr : &inOrderExecInfo), semaphoreCommand, nullptr, waitValue, InOrderPatchCommandHelpers::PatchCmdType::Semaphore);
+                addCmdForPatching((implicitDependency ? nullptr : &inOrderExecInfo), semaphoreCommand, nullptr, waitValue, NEO::InOrderPatchCommandHelpers::PatchCmdType::Semaphore);
             }
         }
 
@@ -2564,7 +2564,7 @@ void CommandListCoreFamily<gfxCoreFamily>::appendSdiInOrderCounterSignalling(uin
     NEO::EncodeStoreMemory<GfxFamily>::programStoreDataImm(miStoreCmd, gpuVa, getLowPart(signalValue), getHighPart(signalValue),
                                                            isQwordInOrderCounter(), (this->partitionCount > 1));
 
-    addCmdForPatching(nullptr, miStoreCmd, nullptr, signalValue, InOrderPatchCommandHelpers::PatchCmdType::Sdi);
+    addCmdForPatching(nullptr, miStoreCmd, nullptr, signalValue, NEO::InOrderPatchCommandHelpers::PatchCmdType::Sdi);
 }
 
 template <GFXCORE_FAMILY gfxCoreFamily>
@@ -3640,7 +3640,7 @@ void CommandListCoreFamily<gfxCoreFamily>::appendWaitOnSingleEvent(Event *event,
 }
 
 template <GFXCORE_FAMILY gfxCoreFamily>
-void CommandListCoreFamily<gfxCoreFamily>::addCmdForPatching(std::shared_ptr<InOrderExecInfo> *externalInOrderExecInfo, void *cmd1, void *cmd2, uint64_t counterValue, InOrderPatchCommandHelpers::PatchCmdType patchCmdType) {
+void CommandListCoreFamily<gfxCoreFamily>::addCmdForPatching(std::shared_ptr<NEO::InOrderExecInfo> *externalInOrderExecInfo, void *cmd1, void *cmd2, uint64_t counterValue, NEO::InOrderPatchCommandHelpers::PatchCmdType patchCmdType) {
     if ((NEO::debugManager.flags.EnableInOrderRegularCmdListPatching.get() != 0) && (this->cmdListType == TYPE_REGULAR)) {
         this->inOrderPatchCmds.emplace_back(externalInOrderExecInfo, cmd1, cmd2, counterValue, patchCmdType);
     }
@@ -3649,7 +3649,7 @@ void CommandListCoreFamily<gfxCoreFamily>::addCmdForPatching(std::shared_ptr<InO
 template <GFXCORE_FAMILY gfxCoreFamily>
 void CommandListCoreFamily<gfxCoreFamily>::patchInOrderCmds() {
     if (isInOrderExecutionEnabled()) {
-        auto implicitAppendCounter = InOrderPatchCommandHelpers::getAppendCounterValue(*inOrderExecInfo);
+        auto implicitAppendCounter = NEO::InOrderPatchCommandHelpers::getAppendCounterValue(*inOrderExecInfo);
 
         for (auto &cmd : inOrderPatchCmds) {
             if (cmd.isExternalDependency() || (inOrderExecInfo->getRegularCmdListSubmissionCounter() > 1)) {

level_zero/core/source/cmdlist/cmdlist_hw_immediate.inl

@@ -20,6 +20,7 @@
 #include "shared/source/helpers/bindless_heaps_helper.h"
 #include "shared/source/helpers/blit_commands_helper.h"
 #include "shared/source/helpers/completion_stamp.h"
+#include "shared/source/helpers/in_order_cmd_helpers.h"
 #include "shared/source/helpers/surface_format_info.h"
 #include "shared/source/memory_manager/internal_allocation_storage.h"
 #include "shared/source/memory_manager/unified_memory_manager.h"
@@ -33,7 +34,6 @@
 #include "level_zero/core/source/driver/driver_handle.h"
 #include "level_zero/core/source/gfx_core_helpers/l0_gfx_core_helper.h"
 #include "level_zero/core/source/helpers/error_code_helper_l0.h"
-#include "level_zero/core/source/helpers/in_order_cmd_helpers.h"
 #include "level_zero/core/source/image/image.h"
 #include "level_zero/core/source/kernel/kernel_imp.h"
 

level_zero/core/source/cmdlist/cmdlist_hw_skl_to_tgllp.inl

@@ -192,7 +192,9 @@ ze_result_t CommandListCoreFamily<gfxCoreFamily>::appendLaunchKernelWithParams(K
     NEO::EncodeDispatchKernelArgs dispatchKernelArgs{
         0,                                                      // eventAddress
         static_cast<uint64_t>(Event::STATE_SIGNALED),           // postSyncImmValue
+        0,                                                      // inOrderCounterValue
         neoDevice,                                              // device
+        nullptr,                                                // inOrderExecInfo
         kernel,                                                 // dispatchInterface
         ssh,                                                    // surfaceStateHeap
         dsh,                                                    // dynamicStateHeap

level_zero/core/source/cmdlist/cmdlist_hw_xehp_and_later.inl

@@ -275,10 +275,27 @@ ze_result_t CommandListCoreFamily<gfxCoreFamily>::appendLaunchKernelWithParams(K
         appendEventForProfilingAllWalkers(compactEvent, true, true);
     }
 
+    bool inOrderExecSignalRequired = (this->isInOrderExecutionEnabled() && !launchParams.isKernelSplitOperation);
+    bool inOrderNonWalkerSignalling = isInOrderNonWalkerSignalingRequired(eventForInOrderExec);
+
+    uint64_t inOrderCounterValue = 0;
+    NEO::InOrderExecInfo *inOrderExecInfo = nullptr;
+
+    if (inOrderExecSignalRequired) {
+        if (inOrderNonWalkerSignalling) {
+            dispatchEventPostSyncOperation(eventForInOrderExec, Event::STATE_CLEARED, false, false, false, false);
+        } else {
+            inOrderCounterValue = this->inOrderExecInfo->getCounterValue() + getInOrderIncrementValue();
+            inOrderExecInfo = this->inOrderExecInfo.get();
+        }
+    }
+
     NEO::EncodeDispatchKernelArgs dispatchKernelArgs{
         eventAddress,                                           // eventAddress
         static_cast<uint64_t>(Event::STATE_SIGNALED),           // postSyncImmValue
+        inOrderCounterValue,                                    // inOrderCounterValue
         neoDevice,                                              // device
+        inOrderExecInfo,                                        // inOrderExecInfo
         kernel,                                                 // dispatchInterface
         ssh,                                                    // surfaceStateHeap
         dsh,                                                    // dynamicStateHeap
@@ -302,21 +319,9 @@ ze_result_t CommandListCoreFamily<gfxCoreFamily>::appendLaunchKernelWithParams(K
         cmdListType == CommandListType::TYPE_IMMEDIATE,         // isKernelDispatchedFromImmediateCmdList
         engineGroupType == NEO::EngineGroupType::renderCompute, // isRcs
         this->dcFlushSupport,                                   // dcFlushEnable
-        this->heaplessModeEnabled                               // isHeaplessModeEnabled
+        this->heaplessModeEnabled,                              // isHeaplessModeEnabled
     };
 
-    bool inOrderExecSignalRequired = (this->isInOrderExecutionEnabled() && !launchParams.isKernelSplitOperation);
-    bool inOrderNonWalkerSignalling = isInOrderNonWalkerSignalingRequired(eventForInOrderExec);
-
-    if (inOrderExecSignalRequired) {
-        if (inOrderNonWalkerSignalling) {
-            dispatchEventPostSyncOperation(eventForInOrderExec, Event::STATE_CLEARED, false, false, false, false);
-        } else {
-            dispatchKernelArgs.eventAddress = inOrderExecInfo->getDeviceCounterAllocation().getGpuAddress() + inOrderExecInfo->getAllocationOffset();
-            dispatchKernelArgs.postSyncImmValue = inOrderExecInfo->getCounterValue() + 1;
-        }
-    }
-
     NEO::EncodeDispatchKernel<GfxFamily>::encodeCommon(commandContainer, dispatchKernelArgs);
 
     if (!this->isFlushTaskSubmissionEnabled) {
@@ -343,7 +348,7 @@ ze_result_t CommandListCoreFamily<gfxCoreFamily>::appendLaunchKernelWithParams(K
             }
         } else {
             UNRECOVERABLE_IF(!dispatchKernelArgs.outWalkerPtr);
-            addCmdForPatching(nullptr, dispatchKernelArgs.outWalkerPtr, nullptr, dispatchKernelArgs.postSyncImmValue, InOrderPatchCommandHelpers::PatchCmdType::Walker);
+            addCmdForPatching(nullptr, dispatchKernelArgs.outWalkerPtr, nullptr, inOrderCounterValue, NEO::InOrderPatchCommandHelpers::PatchCmdType::Walker);
         }
     }
 

level_zero/core/source/cmdlist/cmdlist_imp.cpp

@@ -247,8 +247,8 @@ void CommandListImp::enableInOrderExecution() {
 
     UNRECOVERABLE_IF(!inOrderDependencyCounterAllocation);
 
-    inOrderExecInfo = std::make_shared<InOrderExecInfo>(*inOrderDependencyCounterAllocation, hostCounterAllocation, *device->getMemoryManager(), this->partitionCount,
-                                                        (this->cmdListType == TYPE_REGULAR), inOrderAtomicSignallingEnabled());
+    inOrderExecInfo = std::make_shared<NEO::InOrderExecInfo>(*inOrderDependencyCounterAllocation, hostCounterAllocation, *device->getMemoryManager(), this->partitionCount,
+                                                             (this->cmdListType == TYPE_REGULAR), inOrderAtomicSignallingEnabled());
 }
 
 void CommandListImp::storeReferenceTsToMappedEvents(bool isClearEnabled) {

level_zero/core/source/cmdlist/cmdlist_imp.h

@@ -6,10 +6,10 @@
  */
 
 #pragma once
+#include "shared/source/helpers/in_order_cmd_helpers.h"
 #include "shared/source/os_interface/os_time.h"
 
 #include "level_zero/core/source/cmdlist/cmdlist.h"
-#include "level_zero/core/source/helpers/in_order_cmd_helpers.h"
 
 #include <memory>
 
@@ -39,7 +39,7 @@ struct CommandListImp : public CommandList {
     virtual void patchInOrderCmds() = 0;
 
   protected:
-    std::shared_ptr<InOrderExecInfo> inOrderExecInfo;
+    std::shared_ptr<NEO::InOrderExecInfo> inOrderExecInfo;
 
     ~CommandListImp() override = default;
 

level_zero/core/source/event/event.cpp

@@ -423,7 +423,7 @@ void Event::setIsCompleted() {
     unsetCmdQueue();
 }
 
-void Event::updateInOrderExecState(std::shared_ptr<InOrderExecInfo> &newInOrderExecInfo, uint64_t signalValue, uint32_t allocationOffset) {
+void Event::updateInOrderExecState(std::shared_ptr<NEO::InOrderExecInfo> &newInOrderExecInfo, uint64_t signalValue, uint32_t allocationOffset) {
     resetCompletionStatus();
 
     if (this->inOrderExecInfo.get() != newInOrderExecInfo.get()) {
@@ -435,7 +435,7 @@ void Event::updateInOrderExecState(std::shared_ptr<InOrderExecInfo> &newInOrderE
 }
 
 uint64_t Event::getInOrderExecSignalValueWithSubmissionCounter() const {
-    return (inOrderExecSignalValue + InOrderPatchCommandHelpers::getAppendCounterValue(*inOrderExecInfo));
+    return (inOrderExecSignalValue + NEO::InOrderPatchCommandHelpers::getAppendCounterValue(*inOrderExecInfo));
 }
 
 void Event::setLatestUsedCmdQueue(CommandQueue *newCmdQ) {

level_zero/core/source/event/event.h

@@ -30,6 +30,7 @@ class CommandStreamReceiver;
 class GraphicsAllocation;
 class MultiGraphicsAllocation;
 struct RootDeviceEnvironment;
+class InOrderExecInfo;
 } // namespace NEO
 
 namespace L0 {
@@ -43,7 +44,6 @@ struct DriverHandle;
 struct DriverHandleImp;
 struct Device;
 struct Kernel;
-class InOrderExecInfo;
 
 #pragma pack(1)
 struct IpcEventPoolData {
@@ -238,7 +238,7 @@ struct Event : _ze_event_handle_t {
     void setMetricNotification(MetricCollectorEventNotify *metricNotification) {
         this->metricNotification = metricNotification;
     }
-    void updateInOrderExecState(std::shared_ptr<InOrderExecInfo> &newInOrderExecInfo, uint64_t signalValue, uint32_t allocationOffset);
+    void updateInOrderExecState(std::shared_ptr<NEO::InOrderExecInfo> &newInOrderExecInfo, uint64_t signalValue, uint32_t allocationOffset);
     bool isCounterBased() const { return ((counterBasedMode == CounterBasedMode::ExplicitlyEnabled) || (counterBasedMode == CounterBasedMode::ImplicitlyEnabled)); }
     bool isCounterBasedExplicitlyEnabled() const { return (counterBasedMode == CounterBasedMode::ExplicitlyEnabled); }
     void enableCounterBasedMode(bool apiRequest);
@@ -254,7 +254,7 @@ struct Event : _ze_event_handle_t {
     void setReferenceTs(uint64_t currentCpuTimeStamp);
     const CommandQueue *getLatestUsedCmdQueue() const { return latestUsedCmdQueue; }
     bool hasKerneMappedTsCapability = false;
-    std::shared_ptr<InOrderExecInfo> &getInOrderExecInfo() { return inOrderExecInfo; }
+    std::shared_ptr<NEO::InOrderExecInfo> &getInOrderExecInfo() { return inOrderExecInfo; }
     void enableKmdWaitMode() { kmdWaitMode = true; }
     bool isKmdWaitModeEnabled() const { return kmdWaitMode; }
     void unsetInOrderExecInfo();
@@ -296,7 +296,7 @@ struct Event : _ze_event_handle_t {
     EventPool *eventPool = nullptr;
     std::weak_ptr<Kernel> kernelWithPrintf = std::weak_ptr<Kernel>{};
     std::mutex *kernelWithPrintfDeviceMutex = nullptr;
-    std::shared_ptr<InOrderExecInfo> inOrderExecInfo;
+    std::shared_ptr<NEO::InOrderExecInfo> inOrderExecInfo;
     CommandQueue *latestUsedCmdQueue = nullptr;
 
     uint32_t maxKernelCount = 0;

level_zero/core/source/helpers/CMakeLists.txt

@@ -11,8 +11,6 @@ target_sources(${L0_STATIC_LIB_NAME}
                ${CMAKE_CURRENT_SOURCE_DIR}/error_code_helper_l0.cpp
                ${CMAKE_CURRENT_SOURCE_DIR}/error_code_helper_l0.h
                ${CMAKE_CURRENT_SOURCE_DIR}/implicit_scaling_l0.cpp
-               ${CMAKE_CURRENT_SOURCE_DIR}/in_order_cmd_helpers.cpp
-               ${CMAKE_CURRENT_SOURCE_DIR}/in_order_cmd_helpers.h
                ${CMAKE_CURRENT_SOURCE_DIR}/l0_gfx_core_helper_factory_init.inl
                ${CMAKE_CURRENT_SOURCE_DIR}/l0_populate_factory.h
                ${CMAKE_CURRENT_SOURCE_DIR}/properties_parser.h