performance: improve relaxed ordering task count tracking

Signed-off-by: Bartosz Dunajski <[email protected]>

Author: BartoszDunajski

level_zero/core/source/cmdlist/cmdlist_hw_immediate.inl

@@ -1448,16 +1448,13 @@ bool CommandListCoreFamilyImmediate<gfxCoreFamily>::isRelaxedOrderingDispatchAll
         auto queueTaskCount = getCmdQImmediate(copyOffload)->getTaskCount();
         auto csrTaskCount = csr->peekTaskCount();
 
-        if ((this->device->getNEODevice()->isInitDeviceWithFirstSubmissionSupported(csr->getType()) || this->heaplessStateInitEnabled) && csr->peekTaskCount() == 1) {
-            DEBUG_BREAK_IF(queueTaskCount != 0);
-            queueTaskCount = 1;
-        }
+        bool skipTaskCountCheck = (csrTaskCount - queueTaskCount == 1) && csr->isLatestFlushIsTaskCountUpdateOnly();
 
         if (NEO::debugManager.flags.DirectSubmissionRelaxedOrderingCounterHeuristicTreshold.get() != -1) {
             relaxedOrderingCounterThreshold = static_cast<uint32_t>(NEO::debugManager.flags.DirectSubmissionRelaxedOrderingCounterHeuristicTreshold.get());
         }
 
-        if (queueTaskCount == csrTaskCount) {
+        if (queueTaskCount == csrTaskCount || skipTaskCountCheck) {
             relaxedOrderingCounter++;
         } else {
             // Submission from another queue. Reset counter and keep relaxed ordering allowed

level_zero/core/source/cmdqueue/cmdqueue.cpp

@@ -127,7 +127,7 @@ NEO::SubmissionStatus CommandQueueImp::submitBatchBuffer(size_t offset, NEO::Res
 
     NEO::BatchBuffer batchBuffer(this->startingCmdBuffer->getGraphicsAllocation(), offset, 0, 0, nullptr, false,
                                  NEO::getThrottleFromPowerSavingUint(csr->getUmdPowerHintValue()), NEO::QueueSliceCount::defaultSliceCount,
-                                 this->startingCmdBuffer->getUsed(), this->startingCmdBuffer, endingCmdPtr, csr->getNumClients(), true, false, true);
+                                 this->startingCmdBuffer->getUsed(), this->startingCmdBuffer, endingCmdPtr, csr->getNumClients(), true, false, true, false);
     batchBuffer.disableFlatRingBuffer = true;
 
     if (this->startingCmdBuffer != &this->commandStream) {

level_zero/core/test/unit_tests/sources/cmdlist/test_in_order_cmdlist_1.cpp

@@ -2060,6 +2060,33 @@ HWTEST2_F(InOrderCmdListTests, givenRelaxedOrderingEnabledWhenSignalEventCalledT
     verifyFlags(false, true); // relaxed ordering disabled == stalling semaphore
 }
 
+HWTEST2_F(InOrderCmdListTests, givenCounterHeuristicForRelaxedOrderingEnabledWhenSmallTaskIsFlushedThenIncrementCounter, IsAtLeastXeHpcCore) {
+    debugManager.flags.DirectSubmissionRelaxedOrdering.set(1);
+
+    auto ultCsr = static_cast<UltCommandStreamReceiver<FamilyType> *>(device->getNEODevice()->getDefaultEngine().commandStreamReceiver);
+
+    auto directSubmission = new MockDirectSubmissionHw<FamilyType, RenderDispatcher<FamilyType>>(*ultCsr);
+    ultCsr->directSubmission.reset(directSubmission);
+
+    auto immCmdList = createImmCmdList<gfxCoreFamily>();
+    auto queue = immCmdList->getCmdQImmediate(false);
+    EXPECT_EQ(0u, queue->getTaskCount());
+    EXPECT_EQ(0u, immCmdList->relaxedOrderingCounter);
+
+    immCmdList->appendLaunchKernel(kernel->toHandle(), groupCount, nullptr, 0, nullptr, launchParams, false);
+    EXPECT_EQ(1u, immCmdList->relaxedOrderingCounter);
+
+    immCmdList->appendLaunchKernel(kernel->toHandle(), groupCount, nullptr, 0, nullptr, launchParams, false);
+    EXPECT_EQ(2u, immCmdList->relaxedOrderingCounter);
+
+    ultCsr->flushTagUpdate();
+    EXPECT_NE(ultCsr->taskCount, queue->getTaskCount());
+
+    immCmdList->appendLaunchKernel(kernel->toHandle(), groupCount, nullptr, 0, nullptr, launchParams, false);
+    EXPECT_EQ(3u, immCmdList->relaxedOrderingCounter);
+    EXPECT_EQ(ultCsr->taskCount, queue->getTaskCount());
+}
+
 HWTEST2_F(InOrderCmdListTests, givenCounterHeuristicForRelaxedOrderingEnabledWhenAppendingThenEnableRelaxedOrderingCorrectly, IsAtLeastXeHpcCore) {
     debugManager.flags.DirectSubmissionRelaxedOrdering.set(1);
 

shared/source/command_stream/command_stream_receiver.h

@@ -566,6 +566,7 @@ class CommandStreamReceiver {
     bool enqueueWaitForPagingFence(uint64_t pagingFenceValue);
     virtual void unblockPagingFenceSemaphore(uint64_t pagingFenceValue) {}
     MOCKABLE_VIRTUAL void drainPagingFenceQueue();
+    bool isLatestFlushIsTaskCountUpdateOnly() const { return latestFlushIsTaskCountUpdateOnly; }
 
   protected:
     void cleanupResources();
@@ -707,6 +708,7 @@ class CommandStreamReceiver {
     bool heaplessModeEnabled = false;
     bool use4GbHeaps = true;
     bool csrSurfaceProgrammingDone = false;
+    bool latestFlushIsTaskCountUpdateOnly = false;
 };
 
 typedef CommandStreamReceiver *(*CommandStreamReceiverCreateFunc)(bool withAubDump,

shared/source/command_stream/command_stream_receiver_hw_base.inl

@@ -266,7 +266,7 @@ CompletionStamp CommandStreamReceiverHw<GfxFamily>::flushBcsTask(LinearStream &c
     BatchBuffer batchBuffer{streamToSubmit.getGraphicsAllocation(), startOffset, 0, taskStartAddress, nullptr,
                             false, getThrottleFromPowerSavingUint(this->getUmdPowerHintValue()), NEO::QueueSliceCount::defaultSliceCount,
                             streamToSubmit.getUsed(), &streamToSubmit, bbEndLocation, this->getNumClients(), (submitCSR || dispatchBcsFlags.hasStallingCmds),
-                            dispatchBcsFlags.hasRelaxedOrderingDependencies, dispatchBcsFlags.flushTaskCount};
+                            dispatchBcsFlags.hasRelaxedOrderingDependencies, dispatchBcsFlags.flushTaskCount, false};
 
     updateStreamTaskCount(streamToSubmit, taskCount + 1);
     this->latestSentTaskCount = taskCount + 1;
@@ -1124,7 +1124,7 @@ TaskCountType CommandStreamReceiverHw<GfxFamily>::flushBcsTask(const BlitPropert
     uint64_t taskStartAddress = commandStream.getGpuBase() + commandStreamStart;
 
     BatchBuffer batchBuffer{commandStream.getGraphicsAllocation(), commandStreamStart, 0, taskStartAddress, nullptr, false, getThrottleFromPowerSavingUint(this->getUmdPowerHintValue()), QueueSliceCount::defaultSliceCount,
-                            commandStream.getUsed(), &commandStream, endingCmdPtr, this->getNumClients(), hasStallingCmds, isRelaxedOrderingDispatch, blocking};
+                            commandStream.getUsed(), &commandStream, endingCmdPtr, this->getNumClients(), hasStallingCmds, isRelaxedOrderingDispatch, blocking, false};
 
     updateStreamTaskCount(commandStream, newTaskCount);
 
@@ -1258,7 +1258,7 @@ SubmissionStatus CommandStreamReceiverHw<GfxFamily>::flushSmallTask(LinearStream
 
     BatchBuffer batchBuffer{commandStreamTask.getGraphicsAllocation(), commandStreamStartTask, 0, taskStartAddress,
                             nullptr, false, getThrottleFromPowerSavingUint(this->getUmdPowerHintValue()), QueueSliceCount::defaultSliceCount,
-                            commandStreamTask.getUsed(), &commandStreamTask, endingCmdPtr, this->getNumClients(), true, false, true};
+                            commandStreamTask.getUsed(), &commandStreamTask, endingCmdPtr, this->getNumClients(), true, false, true, true};
 
     this->latestSentTaskCount = taskCount + 1;
     auto submissionStatus = flushHandler(batchBuffer, getResidencyAllocations());
@@ -1275,6 +1275,7 @@ SubmissionStatus CommandStreamReceiverHw<GfxFamily>::sendRenderStateCacheFlush()
 
 template <typename GfxFamily>
 inline SubmissionStatus CommandStreamReceiverHw<GfxFamily>::flushHandler(BatchBuffer &batchBuffer, ResidencyContainer &allocationsForResidency) {
+    this->latestFlushIsTaskCountUpdateOnly = batchBuffer.taskCountUpdateOnly;
     auto status = flush(batchBuffer, allocationsForResidency);
     makeSurfacePackNonResident(allocationsForResidency, true);
     return status;
@@ -2219,7 +2220,7 @@ CompletionStamp CommandStreamReceiverHw<GfxFamily>::handleImmediateFlushSendBatc
     BatchBuffer batchBuffer{streamToSubmit.getGraphicsAllocation(), startOffset, chainedBatchBufferStartOffset, taskStartAddress, chainedBatchBuffer,
                             immediateLowPriority, immediateThrottle, immediateSliceCount,
                             streamToSubmit.getUsed(), &streamToSubmit, flushData.endPtr, this->getNumClients(), hasStallingCmds,
-                            dispatchFlags.hasRelaxedOrderingDependencies, dispatchFlags.blockingAppend};
+                            dispatchFlags.hasRelaxedOrderingDependencies, dispatchFlags.blockingAppend, false};
     updateStreamTaskCount(streamToSubmit, taskCount + 1);
 
     auto submissionStatus = flushHandler(batchBuffer, this->getResidencyAllocations());
@@ -2322,7 +2323,7 @@ inline BatchBuffer CommandStreamReceiverHw<GfxFamily>::prepareBatchBufferForSubm
     BatchBuffer batchBuffer{streamToSubmit.getGraphicsAllocation(), startOffset, chainedBatchBufferStartOffset, taskStartAddress, chainedBatchBuffer,
                             dispatchFlags.lowPriority, dispatchFlags.throttle, dispatchFlags.sliceCount,
                             streamToSubmit.getUsed(), &streamToSubmit, bbEndLocation, this->getNumClients(), (submitCSR || dispatchFlags.hasStallingCmds || hasStallingCmdsOnTaskStream),
-                            dispatchFlags.hasRelaxedOrderingDependencies, hasStallingCmdsOnTaskStream};
+                            dispatchFlags.hasRelaxedOrderingDependencies, hasStallingCmdsOnTaskStream, false};
 
     updateStreamTaskCount(streamToSubmit, taskCount + 1);
 

shared/source/command_stream/submissions_aggregator.cpp

@@ -97,13 +97,13 @@ NEO::BatchBuffer::BatchBuffer(GraphicsAllocation *commandBufferAllocation, size_
                               size_t chainedBatchBufferStartOffset, uint64_t taskStartAddress, GraphicsAllocation *chainedBatchBuffer,
                               bool lowPriority, QueueThrottle throttle, uint64_t sliceCount,
                               size_t usedSize, LinearStream *stream, void *endCmdPtr, uint32_t numCsrClients, bool hasStallingCmds,
-                              bool hasRelaxedOrderingDependencies, bool dispatchMonitorFence)
+                              bool hasRelaxedOrderingDependencies, bool dispatchMonitorFence, bool taskCountUpdateOnly)
     : commandBufferAllocation(commandBufferAllocation), startOffset(startOffset),
       chainedBatchBufferStartOffset(chainedBatchBufferStartOffset), taskStartAddress(taskStartAddress), chainedBatchBuffer(chainedBatchBuffer),
       lowPriority(lowPriority),
       throttle(throttle), sliceCount(sliceCount),
       usedSize(usedSize), stream(stream), endCmdPtr(endCmdPtr), numCsrClients(numCsrClients), hasStallingCmds(hasStallingCmds),
-      hasRelaxedOrderingDependencies(hasRelaxedOrderingDependencies), dispatchMonitorFence(dispatchMonitorFence) {}
+      hasRelaxedOrderingDependencies(hasRelaxedOrderingDependencies), dispatchMonitorFence(dispatchMonitorFence), taskCountUpdateOnly(taskCountUpdateOnly) {}
 
 NEO::CommandBuffer::CommandBuffer(Device &device) : device(device) {
     flushStamp.reset(new FlushStampTracker(false));

shared/source/command_stream/submissions_aggregator.h

@@ -43,7 +43,8 @@ struct BatchBuffer {
                 uint32_t numCsrClients,
                 bool hasStallingCmds,
                 bool hasRelaxedOrderingDependencies,
-                bool dispatchMonitorFence);
+                bool dispatchMonitorFence,
+                bool taskCountUpdateOnly);
     BatchBuffer() {}
     GraphicsAllocation *commandBufferAllocation = nullptr;
     ResidencyContainer *allocationsForResidency = nullptr;
@@ -68,6 +69,7 @@ struct BatchBuffer {
     bool hasRelaxedOrderingDependencies = false;
     bool disableFlatRingBuffer = false;
     bool dispatchMonitorFence = false;
+    bool taskCountUpdateOnly = false;
 };
 
 struct CommandBuffer : public IDNode<CommandBuffer> {

shared/test/common/helpers/batch_buffer_helper.h

@@ -1,5 +1,5 @@
 /*
- * Copyright (C) 2022-2023 Intel Corporation
+ * Copyright (C) 2022-2024 Intel Corporation
  *
  * SPDX-License-Identifier: MIT
  *
@@ -27,7 +27,8 @@ struct BatchBufferHelper {
             0,                                  // numCsrClients
             false,                              // hasStallingCmds
             false,                              // hasRelaxedOrderingDependencies
-            false                               // dispatchMonitorFence
+            false,                              // dispatchMonitorFence
+            false                               // taskCountUpdateOnly
         );
     }