Submit Semaphore dependency for enqueue read/write without Kernel

BartoszDunajski · Compute-Runtime-Automation · commit 8263d488c603 · 2019-06-25T14:21:57.000+02:00
Change-Id: I22e1743b4cbd6e8285527fdfe25424a6cb3ff462
Signed-off-by: Dunajski, Bartosz &lt;bartosz.dunajski@intel.com&gt;
Related-To: NEO-3020
diff --git a/runtime/command_queue/command_queue.cpp b/runtime/command_queue/command_queue.cpp
@@ -584,26 +584,6 @@ bool CommandQueue::bufferCpuCopyAllowed(Buffer *buffer, cl_command_type commandT
            buffer->isReadWriteOnCpuAllowed(blocking, numEventsInWaitList, ptr, size);
 }
 
-cl_int CommandQueue::enqueueReadWriteBufferWithBlitTransfer(cl_command_type commandType, Buffer *buffer, bool blocking,
-                                                            size_t offset, size_t size, void *ptr, cl_uint numEventsInWaitList,
-                                                            const cl_event *eventWaitList, cl_event *event) {
-    auto blitCommandStreamReceiver = context->getCommandStreamReceiverForBlitOperation(*buffer);
-    EventsRequest eventsRequest(numEventsInWaitList, eventWaitList, event);
-    TimestampPacketContainer previousTimestampPacketNodes;
-    CsrDependencies csrDependencies;
-
-    csrDependencies.fillFromEventsRequestAndMakeResident(eventsRequest, *blitCommandStreamReceiver,
-                                                         CsrDependencies::DependenciesType::All);
-
-    obtainNewTimestampPacketNodes(1, previousTimestampPacketNodes, queueDependenciesClearRequired());
-    csrDependencies.push_back(&previousTimestampPacketNodes);
-
-    auto copyDirection = (CL_COMMAND_WRITE_BUFFER == commandType) ? BlitterConstants::BlitWithHostPtrDirection::FromHostPtr
-                                                                  : BlitterConstants::BlitWithHostPtrDirection::ToHostPtr;
-    blitCommandStreamReceiver->blitWithHostPtr(*buffer, ptr, blocking, offset, size, copyDirection, csrDependencies, *timestampPacketContainer);
-    return CL_SUCCESS;
-}
-
 bool CommandQueue::queueDependenciesClearRequired() const {
     return isOOQEnabled() || DebugManager.flags.OmitTimestampPacketDependencies.get();
 }
diff --git a/runtime/command_queue/command_queue.h b/runtime/command_queue/command_queue.h
@@ -439,10 +439,6 @@ class CommandQueue : public BaseObject<_cl_command_queue> {
     void *enqueueMapMemObject(TransferProperties &transferProperties, EventsRequest &eventsRequest, cl_int &errcodeRet);
     cl_int enqueueUnmapMemObject(TransferProperties &transferProperties, EventsRequest &eventsRequest);
 
-    cl_int enqueueReadWriteBufferWithBlitTransfer(cl_command_type commandType, Buffer *buffer, bool blocking,
-                                                  size_t offset, size_t size, void *ptr, cl_uint numEventsInWaitList,
-                                                  const cl_event *eventWaitList, cl_event *event);
-
     virtual void obtainTaskLevelAndBlockedStatus(unsigned int &taskLevel, cl_uint &numEventsInWaitList, const cl_event *&eventWaitList, bool &blockQueueStatus, unsigned int commandType, bool updateQueueTaskLevel){};
 
     MOCKABLE_VIRTUAL void dispatchAuxTranslation(MultiDispatchInfo &multiDispatchInfo, MemObjsForAuxTranslation &memObjsForAuxTranslation,
diff --git a/runtime/command_queue/command_queue_hw.h b/runtime/command_queue/command_queue_hw.h
@@ -289,6 +289,7 @@ class CommandQueueHw : public CommandQueue {
     void enqueueHandler(Surface **surfacesForResidency,
                         size_t numSurfaceForResidency,
                         bool blocking,
+                        bool blitEnqueue,
                         const MultiDispatchInfo &dispatchInfo,
                         cl_uint numEventsInWaitList,
                         const cl_event *eventWaitList,
@@ -301,7 +302,7 @@ class CommandQueueHw : public CommandQueue {
                         cl_uint numEventsInWaitList,
                         const cl_event *eventWaitList,
                         cl_event *event) {
-        enqueueHandler<enqueueType>(surfacesForResidency, size, blocking, dispatchInfo, numEventsInWaitList, eventWaitList, event);
+        enqueueHandler<enqueueType>(surfacesForResidency, size, blocking, false, dispatchInfo, numEventsInWaitList, eventWaitList, event);
     }
 
     template <uint32_t enqueueType, size_t size>
@@ -376,6 +377,10 @@ class CommandQueueHw : public CommandQueue {
     cl_int enqueueMarkerForReadWriteOperation(MemObj *memObj, void *ptr, cl_command_type commandType, cl_bool blocking, cl_uint numEventsInWaitList,
                                               const cl_event *eventWaitList, cl_event *event);
 
+    cl_int enqueueReadWriteBufferWithBlitTransfer(cl_command_type commandType, Buffer *buffer, bool blocking,
+                                                  size_t offset, size_t size, void *ptr, cl_uint numEventsInWaitList,
+                                                  const cl_event *eventWaitList, cl_event *event);
+
   private:
     bool isTaskLevelUpdateRequired(const uint32_t &taskLevel, const cl_event *eventWaitList, const cl_uint &numEventsInWaitList, unsigned int commandType);
     void obtainTaskLevelAndBlockedStatus(unsigned int &taskLevel, cl_uint &numEventsInWaitList, const cl_event *&eventWaitList, bool &blockQueueStatus, unsigned int commandType, bool updateQueueTaskLevel) override;
diff --git a/runtime/command_queue/command_queue_hw_base.inl b/runtime/command_queue/command_queue_hw_base.inl
@@ -99,4 +99,34 @@ cl_int CommandQueueHw<Family>::enqueueMarkerForReadWriteOperation(MemObj *memObj
     return CL_SUCCESS;
 }
 
+template <typename Family>
+cl_int CommandQueueHw<Family>::enqueueReadWriteBufferWithBlitTransfer(cl_command_type commandType, Buffer *buffer, bool blocking,
+                                                                      size_t offset, size_t size, void *ptr, cl_uint numEventsInWaitList,
+                                                                      const cl_event *eventWaitList, cl_event *event) {
+    auto blitCommandStreamReceiver = context->getCommandStreamReceiverForBlitOperation(*buffer);
+    EventsRequest eventsRequest(numEventsInWaitList, eventWaitList, event);
+    TimestampPacketContainer previousTimestampPacketNodes;
+    CsrDependencies csrDependencies;
+
+    csrDependencies.fillFromEventsRequestAndMakeResident(eventsRequest, *blitCommandStreamReceiver,
+                                                         CsrDependencies::DependenciesType::All);
+
+    obtainNewTimestampPacketNodes(1, previousTimestampPacketNodes, queueDependenciesClearRequired());
+    csrDependencies.push_back(&previousTimestampPacketNodes);
+
+    auto copyDirection = (CL_COMMAND_WRITE_BUFFER == commandType) ? BlitterConstants::BlitWithHostPtrDirection::FromHostPtr
+                                                                  : BlitterConstants::BlitWithHostPtrDirection::ToHostPtr;
+    blitCommandStreamReceiver->blitWithHostPtr(*buffer, ptr, blocking, offset, size, copyDirection, csrDependencies, *timestampPacketContainer);
+
+    MultiDispatchInfo multiDispatchInfo;
+
+    if (CL_COMMAND_WRITE_BUFFER == commandType) {
+        enqueueHandler<CL_COMMAND_WRITE_BUFFER>(nullptr, 0, blocking, true, multiDispatchInfo, numEventsInWaitList, eventWaitList, event);
+    } else {
+        enqueueHandler<CL_COMMAND_READ_BUFFER>(nullptr, 0, blocking, true, multiDispatchInfo, numEventsInWaitList, eventWaitList, event);
+    }
+
+    return CL_SUCCESS;
+}
+
 } // namespace NEO
diff --git a/runtime/command_queue/enqueue_common.h b/runtime/command_queue/enqueue_common.h
@@ -123,12 +123,13 @@ template <uint32_t commandType>
 void CommandQueueHw<GfxFamily>::enqueueHandler(Surface **surfacesForResidency,
                                                size_t numSurfaceForResidency,
                                                bool blocking,
+                                               bool blitEnqueue,
                                                const MultiDispatchInfo &multiDispatchInfo,
                                                cl_uint numEventsInWaitList,
                                                const cl_event *eventWaitList,
                                                cl_event *event) {
-    if (multiDispatchInfo.empty() && !isCommandWithoutKernel(commandType)) {
-        enqueueHandler<CL_COMMAND_MARKER>(surfacesForResidency, numSurfaceForResidency, blocking, multiDispatchInfo,
+    if (multiDispatchInfo.empty() && !isCommandWithoutKernel(commandType) && !blitEnqueue) {
+        enqueueHandler<CL_COMMAND_MARKER>(surfacesForResidency, numSurfaceForResidency, blocking, false, multiDispatchInfo,
                                           numEventsInWaitList, eventWaitList, event);
         if (event) {
             castToObjectOrAbort<Event>(*event)->setCmdType(commandType);
@@ -231,6 +232,9 @@ void CommandQueueHw<GfxFamily>::enqueueHandler(Surface **surfacesForResidency,
         processDispatchForKernels<commandType>(multiDispatchInfo, printfHandler, eventBuilder.getEvent(),
                                                hwTimeStamps, parentKernel, blockQueue, devQueueHw, csrDeps, blockedCommandsData,
                                                previousTimestampPacketNodes, preemption);
+    } else if (blitEnqueue) {
+        auto currentTimestampPacketNode = timestampPacketContainer->peekNodes().at(0);
+        TimestampPacketHelper::programSemaphoreWithImplicitDependency<GfxFamily>(commandStream, *currentTimestampPacketNode);
     } else if (isCacheFlushCommand(commandType)) {
         processDispatchForCacheFlush(surfacesForResidency, numSurfaceForResidency, &commandStream, csrDeps);
     } else if (getCommandStreamReceiver().peekTimestampPacketWriteEnabled()) {
@@ -253,9 +257,9 @@ void CommandQueueHw<GfxFamily>::enqueueHandler(Surface **surfacesForResidency,
             processDeviceEnqueue(parentKernel, devQueueHw, multiDispatchInfo, hwTimeStamps, preemption, blocking);
         }
 
-        auto submissionRequired = !isCommandWithoutKernel(commandType);
+        auto kernelSubmissionRequired = !isCommandWithoutKernel(commandType) && !blitEnqueue;
 
-        if (submissionRequired) {
+        if (kernelSubmissionRequired) {
             completionStamp = enqueueNonBlocked<commandType>(
                 surfacesForResidency,
                 numSurfaceForResidency,
@@ -292,7 +296,7 @@ void CommandQueueHw<GfxFamily>::enqueueHandler(Surface **surfacesForResidency,
                                                       devQueueHw->getDebugQueue());
                 }
             }
-        } else if (isCacheFlushCommand(commandType)) {
+        } else if (isCacheFlushCommand(commandType) || blitEnqueue) {
             completionStamp = enqueueCommandWithoutKernel(
                 surfacesForResidency,
                 numSurfaceForResidency,
diff --git a/runtime/command_queue/gpgpu_walker_base.inl b/runtime/command_queue/gpgpu_walker_base.inl
@@ -320,8 +320,14 @@ size_t EnqueueOperation<GfxFamily>::getTotalSizeRequiredCS(uint32_t eventType, c
         expectedSizeCS += EnqueueOperation<GfxFamily>::getSizeRequiredCS(eventType, reserveProfilingCmdsSpace, reservePerfCounters, commandQueue, &scheduler);
     }
     if (commandQueue.getCommandStreamReceiver().peekTimestampPacketWriteEnabled()) {
-        expectedSizeCS += EnqueueOperation<GfxFamily>::getSizeRequiredForTimestampPacketWrite();
+        bool isReadWriteBufferOperationWithoutKernel = (CL_COMMAND_READ_BUFFER == eventType || CL_COMMAND_WRITE_BUFFER == eventType) &&
+                                                       multiDispatchInfo.empty();
         expectedSizeCS += TimestampPacketHelper::getRequiredCmdStreamSize<GfxFamily>(csrDeps);
+        if (isReadWriteBufferOperationWithoutKernel) {
+            expectedSizeCS += TimestampPacketHelper::getRequiredCmdStreamSizeForNodeDependency<GfxFamily>();
+        } else {
+            expectedSizeCS += EnqueueOperation<GfxFamily>::getSizeRequiredForTimestampPacketWrite();
+        }
     }
     return expectedSizeCS;
 }
diff --git a/runtime/helpers/timestamp_packet.h b/runtime/helpers/timestamp_packet.h
@@ -110,14 +110,19 @@ struct TimestampPacketHelper {
         }
     }
 
+    template <typename GfxFamily>
+    static size_t getRequiredCmdStreamSizeForNodeDependency() {
+        return sizeof(typename GfxFamily::MI_SEMAPHORE_WAIT) + sizeof(typename GfxFamily::MI_ATOMIC);
+    }
+
     template <typename GfxFamily>
     static size_t getRequiredCmdStreamSize(const CsrDependencies &csrDependencies) {
         size_t totalNodesCount = 0;
         for (auto timestampPacketContainer : csrDependencies) {
             totalNodesCount += timestampPacketContainer->peekNodes().size();
         }
 
-        return totalNodesCount * (sizeof(typename GfxFamily::MI_SEMAPHORE_WAIT) + sizeof(typename GfxFamily::MI_ATOMIC));
+        return totalNodesCount * getRequiredCmdStreamSizeForNodeDependency<GfxFamily>();
     }
 };
 } // namespace NEO
diff --git a/unit_tests/command_queue/enqueue_command_without_kernel_tests.cpp b/unit_tests/command_queue/enqueue_command_without_kernel_tests.cpp
@@ -49,7 +49,7 @@ HWTEST_F(EnqueueHandlerTest, GivenCommandStreamWithoutKernelAndZeroSurfacesWhenE
     std::unique_ptr<MockCommandQueueWithCacheFlush<FamilyType>> mockCmdQ(new MockCommandQueueWithCacheFlush<FamilyType>(context, pDevice, 0));
 
     mockCmdQ->commandRequireCacheFlush = true;
-    mockCmdQ->template enqueueHandler<CL_COMMAND_MARKER>(nullptr, 0, false, nullptr, 0, nullptr, nullptr);
+    mockCmdQ->template enqueueHandler<CL_COMMAND_MARKER>(nullptr, 0, false, false, nullptr, 0, nullptr, nullptr);
     EXPECT_EQ(mockCmdQ->getCS(0).getUsed(), 0u);
 }
 HWTEST_F(EnqueueHandlerTest, givenTimestampPacketWriteEnabledAndCommandWithCacheFlushWhenEnqueueingHandlerThenObtainNewStamp) {
@@ -63,7 +63,7 @@ HWTEST_F(EnqueueHandlerTest, givenTimestampPacketWriteEnabledAndCommandWithCache
 
     cl_event event;
 
-    mockCmdQ->template enqueueHandler<CL_COMMAND_MARKER>(nullptr, 0, false, nullptr, 0, nullptr, &event);
+    mockCmdQ->template enqueueHandler<CL_COMMAND_MARKER>(nullptr, 0, false, false, nullptr, 0, nullptr, &event);
     auto node1 = mockCmdQ->timestampPacketContainer->peekNodes().at(0);
     EXPECT_NE(nullptr, node1);
     clReleaseEvent(event);
@@ -79,7 +79,7 @@ HWTEST_F(EnqueueHandlerTest, givenTimestampPacketWriteDisabledAndCommandWithCach
 
     cl_event event;
 
-    mockCmdQ->template enqueueHandler<CL_COMMAND_MARKER>(nullptr, 0, false, nullptr, 0, nullptr, &event);
+    mockCmdQ->template enqueueHandler<CL_COMMAND_MARKER>(nullptr, 0, false, false, nullptr, 0, nullptr, &event);
     auto container = mockCmdQ->timestampPacketContainer.get();
     EXPECT_EQ(nullptr, container);
     clReleaseEvent(event);
diff --git a/unit_tests/command_queue/enqueue_handler_tests.cpp b/unit_tests/command_queue/enqueue_handler_tests.cpp
@@ -67,7 +67,7 @@ HWTEST_F(EnqueueHandlerTest, givenEnqueueHandlerWithKernelSplitWhenAubCsrIsActiv
     auto mockCmdQ = std::unique_ptr<MockCommandQueueHw<FamilyType>>(new MockCommandQueueHw<FamilyType>(context, pDevice, 0));
     MockMultiDispatchInfo multiDispatchInfo(std::vector<Kernel *>({kernel1.mockKernel, kernel2.mockKernel}));
 
-    mockCmdQ->template enqueueHandler<CL_COMMAND_WRITE_BUFFER>(nullptr, 0, true, multiDispatchInfo, 0, nullptr, nullptr);
+    mockCmdQ->template enqueueHandler<CL_COMMAND_WRITE_BUFFER>(nullptr, 0, true, false, multiDispatchInfo, 0, nullptr, nullptr);
 
     EXPECT_TRUE(aubCsr->addAubCommentCalled);
 
@@ -301,6 +301,7 @@ HWTEST_F(EnqueueHandlerTest, enqueueBlockedWithoutReturnEventCreatesVirtualEvent
     mockCmdQ->template enqueueHandler<CL_COMMAND_NDRANGE_KERNEL>(nullptr,
                                                                  0,
                                                                  blocking,
+                                                                 false,
                                                                  multiDispatchInfo,
                                                                  0,
                                                                  nullptr,
@@ -333,6 +334,7 @@ HWTEST_F(EnqueueHandlerTest, enqueueBlockedSetsVirtualEventAsCurrentCmdQVirtualE
     mockCmdQ->template enqueueHandler<CL_COMMAND_NDRANGE_KERNEL>(nullptr,
                                                                  0,
                                                                  blocking,
+                                                                 false,
                                                                  multiDispatchInfo,
                                                                  0,
                                                                  nullptr,
@@ -357,6 +359,7 @@ HWTEST_F(EnqueueHandlerTest, enqueueWithOutputEventRegistersEvent) {
     mockCmdQ->template enqueueHandler<CL_COMMAND_NDRANGE_KERNEL>(nullptr,
                                                                  0,
                                                                  blocking,
+                                                                 false,
                                                                  multiDispatchInfo,
                                                                  0,
                                                                  nullptr,
@@ -433,6 +436,7 @@ HWTEST_F(EnqueueHandlerTest, givenExternallySynchronizedParentEventWhenRequestin
     mockCmdQ->template enqueueHandler<CL_COMMAND_MARKER>(nullptr,
                                                          0,
                                                          blocking,
+                                                         false,
                                                          emptyDispatchInfo,
                                                          1U,
                                                          &inEv,
@@ -458,6 +462,7 @@ HWTEST_F(EnqueueHandlerTest, givenEnqueueHandlerWhenSubCaptureIsOffThenActivateS
     mockCmdQ->template enqueueHandler<CL_COMMAND_NDRANGE_KERNEL>(nullptr,
                                                                  0,
                                                                  false,
+                                                                 false,
                                                                  multiDispatchInfo,
                                                                  0,
                                                                  nullptr,
@@ -480,6 +485,7 @@ HWTEST_F(EnqueueHandlerTest, givenEnqueueHandlerWhenSubCaptureIsOnThenActivateSu
     mockCmdQ->template enqueueHandler<CL_COMMAND_NDRANGE_KERNEL>(nullptr,
                                                                  0,
                                                                  false,
+                                                                 false,
                                                                  multiDispatchInfo,
                                                                  0,
                                                                  nullptr,
@@ -505,6 +511,7 @@ HWTEST_F(EnqueueHandlerTestBasic, givenEnqueueHandlerWhenCommandIsBlokingThenCom
     mockCmdQ->template enqueueHandler<CL_COMMAND_WRITE_BUFFER>(nullptr,
                                                                0,
                                                                true,
+                                                               false,
                                                                multiDispatchInfo,
                                                                0,
                                                                nullptr,
diff --git a/unit_tests/helpers/timestamp_packet_tests.cpp b/unit_tests/helpers/timestamp_packet_tests.cpp
@@ -351,6 +351,11 @@ HWTEST_F(TimestampPacketTests, givenEventsRequestWithEventsWithoutTimestampsWhen
     EXPECT_EQ(expectedSize, TimestampPacketHelper::getRequiredCmdStreamSize<FamilyType>(csrDepsSize3));
 }
 
+HWTEST_F(TimestampPacketTests, whenEstimatingSizeForNodeDependencyThenReturnCorrectValue) {
+    size_t expectedSize = sizeof(typename FamilyType::MI_SEMAPHORE_WAIT) + sizeof(typename FamilyType::MI_ATOMIC);
+    EXPECT_EQ(expectedSize, TimestampPacketHelper::getRequiredCmdStreamSizeForNodeDependency<FamilyType>());
+}
+
 HWCMDTEST_F(IGFX_GEN8_CORE, TimestampPacketTests, givenTimestampPacketWhenDispatchingGpuWalkerThenAddTwoPcForLastWalker) {
     using GPGPU_WALKER = typename FamilyType::GPGPU_WALKER;
     using PIPE_CONTROL = typename FamilyType::PIPE_CONTROL;
diff --git a/unit_tests/mem_obj/buffer_tests.cpp b/unit_tests/mem_obj/buffer_tests.cpp

Original file line number	Diff line number	Diff line change
`@@ -320,8 +320,14 @@ size_t EnqueueOperation<GfxFamily>::getTotalSizeRequiredCS(uint32_t eventType, c`
`320`	`320`	`expectedSizeCS += EnqueueOperation<GfxFamily>::getSizeRequiredCS(eventType, reserveProfilingCmdsSpace, reservePerfCounters, commandQueue, &scheduler);`
`321`	`321`	`}`
`322`	`322`	`if (commandQueue.getCommandStreamReceiver().peekTimestampPacketWriteEnabled()) {`
`323`		`- expectedSizeCS += EnqueueOperation<GfxFamily>::getSizeRequiredForTimestampPacketWrite();`
	`323`	`+ bool isReadWriteBufferOperationWithoutKernel = (CL_COMMAND_READ_BUFFER == eventType \|\| CL_COMMAND_WRITE_BUFFER == eventType) &&`
	`324`	`+ multiDispatchInfo.empty();`
`324`	`325`	`expectedSizeCS += TimestampPacketHelper::getRequiredCmdStreamSize<GfxFamily>(csrDeps);`
	`326`	`+ if (isReadWriteBufferOperationWithoutKernel) {`
	`327`	`+ expectedSizeCS += TimestampPacketHelper::getRequiredCmdStreamSizeForNodeDependency<GfxFamily>();`
	`328`	`+ } else {`
	`329`	`+ expectedSizeCS += EnqueueOperation<GfxFamily>::getSizeRequiredForTimestampPacketWrite();`
	`330`	`+ }`
`325`	`331`	`}`
`326`	`332`	`return expectedSizeCS;`
`327`	`333`	`}`
Original file line number	Diff line number	Diff line change
`@@ -110,14 +110,19 @@ struct TimestampPacketHelper {`
`110`	`110`	`}`
`111`	`111`	`}`
`112`	`112`
	`113`	`+ template <typename GfxFamily>`
	`114`	`+ static size_t getRequiredCmdStreamSizeForNodeDependency() {`
	`115`	`+ return sizeof(typename GfxFamily::MI_SEMAPHORE_WAIT) + sizeof(typename GfxFamily::MI_ATOMIC);`
	`116`	`+ }`
	`117`	`+`
`113`	`118`	`template <typename GfxFamily>`
`114`	`119`	`static size_t getRequiredCmdStreamSize(const CsrDependencies &csrDependencies) {`
`115`	`120`	`size_t totalNodesCount = 0;`
`116`	`121`	`for (auto timestampPacketContainer : csrDependencies) {`
`117`	`122`	`totalNodesCount += timestampPacketContainer->peekNodes().size();`
`118`	`123`	`}`
`119`	`124`
`120`		`- return totalNodesCount * (sizeof(typename GfxFamily::MI_SEMAPHORE_WAIT) + sizeof(typename GfxFamily::MI_ATOMIC));`
	`125`	`+ return totalNodesCount * getRequiredCmdStreamSizeForNodeDependency<GfxFamily>();`
`121`	`126`	`}`
`122`	`127`	`};`
`123`	`128`	`} // namespace NEO`