Add compute

Author: solidpixel

layer_gpu_timeline/source/layer_device_functions_command_buffer.cpp

@@ -79,6 +79,7 @@ VKAPI_ATTR VkResult layer_vkBeginCommandBuffer<user_tag>(
     auto& tracker = layer->getStateTracker();
     auto& cmdBuffer = tracker.getCommandBuffer(commandBuffer);
     cmdBuffer.reset();
+    cmdBuffer.begin(pBeginInfo->flags & VK_COMMAND_BUFFER_USAGE_ONE_TIME_SUBMIT_BIT);
 
     // Release the lock to call into the driver
     lock.unlock();

layer_gpu_timeline/source/layer_device_functions_dispatch.cpp

@@ -32,13 +32,16 @@
 
 extern std::mutex g_vulkanLock;
 
-static void registerDispatch(
+static uint64_t registerDispatch(
     Device* layer,
-    VkCommandBuffer commandBuffer
+    VkCommandBuffer commandBuffer,
+    int64_t groupX,
+    int64_t groupY,
+    int64_t groupZ
 ) {
-    auto& state = layer->getStateTracker();
-    auto& stats = state.getCommandBuffer(commandBuffer).getStats();
-    stats.incDispatchCount();
+    auto& tracker = layer->getStateTracker();
+    auto& cb = tracker.getCommandBuffer(commandBuffer);
+    return cb.dispatch(groupX, groupY, groupZ);
 }
 
 /* See Vulkan API for documentation. */
@@ -55,11 +58,27 @@ VKAPI_ATTR void VKAPI_CALL layer_vkCmdDispatch<user_tag>(
     std::unique_lock<std::mutex> lock { g_vulkanLock };
     auto* layer = Device::retrieve(commandBuffer);
 
-    registerDispatch(layer, commandBuffer);
+    uint64_t tagID = registerDispatch(
+        layer,
+        commandBuffer,
+        static_cast<int64_t>(groupCountX),
+        static_cast<int64_t>(groupCountY),
+        static_cast<int64_t>(groupCountZ));
+
+    // Emit the unique workload tag into the command stream
+    std::string tagLabel = formatString("t%" PRIu64, tagID);
+    VkDebugUtilsLabelEXT tagInfo {
+        .sType = VK_STRUCTURE_TYPE_DEBUG_UTILS_LABEL_EXT,
+        .pNext = nullptr,
+        .pLabelName = tagLabel.c_str(),
+        .color = { 0.0f, 0.0f, 0.0f, 0.0f }
+    };
 
     // Release the lock to call into the driver
     lock.unlock();
+    layer->driver.vkCmdBeginDebugUtilsLabelEXT(commandBuffer, &tagInfo);
     layer->driver.vkCmdDispatch(commandBuffer, groupCountX, groupCountY, groupCountZ);
+    layer->driver.vkCmdEndDebugUtilsLabelEXT(commandBuffer);
 }
 
 /* See Vulkan API for documentation. */
@@ -79,11 +98,27 @@ VKAPI_ATTR void VKAPI_CALL layer_vkCmdDispatchBase<user_tag>(
     std::unique_lock<std::mutex> lock { g_vulkanLock };
     auto* layer = Device::retrieve(commandBuffer);
 
-    registerDispatch(layer, commandBuffer);
+    uint64_t tagID = registerDispatch(
+        layer,
+        commandBuffer,
+        static_cast<int64_t>(groupCountX),
+        static_cast<int64_t>(groupCountY),
+        static_cast<int64_t>(groupCountZ));
+
+    // Emit the unique workload tag into the command stream
+    std::string tagLabel = formatString("t%" PRIu64, tagID);
+    VkDebugUtilsLabelEXT tagInfo {
+        .sType = VK_STRUCTURE_TYPE_DEBUG_UTILS_LABEL_EXT,
+        .pNext = nullptr,
+        .pLabelName = tagLabel.c_str(),
+        .color = { 0.0f, 0.0f, 0.0f, 0.0f }
+    };
 
     // Release the lock to call into the driver
     lock.unlock();
+    layer->driver.vkCmdBeginDebugUtilsLabelEXT(commandBuffer, &tagInfo);
     layer->driver.vkCmdDispatchBase(commandBuffer, baseGroupX, baseGroupY, baseGroupZ, groupCountX, groupCountY, groupCountZ);
+    layer->driver.vkCmdEndDebugUtilsLabelEXT(commandBuffer);
 }
 
 /* See Vulkan API for documentation. */
@@ -103,11 +138,27 @@ VKAPI_ATTR void VKAPI_CALL layer_vkCmdDispatchBaseKHR<user_tag>(
     std::unique_lock<std::mutex> lock { g_vulkanLock };
     auto* layer = Device::retrieve(commandBuffer);
 
-    registerDispatch(layer, commandBuffer);
+    uint64_t tagID = registerDispatch(
+        layer,
+        commandBuffer,
+        static_cast<int64_t>(groupCountX),
+        static_cast<int64_t>(groupCountY),
+        static_cast<int64_t>(groupCountZ));
+
+    // Emit the unique workload tag into the command stream
+    std::string tagLabel = formatString("t%" PRIu64, tagID);
+    VkDebugUtilsLabelEXT tagInfo {
+        .sType = VK_STRUCTURE_TYPE_DEBUG_UTILS_LABEL_EXT,
+        .pNext = nullptr,
+        .pLabelName = tagLabel.c_str(),
+        .color = { 0.0f, 0.0f, 0.0f, 0.0f }
+    };
 
     // Release the lock to call into the driver
     lock.unlock();
+    layer->driver.vkCmdBeginDebugUtilsLabelEXT(commandBuffer, &tagInfo);
     layer->driver.vkCmdDispatchBaseKHR(commandBuffer, baseGroupX, baseGroupY, baseGroupZ, groupCountX, groupCountY, groupCountZ);
+    layer->driver.vkCmdEndDebugUtilsLabelEXT(commandBuffer);
 }
 
 /* See Vulkan API for documentation. */
@@ -123,9 +174,20 @@ VKAPI_ATTR void VKAPI_CALL layer_vkCmdDispatchIndirect<user_tag>(
     std::unique_lock<std::mutex> lock { g_vulkanLock };
     auto* layer = Device::retrieve(commandBuffer);
 
-    registerDispatch(layer, commandBuffer);
+    uint64_t tagID = registerDispatch(layer, commandBuffer, -1, -1, -1);
+
+    // Emit the unique workload tag into the command stream
+    std::string tagLabel = formatString("t%" PRIu64, tagID);
+    VkDebugUtilsLabelEXT tagInfo {
+        .sType = VK_STRUCTURE_TYPE_DEBUG_UTILS_LABEL_EXT,
+        .pNext = nullptr,
+        .pLabelName = tagLabel.c_str(),
+        .color = { 0.0f, 0.0f, 0.0f, 0.0f }
+    };
 
     // Release the lock to call into the driver
     lock.unlock();
+    layer->driver.vkCmdBeginDebugUtilsLabelEXT(commandBuffer, &tagInfo);
     layer->driver.vkCmdDispatchIndirect(commandBuffer, buffer, offset);
+    layer->driver.vkCmdEndDebugUtilsLabelEXT(commandBuffer);
 }

layer_gpu_timeline/source/layer_device_functions_render_pass.cpp

@@ -168,7 +168,7 @@ VKAPI_ATTR void VKAPI_CALL layer_vkCmdBeginRenderPass<user_tag>(
 
     // Emit the unique workload tag into the command stream
     std::string tagLabel = formatString("t%" PRIu64, tagID);
-    [[maybe_unused]] VkDebugUtilsLabelEXT tagInfo {
+    VkDebugUtilsLabelEXT tagInfo {
         .sType = VK_STRUCTURE_TYPE_DEBUG_UTILS_LABEL_EXT,
         .pNext = nullptr,
         .pLabelName = tagLabel.c_str(),

lglpy/service_gpu_timeline.py

@@ -33,9 +33,7 @@ class GPUTimelineService:
     def __init__(self):
         self.frame = {
             "frame": 0,
-            "workloads": [
-
-            ]
+            "workloads": []
         }
 
         # TODO: Make file name configurable
@@ -45,8 +43,6 @@ def get_service_name(self) -> str:
         return 'GPUTimeline'
 
     def handle_frame(self, msg):
-        print(json.dumps(self.frame, indent=4))
-
         # Write frame packet to the file
         lastFrame = json.dumps(self.frame).encode('utf-8')
         length = struct.pack('<I', len(lastFrame))
@@ -55,11 +51,15 @@ def handle_frame(self, msg):
         self.fileHandle.write(lastFrame)
 
         # Reset the local frame state for the next frame
+        nextFrame = msg['fid']
         self.frame = {
-            'frame': msg['fid'],
+            'frame': nextFrame,
             'workloads': []
         }
 
+        if nextFrame % 100 == 0:
+            print(f'Starting frame {nextFrame} ...')
+
     def handle_renderpass(self, msg):
         # Find the last workload
         lastRenderPass = None
@@ -70,11 +70,17 @@ def handle_renderpass(self, msg):
 
         # Continuation
         if lastRenderPass and lastRenderPass['tid'] == msg['tid']:
-            lastRenderPass['drawCallCount'] += msg['drawCallCount']
+            # Don't accumulate if tagID is not unique metadata tag
+            if lastRenderPass['drawCallCount'] != -1:
+                lastRenderPass['drawCallCount'] += msg['drawCallCount']
         # New render pass
         else:
             self.frame['workloads'].append(msg)
 
+    def handle_dispatch(self, msg):
+        # Find the last workload
+        self.frame['workloads'].append(msg)
+
     def handle_message(self, message: Message):
         payload = message.payload.decode('utf-8')
         parsedPayload = json.loads(payload)
@@ -87,6 +93,9 @@ def handle_message(self, message: Message):
         elif payloadType == 'renderpass':
             self.handle_renderpass(parsedPayload)
 
+        elif payloadType == 'dispatch':
+            self.handle_dispatch(parsedPayload)
+
         else:
             assert False, f'Unknown payload type {payloadType}'
 

source_common/trackers/command_buffer.cpp

@@ -43,14 +43,21 @@ CommandBuffer::CommandBuffer(
 /* See header for documentation. */
 void CommandBuffer::reset()
 {
+    oneTimeSubmit = false;
     stats.reset();
     workloads.clear();
     workloadCommandStream.clear();
 }
 
-/**
- * @brief Begin a user debug marker range.
- */
+/* See header for documentation. */
+void CommandBuffer::begin(
+    bool _oneTimeSubmit
+)
+{
+    oneTimeSubmit = _oneTimeSubmit;
+}
+
+/* See header for documentation. */
 void CommandBuffer::debugMarkerBegin(
     std::string marker
 ) {
@@ -63,9 +70,7 @@ void CommandBuffer::debugMarkerBegin(
     workloadCommandStream.push_back(instr);
 }
 
-/**
- * @brief End a user debug marker range.
- */
+/* See header for documentation. */
 void CommandBuffer::debugMarkerEnd()
 {
     // Add command with empty workload to update queue debug stack on submit
@@ -74,9 +79,7 @@ void CommandBuffer::debugMarkerEnd()
     workloadCommandStream.push_back(instr);
 }
 
-/**
- * @brief End a user render pass.
- */
+/* See header for documentation. */
 uint64_t CommandBuffer::renderPassBegin(
     const RenderPass& renderPass,
     uint32_t width,
@@ -99,7 +102,7 @@ uint64_t CommandBuffer::renderPassBegin(
     renderPassStartDrawCount = stats.getDrawCallCount();
 
     auto workload = std::make_shared<LCSRenderPass>(
-        tagID, renderPass, width, height, suspending);
+        tagID, renderPass, width, height, suspending, oneTimeSubmit);
 
     currentRenderPass = workload;
     workloads.push_back(workload);
@@ -111,9 +114,7 @@ uint64_t CommandBuffer::renderPassBegin(
     return tagID;
 }
 
-/**
- * @brief End a user render pass.
- */
+/* See header for documentation. */
 bool CommandBuffer::renderPassEnd()
 {
     assert(currentRenderPass);
@@ -130,6 +131,28 @@ bool CommandBuffer::renderPassEnd()
     return suspending;
 }
 
+/* See header for documentation. */
+uint64_t CommandBuffer::dispatch(
+    int64_t xGroups,
+    int64_t yGroups,
+    int64_t zGroups
+) {
+    LAYER_LOG("Creating LCSDispatch workload");
+    uint64_t tagID = Tracker::LCSWorkload::assignTagID();
+    stats.incDispatchCount();
+
+    // Add a workload to the render pass
+    auto workload = std::make_shared<LCSDispatch>(
+        tagID, xGroups, yGroups, zGroups);
+    workloads.push_back(workload);
+
+    // Add a command to the layer-side command stream
+    auto instr = std::make_pair(LCSOpcode::DISPATCH, workload);
+    workloadCommandStream.push_back(instr);
+
+    return tagID;
+}
+
 /* See header for documentation. */
 void CommandBuffer::executeCommands(
     CommandBuffer& secondary
@@ -142,7 +165,7 @@ void CommandBuffer::executeCommands(
     vecAppend(workloadCommandStream, secondary.workloadCommandStream);
 }
 
-
+/* See header for documentation. */
 CommandPool::CommandPool(
     VkCommandPool _handle) :
     handle(_handle)

source_common/trackers/command_buffer.hpp

@@ -114,6 +114,20 @@ class CommandBuffer
      */
     bool renderPassEnd();
 
+    /**
+     * @brief Capture a compute dispatch.
+     *
+     * @param xGroups   Number of groups in X dimension, or -1 if unknown.
+     * @param yGroups   Number of groups in Y dimension, or -1 if unknown.
+     * @param zGroups   Number of groups in Z dimension, or -1 if unknown.
+     *
+     * @return Returns the tagID assigned to this workload.
+     */
+    uint64_t dispatch(
+        int64_t xGroups,
+        int64_t yGroups,
+        int64_t zGroups);
+
     /**
      * @brief Begin a user debug marker range.
      */
@@ -136,12 +150,25 @@ class CommandBuffer
      */
     void reset();
 
+    /**
+     * @brief Begin recording back into the @a Recording state.
+     *
+     * @param oneTimeSubmit   Is this a one-time submit recording.
+     */
+    void begin(
+        bool oneTimeSubmit);
+
 private:
     /**
      * @brief The Vulkan API handle of this command buffer.
      */
     const VkCommandBuffer handle;
 
+    /**
+     * @brief Is this command buffer recording one-time-submit?
+     */
+    bool oneTimeSubmit { false };
+
     /**
      * @brief The command buffer draw count at the start of the render pass.
      */