Improve renderpass handling

Author: solidpixel

layer_gpu_timeline/android_build.sh

@@ -67,7 +67,7 @@ cmake \
     -DCMAKE_TOOLCHAIN_FILE="${ANDROID_NDK_HOME}/build/cmake/android.toolchain.cmake" \
     ..
 
-make -j1
+make -j8
 
 popd
 

layer_gpu_timeline/docs/command_buffer_model.md

@@ -21,7 +21,7 @@ performed.
 
 * Pop the latest marker from the queue debug label stack.
 
-**RENDERPASS_BEGIN(const json\*):**
+**RENDER_PASS(const json\*):**
 
 * Set the current workload to a new render pass with the passed metadata.
 

layer_gpu_timeline/source/CMakeLists.txt

@@ -57,6 +57,7 @@ add_library(
 target_include_directories(
     ${VK_LAYER} PRIVATE
         ${PROJECT_SOURCE_DIR}/../source_common
+        ${PROJECT_SOURCE_DIR}/../source_third_party
         ${CMAKE_CURRENT_BINARY_DIR}
         .)
 

layer_gpu_timeline/source/device.hpp

@@ -130,6 +130,14 @@ class Device
      */
     ~Device();
 
+    /**
+     * @brief Callback for sending messages
+     */
+    void onFrame(const std::string& message)
+    {
+        commsWrapper->txMessage(message);
+    }
+
     /**
      * @brief Callback for sending messages
      */

layer_gpu_timeline/source/layer_device_functions_queue.cpp

@@ -27,9 +27,15 @@
 #include <mutex>
 #include <thread>
 
+#include "utils/misc.hpp"
+#include "nlohmann/json.hpp"
+
 #include "device.hpp"
 #include "layer_device_functions.hpp"
-#include "utils/misc.hpp"
+
+using json = nlohmann::json;
+
+using namespace std::placeholders;
 
 extern std::mutex g_vulkanLock;
 
@@ -48,6 +54,15 @@ VKAPI_ATTR VkResult VKAPI_CALL layer_vkQueuePresentKHR<user_tag>(
     auto& tracker = layer->getStateTracker();
     tracker.queuePresent();
 
+    // This is run with the lock held to ensure that all queue submit
+    // messages are sent sequentially to the host tool
+    json frame {
+        { "type", "frame" },
+        { "fid", tracker.totalStats.getFrameCount() }
+    };
+
+    layer->onFrame(frame.dump());
+
     // Release the lock to call into the driver
     lock.unlock();
     return layer->driver.vkQueuePresentKHR(queue, pPresentInfo);
@@ -67,20 +82,20 @@ VKAPI_ATTR VkResult VKAPI_CALL layer_vkQueueSubmit<user_tag>(
     std::unique_lock<std::mutex> lock { g_vulkanLock };
     auto* layer = Device::retrieve(queue);
 
-    using namespace std::placeholders;
     auto onSubmit = std::bind(&Device::onWorkloadSubmit, layer, _1);
 
     auto& tracker = layer->getStateTracker();
     auto& trackQueue = tracker.getQueue(queue);
 
+    // This is run with the lock held to ensure that all queue submit
+    // messages are sent sequentially to the host tool
     for (uint32_t i = 0; i < submitCount; i++)
     {
         const auto& submit = pSubmits[i];
         for (uint32_t j = 0; j < submit.commandBufferCount; j++)
         {
             auto& trackCB = tracker.getCommandBuffer(submit.pCommandBuffers[j]);
             const auto& LCS = trackCB.getSubmitCommandStream();
-
             trackQueue.runSubmitCommandStream(LCS, onSubmit);
         }
     }
@@ -104,6 +119,24 @@ VKAPI_ATTR VkResult VKAPI_CALL layer_vkQueueSubmit2<user_tag>(
     std::unique_lock<std::mutex> lock { g_vulkanLock };
     auto* layer = Device::retrieve(queue);
 
+    auto onSubmit = std::bind(&Device::onWorkloadSubmit, layer, _1);
+
+    auto& tracker = layer->getStateTracker();
+    auto& trackQueue = tracker.getQueue(queue);
+
+    // This is run with the lock held to ensure that all queue submit
+    // messages are sent sequentially to the host tool
+    for (uint32_t i = 0; i < submitCount; i++)
+    {
+        const auto& submit = pSubmits[i];
+        for (uint32_t j = 0; j < submit.commandBufferInfoCount; j++)
+        {
+            auto& trackCB = tracker.getCommandBuffer(submit.pCommandBufferInfos[j].commandBuffer);
+            const auto& LCS = trackCB.getSubmitCommandStream();
+            trackQueue.runSubmitCommandStream(LCS, onSubmit);
+        }
+    }
+
     // Release the lock to call into the driver
     lock.unlock();
     return layer->driver.vkQueueSubmit2(queue, submitCount, pSubmits, fence);
@@ -123,6 +156,24 @@ VKAPI_ATTR VkResult VKAPI_CALL layer_vkQueueSubmit2KHR<user_tag>(
     std::unique_lock<std::mutex> lock { g_vulkanLock };
     auto* layer = Device::retrieve(queue);
 
+    auto onSubmit = std::bind(&Device::onWorkloadSubmit, layer, _1);
+
+    auto& tracker = layer->getStateTracker();
+    auto& trackQueue = tracker.getQueue(queue);
+
+    // This is run with the lock held to ensure that all queue submit
+    // messages are sent sequentially to the host tool
+    for (uint32_t i = 0; i < submitCount; i++)
+    {
+        const auto& submit = pSubmits[i];
+        for (uint32_t j = 0; j < submit.commandBufferInfoCount; j++)
+        {
+            auto& trackCB = tracker.getCommandBuffer(submit.pCommandBufferInfos[j].commandBuffer);
+            const auto& LCS = trackCB.getSubmitCommandStream();
+            trackQueue.runSubmitCommandStream(LCS, onSubmit);
+        }
+    }
+
     // Release the lock to call into the driver
     lock.unlock();
     return layer->driver.vkQueueSubmit2KHR(queue, submitCount, pSubmits, fence);

lgl_host_server.py

@@ -57,14 +57,15 @@ def main():
     print()
 
     # Start it running
-    serverThread = threading.Thread(target=server.run)
+    serverThread = threading.Thread(target=server.run, daemon=True)
     serverThread.start()
 
     # Press to exit
     try:
-        input("Press any key to exit ...")
+        input("Press any key to exit ...\n\n")
     except KeyboardInterrupt:
-        server.stop()
+        print("Exiting ...")
+        sys.exit(0)
 
     return 0
 

lglpy/server.py

@@ -123,7 +123,7 @@ def run(self):
 
         # Accept connections from outside
         while not self.shutdown:
-            print('Waiting for connection')
+            print('Waiting for client connection')
             try:
                 sockfd, _ = listen_sockfd.accept()
             except OSError:

lglpy/service_gpu_timeline.py

@@ -25,18 +25,69 @@
 # implements a basic message endpoint for testing.
 
 from lglpy.server import Message
+import json
+import struct
 
 class GPUTimelineService:
 
     def __init__(self):
-        pass
+        self.frame = {
+            "frame": 0,
+            "workloads": [
+
+            ]
+        }
+
+        # TODO: Make file name configurable
+        self.fileHandle = open('malivision.gputl', 'wb')
 
     def get_service_name(self) -> str:
         return 'GPUTimeline'
 
+    def handle_frame(self, msg):
+        print(json.dumps(self.frame, indent=4))
+
+        # Write frame packet to the file
+        lastFrame = json.dumps(self.frame).encode('utf-8')
+        length = struct.pack('<I', len(lastFrame))
+
+        self.fileHandle.write(length)
+        self.fileHandle.write(lastFrame)
+
+        # Reset the local frame state for the next frame
+        self.frame = {
+            'frame': msg['fid'],
+            'workloads': []
+        }
+
+    def handle_renderpass(self, msg):
+        # Find the last workload
+        lastRenderPass = None
+        if len(self.frame['workloads']):
+            lastWorkload = self.frame['workloads'][-1]
+            if lastWorkload['type'] == 'renderpass':
+                lastRenderPass = lastWorkload
+
+        # Continuation
+        if lastRenderPass and lastRenderPass['tid'] == msg['tid']:
+            lastRenderPass['drawCallCount'] += msg['drawCallCount']
+        # New render pass
+        else:
+            self.frame['workloads'].append(msg)
+
     def handle_message(self, message: Message):
         payload = message.payload.decode('utf-8')
+        parsedPayload = json.loads(payload)
+
+        payloadType = parsedPayload['type']
+
+        if payloadType == 'frame':
+            self.handle_frame(parsedPayload)
+
+        elif payloadType == 'renderpass':
+            self.handle_renderpass(parsedPayload)
 
-        print(f'{message.message_type.name}: {payload} ({len(payload)} bytes)')
+        else:
+            assert False, f'Unknown payload type {payloadType}'
 
         return None

source_common/trackers/command_buffer.cpp

@@ -91,7 +91,7 @@ uint64_t CommandBuffer::renderPassBegin(
     // Assign ID and update the stats tracker for new render passes only
     if (!resuming)
     {
-        tagID = Tracker::LCSWorkload::getTagID();
+        tagID = Tracker::LCSWorkload::assignTagID();
         stats.incRenderPassCount();
     }
 
@@ -105,7 +105,7 @@ uint64_t CommandBuffer::renderPassBegin(
     workloads.push_back(workload);
 
     // Add a command to the layer-side command stream
-    auto instr = std::make_pair(LCSOpcode::RENDERPASS_BEGIN, workload);
+    auto instr = std::make_pair(LCSOpcode::RENDER_PASS, workload);
     workloadCommandStream.push_back(instr);
 
     return tagID;

source_common/trackers/layer_command_stream.cpp

@@ -71,6 +71,7 @@ LCSRenderPass::LCSRenderPass(
 
 /* See header for details. */
 std::string LCSRenderPass::getBeginMetadata(
+    const std::string* debugLabel,
     uint64_t submitID) const
 {
     json metadata = {
@@ -86,6 +87,11 @@ std::string LCSRenderPass::getBeginMetadata(
         metadata["sid"] = submitID;
     }
 
+    if (debugLabel && debugLabel->size())
+    {
+        metadata["label"] = *debugLabel;
+    }
+
     // Default is 1, so only store if we need it
     if (subpassCount != 1)
     {
@@ -126,6 +132,7 @@ std::string LCSRenderPass::getBeginMetadata(
 
 /* See header for details. */
 std::string LCSRenderPass::getContinuationMetadata(
+    const std::string* debugLabel,
     uint64_t tagIDContinuation,
     uint64_t submitID) const
 {
@@ -135,6 +142,11 @@ std::string LCSRenderPass::getContinuationMetadata(
         { "drawCallCount", drawCallCount }
     };
 
+    if (debugLabel && debugLabel->size())
+    {
+        metadata["label"] = *debugLabel;
+    }
+
     if (submitID != 0)
     {
         metadata["sid"] = submitID;
@@ -145,17 +157,18 @@ std::string LCSRenderPass::getContinuationMetadata(
 
 /* See header for details. */
 std::string LCSRenderPass::getMetadata(
+    const std::string* debugLabel,
     uint64_t tagIDContinuation,
     uint64_t submitID) const
 {
     if (tagID)
     {
         assert(tagIDContinuation == 0);
-        return getBeginMetadata(submitID);
+        return getBeginMetadata(debugLabel, submitID);
     }
 
     assert(tagIDContinuation != 0);
-    return getContinuationMetadata(tagIDContinuation, submitID);
+    return getContinuationMetadata(debugLabel, tagIDContinuation, submitID);
 }
 
 }