Add command buffer recorder

Author: solidpixel

generator/vk_codegen/source_CMakeLists.txt

@@ -57,11 +57,6 @@ target_include_directories(
 
 lgl_set_build_options(${VK_LAYER})
 
-target_compile_definitions(
-    ${VK_LAYER} PRIVATE
-        $<$<PLATFORM_ID:Android>:VK_USE_PLATFORM_ANDROID_KHR=1>
-        $<$<PLATFORM_ID:Android>:LGL_LOG_TAG="${LGL_LOG_TAG}">)
-
 target_link_libraries(
     ${VK_LAYER}
         lib_layer_framework

layer_gpu_timeline/docs/command_buffer_model.md

@@ -25,7 +25,7 @@ performed.
 
 * Set the current workload to a new render pass with the passed metadata.
 
-**RENDERPASS_CONTINUE(const json\*):**
+**RENDERPASS_RESUME(const json\*):**
 
 * Update the current workload, which must be a render pass, with extra
   draw count metadata.
@@ -72,13 +72,70 @@ real applications.
 The command stream for a secondary command buffer is inlined into the primary
 command buffer during recording.
 
+###  Recording sequence
+
+When application records a new workload:
+
+  * A `tagID` is assigned and recorded using `vkCmdMarkerBegin()` label in the
+    Vulkan command stream _before_ the new workload is written to the command
+    stream.
+  * If workload is using indirect parameters, then a transfer job to copy
+    indirect parameters into a layer-owned buffer is emitted _before_ the new
+    workload. No additional barrier is needed because application barriers must
+    have already ensured that the indirect parameter buffer is valid.
+  * A proxy workload object is created in the layer storing the assigned
+    `tagID` and all settings that are known at command recording time.
+  * A layer command stream command is recorded into the submit time stream
+    indicating `<TYPE>_BEGIN` with a pointer to the proxy workload. Note that
+    this JSON may be modified later for some workloads.
+  * If workload is using indirect parameters, a layer command stream command is
+    recorded into the resolve time stream, which will handle cleanup and
+    emitting the `submitID.tagID` annex message for the indirect data.
+  * If the command buffer is not ONE_TIME_SUBMIT, if any workload is using
+    indirect parameters, or contains incomplete render passes, the command
+    buffer is marked as needing a `submitID` wrapper.
+  * The user command is written to the Vulkan command stream.
+
+When application resumes a render pass workload:
+
+  * A `tagID` of zero is assigned, but not emitted to the command stream.
+  * A layer command stream command is recorded into the submit time stream
+    indicating `<TYPE>_RESUME` with a pointer to the proxy workload. Note that
+    this JSON may be modified later for some workloads.
+  * The user command is written to the Vulkan command stream.
+
+When application ends a workload:
+
+  * For render pass workloads, any statistics accumulated since the last begin
+    are rolled up into the proxy workload object.
+  * For render pass workloads, the user command is written to the Vulkan
+    command stream.
+  * The command steam label scope is closed using `vkCmdMarkerEnd()`.
+
 ## Layer command playback
 
 The persistent state for command playback belongs to the queues the command
 buffers are submitted to. The command stream bytecode is run by a bytecode
 interpreter associated with the state of the current queue, giving the
 interpreter access to the current `submitID` and queue debug label stack.
 
+###  Submitting sequence
+
+For each command buffer in the user submit:
+
+* If the command buffer needs a `submitID` we allocate a unique `submitID` and
+  create two new command buffers that will wrap the user command buffer with an
+  additional stack layer of debug label containing the `s<ID>` string. We will
+  inject a layer command stream async command to handle freeing the command
+  buffers.
+* The tool will process the submit-time layer commands, executing each command
+  to either update some state or emit
+* If there are any async layer commands, either recorded in the command buffer
+  or from the wrapping command buffers, we will need to add an async handler.
+  This cannot safely use the user fence or depend on any user object lifetime,
+  so we will add a layer-owned timeline semaphore to the submit which we can
+  wait on to determine when it is safe trigger the async work.
+
 ## Future: Async commands
 
 One of our longer-term goals is to be able to capture indirect parameters,

layer_gpu_timeline/source/layer_device_functions.hpp

@@ -27,6 +27,16 @@
 
 #include "framework/utils.hpp"
 
+// Functions for device setup
+
+/* See Vulkan API for documentation. */
+template <>
+VKAPI_ATTR void VKAPI_CALL layer_vkGetDeviceQueue<user_tag>(
+    VkDevice device,
+    uint32_t queueFamilyIndex,
+    uint32_t queueIndex,
+    VkQueue* pQueue);
+
 // Functions for command pools
 
 /* See Vulkan API for documentation. */
@@ -153,6 +163,21 @@ VKAPI_ATTR void VKAPI_CALL layer_vkCmdBeginRenderingKHR<user_tag>(
     VkCommandBuffer commandBuffer,
     const VkRenderingInfo* pRenderingInfo);
 
+/* See Vulkan API for documentation. */
+template <>
+VKAPI_ATTR void VKAPI_CALL layer_vkCmdEndRenderPass<user_tag>(
+    VkCommandBuffer commandBuffer);
+
+/* See Vulkan API for documentation. */
+template <>
+VKAPI_ATTR void VKAPI_CALL  layer_vkCmdEndRendering<user_tag>(
+    VkCommandBuffer commandBuffer);
+
+/* See Vulkan API for documentation. */
+template <>
+VKAPI_ATTR void VKAPI_CALL  layer_vkCmdEndRenderingKHR<user_tag>(
+    VkCommandBuffer commandBuffer);
+
 // Functions for draw calls
 
 /* See Vulkan API for documentation. */

layer_gpu_timeline/source/layer_device_functions_debug.cpp

@@ -44,9 +44,15 @@ VKAPI_ATTR void VKAPI_CALL layer_vkCmdDebugMarkerBeginEXT<user_tag>(
     std::unique_lock<std::mutex> lock { g_vulkanLock };
     auto* layer = Device::retrieve(commandBuffer);
 
-    // Release the lock to call into the driver
-    lock.unlock();
-    layer->driver.vkCmdDebugMarkerBeginEXT(commandBuffer, pMarkerInfo);
+    auto& tracker = layer->getStateTracker();
+    auto& cb = tracker.getCommandBuffer(commandBuffer);
+
+    // Increment the render pass counter in the tracker
+    cb.debugMarkerBegin(pMarkerInfo->pMarkerName);
+
+    // Note that we do not call the driver for user labels - they are
+    // emitted via the comms side-channel for each workload to avoid
+    // polluting the driver tag labelling
 }
 
 /* See Vulkan API for documentation. */
@@ -60,7 +66,13 @@ VKAPI_ATTR void VKAPI_CALL layer_vkCmdDebugMarkerEndEXT<user_tag>(
     std::unique_lock<std::mutex> lock { g_vulkanLock };
     auto* layer = Device::retrieve(commandBuffer);
 
-    // Release the lock to call into the driver
-    lock.unlock();
-    layer->driver.vkCmdDebugMarkerEndEXT(commandBuffer);
+    auto& tracker = layer->getStateTracker();
+    auto& cb = tracker.getCommandBuffer(commandBuffer);
+
+    // Increment the render pass counter in the tracker
+    cb.debugMarkerEnd();
+
+    // Note that we do not call the driver for user labels - they are
+    // emitted via the comms side-channel for each workload to avoid
+    // polluting the driver tag labelling
 }

layer_gpu_timeline/source/layer_device_functions_queue.cpp

@@ -32,6 +32,30 @@
 
 extern std::mutex g_vulkanLock;
 
+/* See Vulkan API for documentation. */
+template <>
+VKAPI_ATTR void VKAPI_CALL layer_vkGetDeviceQueue<user_tag>(
+    VkDevice device,
+    uint32_t queueFamilyIndex,
+    uint32_t queueIndex,
+    VkQueue* pQueue
+) {
+    LAYER_TRACE(__func__);
+
+    // Hold the lock to access layer-wide global store
+    std::unique_lock<std::mutex> lock { g_vulkanLock };
+    auto* layer = Device::retrieve(device);
+
+    // Release the lock to call into the driver
+    lock.unlock();
+    layer->driver.vkGetDeviceQueue(device, queueFamilyIndex, queueIndex, pQueue);
+
+    // Retake the lock to access layer-wide global store
+    lock.lock();
+    auto& tracker = layer->getStateTracker();
+    tracker.createQueue(*pQueue);
+}
+
 /* See Vulkan API for documentation. */
 template<>
 VKAPI_ATTR VkResult VKAPI_CALL layer_vkQueuePresentKHR<user_tag>(