Fix and optimize synchronization

Use a fence to fully ensure a RenderProcess is done before re-
using it.

Optimize by removing the GPU-to-CPU synchronization with the
previously used fence that was waited upon right after submitting
draw calls to the draw queue. Use proper subpass dependencies
instead.

Author: janhsimon

src/Headset.cpp

@@ -80,6 +80,33 @@ Headset::Headset(const Context* context) : context(context)
     resolveAttachmentReference.attachment = 2u;
     resolveAttachmentReference.layout = VK_IMAGE_LAYOUT_COLOR_ATTACHMENT_OPTIMAL;
 
+    // This subpass dependency waits with the transition to the color attachment optimal layout that takes place when
+    // calling vkCmdBeginRenderPass() until the draw calls in the render pass are in the appropriate pipeline stages
+    VkSubpassDependency subpassDependencyRenderPassBegin;
+    subpassDependencyRenderPassBegin.dependencyFlags = VK_DEPENDENCY_BY_REGION_BIT;
+    subpassDependencyRenderPassBegin.srcSubpass = VK_SUBPASS_EXTERNAL;
+    subpassDependencyRenderPassBegin.dstSubpass = 0;
+    subpassDependencyRenderPassBegin.srcAccessMask = VK_ACCESS_NONE;
+    subpassDependencyRenderPassBegin.dstAccessMask =
+      VK_ACCESS_COLOR_ATTACHMENT_WRITE_BIT | VK_ACCESS_DEPTH_STENCIL_ATTACHMENT_WRITE_BIT;
+    subpassDependencyRenderPassBegin.srcStageMask =
+      VK_PIPELINE_STAGE_COLOR_ATTACHMENT_OUTPUT_BIT | VK_PIPELINE_STAGE_EARLY_FRAGMENT_TESTS_BIT;
+    subpassDependencyRenderPassBegin.dstStageMask =
+      VK_PIPELINE_STAGE_COLOR_ATTACHMENT_OUTPUT_BIT | VK_PIPELINE_STAGE_EARLY_FRAGMENT_TESTS_BIT;
+
+    // This subpass dependency ensures that all color and depth/stencil attachment writes in the color attachment output
+    // and late fragment tests stages have finished before any draw calls after vkCmdEndRenderPass() begin
+    VkSubpassDependency subpassDependencyRenderPassEnd;
+    subpassDependencyRenderPassEnd.dependencyFlags = VK_DEPENDENCY_BY_REGION_BIT;
+    subpassDependencyRenderPassEnd.srcSubpass = 0;
+    subpassDependencyRenderPassEnd.dstSubpass = VK_SUBPASS_EXTERNAL;
+    subpassDependencyRenderPassEnd.srcAccessMask =
+      VK_ACCESS_COLOR_ATTACHMENT_WRITE_BIT | VK_ACCESS_DEPTH_STENCIL_ATTACHMENT_WRITE_BIT;
+    subpassDependencyRenderPassEnd.dstAccessMask = VK_ACCESS_NONE;
+    subpassDependencyRenderPassEnd.srcStageMask =
+      VK_PIPELINE_STAGE_COLOR_ATTACHMENT_OUTPUT_BIT | VK_PIPELINE_STAGE_LATE_FRAGMENT_TESTS_BIT;
+    subpassDependencyRenderPassEnd.dstStageMask = VK_PIPELINE_STAGE_TOP_OF_PIPE_BIT;
+
     VkSubpassDescription subpassDescription{};
     subpassDescription.pipelineBindPoint = VK_PIPELINE_BIND_POINT_GRAPHICS;
     subpassDescription.colorAttachmentCount = 1u;
@@ -90,12 +117,16 @@ Headset::Headset(const Context* context) : context(context)
     const std::array attachments = { colorAttachmentDescription, depthAttachmentDescription,
                                      resolveAttachmentDescription };
 
+    const std::array subpassDependencies = { subpassDependencyRenderPassBegin, subpassDependencyRenderPassEnd };
+
     VkRenderPassCreateInfo renderPassCreateInfo{ VK_STRUCTURE_TYPE_RENDER_PASS_CREATE_INFO };
     renderPassCreateInfo.pNext = &renderPassMultiviewCreateInfo;
     renderPassCreateInfo.attachmentCount = static_cast<uint32_t>(attachments.size());
     renderPassCreateInfo.pAttachments = attachments.data();
     renderPassCreateInfo.subpassCount = 1u;
     renderPassCreateInfo.pSubpasses = &subpassDescription;
+    renderPassCreateInfo.dependencyCount = static_cast<uint32_t>(subpassDependencies.size());
+    renderPassCreateInfo.pDependencies = subpassDependencies.data();
     if (vkCreateRenderPass(device, &renderPassCreateInfo, nullptr, &renderPass) != VK_SUCCESS)
     {
       util::error(Error::GenericVulkan);

src/MirrorView.cpp

@@ -179,7 +179,7 @@ MirrorView::RenderResult MirrorView::render(uint32_t swapchainImageIndex)
     sourceImageMemoryBarrier.dstAccessMask = VK_ACCESS_TRANSFER_READ_BIT;
     sourceImageMemoryBarrier.subresourceRange.aspectMask = VK_IMAGE_ASPECT_COLOR_BIT;
     sourceImageMemoryBarrier.subresourceRange.layerCount = 1u;
-    sourceImageMemoryBarrier.subresourceRange.baseArrayLayer = mirrorEyeIndex;
+    sourceImageMemoryBarrier.subresourceRange.baseArrayLayer = static_cast<uint32_t>(mirrorEyeIndex);
     sourceImageMemoryBarrier.subresourceRange.levelCount = 1u;
     sourceImageMemoryBarrier.subresourceRange.baseMipLevel = 0u;
 
@@ -229,7 +229,7 @@ MirrorView::RenderResult MirrorView::render(uint32_t swapchainImageIndex)
                               static_cast<int32_t>(cropOffset.y + cropResolution.y), 1 };
   imageBlit.srcSubresource.aspectMask = VK_IMAGE_ASPECT_COLOR_BIT;
   imageBlit.srcSubresource.mipLevel = 0u;
-  imageBlit.srcSubresource.baseArrayLayer = mirrorEyeIndex;
+  imageBlit.srcSubresource.baseArrayLayer = static_cast<uint32_t>(mirrorEyeIndex);
   imageBlit.srcSubresource.layerCount = 1u;
 
   imageBlit.dstOffsets[0] = { 0, 0, 0 };
@@ -253,10 +253,10 @@ MirrorView::RenderResult MirrorView::render(uint32_t swapchainImageIndex)
     sourceImageMemoryBarrier.oldLayout = VK_IMAGE_LAYOUT_TRANSFER_SRC_OPTIMAL;
     sourceImageMemoryBarrier.newLayout = VK_IMAGE_LAYOUT_COLOR_ATTACHMENT_OPTIMAL;
     sourceImageMemoryBarrier.srcAccessMask = VK_ACCESS_TRANSFER_READ_BIT;
-    sourceImageMemoryBarrier.dstAccessMask = VK_ACCESS_NONE;
+    sourceImageMemoryBarrier.dstAccessMask = VK_ACCESS_COLOR_ATTACHMENT_WRITE_BIT | VK_ACCESS_COLOR_ATTACHMENT_READ_BIT;
     sourceImageMemoryBarrier.subresourceRange.aspectMask = VK_IMAGE_ASPECT_COLOR_BIT;
     sourceImageMemoryBarrier.subresourceRange.layerCount = 1u;
-    sourceImageMemoryBarrier.subresourceRange.baseArrayLayer = mirrorEyeIndex;
+    sourceImageMemoryBarrier.subresourceRange.baseArrayLayer = static_cast<uint32_t>(mirrorEyeIndex);
     sourceImageMemoryBarrier.subresourceRange.levelCount = 1u;
     sourceImageMemoryBarrier.subresourceRange.baseMipLevel = 0u;
 

src/RenderProcess.cpp

@@ -69,6 +69,7 @@ RenderProcess::RenderProcess(const Context* context,
 
   // Create a fence
   VkFenceCreateInfo fenceCreateInfo{ VK_STRUCTURE_TYPE_FENCE_CREATE_INFO };
+  fenceCreateInfo.flags = VK_FENCE_CREATE_SIGNALED_BIT; // Make sure the fence starts off signaled
   if (vkCreateFence(device, &fenceCreateInfo, nullptr, &busyFence) != VK_SUCCESS)
   {
     util::error(Error::GenericVulkan);

src/Renderer.cpp

@@ -246,6 +246,20 @@ void Renderer::render(const glm::mat4& cameraMatrix, size_t swapchainImageIndex,
 
   RenderProcess* renderProcess = renderProcesses.at(currentRenderProcessIndex);
 
+  const VkDevice device = context->getVkDevice();
+  const VkFence busyFence = renderProcess->getBusyFence();
+
+  // Wait until the render process is free to use again
+  if (vkWaitForFences(device, 1u, &busyFence, VK_TRUE, UINT64_MAX) != VK_SUCCESS)
+  {
+    return;
+  }
+
+  if (vkResetFences(device, 1u, &busyFence) != VK_SUCCESS)
+  {
+    return;
+  }
+
   const VkCommandBuffer commandBuffer = renderProcess->getCommandBuffer();
   if (vkResetCommandBuffer(commandBuffer, 0u) != VK_SUCCESS)
   {
@@ -375,19 +389,6 @@ void Renderer::submit(bool useSemaphores) const
   {
     return;
   }
-
-  // Now that all draw calls have been submitted to the draw queue, we use a fence to ensure that they are all finished
-  // executing before moving on and giving control back to the OpenXR runtime by calling xrReleaseSwapchainImage()
-  const VkDevice device = context->getVkDevice();
-  if (vkWaitForFences(device, 1u, &busyFence, VK_TRUE, UINT64_MAX) != VK_SUCCESS)
-  {
-    return;
-  }
-
-  if (vkResetFences(device, 1u, &busyFence) != VK_SUCCESS)
-  {
-    return;
-  }
 }
 
 bool Renderer::isValid() const