Vulkan: moved dynamic buffer allocations to device context (#420)

Author: TheMostDiligent

Graphics/GraphicsEngine/include/BufferBase.hpp

@@ -1,5 +1,5 @@
 /*
- *  Copyright 2019-2022 Diligent Graphics LLC
+ *  Copyright 2019-2025 Diligent Graphics LLC
  *  Copyright 2015-2019 Egor Yusov
  *
  *  Licensed under the Apache License, Version 2.0 (the "License");
@@ -94,13 +94,26 @@ class BufferBase : public DeviceObjectBase<typename EngineImplTraits::BufferInte
         m_pDefaultUAV{nullptr, STDDeleter<BufferViewImplType, TBuffViewObjAllocator>(BuffViewObjAllocator)},
         m_pDefaultSRV{nullptr, STDDeleter<BufferViewImplType, TBuffViewObjAllocator>(BuffViewObjAllocator)}
     {
-        ValidateBufferDesc(this->m_Desc, this->GetDevice());
+        ValidateBufferDesc(this->m_Desc, this->GetDevice()); // May throw
 
         Uint64 DeviceQueuesMask = this->GetDevice()->GetCommandQueueMask();
         DEV_CHECK_ERR((this->m_Desc.ImmediateContextMask & DeviceQueuesMask) != 0,
                       "No bits in the immediate context mask (0x", std::hex, this->m_Desc.ImmediateContextMask,
                       ") correspond to one of ", this->GetDevice()->GetCommandQueueCount(), " available software command queues");
         this->m_Desc.ImmediateContextMask &= DeviceQueuesMask;
+
+        if (this->m_Desc.Usage == USAGE_DYNAMIC)
+        {
+            m_DynamicBufferId = pDevice->AllocateDynamicBufferId();
+        }
+    }
+
+    ~BufferBase()
+    {
+        if (m_DynamicBufferId != ~0u)
+        {
+            this->m_pDevice->RecycleDynamicBufferId(m_DynamicBufferId);
+        }
     }
 
     IMPLEMENT_QUERY_INTERFACE_IN_PLACE(IID_Buffer, TDeviceObjectBase)
@@ -228,6 +241,11 @@ class BufferBase : public DeviceObjectBase<typename EngineImplTraits::BufferInte
         return (this->m_State & State) == State;
     }
 
+    Uint32 GetDynamicBufferId() const
+    {
+        return m_DynamicBufferId;
+    }
+
 protected:
     /// Pure virtual function that creates buffer view for the specific engine implementation.
     virtual void CreateViewInternal(const struct BufferViewDesc& ViewDesc, IBufferView** ppView, bool bIsDefaultView) = 0;
@@ -261,6 +279,9 @@ class BufferBase : public DeviceObjectBase<typename EngineImplTraits::BufferInte
 
     MEMORY_PROPERTIES m_MemoryProperties = MEMORY_PROPERTY_UNKNOWN;
 
+    // Dynamic buffer Id is used by device contexts to index dynamic allocations
+    Uint32 m_DynamicBufferId = ~0u;
+
     /// Default UAV addressing the entire buffer
     std::unique_ptr<BufferViewImplType, STDDeleter<BufferViewImplType, TBuffViewObjAllocator>> m_pDefaultUAV;
 

Graphics/GraphicsEngine/include/RenderDeviceBase.hpp

@@ -32,6 +32,8 @@
 
 #include <atomic>
 #include <thread>
+#include <vector>
+#include <unordered_set>
 
 #include "RenderDevice.h"
 #include "DeviceObjectBase.hpp"
@@ -48,6 +50,7 @@
 #include "STDAllocator.hpp"
 #include "IndexWrapper.hpp"
 #include "ThreadPool.hpp"
+#include "SpinLock.hpp"
 
 namespace Diligent
 {
@@ -211,6 +214,9 @@ class RenderDeviceBase : public ObjectBase<typename EngineImplTraits::RenderDevi
 
     ~RenderDeviceBase()
     {
+        VERIFY(m_RecycledDynamicBufferIds.size() == m_NextDynamicBufferId, "Not all dynamic buffer IDs have been recycled");
+        VERIFY(m_RecycledDynamicBufferIds.size() == m_DbgRecycledDynamicBufferIds.size(),
+               "Recycled dynamic buffer ID set does not match the actual number of recycled IDs. This may happen if there were duplicate IDs or because of a bug.");
     }
 
     IMPLEMENT_QUERY_INTERFACE_IN_PLACE(IID_RenderDevice, ObjectBase<BaseInterface>)
@@ -346,6 +352,33 @@ class RenderDeviceBase : public ObjectBase<typename EngineImplTraits::RenderDevi
         return m_pShaderCompilationThreadPool;
     }
 
+    Uint32 AllocateDynamicBufferId()
+    {
+        Threading::SpinLockGuard Guard{m_RecycledDynamicBufferIdsLock};
+        if (!m_RecycledDynamicBufferIds.empty())
+        {
+            Uint32 Id = m_RecycledDynamicBufferIds.back();
+            m_RecycledDynamicBufferIds.pop_back();
+#ifdef DILIGENT_DEBUG
+            m_DbgRecycledDynamicBufferIds.erase(Id);
+#endif
+            return Id;
+        }
+        else
+        {
+            return m_NextDynamicBufferId.fetch_add(1);
+        }
+    }
+
+    void RecycleDynamicBufferId(Uint32 Id)
+    {
+        Threading::SpinLockGuard Guard{m_RecycledDynamicBufferIdsLock};
+        m_RecycledDynamicBufferIds.push_back(Id);
+#ifdef DILIGENT_DEBUG
+        VERIFY(m_DbgRecycledDynamicBufferIds.emplace(Id).second, "Dynamic buffer ID ", Id, " has already been recycled. This appears to be a bug.");
+#endif
+    }
+
 protected:
     virtual void TestTextureFormat(TEXTURE_FORMAT TexFormat) = 0;
 
@@ -644,6 +677,14 @@ class RenderDeviceBase : public ObjectBase<typename EngineImplTraits::RenderDevi
     RefCntAutoPtr<IThreadPool> m_pShaderCompilationThreadPool;
 
     std::atomic<UniqueIdentifier> m_UniqueId{0};
+
+    // Dynamic buffer Ids are used by device contexts to index dynamic allocations
+    std::atomic<Uint32> m_NextDynamicBufferId{0};
+    Threading::SpinLock m_RecycledDynamicBufferIdsLock;
+    std::vector<Uint32> m_RecycledDynamicBufferIds;
+#ifdef DILIGENT_DEBUG
+    std::unordered_set<Uint32> m_DbgRecycledDynamicBufferIds;
+#endif
 };
 
 } // namespace Diligent

Graphics/GraphicsEngineVulkan/include/BufferVkImpl.hpp

@@ -1,5 +1,5 @@
 /*
- *  Copyright 2019-2022 Diligent Graphics LLC
+ *  Copyright 2019-2025 Diligent Graphics LLC
  *  Copyright 2015-2019 Egor Yusov
  *
  *  Licensed under the Apache License, Version 2.0 (the "License");
@@ -64,31 +64,6 @@ class BufferVkImpl final : public BufferBase<EngineVkImplTraits>
 
     IMPLEMENT_QUERY_INTERFACE_IN_PLACE(IID_BufferVk, TBufferBase)
 
-#ifdef DILIGENT_DEVELOPMENT
-    void DvpVerifyDynamicAllocation(const DeviceContextVkImpl* pCtx) const;
-#endif
-
-    size_t GetDynamicOffset(DeviceContextIndex CtxId, const DeviceContextVkImpl* pCtx) const
-    {
-        if (m_VulkanBuffer != VK_NULL_HANDLE)
-        {
-            return 0;
-        }
-        else
-        {
-            VERIFY(m_Desc.Usage == USAGE_DYNAMIC, "Dynamic buffer is expected");
-            VERIFY_EXPR(!m_DynamicData.empty());
-#ifdef DILIGENT_DEVELOPMENT
-            if (pCtx != nullptr)
-            {
-                DvpVerifyDynamicAllocation(pCtx);
-            }
-#endif
-            auto& DynAlloc = m_DynamicData[CtxId];
-            return DynAlloc.AlignedOffset;
-        }
-    }
-
     /// Implementation of IBufferVk::GetVkBuffer().
     virtual VkBuffer DILIGENT_CALL_TYPE GetVkBuffer() const override final;
 
@@ -136,24 +111,6 @@ class BufferVkImpl final : public BufferBase<EngineVkImplTraits>
     Uint32       m_DynamicOffsetAlignment    = 0;
     VkDeviceSize m_BufferMemoryAlignedOffset = 0;
 
-    // TODO (assiduous): move dynamic allocations to device context.
-    static constexpr size_t CacheLineSize = 64;
-    struct alignas(CacheLineSize) CtxDynamicData : VulkanDynamicAllocation
-    {
-        CtxDynamicData() noexcept {}
-        CtxDynamicData(CtxDynamicData&&) = default;
-
-        CtxDynamicData& operator=(VulkanDynamicAllocation&& Allocation)
-        {
-            *static_cast<VulkanDynamicAllocation*>(this) = std::move(Allocation);
-            return *this;
-        }
-
-        Uint8 Padding[CacheLineSize - sizeof(VulkanDynamicAllocation)] = {};
-    };
-    static_assert(sizeof(CtxDynamicData) == CacheLineSize, "Unexpected sizeof(CtxDynamicData)");
-    std::vector<CtxDynamicData, STDAllocatorRawMem<CtxDynamicData>> m_DynamicData;
-
     VulkanUtilities::BufferWrapper          m_VulkanBuffer;
     VulkanUtilities::VulkanMemoryAllocation m_MemoryAllocation;
 };

Graphics/GraphicsEngineVulkan/include/DeviceContextVkImpl.hpp

@@ -407,6 +407,12 @@ class DeviceContextVkImpl final : public DeviceContextNextGenBase<EngineVkImplTr
 
     QueryManagerVk* GetQueryManager() { return m_pQueryMgr; }
 
+    size_t GetDynamicBufferOffset(const BufferVkImpl* pBuffer, bool VerifyAllocation = true);
+
+#ifdef DILIGENT_DEVELOPMENT
+    void DvpVerifyDynamicAllocation(const BufferVkImpl* pBuffer) const;
+#endif
+
 private:
     void               TransitionRenderTargets(RESOURCE_STATE_TRANSITION_MODE StateTransitionMode);
     __forceinline void CommitRenderPassAndFramebuffer(bool VerifyStates);
@@ -593,8 +599,6 @@ class DeviceContextVkImpl final : public DeviceContextNextGenBase<EngineVkImplTr
     std::vector<std::pair<Uint64, RefCntAutoPtr<FenceVkImpl>>> m_SignalFences;
     std::vector<std::pair<Uint64, RefCntAutoPtr<FenceVkImpl>>> m_WaitFences;
 
-    std::unordered_map<BufferVkImpl*, VulkanUploadAllocation> m_UploadAllocations;
-
     struct MappedTextureKey
     {
         TextureVkImpl* const Texture;
@@ -622,6 +626,16 @@ class DeviceContextVkImpl final : public DeviceContextNextGenBase<EngineVkImplTr
     };
     std::unordered_map<MappedTextureKey, MappedTexture, MappedTextureKey::Hasher> m_MappedTextures;
 
+    struct MappedBuffer
+    {
+        VulkanDynamicAllocation Allocation;
+#ifdef DILIGENT_DEVELOPMENT
+        UniqueIdentifier DvpBufferUID = -1;
+#endif
+    };
+    // NB: using absl::flat_hash_map<const BufferVkImpl*, MappedBuffer> is considerably slower.
+    std::vector<MappedBuffer> m_MappedBuffers;
+
     // Command pools for every queue family
     std::unique_ptr<std::unique_ptr<VulkanUtilities::VulkanCommandBufferPool>[]> m_QueueFamilyCmdPools;
     // Command pool for the family for which we are recording commands

Graphics/GraphicsEngineVulkan/include/ShaderResourceCacheVk.hpp

@@ -1,5 +1,5 @@
 /*
- *  Copyright 2019-2024 Diligent Graphics LLC
+ *  Copyright 2019-2025 Diligent Graphics LLC
  *  Copyright 2015-2019 Egor Yusov
  *
  *  Licensed under the Apache License, Version 2.0 (the "License");
@@ -194,7 +194,7 @@ class ShaderResourceCacheVk : public ShaderResourceCacheBase
 
     void AssignDescriptorSetAllocation(Uint32 SetIndex, DescriptorSetAllocation&& Allocation)
     {
-        auto& DescrSet = GetDescriptorSet(SetIndex);
+        DescriptorSet& DescrSet = GetDescriptorSet(SetIndex);
         VERIFY(DescrSet.GetSize() > 0, "Descriptor set is empty");
         VERIFY(!DescrSet.m_DescriptorSetAllocation, "Descriptor set allocation has already been initialized");
         DescrSet.m_DescriptorSetAllocation = std::move(Allocation);
@@ -260,9 +260,9 @@ class ShaderResourceCacheVk : public ShaderResourceCacheBase
     template <bool VerifyOnly>
     void TransitionResources(DeviceContextVkImpl* pCtxVkImpl);
 
-    __forceinline Uint32 GetDynamicBufferOffsets(DeviceContextIndex     CtxId,
-                                                 std::vector<uint32_t>& Offsets,
-                                                 Uint32                 StartInd) const;
+    Uint32 GetDynamicBufferOffsets(DeviceContextVkImpl*   pCtx,
+                                   std::vector<uint32_t>& Offsets,
+                                   Uint32                 StartInd) const;
 
 private:
     Resource* GetFirstResourcePtr()
@@ -316,83 +316,4 @@ __forceinline auto ShaderResourceCacheVk::Resource::GetDescriptorWriteInfo<Descr
 template <>
 __forceinline auto ShaderResourceCacheVk::Resource::GetDescriptorWriteInfo<DescriptorType::AccelerationStructure>() const { return GetAccelerationStructureWriteInfo(); }
 
-
-__forceinline Uint32 ShaderResourceCacheVk::GetDynamicBufferOffsets(DeviceContextIndex     CtxId,
-                                                                    std::vector<uint32_t>& Offsets,
-                                                                    Uint32                 StartInd) const
-{
-    // If any of the sets being bound include dynamic uniform or storage buffers, then
-    // pDynamicOffsets includes one element for each array element in each dynamic descriptor
-    // type binding in each set. Values are taken from pDynamicOffsets in an order such that
-    // all entries for set N come before set N+1; within a set, entries are ordered by the binding
-    // numbers (unclear if this is SPIRV binding or VkDescriptorSetLayoutBinding number) in the
-    // descriptor set layouts; and within a binding array, elements are in order. (13.2.5)
-
-    // In each descriptor set, all uniform buffers with dynamic offsets (DescriptorType::UniformBufferDynamic)
-    // for every shader stage come first, followed by all storage buffers with dynamic offsets
-    // (DescriptorType::StorageBufferDynamic and DescriptorType::StorageBufferDynamic_ReadOnly) for every shader stage,
-    // followed by all other resources.
-    Uint32 OffsetInd = StartInd;
-    for (Uint32 set = 0; set < m_NumSets; ++set)
-    {
-        const auto& DescrSet = GetDescriptorSet(set);
-        const auto  SetSize  = DescrSet.GetSize();
-
-        Uint32 res = 0;
-        while (res < SetSize)
-        {
-            const auto& Res = DescrSet.GetResource(res);
-            if (Res.Type == DescriptorType::UniformBufferDynamic)
-            {
-                const auto* pBufferVk = Res.pObject.ConstPtr<BufferVkImpl>();
-                // Do not verify dynamic allocation here as there may be some buffers that are not used by the PSO.
-                // The allocations of the buffers that are actually used will be verified by
-                // PipelineResourceSignatureVkImpl::DvpValidateCommittedResource().
-                const auto Offset = pBufferVk != nullptr ? pBufferVk->GetDynamicOffset(CtxId, nullptr /* Do not verify allocation*/) : 0;
-                // The effective offset used for dynamic uniform and storage buffer bindings is the sum of the relative
-                // offset taken from pDynamicOffsets, and the base address of the buffer plus base offset in the descriptor set.
-                // The range of the dynamic uniform and storage buffer bindings is the buffer range as specified in the descriptor set.
-                Offsets[OffsetInd++] = StaticCast<Uint32>(Res.BufferDynamicOffset + Offset);
-                ++res;
-            }
-            else
-                break;
-        }
-
-        while (res < SetSize)
-        {
-            const auto& Res = DescrSet.GetResource(res);
-            if (Res.Type == DescriptorType::StorageBufferDynamic ||
-                Res.Type == DescriptorType::StorageBufferDynamic_ReadOnly)
-            {
-                const auto* pBufferVkView = Res.pObject.ConstPtr<BufferViewVkImpl>();
-                const auto* pBufferVk     = pBufferVkView != nullptr ? pBufferVkView->GetBuffer<const BufferVkImpl>() : nullptr;
-                // Do not verify dynamic allocation here as there may be some buffers that are not used by the PSO.
-                // The allocations of the buffers that are actually used will be verified by
-                // PipelineResourceSignatureVkImpl::DvpValidateCommittedResource().
-                const auto Offset = pBufferVk != nullptr ? pBufferVk->GetDynamicOffset(CtxId, nullptr /* Do not verify allocation*/) : 0;
-                // The effective offset used for dynamic uniform and storage buffer bindings is the sum of the relative
-                // offset taken from pDynamicOffsets, and the base address of the buffer plus base offset in the descriptor set.
-                // The range of the dynamic uniform and storage buffer bindings is the buffer range as specified in the descriptor set.
-                Offsets[OffsetInd++] = StaticCast<Uint32>(Res.BufferDynamicOffset + Offset);
-                ++res;
-            }
-            else
-                break;
-        }
-
-#ifdef DILIGENT_DEBUG
-        for (; res < SetSize; ++res)
-        {
-            const auto& Res = DescrSet.GetResource(res);
-            VERIFY((Res.Type != DescriptorType::UniformBufferDynamic &&
-                    Res.Type != DescriptorType::StorageBufferDynamic &&
-                    Res.Type != DescriptorType::StorageBufferDynamic_ReadOnly),
-                   "All dynamic uniform and storage buffers are expected to go first in the beginning of each descriptor set");
-        }
-#endif
-    }
-    return OffsetInd - StartInd;
-}
-
 } // namespace Diligent