Archiver: implemented asynchronous pipeline state compilation

Author: TheMostDiligent

Graphics/Archiver/include/SerializedPipelineStateImpl.hpp

@@ -87,9 +87,10 @@ class SerializedPipelineStateImpl final : public ObjectBase<ISerializedPipelineS
 
     UNSUPPORTED_CONST_METHOD(Uint32,                      GetResourceSignatureCount)
     UNSUPPORTED_CONST_METHOD(IPipelineResourceSignature*, GetResourceSignature, Uint32 Index)
-    UNSUPPORTED_METHOD      (PIPELINE_STATE_STATUS,       GetStatus, bool WaitForCompletion)
     // clang-format on
 
+    virtual PIPELINE_STATE_STATUS DILIGENT_CALL_TYPE GetStatus(bool WaitForCompletion = false) override;
+
     virtual Uint32 DILIGENT_CALL_TYPE GetPatchedShaderCount(ARCHIVE_DEVICE_DATA_FLAGS DeviceType) const override final;
 
     virtual ShaderCreateInfo DILIGENT_CALL_TYPE GetPatchedShaderCreateInfo(
@@ -118,10 +119,17 @@ class SerializedPipelineStateImpl final : public ObjectBase<ISerializedPipelineS
         bool DoNotPackSignatures = false;
     };
 
-    const Data& GetData() const { return m_Data; }
-
-    const SerializedData& GetCommonData() const { return m_Data.Common; };
+    const Data& GetData() const
+    {
+        DEV_CHECK_ERR(m_Status.load() == PIPELINE_STATE_STATUS_READY, "Pipeline state '", m_Desc.Name, "' is not ready. Use GetStatus() to check the pipeline state status.");
+        return m_Data;
+    }
 
+    const SerializedData& GetCommonData() const
+    {
+        DEV_CHECK_ERR(m_Status.load() == PIPELINE_STATE_STATUS_READY, "Pipeline state '", m_Desc.Name, "' is not ready. Use GetStatus() to check the pipeline state status.");
+        return m_Data.Common;
+    }
 
     using RayTracingShaderMapType = std::unordered_map<const IShader*, /*Index in TShaderIndices*/ Uint32>;
 
@@ -148,9 +156,17 @@ class SerializedPipelineStateImpl final : public ObjectBase<ISerializedPipelineS
     }
 
     using SignaturesVector = std::vector<RefCntAutoPtr<IPipelineResourceSignature>>;
-    const SignaturesVector& GetSignatures() { return m_Signatures; }
+    const SignaturesVector& GetSignatures()
+    {
+        DEV_CHECK_ERR(m_Status.load() == PIPELINE_STATE_STATUS_READY, "Pipeline state '", m_Desc.Name, "' is not ready. Use GetStatus() to check the pipeline state status.");
+        return m_Signatures;
+    }
 
 private:
+    template <typename PSOCreateInfoType>
+    void Initialize(const PSOCreateInfoType&        CreateInfo,
+                    const PipelineStateArchiveInfo& ArchiveInfo);
+
     template <typename CreateInfoType>
     void PatchShadersVk(const CreateInfoType& CreateInfo) noexcept(false);
 
@@ -185,7 +201,7 @@ class SerializedPipelineStateImpl final : public ObjectBase<ISerializedPipelineS
                                         const ShaderStagesArrayType& ShaderStages,
                                         const ExtraArgsType&... ExtraArgs);
 
-protected:
+private:
     SerializationDeviceImpl* const m_pSerializationDevice;
 
     Data m_Data;
@@ -196,6 +212,9 @@ class SerializedPipelineStateImpl final : public ObjectBase<ISerializedPipelineS
     RefCntAutoPtr<IRenderPass>                     m_pRenderPass;
     RefCntAutoPtr<SerializedResourceSignatureImpl> m_pDefaultSignature;
     SignaturesVector                               m_Signatures;
+
+    std::atomic<PIPELINE_STATE_STATUS> m_Status{PIPELINE_STATE_STATUS_UNINITIALIZED};
+    std::unique_ptr<AsyncInitializer>  m_AsyncInitializer;
 };
 
 #define INSTANTIATE_SERIALIZED_PSO_CTOR(CreateInfoType)                                                             \

Graphics/Archiver/include/SerializedShaderImpl.hpp

@@ -80,6 +80,8 @@ class SerializedShaderImpl final : public ObjectBase<ISerializedShader>
         virtual IShader* GetDeviceShader() = 0;
 
         virtual bool IsCompiling() const = 0;
+
+        virtual RefCntAutoPtr<IAsyncTask> GetCompileTask() const = 0;
     };
 
     template <typename CompiledShaderType>
@@ -104,6 +106,8 @@ class SerializedShaderImpl final : public ObjectBase<ISerializedShader>
         return !(*this == Rhs);
     }
 
+    std::vector<RefCntAutoPtr<IAsyncTask>> GetCompileTasks() const;
+
 private:
     SerializationDeviceImpl* m_pDevice;
     ShaderCreateInfoWrapper  m_CreateInfo;

Graphics/Archiver/src/ArchiverImpl.cpp

@@ -85,7 +85,29 @@ Bool ArchiverImpl::SerializeToBlob(Uint32 ContentVersion, IDataBlob** ppBlob)
     {
         const auto* Name    = pso_it.first.GetName();
         const auto  ResType = pso_it.first.GetType();
-        const auto& SrcPSO  = *pso_it.second;
+        auto&       SrcPSO  = *pso_it.second;
+
+        const PIPELINE_STATE_STATUS PSOStatus = SrcPSO.GetStatus(/*WaitForCompletion = */ true);
+        if (PSOStatus != PIPELINE_STATE_STATUS_READY)
+        {
+            LOG_ERROR_MESSAGE("Pipeline state '", Name, "' is in ", GetPipelineStateStatusString(PSOStatus),
+                              " state and cannot be serialized. Only ready pipeline states can be serialized."
+                              " Use GetStatus() to check the pipeline state status before calling SerializeToBlob().");
+            continue;
+        }
+
+        if (!SrcPSO.GetData().DoNotPackSignatures)
+        {
+            const auto& Signatures = SrcPSO.GetSignatures();
+            for (auto& pSign : Signatures)
+            {
+                if (!AddPipelineResourceSignature(pSign))
+                {
+                    LOG_ERROR_MESSAGE("Failed to add pipeline resource signature '", pSign->GetDesc().Name, "' to the archive.");
+                }
+            }
+        }
+
         const auto& SrcData = SrcPSO.GetData();
         VERIFY_EXPR(SafeStrEqual(Name, SrcPSO.GetDesc().Name));
         VERIFY_EXPR(ResType == PipelineTypeToArchiveResourceType(SrcPSO.GetDesc().PipelineType));
@@ -326,16 +348,6 @@ Bool ArchiverImpl::AddPipelineState(IPipelineState* pPSO)
             Res = false;
     }
 
-    if (!pSerializedPSO->GetData().DoNotPackSignatures)
-    {
-        const auto& Signatures = pSerializedPSO->GetSignatures();
-        for (auto& pSign : Signatures)
-        {
-            if (!AddPipelineResourceSignature(pSign))
-                Res = false;
-        }
-    }
-
     return Res;
 }
 

Graphics/Archiver/src/Archiver_D3D11.cpp

@@ -83,6 +83,11 @@ struct CompiledShaderD3D11 final : SerializedShaderImpl::CompiledShader
     {
         return ShaderD3D11.IsCompiling();
     }
+
+    virtual RefCntAutoPtr<IAsyncTask> GetCompileTask() const override final
+    {
+        return ShaderD3D11.GetCompileTask();
+    }
 };
 
 struct ShaderStageInfoD3D11

Graphics/Archiver/src/Archiver_D3D12.cpp

@@ -72,6 +72,11 @@ struct CompiledShaderD3D12 final : SerializedShaderImpl::CompiledShader
     {
         return ShaderD3D12.IsCompiling();
     }
+
+    virtual RefCntAutoPtr<IAsyncTask> GetCompileTask() const override final
+    {
+        return ShaderD3D12.GetCompileTask();
+    }
 };
 
 inline const ShaderD3D12Impl* GetShaderD3D12(const SerializedShaderImpl* pShader)

Graphics/Archiver/src/Archiver_GL.cpp

@@ -279,6 +279,11 @@ struct CompiledShaderGL final : SerializedShaderImpl::CompiledShader
         return false;
     }
 
+    virtual RefCntAutoPtr<IAsyncTask> GetCompileTask() const override final
+    {
+        return {};
+    }
+
 private:
     static String UnrollSource(const ShaderCreateInfo& CI)
     {

Graphics/Archiver/src/Archiver_Mtl.mm

@@ -309,6 +309,11 @@ virtual bool IsCompiling() const override final
     {
         return ShaderMtl.IsCompiling();
     }
+
+    virtual RefCntAutoPtr<IAsyncTask> GetCompileTask() const override final
+    {
+        return ShaderMtl.GetCompileTask();
+    }
 };
 
 inline const ParsedMSLInfo* GetParsedMsl(const SerializedShaderImpl* pShader, SerializedShaderImpl::DeviceType Type)

Graphics/Archiver/src/Archiver_Vk.cpp

@@ -72,6 +72,11 @@ struct CompiledShaderVk : SerializedShaderImpl::CompiledShader
     {
         return ShaderVk.IsCompiling();
     }
+
+    virtual RefCntAutoPtr<IAsyncTask> GetCompileTask() const override final
+    {
+        return ShaderVk.GetCompileTask();
+    }
 };
 
 inline const ShaderVkImpl* GetShaderVk(const SerializedShaderImpl* pShader)

Graphics/Archiver/src/Archiver_WebGPU.cpp

@@ -76,6 +76,11 @@ struct CompiledShaderWebGPU final : SerializedShaderImpl::CompiledShader
     {
         return ShaderWebGPU.IsCompiling();
     }
+
+    virtual RefCntAutoPtr<IAsyncTask> GetCompileTask() const override final
+    {
+        return ShaderWebGPU.GetCompileTask();
+    }
 };