ArchiverGL: implemented async shader source processing

TheMostDiligent · TheMostDiligent · commit 986c9d6a3806 · 2024-08-15T00:03:37.000-07:00
diff --git a/Graphics/Archiver/include/SerializationDeviceImpl.hpp b/Graphics/Archiver/include/SerializationDeviceImpl.hpp
@@ -172,7 +172,7 @@ class SerializationDeviceImpl final : public RenderDeviceBase<SerializationEngin
     const VkProperties&    GetVkProperties() const { return m_VkProps; }
     const MtlProperties&   GetMtlProperties() const { return m_MtlProps; }
 
-    IRenderDevice* GetRenderDevice(RENDER_DEVICE_TYPE Type)
+    IRenderDevice* GetRenderDevice(RENDER_DEVICE_TYPE Type) const
     {
         return m_RenderDevices[Type];
     }
diff --git a/Graphics/Archiver/include/SerializedShaderImpl.hpp b/Graphics/Archiver/include/SerializedShaderImpl.hpp
@@ -79,6 +79,12 @@ class SerializedShaderImpl final : public ObjectBase<ISerializedShader>
 
         virtual IShader* GetDeviceShader() = 0;
 
+        virtual SHADER_STATUS GetStatus(bool WaitForCompletion)
+        {
+            IShader* pShader = GetDeviceShader();
+            return pShader != nullptr ? pShader->GetStatus(WaitForCompletion) : SHADER_STATUS_UNINITIALIZED;
+        }
+
         virtual bool IsCompiling() const = 0;
 
         virtual RefCntAutoPtr<IAsyncTask> GetCompileTask() const = 0;
diff --git a/Graphics/Archiver/src/ArchiverImpl.cpp b/Graphics/Archiver/src/ArchiverImpl.cpp
@@ -193,7 +193,7 @@ Bool ArchiverImpl::SerializeToBlob(Uint32 ContentVersion, IDataBlob** ppBlob)
         const auto* Name      = shader_it.first.GetStr();
         auto&       SrcShader = *shader_it.second;
         {
-            const SHADER_STATUS Status = SrcShader.GetStatus(/*WaitForCompletion = */ false);
+            const SHADER_STATUS Status = SrcShader.GetStatus(/*WaitForCompletion = */ true);
             if (Status != SHADER_STATUS_READY)
             {
                 LOG_ERROR_MESSAGE("Shader '", Name, "' is in ", GetShaderStatusString(Status),
diff --git a/Graphics/Archiver/src/Archiver_GL.cpp b/Graphics/Archiver/src/Archiver_GL.cpp
@@ -206,56 +206,90 @@ static void AppendGLES30Stubs(std::string& Source)
 
 #endif
 
-struct CompiledShaderGL final : SerializedShaderImpl::CompiledShader
+class CompiledShaderGL final : public SerializedShaderImpl::CompiledShader
 {
-    String                 UnrolledSource;
-    RefCntAutoPtr<IShader> pShaderGL;
-    bool                   IsOptimized = false;
-
-    CompiledShaderGL(IReferenceCounters*                          pRefCounters,
-                     const ShaderCreateInfo&                      ShaderCI,
-                     const ShaderGLImpl::CreateInfo&              GLShaderCI,
-                     IRenderDevice*                               pRenderDeviceGL,
-                     RENDER_DEVICE_TYPE                           DeviceType,
-                     const SerializationDeviceImpl::GLProperties& GLProps)
+public:
+    CompiledShaderGL(IReferenceCounters*             pRefCounters,
+                     const ShaderCreateInfo&         ShaderCI,
+                     const ShaderGLImpl::CreateInfo& GLShaderCI,
+                     const SerializationDeviceImpl*  pSerializationDevice,
+                     RENDER_DEVICE_TYPE              DeviceType) :
+        m_pSerializationDevice{pSerializationDevice},
+        m_ShaderCI{ShaderCI, GetRawAllocator()},
+        m_GLShaderCI{GLShaderCI},
+        m_DeviceType{DeviceType}
     {
+        IThreadPool* pCompilationThreadPool = pSerializationDevice->GetShaderCompilationThreadPool();
+
+        m_Status.store(SHADER_STATUS_COMPILING);
+        if (pCompilationThreadPool == nullptr || (ShaderCI.CompileFlags & SHADER_COMPILE_FLAG_ASYNCHRONOUS) == 0)
+        {
+            InitializeSource();
+            CreateGLShader();
+        }
+        else
+        {
+            this->m_AsyncInitializer = AsyncInitializer::Start(
+                pCompilationThreadPool,
+                [this](Uint32 ThreadId) //
+                {
+                    try
+                    {
+                        InitializeSource();
+                    }
+                    catch (...)
+                    {
+                        m_Status.store(SHADER_STATUS_FAILED);
+                    }
+                });
+        }
+    }
+
+    void InitializeSource()
+    {
+        const SerializationDeviceImpl::GLProperties& GLProps = m_pSerializationDevice->GetGLProperties();
         if (GLProps.OptimizeShaders)
         {
-            UnrolledSource = TransformSource(ShaderCI, GLShaderCI, DeviceType, GLProps);
-            IsOptimized    = !UnrolledSource.empty();
+            m_UnrolledSource = TransformSource(m_ShaderCI, m_GLShaderCI, m_DeviceType, GLProps);
+            m_IsOptimized    = !m_UnrolledSource.empty();
         }
-        if (UnrolledSource.empty())
+        if (m_UnrolledSource.empty())
         {
-            UnrolledSource = UnrollSource(ShaderCI);
+            m_UnrolledSource = UnrollSource(m_ShaderCI);
         }
-        VERIFY_EXPR(!UnrolledSource.empty());
+        VERIFY_EXPR(!m_UnrolledSource.empty());
+
+        m_Status.store(SHADER_STATUS_READY);
+    }
 
+    void CreateGLShader()
+    {
         // Use serialization CI to be consistent with what will be saved in the archive.
-        const auto SerializationCI = GetSerializationCI(ShaderCI);
-        if (pRenderDeviceGL)
+        const ShaderCreateInfo SerializationCI = GetSerializationCI(m_ShaderCI);
+        if (IRenderDevice* pRenderDeviceGL = m_pSerializationDevice->GetRenderDevice(m_DeviceType))
         {
             // GL shader must be created through the render device as GL functions
             // are not loaded by the archiver.
-            pRenderDeviceGL->CreateShader(SerializationCI, &pShaderGL);
-            if (!pShaderGL)
-                LOG_ERROR_AND_THROW("Failed to create GL shader '", (ShaderCI.Desc.Name ? ShaderCI.Desc.Name : ""), "'.");
+            pRenderDeviceGL->CreateShader(SerializationCI, &m_pShaderGL);
+            if (!m_pShaderGL)
+                LOG_ERROR_AND_THROW("Failed to create GL shader '", (m_ShaderCI.Get().Desc.Name ? m_ShaderCI.Get().Desc.Name : ""), "'.");
         }
         else
         {
-            pShaderGL = NEW_RC_OBJ(GetRawAllocator(), "Shader instance", ShaderGLImpl)(nullptr, SerializationCI, GLShaderCI, true /*bIsDeviceInternal*/);
+            m_pShaderGL = NEW_RC_OBJ(GetRawAllocator(), "Shader instance", ShaderGLImpl)(nullptr, SerializationCI, m_GLShaderCI, true /*bIsDeviceInternal*/);
         }
     }
 
     ShaderCreateInfo GetSerializationCI(ShaderCreateInfo ShaderCI) const
     {
         ShaderCI.FilePath       = nullptr;
         ShaderCI.ByteCode       = nullptr;
-        ShaderCI.Source         = UnrolledSource.c_str();
-        ShaderCI.SourceLength   = UnrolledSource.length();
+        ShaderCI.Source         = m_UnrolledSource.c_str();
+        ShaderCI.SourceLength   = m_UnrolledSource.length();
         ShaderCI.ShaderCompiler = SHADER_COMPILER_DEFAULT;
         ShaderCI.Macros         = {}; // Macros are inlined into unrolled source
 
-        if (IsOptimized)
+        if (m_IsOptimized)
         {
             ShaderCI.SourceLanguage = SHADER_SOURCE_LANGUAGE_GLSL;
             ShaderCI.EntryPoint     = "main";
@@ -271,17 +305,42 @@ struct CompiledShaderGL final : SerializedShaderImpl::CompiledShader
 
     virtual IShader* GetDeviceShader() override final
     {
-        return pShaderGL;
+        if (m_pShaderGL == nullptr)
+        {
+            if (m_Status.load() == SHADER_STATUS_READY)
+            {
+                try
+                {
+                    CreateGLShader();
+                }
+                catch (...)
+                {
+                    m_Status.store(SHADER_STATUS_FAILED);
+                }
+            }
+        }
+        return m_pShaderGL;
     }
 
     virtual bool IsCompiling() const override final
     {
-        return false;
+        return m_Status.load() <= SHADER_STATUS_COMPILING;
+    }
+
+    virtual SHADER_STATUS GetStatus(bool WaitForCompletion) override final
+    {
+        VERIFY_EXPR(m_Status.load() != SHADER_STATUS_UNINITIALIZED);
+        ASYNC_TASK_STATUS InitTaskStatus = AsyncInitializer::Update(m_AsyncInitializer, WaitForCompletion);
+        if (InitTaskStatus == ASYNC_TASK_STATUS_COMPLETE)
+        {
+            VERIFY(m_Status.load() > SHADER_STATUS_COMPILING, "Shader status must be atomically set by the compiling task before it finishes");
+        }
+        return m_Status.load();
     }
 
     virtual RefCntAutoPtr<IAsyncTask> GetCompileTask() const override final
     {
-        return {};
+        return AsyncInitializer::GetAsyncTask(m_AsyncInitializer);
     }
 
 private:
@@ -424,6 +483,19 @@ struct CompiledShaderGL final : SerializedShaderImpl::CompiledShader
 
         return OptimizedGLSL;
     }
+
+private:
+    const SerializationDeviceImpl* const m_pSerializationDevice;
+    const ShaderCreateInfoWrapper        m_ShaderCI;
+    const ShaderGLImpl::CreateInfo       m_GLShaderCI;
+    const RENDER_DEVICE_TYPE             m_DeviceType;
+
+    std::unique_ptr<AsyncInitializer> m_AsyncInitializer;
+    std::atomic<SHADER_STATUS>        m_Status{SHADER_STATUS_UNINITIALIZED};
+
+    String                 m_UnrolledSource;
+    RefCntAutoPtr<IShader> m_pShaderGL;
+    bool                   m_IsOptimized = false;
 };
 
 struct ShaderStageInfoGL
@@ -541,8 +613,7 @@ void SerializedShaderImpl::CreateShaderGL(IReferenceCounters*     pRefCounters,
         ppCompilerOutput == nullptr || *ppCompilerOutput == nullptr ? ppCompilerOutput : nullptr,
     };
 
-    CreateShader<CompiledShaderGL>(DeviceType::OpenGL, pRefCounters, ShaderCI, GLShaderCI, m_pDevice->GetRenderDevice(DeviceType),
-                                   DeviceType, m_pDevice->GetGLProperties());
+    CreateShader<CompiledShaderGL>(DeviceType::OpenGL, pRefCounters, ShaderCI, GLShaderCI, m_pDevice, DeviceType);
 }
 
 } // namespace Diligent
diff --git a/Graphics/Archiver/src/SerializedShaderImpl.cpp b/Graphics/Archiver/src/SerializedShaderImpl.cpp
@@ -198,14 +198,10 @@ SHADER_STATUS SerializedShaderImpl::GetStatus(bool WaitForCompletion)
     for (size_t type = 0; type < static_cast<size_t>(DeviceType::Count); ++type)
     {
         const auto& pCompiledShader = m_Shaders[type];
-
-        IShader* pShader = pCompiledShader ? pCompiledShader->GetDeviceShader() : nullptr;
-        if (pShader == nullptr)
+        if (!pCompiledShader)
             continue;
 
-        const SHADER_STATUS Status = static_cast<DeviceType>(type) != DeviceType::OpenGL ?
-            pShader->GetStatus(WaitForCompletion) :
-            SHADER_STATUS_READY;
+        const SHADER_STATUS Status = pCompiledShader->GetStatus(WaitForCompletion);
         switch (Status)
         {
             case SHADER_STATUS_UNINITIALIZED:
diff --git a/Tests/DiligentCoreAPITest/src/ArchiveTest.cpp b/Tests/DiligentCoreAPITest/src/ArchiveTest.cpp
@@ -421,7 +421,7 @@ TEST_P(TestBrokenShader, CompileFailure)
     RefCntAutoPtr<IShader>   pSerializedShader;
     RefCntAutoPtr<IDataBlob> pCompilerOutput;
     pSerializationDevice->CreateShader(ShaderCI, ShaderArchiveInfo{DataFlag}, &pSerializedShader, pCompilerOutput.RawDblPtr());
-    if (CompileAsync && DataFlag != ARCHIVE_DEVICE_DATA_FLAG_GL && DataFlag != ARCHIVE_DEVICE_DATA_FLAG_GLES)
+    if (CompileAsync)
     {
         ASSERT_NE(pSerializedShader, nullptr);
         SHADER_STATUS Status = SHADER_STATUS_UNINITIALIZED;
@@ -477,7 +477,7 @@ TEST_P(TestBrokenShader, MissingSourceFile)
 
     RefCntAutoPtr<IShader> pSerializedShader;
     pSerializationDevice->CreateShader(ShaderCI, ShaderArchiveInfo{DataFlag}, pSerializedShader.RawDblPtr());
-    if (CompileAsync && DataFlag != ARCHIVE_DEVICE_DATA_FLAG_GL && DataFlag != ARCHIVE_DEVICE_DATA_FLAG_GLES)
+    if (CompileAsync)
     {
         ASSERT_NE(pSerializedShader, nullptr);
         SHADER_STATUS Status = SHADER_STATUS_UNINITIALIZED;

Original file line number	Diff line number	Diff line change
`@@ -172,7 +172,7 @@ class SerializationDeviceImpl final : public RenderDeviceBase<SerializationEngin`
`172`	`172`	`const VkProperties& GetVkProperties() const { return m_VkProps; }`
`173`	`173`	`const MtlProperties& GetMtlProperties() const { return m_MtlProps; }`
`174`	`174`
`175`		`- IRenderDevice* GetRenderDevice(RENDER_DEVICE_TYPE Type)`
	`175`	`+ IRenderDevice* GetRenderDevice(RENDER_DEVICE_TYPE Type) const`
`176`	`176`	`{`
`177`	`177`	`return m_RenderDevices[Type];`
`178`	`178`	`}`
Original file line number	Diff line number	Diff line change
`@@ -193,7 +193,7 @@ Bool ArchiverImpl::SerializeToBlob(Uint32 ContentVersion, IDataBlob** ppBlob)`
`193`	`193`	`const auto* Name = shader_it.first.GetStr();`
`194`	`194`	`auto& SrcShader = *shader_it.second;`
`195`	`195`	`{`
`196`		`- const SHADER_STATUS Status = SrcShader.GetStatus(/WaitForCompletion = / false);`
	`196`	`+ const SHADER_STATUS Status = SrcShader.GetStatus(/WaitForCompletion = / true);`
`197`	`197`	`if (Status != SHADER_STATUS_READY)`
`198`	`198`	`{`
`199`	`199`	`LOG_ERROR_MESSAGE("Shader '", Name, "' is in ", GetShaderStatusString(Status),`