Further reduce the likelihood of alt-tab freezes (Not fully eliminated yet)

Author: Dutchman101

Client/core/CAdditionalVertexStreamManager.cpp

@@ -290,15 +290,21 @@ void CAdditionalVertexStreamManager::MaybeUnsetAdditionalVertexStream()
     if (!m_pDevice)
         return;
 
-    HRESULT hr;
     if (m_pOldVertexDeclaration)
     {
-        // Set prev declaration
-        hr = g_pProxyDevice->SetVertexDeclaration(m_pOldVertexDeclaration);
-        SAFE_RELEASE(m_pOldVertexDeclaration);
+        HRESULT hr = m_pDevice->TestCooperativeLevel();
+        const bool bDeviceOperational = (hr == D3D_OK);
+
+        if (bDeviceOperational)
+        {
+            // Set prev declaration
+            g_pProxyDevice->SetVertexDeclaration(m_pOldVertexDeclaration);
 
-        // Unset additional stream
-        hr = m_pDevice->SetStreamSource(2, nullptr, 0, 0);
+            // Unset additional stream
+            m_pDevice->SetStreamSource(2, nullptr, 0, 0);
+        }
+
+        SAFE_RELEASE(m_pOldVertexDeclaration);
     }
 }
 

Client/core/DXHook/CDirect3DEvents9.cpp

@@ -80,33 +80,41 @@ void CDirect3DEvents9::OnInvalidate(IDirect3DDevice9* pDevice)
 {
     WriteDebugEvent("CDirect3DEvents9::OnInvalidate");
 
-    // Ensure device is in a valid state before invalidation
-    // For example, Nvidia drivers can hang if device operations are attempted during invalid states
-    if (pDevice->TestCooperativeLevel() == D3DERR_DEVICELOST)
-    {
-        WriteDebugEvent("OnInvalidate: Device already lost, skipping operations");
-        return;
-    }
+    const HRESULT hrCooperativeLevel = pDevice->TestCooperativeLevel();
+    const bool    bDeviceOperational = (hrCooperativeLevel == D3D_OK);
+    const bool    bDeviceTemporarilyLost =
+        (hrCooperativeLevel == D3DERR_DEVICELOST || hrCooperativeLevel == D3DERR_DEVICENOTRESET);
 
-    // Flush any pending operations before invalidation
-    g_pCore->GetGraphics()->GetRenderItemManager()->SaveReadableDepthBuffer();
-    g_pCore->GetGraphics()->GetRenderItemManager()->FlushNonAARenderTarget();
-    
-    // Force completion of all GPU operations if a scene is currently active
-    if (g_bInMTAScene || g_bInGTAScene)
+    if (!bDeviceOperational && !bDeviceTemporarilyLost)
+        WriteDebugEvent(SString("OnInvalidate: unexpected cooperative level %08x", hrCooperativeLevel));
+
+    if (bDeviceOperational)
     {
-        const HRESULT hrEndScene = pDevice->EndScene();
-        if (SUCCEEDED(hrEndScene))
-        {
-            g_bInMTAScene = false;
-            g_bInGTAScene = false;
-        }
-        else
+        // Flush any pending operations before invalidation while the device still accepts work
+        g_pCore->GetGraphics()->GetRenderItemManager()->SaveReadableDepthBuffer();
+        g_pCore->GetGraphics()->GetRenderItemManager()->FlushNonAARenderTarget();
+
+        if (g_bInMTAScene || g_bInGTAScene)
         {
-            WriteDebugEvent(SString("OnInvalidate: EndScene failed: %08x", hrEndScene));
+            const HRESULT hrEndScene = pDevice->EndScene();
+            if (FAILED(hrEndScene))
+                WriteDebugEvent(SString("OnInvalidate: EndScene failed: %08x", hrEndScene));
         }
+
+        CloseActiveShader();
+    }
+    else
+    {
+        if (g_bInMTAScene || g_bInGTAScene)
+            WriteDebugEvent("OnInvalidate: device lost, skipping EndScene and pending GPU work");
+
+        // Prevent reuse of partially configured shader state across device resets without touching the lost device
+        CloseActiveShader(false);
     }
-    
+
+    g_bInMTAScene = false;
+    g_bInGTAScene = false;
+
     // Invalidate the VMR9 Manager
     // CVideoManager::GetSingleton ().OnLostDevice ();
 
@@ -591,26 +599,43 @@ HRESULT CDirect3DEvents9::DrawIndexedPrimitiveShader(IDirect3DDevice9* pDevice,
 // Finish the active shader if there is one
 //
 /////////////////////////////////////////////////////////////
-void CDirect3DEvents9::CloseActiveShader()
+void CDirect3DEvents9::CloseActiveShader(bool bDeviceOperational)
 {
     if (!g_pActiveShader)
         return;
 
     ID3DXEffect* pD3DEffect = g_pActiveShader->m_pShaderInstance->m_pEffectWrap->m_pD3DEffect;
+    IDirect3DDevice9* pDevice = g_pGraphics ? g_pGraphics->GetDevice() : nullptr;
+    HRESULT            hrCooperativeLevel = D3D_OK;
+    if (pDevice)
+        hrCooperativeLevel = pDevice->TestCooperativeLevel();
+
+    bool bAllowDeviceWork = bDeviceOperational;
+    if (hrCooperativeLevel == D3D_OK)
+        bAllowDeviceWork = true;
+    else if (hrCooperativeLevel == D3DERR_DEVICELOST || hrCooperativeLevel == D3DERR_DEVICENOTRESET)
+        bAllowDeviceWork = false;
+
+    if (pD3DEffect)
+    {
+        HRESULT hrEndPass = pD3DEffect->EndPass();
+        if (FAILED(hrEndPass) && hrEndPass != D3DERR_DEVICELOST && hrEndPass != D3DERR_DEVICENOTRESET)
+            WriteDebugEvent(SString("CloseActiveShader: EndPass failed: %08x", hrEndPass));
+    }
 
-    pD3DEffect->EndPass();
-
-    g_pActiveShader->m_pShaderInstance->m_pEffectWrap->End();
-    g_pActiveShader = NULL;
-
-    // We didn't get the effect to save the shader state, clear some things here
-    IDirect3DDevice9* pDevice = g_pGraphics->GetDevice();
-    pDevice->SetVertexShader(NULL);
-    pDevice->SetPixelShader(NULL);
+    // When the device is lost we intentionally skip touching the GPU beyond the required End call; the effect will be reset later.
+    g_pActiveShader->m_pShaderInstance->m_pEffectWrap->End(bAllowDeviceWork);
+    g_pActiveShader = nullptr;
 
-    // Unset additional vertex stream
     if (CAdditionalVertexStreamManager* pAdditionalStreamManager = CAdditionalVertexStreamManager::GetExistingSingleton())
         pAdditionalStreamManager->MaybeUnsetAdditionalVertexStream();
+
+    if (bAllowDeviceWork && pDevice)
+    {
+        // We didn't get the effect to save the shader state, clear some things here
+        pDevice->SetVertexShader(nullptr);
+        pDevice->SetPixelShader(nullptr);
+    }
 }
 
 /////////////////////////////////////////////////////////////

Client/core/DXHook/CDirect3DEvents9.h

@@ -45,7 +45,7 @@ class CDirect3DEvents9
     static HRESULT SetVertexDeclaration(IDirect3DDevice9* pDevice, IDirect3DVertexDeclaration9* pDecl);
     static void    CheckForScreenShot();
     static ERenderFormat DiscoverReadableDepthFormat(IDirect3DDevice9* pDevice, D3DMULTISAMPLE_TYPE multisampleType, bool bWindowed);
-    static void          CloseActiveShader();
+    static void          CloseActiveShader(bool bDeviceOperational = true);
 
     static IDirect3DVertexBuffer9* GetRealVertexBuffer(IDirect3DVertexBuffer9* pVertexBuffer);
     static IDirect3DIndexBuffer9*  GetRealIndexBuffer(IDirect3DIndexBuffer9* pIndexBuffer);

Client/core/Graphics/CRenderItem.EffectParameters.cpp

@@ -397,15 +397,37 @@ HRESULT CEffectParameters::Begin(UINT* pPasses, DWORD Flags, bool bWorldRender)
 // Ensures secondary render targets are unset
 //
 ////////////////////////////////////////////////////////////////
-HRESULT CEffectParameters::End()
+HRESULT CEffectParameters::End(bool bDeviceOperational)
 {
+    if (!m_pD3DEffect)
+        return D3D_OK;
+
     HRESULT hResult = m_pD3DEffect->End();
-    for (uint i = 0; i < m_SecondaryRenderTargetList.size(); i++)
+
+    if (!m_SecondaryRenderTargetList.empty())
     {
-        LPDIRECT3DDEVICE9 pDevice;
+        LPDIRECT3DDEVICE9 pDevice = nullptr;
         m_pD3DEffect->GetDevice(&pDevice);
-        pDevice->SetRenderTarget(i + 1, NULL);
+
+        if (pDevice)
+        {
+            bool bCanTouchDevice = bDeviceOperational;
+            if (!bCanTouchDevice)
+            {
+                const HRESULT hrCooperativeLevel = pDevice->TestCooperativeLevel();
+                bCanTouchDevice = (hrCooperativeLevel == D3D_OK);
+            }
+
+            if (bCanTouchDevice)
+            {
+                for (uint i = 0; i < m_SecondaryRenderTargetList.size(); i++)
+                    pDevice->SetRenderTarget(i + 1, nullptr);
+            }
+
+            SAFE_RELEASE(pDevice);
+        }
     }
+
     return hResult;
 }
 

Client/core/Graphics/CRenderItem.EffectParameters.h

@@ -95,7 +95,7 @@ class CEffectParameters : public CRenderItem
     bool                        ApplyCommonHandles();
     bool                        ApplyMappedHandles();
     HRESULT                     Begin(UINT* pPasses, DWORD Flags, bool bWorldRender = true);
-    HRESULT                     End();
+    HRESULT                     End(bool bDeviceOperational = true);
     void                        NotifyModifiedParameter(D3DXHANDLE hParameter);
     const std::set<D3DXHANDLE>& GetModifiedParameters();
     void                        RestoreParametersDefaultValue(const std::vector<D3DXHANDLE>& parameterList);

Client/core/Graphics/CRenderItem.EffectWrap.cpp

@@ -95,9 +95,9 @@ HRESULT CEffectWrap::Begin(UINT* pPasses, DWORD Flags, bool bWorldRender)
 // Ensures secondary render targets are unset
 //
 ////////////////////////////////////////////////////////////////
-HRESULT CEffectWrap::End()
+HRESULT CEffectWrap::End(bool bDeviceOperational)
 {
-    return m_pEffectTemplate->End();
+    return m_pEffectTemplate->End(bDeviceOperational);
 }
 
 ////////////////////////////////////////////////////////////////

Client/sdk/core/CRenderItemManagerInterface.h

@@ -332,7 +332,7 @@ class CEffectWrap : public CRenderItem
     virtual void OnLostDevice();
     virtual void OnResetDevice();
     HRESULT      Begin(UINT* pPasses, DWORD Flags, bool bWorldRender = true);
-    HRESULT      End();
+    HRESULT      End(bool bDeviceOperational = true);
     bool         ApplyCommonHandles();
     bool         ApplyMappedHandles();