DWM Crash Fix Backport (#11621)

Author: himgoyalmicro

src/Microsoft.DotNet.Wpf/src/PresentationFramework/MS/Internal/FrameworkAppContextSwitches.cs

@@ -147,6 +147,18 @@ public static bool DisableDynamicResourceOptimization
                 return LocalAppContext.GetCachedSwitchValue(DisableDynamicResourceOptimizationSwitchName, ref _DisableDynamicResourceOptimization);
             }
         }
+
+        // Swtich to disable DWM crash containment
+        internal const string DisableDWMCrashContainmentSwitchName = "Switch.System.Windows.Media.DisableDWMCrashContainment";
+        private static int _disableDWMCrashContainment;
+        public static bool DisableDWMCrashContainment
+        {
+            [MethodImpl(MethodImplOptions.AggressiveInlining)]
+            get
+            {
+                return LocalAppContext.GetCachedSwitchValue(DisableDWMCrashContainmentSwitchName, ref _disableDWMCrashContainment);
+            }
+        }
     }
 }
 

src/Microsoft.DotNet.Wpf/src/PresentationFramework/System/AppContextDefaultValues.cs

@@ -32,7 +32,7 @@ static partial void PopulateDefaultValuesPartial(string platformIdentifier, stri
             LocalAppContext.DefineSwitchDefault(FrameworkAppContextSwitches.ItemAutomationPeerKeepsItsItemAliveSwitchName, false);
             LocalAppContext.DefineSwitchDefault(FrameworkAppContextSwitches.DisableFluentThemeWindowBackdropSwitchName, false);
             LocalAppContext.DefineSwitchDefault(FrameworkAppContextSwitches.DisableDynamicResourceOptimizationSwitchName, false);
-
+             LocalAppContext.DefineSwitchDefault(FrameworkAppContextSwitches.DisableDWMCrashContainmentSwitchName, false);
 #pragma warning restore CS0436 // Type conflicts with imported type
         }
     }

src/Microsoft.DotNet.Wpf/src/PresentationFramework/System/Windows/Appearance/WindowBackdropManager.cs

@@ -131,8 +131,7 @@ private static bool UpdateGlassFrame(IntPtr hwnd, WindowBackdropType backdropTyp
             margins = new MARGINS { cxLeftWidth = -1, cxRightWidth = -1, cyTopHeight = -1, cyBottomHeight = -1 };
         }
 
-        var dwmApiResult = NativeMethods.DwmExtendFrameIntoClientArea(hwnd, ref margins);
-        return new HRESULT((uint)dwmApiResult) == HRESULT.S_OK;
+        return NativeMethods.DwmExtendFrameIntoClientArea(hwnd, ref margins);
     }
 
     #endregion

src/Microsoft.DotNet.Wpf/src/PresentationFramework/System/Windows/Shell/WindowChromeWorker.cs

@@ -1002,7 +1002,11 @@ private void _ExtendGlassFrame()
                     cyBottomHeight = (int)Math.Ceiling(deviceGlassThickness.Bottom),
                 };
 
-                NativeMethods.DwmExtendFrameIntoClientArea(_hwnd, ref dwmMargin);
+                bool dwmApiResult = NativeMethods.DwmExtendFrameIntoClientArea(_hwnd, ref dwmMargin);
+                if(!dwmApiResult)
+                {
+                    _hwndSource.CompositionTarget.BackgroundColor = SystemColors.WindowColor;
+                }
             }
         }
 

src/Microsoft.DotNet.Wpf/src/PresentationFramework/System/Windows/Standard/NativeMethods.cs

@@ -9,7 +9,9 @@
 using System.Runtime.InteropServices;
 using System.Runtime.InteropServices.ComTypes;
 using System.Text;
+using System.Threading;
 using Microsoft.Win32.SafeHandles;
+using MS.Internal;
 
 // Some COM interfaces and Win32 structures are already declared in the framework.
 // Interesting ones to remember in System.Runtime.InteropServices.ComTypes are:
@@ -2501,8 +2503,8 @@ public static IntPtr CreateWindowEx(
         [DllImport("dwmapi.dll", PreserveSig = false)]
         public static extern int DwmGetWindowAttribute(IntPtr hWnd, DWMWA dwAttributeToGet, ref int pvAttributeValue, int cbAttribute);
 
-        [DllImport("dwmapi.dll", PreserveSig = false)]
-        public static extern int DwmExtendFrameIntoClientArea(IntPtr hwnd, ref MARGINS pMarInset);
+        [DllImport("dwmapi.dll", EntryPoint = "DwmExtendFrameIntoClientArea", PreserveSig = true, SetLastError = true)]
+        private static extern HRESULT _DwmExtendFrameIntoClientArea(IntPtr hwnd, ref MARGINS pMarInset);
 
         [DllImport("dwmapi.dll", EntryPoint = "DwmIsCompositionEnabled", PreserveSig = false)]
         [return: MarshalAs(UnmanagedType.Bool)]
@@ -2511,6 +2513,29 @@ public static IntPtr CreateWindowEx(
         [DllImport("dwmapi.dll", EntryPoint = "DwmGetColorizationColor", PreserveSig = true)]
         private static extern HRESULT _DwmGetColorizationColor(out uint pcrColorization, [Out, MarshalAs(UnmanagedType.Bool)] out bool pfOpaqueBlend);
 
+        public static bool DwmExtendFrameIntoClientArea(IntPtr hwnd, ref MARGINS pMarInset)
+        {
+            HRESULT hr = _DwmExtendFrameIntoClientArea(hwnd, ref pMarInset);
+            if(hr == HRESULT.S_OK)
+            {
+                return true;
+            }
+            
+            if(FrameworkAppContextSwitches.DisableDWMCrashContainment)
+            {
+                // Original behavior: just call the native method and let any exceptions propagate.
+                HRESULT.ThrowLastError();
+            }
+
+            // Crash only if the arguments are invalid.
+            if(hr == HRESULT.E_INVALIDARG)
+            {
+                HRESULT.ThrowLastError();
+            }
+
+            return false;
+        }
+
         public static bool DwmGetColorizationColor(out uint pcrColorization, out bool pfOpaqueBlend)
         {
             // Make this call safe to make on downlevel OSes...