Make ExceptionHandling.RaiseAppDomainUnhandledExceptionEvent() thread safe. (#117844)

* Make the ExceptionHandling.RaiseAppDomainUnhandledExceptionEvent() API thread safe.

The RaiseAppDomainUnhandledExceptionEvent() API
is permitted to be called by multiple threads but only
one should be triggering the event handlers.
---------

Co-authored-by: Jan Kotas <[email protected]>

Author: AaronRobinsonMSFT

src/libraries/System.Private.CoreLib/src/System/AppContext.cs

@@ -82,28 +82,50 @@ public static void SetData(string name, object? data)
         internal static event EventHandler<FirstChanceExceptionEventArgs>? FirstChanceException;
 #pragma warning restore CS0067
 
+        private static ulong s_crashingThreadId;
+
 #if NATIVEAOT
         [System.Runtime.RuntimeExport("OnUnhandledException")]
 #endif
         internal static void OnUnhandledException(object e)
         {
+            ulong currentThreadId = Thread.CurrentOSThreadId;
+            ulong previousCrashingThreadId = Interlocked.CompareExchange(ref s_crashingThreadId, currentThreadId, 0);
+            if (previousCrashingThreadId == 0)
+            {
 #if NATIVEAOT
-            RuntimeExceptionHelpers.SerializeCrashInfo(System.Runtime.RhFailFastReason.UnhandledException, (e as Exception)?.Message, e as Exception);
+                RuntimeExceptionHelpers.SerializeCrashInfo(System.Runtime.RhFailFastReason.UnhandledException, (e as Exception)?.Message, e as Exception);
 #endif
-            if (UnhandledException is UnhandledExceptionEventHandler handlers)
-            {
-                UnhandledExceptionEventArgs args = new(e, isTerminating: true);
-                foreach (UnhandledExceptionEventHandler handler in Delegate.EnumerateInvocationList(handlers))
+                if (UnhandledException is UnhandledExceptionEventHandler handlers)
                 {
-                    try
-                    {
-                        handler(/* AppDomain */ null!, args);
-                    }
-                    catch
+                    UnhandledExceptionEventArgs args = new(e, isTerminating: true);
+                    foreach (UnhandledExceptionEventHandler handler in Delegate.EnumerateInvocationList(handlers))
                     {
+                        try
+                        {
+                            handler(/* AppDomain */ null!, args);
+                        }
+                        catch
+                        {
+                        }
                     }
                 }
             }
+            else
+            {
+                if (s_crashingThreadId == previousCrashingThreadId)
+                {
+                    Environment.FailFast("OnUnhandledException called recursively");
+                }
+
+                // If we are already in the process of handling an unhandled
+                // exception, we do not want to raise the event again. We wait
+                // here while the other thread raises the unhandled exception.
+                // Waiting is important because it is possible upon returning, this thread
+                // could call some rude abort method that would terminate the process
+                // before the other thread finishes raising the unhandled exception.
+                Thread.Sleep(-1);
+            }
         }
 
         internal static void OnProcessExit()

src/libraries/System.Private.CoreLib/src/System/Runtime/ExceptionServices/ExceptionHandling.cs

@@ -48,6 +48,10 @@ public static void SetUnhandledExceptionHandler(Func<Exception, bool> handler)
         /// event and then return.
         ///
         /// It will not raise the the handler registered with <see cref="SetUnhandledExceptionHandler"/>.
+        ///
+        /// This API is thread safe and can be called from multiple threads. However, only one thread
+        /// will trigger the event handlers, while other threads will wait indefinitely without raising
+        /// the event.
         /// </remarks>
         public static void RaiseAppDomainUnhandledExceptionEvent(object exception)
         {

src/tests/baseservices/exceptions/RaiseAppDomainUnhandledExceptionEvent/RaiseEvent.cs

@@ -1,6 +1,7 @@
 // Licensed to the .NET Foundation under one or more agreements.
 // The .NET Foundation licenses this file to you under the MIT license.
 using System;
+using System.Runtime.CompilerServices;
 using System.Runtime.ExceptionServices;
 using TestLibrary;
 using Xunit;
@@ -19,7 +20,14 @@ public HandlerRegistration(UnhandledExceptionEventHandler handler)
         public void Dispose()
         {
             AppDomain.CurrentDomain.UnhandledException -= _handler;
+
+            // See usage of s_crashingThreadId in the ExceptionHandling class.
+            // This is to ensure that the static field is reset after the test.
+            GetCrashingThreadId(null) = 0;
         }
+
+        [UnsafeAccessor(UnsafeAccessorKind.StaticField, Name = "s_crashingThreadId")]
+        private static extern ref ulong GetCrashingThreadId([UnsafeAccessorType("System.AppContext")]object? obj);
     }
 
     [ThreadStatic]