[RTL/x64] Improve RtlUnwindInternal

- Implement RtlpExecuteHandlerForUnwind in asm
- Implement RtlpExecuteHandlerForUnwindHandler
- Use the correct context in DISPATCHER_CONTEXT when unwinding
- Handle collided unwinds

Author: tkreuzer

sdk/lib/rtl/CMakeLists.txt

@@ -108,7 +108,8 @@ elseif(ARCH STREQUAL "amd64")
     list(APPEND ASM_SOURCE
         amd64/debug_asm.S
         amd64/except_asm.S
-        amd64/slist.S)
+        amd64/slist.S
+        amd64/unwind-asm.s)
     list(APPEND SOURCE
         bitmap64.c
         byteswap.c

sdk/lib/rtl/amd64/unwind-asm.s

@@ -0,0 +1,61 @@
+/*
+ * PROJECT:     ReactOS runtime library
+ * LICENSE:     MIT (https://spdx.org/licenses/MIT)
+ * PURPOSE:     Unwinding related x64 asm functions
+ * COPYRIGHT:   Copyright 2025 Timo Kreuzer <[email protected]>
+ */
+
+#include <asm.inc>
+#include <ksamd64.inc>
+
+.code64
+
+EXTERN RtlpExecuteHandlerForUnwindHandler:PROC
+
+//
+// EXCEPTION_DISPOSITION
+// RtlpExecuteHandlerForUnwind(
+//    _Inout_ PEXCEPTION_RECORD* ExceptionRecord,
+//    _In_ PVOID EstablisherFrame,
+//    _Inout_ PCONTEXT ContextRecord,
+//    _In_ PDISPATCHER_CONTEXT DispatcherContext);
+//
+PUBLIC RtlpExecuteHandlerForUnwind
+#ifdef _USE_ML
+RtlpExecuteHandlerForUnwind PROC FRAME:RtlpExecuteHandlerForUnwindHandler
+#else
+.PROC RtlpExecuteHandlerForUnwind
+.seh_handler RtlpExecuteHandlerForUnwindHandler, @unwind, @except
+.seh_handlerdata
+.long 0 // Count
+.seh_code
+#endif
+
+    /* Save ExceptionRecord->ExceptionFlags in the home space */
+    mov eax, [rcx + ErExceptionFlags]
+    mov [rsp + P1Home], eax
+
+    /* Save DispatcherContext in the home space */
+    mov [rsp + P4Home], r9
+
+    sub rsp, 40
+    .ALLOCSTACK 40
+    .ENDPROLOG
+
+    /* Call the language handler */
+    call qword ptr [r9 + DcLanguageHandler]
+
+    /* This nop prevents RtlVirtualUnwind from unwinding the epilog,
+       which would lead to ignoring the handler */
+    nop
+
+    add rsp, 40
+    ret
+
+#ifdef _USE_ML
+RtlpExecuteHandlerForUnwind ENDP
+#else
+.ENDP
+#endif
+
+END

sdk/lib/rtl/amd64/unwind.c

@@ -661,6 +661,46 @@ RtlpIsStackPointerValid(
            ((StackPointer & 7) == 0);
 }
 
+EXCEPTION_DISPOSITION
+RtlpExecuteHandlerForUnwindHandler(
+    _Inout_ PEXCEPTION_RECORD ExceptionRecord,
+    _In_ PVOID EstablisherFrame,
+    _Inout_ PCONTEXT ContextRecord,
+    _In_ PDISPATCHER_CONTEXT DispatcherContext)
+{
+    /* Get a pointer to the register home space for RtlpExecuteHandlerForUnwind */
+    PULONG64 HomeSpace = (PULONG64)EstablisherFrame + 6;
+
+    /* Get the ExceptionFlags value, which was saved in the home space */
+    ULONG ExceptionFlags = (ULONG)HomeSpace[0];
+
+    /* Get the DispatcherContext, which was saved in the home space */
+    PDISPATCHER_CONTEXT PreviousDispatcherContext = (PDISPATCHER_CONTEXT)HomeSpace[3];
+
+    /* Check if the original call to RtlpExecuteHandlerForUnwind was an unwind */
+    if (IS_UNWINDING(ExceptionFlags))
+    {
+        /* Check if the current call to this function is due to unwinding */
+        if (IS_UNWINDING(ExceptionRecord->ExceptionFlags))
+        {
+            /* We are unwinding over the call to a termination handler. This
+               could be due to an exception or longjmp. We need to make sure
+               to not run this termination handler again. To achieve that,
+               we copy the contents of the original dispatcher context back
+               over the current dispatcher context and return
+               ExceptionCollidedUnwind. RtlUnwindInternal will take the
+               original context, and continue unwing there. */
+            *DispatcherContext = *PreviousDispatcherContext;
+            return ExceptionCollidedUnwind;
+        }
+    }
+
+    // TODO: properly handle nested exceptions
+
+    return ExceptionContinueSearch;
+}
+
+
 /*!
     \remark The implementation is based on the description in this blog: http://www.nynaeve.net/?p=106
 
@@ -704,7 +744,8 @@ RtlpUnwindInternal(
     UnwindContext = *ContextRecord;
 
     /* Set up the constant fields of the dispatcher context */
-    DispatcherContext.ContextRecord = &UnwindContext;
+    DispatcherContext.ContextRecord =
+        (HandlerType == UNW_FLAG_UHANDLER) ? ContextRecord : &UnwindContext;
     DispatcherContext.HistoryTable = HistoryTable;
     DispatcherContext.TargetIp = (ULONG64)TargetIp;
 
@@ -793,12 +834,11 @@ RtlpUnwindInternal(
              /* Loop all nested handlers */
             do
             {
-                /// TODO: call RtlpExecuteHandlerForUnwind instead
                 /* Call the language specific handler */
-                Disposition = ExceptionRoutine(ExceptionRecord,
-                                               (PVOID)EstablisherFrame,
-                                               ContextRecord,
-                                               &DispatcherContext);
+                Disposition = RtlpExecuteHandlerForUnwind(ExceptionRecord,
+                                                          (PVOID)EstablisherFrame,
+                                                          ContextRecord,
+                                                          &DispatcherContext);
 
                 /* Clear exception flags for the next iteration */
                 ExceptionRecord->ExceptionFlags &= ~(EXCEPTION_TARGET_UNWIND |
@@ -828,12 +868,39 @@ RtlpUnwindInternal(
 
                 if (Disposition == ExceptionCollidedUnwind)
                 {
-                    /// TODO
-                    __debugbreak();
+                    /* We collided with another unwind, so we need to continue
+                       "after" the handler, skipping the termination handler
+                       that resulted in this unwind. The installed handler has
+                       already copied the original dispatcher context, we now
+                       need to copy back the original context. */
+                    UnwindContext = *ContextRecord = *DispatcherContext.ContextRecord;
+
+                    /* The original context was from "before" the unwind, so we
+                       need to do an additional virtual unwind to restore the
+                       unwind contxt. */
+                    RtlVirtualUnwind(HandlerType,
+                                     DispatcherContext.ImageBase,
+                                     UnwindContext.Rip,
+                                     DispatcherContext.FunctionEntry,
+                                     &UnwindContext,
+                                     &DispatcherContext.HandlerData,
+                                     &EstablisherFrame,
+                                     NULL);
+
+                    /* Restore the context pointer and establisher frame. */
+                    DispatcherContext.ContextRecord = &UnwindContext;
+                    EstablisherFrame = DispatcherContext.EstablisherFrame;
+
+                    /* Set the exception flags to indicate that we collided
+                       with an unwind and continue the handler loop, which
+                       will run any additional handlers from the previous
+                       unwind. */
+                    ExceptionRecord->ExceptionFlags |= EXCEPTION_COLLIDED_UNWIND;
+                    continue;
                 }
 
                 /* This must be ExceptionContinueSearch now */
-                if (Disposition != ExceptionContinueSearch)
+                else if (Disposition != ExceptionContinueSearch)
                 {
                     __debugbreak();
                     RtlRaiseStatus(STATUS_INVALID_DISPOSITION);

sdk/lib/rtl/rtlp.h

@@ -184,7 +184,6 @@ RtlpExecuteHandlerForException(PEXCEPTION_RECORD ExceptionRecord,
                                PCONTEXT Context,
                                PVOID DispatcherContext,
                                PEXCEPTION_ROUTINE ExceptionHandler);
-#endif
 
 EXCEPTION_DISPOSITION
 NTAPI
@@ -193,6 +192,16 @@ RtlpExecuteHandlerForUnwind(PEXCEPTION_RECORD ExceptionRecord,
                             PCONTEXT Context,
                             PVOID DispatcherContext,
                             PEXCEPTION_ROUTINE ExceptionHandler);
+#else
+EXCEPTION_DISPOSITION
+NTAPI
+RtlpExecuteHandlerForUnwind(
+    _Inout_ struct _EXCEPTION_RECORD *ExceptionRecord,
+    _In_ PVOID EstablisherFrame,
+    _Inout_ struct _CONTEXT *ContextRecord,
+    _In_ PVOID DispatcherContext);
+
+#endif
 
 VOID
 NTAPI