From 299880a933954069b944d4ab2a943d7646639375 Mon Sep 17 00:00:00 2001 From: Ray Kong Date: Tue, 3 Mar 2026 11:21:47 -0800 Subject: [PATCH 1/6] Fix reentrant bugcheck in SetParam(RESUMPTION_TICKET) with custom executions When MsQuicLib.CustomExecutions is true, SetParam calls execute inline with InlineApiExecution = TRUE. For QUIC_PARAM_CONN_RESUMPTION_TICKET, the call chain reaches QuicStreamSetIndicateStreamsAvailable -> QuicConnIndicateEvent, Since the resumption ticket is set before ConnectionStart, there are no streams yet, so the STREAMS_AVAILABLE event is meaningless. Gate the indication on FlushIfUnblocked (which is already FALSE for the resumption ticket path), consistent with the existing flush guard. --- src/core/stream_set.c | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/core/stream_set.c b/src/core/stream_set.c index ba32718a3d..6abc873f81 100644 --- a/src/core/stream_set.c +++ b/src/core/stream_set.c @@ -423,7 +423,7 @@ QuicStreamSetInitializeTransportParameters( } } - if (UpdateAvailableStreams) { + if (UpdateAvailableStreams && FlushIfUnblocked) { QuicStreamSetIndicateStreamsAvailable(StreamSet); } From 99e5e97dc9299dd3613cbaf0d877c13bc4675dea Mon Sep 17 00:00:00 2001 From: Ray Kong Date: Tue, 3 Mar 2026 20:15:50 -0800 Subject: [PATCH 2/6] Fix reentrant bugcheck for datagram state change in SetParam(RESUMPTION_TICKET) --- src/core/connection.c | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/src/core/connection.c b/src/core/connection.c index 490d742d46..d4c96f30be 100644 --- a/src/core/connection.c +++ b/src/core/connection.c @@ -3089,7 +3089,9 @@ QuicConnProcessPeerTransportParameters( Connection->PeerTransportParams.InitialMaxUniStreams, !FromResumptionTicket); - QuicDatagramOnSendStateChanged(&Connection->Datagram); + if (!FromResumptionTicket) { + QuicDatagramOnSendStateChanged(&Connection->Datagram); + } if (Connection->State.Started) { if (Connection->State.Disable1RttEncrytion && From fef99c376e9bdc304fc691b4c60e1ea6883843d3 Mon Sep 17 00:00:00 2001 From: Ray Kong Date: Wed, 4 Mar 2026 11:59:59 -0800 Subject: [PATCH 3/6] Defer notifications instead of skipping during resumption ticket processing Instead of skipping STREAMS_AVAILABLE and DATAGRAM_STATE_CHANGED notifications when called from the resumption ticket path (which breaks 0-RTT scenarios), defer them by queuing operations that get processed when the worker drains the operation queue, outside the InlineApiExecution context. - Add QUIC_OPER_TYPE_STREAMS_AVAILABLE and QUIC_OPER_TYPE_DATAGRAM_STATE_CHANGED operation types. instead of calling QuicStreamSetIndicateStreamsAvailable directly. - When FromResumptionTicket, enqueue DATAGRAM_STATE_CHANGED instead of calling QuicDatagramOnSendStateChanged directly. - Handle both new operation types in QuicConnDrainOperations. - Make QuicSendQueueFlush unconditional since it only enqueues a FLUSH_SEND operation and does not trigger app callbacks. - Export QuicStreamSetIndicateStreamsAvailable declaration in stream_set.h. --- src/core/connection.c | 25 ++++++++++++++++++++++++- src/core/operation.h | 2 ++ src/core/stream_set.c | 15 +++++++++++++-- src/core/stream_set.h | 9 +++++++++ 4 files changed, 48 insertions(+), 3 deletions(-) diff --git a/src/core/connection.c b/src/core/connection.c index d4c96f30be..a9de3e1dbc 100644 --- a/src/core/connection.c +++ b/src/core/connection.c @@ -3089,7 +3089,16 @@ QuicConnProcessPeerTransportParameters( Connection->PeerTransportParams.InitialMaxUniStreams, !FromResumptionTicket); - if (!FromResumptionTicket) { + if (FromResumptionTicket) { + // + // Defer the datagram state change notification to avoid reentrant + // callbacks when called from the resumption ticket path. + // + QUIC_OPERATION* Oper; + if ((Oper = QuicConnAllocOperation(Connection, QUIC_OPER_TYPE_DATAGRAM_STATE_CHANGED)) != NULL) { + QuicConnQueueOper(Connection, Oper); + } + } else { QuicDatagramOnSendStateChanged(&Connection->Datagram); } @@ -7961,6 +7970,20 @@ QuicConnDrainOperations( Connection, Oper->ROUTE.PhysicalAddress, Oper->ROUTE.PathId, Oper->ROUTE.Succeeded); break; + case QUIC_OPER_TYPE_STREAMS_AVAILABLE: + if (Connection->State.ShutdownComplete) { + break; // Ignore if already shutdown + } + QuicStreamSetIndicateStreamsAvailable(&Connection->Streams); + break; + + case QUIC_OPER_TYPE_DATAGRAM_STATE_CHANGED: + if (Connection->State.ShutdownComplete) { + break; // Ignore if already shutdown + } + QuicDatagramOnSendStateChanged(&Connection->Datagram); + break; + default: CXPLAT_FRE_ASSERT(FALSE); break; diff --git a/src/core/operation.h b/src/core/operation.h index e8e7a9246a..a29fc309f8 100644 --- a/src/core/operation.h +++ b/src/core/operation.h @@ -31,6 +31,8 @@ typedef enum QUIC_OPERATION_TYPE { QUIC_OPER_TYPE_TIMER_EXPIRED, // A timer expired. QUIC_OPER_TYPE_TRACE_RUNDOWN, // A trace rundown was triggered. QUIC_OPER_TYPE_ROUTE_COMPLETION, // Process route completion event. + QUIC_OPER_TYPE_STREAMS_AVAILABLE, // Indicate streams available to the app. + QUIC_OPER_TYPE_DATAGRAM_STATE_CHANGED, // Indicate datagram state changed to the app. // // All stateless operations follow. diff --git a/src/core/stream_set.c b/src/core/stream_set.c index 6abc873f81..42982aa4bf 100644 --- a/src/core/stream_set.c +++ b/src/core/stream_set.c @@ -423,8 +423,19 @@ QuicStreamSetInitializeTransportParameters( } } - if (UpdateAvailableStreams && FlushIfUnblocked) { - QuicStreamSetIndicateStreamsAvailable(StreamSet); + if (UpdateAvailableStreams) { + if (FlushIfUnblocked) { + QuicStreamSetIndicateStreamsAvailable(StreamSet); + } else { + // + // Defer the streams-available notification to avoid reentrant + // callbacks when called from the resumption ticket path. + // + QUIC_OPERATION* Oper; + if ((Oper = QuicConnAllocOperation(Connection, QUIC_OPER_TYPE_STREAMS_AVAILABLE)) != NULL) { + QuicConnQueueOper(Connection, Oper); + } + } } if (MightBeUnblocked && FlushIfUnblocked) { diff --git a/src/core/stream_set.h b/src/core/stream_set.h index 35fc171080..44589c5c3f 100644 --- a/src/core/stream_set.h +++ b/src/core/stream_set.h @@ -134,6 +134,15 @@ QuicStreamSetInitializeTransportParameters( _In_ BOOLEAN FlushIfUnblocked ); +// +// Indicates available streams to the app via a connection event. +// +_IRQL_requires_max_(PASSIVE_LEVEL) +void +QuicStreamSetIndicateStreamsAvailable( + _Inout_ QUIC_STREAM_SET* StreamSet + ); + // // Invoked when the peer sends a MAX_STREAMS frame. // From 9467c267876df786f6cf2313e281ca20a274d7fb Mon Sep 17 00:00:00 2001 From: Ray Kong Date: Wed, 4 Mar 2026 16:28:01 -0800 Subject: [PATCH 4/6] Rename FlushIfUnblocked to FromResumptionTicket --- src/core/connection.c | 2 +- src/core/stream_set.c | 6 +++--- src/core/stream_set.h | 2 +- 3 files changed, 5 insertions(+), 5 deletions(-) diff --git a/src/core/connection.c b/src/core/connection.c index a9de3e1dbc..6a3c172c77 100644 --- a/src/core/connection.c +++ b/src/core/connection.c @@ -3087,7 +3087,7 @@ QuicConnProcessPeerTransportParameters( &Connection->Streams, Connection->PeerTransportParams.InitialMaxBidiStreams, Connection->PeerTransportParams.InitialMaxUniStreams, - !FromResumptionTicket); + FromResumptionTicket); if (FromResumptionTicket) { // diff --git a/src/core/stream_set.c b/src/core/stream_set.c index 42982aa4bf..6702873210 100644 --- a/src/core/stream_set.c +++ b/src/core/stream_set.c @@ -347,7 +347,7 @@ QuicStreamSetInitializeTransportParameters( _Inout_ QUIC_STREAM_SET* StreamSet, _In_ uint64_t BidiStreamCount, _In_ uint64_t UnidiStreamCount, - _In_ BOOLEAN FlushIfUnblocked + _In_ BOOLEAN FromResumptionTicket ) { QUIC_CONNECTION* Connection = QuicStreamSetGetConnection(StreamSet); @@ -424,7 +424,7 @@ QuicStreamSetInitializeTransportParameters( } if (UpdateAvailableStreams) { - if (FlushIfUnblocked) { + if (!FromResumptionTicket) { QuicStreamSetIndicateStreamsAvailable(StreamSet); } else { // @@ -438,7 +438,7 @@ QuicStreamSetInitializeTransportParameters( } } - if (MightBeUnblocked && FlushIfUnblocked) { + if (MightBeUnblocked && !FromResumptionTicket) { // // We opened the window, so start send. Rather than checking // the streams to see if one is actually unblocked, we risk starting diff --git a/src/core/stream_set.h b/src/core/stream_set.h index 44589c5c3f..99e367deb2 100644 --- a/src/core/stream_set.h +++ b/src/core/stream_set.h @@ -131,7 +131,7 @@ QuicStreamSetInitializeTransportParameters( _Inout_ QUIC_STREAM_SET* StreamSet, _In_ uint64_t BidiStreamCount, _In_ uint64_t UnidiStreamCount, - _In_ BOOLEAN FlushIfUnblocked + _In_ BOOLEAN FromResumptionTicket ); // From 07a68661eaa0537b549d0151d4058599248edf1e Mon Sep 17 00:00:00 2001 From: Ray Kong Date: Wed, 11 Mar 2026 13:26:19 -0700 Subject: [PATCH 5/6] Relax reentrant assertion for custom executions instead of queuing notifications --- src/core/connection.c | 36 +++++++----------------------------- src/core/operation.h | 2 -- src/core/stream_set.c | 17 +++-------------- src/core/stream_set.h | 11 +---------- 4 files changed, 11 insertions(+), 55 deletions(-) diff --git a/src/core/connection.c b/src/core/connection.c index 6a3c172c77..c306a6088d 100644 --- a/src/core/connection.c +++ b/src/core/connection.c @@ -692,11 +692,14 @@ QuicConnIndicateEvent( // MsQuic shouldn't indicate reentrancy to the app when at all possible. // The general exception to this rule is when the connection is being // closed because the API MUST block until all work is completed, so we - // have to execute the event callbacks inline. + // have to execute the event callbacks inline. Custom executions also + // allow reentrancy because set/get param must be executed inline to not + // block the thread, and the app is expected to handle reentrancy. // CXPLAT_DBG_ASSERT( !Connection->State.InlineApiExecution || - Connection->State.HandleClosed); + Connection->State.HandleClosed || + MsQuicLib.CustomExecutions); Status = Connection->ClientCallbackHandler( (HQUIC)Connection, @@ -3087,20 +3090,9 @@ QuicConnProcessPeerTransportParameters( &Connection->Streams, Connection->PeerTransportParams.InitialMaxBidiStreams, Connection->PeerTransportParams.InitialMaxUniStreams, - FromResumptionTicket); + !FromResumptionTicket); - if (FromResumptionTicket) { - // - // Defer the datagram state change notification to avoid reentrant - // callbacks when called from the resumption ticket path. - // - QUIC_OPERATION* Oper; - if ((Oper = QuicConnAllocOperation(Connection, QUIC_OPER_TYPE_DATAGRAM_STATE_CHANGED)) != NULL) { - QuicConnQueueOper(Connection, Oper); - } - } else { - QuicDatagramOnSendStateChanged(&Connection->Datagram); - } + QuicDatagramOnSendStateChanged(&Connection->Datagram); if (Connection->State.Started) { if (Connection->State.Disable1RttEncrytion && @@ -7970,20 +7962,6 @@ QuicConnDrainOperations( Connection, Oper->ROUTE.PhysicalAddress, Oper->ROUTE.PathId, Oper->ROUTE.Succeeded); break; - case QUIC_OPER_TYPE_STREAMS_AVAILABLE: - if (Connection->State.ShutdownComplete) { - break; // Ignore if already shutdown - } - QuicStreamSetIndicateStreamsAvailable(&Connection->Streams); - break; - - case QUIC_OPER_TYPE_DATAGRAM_STATE_CHANGED: - if (Connection->State.ShutdownComplete) { - break; // Ignore if already shutdown - } - QuicDatagramOnSendStateChanged(&Connection->Datagram); - break; - default: CXPLAT_FRE_ASSERT(FALSE); break; diff --git a/src/core/operation.h b/src/core/operation.h index a29fc309f8..e8e7a9246a 100644 --- a/src/core/operation.h +++ b/src/core/operation.h @@ -31,8 +31,6 @@ typedef enum QUIC_OPERATION_TYPE { QUIC_OPER_TYPE_TIMER_EXPIRED, // A timer expired. QUIC_OPER_TYPE_TRACE_RUNDOWN, // A trace rundown was triggered. QUIC_OPER_TYPE_ROUTE_COMPLETION, // Process route completion event. - QUIC_OPER_TYPE_STREAMS_AVAILABLE, // Indicate streams available to the app. - QUIC_OPER_TYPE_DATAGRAM_STATE_CHANGED, // Indicate datagram state changed to the app. // // All stateless operations follow. diff --git a/src/core/stream_set.c b/src/core/stream_set.c index 6702873210..ba32718a3d 100644 --- a/src/core/stream_set.c +++ b/src/core/stream_set.c @@ -347,7 +347,7 @@ QuicStreamSetInitializeTransportParameters( _Inout_ QUIC_STREAM_SET* StreamSet, _In_ uint64_t BidiStreamCount, _In_ uint64_t UnidiStreamCount, - _In_ BOOLEAN FromResumptionTicket + _In_ BOOLEAN FlushIfUnblocked ) { QUIC_CONNECTION* Connection = QuicStreamSetGetConnection(StreamSet); @@ -424,21 +424,10 @@ QuicStreamSetInitializeTransportParameters( } if (UpdateAvailableStreams) { - if (!FromResumptionTicket) { - QuicStreamSetIndicateStreamsAvailable(StreamSet); - } else { - // - // Defer the streams-available notification to avoid reentrant - // callbacks when called from the resumption ticket path. - // - QUIC_OPERATION* Oper; - if ((Oper = QuicConnAllocOperation(Connection, QUIC_OPER_TYPE_STREAMS_AVAILABLE)) != NULL) { - QuicConnQueueOper(Connection, Oper); - } - } + QuicStreamSetIndicateStreamsAvailable(StreamSet); } - if (MightBeUnblocked && !FromResumptionTicket) { + if (MightBeUnblocked && FlushIfUnblocked) { // // We opened the window, so start send. Rather than checking // the streams to see if one is actually unblocked, we risk starting diff --git a/src/core/stream_set.h b/src/core/stream_set.h index 99e367deb2..35fc171080 100644 --- a/src/core/stream_set.h +++ b/src/core/stream_set.h @@ -131,16 +131,7 @@ QuicStreamSetInitializeTransportParameters( _Inout_ QUIC_STREAM_SET* StreamSet, _In_ uint64_t BidiStreamCount, _In_ uint64_t UnidiStreamCount, - _In_ BOOLEAN FromResumptionTicket - ); - -// -// Indicates available streams to the app via a connection event. -// -_IRQL_requires_max_(PASSIVE_LEVEL) -void -QuicStreamSetIndicateStreamsAvailable( - _Inout_ QUIC_STREAM_SET* StreamSet + _In_ BOOLEAN FlushIfUnblocked ); // From b2c600d328d2acf4ebe2f3fd39c2b922dcdbfecf Mon Sep 17 00:00:00 2001 From: Ray Kong Date: Wed, 11 Mar 2026 18:26:36 -0700 Subject: [PATCH 6/6] Also relax reentrant assertion in QuicStreamIndicateEvent for custom executions --- src/core/connection.c | 6 +++--- src/core/stream.c | 7 +++++-- 2 files changed, 8 insertions(+), 5 deletions(-) diff --git a/src/core/connection.c b/src/core/connection.c index c306a6088d..bdb505593b 100644 --- a/src/core/connection.c +++ b/src/core/connection.c @@ -692,9 +692,9 @@ QuicConnIndicateEvent( // MsQuic shouldn't indicate reentrancy to the app when at all possible. // The general exception to this rule is when the connection is being // closed because the API MUST block until all work is completed, so we - // have to execute the event callbacks inline. Custom executions also - // allow reentrancy because set/get param must be executed inline to not - // block the thread, and the app is expected to handle reentrancy. + // have to execute the event callbacks inline. Custom executions always + // have InlineApiExecution set, which makes this reentrancy check + // unreliable, so it is skipped for custom executions. // CXPLAT_DBG_ASSERT( !Connection->State.InlineApiExecution || diff --git a/src/core/stream.c b/src/core/stream.c index 350dcb557e..ec426b406c 100644 --- a/src/core/stream.c +++ b/src/core/stream.c @@ -470,13 +470,16 @@ QuicStreamIndicateEvent( // or stream is being closed because the API MUST block until all work // is completed, so we have to execute the event callbacks inline. There // is also one additional exception for start complete when StreamStart - // is called synchronously on an MsQuic thread. + // is called synchronously on an MsQuic thread. Custom executions + // always have InlineApiExecution set, which makes this reentrancy + // check unreliable, so it is skipped for custom executions. // CXPLAT_DBG_ASSERT( !Stream->Connection->State.InlineApiExecution || Stream->Connection->State.HandleClosed || Stream->Flags.HandleClosed || - Event->Type == QUIC_STREAM_EVENT_START_COMPLETE); + Event->Type == QUIC_STREAM_EVENT_START_COMPLETE || + MsQuicLib.CustomExecutions); Status = Stream->ClientCallbackHandler( (HQUIC)Stream,