quic: timeout state fixes

Author: akhinvasara-jumptrading

src/waltz/quic/fd_quic.c

@@ -834,6 +834,30 @@ fd_quic_log_full_hdr( fd_quic_conn_t const * conn,
   return hdr;
 }
 
+static inline void
+fd_quic_set_conn_state( fd_quic_conn_t * conn,
+                        uint             new_state ) {
+
+  uint old_state = conn->state;
+
+  FD_COMPILER_MFENCE();
+
+  int was_timed_out = !!(old_state == FD_QUIC_CONN_STATE_TIMED_OUT);
+  int freeing       = was_timed_out & !!(new_state == FD_QUIC_CONN_STATE_INVALID);
+  int reviving      = was_timed_out & !freeing & !(new_state == FD_QUIC_CONN_STATE_TIMED_OUT);
+
+  fd_quic_metrics_t * metrics = &conn->quic->metrics;
+  metrics->conn_timeout_freed_cnt   += (ulong)freeing;
+  metrics->conn_timeout_revived_cnt += (ulong)reviving;
+
+  metrics->conn_state_cnt[ old_state ]--;
+  metrics->conn_state_cnt[ new_state ]++;
+
+  conn->state = new_state;
+
+  FD_COMPILER_MFENCE();
+}
+
 /* fd_quic_conn_error sets the connection state to aborted.  This does
    not destroy the connection object.  Rather, it will eventually cause
    the connection to be freed during a later fd_quic_service call.
@@ -2277,6 +2301,11 @@ fd_quic_handle_v1_one_rtt( fd_quic_t *      quic,
   /* update expected packet number */
   conn->exp_pkt_number[2] = fd_ulong_max( conn->exp_pkt_number[2], pkt_number+1UL );
 
+  /* Revive conn if it was timed out */
+  if( FD_UNLIKELY( conn->state == FD_QUIC_CONN_STATE_TIMED_OUT ) ) {
+    fd_quic_set_conn_state( conn, FD_QUIC_CONN_STATE_ACTIVE );
+  }
+
   return tot_sz;
 }
 
@@ -2381,13 +2410,6 @@ fd_quic_process_quic_packet_v1( fd_quic_t *     quic,
     return FD_QUIC_PARSE_FAIL;
   }
 
-  /* If connection is timed out, revive it if packet successfully handled. */
-  if( FD_UNLIKELY( conn && conn->state == FD_QUIC_CONN_STATE_TIMED_OUT ) ) {
-    quic->metrics.conn_timeout_revived_cnt++;
-    fd_quic_set_conn_state( conn, FD_QUIC_CONN_STATE_ACTIVE );
-    fd_quic_svc_schedule( state, conn, FD_QUIC_SVC_WAIT );
-  }
-
   cur_ptr += rc;
 
   /* if we get here we parsed all the frames, so ack the packet */
@@ -2868,6 +2890,7 @@ fd_quic_tls_cb_handshake_complete( fd_quic_tls_hs_t * hs,
       return;
 
     case FD_QUIC_CONN_STATE_HANDSHAKE:
+    case FD_QUIC_CONN_STATE_TIMED_OUT:
       if( FD_UNLIKELY( !conn->transport_params_set ) ) { /* unreachable */
         FD_LOG_WARNING(( "Handshake marked as completed but transport params are not set. This is a bug!" ));
         fd_quic_conn_error( conn, FD_QUIC_CONN_REASON_INTERNAL_ERROR, __LINE__ );
@@ -4151,7 +4174,6 @@ fd_quic_free_timed_out( fd_quic_t * quic ) {
   fd_quic_conn_t * conn = fd_quic_conn_at_idx( state, queue->head );
   fd_quic_svc_pop_head( state, queue, conn );
 
-  quic->metrics.conn_timeout_freed_cnt++;
   fd_quic_conn_free( quic, conn );
 }
 

src/waltz/quic/fd_quic_conn.h

@@ -234,17 +234,6 @@ struct fd_quic_conn {
   ulong used_pkt_meta;
 };
 
-inline static void
-fd_quic_set_conn_state( fd_quic_conn_t * conn,
-                        uint             state ) {
-  FD_COMPILER_MFENCE();
-  uint old_state = conn->state;
-  conn->quic->metrics.conn_state_cnt[ old_state ]--;
-  conn->quic->metrics.conn_state_cnt[ state     ]++;
-  conn->state = state;
-  FD_COMPILER_MFENCE();
-}
-
 FD_PROTOTYPES_BEGIN
 
 FD_FN_CONST static inline ulong

src/waltz/quic/tests/test_quic_keep_alive.c

@@ -151,6 +151,44 @@ test_quic_revive( fd_quic_t * client_quic, fd_quic_t * server_quic ) {
   fd_quic_stream_send( stream, "hello", 5, 1 );
   fd_quic_service( client_quic ); /* Send it, aio will receive on server */
   FD_TEST( server_conn->state == FD_QUIC_CONN_STATE_ACTIVE );
+  FD_TEST( server_conn->svc_type == FD_QUIC_SVC_INSTANT || server_conn->svc_type == FD_QUIC_SVC_ACK_TX );
+  FD_TEST( server_quic->metrics.conn_timeout_revived_cnt == 1 );
+}
+
+static void
+test_hs_with_timeout( fd_quic_t * client_quic, fd_quic_t * server_quic ) {
+
+  fd_memset( &server_quic->metrics, 0, sizeof(fd_quic_metrics_t) );
+  fd_memset( &client_quic->metrics, 0, sizeof(fd_quic_metrics_t) );
+
+  FD_TEST( fd_quic_init( server_quic ) );
+  FD_TEST( fd_quic_init( client_quic ) );
+  fd_quic_svc_validate( server_quic );
+  fd_quic_svc_validate( client_quic );
+
+  fd_quic_conn_t * client_conn = fd_quic_connect( client_quic, 0U, 0, 0U, 0 );
+  FD_TEST( client_conn );
+
+  fd_quic_service( client_quic ); /* client initial */
+  fd_quic_service( server_quic ); /* server initial, hs */
+
+  /* trigger a timeout */
+  now += server_quic->config.idle_timeout;
+  fd_quic_service( server_quic );
+
+  FD_TEST( server_conn->state == FD_QUIC_CONN_STATE_TIMED_OUT );
+  FD_TEST( server_conn->svc_time == LONG_MAX );
+  FD_TEST( fd_quic_get_state( server_quic )->svc_queue[ FD_QUIC_SVC_TIMEOUT ].head == server_conn->conn_idx );
+  FD_TEST( server_conn->svc_next == UINT_MAX );
+  FD_TEST( server_conn->svc_prev == UINT_MAX );
+  FD_TEST( server_quic->metrics.conn_timeout_cnt == 1 );
+
+  client_conn->last_activity = now; /* to not trigger self timeout */
+  fd_quic_service( client_quic );   /* client send, trigger revival on server */
+
+  FD_TEST( server_conn->state == FD_QUIC_CONN_STATE_HANDSHAKE_COMPLETE );
+  FD_TEST( server_conn->svc_type == FD_QUIC_SVC_INSTANT );
+  FD_TEST( server_quic->metrics.conn_timeout_revived_cnt == 1 );
 }
 
 static void
@@ -210,6 +248,7 @@ test_quic_timeout_store( fd_quic_t * client_quic, fd_quic_t * server_quic ) {
   client_quic->config.keep_timed_out = 1;
   server_quic->config.keep_timed_out = 1;
   test_quic_revive( client_quic, server_quic );
+  test_hs_with_timeout( client_quic, server_quic );
   test_quic_free_timed_out( client_quic, server_quic );
   client_quic->config.keep_timed_out = 0;
   server_quic->config.keep_timed_out = 0;