feat(replay): publish slot dead frags

Author: lidatong

src/choreo/hfork/fd_hfork.c

@@ -5,13 +5,13 @@ static void
 check( fd_hfork_t *  hfork,
        ulong         total_stake,
        candidate_t * candidate,
-       int           invalid,
+       int           dead,
        fd_hash_t *   our_bank_hash ) {
 
   if( FD_LIKELY( candidate->checked ) ) return; /* already checked this bank hash against our own */
   if( FD_LIKELY( candidate->stake * 100UL / total_stake < 52UL ) ) return; /* not enough stake to compare */
 
-  if( FD_UNLIKELY( invalid ) ) {
+  if( FD_UNLIKELY( dead ) ) {
     char msg[ 4096UL ];
     FD_BASE58_ENCODE_32_BYTES( candidate->key.block_id.uc, _block_id );
     FD_TEST( fd_cstr_printf_check( msg, sizeof( msg ), NULL,
@@ -183,7 +183,10 @@ fd_hfork_count_vote( fd_hfork_t *         hfork,
   /* Get the vtr. */
 
   vtr_t * vtr = vtr_map_query( hfork->vtr_map, *vote_acc, NULL );
-  if( FD_UNLIKELY( !vtr ) ) vtr = vtr_map_insert( hfork->vtr_map, *vote_acc );
+  if( FD_UNLIKELY( !vtr ) ) {
+    FD_TEST( vtr_map_key_cnt( hfork->vtr_map ) < vtr_map_key_max( hfork->vtr_map ) );
+    vtr = vtr_map_insert( hfork->vtr_map, *vote_acc );
+  }
 
   /* Ignore out of order or duplicate votes. */
 
@@ -240,11 +243,10 @@ fd_hfork_count_vote( fd_hfork_t *         hfork,
   blk_t * blk = blk_map_query( hfork->blk_map, *block_id, NULL );
   if( FD_UNLIKELY( !blk ) ) {
     FD_TEST( blk_map_key_cnt( hfork->blk_map ) < blk_map_key_max( hfork->blk_map ) ); /* invariant violation: blk_map full */
-    blk                  = blk_map_insert( hfork->blk_map, *block_id );
-    FD_TEST( blk );
-    blk->bank_hashes     = NULL;
-    blk->replayed        = 0;
-    blk->invalid         = 0;
+    blk              = blk_map_insert( hfork->blk_map, *block_id );
+    blk->bank_hashes = NULL;
+    blk->replayed    = 0;
+    blk->dead        = 0;
   }
   int           found = 0;
   ulong         cnt   = 0;
@@ -256,8 +258,8 @@ fd_hfork_count_vote( fd_hfork_t *         hfork,
     curr = bank_hash_pool_ele( hfork->bank_hash_pool, curr->next );
     cnt++;
   }
-
   if( FD_UNLIKELY( !found ) ) {
+    FD_TEST( bank_hash_pool_free( hfork->bank_hash_pool ) );
     bank_hash_t * ele = bank_hash_pool_ele_acquire( hfork->bank_hash_pool );
     ele->bank_hash    = *bank_hash;
     ele->next         = bank_hash_pool_idx_null( hfork->bank_hash_pool );
@@ -274,7 +276,7 @@ fd_hfork_count_vote( fd_hfork_t *         hfork,
 
   /* Check for hard forks. */
 
-  if( FD_LIKELY( blk->replayed ) ) check( hfork, total_stake, candidate, blk->invalid, &blk->our_bank_hash );
+  if( FD_LIKELY( blk->replayed ) ) check( hfork, total_stake, candidate, blk->dead, &blk->our_bank_hash );
 }
 
 void
@@ -287,14 +289,14 @@ fd_hfork_record_our_bank_hash( fd_hfork_t * hfork,
     blk           = blk_map_insert( hfork->blk_map, *block_id );
     blk->replayed = 1;
   }
-  if( FD_LIKELY( bank_hash ) ) { blk->invalid = 0; blk->our_bank_hash = *bank_hash; }
-  else                           blk->invalid = 1;
+  if( FD_LIKELY( bank_hash ) ) { blk->dead = 0; blk->our_bank_hash = *bank_hash; }
+  else                           blk->dead = 1;
 
   bank_hash_t * curr  = blk->bank_hashes;
   while( FD_LIKELY( curr ) ) {
     candidate_key_t key       = { .block_id = *block_id, .bank_hash = curr->bank_hash };
     candidate_t *   candidate = candidate_map_query( hfork->candidate_map, key, NULL );
-    if( FD_LIKELY( candidate ) ) check( hfork, total_stake, candidate, blk->invalid, &blk->our_bank_hash );
+    if( FD_LIKELY( candidate ) ) check( hfork, total_stake, candidate, blk->dead, &blk->our_bank_hash );
     curr = bank_hash_pool_ele( hfork->bank_hash_pool, curr->next );
   }
 }

src/choreo/hfork/fd_hfork_private.h

@@ -15,7 +15,7 @@ struct blk {
   uint          hash;
   int           forked;        /* whether this block id has hard forked (multiple candidate bank hashes) */
   int           replayed;      /* whether we've replayed */
-  int           invalid;       /* whether we marked the block as invalid during replay (must ignore our_bank_hash) */
+  int           dead;          /* whether we marked the block as dead during replay (must ignore our_bank_hash) */
   fd_hash_t     our_bank_hash; /* our bank hash for this block_id after replay */
   bank_hash_t * bank_hashes;
 };

src/choreo/hfork/test_hfork.c

@@ -4,7 +4,7 @@
 void
 test_hfork_simple( fd_wksp_t * wksp ) {
   ulong  max_live_slots    = 2;
-  ulong  max_vote_accounts = 2;
+  ulong  max_vote_accounts = 4;
 
   void *       mem   = fd_wksp_alloc_laddr( wksp, fd_hfork_align(), fd_hfork_footprint( max_live_slots, max_vote_accounts ), 1UL );
   fd_hfork_t * hfork = fd_hfork_join( fd_hfork_new( mem, max_live_slots, max_vote_accounts, 42, 0 ) );
@@ -22,9 +22,15 @@ test_hfork_simple( fd_wksp_t * wksp ) {
   fd_hash_t block_id2  = { .ul = { slot2 } };
   fd_hash_t bank_hash2 = { .ul = { slot2 } };
 
-  fd_hash_t voters[2] = {
+  ulong     slot3      = 368778156;
+  // fd_hash_t block_id3  = { .ul = { slot3 } };
+  fd_hash_t bank_hash3 = { .ul = { slot3 } };
+
+  fd_hash_t voters[4] = {
     (fd_hash_t){ .ul = { 1 } },
     (fd_hash_t){ .ul = { 2 } },
+    (fd_hash_t){ .ul = { 3 } },
+    (fd_hash_t){ .ul = { 4 } },
   };
 
   fd_hfork_metrics_t metrics = { 0 };
@@ -58,6 +64,22 @@ test_hfork_simple( fd_wksp_t * wksp ) {
   FD_TEST( candidate->stake==51 );
   FD_TEST( candidate->cnt  ==1 );
 
+  /* max bank hashes for a given block_id */
+
+  fd_hfork_count_vote( hfork, &voters[0], &block_id, &bank_hash,  slot3, 1,  100, &metrics );
+  fd_hfork_count_vote( hfork, &voters[1], &block_id, &bank_hash1, slot3, 51, 100, &metrics );
+  fd_hfork_count_vote( hfork, &voters[2], &block_id, &bank_hash2, slot3, 2,  100, &metrics );
+  fd_hfork_count_vote( hfork, &voters[3], &block_id, &bank_hash3, slot3, 3,  100, &metrics );
+
+  blk_t * blk = blk_map_query( hfork->blk_map, block_id, NULL );
+  bank_hash_t * curr = blk->bank_hashes;
+  ulong cnt = 0;
+  while( FD_LIKELY( curr ) ) {
+    curr = bank_hash_pool_ele( hfork->bank_hash_pool, curr->next );
+    cnt++;
+  }
+  FD_TEST( cnt==4 );
+
   fd_wksp_free_laddr( fd_hfork_delete( fd_hfork_leave( hfork ) ) );
 }
 

src/discof/replay/fd_replay_tile.c

@@ -735,6 +735,17 @@ publish_slot_completed( fd_replay_tile_t *  ctx,
   ctx->replay_out->chunk = fd_dcache_compact_next( ctx->replay_out->chunk, sizeof(fd_replay_slot_completed_t), ctx->replay_out->chunk0, ctx->replay_out->wmark );
 }
 
+static void
+publish_slot_dead( fd_replay_tile_t *  ctx,
+                   fd_stem_context_t * stem,
+                   fd_bank_t *         bank ) {
+  fd_replay_slot_dead_t * slot_dead = fd_chunk_to_laddr( ctx->replay_out->mem, ctx->replay_out->chunk );
+  slot_dead->slot                   = fd_bank_slot_get( bank );
+  slot_dead->block_id               = ctx->block_id_arr[ bank->idx ].block_id;
+  fd_stem_publish( stem, ctx->replay_out->idx, REPLAY_SIG_SLOT_DEAD, ctx->replay_out->chunk, sizeof(fd_replay_slot_dead_t), 0UL, 0UL, fd_frag_meta_ts_comp( fd_tickcount() ) );
+  ctx->replay_out->chunk = fd_dcache_compact_next( ctx->replay_out->chunk, sizeof(fd_replay_slot_dead_t), ctx->replay_out->chunk0, ctx->replay_out->wmark );
+}
+
 static void
 replay_block_finalize( fd_replay_tile_t *  ctx,
                        fd_stem_context_t * stem,
@@ -1610,8 +1621,9 @@ can_process_fec( fd_replay_tile_t * ctx ) {
 }
 
 static void
-process_fec_set( fd_replay_tile_t * ctx,
-                 fd_reasm_fec_t *   reasm_fec ) {
+process_fec_set( fd_replay_tile_t *  ctx,
+                 fd_stem_context_t * stem,
+                 fd_reasm_fec_t *    reasm_fec ) {
   long now = fd_log_wallclock();
 
   /* Linking only requires a shared lock because the fields that are
@@ -1722,7 +1734,9 @@ process_fec_set( fd_replay_tile_t * ctx,
   sched_fec->alut_ctx->els          = ctx->published_root_slot;
 
   if( FD_UNLIKELY( !fd_sched_fec_ingest( ctx->sched, sched_fec ) ) ) {
-    fd_banks_mark_bank_dead( ctx->banks, fd_banks_bank_query( ctx->banks, sched_fec->bank_idx ) );
+    fd_bank_t * bank = fd_banks_bank_query( ctx->banks, sched_fec->bank_idx );
+    publish_slot_dead( ctx, stem, bank );
+    fd_banks_mark_bank_dead( ctx->banks, bank );
   }
 }
 
@@ -1861,12 +1875,12 @@ after_credit( fd_replay_tile_t *  ctx,
 
       /* Now we can process all of the FECs. */
       for( ulong i=fec_cnt; i>0UL; i-- ) {
-        process_fec_set( ctx, fecs[i-1UL] );
+        process_fec_set( ctx, stem, fecs[i-1UL] );
       }
     } else {
       /* Standard case. */
       fec = fd_reasm_out( ctx->reasm );
-      process_fec_set( ctx, fec );
+      process_fec_set( ctx, stem, fec );
     }
 
     *charge_busy = 1;
@@ -1915,6 +1929,7 @@ process_solcap_account_update( fd_replay_tile_t *                          ctx,
 
 static void
 process_exec_task_done( fd_replay_tile_t *        ctx,
+                        fd_stem_context_t *       stem,
                         fd_exec_task_done_msg_t * msg,
                         ulong                     sig ) {
   if( FD_UNLIKELY( sig==0UL ) ) {
@@ -1945,6 +1960,7 @@ process_exec_task_done( fd_replay_tile_t *        ctx,
         /* Every transaction in a valid block has to execute.
            Otherwise, we should mark the block as dead.  Also freeze the
            bank if possible. */
+        publish_slot_dead( ctx, stem, bank );
         fd_banks_mark_bank_dead( ctx->banks, bank );
         fd_sched_block_abandon( ctx->sched, bank->idx );
       }
@@ -1959,6 +1975,7 @@ process_exec_task_done( fd_replay_tile_t *        ctx,
         /* Every transaction in a valid block has to sigverify.
            Otherwise, we should mark the block as dead.  Also freeze the
            bank if possible. */
+        publish_slot_dead( ctx, stem, bank );
         fd_banks_mark_bank_dead( ctx->banks, bank );
         fd_sched_block_abandon( ctx->sched, bank->idx );
       }
@@ -2223,7 +2240,7 @@ returnable_frag( fd_replay_tile_t *  ctx,
       maybe_verify_shred_version( ctx );
       break;
     case IN_KIND_EXEC: {
-      process_exec_task_done( ctx, fd_chunk_to_laddr( ctx->in[ in_idx ].mem, chunk ), sig );
+      process_exec_task_done( ctx, stem, fd_chunk_to_laddr( ctx->in[ in_idx ].mem, chunk ), sig );
       break;
     }
     case IN_KIND_POH: {

src/discof/replay/fd_replay_tile.h

@@ -6,7 +6,8 @@
 #include "../../flamenco/types/fd_types_custom.h"
 
 #define REPLAY_SIG_SLOT_COMPLETED (0)
-#define REPLAY_SIG_ROOT_ADVANCED  (1)
+#define REPLAY_SIG_SLOT_DEAD      (1)
+#define REPLAY_SIG_ROOT_ADVANCED  (2)
 #define REPLAY_SIG_RESET          (3)
 #define REPLAY_SIG_BECAME_LEADER  (4)
 
@@ -67,6 +68,12 @@ struct fd_replay_slot_completed {
 
 typedef struct fd_replay_slot_completed fd_replay_slot_completed_t;
 
+struct fd_replay_slot_dead {
+  ulong     slot;
+  fd_hash_t block_id;
+};
+typedef struct fd_replay_slot_dead fd_replay_slot_dead_t;
+
 struct fd_replay_root_advanced {
   ulong bank_idx;
 };