runtime: replacing vote caches with fd_vote_states_t

Author: ibhatt-jumptrading

config/extra/with-handholding.mk

@@ -10,3 +10,4 @@ CPPFLAGS+=-DFD_FUNK_HANDHOLDING=1
 CPPFLAGS+=-DFD_RUNTIME_ERR_HANDHOLDING=1
 CPPFLAGS+=-DFD_FOREST_USE_HANDHOLDING=1
 CPPFLAGS+=-DFD_STAKES_USE_HANDHOLDING=1
+CPPFLAGS+=-DFD_VOTE_STATES_USE_HANDHOLDING=1

src/discof/replay/fd_exec.h

@@ -18,11 +18,11 @@
 static inline ulong
 generate_stake_weight_msg( fd_exec_slot_ctx_t *              slot_ctx,
                            ulong                             epoch,
-                           fd_vote_accounts_global_t const * vote_accounts,
+                           fd_vote_states_t const *          vote_states,
                            ulong *                           stake_weight_msg_out ) {
   fd_stake_weight_msg_t *     stake_weight_msg = (fd_stake_weight_msg_t *)fd_type_pun( stake_weight_msg_out );
   fd_vote_stake_weight_t *    stake_weights    = stake_weight_msg->weights;
-  ulong                       staked_cnt       = fd_stake_weights_by_node( vote_accounts, stake_weights );
+  ulong                       staked_cnt       = fd_stake_weights_by_node( vote_states, stake_weights );
   fd_epoch_schedule_t const * epoch_schedule = fd_bank_epoch_schedule_query( slot_ctx->bank );
 
   stake_weight_msg->epoch          = epoch;

src/discof/replay/fd_replay_tile.c

@@ -358,33 +358,29 @@ publish_stake_weights( fd_replay_tile_ctx_t * ctx,
                        fd_exec_slot_ctx_t *   slot_ctx ) {
   fd_epoch_schedule_t const * epoch_schedule = fd_bank_epoch_schedule_query( slot_ctx->bank );
 
-  fd_vote_accounts_global_t const * epoch_stakes = fd_bank_epoch_stakes_locking_query( slot_ctx->bank );
-  fd_vote_accounts_pair_global_t_mapnode_t * epoch_stakes_root = fd_vote_accounts_vote_accounts_root_join( epoch_stakes );
-
-  if( epoch_stakes_root!=NULL ) {
+  fd_vote_states_t const * vote_states_prev_prev = fd_bank_vote_states_prev_prev_locking_query( slot_ctx->bank );
+  do {
     ulong * stake_weights_msg = fd_chunk_to_laddr( ctx->stake_out->mem, ctx->stake_out->chunk );
-    ulong epoch = fd_slot_to_leader_schedule_epoch( epoch_schedule, fd_bank_slot_get( slot_ctx->bank ) );
-    ulong stake_weights_sz = generate_stake_weight_msg( slot_ctx, epoch - 1, epoch_stakes, stake_weights_msg );
-    ulong stake_weights_sig = 4UL;
+    ulong epoch               = fd_slot_to_leader_schedule_epoch( epoch_schedule, fd_bank_slot_get( slot_ctx->bank ) );
+    ulong stake_weights_sz    = generate_stake_weight_msg( slot_ctx, epoch - 1, vote_states_prev_prev, stake_weights_msg );
+    ulong stake_weights_sig   = 4UL;
     fd_stem_publish( stem, 0UL, stake_weights_sig, ctx->stake_out->chunk, stake_weights_sz, 0UL, 0UL, fd_frag_meta_ts_comp( fd_tickcount() ) );
     ctx->stake_out->chunk = fd_dcache_compact_next( ctx->stake_out->chunk, stake_weights_sz, ctx->stake_out->chunk0, ctx->stake_out->wmark );
     FD_LOG_NOTICE(("sending current epoch stake weights - epoch: %lu, stake_weight_cnt: %lu, start_slot: %lu, slot_cnt: %lu", stake_weights_msg[0], stake_weights_msg[1], stake_weights_msg[2], stake_weights_msg[3]));
-  }
-
-  fd_bank_epoch_stakes_end_locking_query( slot_ctx->bank );
-  fd_vote_accounts_global_t const * next_epoch_stakes = fd_bank_next_epoch_stakes_locking_query( slot_ctx->bank );
-  fd_vote_accounts_pair_global_t_mapnode_t * next_epoch_stakes_root = fd_vote_accounts_vote_accounts_root_join( next_epoch_stakes );
+  } while( 0 );
+  fd_bank_vote_states_prev_prev_end_locking_query( slot_ctx->bank );
 
-  if( next_epoch_stakes_root!=NULL ) {
+  fd_vote_states_t const * vote_states_prev = fd_bank_vote_states_prev_locking_query( slot_ctx->bank );
+  do {
     ulong * stake_weights_msg = fd_chunk_to_laddr( ctx->stake_out->mem, ctx->stake_out->chunk );
     ulong   epoch             = fd_slot_to_leader_schedule_epoch( epoch_schedule, fd_bank_slot_get( slot_ctx->bank ) ); /* epoch */
-    ulong stake_weights_sz = generate_stake_weight_msg( slot_ctx, epoch, next_epoch_stakes, stake_weights_msg );
-    ulong stake_weights_sig = 4UL;
+    ulong   stake_weights_sz  = generate_stake_weight_msg( slot_ctx, epoch, vote_states_prev, stake_weights_msg );
+    ulong   stake_weights_sig = 4UL;
     fd_stem_publish( stem, 0UL, stake_weights_sig, ctx->stake_out->chunk, stake_weights_sz, 0UL, 0UL, fd_frag_meta_ts_comp( fd_tickcount() ) );
     ctx->stake_out->chunk = fd_dcache_compact_next( ctx->stake_out->chunk, stake_weights_sz, ctx->stake_out->chunk0, ctx->stake_out->wmark );
     FD_LOG_NOTICE(("sending next epoch stake weights - epoch: %lu, stake_weight_cnt: %lu, start_slot: %lu, slot_cnt: %lu", stake_weights_msg[0], stake_weights_msg[1], stake_weights_msg[2], stake_weights_msg[3]));
-  }
-  fd_bank_next_epoch_stakes_end_locking_query( slot_ctx->bank );
+  } while( 0 );
+  fd_bank_vote_states_prev_end_locking_query( slot_ctx->bank );
 }
 
 static void
@@ -529,7 +525,6 @@ restore_slot_ctx( fd_replay_tile_ctx_t * ctx,
 
   fd_exec_slot_ctx_t * recovered_slot_ctx = fd_exec_slot_ctx_recover( ctx->slot_ctx,
                                                                       manifest_global );
-
   if( !recovered_slot_ctx ) {
     FD_LOG_ERR(( "Failed to restore slot context from snapshot manifest!" ));
   }
@@ -692,20 +687,18 @@ init_after_snapshot( fd_replay_tile_ctx_t * ctx ) {
   memset( block_id, 0, sizeof(fd_hash_t) );
   block_id->key[0] = UCHAR_MAX; /* TODO: would be good to have the actual block id of the snapshot slot */
 
-  fd_vote_accounts_global_t const * vote_accounts = fd_bank_curr_epoch_stakes_locking_query( ctx->slot_ctx->bank );
-
-  fd_vote_accounts_pair_global_t_mapnode_t * vote_accounts_pool = fd_vote_accounts_vote_accounts_pool_join( vote_accounts );
-  fd_vote_accounts_pair_global_t_mapnode_t * vote_accounts_root = fd_vote_accounts_vote_accounts_root_join( vote_accounts );
+  fd_vote_states_t const * vote_states = fd_bank_vote_states_locking_query( ctx->slot_ctx->bank );
 
   fd_bank_hash_cmp_t * bank_hash_cmp = ctx->bank_hash_cmp;
-  for( fd_vote_accounts_pair_global_t_mapnode_t * curr = fd_vote_accounts_pair_global_t_map_minimum( vote_accounts_pool, vote_accounts_root );
-       curr;
-       curr = fd_vote_accounts_pair_global_t_map_successor( vote_accounts_pool, curr ) ) {
-    bank_hash_cmp->total_stake += curr->elem.stake;
+
+  uchar iter_mem[FD_VOTE_STATE_ITER_FOOTPRINT];
+  for( fd_vote_states_iter_t * iter = fd_vote_states_iter_init( vote_states, iter_mem ); !fd_vote_states_iter_done( iter ); fd_vote_states_iter_next( iter ) ) {
+    fd_vote_state_ele_t const * vote_state = fd_vote_states_iter_ele( iter );
+    bank_hash_cmp->total_stake += vote_state->stake;
   }
   bank_hash_cmp->watermark = snapshot_slot;
 
-  fd_bank_curr_epoch_stakes_end_locking_query( ctx->slot_ctx->bank );
+  fd_bank_vote_states_end_locking_query( ctx->slot_ctx->bank );
 
   ulong root = snapshot_slot;
   if( FD_LIKELY( root > fd_fseq_query( ctx->published_wmark ) ) ) {
@@ -750,8 +743,8 @@ init_from_snapshot( fd_replay_tile_ctx_t * ctx,
   fd_bank_lthash_end_locking_query( ctx->slot_ctx->bank );
 
   fd_runtime_update_leaders( ctx->slot_ctx->bank,
-    fd_bank_slot_get( ctx->slot_ctx->bank ),
-    ctx->runtime_spad );
+      fd_bank_slot_get( ctx->slot_ctx->bank ),
+      ctx->runtime_spad );
 
   fd_runtime_read_genesis( ctx->slot_ctx,
                            ctx->genesis,
@@ -1055,86 +1048,30 @@ publish_votes_to_plugin( fd_replay_tile_ctx_t * ctx,
   fd_fork_t * fork = fd_fork_frontier_ele_query( ctx->forks->frontier, &bank_slot, NULL, ctx->forks->pool );
   if( FD_UNLIKELY ( !fork  ) ) return;
 
-  fd_vote_accounts_global_t const *          epoch_stakes      = fd_bank_epoch_stakes_locking_query( ctx->slot_ctx->bank );
-  fd_vote_accounts_pair_global_t_mapnode_t * epoch_stakes_pool = fd_vote_accounts_vote_accounts_pool_join( epoch_stakes );
-  fd_vote_accounts_pair_global_t_mapnode_t * epoch_stakes_root = fd_vote_accounts_vote_accounts_root_join( epoch_stakes );
-
+  fd_vote_states_t const * vote_states = fd_bank_vote_states_locking_query( ctx->slot_ctx->bank );
   ulong i = 0;
   FD_SPAD_FRAME_BEGIN( ctx->runtime_spad ) {
-  for( fd_vote_accounts_pair_global_t_mapnode_t const * n = fd_vote_accounts_pair_global_t_map_minimum_const( epoch_stakes_pool, epoch_stakes_root );
-       n && i < FD_CLUSTER_NODE_CNT;
-       n = fd_vote_accounts_pair_global_t_map_successor_const( epoch_stakes_pool, n ) ) {
-    if( n->elem.stake == 0 ) continue;
-
-    uchar * data     = (uchar *)&n->elem.value + n->elem.value.data_offset;
-    ulong   data_len = n->elem.value.data_len;
-
-    int err;
-    fd_vote_state_versioned_t * vsv = fd_bincode_decode_spad(
-        vote_state_versioned, ctx->runtime_spad,
-        data,
-        data_len,
-        &err );
-    if( FD_UNLIKELY( err ) ) {
-      FD_LOG_ERR(( "Unexpected failure in decoding vote state %d", err ));
-    }
-
-    fd_pubkey_t node_pubkey;
-    ulong       commission;
-    ulong       epoch_credits;
-    fd_vote_epoch_credits_t const * _epoch_credits;
-    ulong       root_slot;
-
-    switch( vsv->discriminant ) {
-      case fd_vote_state_versioned_enum_v0_23_5:
-        node_pubkey   = vsv->inner.v0_23_5.node_pubkey;
-        commission    = vsv->inner.v0_23_5.commission;
-        _epoch_credits = deq_fd_vote_epoch_credits_t_cnt( vsv->inner.v0_23_5.epoch_credits ) == 0 ? NULL : deq_fd_vote_epoch_credits_t_peek_tail_const( vsv->inner.v0_23_5.epoch_credits );
-        epoch_credits = _epoch_credits==NULL ? 0UL : _epoch_credits->credits - _epoch_credits->prev_credits;
-        root_slot     = vsv->inner.v0_23_5.root_slot;
-        break;
-      case fd_vote_state_versioned_enum_v1_14_11:
-        node_pubkey   = vsv->inner.v1_14_11.node_pubkey;
-        commission    = vsv->inner.v1_14_11.commission;
-        _epoch_credits = deq_fd_vote_epoch_credits_t_cnt( vsv->inner.v1_14_11.epoch_credits ) == 0 ? NULL : deq_fd_vote_epoch_credits_t_peek_tail_const( vsv->inner.v1_14_11.epoch_credits );
-        epoch_credits = _epoch_credits==NULL ? 0UL : _epoch_credits->credits - _epoch_credits->prev_credits;
-        root_slot     = vsv->inner.v1_14_11.root_slot;
-        break;
-      case fd_vote_state_versioned_enum_current:
-        node_pubkey   = vsv->inner.current.node_pubkey;
-        commission    = vsv->inner.current.commission;
-        _epoch_credits = deq_fd_vote_epoch_credits_t_cnt( vsv->inner.current.epoch_credits ) == 0 ? NULL : deq_fd_vote_epoch_credits_t_peek_tail_const( vsv->inner.current.epoch_credits );
-        epoch_credits = _epoch_credits==NULL ? 0UL : _epoch_credits->credits - _epoch_credits->prev_credits;
-        root_slot     = vsv->inner.v0_23_5.root_slot;
-        break;
-      default:
-        __builtin_unreachable();
-    }
-
-    fd_clock_timestamp_vote_t_mapnode_t query;
-    memcpy( query.elem.pubkey.uc, n->elem.key.uc, 32UL );
-    fd_clock_timestamp_votes_global_t const * clock_timestamp_votes = fd_bank_clock_timestamp_votes_locking_query( ctx->slot_ctx->bank );
-    fd_clock_timestamp_vote_t_mapnode_t * timestamp_votes_root  = fd_clock_timestamp_votes_votes_root_join( clock_timestamp_votes );
-    fd_clock_timestamp_vote_t_mapnode_t * timestamp_votes_pool  = fd_clock_timestamp_votes_votes_pool_join( clock_timestamp_votes );
-
-    fd_clock_timestamp_vote_t_mapnode_t * res = fd_clock_timestamp_vote_t_map_find( timestamp_votes_pool, timestamp_votes_root, &query );
+  uchar iter_mem[FD_VOTE_STATE_ITER_FOOTPRINT];
+  for( fd_vote_states_iter_t * iter = fd_vote_states_iter_init( vote_states, iter_mem ); !fd_vote_states_iter_done( iter ); fd_vote_states_iter_next( iter ) ) {
+    fd_vote_state_ele_t const * vote_state = fd_vote_states_iter_ele( iter );
 
     fd_vote_update_msg_t * msg = (fd_vote_update_msg_t *)(dst + sizeof(ulong) + i*112U);
     memset( msg, 0, 112U );
-    memcpy( msg->vote_pubkey, n->elem.key.uc, sizeof(fd_pubkey_t) );
-    memcpy( msg->node_pubkey, node_pubkey.uc, sizeof(fd_pubkey_t) );
-    msg->activated_stake = n->elem.stake;
-    msg->last_vote       = res == NULL ? 0UL : res->elem.slot;
-    msg->root_slot       = root_slot;
-    msg->epoch_credits   = epoch_credits;
-    msg->commission      = (uchar)commission;
+    memcpy( msg->vote_pubkey, &vote_state->vote_account, sizeof(fd_pubkey_t) );
+    memcpy( msg->node_pubkey, &vote_state->node_account, sizeof(fd_pubkey_t) );
+    msg->activated_stake = vote_state->stake;
+    msg->last_vote       = vote_state->last_vote_slot;
+    msg->root_slot       = 0UL; /* TODO: needs to be populated in vote_state */
+    msg->epoch_credits   = 0UL; /* TODO: needs to be populated in vote_state */
+    msg->commission      = (uchar)vote_state->commission;
     msg->is_delinquent   = (uchar)fd_int_if(fd_bank_slot_get( ctx->slot_ctx->bank ) >= 128UL, msg->last_vote <= fd_bank_slot_get( ctx->slot_ctx->bank ) - 128UL, msg->last_vote == 0);
+
+
     ++i;
-    fd_bank_clock_timestamp_votes_end_locking_query( ctx->slot_ctx->bank );
   }
   } FD_SPAD_FRAME_END;
 
-  fd_bank_epoch_stakes_end_locking_query( ctx->slot_ctx->bank );
+  fd_bank_vote_states_end_locking_query( ctx->slot_ctx->bank );
 
   *(ulong *)dst = i;
 

src/discof/restore/utils/fd_ssload.c

@@ -136,23 +136,6 @@ fd_ssload_recover( fd_snapshot_manifest_t * manifest,
   fd_bank_execution_fees_set( slot_ctx->bank, manifest->collector_fees );
   fd_bank_priority_fees_set( slot_ctx->bank, 0UL );
 
-  /* FIXME: Remove the magic number here. */
-  fd_clock_timestamp_votes_global_t * clock_timestamp_votes = fd_bank_clock_timestamp_votes_locking_modify( slot_ctx->bank );
-  uchar * clock_pool_mem = (uchar *)fd_ulong_align_up( (ulong)clock_timestamp_votes + sizeof(fd_clock_timestamp_votes_global_t), fd_clock_timestamp_vote_t_map_align() );
-  fd_clock_timestamp_vote_t_mapnode_t * clock_pool = fd_clock_timestamp_vote_t_map_join( fd_clock_timestamp_vote_t_map_new(clock_pool_mem, 30000UL ) );
-  clock_timestamp_votes->votes_pool_offset = (ulong)fd_clock_timestamp_vote_t_map_leave( clock_pool) - (ulong)clock_timestamp_votes;
-  clock_timestamp_votes->votes_root_offset = 0UL;
-  fd_bank_clock_timestamp_votes_end_locking_modify( slot_ctx->bank );
-
-  for( ulong i=0UL; i<manifest->vote_accounts_len; i++ ) {
-    fd_snapshot_manifest_vote_account_t * account = &manifest->vote_accounts[ i ];
-    fd_pubkey_t vote_account_pubkey;
-    fd_memcpy( vote_account_pubkey.uc, account->vote_account_pubkey, 32UL );
-    if( FD_LIKELY( account->last_slot || account->stake ) ) {
-      fd_vote_record_timestamp_vote_with_slot( &vote_account_pubkey, account->last_timestamp, account->last_slot, slot_ctx->bank );
-    }
-  }
-
   /* Update last restart slot
      https://github.com/solana-labs/solana/blob/30531d7a5b74f914dde53bfbb0bc2144f2ac92bb/runtime/src/bank.rs#L2152
 

src/discof/writer/fd_writer_tile.c

@@ -272,7 +272,6 @@ after_frag( fd_writer_tile_ctx_t * ctx,
           ctx->funk,
           ctx->funk_txn,
           txn_ctx,
-          ctx->spad,
           ctx->bank,
           ctx->capture_ctx );