runtime: new vote cache types [wip]

src/flamenco/runtime/fd_bank.h

@@ -143,7 +143,6 @@ FD_PROTOTYPES_BEGIN
 #define FD_BANKS_ITER(X)                                                                                                                                                                                                                               \
   /* type,                             name,                        footprint,                                 align,                                      CoW, limit fork width, has lock */                                                          \
   X(fd_clock_timestamp_votes_global_t, clock_timestamp_votes,       5000000UL,                                 128UL,                                      1,   0,                1    )  /* TODO: This needs to get sized out */                      \
-  X(fd_account_keys_global_t,          vote_account_keys,           3200000UL,                                 128UL,                                      1,   0,                1    )  /* Supports roughly 100k vote accounts */                    \
   X(fd_blockhashes_t,                  block_hash_queue,            sizeof(fd_blockhashes_t),                  alignof(fd_blockhashes_t),                  0,   0,                0    )  /* Block hash queue */                                       \
   X(fd_fee_rate_governor_t,            fee_rate_governor,           sizeof(fd_fee_rate_governor_t),            alignof(fd_fee_rate_governor_t),            0,   0,                0    )  /* Fee rate governor */                                      \
   X(ulong,                             capitalization,              sizeof(ulong),                             alignof(ulong),                             0,   0,                0    )  /* Capitalization */                                         \
@@ -244,12 +243,6 @@ FD_PROTOTYPES_BEGIN
 #undef POOL_NAME
 #undef POOL_T
 
-#define POOL_NAME fd_bank_vote_account_keys_pool
-#define POOL_T    fd_bank_vote_account_keys_t
-#include "../../util/tmpl/fd_pool.c"
-#undef POOL_NAME
-#undef POOL_T
-
 #define POOL_NAME fd_bank_next_epoch_stakes_pool
 #define POOL_T    fd_bank_next_epoch_stakes_t
 #include "../../util/tmpl/fd_pool.c"

src/flamenco/runtime/fd_runtime.c

@@ -1580,41 +1580,6 @@ fd_runtime_prepare_and_execute_txn( fd_exec_slot_ctx_t const *   slot_ctx,
 /* Epoch Boundary                                                             */
 /******************************************************************************/
 
-/* Update the epoch bank stakes cache with the delegated stake values from the slot bank cache.
-The slot bank cache will have been accumulating this epoch, and now we are at an epoch boundary
-we can safely update the epoch stakes cache with the latest values.
-
-In Solana, the stakes cache is updated after every transaction
-  (https://github.com/solana-labs/solana/blob/c091fd3da8014c0ef83b626318018f238f506435/runtime/src/bank.rs#L7587).
-As delegations have to warm up, the contents of the cache will not change inter-epoch. We can therefore update
-the cache only at epoch boundaries.
-
-https://github.com/solana-labs/solana/blob/c091fd3da8014c0ef83b626318018f238f506435/runtime/src/stakes.rs#L65 */
-static void
-fd_update_stake_delegations( fd_exec_slot_ctx_t * slot_ctx,
-                             fd_epoch_info_t *    temp_info ) {
-
-  fd_stake_delegations_t * stake_delegations = fd_stake_delegations_join( fd_bank_stake_delegations_locking_modify( slot_ctx->bank ) );
-
-  /* In one pass, iterate over all the new stake infos and insert the updated values into the epoch stakes cache
-      This assumes that there is enough memory pre-allocated for the stakes cache. */
-  for( ulong idx=temp_info->stake_infos_new_keys_start_idx; idx<temp_info->stake_infos_len; idx++ ) {
-    // Fetch and store the delegation associated with this stake account
-    fd_stake_delegations_update(
-        stake_delegations,
-        &temp_info->stake_infos[idx].account,
-        &temp_info->stake_infos[idx].stake.delegation.voter_pubkey,
-        temp_info->stake_infos[idx].stake.delegation.stake,
-        temp_info->stake_infos[idx].stake.delegation.activation_epoch,
-        temp_info->stake_infos[idx].stake.delegation.deactivation_epoch,
-        temp_info->stake_infos[idx].stake.credits_observed,
-        temp_info->stake_infos[idx].stake.delegation.warmup_cooldown_rate
-    );
-  }
-
-  fd_bank_stake_delegations_end_locking_modify( slot_ctx->bank );
-}
-
 /* Replace the stakes in T-2 (epoch_stakes) by the stakes at T-1 (next_epoch_stakes) */
 static void
 fd_update_epoch_stakes( fd_exec_slot_ctx_t * slot_ctx ) {
@@ -2246,12 +2211,11 @@ fd_runtime_process_new_epoch( fd_exec_slot_ctx_t * slot_ctx,
   }
 
   /* Updates stake history sysvar accumulated values. */
-  fd_stakes_activate_epoch( slot_ctx,
-                            new_rate_activation_epoch,
-                            &temp_info,
-                            runtime_spad );
-
-  fd_update_stake_delegations( slot_ctx, &temp_info );
+  fd_stakes_activate_epoch(
+      slot_ctx,
+      new_rate_activation_epoch,
+      &temp_info,
+      runtime_spad );
 
   /* Refresh vote accounts in stakes cache using updated stake weights, and merges slot bank vote accounts with the epoch bank vote accounts.
     https://github.com/anza-xyz/agave/blob/v2.1.6/runtime/src/stakes.rs#L363-L370 */
@@ -2842,16 +2806,6 @@ fd_runtime_read_genesis( fd_exec_slot_ctx_t *   slot_ctx,
       FD_LOG_ERR(( "Genesis slot 0 execute failed with error %d", err ));
     }
   }
-
-  fd_account_keys_global_t *         vote_account_keys      = fd_bank_vote_account_keys_locking_modify( slot_ctx->bank );
-  uchar *                            pool_mem               = (uchar *)fd_ulong_align_up( (ulong)vote_account_keys + sizeof(fd_account_keys_global_t), fd_account_keys_pair_t_map_align() );
-  fd_account_keys_pair_t_mapnode_t * vote_account_keys_pool = fd_account_keys_pair_t_map_join( fd_account_keys_pair_t_map_new( pool_mem, 100000UL ) );
-  fd_account_keys_pair_t_mapnode_t * vote_account_keys_root = NULL;
-
-  fd_account_keys_account_keys_pool_update( vote_account_keys, vote_account_keys_pool );
-  fd_account_keys_account_keys_root_update( vote_account_keys, vote_account_keys_root );
-
-  fd_bank_vote_account_keys_end_locking_modify( slot_ctx->bank );
 }
 
 /******************************************************************************/

src/flamenco/runtime/program/fd_vote_program.c

@@ -2850,12 +2850,10 @@ remove_vote_account( fd_txn_account_t *   vote_account,
   fd_vote_accounts_pair_global_t_mapnode_t * epoch_vote_accounts_root = fd_vote_accounts_vote_accounts_root_join( epoch_vote_accounts );
 
   if( FD_UNLIKELY( epoch_vote_accounts_pool==NULL ) ) {
-    FD_LOG_DEBUG(("Vote accounts pool does not exist"));
     fd_bank_curr_epoch_stakes_end_locking_modify( bank );
     return;
   }
 
-
   fd_vote_accounts_pair_global_t_mapnode_t vote_acc;
   fd_memcpy( vote_acc.elem.key.uc, vote_account->pubkey->uc, sizeof(fd_pubkey_t) );
   fd_vote_accounts_pair_global_t_mapnode_t * vote_account_entry = fd_vote_accounts_pair_global_t_map_find( epoch_vote_accounts_pool, epoch_vote_accounts_root, &vote_acc );
@@ -2866,48 +2864,16 @@ remove_vote_account( fd_txn_account_t *   vote_account,
   fd_vote_accounts_vote_accounts_pool_update( epoch_vote_accounts, epoch_vote_accounts_pool );
   fd_vote_accounts_vote_accounts_root_update( epoch_vote_accounts, epoch_vote_accounts_root );
   fd_bank_curr_epoch_stakes_end_locking_modify( bank );
-
-  fd_account_keys_global_t * vote_account_keys = fd_bank_vote_account_keys_locking_modify( bank );
-  fd_account_keys_pair_t_mapnode_t * vote_account_keys_pool = fd_account_keys_account_keys_pool_join( vote_account_keys );
-  fd_account_keys_pair_t_mapnode_t * vote_account_keys_root = fd_account_keys_account_keys_root_join( vote_account_keys );
-
-  if( FD_UNLIKELY( vote_account_keys_pool==NULL ) ) {
-    fd_bank_vote_account_keys_end_locking_modify( bank );
-    FD_LOG_DEBUG(("Vote accounts pool does not exist"));
-    return;
-  }
-
-  fd_account_keys_pair_t_mapnode_t account_key;
-  fd_memcpy( account_key.elem.key.uc, vote_account->pubkey->uc, sizeof(fd_pubkey_t) );
-  fd_account_keys_pair_t_mapnode_t * account_key_entry = fd_account_keys_pair_t_map_find( vote_account_keys_pool, vote_account_keys_root, &account_key );
-  if( account_key_entry ) {
-    fd_account_keys_pair_t_map_remove( vote_account_keys_pool, &vote_account_keys_root, account_key_entry );
-  }
-
-  fd_account_keys_account_keys_pool_update( vote_account_keys, vote_account_keys_pool );
-
-  fd_bank_vote_account_keys_end_locking_modify( bank );
 }
 
 static void
 upsert_vote_account( fd_txn_account_t *   vote_account,
                      fd_bank_t *          bank ) {
 
-  fd_vote_accounts_global_t const * vote_accounts = fd_bank_curr_epoch_stakes_locking_query( bank );
+  fd_vote_accounts_global_t * vote_accounts = fd_bank_curr_epoch_stakes_locking_modify( bank );
   fd_vote_accounts_pair_global_t_mapnode_t * stakes_vote_accounts_pool = fd_vote_accounts_vote_accounts_pool_join( vote_accounts );
   fd_vote_accounts_pair_global_t_mapnode_t * stakes_vote_accounts_root = fd_vote_accounts_vote_accounts_root_join( vote_accounts );
 
-  fd_account_keys_global_t *         vote_account_keys      = fd_bank_vote_account_keys_locking_modify( bank );
-  fd_account_keys_pair_t_mapnode_t * vote_account_keys_pool = fd_account_keys_account_keys_pool_join( vote_account_keys );
-  fd_account_keys_pair_t_mapnode_t * vote_account_keys_root = fd_account_keys_account_keys_root_join( vote_account_keys );
-
-  if( FD_UNLIKELY( vote_account_keys_pool==NULL ) ) {
-    fd_bank_vote_account_keys_end_locking_modify( bank );
-    fd_bank_curr_epoch_stakes_end_locking_query( bank );
-    FD_LOG_DEBUG(( "Vote accounts pool does not exist" ));
-    return;
-  }
-
   if( fd_vote_state_versions_is_correct_and_initialized( vote_account ) ) {
     fd_account_keys_pair_t_mapnode_t key;
     fd_memcpy( &key.elem.key, vote_account->pubkey->uc, sizeof(fd_pubkey_t) );
@@ -2916,25 +2882,22 @@ upsert_vote_account( fd_txn_account_t *   vote_account,
     fd_memcpy( &vote_acc.elem.key, vote_account->pubkey->uc, sizeof(fd_pubkey_t) );
 
     // Skip duplicates
-    if( FD_LIKELY( fd_account_keys_pair_t_map_find( vote_account_keys_pool, vote_account_keys_root, &key ) ||
-                   fd_vote_accounts_pair_global_t_map_find( stakes_vote_accounts_pool, stakes_vote_accounts_root, &vote_acc )  ) ) {
-      fd_bank_vote_account_keys_end_locking_modify( bank );
-      fd_bank_curr_epoch_stakes_end_locking_query( bank );
+    if( FD_LIKELY( fd_vote_accounts_pair_global_t_map_find( stakes_vote_accounts_pool, stakes_vote_accounts_root, &vote_acc )  ) ) {
+      fd_bank_curr_epoch_stakes_end_locking_modify( bank );
       return;
     }
-    fd_bank_curr_epoch_stakes_end_locking_query( bank );
 
-    fd_account_keys_pair_t_mapnode_t * new_node = fd_account_keys_pair_t_map_acquire( vote_account_keys_pool );
+    fd_vote_accounts_pair_global_t_mapnode_t * new_node = fd_vote_accounts_pair_global_t_map_acquire( stakes_vote_accounts_pool );
     if( FD_UNLIKELY( !new_node ) ) {
-      FD_LOG_ERR(("Map full"));
+      FD_LOG_CRIT(( "Map full" ));
     }
 
-    fd_memcpy( &new_node->elem.key, vote_account->pubkey, sizeof(fd_pubkey_t));
-    fd_account_keys_pair_t_map_insert( vote_account_keys_pool, &vote_account_keys_root, new_node );
-    fd_bank_vote_account_keys_end_locking_modify( bank );
+    fd_memcpy( &new_node->elem.key, vote_account->pubkey, sizeof(fd_pubkey_t) );
+
+    fd_vote_accounts_pair_global_t_map_insert( stakes_vote_accounts_pool, &stakes_vote_accounts_root, new_node );
+    fd_bank_curr_epoch_stakes_end_locking_modify( bank );
   } else {
-    fd_bank_vote_account_keys_end_locking_modify( bank );
-    fd_bank_curr_epoch_stakes_end_locking_query( bank );
+    fd_bank_curr_epoch_stakes_end_locking_modify( bank );
     remove_vote_account( vote_account, bank );
   }
 }

src/flamenco/runtime/test_bank.c

@@ -115,14 +115,13 @@ main( int argc, char ** argv ) {
 
   /* Set some CoW fields. */
 
-  fd_account_keys_global_t * keys = fd_bank_vote_account_keys_locking_modify( bank9 );
-  keys->account_keys_pool_offset = 100UL;
-  keys->account_keys_root_offset = 100UL;
-  fd_bank_vote_account_keys_end_locking_modify( bank9 );
+  fd_stake_delegations_t * keys = fd_bank_stake_delegations_locking_modify( bank9 );
+  keys->magic = 101UL;
+  fd_bank_stake_delegations_end_locking_modify( bank9 );
 
   /* Check that are now 15 free pool elements. */
 
-  FD_TEST( fd_bank_vote_account_keys_pool_free( fd_bank_get_vote_account_keys_pool( bank9 ) ) == 15UL );
+  FD_TEST( fd_bank_stake_delegations_pool_free( fd_bank_get_stake_delegations_pool( bank9 ) ) == 15UL );
 
   fd_stake_delegations_t * keys2 = fd_bank_stake_delegations_locking_modify( bank9 );
   keys2->magic = 101UL;
@@ -170,19 +169,17 @@ main( int argc, char ** argv ) {
   /* Again, there are no free epoch leader pool elements. */
   FD_TEST( !fd_bank_epoch_leaders_pool_free( fd_bank_get_epoch_leaders_pool( bank11 ) ) );
 
-  fd_account_keys_global_t const * keys3 = fd_bank_vote_account_keys_locking_query( bank11 );
-  FD_TEST( keys3->account_keys_pool_offset == 100UL );
-  FD_TEST( keys3->account_keys_root_offset == 100UL );
-  fd_bank_vote_account_keys_end_locking_query( bank11 );
+  fd_stake_delegations_t const * keys3 = fd_bank_stake_delegations_locking_query( bank11 );
+  FD_TEST( keys3->magic == 101UL );
+  fd_bank_stake_delegations_end_locking_query( bank11 );
 
   fd_stake_delegations_t const * keys4 = fd_bank_stake_delegations_locking_query( bank11 );
   FD_TEST( keys4->magic == 101UL );
   fd_bank_stake_delegations_end_locking_query( bank11 );
 
-  keys = fd_bank_vote_account_keys_locking_modify( bank11 );
-  keys->account_keys_pool_offset = 200UL;
-  keys->account_keys_root_offset = 200UL;
-  fd_bank_vote_account_keys_end_locking_modify( bank11 );
+  keys = fd_bank_stake_delegations_locking_modify( bank11 );
+  keys->magic = 101UL;
+  fd_bank_stake_delegations_end_locking_modify( bank11 );
 
   fd_clock_timestamp_votes_global_t const * votes_const = fd_bank_clock_timestamp_votes_locking_query( bank11 );
   FD_TEST( !votes_const );
@@ -212,10 +209,9 @@ main( int argc, char ** argv ) {
 
   /* Verify that the CoW fields are properly set for bank11 */
 
-  keys3 = fd_bank_vote_account_keys_locking_query( bank11 );
-  FD_TEST( keys3->account_keys_pool_offset == 200UL );
-  FD_TEST( keys3->account_keys_root_offset == 200UL );
-  fd_bank_vote_account_keys_end_locking_query( bank11 );
+  keys3 = fd_bank_stake_delegations_locking_query( bank11 );
+  FD_TEST( keys3->magic == 101UL );
+  fd_bank_stake_delegations_end_locking_query( bank11 );
 
   keys4 = fd_bank_stake_delegations_locking_query( bank11 );
   FD_TEST( keys4->magic == 101UL );
@@ -239,10 +235,9 @@ main( int argc, char ** argv ) {
   FD_TEST( fd_bank_slot_get( bank11 ) == 11UL );
   FD_TEST( fd_bank_capitalization_get( bank11 ) == 0UL );
 
-  keys3 = fd_bank_vote_account_keys_locking_query( bank11 );
-  FD_TEST( keys3->account_keys_pool_offset == 100UL );
-  FD_TEST( keys3->account_keys_root_offset == 100UL );
-  fd_bank_vote_account_keys_end_locking_query( bank11 );
+  keys3 = fd_bank_stake_delegations_locking_query( bank11 );
+  FD_TEST( keys3->magic == 101UL );
+  fd_bank_stake_delegations_end_locking_query( bank11 );
 
   keys4 = fd_bank_stake_delegations_locking_query( bank11 );
   FD_TEST( keys4->magic == 101UL );

src/flamenco/runtime/tests/fd_dump_pb.c

@@ -547,22 +547,6 @@ create_block_context_protobuf_from_block( fd_exec_test_block_context_t * block_c
 
   fd_bank_curr_epoch_stakes_end_locking_query( slot_ctx->bank );
 
-  fd_account_keys_global_t const * vote_account_keys = fd_bank_vote_account_keys_locking_query( slot_ctx->bank );
-  fd_account_keys_pair_t_mapnode_t * vote_account_keys_pool = fd_account_keys_account_keys_pool_join( vote_account_keys );
-  fd_account_keys_pair_t_mapnode_t * vote_account_keys_root = fd_account_keys_account_keys_root_join( vote_account_keys );
-
-
-  /* Dump all new vote accounts */
-  for( fd_account_keys_pair_t_mapnode_t const * curr = fd_account_keys_pair_t_map_minimum_const(
-          vote_account_keys_pool,
-          vote_account_keys_root );
-       curr;
-       curr = fd_account_keys_pair_t_map_successor_const( vote_account_keys_pool, curr ) ) {
-    dump_account_if_not_already_dumped( slot_ctx->funk, slot_ctx->funk_txn, &curr->elem.key, spad, block_context->acct_states, &block_context->acct_states_count, NULL );
-  }
-
-  fd_bank_vote_account_keys_end_locking_query( slot_ctx->bank );
-
   // BlockContext -> EpochContext -> vote_accounts_t_1 (vote accounts at epoch T-1)
   fd_vote_accounts_global_t const * next_epoch_stakes_vaccs = fd_bank_next_epoch_stakes_locking_query( slot_ctx->bank );
   dump_vote_accounts( slot_ctx,

src/flamenco/runtime/tests/harness/fd_block_harness.c

@@ -277,16 +277,7 @@ fd_runtime_fuzz_block_ctx_create( fd_runtime_fuzz_runner_t *           runner,
 
   fd_bank_block_height_set( slot_ctx->bank, test_ctx->slot_ctx.block_height );
 
-  // /* Initialize the current running epoch stake and vote accounts */
-
-  fd_account_keys_global_t * vote_account_keys = fd_bank_vote_account_keys_locking_modify( slot_ctx->bank );
-  pool_mem = (uchar *)fd_ulong_align_up( (ulong)vote_account_keys + sizeof(fd_account_keys_global_t), fd_account_keys_pair_t_map_align() );
-  fd_account_keys_pair_t_mapnode_t * vote_account_keys_pool = fd_account_keys_pair_t_map_join( fd_account_keys_pair_t_map_new( pool_mem, vote_acct_max ) );
-  fd_account_keys_pair_t_mapnode_t * vote_account_keys_root = NULL;
-  fd_account_keys_account_keys_pool_update( vote_account_keys, vote_account_keys_pool );
-  fd_account_keys_account_keys_root_update( vote_account_keys, vote_account_keys_root );
-  fd_bank_vote_account_keys_end_locking_modify( slot_ctx->bank );
-
+  /* Initialize the current running epoch stake and vote accounts */
 
   /* SETUP STAKES HERE */
   fd_vote_accounts_global_t * curr_stakes = fd_bank_curr_epoch_stakes_locking_modify( slot_ctx->bank );

src/flamenco/stakes/Local.mk

@@ -7,6 +7,11 @@ $(call add-objs,fd_stake_delegations,fd_flamenco)
 $(call make-unit-test,test_stake_delegations,test_stake_delegations,fd_flamenco fd_ballet fd_util)
 $(call run-unit-test,test_stake_delegations)
 
+$(call add-hdrs,fd_vote_states.h)
+$(call add-objs,fd_vote_states,fd_flamenco)
+$(call make-unit-test,test_vote_states,test_vote_states,fd_flamenco fd_ballet fd_util)
+$(call run-unit-test,test_vote_states)
+
 # TODO this should not depend on fd_funk
 ifdef FD_HAS_HOSTED
 $(call make-bin,fd_stakes_from_snapshot,fd_stakes_from_snapshot,fd_flamenco fd_funk fd_ballet fd_util)