feat(choreo): dup handling, gossip int, bug fixes, missing logic

Author: lidatong

book/api/metrics-generated.md

@@ -1152,3 +1152,20 @@
 | <span class="metrics-name">snapls_&#8203;incremental_&#8203;accounts_&#8203;hashed</span> | gauge | Number of accounts hashed for the incremental snapshot during snapshot loading. Might decrease if snapshot load is aborted and restarted |
 
 </div>
+
+## Tower Tile
+
+<div class="metrics">
+
+| Metric | Type | Description |
+|--------|------|-------------|
+| <span class="metrics-name">tower_&#8203;ancestor_&#8203;rollback</span> | counter | Rollback to an ancestor of our prev vote (can't vote) |
+| <span class="metrics-name">tower_&#8203;sibling_&#8203;confirmed</span> | counter | Duplicate sibling got confirmed (can't vote) |
+| <span class="metrics-name">tower_&#8203;same_&#8203;fork</span> | counter | Same fork as prev vote (can vote) |
+| <span class="metrics-name">tower_&#8203;switch_&#8203;pass</span> | counter | Prev vote was on a different fork, but we are allowed to switch (can vote) |
+| <span class="metrics-name">tower_&#8203;switch_&#8203;fail</span> | counter | Prev vote was on a different fork, and we are not allowed to switch (can't vote) |
+| <span class="metrics-name">tower_&#8203;lockout_&#8203;fail</span> | counter | Locked out (can't vote) |
+| <span class="metrics-name">tower_&#8203;threshold_&#8203;fail</span> | counter | Did not pass threshold check (can't vote) |
+| <span class="metrics-name">tower_&#8203;propagated_&#8203;fail</span> | counter | Prev leader block did not propagate (can't vote) |
+
+</div>

src/app/firedancer-dev/commands/send_test/send_test_helpers.c

@@ -215,7 +215,7 @@ encode_vote( send_test_ctx_t * ctx, fd_tower_slot_done_t * slot_done ) {
   /* Create minimal mock tower with one vote */
   uchar tower_mem[ FD_TOWER_FOOTPRINT ] __attribute__((aligned(FD_TOWER_ALIGN)));
   fd_tower_t * tower = fd_tower_join( fd_tower_new( tower_mem ) );
-  fd_tower_votes_push_tail( tower, (fd_tower_vote_t){ .slot = vote_slot, .conf = 1 } );
+  fd_tower_push_tail( tower, (fd_tower_vote_t){ .slot = vote_slot, .conf = 1 } );
 
   /* Mock values */
   fd_lockout_offset_t lockouts_scratch[1];

src/app/firedancer/config/default.toml

@@ -472,6 +472,15 @@ user = ""
     # abort with an error.
     expected_genesis_hash = ""
 
+    # Firedancer can process at most this many slots without rooting in
+    # the consensus rules before it must begin evicting.
+    #
+    # This is an estimate and should be set as generously as possible to
+    # allow for brief anomalies such as the validator disconnecting from
+    # part of the cluster due to data center issues.  Roughly, at 400 ms
+    # per slot, the default allows for 30 minutes without rooting.
+    max_unrooted_slots = 4096
+
 # This section configures the "funk" account database.  Currently, funk
 # stores all Solana accounts.  In future versions of Firedancer, most
 # accounts will be offloaded to the "groove" database.

src/app/firedancer/topology.c

@@ -1,6 +1,7 @@
 #include "topology.h"
 
 #include "../../ballet/lthash/fd_lthash.h"
+#include "../../choreo/fd_choreo_base.h"
 #include "../../discof/reasm/fd_reasm.h"
 #include "../../discof/poh/fd_poh.h"
 #include "../../discof/replay/fd_exec.h"
@@ -408,6 +409,7 @@ fd_topo_initialize( config_t * config ) {
 
   /* TODO: Explain this .... USHORT_MAX is not dcache max */
   ulong pending_fec_shreds_depth = fd_ulong_min( fd_ulong_pow2_up( config->tiles.shred.max_pending_shred_sets * FD_REEDSOL_DATA_SHREDS_MAX ), USHORT_MAX + 1 /* dcache max */ );
+  ulong max_unrooted_slots       = config->firedancer.consensus.max_unrooted_slots;
 
   /*                                  topo, link_name,      wksp_name,      depth,                                    mtu,                           burst */
   /**/                 fd_topob_link( topo, "gossip_net",   "net_gossip",   32768UL,                                  FD_NET_MTU,                    1UL );
@@ -481,7 +483,7 @@ fd_topo_initialize( config_t * config ) {
 
   FOR(shred_tile_cnt)  fd_topob_link( topo, "shred_out",    "shred_out",    pending_fec_shreds_depth,                 FD_SHRED_OUT_MTU,              3UL ); /* TODO: Pretty sure burst of 3 is incorrect here */
   FOR(shred_tile_cnt)  fd_topob_link( topo, "repair_shred", "shred_out",    pending_fec_shreds_depth,                 sizeof(fd_ed25519_sig_t),      1UL ); /* TODO: Also pending_fec_shreds_depth? Seems wrong */
-  /**/                 fd_topob_link( topo, "tower_out",    "tower_out",    1024UL,                                   sizeof(fd_tower_slot_done_t),  1UL );
+  /**/                 fd_topob_link( topo, "tower_out",    "tower_out",    max_unrooted_slots,                       sizeof(fd_tower_msg_t),        1UL );
   /**/                 fd_topob_link( topo, "send_out",     "send_out",     128UL,                                    FD_TPU_RAW_MTU,                1UL );
 
                        fd_topob_link( topo, "replay_exec",  "replay_exec",  16384UL,                                  sizeof(fd_exec_task_msg_t),    1UL );
@@ -668,6 +670,7 @@ fd_topo_initialize( config_t * config ) {
   /**/                 fd_topob_tile_in(    topo, "replay",  0UL,          "metric_in", "poh_replay",   0UL,          FD_TOPOB_RELIABLE,   FD_TOPOB_POLLED );
   FOR(exec_tile_cnt)   fd_topob_tile_in(    topo, "exec",    i,            "metric_in", "replay_exec",  0UL,          FD_TOPOB_RELIABLE,   FD_TOPOB_POLLED );
   /**/                 fd_topob_tile_in (   topo, "tower",   0UL,          "metric_in", "genesi_out",   0UL,          FD_TOPOB_RELIABLE,   FD_TOPOB_POLLED );
+  /**/                 fd_topob_tile_in (   topo, "tower",   0UL,          "metric_in", "gossip_out",   0UL,          FD_TOPOB_RELIABLE,   FD_TOPOB_POLLED );
   /**/                 fd_topob_tile_in (   topo, "tower",   0UL,          "metric_in", "replay_out",   0UL,          FD_TOPOB_RELIABLE,   FD_TOPOB_POLLED );
   if( snapshots_enabled ) {
                        fd_topob_tile_in (   topo, "tower",   0UL,          "metric_in", "snapin_manif", 0UL,          FD_TOPOB_RELIABLE,   FD_TOPOB_POLLED );
@@ -1205,9 +1208,10 @@ fd_topo_configure_tile( fd_topo_tile_t * tile,
 
   } else if( FD_UNLIKELY( !strcmp( tile->name, "tower" ) ) ) {
 
-    strncpy( tile->tower.identity_key_path, config->paths.identity_key, sizeof(tile->tower.identity_key_path) );
-    strncpy( tile->tower.vote_acc_path, config->paths.vote_account, sizeof(tile->tower.vote_acc_path) );
-    strncpy( tile->tower.ledger_path, config->paths.base, sizeof(tile->tower.ledger_path) );
+    tile->tower.slot_max = config->firedancer.consensus.max_unrooted_slots;
+    strncpy( tile->tower.identity_key, config->paths.identity_key, sizeof(tile->tower.identity_key) );
+    strncpy( tile->tower.vote_account, config->paths.vote_account, sizeof(tile->tower.vote_account) );
+    strncpy( tile->tower.base_path, config->paths.base, sizeof(tile->tower.base_path) );
 
   } else if( FD_UNLIKELY( !strcmp( tile->name, "send" ) ) ) {
 

src/app/shared/fd_config.h

@@ -133,6 +133,10 @@ struct fd_configf {
     char host[ 256 ];
   } gossip;
 
+  struct {
+    ulong max_unrooted_slots;
+  } consensus;
+
   struct {
     struct {
       uint max_local_full_effective_age;

src/app/shared/fd_config_parse.c

@@ -80,6 +80,8 @@ fd_config_extract_podf( uchar *        pod,
                         fd_configf_t * config ) {
   CFG_POP      ( cstr,   gossip.host                                         );
 
+  CFG_POP      ( ulong,  consensus.max_unrooted_slots                        );
+
   CFG_POP      ( uint,   layout.exec_tile_count                              );
   CFG_POP      ( uint,   layout.sign_tile_count                              );
   CFG_POP      ( uint,   layout.gossvf_tile_count                            );

src/app/shared_dev/commands/dev.c

@@ -109,13 +109,6 @@ update_config_for_dev( fd_config_t * config ) {
       shred->shred.expected_shred_version = shred_version;
     }
   }
-  ulong store_id = fd_topo_find_tile( &config->topo, "storei", 0 );
-  if( FD_UNLIKELY( store_id!=ULONG_MAX ) ) {
-    fd_topo_tile_t * storei = &config->topo.tiles[ store_id ];
-    if( FD_LIKELY( storei->store_int.expected_shred_version==(ushort)0 ) ) {
-      storei->store_int.expected_shred_version = shred_version;
-    }
-  }
 }
 
 /* Run Firedancer entirely in a single process for development and