tower: various fixes

src/app/firedancer-dev/Local.mk

@@ -15,6 +15,7 @@ $(call add-objs,commands/sim,fd_firedancer_dev)
 $(call add-objs,commands/backtest,fd_firedancer_dev)
 $(call add-objs,commands/snapshot_load,fd_firedancer_dev)
 $(call add-objs,commands/repair,fd_firedancer_dev)
+$(call add-objs,commands/tower,fd_firedancer_dev)
 $(call add-objs,commands/ipecho_server,fd_firedancer_dev)
 $(call add-objs,commands/gossip_dump,fd_firedancer_dev)
 

src/app/firedancer-dev/commands/tower.c

@@ -0,0 +1,183 @@
+/* The tower command prints the tower forks tree structure and leaves.
+   This is a standalone application that can be run to inspect the tower
+   tile's fork structure. */
+
+#include "../../shared/fd_config.h" /* config_t */
+#include "../../shared_dev/commands/dev.h"
+#include "../../../discof/tower/fd_tower_tile.c"
+#include "../../../choreo/tower/fd_tower_forks.h"
+
+#include <stdio.h>
+#include <unistd.h>
+
+fd_topo_run_tile_t
+fdctl_tile_run( fd_topo_tile_t const * tile );
+
+/* ctx_t is defined in fd_tower_tile.c, we just need to access it */
+
+static void
+tower_ctx_wksp( args_t *          args,
+                config_t *        config,
+                ctx_t **          tower_ctx,
+                fd_topo_wksp_t ** tower_wksp ) {
+  (void)args;
+
+  fd_topo_t * topo = &config->topo;
+
+  ulong tile_id = fd_topo_find_tile( topo, "tower", 0UL );
+  if( FD_UNLIKELY( tile_id==ULONG_MAX ) ) FD_LOG_ERR(( "tower tile not found" ));
+
+  fd_topo_tile_t * tile = &topo->tiles[ tile_id ];
+
+  /* Get the workspace that contains the tile's scratch memory */
+  ulong scratch_wksp_id = topo->objs[ tile->tile_obj_id ].wksp_id;
+  if( FD_UNLIKELY( scratch_wksp_id>=topo->wksp_cnt ) ) FD_LOG_ERR(( "invalid workspace id %lu for tile scratch", scratch_wksp_id ));
+
+  fd_topo_wksp_t * _tower_wksp = &topo->workspaces[ scratch_wksp_id ];
+  fd_topo_join_workspace( topo, _tower_wksp, FD_SHMEM_JOIN_MODE_READ_ONLY );
+
+  /* Access the tower tile scratch memory where tower_tile_ctx is stored */
+  void * scratch = fd_topo_obj_laddr( topo, tile->tile_obj_id );
+  if( FD_UNLIKELY( !scratch ) ) FD_LOG_ERR(( "Failed to access tower tile scratch memory" ));
+
+  FD_SCRATCH_ALLOC_INIT( l, scratch );
+  ctx_t * _tower_ctx = FD_SCRATCH_ALLOC_APPEND( l, alignof(ctx_t), sizeof(ctx_t) );
+
+  *tower_ctx  = _tower_ctx;
+  *tower_wksp = _tower_wksp;
+}
+
+static void
+print_all_forks( fd_wksp_t * wksp, ctx_t * tower_ctx, fd_forks_t * forks ) {
+  printf( "\n[Tower Forks]\n" );
+  printf( "=============\n" );
+  printf( "%-15s | %-15s | %-10s | %-10s\n", "Slot", "Parent Slot", "Voted", "Confirmed" );
+  printf( "%-15s-+-%-15s-+-%-10s-+-%-10s\n", "---------------", "---------------", "----------", "----------" );
+
+  /* Iterate through all map slots */
+  ulong tower_forks_gaddr = fd_wksp_gaddr_fast( tower_ctx->wksp, forks->tower_forks );
+  fd_tower_forks_t * map = (fd_tower_forks_t *)fd_wksp_laddr_fast( wksp, tower_forks_gaddr );
+  ulong slot_count = 0;
+
+  for( ulong slot_idx = 0UL; slot_idx < fd_tower_forks_slot_cnt( map ); slot_idx++ ) {
+    fd_tower_forks_t * fork = &map[ slot_idx ];
+    /* Check if key is valid (not MAP_KEY_NULL which is ULONG_MAX) */
+    if( !fd_tower_forks_key_inval( fork->slot ) ) {
+      printf( "%-15lu | ", fork->slot );
+      if( fork->parent_slot == ULONG_MAX ) {
+        printf( "%-15s | ", "NULL" );
+      } else {
+        printf( "%-15lu | ", fork->parent_slot );
+      }
+      printf( "%-10s | ", fork->voted ? "Yes" : "No" );
+      printf( "%-10s\n", fork->confirmed ? "Yes" : "No" );
+      slot_count++;
+    }
+  }
+
+  printf( "\n[Tower Leaves]\n" );
+  printf( "==============\n" );
+
+  ulong         tower_leaves_dlist_gaddr = fd_wksp_gaddr_fast( tower_ctx->wksp, forks->tower_leaves_dlist );
+  fd_tower_leaves_dlist_t * leaves_dlist = (fd_tower_leaves_dlist_t *)fd_wksp_laddr_fast( wksp, tower_leaves_dlist_gaddr );
+  ulong tower_leaves_pool_gaddr = fd_wksp_gaddr_fast( tower_ctx->wksp, forks->tower_leaves_pool );
+  fd_tower_leaf_t * leaves_pool = (fd_tower_leaf_t *)fd_wksp_laddr_fast( wksp, tower_leaves_pool_gaddr );
+
+  ulong leaf_count = 0;
+  for( fd_tower_leaves_dlist_iter_t iter = fd_tower_leaves_dlist_iter_fwd_init( leaves_dlist, leaves_pool );
+                                          !fd_tower_leaves_dlist_iter_done( iter, leaves_dlist, leaves_pool );
+                                    iter = fd_tower_leaves_dlist_iter_fwd_next( iter, leaves_dlist, leaves_pool ) ) {
+    fd_tower_leaf_t * leaf = fd_tower_leaves_dlist_iter_ele( iter, leaves_dlist, leaves_pool );
+    if( FD_LIKELY( leaf ) ) {
+      fd_tower_forks_t * fork = fd_tower_forks_query( map, leaf->slot, NULL );
+      printf( "Leaf slot: %lu", leaf->slot );
+      if( fork->voted )     printf( " [voted]"     );
+      if( fork->confirmed ) printf( " [confirmed]" );
+      printf( "\n" );
+      leaf_count++;
+    }
+  }
+  printf( "\nTotal leaves: %lu\n", leaf_count );
+  printf( "Total slots: %lu\n", slot_count );
+  printf( "\n" );
+}
+
+static void
+tower_cmd_fn_forks( args_t *   args,
+                    config_t * config ) {
+  ctx_t *          tower_ctx;
+  fd_topo_wksp_t * tower_wksp;
+  tower_ctx_wksp( args, config, &tower_ctx, &tower_wksp );
+
+  ulong forks_laddr = fd_wksp_gaddr_fast( tower_ctx->wksp, tower_ctx->forks );
+  fd_forks_t * forks = (fd_forks_t *)fd_wksp_laddr( tower_wksp->wksp, forks_laddr );
+
+  for( ;; ) {
+    print_all_forks( tower_wksp->wksp, tower_ctx, forks );
+    sleep( 1 );
+  }
+}
+
+static const char * HELP =
+  "\n\n"
+  "usage: tower [-h] {forks}\n"
+  "\n"
+  "positional arguments:\n"
+  "  {forks}\n"
+  "    forks              prints the tower forks tree structure and leaves\n"
+  "\n"
+  "optional arguments:\n"
+  "  -h, --help            show this help message and exit\n";
+
+static const char * FORKS_HELP =
+  "\n\n"
+  "usage: tower forks [-h]\n"
+  "\n"
+  "optional arguments:\n"
+  "  -h, --help            show this help message and exit\n";
+
+void
+tower_cmd_help( char const * arg ) {
+  if      ( FD_LIKELY( !arg                    ) ) FD_LOG_NOTICE(( "%s", HELP           ));
+  else if ( FD_LIKELY( !strcmp( arg, "forks" ) ) ) FD_LOG_NOTICE(( "%s", FORKS_HELP    ));
+  else                                                 FD_LOG_NOTICE(( "%s", HELP           ));
+}
+
+void
+tower_cmd_args( int *    pargc,
+                char *** pargv,
+                args_t * args ) {
+
+  /* help */
+  args->tower.help = fd_env_strip_cmdline_contains( pargc, pargv, "--help" );
+  args->tower.help = args->tower.help || fd_env_strip_cmdline_contains( pargc, pargv, "-h" );
+
+  /* positional arg */
+  args->tower.pos_arg = (*pargv)[0];
+  if( FD_UNLIKELY( !args->tower.pos_arg ) ) {
+    args->tower.help = 1;
+    return;
+  }
+
+  (*pargc)--;
+}
+
+static void
+tower_cmd_fn( args_t *   args,
+              config_t * config ) {
+
+  if( args->tower.help ) {
+    tower_cmd_help( args->tower.pos_arg );
+    return;
+  }
+
+  if     ( !strcmp( args->tower.pos_arg, "forks" ) ) tower_cmd_fn_forks( args, config );
+  else                                                    tower_cmd_help( NULL );
+}
+
+action_t fd_action_tower = {
+  .name = "tower",
+  .args = tower_cmd_args,
+  .fn   = tower_cmd_fn,
+  .perm = dev_cmd_perm,
+};

src/app/firedancer-dev/main.c

@@ -203,6 +203,7 @@ extern action_t fd_action_sim;
 extern action_t fd_action_backtest;
 extern action_t fd_action_snapshot_load;
 extern action_t fd_action_repair;
+extern action_t fd_action_tower;
 extern action_t fd_action_shred_version;
 extern action_t fd_action_ipecho_server;
 extern action_t fd_action_send_test;
@@ -238,6 +239,7 @@ action_t * ACTIONS[] = {
   &fd_action_backtest,
   &fd_action_snapshot_load,
   &fd_action_repair,
+  &fd_action_tower,
   &fd_action_shred_version,
   &fd_action_ipecho_server,
   &fd_action_send_test,

src/app/shared/fd_action.h

@@ -147,6 +147,11 @@ union fdctl_args {
     ulong max_contact;
     int   compact_mode;
   } gossip;
+
+  struct {
+    char const * pos_arg;
+    int          help;
+  } tower;
 };
 
 typedef union fdctl_args args_t;

src/choreo/tower/fd_tower.c

@@ -236,8 +236,9 @@ switch_check( fd_tower_t const  * tower,
     /* Iterate over all the leaves of all forks */
     fd_tower_leaf_t  * leaf = fd_tower_leaves_dlist_iter_ele( iter, forks->tower_leaves_dlist, forks->tower_leaves_pool );
     ulong candidate_slot = leaf->slot;
+    ulong lca = fd_forks_lowest_common_ancestor( forks, candidate_slot, last_vote_slot );
 
-    if( fd_forks_is_slot_descendant( forks, fd_forks_lowest_common_ancestor( forks, candidate_slot, last_vote_slot ), switch_slot ) ) {
+    if( lca != ULONG_MAX && fd_forks_is_slot_descendant( forks, lca, switch_slot ) ) {
 
       /* This candidate slot may be considered for the switch proof, if
          it passes the following conditions:
@@ -811,6 +812,9 @@ fd_tower_verify( fd_tower_t const * tower ) {
 void
 fd_tower_print( fd_tower_t const * tower, ulong root ) {
   FD_LOG_NOTICE( ( "\n\n[Tower]" ) );
+
+  if( FD_UNLIKELY( fd_tower_empty( tower ) ) ) return;
+
   ulong max_slot = 0;
 
   /* Determine spacing. */

src/choreo/tower/fd_tower_forks.c

@@ -108,7 +108,10 @@ fd_forks_lowest_common_ancestor( fd_forks_t * forks,
     if( fork1->slot > fork2->slot                 ) fork1 = fd_tower_forks_query( forks->tower_forks, fork1->parent_slot, NULL );
     else                                            fork2 = fd_tower_forks_query( forks->tower_forks, fork2->parent_slot, NULL );
   }
-  FD_LOG_CRIT(( "invalid forks" ));
+  /* If we reach here, then one of the slots is on a minority fork who's
+     ancestor that connected it to the main fork has been pruned (i.e.)
+     we have a dangling leaf right now! There is no LCA in this case. */
+  return ULONG_MAX;
 }
 
 fd_hash_t const *

src/choreo/voter/fd_voter.h

@@ -79,8 +79,8 @@ fd_voter_root_slot( uchar const * vote_account_data ) {
   fd_voter_t const * voter = (fd_voter_t const *)fd_type_pun_const( vote_account_data );
   ulong              cnt   = voter->votes_cnt;
   switch( voter->kind ) {
-  case FD_VOTER_V3: { uchar root_option = fd_uchar_load_1_fast( (uchar *)&voter->votes_v3[cnt] ); return root_option ? fd_ulong_load_8_fast( (uchar *)&voter->votes_v3[cnt] ) : ULONG_MAX; }
-  case FD_VOTER_V2: { uchar root_option = fd_uchar_load_1_fast( (uchar *)&voter->votes_v2[cnt] ); return root_option ? fd_ulong_load_8_fast( (uchar *)&voter->votes_v2[cnt] ) : ULONG_MAX; }
+  case FD_VOTER_V3: { uchar root_option = fd_uchar_load_1_fast( (uchar *)&voter->votes_v3[cnt] ); return root_option ? fd_ulong_load_8_fast( (uchar *)&voter->votes_v3[cnt] + 1UL ) : ULONG_MAX; }
+  case FD_VOTER_V2: { uchar root_option = fd_uchar_load_1_fast( (uchar *)&voter->votes_v2[cnt] ); return root_option ? fd_ulong_load_8_fast( (uchar *)&voter->votes_v2[cnt] + 1UL ) : ULONG_MAX; }
   default: FD_LOG_CRIT(( "unhandled kind %u", voter->kind ));
   }
 }

src/discof/tower/fd_tower_tile.c

@@ -77,6 +77,8 @@ typedef struct {
 } in_ctx_t;
 
 typedef struct {
+  fd_wksp_t * wksp; /* workspace */
+
   ulong       seed; /* map seed */
   int         checkpt_fd;
   int         restore_fd;
@@ -153,21 +155,18 @@ scratch_align( void ) {
 FD_FN_PURE static inline ulong
 scratch_footprint( FD_PARAM_UNUSED fd_topo_tile_t const * tile ) {
   ulong slot_max     = tile->tower.max_live_slots;
-  int   lg_slot_max  = fd_ulong_find_msb( fd_ulong_pow2_up( slot_max ) ) + 1;
   FD_LOG_DEBUG(( "hfork footprint %lu", fd_hfork_footprint( slot_max, FD_VOTER_MAX ) ));
   ulong l = FD_LAYOUT_INIT;
-  l = FD_LAYOUT_APPEND( l, alignof(ctx_t),         sizeof(ctx_t)                                        );
-  l = FD_LAYOUT_APPEND( l, fd_ghost_align(),       fd_ghost_footprint( 2*slot_max, FD_VOTER_MAX )       );
-  l = FD_LAYOUT_APPEND( l, fd_hfork_align(),       fd_hfork_footprint( slot_max, FD_VOTER_MAX )         );
-  l = FD_LAYOUT_APPEND( l, fd_notar_align(),       fd_notar_footprint( tile->tower.max_vote_lookahead ) );
-  l = FD_LAYOUT_APPEND( l, fd_tower_align(),       fd_tower_footprint()                                 );
-  l = FD_LAYOUT_APPEND( l, fd_tower_accts_align(), fd_tower_accts_footprint( FD_VOTER_MAX )             );
-  l = FD_LAYOUT_APPEND( l, fd_forks_align(),       fd_forks_footprint( slot_max, FD_VOTER_MAX )         );
-  l = FD_LAYOUT_APPEND( l, fd_tower_align(),       fd_tower_footprint()                                 );
-  l = FD_LAYOUT_APPEND( l, fd_epoch_stakes_align(), fd_epoch_stakes_footprint( slot_max )               );
-  l = FD_LAYOUT_APPEND( l, fd_tower_forks_align(), fd_tower_forks_footprint( lg_slot_max )              );
-  l = FD_LAYOUT_APPEND( l, fd_tower_align(),       fd_tower_footprint()                                 ); /* ctx->tower_spare */
-  l = FD_LAYOUT_APPEND( l, notif_align(),          notif_footprint( slot_max )                          );
+  l = FD_LAYOUT_APPEND( l, alignof(ctx_t),          sizeof(ctx_t)                                        );
+  l = FD_LAYOUT_APPEND( l, fd_ghost_align(),        fd_ghost_footprint( 2*slot_max, FD_VOTER_MAX )       );
+  l = FD_LAYOUT_APPEND( l, fd_hfork_align(),        fd_hfork_footprint( slot_max, FD_VOTER_MAX )         );
+  l = FD_LAYOUT_APPEND( l, fd_notar_align(),        fd_notar_footprint( tile->tower.max_vote_lookahead ) );
+  l = FD_LAYOUT_APPEND( l, fd_tower_align(),        fd_tower_footprint()                                 );
+  l = FD_LAYOUT_APPEND( l, fd_tower_accts_align(),  fd_tower_accts_footprint( FD_VOTER_MAX )             );
+  l = FD_LAYOUT_APPEND( l, fd_forks_align(),        fd_forks_footprint( slot_max, FD_VOTER_MAX )         );
+  l = FD_LAYOUT_APPEND( l, fd_tower_align(),        fd_tower_footprint()                                 ); /* ctx->tower_spare */
+  l = FD_LAYOUT_APPEND( l, fd_epoch_stakes_align(), fd_epoch_stakes_footprint( slot_max )                );
+  l = FD_LAYOUT_APPEND( l, notif_align(),           notif_footprint( slot_max )                          );
   return FD_LAYOUT_FINI( l, scratch_align() );
 }
 
@@ -474,7 +473,12 @@ replay_slot_completed( ctx_t *                      ctx,
   /* Query our on-chain vote acct and reconcile with our local tower. */
 
   int found = query_vote_state_from_accdb( ctx->accdb, &xid, ctx->vote_account, ctx->our_vote_acct );
-  if( FD_LIKELY( found ) ) fd_tower_reconcile( ctx->tower, ctx->root_slot, ctx->our_vote_acct );
+  if( FD_LIKELY( found ) ) {
+    fd_tower_reconcile( ctx->tower, ctx->root_slot, ctx->our_vote_acct );
+    /* Sanity check that most recent vote in tower exists in tower forks */
+    fd_tower_vote_t const * last_vote = fd_tower_peek_tail_const( ctx->tower );
+    FD_TEST( !last_vote || fd_forks_query( ctx->forks, last_vote->slot ) );
+  }
 
   /* Insert the vote acct addrs and stakes from the bank into accts. */
 
@@ -782,19 +786,20 @@ unprivileged_init( fd_topo_t *      topo,
   void  * accts = FD_SCRATCH_ALLOC_APPEND( l, fd_tower_accts_align(),   fd_tower_accts_footprint( FD_VOTER_MAX )             );
   void  * forks = FD_SCRATCH_ALLOC_APPEND( l, fd_forks_align(),         fd_forks_footprint( slot_max, FD_VOTER_MAX )         );
   void  * spare = FD_SCRATCH_ALLOC_APPEND( l, fd_tower_align(),         fd_tower_footprint()                                 );
-  void  * stake = FD_SCRATCH_ALLOC_APPEND( l, fd_epoch_stakes_align(), fd_epoch_stakes_footprint( slot_max )                 );
+  void  * stake = FD_SCRATCH_ALLOC_APPEND( l, fd_epoch_stakes_align(),  fd_epoch_stakes_footprint( slot_max )                 );
   void  * notif = FD_SCRATCH_ALLOC_APPEND( l, notif_align(),            notif_footprint( slot_max )                          );
   FD_SCRATCH_ALLOC_FINI( l, scratch_align() );
 
-  ctx->ghost       = fd_ghost_join      ( fd_ghost_new      ( ghost, 2*slot_max, FD_VOTER_MAX, 42UL ) ); /* FIXME seed */
-  ctx->hfork       = fd_hfork_join      ( fd_hfork_new      ( hfork, slot_max, FD_VOTER_MAX, ctx->seed, tile->tower.hard_fork_fatal ) );
-  ctx->notar       = fd_notar_join      ( fd_notar_new      ( notar, tile->tower.max_vote_lookahead ) );
-  ctx->tower       = fd_tower_join      ( fd_tower_new      ( tower                                 ) );
-  ctx->tower_accts = fd_tower_accts_join( fd_tower_accts_new( accts, FD_VOTER_MAX                   ) );
-  ctx->forks       = fd_forks_join      ( fd_forks_new      ( forks, slot_max, FD_VOTER_MAX         ) );
-  ctx->tower_spare = fd_tower_join      ( fd_tower_new      ( spare                                 ) );
-  ctx->slot_stakes = fd_epoch_stakes_join( fd_epoch_stakes_new( stake, slot_max                   ) );
-  ctx->notif       = notif_join         ( notif_new         ( notif, slot_max                       ) );
+  ctx->wksp        = topo->workspaces[ topo->objs[ tile->tile_obj_id ].wksp_id ].wksp;
+  ctx->ghost       = fd_ghost_join       ( fd_ghost_new       ( ghost, 2*slot_max, FD_VOTER_MAX, 42UL ) ); /* FIXME seed */
+  ctx->hfork       = fd_hfork_join       ( fd_hfork_new       ( hfork, slot_max, FD_VOTER_MAX, ctx->seed, tile->tower.hard_fork_fatal ) );
+  ctx->notar       = fd_notar_join       ( fd_notar_new       ( notar, tile->tower.max_vote_lookahead ) );
+  ctx->tower       = fd_tower_join       ( fd_tower_new       ( tower                                 ) );
+  ctx->tower_accts = fd_tower_accts_join ( fd_tower_accts_new ( accts, FD_VOTER_MAX                   ) );
+  ctx->forks       = fd_forks_join       ( fd_forks_new       ( forks, slot_max, FD_VOTER_MAX         ) );
+  ctx->tower_spare = fd_tower_join       ( fd_tower_new       ( spare                                 ) );
+  ctx->slot_stakes = fd_epoch_stakes_join( fd_epoch_stakes_new( stake, slot_max                       ) );
+  ctx->notif       = notif_join          ( notif_new          ( notif, slot_max                       ) );
   FD_TEST( ctx->ghost );
   FD_TEST( ctx->hfork );
   FD_TEST( ctx->notar );