Merge remote-tracking branch 'origin/main' into asiegel/rpc-resurrect

Author: asiegel-jt

CODEOWNERS

@@ -6,6 +6,8 @@
 /src/choreo @lidatong @emwang-jump
 /src/disco/pack @ptaffet-jump @mmcgee-jump
 /src/disco/shred @ptaffet-jump @mmcgee-jump
+/src/disco/reasm @lidatong @emwang-jump
+/src/disco/store @lidatong @emwang-jump
 /src/discof/repair @lidatong @emwang-jump
 /src/waltz/quic @nbridge-jump @ripatel-fd @akhinvasara-jumptrading
 /src/waltz/tls @ripatel-fd @mmcgee-jump

book/api/cli.md

@@ -142,6 +142,16 @@ $ fdctl version
 0.101.11814
 ```
 
+## `shred-version`
+Prints the current shred version of the cluster being joined, according
+to the entrypoints, to standard output and exits. The command writes
+diagnostic messages from logs to `stderr`.
+
+```sh [bash]
+$ fdctl shred-version
+9065
+```
+
 ## `set-identity`
 Changes the identity key of a running validator. The `<keypair>`
 argument is required and must be the path to an Agave style

book/api/metrics-generated.md

@@ -899,3 +899,18 @@
 | <span class="metrics-name">snapin_&#8203;accounts_&#8203;inserted</span> | gauge | Number of accounts inserted during snpashot loading. Might decrease if snapshot load is aborted and restarted |
 
 </div>
+
+## Ipecho Tile
+
+<div class="metrics">
+
+| Metric | Type | Description |
+|--------|------|-------------|
+| <span class="metrics-name">ipecho_&#8203;shred_&#8203;version</span> | gauge | The current shred version used by the validator |
+| <span class="metrics-name">ipecho_&#8203;connection_&#8203;count</span> | gauge | The number of active connections to the ipecho service |
+| <span class="metrics-name">ipecho_&#8203;connections_&#8203;closed_&#8203;ok</span> | counter | The number of connections to the ipecho service that have been made and closed normally |
+| <span class="metrics-name">ipecho_&#8203;connections_&#8203;closed_&#8203;error</span> | counter | The number of connections to the ipecho service that have been made and closed abnormally |
+| <span class="metrics-name">ipecho_&#8203;bytes_&#8203;read</span> | counter | The total number of bytes read from all connections to the ipecho service |
+| <span class="metrics-name">ipecho_&#8203;bytes_&#8203;written</span> | counter | The total number of bytes written to all connections to the ipecho service |
+
+</div>

src/app/fdctl/main.c

@@ -1,5 +1,6 @@
 #include "topology.h"
 #include "config.h"
+#include "../shared/fd_config_file.h"
 #include "../shared/boot/fd_boot.h"
 #include "../shared/commands/configure/configure.h"
 
@@ -116,7 +117,18 @@ action_t * ACTIONS[] = {
 int
 main( int     argc,
       char ** argv ) {
-  return fd_main( argc, argv, 0, (char const *)fdctl_default_config, fdctl_default_config_sz, fd_topo_initialize );
+  fd_config_file_t _default = (fd_config_file_t){
+    .name    = "default",
+    .data    = fdctl_default_config,
+    .data_sz = fdctl_default_config_sz,
+  };
+
+  fd_config_file_t * configs[] = {
+    &_default,
+    NULL
+  };
+
+  return fd_main( argc, argv, 0, configs, fd_topo_initialize );
 }
 
 /* Kind of a hack for now, we sometimes want to view bench generation

src/app/fddev/main.c

@@ -3,5 +3,16 @@
 int
 main( int     argc,
       char ** argv ) {
-  return fd_dev_main( argc, argv, 0, (char const *)fdctl_default_config, fdctl_default_config_sz, fd_topo_initialize );
+  fd_config_file_t _default = (fd_config_file_t){
+    .name    = "default",
+    .data    = fdctl_default_config,
+    .data_sz = fdctl_default_config_sz,
+  };
+
+  fd_config_file_t * configs[] = {
+    &_default,
+    NULL
+  };
+
+  return fd_dev_main( argc, argv, 0, configs, fd_topo_initialize );
 }

src/app/fddev/tests/test_fddev.c

@@ -174,7 +174,7 @@ fddev_test_run( int     argc,
       fd_log_thread_set( "supervisor" );
 
       static config_t config[1];
-      fd_config_load( 0, 0, 1, (char const *)fdctl_default_config, fdctl_default_config_sz, NULL, 0UL, NULL, config );
+      fd_config_load( 0, 0, 1, (char const *)fdctl_default_config, fdctl_default_config_sz, NULL, NULL, 0UL, NULL, 0UL, NULL, config );
       fd_topo_initialize( config );
       config->log.log_fd = fd_log_private_logfile_fd();
       config->log.lock_fd = init_log_memfd();
@@ -195,7 +195,18 @@ fddev_test_run( int     argc,
       else if( FD_UNLIKELY( WEXITSTATUS( wstatus ) ) ) return WEXITSTATUS( wstatus );
     }
   } else {
-    return fd_dev_main( argc, argv, 0, (char const *)fdctl_default_config, fdctl_default_config_sz, fd_topo_initialize );
+    fd_config_file_t _default = (fd_config_file_t){
+      .name    = "default",
+      .data    = fdctl_default_config,
+      .data_sz = fdctl_default_config_sz,
+    };
+
+    fd_config_file_t * configs[] = {
+      &_default,
+      NULL
+    };
+
+    return fd_dev_main( argc, argv, 0, configs, fd_topo_initialize );
   }
 
   return 0;

src/app/firedancer-dev/Local.mk

@@ -15,6 +15,7 @@ $(call add-objs,commands/sim,fd_firedancer_dev)
 $(call add-objs,commands/backtest,fd_firedancer_dev)
 $(call add-objs,commands/snapshot_load,fd_firedancer_dev)
 $(call add-objs,commands/repair,fd_firedancer_dev)
+$(call add-objs,commands/ipecho_server,fd_firedancer_dev)
 
 $(call make-bin,firedancer-dev,main,fd_firedancer_dev fd_firedancer fddev_shared fdctl_shared fdctl_platform fd_discof fd_disco fd_choreo fd_flamenco fd_funk fd_quic fd_tls fd_reedsol fd_waltz fd_tango fd_ballet fd_util firedancer_version,$(SECP256K1_LIBS) $(ROCKSDB_LIBS) $(OPENSSL_LIBS))
 

src/app/firedancer-dev/commands/backtest.c

@@ -23,7 +23,7 @@
 #include "../../../disco/metrics/fd_metrics.h"
 #include "../../../util/pod/fd_pod_format.h"
 #include "../../../discof/replay/fd_replay_notif.h"
-#include "../../../discof/repair/fd_fec_chainer.h"
+#include "../../../discof/reasm/fd_reasm.h"
 #include "../../../flamenco/runtime/fd_runtime_public.h" /* FD_RUNTIME_PUBLIC_ACCOUNT_UPDATE_MSG_MTU */
 #include "../main.h"
 
@@ -109,7 +109,7 @@ backtest_topo( config_t * config ) {
      batches from the CLI-specified source (eg. RocksDB). */
 
   fd_topob_wksp( topo, "repair_repla" );
-  fd_topob_link( topo, "repair_repla", "repair_repla", 65536UL, sizeof(fd_fec_out_t), 1UL );
+  fd_topob_link( topo, "repair_repla", "repair_repla", 65536UL, sizeof(fd_reasm_fec_t), 1UL );
   fd_topob_tile_in( topo, "replay", 0UL, "metric_in", "repair_repla", 0UL, FD_TOPOB_RELIABLE, FD_TOPOB_POLLED );
   fd_topob_tile_out( topo, "back", 0UL, "repair_repla", 0UL );
 
@@ -155,10 +155,10 @@ backtest_topo( config_t * config ) {
      This allows the replay tile to advance its watermark, and publish
      various data structures.  This is an oversimplified barebones mock
      of the tower tile. */
-  fd_topob_wksp( topo, "tower_replay" );
-  fd_topob_link( topo, "tower_replay", "tower_replay", 128UL, 0UL, 1UL );
-  fd_topob_tile_in( topo, "replay", 0UL, "metric_in", "tower_replay", 0UL, FD_TOPOB_RELIABLE, FD_TOPOB_POLLED );
-  fd_topob_tile_out( topo, "back", 0UL, "tower_replay", 0UL );
+  fd_topob_wksp( topo, "root_out" );
+  fd_topob_link( topo, "root_out", "root_out", 128UL, 0UL, 1UL );
+  fd_topob_tile_in( topo, "replay", 0UL, "metric_in", "root_out", 0UL, FD_TOPOB_RELIABLE, FD_TOPOB_POLLED );
+  fd_topob_tile_out( topo, "back", 0UL, "root_out", 0UL );
 
   /**********************************************************************/
   /* Setup replay->stake/send/poh links in topo w/o consumers         */
@@ -350,7 +350,7 @@ backtest_cmd_topo( config_t * config ) {
   backtest_topo( config );
 }
 
-args_t
+static args_t
 configure_args( void ) {
   args_t args = {
     .configure.command = CONFIGURE_CMD_INIT,

src/app/firedancer-dev/commands/ipecho_server.c

@@ -0,0 +1,121 @@
+
+#include "../../shared/commands/configure/configure.h"
+#include "../../shared/commands/run/run.h"
+#include "../../../disco/metrics/fd_metrics.h"
+#include "../../../disco/topo/fd_topob.h"
+
+#include <time.h>
+#include <stdio.h>
+#include <unistd.h>
+
+#define NAME "ipecho-server"
+
+extern fd_topo_obj_callbacks_t * CALLBACKS[];
+
+fd_topo_run_tile_t
+fdctl_tile_run( fd_topo_tile_t const * tile );
+
+static void
+ipecho_topo( fd_topo_t *  topo,
+             char const * name ) {
+  fd_topob_new( topo, name );
+  topo->max_page_size = 1UL<<21UL;
+
+  fd_topob_wksp( topo, "all" );
+  fd_topo_link_t * link = fd_topob_link( topo, "ipecho_out", "all", 4UL, 0UL, 1UL );
+  link->permit_no_consumers = 1;
+  fd_topo_tile_t * tile = fd_topob_tile( topo, "ipecho", "all", "all", 0UL, 0, 0 );
+  tile->ipecho.expected_shred_version = 32;
+  tile->ipecho.entrypoints_cnt = 0UL;
+  tile->ipecho.bind_address = FD_IP4_ADDR(127,0,0,1);
+  tile->ipecho.bind_port    = 12008;
+  fd_topob_tile_out( topo, "ipecho", 0UL, "ipecho_out", 0UL );
+
+  fd_topob_auto_layout( topo, 0 );
+  fd_topob_finish( topo, CALLBACKS );
+}
+
+extern int * fd_log_private_shared_lock;
+
+static void
+ipecho_server_cmd_topo( config_t * config ) {
+  ipecho_topo( &config->topo, config->name );
+}
+
+static args_t
+configure_args( void ) {
+  args_t args = {
+    .configure.command = CONFIGURE_CMD_INIT,
+  };
+
+  ulong stage_idx = 0UL;
+  args.configure.stages[ stage_idx++ ] = &fd_cfg_stage_hugetlbfs;
+  args.configure.stages[ stage_idx++ ] = NULL;
+
+  return args;
+}
+
+void
+ipecho_server_cmd_perm( args_t *    args FD_PARAM_UNUSED,
+                   fd_cap_chk_t *   chk,
+                   config_t const * config ) {
+  args_t c_args = configure_args();
+  configure_cmd_perm( &c_args, chk, config );
+  run_cmd_perm( NULL, chk, config );
+}
+
+static void
+ipecho_server_cmd_fn( args_t *   args,
+                      config_t * config ) {
+  (void)args;
+
+  args_t c_args = configure_args();
+  configure_cmd_fn( &c_args, config );
+
+  run_firedancer_init( config, 1 );
+
+  fd_log_private_shared_lock[ 1 ] = 0;
+  fd_topo_join_workspaces( &config->topo, FD_SHMEM_JOIN_MODE_READ_WRITE );
+  fd_topo_fill( &config->topo );
+
+  ulong tile_idx1 = fd_topo_find_tile( &config->topo, "ipecho", 0UL );
+  fd_topo_tile_t * ipecho_tile1 = &config->topo.tiles[ tile_idx1 ];
+  ulong volatile * const ipecho_metrics1 = fd_metrics_tile( ipecho_tile1->metrics );
+  (void)ipecho_metrics1;
+
+  fd_topo_run_single_process( &config->topo, 2, config->uid, config->gid, fdctl_tile_run );
+
+  ulong tile_idx = fd_topo_find_tile( &config->topo, "ipecho", 0UL );
+  FD_TEST( tile_idx!=ULONG_MAX );
+  fd_topo_tile_t * ipecho_tile = &config->topo.tiles[ tile_idx ];
+
+  sleep(1);
+  ulong volatile * const ipecho_metrics = fd_metrics_tile( ipecho_tile->metrics );
+
+  ulong last_conns = ULONG_MAX;
+  ulong last_closed_ok = ULONG_MAX;
+  ulong last_closed_error = ULONG_MAX;
+
+  for(;;) {
+    ulong ipecho_conns = FD_VOLATILE_CONST( ipecho_metrics[ MIDX( GAUGE, IPECHO, CONNECTION_COUNT ) ] );
+    ulong ipecho_closed_ok = FD_VOLATILE_CONST( ipecho_metrics[ MIDX( COUNTER, IPECHO, CONNECTIONS_CLOSED_OK ) ] );
+    ulong ipecho_closed_error = FD_VOLATILE_CONST( ipecho_metrics[ MIDX( COUNTER, IPECHO, CONNECTIONS_CLOSED_ERROR ) ] );
+
+    if( FD_UNLIKELY( ipecho_conns!=last_conns || ipecho_closed_ok!=last_closed_ok || ipecho_closed_error!=last_closed_error ) ) {
+      FD_LOG_NOTICE(( "connections=%lu closed_ok=%lu closed_err=%lu", ipecho_conns, ipecho_closed_ok, ipecho_closed_error ));
+      last_conns = ipecho_conns;
+      last_closed_ok = ipecho_closed_ok;
+      last_closed_error = ipecho_closed_error;
+    }
+
+    nanosleep( &(struct timespec){ .tv_sec=0, .tv_nsec=1000L*1000L }, NULL );
+  }
+}
+
+action_t fd_action_ipecho_server = {
+  .name = NAME,
+  .args = NULL,
+  .perm = ipecho_server_cmd_perm,
+  .fn   = ipecho_server_cmd_fn,
+  .topo = ipecho_server_cmd_topo,
+};

src/app/firedancer-dev/commands/repair.c

@@ -102,6 +102,7 @@ repair_topo( config_t * config ) {
   fd_topob_wksp( topo, "gossip"       );
   fd_topob_wksp( topo, "metric"       );
   fd_topob_wksp( topo, "fec_sets"     );
+  fd_topob_wksp( topo, "snap_out"     );
 
   fd_topob_wksp( topo, "slot_fseqs"   ); /* fseqs for marked slots eg. turbine slot */
 
@@ -132,14 +133,17 @@ repair_topo( config_t * config ) {
 
   /**/                 fd_topob_link( topo, "repair_net",   "net_repair",   config->net.ingress_buffer_size,          FD_NET_MTU,                    1UL );
   /**/                 fd_topob_link( topo, "repair_sign",  "repair_sign",  128UL,                                    2048UL,                        1UL );
-  FOR(shred_tile_cnt)  fd_topob_link( topo, "shred_repair", "shred_repair", pending_fec_shreds_depth,                 FD_SHRED_REPAIR_MTU,           2UL /* at most 2 msgs per after_frag */ );
+  FOR(shred_tile_cnt)  fd_topob_link( topo, "shred_repair", "shred_repair", pending_fec_shreds_depth,                 FD_SHRED_REPAIR_MTU,           2UL );
 
   FOR(shred_tile_cnt)  fd_topob_link( topo, "repair_shred", "shred_repair", pending_fec_shreds_depth,                 sizeof(fd_ed25519_sig_t),      1UL );
   /**/                 fd_topob_link( topo, "sign_repair",  "sign_repair",  128UL,                                    64UL,                          1UL );
-  /**/                 fd_topob_link( topo, "repair_repla", "repair_repla", 65536UL,                                  sizeof(fd_fec_out_t),    1UL );
+  /**/                 fd_topob_link( topo, "repair_repla", "repair_repla", 65536UL,                                  sizeof(fd_reasm_fec_t),        1UL );
   /**/                 fd_topob_link( topo, "poh_shred",    "poh_shred",    16384UL,                                  USHORT_MAX,                    1UL );
 
-  /**/                 fd_topob_link( topo, "send_txns",    "send_txns",  128UL,                                    FD_TXN_MTU,                    1UL   );
+  /**/                 fd_topob_link( topo, "send_txns",    "send_txns",    128UL,                                    FD_TXN_MTU,                    1UL );
+
+  FD_TEST( sizeof(fd_snapshot_manifest_t)<=(5UL*(1UL<<30UL)) );
+  /**/                 fd_topob_link( topo, "snap_out",     "snap_out",     2UL,                                      5UL*(1UL<<30UL),               1UL );
 
   ushort parsed_tile_to_cpu[ FD_TILE_MAX ];
   /* Unassigned tiles will be floating, unless auto topology is enabled. */
@@ -296,6 +300,7 @@ repair_topo( config_t * config ) {
   FOR(net_tile_cnt)    fd_topob_tile_in(  topo, "repair",  0UL,          "metric_in", "net_repair",    i,            FD_TOPOB_UNRELIABLE, FD_TOPOB_POLLED   ); /* No reliable consumers of networking fragments, may be dropped or overrun */
   /**/                 fd_topob_tile_in(  topo, "repair",  0UL,          "metric_in", "gossip_repai",  0UL,          FD_TOPOB_UNRELIABLE, FD_TOPOB_POLLED   );
   /**/                 fd_topob_tile_in(  topo, "repair",  0UL,          "metric_in", "stake_out",     0UL,          FD_TOPOB_UNRELIABLE, FD_TOPOB_POLLED   );
+                       fd_topob_tile_in(  topo, "repair",  0UL,          "metric_in", "snap_out",      0UL,          FD_TOPOB_RELIABLE,   FD_TOPOB_POLLED   );
   FOR(shred_tile_cnt)  fd_topob_tile_in(  topo, "repair",  0UL,          "metric_in", "shred_repair",  i,            FD_TOPOB_RELIABLE,   FD_TOPOB_POLLED   );
 
   /**/                 fd_topob_tile_in(  topo, "sign",   0UL,         "metric_in",  "repair_sign",  0UL,    FD_TOPOB_UNRELIABLE, FD_TOPOB_POLLED   );
@@ -330,7 +335,8 @@ repair_topo( config_t * config ) {
     fd_topob_tile_in( topo, "scap", 0UL, "metric_in", "replay_scap", 0UL, FD_TOPOB_RELIABLE, FD_TOPOB_POLLED );
 
     fd_topob_tile_uses( topo, scap_tile, root_slot_obj, FD_SHMEM_JOIN_MODE_READ_WRITE );
-    fd_topob_tile_out( topo, "scap",  0UL, "stake_out", 0UL );
+    fd_topob_tile_out( topo, "scap", 0UL, "stake_out", 0UL );
+    fd_topob_tile_out( topo, "scap", 0UL, "snap_out",  0UL );
   }
 
   FD_TEST( link_permit_no_producers( topo, "quic_net"     ) == quic_tile_cnt );
@@ -396,6 +402,7 @@ repair_cmd_fn( args_t *   args,
   }
 
   FD_LOG_NOTICE(( "Repair profiler init" ));
+  fd_topo_print_log( 1, &config->topo );
 
   args_t configure_args = {
     .configure.command = CONFIGURE_CMD_INIT,