solcap: v2

Author: nipatel-jump

contrib/test/run_solcap_tests.sh

@@ -82,8 +82,5 @@ $OBJDIR/bin/firedancer-dev configure init all --config $DUMP/$LEDGER/devnet-3987
 $OBJDIR/bin/firedancer-dev backtest --config $DUMP/$LEDGER/devnet-398736132_current.toml
 $OBJDIR/bin/firedancer-dev configure fini all --config $DUMP/$LEDGER/devnet-398736132_current.toml
 
-$OBJDIR/bin/fd_solcap_import $DUMP/$LEDGER/bank_hash_details/ $DUMP/$LEDGER/solana.solcap
-$OBJDIR/bin/fd_solcap_diff $DUMP/$LEDGER/solana.solcap $DUMP/$LEDGER/fd.solcap -v 4
-
 # check that the ledger is not corrupted after a run
 check_ledger_checksum

src/app/firedancer-dev/commands/backtest.c

@@ -27,7 +27,7 @@
 #include "../../../discof/tower/fd_tower_tile.h"
 #include "../../../discof/replay/fd_exec.h"
 #include "../../../ballet/lthash/fd_lthash.h"
-#include "../../../flamenco/runtime/context/fd_capture_ctx.h"
+#include "../../../discof/capture/fd_capture_ctx.h"
 #include "../../../disco/pack/fd_pack_cost.h"
 #include "../../../flamenco/progcache/fd_progcache_admin.h"
 
@@ -102,6 +102,14 @@ backtest_topo( config_t * config ) {
   #define FOR(cnt) for( ulong i=0UL; i<cnt; i++ )
   FOR(exec_tile_cnt) fd_topob_tile( topo, "exec", "exec", "metric_in", cpu_idx++, 0, 0 );
 
+  /**********************************************************************/
+  /* Add the capture tile to topo                                       */
+  /**********************************************************************/
+  if( solcap_enabled ) {
+    fd_topob_wksp( topo, "captur" );
+    fd_topob_tile( topo, "captur", "captur", "metric_in", cpu_idx++, 0, 0 );
+  }
+
   /**********************************************************************/
   /* Add the snapshot tiles to topo                                       */
   /**********************************************************************/
@@ -283,15 +291,7 @@ backtest_topo( config_t * config ) {
   /**********************************************************************/
   fd_topob_wksp( topo, "exec_replay" );
 
-  /* If solcap is enabled, we need to overload this link to also send
-     solcap account updates to the replay tile. We can't use a separate
-     link for this without introducing a race. This will get removed with solcap V2. */
-  if( FD_UNLIKELY( solcap_enabled ) ) {
-    /* TODO: remove this with solcap V2 */
-    FOR(exec_tile_cnt) fd_topob_link( topo, "exec_replay", "exec_replay", 1024UL, FD_CAPTURE_CTX_ACCOUNT_UPDATE_MSG_FOOTPRINT, 1UL );
-  } else {
-    FOR(exec_tile_cnt) fd_topob_link( topo, "exec_replay", "exec_replay", 16384UL, sizeof(fd_exec_task_done_msg_t), 1UL );
-  }
+  FOR(exec_tile_cnt) fd_topob_link( topo, "exec_replay", "exec_replay", 16384UL, sizeof(fd_exec_task_done_msg_t), 1UL );
 
   FOR(exec_tile_cnt) fd_topob_tile_out( topo, "exec", i, "exec_replay", i );
   FOR(exec_tile_cnt) fd_topob_tile_in( topo, "replay", 0UL, "metric_in", "exec_replay", i, FD_TOPOB_RELIABLE, FD_TOPOB_POLLED );
@@ -300,6 +300,18 @@ backtest_topo( config_t * config ) {
   /* Setup the shared objs used by replay and exec tiles                */
   /**********************************************************************/
 
+  if( FD_UNLIKELY( solcap_enabled ) ) {
+    /* 32 sections of SOLCAP_WRITE_ACCOUNT_DATA_MTU bytes each ≈ 4MB.
+       This is done to ideally avoid cache thrashing and allow for all
+       the links to sit on L3 cache. */
+    fd_topob_link( topo, "cap_repl", "captur", 32UL, SOLCAP_WRITE_ACCOUNT_DATA_MTU, 1UL );
+    fd_topob_tile_out( topo, "replay", 0UL, "cap_repl", 0UL );
+    fd_topob_tile_in( topo, "captur", 0UL, "metric_in", "cap_repl", 0UL, FD_TOPOB_RELIABLE, FD_TOPOB_POLLED );
+    FOR(exec_tile_cnt) fd_topob_link( topo, "cap_exec", "captur", 32UL, SOLCAP_WRITE_ACCOUNT_DATA_MTU, 1UL );
+    FOR(exec_tile_cnt) fd_topob_tile_out( topo, "exec", i, "cap_exec", i );
+    FOR(exec_tile_cnt) fd_topob_tile_in( topo, "captur", 0UL, "metric_in", "cap_exec", i, FD_TOPOB_RELIABLE, FD_TOPOB_POLLED );
+  }
+
   fd_topob_wksp( topo, "store" );
   fd_topo_obj_t * store_obj = setup_topo_store( topo, "store", config->firedancer.store.max_completed_shred_sets, 1 );
   fd_topob_tile_uses( topo, backt_tile, store_obj, FD_SHMEM_JOIN_MODE_READ_WRITE );

src/app/firedancer-dev/main.c

@@ -110,6 +110,7 @@ extern fd_topo_run_tile_t fd_tile_archiver_writer;
 extern fd_topo_run_tile_t fd_tile_archiver_playback;
 extern fd_topo_run_tile_t fd_tile_shredcap;
 extern fd_topo_run_tile_t fd_tile_vinyl;
+extern fd_topo_run_tile_t fd_tile_capture;
 
 extern fd_topo_run_tile_t fd_tile_snapct;
 extern fd_topo_run_tile_t fd_tile_snapld;
@@ -172,6 +173,7 @@ fd_topo_run_tile_t * TILES[] = {
   &fd_tile_genesi,
   &fd_tile_ipecho,
   &fd_tile_vinyl,
+  &fd_tile_capture,
   NULL,
 };
 

src/app/firedancer/topology.c

@@ -22,6 +22,7 @@
 #include "../../util/tile/fd_tile_private.h"
 #include "../../discof/restore/utils/fd_ssctrl.h"
 #include "../../discof/restore/utils/fd_ssmsg.h"
+#include "../../flamenco/capture/fd_solcap_writer.h"
 #include "../../flamenco/progcache/fd_progcache_admin.h"
 #include "../../vinyl/meta/fd_vinyl_meta.h"
 #include "../../vinyl/io/fd_vinyl_io.h" /* FD_VINYL_IO_TYPE_* */
@@ -285,6 +286,8 @@ fd_topo_initialize( config_t * config ) {
   topo->max_page_size = fd_cstr_to_shmem_page_sz( config->hugetlbfs.max_page_size );
   topo->gigantic_page_threshold = config->hugetlbfs.gigantic_page_threshold_mib << 20;
 
+  int solcap_enabled = strlen( config->capture.solcap_capture ) > 0;
+
   /*             topo, name */
   fd_topob_wksp( topo, "metric"       );
   fd_topob_wksp( topo, "genesi"       );
@@ -560,6 +563,11 @@ fd_topo_initialize( config_t * config ) {
   /**/                 fd_topob_tile( topo, "poh",     "poh",     "metric_in",  tile_to_cpu[ topo->tile_cnt ], 0,        1 );
   FOR(sign_tile_cnt)   fd_topob_tile( topo, "sign",    "sign",    "metric_in",  tile_to_cpu[ topo->tile_cnt ], 0,        1 );
 
+  if( FD_UNLIKELY( solcap_enabled ) ) {
+    fd_topob_wksp( topo, "captur" );
+    fd_topob_tile( topo, "captur", "captur", "metric_in", tile_to_cpu[ topo->tile_cnt ], 0, 0 );
+  }
+
   /*                                        topo, tile_name, tile_kind_id, fseq_wksp,   link_name,      link_kind_id, reliable,            polled */
   FOR(gossvf_tile_cnt) for( ulong j=0UL; j<net_tile_cnt; j++ )
                       fd_topob_tile_in(     topo, "gossvf",  i,            "metric_in", "net_gossvf",   j,            FD_TOPOB_UNRELIABLE, FD_TOPOB_POLLED ); /* No reliable consumers of networking fragments, may be dropped or overrun */
@@ -839,6 +847,15 @@ fd_topo_initialize( config_t * config ) {
     fd_topob_tile_in( topo, "replay", 0UL, "metric_in", "rpc_replay", 0UL, FD_TOPOB_RELIABLE, FD_TOPOB_POLLED );
   }
 
+  if( FD_UNLIKELY( solcap_enabled ) ) {
+    fd_topob_link( topo, "cap_repl", "captur", 32UL, SOLCAP_WRITE_ACCOUNT_DATA_MTU, 1UL );
+    fd_topob_tile_out( topo, "replay", 0UL, "cap_repl", 0UL );
+    fd_topob_tile_in( topo, "captur", 0UL, "metric_in", "cap_repl", 0UL, FD_TOPOB_RELIABLE, FD_TOPOB_POLLED );
+    FOR(exec_tile_cnt) fd_topob_link( topo, "cap_exec", "captur", 32UL, SOLCAP_WRITE_ACCOUNT_DATA_MTU, 1UL );
+    FOR(exec_tile_cnt) fd_topob_tile_out( topo, "exec", i, "cap_exec", i );
+    FOR(exec_tile_cnt) fd_topob_tile_in( topo, "captur", 0UL, "metric_in", "cap_exec", i, FD_TOPOB_RELIABLE, FD_TOPOB_POLLED );
+  }
+
   if( FD_LIKELY( !is_auto_affinity ) ) {
     if( FD_UNLIKELY( affinity_tile_cnt<topo->tile_cnt ) )
       FD_LOG_ERR(( "The topology you are using has %lu tiles, but the CPU affinity specified in the config tile as [layout.affinity] only provides for %lu cores. "
@@ -1394,6 +1411,13 @@ fd_topo_configure_tile( fd_topo_tile_t * tile,
     tile->bundle.tls_cert_verify = !!config->tiles.bundle.tls_cert_verify;
 
   } else if( FD_UNLIKELY( !strcmp( tile->name, "vinyl" ) ) ) {
+  
+  } else if( FD_UNLIKELY( !strcmp( tile->name, "captur" ) ) ) {
+
+    tile->capctx.capture_start_slot = config->capture.capture_start_slot;
+    strncpy( tile->capctx.solcap_capture, config->capture.solcap_capture, sizeof(tile->capctx.solcap_capture) );
+    tile->capctx.recent_only = config->capture.recent_only;
+    tile->capctx.recent_slots_per_file = config->capture.recent_slots_per_file;
 
     tile->vinyl.vinyl_meta_map_obj_id  = fd_pod_query_ulong( config->topo.props, "vinyl.meta_map",  ULONG_MAX );
     tile->vinyl.vinyl_meta_pool_obj_id = fd_pod_query_ulong( config->topo.props, "vinyl.meta_pool", ULONG_MAX );

src/app/ledger/main.c

@@ -1,6 +1,6 @@
 #include "../../flamenco/types/fd_types.h"
 #include "../../flamenco/runtime/fd_rocksdb.h"
-#include "../../flamenco/runtime/context/fd_capture_ctx.h"
+#include "../../discof/capture/fd_capture_ctx.h"
 #include <unistd.h>
 #include <sys/stat.h>
 

src/app/shared/fd_config.h

@@ -168,6 +168,10 @@ struct fd_configf {
   struct {
     ulong max_completed_shred_sets;
   } store;
+
+  struct {
+    char path[ PATH_MAX ];
+  } capctx;
 };
 
 typedef struct fd_configf fd_configf_t;
@@ -499,6 +503,8 @@ struct fd_config {
     ulong capture_start_slot;
     char  dump_proto_dir[ PATH_MAX ];
     char  solcap_capture[ PATH_MAX ];
+    int   recent_only;
+    ulong recent_slots_per_file;
     int   dump_elf_to_pb;
     int   dump_syscall_to_pb;
     int   dump_instr_to_pb;

src/app/shared/fd_config_parse.c

@@ -249,6 +249,8 @@ fd_config_extract_pod( uchar *       pod,
 
   CFG_POP      ( ulong,  capture.capture_start_slot                       );
   CFG_POP      ( cstr,   capture.solcap_capture                           );
+  CFG_POP      ( bool,   capture.recent_only                              );
+  CFG_POP      ( ulong,  capture.recent_slots_per_file                    );
   CFG_POP      ( cstr,   capture.dump_proto_dir                           );
   CFG_POP      ( bool,   capture.dump_elf_to_pb                           );
   CFG_POP      ( bool,   capture.dump_syscall_to_pb                       );

src/disco/topo/fd_topo.h

@@ -616,6 +616,16 @@ struct fd_topo_tile {
       int  io_type; /* FD_VINYL_IO_TYPE_* */
       uint uring_depth;
     } vinyl;
+
+    struct {
+      ulong capture_start_slot;
+      char solcap_capture[ PATH_MAX ];
+      int recent_only;
+      ulong recent_slots_per_file;
+      int solcap_fd;
+      int solcap_fd_0;  /* First FD (single file mode) or recent_0.solcap */
+      int solcap_fd_1;  /* Second FD (recent_1.solcap, or same as fd_0) */
+    } capctx;
   };
 };
 

src/discof/capture/Local.mk

@@ -0,0 +1,6 @@
+ifdef FD_HAS_INT128
+ifdef FD_HAS_ALLOCA
+$(call add-hdrs,fd_capture_ctx.h)
+$(call add-objs,fd_capture_ctx fd_capture_tile,fd_discof)
+endif
+endif