diff --git a/book/api/metrics-generated.md b/book/api/metrics-generated.md
index 451830dd950..83a9009ac04 100644
--- a/book/api/metrics-generated.md
+++ b/book/api/metrics-generated.md
@@ -900,6 +900,17 @@
+## Snaphs Tile
+
+
+
+| Metric | Type | Description |
+|--------|------|-------------|
+| snaphs_state | gauge | State of the tile. 0=hashing, 1=done, 2=shutdown |
+| snaphs_accounts_hashed | gauge | Number of accounts hashed so far during snapshot loading. Might decrease if snapshot load is aborted and restarted |
+
+
+
## Ipecho Tile
diff --git a/src/app/firedancer-dev/commands/backtest.c b/src/app/firedancer-dev/commands/backtest.c
index 18fe8f56acb..88b68997329 100644
--- a/src/app/firedancer-dev/commands/backtest.c
+++ b/src/app/firedancer-dev/commands/backtest.c
@@ -24,6 +24,7 @@
#include "../../../util/pod/fd_pod_format.h"
#include "../../../discof/replay/fd_replay_notif.h"
#include "../../../discof/reasm/fd_reasm.h"
+#include "../../../discof/restore/utils/fd_ssctrl.h"
#include "../../../flamenco/runtime/fd_runtime_public.h" /* FD_RUNTIME_PUBLIC_ACCOUNT_UPDATE_MSG_MTU */
#include "../main.h"
@@ -36,6 +37,7 @@ static void
backtest_topo( config_t * config ) {
ulong exec_tile_cnt = config->firedancer.layout.exec_tile_count;
ulong writer_tile_cnt = config->firedancer.layout.writer_tile_count;
+ ulong hash_tile_cnt = config->firedancer.layout.hash_tile_count;
fd_topo_t * topo = { fd_topob_new( &config->topo, config->name ) };
topo->max_page_size = fd_cstr_to_shmem_page_sz( config->hugetlbfs.max_page_size );
@@ -99,6 +101,15 @@ backtest_topo( config_t * config ) {
snapdc_tile->allow_shutdown = 1;
snapin_tile->allow_shutdown = 1;
+ if( FD_LIKELY( hash_tile_cnt ) ) {
+ fd_topob_wksp( topo, "snaphs" );
+ }
+
+ for( ulong i=0UL; iallow_shutdown = 1;
+ }
+
/**********************************************************************/
/* Setup backtest->replay link (repair_repla) in topo */
/**********************************************************************/
@@ -121,7 +132,12 @@ backtest_topo( config_t * config ) {
fd_topob_wksp( topo, "snapdc_rd" );
fd_topob_wksp( topo, "snapin_rd" );
fd_topob_wksp( topo, "snap_out" );
- fd_topob_wksp( topo, "replay_manif" );
+
+ if( FD_LIKELY( hash_tile_cnt ) ) {
+ fd_topob_wksp( topo, "snapin_hsh" );
+ fd_topob_wksp( topo, "snaphsh_out" );
+ }
+
/* TODO: Should be depth of 1 or 2, not 4, but it causes backpressure
from the replay tile parsing the manifest, remove when this is
fixed. */
@@ -129,8 +145,10 @@ backtest_topo( config_t * config ) {
fd_topob_link( topo, "snap_zstd", "snap_zstd", 8192UL, 16384UL, 1UL );
fd_topob_link( topo, "snap_stream", "snap_stream", 2048UL, USHORT_MAX, 1UL );
- fd_topob_link( topo, "snapdc_rd", "snapdc_rd", 128UL, 0UL, 1UL );
- fd_topob_link( topo, "snapin_rd", "snapin_rd", 128UL, 0UL, 1UL );
+ fd_topob_link( topo, "snapdc_rd", "snapdc_rd", 128UL, 0UL, 1UL );
+ fd_topob_link( topo, "snapin_rd", "snapin_rd", 128UL, 0UL, 1UL );
+ FOR(hash_tile_cnt) fd_topob_link( topo, "snapin_hsh", "snapin_hsh", 128UL, sizeof(fd_snapshot_existing_account_t), 1UL );
+ FOR(hash_tile_cnt) fd_topob_link( topo, "snaphsh_out", "snaphsh_out", 128UL, 2048UL, 1UL );
fd_topob_tile_out( topo, "snaprd", 0UL, "snap_zstd", 0UL );
fd_topob_tile_in ( topo, "snapdc", 0UL, "metric_in", "snap_zstd", 0UL, FD_TOPOB_RELIABLE, FD_TOPOB_POLLED );
@@ -144,6 +162,12 @@ backtest_topo( config_t * config ) {
fd_topob_tile_in( topo, "snaprd", 0UL, "metric_in", "snapin_rd", 0UL, FD_TOPOB_RELIABLE, FD_TOPOB_POLLED );
fd_topob_tile_out( topo, "snapin", 0UL, "snapin_rd", 0UL );
+ FOR(hash_tile_cnt) fd_topob_tile_out( topo, "snapin", 0UL, "snapin_hsh", i );
+ FOR(hash_tile_cnt) fd_topob_tile_in( topo, "snapin", 0UL, "metric_in", "snaphsh_out", i, FD_TOPOB_RELIABLE, FD_TOPOB_POLLED );
+
+ FOR(hash_tile_cnt) fd_topob_tile_in( topo, "snaphs", i, "metric_in", "snapin_hsh", i, FD_TOPOB_RELIABLE, FD_TOPOB_POLLED );
+ FOR(hash_tile_cnt) fd_topob_tile_out( topo, "snaphs", i, "snaphsh_out", i );
+
/**********************************************************************/
/* More backtest->replay links in topo */
/**********************************************************************/
@@ -312,15 +336,7 @@ backtest_topo( config_t * config ) {
FD_TEST( fd_pod_insertf_ulong( topo->props, busy_obj->id, "bank_busy.%lu", i ) );
}
- /* Replay decoded manifest dcache topo obj */
- fd_topo_obj_t * replay_manifest_dcache = fd_topob_obj( topo, "dcache", "replay_manif" );
- fd_pod_insertf_ulong( topo->props, 2UL << 30UL, "obj.%lu.data_sz", replay_manifest_dcache->id );
- fd_pod_insert_ulong( topo->props, "manifest_dcache", replay_manifest_dcache->id );
-
fd_topob_tile_uses( topo, snapin_tile, funk_obj, FD_SHMEM_JOIN_MODE_READ_WRITE );
- fd_topob_tile_uses( topo, snapin_tile, replay_manifest_dcache, FD_SHMEM_JOIN_MODE_READ_WRITE );
- fd_topob_tile_uses( topo, replay_tile, replay_manifest_dcache, FD_SHMEM_JOIN_MODE_READ_ONLY );
-
for( ulong i=0UL; itile_cnt; i++ ) {
fd_topo_tile_t * tile = &topo->tiles[ i ];
if( !fd_topo_configure_tile( tile, config ) ) {
@@ -398,9 +414,13 @@ backtest_cmd_fn( args_t * args FD_PARAM_UNUSED,
fd_topo_tile_t * snapdc_tile = &topo->tiles[ fd_topo_find_tile( topo, "snapdc", 0UL ) ];
fd_topo_tile_t * snapin_tile = &topo->tiles[ fd_topo_find_tile( topo, "snapin", 0UL ) ];
+ ulong snaphs_tile_idx = fd_topo_find_tile( topo, "snaphs", 0UL );
+ fd_topo_tile_t * snaphs_tile = snaphs_tile_idx!=ULONG_MAX ? &topo->tiles[ snaphs_tile_idx ] : NULL;
+
ulong volatile * const snaprd_metrics = fd_metrics_tile( snaprd_tile->metrics );
ulong volatile * const snapdc_metrics = fd_metrics_tile( snapdc_tile->metrics );
ulong volatile * const snapin_metrics = fd_metrics_tile( snapin_tile->metrics );
+ ulong volatile * const snaphs_metrics = snaphs_tile ? fd_metrics_tile( snaphs_tile->metrics ) : NULL;
ulong total_off_old = 0UL;
ulong snaprd_backp_old = 0UL;
@@ -409,6 +429,8 @@ backtest_cmd_fn( args_t * args FD_PARAM_UNUSED,
ulong snapdc_wait_old = 0UL;
ulong snapin_backp_old = 0UL;
ulong snapin_wait_old = 0UL;
+ ulong snaphs_backp_old = 0UL;
+ ulong snaphs_wait_old = 0UL;
ulong acc_cnt_old = 0UL;
sleep( 1 );
puts( "-------------backp=(snaprd,snapdc,snapin) busy=(snaprd,snapdc,snapin)---------------" );
@@ -417,8 +439,9 @@ backtest_cmd_fn( args_t * args FD_PARAM_UNUSED,
ulong snaprd_status = FD_VOLATILE_CONST( snaprd_metrics[ MIDX( GAUGE, TILE, STATUS ) ] );
ulong snapdc_status = FD_VOLATILE_CONST( snapdc_metrics[ MIDX( GAUGE, TILE, STATUS ) ] );
ulong snapin_status = FD_VOLATILE_CONST( snapin_metrics[ MIDX( GAUGE, TILE, STATUS ) ] );
+ ulong snaphs_status = snaphs_metrics ? FD_VOLATILE_CONST( snaphs_metrics[ MIDX( GAUGE, TILE, STATUS ) ] ) : 2UL;
- if( FD_UNLIKELY( snaprd_status==2UL && snapdc_status==2UL && snapin_status == 2UL ) ) break;
+ if( FD_UNLIKELY( snaprd_status==2UL && snapdc_status==2UL && snapin_status == 2UL && snaphs_status==2UL ) ) break;
long cur = fd_log_wallclock();
if( FD_UNLIKELY( cur
@@ -26,6 +27,7 @@ snapshot_load_topo( config_t * config,
fd_topo_t * topo = &config->topo;
fd_topob_new( &config->topo, config->name );
topo->max_page_size = fd_cstr_to_shmem_page_sz( config->hugetlbfs.max_page_size );
+ ulong hash_tile_cnt = config->firedancer.layout.hash_tile_count;
fd_topob_wksp( topo, "funk" );
fd_topo_obj_t * funk_obj = setup_topo_funk( topo, "funk",
@@ -37,7 +39,7 @@ snapshot_load_topo( config_t * config,
static ushort tile_to_cpu[ FD_TILE_MAX ] = {0};
if( args->snapshot_load.tile_cpus[0] ) {
ulong cpu_cnt = fd_tile_private_cpus_parse( args->snapshot_load.tile_cpus, tile_to_cpu );
- if( FD_UNLIKELY( cpu_cnt<4UL ) ) FD_LOG_ERR(( "--tile-cpus specifies %lu CPUs, but need at least 4", cpu_cnt ));
+ if( FD_UNLIKELY( cpu_cnt<4UL + hash_tile_cnt ) ) FD_LOG_ERR(( "--tile-cpus specifies %lu CPUs, but need at least %lu", cpu_cnt, 4UL + hash_tile_cnt ));
}
/* metrics tile *****************************************************/
@@ -77,6 +79,20 @@ snapshot_load_topo( config_t * config,
fd_topo_tile_t * snapin_tile = fd_topob_tile( topo, "snapin", "snapin", "snapin", tile_to_cpu[3], 0, 0 );
snapin_tile->allow_shutdown = 1;
+ if( FD_LIKELY( hash_tile_cnt ) ) {
+ fd_topob_wksp( topo, "snaphs" );
+ fd_topob_wksp( topo, "snapin_hsh" );
+ fd_topob_wksp( topo, "snaphsh_out" );
+ }
+ #define FOR(cnt) for( ulong i=0UL; iallow_shutdown = 1;
+ }
+
+ FOR(hash_tile_cnt) fd_topob_link( topo, "snapin_hsh", "snapin_hsh", 128UL, sizeof(fd_snapshot_existing_account_t), 1UL );
+ FOR(hash_tile_cnt) fd_topob_link( topo, "snaphsh_out", "snaphsh_out", 128UL, 2048UL, 1UL );
+
/* uncompressed stream -> snapin tile */
fd_topob_tile_in ( topo, "snapin", 0UL, "metric_in", "snap_stream", 0UL, FD_TOPOB_RELIABLE, FD_TOPOB_POLLED );
@@ -100,6 +116,11 @@ snapshot_load_topo( config_t * config,
fd_topob_tile_in( topo, "snaprd", 0UL, "metric_in", "snapin_rd", 0UL, FD_TOPOB_RELIABLE, FD_TOPOB_POLLED );
fd_topob_tile_out( topo, "snapin", 0UL, "snapin_rd", 0UL );
+ FOR(hash_tile_cnt) fd_topob_tile_out( topo, "snapin", 0UL, "snapin_hsh", i );
+ FOR(hash_tile_cnt) fd_topob_tile_in( topo, "snapin", 0UL, "metric_in", "snaphsh_out", i, FD_TOPOB_RELIABLE, FD_TOPOB_POLLED );
+ FOR(hash_tile_cnt) fd_topob_tile_in( topo, "snaphs", i, "metric_in", "snapin_hsh", i, FD_TOPOB_RELIABLE, FD_TOPOB_POLLED );
+ FOR(hash_tile_cnt) fd_topob_tile_out( topo, "snaphs", i, "snaphsh_out", i );
+
for( ulong i=0UL; itile_cnt; i++ ) {
fd_topo_tile_t * tile = &topo->tiles[ i ];
if( !fd_topo_configure_tile( tile, config ) ) {
@@ -158,9 +179,13 @@ snapshot_load_cmd_fn( args_t * args,
fd_topo_tile_t * snapdc_tile = &topo->tiles[ fd_topo_find_tile( topo, "snapdc", 0UL ) ];
fd_topo_tile_t * snapin_tile = &topo->tiles[ fd_topo_find_tile( topo, "snapin", 0UL ) ];
+ ulong snaphs_tile_idx = fd_topo_find_tile( topo, "snaphs", 0UL );
+ fd_topo_tile_t * snaphs_tile = snaphs_tile_idx!=ULONG_MAX ? &topo->tiles[ snaphs_tile_idx ] : NULL;
+
ulong volatile * const snaprd_metrics = fd_metrics_tile( snaprd_tile->metrics );
ulong volatile * const snapdc_metrics = fd_metrics_tile( snapdc_tile->metrics );
ulong volatile * const snapin_metrics = fd_metrics_tile( snapin_tile->metrics );
+ ulong volatile * const snaphs_metrics = snaphs_tile ? fd_metrics_tile( snaphs_tile->metrics ) : NULL;
ulong total_off_old = 0UL;
ulong snaprd_backp_old = 0UL;
@@ -169,6 +194,8 @@ snapshot_load_cmd_fn( args_t * args,
ulong snapdc_wait_old = 0UL;
ulong snapin_backp_old = 0UL;
ulong snapin_wait_old = 0UL;
+ ulong snaphs_backp_old = 0UL;
+ ulong snaphs_wait_old = 0UL;
ulong acc_cnt_old = 0UL;
sleep( 1 );
puts( "" );
@@ -178,14 +205,15 @@ snapshot_load_cmd_fn( args_t * args,
puts( "- stall: Waiting on upstream tile" );
puts( "- acc: Number of accounts" );
puts( "" );
- puts( "-------------backp=(snaprd,snapdc,snapin) busy=(snaprd,snapdc,snapin)---------------" );
+ puts( "-------------backp=(snaprd,snapdc,snapin,snaphs) busy=(snaprd,snapdc,snapin,snaphs)---------------" );
long next = start+1000L*1000L*1000L;
for(;;) {
ulong snaprd_status = FD_VOLATILE_CONST( snaprd_metrics[ MIDX( GAUGE, TILE, STATUS ) ] );
ulong snapdc_status = FD_VOLATILE_CONST( snapdc_metrics[ MIDX( GAUGE, TILE, STATUS ) ] );
ulong snapin_status = FD_VOLATILE_CONST( snapin_metrics[ MIDX( GAUGE, TILE, STATUS ) ] );
+ ulong snaphs_status = snaphs_metrics ? FD_VOLATILE_CONST( snaphs_metrics[ MIDX( GAUGE, TILE, STATUS ) ] ) : 2UL;
- if( FD_UNLIKELY( snaprd_status==2UL && snapdc_status==2UL && snapin_status == 2UL ) ) break;
+ if( FD_UNLIKELY( snaprd_status==2UL && snapdc_status==2UL && snapin_status == 2UL && snaphs_status==2UL ) ) break;
long cur = fd_log_wallclock();
if( FD_UNLIKELY( cur
@@ -201,6 +202,7 @@ fd_topo_initialize( config_t * config ) {
ulong bank_tile_cnt = config->layout.bank_tile_count;
ulong exec_tile_cnt = config->firedancer.layout.exec_tile_count;
ulong writer_tile_cnt = config->firedancer.layout.writer_tile_count;
+ ulong hash_tile_cnt = config->firedancer.layout.hash_tile_count;
ulong resolv_tile_cnt = config->layout.resolv_tile_count;
ulong sign_tile_cnt = config->firedancer.layout.sign_tile_count;
@@ -293,7 +295,12 @@ fd_topo_initialize( config_t * config ) {
fd_topob_wksp( topo, "snap_stream" );
fd_topob_wksp( topo, "snap_zstd" );
fd_topob_wksp( topo, "snap_out" );
- fd_topob_wksp( topo, "replay_manif" );
+
+ if( FD_LIKELY( hash_tile_cnt ) ) {
+ fd_topob_wksp( topo, "snaphs" );
+ fd_topob_wksp( topo, "snapin_hsh" );
+ fd_topob_wksp( topo, "snaphsh_out" );
+ }
fd_topob_wksp( topo, "slot_fseqs" ); /* fseqs for marked slots eg. turbine slot */
if( enable_rpc ) fd_topob_wksp( topo, "rpcsrv" );
@@ -376,11 +383,8 @@ fd_topo_initialize( config_t * config ) {
/**/ fd_topob_link( topo, "snap_out", "snap_out", 2UL, 5UL*(1UL<<30UL), 1UL );
/**/ fd_topob_link( topo, "snapdc_rd", "snapdc_rd", 128UL, 0UL, 1UL );
/**/ fd_topob_link( topo, "snapin_rd", "snapin_rd", 128UL, 0UL, 1UL );
-
- /* Replay decoded manifest dcache topo obj */
- fd_topo_obj_t * replay_manifest_dcache = fd_topob_obj( topo, "dcache", "replay_manif" );
- fd_pod_insertf_ulong( topo->props, 2UL << 30UL, "obj.%lu.data_sz", replay_manifest_dcache->id );
- fd_pod_insert_ulong( topo->props, "manifest_dcache", replay_manifest_dcache->id );
+ FOR(hash_tile_cnt) fd_topob_link( topo, "snapin_hsh", "snapin_hsh", 128UL, sizeof(fd_snapshot_existing_account_t), 1UL );
+ FOR(hash_tile_cnt) fd_topob_link( topo, "snaphsh_out", "snaphsh_out", 128UL, 2048UL, 1UL );
ushort parsed_tile_to_cpu[ FD_TILE_MAX ];
/* Unassigned tiles will be floating, unless auto topology is enabled. */
@@ -446,6 +450,11 @@ fd_topo_initialize( config_t * config ) {
fd_topo_tile_t * snapin_tile = fd_topob_tile( topo, "snapin", "snapin", "metric_in", tile_to_cpu[ topo->tile_cnt ], 0, 0 );
snapin_tile->allow_shutdown = 1;
+ for( ulong i=0UL; itile_cnt ], 0, 0 );
+ snaphsh_tile->allow_shutdown = 1;
+ }
+
/* Database cache */
fd_topo_obj_t * funk_obj = setup_topo_funk( topo, "funk",
@@ -544,9 +553,6 @@ fd_topo_initialize( config_t * config ) {
}
fd_topob_tile_uses( topo, snapin_tile, funk_obj, FD_SHMEM_JOIN_MODE_READ_WRITE );
- fd_topob_tile_uses( topo, snapin_tile, runtime_pub_obj, FD_SHMEM_JOIN_MODE_READ_WRITE );
- fd_topob_tile_uses( topo, snapin_tile, replay_manifest_dcache, FD_SHMEM_JOIN_MODE_READ_WRITE );
- fd_topob_tile_uses( topo, replay_tile, replay_manifest_dcache, FD_SHMEM_JOIN_MODE_READ_ONLY );
/* There's another special fseq that's used to communicate the shred
version from the Agave boot path to the shred tile. */
@@ -736,6 +742,12 @@ fd_topo_initialize( config_t * config ) {
fd_topob_tile_in ( topo, "snaprd", 0UL, "metric_in", "snapin_rd", 0UL, FD_TOPOB_RELIABLE, FD_TOPOB_POLLED );
fd_topob_tile_out( topo, "snapin", 0UL, "snapin_rd", 0UL );
+ FOR(hash_tile_cnt) fd_topob_tile_out( topo, "snapin", 0UL, "snapin_hsh", i );
+ FOR(hash_tile_cnt) fd_topob_tile_in( topo, "snapin", 0UL, "metric_in", "snaphsh_out", i, FD_TOPOB_RELIABLE, FD_TOPOB_POLLED );
+
+ FOR(hash_tile_cnt) fd_topob_tile_in( topo, "snaphs", i, "metric_in", "snapin_hsh", i, FD_TOPOB_RELIABLE, FD_TOPOB_POLLED );
+ FOR(hash_tile_cnt) fd_topob_tile_out( topo, "snaphs", i, "snaphsh_out", i );
+
if( config->tiles.archiver.enabled ) {
fd_topob_wksp( topo, "arch_f" );
fd_topob_wksp( topo, "arch_w" );
@@ -1057,6 +1069,8 @@ fd_topo_configure_tile( fd_topo_tile_t * tile,
} else if( FD_UNLIKELY( !strcmp( tile->name, "snapin" ) ) ) {
tile->snapin.funk_obj_id = fd_pod_query_ulong( config->topo.props, "funk", ULONG_MAX );
+ } else if( FD_UNLIKELY( !strcmp( tile->name, "snaphs" ) ) ) {
+
} else if( FD_UNLIKELY( !strcmp( tile->name, "arch_f" ) ||
!strcmp( tile->name, "arch_w" ) ) ) {
strncpy( tile->archiver.rocksdb_path, config->tiles.archiver.rocksdb_path, sizeof(tile->archiver.rocksdb_path) );
diff --git a/src/app/shared/fd_config.h b/src/app/shared/fd_config.h
index 120b2a542fe..cfbd22241f1 100644
--- a/src/app/shared/fd_config.h
+++ b/src/app/shared/fd_config.h
@@ -113,6 +113,7 @@ struct fd_configf {
uint exec_tile_count; /* TODO: redundant ish with bank tile cnt */
uint writer_tile_count;
uint sign_tile_count;
+ uint hash_tile_count;
} layout;
struct {
diff --git a/src/app/shared/fd_config_parse.c b/src/app/shared/fd_config_parse.c
index 073a8811a7a..8d0176763dc 100644
--- a/src/app/shared/fd_config_parse.c
+++ b/src/app/shared/fd_config_parse.c
@@ -79,6 +79,7 @@ fd_config_extract_podf( uchar * pod,
CFG_POP ( uint, layout.exec_tile_count );
CFG_POP ( uint, layout.writer_tile_count );
CFG_POP ( uint, layout.sign_tile_count );
+ CFG_POP ( uint, layout.hash_tile_count );
CFG_POP ( ulong, blockstore.shred_max );
CFG_POP ( ulong, blockstore.block_max );
diff --git a/src/disco/metrics/generate/types.py b/src/disco/metrics/generate/types.py
index adfeded3c69..6126507c650 100644
--- a/src/disco/metrics/generate/types.py
+++ b/src/disco/metrics/generate/types.py
@@ -30,7 +30,8 @@ class Tile(Enum):
SNAPRD = 24
SNAPDC = 25
SNAPIN = 26
- IPECHO = 27
+ SNAPHS = 27
+ IPECHO = 28
class MetricType(Enum):
diff --git a/src/disco/metrics/generated/fd_metrics_all.c b/src/disco/metrics/generated/fd_metrics_all.c
index 158568644d5..b6ec7a80613 100644
--- a/src/disco/metrics/generated/fd_metrics_all.c
+++ b/src/disco/metrics/generated/fd_metrics_all.c
@@ -59,6 +59,7 @@ const char * FD_METRICS_TILE_KIND_NAMES[FD_METRICS_TILE_KIND_CNT] = {
"snaprd",
"snapdc",
"snapin",
+ "snaphs",
"ipecho",
};
@@ -86,6 +87,7 @@ const ulong FD_METRICS_TILE_KIND_SIZES[FD_METRICS_TILE_KIND_CNT] = {
FD_METRICS_SNAPRD_TOTAL,
FD_METRICS_SNAPDC_TOTAL,
FD_METRICS_SNAPIN_TOTAL,
+ FD_METRICS_SNAPHS_TOTAL,
FD_METRICS_IPECHO_TOTAL,
};
const fd_metrics_meta_t * FD_METRICS_TILE_KIND_METRICS[FD_METRICS_TILE_KIND_CNT] = {
@@ -112,5 +114,6 @@ const fd_metrics_meta_t * FD_METRICS_TILE_KIND_METRICS[FD_METRICS_TILE_KIND_CNT]
FD_METRICS_SNAPRD,
FD_METRICS_SNAPDC,
FD_METRICS_SNAPIN,
+ FD_METRICS_SNAPHS,
FD_METRICS_IPECHO,
};
diff --git a/src/disco/metrics/generated/fd_metrics_all.h b/src/disco/metrics/generated/fd_metrics_all.h
index 00962b38ed8..a08d61a6865 100644
--- a/src/disco/metrics/generated/fd_metrics_all.h
+++ b/src/disco/metrics/generated/fd_metrics_all.h
@@ -24,6 +24,7 @@
#include "fd_metrics_snaprd.h"
#include "fd_metrics_snapdc.h"
#include "fd_metrics_snapin.h"
+#include "fd_metrics_snaphs.h"
#include "fd_metrics_metric.h"
#include "fd_metrics_ipecho.h"
/* Start of LINK OUT metrics */
@@ -162,7 +163,7 @@ extern const fd_metrics_meta_t FD_METRICS_ALL_LINK_OUT[FD_METRICS_ALL_LINK_OUT_T
#define FD_METRICS_TOTAL_SZ (8UL*253UL)
-#define FD_METRICS_TILE_KIND_CNT 24
+#define FD_METRICS_TILE_KIND_CNT 25
extern const char * FD_METRICS_TILE_KIND_NAMES[FD_METRICS_TILE_KIND_CNT];
extern const ulong FD_METRICS_TILE_KIND_SIZES[FD_METRICS_TILE_KIND_CNT];
extern const fd_metrics_meta_t * FD_METRICS_TILE_KIND_METRICS[FD_METRICS_TILE_KIND_CNT];
diff --git a/src/disco/metrics/generated/fd_metrics_snaphs.c b/src/disco/metrics/generated/fd_metrics_snaphs.c
new file mode 100644
index 00000000000..a3fe0b94610
--- /dev/null
+++ b/src/disco/metrics/generated/fd_metrics_snaphs.c
@@ -0,0 +1,7 @@
+/* THIS FILE IS GENERATED BY gen_metrics.py. DO NOT HAND EDIT. */
+#include "fd_metrics_snaphs.h"
+
+const fd_metrics_meta_t FD_METRICS_SNAPHS[FD_METRICS_SNAPHS_TOTAL] = {
+ DECLARE_METRIC( SNAPHS_STATE, GAUGE ),
+ DECLARE_METRIC( SNAPHS_ACCOUNTS_HASHED, GAUGE ),
+};
diff --git a/src/disco/metrics/generated/fd_metrics_snaphs.h b/src/disco/metrics/generated/fd_metrics_snaphs.h
new file mode 100644
index 00000000000..da85e9cc8a3
--- /dev/null
+++ b/src/disco/metrics/generated/fd_metrics_snaphs.h
@@ -0,0 +1,19 @@
+/* THIS FILE IS GENERATED BY gen_metrics.py. DO NOT HAND EDIT. */
+
+#include "../fd_metrics_base.h"
+#include "fd_metrics_enums.h"
+
+#define FD_METRICS_GAUGE_SNAPHS_STATE_OFF (16UL)
+#define FD_METRICS_GAUGE_SNAPHS_STATE_NAME "snaphs_state"
+#define FD_METRICS_GAUGE_SNAPHS_STATE_TYPE (FD_METRICS_TYPE_GAUGE)
+#define FD_METRICS_GAUGE_SNAPHS_STATE_DESC "State of the tile. 0=hashing, 1=done, 2=shutdown"
+#define FD_METRICS_GAUGE_SNAPHS_STATE_CVT (FD_METRICS_CONVERTER_NONE)
+
+#define FD_METRICS_GAUGE_SNAPHS_ACCOUNTS_HASHED_OFF (17UL)
+#define FD_METRICS_GAUGE_SNAPHS_ACCOUNTS_HASHED_NAME "snaphs_accounts_hashed"
+#define FD_METRICS_GAUGE_SNAPHS_ACCOUNTS_HASHED_TYPE (FD_METRICS_TYPE_GAUGE)
+#define FD_METRICS_GAUGE_SNAPHS_ACCOUNTS_HASHED_DESC "Number of accounts hashed so far during snapshot loading. Might decrease if snapshot load is aborted and restarted"
+#define FD_METRICS_GAUGE_SNAPHS_ACCOUNTS_HASHED_CVT (FD_METRICS_CONVERTER_NONE)
+
+#define FD_METRICS_SNAPHS_TOTAL (2UL)
+extern const fd_metrics_meta_t FD_METRICS_SNAPHS[FD_METRICS_SNAPHS_TOTAL];
diff --git a/src/disco/metrics/generated/fd_metrics_snaphsh.c b/src/disco/metrics/generated/fd_metrics_snaphsh.c
new file mode 100644
index 00000000000..784d4dac2b3
--- /dev/null
+++ b/src/disco/metrics/generated/fd_metrics_snaphsh.c
@@ -0,0 +1,8 @@
+/* THIS FILE IS GENERATED BY gen_metrics.py. DO NOT HAND EDIT. */
+#include "fd_metrics_snaphsh.h"
+
+const fd_metrics_meta_t FD_METRICS_SNAPHSH[FD_METRICS_SNAPHSH_TOTAL] = {
+ DECLARE_METRIC( SNAPHSH_STATE, GAUGE ),
+ DECLARE_METRIC( SNAPHSH_FULL_ACCOUNTS_HASHED, GAUGE ),
+ DECLARE_METRIC( SNAPHSH_INCREMENTAL_ACCOUNTS_HASHED, GAUGE ),
+};
diff --git a/src/disco/metrics/generated/fd_metrics_snaphsh.h b/src/disco/metrics/generated/fd_metrics_snaphsh.h
new file mode 100644
index 00000000000..4193f7dd2a1
--- /dev/null
+++ b/src/disco/metrics/generated/fd_metrics_snaphsh.h
@@ -0,0 +1,25 @@
+/* THIS FILE IS GENERATED BY gen_metrics.py. DO NOT HAND EDIT. */
+
+#include "../fd_metrics_base.h"
+#include "fd_metrics_enums.h"
+
+#define FD_METRICS_GAUGE_SNAPHSH_STATE_OFF (16UL)
+#define FD_METRICS_GAUGE_SNAPHSH_STATE_NAME "snaphsh_state"
+#define FD_METRICS_GAUGE_SNAPHSH_STATE_TYPE (FD_METRICS_TYPE_GAUGE)
+#define FD_METRICS_GAUGE_SNAPHSH_STATE_DESC "State of the tile. 0=hashing, 1=done, 2=shutdown"
+#define FD_METRICS_GAUGE_SNAPHSH_STATE_CVT (FD_METRICS_CONVERTER_NONE)
+
+#define FD_METRICS_GAUGE_SNAPHSH_FULL_ACCOUNTS_HASHED_OFF (17UL)
+#define FD_METRICS_GAUGE_SNAPHSH_FULL_ACCOUNTS_HASHED_NAME "snaphsh_full_accounts_hashed"
+#define FD_METRICS_GAUGE_SNAPHSH_FULL_ACCOUNTS_HASHED_TYPE (FD_METRICS_TYPE_GAUGE)
+#define FD_METRICS_GAUGE_SNAPHSH_FULL_ACCOUNTS_HASHED_DESC "Number of accounts hashed so far from the full snapshot. Might decrease if snapshot load is aborted and restarted"
+#define FD_METRICS_GAUGE_SNAPHSH_FULL_ACCOUNTS_HASHED_CVT (FD_METRICS_CONVERTER_NONE)
+
+#define FD_METRICS_GAUGE_SNAPHSH_INCREMENTAL_ACCOUNTS_HASHED_OFF (18UL)
+#define FD_METRICS_GAUGE_SNAPHSH_INCREMENTAL_ACCOUNTS_HASHED_NAME "snaphsh_incremental_accounts_hashed"
+#define FD_METRICS_GAUGE_SNAPHSH_INCREMENTAL_ACCOUNTS_HASHED_TYPE (FD_METRICS_TYPE_GAUGE)
+#define FD_METRICS_GAUGE_SNAPHSH_INCREMENTAL_ACCOUNTS_HASHED_DESC "Number of accounts hashed so far from the incremental snapshot. Might decrease if snapshot load is aborted and restarted"
+#define FD_METRICS_GAUGE_SNAPHSH_INCREMENTAL_ACCOUNTS_HASHED_CVT (FD_METRICS_CONVERTER_NONE)
+
+#define FD_METRICS_SNAPHSH_TOTAL (3UL)
+extern const fd_metrics_meta_t FD_METRICS_SNAPHSH[FD_METRICS_SNAPHSH_TOTAL];
diff --git a/src/disco/metrics/metrics.xml b/src/disco/metrics/metrics.xml
index 0de555fb6a1..40b056e9495 100644
--- a/src/disco/metrics/metrics.xml
+++ b/src/disco/metrics/metrics.xml
@@ -928,6 +928,11 @@ metric introduced.
+
+
+
+
+
diff --git a/src/discof/restore/Local.mk b/src/discof/restore/Local.mk
index a4c1d484542..e666c587667 100644
--- a/src/discof/restore/Local.mk
+++ b/src/discof/restore/Local.mk
@@ -6,6 +6,7 @@ endif
ifdef FD_HAS_INT128
$(call add-objs,fd_snapin_tile,fd_discof)
endif
+$(call add-objs,fd_snaphs_tile,fd_discof)
endif
ifdef FD_HAS_INT128
$(call add-objs,utils/fd_snapshot_parser,fd_discof)
diff --git a/src/discof/restore/fd_snaphs_tile.c b/src/discof/restore/fd_snaphs_tile.c
new file mode 100644
index 00000000000..71131d71e9b
--- /dev/null
+++ b/src/discof/restore/fd_snaphs_tile.c
@@ -0,0 +1,256 @@
+#include "../../disco/topo/fd_topo.h"
+#include "../../disco/metrics/fd_metrics.h"
+#include "../../ballet/lthash/fd_lthash.h"
+#include "../../flamenco/runtime/fd_hashes.h"
+#include "../../flamenco/runtime/fd_acc_mgr.h"
+
+#include "utils/fd_ssctrl.h"
+
+#define NAME "snaphs"
+
+/* The snaphs tile is a state machine that hashes accounts from an
+ account input stream that it receives from the snapin tile.
+
+ An account input stream starts with a SNAPSHOT_HASH_MSG_RESET
+ message, which indicates the start of an account input stream.
+ An account input stream ends with a SNAPSHOT_HASH_MSG_FINI message,
+ indicating snaphs should send its calculated accounts hash to snapin
+ with a SNAPSHOT_HASH_MSG_RESULT message. */
+
+#define FD_SNAPHS_STATE_WAITING (0)
+#define FD_SNAPHS_STATE_HASHING (1)
+#define FD_SNAPHS_STATE_DONE (2)
+#define FD_SNAPHS_STATE_SHUTDOWN (3)
+
+struct fd_snaphs_tile {
+ int state;
+
+ fd_lthash_value_t running_lthash;
+
+ fd_snapshot_account_t account;
+ ulong acc_data_sz;
+ uchar acc_data[ FD_RUNTIME_ACC_SZ_MAX ];
+
+ struct {
+ ulong accounts_hashed;
+ } metrics;
+
+ struct {
+ fd_wksp_t * wksp;
+ ulong chunk0;
+ ulong wmark;
+ ulong mtu;
+ } in;
+
+ struct {
+ fd_wksp_t * wksp;
+ ulong chunk0;
+ ulong wmark;
+ ulong chunk;
+ ulong mtu;
+ } out;
+
+};
+
+typedef struct fd_snaphs_tile fd_snaphs_tile_t;
+
+static inline int
+should_shutdown( fd_snaphs_tile_t * ctx ) {
+ return ctx->state==FD_SNAPHS_STATE_SHUTDOWN;
+}
+
+static ulong
+scratch_align( void ) {
+ return 128UL;
+}
+
+static ulong
+scratch_footprint( fd_topo_tile_t const * tile ) {
+ (void)tile;
+ ulong l = FD_LAYOUT_INIT;
+ l = FD_LAYOUT_APPEND( l, alignof(fd_snaphs_tile_t), sizeof(fd_snaphs_tile_t) );
+ return FD_LAYOUT_FINI( l, alignof(fd_snaphs_tile_t) );
+}
+
+static void
+metrics_write( fd_snaphs_tile_t * ctx ) {
+ FD_MGAUGE_SET( SNAPHS, ACCOUNTS_HASHED, ctx->metrics.accounts_hashed );
+ FD_MGAUGE_SET( SNAPHS, STATE, (ulong)(ctx->state) );
+}
+
+static void
+during_frag( fd_snaphs_tile_t * ctx,
+ ulong in_idx,
+ ulong seq,
+ ulong sig,
+ ulong chunk,
+ ulong sz,
+ ulong ctl ) {
+ (void)in_idx;
+ (void)seq;
+ (void)ctl;
+
+ FD_TEST( ctx->state!=FD_SNAPHS_STATE_SHUTDOWN );
+ if( FD_LIKELY( sz ) ) FD_TEST( chunk>=ctx->in.chunk0 && chunk<=ctx->in.wmark && sz<=ctx->in.mtu );
+
+ switch( sig ) {
+ case FD_SNAPSHOT_HASH_MSG_SUB: {
+ FD_TEST( ctx->state==FD_SNAPHS_STATE_HASHING );
+
+ fd_snapshot_existing_account_t * prev_acc = (fd_snapshot_existing_account_t *)fd_chunk_to_laddr_const( ctx->in.wksp, chunk );
+
+ fd_lthash_value_t prev_lthash[1];
+ fd_hashes_account_lthash_simple( prev_acc->hdr.pubkey,
+ prev_acc->hdr.owner,
+ prev_acc->hdr.lamports,
+ prev_acc->hdr.executable,
+ prev_acc->data,
+ prev_acc->hdr.data_len,
+ prev_lthash );
+ fd_lthash_sub( &ctx->running_lthash, prev_lthash );
+ break;
+ }
+ case FD_SNAPSHOT_HASH_MSG_ACCOUNT_HDR: {
+ FD_TEST( ctx->state==FD_SNAPHS_STATE_HASHING );
+ fd_snapshot_account_t * account = (fd_snapshot_account_t *)fd_chunk_to_laddr_const( ctx->in.wksp, chunk );
+ fd_memcpy( &ctx->account, account, sizeof(fd_snapshot_account_t) );
+ break;
+ }
+ case FD_SNAPSHOT_HASH_MSG_ACCOUNT_DATA: {
+ FD_TEST( ctx->state==FD_SNAPHS_STATE_HASHING );
+ fd_memcpy( ctx->acc_data + ctx->acc_data_sz,
+ fd_chunk_to_laddr_const( ctx->in.wksp, chunk ),
+ sz );
+ ctx->acc_data_sz += sz;
+ break;
+ }
+ case FD_SNAPSHOT_HASH_MSG_RESET:
+ case FD_SNAPSHOT_HASH_MSG_FINI:
+ case FD_SNAPSHOT_HASH_MSG_SHUTDOWN: {
+ break;
+ }
+ default:
+ FD_LOG_ERR(( "unexpected sig %lu in during_frag", sig ));
+ return;
+ }
+}
+
+static void
+after_frag( fd_snaphs_tile_t * ctx,
+ ulong in_idx,
+ ulong seq,
+ ulong sig,
+ ulong sz,
+ ulong tsorig,
+ ulong _tspub,
+ fd_stem_context_t * stem ) {
+ (void)in_idx;
+ (void)seq;
+ (void)tsorig;
+ (void)_tspub;
+ (void)sz;
+
+ switch( sig ) {
+ case FD_SNAPSHOT_HASH_MSG_ACCOUNT_HDR:
+ case FD_SNAPSHOT_HASH_MSG_ACCOUNT_DATA: {
+ FD_TEST( ctx->state==FD_SNAPHS_STATE_HASHING );
+ if( FD_LIKELY( ctx->acc_data_sz==ctx->account.data_len ) ) {
+ /* hash account here */
+ fd_lthash_value_t account_lthash[1];
+ fd_hashes_account_lthash_simple( ctx->account.pubkey,
+ ctx->account.owner,
+ ctx->account.lamports,
+ ctx->account.executable,
+ ctx->acc_data,
+ ctx->acc_data_sz,
+ account_lthash );
+ fd_lthash_add( &ctx->running_lthash, account_lthash );
+ ctx->acc_data_sz = 0UL;
+ ctx->metrics.accounts_hashed++;
+ }
+ break;
+ }
+ case FD_SNAPSHOT_HASH_MSG_FINI: {
+ FD_TEST( ctx->state==FD_SNAPHS_STATE_HASHING );
+ uchar * lthash_out = fd_chunk_to_laddr( ctx->out.wksp, ctx->out.chunk );
+ fd_memcpy( lthash_out, &ctx->running_lthash, sizeof(fd_lthash_value_t) );
+ fd_stem_publish( stem, 0UL, FD_SNAPSHOT_HASH_MSG_RESULT, ctx->out.chunk, FD_LTHASH_LEN_BYTES, 0UL, 0UL, 0UL );
+ ctx->out.chunk = fd_dcache_compact_next( ctx->out.chunk, FD_SNAPSHOT_HASH_MSG_RESULT, ctx->out.chunk0, ctx->out.wmark );
+ ctx->state = FD_SNAPHS_STATE_DONE;
+ break;
+ }
+ case FD_SNAPSHOT_HASH_MSG_SHUTDOWN: {
+ FD_LOG_INFO(("num hashed accounts is %lu", ctx->metrics.accounts_hashed));
+ FD_TEST( ctx->state==FD_SNAPHS_STATE_DONE );
+ ctx->state = FD_SNAPHS_STATE_SHUTDOWN;
+ break;
+ }
+ case FD_SNAPSHOT_HASH_MSG_RESET: {
+ ctx->state = FD_SNAPHS_STATE_HASHING;
+ fd_lthash_zero( &ctx->running_lthash );
+ ctx->metrics.accounts_hashed = 0UL;
+ break;
+ }
+ case FD_SNAPSHOT_HASH_MSG_SUB:
+ /* These messages are handled in during_frag, do nothing here. */
+ break;
+ default:
+ FD_LOG_ERR(( "unexpected sig %lu in during_frag", sig ));
+ return;
+ }
+}
+
+static void
+unprivileged_init( fd_topo_t * topo,
+ fd_topo_tile_t * tile ) {
+ void * scratch = fd_topo_obj_laddr( topo, tile->tile_obj_id );
+
+ FD_SCRATCH_ALLOC_INIT( l, scratch );
+ fd_snaphs_tile_t * ctx = FD_SCRATCH_ALLOC_APPEND( l, alignof(fd_snaphs_tile_t), sizeof(fd_snaphs_tile_t) );
+
+ if( FD_UNLIKELY( tile->in_cnt!=1UL ) ) FD_LOG_ERR(( "tile `" NAME "` has %lu ins, expected 1", tile->in_cnt ));
+ if( FD_UNLIKELY( tile->out_cnt!=1UL ) ) FD_LOG_ERR(( "tile `" NAME "` has %lu outs, expected 1", tile->out_cnt ));
+
+ fd_topo_link_t const * in_link = &topo->links[ tile->in_link_id[ 0UL ] ];
+ fd_topo_wksp_t const * in_wksp = &topo->workspaces[ topo->objs[ in_link->dcache_obj_id ].wksp_id ];
+ ctx->in.wksp = in_wksp->wksp;;
+ ctx->in.chunk0 = fd_dcache_compact_chunk0( ctx->in.wksp, in_link->dcache );
+ ctx->in.wmark = fd_dcache_compact_wmark( ctx->in.wksp, in_link->dcache, in_link->mtu );
+ ctx->in.mtu = in_link->mtu;
+
+ fd_topo_link_t * writer_link = &topo->links[ tile->out_link_id[ 0UL ] ];
+ ctx->out.wksp = topo->workspaces[ topo->objs[ writer_link->dcache_obj_id ].wksp_id ].wksp;
+ ctx->out.chunk0 = fd_dcache_compact_chunk0( fd_wksp_containing( writer_link->dcache ), writer_link->dcache );
+ ctx->out.wmark = fd_dcache_compact_wmark ( ctx->out.wksp, writer_link->dcache, writer_link->mtu );
+ ctx->out.chunk = ctx->out.chunk0;
+ ctx->out.mtu = writer_link->mtu;
+
+ ctx->metrics.accounts_hashed = 0UL;
+ ctx->state = FD_SNAPHS_STATE_WAITING;
+ ctx->acc_data_sz = 0UL;
+
+ fd_lthash_zero( &ctx->running_lthash );
+}
+
+#define STEM_BURST 1UL
+#define STEM_LAZY 1000L
+
+#define STEM_CALLBACK_CONTEXT_TYPE fd_snaphs_tile_t
+#define STEM_CALLBACK_CONTEXT_ALIGN alignof(fd_snaphs_tile_t)
+
+#define STEM_CALLBACK_SHOULD_SHUTDOWN should_shutdown
+#define STEM_CALLBACK_METRICS_WRITE metrics_write
+#define STEM_CALLBACK_DURING_FRAG during_frag
+#define STEM_CALLBACK_AFTER_FRAG after_frag
+
+#include "../../disco/stem/fd_stem.c"
+
+fd_topo_run_tile_t fd_tile_snaphs = {
+ .name = NAME,
+ .scratch_align = scratch_align,
+ .scratch_footprint = scratch_footprint,
+ .unprivileged_init = unprivileged_init,
+ .run = stem_run,
+};
+
+#undef NAME
diff --git a/src/discof/restore/fd_snapin_tile.c b/src/discof/restore/fd_snapin_tile.c
index 85c27470308..4038f756b6e 100644
--- a/src/discof/restore/fd_snapin_tile.c
+++ b/src/discof/restore/fd_snapin_tile.c
@@ -7,6 +7,8 @@
#include "../../flamenco/runtime/fd_acc_mgr.h"
#include "../../flamenco/types/fd_types.h"
#include "../../funk/fd_funk.h"
+#include "../../ballet/lthash/fd_lthash.h"
+#include "../../flamenco/runtime/fd_hashes.h"
#define NAME "snapin"
@@ -20,9 +22,13 @@
#define FD_SNAPIN_STATE_MALFORMED (1) /* The snapshot is malformed, we are waiting for a reset notification */
#define FD_SNAPIN_STATE_SHUTDOWN (2) /* The tile is done, been told to shut down, and has likely already exited */
+#define FD_SNAPIN_HSH_IDX (2UL)
+#define FD_MAX_HASH_TILES (30UL)
+
struct fd_snapin_tile {
- int full;
- int state;
+ int full;
+ int state;
+ int pending_ack;
ulong seed;
long boot_timestamp;
@@ -30,10 +36,20 @@ struct fd_snapin_tile {
fd_funk_t funk[1];
fd_funk_txn_t * funk_txn;
uchar * acc_data;
+ ulong acc_data_hash_tile_idx;
- fd_stem_context_t * stem;
+ fd_stem_context_t * stem;
fd_snapshot_parser_t * ssparse;
+ struct {
+ int enabled;
+ ulong received_lthashes;
+ ulong num_hash_tiles;
+ fd_lthash_value_t full_lthash;
+ fd_lthash_value_t incremental_lthash;
+ fd_lthash_value_t result_lthash;
+ } hash_info;
+
struct {
ulong full_bytes_read;
ulong incremental_bytes_read;
@@ -45,6 +61,7 @@ struct fd_snapin_tile {
ulong chunk0;
ulong wmark;
ulong mtu;
+ ulong chunk_offset;
} in;
struct {
@@ -54,6 +71,21 @@ struct fd_snapin_tile {
ulong chunk;
ulong mtu;
} manifest_out;
+
+ struct {
+ fd_wksp_t * wksp;
+ ulong chunk0;
+ ulong wmark;
+ ulong chunk;
+ ulong mtu;
+ } hash_out[ FD_MAX_HASH_TILES ];
+
+ struct {
+ fd_wksp_t * wksp;
+ ulong chunk0;
+ ulong wmark;
+ ulong mtu;
+ } hash_in[ FD_MAX_HASH_TILES ];
};
typedef struct fd_snapin_tile fd_snapin_tile_t;
@@ -89,17 +121,40 @@ metrics_write( fd_snapin_tile_t * ctx ) {
FD_MGAUGE_SET( SNAPIN, STATE, (ulong)ctx->state );
}
+static ulong
+hash_account_to_hash_tile( fd_snapin_tile_t * ctx,
+ uchar const account_pubkey[ static FD_HASH_FOOTPRINT ] ) {
+ return fd_hash( account_pubkey[ 4UL ], account_pubkey, sizeof(fd_pubkey_t) ) % ctx->hash_info.num_hash_tiles;
+}
+
static void
manifest_cb( void * _ctx,
ulong manifest_sz ) {
fd_snapin_tile_t * ctx = (fd_snapin_tile_t*)_ctx;
ulong sz = sizeof(fd_snapshot_manifest_t)+manifest_sz;
+
+ fd_snapshot_manifest_t * manifest = (fd_snapshot_manifest_t * )fd_chunk_to_laddr_const( ctx->manifest_out.wksp, ctx->manifest_out.chunk );
+ if( FD_LIKELY( manifest->has_accounts_lthash ) ) {
+ uchar * manifest_lthash = ctx->full ? ctx->hash_info.full_lthash.bytes : ctx->hash_info.incremental_lthash.bytes;
+ fd_memcpy( manifest_lthash, manifest->accounts_lthash, sizeof(fd_lthash_value_t) );
+ } else {
+ /* disable lthash verification if there's no lthash in the
+ manifest. */
+ ctx->hash_info.enabled = 0;
+ }
+
FD_TEST( sz<=ctx->manifest_out.mtu );
ulong sig = ctx->full ? fd_ssmsg_sig( FD_SSMSG_MANIFEST_FULL, manifest_sz ) :
fd_ssmsg_sig( FD_SSMSG_MANIFEST_INCREMENTAL, manifest_sz );
fd_stem_publish( ctx->stem, 0UL, sig, ctx->manifest_out.chunk, sz, 0UL, 0UL, 0UL );
ctx->manifest_out.chunk = fd_dcache_compact_next( ctx->manifest_out.chunk, sz, ctx->manifest_out.chunk0, ctx->manifest_out.wmark );
+
+ if( FD_LIKELY( ctx->hash_info.enabled ) ) {
+ for( ulong i=0UL; ihash_info.num_hash_tiles; i++ ) {
+ fd_stem_publish( ctx->stem, FD_SNAPIN_HSH_IDX + i, FD_SNAPSHOT_HASH_MSG_RESET, 0UL, 0UL, 0UL, 0UL, 0UL );
+ }
+ }
}
static int
@@ -114,9 +169,17 @@ is_duplicate_account( fd_snapin_tile_t * ctx,
if( FD_UNLIKELY( rec_meta ) ) {
if( FD_LIKELY( rec_meta->slot>ctx->ssparse->accv_slot ) ) return 1;
- /* TODO: Reaching here means the existing value is a duplicate
- account. We need to hash the existing account and subtract that
- hash from the running lthash. */
+ if( FD_LIKELY( ctx->hash_info.enabled ) ) {
+ /* Send the existing account to the hash tile */
+ /* TODO: cache the previous lthash somewhere in funk to avoid
+ recalculating hash. */
+ ulong hash_tile_idx = hash_account_to_hash_tile( ctx, account_pubkey );
+ fd_snapshot_existing_account_t * account = (fd_snapshot_existing_account_t *)fd_chunk_to_laddr( ctx->hash_out[ hash_tile_idx ].wksp, ctx->hash_out[ hash_tile_idx ].chunk );
+ fd_snapshot_account_init( &account->hdr, account_pubkey, rec_meta->info.owner, rec_meta->info.lamports, rec_meta->info.executable, rec_meta->dlen );
+ fd_memcpy( account->data, fd_account_meta_get_data_const( rec_meta ), rec_meta->dlen );
+ fd_stem_publish( ctx->stem, FD_SNAPIN_HSH_IDX + hash_tile_idx, FD_SNAPSHOT_HASH_MSG_SUB, ctx->hash_out[ hash_tile_idx ].chunk, sizeof(fd_snapshot_existing_account_t), 0UL, 0UL, 0UL );
+ ctx->hash_out[ hash_tile_idx ].chunk = fd_dcache_compact_next( ctx->hash_out[ hash_tile_idx ].chunk, sizeof(fd_snapshot_existing_account_t), ctx->hash_out[ hash_tile_idx ].chunk0, ctx->hash_out[ hash_tile_idx ].wmark );
+ }
}
return 0;
@@ -151,6 +214,17 @@ account_cb( void * _ctx,
ctx->acc_data = fd_txn_account_get_data_mut( rec );
ctx->metrics.accounts_inserted++;
fd_txn_account_mutable_fini( rec, ctx->funk, ctx->funk_txn, &prepare );
+
+ if( FD_LIKELY( ctx->hash_info.enabled ) ) {
+ /* send account hdr to snaphsh tile */
+ ulong hash_tile_idx = hash_account_to_hash_tile( ctx, hdr->meta.pubkey );
+ fd_snapshot_account_t * account = (fd_snapshot_account_t *)fd_chunk_to_laddr( ctx->hash_out[ hash_tile_idx ].wksp, ctx->hash_out[ hash_tile_idx ].chunk );
+ fd_snapshot_account_init( account, hdr->meta.pubkey, hdr->info.owner, hdr->info.lamports, hdr->info.executable, hdr->meta.data_len );
+ fd_stem_publish( ctx->stem, FD_SNAPIN_HSH_IDX + hash_tile_idx, FD_SNAPSHOT_HASH_MSG_ACCOUNT_HDR, ctx->hash_out[ hash_tile_idx ].chunk, sizeof(fd_snapshot_account_t), 0UL, 0UL, 0UL );
+ ctx->hash_out[ hash_tile_idx ].chunk = fd_dcache_compact_next( ctx->hash_out[ hash_tile_idx ].chunk, sizeof(fd_snapshot_account_t), ctx->hash_out[ hash_tile_idx ].chunk0, ctx->hash_out[ hash_tile_idx ].wmark );
+ ctx->acc_data_hash_tile_idx = hash_tile_idx;
+ }
+
}
static void
@@ -162,21 +236,30 @@ account_data_cb( void * _ctx,
fd_memcpy( ctx->acc_data, buf, data_sz );
ctx->acc_data += data_sz;
+
+ if( FD_LIKELY( ctx->hash_info.enabled ) ) {
+ FD_TEST( data_sz<=ctx->hash_out[ ctx->acc_data_hash_tile_idx ].mtu );
+ /* send acc data to snaphsh tile */
+ uchar * snaphsh_acc_data = fd_chunk_to_laddr( ctx->hash_out[ ctx->acc_data_hash_tile_idx ].wksp, ctx->hash_out[ ctx->acc_data_hash_tile_idx ].chunk );
+ fd_memcpy( snaphsh_acc_data, buf, data_sz );
+ fd_stem_publish( ctx->stem, FD_SNAPIN_HSH_IDX + ctx->acc_data_hash_tile_idx, FD_SNAPSHOT_HASH_MSG_ACCOUNT_DATA, ctx->hash_out[ ctx->acc_data_hash_tile_idx ].chunk, data_sz, 0UL, 0UL, 0UL );
+ ctx->hash_out[ ctx->acc_data_hash_tile_idx ].chunk = fd_dcache_compact_next( ctx->hash_out[ ctx->acc_data_hash_tile_idx ].chunk, data_sz, ctx->hash_out[ ctx->acc_data_hash_tile_idx ].chunk0, ctx->hash_out[ ctx->acc_data_hash_tile_idx ].wmark );
+ }
}
static void
transition_malformed( fd_snapin_tile_t * ctx,
- fd_stem_context_t * stem ) {
+ fd_stem_context_t * stem ) {
ctx->state = FD_SNAPIN_STATE_MALFORMED;
fd_stem_publish( stem, 1UL, FD_SNAPSHOT_MSG_CTRL_MALFORMED, 0UL, 0UL, 0UL, 0UL, 0UL );
}
-static void
+static int
handle_data_frag( fd_snapin_tile_t * ctx,
ulong chunk,
ulong sz,
fd_stem_context_t * stem ) {
- if( FD_UNLIKELY( ctx->state==FD_SNAPIN_STATE_MALFORMED ) ) return;
+ if( FD_UNLIKELY( ctx->state==FD_SNAPIN_STATE_MALFORMED ) ) return 0;
FD_TEST( ctx->state==FD_SNAPIN_STATE_LOADING || ctx->state==FD_SNAPIN_STATE_DONE );
FD_TEST( chunk>=ctx->in.chunk0 && chunk<=ctx->in.wmark && sz<=ctx->in.mtu );
@@ -184,42 +267,50 @@ handle_data_frag( fd_snapin_tile_t * ctx,
if( FD_UNLIKELY( ctx->state==FD_SNAPIN_STATE_DONE ) ) {
FD_LOG_WARNING(( "received data fragment while in done state" ));
transition_malformed( ctx, stem );
- return;
+ return 0;
}
uchar const * const chunk_start = fd_chunk_to_laddr_const( ctx->in.wksp, chunk );
uchar const * const chunk_end = chunk_start + sz;
- uchar const * cur = chunk_start;
-
- for(;;) {
- if( FD_UNLIKELY( cur>=chunk_end ) ) {
- break;
- }
+ uchar const * cur = chunk_start + ctx->in.chunk_offset;
- cur = fd_snapshot_parser_process_chunk( ctx->ssparse, cur, (ulong)( chunk_end-cur ) );
- if( FD_UNLIKELY( ctx->ssparse->flags ) ) {
- if( FD_UNLIKELY( ctx->ssparse->flags & SNAP_FLAG_FAILED ) ) {
- transition_malformed( ctx, stem );
- return;
- }
+ cur = fd_snapshot_parser_process_chunk( ctx->ssparse, cur, (ulong)( chunk_end-cur ) );
+ if( FD_UNLIKELY( ctx->ssparse->flags ) ) {
+ if( FD_UNLIKELY( ctx->ssparse->flags & SNAP_FLAG_FAILED ) ) {
+ transition_malformed( ctx, stem );
+ return 0;
}
}
+ ctx->in.chunk_offset = (ulong)(cur - chunk_start);
+
if( FD_UNLIKELY( ctx->ssparse->flags & SNAP_FLAG_DONE ) ) ctx->state = FD_SNAPIN_STATE_DONE;
if( FD_LIKELY( ctx->full ) ) ctx->metrics.full_bytes_read += sz;
else ctx->metrics.incremental_bytes_read += sz;
+
+ if( FD_UNLIKELY( ctx->in.chunk_offset==sz ) ) {
+ ctx->in.chunk_offset = 0UL;
+ return 0;
+ }
+
+ return 1;
}
static void
handle_control_frag( fd_snapin_tile_t * ctx,
fd_stem_context_t * stem,
- ulong sig ) {
+ ulong sig,
+ ulong in_idx,
+ ulong chunk ) {
switch( sig ) {
case FD_SNAPSHOT_MSG_CTRL_RESET_FULL:
ctx->full = 1;
fd_snapshot_parser_reset( ctx->ssparse, fd_chunk_to_laddr( ctx->manifest_out.wksp, ctx->manifest_out.chunk ), ctx->manifest_out.mtu );
fd_funk_txn_cancel_root( ctx->funk );
+ fd_lthash_zero( &ctx->hash_info.full_lthash );
+ fd_lthash_zero( &ctx->hash_info.incremental_lthash );
+ fd_lthash_zero( &ctx->hash_info.result_lthash );
ctx->state = FD_SNAPIN_STATE_LOADING;
break;
case FD_SNAPSHOT_MSG_CTRL_RESET_INCREMENTAL:
@@ -227,6 +318,8 @@ handle_control_frag( fd_snapin_tile_t * ctx,
fd_snapshot_parser_reset( ctx->ssparse, fd_chunk_to_laddr( ctx->manifest_out.wksp, ctx->manifest_out.chunk ), ctx->manifest_out.mtu );
if( FD_UNLIKELY( !ctx->funk_txn ) ) fd_funk_txn_cancel_root( ctx->funk );
else fd_funk_txn_cancel( ctx->funk, ctx->funk_txn, 0 );
+ fd_lthash_zero( &ctx->hash_info.incremental_lthash );
+ fd_lthash_zero( &ctx->hash_info.result_lthash );
ctx->state = FD_SNAPIN_STATE_LOADING;
break;
case FD_SNAPSHOT_MSG_CTRL_EOF_FULL:
@@ -237,12 +330,17 @@ handle_control_frag( fd_snapin_tile_t * ctx,
break;
}
+ if( FD_LIKELY( ctx->hash_info.enabled ) ) {
+ for( ulong i=0UL; ihash_info.num_hash_tiles; i++ ) {
+ fd_stem_publish( stem, FD_SNAPIN_HSH_IDX + i, FD_SNAPSHOT_HASH_MSG_FINI, 0UL, 0UL, 0UL, 0UL, 0UL );
+ }
+ ctx->pending_ack = 1;
+ }
fd_snapshot_parser_reset( ctx->ssparse, fd_chunk_to_laddr( ctx->manifest_out.wksp, ctx->manifest_out.chunk ), ctx->manifest_out.mtu );
-
fd_funk_txn_xid_t incremental_xid = fd_funk_generate_xid();
- ctx->funk_txn = fd_funk_txn_prepare( ctx->funk, ctx->funk_txn, &incremental_xid, 0 );
- ctx->full = 0;
- ctx->state = FD_SNAPIN_STATE_LOADING;
+ ctx->funk_txn = fd_funk_txn_prepare( ctx->funk, ctx->funk_txn, &incremental_xid, 0 );
+ ctx->full = 0;
+ ctx->state = FD_SNAPIN_STATE_LOADING;
break;
case FD_SNAPSHOT_MSG_CTRL_DONE:
if( FD_UNLIKELY( ctx->state!=FD_SNAPIN_STATE_DONE ) ) {
@@ -251,17 +349,58 @@ handle_control_frag( fd_snapin_tile_t * ctx,
break;
}
+ if( FD_LIKELY( ctx->hash_info.enabled ) ) {
+ for( ulong i=0UL; ihash_info.num_hash_tiles; i++ ) {
+ fd_stem_publish( stem, FD_SNAPIN_HSH_IDX + i, FD_SNAPSHOT_HASH_MSG_FINI, 0UL, 0UL, 0UL, 0UL, 0UL );
+ }
+ ctx->pending_ack = 1;
+ }
if( FD_LIKELY( ctx->funk_txn ) ) fd_funk_txn_publish_into_parent( ctx->funk, ctx->funk_txn, 0 );
fd_stem_publish( stem, 0UL, fd_ssmsg_sig( FD_SSMSG_DONE, 0UL ), 0UL, 0UL, 0UL, 0UL, 0UL );
break;
case FD_SNAPSHOT_MSG_CTRL_SHUTDOWN:
+ FD_TEST( ctx->pending_ack==0 );
ctx->state = FD_SNAPIN_STATE_SHUTDOWN;
+
+ if( FD_LIKELY( ctx->hash_info.enabled ) ) {
+ for( ulong i=0UL; ihash_info.num_hash_tiles; i++ ) {
+ fd_stem_publish( stem, FD_SNAPIN_HSH_IDX + i, FD_SNAPSHOT_HASH_MSG_SHUTDOWN, 0UL, 0UL, 0UL, 0UL, 0UL );
+ }
+ }
+ break;
+ case FD_SNAPSHOT_HASH_MSG_RESULT: {
+ FD_TEST( ctx->hash_info.enabled && ctx->pending_ack );
+ /* TODO: more robust in indexing */
+ fd_lthash_value_t const * result_lthash = (fd_lthash_value_t const *)fd_chunk_to_laddr_const( ctx->hash_in[ in_idx - 1UL ].wksp, chunk );
+ fd_lthash_add( &ctx->hash_info.result_lthash, result_lthash );
+ ctx->hash_info.received_lthashes++;
+
+ if( FD_LIKELY( ctx->hash_info.received_lthashes!=ctx->hash_info.num_hash_tiles ) ) break;
+
+ fd_lthash_value_t * ref_lthash = ctx->full ? &ctx->hash_info.full_lthash : &ctx->hash_info.incremental_lthash;
+ if( FD_UNLIKELY( memcmp( ref_lthash, &ctx->hash_info.result_lthash, sizeof(fd_lthash_value_t) ) ) ) {
+ FD_LOG_WARNING(( "calculated accounts lthash %s does not match accounts lthash %s in snapshot manifest",
+ FD_LTHASH_ENC_32_ALLOCA( &ctx->hash_info.result_lthash ),
+ FD_LTHASH_ENC_32_ALLOCA( ref_lthash ) ));
+ transition_malformed( ctx, stem );
+ } else {
+ FD_LOG_NOTICE(( "calculated accounts lthash %s matches accounts lthash %s in snapshot manifest",
+ FD_LTHASH_ENC_32_ALLOCA( ref_lthash ),
+ FD_LTHASH_ENC_32_ALLOCA( &ctx->hash_info.result_lthash ) ));
+ }
break;
+ }
default:
FD_LOG_ERR(( "unexpected control sig %lu", sig ));
return;
}
+ /* snapin waits for result lthashes to come back from the snaphs tiles.
+ Until these hashes come back, snapin cannot ack the control message
+ sent from snaprd because snaprd would advance before snapin
+ is ready to receive another snapshot byte stream or shutdown. */
+ if( FD_UNLIKELY( ctx->pending_ack ) ) return;
+
/* We must acknowledge after handling the control frag, because if it
causes us to generate a malformed transition, that must be sent
back to the snaprd controller before the acknowledgement. */
@@ -287,12 +426,32 @@ returnable_frag( fd_snapin_tile_t * ctx,
FD_TEST( ctx->state!=FD_SNAPIN_STATE_SHUTDOWN );
- if( FD_UNLIKELY( sig==FD_SNAPSHOT_MSG_DATA ) ) handle_data_frag( ctx, chunk, sz, stem );
- else handle_control_frag( ctx, stem, sig );
+ if( FD_UNLIKELY( sig==FD_SNAPSHOT_MSG_DATA ) ) return handle_data_frag( ctx, chunk, sz, stem );
+ else handle_control_frag( ctx, stem, sig, in_idx, chunk );
return 0;
}
+static void
+after_credit( fd_snapin_tile_t * ctx,
+ fd_stem_context_t * stem,
+ int * opt_poll_in,
+ int * charge_busy ) {
+ (void)opt_poll_in;
+ (void)charge_busy;
+
+ if( FD_UNLIKELY( ctx->pending_ack ) ) {
+ FD_TEST( ctx->hash_info.enabled );
+ /* A pending ack is waiting for lthash results to come back from
+ the hashing tiles. Once these hashes come back, the ack can be
+ sent. */
+ if( FD_LIKELY( ctx->hash_info.received_lthashes==ctx->hash_info.num_hash_tiles ) ) {
+ fd_stem_publish( stem, 1UL, FD_SNAPSHOT_MSG_CTRL_ACK, 0UL, 0UL, 0UL, 0UL, 0UL );
+ ctx->pending_ack = 0;
+ }
+ }
+}
+
static void
privileged_init( fd_topo_t * topo,
fd_topo_tile_t * tile ) {
@@ -313,8 +472,9 @@ unprivileged_init( fd_topo_t * topo,
fd_snapin_tile_t * ctx = FD_SCRATCH_ALLOC_APPEND( l, alignof(fd_snapin_tile_t), sizeof(fd_snapin_tile_t) );
void * _ssparse = FD_SCRATCH_ALLOC_APPEND( l, fd_snapshot_parser_align(), fd_snapshot_parser_footprint( 1UL<<24UL ) );
- ctx->full = 1;
- ctx->state = FD_SNAPIN_STATE_LOADING;
+ ctx->full = 1;
+ ctx->state = FD_SNAPIN_STATE_LOADING;
+ ctx->pending_ack = 0;
ctx->boot_timestamp = fd_log_wallclock();
@@ -327,9 +487,14 @@ unprivileged_init( fd_topo_t * topo,
fd_memset( &ctx->metrics, 0, sizeof(ctx->metrics) );
+ ulong num_hash_tiles = fd_topo_tile_name_cnt( topo, "snaphs" );
+ ctx->hash_info.num_hash_tiles = num_hash_tiles;
+ ctx->hash_info.received_lthashes = 0UL;
+ ctx->hash_info.enabled = num_hash_tiles>0UL;
+
if( FD_UNLIKELY( tile->kind_id ) ) FD_LOG_ERR(( "There can only be one `" NAME "` tile" ));
- if( FD_UNLIKELY( tile->in_cnt!=1UL ) ) FD_LOG_ERR(( "tile `" NAME "` has %lu ins, expected 1", tile->in_cnt ));
- if( FD_UNLIKELY( tile->out_cnt!=2UL ) ) FD_LOG_ERR(( "tile `" NAME "` has %lu outs, expected 2", tile->out_cnt ));
+ if( FD_UNLIKELY( tile->in_cnt!=1UL + num_hash_tiles ) ) FD_LOG_ERR(( "tile `" NAME "` has %lu ins, expected 2", tile->in_cnt ));
+ if( FD_UNLIKELY( tile->out_cnt!=2UL + num_hash_tiles ) ) FD_LOG_ERR(( "tile `" NAME "` has %lu outs, expected 3", tile->out_cnt ));
fd_topo_link_t * writer_link = &topo->links[ tile->out_link_id[ 0UL ] ];
ctx->manifest_out.wksp = topo->workspaces[ topo->objs[ writer_link->dcache_obj_id ].wksp_id ].wksp;
@@ -346,6 +511,29 @@ unprivileged_init( fd_topo_t * topo,
ctx->in.chunk0 = fd_dcache_compact_chunk0( ctx->in.wksp, in_link->dcache );
ctx->in.wmark = fd_dcache_compact_wmark( ctx->in.wksp, in_link->dcache, in_link->mtu );
ctx->in.mtu = in_link->mtu;
+ ctx->in.chunk_offset = 0UL;
+
+ for( ulong i=0UL; ilinks[ tile->in_link_id[ i+1UL ] ];
+ FD_TEST( strcmp( hash_in_link->name, "snaphsh_out" )==0 );
+ fd_topo_wksp_t const * hash_in_wksp = &topo->workspaces[ topo->objs[ hash_in_link->dcache_obj_id ].wksp_id ];
+ ctx->hash_in[ i ].wksp = hash_in_wksp->wksp;
+ ctx->hash_in[ i ].chunk0 = fd_dcache_compact_chunk0( ctx->hash_in[ i ].wksp, hash_in_link->dcache );
+ ctx->hash_in[ i ].wmark = fd_dcache_compact_wmark( ctx->hash_in[ i ].wksp, hash_in_link->dcache, hash_in_link->mtu );
+ ctx->hash_in[ i ].mtu = hash_in_link->mtu;
+
+ fd_topo_link_t const * hash_out_link = &topo->links[ tile->out_link_id[ FD_SNAPIN_HSH_IDX + i ] ];
+ FD_TEST( strcmp( hash_out_link->name, "snapin_hsh" )==0 );
+ ctx->hash_out[ i ].wksp = topo->workspaces[ topo->objs[ hash_out_link->dcache_obj_id ].wksp_id ].wksp;
+ ctx->hash_out[ i ].chunk0 = fd_dcache_compact_chunk0( fd_wksp_containing( hash_out_link->dcache ), hash_out_link->dcache );
+ ctx->hash_out[ i ].wmark = fd_dcache_compact_wmark ( ctx->hash_out[ i ].wksp, hash_out_link->dcache, hash_out_link->mtu );
+ ctx->hash_out[ i ].chunk = ctx->hash_out[ i ].chunk0;
+ ctx->hash_out[ i ].mtu = hash_out_link->mtu;
+ }
+
+ fd_lthash_zero( &ctx->hash_info.full_lthash );
+ fd_lthash_zero( &ctx->hash_info.incremental_lthash );
+ fd_lthash_zero( &ctx->hash_info.result_lthash );
}
#define STEM_BURST 2UL /* For control fragments, one acknowledgement, and one malformed message */
@@ -357,6 +545,7 @@ unprivileged_init( fd_topo_t * topo,
#define STEM_CALLBACK_SHOULD_SHUTDOWN should_shutdown
#define STEM_CALLBACK_METRICS_WRITE metrics_write
#define STEM_CALLBACK_RETURNABLE_FRAG returnable_frag
+#define STEM_CALLBACK_AFTER_CREDIT after_credit
#include "../../disco/stem/fd_stem.c"
diff --git a/src/discof/restore/utils/fd_ssctrl.h b/src/discof/restore/utils/fd_ssctrl.h
index f21fe288b49..946c3e50c71 100644
--- a/src/discof/restore/utils/fd_ssctrl.h
+++ b/src/discof/restore/utils/fd_ssctrl.h
@@ -1,6 +1,9 @@
#ifndef HEADER_fd_src_discof_restore_utils_fd_ssctrl_h
#define HEADER_fd_src_discof_restore_utils_fd_ssctrl_h
+#include "../../../flamenco/types/fd_types_custom.h"
+#include "../../../flamenco/runtime/fd_runtime_const.h"
+
/* The snapshot tiles have a somewhat involved state machine, which is
controlled by snaprd. Imagine first the following sequence:
@@ -49,4 +52,58 @@
#define FD_SNAPSHOT_MSG_CTRL_ACK (6UL) /* Sent from tiles back to snaprd, meaning they ACK whatever control message was pending */
#define FD_SNAPSHOT_MSG_CTRL_MALFORMED (7UL) /* Sent from tiles back to snaprd, meaning they consider the current snapshot malformed */
+/* The following message signatures define the sequence of control and
+ data messages between the snapin and snaphs tile. The snaphs tile
+ does not participate in control messages between snaprd, snapdc, and
+ snapin. It acts as a hashing accelerator for snapin. */
+#define FD_SNAPSHOT_HASH_MSG_RESET (8UL) /* Indicates snapin has a new account stream incoming */
+#define FD_SNAPSHOT_HASH_MSG_SUB (9UL) /* Indicates snapin has encountered a duplicate account whose hash must be subtracted */
+#define FD_SNAPSHOT_HASH_MSG_ACCOUNT_HDR (10UL) /* Indicates snapin has encountered a new account metadata */
+#define FD_SNAPSHOT_HASH_MSG_ACCOUNT_DATA (11UL) /* Account data that is sent as snapin processes a new account */
+#define FD_SNAPSHOT_HASH_MSG_FINI (12UL) /* Indicates the account stream from snapin is done, awaiting hash result */
+#define FD_SNAPSHOT_HASH_MSG_RESULT (13UL) /* Hash result sent from snaphs to snapin */
+#define FD_SNAPSHOT_HASH_MSG_SHUTDOWN (14UL) /* Snapin is shutting down, snaphs can shutdown too */
+
+/* fd_snapshot_account is the contents of the
+ SNAPSHOT_HASH_MSG_ACCOUNT_HDR message. It contains account metadata
+ that is contained in the accounts hash. */
+struct fd_snapshot_account {
+ uchar pubkey[ FD_HASH_FOOTPRINT ];
+ uchar owner[ FD_HASH_FOOTPRINT ];
+ ulong lamports;
+ uchar executable;
+ ulong data_len;
+};
+typedef struct fd_snapshot_account fd_snapshot_account_t;
+
+/* fd_snapshot_account_init initializes a fd_snapshot_account_t struct
+ with the appropriate account metadata fields. */
+static inline void
+fd_snapshot_account_init( fd_snapshot_account_t * account,
+ uchar const pubkey[ FD_HASH_FOOTPRINT ],
+ uchar const owner[ FD_PUBKEY_FOOTPRINT ],
+ ulong lamports,
+ uchar executable,
+ ulong data_len ) {
+ fd_memcpy( account->pubkey, pubkey, FD_HASH_FOOTPRINT );
+ fd_memcpy( account->owner, owner, FD_PUBKEY_FOOTPRINT );
+ account->lamports = lamports;
+ account->executable = executable;
+ account->data_len = data_len;
+}
+
+/* fd_snapshot_existing_account is the contents of the
+ SNAPSHOT_HASH_MSG_SUB message. It contains a fd_snapshot_account_t
+ header and the corresponding account data in a single message.
+
+ For simplicity and conformance to burst limitations in snapin, the
+ entire duplicate account is sent in one message (one frag). Consider
+ caching the lthash of the duplicate account so we do not have to
+ send the entire account over. */
+struct fd_snapshot_existing_account {
+ fd_snapshot_account_t hdr;
+ uchar data[ FD_RUNTIME_ACC_SZ_MAX ];
+};
+typedef struct fd_snapshot_existing_account fd_snapshot_existing_account_t;
+
#endif /* HEADER_fd_src_discof_restore_utils_fd_ssctrl_h */
diff --git a/src/flamenco/runtime/fd_hashes.c b/src/flamenco/runtime/fd_hashes.c
index c567bcbbb92..9d962ebb69f 100644
--- a/src/flamenco/runtime/fd_hashes.c
+++ b/src/flamenco/runtime/fd_hashes.c
@@ -12,21 +12,38 @@ fd_hashes_account_lthash( fd_pubkey_t const * pubkey,
fd_account_meta_t const * account,
uchar const * data,
fd_lthash_value_t * lthash_out ) {
+ fd_hashes_account_lthash_simple( pubkey->uc,
+ account->info.owner,
+ account->info.lamports,
+ account->info.executable,
+ data,
+ account->dlen,
+ lthash_out );
+}
+
+void
+fd_hashes_account_lthash_simple( uchar const pubkey[ static FD_HASH_FOOTPRINT ],
+ uchar const owner[ static FD_PUBKEY_FOOTPRINT ],
+ ulong lamports,
+ uchar executable,
+ uchar const * data,
+ ulong data_len,
+ fd_lthash_value_t * lthash_out ) {
fd_lthash_zero( lthash_out );
/* Accounts with zero lamports are not included in the hash, so they should always be treated as zero */
- if( FD_UNLIKELY( account->info.lamports == 0 ) ) {
+ if( FD_UNLIKELY( lamports == 0 ) ) {
return;
}
- uchar executable = account->info.executable & 0x1;
+ uchar executable_flag = executable & 0x1;
fd_blake3_t b3[1];
fd_blake3_init( b3 );
- fd_blake3_append( b3, &account->info.lamports, sizeof( ulong ) );
- fd_blake3_append( b3, data, account->dlen );
- fd_blake3_append( b3, &executable, sizeof( uchar ) );
- fd_blake3_append( b3, account->info.owner, FD_PUBKEY_FOOTPRINT );
+ fd_blake3_append( b3, &lamports, sizeof( ulong ) );
+ fd_blake3_append( b3, data, data_len );
+ fd_blake3_append( b3, &executable_flag, sizeof( uchar ) );
+ fd_blake3_append( b3, owner, FD_PUBKEY_FOOTPRINT );
fd_blake3_append( b3, pubkey, FD_PUBKEY_FOOTPRINT );
fd_blake3_fini_varlen( b3, lthash_out->bytes, FD_LTHASH_LEN_BYTES );
}
diff --git a/src/flamenco/runtime/fd_hashes.h b/src/flamenco/runtime/fd_hashes.h
index a0fe9cbd1a7..ea45598addb 100644
--- a/src/flamenco/runtime/fd_hashes.h
+++ b/src/flamenco/runtime/fd_hashes.h
@@ -58,6 +58,29 @@ fd_hashes_account_lthash( fd_pubkey_t const * pubkey,
uchar const * data,
fd_lthash_value_t * lthash_out );
+/* fd_hashes_account_lthash_simple is functionally the same as
+ fd_hashes_account_lthash, but with simpler arguments that detail
+ the exact parameters that go into the lthash.
+
+ pubkey points to the account's public key (32 bytes). Owner points
+ to the account's owner (32 bytes). lamports is the account's
+ lamports. executable is the account's executable flag. data points
+ to the account data. data_len is the length of the account data.
+ lthash_out points to where the computed lthash value will be written
+ (2048 bytes).
+
+ On return, lthash_out contains the computed lthash. This function
+ assumes all pointers are valid and properly aligned. The account
+ data pointer must be readable for account->dlen bytes. */
+void
+fd_hashes_account_lthash_simple( uchar const pubkey[ static FD_HASH_FOOTPRINT ],
+ uchar const owner[ static FD_PUBKEY_FOOTPRINT ],
+ ulong lamports,
+ uchar executable,
+ uchar const * data,
+ ulong data_len,
+ fd_lthash_value_t * lthash_out );
+
/* fd_hashes_update_lthash updates the bank's incremental lthash when an
account is modified during transaction execution. The bank lthash is
maintained incrementally by subtracting the old account hash and
diff --git a/src/flamenco/runtime/fd_runtime_const.h b/src/flamenco/runtime/fd_runtime_const.h
index bd55afa5563..997b718d157 100644
--- a/src/flamenco/runtime/fd_runtime_const.h
+++ b/src/flamenco/runtime/fd_runtime_const.h
@@ -14,6 +14,8 @@ FD_PROTOTYPES_BEGIN
#define FD_RUNTIME_SLOTS_PER_EPOCH (432000UL) /* 432k slots per epoch */
+#define FD_RUNTIME_ACC_SZ_MAX (10UL<<20) /* Account data is bounded by 10MiB */
+
#define FD_RUNTIME_MAX_EPOCH_LEADERS (FD_EPOCH_LEADERS_FOOTPRINT(FD_RUNTIME_MAX_VOTE_ACCOUNTS, FD_RUNTIME_SLOTS_PER_EPOCH))
FD_PROTOTYPES_END
diff --git a/src/flamenco/runtime/tests/run_backtest_tests_all.sh b/src/flamenco/runtime/tests/run_backtest_tests_all.sh
index 7526d39cf5c..64ac363f971 100755
--- a/src/flamenco/runtime/tests/run_backtest_tests_all.sh
+++ b/src/flamenco/runtime/tests/run_backtest_tests_all.sh
@@ -1,87 +1,87 @@
#!/bin/bash
set -e
-src/flamenco/runtime/tests/run_ledger_backtest.sh -l testnet-519-v2.3.0 -y 3 -m 2000000 -e 255312007 -c 2.3.0
-src/flamenco/runtime/tests/run_ledger_backtest.sh -l mainnet-257066033-v2.3.0 -y 3 -m 2000000 -e 257066038 -c 2.3.0
-src/flamenco/runtime/tests/run_ledger_backtest.sh -l mainnet-257066844-v2.3.0 -y 3 -m 2000000 -e 257066849 -c 2.3.0
-src/flamenco/runtime/tests/run_ledger_backtest.sh -l mainnet-257067457-v2.3.0 -y 3 -m 2000000 -e 257067461 -c 2.3.0
-src/flamenco/runtime/tests/run_ledger_backtest.sh -l mainnet-257068890-v2.3.0 -y 3 -m 2000000 -e 257068895 -c 2.3.0
-src/flamenco/runtime/tests/run_ledger_backtest.sh -l mainnet-257181622-v2.3.0 -y 3 -m 2000000 -e 257181624 -c 2.3.0
-src/flamenco/runtime/tests/run_ledger_backtest.sh -l mainnet-254462437-v2.3.0 -y 16 -m 10000000 -e 254462598 -c 2.3.0
-src/flamenco/runtime/tests/run_ledger_backtest.sh -l local-cluster-v2.3.0 -y 1 -m 2000000 -e 5010 -c 2.3.0
-src/flamenco/runtime/tests/run_ledger_backtest.sh -l mainnet-262654839-v2.3.0 -y 3 -m 10000000 -e 262654840 -c 2.3.0
-src/flamenco/runtime/tests/run_ledger_backtest.sh -l mainnet-257037451-v2.3.0 -y 3 -m 2000000 -e 257037454 -c 2.3.0
-src/flamenco/runtime/tests/run_ledger_backtest.sh -l mainnet-257035225-v2.3.0 -y 4 -m 2000000 -e 257035233 -c 2.3.0
-src/flamenco/runtime/tests/run_ledger_backtest.sh -l mainnet-257465453-v2.3.0 -y 4 -m 10000000 -e 257465454 -c 2.3.0
-src/flamenco/runtime/tests/run_ledger_backtest.sh -l mainnet-257058865-v2.3.0 -y 3 -m 2000000 -e 257058870 -c 2.3.0
-src/flamenco/runtime/tests/run_ledger_backtest.sh -l mainnet-257059815-v2.3.0 -y 3 -m 2000000 -e 257059818 -c 2.3.0
-src/flamenco/runtime/tests/run_ledger_backtest.sh -l mainnet-257061172-v2.3.0 -y 3 -m 2000000 -e 257061175 -c 2.3.0
-src/flamenco/runtime/tests/run_ledger_backtest.sh -l mainnet-257222682-v2.3.0 -y 3 -m 2000000 -e 257222688 -c 2.3.0
-src/flamenco/runtime/tests/run_ledger_backtest.sh -l mainnet-264890264-v2.3.0 -y 3 -m 2000000 -e 264890265 -c 2.3.0
-src/flamenco/runtime/tests/run_ledger_backtest.sh -l mainnet-257229353-v2.3.0 -y 4 -m 2000000 -e 257229357 -c 2.3.0
-src/flamenco/runtime/tests/run_ledger_backtest.sh -l mainnet-257257983-v2.3.0 -y 3 -m 2000000 -e 257257986 -c 2.3.0
-src/flamenco/runtime/tests/run_ledger_backtest.sh -l mainnet-267728520-v2.3.0 -y 3 -m 2000000 -e 267728522 -c 2.3.0
-src/flamenco/runtime/tests/run_ledger_backtest.sh -l mainnet-267651942-v2.3.0 -y 3 -m 2000000 -e 267651943 -c 2.3.0
-src/flamenco/runtime/tests/run_ledger_backtest.sh -l mainnet-267081197-v2.3.0 -y 3 -m 2000000 -e 267081198 -c 2.3.0
-src/flamenco/runtime/tests/run_ledger_backtest.sh -l mainnet-267085604-v2.3.0 -y 3 -m 2000000 -e 267085605 -c 2.3.0
-src/flamenco/runtime/tests/run_ledger_backtest.sh -l mainnet-265688706-v2.3.0 -y 3 -m 2000000 -e 265688707 -c 2.3.0
-src/flamenco/runtime/tests/run_ledger_backtest.sh -l mainnet-265330432-v2.3.0 -y 3 -m 2000000 -e 265330433 -c 2.3.0
-src/flamenco/runtime/tests/run_ledger_backtest.sh -l mainnet-268575190-v2.3.0 -y 3 -m 2000000 -e 268575191 -c 2.3.0
-src/flamenco/runtime/tests/run_ledger_backtest.sh -l mainnet-268129380-v2.3.0 -y 3 -m 2000000 -e 268129380 -c 2.3.0
-src/flamenco/runtime/tests/run_ledger_backtest.sh -l mainnet-268163043-v2.3.0 -y 3 -m 2000000 -e 268163043 -c 2.3.0
-src/flamenco/runtime/tests/run_ledger_backtest.sh -l mainnet-269511381-v2.3.0 -y 3 -m 2000000 -e 269511381 -c 2.3.0
-src/flamenco/runtime/tests/run_ledger_backtest.sh -l mainnet-269567236-v2.3.0 -y 3 -m 2000000 -e 269567236 -c 2.3.0
-src/flamenco/runtime/tests/run_ledger_backtest.sh -l mainnet-266134813-v2.3.0 -y 3 -m 2000000 -e 266134814 -c 2.3.0
-src/flamenco/runtime/tests/run_ledger_backtest.sh -l mainnet-266545736-v2.3.0 -y 3 -m 2000000 -e 266545737 -c 2.3.0
-src/flamenco/runtime/tests/run_ledger_backtest.sh -l mainnet-267059180-v2.3.0 -y 3 -m 2000000 -e 267059181 -c 2.3.0
-src/flamenco/runtime/tests/run_ledger_backtest.sh -l mainnet-267580466-v2.3.0 -y 3 -m 2000000 -e 267580467 -c 2.3.0
-src/flamenco/runtime/tests/run_ledger_backtest.sh -l mainnet-268196194-v2.3.0 -y 3 -m 2000000 -e 268196195 -c 2.3.0
-src/flamenco/runtime/tests/run_ledger_backtest.sh -l mainnet-267766641-v2.3.0 -y 3 -m 2000000 -e 267766642 -c 2.3.0
-src/flamenco/runtime/tests/run_ledger_backtest.sh -l mainnet-269648145-v2.3.0 -y 3 -m 2000000 -e 269648146 -c 2.3.0
-src/flamenco/runtime/tests/run_ledger_backtest.sh -l testnet-281688085-v2.3.0 -y 3 -m 2000000 -e 281688086 -c 2.3.0
-src/flamenco/runtime/tests/run_ledger_backtest.sh -l mainnet-277660422-v2.3.0 -y 3 -m 2000000 -e 277660423 -c 2.3.0
-src/flamenco/runtime/tests/run_ledger_backtest.sh -l mainnet-277876060-v2.3.0 -y 3 -m 2000000 -e 277876061 -c 2.3.0
-src/flamenco/runtime/tests/run_ledger_backtest.sh -l mainnet-277927063-v2.3.0 -y 3 -m 2000000 -e 277927065 -c 2.3.0
-src/flamenco/runtime/tests/run_ledger_backtest.sh -l mainnet-281375356-v2.3.0 -y 3 -m 2000000 -e 281375359 -c 2.3.0
-src/flamenco/runtime/tests/run_ledger_backtest.sh -l mainnet-251418170-v2.3.0 -y 5 -m 2000000 -e 251418233 -c 2.3.0
-src/flamenco/runtime/tests/run_ledger_backtest.sh -l mainnet-282232100-v2.3.0 -y 3 -m 2000000 -e 282232101 -c 2.3.0
-src/flamenco/runtime/tests/run_ledger_backtest.sh -l mainnet-282151715-v2.3.0 -y 3 -m 2000000 -e 282151717 -c 2.3.0
-src/flamenco/runtime/tests/run_ledger_backtest.sh -l mainnet-286450148-v2.3.0 -y 3 -m 2000000 -e 286450151 -c 2.3.0
-src/flamenco/runtime/tests/run_ledger_backtest.sh -l multi-epoch-per-200-v2.3.0 -y 1 -m 2000000 -e 984 -c 2.3.0
-src/flamenco/runtime/tests/run_ledger_backtest.sh -l multi-epoch-per-300-v2.3.0 -y 1 -m 2000000 -e 984 -c 2.3.0
-src/flamenco/runtime/tests/run_ledger_backtest.sh -l multi-epoch-per-500-v2.3.0 -y 1 -m 2000000 -e 984 -c 2.3.0
-src/flamenco/runtime/tests/run_ledger_backtest.sh -l testnet-297489336-v2.3.0 -y 3 -m 2000000 -e 297489363 -c 2.3.0
-src/flamenco/runtime/tests/run_ledger_backtest.sh -l mainnet-300377724-v2.3.0 -y 5 -m 2000000 -e 300377728 -c 2.3.0
-src/flamenco/runtime/tests/run_ledger_backtest.sh -l mainnet-300645644-v2.3.0 -y 5 -m 2000000 -e 300645644 -c 2.3.0
-src/flamenco/runtime/tests/run_ledger_backtest.sh -l mainnet-300648964-v2.3.0 -y 5 -m 2000000 -e 300648964 -c 2.3.0
-src/flamenco/runtime/tests/run_ledger_backtest.sh -l mainnet-301359740-v2.3.0 -y 5 -m 2000000 -e 301359740 -c 2.3.0
-src/flamenco/runtime/tests/run_ledger_backtest.sh -l mainnet-257181032-v2.3.0 -y 3 -m 2000000 -e 257181035 -c 2.3.0
-src/flamenco/runtime/tests/run_ledger_backtest.sh -l mainnet-257047660-v2.3.0 -y 3 -m 2000000 -e 257047662 -c 2.3.0
-src/flamenco/runtime/tests/run_ledger_backtest.sh -l mainnet-257047659-v2.3.0 -y 3 -m 2000000 -e 257047660 -c 2.3.0
-src/flamenco/runtime/tests/run_ledger_backtest.sh -l mainnet-308445707-v2.3.0 -y 5 -m 2000000 -e 308445711 -c 2.3.0
-src/flamenco/runtime/tests/run_ledger_backtest.sh -l testnet-307395181-v2.3.0 -y 3 -m 2000000 -e 307395190 -c 2.3.0
-src/flamenco/runtime/tests/run_ledger_backtest.sh -l mainnet-308392063-v2.3.0 -y 5 -m 2000000 -e 308392090 -c 2.3.0
-src/flamenco/runtime/tests/run_ledger_backtest.sh -l devnet-350814254-v2.3.0 -y 3 -m 2000000 -e 350814284 -c 2.3.0
-src/flamenco/runtime/tests/run_ledger_backtest.sh -l testnet-311586340-v2.3.0 -y 3 -m 2000000 -e 311586380 -c 2.3.0
-src/flamenco/runtime/tests/run_ledger_backtest.sh -l testnet-281546597-v2.3.0 -y 3 -m 2000000 -e 281546597 -c 2.3.0
-src/flamenco/runtime/tests/run_ledger_backtest.sh -l mainnet-324823213-v2.3.0 -y 4 -m 2000000 -e 324823214 -c 2.3.0
-src/flamenco/runtime/tests/run_ledger_backtest.sh -l mainnet-325467935-v2.3.0 -y 4 -m 2000000 -e 325467936 -c 2.3.0
-src/flamenco/runtime/tests/run_ledger_backtest.sh -l testnet-283927487-v2.3.0 -y 3 -m 2000000 -e 283927497 -c 2.3.0
-src/flamenco/runtime/tests/run_ledger_backtest.sh -l testnet-321168308-v2.3.0 -y 3 -m 2000000 -e 321168308 -c 2.3.0
-src/flamenco/runtime/tests/run_ledger_backtest.sh -l mainnet-327324660-v2.3.0 -y 4 -m 2000000 -e 327324660 -c 2.3.0
-src/flamenco/runtime/tests/run_ledger_backtest.sh -l devnet-370199634-v2.3.0 -y 3 -m 200000 -e 370199634 -c 2.3.0
-src/flamenco/runtime/tests/run_ledger_backtest.sh -l mainnet-330219081-v2.3.0 -y 4 -m 2000000 -e 330219082 -c 2.3.0
-src/flamenco/runtime/tests/run_ledger_backtest.sh -l devnet-372721907-v2.3.0 -y 3 -m 2000000 -e 372721910 -c 2.3.0
-src/flamenco/runtime/tests/run_ledger_backtest.sh -l mainnet-331691646-v2.3.0 -y 4 -m 2000000 -e 331691647 -c 2.3.0
-src/flamenco/runtime/tests/run_ledger_backtest.sh -l devnet-378683870-v2.3.0 -y 3 -m 2000000 -e 378683872 -c 2.3.0
-src/flamenco/runtime/tests/run_ledger_backtest.sh -l slashing-ledger-v2.3.0 -y 1 -m 2000000 -e 44 -c 2.3.0
-src/flamenco/runtime/tests/run_ledger_backtest.sh -l devnet-380592002-v2.3.0 -y 3 -m 2000000 -e 380592006 -c 2.3.0
+src/flamenco/runtime/tests/run_ledger_backtest.sh -l testnet-519-v2.3.0 -y 3 -m 2000000 -e 255312007 -c 2.3.0 -lt
+src/flamenco/runtime/tests/run_ledger_backtest.sh -l mainnet-257066033-v2.3.0 -y 3 -m 2000000 -e 257066038 -c 2.3.0 -lt
+src/flamenco/runtime/tests/run_ledger_backtest.sh -l mainnet-257066844-v2.3.0 -y 3 -m 2000000 -e 257066849 -c 2.3.0 -lt
+src/flamenco/runtime/tests/run_ledger_backtest.sh -l mainnet-257067457-v2.3.0 -y 3 -m 2000000 -e 257067461 -c 2.3.0 -lt
+src/flamenco/runtime/tests/run_ledger_backtest.sh -l mainnet-257068890-v2.3.0 -y 3 -m 2000000 -e 257068895 -c 2.3.0 -lt
+src/flamenco/runtime/tests/run_ledger_backtest.sh -l mainnet-257181622-v2.3.0 -y 3 -m 2000000 -e 257181624 -c 2.3.0 -lt
+src/flamenco/runtime/tests/run_ledger_backtest.sh -l mainnet-254462437-v2.3.0 -y 16 -m 10000000 -e 254462598 -c 2.3.0 -lt
+src/flamenco/runtime/tests/run_ledger_backtest.sh -l local-cluster-v2.3.0 -y 1 -m 2000000 -e 5010 -c 2.3.0 -lt
+src/flamenco/runtime/tests/run_ledger_backtest.sh -l mainnet-262654839-v2.3.0 -y 3 -m 10000000 -e 262654840 -c 2.3.0 -lt
+src/flamenco/runtime/tests/run_ledger_backtest.sh -l mainnet-257037451-v2.3.0 -y 3 -m 2000000 -e 257037454 -c 2.3.0 -lt
+src/flamenco/runtime/tests/run_ledger_backtest.sh -l mainnet-257035225-v2.3.0 -y 4 -m 2000000 -e 257035233 -c 2.3.0 -lt
+src/flamenco/runtime/tests/run_ledger_backtest.sh -l mainnet-257465453-v2.3.0 -y 4 -m 10000000 -e 257465454 -c 2.3.0 -lt
+src/flamenco/runtime/tests/run_ledger_backtest.sh -l mainnet-257058865-v2.3.0 -y 3 -m 2000000 -e 257058870 -c 2.3.0 -lt
+src/flamenco/runtime/tests/run_ledger_backtest.sh -l mainnet-257059815-v2.3.0 -y 3 -m 2000000 -e 257059818 -c 2.3.0 -lt
+src/flamenco/runtime/tests/run_ledger_backtest.sh -l mainnet-257061172-v2.3.0 -y 3 -m 2000000 -e 257061175 -c 2.3.0 -lt
+src/flamenco/runtime/tests/run_ledger_backtest.sh -l mainnet-257222682-v2.3.0 -y 3 -m 2000000 -e 257222688 -c 2.3.0 -lt
+src/flamenco/runtime/tests/run_ledger_backtest.sh -l mainnet-264890264-v2.3.0 -y 3 -m 2000000 -e 264890265 -c 2.3.0 -lt
+src/flamenco/runtime/tests/run_ledger_backtest.sh -l mainnet-257229353-v2.3.0 -y 4 -m 2000000 -e 257229357 -c 2.3.0 -lt
+src/flamenco/runtime/tests/run_ledger_backtest.sh -l mainnet-257257983-v2.3.0 -y 3 -m 2000000 -e 257257986 -c 2.3.0 -lt
+src/flamenco/runtime/tests/run_ledger_backtest.sh -l mainnet-267728520-v2.3.0 -y 3 -m 2000000 -e 267728522 -c 2.3.0 -lt
+src/flamenco/runtime/tests/run_ledger_backtest.sh -l mainnet-267651942-v2.3.0 -y 3 -m 2000000 -e 267651943 -c 2.3.0 -lt
+src/flamenco/runtime/tests/run_ledger_backtest.sh -l mainnet-267081197-v2.3.0 -y 3 -m 2000000 -e 267081198 -c 2.3.0 -lt
+src/flamenco/runtime/tests/run_ledger_backtest.sh -l mainnet-267085604-v2.3.0 -y 3 -m 2000000 -e 267085605 -c 2.3.0 -lt
+src/flamenco/runtime/tests/run_ledger_backtest.sh -l mainnet-265688706-v2.3.0 -y 3 -m 2000000 -e 265688707 -c 2.3.0 -lt
+src/flamenco/runtime/tests/run_ledger_backtest.sh -l mainnet-265330432-v2.3.0 -y 3 -m 2000000 -e 265330433 -c 2.3.0 -lt
+src/flamenco/runtime/tests/run_ledger_backtest.sh -l mainnet-268575190-v2.3.0 -y 3 -m 2000000 -e 268575191 -c 2.3.0 -lt
+src/flamenco/runtime/tests/run_ledger_backtest.sh -l mainnet-268129380-v2.3.0 -y 3 -m 2000000 -e 268129380 -c 2.3.0 -lt
+src/flamenco/runtime/tests/run_ledger_backtest.sh -l mainnet-268163043-v2.3.0 -y 3 -m 2000000 -e 268163043 -c 2.3.0 -lt
+src/flamenco/runtime/tests/run_ledger_backtest.sh -l mainnet-269511381-v2.3.0 -y 3 -m 2000000 -e 269511381 -c 2.3.0 -lt
+src/flamenco/runtime/tests/run_ledger_backtest.sh -l mainnet-269567236-v2.3.0 -y 3 -m 2000000 -e 269567236 -c 2.3.0 -lt
+src/flamenco/runtime/tests/run_ledger_backtest.sh -l mainnet-266134813-v2.3.0 -y 3 -m 2000000 -e 266134814 -c 2.3.0 -lt
+src/flamenco/runtime/tests/run_ledger_backtest.sh -l mainnet-266545736-v2.3.0 -y 3 -m 2000000 -e 266545737 -c 2.3.0 -lt
+src/flamenco/runtime/tests/run_ledger_backtest.sh -l mainnet-267059180-v2.3.0 -y 3 -m 2000000 -e 267059181 -c 2.3.0 -lt
+src/flamenco/runtime/tests/run_ledger_backtest.sh -l mainnet-267580466-v2.3.0 -y 3 -m 2000000 -e 267580467 -c 2.3.0 -lt
+src/flamenco/runtime/tests/run_ledger_backtest.sh -l mainnet-268196194-v2.3.0 -y 3 -m 2000000 -e 268196195 -c 2.3.0 -lt
+src/flamenco/runtime/tests/run_ledger_backtest.sh -l mainnet-267766641-v2.3.0 -y 3 -m 2000000 -e 267766642 -c 2.3.0 -lt
+src/flamenco/runtime/tests/run_ledger_backtest.sh -l mainnet-269648145-v2.3.0 -y 3 -m 2000000 -e 269648146 -c 2.3.0 -lt
+src/flamenco/runtime/tests/run_ledger_backtest.sh -l testnet-281688085-v2.3.0 -y 3 -m 2000000 -e 281688086 -c 2.3.0 -lt
+src/flamenco/runtime/tests/run_ledger_backtest.sh -l mainnet-277660422-v2.3.0 -y 3 -m 2000000 -e 277660423 -c 2.3.0 -lt
+src/flamenco/runtime/tests/run_ledger_backtest.sh -l mainnet-277876060-v2.3.0 -y 3 -m 2000000 -e 277876061 -c 2.3.0 -lt
+src/flamenco/runtime/tests/run_ledger_backtest.sh -l mainnet-277927063-v2.3.0 -y 3 -m 2000000 -e 277927065 -c 2.3.0 -lt
+src/flamenco/runtime/tests/run_ledger_backtest.sh -l mainnet-281375356-v2.3.0 -y 3 -m 2000000 -e 281375359 -c 2.3.0 -lt
+src/flamenco/runtime/tests/run_ledger_backtest.sh -l mainnet-251418170-v2.3.0 -y 5 -m 2000000 -e 251418233 -c 2.3.0 -lt
+src/flamenco/runtime/tests/run_ledger_backtest.sh -l mainnet-282232100-v2.3.0 -y 3 -m 2000000 -e 282232101 -c 2.3.0 -lt
+src/flamenco/runtime/tests/run_ledger_backtest.sh -l mainnet-282151715-v2.3.0 -y 3 -m 2000000 -e 282151717 -c 2.3.0 -lt
+src/flamenco/runtime/tests/run_ledger_backtest.sh -l mainnet-286450148-v2.3.0 -y 3 -m 2000000 -e 286450151 -c 2.3.0 -lt
+src/flamenco/runtime/tests/run_ledger_backtest.sh -l multi-epoch-per-200-v2.3.0 -y 1 -m 2000000 -e 984 -c 2.3.0 -lt
+src/flamenco/runtime/tests/run_ledger_backtest.sh -l multi-epoch-per-300-v2.3.0 -y 1 -m 2000000 -e 984 -c 2.3.0 -lt
+src/flamenco/runtime/tests/run_ledger_backtest.sh -l multi-epoch-per-500-v2.3.0 -y 1 -m 2000000 -e 984 -c 2.3.0 -lt
+src/flamenco/runtime/tests/run_ledger_backtest.sh -l testnet-297489336-v2.3.0 -y 3 -m 2000000 -e 297489363 -c 2.3.0 -lt
+src/flamenco/runtime/tests/run_ledger_backtest.sh -l mainnet-300377724-v2.3.0 -y 5 -m 2000000 -e 300377728 -c 2.3.0 -lt
+src/flamenco/runtime/tests/run_ledger_backtest.sh -l mainnet-300645644-v2.3.0 -y 5 -m 2000000 -e 300645644 -c 2.3.0 -lt
+src/flamenco/runtime/tests/run_ledger_backtest.sh -l mainnet-300648964-v2.3.0 -y 5 -m 2000000 -e 300648964 -c 2.3.0 -lt
+src/flamenco/runtime/tests/run_ledger_backtest.sh -l mainnet-301359740-v2.3.0 -y 5 -m 2000000 -e 301359740 -c 2.3.0 -lt
+src/flamenco/runtime/tests/run_ledger_backtest.sh -l mainnet-257181032-v2.3.0 -y 3 -m 2000000 -e 257181035 -c 2.3.0 -lt
+src/flamenco/runtime/tests/run_ledger_backtest.sh -l mainnet-257047660-v2.3.0 -y 3 -m 2000000 -e 257047662 -c 2.3.0 -lt
+src/flamenco/runtime/tests/run_ledger_backtest.sh -l mainnet-257047659-v2.3.0 -y 3 -m 2000000 -e 257047660 -c 2.3.0 -lt
+src/flamenco/runtime/tests/run_ledger_backtest.sh -l mainnet-308445707-v2.3.0 -y 5 -m 2000000 -e 308445711 -c 2.3.0 -lt
+src/flamenco/runtime/tests/run_ledger_backtest.sh -l testnet-307395181-v2.3.0 -y 3 -m 2000000 -e 307395190 -c 2.3.0 -lt
+src/flamenco/runtime/tests/run_ledger_backtest.sh -l mainnet-308392063-v2.3.0 -y 5 -m 2000000 -e 308392090 -c 2.3.0 -lt
+src/flamenco/runtime/tests/run_ledger_backtest.sh -l devnet-350814254-v2.3.0 -y 3 -m 2000000 -e 350814284 -c 2.3.0 -lt
+src/flamenco/runtime/tests/run_ledger_backtest.sh -l testnet-311586340-v2.3.0 -y 3 -m 2000000 -e 311586380 -c 2.3.0 -lt
+src/flamenco/runtime/tests/run_ledger_backtest.sh -l testnet-281546597-v2.3.0 -y 3 -m 2000000 -e 281546597 -c 2.3.0 -lt
+src/flamenco/runtime/tests/run_ledger_backtest.sh -l mainnet-324823213-v2.3.0 -y 4 -m 2000000 -e 324823214 -c 2.3.0 -lt
+src/flamenco/runtime/tests/run_ledger_backtest.sh -l mainnet-325467935-v2.3.0 -y 4 -m 2000000 -e 325467936 -c 2.3.0 -lt
+src/flamenco/runtime/tests/run_ledger_backtest.sh -l testnet-283927487-v2.3.0 -y 3 -m 2000000 -e 283927497 -c 2.3.0 -lt
+src/flamenco/runtime/tests/run_ledger_backtest.sh -l testnet-321168308-v2.3.0 -y 3 -m 2000000 -e 321168308 -c 2.3.0 -lt
+src/flamenco/runtime/tests/run_ledger_backtest.sh -l mainnet-327324660-v2.3.0 -y 4 -m 2000000 -e 327324660 -c 2.3.0 -lt
+src/flamenco/runtime/tests/run_ledger_backtest.sh -l devnet-370199634-v2.3.0 -y 3 -m 200000 -e 370199634 -c 2.3.0 -lt
+src/flamenco/runtime/tests/run_ledger_backtest.sh -l mainnet-330219081-v2.3.0 -y 4 -m 2000000 -e 330219082 -c 2.3.0 -lt
+src/flamenco/runtime/tests/run_ledger_backtest.sh -l devnet-372721907-v2.3.0 -y 3 -m 2000000 -e 372721910 -c 2.3.0 -lt
+src/flamenco/runtime/tests/run_ledger_backtest.sh -l mainnet-331691646-v2.3.0 -y 4 -m 2000000 -e 331691647 -c 2.3.0 -lt
+src/flamenco/runtime/tests/run_ledger_backtest.sh -l devnet-378683870-v2.3.0 -y 3 -m 2000000 -e 378683872 -c 2.3.0 -lt
+src/flamenco/runtime/tests/run_ledger_backtest.sh -l slashing-ledger-v2.3.0 -y 1 -m 2000000 -e 44 -c 2.3.0 -lt
+src/flamenco/runtime/tests/run_ledger_backtest.sh -l devnet-380592002-v2.3.0 -y 3 -m 2000000 -e 380592006 -c 2.3.0 -lt
src/flamenco/runtime/tests/run_ledger_backtest.sh -l testnet-336218682-v2.3.0 -y 5 -m 2000000 -e 336218683 -c 2.3.0
src/flamenco/runtime/tests/run_ledger_backtest.sh -l testnet-340269866-v2.3.0 -y 5 -m 2000000 -e 340269872 -c 2.3.0
src/flamenco/runtime/tests/run_ledger_backtest.sh -l testnet-340272018-v2.3.0 -y 5 -m 2000000 -e 340272024 -c 2.3.0
src/flamenco/runtime/tests/run_ledger_backtest.sh -l devnet-390056400-v2.3.0 -y 10 -m 2000000 -e 390056406 -c 2.3.0
# src/flamenco/runtime/tests/run_ledger_backtest.sh -l localnet-1678201-v2.3.0 -y 1 -m 2000000 -e 1679200 -c 2.3.0 -i shredcap
# src/flamenco/runtime/tests/run_ledger_backtest.sh -l testnet-346796573-v2.3.0 -y 30 -m 90000000 -e 346796800 -c 2.3.0 -i shredcap -v true
-src/flamenco/runtime/tests/run_ledger_backtest.sh -l testnet-346556000 -y 3 -m 2000000 -e 346556337 -c 2.3.0
-src/flamenco/runtime/tests/run_ledger_backtest.sh -l testnet-346179946 -y 30 -m 90000000 -e 346179950 -c 2.3.0
-src/flamenco/runtime/tests/run_ledger_backtest.sh -l multi-bpf-loader-v2.3.0 -y 1 -m 1000 -e 108 -c 2.3.0
+src/flamenco/runtime/tests/run_ledger_backtest.sh -l testnet-346556000 -y 3 -m 2000000 -e 346556337 -c 2.3.0 -lt
+src/flamenco/runtime/tests/run_ledger_backtest.sh -l testnet-346179946 -y 30 -m 90000000 -e 346179950 -c 2.3.0 -lt
+src/flamenco/runtime/tests/run_ledger_backtest.sh -l multi-bpf-loader-v2.3.0 -y 1 -m 1000 -e 108 -c 2.3.0 -lt
diff --git a/src/flamenco/runtime/tests/run_ledger_backtest.sh b/src/flamenco/runtime/tests/run_ledger_backtest.sh
index 89149ca9492..f643085a9cc 100755
--- a/src/flamenco/runtime/tests/run_ledger_backtest.sh
+++ b/src/flamenco/runtime/tests/run_ledger_backtest.sh
@@ -27,6 +27,7 @@ ONE_OFFS=""
HUGE_TLBFS_MOUNT_PATH=${HUGE_TLBFS_MOUNT_PATH:="/mnt/.fd"}
HUGE_TLBFS_ALLOW_HUGEPAGE_INCREASE=${HUGE_TLBFS_ALLOW_HUGEPAGE_INCREASE:="true"}
HAS_INCREMENTAL="false"
+ENABLE_LTHASH_VERIFICATION=false
while [[ $# -gt 0 ]]; do
case $1 in
@@ -97,6 +98,10 @@ while [[ $# -gt 0 ]]; do
HAS_INCREMENTAL="$2"
shift
;;
+ -lt|--enable-lthash-verification)
+ ENABLE_LTHASH_VERIFICATION=true
+ shift
+ ;;
-*|--*)
echo "unknown option $1"
exit 1
@@ -116,6 +121,12 @@ mkdir -p $OBJDIR/cov/raw
DUMP=$(realpath $DUMP_DIR)
mkdir -p $DUMP
+if $ENABLE_LTHASH_VERIFICATION; then
+HASH_TILE_COUNT=1
+else
+HASH_TILE_COUNT=0
+fi
+
if [[ ! -e $DUMP/$LEDGER && SKIP_INGEST -eq 0 ]]; then
if [[ -n "$ZST" ]]; then
echo "Downloading gs://firedancer-ci-resources/$LEDGER.tar.zst"
@@ -153,6 +164,7 @@ echo "
bank_tile_count = 1
shred_tile_count = 4
exec_tile_count = 4
+ hash_tile_count = $HASH_TILE_COUNT
[tiles]
[tiles.archiver]
enabled = true