Skip to content

Commit 319aac7

Browse files
cali-jumptradingcali-jumptrading
committed
snapshots: lthash tile
1 parent 47eeb98 commit 319aac7

28 files changed

+966
-186
lines changed

‎book/api/metrics-generated.md

Lines changed: 11 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -900,6 +900,17 @@
900900

901901
</div>
902902

903+
## Snaplt Tile
904+
905+
<div class="metrics">
906+
907+
| Metric | Type | Description |
908+
|--------|------|-------------|
909+
| <span class="metrics-name">snaplt_&#8203;state</span> | gauge | State of the tile. 0=hashing, 1=done, 2=shutdown |
910+
| <span class="metrics-name">snaplt_&#8203;accounts_&#8203;hashed</span> | gauge | Number of accounts hashed so far during snapshot loading. Might decrease if snapshot load is aborted and restarted |
911+
912+
</div>
913+
903914
## Ipecho Tile
904915

905916
<div class="metrics">

‎src/app/firedancer-dev/commands/backtest.c

Lines changed: 77 additions & 15 deletions
Original file line numberDiff line numberDiff line change
@@ -24,6 +24,7 @@
2424
#include "../../../util/pod/fd_pod_format.h"
2525
#include "../../../discof/replay/fd_replay_notif.h"
2626
#include "../../../discof/reasm/fd_reasm.h"
27+
#include "../../../discof/restore/utils/fd_ssctrl.h"
2728
#include "../../../flamenco/runtime/fd_runtime_public.h" /* FD_RUNTIME_PUBLIC_ACCOUNT_UPDATE_MSG_MTU */
2829
#include "../main.h"
2930

@@ -36,6 +37,8 @@ static void
3637
backtest_topo( config_t * config ) {
3738
ulong exec_tile_cnt = config->firedancer.layout.exec_tile_count;
3839
ulong writer_tile_cnt = config->firedancer.layout.writer_tile_count;
40+
ulong snaplt_tile_cnt = config->firedancer.layout.snaplt_tile_count;
41+
int snaplt_enabled = !config->development.snapshots.disable_lthash_verification;
3942

4043
fd_topo_t * topo = { fd_topob_new( &config->topo, config->name ) };
4144
topo->max_page_size = fd_cstr_to_shmem_page_sz( config->hugetlbfs.max_page_size );
@@ -99,6 +102,14 @@ backtest_topo( config_t * config ) {
99102
snapdc_tile->allow_shutdown = 1;
100103
snapin_tile->allow_shutdown = 1;
101104

105+
if( FD_LIKELY( snaplt_enabled ) ) {
106+
fd_topob_wksp( topo, "snaplt" );
107+
for( ulong i=0UL; i<snaplt_tile_cnt; i++ ) {
108+
fd_topo_tile_t * snaplt_tile = fd_topob_tile( topo, "snaplt", "snaplt", "metric_in", cpu_idx++, 0, 0 );
109+
snaplt_tile->allow_shutdown = 1;
110+
}
111+
}
112+
102113
/**********************************************************************/
103114
/* Setup backtest->replay link (repair_repla) in topo */
104115
/**********************************************************************/
@@ -121,16 +132,26 @@ backtest_topo( config_t * config ) {
121132
fd_topob_wksp( topo, "snapdc_rd" );
122133
fd_topob_wksp( topo, "snapin_rd" );
123134
fd_topob_wksp( topo, "snap_out" );
124-
fd_topob_wksp( topo, "replay_manif" );
135+
136+
if( FD_LIKELY( snaplt_enabled ) ) {
137+
fd_topob_wksp( topo, "snapin_lt" );
138+
fd_topob_wksp( topo, "snaplt_out" );
139+
}
140+
125141
/* TODO: Should be depth of 1 or 2, not 4, but it causes backpressure
126142
from the replay tile parsing the manifest, remove when this is
127143
fixed. */
128144
fd_topob_link( topo, "snap_out", "snap_out", 4UL, 5UL*(1UL<<30UL), 1UL );
129145

130146
fd_topob_link( topo, "snap_zstd", "snap_zstd", 8192UL, 16384UL, 1UL );
131147
fd_topob_link( topo, "snap_stream", "snap_stream", 2048UL, USHORT_MAX, 1UL );
132-
fd_topob_link( topo, "snapdc_rd", "snapdc_rd", 128UL, 0UL, 1UL );
133-
fd_topob_link( topo, "snapin_rd", "snapin_rd", 128UL, 0UL, 1UL );
148+
fd_topob_link( topo, "snapdc_rd", "snapdc_rd", 128UL, 0UL, 1UL );
149+
fd_topob_link( topo, "snapin_rd", "snapin_rd", 128UL, 0UL, 1UL );
150+
151+
if( FD_LIKELY( snaplt_enabled ) ) {
152+
fd_topob_link( topo, "snapin_lt", "snapin_lt", 128UL, sizeof(fd_snapshot_existing_account_t), 1UL );
153+
FOR(snaplt_tile_cnt) fd_topob_link( topo, "snaplt_out", "snaplt_out", 128UL, 2048UL, 1UL );
154+
}
134155

135156
fd_topob_tile_out( topo, "snaprd", 0UL, "snap_zstd", 0UL );
136157
fd_topob_tile_in ( topo, "snapdc", 0UL, "metric_in", "snap_zstd", 0UL, FD_TOPOB_RELIABLE, FD_TOPOB_POLLED );
@@ -144,6 +165,13 @@ backtest_topo( config_t * config ) {
144165
fd_topob_tile_in( topo, "snaprd", 0UL, "metric_in", "snapin_rd", 0UL, FD_TOPOB_RELIABLE, FD_TOPOB_POLLED );
145166
fd_topob_tile_out( topo, "snapin", 0UL, "snapin_rd", 0UL );
146167

168+
if( FD_LIKELY( snaplt_enabled ) ) {
169+
fd_topob_tile_out( topo, "snapin", 0UL, "snapin_lt", 0UL );
170+
FOR(snaplt_tile_cnt) fd_topob_tile_in( topo, "snapin", 0UL, "metric_in", "snaplt_out", i, FD_TOPOB_RELIABLE, FD_TOPOB_POLLED );
171+
FOR(snaplt_tile_cnt) fd_topob_tile_in( topo, "snaplt", i, "metric_in", "snapin_lt", 0UL, FD_TOPOB_RELIABLE, FD_TOPOB_POLLED );
172+
FOR(snaplt_tile_cnt) fd_topob_tile_out( topo, "snaplt", i, "snaplt_out", i );
173+
}
174+
147175
/**********************************************************************/
148176
/* More backtest->replay links in topo */
149177
/**********************************************************************/
@@ -294,15 +322,7 @@ backtest_topo( config_t * config ) {
294322
FD_TEST( fd_pod_insertf_ulong( topo->props, busy_obj->id, "bank_busy.%lu", i ) );
295323
}
296324

297-
/* Replay decoded manifest dcache topo obj */
298-
fd_topo_obj_t * replay_manifest_dcache = fd_topob_obj( topo, "dcache", "replay_manif" );
299-
fd_pod_insertf_ulong( topo->props, 2UL << 30UL, "obj.%lu.data_sz", replay_manifest_dcache->id );
300-
fd_pod_insert_ulong( topo->props, "manifest_dcache", replay_manifest_dcache->id );
301-
302325
fd_topob_tile_uses( topo, snapin_tile, funk_obj, FD_SHMEM_JOIN_MODE_READ_WRITE );
303-
fd_topob_tile_uses( topo, snapin_tile, replay_manifest_dcache, FD_SHMEM_JOIN_MODE_READ_WRITE );
304-
fd_topob_tile_uses( topo, replay_tile, replay_manifest_dcache, FD_SHMEM_JOIN_MODE_READ_ONLY );
305-
306326
for( ulong i=0UL; i<topo->tile_cnt; i++ ) {
307327
fd_topo_tile_t * tile = &topo->tiles[ i ];
308328
if( !fd_topo_configure_tile( tile, config ) ) {
@@ -380,6 +400,16 @@ backtest_cmd_fn( args_t * args FD_PARAM_UNUSED,
380400
fd_topo_tile_t * snapdc_tile = &topo->tiles[ fd_topo_find_tile( topo, "snapdc", 0UL ) ];
381401
fd_topo_tile_t * snapin_tile = &topo->tiles[ fd_topo_find_tile( topo, "snapin", 0UL ) ];
382402

403+
ulong volatile * snaplt_metrics[ FD_MAX_SNAPLT_TILES ];
404+
ulong snaplt_tile_cnt = fd_topo_tile_name_cnt( topo, "snaplt" );
405+
406+
for( ulong i=0UL; i<snaplt_tile_cnt; i++ ) {
407+
ulong snaplt_tile_idx = fd_topo_find_tile( topo, "snaplt", i );
408+
FD_TEST( snaplt_tile_idx!=ULONG_MAX );
409+
fd_topo_tile_t * snaplt_tile = &topo->tiles[ snaplt_tile_idx ];
410+
snaplt_metrics[ i ] = fd_metrics_tile( snaplt_tile->metrics );
411+
}
412+
383413
ulong volatile * const snaprd_metrics = fd_metrics_tile( snaprd_tile->metrics );
384414
ulong volatile * const snapdc_metrics = fd_metrics_tile( snapdc_tile->metrics );
385415
ulong volatile * const snapin_metrics = fd_metrics_tile( snapin_tile->metrics );
@@ -391,16 +421,34 @@ backtest_cmd_fn( args_t * args FD_PARAM_UNUSED,
391421
ulong snapdc_wait_old = 0UL;
392422
ulong snapin_backp_old = 0UL;
393423
ulong snapin_wait_old = 0UL;
424+
ulong snaplt_backp_old = 0UL;
425+
ulong snaplt_wait_old = 0UL;
394426
ulong acc_cnt_old = 0UL;
395427
sleep( 1 );
396-
puts( "-------------backp=(snaprd,snapdc,snapin) busy=(snaprd,snapdc,snapin)---------------" );
428+
puts( "" );
429+
puts( "Columns:" );
430+
puts( "- bw: Uncompressed bandwidth" );
431+
puts( "- backp: Backpressured by downstream tile" );
432+
puts( "- stall: Waiting on upstream tile" );
433+
puts( "- acc: Number of accounts" );
434+
puts( "" );
435+
puts( "-------------backp=(snaprd,snapdc,snapin,snaplt) busy=(snaprd,snapdc,snapin,snaplt)---------------" );
397436
long next = start+1000L*1000L*1000L;
398437
for(;;) {
399438
ulong snaprd_status = FD_VOLATILE_CONST( snaprd_metrics[ MIDX( GAUGE, TILE, STATUS ) ] );
400439
ulong snapdc_status = FD_VOLATILE_CONST( snapdc_metrics[ MIDX( GAUGE, TILE, STATUS ) ] );
401440
ulong snapin_status = FD_VOLATILE_CONST( snapin_metrics[ MIDX( GAUGE, TILE, STATUS ) ] );
441+
ulong snaplt_status = ULONG_MAX;
402442

403-
if( FD_UNLIKELY( snaprd_status==2UL && snapdc_status==2UL && snapin_status == 2UL ) ) break;
443+
ulong snaplt_status_sum = 0UL;
444+
for( ulong i=0UL; i<snaplt_tile_cnt; i++ ) {
445+
ulong snaplt_status = FD_VOLATILE_CONST( snaplt_metrics[ i ][ MIDX( GAUGE, TILE, STATUS ) ] );
446+
snaplt_status_sum += snaplt_status;
447+
}
448+
if( FD_UNLIKELY( snaplt_status_sum==2UL*snaplt_tile_cnt ) ) snaplt_status = 2UL;
449+
else snaplt_status = snaplt_tile_cnt>0UL ? 1UL : 2UL;
450+
451+
if( FD_UNLIKELY( snaprd_status==2UL && snapdc_status==2UL && snapin_status == 2UL && snaplt_status==2UL ) ) break;
404452

405453
long cur = fd_log_wallclock();
406454
if( FD_UNLIKELY( cur<next ) ) {
@@ -420,16 +468,28 @@ backtest_cmd_fn( args_t * args FD_PARAM_UNUSED,
420468
ulong snapin_backp = snapin_metrics[ MIDX( COUNTER, TILE, REGIME_DURATION_NANOS_BACKPRESSURE_PREFRAG ) ];
421469
ulong snapin_wait = snapin_metrics[ MIDX( COUNTER, TILE, REGIME_DURATION_NANOS_CAUGHT_UP_PREFRAG ) ] +
422470
snapin_metrics[ MIDX( COUNTER, TILE, REGIME_DURATION_NANOS_CAUGHT_UP_POSTFRAG ) ] + snapin_backp;
471+
ulong snaplt_backp = 0UL;
472+
ulong snaplt_wait = 0UL;
473+
474+
for( ulong i=0UL; i<snaplt_tile_cnt; i++ ) {
475+
snaplt_backp += snaplt_metrics[ i ][ MIDX( COUNTER, TILE, REGIME_DURATION_NANOS_BACKPRESSURE_PREFRAG ) ];
476+
}
477+
for( ulong i=0UL; i<snaplt_tile_cnt; i++ ) {
478+
snaplt_wait += snaplt_metrics[ i ][ MIDX( COUNTER, TILE, REGIME_DURATION_NANOS_CAUGHT_UP_PREFRAG ) ] +
479+
snaplt_metrics[ i ][ MIDX( COUNTER, TILE, REGIME_DURATION_NANOS_CAUGHT_UP_POSTFRAG ) ] + snaplt_backp;
480+
}
423481

424-
ulong acc_cnt = snapin_metrics[ MIDX( GAUGE, SNAPIN, ACCOUNTS_INSERTED ) ];
425-
printf( "bw=%4.0f MB/s backp=(%3.0f%%,%3.0f%%,%3.0f%%) busy=(%3.0f%%,%3.0f%%,%3.0f%%) acc=%3.1f M/s\n",
482+
ulong acc_cnt = snapin_metrics[ MIDX( GAUGE, SNAPIN, ACCOUNTS_INSERTED ) ];
483+
printf( "bw=%4.0f MB/s backp=(%3.0f%%,%3.0f%%,%3.0f%%,%3.0f%%) busy=(%3.0f%%,%3.0f%%,%3.0f%%,%3.0f%%) acc=%3.1f M/s\n",
426484
(double)( total_off-total_off_old )/1e6,
427485
( (double)( snaprd_backp-snaprd_backp_old )*ns_per_tick )/1e7,
428486
( (double)( snapdc_backp-snapdc_backp_old )*ns_per_tick )/1e7,
429487
( (double)( snapin_backp-snapin_backp_old )*ns_per_tick )/1e7,
488+
( (double)( snaplt_backp-snaplt_backp_old )*ns_per_tick )/1e7/(double)snaplt_tile_cnt,
430489
100-( ( (double)( snaprd_wait-snaprd_wait_old )*ns_per_tick )/1e7 ),
431490
100-( ( (double)( snapdc_wait-snapdc_wait_old )*ns_per_tick )/1e7 ),
432491
100-( ( (double)( snapin_wait-snapin_wait_old )*ns_per_tick )/1e7 ),
492+
100-( ( (double)( snaplt_wait-snaplt_wait_old )*ns_per_tick )/1e7/(double)snaplt_tile_cnt ),
433493
(double)( acc_cnt-acc_cnt_old )/1e6 );
434494
fflush( stdout );
435495
total_off_old = total_off;
@@ -439,6 +499,8 @@ backtest_cmd_fn( args_t * args FD_PARAM_UNUSED,
439499
snapdc_wait_old = snapdc_wait;
440500
snapin_backp_old = snapin_backp;
441501
snapin_wait_old = snapin_wait;
502+
snaplt_backp_old = snaplt_backp;
503+
snaplt_wait_old = snaplt_wait;
442504
acc_cnt_old = acc_cnt;
443505

444506
next+=1000L*1000L*1000L;

‎src/app/firedancer-dev/commands/snapshot_load.c

Lines changed: 68 additions & 6 deletions
Original file line numberDiff line numberDiff line change
@@ -6,6 +6,7 @@
66
#include "../../../disco/metrics/fd_metrics.h"
77
#include "../../../disco/topo/fd_topob.h"
88
#include "../../../util/tile/fd_tile_private.h"
9+
#include "../../../discof/restore/utils/fd_ssctrl.h"
910
#include "../../../discof/restore/utils/fd_ssmsg.h"
1011

1112
#include <sys/resource.h>
@@ -25,7 +26,9 @@ snapshot_load_topo( config_t * config,
2526
args_t const * args ) {
2627
fd_topo_t * topo = &config->topo;
2728
fd_topob_new( &config->topo, config->name );
28-
topo->max_page_size = fd_cstr_to_shmem_page_sz( config->hugetlbfs.max_page_size );
29+
topo->max_page_size = fd_cstr_to_shmem_page_sz( config->hugetlbfs.max_page_size );
30+
ulong snaplt_tile_cnt = config->firedancer.layout.snaplt_tile_count;
31+
int snaplt_enabled = !config->development.snapshots.disable_lthash_verification;
2932

3033
fd_topob_wksp( topo, "funk" );
3134
fd_topo_obj_t * funk_obj = setup_topo_funk( topo, "funk",
@@ -37,7 +40,7 @@ snapshot_load_topo( config_t * config,
3740
static ushort tile_to_cpu[ FD_TILE_MAX ] = {0};
3841
if( args->snapshot_load.tile_cpus[0] ) {
3942
ulong cpu_cnt = fd_tile_private_cpus_parse( args->snapshot_load.tile_cpus, tile_to_cpu );
40-
if( FD_UNLIKELY( cpu_cnt<4UL ) ) FD_LOG_ERR(( "--tile-cpus specifies %lu CPUs, but need at least 4", cpu_cnt ));
43+
if( FD_UNLIKELY( cpu_cnt<4UL + (snaplt_enabled?snaplt_tile_cnt:0) ) ) FD_LOG_ERR(( "--tile-cpus specifies %lu CPUs, but need at least %lu", cpu_cnt, 4UL + (snaplt_enabled?snaplt_tile_cnt:0) ));
4144
}
4245

4346
/* metrics tile *****************************************************/
@@ -77,6 +80,23 @@ snapshot_load_topo( config_t * config,
7780
fd_topo_tile_t * snapin_tile = fd_topob_tile( topo, "snapin", "snapin", "snapin", tile_to_cpu[3], 0, 0 );
7881
snapin_tile->allow_shutdown = 1;
7982

83+
if( FD_LIKELY( snaplt_enabled ) ) {
84+
fd_topob_wksp( topo, "snaplt" );
85+
fd_topob_wksp( topo, "snapin_lt" );
86+
fd_topob_wksp( topo, "snaplt_out" );
87+
88+
#define FOR(cnt) for( ulong i=0UL; i<cnt; i++ )
89+
for( ulong i=0UL; i<snaplt_tile_cnt; i++ ) {
90+
fd_topo_tile_t * snaplt_tile = fd_topob_tile( topo, "snaplt", "snaplt", "metric_in", tile_to_cpu[4 + i], 0, 0 );
91+
snaplt_tile->allow_shutdown = 1;
92+
}
93+
}
94+
95+
if( FD_LIKELY( snaplt_enabled ) ) {
96+
fd_topob_link( topo, "snapin_lt", "snapin_lt", 128UL, sizeof(fd_snapshot_existing_account_t), 1UL );
97+
FOR(snaplt_tile_cnt) fd_topob_link( topo, "snaplt_out", "snaplt_out", 128UL, 2048UL, 1UL );
98+
}
99+
80100
/* uncompressed stream -> snapin tile */
81101
fd_topob_tile_in ( topo, "snapin", 0UL, "metric_in", "snap_stream", 0UL, FD_TOPOB_RELIABLE, FD_TOPOB_POLLED );
82102

@@ -100,6 +120,13 @@ snapshot_load_topo( config_t * config,
100120
fd_topob_tile_in( topo, "snaprd", 0UL, "metric_in", "snapin_rd", 0UL, FD_TOPOB_RELIABLE, FD_TOPOB_POLLED );
101121
fd_topob_tile_out( topo, "snapin", 0UL, "snapin_rd", 0UL );
102122

123+
if( FD_LIKELY( snaplt_enabled ) ) {
124+
fd_topob_tile_out( topo, "snapin", 0UL, "snapin_lt", 0UL );
125+
FOR(snaplt_tile_cnt) fd_topob_tile_in( topo, "snapin", 0UL, "metric_in", "snaplt_out", i, FD_TOPOB_RELIABLE, FD_TOPOB_POLLED );
126+
FOR(snaplt_tile_cnt) fd_topob_tile_in( topo, "snaplt", i, "metric_in", "snapin_lt", 0UL, FD_TOPOB_RELIABLE, FD_TOPOB_POLLED );
127+
FOR(snaplt_tile_cnt) fd_topob_tile_out( topo, "snaplt", i, "snaplt_out", i );
128+
}
129+
103130
for( ulong i=0UL; i<topo->tile_cnt; i++ ) {
104131
fd_topo_tile_t * tile = &topo->tiles[ i ];
105132
if( !fd_topo_configure_tile( tile, config ) ) {
@@ -158,6 +185,16 @@ snapshot_load_cmd_fn( args_t * args,
158185
fd_topo_tile_t * snapdc_tile = &topo->tiles[ fd_topo_find_tile( topo, "snapdc", 0UL ) ];
159186
fd_topo_tile_t * snapin_tile = &topo->tiles[ fd_topo_find_tile( topo, "snapin", 0UL ) ];
160187

188+
ulong volatile * snaplt_metrics[ FD_MAX_SNAPLT_TILES ];
189+
ulong snaplt_tile_cnt = fd_topo_tile_name_cnt( topo, "snaplt" );
190+
191+
for( ulong i=0UL; i<snaplt_tile_cnt; i++ ) {
192+
ulong snaplt_tile_idx = fd_topo_find_tile( topo, "snaplt", i );
193+
FD_TEST( snaplt_tile_idx!=ULONG_MAX );
194+
fd_topo_tile_t * snaplt_tile = &topo->tiles[ snaplt_tile_idx ];
195+
snaplt_metrics[ i ] = fd_metrics_tile( snaplt_tile->metrics );
196+
}
197+
161198
ulong volatile * const snaprd_metrics = fd_metrics_tile( snaprd_tile->metrics );
162199
ulong volatile * const snapdc_metrics = fd_metrics_tile( snapdc_tile->metrics );
163200
ulong volatile * const snapin_metrics = fd_metrics_tile( snapin_tile->metrics );
@@ -169,6 +206,8 @@ snapshot_load_cmd_fn( args_t * args,
169206
ulong snapdc_wait_old = 0UL;
170207
ulong snapin_backp_old = 0UL;
171208
ulong snapin_wait_old = 0UL;
209+
ulong snaplt_backp_old = 0UL;
210+
ulong snaplt_wait_old = 0UL;
172211
ulong acc_cnt_old = 0UL;
173212
sleep( 1 );
174213
puts( "" );
@@ -178,14 +217,23 @@ snapshot_load_cmd_fn( args_t * args,
178217
puts( "- stall: Waiting on upstream tile" );
179218
puts( "- acc: Number of accounts" );
180219
puts( "" );
181-
puts( "-------------backp=(snaprd,snapdc,snapin) busy=(snaprd,snapdc,snapin)---------------" );
220+
puts( "-------------backp=(snaprd,snapdc,snapin,snaplt) busy=(snaprd,snapdc,snapin,snaplt)---------------" );
182221
long next = start+1000L*1000L*1000L;
183222
for(;;) {
184223
ulong snaprd_status = FD_VOLATILE_CONST( snaprd_metrics[ MIDX( GAUGE, TILE, STATUS ) ] );
185224
ulong snapdc_status = FD_VOLATILE_CONST( snapdc_metrics[ MIDX( GAUGE, TILE, STATUS ) ] );
186225
ulong snapin_status = FD_VOLATILE_CONST( snapin_metrics[ MIDX( GAUGE, TILE, STATUS ) ] );
226+
ulong snaplt_status = ULONG_MAX;
227+
228+
ulong snaplt_status_sum = 0UL;
229+
for( ulong i=0UL; i<snaplt_tile_cnt; i++ ) {
230+
ulong snaplt_status = FD_VOLATILE_CONST( snaplt_metrics[ i ][ MIDX( GAUGE, TILE, STATUS ) ] );
231+
snaplt_status_sum += snaplt_status;
232+
}
233+
if( FD_UNLIKELY( snaplt_status_sum==2UL*snaplt_tile_cnt ) ) snaplt_status = 2UL;
234+
else snaplt_status = snaplt_tile_cnt>0UL ? 1UL : 2UL;
187235

188-
if( FD_UNLIKELY( snaprd_status==2UL && snapdc_status==2UL && snapin_status == 2UL ) ) break;
236+
if( FD_UNLIKELY( snaprd_status==2UL && snapdc_status==2UL && snapin_status == 2UL && snaplt_status==2UL ) ) break;
189237

190238
long cur = fd_log_wallclock();
191239
if( FD_UNLIKELY( cur<next ) ) {
@@ -205,16 +253,28 @@ snapshot_load_cmd_fn( args_t * args,
205253
ulong snapin_backp = snapin_metrics[ MIDX( COUNTER, TILE, REGIME_DURATION_NANOS_BACKPRESSURE_PREFRAG ) ];
206254
ulong snapin_wait = snapin_metrics[ MIDX( COUNTER, TILE, REGIME_DURATION_NANOS_CAUGHT_UP_PREFRAG ) ] +
207255
snapin_metrics[ MIDX( COUNTER, TILE, REGIME_DURATION_NANOS_CAUGHT_UP_POSTFRAG ) ] + snapin_backp;
256+
ulong snaplt_backp = 0UL;
257+
ulong snaplt_wait = 0UL;
258+
259+
for( ulong i=0UL; i<snaplt_tile_cnt; i++ ) {
260+
snaplt_backp += snaplt_metrics[ i ][ MIDX( COUNTER, TILE, REGIME_DURATION_NANOS_BACKPRESSURE_PREFRAG ) ];
261+
}
262+
for( ulong i=0UL; i<snaplt_tile_cnt; i++ ) {
263+
snaplt_wait += snaplt_metrics[ i ][ MIDX( COUNTER, TILE, REGIME_DURATION_NANOS_CAUGHT_UP_PREFRAG ) ] +
264+
snaplt_metrics[ i ][ MIDX( COUNTER, TILE, REGIME_DURATION_NANOS_CAUGHT_UP_POSTFRAG ) ] + snaplt_backp;
265+
}
208266

209-
ulong acc_cnt = snapin_metrics[ MIDX( GAUGE, SNAPIN, ACCOUNTS_INSERTED ) ];
210-
printf( "bw=%4.0f MB/s backp=(%3.0f%%,%3.0f%%,%3.0f%%) busy=(%3.0f%%,%3.0f%%,%3.0f%%) acc=%3.1f M/s\n",
267+
ulong acc_cnt = snapin_metrics[ MIDX( GAUGE, SNAPIN, ACCOUNTS_INSERTED ) ];
268+
printf( "bw=%4.0f MB/s backp=(%3.0f%%,%3.0f%%,%3.0f%%,%3.0f%%) busy=(%3.0f%%,%3.0f%%,%3.0f%%,%3.0f%%) acc=%3.1f M/s\n",
211269
(double)( total_off-total_off_old )/1e6,
212270
( (double)( snaprd_backp-snaprd_backp_old )*ns_per_tick )/1e7,
213271
( (double)( snapdc_backp-snapdc_backp_old )*ns_per_tick )/1e7,
214272
( (double)( snapin_backp-snapin_backp_old )*ns_per_tick )/1e7,
273+
( (double)( snaplt_backp-snaplt_backp_old )*ns_per_tick )/1e7/(double)snaplt_tile_cnt,
215274
100-( ( (double)( snaprd_wait-snaprd_wait_old )*ns_per_tick )/1e7 ),
216275
100-( ( (double)( snapdc_wait-snapdc_wait_old )*ns_per_tick )/1e7 ),
217276
100-( ( (double)( snapin_wait-snapin_wait_old )*ns_per_tick )/1e7 ),
277+
100-( ( (double)( snaplt_wait-snaplt_wait_old )*ns_per_tick )/1e7/(double)snaplt_tile_cnt ),
218278
(double)( acc_cnt-acc_cnt_old )/1e6 );
219279
fflush( stdout );
220280
total_off_old = total_off;
@@ -224,6 +284,8 @@ snapshot_load_cmd_fn( args_t * args,
224284
snapdc_wait_old = snapdc_wait;
225285
snapin_backp_old = snapin_backp;
226286
snapin_wait_old = snapin_wait;
287+
snaplt_backp_old = snaplt_backp;
288+
snaplt_wait_old = snaplt_wait;
227289
acc_cnt_old = acc_cnt;
228290

229291
next+=1000L*1000L*1000L;

‎src/app/firedancer-dev/main.c

Lines changed: 2 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -106,6 +106,7 @@ extern fd_topo_run_tile_t fd_tile_shredcap;
106106
extern fd_topo_run_tile_t fd_tile_snaprd;
107107
extern fd_topo_run_tile_t fd_tile_snapdc;
108108
extern fd_topo_run_tile_t fd_tile_snapin;
109+
extern fd_topo_run_tile_t fd_tile_snaplt;
109110

110111
fd_topo_run_tile_t * TILES[] = {
111112
&fd_tile_net,
@@ -150,6 +151,7 @@ fd_topo_run_tile_t * TILES[] = {
150151
&fd_tile_snaprd,
151152
&fd_tile_snapdc,
152153
&fd_tile_snapin,
154+
&fd_tile_snaplt,
153155
&fd_tile_ipecho,
154156
NULL,
155157
};

0 commit comments

Comments
 (0)