Skip to content

firedancer: add tile memory coredump dumping option [wip] #7368

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Open
wants to merge 2 commits into
base: main
Choose a base branch
from
Open

firedancer: add tile memory coredump dumping option [wip] #7368

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
2 commits
Select commit Hold shift + click to select a range
0740281
firedancer: add tile memory coredump dumping option
ibhatt-jumptrading Nov 20, 2025
3aa976d
wip
ibhatt-jumptrading Nov 20, 2025
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
8 changes: 8 additions & 0 deletions src/app/fdctl/config/default.toml
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -1494,6 +1494,14 @@ dynamic_port_range = "8900-9000"
# enabled during routine running of the validator.
core_dump = false

# By default, if core_dump is set to true, the workspace memory is
# not included in the dump. This is done via a call to
# madvise( shmem, sz, MADV_DONTDUMP ) which effectively sets it so
# the shared memory region that corresponds to the wksp's memory is
# not included in the dump. This flag is currently a no-op in the
# Frankedancer validator.
dump_wksp_mem = false

# Firedancer currently hosts a Agave client as a child process
# when it starts up, to provide functionality that has not yet been
# implemented. For development sometimes it is desirable to not
Expand Down
90 changes: 45 additions & 45 deletions src/app/fdctl/topology.c
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -47,40 +47,40 @@ fd_topo_initialize( config_t * config ) {
topo->max_page_size = fd_cstr_to_shmem_page_sz( config->hugetlbfs.max_page_size );
topo->gigantic_page_threshold = config->hugetlbfs.gigantic_page_threshold_mib << 20;

/* topo, name */
fd_topob_wksp( topo, "metric_in" );
fd_topob_wksp( topo, "net_quic" );
fd_topob_wksp( topo, "net_shred" );
fd_topob_wksp( topo, "quic_verify" );
fd_topob_wksp( topo, "verify_dedup" );
fd_topob_wksp( topo, "dedup_resolv" );
fd_topob_wksp( topo, "resolv_pack" );
fd_topob_wksp( topo, "pack_bank" );
fd_topob_wksp( topo, "pack_poh" );
fd_topob_wksp( topo, "bank_pack" );
fd_topob_wksp( topo, "bank_poh" );
fd_topob_wksp( topo, "bank_busy" );
fd_topob_wksp( topo, "poh_shred" );
fd_topob_wksp( topo, "gossip_dedup" );
fd_topob_wksp( topo, "shred_store" );
fd_topob_wksp( topo, "stake_out" );
fd_topob_wksp( topo, "executed_txn" );

fd_topob_wksp( topo, "shred_sign" );
fd_topob_wksp( topo, "sign_shred" );

fd_topob_wksp( topo, "quic" );
fd_topob_wksp( topo, "verify" );
fd_topob_wksp( topo, "dedup" );
fd_topob_wksp( topo, "resolv" );
fd_topob_wksp( topo, "pack" );
fd_topob_wksp( topo, "bank" );
fd_topob_wksp( topo, "poh" );
fd_topob_wksp( topo, "shred" );
fd_topob_wksp( topo, "store" );
fd_topob_wksp( topo, "sign" );
fd_topob_wksp( topo, "metric" );
fd_topob_wksp( topo, "cswtch" );
/* topo, name, dumpable */
fd_topob_wksp( topo, "metric_in", 0 );
fd_topob_wksp( topo, "net_quic", 0 );
fd_topob_wksp( topo, "net_shred", 0 );
fd_topob_wksp( topo, "quic_verify", 0 );
fd_topob_wksp( topo, "verify_dedup", 0 );
fd_topob_wksp( topo, "dedup_resolv", 0 );
fd_topob_wksp( topo, "resolv_pack", 0 );
fd_topob_wksp( topo, "pack_bank", 0 );
fd_topob_wksp( topo, "pack_poh", 0 );
fd_topob_wksp( topo, "bank_pack", 0 );
fd_topob_wksp( topo, "bank_poh", 0 );
fd_topob_wksp( topo, "bank_busy", 0 );
fd_topob_wksp( topo, "poh_shred", 0 );
fd_topob_wksp( topo, "gossip_dedup", 0 );
fd_topob_wksp( topo, "shred_store", 0 );
fd_topob_wksp( topo, "stake_out", 0 );
fd_topob_wksp( topo, "executed_txn", 0 );

fd_topob_wksp( topo, "shred_sign", 0 );
fd_topob_wksp( topo, "sign_shred", 0 );

fd_topob_wksp( topo, "quic", 0 );
fd_topob_wksp( topo, "verify", 0 );
fd_topob_wksp( topo, "dedup", 0 );
fd_topob_wksp( topo, "resolv", 0 );
fd_topob_wksp( topo, "pack", 0 );
fd_topob_wksp( topo, "bank", 0 );
fd_topob_wksp( topo, "poh", 0 );
fd_topob_wksp( topo, "shred", 0 );
fd_topob_wksp( topo, "store", 0 );
fd_topob_wksp( topo, "sign", 0 );
fd_topob_wksp( topo, "metric", 0 );
fd_topob_wksp( topo, "cswtch", 0 );

#define FOR(cnt) for( ulong i=0UL; i<cnt; i++ )

Expand Down Expand Up @@ -238,9 +238,9 @@ fd_topo_initialize( config_t * config ) {
/* For now the only plugin consumer is the GUI */
int plugins_enabled = config->tiles.gui.enabled;
if( FD_LIKELY( plugins_enabled ) ) {
fd_topob_wksp( topo, "plugin_in" );
fd_topob_wksp( topo, "plugin_out" );
fd_topob_wksp( topo, "plugin" );
fd_topob_wksp( topo, "plugin_in", 0 );
fd_topob_wksp( topo, "plugin_out", 0 );
fd_topob_wksp( topo, "plugin", 0 );

/**/ fd_topob_link( topo, "plugin_out", "plugin_out", 128UL, 8UL+40200UL*(58UL+12UL*34UL), 1UL );
/**/ fd_topob_link( topo, "replay_plugi", "plugin_in", 128UL, 4098*8UL, 1UL );
Expand Down Expand Up @@ -270,7 +270,7 @@ fd_topo_initialize( config_t * config ) {
}

if( FD_LIKELY( config->tiles.gui.enabled ) ) {
fd_topob_wksp( topo, "gui" );
fd_topob_wksp( topo, "gui", 0 );
/**/ fd_topob_tile( topo, "gui", "gui", "metric_in", tile_to_cpu[ topo->tile_cnt ], 0, 1 );
/**/ fd_topob_tile_in( topo, "gui", 0UL, "metric_in", "plugin_out", 0UL, FD_TOPOB_RELIABLE, FD_TOPOB_POLLED );
/**/ fd_topob_tile_in( topo, "gui", 0UL, "metric_in", "poh_pack", 0UL, FD_TOPOB_RELIABLE, FD_TOPOB_POLLED );
Expand All @@ -280,12 +280,12 @@ fd_topo_initialize( config_t * config ) {
}

if( FD_UNLIKELY( config->tiles.bundle.enabled ) ) {
fd_topob_wksp( topo, "bundle_verif" );
fd_topob_wksp( topo, "bundle_sign" );
fd_topob_wksp( topo, "sign_bundle" );
fd_topob_wksp( topo, "pack_sign" );
fd_topob_wksp( topo, "sign_pack" );
fd_topob_wksp( topo, "bundle" );
fd_topob_wksp( topo, "bundle_verif", 0 );
fd_topob_wksp( topo, "bundle_sign", 0 );
fd_topob_wksp( topo, "sign_bundle", 0 );
fd_topob_wksp( topo, "pack_sign", 0 );
fd_topob_wksp( topo, "sign_pack", 0 );
fd_topob_wksp( topo, "bundle", 0 );

/**/ fd_topob_link( topo, "bundle_verif", "bundle_verif", config->tiles.verify.receive_buffer_size, FD_TPU_PARSED_MTU, 1UL );
/**/ fd_topob_link( topo, "bundle_sign", "bundle_sign", 65536UL, 9UL, 1UL );
Expand All @@ -309,7 +309,7 @@ fd_topo_initialize( config_t * config ) {
/**/ fd_topob_tile_out( topo, "sign", 0UL, "sign_pack", 0UL );

if( plugins_enabled ) {
fd_topob_wksp( topo, "bundle_plugi" );
fd_topob_wksp( topo, "bundle_plugi", 0 );
/* bundle_plugi must be kind of deep, to prevent exhausting shared
flow control credits when publishing many packets at once. */
fd_topob_link( topo, "bundle_plugi", "bundle_plugi", 65536UL, sizeof(fd_plugin_msg_block_engine_update_t), 1UL );
Expand Down
68 changes: 34 additions & 34 deletions src/app/firedancer-dev/commands/backtest.c
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -60,30 +60,30 @@ backtest_topo( config_t * config ) {

ulong cpu_idx = 0;

fd_topob_wksp( topo, "metric_in" );
fd_topob_wksp( topo, "metric_in", 0 );

/**********************************************************************/
/* Add the backtest tile to topo */
/**********************************************************************/
fd_topob_wksp( topo, "backt" );
fd_topob_wksp( topo, "backt", 0 );
fd_topo_tile_t * backt_tile = fd_topob_tile( topo, "backt", "backt", "metric_in", cpu_idx++, 0, 0 );

/**********************************************************************/
/* Add the replay tile to topo */
/**********************************************************************/
fd_topob_wksp( topo, "replay" );
fd_topob_wksp( topo, "replay", 0 );
fd_topo_tile_t * replay_tile = fd_topob_tile( topo, "replay", "replay", "metric_in", cpu_idx++, 0, 0 );

/* specified by [tiles.replay] */

fd_topob_wksp( topo, "funk" );
fd_topob_wksp( topo, "funk", 0 );
fd_topo_obj_t * funk_obj = setup_topo_funk( topo, "funk",
config->firedancer.funk.max_account_records,
config->firedancer.funk.max_database_transactions,
config->firedancer.funk.heap_size_gib );
fd_topob_tile_uses( topo, replay_tile, funk_obj, FD_SHMEM_JOIN_MODE_READ_WRITE );

fd_topob_wksp( topo, "progcache" );
fd_topob_wksp( topo, "progcache", 0 );
fd_topo_obj_t * progcache_obj = setup_topo_progcache( topo, "progcache",
fd_progcache_est_rec_max( config->firedancer.runtime.program_cache.heap_size_mib<<20,
config->firedancer.runtime.program_cache.mean_cache_entry_size ),
Expand All @@ -98,7 +98,7 @@ backtest_topo( config_t * config ) {
/**********************************************************************/
/* Add the executor tiles to topo */
/**********************************************************************/
fd_topob_wksp( topo, "exec" );
fd_topob_wksp( topo, "exec", 0 );
#define FOR(cnt) for( ulong i=0UL; i<cnt; i++ )
FOR(exec_tile_cnt) fd_topob_tile( topo, "exec", "exec", "metric_in", cpu_idx++, 0, 0 );

Expand All @@ -107,14 +107,14 @@ backtest_topo( config_t * config ) {
/**********************************************************************/
fd_topo_tile_t * snapin_tile = NULL;
if( FD_UNLIKELY( !disable_snap_loader ) ) {
fd_topob_wksp( topo, "snapct" );
fd_topob_wksp( topo, "snapld" );
fd_topob_wksp( topo, "snapdc" );
fd_topob_wksp( topo, "snapin" );
fd_topob_wksp( topo, "snapct", 0 );
fd_topob_wksp( topo, "snapld", 0 );
fd_topob_wksp( topo, "snapdc", 0 );
fd_topob_wksp( topo, "snapin", 0 );

if( FD_LIKELY( !snapshot_lthash_disabled ) ) {
fd_topob_wksp( topo, "snapla" );
fd_topob_wksp( topo, "snapls" );
fd_topob_wksp( topo, "snapla", 0 );
fd_topob_wksp( topo, "snapls", 0 );
}

fd_topo_tile_t * snapct_tile = fd_topob_tile( topo, "snapct", "snapct", "metric_in", cpu_idx++, 0, 0 );
Expand All @@ -130,7 +130,7 @@ backtest_topo( config_t * config ) {
snapdc_tile->allow_shutdown = 1;
snapin_tile->allow_shutdown = 1;
} else {
fd_topob_wksp( topo, "genesi" );
fd_topob_wksp( topo, "genesi", 0 );
fd_topob_tile( topo, "genesi", "genesi", "metric_in", cpu_idx++, 0, 0 )->allow_shutdown = 1;
}

Expand All @@ -143,7 +143,7 @@ backtest_topo( config_t * config ) {
which is provided by the backtest tile, which reads in the entry
batches from the CLI-specified source (eg. RocksDB). */

fd_topob_wksp( topo, "shred_out" );
fd_topob_wksp( topo, "shred_out", 0 );
fd_topob_link( topo, "shred_out", "shred_out", 65536UL, FD_SHRED_OUT_MTU, 1UL );
fd_topob_tile_in( topo, "replay", 0UL, "metric_in", "shred_out", 0UL, FD_TOPOB_RELIABLE, FD_TOPOB_POLLED );
fd_topob_tile_out( topo, "backt", 0UL, "shred_out", 0UL );
Expand All @@ -152,21 +152,21 @@ backtest_topo( config_t * config ) {
/* Setup snapshot links in topo */
/**********************************************************************/
if( FD_LIKELY( !disable_snap_loader ) ) {
fd_topob_wksp( topo, "snapct_ld" );
fd_topob_wksp( topo, "snapld_dc" );
fd_topob_wksp( topo, "snapdc_in" );
fd_topob_wksp( topo, "snapct_ld", 0 );
fd_topob_wksp( topo, "snapld_dc", 0 );
fd_topob_wksp( topo, "snapdc_in", 0 );
if( FD_UNLIKELY( snapshot_lthash_disabled ) ) {
fd_topob_wksp( topo, "snapin_ct" );
fd_topob_wksp( topo, "snapin_ct", 0 );
} else {
fd_topob_wksp( topo, "snapls_ct" );
fd_topob_wksp( topo, "snapls_ct", 0 );
}

fd_topob_wksp( topo, "snapin_manif" );
fd_topob_wksp( topo, "snapct_repr" );
fd_topob_wksp( topo, "snapin_manif", 0 );
fd_topob_wksp( topo, "snapct_repr", 0 );

if( FD_LIKELY( !snapshot_lthash_disabled ) ) {
fd_topob_wksp( topo, "snapla_ls" );
fd_topob_wksp( topo, "snapin_ls" );
fd_topob_wksp( topo, "snapla_ls", 0 );
fd_topob_wksp( topo, "snapin_ls", 0 );
}

fd_topob_link( topo, "snapct_ld", "snapct_ld", 128UL, sizeof(fd_ssctrl_init_t), 1UL );
Expand Down Expand Up @@ -214,7 +214,7 @@ backtest_topo( config_t * config ) {
/**/ fd_topob_tile_out( topo, "snapls", 0UL, "snapls_ct", 0UL );
}
} else {
fd_topob_wksp( topo, "genesi_out" );
fd_topob_wksp( topo, "genesi_out", 0 );
fd_topob_link( topo, "genesi_out", "genesi_out", 2UL, 10UL*1024UL*1024UL+32UL+sizeof(fd_lthash_value_t), 1UL );
fd_topob_tile_out( topo, "genesi", 0UL, "genesi_out", 0UL );
fd_topob_tile_in ( topo, "replay", 0UL, "metric_in", "genesi_out", 0UL, FD_TOPOB_RELIABLE, FD_TOPOB_POLLED );
Expand All @@ -231,16 +231,16 @@ backtest_topo( config_t * config ) {
This allows the replay tile to advance its watermark, and publish
various data structures. This is an oversimplified barebones mock
of the tower tile. */
fd_topob_wksp( topo, "tower_out" );
fd_topob_wksp( topo, "tower_out", 0 );
fd_topob_link( topo, "tower_out", "tower_out", 1024UL, sizeof(fd_tower_slot_done_t), 1UL );
fd_topob_tile_in( topo, "replay", 0UL, "metric_in", "tower_out", 0UL, FD_TOPOB_RELIABLE, FD_TOPOB_POLLED );
fd_topob_tile_out( topo, "backt", 0UL, "tower_out", 0UL );

/**********************************************************************/
/* Setup replay->stake/send/poh links in topo w/o consumers */
/**********************************************************************/
fd_topob_wksp( topo, "replay_stake" );
fd_topob_wksp( topo, "replay_poh" );
fd_topob_wksp( topo, "replay_stake", 0 );
fd_topob_wksp( topo, "replay_poh", 0 );

fd_topob_link( topo, "replay_stake", "replay_stake", 128UL, 40UL + 40200UL * 40UL, 1UL );
ulong bank_tile_cnt = config->layout.bank_tile_count;
Expand All @@ -256,7 +256,7 @@ backtest_topo( config_t * config ) {
/* Setup replay->backtest link (replay_notif) in topo */
/**********************************************************************/

fd_topob_wksp( topo, "replay_out" );
fd_topob_wksp( topo, "replay_out", 0 );
fd_topob_link( topo, "replay_out", "replay_out", 8192UL, sizeof( fd_replay_message_t ), 1UL );
fd_topob_tile_out( topo, "replay", 0UL, "replay_out", 0UL );
fd_topob_tile_in ( topo, "backt", 0UL, "metric_in", "replay_out", 0UL, FD_TOPOB_RELIABLE, FD_TOPOB_POLLED );
Expand All @@ -269,7 +269,7 @@ backtest_topo( config_t * config ) {
/**********************************************************************/
/* Setup replay->exec link in topo */
/**********************************************************************/
fd_topob_wksp( topo, "replay_exec" );
fd_topob_wksp( topo, "replay_exec", 0 );
fd_topob_link( topo, "replay_exec", "replay_exec", 16384UL, 2240UL, 1UL );
fd_topob_tile_out( topo, "replay", 0UL, "replay_exec", 0UL );
for( ulong i=0UL; i<exec_tile_cnt; i++ ) {
Expand All @@ -281,7 +281,7 @@ backtest_topo( config_t * config ) {
so that they are serialized, and to notify replay tile that a txn
has been finalized by the exec tile. */
/**********************************************************************/
fd_topob_wksp( topo, "exec_replay" );
fd_topob_wksp( topo, "exec_replay", 0 );

/* If solcap is enabled, we need to overload this link to also send
solcap account updates to the replay tile. We can't use a separate
Expand All @@ -300,22 +300,22 @@ backtest_topo( config_t * config ) {
/* Setup the shared objs used by replay and exec tiles */
/**********************************************************************/

fd_topob_wksp( topo, "store" );
fd_topob_wksp( topo, "store", 0 );
fd_topo_obj_t * store_obj = setup_topo_store( topo, "store", config->firedancer.store.max_completed_shred_sets, 1 );
fd_topob_tile_uses( topo, backt_tile, store_obj, FD_SHMEM_JOIN_MODE_READ_WRITE );
fd_topob_tile_uses( topo, replay_tile, store_obj, FD_SHMEM_JOIN_MODE_READ_WRITE );
FD_TEST( fd_pod_insertf_ulong( topo->props, store_obj->id, "store" ) );

/* banks_obj shared by replay and exec tiles */
fd_topob_wksp( topo, "banks" );
fd_topob_wksp( topo, "banks", 0 );
fd_topo_obj_t * banks_obj = setup_topo_banks( topo, "banks", config->firedancer.runtime.max_live_slots, config->firedancer.runtime.max_fork_width, 0 );
fd_topob_tile_uses( topo, replay_tile, banks_obj, FD_SHMEM_JOIN_MODE_READ_WRITE );
FOR(exec_tile_cnt) fd_topob_tile_uses( topo, &topo->tiles[ fd_topo_find_tile( topo, "exec", i ) ], banks_obj, FD_SHMEM_JOIN_MODE_READ_WRITE );
FD_TEST( fd_pod_insertf_ulong( topo->props, banks_obj->id, "banks" ) );

/* txncache_obj, busy_obj and poh_slot_obj only by replay tile */
fd_topob_wksp( topo, "txncache" );
fd_topob_wksp( topo, "bank_busy" );
fd_topob_wksp( topo, "txncache", 0 );
fd_topob_wksp( topo, "bank_busy", 0 );
fd_topo_obj_t * txncache_obj = setup_topo_txncache( topo, "txncache",
config->firedancer.runtime.max_live_slots,
fd_ulong_pow2_up( FD_PACK_MAX_TXNCACHE_TXN_PER_SLOT ) );
Expand Down
6 changes: 3 additions & 3 deletions src/app/firedancer-dev/commands/core_subtopo.h
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -27,8 +27,8 @@ fd_core_subtopo( config_t * config, ulong tile_to_cpu[ FD_TILE_MAX ] ) {
ulong net_tile_cnt = config->layout.net_tile_count;
ulong sign_tile_cnt = config->firedancer.layout.sign_tile_count;

fd_topob_wksp( topo, "metric" );
fd_topob_wksp( topo, "metric_in" );
fd_topob_wksp( topo, "metric", 0 );
fd_topob_wksp( topo, "metric_in", 0 );
fd_topo_tile_t * metric_tile = fd_topob_tile( topo, "metric", "metric", "metric_in", ULONG_MAX, 0, 0 );
if( FD_UNLIKELY( !fd_cstr_to_ip4_addr( config->tiles.metric.prometheus_listen_address, &metric_tile->metric.prometheus_listen_addr ) ) )
FD_LOG_ERR(( "failed to parse prometheus listen address `%s`", config->tiles.metric.prometheus_listen_address ));
Expand All @@ -43,7 +43,7 @@ fd_core_subtopo( config_t * config, ulong tile_to_cpu[ FD_TILE_MAX ] ) {
net_tile->net.gossip_listen_port = config->gossip.port;
}

fd_topob_wksp( topo, "sign" );
fd_topob_wksp( topo, "sign", 0 );
for( ulong i=0UL; i<sign_tile_cnt; i++ ) {
fd_topo_tile_t * sign_tile = fd_topob_tile( topo, "sign", "sign", "metric_in", tile_to_cpu[ topo->tile_cnt ], 0, 1 );
strncpy( sign_tile->sign.identity_key_path, config->paths.identity_key, sizeof(sign_tile->sign.identity_key_path) );
Expand Down
Loading
Loading