firedancer-io · ripatel-fd · Jun 6, 2025 · Jun 24, 2025 · Jun 21, 2025 · Jun 21, 2025
diff --git a/book/api/metrics-generated.md b/book/api/metrics-generated.md
@@ -1204,3 +1204,16 @@
 | <span class="metrics-name">tower_&#8203;hard_&#8203;forks_&#8203;active</span> | gauge | Currently active hard forks |
 
 </div>
+
+## Ibeth Tile
+
+<div class="metrics">
+
+| Metric | Type | Description |
+|--------|------|-------------|
+| <span class="metrics-name">ibeth_&#8203;rx_&#8203;pkt_&#8203;cnt</span> | counter | Packet receive count. |
+| <span class="metrics-name">ibeth_&#8203;rx_&#8203;bytes_&#8203;total</span> | counter | Total number of bytes received (including Ethernet header). |
+| <span class="metrics-name">ibeth_&#8203;tx_&#8203;pkt_&#8203;cnt</span> | counter | Number of packet transmit jobs marked as completed by the kernel. |
+| <span class="metrics-name">ibeth_&#8203;tx_&#8203;bytes_&#8203;total</span> | counter | Total number of bytes transmitted (including Ethernet header). |
+
+</div>
@@ -0,0 +1,3 @@
+FD_HAS_IBVERBS:=1
+CPPFLAGS+=-DFD_HAS_IBVERBS=1
+LDFLAGS+=-libverbs
diff --git a/src/app/fdctl/config/default.toml b/src/app/fdctl/config/default.toml
@@ -894,8 +894,13 @@ dynamic_port_range = "8900-9000"
     #   do not support XDP, and to support experimental features that are
     #   not yet implemented in the XDP stack.
     #
+    #  "ibverbs" (experimental)
+    #   Use libibverbs (RDMA) in 'raw packet' mode for networking.
+    #   Compatible with Mellanox ConnectX 5 and newer.  Requires a
+    #   Firedancer build with 'make EXTRAS=ibverbs'.
+    #
     # Using the XDP networking stack is strongly preferred where possible,
-    # as it is faster and better tested by the development team.
+    # as it provides good performance and is battle tested.
     provider = "xdp"
 
     # Which interface to bind to for network traffic.  Currently,

diff --git a/src/app/fdctl/main.c b/src/app/fdctl/main.c
@@ -44,6 +44,7 @@ configure_stage_t * STAGES[] = {
 extern fd_topo_run_tile_t fd_tile_net;
 extern fd_topo_run_tile_t fd_tile_netlnk;
 extern fd_topo_run_tile_t fd_tile_sock;
+extern fd_topo_run_tile_t fd_tile_ibeth;
 extern fd_topo_run_tile_t fd_tile_quic;
 extern fd_topo_run_tile_t fd_tile_bundle;
 extern fd_topo_run_tile_t fd_tile_verify;
@@ -64,6 +65,9 @@ fd_topo_run_tile_t * TILES[] = {
   &fd_tile_net,
   &fd_tile_netlnk,
   &fd_tile_sock,
+# if FD_HAS_IBVERBS
+  &fd_tile_ibeth,
+# endif
   &fd_tile_quic,
   &fd_tile_bundle,
   &fd_tile_verify,

diff --git a/src/app/fddev/main.h b/src/app/fddev/main.h
@@ -54,6 +54,7 @@ configure_stage_t * STAGES[] = {
 extern fd_topo_run_tile_t fd_tile_net;
 extern fd_topo_run_tile_t fd_tile_netlnk;
 extern fd_topo_run_tile_t fd_tile_sock;
+extern fd_topo_run_tile_t fd_tile_ibeth;
 extern fd_topo_run_tile_t fd_tile_quic;
 extern fd_topo_run_tile_t fd_tile_bundle;
 extern fd_topo_run_tile_t fd_tile_verify;
@@ -79,6 +80,9 @@ fd_topo_run_tile_t * TILES[] = {
   &fd_tile_net,
   &fd_tile_netlnk,
   &fd_tile_sock,
+# if FD_HAS_IBVERBS
+  &fd_tile_ibeth,
+# endif
   &fd_tile_quic,
   &fd_tile_bundle,
   &fd_tile_verify,

diff --git a/src/app/firedancer-dev/main.c b/src/app/firedancer-dev/main.c
@@ -72,6 +72,7 @@ configure_stage_t * STAGES[] = {
 extern fd_topo_run_tile_t fd_tile_net;
 extern fd_topo_run_tile_t fd_tile_netlnk;
 extern fd_topo_run_tile_t fd_tile_sock;
+extern fd_topo_run_tile_t fd_tile_ibeth;
 extern fd_topo_run_tile_t fd_tile_quic;
 extern fd_topo_run_tile_t fd_tile_verify;
 extern fd_topo_run_tile_t fd_tile_dedup;
@@ -122,6 +123,9 @@ fd_topo_run_tile_t * TILES[] = {
   &fd_tile_net,
   &fd_tile_netlnk,
   &fd_tile_sock,
+# if FD_HAS_IBVERBS
+  &fd_tile_ibeth,
+# endif
   &fd_tile_quic,
   &fd_tile_verify,
   &fd_tile_dedup,

diff --git a/src/app/firedancer/config/default.toml b/src/app/firedancer/config/default.toml
@@ -889,6 +889,11 @@ user = ""
     #   for hosts which do not support XDP, and to support experimental
     #   features that are not yet implemented in the XDP stack.
     #
+    #  "ibverbs" (experimental)
+    #   Use libibverbs (RDMA) in 'raw packet' mode for networking.
+    #   Compatible with Mellanox ConnectX 5 and newer.  Requires a
+    #   Firedancer build with 'make EXTRAS=ibverbs'.
+    #
     # Using the XDP networking stack is strongly preferred where
     # possible, as it is faster and better tested by the development
     # team.

diff --git a/src/app/firedancer/main.c b/src/app/firedancer/main.c
@@ -57,6 +57,7 @@ configure_stage_t * STAGES[] = {
 extern fd_topo_run_tile_t fd_tile_net;
 extern fd_topo_run_tile_t fd_tile_netlnk;
 extern fd_topo_run_tile_t fd_tile_sock;
+extern fd_topo_run_tile_t fd_tile_ibeth;
 extern fd_topo_run_tile_t fd_tile_quic;
 extern fd_topo_run_tile_t fd_tile_verify;
 extern fd_topo_run_tile_t fd_tile_dedup;
@@ -96,6 +97,9 @@ fd_topo_run_tile_t * TILES[] = {
   &fd_tile_net,
   &fd_tile_netlnk,
   &fd_tile_sock,
+# if FD_HAS_IBVERBS
+  &fd_tile_ibeth,
+# endif
   &fd_tile_quic,
   &fd_tile_verify,
   &fd_tile_dedup,

diff --git a/src/app/firedancer/topology.c b/src/app/firedancer/topology.c
@@ -1044,8 +1044,11 @@ fd_topo_configure_tile( fd_topo_tile_t * tile,
       FD_LOG_ERR(( "failed to parse prometheus listen address `%s`", config->tiles.metric.prometheus_listen_address ));
     tile->metric.prometheus_listen_port = config->tiles.metric.prometheus_listen_port;
 
-  } else  if( FD_UNLIKELY( !strcmp( tile->name, "net" ) || !strcmp( tile->name, "sock" ) ) ) {
+  } else if( FD_UNLIKELY( !strcmp( tile->name, "net"   ) ||
+                          !strcmp( tile->name, "sock"  ) ||
+                          !strcmp( tile->name, "ibeth" ) ) ) {
 
+    tile->net.bind_address                   = config->net.bind_address_parsed;
     tile->net.shred_listen_port              = config->tiles.shred.shred_listen_port;
     tile->net.quic_transaction_listen_port   = config->tiles.quic.quic_transaction_listen_port;
     tile->net.legacy_transaction_listen_port = config->tiles.quic.regular_transaction_listen_port;

diff --git a/src/app/shared/commands/configure/sysctl.c b/src/app/shared/commands/configure/sysctl.c
@@ -184,7 +184,7 @@ check( config_t const * config,
     sock_params[ 1 ].value = config->net.socket.send_buffer_size;
     r = check_param_list( sock_params );
   } else {
-    FD_LOG_ERR(( "unknown net provider: %s", config->net.provider ));
+    CONFIGURE_OK();
   }
   if( r.result!=CONFIGURE_OK ) return r;
 

diff --git a/src/app/shared/fd_action.h b/src/app/shared/fd_action.h
@@ -147,6 +147,10 @@ union fdctl_args {
     ulong max_contact;
     int   compact_mode;
   } gossip;
+
+  struct {
+    ushort listen_port;
+  } pktgen;
 };
 
 typedef union fdctl_args args_t;

diff --git a/src/app/shared/fd_config.c b/src/app/shared/fd_config.c
@@ -546,8 +546,10 @@ fd_config_validate( fd_config_t const * config ) {
   } else if( 0==strcmp( config->net.provider, "socket" ) ) {
     CFG_HAS_NON_ZERO( net.socket.receive_buffer_size );
     CFG_HAS_NON_ZERO( net.socket.send_buffer_size );
+  } else if( 0==strcmp( config->net.provider, "ibverbs" ) ) {
+    /**/
   } else {
-    FD_LOG_ERR(( "invalid `net.provider`: must be \"xdp\" or \"socket\"" ));
+    FD_LOG_ERR(( "invalid `net.provider`: must be \"xdp\", \"socket\", or \"ibverbs\"" ));
   }
 
   CFG_HAS_NON_ZERO( tiles.netlink.max_routes           );

diff --git a/src/app/shared/fd_config.h b/src/app/shared/fd_config.h
@@ -173,7 +173,7 @@ struct fd_configf {
 typedef struct fd_configf fd_configf_t;
 
 struct fd_config_net {
-  char provider[ 8 ]; /* "xdp" or "socket" */
+  char provider[ 8 ]; /* "xdp", "socket", or "ibverbs" */
 
   char interface[ IF_NAMESIZE ];
   char bind_address[ 16 ];

diff --git a/src/app/shared_dev/commands/pktgen/pktgen.c b/src/app/shared_dev/commands/pktgen/pktgen.c
@@ -18,6 +18,19 @@ extern fd_topo_obj_callbacks_t * CALLBACKS[];
 fd_topo_run_tile_t
 fdctl_tile_run( fd_topo_tile_t const * tile );
 
+static char const *
+net_tile_name( char const * provider ) {
+  if( 0==strcmp( provider, "xdp" ) ) {
+    return "net";
+  } else if( 0==strcmp( provider, "socket" ) ) {
+    return "socket";
+  } else if( 0==strcmp( provider, "ibverbs" ) ) {
+    return "ibeth";
+  } else {
+    FD_LOG_ERR(( "Invalid [net.provider]: %s", provider ));
+  }
+}
+
 static void
 pktgen_topo( config_t * config ) {
   char const * affinity = config->development.pktgen.affinity;
@@ -43,7 +56,7 @@ pktgen_topo( config_t * config ) {
   }
   if( FD_LIKELY( !is_auto_affinity ) ) {
     if( FD_UNLIKELY( affinity_tile_cnt!=4UL ) )
-      FD_LOG_ERR(( "Invalid [development.pktgen.affinity]: must include exactly three CPUs" ));
+      FD_LOG_ERR(( "Invalid [development.pktgen.affinity]: must include exactly 4 CPUs" ));
   }
 
   /* Reset topology from scratch */
@@ -56,14 +69,16 @@ pktgen_topo( config_t * config ) {
   fd_topos_net_tiles( topo, config->layout.net_tile_count, &config->net, config->tiles.netlink.max_routes, config->tiles.netlink.max_peer_routes, config->tiles.netlink.max_neighbors, tile_to_cpu );
   fd_topob_tile( topo, "metric",  "metric", "metric_in", tile_to_cpu[ topo->tile_cnt ], 0, 0 );
 
+  char const * net_tile = net_tile_name( config->net.provider );
+
   fd_topob_wksp( topo, "pktgen" );
   fd_topo_tile_t * pktgen_tile = fd_topob_tile( topo, "pktgen", "pktgen", "pktgen", tile_to_cpu[ topo->tile_cnt ], 0, 0 );
   if( FD_UNLIKELY( !fd_cstr_to_ip4_addr( config->development.pktgen.fake_dst_ip, &pktgen_tile->pktgen.fake_dst_ip ) ) ) {
     FD_LOG_ERR(( "Invalid [development.pktgen.fake_dst_ip]" ));
   }
   fd_topob_link( topo, "pktgen_out", "pktgen", 2048UL, FD_NET_MTU, 1UL );
   fd_topob_tile_out( topo, "pktgen", 0UL, "pktgen_out", 0UL );
-  fd_topob_tile_in( topo, "net", 0UL, "metric_in", "pktgen_out", 0UL, FD_TOPOB_UNRELIABLE, FD_TOPOB_POLLED );
+  fd_topob_tile_in( topo, net_tile, 0UL, "metric_in", "pktgen_out", 0UL, FD_TOPOB_UNRELIABLE, FD_TOPOB_POLLED );
 
   /* Create dummy RX link */
   fd_topos_net_rx_link( topo, "net_quic", 0UL, config->net.ingress_buffer_size );
@@ -80,8 +95,7 @@ void
 pktgen_cmd_args( int *    pargc,
                  char *** pargv,
                  args_t * args ) {
-  /* FIXME add config options here */
-  (void)pargc; (void)pargv; (void)args;
+  args->pktgen.listen_port = fd_env_strip_cmdline_ushort( pargc, pargv, "--listen-port", NULL, 9000 );
 }
 
 /* Hacky: Since the pktgen runs in the same process, use globals to
@@ -91,8 +105,20 @@ extern uint fd_pktgen_active;
 /* render_status prints statistics at the top of the screen.
    Should be called at a low rate (~500ms). */
 
+union net_abstract_metrics {
+  ulong volatile const * a[4];
+  struct {
+    ulong volatile const * rx_pkt_cnt;
+    ulong volatile const * rx_bytes_total;
+    ulong volatile const * tx_pkt_cnt;
+    ulong volatile const * tx_bytes_total;
+  };
+};
+typedef union net_abstract_metrics net_abstract_metrics_t;
+
 static void
-render_status( ulong volatile const * net_metrics ) {
+render_status( ulong volatile const *         net_metrics,
+               net_abstract_metrics_t const * abstract ) {
   fputs( "\0337"      /* save cursor position */
          "\033[H"     /* move cursor to (0,0) */
          "\033[2K\n", /* create an empty line to avoid spamming look back buffer */
@@ -130,15 +156,15 @@ render_status( ulong volatile const * net_metrics ) {
     /* */ cum_tick_now += net_metrics[ MIDX( COUNTER, TILE, REGIME_DURATION_NANOS_PROCESSING_PREFRAG        ) ];
     /* */ cum_tick_now += net_metrics[ MIDX( COUNTER, TILE, REGIME_DURATION_NANOS_BACKPRESSURE_PREFRAG      ) ];
     /* */ cum_tick_now += net_metrics[ MIDX( COUNTER, TILE, REGIME_DURATION_NANOS_PROCESSING_POSTFRAG       ) ];
-    ulong rx_ok_now     = net_metrics[ MIDX( COUNTER, NET, RX_PKT_CNT           ) ];
-    ulong rx_byte_now   = net_metrics[ MIDX( COUNTER, NET, RX_BYTES_TOTAL       ) ];
+    ulong rx_ok_now     = abstract->rx_pkt_cnt[0];
+    ulong rx_byte_now   = abstract->rx_bytes_total[0];
     ulong rx_drop_now   = net_metrics[ MIDX( COUNTER, NET, RX_FILL_BLOCKED_CNT  ) ];
     /* */ rx_drop_now  += net_metrics[ MIDX( COUNTER, NET, RX_BACKPRESSURE_CNT  ) ];
     /* */ rx_drop_now  += net_metrics[ MIDX( COUNTER, NET, XDP_RX_DROPPED_OTHER ) ];
     /* */ rx_drop_now  += net_metrics[ MIDX( COUNTER, NET, XDP_RX_INVALID_DESCS ) ];
     /* */ rx_drop_now  += net_metrics[ MIDX( COUNTER, NET, XDP_RX_RING_FULL     ) ];
-    ulong tx_ok_now     = net_metrics[ MIDX( COUNTER, NET, TX_COMPLETE_CNT      ) ];
-    ulong tx_byte_now   = net_metrics[ MIDX( COUNTER, NET, TX_BYTES_TOTAL       ) ];
+    ulong tx_ok_now     = abstract->tx_pkt_cnt[0];
+    ulong tx_byte_now   = abstract->tx_bytes_total[0];
 
     ulong cum_idle_delta = cum_idle_now-cum_idle_last;
     ulong cum_tick_delta = cum_tick_now-cum_tick_last;
@@ -189,15 +215,14 @@ render_status( ulong volatile const * net_metrics ) {
 /* FIXME fixup screen on window size changes */
 
 void
-pktgen_cmd_fn( args_t *   args FD_PARAM_UNUSED,
+pktgen_cmd_fn( args_t *   args,
                config_t * config ) {
   pktgen_topo( config );
   fd_topo_t *      topo        = &config->topo;
-  fd_topo_tile_t * net_tile    = &topo->tiles[ fd_topo_find_tile( topo, "net",    0UL ) ];
+  fd_topo_tile_t * net_tile    = &topo->tiles[ fd_topo_find_tile( topo, net_tile_name( config->net.provider ), 0UL ) ];
   fd_topo_tile_t * metric_tile = &topo->tiles[ fd_topo_find_tile( topo, "metric", 0UL ) ];
 
-  ushort const listen_port = 9000;
-  net_tile->net.legacy_transaction_listen_port = listen_port;
+  net_tile->net.legacy_transaction_listen_port = args->pktgen.listen_port;
 
   if( FD_UNLIKELY( !fd_cstr_to_ip4_addr( config->tiles.metric.prometheus_listen_address, &metric_tile->metric.prometheus_listen_addr ) ) )
     FD_LOG_ERR(( "failed to parse prometheus listen address `%s`", config->tiles.metric.prometheus_listen_address ));
@@ -209,7 +234,6 @@ pktgen_cmd_fn( args_t *   args FD_PARAM_UNUSED,
   configure_stage( &fd_cfg_stage_ethtool_channels, CONFIGURE_CMD_INIT, config );
   configure_stage( &fd_cfg_stage_ethtool_offloads, CONFIGURE_CMD_INIT, config );
 
-  fdctl_check_configure( config );
   /* FIXME this allocates lots of memory unnecessarily */
   initialize_workspaces( config );
   initialize_stacks( config );
@@ -240,22 +264,39 @@ pktgen_cmd_fn( args_t *   args FD_PARAM_UNUSED,
     for( ulong i=0UL; i<w.ws_row; i++ ) putc( '\n', stdout );
   }
 
+  net_abstract_metrics_t abstract;
+  for( ulong j=0UL; j<(sizeof(abstract.a)/sizeof(ulong)); j++ ) {
+    static ulong const zero = 0UL;
+    abstract.a[j] = &zero;
+  }
+  if( 0==strcmp( config->net.provider, "xdp" ) ) {
+    abstract.rx_pkt_cnt     = &net_metrics[ MIDX( COUNTER, NET, RX_PKT_CNT           ) ];
+    abstract.rx_bytes_total = &net_metrics[ MIDX( COUNTER, NET, RX_BYTES_TOTAL       ) ];
+    abstract.tx_pkt_cnt     = &net_metrics[ MIDX( COUNTER, NET, TX_COMPLETE_CNT      ) ];
+    abstract.tx_bytes_total = &net_metrics[ MIDX( COUNTER, NET, TX_BYTES_TOTAL       ) ];
+  } else if( 0==strcmp( config->net.provider, "ibverbs" ) ) {
+    abstract.rx_pkt_cnt     = &net_metrics[ MIDX( COUNTER, IBETH, RX_PKT_CNT           ) ];
+    abstract.rx_bytes_total = &net_metrics[ MIDX( COUNTER, IBETH, RX_BYTES_TOTAL       ) ];
+    abstract.tx_pkt_cnt     = &net_metrics[ MIDX( COUNTER, IBETH, TX_PKT_CNT           ) ];
+    abstract.tx_bytes_total = &net_metrics[ MIDX( COUNTER, IBETH, TX_BYTES_TOTAL       ) ];
+  }
+
   /* Simple REPL loop */
   puts( "Running fddev pktgen" );
-  printf( "XDP socket listening on port %u\n", (uint)listen_port );
+  printf( "%s listening on port %u\n", config->net.provider, (uint)net_tile->net.legacy_transaction_listen_port );
   puts( "Available commands: start, stop, quit" );
   puts( "" );
   char input[ 256 ] = {0};
   for(;;) {
-    render_status( net_metrics );
+    render_status( net_metrics, &abstract );
     fputs( "pktgen> ", stdout );
     fflush( stdout );
 
     for(;;) {
       struct pollfd fds[1] = {{ .fd=STDIN_FILENO, .events=POLLIN }};
       int poll_res = poll( fds, 1, 500 );
       if( poll_res==0 ) {
-        render_status( net_metrics );
+        render_status( net_metrics, &abstract );
         continue;
       } else if( poll_res>0 ) {
         break;

diff --git a/src/disco/metrics/generate/types.py b/src/disco/metrics/generate/types.py
@@ -43,6 +43,7 @@ class Tile(Enum):
     SNAPLA = 37
     SNAPLS = 38
     TOWER = 39
+    IBETH = 40
 
 class MetricType(Enum):
     COUNTER = 0

diff --git a/src/disco/metrics/generated/fd_metrics_all.c b/src/disco/metrics/generated/fd_metrics_all.c
@@ -73,6 +73,7 @@ const char * FD_METRICS_TILE_KIND_NAMES[FD_METRICS_TILE_KIND_CNT] = {
     "snapla",
     "snapls",
     "tower",
+    "ibeth",
 };
 
 const ulong FD_METRICS_TILE_KIND_SIZES[FD_METRICS_TILE_KIND_CNT] = {
@@ -113,6 +114,7 @@ const ulong FD_METRICS_TILE_KIND_SIZES[FD_METRICS_TILE_KIND_CNT] = {
     FD_METRICS_SNAPLA_TOTAL,
     FD_METRICS_SNAPLS_TOTAL,
     FD_METRICS_TOWER_TOTAL,
+    FD_METRICS_IBETH_TOTAL,
 };
 const fd_metrics_meta_t * FD_METRICS_TILE_KIND_METRICS[FD_METRICS_TILE_KIND_CNT] = {
     FD_METRICS_NET,
@@ -152,4 +154,5 @@ const fd_metrics_meta_t * FD_METRICS_TILE_KIND_METRICS[FD_METRICS_TILE_KIND_CNT]
     FD_METRICS_SNAPLA,
     FD_METRICS_SNAPLS,
     FD_METRICS_TOWER,
+    FD_METRICS_IBETH,
 };
diff --git a/src/disco/metrics/generated/fd_metrics_all.h b/src/disco/metrics/generated/fd_metrics_all.h
@@ -42,6 +42,7 @@
 #include "fd_metrics_benchs.h"
 #include "fd_metrics_tower.h"
 #include "fd_metrics_gui.h"
+#include "fd_metrics_ibeth.h"
 /* Start of LINK OUT metrics */
 
 #define FD_METRICS_COUNTER_LINK_SLOW_COUNT_OFF  (0UL)
@@ -178,7 +179,7 @@ extern const fd_metrics_meta_t FD_METRICS_ALL_LINK_OUT[FD_METRICS_ALL_LINK_OUT_T
 
 #define FD_METRICS_TOTAL_SZ (8UL*254UL)
 
-#define FD_METRICS_TILE_KIND_CNT 37
+#define FD_METRICS_TILE_KIND_CNT 38
 extern const char * FD_METRICS_TILE_KIND_NAMES[FD_METRICS_TILE_KIND_CNT];
 extern const ulong FD_METRICS_TILE_KIND_SIZES[FD_METRICS_TILE_KIND_CNT];
 extern const fd_metrics_meta_t * FD_METRICS_TILE_KIND_METRICS[FD_METRICS_TILE_KIND_CNT];

diff --git a/src/disco/metrics/generated/fd_metrics_ibeth.c b/src/disco/metrics/generated/fd_metrics_ibeth.c
@@ -0,0 +1,9 @@
+/* THIS FILE IS GENERATED BY gen_metrics.py. DO NOT HAND EDIT. */
+#include "fd_metrics_ibeth.h"
+
+const fd_metrics_meta_t FD_METRICS_IBETH[FD_METRICS_IBETH_TOTAL] = {
+    DECLARE_METRIC( IBETH_RX_PKT_CNT, COUNTER ),
+    DECLARE_METRIC( IBETH_RX_BYTES_TOTAL, COUNTER ),
+    DECLARE_METRIC( IBETH_TX_PKT_CNT, COUNTER ),
+    DECLARE_METRIC( IBETH_TX_BYTES_TOTAL, COUNTER ),
+};