net 3.0: remove neighbor solicit code

Thanks to the socket fallback, out-of-band neighbor solicitation
is no longer required.  When starting up (with an empty neighbor
table), Firedancer steers all packets through sockets.

This socket traffic will slowely populate the neighbor table with
entries, which then causes traffic to get steered over the high
performance path.

Author: riptl

book/api/metrics-generated.md

@@ -700,10 +700,6 @@
 | <span class="metrics-name">netlnk_&#8203;interface_&#8203;count</span> | gauge | Number of network interfaces |
 | <span class="metrics-name">netlnk_&#8203;route_&#8203;count</span><br/>{route_&#8203;table="<span class="metrics-enum">local</span>"} | gauge | Number of IPv4 routes (Local) |
 | <span class="metrics-name">netlnk_&#8203;route_&#8203;count</span><br/>{route_&#8203;table="<span class="metrics-enum">main</span>"} | gauge | Number of IPv4 routes (Main) |
-| <span class="metrics-name">netlnk_&#8203;neigh_&#8203;probe_&#8203;sent</span> | counter | Number of neighbor solicit requests sent to kernel |
-| <span class="metrics-name">netlnk_&#8203;neigh_&#8203;probe_&#8203;fails</span> | counter | Number of neighbor solicit requests that failed to send (kernel too slow) |
-| <span class="metrics-name">netlnk_&#8203;neigh_&#8203;probe_&#8203;rate_&#8203;limit_&#8203;host</span> | counter | Number of neighbor solicit that exceeded the per-host rate limit |
-| <span class="metrics-name">netlnk_&#8203;neigh_&#8203;probe_&#8203;rate_&#8203;limit_&#8203;global</span> | counter | Number of neighbor solicit that exceeded the global rate limit |
 
 </div>
 

book/guide/internals/netlink.md

@@ -71,33 +71,12 @@ inputs.
    Neighbor table updates are forwarded ot the netlink tile.  This path
    has limited throughput (few ~100K updates per second).
 
-- `[untrusted traffic] --> [net tile] --> [app tile]` <br/>
-   `--> [net tile] --> [netlink tile] --> [neighbor discovery]` <br/>
-   App tiles will blindly respond to the source IP found in untrusted
-   packets.  This source IP can be spoofed.  Neighbor solicitation might
-   be required in order to find out the MAC address of that IP.  On IPv4,
-   these are ARP requests broadcasted to the local network.
-
-   Net tiles cannot solicit neighbors directly, so they notify the
-   netlink tile that neighbor solicitation is needed.  (Potentially at
-   line rate if network configuration is part of a huge subnet)
-
-   The netlink tile will deduplicate these requests and forward them to
-   the kernel.
-
-   This path is the only direct 'untrusted traffic' -> 'netlink tile'
-   data flow, so the internal neighbor solicit message format is kept
-   as simple as possible for security.
-
 ### Neighbor discovery (ARP)
 
 A concurrent open addressed hash table is used to store ARP entries
 (henceforth called "neighbor table").  This table attempts to
 continuously stay in sync with the kernel.
 
-The netlink tile requests neighbor solicitations via the netlink
-equivalent of `ip neigh add dev DEVICE IP use`.
-
 ### Routing
 
 The Firedancer network stack supports very simple routing tables as

src/disco/metrics/generated/fd_metrics_netlnk.c

@@ -11,8 +11,4 @@ const fd_metrics_meta_t FD_METRICS_NETLNK[FD_METRICS_NETLNK_TOTAL] = {
     DECLARE_METRIC( NETLNK_INTERFACE_COUNT, GAUGE ),
     DECLARE_METRIC_ENUM( NETLNK_ROUTE_COUNT, GAUGE, ROUTE_TABLE, LOCAL ),
     DECLARE_METRIC_ENUM( NETLNK_ROUTE_COUNT, GAUGE, ROUTE_TABLE, MAIN ),
-    DECLARE_METRIC( NETLNK_NEIGH_PROBE_SENT, COUNTER ),
-    DECLARE_METRIC( NETLNK_NEIGH_PROBE_FAILS, COUNTER ),
-    DECLARE_METRIC( NETLNK_NEIGH_PROBE_RATE_LIMIT_HOST, COUNTER ),
-    DECLARE_METRIC( NETLNK_NEIGH_PROBE_RATE_LIMIT_GLOBAL, COUNTER ),
 };

src/disco/metrics/generated/fd_metrics_netlnk.h

@@ -48,29 +48,5 @@
 #define FD_METRICS_GAUGE_NETLNK_ROUTE_COUNT_LOCAL_OFF (23UL)
 #define FD_METRICS_GAUGE_NETLNK_ROUTE_COUNT_MAIN_OFF (24UL)
 
-#define FD_METRICS_COUNTER_NETLNK_NEIGH_PROBE_SENT_OFF  (25UL)
-#define FD_METRICS_COUNTER_NETLNK_NEIGH_PROBE_SENT_NAME "netlnk_neigh_probe_sent"
-#define FD_METRICS_COUNTER_NETLNK_NEIGH_PROBE_SENT_TYPE (FD_METRICS_TYPE_COUNTER)
-#define FD_METRICS_COUNTER_NETLNK_NEIGH_PROBE_SENT_DESC "Number of neighbor solicit requests sent to kernel"
-#define FD_METRICS_COUNTER_NETLNK_NEIGH_PROBE_SENT_CVT  (FD_METRICS_CONVERTER_NONE)
-
-#define FD_METRICS_COUNTER_NETLNK_NEIGH_PROBE_FAILS_OFF  (26UL)
-#define FD_METRICS_COUNTER_NETLNK_NEIGH_PROBE_FAILS_NAME "netlnk_neigh_probe_fails"
-#define FD_METRICS_COUNTER_NETLNK_NEIGH_PROBE_FAILS_TYPE (FD_METRICS_TYPE_COUNTER)
-#define FD_METRICS_COUNTER_NETLNK_NEIGH_PROBE_FAILS_DESC "Number of neighbor solicit requests that failed to send (kernel too slow)"
-#define FD_METRICS_COUNTER_NETLNK_NEIGH_PROBE_FAILS_CVT  (FD_METRICS_CONVERTER_NONE)
-
-#define FD_METRICS_COUNTER_NETLNK_NEIGH_PROBE_RATE_LIMIT_HOST_OFF  (27UL)
-#define FD_METRICS_COUNTER_NETLNK_NEIGH_PROBE_RATE_LIMIT_HOST_NAME "netlnk_neigh_probe_rate_limit_host"
-#define FD_METRICS_COUNTER_NETLNK_NEIGH_PROBE_RATE_LIMIT_HOST_TYPE (FD_METRICS_TYPE_COUNTER)
-#define FD_METRICS_COUNTER_NETLNK_NEIGH_PROBE_RATE_LIMIT_HOST_DESC "Number of neighbor solicit that exceeded the per-host rate limit"
-#define FD_METRICS_COUNTER_NETLNK_NEIGH_PROBE_RATE_LIMIT_HOST_CVT  (FD_METRICS_CONVERTER_NONE)
-
-#define FD_METRICS_COUNTER_NETLNK_NEIGH_PROBE_RATE_LIMIT_GLOBAL_OFF  (28UL)
-#define FD_METRICS_COUNTER_NETLNK_NEIGH_PROBE_RATE_LIMIT_GLOBAL_NAME "netlnk_neigh_probe_rate_limit_global"
-#define FD_METRICS_COUNTER_NETLNK_NEIGH_PROBE_RATE_LIMIT_GLOBAL_TYPE (FD_METRICS_TYPE_COUNTER)
-#define FD_METRICS_COUNTER_NETLNK_NEIGH_PROBE_RATE_LIMIT_GLOBAL_DESC "Number of neighbor solicit that exceeded the global rate limit"
-#define FD_METRICS_COUNTER_NETLNK_NEIGH_PROBE_RATE_LIMIT_GLOBAL_CVT  (FD_METRICS_CONVERTER_NONE)
-
-#define FD_METRICS_NETLNK_TOTAL (13UL)
+#define FD_METRICS_NETLNK_TOTAL (9UL)
 extern const fd_metrics_meta_t FD_METRICS_NETLNK[FD_METRICS_NETLNK_TOTAL];

src/disco/metrics/metrics.xml

@@ -898,10 +898,6 @@ metric introduced.
     <counter name="Updates" enum="NetlinkMsg" summary="Number of netlink live updates processed" />
     <gauge name="InterfaceCount" summary="Number of network interfaces" />
     <gauge name="RouteCount" enum="RouteTable" summary="Number of IPv4 routes" />
-    <counter name="NeighProbeSent" summary="Number of neighbor solicit requests sent to kernel" />
-    <counter name="NeighProbeFails" summary="Number of neighbor solicit requests that failed to send (kernel too slow)" />
-    <counter name="NeighProbeRateLimitHost" summary="Number of neighbor solicit that exceeded the per-host rate limit" />
-    <counter name="NeighProbeRateLimitGlobal" summary="Number of neighbor solicit that exceeded the global rate limit" />
 </tile>
 
 <tile name="snaprd">

src/disco/net/fd_net_tile_topo.c

@@ -15,9 +15,6 @@ setup_xdp_tile( fd_topo_t *             topo,
                 ulong const *           tile_to_cpu,
                 fd_config_net_t const * net_cfg ) {
   fd_topo_tile_t * tile = fd_topob_tile( topo, "net", "net", "metric_in", tile_to_cpu[ topo->tile_cnt ], 0, 0 );
-  fd_topob_link( topo, "net_netlnk", "net_netlnk", 128UL, 0UL, 0UL );
-  fd_topob_tile_in(  topo, "netlnk", 0UL, "metric_in", "net_netlnk", i, FD_TOPOB_UNRELIABLE, FD_TOPOB_POLLED );
-  fd_topob_tile_out( topo, "net",    i,                "net_netlnk", i );
   fd_netlink_topo_join( topo, netlink_tile, tile );
 
   fd_topo_obj_t * umem_obj = fd_topob_obj( topo, "dcache", "net_umem" );
@@ -81,8 +78,6 @@ fd_topos_net_tiles( fd_topo_t *             topo,
     fd_topob_wksp( topo, "netlnk" );
     /* netbase: shared network config (config plane) */
     fd_topob_wksp( topo, "netbase" );
-    /* net_netlnk: net->netlnk ARP requests */
-    fd_topob_wksp( topo, "net_netlnk" );
 
     fd_topo_tile_t * netlink_tile = fd_topob_tile( topo, "netlnk", "netlnk", "metric_in", tile_to_cpu[ topo->tile_cnt ], 0, 0 );
     fd_netlink_topo_create( netlink_tile, topo, netlnk_max_routes, netlnk_max_peer_routes, netlnk_max_neighbors, net_cfg->interface );

src/disco/net/xdp/fd_xdp_tile.c

@@ -11,7 +11,7 @@
 
 #include "../fd_find_16x16.h"
 #include "../../metrics/fd_metrics.h"
-#include "../../netlink/fd_netlink_tile.h" /* neigh4_solicit */
+#include "../../netlink/fd_netlink_tile.h"
 #include "../../topo/fd_topo.h"
 
 #include "../../../waltz/ip/fd_fib4.h"
@@ -247,7 +247,6 @@ typedef struct {
   fd_fib4_t const * fib_local;
   fd_fib4_t const * fib_main;
   fd_neigh4_hmap_t  neigh4[1];
-  fd_netlink_neigh4_solicit_link_t neigh4_solicit[1];
 
   /* Netdev table */
   fd_dbl_buf_t *       netdev_dbl_buf;    /* remote copy of device table */

src/disco/netlink/fd_netlink_tile.c

@@ -110,7 +110,7 @@ populate_allowed_seccomp( fd_topo_t const *      topo,
                           struct sock_filter *   out ) {
   fd_netlink_tile_ctx_t * ctx = fd_topo_obj_laddr( topo, tile->tile_obj_id );
   FD_TEST( ctx->magic==FD_NETLINK_TILE_CTX_MAGIC );
-  populate_sock_filter_policy_netlink( out_cnt, out, (uint)fd_log_private_logfile_fd(), (uint)ctx->nl_monitor->fd, (uint)ctx->nl_req->fd, (uint)ctx->prober->sock_fd );
+  populate_sock_filter_policy_netlink( out_cnt, out, (uint)fd_log_private_logfile_fd(), (uint)ctx->nl_monitor->fd, (uint)ctx->nl_req->fd );
   return sock_filter_policy_netlink_instr_cnt;
 }
 
@@ -122,15 +122,14 @@ populate_allowed_fds( fd_topo_t const *      topo,
   fd_netlink_tile_ctx_t * ctx = fd_topo_obj_laddr( topo, tile->tile_obj_id );
   FD_TEST( ctx->magic==FD_NETLINK_TILE_CTX_MAGIC );
 
-  if( FD_UNLIKELY( out_fds_cnt<5UL ) ) FD_LOG_ERR(( "out_fds_cnt too low (%lu)", out_fds_cnt ));
+  if( FD_UNLIKELY( out_fds_cnt<4UL ) ) FD_LOG_ERR(( "out_fds_cnt too low (%lu)", out_fds_cnt ));
 
   ulong out_cnt = 0UL;
   out_fds[ out_cnt++ ] = 2; /* stderr */
   if( FD_LIKELY( -1!=fd_log_private_logfile_fd() ) )
     out_fds[ out_cnt++ ] = fd_log_private_logfile_fd(); /* logfile */
   out_fds[ out_cnt++ ] = ctx->nl_monitor->fd;
   out_fds[ out_cnt++ ] = ctx->nl_req->fd;
-  out_fds[ out_cnt++ ] = ctx->prober->sock_fd;
   return out_cnt;
 }
 
@@ -168,11 +167,6 @@ privileged_init( fd_topo_t *      topo,
     FD_LOG_ERR(( "bind(sock,RT_NETLINK,RTMGRP_{LINK,NEIGH,IPV4_ROUTE}) failed (%i-%s)", errno, fd_io_strerror( errno ) ));
   }
 
-  float const max_probes_per_second =   3.f;
-  ulong const max_probe_burst       = 128UL;
-  float const probe_delay_seconds   =  15.f;
-  fd_neigh4_prober_init( ctx->prober, max_probes_per_second, max_probe_burst, probe_delay_seconds );
-
   /* Set duration of blocking reads in before_credit */
   struct timeval tv = { .tv_usec = 2000 }; /* 2ms */
   if( FD_UNLIKELY( 0!=setsockopt( ctx->nl_monitor->fd, SOL_SOCKET, SO_RCVTIMEO, &tv, sizeof(struct timeval) ) ) ) {
@@ -204,10 +198,7 @@ unprivileged_init( fd_topo_t *      topo,
   ctx->fib4_local = fd_fib4_join( fd_topo_obj_laddr( topo, tile->netlink.fib4_local_obj_id ) ); FD_TEST( ctx->fib4_local );
   ctx->fib4_main  = fd_fib4_join( fd_topo_obj_laddr( topo, tile->netlink.fib4_main_obj_id  ) ); FD_TEST( ctx->fib4_main  );
 
-  for( ulong i=0UL; i<tile->in_cnt; i++ ) {
-    fd_topo_link_t * link = &topo->links[ tile->in_link_id[ i ] ];
-    if( FD_UNLIKELY( link->mtu!=0UL ) ) FD_LOG_ERR(( "netlink solicit links must have an MTU of zero" ));
-  }
+  if( FD_UNLIKELY( tile->in_cnt!=0 ) ) FD_LOG_ERR(( "netlink tile had unexpected input links" ));
 
   ctx->action |= FD_NET_TILE_ACTION_LINK_UPDATE;
   ctx->action |= FD_NET_TILE_ACTION_ROUTE4_UPDATE;
@@ -232,10 +223,6 @@ metrics_write( fd_netlink_tile_ctx_t * ctx ) {
   FD_MGAUGE_SET(     NETLNK, INTERFACE_COUNT,         ctx->netdev_tbl->hdr->dev_cnt     );
   FD_MGAUGE_SET(     NETLNK, ROUTE_COUNT_LOCAL,       fd_fib4_cnt( ctx->fib4_local )    );
   FD_MGAUGE_SET(     NETLNK, ROUTE_COUNT_MAIN,        fd_fib4_cnt( ctx->fib4_main  )    );
-  FD_MCNT_SET(       NETLNK, NEIGH_PROBE_SENT,        ctx->metrics.neigh_solicits_sent  );
-  FD_MCNT_SET(       NETLNK, NEIGH_PROBE_FAILS,       ctx->metrics.neigh_solicits_fails );
-  FD_MCNT_SET(       NETLNK, NEIGH_PROBE_RATE_LIMIT_HOST,   ctx->prober->local_rate_limited_cnt  );
-  FD_MCNT_SET(       NETLNK, NEIGH_PROBE_RATE_LIMIT_GLOBAL, ctx->prober->global_rate_limited_cnt );
 }
 
 /* netlink_monitor_read calls recvfrom to process a link, route, or
@@ -337,81 +324,6 @@ before_credit( fd_netlink_tile_ctx_t * ctx,
 
 }
 
-/* after_poll_overrun is called when fd_stem.c was overrun while
-   checking for new fragments.  This typically happens when
-   before_credit takes too long (e.g. we were in a blocking netlink
-   read) */
-
-static void
-after_poll_overrun( fd_netlink_tile_ctx_t * ctx ) {
-  ctx->idle_cnt = -1L;
-}
-
-/* after_frag handles a neighbor solicit request */
-
-static void
-after_frag( fd_netlink_tile_ctx_t * ctx,
-            ulong                   in_idx,
-            ulong                   seq,
-            ulong                   sig,
-            ulong                   sz,
-            ulong                   tsorig,
-            ulong                   tspub,
-            fd_stem_context_t *     stem ) {
-  (void)in_idx; (void)seq; (void)tsorig; (void)tspub; (void)stem;
-
-  long now = fd_tickcount();
-  ctx->idle_cnt = -1L;
-
-  /* Parse request (fully contained in sig field) */
-
-  if( FD_UNLIKELY( sz!=0UL ) ) {
-    FD_LOG_WARNING(( "unexpected sz %lu", sz ));
-  }
-  if( FD_UNLIKELY( sig>>48 ) ) {
-    FD_LOG_WARNING(( "unexpected high bits in sig %016lx", sig ));
-  }
-  ushort if_idx   = (ushort)(sig>>32);
-  uint   ip4_addr = (uint)sig;
-  if( FD_UNLIKELY( if_idx!=ctx->neigh4_ifidx ) ) {
-    ctx->metrics.neigh_solicits_fails++;
-    FD_LOG_ERR(( "received neighbor solicit request for invalid interface index %u", if_idx ));
-    return;
-  }
-
-  /* Drop if the kernel is already working on the request */
-
-  fd_neigh4_hmap_query_t query[1];
-  int spec_res = fd_neigh4_hmap_query_try( ctx->neigh4, &ip4_addr, NULL, query, 0 );
-  if( spec_res==FD_MAP_SUCCESS ) {
-    ctx->metrics.neigh_solicits_fails++;
-    return;
-  }
-
-  /* Insert placeholder (take above branch next time) */
-
-  int prepare_res = fd_neigh4_hmap_prepare( ctx->neigh4, &ip4_addr, NULL, query, 0 );
-  if( FD_UNLIKELY( prepare_res!=FD_MAP_SUCCESS ) ) {
-    ctx->metrics.neigh_solicits_fails++;
-    return;
-  }
-  fd_neigh4_entry_t * ele = fd_neigh4_hmap_query_ele( query );
-  ele->state    = FD_NEIGH4_STATE_INCOMPLETE;
-  ele->ip4_addr = ip4_addr;
-  memset( ele->mac_addr, 0, 6UL );
-  fd_neigh4_hmap_publish( query );
-
-  /* Trigger neighbor solicit via netlink */
-
-  int probe_res = fd_neigh4_probe_rate_limited( ctx->prober, ele, ip4_addr, now );
-  if( probe_res==0 ) {
-    ctx->metrics.neigh_solicits_sent++;
-  } else if( probe_res>0 ) {
-    ctx->metrics.neigh_solicits_fails++;
-  }
-
-}
-
 #define STEM_BURST (1UL)
 #define STEM_LAZY ((ulong)13e6) /* 13ms */
 
@@ -421,8 +333,6 @@ after_frag( fd_netlink_tile_ctx_t * ctx,
 #define STEM_CALLBACK_METRICS_WRITE       metrics_write
 #define STEM_CALLBACK_DURING_HOUSEKEEPING during_housekeeping
 #define STEM_CALLBACK_BEFORE_CREDIT       before_credit
-#define STEM_CALLBACK_AFTER_POLL_OVERRUN  after_poll_overrun
-#define STEM_CALLBACK_AFTER_FRAG          after_frag
 
 #include "../stem/fd_stem.c"
 

src/disco/netlink/fd_netlink_tile.h

@@ -16,17 +16,6 @@
 
 extern fd_topo_run_tile_t fd_tile_netlnk;
 
-/* fd_netlink_neigh4_solicit_link_t holds information required to send
-   neighbor solicitation requests to the netlink tile. */
-
-struct fd_netlink_neigh4_solicit_link {
-  fd_frag_meta_t * mcache;
-  ulong            depth;
-  ulong            seq;
-};
-
-typedef struct fd_netlink_neigh4_solicit_link fd_netlink_neigh4_solicit_link_t;
-
 struct fdctl_config;
 
 FD_PROTOTYPES_BEGIN
@@ -44,21 +33,6 @@ fd_netlink_topo_join( fd_topo_t *      topo,
                       fd_topo_tile_t * netlink_tile,
                       fd_topo_tile_t * join_tile );
 
-/* fd_netlink_neigh4_solicit requests a neighbor solicitation (i.e. ARP
-   request) for an IPv4 address.  Safe to call at a high rate.  The
-   netlink tile will deduplicate requests.  ip4_addr is big endian. */
-
-static inline void
-fd_netlink_neigh4_solicit( fd_netlink_neigh4_solicit_link_t * link,
-                           uint                               ip4_addr,
-                           uint                               if_idx,
-                           ulong                              tspub_comp ) {
-  ulong seq = link->seq;
-  ulong sig = (ulong)ip4_addr | ( (ulong)if_idx<<32 );
-  fd_mcache_publish( link->mcache, link->depth, seq, sig, 0UL, 0UL, 0UL, 0UL, tspub_comp );
-  link->seq = fd_seq_inc( seq, 1UL );
-}
-
 FD_PROTOTYPES_END
 
 #endif /* HEADER_fd_src_disco_netlink_fd_netlink_tile_h */

src/disco/netlink/fd_netlink_tile_private.h

@@ -7,7 +7,6 @@
 #include "../../waltz/mib/fd_dbl_buf.h"
 #include "../../waltz/mib/fd_netdev_tbl.h"
 #include "../../waltz/neigh/fd_neigh4_map.h"
-#include "../../waltz/neigh/fd_neigh4_probe.h"
 
 /* FD_NETLINK_TILE_CTX_MAGIC uniquely identifies a fd_netlink_tile_ctx_t.
    CHange this whenever the fd_netlink_tile_ctx_t struct changes. */
@@ -46,15 +45,10 @@ struct fd_netlink_tile_ctx {
   uint             neigh4_ifidx;
   long             idle_cnt;
 
-  /* Neighbor table prober */
-  fd_neigh4_prober_t prober[1];
-
   struct {
     ulong link_full_syncs;
     ulong route_full_syncs;
     ulong update_cnt[ FD_METRICS_COUNTER_NETLNK_UPDATES_CNT ];
-    ulong neigh_solicits_sent;
-    ulong neigh_solicits_fails;
   } metrics;
 };