net 3.0: clean up routing logic

- Build entire packet in during_frag callback (fixes the need to
  save and restore state when transitioning from during_frag to
  after_frag)

- Publish non-routable packets to 'fallback' link instead of
  dropping them

Author: riptl

book/api/metrics-generated.md

@@ -49,8 +49,8 @@
 
 | Metric | Type | Description |
 |--------|------|-------------|
-| <span class="metrics-name">net_&#8203;rx_&#8203;pkt_&#8203;cnt</span><br/>{pkt_&#8203;kind="<span class="metrics-enum">ip4_&#8203;udp</span>"} | counter | Packet receive count. (IPv4 UDP packet (no options)) |
-| <span class="metrics-name">net_&#8203;rx_&#8203;pkt_&#8203;cnt</span><br/>{pkt_&#8203;kind="<span class="metrics-enum">ip4_&#8203;opt_&#8203;udp</span>"} | counter | Packet receive count. (IPv4 UDP packet (with options)) |
+| <span class="metrics-name">net_&#8203;rx_&#8203;pkt_&#8203;cnt</span><br/>{pkt_&#8203;kind="<span class="metrics-enum">ip4_&#8203;udp</span>"} | counter | Packet receive count (ignoring tunnels) (IPv4 UDP packet (no options)) |
+| <span class="metrics-name">net_&#8203;rx_&#8203;pkt_&#8203;cnt</span><br/>{pkt_&#8203;kind="<span class="metrics-enum">ip4_&#8203;opt_&#8203;udp</span>"} | counter | Packet receive count (ignoring tunnels) (IPv4 UDP packet (with options)) |
 | <span class="metrics-name">net_&#8203;rx_&#8203;bytes_&#8203;total</span> | counter | Total number of bytes received (including Ethernet header). |
 | <span class="metrics-name">net_&#8203;rx_&#8203;undersz_&#8203;cnt</span> | counter | Number of incoming packets dropped due to being too small. |
 | <span class="metrics-name">net_&#8203;rx_&#8203;fill_&#8203;blocked_&#8203;cnt</span> | counter | Number of incoming packets dropped due to fill ring being full. |
@@ -60,8 +60,8 @@
 | <span class="metrics-name">net_&#8203;tx_&#8203;submit_&#8203;cnt</span> | counter | Number of packet transmit jobs submitted. |
 | <span class="metrics-name">net_&#8203;tx_&#8203;complete_&#8203;cnt</span> | counter | Number of packet transmit jobs marked as completed by the kernel. |
 | <span class="metrics-name">net_&#8203;tx_&#8203;bytes_&#8203;total</span> | counter | Total number of bytes transmitted (including Ethernet header). |
-| <span class="metrics-name">net_&#8203;tx_&#8203;route_&#8203;fail_&#8203;cnt</span> | counter | Number of packet transmit jobs dropped due to route failure. |
-| <span class="metrics-name">net_&#8203;tx_&#8203;neighbor_&#8203;fail_&#8203;cnt</span> | counter | Number of packet transmit jobs dropped due to unresolved neighbor. |
+| <span class="metrics-name">net_&#8203;tx_&#8203;corrupt_&#8203;cnt</span> | counter | Number of packet transmit jobs dropped due to malformed content. |
+| <span class="metrics-name">net_&#8203;tx_&#8203;fallback_&#8203;cnt</span> | counter | Number of packet transmit jobs handled via sockets fallback instead of XDP. |
 | <span class="metrics-name">net_&#8203;tx_&#8203;full_&#8203;fail_&#8203;cnt</span> | counter | Number of packet transmit jobs dropped due to XDP TX ring full or missing completions. |
 | <span class="metrics-name">net_&#8203;tx_&#8203;busy_&#8203;cnt</span> | gauge | Number of transmit buffers currently busy. |
 | <span class="metrics-name">net_&#8203;tx_&#8203;idle_&#8203;cnt</span> | gauge | Number of transmit buffers currently idle. |
@@ -77,7 +77,6 @@
 | <span class="metrics-name">net_&#8203;rx_&#8203;gre_&#8203;invalid_&#8203;cnt</span> | counter | Number of invalid GRE packets received |
 | <span class="metrics-name">net_&#8203;rx_&#8203;gre_&#8203;ignored_&#8203;cnt</span> | counter | Number of received but ignored GRE packets |
 | <span class="metrics-name">net_&#8203;tx_&#8203;gre_&#8203;cnt</span> | counter | Number of GRE packet transmit jobs submitted |
-| <span class="metrics-name">net_&#8203;tx_&#8203;gre_&#8203;route_&#8203;fail_&#8203;cnt</span> | counter | Number of GRE packets transmit jobs dropped due to route failure |
 
 </div>
 

src/disco/metrics/generated/fd_metrics_net.c

@@ -13,8 +13,8 @@ const fd_metrics_meta_t FD_METRICS_NET[FD_METRICS_NET_TOTAL] = {
     DECLARE_METRIC( NET_TX_SUBMIT_CNT, COUNTER ),
     DECLARE_METRIC( NET_TX_COMPLETE_CNT, COUNTER ),
     DECLARE_METRIC( NET_TX_BYTES_TOTAL, COUNTER ),
-    DECLARE_METRIC( NET_TX_ROUTE_FAIL_CNT, COUNTER ),
-    DECLARE_METRIC( NET_TX_NEIGHBOR_FAIL_CNT, COUNTER ),
+    DECLARE_METRIC( NET_TX_CORRUPT_CNT, COUNTER ),
+    DECLARE_METRIC( NET_TX_FALLBACK_CNT, COUNTER ),
     DECLARE_METRIC( NET_TX_FULL_FAIL_CNT, COUNTER ),
     DECLARE_METRIC( NET_TX_BUSY_CNT, GAUGE ),
     DECLARE_METRIC( NET_TX_IDLE_CNT, GAUGE ),
@@ -30,5 +30,4 @@ const fd_metrics_meta_t FD_METRICS_NET[FD_METRICS_NET_TOTAL] = {
     DECLARE_METRIC( NET_RX_GRE_INVALID_CNT, COUNTER ),
     DECLARE_METRIC( NET_RX_GRE_IGNORED_CNT, COUNTER ),
     DECLARE_METRIC( NET_TX_GRE_CNT, COUNTER ),
-    DECLARE_METRIC( NET_TX_GRE_ROUTE_FAIL_CNT, COUNTER ),
 };

src/disco/metrics/generated/fd_metrics_net.h

@@ -6,7 +6,7 @@
 #define FD_METRICS_COUNTER_NET_RX_PKT_CNT_OFF  (16UL)
 #define FD_METRICS_COUNTER_NET_RX_PKT_CNT_NAME "net_rx_pkt_cnt"
 #define FD_METRICS_COUNTER_NET_RX_PKT_CNT_TYPE (FD_METRICS_TYPE_COUNTER)
-#define FD_METRICS_COUNTER_NET_RX_PKT_CNT_DESC "Packet receive count."
+#define FD_METRICS_COUNTER_NET_RX_PKT_CNT_DESC "Packet receive count (ignoring tunnels)"
 #define FD_METRICS_COUNTER_NET_RX_PKT_CNT_CVT  (FD_METRICS_CONVERTER_NONE)
 #define FD_METRICS_COUNTER_NET_RX_PKT_CNT_CNT  (2UL)
 
@@ -67,17 +67,17 @@
 #define FD_METRICS_COUNTER_NET_TX_BYTES_TOTAL_DESC "Total number of bytes transmitted (including Ethernet header)."
 #define FD_METRICS_COUNTER_NET_TX_BYTES_TOTAL_CVT  (FD_METRICS_CONVERTER_NONE)
 
-#define FD_METRICS_COUNTER_NET_TX_ROUTE_FAIL_CNT_OFF  (27UL)
-#define FD_METRICS_COUNTER_NET_TX_ROUTE_FAIL_CNT_NAME "net_tx_route_fail_cnt"
-#define FD_METRICS_COUNTER_NET_TX_ROUTE_FAIL_CNT_TYPE (FD_METRICS_TYPE_COUNTER)
-#define FD_METRICS_COUNTER_NET_TX_ROUTE_FAIL_CNT_DESC "Number of packet transmit jobs dropped due to route failure."
-#define FD_METRICS_COUNTER_NET_TX_ROUTE_FAIL_CNT_CVT  (FD_METRICS_CONVERTER_NONE)
+#define FD_METRICS_COUNTER_NET_TX_CORRUPT_CNT_OFF  (27UL)
+#define FD_METRICS_COUNTER_NET_TX_CORRUPT_CNT_NAME "net_tx_corrupt_cnt"
+#define FD_METRICS_COUNTER_NET_TX_CORRUPT_CNT_TYPE (FD_METRICS_TYPE_COUNTER)
+#define FD_METRICS_COUNTER_NET_TX_CORRUPT_CNT_DESC "Number of packet transmit jobs dropped due to malformed content."
+#define FD_METRICS_COUNTER_NET_TX_CORRUPT_CNT_CVT  (FD_METRICS_CONVERTER_NONE)
 
-#define FD_METRICS_COUNTER_NET_TX_NEIGHBOR_FAIL_CNT_OFF  (28UL)
-#define FD_METRICS_COUNTER_NET_TX_NEIGHBOR_FAIL_CNT_NAME "net_tx_neighbor_fail_cnt"
-#define FD_METRICS_COUNTER_NET_TX_NEIGHBOR_FAIL_CNT_TYPE (FD_METRICS_TYPE_COUNTER)
-#define FD_METRICS_COUNTER_NET_TX_NEIGHBOR_FAIL_CNT_DESC "Number of packet transmit jobs dropped due to unresolved neighbor."
-#define FD_METRICS_COUNTER_NET_TX_NEIGHBOR_FAIL_CNT_CVT  (FD_METRICS_CONVERTER_NONE)
+#define FD_METRICS_COUNTER_NET_TX_FALLBACK_CNT_OFF  (28UL)
+#define FD_METRICS_COUNTER_NET_TX_FALLBACK_CNT_NAME "net_tx_fallback_cnt"
+#define FD_METRICS_COUNTER_NET_TX_FALLBACK_CNT_TYPE (FD_METRICS_TYPE_COUNTER)
+#define FD_METRICS_COUNTER_NET_TX_FALLBACK_CNT_DESC "Number of packet transmit jobs handled via sockets fallback instead of XDP."
+#define FD_METRICS_COUNTER_NET_TX_FALLBACK_CNT_CVT  (FD_METRICS_CONVERTER_NONE)
 
 #define FD_METRICS_COUNTER_NET_TX_FULL_FAIL_CNT_OFF  (29UL)
 #define FD_METRICS_COUNTER_NET_TX_FULL_FAIL_CNT_NAME "net_tx_full_fail_cnt"
@@ -169,11 +169,5 @@
 #define FD_METRICS_COUNTER_NET_TX_GRE_CNT_DESC "Number of GRE packet transmit jobs submitted"
 #define FD_METRICS_COUNTER_NET_TX_GRE_CNT_CVT  (FD_METRICS_CONVERTER_NONE)
 
-#define FD_METRICS_COUNTER_NET_TX_GRE_ROUTE_FAIL_CNT_OFF  (44UL)
-#define FD_METRICS_COUNTER_NET_TX_GRE_ROUTE_FAIL_CNT_NAME "net_tx_gre_route_fail_cnt"
-#define FD_METRICS_COUNTER_NET_TX_GRE_ROUTE_FAIL_CNT_TYPE (FD_METRICS_TYPE_COUNTER)
-#define FD_METRICS_COUNTER_NET_TX_GRE_ROUTE_FAIL_CNT_DESC "Number of GRE packets transmit jobs dropped due to route failure"
-#define FD_METRICS_COUNTER_NET_TX_GRE_ROUTE_FAIL_CNT_CVT  (FD_METRICS_CONVERTER_NONE)
-
-#define FD_METRICS_NET_TOTAL (29UL)
+#define FD_METRICS_NET_TOTAL (28UL)
 extern const fd_metrics_meta_t FD_METRICS_NET[FD_METRICS_NET_TOTAL];

src/disco/metrics/metrics.xml

@@ -68,8 +68,8 @@ metric introduced.
     <counter name="TxSubmitCnt" summary="Number of packet transmit jobs submitted." />
     <counter name="TxCompleteCnt" summary="Number of packet transmit jobs marked as completed by the kernel." />
     <counter name="TxBytesTotal" summary="Total number of bytes transmitted (including Ethernet header)." />
-    <counter name="TxRouteFailCnt" summary="Number of packet transmit jobs dropped due to route failure." />
-    <counter name="TxNeighborFailCnt" summary="Number of packet transmit jobs dropped due to unresolved neighbor." />
+    <counter name="TxCorruptCnt" summary="Number of packet transmit jobs dropped due to malformed content." />
+    <counter name="TxFallbackCnt" summary="Number of packet transmit jobs handled via sockets fallback instead of XDP." />
     <counter name="TxFullFailCnt" summary="Number of packet transmit jobs dropped due to XDP TX ring full or missing completions." />
     <gauge name="TxBusyCnt" summary="Number of transmit buffers currently busy." />
     <gauge name="TxIdleCnt" summary="Number of transmit buffers currently idle." />
@@ -90,7 +90,6 @@ metric introduced.
     <counter name="RxGreInvalidCnt" summary="Number of invalid GRE packets received" />
     <counter name="RxGreIgnoredCnt" summary="Number of received but ignored GRE packets" />
     <counter name="TxGreCnt" summary="Number of GRE packet transmit jobs submitted" />
-    <counter name="TxGreRouteFailCnt" summary="Number of GRE packets transmit jobs dropped due to route failure" />
 </tile>
 
 <enum name="SockErr">

src/disco/net/fd_net_router.h

@@ -0,0 +1,101 @@
+#ifndef HEADER_fd_src_disco_net_fd_net_router_h
+#define HEADER_fd_src_disco_net_fd_net_router_h
+
+/* fd_net_router.h provides an internal API for userland routing. */
+
+#include "../../waltz/mib/fd_netdev_tbl.h"
+#include "../../waltz/ip/fd_fib4.h"
+#include "../../waltz/neigh/fd_neigh4_map.h"
+
+#include <linux/if_arp.h>
+
+struct fd_net_router {
+  /* Route and neighbor tables */
+  fd_fib4_t const *    fib_local;
+  fd_fib4_t const *    fib_main;
+  fd_neigh4_hmap_t     neigh4[1];
+  fd_netdev_tbl_join_t netdev_tbl;
+
+  uint if_idx;
+  uint bind_address;
+  uint default_address;
+};
+
+typedef struct fd_net_router fd_net_router_t;
+
+struct fd_net_next_hop {
+  uchar mac_addrs[12]; /* First 12 bytes of Ethernet header */
+  uint  src_ip;
+
+  uint  gre_src_ip;
+  uint  gre_dst_ip;
+};
+
+typedef struct fd_net_next_hop fd_next_hop_t;
+
+/* FD_NET_HOP_* give the result types of a route lookup. */
+
+#define FD_NET_HOP_RAW      0
+#define FD_NET_HOP_GRE      1
+#define FD_NET_HOP_FALLBACK 2
+
+/* fd_net_tx_route routes an outgoing packet based on its destination IP
+   address.  Returns an action FD_NET_HOP_*.
+
+   Saves out routing instructions to net_ctx->tx_op, including:
+   - XSK index
+   - source IP address
+   - source and dest MAC addresses
+   - GRE tunnelling info */
+
+static FD_FN_UNUSED uint
+fd_net_tx_route( fd_net_router_t const * router,
+                 fd_next_hop_t *     out,
+                 uint                    dst_ip ) {
+  /* Route lookup */
+
+  fd_fib4_hop_t hop[2] = {0};
+  fd_fib4_lookup( router->fib_local, hop+0, dst_ip, 0UL );
+  fd_fib4_lookup( router->fib_main,  hop+1, dst_ip, 0UL );
+  fd_fib4_hop_t const * next_hop = fd_fib4_hop_or( hop+0, hop+1 );
+
+  uint rtype   = next_hop->rtype;
+  uint if_idx  = next_hop->if_idx;
+  uint ip4_src = next_hop->ip4_src;
+
+  if( FD_UNLIKELY( rtype!=FD_FIB4_RTYPE_UNICAST           ) ) return FD_NET_HOP_FALLBACK;
+  if( FD_UNLIKELY( if_idx>router->netdev_tbl.hdr->dev_cnt ) ) return FD_NET_HOP_FALLBACK;
+  fd_netdev_t const * netdev = &router->netdev_tbl.dev_tbl[ if_idx ];
+
+  ip4_src = fd_uint_if( !!router->bind_address, router->bind_address, ip4_src );
+  out->src_ip = ip4_src;
+
+  if( netdev->dev_type==ARPHRD_IPGRE ) {
+    /* Packet targets a GRE tunnel */
+    if( netdev->gre_src_ip ) out->gre_src_ip = netdev->gre_src_ip;
+    out->gre_dst_ip = netdev->gre_dst_ip;
+    return FD_NET_HOP_GRE;
+  }
+
+  if( FD_UNLIKELY( if_idx!=router->if_idx ) ) return FD_NET_HOP_FALLBACK;
+
+  /* Neighbor resolve */
+  uint neigh_ip = next_hop->ip4_gw;
+  if( !neigh_ip ) neigh_ip = dst_ip;
+
+  fd_neigh4_hmap_query_t neigh_query[1];
+  int neigh_res = fd_neigh4_hmap_query_try( router->neigh4, &neigh_ip, NULL, neigh_query, 0 );
+  if( FD_UNLIKELY( neigh_res!=FD_MAP_SUCCESS ) ) return FD_NET_HOP_FALLBACK;
+  fd_neigh4_entry_t const * neigh = fd_neigh4_hmap_query_ele_const( neigh_query );
+  if( FD_UNLIKELY( neigh->state != FD_NEIGH4_STATE_ACTIVE ) ) return FD_NET_HOP_FALLBACK;
+  ip4_src = fd_uint_if( !ip4_src, router->default_address, ip4_src );
+  out->src_ip = ip4_src;
+  memcpy( out->mac_addrs+0, neigh->mac_addr,  6 );
+  memcpy( out->mac_addrs+6, netdev->mac_addr, 6 );
+
+  if( FD_UNLIKELY( fd_neigh4_hmap_query_test( neigh_query ) ) ) return FD_NET_HOP_FALLBACK;
+
+  return FD_NET_HOP_RAW;
+}
+
+#endif /* HEADER_fd_src_disco_net_xdp_fd_xdp_route_h */