diff --git a/Documentation/netlink/specs/netdev.yaml b/Documentation/netlink/specs/netdev.yaml index c035dc0f64fd6..0d0e05e059472 100644 --- a/Documentation/netlink/specs/netdev.yaml +++ b/Documentation/netlink/specs/netdev.yaml @@ -61,6 +61,11 @@ definitions: doc: | Device is capable of exposing receive packet VLAN tag via bpf_xdp_metadata_rx_vlan_tag(). + - + name: checksum + doc: | + Device is capable of exposing receive checksum result via + bpf_xdp_metadata_rx_checksum(). - type: flags name: xsk-flags diff --git a/drivers/net/ethernet/intel/ice/ice_base.c b/drivers/net/ethernet/intel/ice/ice_base.c index c5da8e9cc0a0e..e8ec419bdc98c 100644 --- a/drivers/net/ethernet/intel/ice/ice_base.c +++ b/drivers/net/ethernet/intel/ice/ice_base.c @@ -591,6 +591,7 @@ static int ice_vsi_cfg_rxq(struct ice_rx_ring *ring) } } + ring->pkt_ctx.rxq_flags = ring->flags; xdp_init_buff(&ring->xdp, ice_get_frame_sz(ring), &ring->xdp_rxq); ring->xdp.data = NULL; ring->xdp_ext.pkt_ctx = &ring->pkt_ctx; diff --git a/drivers/net/ethernet/intel/ice/ice_txrx.h b/drivers/net/ethernet/intel/ice/ice_txrx.h index fef750c5f288f..ed6c8b6b2390d 100644 --- a/drivers/net/ethernet/intel/ice/ice_txrx.h +++ b/drivers/net/ethernet/intel/ice/ice_txrx.h @@ -260,6 +260,7 @@ enum ice_rx_dtype { struct ice_pkt_ctx { u64 cached_phctime; __be16 vlan_proto; + u8 rxq_flags; }; struct ice_xdp_buff { diff --git a/drivers/net/ethernet/intel/ice/ice_txrx_lib.c b/drivers/net/ethernet/intel/ice/ice_txrx_lib.c index 45cfaabc41cbe..56c3f09c81cef 100644 --- a/drivers/net/ethernet/intel/ice/ice_txrx_lib.c +++ b/drivers/net/ethernet/intel/ice/ice_txrx_lib.c @@ -80,69 +80,46 @@ ice_rx_hash_to_skb(const struct ice_rx_ring *rx_ring, libeth_rx_pt_set_hash(skb, hash, decoded); } -/** - * ice_rx_gcs - Set generic checksum in skb - * @skb: skb currently being received and modified - * @rx_desc: receive descriptor - */ -static void ice_rx_gcs(struct sk_buff *skb, - const union ice_32b_rx_flex_desc *rx_desc) -{ - const struct ice_32b_rx_flex_desc_nic *desc; - u16 csum; - - desc = (struct ice_32b_rx_flex_desc_nic *)rx_desc; - skb->ip_summed = CHECKSUM_COMPLETE; - csum = (__force u16)desc->raw_csum; - skb->csum = csum_unfold((__force __sum16)swab16(csum)); -} - -/** - * ice_rx_csum - Indicate in skb if checksum is good - * @ring: the ring we care about - * @skb: skb currently being received and modified - * @rx_desc: the receive descriptor - * @ptype: the packet type decoded by hardware - * - * skb->protocol must be set before this function is called - */ static void -ice_rx_csum(struct ice_rx_ring *ring, struct sk_buff *skb, - union ice_32b_rx_flex_desc *rx_desc, u16 ptype) +ice_get_rx_csum(const union ice_32b_rx_flex_desc *rx_desc, u16 ptype, + struct net_device *dev, struct ice_pf *pf, u8 rxq_flag, + enum xdp_checksum *ip_summed, u32 *cksum_meta) { - struct libeth_rx_pt decoded; + struct libeth_rx_pt decoded = libie_rx_pt_parse(ptype); u16 rx_status0, rx_status1; bool ipv4, ipv6; - /* Start with CHECKSUM_NONE and by default csum_level = 0 */ - skb->ip_summed = CHECKSUM_NONE; - - decoded = libie_rx_pt_parse(ptype); - if (!libeth_rx_pt_has_checksum(ring->netdev, decoded)) - return; + if (!libeth_rx_pt_has_checksum(dev, decoded)) + goto checksum_none; rx_status0 = le16_to_cpu(rx_desc->wb.status_error0); rx_status1 = le16_to_cpu(rx_desc->wb.status_error1); - - if ((ring->flags & ICE_RX_FLAGS_RING_GCS) && + if ((rxq_flag & ICE_RX_FLAGS_RING_GCS) && rx_desc->wb.rxdid == ICE_RXDID_FLEX_NIC && (decoded.inner_prot == LIBETH_RX_PT_INNER_TCP || decoded.inner_prot == LIBETH_RX_PT_INNER_UDP || decoded.inner_prot == LIBETH_RX_PT_INNER_ICMP)) { - ice_rx_gcs(skb, rx_desc); + const struct ice_32b_rx_flex_desc_nic *desc; + u16 csum; + + desc = (struct ice_32b_rx_flex_desc_nic *)rx_desc; + *ip_summed = XDP_CHECKSUM_COMPLETE; + csum = (__force u16)desc->raw_csum; + *cksum_meta = csum_unfold((__force __sum16)swab16(csum)); return; } /* check if HW has decoded the packet and checksum */ if (!(rx_status0 & BIT(ICE_RX_FLEX_DESC_STATUS0_L3L4P_S))) - return; + goto checksum_none; ipv4 = libeth_rx_pt_get_ip_ver(decoded) == LIBETH_RX_PT_OUTER_IPV4; ipv6 = libeth_rx_pt_get_ip_ver(decoded) == LIBETH_RX_PT_OUTER_IPV6; if (ipv4 && (rx_status0 & (BIT(ICE_RX_FLEX_DESC_STATUS0_XSUM_EIPE_S)))) { - ring->vsi->back->hw_rx_eipe_error++; - return; + if (pf) + pf->hw_rx_eipe_error++; + goto checksum_none; } if (ipv4 && (rx_status0 & (BIT(ICE_RX_FLEX_DESC_STATUS0_XSUM_IPE_S)))) @@ -167,13 +144,48 @@ ice_rx_csum(struct ice_rx_ring *ring, struct sk_buff *skb, * we are indicating we validated the inner checksum. */ if (decoded.tunnel_type >= LIBETH_RX_PT_TUNNEL_IP_GRENAT) - skb->csum_level = 1; + *cksum_meta = 1; - skb->ip_summed = CHECKSUM_UNNECESSARY; + *ip_summed = XDP_CHECKSUM_UNNECESSARY; return; checksum_fail: - ring->vsi->back->hw_csum_rx_error++; + if (pf) + pf->hw_csum_rx_error++; +checksum_none: + *ip_summed = XDP_CHECKSUM_NONE; + *cksum_meta = 0; +} + +/** + * ice_rx_csum - Indicate in skb if checksum is good + * @ring: the ring we care about + * @skb: skb currently being received and modified + * @rx_desc: the receive descriptor + * @ptype: the packet type decoded by hardware + * + * skb->protocol must be set before this function is called + */ +static void +ice_rx_csum(struct ice_rx_ring *ring, struct sk_buff *skb, + union ice_32b_rx_flex_desc *rx_desc, u16 ptype) +{ + enum xdp_checksum ip_summed; + u32 cksum_meta; + + ice_get_rx_csum(rx_desc, ptype, ring->netdev, ring->vsi->back, + ring->flags, &ip_summed, &cksum_meta); + switch (ip_summed) { + case XDP_CHECKSUM_UNNECESSARY: + skb->csum_level = cksum_meta; + break; + case XDP_CHECKSUM_COMPLETE: + skb->csum = cksum_meta; + break; + default: + break; + } + skb->ip_summed = ip_summed; } /** @@ -555,6 +567,30 @@ static int ice_xdp_rx_hash(const struct xdp_md *ctx, u32 *hash, return 0; } +/** + * ice_xdp_rx_checksum - RX checksum XDP hint handler + * @ctx: XDP buff pointer + * @ip_summed: RX checksum result destination address + * @cksum_meta: XDP RX checksum metadata destination address + * + * Copy RX checksum result (if available) and its metadata to the + * destination address. + */ +static int ice_xdp_rx_checksum(const struct xdp_md *ctx, + enum xdp_checksum *ip_summed, + u32 *cksum_meta) +{ + const struct ice_xdp_buff *xdp_ext = (void *)ctx; + const union ice_32b_rx_flex_desc *rx_desc = xdp_ext->eop_desc; + + ice_get_rx_csum(rx_desc, ice_get_ptype(rx_desc), + xdp_ext->xdp_buff.rxq->dev, NULL, + xdp_ext->pkt_ctx->rxq_flags, + ip_summed, cksum_meta); + + return 0; +} + /** * ice_xdp_rx_vlan_tag - VLAN tag XDP hint handler * @ctx: XDP buff pointer @@ -584,4 +620,5 @@ const struct xdp_metadata_ops ice_xdp_md_ops = { .xmo_rx_timestamp = ice_xdp_rx_hw_ts, .xmo_rx_hash = ice_xdp_rx_hash, .xmo_rx_vlan_tag = ice_xdp_rx_vlan_tag, + .xmo_rx_checksum = ice_xdp_rx_checksum, }; diff --git a/drivers/net/veth.c b/drivers/net/veth.c index a3046142cb8e2..a4de941edba91 100644 --- a/drivers/net/veth.c +++ b/drivers/net/veth.c @@ -1692,6 +1692,25 @@ static int veth_xdp_rx_vlan_tag(const struct xdp_md *ctx, __be16 *vlan_proto, return err; } +static int veth_xdp_rx_checksum(const struct xdp_md *ctx, + enum xdp_checksum *ip_summed, + u32 *cksum_meta) +{ + const struct veth_xdp_buff *_ctx = (void *)ctx; + const struct sk_buff *skb = _ctx->skb; + + if (!skb) + return -ENODATA; + + /* For locally generated packets ip_summed is set to + * CHECKSUM_PARTIAL. + */ + *ip_summed = skb->ip_summed; + *cksum_meta = 0; + + return 0; +} + static const struct net_device_ops veth_netdev_ops = { .ndo_init = veth_dev_init, .ndo_open = veth_open, @@ -1717,6 +1736,7 @@ static const struct xdp_metadata_ops veth_xdp_metadata_ops = { .xmo_rx_timestamp = veth_xdp_rx_timestamp, .xmo_rx_hash = veth_xdp_rx_hash, .xmo_rx_vlan_tag = veth_xdp_rx_vlan_tag, + .xmo_rx_checksum = veth_xdp_rx_checksum, }; #define VETH_FEATURES (NETIF_F_SG | NETIF_F_FRAGLIST | NETIF_F_HW_CSUM | \ diff --git a/include/net/xdp.h b/include/net/xdp.h index f288c348a6c13..84a1da5b64318 100644 --- a/include/net/xdp.h +++ b/include/net/xdp.h @@ -552,6 +552,10 @@ void xdp_attachment_setup(struct xdp_attachment_info *info, NETDEV_XDP_RX_METADATA_VLAN_TAG, \ bpf_xdp_metadata_rx_vlan_tag, \ xmo_rx_vlan_tag) \ + XDP_METADATA_KFUNC(XDP_METADATA_KFUNC_RX_CHECKSUM, \ + NETDEV_XDP_RX_METADATA_CHECKSUM, \ + bpf_xdp_metadata_rx_checksum, \ + xmo_rx_checksum) enum xdp_rx_metadata { #define XDP_METADATA_KFUNC(name, _, __, ___) name, @@ -609,12 +613,22 @@ enum xdp_rss_hash_type { XDP_RSS_TYPE_L4_IPV6_SCTP_EX = XDP_RSS_TYPE_L4_IPV6_SCTP | XDP_RSS_L3_DYNHDR, }; +enum xdp_checksum { + XDP_CHECKSUM_NONE = CHECKSUM_NONE, + XDP_CHECKSUM_UNNECESSARY = CHECKSUM_UNNECESSARY, + XDP_CHECKSUM_COMPLETE = CHECKSUM_COMPLETE, + XDP_CHECKSUM_PARTIAL = CHECKSUM_PARTIAL, +}; + struct xdp_metadata_ops { int (*xmo_rx_timestamp)(const struct xdp_md *ctx, u64 *timestamp); int (*xmo_rx_hash)(const struct xdp_md *ctx, u32 *hash, enum xdp_rss_hash_type *rss_type); int (*xmo_rx_vlan_tag)(const struct xdp_md *ctx, __be16 *vlan_proto, u16 *vlan_tci); + int (*xmo_rx_checksum)(const struct xdp_md *ctx, + enum xdp_checksum *ip_summed, + u32 *cksum_meta); }; #ifdef CONFIG_NET diff --git a/net/core/xdp.c b/net/core/xdp.c index 491334b9b8bec..6d87a4e22a769 100644 --- a/net/core/xdp.c +++ b/net/core/xdp.c @@ -962,6 +962,35 @@ __bpf_kfunc int bpf_xdp_metadata_rx_vlan_tag(const struct xdp_md *ctx, return -EOPNOTSUPP; } +/** + * bpf_xdp_metadata_rx_checksum - Read XDP frame RX checksum. + * @ctx: XDP context pointer. + * @ip_summed: Return value pointer indicating checksum result. + * @cksum_meta: Return value pointer indicating checksum result metadata. + * + * In case of success, ``ip_summed`` is set to the RX checksum result. Possible + * values are: + * ``XDP_CHECKSUM_NONE`` + * ``XDP_CHECKSUM_UNNECESSARY`` + * ``XDP_CHECKSUM_COMPLETE`` + * ``XDP_CHECKSUM_PARTIAL`` + * + * In case of success, ``cksum_meta`` contains the hw computed checksum value + * for ``XDP_CHECKSUM_COMPLETE`` or the ``csum_level`` for + * ``XDP_CHECKSUM_UNNECESSARY``. It is set to 0 for ``XDP_CHECKSUM_NONE`` and + * ``XDP_CHECKSUM_PARTIAL``. + * + * Return: + * * Returns 0 on success or ``-errno`` on error. + * * ``-EOPNOTSUPP`` : means device driver does not implement kfunc + * * ``-ENODATA`` : means no RX-timestamp available for this frame + */ +__bpf_kfunc int bpf_xdp_metadata_rx_checksum(const struct xdp_md *ctx, + u8 *ip_summed, u32 *cksum_meta) +{ + return -EOPNOTSUPP; +} + __bpf_kfunc_end_defs(); BTF_KFUNCS_START(xdp_metadata_kfunc_ids) diff --git a/tools/testing/selftests/bpf/prog_tests/xdp_metadata.c b/tools/testing/selftests/bpf/prog_tests/xdp_metadata.c index 19f92affc2daa..707c98e664745 100644 --- a/tools/testing/selftests/bpf/prog_tests/xdp_metadata.c +++ b/tools/testing/selftests/bpf/prog_tests/xdp_metadata.c @@ -258,6 +258,7 @@ static void refill_rx(struct xsk *xsk, __u64 addr) static int verify_xsk_metadata(struct xsk *xsk, bool sent_from_af_xdp) { + __u8 ip_summed = sent_from_af_xdp ? XDP_CHECKSUM_NONE : XDP_CHECKSUM_PARTIAL; const struct xdp_desc *rx_desc; struct pollfd fds = {}; struct xdp_meta *meta; @@ -310,6 +311,12 @@ static int verify_xsk_metadata(struct xsk *xsk, bool sent_from_af_xdp) if (!ASSERT_NEQ(meta->rx_hash, 0, "rx_hash")) return -1; + if (!ASSERT_EQ(meta->ip_summed, ip_summed, "rx_ip_summed")) + return -1; + + if (!ASSERT_EQ(meta->cksum_meta, 0, "rx_cksum_meta")) + return -1; + if (!sent_from_af_xdp) { if (!ASSERT_NEQ(meta->rx_hash_type & XDP_RSS_TYPE_L4, 0, "rx_hash_type")) return -1; diff --git a/tools/testing/selftests/bpf/progs/xdp_hw_metadata.c b/tools/testing/selftests/bpf/progs/xdp_hw_metadata.c index 330ece2eabdb4..dc62d572e3ac6 100644 --- a/tools/testing/selftests/bpf/progs/xdp_hw_metadata.c +++ b/tools/testing/selftests/bpf/progs/xdp_hw_metadata.c @@ -110,6 +110,13 @@ int rx(struct xdp_md *ctx) else meta->hint_valid |= XDP_META_FIELD_VLAN_TAG; + err = bpf_xdp_metadata_rx_checksum(ctx, &meta->ip_summed, + &meta->cksum_meta); + if (err) + meta->rx_cksum_err = err; + else + meta->hint_valid |= XDP_META_FIELD_CHECKSUM; + __sync_add_and_fetch(&pkts_redir, 1); return bpf_redirect_map(&xsk, ctx->rx_queue_index, XDP_PASS); } diff --git a/tools/testing/selftests/bpf/progs/xdp_metadata.c b/tools/testing/selftests/bpf/progs/xdp_metadata.c index 09bb8a038d528..ef6a5584a1876 100644 --- a/tools/testing/selftests/bpf/progs/xdp_metadata.c +++ b/tools/testing/selftests/bpf/progs/xdp_metadata.c @@ -98,6 +98,7 @@ int rx(struct xdp_md *ctx) bpf_xdp_metadata_rx_hash(ctx, &meta->rx_hash, &meta->rx_hash_type); bpf_xdp_metadata_rx_vlan_tag(ctx, &meta->rx_vlan_proto, &meta->rx_vlan_tci); + bpf_xdp_metadata_rx_checksum(ctx, &meta->ip_summed, &meta->cksum_meta); return bpf_redirect_map(&xsk, ctx->rx_queue_index, XDP_PASS); } diff --git a/tools/testing/selftests/bpf/xdp_hw_metadata.c b/tools/testing/selftests/bpf/xdp_hw_metadata.c index 3d8de0d4c96a7..c8c25ef383b1f 100644 --- a/tools/testing/selftests/bpf/xdp_hw_metadata.c +++ b/tools/testing/selftests/bpf/xdp_hw_metadata.c @@ -219,6 +219,28 @@ static void print_vlan_tci(__u16 tag) printf("PCP=%u, DEI=%d, VID=0x%X\n", pcp, dei, vlan_id); } +static void print_rx_cksum(__u8 ip_summed, __u32 cksum_meta) +{ + const char *cksum = "CHECKSUM_NONE"; + + switch (ip_summed) { + case XDP_CHECKSUM_UNNECESSARY: + cksum = "CHECKSUM_UNNECESSARY"; + break; + case XDP_CHECKSUM_COMPLETE: + cksum = "CHECKSUM_COMPLETE"; + break; + case XDP_CHECKSUM_PARTIAL: + cksum = "CHECKSUM_PARTIAL"; + break; + case XDP_CHECKSUM_NONE: + default: + break; + } + + printf("rx-cksum: %s, csum_meta=0x%x\n", cksum, cksum_meta); +} + static void verify_xdp_metadata(void *data, clockid_t clock_id) { struct xdp_meta *meta; @@ -254,6 +276,11 @@ static void verify_xdp_metadata(void *data, clockid_t clock_id) printf("No rx_vlan_tci or rx_vlan_proto, err=%d\n", meta->rx_vlan_tag_err); } + + if (meta->hint_valid & XDP_META_FIELD_CHECKSUM) + print_rx_cksum(meta->ip_summed, meta->cksum_meta); + else + printf("No rx_chsum, err=%d\n", meta->rx_cksum_err); } static void verify_skb_metadata(int fd) diff --git a/tools/testing/selftests/bpf/xdp_metadata.h b/tools/testing/selftests/bpf/xdp_metadata.h index 87318ad1117a1..b7ab112da8032 100644 --- a/tools/testing/selftests/bpf/xdp_metadata.h +++ b/tools/testing/selftests/bpf/xdp_metadata.h @@ -28,8 +28,14 @@ enum xdp_meta_field { XDP_META_FIELD_TS = BIT(0), XDP_META_FIELD_RSS = BIT(1), XDP_META_FIELD_VLAN_TAG = BIT(2), + XDP_META_FIELD_CHECKSUM = BIT(3), }; +#define XDP_CHECKSUM_NONE 0 +#define XDP_CHECKSUM_UNNECESSARY 1 +#define XDP_CHECKSUM_COMPLETE 2 +#define XDP_CHECKSUM_PARTIAL 3 + struct xdp_meta { union { __u64 rx_timestamp; @@ -48,5 +54,12 @@ struct xdp_meta { }; __s32 rx_vlan_tag_err; }; + union { + struct { + __u8 ip_summed; + __u32 cksum_meta; + }; + __s32 rx_cksum_err; + }; enum xdp_meta_field hint_valid; };