diff --git a/drivers/net/veth.c b/drivers/net/veth.c index a3046142cb8e2..187f30e2cb4b9 100644 --- a/drivers/net/veth.c +++ b/drivers/net/veth.c @@ -733,7 +733,7 @@ static void veth_xdp_rcv_bulk_skb(struct veth_rq *rq, void **frames, } } -static void veth_xdp_get(struct xdp_buff *xdp) +static void veth_xdp_get_shared(struct xdp_buff *xdp) { struct skb_shared_info *sinfo = xdp_get_shared_info_from_buff(xdp); int i; @@ -746,12 +746,33 @@ static void veth_xdp_get(struct xdp_buff *xdp) __skb_frag_ref(&sinfo->frags[i]); } +static void veth_xdp_get_pp(struct xdp_buff *xdp) +{ + struct skb_shared_info *sinfo = xdp_get_shared_info_from_buff(xdp); + int i; + + page_pool_ref_page(virt_to_page(xdp->data)); + if (likely(!xdp_buff_has_frags(xdp))) + return; + + for (i = 0; i < sinfo->nr_frags; i++) { + skb_frag_t *frag = &sinfo->frags[i]; + + page_pool_ref_page(netmem_to_page(frag->netmem)); + } +} + +static void veth_xdp_get(struct xdp_buff *xdp) +{ + xdp->rxq->mem.type == MEM_TYPE_PAGE_POOL ? + veth_xdp_get_pp(xdp) : veth_xdp_get_shared(xdp); +} + static int veth_convert_skb_to_xdp_buff(struct veth_rq *rq, struct xdp_buff *xdp, struct sk_buff **pskb) { struct sk_buff *skb = *pskb; - u32 frame_sz; if (skb_shared(skb) || skb_head_is_locked(skb) || skb_shinfo(skb)->nr_frags || @@ -762,19 +783,7 @@ static int veth_convert_skb_to_xdp_buff(struct veth_rq *rq, skb = *pskb; } - /* SKB "head" area always have tailroom for skb_shared_info */ - frame_sz = skb_end_pointer(skb) - skb->head; - frame_sz += SKB_DATA_ALIGN(sizeof(struct skb_shared_info)); - xdp_init_buff(xdp, frame_sz, &rq->xdp_rxq); - xdp_prepare_buff(xdp, skb->head, skb_headroom(skb), - skb_headlen(skb), true); - - if (skb_is_nonlinear(skb)) { - skb_shinfo(skb)->xdp_frags_size = skb->data_len; - xdp_buff_set_frags_flag(xdp); - } else { - xdp_buff_clear_frags_flag(xdp); - } + xdp_convert_skb_to_buff(skb, xdp, &rq->xdp_rxq); *pskb = skb; return 0; @@ -822,24 +831,24 @@ static struct sk_buff *veth_xdp_rcv_skb(struct veth_rq *rq, case XDP_TX: veth_xdp_get(xdp); consume_skb(skb); - xdp->rxq->mem = rq->xdp_mem; if (unlikely(veth_xdp_tx(rq, xdp, bq) < 0)) { trace_xdp_exception(rq->dev, xdp_prog, act); stats->rx_drops++; goto err_xdp; } stats->xdp_tx++; + rq->xdp_rxq.mem = rq->xdp_mem; rcu_read_unlock(); goto xdp_xmit; case XDP_REDIRECT: veth_xdp_get(xdp); consume_skb(skb); - xdp->rxq->mem = rq->xdp_mem; if (xdp_do_redirect(rq->dev, xdp, xdp_prog)) { stats->rx_drops++; goto err_xdp; } stats->xdp_redirect++; + rq->xdp_rxq.mem = rq->xdp_mem; rcu_read_unlock(); goto xdp_xmit; default: diff --git a/include/net/xdp.h b/include/net/xdp.h index aa742f413c358..be7cc2eb956da 100644 --- a/include/net/xdp.h +++ b/include/net/xdp.h @@ -384,6 +384,31 @@ struct sk_buff *xdp_build_skb_from_frame(struct xdp_frame *xdpf, struct net_device *dev); struct xdp_frame *xdpf_clone(struct xdp_frame *xdpf); +static inline +void xdp_convert_skb_to_buff(struct sk_buff *skb, struct xdp_buff *xdp, + struct xdp_rxq_info *xdp_rxq) +{ + u32 frame_sz, pkt_len; + + /* SKB "head" area always have tailroom for skb_shared_info */ + frame_sz = skb_end_pointer(skb) - skb->head; + frame_sz += SKB_DATA_ALIGN(sizeof(struct skb_shared_info)); + pkt_len = skb_tail_pointer(skb) - skb_mac_header(skb); + + xdp_init_buff(xdp, frame_sz, xdp_rxq); + xdp_prepare_buff(xdp, skb->head, skb->mac_header, pkt_len, true); + + if (skb_is_nonlinear(skb)) { + skb_shinfo(skb)->xdp_frags_size = skb->data_len; + xdp_buff_set_frags_flag(xdp); + } else { + xdp_buff_clear_frags_flag(xdp); + } + + xdp->rxq->mem.type = skb->pp_recycle ? MEM_TYPE_PAGE_POOL : + MEM_TYPE_PAGE_SHARED; +} + static inline void xdp_convert_frame_to_buff(const struct xdp_frame *frame, struct xdp_buff *xdp) diff --git a/net/core/dev.c b/net/core/dev.c index 2acfa44927daa..a71da4edc493d 100644 --- a/net/core/dev.c +++ b/net/core/dev.c @@ -5320,35 +5320,18 @@ static struct netdev_rx_queue *netif_get_rxqueue(struct sk_buff *skb) u32 bpf_prog_run_generic_xdp(struct sk_buff *skb, struct xdp_buff *xdp, const struct bpf_prog *xdp_prog) { - void *orig_data, *orig_data_end, *hard_start; + void *orig_data, *orig_data_end; struct netdev_rx_queue *rxqueue; bool orig_bcast, orig_host; - u32 mac_len, frame_sz; + u32 metalen, act, mac_len; __be16 orig_eth_type; struct ethhdr *eth; - u32 metalen, act; int off; - /* The XDP program wants to see the packet starting at the MAC - * header. - */ + rxqueue = netif_get_rxqueue(skb); mac_len = skb->data - skb_mac_header(skb); - hard_start = skb->data - skb_headroom(skb); - - /* SKB "head" area always have tailroom for skb_shared_info */ - frame_sz = (void *)skb_end_pointer(skb) - hard_start; - frame_sz += SKB_DATA_ALIGN(sizeof(struct skb_shared_info)); - rxqueue = netif_get_rxqueue(skb); - xdp_init_buff(xdp, frame_sz, &rxqueue->xdp_rxq); - xdp_prepare_buff(xdp, hard_start, skb_headroom(skb) - mac_len, - skb_headlen(skb) + mac_len, true); - if (skb_is_nonlinear(skb)) { - skb_shinfo(skb)->xdp_frags_size = skb->data_len; - xdp_buff_set_frags_flag(xdp); - } else { - xdp_buff_clear_frags_flag(xdp); - } + xdp_convert_skb_to_buff(skb, xdp, &rxqueue->xdp_rxq); orig_data_end = xdp->data_end; orig_data = xdp->data;