diff --git a/net/xdp/xsk.c b/net/xdp/xsk.c index 9c3acecc14b1a..012991de9df2e 100644 --- a/net/xdp/xsk.c +++ b/net/xdp/xsk.c @@ -36,6 +36,13 @@ #define TX_BATCH_SIZE 32 #define MAX_PER_SOCKET_BUDGET 32 +struct xsk_addrs { + u32 num_descs; + u64 addrs[MAX_SKB_FRAGS + 1]; +}; + +static struct kmem_cache *xsk_tx_generic_cache; + void xsk_set_rx_need_wakeup(struct xsk_buff_pool *pool) { if (pool->cached_need_wakeup & XDP_WAKEUP_RX) @@ -532,25 +539,39 @@ static int xsk_wakeup(struct xdp_sock *xs, u8 flags) return dev->netdev_ops->ndo_xsk_wakeup(dev, xs->queue_id, flags); } -static int xsk_cq_reserve_addr_locked(struct xsk_buff_pool *pool, u64 addr) +static int xsk_cq_reserve_locked(struct xsk_buff_pool *pool) { unsigned long flags; int ret; spin_lock_irqsave(&pool->cq_lock, flags); - ret = xskq_prod_reserve_addr(pool->cq, addr); + ret = xskq_prod_reserve(pool->cq); spin_unlock_irqrestore(&pool->cq_lock, flags); return ret; } -static void xsk_cq_submit_locked(struct xsk_buff_pool *pool, u32 n) +static void xsk_cq_submit_addr_locked(struct xdp_sock *xs, + struct sk_buff *skb) { + struct xsk_buff_pool *pool = xs->pool; + struct xsk_addrs *xsk_addrs; unsigned long flags; + u32 num_desc, i; + u32 idx; + + xsk_addrs = (struct xsk_addrs *)skb_shinfo(skb)->destructor_arg; + num_desc = xsk_addrs->num_descs; spin_lock_irqsave(&pool->cq_lock, flags); - xskq_prod_submit_n(pool->cq, n); + idx = xskq_get_prod(pool->cq); + + for (i = 0; i < num_desc; i++) + xskq_prod_write_addr(pool->cq, idx + i, xsk_addrs->addrs[i]); + xskq_prod_submit_n(pool->cq, num_desc); + spin_unlock_irqrestore(&pool->cq_lock, flags); + kmem_cache_free(xsk_tx_generic_cache, xsk_addrs); } static void xsk_cq_cancel_locked(struct xsk_buff_pool *pool, u32 n) @@ -562,11 +583,6 @@ static void xsk_cq_cancel_locked(struct xsk_buff_pool *pool, u32 n) spin_unlock_irqrestore(&pool->cq_lock, flags); } -static u32 xsk_get_num_desc(struct sk_buff *skb) -{ - return skb ? (long)skb_shinfo(skb)->destructor_arg : 0; -} - static void xsk_destruct_skb(struct sk_buff *skb) { struct xsk_tx_metadata_compl *compl = &skb_shinfo(skb)->xsk_meta; @@ -576,21 +592,37 @@ static void xsk_destruct_skb(struct sk_buff *skb) *compl->tx_timestamp = ktime_get_tai_fast_ns(); } - xsk_cq_submit_locked(xdp_sk(skb->sk)->pool, xsk_get_num_desc(skb)); + xsk_cq_submit_addr_locked(xdp_sk(skb->sk), skb); sock_wfree(skb); } -static void xsk_set_destructor_arg(struct sk_buff *skb) +static u32 xsk_get_num_desc(struct sk_buff *skb) { - long num = xsk_get_num_desc(xdp_sk(skb->sk)->skb) + 1; + struct xsk_addrs *addrs; - skb_shinfo(skb)->destructor_arg = (void *)num; + addrs = (struct xsk_addrs *)skb_shinfo(skb)->destructor_arg; + return addrs->num_descs; +} + +static void xsk_set_destructor_arg(struct sk_buff *skb, struct xsk_addrs *addrs) +{ + skb_shinfo(skb)->destructor_arg = (void *)addrs; +} + +static void xsk_inc_skb_descs(struct sk_buff *skb) +{ + struct xsk_addrs *addrs; + + addrs = (struct xsk_addrs *)skb_shinfo(skb)->destructor_arg; + addrs->num_descs++; } static void xsk_consume_skb(struct sk_buff *skb) { struct xdp_sock *xs = xdp_sk(skb->sk); + kmem_cache_free(xsk_tx_generic_cache, + (struct xsk_addrs *)skb_shinfo(skb)->destructor_arg); skb->destructor = sock_wfree; xsk_cq_cancel_locked(xs->pool, xsk_get_num_desc(skb)); /* Free skb without triggering the perf drop trace */ @@ -609,6 +641,7 @@ static struct sk_buff *xsk_build_skb_zerocopy(struct xdp_sock *xs, { struct xsk_buff_pool *pool = xs->pool; u32 hr, len, ts, offset, copy, copied; + struct xsk_addrs *addrs = NULL; struct sk_buff *skb = xs->skb; struct page *page; void *buffer; @@ -623,6 +656,12 @@ static struct sk_buff *xsk_build_skb_zerocopy(struct xdp_sock *xs, return ERR_PTR(err); skb_reserve(skb, hr); + + addrs = kmem_cache_zalloc(xsk_tx_generic_cache, GFP_KERNEL); + if (!addrs) + return ERR_PTR(-ENOMEM); + + xsk_set_destructor_arg(skb, addrs); } addr = desc->addr; @@ -662,6 +701,7 @@ static struct sk_buff *xsk_build_skb(struct xdp_sock *xs, { struct xsk_tx_metadata *meta = NULL; struct net_device *dev = xs->dev; + struct xsk_addrs *addrs = NULL; struct sk_buff *skb = xs->skb; bool first_frag = false; int err; @@ -694,6 +734,15 @@ static struct sk_buff *xsk_build_skb(struct xdp_sock *xs, err = skb_store_bits(skb, 0, buffer, len); if (unlikely(err)) goto free_err; + + addrs = kmem_cache_zalloc(xsk_tx_generic_cache, GFP_KERNEL); + if (!addrs) { + err = -ENOMEM; + goto free_err; + } + + xsk_set_destructor_arg(skb, addrs); + } else { int nr_frags = skb_shinfo(skb)->nr_frags; struct page *page; @@ -759,7 +808,9 @@ static struct sk_buff *xsk_build_skb(struct xdp_sock *xs, skb->mark = READ_ONCE(xs->sk.sk_mark); skb->destructor = xsk_destruct_skb; xsk_tx_metadata_to_compl(meta, &skb_shinfo(skb)->xsk_meta); - xsk_set_destructor_arg(skb); + + addrs = (struct xsk_addrs *)skb_shinfo(skb)->destructor_arg; + addrs->addrs[addrs->num_descs++] = desc->addr; return skb; @@ -769,7 +820,7 @@ static struct sk_buff *xsk_build_skb(struct xdp_sock *xs, if (err == -EOVERFLOW) { /* Drop the packet */ - xsk_set_destructor_arg(xs->skb); + xsk_inc_skb_descs(xs->skb); xsk_drop_skb(xs->skb); xskq_cons_release(xs->tx); } else { @@ -812,7 +863,7 @@ static int __xsk_generic_xmit(struct sock *sk) * if there is space in it. This avoids having to implement * any buffering in the Tx path. */ - err = xsk_cq_reserve_addr_locked(xs->pool, desc.addr); + err = xsk_cq_reserve_locked(xs->pool); if (err) { err = -EAGAIN; goto out; @@ -1815,6 +1866,14 @@ static int __init xsk_init(void) if (err) goto out_pernet; + xsk_tx_generic_cache = kmem_cache_create("xsk_generic_xmit_cache", + sizeof(struct xsk_addrs), 0, + SLAB_HWCACHE_ALIGN, NULL); + if (!xsk_tx_generic_cache) { + err = -ENOMEM; + goto out_pernet; + } + return 0; out_pernet: diff --git a/net/xdp/xsk_queue.h b/net/xdp/xsk_queue.h index 46d87e961ad6d..f16f390370dc4 100644 --- a/net/xdp/xsk_queue.h +++ b/net/xdp/xsk_queue.h @@ -344,6 +344,11 @@ static inline u32 xskq_cons_present_entries(struct xsk_queue *q) /* Functions for producers */ +static inline u32 xskq_get_prod(struct xsk_queue *q) +{ + return READ_ONCE(q->ring->producer); +} + static inline u32 xskq_prod_nb_free(struct xsk_queue *q, u32 max) { u32 free_entries = q->nentries - (q->cached_prod - q->cached_cons); @@ -390,6 +395,13 @@ static inline int xskq_prod_reserve_addr(struct xsk_queue *q, u64 addr) return 0; } +static inline void xskq_prod_write_addr(struct xsk_queue *q, u32 idx, u64 addr) +{ + struct xdp_umem_ring *ring = (struct xdp_umem_ring *)q->ring; + + ring->desc[idx & q->ring_mask] = addr; +} + static inline void xskq_prod_write_addr_batch(struct xsk_queue *q, struct xdp_desc *descs, u32 nb_entries) {