Skip to content

xsk: refactors around generic xmit side #6016

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Closed
wants to merge 3 commits into from
Closed

xsk: refactors around generic xmit side #6016

Changes from all commits
Commits
Show all changes
3 commits
Select commit Hold shift + click to select a range
e8786b6
xsk: avoid overwriting skb fields for multi-buffer traffic
mfijalko Sep 22, 2025
b3cd7ab
xsk: remove @first_frag from xsk_build_skb()
mfijalko Sep 22, 2025
e037c90
xsk: wrap generic metadata handling onto separate function
mfijalko Sep 22, 2025
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
109 changes: 60 additions & 49 deletions net/xdp/xsk.c
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -605,6 +605,13 @@ static u32 xsk_get_num_desc(struct sk_buff *skb)
return XSKCB(skb)->num_descs;
}

static void xsk_init_cb(struct sk_buff *skb)
{
BUILD_BUG_ON(sizeof(struct xsk_addr_head) > sizeof(skb->cb));
INIT_LIST_HEAD(&XSKCB(skb)->addrs_list);
XSKCB(skb)->num_descs = 0;
}

static void xsk_destruct_skb(struct sk_buff *skb)
{
struct xsk_tx_metadata_compl *compl = &skb_shinfo(skb)->xsk_meta;
Expand All @@ -620,9 +627,6 @@ static void xsk_destruct_skb(struct sk_buff *skb)

static void xsk_set_destructor_arg(struct sk_buff *skb, u64 addr)
{
BUILD_BUG_ON(sizeof(struct xsk_addr_head) > sizeof(skb->cb));
INIT_LIST_HEAD(&XSKCB(skb)->addrs_list);
XSKCB(skb)->num_descs = 0;
skb_shinfo(skb)->destructor_arg = (void *)(uintptr_t)addr;
}

Expand Down Expand Up @@ -672,7 +676,7 @@ static struct sk_buff *xsk_build_skb_zerocopy(struct xdp_sock *xs,
return ERR_PTR(err);

skb_reserve(skb, hr);

xsk_init_cb(skb);
xsk_set_destructor_arg(skb, desc->addr);
} else {
xsk_addr = kmem_cache_zalloc(xsk_tx_generic_cache, GFP_KERNEL);
Expand Down Expand Up @@ -719,13 +723,50 @@ static struct sk_buff *xsk_build_skb_zerocopy(struct xdp_sock *xs,
return skb;
}

static int xsk_skb_metadata(struct sk_buff *skb, void *buffer,
struct xdp_desc *desc, struct xsk_buff_pool *pool,
u32 hr)
{
struct xsk_tx_metadata *meta = NULL;

if (unlikely(pool->tx_metadata_len == 0))
return -EINVAL;

meta = buffer - pool->tx_metadata_len;
if (unlikely(!xsk_buff_valid_tx_metadata(meta)))
return -EINVAL;

if (meta->flags & XDP_TXMD_FLAGS_CHECKSUM) {
if (unlikely(meta->request.csum_start +
meta->request.csum_offset +
sizeof(__sum16) > desc->len))
return -EINVAL;

skb->csum_start = hr + meta->request.csum_start;
skb->csum_offset = meta->request.csum_offset;
skb->ip_summed = CHECKSUM_PARTIAL;

if (unlikely(pool->tx_sw_csum)) {
int err;

err = skb_checksum_help(skb);
if (err)
return err;
}
}

if (meta->flags & XDP_TXMD_FLAGS_LAUNCH_TIME)
skb->skb_mstamp_ns = meta->request.launch_time;
xsk_tx_metadata_to_compl(meta, &skb_shinfo(skb)->xsk_meta);

return 0;
}

static struct sk_buff *xsk_build_skb(struct xdp_sock *xs,
struct xdp_desc *desc)
{
struct xsk_tx_metadata *meta = NULL;
struct net_device *dev = xs->dev;
struct sk_buff *skb = xs->skb;
bool first_frag = false;
int err;

if (dev->priv_flags & IFF_TX_SKB_NO_LINEAR) {
Expand All @@ -742,8 +783,6 @@ static struct sk_buff *xsk_build_skb(struct xdp_sock *xs,
len = desc->len;

if (!skb) {
first_frag = true;

hr = max(NET_SKB_PAD, L1_CACHE_ALIGN(dev->needed_headroom));
tr = dev->needed_tailroom;
skb = sock_alloc_send_skb(&xs->sk, hr + len + tr, 1, &err);
Expand All @@ -752,12 +791,24 @@ static struct sk_buff *xsk_build_skb(struct xdp_sock *xs,

skb_reserve(skb, hr);
skb_put(skb, len);
xsk_init_cb(skb);

err = skb_store_bits(skb, 0, buffer, len);
if (unlikely(err))
goto free_err;

xsk_set_destructor_arg(skb, desc->addr);
skb->dev = dev;
skb->priority = READ_ONCE(xs->sk.sk_priority);
skb->mark = READ_ONCE(xs->sk.sk_mark);
skb->destructor = xsk_destruct_skb;

if (desc->options & XDP_TX_METADATA) {
err = xsk_skb_metadata(skb, buffer, desc,
xs->pool, hr);
if (unlikely(err))
goto free_err;
}
} else {
int nr_frags = skb_shinfo(skb)->nr_frags;
struct xsk_addr_node *xsk_addr;
Expand Down Expand Up @@ -792,54 +843,14 @@ static struct sk_buff *xsk_build_skb(struct xdp_sock *xs,
xsk_addr->addr = desc->addr;
list_add_tail(&xsk_addr->addr_node, &XSKCB(skb)->addrs_list);
}

if (first_frag && desc->options & XDP_TX_METADATA) {
if (unlikely(xs->pool->tx_metadata_len == 0)) {
err = -EINVAL;
goto free_err;
}

meta = buffer - xs->pool->tx_metadata_len;
if (unlikely(!xsk_buff_valid_tx_metadata(meta))) {
err = -EINVAL;
goto free_err;
}

if (meta->flags & XDP_TXMD_FLAGS_CHECKSUM) {
if (unlikely(meta->request.csum_start +
meta->request.csum_offset +
sizeof(__sum16) > len)) {
err = -EINVAL;
goto free_err;
}

skb->csum_start = hr + meta->request.csum_start;
skb->csum_offset = meta->request.csum_offset;
skb->ip_summed = CHECKSUM_PARTIAL;

if (unlikely(xs->pool->tx_sw_csum)) {
err = skb_checksum_help(skb);
if (err)
goto free_err;
}
}

if (meta->flags & XDP_TXMD_FLAGS_LAUNCH_TIME)
skb->skb_mstamp_ns = meta->request.launch_time;
}
}

skb->dev = dev;
skb->priority = READ_ONCE(xs->sk.sk_priority);
skb->mark = READ_ONCE(xs->sk.sk_mark);
skb->destructor = xsk_destruct_skb;
xsk_tx_metadata_to_compl(meta, &skb_shinfo(skb)->xsk_meta);
xsk_inc_num_desc(skb);

return skb;

free_err:
if (first_frag && skb)
if (skb && !xsk_get_num_desc(skb))
kfree_skb(skb);

if (err == -EOVERFLOW) {
Expand Down
Loading