Skip to content

Commit 3830fff

Browse files
axboeaxboe
committed
Merge branch 'for-uring-ubufops' of git://git.kernel.org/pub/scm/linux/kernel/git/kuba/linux into for-6.10/io_uring
Merge net changes required for the upcoming send zerocopy improvements. * 'for-uring-ubufops' of git://git.kernel.org/pub/scm/linux/kernel/git/kuba/linux: net: add callback for setting a ubuf_info to skb net: extend ubuf_info callback to ops structure Signed-off-by: Jens Axboe <[email protected]>
2 parents 2f9c951 + 65bada8 commit 3830fff

File tree

9 files changed

+62
-33
lines changed

9 files changed

+62
-33
lines changed

‎drivers/net/tap.c

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -754,7 +754,7 @@ static ssize_t tap_get_user(struct tap_queue *q, void *msg_control,
754754
skb_zcopy_init(skb, msg_control);
755755
} else if (msg_control) {
756756
struct ubuf_info *uarg = msg_control;
757-
uarg->callback(NULL, uarg, false);
757+
uarg->ops->complete(NULL, uarg, false);
758758
}
759759

760760
dev_queue_xmit(skb);

‎drivers/net/tun.c

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -1906,7 +1906,7 @@ static ssize_t tun_get_user(struct tun_struct *tun, struct tun_file *tfile,
19061906
skb_zcopy_init(skb, msg_control);
19071907
} else if (msg_control) {
19081908
struct ubuf_info *uarg = msg_control;
1909-
uarg->callback(NULL, uarg, false);
1909+
uarg->ops->complete(NULL, uarg, false);
19101910
}
19111911

19121912
skb_reset_network_header(skb);

‎drivers/net/xen-netback/common.h

Lines changed: 2 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -390,9 +390,8 @@ bool xenvif_rx_queue_tail(struct xenvif_queue *queue, struct sk_buff *skb);
390390

391391
void xenvif_carrier_on(struct xenvif *vif);
392392

393-
/* Callback from stack when TX packet can be released */
394-
void xenvif_zerocopy_callback(struct sk_buff *skb, struct ubuf_info *ubuf,
395-
bool zerocopy_success);
393+
/* Callbacks from stack when TX packet can be released */
394+
extern const struct ubuf_info_ops xenvif_ubuf_ops;
396395

397396
static inline pending_ring_idx_t nr_pending_reqs(struct xenvif_queue *queue)
398397
{

‎drivers/net/xen-netback/interface.c

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -593,7 +593,7 @@ int xenvif_init_queue(struct xenvif_queue *queue)
593593

594594
for (i = 0; i < MAX_PENDING_REQS; i++) {
595595
queue->pending_tx_info[i].callback_struct = (struct ubuf_info_msgzc)
596-
{ { .callback = xenvif_zerocopy_callback },
596+
{ { .ops = &xenvif_ubuf_ops },
597597
{ { .ctx = NULL,
598598
.desc = i } } };
599599
queue->grant_tx_handle[i] = NETBACK_INVALID_HANDLE;

‎drivers/net/xen-netback/netback.c

Lines changed: 8 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -1156,7 +1156,7 @@ static int xenvif_handle_frag_list(struct xenvif_queue *queue, struct sk_buff *s
11561156
uarg = skb_shinfo(skb)->destructor_arg;
11571157
/* increase inflight counter to offset decrement in callback */
11581158
atomic_inc(&queue->inflight_packets);
1159-
uarg->callback(NULL, uarg, true);
1159+
uarg->ops->complete(NULL, uarg, true);
11601160
skb_shinfo(skb)->destructor_arg = NULL;
11611161

11621162
/* Fill the skb with the new (local) frags. */
@@ -1278,8 +1278,9 @@ static int xenvif_tx_submit(struct xenvif_queue *queue)
12781278
return work_done;
12791279
}
12801280

1281-
void xenvif_zerocopy_callback(struct sk_buff *skb, struct ubuf_info *ubuf_base,
1282-
bool zerocopy_success)
1281+
static void xenvif_zerocopy_callback(struct sk_buff *skb,
1282+
struct ubuf_info *ubuf_base,
1283+
bool zerocopy_success)
12831284
{
12841285
unsigned long flags;
12851286
pending_ring_idx_t index;
@@ -1312,6 +1313,10 @@ void xenvif_zerocopy_callback(struct sk_buff *skb, struct ubuf_info *ubuf_base,
13121313
xenvif_skb_zerocopy_complete(queue);
13131314
}
13141315

1316+
const struct ubuf_info_ops xenvif_ubuf_ops = {
1317+
.complete = xenvif_zerocopy_callback,
1318+
};
1319+
13151320
static inline void xenvif_tx_dealloc_action(struct xenvif_queue *queue)
13161321
{
13171322
struct gnttab_unmap_grant_ref *gop;

‎drivers/vhost/net.c

Lines changed: 6 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -380,7 +380,7 @@ static void vhost_zerocopy_signal_used(struct vhost_net *net,
380380
}
381381
}
382382

383-
static void vhost_zerocopy_callback(struct sk_buff *skb,
383+
static void vhost_zerocopy_complete(struct sk_buff *skb,
384384
struct ubuf_info *ubuf_base, bool success)
385385
{
386386
struct ubuf_info_msgzc *ubuf = uarg_to_msgzc(ubuf_base);
@@ -408,6 +408,10 @@ static void vhost_zerocopy_callback(struct sk_buff *skb,
408408
rcu_read_unlock_bh();
409409
}
410410

411+
static const struct ubuf_info_ops vhost_ubuf_ops = {
412+
.complete = vhost_zerocopy_complete,
413+
};
414+
411415
static inline unsigned long busy_clock(void)
412416
{
413417
return local_clock() >> 10;
@@ -879,7 +883,7 @@ static void handle_tx_zerocopy(struct vhost_net *net, struct socket *sock)
879883
vq->heads[nvq->upend_idx].len = VHOST_DMA_IN_PROGRESS;
880884
ubuf->ctx = nvq->ubufs;
881885
ubuf->desc = nvq->upend_idx;
882-
ubuf->ubuf.callback = vhost_zerocopy_callback;
886+
ubuf->ubuf.ops = &vhost_ubuf_ops;
883887
ubuf->ubuf.flags = SKBFL_ZEROCOPY_FRAG;
884888
refcount_set(&ubuf->ubuf.refcnt, 1);
885889
msg.msg_control = &ctl;

‎include/linux/skbuff.h

Lines changed: 13 additions & 8 deletions
Original file line numberDiff line numberDiff line change
@@ -527,6 +527,13 @@ enum {
527527
#define SKBFL_ALL_ZEROCOPY (SKBFL_ZEROCOPY_FRAG | SKBFL_PURE_ZEROCOPY | \
528528
SKBFL_DONT_ORPHAN | SKBFL_MANAGED_FRAG_REFS)
529529

530+
struct ubuf_info_ops {
531+
void (*complete)(struct sk_buff *, struct ubuf_info *,
532+
bool zerocopy_success);
533+
/* has to be compatible with skb_zcopy_set() */
534+
int (*link_skb)(struct sk_buff *skb, struct ubuf_info *uarg);
535+
};
536+
530537
/*
531538
* The callback notifies userspace to release buffers when skb DMA is done in
532539
* lower device, the skb last reference should be 0 when calling this.
@@ -536,8 +543,7 @@ enum {
536543
* The desc field is used to track userspace buffer index.
537544
*/
538545
struct ubuf_info {
539-
void (*callback)(struct sk_buff *, struct ubuf_info *,
540-
bool zerocopy_success);
546+
const struct ubuf_info_ops *ops;
541547
refcount_t refcnt;
542548
u8 flags;
543549
};
@@ -1662,14 +1668,13 @@ static inline void skb_set_end_offset(struct sk_buff *skb, unsigned int offset)
16621668
}
16631669
#endif
16641670

1671+
extern const struct ubuf_info_ops msg_zerocopy_ubuf_ops;
1672+
16651673
struct ubuf_info *msg_zerocopy_realloc(struct sock *sk, size_t size,
16661674
struct ubuf_info *uarg);
16671675

16681676
void msg_zerocopy_put_abort(struct ubuf_info *uarg, bool have_uref);
16691677

1670-
void msg_zerocopy_callback(struct sk_buff *skb, struct ubuf_info *uarg,
1671-
bool success);
1672-
16731678
int __zerocopy_sg_from_iter(struct msghdr *msg, struct sock *sk,
16741679
struct sk_buff *skb, struct iov_iter *from,
16751680
size_t length);
@@ -1757,13 +1762,13 @@ static inline void *skb_zcopy_get_nouarg(struct sk_buff *skb)
17571762
static inline void net_zcopy_put(struct ubuf_info *uarg)
17581763
{
17591764
if (uarg)
1760-
uarg->callback(NULL, uarg, true);
1765+
uarg->ops->complete(NULL, uarg, true);
17611766
}
17621767

17631768
static inline void net_zcopy_put_abort(struct ubuf_info *uarg, bool have_uref)
17641769
{
17651770
if (uarg) {
1766-
if (uarg->callback == msg_zerocopy_callback)
1771+
if (uarg->ops == &msg_zerocopy_ubuf_ops)
17671772
msg_zerocopy_put_abort(uarg, have_uref);
17681773
else if (have_uref)
17691774
net_zcopy_put(uarg);
@@ -1777,7 +1782,7 @@ static inline void skb_zcopy_clear(struct sk_buff *skb, bool zerocopy_success)
17771782

17781783
if (uarg) {
17791784
if (!skb_zcopy_is_nouarg(skb))
1780-
uarg->callback(skb, uarg, zerocopy_success);
1785+
uarg->ops->complete(skb, uarg, zerocopy_success);
17811786

17821787
skb_shinfo(skb)->flags &= ~SKBFL_ALL_ZEROCOPY;
17831788
}

‎io_uring/notif.c

Lines changed: 6 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -23,7 +23,7 @@ void io_notif_tw_complete(struct io_kiocb *notif, struct io_tw_state *ts)
2323
io_req_task_complete(notif, ts);
2424
}
2525

26-
static void io_tx_ubuf_callback(struct sk_buff *skb, struct ubuf_info *uarg,
26+
static void io_tx_ubuf_complete(struct sk_buff *skb, struct ubuf_info *uarg,
2727
bool success)
2828
{
2929
struct io_notif_data *nd = container_of(uarg, struct io_notif_data, uarg);
@@ -43,6 +43,10 @@ static void io_tx_ubuf_callback(struct sk_buff *skb, struct ubuf_info *uarg,
4343
__io_req_task_work_add(notif, IOU_F_TWQ_LAZY_WAKE);
4444
}
4545

46+
static const struct ubuf_info_ops io_ubuf_ops = {
47+
.complete = io_tx_ubuf_complete,
48+
};
49+
4650
struct io_kiocb *io_alloc_notif(struct io_ring_ctx *ctx)
4751
__must_hold(&ctx->uring_lock)
4852
{
@@ -62,7 +66,7 @@ struct io_kiocb *io_alloc_notif(struct io_ring_ctx *ctx)
6266
nd->zc_report = false;
6367
nd->account_pages = 0;
6468
nd->uarg.flags = IO_NOTIF_UBUF_FLAGS;
65-
nd->uarg.callback = io_tx_ubuf_callback;
69+
nd->uarg.ops = &io_ubuf_ops;
6670
refcount_set(&nd->uarg.refcnt, 1);
6771
return notif;
6872
}

‎net/core/skbuff.c

Lines changed: 24 additions & 12 deletions
Original file line numberDiff line numberDiff line change
@@ -1708,7 +1708,7 @@ static struct ubuf_info *msg_zerocopy_alloc(struct sock *sk, size_t size)
17081708
return NULL;
17091709
}
17101710

1711-
uarg->ubuf.callback = msg_zerocopy_callback;
1711+
uarg->ubuf.ops = &msg_zerocopy_ubuf_ops;
17121712
uarg->id = ((u32)atomic_inc_return(&sk->sk_zckey)) - 1;
17131713
uarg->len = 1;
17141714
uarg->bytelen = size;
@@ -1734,7 +1734,7 @@ struct ubuf_info *msg_zerocopy_realloc(struct sock *sk, size_t size,
17341734
u32 bytelen, next;
17351735

17361736
/* there might be non MSG_ZEROCOPY users */
1737-
if (uarg->callback != msg_zerocopy_callback)
1737+
if (uarg->ops != &msg_zerocopy_ubuf_ops)
17381738
return NULL;
17391739

17401740
/* realloc only when socket is locked (TCP, UDP cork),
@@ -1845,8 +1845,8 @@ static void __msg_zerocopy_callback(struct ubuf_info_msgzc *uarg)
18451845
sock_put(sk);
18461846
}
18471847

1848-
void msg_zerocopy_callback(struct sk_buff *skb, struct ubuf_info *uarg,
1849-
bool success)
1848+
static void msg_zerocopy_complete(struct sk_buff *skb, struct ubuf_info *uarg,
1849+
bool success)
18501850
{
18511851
struct ubuf_info_msgzc *uarg_zc = uarg_to_msgzc(uarg);
18521852

@@ -1855,7 +1855,6 @@ void msg_zerocopy_callback(struct sk_buff *skb, struct ubuf_info *uarg,
18551855
if (refcount_dec_and_test(&uarg->refcnt))
18561856
__msg_zerocopy_callback(uarg_zc);
18571857
}
1858-
EXPORT_SYMBOL_GPL(msg_zerocopy_callback);
18591858

18601859
void msg_zerocopy_put_abort(struct ubuf_info *uarg, bool have_uref)
18611860
{
@@ -1865,22 +1864,34 @@ void msg_zerocopy_put_abort(struct ubuf_info *uarg, bool have_uref)
18651864
uarg_to_msgzc(uarg)->len--;
18661865

18671866
if (have_uref)
1868-
msg_zerocopy_callback(NULL, uarg, true);
1867+
msg_zerocopy_complete(NULL, uarg, true);
18691868
}
18701869
EXPORT_SYMBOL_GPL(msg_zerocopy_put_abort);
18711870

1871+
const struct ubuf_info_ops msg_zerocopy_ubuf_ops = {
1872+
.complete = msg_zerocopy_complete,
1873+
};
1874+
EXPORT_SYMBOL_GPL(msg_zerocopy_ubuf_ops);
1875+
18721876
int skb_zerocopy_iter_stream(struct sock *sk, struct sk_buff *skb,
18731877
struct msghdr *msg, int len,
18741878
struct ubuf_info *uarg)
18751879
{
18761880
struct ubuf_info *orig_uarg = skb_zcopy(skb);
18771881
int err, orig_len = skb->len;
18781882

1879-
/* An skb can only point to one uarg. This edge case happens when
1880-
* TCP appends to an skb, but zerocopy_realloc triggered a new alloc.
1881-
*/
1882-
if (orig_uarg && uarg != orig_uarg)
1883-
return -EEXIST;
1883+
if (uarg->ops->link_skb) {
1884+
err = uarg->ops->link_skb(skb, uarg);
1885+
if (err)
1886+
return err;
1887+
} else {
1888+
/* An skb can only point to one uarg. This edge case happens
1889+
* when TCP appends to an skb, but zerocopy_realloc triggered
1890+
* a new alloc.
1891+
*/
1892+
if (orig_uarg && uarg != orig_uarg)
1893+
return -EEXIST;
1894+
}
18841895

18851896
err = __zerocopy_sg_from_iter(msg, sk, skb, &msg->msg_iter, len);
18861897
if (err == -EFAULT || (err == -EMSGSIZE && skb->len == orig_len)) {
@@ -1894,7 +1905,8 @@ int skb_zerocopy_iter_stream(struct sock *sk, struct sk_buff *skb,
18941905
return err;
18951906
}
18961907

1897-
skb_zcopy_set(skb, uarg, NULL);
1908+
if (!uarg->ops->link_skb)
1909+
skb_zcopy_set(skb, uarg, NULL);
18981910
return skb->len - orig_len;
18991911
}
19001912
EXPORT_SYMBOL_GPL(skb_zerocopy_iter_stream);

0 commit comments

Comments
 (0)