Skip to content

Commit 7ab4f16

Browse files
isilencekuba-moo
isilence
authored and
kuba-moo
committed
net: extend ubuf_info callback to ops structure
We'll need to associate additional callbacks with ubuf_info, introduce a structure holding ubuf_info callbacks. Apart from a more smarter io_uring notification management introduced in next patches, it can be used to generalise msg_zerocopy_put_abort() and also store ->sg_from_iter, which is currently passed in struct msghdr. Reviewed-by: Jens Axboe <[email protected]> Reviewed-by: David Ahern <[email protected]> Signed-off-by: Pavel Begunkov <[email protected]> Reviewed-by: Willem de Bruijn <[email protected]> Link: https://lore.kernel.org/all/a62015541de49c0e2a8a0377a1d5d0a5aeb07016.1713369317.git.asml.silence@gmail.com Signed-off-by: Jakub Kicinski <[email protected]>
1 parent 0bbac3f commit 7ab4f16

File tree

9 files changed

+53
-30
lines changed

9 files changed

+53
-30
lines changed

‎drivers/net/tap.c

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -754,7 +754,7 @@ static ssize_t tap_get_user(struct tap_queue *q, void *msg_control,
754754
skb_zcopy_init(skb, msg_control);
755755
} else if (msg_control) {
756756
struct ubuf_info *uarg = msg_control;
757-
uarg->callback(NULL, uarg, false);
757+
uarg->ops->complete(NULL, uarg, false);
758758
}
759759

760760
dev_queue_xmit(skb);

‎drivers/net/tun.c

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -1906,7 +1906,7 @@ static ssize_t tun_get_user(struct tun_struct *tun, struct tun_file *tfile,
19061906
skb_zcopy_init(skb, msg_control);
19071907
} else if (msg_control) {
19081908
struct ubuf_info *uarg = msg_control;
1909-
uarg->callback(NULL, uarg, false);
1909+
uarg->ops->complete(NULL, uarg, false);
19101910
}
19111911

19121912
skb_reset_network_header(skb);

‎drivers/net/xen-netback/common.h

Lines changed: 2 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -390,9 +390,8 @@ bool xenvif_rx_queue_tail(struct xenvif_queue *queue, struct sk_buff *skb);
390390

391391
void xenvif_carrier_on(struct xenvif *vif);
392392

393-
/* Callback from stack when TX packet can be released */
394-
void xenvif_zerocopy_callback(struct sk_buff *skb, struct ubuf_info *ubuf,
395-
bool zerocopy_success);
393+
/* Callbacks from stack when TX packet can be released */
394+
extern const struct ubuf_info_ops xenvif_ubuf_ops;
396395

397396
static inline pending_ring_idx_t nr_pending_reqs(struct xenvif_queue *queue)
398397
{

‎drivers/net/xen-netback/interface.c

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -593,7 +593,7 @@ int xenvif_init_queue(struct xenvif_queue *queue)
593593

594594
for (i = 0; i < MAX_PENDING_REQS; i++) {
595595
queue->pending_tx_info[i].callback_struct = (struct ubuf_info_msgzc)
596-
{ { .callback = xenvif_zerocopy_callback },
596+
{ { .ops = &xenvif_ubuf_ops },
597597
{ { .ctx = NULL,
598598
.desc = i } } };
599599
queue->grant_tx_handle[i] = NETBACK_INVALID_HANDLE;

‎drivers/net/xen-netback/netback.c

Lines changed: 8 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -1156,7 +1156,7 @@ static int xenvif_handle_frag_list(struct xenvif_queue *queue, struct sk_buff *s
11561156
uarg = skb_shinfo(skb)->destructor_arg;
11571157
/* increase inflight counter to offset decrement in callback */
11581158
atomic_inc(&queue->inflight_packets);
1159-
uarg->callback(NULL, uarg, true);
1159+
uarg->ops->complete(NULL, uarg, true);
11601160
skb_shinfo(skb)->destructor_arg = NULL;
11611161

11621162
/* Fill the skb with the new (local) frags. */
@@ -1278,8 +1278,9 @@ static int xenvif_tx_submit(struct xenvif_queue *queue)
12781278
return work_done;
12791279
}
12801280

1281-
void xenvif_zerocopy_callback(struct sk_buff *skb, struct ubuf_info *ubuf_base,
1282-
bool zerocopy_success)
1281+
static void xenvif_zerocopy_callback(struct sk_buff *skb,
1282+
struct ubuf_info *ubuf_base,
1283+
bool zerocopy_success)
12831284
{
12841285
unsigned long flags;
12851286
pending_ring_idx_t index;
@@ -1312,6 +1313,10 @@ void xenvif_zerocopy_callback(struct sk_buff *skb, struct ubuf_info *ubuf_base,
13121313
xenvif_skb_zerocopy_complete(queue);
13131314
}
13141315

1316+
const struct ubuf_info_ops xenvif_ubuf_ops = {
1317+
.complete = xenvif_zerocopy_callback,
1318+
};
1319+
13151320
static inline void xenvif_tx_dealloc_action(struct xenvif_queue *queue)
13161321
{
13171322
struct gnttab_unmap_grant_ref *gop;

‎drivers/vhost/net.c

Lines changed: 6 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -380,7 +380,7 @@ static void vhost_zerocopy_signal_used(struct vhost_net *net,
380380
}
381381
}
382382

383-
static void vhost_zerocopy_callback(struct sk_buff *skb,
383+
static void vhost_zerocopy_complete(struct sk_buff *skb,
384384
struct ubuf_info *ubuf_base, bool success)
385385
{
386386
struct ubuf_info_msgzc *ubuf = uarg_to_msgzc(ubuf_base);
@@ -408,6 +408,10 @@ static void vhost_zerocopy_callback(struct sk_buff *skb,
408408
rcu_read_unlock_bh();
409409
}
410410

411+
static const struct ubuf_info_ops vhost_ubuf_ops = {
412+
.complete = vhost_zerocopy_complete,
413+
};
414+
411415
static inline unsigned long busy_clock(void)
412416
{
413417
return local_clock() >> 10;
@@ -879,7 +883,7 @@ static void handle_tx_zerocopy(struct vhost_net *net, struct socket *sock)
879883
vq->heads[nvq->upend_idx].len = VHOST_DMA_IN_PROGRESS;
880884
ubuf->ctx = nvq->ubufs;
881885
ubuf->desc = nvq->upend_idx;
882-
ubuf->ubuf.callback = vhost_zerocopy_callback;
886+
ubuf->ubuf.ops = &vhost_ubuf_ops;
883887
ubuf->ubuf.flags = SKBFL_ZEROCOPY_FRAG;
884888
refcount_set(&ubuf->ubuf.refcnt, 1);
885889
msg.msg_control = &ctl;

‎include/linux/skbuff.h

Lines changed: 11 additions & 8 deletions
Original file line numberDiff line numberDiff line change
@@ -527,6 +527,11 @@ enum {
527527
#define SKBFL_ALL_ZEROCOPY (SKBFL_ZEROCOPY_FRAG | SKBFL_PURE_ZEROCOPY | \
528528
SKBFL_DONT_ORPHAN | SKBFL_MANAGED_FRAG_REFS)
529529

530+
struct ubuf_info_ops {
531+
void (*complete)(struct sk_buff *, struct ubuf_info *,
532+
bool zerocopy_success);
533+
};
534+
530535
/*
531536
* The callback notifies userspace to release buffers when skb DMA is done in
532537
* lower device, the skb last reference should be 0 when calling this.
@@ -536,8 +541,7 @@ enum {
536541
* The desc field is used to track userspace buffer index.
537542
*/
538543
struct ubuf_info {
539-
void (*callback)(struct sk_buff *, struct ubuf_info *,
540-
bool zerocopy_success);
544+
const struct ubuf_info_ops *ops;
541545
refcount_t refcnt;
542546
u8 flags;
543547
};
@@ -1662,14 +1666,13 @@ static inline void skb_set_end_offset(struct sk_buff *skb, unsigned int offset)
16621666
}
16631667
#endif
16641668

1669+
extern const struct ubuf_info_ops msg_zerocopy_ubuf_ops;
1670+
16651671
struct ubuf_info *msg_zerocopy_realloc(struct sock *sk, size_t size,
16661672
struct ubuf_info *uarg);
16671673

16681674
void msg_zerocopy_put_abort(struct ubuf_info *uarg, bool have_uref);
16691675

1670-
void msg_zerocopy_callback(struct sk_buff *skb, struct ubuf_info *uarg,
1671-
bool success);
1672-
16731676
int __zerocopy_sg_from_iter(struct msghdr *msg, struct sock *sk,
16741677
struct sk_buff *skb, struct iov_iter *from,
16751678
size_t length);
@@ -1757,13 +1760,13 @@ static inline void *skb_zcopy_get_nouarg(struct sk_buff *skb)
17571760
static inline void net_zcopy_put(struct ubuf_info *uarg)
17581761
{
17591762
if (uarg)
1760-
uarg->callback(NULL, uarg, true);
1763+
uarg->ops->complete(NULL, uarg, true);
17611764
}
17621765

17631766
static inline void net_zcopy_put_abort(struct ubuf_info *uarg, bool have_uref)
17641767
{
17651768
if (uarg) {
1766-
if (uarg->callback == msg_zerocopy_callback)
1769+
if (uarg->ops == &msg_zerocopy_ubuf_ops)
17671770
msg_zerocopy_put_abort(uarg, have_uref);
17681771
else if (have_uref)
17691772
net_zcopy_put(uarg);
@@ -1777,7 +1780,7 @@ static inline void skb_zcopy_clear(struct sk_buff *skb, bool zerocopy_success)
17771780

17781781
if (uarg) {
17791782
if (!skb_zcopy_is_nouarg(skb))
1780-
uarg->callback(skb, uarg, zerocopy_success);
1783+
uarg->ops->complete(skb, uarg, zerocopy_success);
17811784

17821785
skb_shinfo(skb)->flags &= ~SKBFL_ALL_ZEROCOPY;
17831786
}

‎io_uring/notif.c

Lines changed: 13 additions & 5 deletions
Original file line numberDiff line numberDiff line change
@@ -24,7 +24,7 @@ static void io_notif_complete_tw_ext(struct io_kiocb *notif, struct io_tw_state
2424
io_req_task_complete(notif, ts);
2525
}
2626

27-
static void io_tx_ubuf_callback(struct sk_buff *skb, struct ubuf_info *uarg,
27+
static void io_tx_ubuf_complete(struct sk_buff *skb, struct ubuf_info *uarg,
2828
bool success)
2929
{
3030
struct io_notif_data *nd = container_of(uarg, struct io_notif_data, uarg);
@@ -45,19 +45,27 @@ static void io_tx_ubuf_callback_ext(struct sk_buff *skb, struct ubuf_info *uarg,
4545
else if (!success && !nd->zc_copied)
4646
WRITE_ONCE(nd->zc_copied, true);
4747
}
48-
io_tx_ubuf_callback(skb, uarg, success);
48+
io_tx_ubuf_complete(skb, uarg, success);
4949
}
5050

51+
static const struct ubuf_info_ops io_ubuf_ops = {
52+
.complete = io_tx_ubuf_complete,
53+
};
54+
55+
static const struct ubuf_info_ops io_ubuf_ops_ext = {
56+
.complete = io_tx_ubuf_callback_ext,
57+
};
58+
5159
void io_notif_set_extended(struct io_kiocb *notif)
5260
{
5361
struct io_notif_data *nd = io_notif_to_data(notif);
5462

55-
if (nd->uarg.callback != io_tx_ubuf_callback_ext) {
63+
if (nd->uarg.ops != &io_ubuf_ops_ext) {
5664
nd->account_pages = 0;
5765
nd->zc_report = false;
5866
nd->zc_used = false;
5967
nd->zc_copied = false;
60-
nd->uarg.callback = io_tx_ubuf_callback_ext;
68+
nd->uarg.ops = &io_ubuf_ops_ext;
6169
notif->io_task_work.func = io_notif_complete_tw_ext;
6270
}
6371
}
@@ -80,7 +88,7 @@ struct io_kiocb *io_alloc_notif(struct io_ring_ctx *ctx)
8088

8189
nd = io_notif_to_data(notif);
8290
nd->uarg.flags = IO_NOTIF_UBUF_FLAGS;
83-
nd->uarg.callback = io_tx_ubuf_callback;
91+
nd->uarg.ops = &io_ubuf_ops;
8492
refcount_set(&nd->uarg.refcnt, 1);
8593
return notif;
8694
}

‎net/core/skbuff.c

Lines changed: 10 additions & 6 deletions
Original file line numberDiff line numberDiff line change
@@ -1708,7 +1708,7 @@ static struct ubuf_info *msg_zerocopy_alloc(struct sock *sk, size_t size)
17081708
return NULL;
17091709
}
17101710

1711-
uarg->ubuf.callback = msg_zerocopy_callback;
1711+
uarg->ubuf.ops = &msg_zerocopy_ubuf_ops;
17121712
uarg->id = ((u32)atomic_inc_return(&sk->sk_zckey)) - 1;
17131713
uarg->len = 1;
17141714
uarg->bytelen = size;
@@ -1734,7 +1734,7 @@ struct ubuf_info *msg_zerocopy_realloc(struct sock *sk, size_t size,
17341734
u32 bytelen, next;
17351735

17361736
/* there might be non MSG_ZEROCOPY users */
1737-
if (uarg->callback != msg_zerocopy_callback)
1737+
if (uarg->ops != &msg_zerocopy_ubuf_ops)
17381738
return NULL;
17391739

17401740
/* realloc only when socket is locked (TCP, UDP cork),
@@ -1845,8 +1845,8 @@ static void __msg_zerocopy_callback(struct ubuf_info_msgzc *uarg)
18451845
sock_put(sk);
18461846
}
18471847

1848-
void msg_zerocopy_callback(struct sk_buff *skb, struct ubuf_info *uarg,
1849-
bool success)
1848+
static void msg_zerocopy_complete(struct sk_buff *skb, struct ubuf_info *uarg,
1849+
bool success)
18501850
{
18511851
struct ubuf_info_msgzc *uarg_zc = uarg_to_msgzc(uarg);
18521852

@@ -1855,7 +1855,6 @@ void msg_zerocopy_callback(struct sk_buff *skb, struct ubuf_info *uarg,
18551855
if (refcount_dec_and_test(&uarg->refcnt))
18561856
__msg_zerocopy_callback(uarg_zc);
18571857
}
1858-
EXPORT_SYMBOL_GPL(msg_zerocopy_callback);
18591858

18601859
void msg_zerocopy_put_abort(struct ubuf_info *uarg, bool have_uref)
18611860
{
@@ -1865,10 +1864,15 @@ void msg_zerocopy_put_abort(struct ubuf_info *uarg, bool have_uref)
18651864
uarg_to_msgzc(uarg)->len--;
18661865

18671866
if (have_uref)
1868-
msg_zerocopy_callback(NULL, uarg, true);
1867+
msg_zerocopy_complete(NULL, uarg, true);
18691868
}
18701869
EXPORT_SYMBOL_GPL(msg_zerocopy_put_abort);
18711870

1871+
const struct ubuf_info_ops msg_zerocopy_ubuf_ops = {
1872+
.complete = msg_zerocopy_complete,
1873+
};
1874+
EXPORT_SYMBOL_GPL(msg_zerocopy_ubuf_ops);
1875+
18721876
int skb_zerocopy_iter_stream(struct sock *sk, struct sk_buff *skb,
18731877
struct msghdr *msg, int len,
18741878
struct ubuf_info *uarg)

0 commit comments

Comments
 (0)