Skip to content

Commit bcf8a02

Browse files
calebsanderaxboe
calebsander
authored and
axboe
committed
io_uring: introduce type alias for io_tw_state
In preparation for changing how io_tw_state is passed, introduce a type alias io_tw_token_t for struct io_tw_state *. This allows for changing the representation in one place, without having to update the many functions that just forward their struct io_tw_state * argument. Also add a comment to struct io_tw_state to explain its purpose. Signed-off-by: Caleb Sander Mateos <[email protected]> Link: https://lore.kernel.org/r/[email protected] Signed-off-by: Jens Axboe <[email protected]>
1 parent 496f56b commit bcf8a02

File tree

13 files changed

+66
-56
lines changed

13 files changed

+66
-56
lines changed

‎include/linux/io_uring_types.h‎

Lines changed: 8 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -436,8 +436,15 @@ struct io_ring_ctx {
436436
struct io_mapped_region param_region;
437437
};
438438

439+
/*
440+
* Token indicating function is called in task work context:
441+
* ctx->uring_lock is held and any completions generated will be flushed.
442+
* ONLY core io_uring.c should instantiate this struct.
443+
*/
439444
struct io_tw_state {
440445
};
446+
/* Alias to use in code that doesn't instantiate struct io_tw_state */
447+
typedef struct io_tw_state *io_tw_token_t;
441448

442449
enum {
443450
REQ_F_FIXED_FILE_BIT = IOSQE_FIXED_FILE_BIT,
@@ -563,7 +570,7 @@ enum {
563570
REQ_F_HAS_METADATA = IO_REQ_FLAG(REQ_F_HAS_METADATA_BIT),
564571
};
565572

566-
typedef void (*io_req_tw_func_t)(struct io_kiocb *req, struct io_tw_state *ts);
573+
typedef void (*io_req_tw_func_t)(struct io_kiocb *req, io_tw_token_t tw);
567574

568575
struct io_task_work {
569576
struct llist_node node;

‎io_uring/futex.c‎

Lines changed: 8 additions & 8 deletions
Original file line numberDiff line numberDiff line change
@@ -44,30 +44,30 @@ void io_futex_cache_free(struct io_ring_ctx *ctx)
4444
io_alloc_cache_free(&ctx->futex_cache, kfree);
4545
}
4646

47-
static void __io_futex_complete(struct io_kiocb *req, struct io_tw_state *ts)
47+
static void __io_futex_complete(struct io_kiocb *req, io_tw_token_t tw)
4848
{
4949
req->async_data = NULL;
5050
hlist_del_init(&req->hash_node);
51-
io_req_task_complete(req, ts);
51+
io_req_task_complete(req, tw);
5252
}
5353

54-
static void io_futex_complete(struct io_kiocb *req, struct io_tw_state *ts)
54+
static void io_futex_complete(struct io_kiocb *req, io_tw_token_t tw)
5555
{
5656
struct io_futex_data *ifd = req->async_data;
5757
struct io_ring_ctx *ctx = req->ctx;
5858

59-
io_tw_lock(ctx, ts);
59+
io_tw_lock(ctx, tw);
6060
if (!io_alloc_cache_put(&ctx->futex_cache, ifd))
6161
kfree(ifd);
62-
__io_futex_complete(req, ts);
62+
__io_futex_complete(req, tw);
6363
}
6464

65-
static void io_futexv_complete(struct io_kiocb *req, struct io_tw_state *ts)
65+
static void io_futexv_complete(struct io_kiocb *req, io_tw_token_t tw)
6666
{
6767
struct io_futex *iof = io_kiocb_to_cmd(req, struct io_futex);
6868
struct futex_vector *futexv = req->async_data;
6969

70-
io_tw_lock(req->ctx, ts);
70+
io_tw_lock(req->ctx, tw);
7171

7272
if (!iof->futexv_unqueued) {
7373
int res;
@@ -79,7 +79,7 @@ static void io_futexv_complete(struct io_kiocb *req, struct io_tw_state *ts)
7979

8080
kfree(req->async_data);
8181
req->flags &= ~REQ_F_ASYNC_DATA;
82-
__io_futex_complete(req, ts);
82+
__io_futex_complete(req, tw);
8383
}
8484

8585
static bool io_futexv_claim(struct io_futex *iof)

‎io_uring/io_uring.c‎

Lines changed: 14 additions & 14 deletions
Original file line numberDiff line numberDiff line change
@@ -543,7 +543,7 @@ static void io_queue_iowq(struct io_kiocb *req)
543543
io_queue_linked_timeout(link);
544544
}
545545

546-
static void io_req_queue_iowq_tw(struct io_kiocb *req, struct io_tw_state *ts)
546+
static void io_req_queue_iowq_tw(struct io_kiocb *req, io_tw_token_t tw)
547547
{
548548
io_queue_iowq(req);
549549
}
@@ -1022,7 +1022,7 @@ static inline struct io_kiocb *io_req_find_next(struct io_kiocb *req)
10221022
return nxt;
10231023
}
10241024

1025-
static void ctx_flush_and_put(struct io_ring_ctx *ctx, struct io_tw_state *ts)
1025+
static void ctx_flush_and_put(struct io_ring_ctx *ctx, io_tw_token_t tw)
10261026
{
10271027
if (!ctx)
10281028
return;
@@ -1277,7 +1277,7 @@ static bool io_run_local_work_continue(struct io_ring_ctx *ctx, int events,
12771277
}
12781278

12791279
static int __io_run_local_work_loop(struct llist_node **node,
1280-
struct io_tw_state *ts,
1280+
io_tw_token_t tw,
12811281
int events)
12821282
{
12831283
int ret = 0;
@@ -1288,7 +1288,7 @@ static int __io_run_local_work_loop(struct llist_node **node,
12881288
io_task_work.node);
12891289
INDIRECT_CALL_2(req->io_task_work.func,
12901290
io_poll_task_func, io_req_rw_complete,
1291-
req, ts);
1291+
req, tw);
12921292
*node = next;
12931293
if (++ret >= events)
12941294
break;
@@ -1297,7 +1297,7 @@ static int __io_run_local_work_loop(struct llist_node **node,
12971297
return ret;
12981298
}
12991299

1300-
static int __io_run_local_work(struct io_ring_ctx *ctx, struct io_tw_state *ts,
1300+
static int __io_run_local_work(struct io_ring_ctx *ctx, io_tw_token_t tw,
13011301
int min_events, int max_events)
13021302
{
13031303
struct llist_node *node;
@@ -1310,7 +1310,7 @@ static int __io_run_local_work(struct io_ring_ctx *ctx, struct io_tw_state *ts,
13101310
atomic_andnot(IORING_SQ_TASKRUN, &ctx->rings->sq_flags);
13111311
again:
13121312
min_events -= ret;
1313-
ret = __io_run_local_work_loop(&ctx->retry_llist.first, ts, max_events);
1313+
ret = __io_run_local_work_loop(&ctx->retry_llist.first, tw, max_events);
13141314
if (ctx->retry_llist.first)
13151315
goto retry_done;
13161316

@@ -1319,7 +1319,7 @@ static int __io_run_local_work(struct io_ring_ctx *ctx, struct io_tw_state *ts,
13191319
* running the pending items.
13201320
*/
13211321
node = llist_reverse_order(llist_del_all(&ctx->work_llist));
1322-
ret += __io_run_local_work_loop(&node, ts, max_events - ret);
1322+
ret += __io_run_local_work_loop(&node, tw, max_events - ret);
13231323
ctx->retry_llist.first = node;
13241324
loops++;
13251325

@@ -1357,15 +1357,15 @@ static int io_run_local_work(struct io_ring_ctx *ctx, int min_events,
13571357
return ret;
13581358
}
13591359

1360-
static void io_req_task_cancel(struct io_kiocb *req, struct io_tw_state *ts)
1360+
static void io_req_task_cancel(struct io_kiocb *req, io_tw_token_t tw)
13611361
{
1362-
io_tw_lock(req->ctx, ts);
1362+
io_tw_lock(req->ctx, tw);
13631363
io_req_defer_failed(req, req->cqe.res);
13641364
}
13651365

1366-
void io_req_task_submit(struct io_kiocb *req, struct io_tw_state *ts)
1366+
void io_req_task_submit(struct io_kiocb *req, io_tw_token_t tw)
13671367
{
1368-
io_tw_lock(req->ctx, ts);
1368+
io_tw_lock(req->ctx, tw);
13691369
if (unlikely(io_should_terminate_tw()))
13701370
io_req_defer_failed(req, -EFAULT);
13711371
else if (req->flags & REQ_F_FORCE_ASYNC)
@@ -1583,7 +1583,7 @@ static int io_iopoll_check(struct io_ring_ctx *ctx, long min)
15831583
return 0;
15841584
}
15851585

1586-
void io_req_task_complete(struct io_kiocb *req, struct io_tw_state *ts)
1586+
void io_req_task_complete(struct io_kiocb *req, io_tw_token_t tw)
15871587
{
15881588
io_req_complete_defer(req);
15891589
}
@@ -1763,9 +1763,9 @@ static int io_issue_sqe(struct io_kiocb *req, unsigned int issue_flags)
17631763
return ret;
17641764
}
17651765

1766-
int io_poll_issue(struct io_kiocb *req, struct io_tw_state *ts)
1766+
int io_poll_issue(struct io_kiocb *req, io_tw_token_t tw)
17671767
{
1768-
io_tw_lock(req->ctx, ts);
1768+
io_tw_lock(req->ctx, tw);
17691769
return io_issue_sqe(req, IO_URING_F_NONBLOCK|IO_URING_F_MULTISHOT|
17701770
IO_URING_F_COMPLETE_DEFER);
17711771
}

‎io_uring/io_uring.h‎

Lines changed: 4 additions & 4 deletions
Original file line numberDiff line numberDiff line change
@@ -90,9 +90,9 @@ void io_req_task_work_add_remote(struct io_kiocb *req, struct io_ring_ctx *ctx,
9090
unsigned flags);
9191
bool io_alloc_async_data(struct io_kiocb *req);
9292
void io_req_task_queue(struct io_kiocb *req);
93-
void io_req_task_complete(struct io_kiocb *req, struct io_tw_state *ts);
93+
void io_req_task_complete(struct io_kiocb *req, io_tw_token_t tw);
9494
void io_req_task_queue_fail(struct io_kiocb *req, int ret);
95-
void io_req_task_submit(struct io_kiocb *req, struct io_tw_state *ts);
95+
void io_req_task_submit(struct io_kiocb *req, io_tw_token_t tw);
9696
struct llist_node *io_handle_tw_list(struct llist_node *node, unsigned int *count, unsigned int max_entries);
9797
struct llist_node *tctx_task_work_run(struct io_uring_task *tctx, unsigned int max_entries, unsigned int *count);
9898
void tctx_task_work(struct callback_head *cb);
@@ -104,7 +104,7 @@ int io_ring_add_registered_file(struct io_uring_task *tctx, struct file *file,
104104
int start, int end);
105105
void io_req_queue_iowq(struct io_kiocb *req);
106106

107-
int io_poll_issue(struct io_kiocb *req, struct io_tw_state *ts);
107+
int io_poll_issue(struct io_kiocb *req, io_tw_token_t tw);
108108
int io_submit_sqes(struct io_ring_ctx *ctx, unsigned int nr);
109109
int io_do_iopoll(struct io_ring_ctx *ctx, bool force_nonspin);
110110
void __io_submit_flush_completions(struct io_ring_ctx *ctx);
@@ -376,7 +376,7 @@ static inline bool io_task_work_pending(struct io_ring_ctx *ctx)
376376
return task_work_pending(current) || io_local_work_pending(ctx);
377377
}
378378

379-
static inline void io_tw_lock(struct io_ring_ctx *ctx, struct io_tw_state *ts)
379+
static inline void io_tw_lock(struct io_ring_ctx *ctx, io_tw_token_t tw)
380380
{
381381
lockdep_assert_held(&ctx->uring_lock);
382382
}

‎io_uring/msg_ring.c‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -71,7 +71,7 @@ static inline bool io_msg_need_remote(struct io_ring_ctx *target_ctx)
7171
return target_ctx->task_complete;
7272
}
7373

74-
static void io_msg_tw_complete(struct io_kiocb *req, struct io_tw_state *ts)
74+
static void io_msg_tw_complete(struct io_kiocb *req, io_tw_token_t tw)
7575
{
7676
struct io_ring_ctx *ctx = req->ctx;
7777

‎io_uring/notif.c‎

Lines changed: 2 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -11,7 +11,7 @@
1111

1212
static const struct ubuf_info_ops io_ubuf_ops;
1313

14-
static void io_notif_tw_complete(struct io_kiocb *notif, struct io_tw_state *ts)
14+
static void io_notif_tw_complete(struct io_kiocb *notif, io_tw_token_t tw)
1515
{
1616
struct io_notif_data *nd = io_notif_to_data(notif);
1717

@@ -29,7 +29,7 @@ static void io_notif_tw_complete(struct io_kiocb *notif, struct io_tw_state *ts)
2929
}
3030

3131
nd = nd->next;
32-
io_req_task_complete(notif, ts);
32+
io_req_task_complete(notif, tw);
3333
} while (nd);
3434
}
3535

‎io_uring/poll.c‎

Lines changed: 9 additions & 9 deletions
Original file line numberDiff line numberDiff line change
@@ -220,7 +220,7 @@ static inline void io_poll_execute(struct io_kiocb *req, int res)
220220
* req->cqe.res. IOU_POLL_REMOVE_POLL_USE_RES indicates to remove multishot
221221
* poll and that the result is stored in req->cqe.
222222
*/
223-
static int io_poll_check_events(struct io_kiocb *req, struct io_tw_state *ts)
223+
static int io_poll_check_events(struct io_kiocb *req, io_tw_token_t tw)
224224
{
225225
int v;
226226

@@ -288,7 +288,7 @@ static int io_poll_check_events(struct io_kiocb *req, struct io_tw_state *ts)
288288
return IOU_POLL_REMOVE_POLL_USE_RES;
289289
}
290290
} else {
291-
int ret = io_poll_issue(req, ts);
291+
int ret = io_poll_issue(req, tw);
292292
if (ret == IOU_STOP_MULTISHOT)
293293
return IOU_POLL_REMOVE_POLL_USE_RES;
294294
else if (ret == IOU_REQUEUE)
@@ -311,11 +311,11 @@ static int io_poll_check_events(struct io_kiocb *req, struct io_tw_state *ts)
311311
return IOU_POLL_NO_ACTION;
312312
}
313313

314-
void io_poll_task_func(struct io_kiocb *req, struct io_tw_state *ts)
314+
void io_poll_task_func(struct io_kiocb *req, io_tw_token_t tw)
315315
{
316316
int ret;
317317

318-
ret = io_poll_check_events(req, ts);
318+
ret = io_poll_check_events(req, tw);
319319
if (ret == IOU_POLL_NO_ACTION) {
320320
io_kbuf_recycle(req, 0);
321321
return;
@@ -335,22 +335,22 @@ void io_poll_task_func(struct io_kiocb *req, struct io_tw_state *ts)
335335
poll = io_kiocb_to_cmd(req, struct io_poll);
336336
req->cqe.res = mangle_poll(req->cqe.res & poll->events);
337337
} else if (ret == IOU_POLL_REISSUE) {
338-
io_req_task_submit(req, ts);
338+
io_req_task_submit(req, tw);
339339
return;
340340
} else if (ret != IOU_POLL_REMOVE_POLL_USE_RES) {
341341
req->cqe.res = ret;
342342
req_set_fail(req);
343343
}
344344

345345
io_req_set_res(req, req->cqe.res, 0);
346-
io_req_task_complete(req, ts);
346+
io_req_task_complete(req, tw);
347347
} else {
348-
io_tw_lock(req->ctx, ts);
348+
io_tw_lock(req->ctx, tw);
349349

350350
if (ret == IOU_POLL_REMOVE_POLL_USE_RES)
351-
io_req_task_complete(req, ts);
351+
io_req_task_complete(req, tw);
352352
else if (ret == IOU_POLL_DONE || ret == IOU_POLL_REISSUE)
353-
io_req_task_submit(req, ts);
353+
io_req_task_submit(req, tw);
354354
else
355355
io_req_defer_failed(req, ret);
356356
}

‎io_uring/poll.h‎

Lines changed: 3 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -1,5 +1,7 @@
11
// SPDX-License-Identifier: GPL-2.0
22

3+
#include <linux/io_uring_types.h>
4+
35
#define IO_POLL_ALLOC_CACHE_MAX 32
46

57
enum {
@@ -43,4 +45,4 @@ int io_arm_poll_handler(struct io_kiocb *req, unsigned issue_flags);
4345
bool io_poll_remove_all(struct io_ring_ctx *ctx, struct io_uring_task *tctx,
4446
bool cancel_all);
4547

46-
void io_poll_task_func(struct io_kiocb *req, struct io_tw_state *ts);
48+
void io_poll_task_func(struct io_kiocb *req, io_tw_token_t tw);

‎io_uring/rw.c‎

Lines changed: 2 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -511,7 +511,7 @@ static inline int io_fixup_rw_res(struct io_kiocb *req, long res)
511511
return res;
512512
}
513513

514-
void io_req_rw_complete(struct io_kiocb *req, struct io_tw_state *ts)
514+
void io_req_rw_complete(struct io_kiocb *req, io_tw_token_t tw)
515515
{
516516
struct io_rw *rw = io_kiocb_to_cmd(req, struct io_rw);
517517
struct kiocb *kiocb = &rw->kiocb;
@@ -528,7 +528,7 @@ void io_req_rw_complete(struct io_kiocb *req, struct io_tw_state *ts)
528528
req->cqe.flags |= io_put_kbuf(req, req->cqe.res, 0);
529529

530530
io_req_rw_cleanup(req, 0);
531-
io_req_task_complete(req, ts);
531+
io_req_task_complete(req, tw);
532532
}
533533

534534
static void io_complete_rw(struct kiocb *kiocb, long res)

‎io_uring/rw.h‎

Lines changed: 2 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -1,5 +1,6 @@
11
// SPDX-License-Identifier: GPL-2.0
22

3+
#include <linux/io_uring_types.h>
34
#include <linux/pagemap.h>
45

56
struct io_meta_state {
@@ -39,7 +40,7 @@ int io_read(struct io_kiocb *req, unsigned int issue_flags);
3940
int io_write(struct io_kiocb *req, unsigned int issue_flags);
4041
void io_readv_writev_cleanup(struct io_kiocb *req);
4142
void io_rw_fail(struct io_kiocb *req);
42-
void io_req_rw_complete(struct io_kiocb *req, struct io_tw_state *ts);
43+
void io_req_rw_complete(struct io_kiocb *req, io_tw_token_t tw);
4344
int io_read_mshot_prep(struct io_kiocb *req, const struct io_uring_sqe *sqe);
4445
int io_read_mshot(struct io_kiocb *req, unsigned int issue_flags);
4546
void io_rw_cache_free(const void *entry);

0 commit comments

Comments
 (0)