Skip to content

Commit 8ccebc1

Browse files
Ming Leiaxboe
Ming Lei
authored and
axboe
committed
selftests: ublk: support UBLK_F_AUTO_BUF_REG
Enable UBLK_F_AUTO_BUF_REG support for ublk utility by argument `--auto_zc`, meantime support this feature in null, loop and stripe target code. Add function test generic_08 for covering basic UBLK_F_AUTO_BUF_REG feature. Also cover UBLK_F_AUTO_BUF_REG in stress_03, stress_04 and stress_05 test too. 'fio/t/io_uring -p0 /dev/ublkb0' shows that F_AUTO_BUF_REG can improve IOPS by 50% compared with F_SUPPORT_ZERO_COPY in my test VM. Signed-off-by: Ming Lei <[email protected]> Link: https://lore.kernel.org/r/[email protected] Signed-off-by: Jens Axboe <[email protected]>
1 parent 53f427e commit 8ccebc1

File tree

10 files changed

+138
-29
lines changed

10 files changed

+138
-29
lines changed

‎tools/testing/selftests/ublk/Makefile

Lines changed: 2 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -15,6 +15,8 @@ TEST_PROGS += test_generic_05.sh
1515
TEST_PROGS += test_generic_06.sh
1616
TEST_PROGS += test_generic_07.sh
1717

18+
TEST_PROGS += test_generic_08.sh
19+
1820
TEST_PROGS += test_null_01.sh
1921
TEST_PROGS += test_null_02.sh
2022
TEST_PROGS += test_loop_01.sh

‎tools/testing/selftests/ublk/file_backed.c

Lines changed: 8 additions & 4 deletions
Original file line numberDiff line numberDiff line change
@@ -29,19 +29,23 @@ static int loop_queue_flush_io(struct ublk_queue *q, const struct ublksrv_io_des
2929
static int loop_queue_tgt_rw_io(struct ublk_queue *q, const struct ublksrv_io_desc *iod, int tag)
3030
{
3131
unsigned ublk_op = ublksrv_get_op(iod);
32-
int zc = ublk_queue_use_zc(q);
33-
enum io_uring_op op = ublk_to_uring_op(iod, zc);
32+
unsigned zc = ublk_queue_use_zc(q);
33+
unsigned auto_zc = ublk_queue_use_auto_zc(q);
34+
enum io_uring_op op = ublk_to_uring_op(iod, zc | auto_zc);
3435
struct io_uring_sqe *sqe[3];
36+
void *addr = (zc | auto_zc) ? NULL : (void *)iod->addr;
3537

36-
if (!zc) {
38+
if (!zc || auto_zc) {
3739
ublk_queue_alloc_sqes(q, sqe, 1);
3840
if (!sqe[0])
3941
return -ENOMEM;
4042

4143
io_uring_prep_rw(op, sqe[0], 1 /*fds[1]*/,
42-
(void *)iod->addr,
44+
addr,
4345
iod->nr_sectors << 9,
4446
iod->start_sector << 9);
47+
if (auto_zc)
48+
sqe[0]->buf_index = tag;
4549
io_uring_sqe_set_flags(sqe[0], IOSQE_FIXED_FILE);
4650
/* bit63 marks us as tgt io */
4751
sqe[0]->user_data = build_user_data(tag, ublk_op, 0, 1);

‎tools/testing/selftests/ublk/kublk.c

Lines changed: 26 additions & 4 deletions
Original file line numberDiff line numberDiff line change
@@ -420,9 +420,12 @@ static int ublk_queue_init(struct ublk_queue *q)
420420
q->cmd_inflight = 0;
421421
q->tid = gettid();
422422

423-
if (dev->dev_info.flags & UBLK_F_SUPPORT_ZERO_COPY) {
423+
if (dev->dev_info.flags & (UBLK_F_SUPPORT_ZERO_COPY | UBLK_F_AUTO_BUF_REG)) {
424424
q->state |= UBLKSRV_NO_BUF;
425-
q->state |= UBLKSRV_ZC;
425+
if (dev->dev_info.flags & UBLK_F_SUPPORT_ZERO_COPY)
426+
q->state |= UBLKSRV_ZC;
427+
if (dev->dev_info.flags & UBLK_F_AUTO_BUF_REG)
428+
q->state |= UBLKSRV_AUTO_BUF_REG;
426429
}
427430

428431
cmd_buf_size = ublk_queue_cmd_buf_sz(q);
@@ -461,7 +464,7 @@ static int ublk_queue_init(struct ublk_queue *q)
461464
goto fail;
462465
}
463466

464-
if (dev->dev_info.flags & UBLK_F_SUPPORT_ZERO_COPY) {
467+
if (dev->dev_info.flags & (UBLK_F_SUPPORT_ZERO_COPY | UBLK_F_AUTO_BUF_REG)) {
465468
ret = io_uring_register_buffers_sparse(&q->ring, q->q_depth);
466469
if (ret) {
467470
ublk_err("ublk dev %d queue %d register spare buffers failed %d",
@@ -525,6 +528,18 @@ static void ublk_dev_unprep(struct ublk_dev *dev)
525528
close(dev->fds[0]);
526529
}
527530

531+
static void ublk_set_auto_buf_reg(struct io_uring_sqe *sqe,
532+
unsigned short buf_idx,
533+
unsigned char flags)
534+
{
535+
struct ublk_auto_buf_reg buf = {
536+
.index = buf_idx,
537+
.flags = flags,
538+
};
539+
540+
sqe->addr = ublk_auto_buf_reg_to_sqe_addr(&buf);
541+
}
542+
528543
int ublk_queue_io_cmd(struct ublk_queue *q, struct ublk_io *io, unsigned tag)
529544
{
530545
struct ublksrv_io_cmd *cmd;
@@ -579,6 +594,9 @@ int ublk_queue_io_cmd(struct ublk_queue *q, struct ublk_io *io, unsigned tag)
579594
else
580595
cmd->addr = 0;
581596

597+
if (q->state & UBLKSRV_AUTO_BUF_REG)
598+
ublk_set_auto_buf_reg(sqe[0], tag, 0);
599+
582600
user_data = build_user_data(tag, _IOC_NR(cmd_op), 0, 0);
583601
io_uring_sqe_set_data64(sqe[0], user_data);
584602

@@ -1206,6 +1224,7 @@ static int cmd_dev_get_features(void)
12061224
[const_ilog2(UBLK_F_USER_COPY)] = "USER_COPY",
12071225
[const_ilog2(UBLK_F_ZONED)] = "ZONED",
12081226
[const_ilog2(UBLK_F_USER_RECOVERY_FAIL_IO)] = "RECOVERY_FAIL_IO",
1227+
[const_ilog2(UBLK_F_AUTO_BUF_REG)] = "AUTO_BUF_REG",
12091228
};
12101229
struct ublk_dev *dev;
12111230
__u64 features = 0;
@@ -1245,7 +1264,7 @@ static void __cmd_create_help(char *exe, bool recovery)
12451264

12461265
printf("%s %s -t [null|loop|stripe|fault_inject] [-q nr_queues] [-d depth] [-n dev_id]\n",
12471266
exe, recovery ? "recover" : "add");
1248-
printf("\t[--foreground] [--quiet] [-z] [--debug_mask mask] [-r 0|1 ] [-g]\n");
1267+
printf("\t[--foreground] [--quiet] [-z] [--auto_zc] [--debug_mask mask] [-r 0|1 ] [-g]\n");
12491268
printf("\t[-e 0|1 ] [-i 0|1]\n");
12501269
printf("\t[target options] [backfile1] [backfile2] ...\n");
12511270
printf("\tdefault: nr_queues=2(max 32), depth=128(max 1024), dev_id=-1(auto allocation)\n");
@@ -1300,6 +1319,7 @@ int main(int argc, char *argv[])
13001319
{ "recovery_fail_io", 1, NULL, 'e'},
13011320
{ "recovery_reissue", 1, NULL, 'i'},
13021321
{ "get_data", 1, NULL, 'g'},
1322+
{ "auto_zc", 0, NULL, 0},
13031323
{ 0, 0, 0, 0 }
13041324
};
13051325
const struct ublk_tgt_ops *ops = NULL;
@@ -1368,6 +1388,8 @@ int main(int argc, char *argv[])
13681388
ublk_dbg_mask = 0;
13691389
if (!strcmp(longopts[option_idx].name, "foreground"))
13701390
ctx.fg = 1;
1391+
if (!strcmp(longopts[option_idx].name, "auto_zc"))
1392+
ctx.flags |= UBLK_F_AUTO_BUF_REG;
13711393
break;
13721394
case '?':
13731395
/*

‎tools/testing/selftests/ublk/kublk.h

Lines changed: 7 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -115,6 +115,7 @@ struct ublk_io {
115115
#define UBLKSRV_NEED_COMMIT_RQ_COMP (1UL << 1)
116116
#define UBLKSRV_IO_FREE (1UL << 2)
117117
#define UBLKSRV_NEED_GET_DATA (1UL << 3)
118+
#define UBLKSRV_NEED_REG_BUF (1UL << 4)
118119
unsigned short flags;
119120
unsigned short refs; /* used by target code only */
120121

@@ -168,6 +169,7 @@ struct ublk_queue {
168169
#define UBLKSRV_QUEUE_IDLE (1U << 1)
169170
#define UBLKSRV_NO_BUF (1U << 2)
170171
#define UBLKSRV_ZC (1U << 3)
172+
#define UBLKSRV_AUTO_BUF_REG (1U << 4)
171173
unsigned state;
172174
pid_t tid;
173175
pthread_t thread;
@@ -387,6 +389,11 @@ static inline int ublk_queue_use_zc(const struct ublk_queue *q)
387389
return q->state & UBLKSRV_ZC;
388390
}
389391

392+
static inline int ublk_queue_use_auto_zc(const struct ublk_queue *q)
393+
{
394+
return q->state & UBLKSRV_AUTO_BUF_REG;
395+
}
396+
390397
extern const struct ublk_tgt_ops null_tgt_ops;
391398
extern const struct ublk_tgt_ops loop_tgt_ops;
392399
extern const struct ublk_tgt_ops stripe_tgt_ops;

‎tools/testing/selftests/ublk/null.c

Lines changed: 32 additions & 11 deletions
Original file line numberDiff line numberDiff line change
@@ -42,10 +42,22 @@ static int ublk_null_tgt_init(const struct dev_ctx *ctx, struct ublk_dev *dev)
4242
return 0;
4343
}
4444

45+
static void __setup_nop_io(int tag, const struct ublksrv_io_desc *iod,
46+
struct io_uring_sqe *sqe)
47+
{
48+
unsigned ublk_op = ublksrv_get_op(iod);
49+
50+
io_uring_prep_nop(sqe);
51+
sqe->buf_index = tag;
52+
sqe->flags |= IOSQE_FIXED_FILE;
53+
sqe->rw_flags = IORING_NOP_FIXED_BUFFER | IORING_NOP_INJECT_RESULT;
54+
sqe->len = iod->nr_sectors << 9; /* injected result */
55+
sqe->user_data = build_user_data(tag, ublk_op, 0, 1);
56+
}
57+
4558
static int null_queue_zc_io(struct ublk_queue *q, int tag)
4659
{
4760
const struct ublksrv_io_desc *iod = ublk_get_iod(q, tag);
48-
unsigned ublk_op = ublksrv_get_op(iod);
4961
struct io_uring_sqe *sqe[3];
5062

5163
ublk_queue_alloc_sqes(q, sqe, 3);
@@ -55,12 +67,8 @@ static int null_queue_zc_io(struct ublk_queue *q, int tag)
5567
ublk_cmd_op_nr(sqe[0]->cmd_op), 0, 1);
5668
sqe[0]->flags |= IOSQE_CQE_SKIP_SUCCESS | IOSQE_IO_HARDLINK;
5769

58-
io_uring_prep_nop(sqe[1]);
59-
sqe[1]->buf_index = tag;
60-
sqe[1]->flags |= IOSQE_FIXED_FILE | IOSQE_IO_HARDLINK;
61-
sqe[1]->rw_flags = IORING_NOP_FIXED_BUFFER | IORING_NOP_INJECT_RESULT;
62-
sqe[1]->len = iod->nr_sectors << 9; /* injected result */
63-
sqe[1]->user_data = build_user_data(tag, ublk_op, 0, 1);
70+
__setup_nop_io(tag, iod, sqe[1]);
71+
sqe[1]->flags |= IOSQE_IO_HARDLINK;
6472

6573
io_uring_prep_buf_unregister(sqe[2], 0, tag, q->q_id, tag);
6674
sqe[2]->user_data = build_user_data(tag, ublk_cmd_op_nr(sqe[2]->cmd_op), 0, 1);
@@ -69,6 +77,16 @@ static int null_queue_zc_io(struct ublk_queue *q, int tag)
6977
return 2;
7078
}
7179

80+
static int null_queue_auto_zc_io(struct ublk_queue *q, int tag)
81+
{
82+
const struct ublksrv_io_desc *iod = ublk_get_iod(q, tag);
83+
struct io_uring_sqe *sqe[1];
84+
85+
ublk_queue_alloc_sqes(q, sqe, 1);
86+
__setup_nop_io(tag, iod, sqe[0]);
87+
return 1;
88+
}
89+
7290
static void ublk_null_io_done(struct ublk_queue *q, int tag,
7391
const struct io_uring_cqe *cqe)
7492
{
@@ -94,15 +112,18 @@ static void ublk_null_io_done(struct ublk_queue *q, int tag,
94112
static int ublk_null_queue_io(struct ublk_queue *q, int tag)
95113
{
96114
const struct ublksrv_io_desc *iod = ublk_get_iod(q, tag);
97-
int zc = ublk_queue_use_zc(q);
115+
unsigned auto_zc = ublk_queue_use_auto_zc(q);
116+
unsigned zc = ublk_queue_use_zc(q);
98117
int queued;
99118

100-
if (!zc) {
119+
if (auto_zc)
120+
queued = null_queue_auto_zc_io(q, tag);
121+
else if (zc)
122+
queued = null_queue_zc_io(q, tag);
123+
else {
101124
ublk_complete_io(q, tag, iod->nr_sectors << 9);
102125
return 0;
103126
}
104-
105-
queued = null_queue_zc_io(q, tag);
106127
ublk_queued_tgt_io(q, tag, queued);
107128
return 0;
108129
}

‎tools/testing/selftests/ublk/stripe.c

Lines changed: 11 additions & 10 deletions
Original file line numberDiff line numberDiff line change
@@ -70,7 +70,7 @@ static void free_stripe_array(struct stripe_array *s)
7070
}
7171

7272
static void calculate_stripe_array(const struct stripe_conf *conf,
73-
const struct ublksrv_io_desc *iod, struct stripe_array *s)
73+
const struct ublksrv_io_desc *iod, struct stripe_array *s, void *base)
7474
{
7575
const unsigned shift = conf->shift - 9;
7676
const unsigned chunk_sects = 1 << shift;
@@ -102,7 +102,7 @@ static void calculate_stripe_array(const struct stripe_conf *conf,
102102
}
103103

104104
assert(this->nr_vec < this->cap);
105-
this->vec[this->nr_vec].iov_base = (void *)(iod->addr + done);
105+
this->vec[this->nr_vec].iov_base = (void *)(base + done);
106106
this->vec[this->nr_vec++].iov_len = nr_sects << 9;
107107

108108
start += nr_sects;
@@ -126,15 +126,17 @@ static inline enum io_uring_op stripe_to_uring_op(
126126
static int stripe_queue_tgt_rw_io(struct ublk_queue *q, const struct ublksrv_io_desc *iod, int tag)
127127
{
128128
const struct stripe_conf *conf = get_chunk_shift(q);
129-
int zc = !!(ublk_queue_use_zc(q) != 0);
130-
enum io_uring_op op = stripe_to_uring_op(iod, zc);
129+
unsigned auto_zc = (ublk_queue_use_auto_zc(q) != 0);
130+
unsigned zc = (ublk_queue_use_zc(q) != 0);
131+
enum io_uring_op op = stripe_to_uring_op(iod, zc | auto_zc);
131132
struct io_uring_sqe *sqe[NR_STRIPE];
132133
struct stripe_array *s = alloc_stripe_array(conf, iod);
133134
struct ublk_io *io = ublk_get_io(q, tag);
134135
int i, extra = zc ? 2 : 0;
136+
void *base = (zc | auto_zc) ? NULL : (void *)iod->addr;
135137

136138
io->private_data = s;
137-
calculate_stripe_array(conf, iod, s);
139+
calculate_stripe_array(conf, iod, s, base);
138140

139141
ublk_queue_alloc_sqes(q, sqe, s->nr + extra);
140142

@@ -153,12 +155,11 @@ static int stripe_queue_tgt_rw_io(struct ublk_queue *q, const struct ublksrv_io_
153155
(void *)t->vec,
154156
t->nr_vec,
155157
t->start << 9);
156-
if (zc) {
158+
io_uring_sqe_set_flags(sqe[i], IOSQE_FIXED_FILE);
159+
if (auto_zc || zc) {
157160
sqe[i]->buf_index = tag;
158-
io_uring_sqe_set_flags(sqe[i],
159-
IOSQE_FIXED_FILE | IOSQE_IO_HARDLINK);
160-
} else {
161-
io_uring_sqe_set_flags(sqe[i], IOSQE_FIXED_FILE);
161+
if (zc)
162+
sqe[i]->flags |= IOSQE_IO_HARDLINK;
162163
}
163164
/* bit63 marks us as tgt io */
164165
sqe[i]->user_data = build_user_data(tag, ublksrv_get_op(iod), i - zc, 1);

‎tools/testing/selftests/ublk/test_generic_08.sh

Lines changed: 32 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,32 @@
1+
#!/bin/bash
2+
# SPDX-License-Identifier: GPL-2.0
3+
4+
. "$(cd "$(dirname "$0")" && pwd)"/test_common.sh
5+
6+
TID="generic_08"
7+
ERR_CODE=0
8+
9+
if ! _have_feature "AUTO_BUF_REG"; then
10+
exit "$UBLK_SKIP_CODE"
11+
fi
12+
13+
_prep_test "generic" "test UBLK_F_AUTO_BUF_REG"
14+
15+
_create_backfile 0 256M
16+
_create_backfile 1 256M
17+
18+
dev_id=$(_add_ublk_dev -t loop -q 2 --auto_zc "${UBLK_BACKFILES[0]}")
19+
_check_add_dev $TID $?
20+
21+
if ! _mkfs_mount_test /dev/ublkb"${dev_id}"; then
22+
_cleanup_test "generic"
23+
_show_result $TID 255
24+
fi
25+
26+
dev_id=$(_add_ublk_dev -t stripe --auto_zc "${UBLK_BACKFILES[0]}" "${UBLK_BACKFILES[1]}")
27+
_check_add_dev $TID $?
28+
_mkfs_mount_test /dev/ublkb"${dev_id}"
29+
ERR_CODE=$?
30+
31+
_cleanup_test "generic"
32+
_show_result $TID $ERR_CODE

‎tools/testing/selftests/ublk/test_stress_03.sh

Lines changed: 6 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -32,6 +32,12 @@ _create_backfile 2 128M
3232
ublk_io_and_remove 8G -t null -q 4 -z &
3333
ublk_io_and_remove 256M -t loop -q 4 -z "${UBLK_BACKFILES[0]}" &
3434
ublk_io_and_remove 256M -t stripe -q 4 -z "${UBLK_BACKFILES[1]}" "${UBLK_BACKFILES[2]}" &
35+
36+
if _have_feature "AUTO_BUF_REG"; then
37+
ublk_io_and_remove 8G -t null -q 4 --auto_zc &
38+
ublk_io_and_remove 256M -t loop -q 4 --auto_zc "${UBLK_BACKFILES[0]}" &
39+
ublk_io_and_remove 256M -t stripe -q 4 --auto_zc "${UBLK_BACKFILES[1]}" "${UBLK_BACKFILES[2]}" &
40+
fi
3541
wait
3642

3743
_cleanup_test "stress"

‎tools/testing/selftests/ublk/test_stress_04.sh

Lines changed: 6 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -31,6 +31,12 @@ _create_backfile 2 128M
3131
ublk_io_and_kill_daemon 8G -t null -q 4 -z &
3232
ublk_io_and_kill_daemon 256M -t loop -q 4 -z "${UBLK_BACKFILES[0]}" &
3333
ublk_io_and_kill_daemon 256M -t stripe -q 4 -z "${UBLK_BACKFILES[1]}" "${UBLK_BACKFILES[2]}" &
34+
35+
if _have_feature "AUTO_BUF_REG"; then
36+
ublk_io_and_kill_daemon 8G -t null -q 4 --auto_zc &
37+
ublk_io_and_kill_daemon 256M -t loop -q 4 --auto_zc "${UBLK_BACKFILES[0]}" &
38+
ublk_io_and_kill_daemon 256M -t stripe -q 4 --auto_zc "${UBLK_BACKFILES[1]}" "${UBLK_BACKFILES[2]}" &
39+
fi
3440
wait
3541

3642
_cleanup_test "stress"

‎tools/testing/selftests/ublk/test_stress_05.sh

Lines changed: 8 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -60,5 +60,13 @@ if _have_feature "ZERO_COPY"; then
6060
done
6161
fi
6262

63+
if _have_feature "AUTO_BUF_REG"; then
64+
for reissue in $(seq 0 1); do
65+
ublk_io_and_remove 8G -t null -q 4 -g --auto_zc -r 1 -i "$reissue" &
66+
ublk_io_and_remove 256M -t loop -q 4 -g --auto_zc -r 1 -i "$reissue" "${UBLK_BACKFILES[1]}" &
67+
wait
68+
done
69+
fi
70+
6371
_cleanup_test "stress"
6472
_show_result $TID $ERR_CODE

0 commit comments

Comments
 (0)