mirror of
git://git.kernel.org/pub/scm/linux/kernel/git/torvalds/linux.git
synced 2025-08-05 16:54:27 +00:00
selftests: ublk: support UBLK_F_AUTO_BUF_REG
Enable UBLK_F_AUTO_BUF_REG support for ublk utility by argument `--auto_zc`, meantime support this feature in null, loop and stripe target code. Add function test generic_08 for covering basic UBLK_F_AUTO_BUF_REG feature. Also cover UBLK_F_AUTO_BUF_REG in stress_03, stress_04 and stress_05 test too. 'fio/t/io_uring -p0 /dev/ublkb0' shows that F_AUTO_BUF_REG can improve IOPS by 50% compared with F_SUPPORT_ZERO_COPY in my test VM. Signed-off-by: Ming Lei <ming.lei@redhat.com> Link: https://lore.kernel.org/r/20250520045455.515691-6-ming.lei@redhat.com Signed-off-by: Jens Axboe <axboe@kernel.dk>
This commit is contained in:
Ming Lei
Jens Axboe
parent
53f427e794
commit
8ccebc19ee
10 changed files with 138 additions and 29 deletions
|
|
@ -15,6 +15,8 @@ TEST_PROGS += test_generic_05.sh
|
|||
TEST_PROGS += test_generic_06.sh
|
||||
TEST_PROGS += test_generic_07.sh
|
||||
|
||||
TEST_PROGS += test_generic_08.sh
|
||||
|
||||
TEST_PROGS += test_null_01.sh
|
||||
TEST_PROGS += test_null_02.sh
|
||||
TEST_PROGS += test_loop_01.sh
|
||||
|
|
|
|||
|
|
@ -29,19 +29,23 @@ static int loop_queue_flush_io(struct ublk_queue *q, const struct ublksrv_io_des
|
|||
static int loop_queue_tgt_rw_io(struct ublk_queue *q, const struct ublksrv_io_desc *iod, int tag)
|
||||
{
|
||||
unsigned ublk_op = ublksrv_get_op(iod);
|
||||
int zc = ublk_queue_use_zc(q);
|
||||
enum io_uring_op op = ublk_to_uring_op(iod, zc);
|
||||
unsigned zc = ublk_queue_use_zc(q);
|
||||
unsigned auto_zc = ublk_queue_use_auto_zc(q);
|
||||
enum io_uring_op op = ublk_to_uring_op(iod, zc | auto_zc);
|
||||
struct io_uring_sqe *sqe[3];
|
||||
void *addr = (zc | auto_zc) ? NULL : (void *)iod->addr;
|
||||
|
||||
if (!zc) {
|
||||
if (!zc || auto_zc) {
|
||||
ublk_queue_alloc_sqes(q, sqe, 1);
|
||||
if (!sqe[0])
|
||||
return -ENOMEM;
|
||||
|
||||
io_uring_prep_rw(op, sqe[0], 1 /*fds[1]*/,
|
||||
(void *)iod->addr,
|
||||
addr,
|
||||
iod->nr_sectors << 9,
|
||||
iod->start_sector << 9);
|
||||
if (auto_zc)
|
||||
sqe[0]->buf_index = tag;
|
||||
io_uring_sqe_set_flags(sqe[0], IOSQE_FIXED_FILE);
|
||||
/* bit63 marks us as tgt io */
|
||||
sqe[0]->user_data = build_user_data(tag, ublk_op, 0, 1);
|
||||
|
|
|
|||
|
|
@ -420,9 +420,12 @@ static int ublk_queue_init(struct ublk_queue *q)
|
|||
q->cmd_inflight = 0;
|
||||
q->tid = gettid();
|
||||
|
||||
if (dev->dev_info.flags & UBLK_F_SUPPORT_ZERO_COPY) {
|
||||
if (dev->dev_info.flags & (UBLK_F_SUPPORT_ZERO_COPY | UBLK_F_AUTO_BUF_REG)) {
|
||||
q->state |= UBLKSRV_NO_BUF;
|
||||
q->state |= UBLKSRV_ZC;
|
||||
if (dev->dev_info.flags & UBLK_F_SUPPORT_ZERO_COPY)
|
||||
q->state |= UBLKSRV_ZC;
|
||||
if (dev->dev_info.flags & UBLK_F_AUTO_BUF_REG)
|
||||
q->state |= UBLKSRV_AUTO_BUF_REG;
|
||||
}
|
||||
|
||||
cmd_buf_size = ublk_queue_cmd_buf_sz(q);
|
||||
|
|
@ -461,7 +464,7 @@ static int ublk_queue_init(struct ublk_queue *q)
|
|||
goto fail;
|
||||
}
|
||||
|
||||
if (dev->dev_info.flags & UBLK_F_SUPPORT_ZERO_COPY) {
|
||||
if (dev->dev_info.flags & (UBLK_F_SUPPORT_ZERO_COPY | UBLK_F_AUTO_BUF_REG)) {
|
||||
ret = io_uring_register_buffers_sparse(&q->ring, q->q_depth);
|
||||
if (ret) {
|
||||
ublk_err("ublk dev %d queue %d register spare buffers failed %d",
|
||||
|
|
@ -525,6 +528,18 @@ static void ublk_dev_unprep(struct ublk_dev *dev)
|
|||
close(dev->fds[0]);
|
||||
}
|
||||
|
||||
static void ublk_set_auto_buf_reg(struct io_uring_sqe *sqe,
|
||||
unsigned short buf_idx,
|
||||
unsigned char flags)
|
||||
{
|
||||
struct ublk_auto_buf_reg buf = {
|
||||
.index = buf_idx,
|
||||
.flags = flags,
|
||||
};
|
||||
|
||||
sqe->addr = ublk_auto_buf_reg_to_sqe_addr(&buf);
|
||||
}
|
||||
|
||||
int ublk_queue_io_cmd(struct ublk_queue *q, struct ublk_io *io, unsigned tag)
|
||||
{
|
||||
struct ublksrv_io_cmd *cmd;
|
||||
|
|
@ -579,6 +594,9 @@ int ublk_queue_io_cmd(struct ublk_queue *q, struct ublk_io *io, unsigned tag)
|
|||
else
|
||||
cmd->addr = 0;
|
||||
|
||||
if (q->state & UBLKSRV_AUTO_BUF_REG)
|
||||
ublk_set_auto_buf_reg(sqe[0], tag, 0);
|
||||
|
||||
user_data = build_user_data(tag, _IOC_NR(cmd_op), 0, 0);
|
||||
io_uring_sqe_set_data64(sqe[0], user_data);
|
||||
|
||||
|
|
@ -1206,6 +1224,7 @@ static int cmd_dev_get_features(void)
|
|||
[const_ilog2(UBLK_F_USER_COPY)] = "USER_COPY",
|
||||
[const_ilog2(UBLK_F_ZONED)] = "ZONED",
|
||||
[const_ilog2(UBLK_F_USER_RECOVERY_FAIL_IO)] = "RECOVERY_FAIL_IO",
|
||||
[const_ilog2(UBLK_F_AUTO_BUF_REG)] = "AUTO_BUF_REG",
|
||||
};
|
||||
struct ublk_dev *dev;
|
||||
__u64 features = 0;
|
||||
|
|
@ -1245,7 +1264,7 @@ static void __cmd_create_help(char *exe, bool recovery)
|
|||
|
||||
printf("%s %s -t [null|loop|stripe|fault_inject] [-q nr_queues] [-d depth] [-n dev_id]\n",
|
||||
exe, recovery ? "recover" : "add");
|
||||
printf("\t[--foreground] [--quiet] [-z] [--debug_mask mask] [-r 0|1 ] [-g]\n");
|
||||
printf("\t[--foreground] [--quiet] [-z] [--auto_zc] [--debug_mask mask] [-r 0|1 ] [-g]\n");
|
||||
printf("\t[-e 0|1 ] [-i 0|1]\n");
|
||||
printf("\t[target options] [backfile1] [backfile2] ...\n");
|
||||
printf("\tdefault: nr_queues=2(max 32), depth=128(max 1024), dev_id=-1(auto allocation)\n");
|
||||
|
|
@ -1300,6 +1319,7 @@ int main(int argc, char *argv[])
|
|||
{ "recovery_fail_io", 1, NULL, 'e'},
|
||||
{ "recovery_reissue", 1, NULL, 'i'},
|
||||
{ "get_data", 1, NULL, 'g'},
|
||||
{ "auto_zc", 0, NULL, 0},
|
||||
{ 0, 0, 0, 0 }
|
||||
};
|
||||
const struct ublk_tgt_ops *ops = NULL;
|
||||
|
|
@ -1368,6 +1388,8 @@ int main(int argc, char *argv[])
|
|||
ublk_dbg_mask = 0;
|
||||
if (!strcmp(longopts[option_idx].name, "foreground"))
|
||||
ctx.fg = 1;
|
||||
if (!strcmp(longopts[option_idx].name, "auto_zc"))
|
||||
ctx.flags |= UBLK_F_AUTO_BUF_REG;
|
||||
break;
|
||||
case '?':
|
||||
/*
|
||||
|
|
|
|||
|
|
@ -115,6 +115,7 @@ struct ublk_io {
|
|||
#define UBLKSRV_NEED_COMMIT_RQ_COMP (1UL << 1)
|
||||
#define UBLKSRV_IO_FREE (1UL << 2)
|
||||
#define UBLKSRV_NEED_GET_DATA (1UL << 3)
|
||||
#define UBLKSRV_NEED_REG_BUF (1UL << 4)
|
||||
unsigned short flags;
|
||||
unsigned short refs; /* used by target code only */
|
||||
|
||||
|
|
@ -168,6 +169,7 @@ struct ublk_queue {
|
|||
#define UBLKSRV_QUEUE_IDLE (1U << 1)
|
||||
#define UBLKSRV_NO_BUF (1U << 2)
|
||||
#define UBLKSRV_ZC (1U << 3)
|
||||
#define UBLKSRV_AUTO_BUF_REG (1U << 4)
|
||||
unsigned state;
|
||||
pid_t tid;
|
||||
pthread_t thread;
|
||||
|
|
@ -387,6 +389,11 @@ static inline int ublk_queue_use_zc(const struct ublk_queue *q)
|
|||
return q->state & UBLKSRV_ZC;
|
||||
}
|
||||
|
||||
static inline int ublk_queue_use_auto_zc(const struct ublk_queue *q)
|
||||
{
|
||||
return q->state & UBLKSRV_AUTO_BUF_REG;
|
||||
}
|
||||
|
||||
extern const struct ublk_tgt_ops null_tgt_ops;
|
||||
extern const struct ublk_tgt_ops loop_tgt_ops;
|
||||
extern const struct ublk_tgt_ops stripe_tgt_ops;
|
||||
|
|
|
|||
|
|
@ -42,10 +42,22 @@ static int ublk_null_tgt_init(const struct dev_ctx *ctx, struct ublk_dev *dev)
|
|||
return 0;
|
||||
}
|
||||
|
||||
static void __setup_nop_io(int tag, const struct ublksrv_io_desc *iod,
|
||||
struct io_uring_sqe *sqe)
|
||||
{
|
||||
unsigned ublk_op = ublksrv_get_op(iod);
|
||||
|
||||
io_uring_prep_nop(sqe);
|
||||
sqe->buf_index = tag;
|
||||
sqe->flags |= IOSQE_FIXED_FILE;
|
||||
sqe->rw_flags = IORING_NOP_FIXED_BUFFER | IORING_NOP_INJECT_RESULT;
|
||||
sqe->len = iod->nr_sectors << 9; /* injected result */
|
||||
sqe->user_data = build_user_data(tag, ublk_op, 0, 1);
|
||||
}
|
||||
|
||||
static int null_queue_zc_io(struct ublk_queue *q, int tag)
|
||||
{
|
||||
const struct ublksrv_io_desc *iod = ublk_get_iod(q, tag);
|
||||
unsigned ublk_op = ublksrv_get_op(iod);
|
||||
struct io_uring_sqe *sqe[3];
|
||||
|
||||
ublk_queue_alloc_sqes(q, sqe, 3);
|
||||
|
|
@ -55,12 +67,8 @@ static int null_queue_zc_io(struct ublk_queue *q, int tag)
|
|||
ublk_cmd_op_nr(sqe[0]->cmd_op), 0, 1);
|
||||
sqe[0]->flags |= IOSQE_CQE_SKIP_SUCCESS | IOSQE_IO_HARDLINK;
|
||||
|
||||
io_uring_prep_nop(sqe[1]);
|
||||
sqe[1]->buf_index = tag;
|
||||
sqe[1]->flags |= IOSQE_FIXED_FILE | IOSQE_IO_HARDLINK;
|
||||
sqe[1]->rw_flags = IORING_NOP_FIXED_BUFFER | IORING_NOP_INJECT_RESULT;
|
||||
sqe[1]->len = iod->nr_sectors << 9; /* injected result */
|
||||
sqe[1]->user_data = build_user_data(tag, ublk_op, 0, 1);
|
||||
__setup_nop_io(tag, iod, sqe[1]);
|
||||
sqe[1]->flags |= IOSQE_IO_HARDLINK;
|
||||
|
||||
io_uring_prep_buf_unregister(sqe[2], 0, tag, q->q_id, tag);
|
||||
sqe[2]->user_data = build_user_data(tag, ublk_cmd_op_nr(sqe[2]->cmd_op), 0, 1);
|
||||
|
|
@ -69,6 +77,16 @@ static int null_queue_zc_io(struct ublk_queue *q, int tag)
|
|||
return 2;
|
||||
}
|
||||
|
||||
static int null_queue_auto_zc_io(struct ublk_queue *q, int tag)
|
||||
{
|
||||
const struct ublksrv_io_desc *iod = ublk_get_iod(q, tag);
|
||||
struct io_uring_sqe *sqe[1];
|
||||
|
||||
ublk_queue_alloc_sqes(q, sqe, 1);
|
||||
__setup_nop_io(tag, iod, sqe[0]);
|
||||
return 1;
|
||||
}
|
||||
|
||||
static void ublk_null_io_done(struct ublk_queue *q, int tag,
|
||||
const struct io_uring_cqe *cqe)
|
||||
{
|
||||
|
|
@ -94,15 +112,18 @@ static void ublk_null_io_done(struct ublk_queue *q, int tag,
|
|||
static int ublk_null_queue_io(struct ublk_queue *q, int tag)
|
||||
{
|
||||
const struct ublksrv_io_desc *iod = ublk_get_iod(q, tag);
|
||||
int zc = ublk_queue_use_zc(q);
|
||||
unsigned auto_zc = ublk_queue_use_auto_zc(q);
|
||||
unsigned zc = ublk_queue_use_zc(q);
|
||||
int queued;
|
||||
|
||||
if (!zc) {
|
||||
if (auto_zc)
|
||||
queued = null_queue_auto_zc_io(q, tag);
|
||||
else if (zc)
|
||||
queued = null_queue_zc_io(q, tag);
|
||||
else {
|
||||
ublk_complete_io(q, tag, iod->nr_sectors << 9);
|
||||
return 0;
|
||||
}
|
||||
|
||||
queued = null_queue_zc_io(q, tag);
|
||||
ublk_queued_tgt_io(q, tag, queued);
|
||||
return 0;
|
||||
}
|
||||
|
|
|
|||
|
|
@ -70,7 +70,7 @@ static void free_stripe_array(struct stripe_array *s)
|
|||
}
|
||||
|
||||
static void calculate_stripe_array(const struct stripe_conf *conf,
|
||||
const struct ublksrv_io_desc *iod, struct stripe_array *s)
|
||||
const struct ublksrv_io_desc *iod, struct stripe_array *s, void *base)
|
||||
{
|
||||
const unsigned shift = conf->shift - 9;
|
||||
const unsigned chunk_sects = 1 << shift;
|
||||
|
|
@ -102,7 +102,7 @@ static void calculate_stripe_array(const struct stripe_conf *conf,
|
|||
}
|
||||
|
||||
assert(this->nr_vec < this->cap);
|
||||
this->vec[this->nr_vec].iov_base = (void *)(iod->addr + done);
|
||||
this->vec[this->nr_vec].iov_base = (void *)(base + done);
|
||||
this->vec[this->nr_vec++].iov_len = nr_sects << 9;
|
||||
|
||||
start += nr_sects;
|
||||
|
|
@ -126,15 +126,17 @@ static inline enum io_uring_op stripe_to_uring_op(
|
|||
static int stripe_queue_tgt_rw_io(struct ublk_queue *q, const struct ublksrv_io_desc *iod, int tag)
|
||||
{
|
||||
const struct stripe_conf *conf = get_chunk_shift(q);
|
||||
int zc = !!(ublk_queue_use_zc(q) != 0);
|
||||
enum io_uring_op op = stripe_to_uring_op(iod, zc);
|
||||
unsigned auto_zc = (ublk_queue_use_auto_zc(q) != 0);
|
||||
unsigned zc = (ublk_queue_use_zc(q) != 0);
|
||||
enum io_uring_op op = stripe_to_uring_op(iod, zc | auto_zc);
|
||||
struct io_uring_sqe *sqe[NR_STRIPE];
|
||||
struct stripe_array *s = alloc_stripe_array(conf, iod);
|
||||
struct ublk_io *io = ublk_get_io(q, tag);
|
||||
int i, extra = zc ? 2 : 0;
|
||||
void *base = (zc | auto_zc) ? NULL : (void *)iod->addr;
|
||||
|
||||
io->private_data = s;
|
||||
calculate_stripe_array(conf, iod, s);
|
||||
calculate_stripe_array(conf, iod, s, base);
|
||||
|
||||
ublk_queue_alloc_sqes(q, sqe, s->nr + extra);
|
||||
|
||||
|
|
@ -153,12 +155,11 @@ static int stripe_queue_tgt_rw_io(struct ublk_queue *q, const struct ublksrv_io_
|
|||
(void *)t->vec,
|
||||
t->nr_vec,
|
||||
t->start << 9);
|
||||
if (zc) {
|
||||
io_uring_sqe_set_flags(sqe[i], IOSQE_FIXED_FILE);
|
||||
if (auto_zc || zc) {
|
||||
sqe[i]->buf_index = tag;
|
||||
io_uring_sqe_set_flags(sqe[i],
|
||||
IOSQE_FIXED_FILE | IOSQE_IO_HARDLINK);
|
||||
} else {
|
||||
io_uring_sqe_set_flags(sqe[i], IOSQE_FIXED_FILE);
|
||||
if (zc)
|
||||
sqe[i]->flags |= IOSQE_IO_HARDLINK;
|
||||
}
|
||||
/* bit63 marks us as tgt io */
|
||||
sqe[i]->user_data = build_user_data(tag, ublksrv_get_op(iod), i - zc, 1);
|
||||
|
|
|
|||
32
tools/testing/selftests/ublk/test_generic_08.sh
Executable file
32
tools/testing/selftests/ublk/test_generic_08.sh
Executable file
|
|
@ -0,0 +1,32 @@
|
|||
#!/bin/bash
|
||||
# SPDX-License-Identifier: GPL-2.0
|
||||
|
||||
. "$(cd "$(dirname "$0")" && pwd)"/test_common.sh
|
||||
|
||||
TID="generic_08"
|
||||
ERR_CODE=0
|
||||
|
||||
if ! _have_feature "AUTO_BUF_REG"; then
|
||||
exit "$UBLK_SKIP_CODE"
|
||||
fi
|
||||
|
||||
_prep_test "generic" "test UBLK_F_AUTO_BUF_REG"
|
||||
|
||||
_create_backfile 0 256M
|
||||
_create_backfile 1 256M
|
||||
|
||||
dev_id=$(_add_ublk_dev -t loop -q 2 --auto_zc "${UBLK_BACKFILES[0]}")
|
||||
_check_add_dev $TID $?
|
||||
|
||||
if ! _mkfs_mount_test /dev/ublkb"${dev_id}"; then
|
||||
_cleanup_test "generic"
|
||||
_show_result $TID 255
|
||||
fi
|
||||
|
||||
dev_id=$(_add_ublk_dev -t stripe --auto_zc "${UBLK_BACKFILES[0]}" "${UBLK_BACKFILES[1]}")
|
||||
_check_add_dev $TID $?
|
||||
_mkfs_mount_test /dev/ublkb"${dev_id}"
|
||||
ERR_CODE=$?
|
||||
|
||||
_cleanup_test "generic"
|
||||
_show_result $TID $ERR_CODE
|
||||
|
|
@ -32,6 +32,12 @@ _create_backfile 2 128M
|
|||
ublk_io_and_remove 8G -t null -q 4 -z &
|
||||
ublk_io_and_remove 256M -t loop -q 4 -z "${UBLK_BACKFILES[0]}" &
|
||||
ublk_io_and_remove 256M -t stripe -q 4 -z "${UBLK_BACKFILES[1]}" "${UBLK_BACKFILES[2]}" &
|
||||
|
||||
if _have_feature "AUTO_BUF_REG"; then
|
||||
ublk_io_and_remove 8G -t null -q 4 --auto_zc &
|
||||
ublk_io_and_remove 256M -t loop -q 4 --auto_zc "${UBLK_BACKFILES[0]}" &
|
||||
ublk_io_and_remove 256M -t stripe -q 4 --auto_zc "${UBLK_BACKFILES[1]}" "${UBLK_BACKFILES[2]}" &
|
||||
fi
|
||||
wait
|
||||
|
||||
_cleanup_test "stress"
|
||||
|
|
|
|||
|
|
@ -31,6 +31,12 @@ _create_backfile 2 128M
|
|||
ublk_io_and_kill_daemon 8G -t null -q 4 -z &
|
||||
ublk_io_and_kill_daemon 256M -t loop -q 4 -z "${UBLK_BACKFILES[0]}" &
|
||||
ublk_io_and_kill_daemon 256M -t stripe -q 4 -z "${UBLK_BACKFILES[1]}" "${UBLK_BACKFILES[2]}" &
|
||||
|
||||
if _have_feature "AUTO_BUF_REG"; then
|
||||
ublk_io_and_kill_daemon 8G -t null -q 4 --auto_zc &
|
||||
ublk_io_and_kill_daemon 256M -t loop -q 4 --auto_zc "${UBLK_BACKFILES[0]}" &
|
||||
ublk_io_and_kill_daemon 256M -t stripe -q 4 --auto_zc "${UBLK_BACKFILES[1]}" "${UBLK_BACKFILES[2]}" &
|
||||
fi
|
||||
wait
|
||||
|
||||
_cleanup_test "stress"
|
||||
|
|
|
|||
|
|
@ -60,5 +60,13 @@ if _have_feature "ZERO_COPY"; then
|
|||
done
|
||||
fi
|
||||
|
||||
if _have_feature "AUTO_BUF_REG"; then
|
||||
for reissue in $(seq 0 1); do
|
||||
ublk_io_and_remove 8G -t null -q 4 -g --auto_zc -r 1 -i "$reissue" &
|
||||
ublk_io_and_remove 256M -t loop -q 4 -g --auto_zc -r 1 -i "$reissue" "${UBLK_BACKFILES[1]}" &
|
||||
wait
|
||||
done
|
||||
fi
|
||||
|
||||
_cleanup_test "stress"
|
||||
_show_result $TID $ERR_CODE
|
||||
|
|
|
|||
Loading…
Add table
Reference in a new issue