* [PATCH 1/4] io_uring: add validate_fixed_range() for validate fixed buffer
2025-03-25 13:51 [PATCH 0/4] io_uring: support vectored fixed kernel buffer Ming Lei
@ 2025-03-25 13:51 ` Ming Lei
2025-03-25 16:00 ` Caleb Sander Mateos
2025-03-25 13:51 ` [PATCH 2/4] block: add for_each_mp_bvec() Ming Lei
` (2 subsequent siblings)
3 siblings, 1 reply; 6+ messages in thread
From: Ming Lei @ 2025-03-25 13:51 UTC (permalink / raw)
To: Jens Axboe, io-uring, Pavel Begunkov; +Cc: Caleb Sander Mateos, Ming Lei
Add helper of validate_fixed_range() for validating fixed buffer
range.
Signed-off-by: Ming Lei <[email protected]>
---
io_uring/rsrc.c | 34 +++++++++++++++++++++++-----------
1 file changed, 23 insertions(+), 11 deletions(-)
diff --git a/io_uring/rsrc.c b/io_uring/rsrc.c
index 3f195e24777e..52e7492e863e 100644
--- a/io_uring/rsrc.c
+++ b/io_uring/rsrc.c
@@ -1002,20 +1002,32 @@ int io_buffer_unregister_bvec(struct io_uring_cmd *cmd, unsigned int index,
}
EXPORT_SYMBOL_GPL(io_buffer_unregister_bvec);
-static int io_import_fixed(int ddir, struct iov_iter *iter,
- struct io_mapped_ubuf *imu,
- u64 buf_addr, size_t len)
+static int validate_fixed_range(u64 buf_addr, size_t len,
+ const struct io_mapped_ubuf *imu)
{
u64 buf_end;
- size_t offset;
- if (WARN_ON_ONCE(!imu))
- return -EFAULT;
if (unlikely(check_add_overflow(buf_addr, (u64)len, &buf_end)))
return -EFAULT;
/* not inside the mapped region */
if (unlikely(buf_addr < imu->ubuf || buf_end > (imu->ubuf + imu->len)))
return -EFAULT;
+
+ return 0;
+}
+
+static int io_import_fixed(int ddir, struct iov_iter *iter,
+ struct io_mapped_ubuf *imu,
+ u64 buf_addr, size_t len)
+{
+ size_t offset;
+ int ret;
+
+ if (WARN_ON_ONCE(!imu))
+ return -EFAULT;
+ ret = validate_fixed_range(buf_addr, len, imu);
+ if (ret)
+ return ret;
if (!(imu->dir & (1 << ddir)))
return -EFAULT;
@@ -1305,12 +1317,12 @@ static int io_vec_fill_bvec(int ddir, struct iov_iter *iter,
u64 buf_addr = (u64)(uintptr_t)iovec[iov_idx].iov_base;
struct bio_vec *src_bvec;
size_t offset;
- u64 buf_end;
+ int ret;
+
+ ret = validate_fixed_range(buf_addr, iov_len, imu);
+ if (unlikely(ret))
+ return ret;
- if (unlikely(check_add_overflow(buf_addr, (u64)iov_len, &buf_end)))
- return -EFAULT;
- if (unlikely(buf_addr < imu->ubuf || buf_end > (imu->ubuf + imu->len)))
- return -EFAULT;
if (unlikely(!iov_len))
return -EFAULT;
if (unlikely(check_add_overflow(total_len, iov_len, &total_len)))
--
2.47.0
^ permalink raw reply related [flat|nested] 6+ messages in thread* Re: [PATCH 1/4] io_uring: add validate_fixed_range() for validate fixed buffer
2025-03-25 13:51 ` [PATCH 1/4] io_uring: add validate_fixed_range() for validate fixed buffer Ming Lei
@ 2025-03-25 16:00 ` Caleb Sander Mateos
0 siblings, 0 replies; 6+ messages in thread
From: Caleb Sander Mateos @ 2025-03-25 16:00 UTC (permalink / raw)
To: Ming Lei; +Cc: Jens Axboe, io-uring, Pavel Begunkov
On Tue, Mar 25, 2025 at 6:52 AM Ming Lei <[email protected]> wrote:
>
> Add helper of validate_fixed_range() for validating fixed buffer
> range.
>
> Signed-off-by: Ming Lei <[email protected]>
> ---
> io_uring/rsrc.c | 34 +++++++++++++++++++++++-----------
> 1 file changed, 23 insertions(+), 11 deletions(-)
>
> diff --git a/io_uring/rsrc.c b/io_uring/rsrc.c
> index 3f195e24777e..52e7492e863e 100644
> --- a/io_uring/rsrc.c
> +++ b/io_uring/rsrc.c
> @@ -1002,20 +1002,32 @@ int io_buffer_unregister_bvec(struct io_uring_cmd *cmd, unsigned int index,
> }
> EXPORT_SYMBOL_GPL(io_buffer_unregister_bvec);
>
> -static int io_import_fixed(int ddir, struct iov_iter *iter,
> - struct io_mapped_ubuf *imu,
> - u64 buf_addr, size_t len)
> +static int validate_fixed_range(u64 buf_addr, size_t len,
> + const struct io_mapped_ubuf *imu)
> {
> u64 buf_end;
> - size_t offset;
>
> - if (WARN_ON_ONCE(!imu))
> - return -EFAULT;
> if (unlikely(check_add_overflow(buf_addr, (u64)len, &buf_end)))
> return -EFAULT;
> /* not inside the mapped region */
> if (unlikely(buf_addr < imu->ubuf || buf_end > (imu->ubuf + imu->len)))
> return -EFAULT;
> +
> + return 0;
It's nice to avoid this code duplication. It looks like
validate_fixed_range() could return a bool instead and leave the
return code up to the caller, but I don't feel strongly either way.
Reviewed-by: Caleb Sander Mateos <[email protected]>
> +}
> +
> +static int io_import_fixed(int ddir, struct iov_iter *iter,
> + struct io_mapped_ubuf *imu,
> + u64 buf_addr, size_t len)
> +{
> + size_t offset;
> + int ret;
> +
> + if (WARN_ON_ONCE(!imu))
> + return -EFAULT;
> + ret = validate_fixed_range(buf_addr, len, imu);
> + if (ret)
> + return ret;
> if (!(imu->dir & (1 << ddir)))
> return -EFAULT;
>
> @@ -1305,12 +1317,12 @@ static int io_vec_fill_bvec(int ddir, struct iov_iter *iter,
> u64 buf_addr = (u64)(uintptr_t)iovec[iov_idx].iov_base;
> struct bio_vec *src_bvec;
> size_t offset;
> - u64 buf_end;
> + int ret;
> +
> + ret = validate_fixed_range(buf_addr, iov_len, imu);
> + if (unlikely(ret))
> + return ret;
>
> - if (unlikely(check_add_overflow(buf_addr, (u64)iov_len, &buf_end)))
> - return -EFAULT;
> - if (unlikely(buf_addr < imu->ubuf || buf_end > (imu->ubuf + imu->len)))
> - return -EFAULT;
> if (unlikely(!iov_len))
> return -EFAULT;
> if (unlikely(check_add_overflow(total_len, iov_len, &total_len)))
> --
> 2.47.0
>
^ permalink raw reply [flat|nested] 6+ messages in thread
* [PATCH 2/4] block: add for_each_mp_bvec()
2025-03-25 13:51 [PATCH 0/4] io_uring: support vectored fixed kernel buffer Ming Lei
2025-03-25 13:51 ` [PATCH 1/4] io_uring: add validate_fixed_range() for validate fixed buffer Ming Lei
@ 2025-03-25 13:51 ` Ming Lei
2025-03-25 13:51 ` [PATCH 3/4] io_uring: support vectored kernel fixed buffer Ming Lei
2025-03-25 13:51 ` [PATCH 4/4] selftests: ublk: enable zero copy for stripe target Ming Lei
3 siblings, 0 replies; 6+ messages in thread
From: Ming Lei @ 2025-03-25 13:51 UTC (permalink / raw)
To: Jens Axboe, io-uring, Pavel Begunkov; +Cc: Caleb Sander Mateos, Ming Lei
Add helper of for_each_mp_bvec() for io_uring to import fixed kernel
buffer.
Signed-off-by: Ming Lei <[email protected]>
---
include/linux/bvec.h | 6 ++++++
1 file changed, 6 insertions(+)
diff --git a/include/linux/bvec.h b/include/linux/bvec.h
index ba8f52d48b94..204b22a99c4b 100644
--- a/include/linux/bvec.h
+++ b/include/linux/bvec.h
@@ -184,6 +184,12 @@ static inline void bvec_iter_advance_single(const struct bio_vec *bv,
((bvl = bvec_iter_bvec((bio_vec), (iter))), 1); \
bvec_iter_advance_single((bio_vec), &(iter), (bvl).bv_len))
+#define for_each_mp_bvec(bvl, bio_vec, iter, start) \
+ for (iter = (start); \
+ (iter).bi_size && \
+ ((bvl = mp_bvec_iter_bvec((bio_vec), (iter))), 1); \
+ bvec_iter_advance_single((bio_vec), &(iter), (bvl).bv_len))
+
/* for iterating one bio from start to end */
#define BVEC_ITER_ALL_INIT (struct bvec_iter) \
{ \
--
2.47.0
^ permalink raw reply related [flat|nested] 6+ messages in thread* [PATCH 3/4] io_uring: support vectored kernel fixed buffer
2025-03-25 13:51 [PATCH 0/4] io_uring: support vectored fixed kernel buffer Ming Lei
2025-03-25 13:51 ` [PATCH 1/4] io_uring: add validate_fixed_range() for validate fixed buffer Ming Lei
2025-03-25 13:51 ` [PATCH 2/4] block: add for_each_mp_bvec() Ming Lei
@ 2025-03-25 13:51 ` Ming Lei
2025-03-25 13:51 ` [PATCH 4/4] selftests: ublk: enable zero copy for stripe target Ming Lei
3 siblings, 0 replies; 6+ messages in thread
From: Ming Lei @ 2025-03-25 13:51 UTC (permalink / raw)
To: Jens Axboe, io-uring, Pavel Begunkov; +Cc: Caleb Sander Mateos, Ming Lei
io_uring has supported fixed kernel buffer via io_buffer_register_bvec()
and io_buffer_unregister_bvec().
The vectored fixed buffer has been ready, so it is natural to support
fixed kernel buffer, one use case is ublk.
Signed-off-by: Ming Lei <[email protected]>
---
io_uring/rsrc.c | 91 +++++++++++++++++++++++++++++++++++++++++++++++--
1 file changed, 88 insertions(+), 3 deletions(-)
diff --git a/io_uring/rsrc.c b/io_uring/rsrc.c
index 52e7492e863e..82a7c2fcf58f 100644
--- a/io_uring/rsrc.c
+++ b/io_uring/rsrc.c
@@ -1361,6 +1361,82 @@ static int io_estimate_bvec_size(struct iovec *iov, unsigned nr_iovs,
return max_segs;
}
+static int io_vec_fill_kern_bvec(int ddir, struct iov_iter *iter,
+ struct io_mapped_ubuf *imu,
+ struct iovec *iovec, unsigned nr_iovs,
+ struct iou_vec *vec)
+{
+ const struct bio_vec *src_bvec = imu->bvec;
+ struct bio_vec *res_bvec = vec->bvec;
+ unsigned res_idx = 0;
+ size_t total_len = 0;
+ unsigned iov_idx;
+
+ for (iov_idx = 0; iov_idx < nr_iovs; iov_idx++) {
+ size_t offset = (size_t)(uintptr_t)iovec[iov_idx].iov_base;
+ size_t iov_len = iovec[iov_idx].iov_len;
+ struct bvec_iter bi = {
+ .bi_size = offset + iov_len,
+ };
+ struct bio_vec bv;
+
+ bvec_iter_advance(src_bvec, &bi, offset);
+ for_each_mp_bvec(bv, src_bvec, bi, bi)
+ res_bvec[res_idx++] = bv;
+ total_len += iov_len;
+ }
+ iov_iter_bvec(iter, ddir, res_bvec, res_idx, total_len);
+ return 0;
+}
+
+static int iov_kern_bvec_size(const struct iovec *iov,
+ const struct io_mapped_ubuf *imu,
+ unsigned int *nr_seg)
+{
+ size_t offset = (size_t)(uintptr_t)iov->iov_base;
+ const struct bio_vec *bvec = imu->bvec;
+ int start = 0, i = 0;
+ size_t off = 0;
+ int ret;
+
+ ret = validate_fixed_range(offset, iov->iov_len, imu);
+ if (unlikely(ret))
+ return ret;
+
+ for (i = 0; off < offset + iov->iov_len && i < imu->nr_bvecs;
+ off += bvec[i].bv_len, i++) {
+ if (offset >= off && offset < off + bvec[i].bv_len)
+ start = i;
+ }
+ *nr_seg = i - start;
+ return 0;
+}
+
+static int io_kern_bvec_size(struct iovec *iov, unsigned nr_iovs,
+ struct io_mapped_ubuf *imu, unsigned *nr_segs)
+{
+ unsigned max_segs = 0;
+ size_t total_len = 0;
+ unsigned i;
+ int ret;
+
+ *nr_segs = 0;
+ for (i = 0; i < nr_iovs; i++) {
+ if (unlikely(!iov[i].iov_len))
+ return -EFAULT;
+ if (unlikely(check_add_overflow(total_len, iov[i].iov_len,
+ &total_len)))
+ return -EOVERFLOW;
+ ret = iov_kern_bvec_size(&iov[i], imu, &max_segs);
+ if (unlikely(ret))
+ return ret;
+ *nr_segs += max_segs;
+ }
+ if (total_len > MAX_RW_COUNT)
+ return -EINVAL;
+ return 0;
+}
+
int io_import_reg_vec(int ddir, struct iov_iter *iter,
struct io_kiocb *req, struct iou_vec *vec,
unsigned nr_iovs, unsigned issue_flags)
@@ -1375,14 +1451,20 @@ int io_import_reg_vec(int ddir, struct iov_iter *iter,
if (!node)
return -EFAULT;
imu = node->buf;
- if (imu->is_kbuf)
- return -EOPNOTSUPP;
if (!(imu->dir & (1 << ddir)))
return -EFAULT;
iovec_off = vec->nr - nr_iovs;
iov = vec->iovec + iovec_off;
- nr_segs = io_estimate_bvec_size(iov, nr_iovs, imu);
+
+ if (imu->is_kbuf) {
+ int ret = io_kern_bvec_size(iov, nr_iovs, imu, &nr_segs);
+
+ if (unlikely(ret))
+ return ret;
+ } else {
+ nr_segs = io_estimate_bvec_size(iov, nr_iovs, imu);
+ }
if (sizeof(struct bio_vec) > sizeof(struct iovec)) {
size_t bvec_bytes;
@@ -1409,6 +1491,9 @@ int io_import_reg_vec(int ddir, struct iov_iter *iter,
req->flags |= REQ_F_NEED_CLEANUP;
}
+ if (imu->is_kbuf)
+ return io_vec_fill_kern_bvec(ddir, iter, imu, iov, nr_iovs, vec);
+
return io_vec_fill_bvec(ddir, iter, imu, iov, nr_iovs, vec);
}
--
2.47.0
^ permalink raw reply related [flat|nested] 6+ messages in thread* [PATCH 4/4] selftests: ublk: enable zero copy for stripe target
2025-03-25 13:51 [PATCH 0/4] io_uring: support vectored fixed kernel buffer Ming Lei
` (2 preceding siblings ...)
2025-03-25 13:51 ` [PATCH 3/4] io_uring: support vectored kernel fixed buffer Ming Lei
@ 2025-03-25 13:51 ` Ming Lei
3 siblings, 0 replies; 6+ messages in thread
From: Ming Lei @ 2025-03-25 13:51 UTC (permalink / raw)
To: Jens Axboe, io-uring, Pavel Begunkov; +Cc: Caleb Sander Mateos, Ming Lei
Use io_uring vectored fixed kernel buffer for handling stripe IO.
Signed-off-by: Ming Lei <[email protected]>
---
tools/testing/selftests/ublk/Makefile | 1 +
tools/testing/selftests/ublk/stripe.c | 69 ++++++++++++++++++++-------
2 files changed, 53 insertions(+), 17 deletions(-)
diff --git a/tools/testing/selftests/ublk/Makefile b/tools/testing/selftests/ublk/Makefile
index d98680d64a2f..c7781efea0f3 100644
--- a/tools/testing/selftests/ublk/Makefile
+++ b/tools/testing/selftests/ublk/Makefile
@@ -17,6 +17,7 @@ TEST_PROGS += test_loop_05.sh
TEST_PROGS += test_stripe_01.sh
TEST_PROGS += test_stripe_02.sh
TEST_PROGS += test_stripe_03.sh
+TEST_PROGS += test_stripe_04.sh
TEST_PROGS += test_stress_01.sh
TEST_PROGS += test_stress_02.sh
diff --git a/tools/testing/selftests/ublk/stripe.c b/tools/testing/selftests/ublk/stripe.c
index 98c564b12f3c..179731c3dd6f 100644
--- a/tools/testing/selftests/ublk/stripe.c
+++ b/tools/testing/selftests/ublk/stripe.c
@@ -111,43 +111,67 @@ static void calculate_stripe_array(const struct stripe_conf *conf,
}
}
-static inline enum io_uring_op stripe_to_uring_op(const struct ublksrv_io_desc *iod)
+static inline enum io_uring_op stripe_to_uring_op(
+ const struct ublksrv_io_desc *iod, int zc)
{
unsigned ublk_op = ublksrv_get_op(iod);
if (ublk_op == UBLK_IO_OP_READ)
- return IORING_OP_READV;
+ return zc ? IORING_OP_READV_FIXED : IORING_OP_READV;
else if (ublk_op == UBLK_IO_OP_WRITE)
- return IORING_OP_WRITEV;
+ return zc ? IORING_OP_WRITEV_FIXED : IORING_OP_WRITEV;
assert(0);
}
static int stripe_queue_tgt_rw_io(struct ublk_queue *q, const struct ublksrv_io_desc *iod, int tag)
{
const struct stripe_conf *conf = get_chunk_shift(q);
- enum io_uring_op op = stripe_to_uring_op(iod);
+ int zc = !!(ublk_queue_use_zc(q) != 0);
+ enum io_uring_op op = stripe_to_uring_op(iod, zc);
struct io_uring_sqe *sqe[NR_STRIPE];
struct stripe_array *s = alloc_stripe_array(conf, iod);
struct ublk_io *io = ublk_get_io(q, tag);
- int i;
+ int i, extra = zc ? 2 : 0;
io->private_data = s;
calculate_stripe_array(conf, iod, s);
- ublk_queue_alloc_sqes(q, sqe, s->nr);
- for (i = 0; i < s->nr; i++) {
- struct stripe *t = &s->s[i];
+ ublk_queue_alloc_sqes(q, sqe, s->nr + extra);
+
+ if (zc) {
+ io_uring_prep_buf_register(sqe[0], 0, tag, q->q_id, tag);
+ sqe[0]->flags |= IOSQE_CQE_SKIP_SUCCESS | IOSQE_IO_HARDLINK;
+ sqe[0]->user_data = build_user_data(tag,
+ ublk_cmd_op_nr(sqe[0]->cmd_op), 0, 1);
+ }
+
+ for (i = zc; i < s->nr + extra - zc; i++) {
+ struct stripe *t = &s->s[i - zc];
io_uring_prep_rw(op, sqe[i],
t->seq + 1,
(void *)t->vec,
t->nr_vec,
t->start << 9);
- io_uring_sqe_set_flags(sqe[i], IOSQE_FIXED_FILE);
+ if (zc) {
+ sqe[i]->buf_index = tag;
+ io_uring_sqe_set_flags(sqe[i],
+ IOSQE_FIXED_FILE | IOSQE_IO_HARDLINK);
+ } else {
+ io_uring_sqe_set_flags(sqe[i], IOSQE_FIXED_FILE);
+ }
/* bit63 marks us as tgt io */
- sqe[i]->user_data = build_user_data(tag, ublksrv_get_op(iod), i, 1);
+ sqe[i]->user_data = build_user_data(tag, ublksrv_get_op(iod), i - zc, 1);
+ }
+ if (zc) {
+ struct io_uring_sqe *unreg = sqe[s->nr + 1];
+
+ io_uring_prep_buf_unregister(unreg, 0, tag, q->q_id, tag);
+ unreg->user_data = build_user_data(tag, ublk_cmd_op_nr(unreg->cmd_op), 0, 1);
}
- return s->nr;
+
+ /* register buffer is skip_success */
+ return s->nr + zc;
}
static int handle_flush(struct ublk_queue *q, const struct ublksrv_io_desc *iod, int tag)
@@ -208,19 +232,27 @@ static void ublk_stripe_io_done(struct ublk_queue *q, int tag,
struct ublk_io *io = ublk_get_io(q, tag);
int res = cqe->res;
- if (res < 0) {
+ if (res < 0 || op != ublk_cmd_op_nr(UBLK_U_IO_UNREGISTER_IO_BUF)) {
if (!io->result)
io->result = res;
- ublk_err("%s: io failure %d tag %u\n", __func__, res, tag);
+ if (res < 0)
+ ublk_err("%s: io failure %d tag %u\n", __func__, res, tag);
}
+ /* buffer register op is IOSQE_CQE_SKIP_SUCCESS */
+ if (op == ublk_cmd_op_nr(UBLK_U_IO_REGISTER_IO_BUF))
+ io->tgt_ios += 1;
+
/* fail short READ/WRITE simply */
if (op == UBLK_IO_OP_READ || op == UBLK_IO_OP_WRITE) {
unsigned seq = user_data_to_tgt_data(cqe->user_data);
struct stripe_array *s = io->private_data;
- if (res < s->s[seq].vec->iov_len)
+ if (res < s->s[seq].nr_sects << 9) {
io->result = -EIO;
+ ublk_err("%s: short rw op %u res %d exp %u tag %u\n",
+ __func__, op, res, s->s[seq].vec->iov_len, tag);
+ }
}
if (ublk_completed_tgt_io(q, tag)) {
@@ -253,7 +285,7 @@ static int ublk_stripe_tgt_init(const struct dev_ctx *ctx, struct ublk_dev *dev)
struct stripe_conf *conf;
unsigned chunk_shift;
loff_t bytes = 0;
- int ret, i;
+ int ret, i, mul = 1;
if ((chunk_size & (chunk_size - 1)) || !chunk_size) {
ublk_err("invalid chunk size %u\n", chunk_size);
@@ -295,8 +327,11 @@ static int ublk_stripe_tgt_init(const struct dev_ctx *ctx, struct ublk_dev *dev)
dev->tgt.dev_size = bytes;
p.basic.dev_sectors = bytes >> 9;
dev->tgt.params = p;
- dev->tgt.sq_depth = dev->dev_info.queue_depth * conf->nr_files;
- dev->tgt.cq_depth = dev->dev_info.queue_depth * conf->nr_files;
+
+ if (dev->dev_info.flags & UBLK_F_SUPPORT_ZERO_COPY)
+ mul = 2;
+ dev->tgt.sq_depth = mul * dev->dev_info.queue_depth * conf->nr_files;
+ dev->tgt.cq_depth = mul * dev->dev_info.queue_depth * conf->nr_files;
printf("%s: shift %u files %u\n", __func__, conf->shift, conf->nr_files);
--
2.47.0
^ permalink raw reply related [flat|nested] 6+ messages in thread