From 78a9c8a3b9d59e7465d6c158283a531a221fa3b2 Mon Sep 17 00:00:00 2001 Date: Tue, 12 Nov 2024 22:58:18 +0000 Subject: [PATCH 1/4] io_uring: export io_mapped_ubuf definition --- include/linux/io_uring/kbuf.h | 19 +++++++++++++++++++ io_uring/rsrc.h | 12 ++---------- 2 files changed, 21 insertions(+), 10 deletions(-) create mode 100644 include/linux/io_uring/kbuf.h diff --git a/include/linux/io_uring/kbuf.h b/include/linux/io_uring/kbuf.h new file mode 100644 index 000000000000..a32578df3d8e --- /dev/null +++ b/include/linux/io_uring/kbuf.h @@ -0,0 +1,19 @@ +/* SPDX-License-Identifier: GPL-2.0-or-later */ +#ifndef _LINUX_IO_URING_KBUF_H +#define _LINUX_IO_URING_KBUF_H + +#include +#include +#include + +struct io_mapped_ubuf { + u64 ubuf; + unsigned int len; + unsigned int nr_bvecs; + unsigned int folio_shift; + refcount_t refs; + unsigned long acct_pages; + struct bio_vec bvec[] __counted_by(nr_bvecs); +}; + +#endif \ No newline at end of file diff --git a/io_uring/rsrc.h b/io_uring/rsrc.h index 7a4668deaa1a..885ccecade08 100644 --- a/io_uring/rsrc.h +++ b/io_uring/rsrc.h @@ -2,6 +2,8 @@ #ifndef IOU_RSRC_H #define IOU_RSRC_H +#include + #define IO_NODE_ALLOC_CACHE_MAX 32 #define IO_RSRC_TAG_TABLE_SHIFT (PAGE_SHIFT - 3) @@ -24,16 +26,6 @@ struct io_rsrc_node { }; }; -struct io_mapped_ubuf { - u64 ubuf; - unsigned int len; - unsigned int nr_bvecs; - unsigned int folio_shift; - refcount_t refs; - unsigned long acct_pages; - struct bio_vec bvec[] __counted_by(nr_bvecs); -}; - struct io_imu_folio_data { /* Head folio can be partially included in the fixed buf */ unsigned int nr_pages_head; -- 2.46.0 From 6839ca1ca94a89ec11362f32af22e2c0cfdfaa81 Mon Sep 17 00:00:00 2001 Date: Tue, 12 Nov 2024 23:12:15 +0000 Subject: [PATCH 2/4] io_uring: add io_mapped_ubuf release callback --- include/linux/io_uring/kbuf.h | 10 ++++++++++ io_uring/rsrc.c | 6 +++++- 2 files changed, 15 insertions(+), 1 deletion(-) diff --git a/include/linux/io_uring/kbuf.h b/include/linux/io_uring/kbuf.h index a32578df3d8e..aa3eeaa1ac25 100644 --- a/include/linux/io_uring/kbuf.h +++ b/include/linux/io_uring/kbuf.h @@ -13,7 +13,17 @@ struct io_mapped_ubuf { unsigned int folio_shift; refcount_t refs; unsigned long acct_pages; + void (*release)(struct io_mapped_ubuf *); struct bio_vec bvec[] __counted_by(nr_bvecs); }; +static inline void iou_init_kbuf(struct io_mapped_ubuf *buf, + void (*release)(struct io_mapped_ubuf *)) +{ + refcount_set(&buf->refs, 1); + buf->acct_pages = 0; + buf->ubuf = 0; + buf->release = release; +} + #endif \ No newline at end of file diff --git a/io_uring/rsrc.c b/io_uring/rsrc.c index adaae8630932..84ea5a480058 100644 --- a/io_uring/rsrc.c +++ b/io_uring/rsrc.c @@ -110,6 +110,10 @@ static void io_buffer_unmap(struct io_ring_ctx *ctx, struct io_rsrc_node *node) if (!refcount_dec_and_test(&imu->refs)) return; + if (imu->release) { + imu->release(imu); + return; + } for (i = 0; i < imu->nr_bvecs; i++) unpin_user_page(imu->bvec[i].bv_page); if (imu->acct_pages) @@ -762,6 +766,7 @@ static struct io_rsrc_node *io_sqe_buffer_register(struct io_ring_ctx *ctx, } size = iov->iov_len; + iou_init_kbuf(imu, NULL); /* store original address for later verification */ imu->ubuf = (unsigned long) iov->iov_base; imu->len = iov->iov_len; @@ -769,7 +774,6 @@ static struct io_rsrc_node *io_sqe_buffer_register(struct io_ring_ctx *ctx, imu->folio_shift = PAGE_SHIFT; if (coalesced) imu->folio_shift = data.folio_shift; - refcount_set(&imu->refs, 1); off = (unsigned long) iov->iov_base & ((1UL << imu->folio_shift) - 1); node->buf = imu; ret = 0; -- 2.46.0 From 55b56ed8c5cb58727c7daabd1e56e6f194749e7f Mon Sep 17 00:00:00 2001 Date: Tue, 12 Nov 2024 23:33:24 +0000 Subject: [PATCH 3/4] io_uring: add a helper for leasing a buffer --- include/linux/io_uring/kbuf.h | 23 +++++++++++++++++++++++ io_uring/rsrc.c | 32 ++++++++++++++++++++++++++++++++ 2 files changed, 55 insertions(+) diff --git a/include/linux/io_uring/kbuf.h b/include/linux/io_uring/kbuf.h index aa3eeaa1ac25..91cfcdc685cc 100644 --- a/include/linux/io_uring/kbuf.h +++ b/include/linux/io_uring/kbuf.h @@ -4,6 +4,7 @@ #include #include +#include #include struct io_mapped_ubuf { @@ -26,4 +27,26 @@ static inline void iou_init_kbuf(struct io_mapped_ubuf *buf, buf->release = release; } +#if defined(CONFIG_IO_URING) +int iou_export_kbuf(struct io_ring_ctx *ctx, unsigned issue_flags, + struct io_mapped_ubuf *buf, unsigned index); +#else +static inline int iou_export_kbuf(struct io_ring_ctx *ctx, + unsigned issue_flags, + struct io_mapped_ubuf *buf, unsigned index) +{ + return -EOPNOTSUPP; +} +#endif + +static inline int io_uring_cmd_export_kbuf(struct io_uring_cmd *cmd, + unsigned issue_flags, + struct io_mapped_ubuf *buf, + unsigned index) +{ + struct io_ring_ctx *ctx = cmd_to_io_kiocb(cmd)->ctx; + + return iou_export_kbuf(ctx, issue_flags, buf, index); +} + #endif \ No newline at end of file diff --git a/io_uring/rsrc.c b/io_uring/rsrc.c index 84ea5a480058..07842a6a8020 100644 --- a/io_uring/rsrc.c +++ b/io_uring/rsrc.c @@ -797,6 +797,38 @@ static struct io_rsrc_node *io_sqe_buffer_register(struct io_ring_ctx *ctx, return node; } +static int __iou_export_kbuf(struct io_ring_ctx *ctx, unsigned issue_flags, + struct io_mapped_ubuf *buf, unsigned idx) +{ + struct io_rsrc_node *node; + + if (unlikely(idx >= ctx->buf_table.nr)) { + if (!ctx->buf_table.nr) + return -ENXIO; + return -EINVAL; + } + idx = array_index_nospec(idx, ctx->buf_table.nr); + + node = io_rsrc_node_alloc(ctx, IORING_RSRC_BUFFER); + if (!node) + return -ENOMEM; + node->buf = buf; + io_reset_rsrc_node(ctx, &ctx->buf_table, idx); + ctx->buf_table.nodes[idx] = node; + return 0; +} + +int iou_export_kbuf(struct io_ring_ctx *ctx, unsigned issue_flags, + struct io_mapped_ubuf *buf, unsigned idx) +{ + int ret; + + io_ring_submit_lock(ctx, issue_flags); + ret = __iou_export_kbuf(ctx, issue_flags, buf, idx); + io_ring_submit_unlock(ctx, issue_flags); + return ret; +} + int io_sqe_buffers_register(struct io_ring_ctx *ctx, void __user *arg, unsigned int nr_args, u64 __user *tags) { -- 2.46.0