[PATCH 0/6] io_uring/nvme: support fixed buffer for metadata

[PATCH 0/6] io_uring/nvme: support fixed buffer for metadata

[Caleb Sander Mateos]

io_uring NVMe passthrough supports using a "fixed" (registered) buffer
for data, but not metadata. On high-IOPS workloads, the pinning and
unpinning overhead for the metadata pages is significant and could be
avoided if fixed metadata buffers were supported.

This patch series adds the necessary plumbing to allow NVMe passthrough
commands to use fixed buffers for their metadata. The metadata and data
fixed buffer indices can be specified (or omitted) independently.
Supporting separate fixed buffers is important as metadata buffers are
often stored in separate memory from the corresponding data buffers. For
ublk zero-copy I/Os, sharing a buffer index would be impossible as the
data buffer is a kernel registered buffer while the metadata buffer is a
userspace registered buffer.

My main question is which layer the metadata fixed buffer should belong
to: core io_uring, io_uring_cmd, or NVMe passthrough? In this initial
implementation, the io_uring_cmd layer stores the metadata buffer node
and the NVMe passthrough layer defines the UAPI.

The main argument for moving the implementation to a more generic layer
would be to reuse it for other io_uring request types. For example,
IORING_RW_ATTR_FLAG_PI could also benefit from a fixed metadata buffer
option, though this series doesn't implement it yet.

On the other hand, core io_uring_sqe and io_kiocb space is very limited,
so it may be undesirable to dedicate it for a somewhat niche use case.
The metadata buffer node storage could be pushed to the NVMe passthrough
layer after my in-flight series [1] to reclaim nvme_uring_cmd_pdu space.
However, managing request-scoped resources from a ->uring_cmd()
implementation is a pain, as the same request can call ->uring_cmd()
multiple times and may or may not complete when ->uring_cmd() returns,
depending on the ->uring_cmd() return value. io_req_uring_cleanup(), in
contrast, provides a single cleanup path for all uring_cmds.

[1]: https://lore.kernel.org/io-uring/20260909155848.2069290-1-csander@purestorage.com/T/

Caleb Sander Mateos (6):
  bio-integrity: remove dead bio_integrity_copy_user() error path
  nvme/ioctl: remove struct nvme_uring_data
  blk-integrity: pass iov_iter to blk_rq_integrity_map_user()
  nvme/ioctl: pass iov_iter to nvme_map_user_request()
  io_uring/cmd: support fixed buffer for metadata
  nvme/ioctl: support fixed buffer for metadata

 block/bio-integrity.c           | 51 ++++++++++--------
 block/blk-integrity.c           |  7 +--
 drivers/nvme/host/ioctl.c       | 91 +++++++++++++++++++--------------
 include/linux/bio-integrity.h   |  1 +
 include/linux/blk-integrity.h   |  6 +--
 include/linux/io_uring/cmd.h    | 12 ++++-
 include/uapi/linux/nvme_ioctl.h |  5 +-
 io_uring/rsrc.c                 | 28 ++++------
 io_uring/rsrc.h                 | 17 ++++++
 io_uring/uring_cmd.c            | 30 ++++++++++-
 10 files changed, 160 insertions(+), 88 deletions(-)

-- 
2.55.0

[PATCH 1/6] bio-integrity: remove dead bio_integrity_copy_user() error path

[Caleb Sander Mateos]

If bio_integrity_add_page() in bio_integrity_copy_user() fails in the
write case, bio_integrity_copy_user() will return an error after having
already unpinned the user pages. The release_pages label in
bio_integrity_map_user() would then unpin the pages a second time.
Thankfully, bio_integrity_add_page() can't fail here because the
bio_integrity_payload is freshly allocated, so bip_vcnt is 0. Replace
the unreachable error path with a WARN_ON_ONCE().

Signed-off-by: Caleb Sander Mateos <csander@purestorage.com>
---
 block/bio-integrity.c | 8 +-------
 1 file changed, 1 insertion(+), 7 deletions(-)

diff --git a/block/bio-integrity.c b/block/bio-integrity.c
index b23e2434d80c..8ba8bfa41f8a 100644
--- a/block/bio-integrity.c
+++ b/block/bio-integrity.c
@@ -299,19 +299,13 @@ static int bio_integrity_copy_user(struct bio *bio, struct bio_vec *bvec,
 	else
 		memcpy(&bip->bip_vec[1], bvec, nr_vecs * sizeof(*bvec));
 
 	ret = bio_integrity_add_page(bio, virt_to_page(buf), len,
 				     offset_in_page(buf));
-	if (ret != len) {
-		ret = -ENOMEM;
-		goto free_bip;
-	}
-
+	WARN_ON_ONCE(ret != len);
 	bip->bip_flags |= BIP_COPY_USER;
 	return 0;
-free_bip:
-	bio_integrity_free(bio);
 free_buf:
 	kfree(buf);
 	return ret;
 }
 
-- 
2.55.0

[PATCH 2/6] nvme/ioctl: remove struct nvme_uring_data

[Caleb Sander Mateos]

This struct is only used once as a local variable type. Make the fields
separate local variables and remove the struct type to shave some lines.

Signed-off-by: Caleb Sander Mateos <csander@purestorage.com>
---
 drivers/nvme/host/ioctl.c | 36 +++++++++++++++---------------------
 1 file changed, 15 insertions(+), 21 deletions(-)

diff --git a/drivers/nvme/host/ioctl.c b/drivers/nvme/host/ioctl.c
index 3aa5d2f2dfbb..748a4cbf7e90 100644
--- a/drivers/nvme/host/ioctl.c
+++ b/drivers/nvme/host/ioctl.c
@@ -406,18 +406,10 @@ static int nvme_user_cmd64(struct nvme_ctrl *ctrl, struct nvme_ns *ns,
 	}
 
 	return status;
 }
 
-struct nvme_uring_data {
-	__u64	metadata;
-	__u64	addr;
-	__u32	data_len;
-	__u32	metadata_len;
-	__u32	timeout_ms;
-};
-
 /*
  * This overlays struct io_uring_cmd pdu.
  * Expect build errors if this grows larger than that.
  */
 struct nvme_uring_cmd_pdu {
@@ -483,17 +475,19 @@ static int nvme_uring_cmd_io(struct nvme_ctrl *ctrl, struct nvme_ns *ns,
 	struct nvme_uring_cmd_pdu *pdu = nvme_uring_cmd_pdu(ioucmd);
 	const struct nvme_uring_cmd *cmd = io_uring_sqe128_cmd(ioucmd->sqe,
 							       struct nvme_uring_cmd);
 	struct request_queue *q = ns ? ns->queue : ctrl->admin_q;
 	bool open_for_write = ioucmd->file->f_mode & FMODE_WRITE;
-	struct nvme_uring_data d;
 	struct nvme_command c;
 	struct iov_iter iter;
 	struct iov_iter *map_iter = NULL;
 	struct request *req;
 	blk_opf_t rq_flags = 0;
 	blk_mq_req_flags_t blk_flags = 0;
+	u32 metadata_len, data_len;
+	u64 metadata, addr;
+	u32 timeout_ms;
 	int ret;
 
 	c.common.opcode = READ_ONCE(cmd->opcode);
 	c.common.flags = READ_ONCE(cmd->flags);
 	if (c.common.flags)
@@ -516,25 +510,25 @@ static int nvme_uring_cmd_io(struct nvme_ctrl *ctrl, struct nvme_ns *ns,
 	c.common.cdw15 = cpu_to_le32(READ_ONCE(cmd->cdw15));
 
 	if (!nvme_cmd_allowed(ctrl, ns, &c, 0, open_for_write))
 		return -EACCES;
 
-	d.metadata = READ_ONCE(cmd->metadata);
-	d.addr = READ_ONCE(cmd->addr);
-	d.data_len = READ_ONCE(cmd->data_len);
-	d.metadata_len = READ_ONCE(cmd->metadata_len);
-	d.timeout_ms = READ_ONCE(cmd->timeout_ms);
+	metadata = READ_ONCE(cmd->metadata);
+	addr = READ_ONCE(cmd->addr);
+	data_len = READ_ONCE(cmd->data_len);
+	metadata_len = READ_ONCE(cmd->metadata_len);
+	timeout_ms = READ_ONCE(cmd->timeout_ms);
 
-	if (d.data_len && (ioucmd->flags & IORING_URING_CMD_FIXED)) {
+	if (data_len && (ioucmd->flags & IORING_URING_CMD_FIXED)) {
 		int ddir = nvme_is_write(&c) ? WRITE : READ;
 
 		if (vec)
 			ret = io_uring_cmd_import_fixed_vec(ioucmd,
-					u64_to_user_ptr(d.addr), d.data_len,
+					u64_to_user_ptr(addr), data_len,
 					ddir, &iter, issue_flags);
 		else
-			ret = io_uring_cmd_import_fixed(d.addr, d.data_len,
+			ret = io_uring_cmd_import_fixed(addr, data_len,
 					ddir, &iter, ioucmd, issue_flags);
 		if (ret < 0)
 			return ret;
 
 		map_iter = &iter;
@@ -548,15 +542,15 @@ static int nvme_uring_cmd_io(struct nvme_ctrl *ctrl, struct nvme_ns *ns,
 		rq_flags |= REQ_POLLED;
 
 	req = nvme_alloc_user_request(q, &c, rq_flags, blk_flags);
 	if (IS_ERR(req))
 		return PTR_ERR(req);
-	req->timeout = d.timeout_ms ? msecs_to_jiffies(d.timeout_ms) : 0;
+	req->timeout = timeout_ms ? msecs_to_jiffies(timeout_ms) : 0;
 
-	if (d.data_len) {
-		ret = nvme_map_user_request(req, d.addr, d.data_len,
-			nvme_to_user_ptr(d.metadata), d.metadata_len,
+	if (data_len) {
+		ret = nvme_map_user_request(req, addr, data_len,
+			nvme_to_user_ptr(metadata), metadata_len,
 			map_iter, vec ? NVME_IOCTL_VEC : 0);
 		if (ret)
 			goto out_free_req;
 	}
 
-- 
2.55.0

[PATCH 3/6] blk-integrity: pass iov_iter to blk_rq_integrity_map_user()

[Caleb Sander Mateos]

To allow blk_rq_integrity_map_user() to accept an io_uring registered
buffer (represented as a ITER_BVEC iov_iter), take the buffer as an
iov_iter argument. Move the conversion from a user address-length pair
into an iov_iter to the caller.

Since a non-ITER_UBUF iov_iter could be passed in the future,
iov_iter_extract_pages() in bio_integrity_map_user() won't necessarily
pin the pages. Condition the page unpins on iov_iter_extract_will_pin().
Add a BIP_PAGE_PINNED bit to bip_flags to track if the pages are pinned.

Signed-off-by: Caleb Sander Mateos <csander@purestorage.com>
---
 block/bio-integrity.c         | 43 +++++++++++++++++++++++------------
 block/blk-integrity.c         |  7 ++----
 drivers/nvme/host/ioctl.c     |  6 ++++-
 include/linux/bio-integrity.h |  1 +
 include/linux/blk-integrity.h |  6 ++---
 5 files changed, 38 insertions(+), 25 deletions(-)

diff --git a/block/bio-integrity.c b/block/bio-integrity.c
index 8ba8bfa41f8a..3a5554d3ea2f 100644
--- a/block/bio-integrity.c
+++ b/block/bio-integrity.c
@@ -187,11 +187,12 @@ static void bio_integrity_uncopy_user(struct bio_integrity_payload *bip)
 
 	iov_iter_bvec(&orig_iter, ITER_DEST, orig_bvecs, orig_nr_vecs, bytes);
 	ret = copy_to_iter(bvec_virt(bounce_bvec), bytes, &orig_iter);
 	WARN_ON_ONCE(ret != bytes);
 
-	bio_integrity_unpin_bvec(orig_bvecs, orig_nr_vecs);
+	if (bip->bip_flags & BIP_PAGE_PINNED)
+		bio_integrity_unpin_bvec(orig_bvecs, orig_nr_vecs);
 }
 
 /**
  * bio_integrity_unmap_user - Unmap user integrity payload
  * @bio:	bio containing bip to be unmapped
@@ -207,11 +208,12 @@ void bio_integrity_unmap_user(struct bio *bio)
 			bio_integrity_uncopy_user(bip);
 		kfree(bvec_virt(bip->bip_vec));
 		return;
 	}
 
-	bio_integrity_unpin_bvec(bip->bip_vec, bip->bip_max_vcnt);
+	if (bip->bip_flags & BIP_PAGE_PINNED)
+		bio_integrity_unpin_bvec(bip->bip_vec, bip->bip_max_vcnt);
 }
 
 /**
  * bio_integrity_add_page - Attach integrity metadata
  * @bio:	bio to update
@@ -257,11 +259,11 @@ int bio_integrity_add_page(struct bio *bio, struct page *page,
 	return len;
 }
 EXPORT_SYMBOL(bio_integrity_add_page);
 
 static int bio_integrity_copy_user(struct bio *bio, struct bio_vec *bvec,
-				   int nr_vecs, unsigned int len)
+				   int nr_vecs, unsigned int len, bool pinned)
 {
 	bool write = op_is_write(bio_op(bio));
 	struct bio_integrity_payload *bip;
 	struct iov_iter iter;
 	void *buf;
@@ -292,14 +294,18 @@ static int bio_integrity_copy_user(struct bio *bio, struct bio_vec *bvec,
 	if (IS_ERR(bip)) {
 		ret = PTR_ERR(bip);
 		goto free_buf;
 	}
 
-	if (write)
-		bio_integrity_unpin_bvec(bvec, nr_vecs);
-	else
+	if (write) {
+		if (pinned)
+			bio_integrity_unpin_bvec(bvec, nr_vecs);
+	} else {
 		memcpy(&bip->bip_vec[1], bvec, nr_vecs * sizeof(*bvec));
+		if (pinned)
+			bip->bip_flags |= BIP_PAGE_PINNED;
+	}
 
 	ret = bio_integrity_add_page(bio, virt_to_page(buf), len,
 				     offset_in_page(buf));
 	WARN_ON_ONCE(ret != len);
 	bip->bip_flags |= BIP_COPY_USER;
@@ -308,27 +314,29 @@ static int bio_integrity_copy_user(struct bio *bio, struct bio_vec *bvec,
 	kfree(buf);
 	return ret;
 }
 
 static int bio_integrity_init_user(struct bio *bio, struct bio_vec *bvec,
-				   int nr_vecs, unsigned int len)
+				   int nr_vecs, unsigned int len, bool pinned)
 {
 	struct bio_integrity_payload *bip;
 
 	bip = bio_integrity_alloc(bio, GFP_KERNEL, nr_vecs);
 	if (IS_ERR(bip))
 		return PTR_ERR(bip);
 
 	memcpy(bip->bip_vec, bvec, nr_vecs * sizeof(*bvec));
 	bip->bip_iter.bi_size = len;
 	bip->bip_vcnt = nr_vecs;
+	if (pinned)
+		bip->bip_flags |= BIP_PAGE_PINNED;
 	return 0;
 }
 
 static unsigned int bvec_from_pages(struct bio_vec *bvec, struct page **pages,
 				    int nr_vecs, ssize_t bytes, ssize_t offset,
-				    bool *is_p2p)
+				    bool *is_p2p, bool pinned)
 {
 	unsigned int nr_bvecs = 0;
 	int i, j;
 
 	for (i = 0; i < nr_vecs; i = j) {
@@ -340,11 +348,12 @@ static unsigned int bvec_from_pages(struct bio_vec *bvec, struct page **pages,
 			size_t next = min_t(size_t, PAGE_SIZE, bytes);
 
 			if (page_folio(pages[j]) != folio ||
 			    pages[j] != pages[j - 1] + 1)
 				break;
-			unpin_user_page(pages[j]);
+			if (pinned)
+				unpin_user_page(pages[j]);
 			size += next;
 			bytes -= next;
 		}
 
 		if (is_pci_p2pdma_page(pages[i]))
@@ -361,10 +370,11 @@ static unsigned int bvec_from_pages(struct bio_vec *bvec, struct page **pages,
 int bio_integrity_map_user(struct bio *bio, struct iov_iter *iter)
 {
 	struct request_queue *q = bdev_get_queue(bio->bi_bdev);
 	struct page *stack_pages[UIO_FASTIOV], **pages = stack_pages;
 	struct bio_vec stack_vec[UIO_FASTIOV], *bvec = stack_vec;
+	bool pinned = iov_iter_extract_will_pin(iter);
 	iov_iter_extraction_t extraction_flags = 0;
 	size_t offset, bytes = iter->count;
 	bool copy, is_p2p = false;
 	unsigned int nr_bvecs;
 	int ret, nr_vecs;
@@ -397,12 +407,12 @@ int bio_integrity_map_user(struct bio *bio, struct iov_iter *iter)
 
 	/*
 	 * Handle partial pinning. This can happen when pin_user_pages_fast()
 	 * returns fewer pages than requested.
 	 */
-	if (user_backed_iter(iter) && unlikely(ret != bytes)) {
-		if (ret > 0) {
+	if (unlikely(ret != bytes)) {
+		if (pinned && ret > 0) {
 			int npinned = DIV_ROUND_UP(offset + ret, PAGE_SIZE);
 			int i;
 
 			for (i = 0; i < npinned; i++)
 				unpin_user_page(pages[i]);
@@ -412,31 +422,34 @@ int bio_integrity_map_user(struct bio *bio, struct iov_iter *iter)
 		ret = -EFAULT;
 		goto free_bvec;
 	}
 
 	nr_bvecs = bvec_from_pages(bvec, pages, nr_vecs, bytes, offset,
-				   &is_p2p);
+				   &is_p2p, pinned);
 	if (pages != stack_pages)
 		kvfree(pages);
 	if (nr_bvecs > queue_max_integrity_segments(q))
 		copy = true;
 	if (is_p2p)
 		bio->bi_opf |= REQ_NOMERGE;
 
 	if (copy)
-		ret = bio_integrity_copy_user(bio, bvec, nr_bvecs, bytes);
+		ret = bio_integrity_copy_user(bio, bvec, nr_bvecs, bytes,
+					      pinned);
 	else
-		ret = bio_integrity_init_user(bio, bvec, nr_bvecs, bytes);
+		ret = bio_integrity_init_user(bio, bvec, nr_bvecs, bytes,
+					      pinned);
 	if (ret)
 		goto release_pages;
 	if (bvec != stack_vec)
 		kfree(bvec);
 
 	return 0;
 
 release_pages:
-	bio_integrity_unpin_bvec(bvec, nr_bvecs);
+	if (pinned)
+		bio_integrity_unpin_bvec(bvec, nr_bvecs);
 free_bvec:
 	if (bvec != stack_vec)
 		kfree(bvec);
 	return ret;
 }
diff --git a/block/blk-integrity.c b/block/blk-integrity.c
index 964eebbee14d..6977bfd96927 100644
--- a/block/blk-integrity.c
+++ b/block/blk-integrity.c
@@ -118,18 +118,15 @@ int blk_get_meta_cap(struct block_device *bdev, unsigned int cmd,
 out:
 	return copy_struct_to_user(argp, usize, &meta_cap, sizeof(meta_cap),
 				   NULL);
 }
 
-int blk_rq_integrity_map_user(struct request *rq, void __user *ubuf,
-			      ssize_t bytes)
+int blk_rq_integrity_map_user(struct request *rq, struct iov_iter *iter)
 {
 	int ret;
-	struct iov_iter iter;
 
-	iov_iter_ubuf(&iter, rq_data_dir(rq), ubuf, bytes);
-	ret = bio_integrity_map_user(rq->bio, &iter);
+	ret = bio_integrity_map_user(rq->bio, iter);
 	if (ret)
 		return ret;
 
 	rq->nr_integrity_segments = blk_rq_count_integrity_sg(rq->q, rq->bio);
 	rq->cmd_flags |= REQ_INTEGRITY;
diff --git a/drivers/nvme/host/ioctl.c b/drivers/nvme/host/ioctl.c
index 748a4cbf7e90..e69c49c334a0 100644
--- a/drivers/nvme/host/ioctl.c
+++ b/drivers/nvme/host/ioctl.c
@@ -175,11 +175,15 @@ static int nvme_map_user_request(struct request *req, u64 ubuffer,
 				0, rq_data_dir(req));
 	if (ret)
 		return ret;
 
 	if (has_metadata) {
-		ret = blk_rq_integrity_map_user(req, meta_buffer, meta_len);
+		struct iov_iter meta_iter;
+
+		iov_iter_ubuf(&meta_iter, rq_data_dir(req), meta_buffer,
+			      meta_len);
+		ret = blk_rq_integrity_map_user(req, &meta_iter);
 		if (ret)
 			goto out_unmap;
 	}
 
 	return ret;
diff --git a/include/linux/bio-integrity.h b/include/linux/bio-integrity.h
index 0ea2a8bf7efb..740acb8c1077 100644
--- a/include/linux/bio-integrity.h
+++ b/include/linux/bio-integrity.h
@@ -12,10 +12,11 @@ enum bip_flags {
 	BIP_COPY_USER		= 1 << 4, /* Kernel bounce buffer in use */
 	BIP_CHECK_GUARD		= 1 << 5, /* guard check */
 	BIP_CHECK_REFTAG	= 1 << 6, /* reftag check */
 	BIP_CHECK_APPTAG	= 1 << 7, /* apptag check */
 
+	BIP_PAGE_PINNED		= 1 << 14, /* pages need to be unpinned */
 	BIP_MEMPOOL		= 1 << 15, /* buffer backed by mempool */
 };
 
 /* flags that require generate/verify action. */
 #define BIP_CHECK_FLAGS (BIP_CHECK_GUARD | BIP_CHECK_REFTAG | BIP_CHECK_APPTAG)
diff --git a/include/linux/blk-integrity.h b/include/linux/blk-integrity.h
index b1b530613c34..8b655ac165e8 100644
--- a/include/linux/blk-integrity.h
+++ b/include/linux/blk-integrity.h
@@ -28,12 +28,11 @@ static inline bool queue_limits_stack_integrity_bdev(struct queue_limits *t,
 
 #ifdef CONFIG_BLK_DEV_INTEGRITY
 int blk_rq_map_integrity_sg(struct request *, struct scatterlist *);
 
 int blk_rq_count_integrity_sg(struct request_queue *, struct bio *);
-int blk_rq_integrity_map_user(struct request *rq, void __user *ubuf,
-			      ssize_t bytes);
+int blk_rq_integrity_map_user(struct request *rq, struct iov_iter *iter);
 int blk_get_meta_cap(struct block_device *bdev, unsigned int cmd,
 		     struct logical_block_metadata_cap __user *argp);
 bool blk_rq_integrity_dma_map_iter_start(struct request *req,
 		struct device *dma_dev,  struct dma_iova_state *state,
 		struct blk_dma_iter *iter);
@@ -116,12 +115,11 @@ static inline int blk_rq_map_integrity_sg(struct request *q,
 					  struct scatterlist *s)
 {
 	return 0;
 }
 static inline int blk_rq_integrity_map_user(struct request *rq,
-					    void __user *ubuf,
-					    ssize_t bytes)
+					    struct iov_iter *iter)
 {
 	return -EINVAL;
 }
 static inline bool blk_rq_integrity_dma_map_iter_start(struct request *req,
 		struct device *dma_dev,  struct dma_iova_state *state,
-- 
2.55.0

[PATCH 4/6] nvme/ioctl: pass iov_iter to nvme_map_user_request()

[Caleb Sander Mateos]

To allow nvme_map_user_request() to accept an io_uring registered buffer
(represented as a ITER_BVEC iov_iter) for the metadata buffer, pass it
as an iov_iter argument. Move the conversions from user address-length
pairs into iov_iters to the callers. A NULL iov_iter pointer represents
no metadata buffer.

Signed-off-by: Caleb Sander Mateos <csander@purestorage.com>
---
 drivers/nvme/host/ioctl.c | 46 +++++++++++++++++++++++----------------
 1 file changed, 27 insertions(+), 19 deletions(-)

diff --git a/drivers/nvme/host/ioctl.c b/drivers/nvme/host/ioctl.c
index e69c49c334a0..8e722184ecf1 100644
--- a/drivers/nvme/host/ioctl.c
+++ b/drivers/nvme/host/ioctl.c
@@ -152,21 +152,20 @@ static struct request *nvme_alloc_user_request(struct request_queue *q,
 	nvme_req(req)->flags |= NVME_REQ_USERCMD;
 	return req;
 }
 
 static int nvme_map_user_request(struct request *req, u64 ubuffer,
-		unsigned bufflen, void __user *meta_buffer, unsigned meta_len,
-		struct iov_iter *iter, unsigned int flags)
+		unsigned bufflen, struct iov_iter *iter,
+		struct iov_iter *meta_iter, unsigned int flags)
 {
 	struct request_queue *q = req->q;
 	struct nvme_ns *ns = q->queuedata;
 	struct block_device *bdev = ns ? ns->disk->part0 : NULL;
 	bool supports_metadata = bdev && blk_get_integrity(bdev->bd_disk);
-	bool has_metadata = meta_buffer && meta_len;
 	int ret;
 
-	if (has_metadata && !supports_metadata)
+	if (meta_iter && !supports_metadata)
 		return -EINVAL;
 
 	if (iter)
 		ret = blk_rq_map_user_iov(q, req, NULL, iter, GFP_KERNEL);
 	else
@@ -174,16 +173,12 @@ static int nvme_map_user_request(struct request *req, u64 ubuffer,
 				bufflen, GFP_KERNEL, flags & NVME_IOCTL_VEC, 0,
 				0, rq_data_dir(req));
 	if (ret)
 		return ret;
 
-	if (has_metadata) {
-		struct iov_iter meta_iter;
-
-		iov_iter_ubuf(&meta_iter, rq_data_dir(req), meta_buffer,
-			      meta_len);
-		ret = blk_rq_integrity_map_user(req, &meta_iter);
+	if (meta_iter) {
+		ret = blk_rq_integrity_map_user(req, meta_iter);
 		if (ret)
 			goto out_unmap;
 	}
 
 	return ret;
@@ -210,12 +205,20 @@ static int nvme_submit_user_cmd(struct request_queue *q,
 	if (IS_ERR(req))
 		return PTR_ERR(req);
 
 	req->timeout = timeout;
 	if (ubuffer && bufflen) {
-		ret = nvme_map_user_request(req, ubuffer, bufflen, meta_buffer,
-				meta_len, NULL, flags);
+		struct iov_iter meta_iter;
+		struct iov_iter *map_meta_iter = NULL;
+
+		if (meta_buffer && meta_len) {
+			iov_iter_ubuf(&meta_iter, rq_data_dir(req), meta_buffer,
+				      meta_len);
+			map_meta_iter = &meta_iter;
+		}
+		ret = nvme_map_user_request(req, ubuffer, bufflen, NULL,
+					    map_meta_iter, flags);
 		if (ret)
 			goto out_free_req;
 	}
 
 	bio = req->bio;
@@ -480,18 +483,19 @@ static int nvme_uring_cmd_io(struct nvme_ctrl *ctrl, struct nvme_ns *ns,
 	const struct nvme_uring_cmd *cmd = io_uring_sqe128_cmd(ioucmd->sqe,
 							       struct nvme_uring_cmd);
 	struct request_queue *q = ns ? ns->queue : ctrl->admin_q;
 	bool open_for_write = ioucmd->file->f_mode & FMODE_WRITE;
 	struct nvme_command c;
-	struct iov_iter iter;
-	struct iov_iter *map_iter = NULL;
+	struct iov_iter iter, meta_iter;
+	struct iov_iter *map_iter = NULL, *map_meta_iter = NULL;
 	struct request *req;
 	blk_opf_t rq_flags = 0;
 	blk_mq_req_flags_t blk_flags = 0;
 	u32 metadata_len, data_len;
 	u64 metadata, addr;
 	u32 timeout_ms;
+	int ddir;
 	int ret;
 
 	c.common.opcode = READ_ONCE(cmd->opcode);
 	c.common.flags = READ_ONCE(cmd->flags);
 	if (c.common.flags)
@@ -520,13 +524,12 @@ static int nvme_uring_cmd_io(struct nvme_ctrl *ctrl, struct nvme_ns *ns,
 	addr = READ_ONCE(cmd->addr);
 	data_len = READ_ONCE(cmd->data_len);
 	metadata_len = READ_ONCE(cmd->metadata_len);
 	timeout_ms = READ_ONCE(cmd->timeout_ms);
 
+	ddir = nvme_is_write(&c) ? WRITE : READ;
 	if (data_len && (ioucmd->flags & IORING_URING_CMD_FIXED)) {
-		int ddir = nvme_is_write(&c) ? WRITE : READ;
-
 		if (vec)
 			ret = io_uring_cmd_import_fixed_vec(ioucmd,
 					u64_to_user_ptr(addr), data_len,
 					ddir, &iter, issue_flags);
 		else
@@ -535,10 +538,15 @@ static int nvme_uring_cmd_io(struct nvme_ctrl *ctrl, struct nvme_ns *ns,
 		if (ret < 0)
 			return ret;
 
 		map_iter = &iter;
 	}
+	if (data_len && metadata && metadata_len) {
+		iov_iter_ubuf(&meta_iter, ddir, nvme_to_user_ptr(metadata),
+			      metadata_len);
+		map_meta_iter = &meta_iter;
+	}
 
 	if (issue_flags & IO_URING_F_NONBLOCK) {
 		rq_flags |= REQ_NOWAIT;
 		blk_flags = BLK_MQ_REQ_NOWAIT;
 	}
@@ -549,13 +557,13 @@ static int nvme_uring_cmd_io(struct nvme_ctrl *ctrl, struct nvme_ns *ns,
 	if (IS_ERR(req))
 		return PTR_ERR(req);
 	req->timeout = timeout_ms ? msecs_to_jiffies(timeout_ms) : 0;
 
 	if (data_len) {
-		ret = nvme_map_user_request(req, addr, data_len,
-			nvme_to_user_ptr(metadata), metadata_len,
-			map_iter, vec ? NVME_IOCTL_VEC : 0);
+		ret = nvme_map_user_request(req, addr, data_len, map_iter,
+					    map_meta_iter,
+					    vec ? NVME_IOCTL_VEC : 0);
 		if (ret)
 			goto out_free_req;
 	}
 
 	/* to free bio on completion, as req->bio will be null at that time */
-- 
2.55.0

[PATCH 5/6] io_uring/cmd: support fixed buffer for metadata

[Caleb Sander Mateos]

Allow a "metadata" io_uring fixed buffer to be imported by uring_cmds in
addition to the existing data buffer node. NVMe passthrough requests
will use the new buffer node for their metadata buffers if requested.

Provide a function io_uring_cmd_import_fixed_metadata() analogous to
io_uring_cmd_import_fixed() that initializes an iov_iter for a user
address range contained within a fixed buffer. The buffer node is stored
in previously unused space in struct io_uring_cmd so it can be reused if
imported multiple times and released once the uring_cmd completes.

Whereas a fixed data buffer is indicated by io_uring_sqe's
uring_cmd_flags bit IORING_URING_CMD_FIXED and buf_index, how the fixed
metadata buffer is specified is left up to the uring_cmd implementation.

io_get_buf_node() is split out of io_find_buf_node() and moved to rsrc.h
so it can be reused for the metadata buffer. io_import_fixed() is made
extern so it can be called from uring_cmd.c, and io_rsrc_node is passed
instead of io_mapped_ubuf so uring_cmd.c doesn't have to reach into
io_rsrc_node.

Signed-off-by: Caleb Sander Mateos <csander@purestorage.com>
---
 include/linux/io_uring/cmd.h | 12 +++++++++++-
 io_uring/rsrc.c              | 28 +++++++++++-----------------
 io_uring/rsrc.h              | 17 +++++++++++++++++
 io_uring/uring_cmd.c         | 30 +++++++++++++++++++++++++++++-
 4 files changed, 68 insertions(+), 19 deletions(-)

diff --git a/include/linux/io_uring/cmd.h b/include/linux/io_uring/cmd.h
index 2e4368d611ee..6df5f3bf5628 100644
--- a/include/linux/io_uring/cmd.h
+++ b/include/linux/io_uring/cmd.h
@@ -16,11 +16,11 @@ struct io_uring_cmd {
 	struct file	*file;
 	const struct io_uring_sqe *sqe;
 	u32		cmd_op;
 	u32		flags;
 	u8		pdu[32]; /* available inline for free use */
-	u8		unused[8];
+	struct io_rsrc_node *metadata_node;
 };
 
 #define io_uring_sqe128_cmd(sqe, type)	({					\
 	BUILD_BUG_ON(sizeof(type) > ((2 * sizeof(struct io_uring_sqe)) -	\
 				     offsetof(struct io_uring_sqe, cmd)));	\
@@ -68,10 +68,14 @@ int io_uring_cmd_import_fixed(u64 ubuf, unsigned long len, int rw,
 int io_uring_cmd_import_fixed_vec(struct io_uring_cmd *ioucmd,
 				  const struct iovec __user *uvec,
 				  size_t uvec_segs,
 				  int ddir, struct iov_iter *iter,
 				  unsigned issue_flags);
+int io_uring_cmd_import_fixed_metadata(struct io_uring_cmd *ioucmd,
+				       u16 buf_index, u64 ubuf, size_t len,
+				       int ddir, struct iov_iter *iter,
+				       unsigned int issue_flags);
 
 /*
  * Completes the request, i.e. posts an io_uring CQE and deallocates @ioucmd
  * and the corresponding io_uring request.
  *
@@ -125,10 +129,16 @@ static inline int io_uring_cmd_import_fixed_vec(struct io_uring_cmd *ioucmd,
 						int ddir, struct iov_iter *iter,
 						unsigned issue_flags)
 {
 	return -EOPNOTSUPP;
 }
+static inline int io_uring_cmd_import_fixed_metadata(
+	struct io_uring_cmd *ioucmd, u16 buf_index, u64 ubuf, size_t len,
+	int ddir, struct iov_iter *iter, unsigned int issue_flags)
+{
+	return -EOPNOTSUPP;
+}
 static inline void __io_uring_cmd_done(struct io_uring_cmd *cmd,
 				       unsigned issue_flags)
 {
 }
 static inline void __io_uring_cmd_do_in_task(struct io_uring_cmd *ioucmd,
diff --git a/io_uring/rsrc.c b/io_uring/rsrc.c
index 174f74cbbf60..4bd69803c470 100644
--- a/io_uring/rsrc.c
+++ b/io_uring/rsrc.c
@@ -1134,14 +1134,14 @@ static int io_import_kbuf(int ddir, struct iov_iter *iter,
 	iov_iter_bvec(iter, ddir, imu->bvec, imu->nr_bvecs, count);
 	iov_iter_advance(iter, offset);
 	return 0;
 }
 
-static int io_import_fixed(int ddir, struct iov_iter *iter,
-			   struct io_mapped_ubuf *imu,
-			   u64 buf_addr, size_t len)
+int io_import_fixed(int ddir, struct iov_iter *iter, struct io_rsrc_node *node,
+		    u64 buf_addr, size_t len)
 {
+	struct io_mapped_ubuf *imu = node->buf;
 	const struct bio_vec *bvec;
 	size_t folio_mask;
 	unsigned nr_segs;
 	size_t offset;
 	int ret;
@@ -1189,28 +1189,22 @@ static int io_import_fixed(int ddir, struct iov_iter *iter,
 }
 
 inline struct io_rsrc_node *io_find_buf_node(struct io_kiocb *req,
 					     unsigned issue_flags)
 {
-	struct io_ring_ctx *ctx = req->ctx;
 	struct io_rsrc_node *node;
 
 	if (req->flags & REQ_F_BUF_NODE)
 		return req->buf_node;
-	req->flags |= REQ_F_BUF_NODE;
 
-	io_ring_submit_lock(ctx, issue_flags);
-	node = io_rsrc_node_lookup(&ctx->buf_table, req->buf_index);
-	if (node) {
-		node->refs++;
-		req->buf_node = node;
-		io_ring_submit_unlock(ctx, issue_flags);
-		return node;
-	}
-	req->flags &= ~REQ_F_BUF_NODE;
-	io_ring_submit_unlock(ctx, issue_flags);
-	return NULL;
+	node = io_get_buf_node(req, req->buf_index, issue_flags);
+	if (!node)
+		return NULL;
+
+	req->flags |= REQ_F_BUF_NODE;
+	req->buf_node = node;
+	return node;
 }
 
 int io_import_reg_buf(struct io_kiocb *req, struct iov_iter *iter,
 			u64 buf_addr, size_t len, int ddir,
 			unsigned issue_flags)
@@ -1218,11 +1212,11 @@ int io_import_reg_buf(struct io_kiocb *req, struct iov_iter *iter,
 	struct io_rsrc_node *node;
 
 	node = io_find_buf_node(req, issue_flags);
 	if (!node)
 		return -EFAULT;
-	return io_import_fixed(ddir, iter, node->buf, buf_addr, len);
+	return io_import_fixed(ddir, iter, node, buf_addr, len);
 }
 
 static int io_buffer_acct_cloned_hpages(struct io_ring_ctx *ctx,
 					struct io_mapped_ubuf *imu)
 {
diff --git a/io_uring/rsrc.h b/io_uring/rsrc.h
index eacfdb70f203..277e2007803d 100644
--- a/io_uring/rsrc.h
+++ b/io_uring/rsrc.h
@@ -4,10 +4,11 @@
 
 #include <linux/bvec.h>
 #include <linux/io_uring_types.h>
 #include <linux/lockdep.h>
 #include <linux/uio.h>
+#include "io_uring.h"
 
 #define IO_VEC_CACHE_SOFT_CAP		256
 
 enum {
 	IORING_RSRC_FILE		= 0,
@@ -64,10 +65,12 @@ void io_free_rsrc_node(struct io_ring_ctx *ctx, struct io_rsrc_node *node);
 void io_rsrc_data_free(struct io_ring_ctx *ctx, struct io_rsrc_data *data);
 int io_rsrc_data_alloc(struct io_rsrc_data *data, unsigned nr);
 
 struct io_rsrc_node *io_find_buf_node(struct io_kiocb *req,
 				      unsigned issue_flags);
+int io_import_fixed(int ddir, struct iov_iter *iter, struct io_rsrc_node *node,
+		    u64 buf_addr, size_t len);
 int io_import_reg_buf(struct io_kiocb *req, struct iov_iter *iter,
 			u64 buf_addr, size_t len, int ddir,
 			unsigned issue_flags);
 int io_import_reg_vec(int ddir, struct iov_iter *iter,
 			struct io_kiocb *req, struct iou_vec *vec,
@@ -100,10 +103,24 @@ static inline struct io_rsrc_node *io_rsrc_node_lookup(struct io_rsrc_data *data
 	if (index < data->nr)
 		return data->nodes[array_index_nospec(index, data->nr)];
 	return NULL;
 }
 
+static inline struct io_rsrc_node *
+io_get_buf_node(const struct io_kiocb *req, u16 buf_index, unsigned issue_flags)
+{
+	struct io_ring_ctx *ctx = req->ctx;
+	struct io_rsrc_node *node;
+
+	io_ring_submit_lock(ctx, issue_flags);
+	node = io_rsrc_node_lookup(&ctx->buf_table, buf_index);
+	if (node)
+		node->refs++;
+	io_ring_submit_unlock(ctx, issue_flags);
+	return node;
+}
+
 static inline void io_put_rsrc_node(struct io_ring_ctx *ctx, struct io_rsrc_node *node)
 {
 	lockdep_assert_held(&ctx->uring_lock);
 	if (!--node->refs)
 		io_free_rsrc_node(ctx, node);
diff --git a/io_uring/uring_cmd.c b/io_uring/uring_cmd.c
index 3d5d8b5f4ebb..e4b384094392 100644
--- a/io_uring/uring_cmd.c
+++ b/io_uring/uring_cmd.c
@@ -25,19 +25,25 @@ void io_cmd_cache_free(const void *entry)
 
 static void io_req_uring_cleanup(struct io_kiocb *req, unsigned int issue_flags)
 {
 	struct io_uring_cmd *ioucmd = io_kiocb_to_cmd(req, struct io_uring_cmd);
 	struct io_async_cmd *ac = req->async_data;
+	struct io_ring_ctx *ctx = req->ctx;
 
 	if (issue_flags & IO_URING_F_UNLOCKED)
 		return;
 
+	if (ioucmd->metadata_node) {
+		io_put_rsrc_node(ctx, ioucmd->metadata_node);
+		ioucmd->metadata_node = NULL;
+	}
+
 	io_alloc_cache_vec_kasan(&ac->vec);
 	if (ac->vec.nr > IO_VEC_CACHE_SOFT_CAP)
 		io_vec_free(&ac->vec);
 
-	if (io_alloc_cache_put(&req->ctx->cmd_cache, ac)) {
+	if (io_alloc_cache_put(&ctx->cmd_cache, ac)) {
 		ioucmd->sqe = NULL;
 		io_req_async_data_clear(req, REQ_F_NEED_CLEANUP);
 	} else {
 		io_vec_free(&ac->vec);
 	}
@@ -194,10 +200,11 @@ int io_uring_cmd_prep(struct io_kiocb *req, const struct io_uring_sqe *sqe)
 	if (!ac)
 		return -ENOMEM;
 	if (ac->vec.iovec)
 		req->flags |= REQ_F_NEED_CLEANUP;
 	ioucmd->sqe = sqe;
+	ioucmd->metadata_node = NULL;
 	return 0;
 }
 
 /*
  * IORING_SETUP_SQE128 contexts allocate twice the normal SQE size for each
@@ -303,10 +310,31 @@ int io_uring_cmd_import_fixed_vec(struct io_uring_cmd *ioucmd,
 	return io_import_reg_vec(ddir, iter, req, &ac->vec, uvec_segs,
 				 issue_flags);
 }
 EXPORT_SYMBOL_GPL(io_uring_cmd_import_fixed_vec);
 
+int io_uring_cmd_import_fixed_metadata(struct io_uring_cmd *ioucmd,
+				       u16 buf_index, u64 ubuf, size_t len,
+				       int ddir, struct iov_iter *iter,
+				       unsigned int issue_flags)
+{
+	struct io_rsrc_node *buf_node = ioucmd->metadata_node;
+
+	if (!buf_node) {
+		struct io_kiocb *req = cmd_to_io_kiocb(ioucmd);
+
+		buf_node = io_get_buf_node(req, buf_index, issue_flags);
+		if (!buf_node)
+			return -EFAULT;
+
+		req->flags |= REQ_F_NEED_CLEANUP;
+		ioucmd->metadata_node = buf_node;
+	}
+	return io_import_fixed(ddir, iter, buf_node, ubuf, len);
+}
+EXPORT_SYMBOL_GPL(io_uring_cmd_import_fixed_metadata);
+
 void io_uring_cmd_issue_blocking(struct io_uring_cmd *ioucmd)
 {
 	struct io_kiocb *req = cmd_to_io_kiocb(ioucmd);
 
 	io_queue_iowq(req);
-- 
2.55.0

[PATCH 6/6] nvme/ioctl: support fixed buffer for metadata

[Caleb Sander Mateos]

io_uring NVMe passthrough supports using a "fixed" (registered) buffer
for data, but not metadata. On high-IOPS workloads, the pinning and
unpinning overhead for the metadata pages is significant and could be
avoided if fixed metadata buffers were supported.

Define a NVME_URING_CMD_FIXED_METADATA bit for the nvme_uring_cmd's
flags field (which is currently required to be 0) to indicate that the
metadata buffer belongs to a fixed buffer registered with the io_uring.
The metadata fixed buffer index is specified in a metadata_buf_index
field replacing the existing rsvd1 in nvme_uring_cmd.

If NVME_URING_CMD_FIXED_METADATA is set, the metadata iov_iter is
obtained from io_uring_cmd_import_fixed_metadata() instead of
iov_iter_ubuf().

Signed-off-by: Caleb Sander Mateos <csander@purestorage.com>
---
 drivers/nvme/host/ioctl.c       | 21 ++++++++++++++++-----
 include/uapi/linux/nvme_ioctl.h |  5 ++++-
 2 files changed, 20 insertions(+), 6 deletions(-)

diff --git a/drivers/nvme/host/ioctl.c b/drivers/nvme/host/ioctl.c
index 8e722184ecf1..4da8cf92c2e0 100644
--- a/drivers/nvme/host/ioctl.c
+++ b/drivers/nvme/host/ioctl.c
@@ -488,21 +488,22 @@ static int nvme_uring_cmd_io(struct nvme_ctrl *ctrl, struct nvme_ns *ns,
 	struct iov_iter iter, meta_iter;
 	struct iov_iter *map_iter = NULL, *map_meta_iter = NULL;
 	struct request *req;
 	blk_opf_t rq_flags = 0;
 	blk_mq_req_flags_t blk_flags = 0;
+	u8 flags = READ_ONCE(cmd->flags);
 	u32 metadata_len, data_len;
 	u64 metadata, addr;
 	u32 timeout_ms;
 	int ddir;
 	int ret;
 
-	c.common.opcode = READ_ONCE(cmd->opcode);
-	c.common.flags = READ_ONCE(cmd->flags);
-	if (c.common.flags)
+	if (flags & ~NVME_URING_CMD_FIXED_METADATA)
 		return -EINVAL;
 
+	c.common.opcode = READ_ONCE(cmd->opcode);
+	c.common.flags = 0;
 	c.common.command_id = 0;
 	c.common.nsid = cpu_to_le32(cmd->nsid);
 	if (!nvme_validate_passthru_nsid(ctrl, ns, le32_to_cpu(c.common.nsid)))
 		return -EINVAL;
 
@@ -539,12 +540,22 @@ static int nvme_uring_cmd_io(struct nvme_ctrl *ctrl, struct nvme_ns *ns,
 			return ret;
 
 		map_iter = &iter;
 	}
 	if (data_len && metadata && metadata_len) {
-		iov_iter_ubuf(&meta_iter, ddir, nvme_to_user_ptr(metadata),
-			      metadata_len);
+		if (flags & NVME_URING_CMD_FIXED_METADATA) {
+			u16 buf_index = READ_ONCE(cmd->metadata_buf_index);
+
+			ret = io_uring_cmd_import_fixed_metadata(
+				ioucmd, buf_index, metadata, metadata_len, ddir,
+				&meta_iter, issue_flags);
+			if (ret < 0)
+				return ret;
+		} else {
+			iov_iter_ubuf(&meta_iter, ddir, nvme_to_user_ptr(metadata),
+				      metadata_len);
+		}
 		map_meta_iter = &meta_iter;
 	}
 
 	if (issue_flags & IO_URING_F_NONBLOCK) {
 		rq_flags |= REQ_NOWAIT;
diff --git a/include/uapi/linux/nvme_ioctl.h b/include/uapi/linux/nvme_ioctl.h
index 2f76cba67166..93973f636b48 100644
--- a/include/uapi/linux/nvme_ioctl.h
+++ b/include/uapi/linux/nvme_ioctl.h
@@ -68,15 +68,18 @@ struct nvme_passthru_cmd64 {
 	__u32	timeout_ms;
 	__u32   rsvd2;
 	__u64	result;
 };
 
+/* struct nvme_uring_cmd flags field bits */
+#define NVME_URING_CMD_FIXED_METADATA (1U << 0)
+
 /* same as struct nvme_passthru_cmd64, minus the 8b result field */
 struct nvme_uring_cmd {
 	__u8	opcode;
 	__u8	flags;
-	__u16	rsvd1;
+	__u16	metadata_buf_index;
 	__u32	nsid;
 	__u32	cdw2;
 	__u32	cdw3;
 	__u64	metadata;
 	__u64	addr;
-- 
2.55.0