* [PATCH 1/6] bio-integrity: remove dead bio_integrity_copy_user() error path
2026-09-09 22:28 [PATCH 0/6] io_uring/nvme: support fixed buffer for metadata Caleb Sander Mateos
@ 2026-09-09 22:28 ` Caleb Sander Mateos
2026-09-09 22:28 ` [PATCH 2/6] nvme/ioctl: remove struct nvme_uring_data Caleb Sander Mateos
` (4 subsequent siblings)
5 siblings, 0 replies; 7+ messages in thread
From: Caleb Sander Mateos @ 2026-09-09 22:28 UTC (permalink / raw)
To: Jens Axboe, Keith Busch, Christoph Hellwig, Sagi Grimberg
Cc: io-uring, linux-nvme, linux-block, linux-kernel,
Caleb Sander Mateos
If bio_integrity_add_page() in bio_integrity_copy_user() fails in the
write case, bio_integrity_copy_user() will return an error after having
already unpinned the user pages. The release_pages label in
bio_integrity_map_user() would then unpin the pages a second time.
Thankfully, bio_integrity_add_page() can't fail here because the
bio_integrity_payload is freshly allocated, so bip_vcnt is 0. Replace
the unreachable error path with a WARN_ON_ONCE().
Signed-off-by: Caleb Sander Mateos <csander@purestorage.com>
---
block/bio-integrity.c | 8 +-------
1 file changed, 1 insertion(+), 7 deletions(-)
diff --git a/block/bio-integrity.c b/block/bio-integrity.c
index b23e2434d80c..8ba8bfa41f8a 100644
--- a/block/bio-integrity.c
+++ b/block/bio-integrity.c
@@ -299,19 +299,13 @@ static int bio_integrity_copy_user(struct bio *bio, struct bio_vec *bvec,
else
memcpy(&bip->bip_vec[1], bvec, nr_vecs * sizeof(*bvec));
ret = bio_integrity_add_page(bio, virt_to_page(buf), len,
offset_in_page(buf));
- if (ret != len) {
- ret = -ENOMEM;
- goto free_bip;
- }
-
+ WARN_ON_ONCE(ret != len);
bip->bip_flags |= BIP_COPY_USER;
return 0;
-free_bip:
- bio_integrity_free(bio);
free_buf:
kfree(buf);
return ret;
}
--
2.55.0
^ permalink raw reply related [flat|nested] 7+ messages in thread* [PATCH 2/6] nvme/ioctl: remove struct nvme_uring_data
2026-09-09 22:28 [PATCH 0/6] io_uring/nvme: support fixed buffer for metadata Caleb Sander Mateos
2026-09-09 22:28 ` [PATCH 1/6] bio-integrity: remove dead bio_integrity_copy_user() error path Caleb Sander Mateos
@ 2026-09-09 22:28 ` Caleb Sander Mateos
2026-09-09 22:28 ` [PATCH 3/6] blk-integrity: pass iov_iter to blk_rq_integrity_map_user() Caleb Sander Mateos
` (3 subsequent siblings)
5 siblings, 0 replies; 7+ messages in thread
From: Caleb Sander Mateos @ 2026-09-09 22:28 UTC (permalink / raw)
To: Jens Axboe, Keith Busch, Christoph Hellwig, Sagi Grimberg
Cc: io-uring, linux-nvme, linux-block, linux-kernel,
Caleb Sander Mateos
This struct is only used once as a local variable type. Make the fields
separate local variables and remove the struct type to shave some lines.
Signed-off-by: Caleb Sander Mateos <csander@purestorage.com>
---
drivers/nvme/host/ioctl.c | 36 +++++++++++++++---------------------
1 file changed, 15 insertions(+), 21 deletions(-)
diff --git a/drivers/nvme/host/ioctl.c b/drivers/nvme/host/ioctl.c
index 3aa5d2f2dfbb..748a4cbf7e90 100644
--- a/drivers/nvme/host/ioctl.c
+++ b/drivers/nvme/host/ioctl.c
@@ -406,18 +406,10 @@ static int nvme_user_cmd64(struct nvme_ctrl *ctrl, struct nvme_ns *ns,
}
return status;
}
-struct nvme_uring_data {
- __u64 metadata;
- __u64 addr;
- __u32 data_len;
- __u32 metadata_len;
- __u32 timeout_ms;
-};
-
/*
* This overlays struct io_uring_cmd pdu.
* Expect build errors if this grows larger than that.
*/
struct nvme_uring_cmd_pdu {
@@ -483,17 +475,19 @@ static int nvme_uring_cmd_io(struct nvme_ctrl *ctrl, struct nvme_ns *ns,
struct nvme_uring_cmd_pdu *pdu = nvme_uring_cmd_pdu(ioucmd);
const struct nvme_uring_cmd *cmd = io_uring_sqe128_cmd(ioucmd->sqe,
struct nvme_uring_cmd);
struct request_queue *q = ns ? ns->queue : ctrl->admin_q;
bool open_for_write = ioucmd->file->f_mode & FMODE_WRITE;
- struct nvme_uring_data d;
struct nvme_command c;
struct iov_iter iter;
struct iov_iter *map_iter = NULL;
struct request *req;
blk_opf_t rq_flags = 0;
blk_mq_req_flags_t blk_flags = 0;
+ u32 metadata_len, data_len;
+ u64 metadata, addr;
+ u32 timeout_ms;
int ret;
c.common.opcode = READ_ONCE(cmd->opcode);
c.common.flags = READ_ONCE(cmd->flags);
if (c.common.flags)
@@ -516,25 +510,25 @@ static int nvme_uring_cmd_io(struct nvme_ctrl *ctrl, struct nvme_ns *ns,
c.common.cdw15 = cpu_to_le32(READ_ONCE(cmd->cdw15));
if (!nvme_cmd_allowed(ctrl, ns, &c, 0, open_for_write))
return -EACCES;
- d.metadata = READ_ONCE(cmd->metadata);
- d.addr = READ_ONCE(cmd->addr);
- d.data_len = READ_ONCE(cmd->data_len);
- d.metadata_len = READ_ONCE(cmd->metadata_len);
- d.timeout_ms = READ_ONCE(cmd->timeout_ms);
+ metadata = READ_ONCE(cmd->metadata);
+ addr = READ_ONCE(cmd->addr);
+ data_len = READ_ONCE(cmd->data_len);
+ metadata_len = READ_ONCE(cmd->metadata_len);
+ timeout_ms = READ_ONCE(cmd->timeout_ms);
- if (d.data_len && (ioucmd->flags & IORING_URING_CMD_FIXED)) {
+ if (data_len && (ioucmd->flags & IORING_URING_CMD_FIXED)) {
int ddir = nvme_is_write(&c) ? WRITE : READ;
if (vec)
ret = io_uring_cmd_import_fixed_vec(ioucmd,
- u64_to_user_ptr(d.addr), d.data_len,
+ u64_to_user_ptr(addr), data_len,
ddir, &iter, issue_flags);
else
- ret = io_uring_cmd_import_fixed(d.addr, d.data_len,
+ ret = io_uring_cmd_import_fixed(addr, data_len,
ddir, &iter, ioucmd, issue_flags);
if (ret < 0)
return ret;
map_iter = &iter;
@@ -548,15 +542,15 @@ static int nvme_uring_cmd_io(struct nvme_ctrl *ctrl, struct nvme_ns *ns,
rq_flags |= REQ_POLLED;
req = nvme_alloc_user_request(q, &c, rq_flags, blk_flags);
if (IS_ERR(req))
return PTR_ERR(req);
- req->timeout = d.timeout_ms ? msecs_to_jiffies(d.timeout_ms) : 0;
+ req->timeout = timeout_ms ? msecs_to_jiffies(timeout_ms) : 0;
- if (d.data_len) {
- ret = nvme_map_user_request(req, d.addr, d.data_len,
- nvme_to_user_ptr(d.metadata), d.metadata_len,
+ if (data_len) {
+ ret = nvme_map_user_request(req, addr, data_len,
+ nvme_to_user_ptr(metadata), metadata_len,
map_iter, vec ? NVME_IOCTL_VEC : 0);
if (ret)
goto out_free_req;
}
--
2.55.0
^ permalink raw reply related [flat|nested] 7+ messages in thread* [PATCH 3/6] blk-integrity: pass iov_iter to blk_rq_integrity_map_user()
2026-09-09 22:28 [PATCH 0/6] io_uring/nvme: support fixed buffer for metadata Caleb Sander Mateos
2026-09-09 22:28 ` [PATCH 1/6] bio-integrity: remove dead bio_integrity_copy_user() error path Caleb Sander Mateos
2026-09-09 22:28 ` [PATCH 2/6] nvme/ioctl: remove struct nvme_uring_data Caleb Sander Mateos
@ 2026-09-09 22:28 ` Caleb Sander Mateos
2026-09-09 22:28 ` [PATCH 4/6] nvme/ioctl: pass iov_iter to nvme_map_user_request() Caleb Sander Mateos
` (2 subsequent siblings)
5 siblings, 0 replies; 7+ messages in thread
From: Caleb Sander Mateos @ 2026-09-09 22:28 UTC (permalink / raw)
To: Jens Axboe, Keith Busch, Christoph Hellwig, Sagi Grimberg
Cc: io-uring, linux-nvme, linux-block, linux-kernel,
Caleb Sander Mateos
To allow blk_rq_integrity_map_user() to accept an io_uring registered
buffer (represented as a ITER_BVEC iov_iter), take the buffer as an
iov_iter argument. Move the conversion from a user address-length pair
into an iov_iter to the caller.
Since a non-ITER_UBUF iov_iter could be passed in the future,
iov_iter_extract_pages() in bio_integrity_map_user() won't necessarily
pin the pages. Condition the page unpins on iov_iter_extract_will_pin().
Add a BIP_PAGE_PINNED bit to bip_flags to track if the pages are pinned.
Signed-off-by: Caleb Sander Mateos <csander@purestorage.com>
---
block/bio-integrity.c | 43 +++++++++++++++++++++++------------
block/blk-integrity.c | 7 ++----
drivers/nvme/host/ioctl.c | 6 ++++-
include/linux/bio-integrity.h | 1 +
include/linux/blk-integrity.h | 6 ++---
5 files changed, 38 insertions(+), 25 deletions(-)
diff --git a/block/bio-integrity.c b/block/bio-integrity.c
index 8ba8bfa41f8a..3a5554d3ea2f 100644
--- a/block/bio-integrity.c
+++ b/block/bio-integrity.c
@@ -187,11 +187,12 @@ static void bio_integrity_uncopy_user(struct bio_integrity_payload *bip)
iov_iter_bvec(&orig_iter, ITER_DEST, orig_bvecs, orig_nr_vecs, bytes);
ret = copy_to_iter(bvec_virt(bounce_bvec), bytes, &orig_iter);
WARN_ON_ONCE(ret != bytes);
- bio_integrity_unpin_bvec(orig_bvecs, orig_nr_vecs);
+ if (bip->bip_flags & BIP_PAGE_PINNED)
+ bio_integrity_unpin_bvec(orig_bvecs, orig_nr_vecs);
}
/**
* bio_integrity_unmap_user - Unmap user integrity payload
* @bio: bio containing bip to be unmapped
@@ -207,11 +208,12 @@ void bio_integrity_unmap_user(struct bio *bio)
bio_integrity_uncopy_user(bip);
kfree(bvec_virt(bip->bip_vec));
return;
}
- bio_integrity_unpin_bvec(bip->bip_vec, bip->bip_max_vcnt);
+ if (bip->bip_flags & BIP_PAGE_PINNED)
+ bio_integrity_unpin_bvec(bip->bip_vec, bip->bip_max_vcnt);
}
/**
* bio_integrity_add_page - Attach integrity metadata
* @bio: bio to update
@@ -257,11 +259,11 @@ int bio_integrity_add_page(struct bio *bio, struct page *page,
return len;
}
EXPORT_SYMBOL(bio_integrity_add_page);
static int bio_integrity_copy_user(struct bio *bio, struct bio_vec *bvec,
- int nr_vecs, unsigned int len)
+ int nr_vecs, unsigned int len, bool pinned)
{
bool write = op_is_write(bio_op(bio));
struct bio_integrity_payload *bip;
struct iov_iter iter;
void *buf;
@@ -292,14 +294,18 @@ static int bio_integrity_copy_user(struct bio *bio, struct bio_vec *bvec,
if (IS_ERR(bip)) {
ret = PTR_ERR(bip);
goto free_buf;
}
- if (write)
- bio_integrity_unpin_bvec(bvec, nr_vecs);
- else
+ if (write) {
+ if (pinned)
+ bio_integrity_unpin_bvec(bvec, nr_vecs);
+ } else {
memcpy(&bip->bip_vec[1], bvec, nr_vecs * sizeof(*bvec));
+ if (pinned)
+ bip->bip_flags |= BIP_PAGE_PINNED;
+ }
ret = bio_integrity_add_page(bio, virt_to_page(buf), len,
offset_in_page(buf));
WARN_ON_ONCE(ret != len);
bip->bip_flags |= BIP_COPY_USER;
@@ -308,27 +314,29 @@ static int bio_integrity_copy_user(struct bio *bio, struct bio_vec *bvec,
kfree(buf);
return ret;
}
static int bio_integrity_init_user(struct bio *bio, struct bio_vec *bvec,
- int nr_vecs, unsigned int len)
+ int nr_vecs, unsigned int len, bool pinned)
{
struct bio_integrity_payload *bip;
bip = bio_integrity_alloc(bio, GFP_KERNEL, nr_vecs);
if (IS_ERR(bip))
return PTR_ERR(bip);
memcpy(bip->bip_vec, bvec, nr_vecs * sizeof(*bvec));
bip->bip_iter.bi_size = len;
bip->bip_vcnt = nr_vecs;
+ if (pinned)
+ bip->bip_flags |= BIP_PAGE_PINNED;
return 0;
}
static unsigned int bvec_from_pages(struct bio_vec *bvec, struct page **pages,
int nr_vecs, ssize_t bytes, ssize_t offset,
- bool *is_p2p)
+ bool *is_p2p, bool pinned)
{
unsigned int nr_bvecs = 0;
int i, j;
for (i = 0; i < nr_vecs; i = j) {
@@ -340,11 +348,12 @@ static unsigned int bvec_from_pages(struct bio_vec *bvec, struct page **pages,
size_t next = min_t(size_t, PAGE_SIZE, bytes);
if (page_folio(pages[j]) != folio ||
pages[j] != pages[j - 1] + 1)
break;
- unpin_user_page(pages[j]);
+ if (pinned)
+ unpin_user_page(pages[j]);
size += next;
bytes -= next;
}
if (is_pci_p2pdma_page(pages[i]))
@@ -361,10 +370,11 @@ static unsigned int bvec_from_pages(struct bio_vec *bvec, struct page **pages,
int bio_integrity_map_user(struct bio *bio, struct iov_iter *iter)
{
struct request_queue *q = bdev_get_queue(bio->bi_bdev);
struct page *stack_pages[UIO_FASTIOV], **pages = stack_pages;
struct bio_vec stack_vec[UIO_FASTIOV], *bvec = stack_vec;
+ bool pinned = iov_iter_extract_will_pin(iter);
iov_iter_extraction_t extraction_flags = 0;
size_t offset, bytes = iter->count;
bool copy, is_p2p = false;
unsigned int nr_bvecs;
int ret, nr_vecs;
@@ -397,12 +407,12 @@ int bio_integrity_map_user(struct bio *bio, struct iov_iter *iter)
/*
* Handle partial pinning. This can happen when pin_user_pages_fast()
* returns fewer pages than requested.
*/
- if (user_backed_iter(iter) && unlikely(ret != bytes)) {
- if (ret > 0) {
+ if (unlikely(ret != bytes)) {
+ if (pinned && ret > 0) {
int npinned = DIV_ROUND_UP(offset + ret, PAGE_SIZE);
int i;
for (i = 0; i < npinned; i++)
unpin_user_page(pages[i]);
@@ -412,31 +422,34 @@ int bio_integrity_map_user(struct bio *bio, struct iov_iter *iter)
ret = -EFAULT;
goto free_bvec;
}
nr_bvecs = bvec_from_pages(bvec, pages, nr_vecs, bytes, offset,
- &is_p2p);
+ &is_p2p, pinned);
if (pages != stack_pages)
kvfree(pages);
if (nr_bvecs > queue_max_integrity_segments(q))
copy = true;
if (is_p2p)
bio->bi_opf |= REQ_NOMERGE;
if (copy)
- ret = bio_integrity_copy_user(bio, bvec, nr_bvecs, bytes);
+ ret = bio_integrity_copy_user(bio, bvec, nr_bvecs, bytes,
+ pinned);
else
- ret = bio_integrity_init_user(bio, bvec, nr_bvecs, bytes);
+ ret = bio_integrity_init_user(bio, bvec, nr_bvecs, bytes,
+ pinned);
if (ret)
goto release_pages;
if (bvec != stack_vec)
kfree(bvec);
return 0;
release_pages:
- bio_integrity_unpin_bvec(bvec, nr_bvecs);
+ if (pinned)
+ bio_integrity_unpin_bvec(bvec, nr_bvecs);
free_bvec:
if (bvec != stack_vec)
kfree(bvec);
return ret;
}
diff --git a/block/blk-integrity.c b/block/blk-integrity.c
index 964eebbee14d..6977bfd96927 100644
--- a/block/blk-integrity.c
+++ b/block/blk-integrity.c
@@ -118,18 +118,15 @@ int blk_get_meta_cap(struct block_device *bdev, unsigned int cmd,
out:
return copy_struct_to_user(argp, usize, &meta_cap, sizeof(meta_cap),
NULL);
}
-int blk_rq_integrity_map_user(struct request *rq, void __user *ubuf,
- ssize_t bytes)
+int blk_rq_integrity_map_user(struct request *rq, struct iov_iter *iter)
{
int ret;
- struct iov_iter iter;
- iov_iter_ubuf(&iter, rq_data_dir(rq), ubuf, bytes);
- ret = bio_integrity_map_user(rq->bio, &iter);
+ ret = bio_integrity_map_user(rq->bio, iter);
if (ret)
return ret;
rq->nr_integrity_segments = blk_rq_count_integrity_sg(rq->q, rq->bio);
rq->cmd_flags |= REQ_INTEGRITY;
diff --git a/drivers/nvme/host/ioctl.c b/drivers/nvme/host/ioctl.c
index 748a4cbf7e90..e69c49c334a0 100644
--- a/drivers/nvme/host/ioctl.c
+++ b/drivers/nvme/host/ioctl.c
@@ -175,11 +175,15 @@ static int nvme_map_user_request(struct request *req, u64 ubuffer,
0, rq_data_dir(req));
if (ret)
return ret;
if (has_metadata) {
- ret = blk_rq_integrity_map_user(req, meta_buffer, meta_len);
+ struct iov_iter meta_iter;
+
+ iov_iter_ubuf(&meta_iter, rq_data_dir(req), meta_buffer,
+ meta_len);
+ ret = blk_rq_integrity_map_user(req, &meta_iter);
if (ret)
goto out_unmap;
}
return ret;
diff --git a/include/linux/bio-integrity.h b/include/linux/bio-integrity.h
index 0ea2a8bf7efb..740acb8c1077 100644
--- a/include/linux/bio-integrity.h
+++ b/include/linux/bio-integrity.h
@@ -12,10 +12,11 @@ enum bip_flags {
BIP_COPY_USER = 1 << 4, /* Kernel bounce buffer in use */
BIP_CHECK_GUARD = 1 << 5, /* guard check */
BIP_CHECK_REFTAG = 1 << 6, /* reftag check */
BIP_CHECK_APPTAG = 1 << 7, /* apptag check */
+ BIP_PAGE_PINNED = 1 << 14, /* pages need to be unpinned */
BIP_MEMPOOL = 1 << 15, /* buffer backed by mempool */
};
/* flags that require generate/verify action. */
#define BIP_CHECK_FLAGS (BIP_CHECK_GUARD | BIP_CHECK_REFTAG | BIP_CHECK_APPTAG)
diff --git a/include/linux/blk-integrity.h b/include/linux/blk-integrity.h
index b1b530613c34..8b655ac165e8 100644
--- a/include/linux/blk-integrity.h
+++ b/include/linux/blk-integrity.h
@@ -28,12 +28,11 @@ static inline bool queue_limits_stack_integrity_bdev(struct queue_limits *t,
#ifdef CONFIG_BLK_DEV_INTEGRITY
int blk_rq_map_integrity_sg(struct request *, struct scatterlist *);
int blk_rq_count_integrity_sg(struct request_queue *, struct bio *);
-int blk_rq_integrity_map_user(struct request *rq, void __user *ubuf,
- ssize_t bytes);
+int blk_rq_integrity_map_user(struct request *rq, struct iov_iter *iter);
int blk_get_meta_cap(struct block_device *bdev, unsigned int cmd,
struct logical_block_metadata_cap __user *argp);
bool blk_rq_integrity_dma_map_iter_start(struct request *req,
struct device *dma_dev, struct dma_iova_state *state,
struct blk_dma_iter *iter);
@@ -116,12 +115,11 @@ static inline int blk_rq_map_integrity_sg(struct request *q,
struct scatterlist *s)
{
return 0;
}
static inline int blk_rq_integrity_map_user(struct request *rq,
- void __user *ubuf,
- ssize_t bytes)
+ struct iov_iter *iter)
{
return -EINVAL;
}
static inline bool blk_rq_integrity_dma_map_iter_start(struct request *req,
struct device *dma_dev, struct dma_iova_state *state,
--
2.55.0
^ permalink raw reply related [flat|nested] 7+ messages in thread* [PATCH 4/6] nvme/ioctl: pass iov_iter to nvme_map_user_request()
2026-09-09 22:28 [PATCH 0/6] io_uring/nvme: support fixed buffer for metadata Caleb Sander Mateos
` (2 preceding siblings ...)
2026-09-09 22:28 ` [PATCH 3/6] blk-integrity: pass iov_iter to blk_rq_integrity_map_user() Caleb Sander Mateos
@ 2026-09-09 22:28 ` Caleb Sander Mateos
2026-09-09 22:28 ` [PATCH 5/6] io_uring/cmd: support fixed buffer for metadata Caleb Sander Mateos
2026-09-09 22:28 ` [PATCH 6/6] nvme/ioctl: " Caleb Sander Mateos
5 siblings, 0 replies; 7+ messages in thread
From: Caleb Sander Mateos @ 2026-09-09 22:28 UTC (permalink / raw)
To: Jens Axboe, Keith Busch, Christoph Hellwig, Sagi Grimberg
Cc: io-uring, linux-nvme, linux-block, linux-kernel,
Caleb Sander Mateos
To allow nvme_map_user_request() to accept an io_uring registered buffer
(represented as a ITER_BVEC iov_iter) for the metadata buffer, pass it
as an iov_iter argument. Move the conversions from user address-length
pairs into iov_iters to the callers. A NULL iov_iter pointer represents
no metadata buffer.
Signed-off-by: Caleb Sander Mateos <csander@purestorage.com>
---
drivers/nvme/host/ioctl.c | 46 +++++++++++++++++++++++----------------
1 file changed, 27 insertions(+), 19 deletions(-)
diff --git a/drivers/nvme/host/ioctl.c b/drivers/nvme/host/ioctl.c
index e69c49c334a0..8e722184ecf1 100644
--- a/drivers/nvme/host/ioctl.c
+++ b/drivers/nvme/host/ioctl.c
@@ -152,21 +152,20 @@ static struct request *nvme_alloc_user_request(struct request_queue *q,
nvme_req(req)->flags |= NVME_REQ_USERCMD;
return req;
}
static int nvme_map_user_request(struct request *req, u64 ubuffer,
- unsigned bufflen, void __user *meta_buffer, unsigned meta_len,
- struct iov_iter *iter, unsigned int flags)
+ unsigned bufflen, struct iov_iter *iter,
+ struct iov_iter *meta_iter, unsigned int flags)
{
struct request_queue *q = req->q;
struct nvme_ns *ns = q->queuedata;
struct block_device *bdev = ns ? ns->disk->part0 : NULL;
bool supports_metadata = bdev && blk_get_integrity(bdev->bd_disk);
- bool has_metadata = meta_buffer && meta_len;
int ret;
- if (has_metadata && !supports_metadata)
+ if (meta_iter && !supports_metadata)
return -EINVAL;
if (iter)
ret = blk_rq_map_user_iov(q, req, NULL, iter, GFP_KERNEL);
else
@@ -174,16 +173,12 @@ static int nvme_map_user_request(struct request *req, u64 ubuffer,
bufflen, GFP_KERNEL, flags & NVME_IOCTL_VEC, 0,
0, rq_data_dir(req));
if (ret)
return ret;
- if (has_metadata) {
- struct iov_iter meta_iter;
-
- iov_iter_ubuf(&meta_iter, rq_data_dir(req), meta_buffer,
- meta_len);
- ret = blk_rq_integrity_map_user(req, &meta_iter);
+ if (meta_iter) {
+ ret = blk_rq_integrity_map_user(req, meta_iter);
if (ret)
goto out_unmap;
}
return ret;
@@ -210,12 +205,20 @@ static int nvme_submit_user_cmd(struct request_queue *q,
if (IS_ERR(req))
return PTR_ERR(req);
req->timeout = timeout;
if (ubuffer && bufflen) {
- ret = nvme_map_user_request(req, ubuffer, bufflen, meta_buffer,
- meta_len, NULL, flags);
+ struct iov_iter meta_iter;
+ struct iov_iter *map_meta_iter = NULL;
+
+ if (meta_buffer && meta_len) {
+ iov_iter_ubuf(&meta_iter, rq_data_dir(req), meta_buffer,
+ meta_len);
+ map_meta_iter = &meta_iter;
+ }
+ ret = nvme_map_user_request(req, ubuffer, bufflen, NULL,
+ map_meta_iter, flags);
if (ret)
goto out_free_req;
}
bio = req->bio;
@@ -480,18 +483,19 @@ static int nvme_uring_cmd_io(struct nvme_ctrl *ctrl, struct nvme_ns *ns,
const struct nvme_uring_cmd *cmd = io_uring_sqe128_cmd(ioucmd->sqe,
struct nvme_uring_cmd);
struct request_queue *q = ns ? ns->queue : ctrl->admin_q;
bool open_for_write = ioucmd->file->f_mode & FMODE_WRITE;
struct nvme_command c;
- struct iov_iter iter;
- struct iov_iter *map_iter = NULL;
+ struct iov_iter iter, meta_iter;
+ struct iov_iter *map_iter = NULL, *map_meta_iter = NULL;
struct request *req;
blk_opf_t rq_flags = 0;
blk_mq_req_flags_t blk_flags = 0;
u32 metadata_len, data_len;
u64 metadata, addr;
u32 timeout_ms;
+ int ddir;
int ret;
c.common.opcode = READ_ONCE(cmd->opcode);
c.common.flags = READ_ONCE(cmd->flags);
if (c.common.flags)
@@ -520,13 +524,12 @@ static int nvme_uring_cmd_io(struct nvme_ctrl *ctrl, struct nvme_ns *ns,
addr = READ_ONCE(cmd->addr);
data_len = READ_ONCE(cmd->data_len);
metadata_len = READ_ONCE(cmd->metadata_len);
timeout_ms = READ_ONCE(cmd->timeout_ms);
+ ddir = nvme_is_write(&c) ? WRITE : READ;
if (data_len && (ioucmd->flags & IORING_URING_CMD_FIXED)) {
- int ddir = nvme_is_write(&c) ? WRITE : READ;
-
if (vec)
ret = io_uring_cmd_import_fixed_vec(ioucmd,
u64_to_user_ptr(addr), data_len,
ddir, &iter, issue_flags);
else
@@ -535,10 +538,15 @@ static int nvme_uring_cmd_io(struct nvme_ctrl *ctrl, struct nvme_ns *ns,
if (ret < 0)
return ret;
map_iter = &iter;
}
+ if (data_len && metadata && metadata_len) {
+ iov_iter_ubuf(&meta_iter, ddir, nvme_to_user_ptr(metadata),
+ metadata_len);
+ map_meta_iter = &meta_iter;
+ }
if (issue_flags & IO_URING_F_NONBLOCK) {
rq_flags |= REQ_NOWAIT;
blk_flags = BLK_MQ_REQ_NOWAIT;
}
@@ -549,13 +557,13 @@ static int nvme_uring_cmd_io(struct nvme_ctrl *ctrl, struct nvme_ns *ns,
if (IS_ERR(req))
return PTR_ERR(req);
req->timeout = timeout_ms ? msecs_to_jiffies(timeout_ms) : 0;
if (data_len) {
- ret = nvme_map_user_request(req, addr, data_len,
- nvme_to_user_ptr(metadata), metadata_len,
- map_iter, vec ? NVME_IOCTL_VEC : 0);
+ ret = nvme_map_user_request(req, addr, data_len, map_iter,
+ map_meta_iter,
+ vec ? NVME_IOCTL_VEC : 0);
if (ret)
goto out_free_req;
}
/* to free bio on completion, as req->bio will be null at that time */
--
2.55.0
^ permalink raw reply related [flat|nested] 7+ messages in thread* [PATCH 5/6] io_uring/cmd: support fixed buffer for metadata
2026-09-09 22:28 [PATCH 0/6] io_uring/nvme: support fixed buffer for metadata Caleb Sander Mateos
` (3 preceding siblings ...)
2026-09-09 22:28 ` [PATCH 4/6] nvme/ioctl: pass iov_iter to nvme_map_user_request() Caleb Sander Mateos
@ 2026-09-09 22:28 ` Caleb Sander Mateos
2026-09-09 22:28 ` [PATCH 6/6] nvme/ioctl: " Caleb Sander Mateos
5 siblings, 0 replies; 7+ messages in thread
From: Caleb Sander Mateos @ 2026-09-09 22:28 UTC (permalink / raw)
To: Jens Axboe, Keith Busch, Christoph Hellwig, Sagi Grimberg
Cc: io-uring, linux-nvme, linux-block, linux-kernel,
Caleb Sander Mateos
Allow a "metadata" io_uring fixed buffer to be imported by uring_cmds in
addition to the existing data buffer node. NVMe passthrough requests
will use the new buffer node for their metadata buffers if requested.
Provide a function io_uring_cmd_import_fixed_metadata() analogous to
io_uring_cmd_import_fixed() that initializes an iov_iter for a user
address range contained within a fixed buffer. The buffer node is stored
in previously unused space in struct io_uring_cmd so it can be reused if
imported multiple times and released once the uring_cmd completes.
Whereas a fixed data buffer is indicated by io_uring_sqe's
uring_cmd_flags bit IORING_URING_CMD_FIXED and buf_index, how the fixed
metadata buffer is specified is left up to the uring_cmd implementation.
io_get_buf_node() is split out of io_find_buf_node() and moved to rsrc.h
so it can be reused for the metadata buffer. io_import_fixed() is made
extern so it can be called from uring_cmd.c, and io_rsrc_node is passed
instead of io_mapped_ubuf so uring_cmd.c doesn't have to reach into
io_rsrc_node.
Signed-off-by: Caleb Sander Mateos <csander@purestorage.com>
---
include/linux/io_uring/cmd.h | 12 +++++++++++-
io_uring/rsrc.c | 28 +++++++++++-----------------
io_uring/rsrc.h | 17 +++++++++++++++++
io_uring/uring_cmd.c | 30 +++++++++++++++++++++++++++++-
4 files changed, 68 insertions(+), 19 deletions(-)
diff --git a/include/linux/io_uring/cmd.h b/include/linux/io_uring/cmd.h
index 2e4368d611ee..6df5f3bf5628 100644
--- a/include/linux/io_uring/cmd.h
+++ b/include/linux/io_uring/cmd.h
@@ -16,11 +16,11 @@ struct io_uring_cmd {
struct file *file;
const struct io_uring_sqe *sqe;
u32 cmd_op;
u32 flags;
u8 pdu[32]; /* available inline for free use */
- u8 unused[8];
+ struct io_rsrc_node *metadata_node;
};
#define io_uring_sqe128_cmd(sqe, type) ({ \
BUILD_BUG_ON(sizeof(type) > ((2 * sizeof(struct io_uring_sqe)) - \
offsetof(struct io_uring_sqe, cmd))); \
@@ -68,10 +68,14 @@ int io_uring_cmd_import_fixed(u64 ubuf, unsigned long len, int rw,
int io_uring_cmd_import_fixed_vec(struct io_uring_cmd *ioucmd,
const struct iovec __user *uvec,
size_t uvec_segs,
int ddir, struct iov_iter *iter,
unsigned issue_flags);
+int io_uring_cmd_import_fixed_metadata(struct io_uring_cmd *ioucmd,
+ u16 buf_index, u64 ubuf, size_t len,
+ int ddir, struct iov_iter *iter,
+ unsigned int issue_flags);
/*
* Completes the request, i.e. posts an io_uring CQE and deallocates @ioucmd
* and the corresponding io_uring request.
*
@@ -125,10 +129,16 @@ static inline int io_uring_cmd_import_fixed_vec(struct io_uring_cmd *ioucmd,
int ddir, struct iov_iter *iter,
unsigned issue_flags)
{
return -EOPNOTSUPP;
}
+static inline int io_uring_cmd_import_fixed_metadata(
+ struct io_uring_cmd *ioucmd, u16 buf_index, u64 ubuf, size_t len,
+ int ddir, struct iov_iter *iter, unsigned int issue_flags)
+{
+ return -EOPNOTSUPP;
+}
static inline void __io_uring_cmd_done(struct io_uring_cmd *cmd,
unsigned issue_flags)
{
}
static inline void __io_uring_cmd_do_in_task(struct io_uring_cmd *ioucmd,
diff --git a/io_uring/rsrc.c b/io_uring/rsrc.c
index 174f74cbbf60..4bd69803c470 100644
--- a/io_uring/rsrc.c
+++ b/io_uring/rsrc.c
@@ -1134,14 +1134,14 @@ static int io_import_kbuf(int ddir, struct iov_iter *iter,
iov_iter_bvec(iter, ddir, imu->bvec, imu->nr_bvecs, count);
iov_iter_advance(iter, offset);
return 0;
}
-static int io_import_fixed(int ddir, struct iov_iter *iter,
- struct io_mapped_ubuf *imu,
- u64 buf_addr, size_t len)
+int io_import_fixed(int ddir, struct iov_iter *iter, struct io_rsrc_node *node,
+ u64 buf_addr, size_t len)
{
+ struct io_mapped_ubuf *imu = node->buf;
const struct bio_vec *bvec;
size_t folio_mask;
unsigned nr_segs;
size_t offset;
int ret;
@@ -1189,28 +1189,22 @@ static int io_import_fixed(int ddir, struct iov_iter *iter,
}
inline struct io_rsrc_node *io_find_buf_node(struct io_kiocb *req,
unsigned issue_flags)
{
- struct io_ring_ctx *ctx = req->ctx;
struct io_rsrc_node *node;
if (req->flags & REQ_F_BUF_NODE)
return req->buf_node;
- req->flags |= REQ_F_BUF_NODE;
- io_ring_submit_lock(ctx, issue_flags);
- node = io_rsrc_node_lookup(&ctx->buf_table, req->buf_index);
- if (node) {
- node->refs++;
- req->buf_node = node;
- io_ring_submit_unlock(ctx, issue_flags);
- return node;
- }
- req->flags &= ~REQ_F_BUF_NODE;
- io_ring_submit_unlock(ctx, issue_flags);
- return NULL;
+ node = io_get_buf_node(req, req->buf_index, issue_flags);
+ if (!node)
+ return NULL;
+
+ req->flags |= REQ_F_BUF_NODE;
+ req->buf_node = node;
+ return node;
}
int io_import_reg_buf(struct io_kiocb *req, struct iov_iter *iter,
u64 buf_addr, size_t len, int ddir,
unsigned issue_flags)
@@ -1218,11 +1212,11 @@ int io_import_reg_buf(struct io_kiocb *req, struct iov_iter *iter,
struct io_rsrc_node *node;
node = io_find_buf_node(req, issue_flags);
if (!node)
return -EFAULT;
- return io_import_fixed(ddir, iter, node->buf, buf_addr, len);
+ return io_import_fixed(ddir, iter, node, buf_addr, len);
}
static int io_buffer_acct_cloned_hpages(struct io_ring_ctx *ctx,
struct io_mapped_ubuf *imu)
{
diff --git a/io_uring/rsrc.h b/io_uring/rsrc.h
index eacfdb70f203..277e2007803d 100644
--- a/io_uring/rsrc.h
+++ b/io_uring/rsrc.h
@@ -4,10 +4,11 @@
#include <linux/bvec.h>
#include <linux/io_uring_types.h>
#include <linux/lockdep.h>
#include <linux/uio.h>
+#include "io_uring.h"
#define IO_VEC_CACHE_SOFT_CAP 256
enum {
IORING_RSRC_FILE = 0,
@@ -64,10 +65,12 @@ void io_free_rsrc_node(struct io_ring_ctx *ctx, struct io_rsrc_node *node);
void io_rsrc_data_free(struct io_ring_ctx *ctx, struct io_rsrc_data *data);
int io_rsrc_data_alloc(struct io_rsrc_data *data, unsigned nr);
struct io_rsrc_node *io_find_buf_node(struct io_kiocb *req,
unsigned issue_flags);
+int io_import_fixed(int ddir, struct iov_iter *iter, struct io_rsrc_node *node,
+ u64 buf_addr, size_t len);
int io_import_reg_buf(struct io_kiocb *req, struct iov_iter *iter,
u64 buf_addr, size_t len, int ddir,
unsigned issue_flags);
int io_import_reg_vec(int ddir, struct iov_iter *iter,
struct io_kiocb *req, struct iou_vec *vec,
@@ -100,10 +103,24 @@ static inline struct io_rsrc_node *io_rsrc_node_lookup(struct io_rsrc_data *data
if (index < data->nr)
return data->nodes[array_index_nospec(index, data->nr)];
return NULL;
}
+static inline struct io_rsrc_node *
+io_get_buf_node(const struct io_kiocb *req, u16 buf_index, unsigned issue_flags)
+{
+ struct io_ring_ctx *ctx = req->ctx;
+ struct io_rsrc_node *node;
+
+ io_ring_submit_lock(ctx, issue_flags);
+ node = io_rsrc_node_lookup(&ctx->buf_table, buf_index);
+ if (node)
+ node->refs++;
+ io_ring_submit_unlock(ctx, issue_flags);
+ return node;
+}
+
static inline void io_put_rsrc_node(struct io_ring_ctx *ctx, struct io_rsrc_node *node)
{
lockdep_assert_held(&ctx->uring_lock);
if (!--node->refs)
io_free_rsrc_node(ctx, node);
diff --git a/io_uring/uring_cmd.c b/io_uring/uring_cmd.c
index 3d5d8b5f4ebb..e4b384094392 100644
--- a/io_uring/uring_cmd.c
+++ b/io_uring/uring_cmd.c
@@ -25,19 +25,25 @@ void io_cmd_cache_free(const void *entry)
static void io_req_uring_cleanup(struct io_kiocb *req, unsigned int issue_flags)
{
struct io_uring_cmd *ioucmd = io_kiocb_to_cmd(req, struct io_uring_cmd);
struct io_async_cmd *ac = req->async_data;
+ struct io_ring_ctx *ctx = req->ctx;
if (issue_flags & IO_URING_F_UNLOCKED)
return;
+ if (ioucmd->metadata_node) {
+ io_put_rsrc_node(ctx, ioucmd->metadata_node);
+ ioucmd->metadata_node = NULL;
+ }
+
io_alloc_cache_vec_kasan(&ac->vec);
if (ac->vec.nr > IO_VEC_CACHE_SOFT_CAP)
io_vec_free(&ac->vec);
- if (io_alloc_cache_put(&req->ctx->cmd_cache, ac)) {
+ if (io_alloc_cache_put(&ctx->cmd_cache, ac)) {
ioucmd->sqe = NULL;
io_req_async_data_clear(req, REQ_F_NEED_CLEANUP);
} else {
io_vec_free(&ac->vec);
}
@@ -194,10 +200,11 @@ int io_uring_cmd_prep(struct io_kiocb *req, const struct io_uring_sqe *sqe)
if (!ac)
return -ENOMEM;
if (ac->vec.iovec)
req->flags |= REQ_F_NEED_CLEANUP;
ioucmd->sqe = sqe;
+ ioucmd->metadata_node = NULL;
return 0;
}
/*
* IORING_SETUP_SQE128 contexts allocate twice the normal SQE size for each
@@ -303,10 +310,31 @@ int io_uring_cmd_import_fixed_vec(struct io_uring_cmd *ioucmd,
return io_import_reg_vec(ddir, iter, req, &ac->vec, uvec_segs,
issue_flags);
}
EXPORT_SYMBOL_GPL(io_uring_cmd_import_fixed_vec);
+int io_uring_cmd_import_fixed_metadata(struct io_uring_cmd *ioucmd,
+ u16 buf_index, u64 ubuf, size_t len,
+ int ddir, struct iov_iter *iter,
+ unsigned int issue_flags)
+{
+ struct io_rsrc_node *buf_node = ioucmd->metadata_node;
+
+ if (!buf_node) {
+ struct io_kiocb *req = cmd_to_io_kiocb(ioucmd);
+
+ buf_node = io_get_buf_node(req, buf_index, issue_flags);
+ if (!buf_node)
+ return -EFAULT;
+
+ req->flags |= REQ_F_NEED_CLEANUP;
+ ioucmd->metadata_node = buf_node;
+ }
+ return io_import_fixed(ddir, iter, buf_node, ubuf, len);
+}
+EXPORT_SYMBOL_GPL(io_uring_cmd_import_fixed_metadata);
+
void io_uring_cmd_issue_blocking(struct io_uring_cmd *ioucmd)
{
struct io_kiocb *req = cmd_to_io_kiocb(ioucmd);
io_queue_iowq(req);
--
2.55.0
^ permalink raw reply related [flat|nested] 7+ messages in thread* [PATCH 6/6] nvme/ioctl: support fixed buffer for metadata
2026-09-09 22:28 [PATCH 0/6] io_uring/nvme: support fixed buffer for metadata Caleb Sander Mateos
` (4 preceding siblings ...)
2026-09-09 22:28 ` [PATCH 5/6] io_uring/cmd: support fixed buffer for metadata Caleb Sander Mateos
@ 2026-09-09 22:28 ` Caleb Sander Mateos
5 siblings, 0 replies; 7+ messages in thread
From: Caleb Sander Mateos @ 2026-09-09 22:28 UTC (permalink / raw)
To: Jens Axboe, Keith Busch, Christoph Hellwig, Sagi Grimberg
Cc: io-uring, linux-nvme, linux-block, linux-kernel,
Caleb Sander Mateos
io_uring NVMe passthrough supports using a "fixed" (registered) buffer
for data, but not metadata. On high-IOPS workloads, the pinning and
unpinning overhead for the metadata pages is significant and could be
avoided if fixed metadata buffers were supported.
Define a NVME_URING_CMD_FIXED_METADATA bit for the nvme_uring_cmd's
flags field (which is currently required to be 0) to indicate that the
metadata buffer belongs to a fixed buffer registered with the io_uring.
The metadata fixed buffer index is specified in a metadata_buf_index
field replacing the existing rsvd1 in nvme_uring_cmd.
If NVME_URING_CMD_FIXED_METADATA is set, the metadata iov_iter is
obtained from io_uring_cmd_import_fixed_metadata() instead of
iov_iter_ubuf().
Signed-off-by: Caleb Sander Mateos <csander@purestorage.com>
---
drivers/nvme/host/ioctl.c | 21 ++++++++++++++++-----
include/uapi/linux/nvme_ioctl.h | 5 ++++-
2 files changed, 20 insertions(+), 6 deletions(-)
diff --git a/drivers/nvme/host/ioctl.c b/drivers/nvme/host/ioctl.c
index 8e722184ecf1..4da8cf92c2e0 100644
--- a/drivers/nvme/host/ioctl.c
+++ b/drivers/nvme/host/ioctl.c
@@ -488,21 +488,22 @@ static int nvme_uring_cmd_io(struct nvme_ctrl *ctrl, struct nvme_ns *ns,
struct iov_iter iter, meta_iter;
struct iov_iter *map_iter = NULL, *map_meta_iter = NULL;
struct request *req;
blk_opf_t rq_flags = 0;
blk_mq_req_flags_t blk_flags = 0;
+ u8 flags = READ_ONCE(cmd->flags);
u32 metadata_len, data_len;
u64 metadata, addr;
u32 timeout_ms;
int ddir;
int ret;
- c.common.opcode = READ_ONCE(cmd->opcode);
- c.common.flags = READ_ONCE(cmd->flags);
- if (c.common.flags)
+ if (flags & ~NVME_URING_CMD_FIXED_METADATA)
return -EINVAL;
+ c.common.opcode = READ_ONCE(cmd->opcode);
+ c.common.flags = 0;
c.common.command_id = 0;
c.common.nsid = cpu_to_le32(cmd->nsid);
if (!nvme_validate_passthru_nsid(ctrl, ns, le32_to_cpu(c.common.nsid)))
return -EINVAL;
@@ -539,12 +540,22 @@ static int nvme_uring_cmd_io(struct nvme_ctrl *ctrl, struct nvme_ns *ns,
return ret;
map_iter = &iter;
}
if (data_len && metadata && metadata_len) {
- iov_iter_ubuf(&meta_iter, ddir, nvme_to_user_ptr(metadata),
- metadata_len);
+ if (flags & NVME_URING_CMD_FIXED_METADATA) {
+ u16 buf_index = READ_ONCE(cmd->metadata_buf_index);
+
+ ret = io_uring_cmd_import_fixed_metadata(
+ ioucmd, buf_index, metadata, metadata_len, ddir,
+ &meta_iter, issue_flags);
+ if (ret < 0)
+ return ret;
+ } else {
+ iov_iter_ubuf(&meta_iter, ddir, nvme_to_user_ptr(metadata),
+ metadata_len);
+ }
map_meta_iter = &meta_iter;
}
if (issue_flags & IO_URING_F_NONBLOCK) {
rq_flags |= REQ_NOWAIT;
diff --git a/include/uapi/linux/nvme_ioctl.h b/include/uapi/linux/nvme_ioctl.h
index 2f76cba67166..93973f636b48 100644
--- a/include/uapi/linux/nvme_ioctl.h
+++ b/include/uapi/linux/nvme_ioctl.h
@@ -68,15 +68,18 @@ struct nvme_passthru_cmd64 {
__u32 timeout_ms;
__u32 rsvd2;
__u64 result;
};
+/* struct nvme_uring_cmd flags field bits */
+#define NVME_URING_CMD_FIXED_METADATA (1U << 0)
+
/* same as struct nvme_passthru_cmd64, minus the 8b result field */
struct nvme_uring_cmd {
__u8 opcode;
__u8 flags;
- __u16 rsvd1;
+ __u16 metadata_buf_index;
__u32 nsid;
__u32 cdw2;
__u32 cdw3;
__u64 metadata;
__u64 addr;
--
2.55.0
^ permalink raw reply related [flat|nested] 7+ messages in thread