* [PATCH 1/4] io_uring: cleanly separate request types for iopoll
2022-09-03 16:52 [PATCHSET v2 0/4] Fixups/improvements for iopoll passthrough Jens Axboe
@ 2022-09-03 16:52 ` Jens Axboe
2022-09-04 15:44 ` Kanchan Joshi
2022-09-03 16:52 ` [PATCH 2/4] io_uring: add local task_work run helper that is entered locked Jens Axboe
` (2 subsequent siblings)
3 siblings, 1 reply; 7+ messages in thread
From: Jens Axboe @ 2022-09-03 16:52 UTC (permalink / raw)
To: io-uring; +Cc: joshi.k, Jens Axboe
After the addition of iopoll support for passthrough, there's a bit of
a mixup here. Clean it up and get rid of the casting for the passthrough
command type.
Signed-off-by: Jens Axboe <[email protected]>
---
io_uring/rw.c | 15 +++++++++------
1 file changed, 9 insertions(+), 6 deletions(-)
diff --git a/io_uring/rw.c b/io_uring/rw.c
index 9698a789b3d5..966c923bc0be 100644
--- a/io_uring/rw.c
+++ b/io_uring/rw.c
@@ -994,7 +994,7 @@ int io_do_iopoll(struct io_ring_ctx *ctx, bool force_nonspin)
wq_list_for_each(pos, start, &ctx->iopoll_list) {
struct io_kiocb *req = container_of(pos, struct io_kiocb, comp_list);
- struct io_rw *rw = io_kiocb_to_cmd(req, struct io_rw);
+ struct file *file = req->file;
int ret;
/*
@@ -1006,12 +1006,15 @@ int io_do_iopoll(struct io_ring_ctx *ctx, bool force_nonspin)
break;
if (req->opcode == IORING_OP_URING_CMD) {
- struct io_uring_cmd *ioucmd = (struct io_uring_cmd *)rw;
+ struct io_uring_cmd *ioucmd;
- ret = req->file->f_op->uring_cmd_iopoll(ioucmd);
- } else
- ret = rw->kiocb.ki_filp->f_op->iopoll(&rw->kiocb, &iob,
- poll_flags);
+ ioucmd = io_kiocb_to_cmd(req, struct io_uring_cmd);
+ ret = file->f_op->uring_cmd_iopoll(ioucmd);
+ } else {
+ struct io_rw *rw = io_kiocb_to_cmd(req, struct io_rw);
+
+ ret = file->f_op->iopoll(&rw->kiocb, &iob, poll_flags);
+ }
if (unlikely(ret < 0))
return ret;
else if (ret)
--
2.35.1
^ permalink raw reply related [flat|nested] 7+ messages in thread
* [PATCH 2/4] io_uring: add local task_work run helper that is entered locked
2022-09-03 16:52 [PATCHSET v2 0/4] Fixups/improvements for iopoll passthrough Jens Axboe
2022-09-03 16:52 ` [PATCH 1/4] io_uring: cleanly separate request types for iopoll Jens Axboe
@ 2022-09-03 16:52 ` Jens Axboe
2022-09-03 16:52 ` [PATCH 3/4] io_uring: ensure iopoll runs local task work as well Jens Axboe
2022-09-03 16:52 ` [PATCH 4/4] fs: add batch and poll flags to the uring_cmd_iopoll() handler Jens Axboe
3 siblings, 0 replies; 7+ messages in thread
From: Jens Axboe @ 2022-09-03 16:52 UTC (permalink / raw)
To: io-uring; +Cc: joshi.k, Jens Axboe
We have a few spots that drop the mutex just to run local task_work,
which immediately tries to grab it again. Add a helper that just passes
in whether we're locked already.
Signed-off-by: Jens Axboe <[email protected]>
---
io_uring/io_uring.c | 23 ++++++++++++++++-------
io_uring/io_uring.h | 1 +
2 files changed, 17 insertions(+), 7 deletions(-)
diff --git a/io_uring/io_uring.c b/io_uring/io_uring.c
index 4edc31d0a3e0..f841f0e126bc 100644
--- a/io_uring/io_uring.c
+++ b/io_uring/io_uring.c
@@ -1161,9 +1161,8 @@ static void __cold io_move_task_work_from_local(struct io_ring_ctx *ctx)
}
}
-int io_run_local_work(struct io_ring_ctx *ctx)
+int __io_run_local_work(struct io_ring_ctx *ctx, bool locked)
{
- bool locked;
struct llist_node *node;
struct llist_node fake;
struct llist_node *current_final = NULL;
@@ -1178,8 +1177,6 @@ int io_run_local_work(struct io_ring_ctx *ctx)
return -EEXIST;
}
- locked = mutex_trylock(&ctx->uring_lock);
-
node = io_llist_xchg(&ctx->work_llist, &fake);
ret = 0;
again:
@@ -1204,12 +1201,24 @@ int io_run_local_work(struct io_ring_ctx *ctx)
goto again;
}
- if (locked) {
+ if (locked)
io_submit_flush_completions(ctx);
- mutex_unlock(&ctx->uring_lock);
- }
trace_io_uring_local_work_run(ctx, ret, loops);
return ret;
+
+}
+
+int io_run_local_work(struct io_ring_ctx *ctx)
+{
+ bool locked;
+ int ret;
+
+ locked = mutex_trylock(&ctx->uring_lock);
+ ret = __io_run_local_work(ctx, locked);
+ if (locked)
+ mutex_unlock(&ctx->uring_lock);
+
+ return ret;
}
static void io_req_tw_post(struct io_kiocb *req, bool *locked)
diff --git a/io_uring/io_uring.h b/io_uring/io_uring.h
index f417d75d7bc1..0f90d1dfa42b 100644
--- a/io_uring/io_uring.h
+++ b/io_uring/io_uring.h
@@ -27,6 +27,7 @@ enum {
struct io_uring_cqe *__io_get_cqe(struct io_ring_ctx *ctx);
bool io_req_cqe_overflow(struct io_kiocb *req);
int io_run_task_work_sig(struct io_ring_ctx *ctx);
+int __io_run_local_work(struct io_ring_ctx *ctx, bool locked);
int io_run_local_work(struct io_ring_ctx *ctx);
void io_req_complete_failed(struct io_kiocb *req, s32 res);
void __io_req_complete(struct io_kiocb *req, unsigned issue_flags);
--
2.35.1
^ permalink raw reply related [flat|nested] 7+ messages in thread
* [PATCH 3/4] io_uring: ensure iopoll runs local task work as well
2022-09-03 16:52 [PATCHSET v2 0/4] Fixups/improvements for iopoll passthrough Jens Axboe
2022-09-03 16:52 ` [PATCH 1/4] io_uring: cleanly separate request types for iopoll Jens Axboe
2022-09-03 16:52 ` [PATCH 2/4] io_uring: add local task_work run helper that is entered locked Jens Axboe
@ 2022-09-03 16:52 ` Jens Axboe
2022-09-03 16:52 ` [PATCH 4/4] fs: add batch and poll flags to the uring_cmd_iopoll() handler Jens Axboe
3 siblings, 0 replies; 7+ messages in thread
From: Jens Axboe @ 2022-09-03 16:52 UTC (permalink / raw)
To: io-uring; +Cc: joshi.k, Jens Axboe
Combine the two checks we have for task_work running and whether or not
we need to shuffle the mutex into one, so we unify how task_work is run
in the iopoll loop. This helps ensure that local task_work is run when
needed, and also optimizes that path to avoid a mutex shuffle if it's
not needed.
Signed-off-by: Jens Axboe <[email protected]>
---
io_uring/io_uring.c | 39 ++++++++++++++++++++-------------------
io_uring/io_uring.h | 6 ++++++
2 files changed, 26 insertions(+), 19 deletions(-)
diff --git a/io_uring/io_uring.c b/io_uring/io_uring.c
index f841f0e126bc..118db2264189 100644
--- a/io_uring/io_uring.c
+++ b/io_uring/io_uring.c
@@ -1428,25 +1428,26 @@ static int io_iopoll_check(struct io_ring_ctx *ctx, long min)
* forever, while the workqueue is stuck trying to acquire the
* very same mutex.
*/
- if (wq_list_empty(&ctx->iopoll_list)) {
- u32 tail = ctx->cached_cq_tail;
-
- mutex_unlock(&ctx->uring_lock);
- ret = io_run_task_work_ctx(ctx);
- mutex_lock(&ctx->uring_lock);
- if (ret < 0)
- break;
-
- /* some requests don't go through iopoll_list */
- if (tail != ctx->cached_cq_tail ||
- wq_list_empty(&ctx->iopoll_list))
- break;
- }
-
- if (task_work_pending(current)) {
- mutex_unlock(&ctx->uring_lock);
- io_run_task_work();
- mutex_lock(&ctx->uring_lock);
+ if (wq_list_empty(&ctx->iopoll_list) ||
+ io_task_work_pending(ctx)) {
+ if (!llist_empty(&ctx->work_llist))
+ __io_run_local_work(ctx, true);
+ if (task_work_pending(current) ||
+ wq_list_empty(&ctx->iopoll_list)) {
+ u32 tail = ctx->cached_cq_tail;
+
+ mutex_unlock(&ctx->uring_lock);
+ ret = io_run_task_work();
+ mutex_lock(&ctx->uring_lock);
+
+ if (ret < 0)
+ break;
+
+ /* some requests don't go through iopoll_list */
+ if (tail != ctx->cached_cq_tail ||
+ wq_list_empty(&ctx->iopoll_list))
+ break;
+ }
}
ret = io_do_iopoll(ctx, !min);
if (ret < 0)
diff --git a/io_uring/io_uring.h b/io_uring/io_uring.h
index 0f90d1dfa42b..9d89425292b7 100644
--- a/io_uring/io_uring.h
+++ b/io_uring/io_uring.h
@@ -236,6 +236,12 @@ static inline int io_run_task_work(void)
return 0;
}
+static inline bool io_task_work_pending(struct io_ring_ctx *ctx)
+{
+ return test_thread_flag(TIF_NOTIFY_SIGNAL) ||
+ !wq_list_empty(&ctx->work_llist);
+}
+
static inline int io_run_task_work_ctx(struct io_ring_ctx *ctx)
{
int ret = 0;
--
2.35.1
^ permalink raw reply related [flat|nested] 7+ messages in thread
* [PATCH 4/4] fs: add batch and poll flags to the uring_cmd_iopoll() handler
2022-09-03 16:52 [PATCHSET v2 0/4] Fixups/improvements for iopoll passthrough Jens Axboe
` (2 preceding siblings ...)
2022-09-03 16:52 ` [PATCH 3/4] io_uring: ensure iopoll runs local task work as well Jens Axboe
@ 2022-09-03 16:52 ` Jens Axboe
2022-09-04 15:50 ` Kanchan Joshi
3 siblings, 1 reply; 7+ messages in thread
From: Jens Axboe @ 2022-09-03 16:52 UTC (permalink / raw)
To: io-uring; +Cc: joshi.k, Jens Axboe
We need the poll_flags to know how to poll for the IO, and we should
have the batch structure in preparation for supporting batched
completions with iopoll.
Signed-off-by: Jens Axboe <[email protected]>
---
drivers/nvme/host/ioctl.c | 12 ++++++++----
drivers/nvme/host/nvme.h | 6 ++++--
include/linux/fs.h | 3 ++-
io_uring/rw.c | 3 ++-
4 files changed, 16 insertions(+), 8 deletions(-)
diff --git a/drivers/nvme/host/ioctl.c b/drivers/nvme/host/ioctl.c
index 7756b439a688..548aca8b5b9f 100644
--- a/drivers/nvme/host/ioctl.c
+++ b/drivers/nvme/host/ioctl.c
@@ -623,7 +623,9 @@ int nvme_ns_chr_uring_cmd(struct io_uring_cmd *ioucmd, unsigned int issue_flags)
return nvme_ns_uring_cmd(ns, ioucmd, issue_flags);
}
-int nvme_ns_chr_uring_cmd_iopoll(struct io_uring_cmd *ioucmd)
+int nvme_ns_chr_uring_cmd_iopoll(struct io_uring_cmd *ioucmd,
+ struct io_comp_batch *iob,
+ unsigned int poll_flags)
{
struct bio *bio;
int ret = 0;
@@ -636,7 +638,7 @@ int nvme_ns_chr_uring_cmd_iopoll(struct io_uring_cmd *ioucmd)
struct nvme_ns, cdev);
q = ns->queue;
if (test_bit(QUEUE_FLAG_POLL, &q->queue_flags) && bio && bio->bi_bdev)
- ret = bio_poll(bio, NULL, 0);
+ ret = bio_poll(bio, iob, poll_flags);
rcu_read_unlock();
return ret;
}
@@ -722,7 +724,9 @@ int nvme_ns_head_chr_uring_cmd(struct io_uring_cmd *ioucmd,
return ret;
}
-int nvme_ns_head_chr_uring_cmd_iopoll(struct io_uring_cmd *ioucmd)
+int nvme_ns_head_chr_uring_cmd_iopoll(struct io_uring_cmd *ioucmd,
+ struct io_comp_batch *iob,
+ unsigned int poll_flags)
{
struct cdev *cdev = file_inode(ioucmd->file)->i_cdev;
struct nvme_ns_head *head = container_of(cdev, struct nvme_ns_head, cdev);
@@ -738,7 +742,7 @@ int nvme_ns_head_chr_uring_cmd_iopoll(struct io_uring_cmd *ioucmd)
q = ns->queue;
if (test_bit(QUEUE_FLAG_POLL, &q->queue_flags) && bio
&& bio->bi_bdev)
- ret = bio_poll(bio, NULL, 0);
+ ret = bio_poll(bio, iob, poll_flags);
rcu_read_unlock();
}
srcu_read_unlock(&head->srcu, srcu_idx);
diff --git a/drivers/nvme/host/nvme.h b/drivers/nvme/host/nvme.h
index fdcbc93dea21..216acbe953b3 100644
--- a/drivers/nvme/host/nvme.h
+++ b/drivers/nvme/host/nvme.h
@@ -821,8 +821,10 @@ long nvme_ns_head_chr_ioctl(struct file *file, unsigned int cmd,
unsigned long arg);
long nvme_dev_ioctl(struct file *file, unsigned int cmd,
unsigned long arg);
-int nvme_ns_chr_uring_cmd_iopoll(struct io_uring_cmd *ioucmd);
-int nvme_ns_head_chr_uring_cmd_iopoll(struct io_uring_cmd *ioucmd);
+int nvme_ns_chr_uring_cmd_iopoll(struct io_uring_cmd *ioucmd,
+ struct io_comp_batch *iob, unsigned int poll_flags);
+int nvme_ns_head_chr_uring_cmd_iopoll(struct io_uring_cmd *ioucmd,
+ struct io_comp_batch *iob, unsigned int poll_flags);
int nvme_ns_chr_uring_cmd(struct io_uring_cmd *ioucmd,
unsigned int issue_flags);
int nvme_ns_head_chr_uring_cmd(struct io_uring_cmd *ioucmd,
diff --git a/include/linux/fs.h b/include/linux/fs.h
index d6badd19784f..01681d061a6a 100644
--- a/include/linux/fs.h
+++ b/include/linux/fs.h
@@ -2132,7 +2132,8 @@ struct file_operations {
loff_t len, unsigned int remap_flags);
int (*fadvise)(struct file *, loff_t, loff_t, int);
int (*uring_cmd)(struct io_uring_cmd *ioucmd, unsigned int issue_flags);
- int (*uring_cmd_iopoll)(struct io_uring_cmd *ioucmd);
+ int (*uring_cmd_iopoll)(struct io_uring_cmd *, struct io_comp_batch *,
+ unsigned int poll_flags);
} __randomize_layout;
struct inode_operations {
diff --git a/io_uring/rw.c b/io_uring/rw.c
index 966c923bc0be..4a061326c664 100644
--- a/io_uring/rw.c
+++ b/io_uring/rw.c
@@ -1009,7 +1009,8 @@ int io_do_iopoll(struct io_ring_ctx *ctx, bool force_nonspin)
struct io_uring_cmd *ioucmd;
ioucmd = io_kiocb_to_cmd(req, struct io_uring_cmd);
- ret = file->f_op->uring_cmd_iopoll(ioucmd);
+ ret = file->f_op->uring_cmd_iopoll(ioucmd, &iob,
+ poll_flags);
} else {
struct io_rw *rw = io_kiocb_to_cmd(req, struct io_rw);
--
2.35.1
^ permalink raw reply related [flat|nested] 7+ messages in thread