[PATCH 09/15] io_uring: keep the submission plug on the io_submit_sqes() stack

[Jens Axboe <axboe@kernel.dk>]

The submission plug is embedded in the ring's submit state, which only
works while a single task runs the whole batch. With a submitter
identity handoff another task finishes the batch, and the block layer
caches current->plug across sleeps. Put the plug on the stack of
io_submit_sqes() instead, it stays with the task that started it.

No functional changes in this patch.

Signed-off-by: Jens Axboe <axboe@kernel.dk>
---
 include/linux/io_uring_types.h |  3 ++-
 io_uring/io_uring.c            | 10 +++++++---
 2 files changed, 9 insertions(+), 4 deletions(-)

diff --git a/include/linux/io_uring_types.h b/include/linux/io_uring_types.h
index 50a4a0ad222f..0b0d73688b8c 100644
--- a/include/linux/io_uring_types.h
+++ b/include/linux/io_uring_types.h
@@ -298,7 +298,8 @@ struct io_submit_state {
 	bool			need_plug;
 	bool			cq_flush;
 	unsigned short		submit_nr;
-	struct blk_plug		plug;
+	/* the submitting task's plug, lives on its stack */
+	struct blk_plug		*plug;
 };
 
 struct io_alloc_cache {
diff --git a/io_uring/io_uring.c b/io_uring/io_uring.c
index b09221e239c0..100ade1eee3e 100644
--- a/io_uring/io_uring.c
+++ b/io_uring/io_uring.c
@@ -1810,7 +1810,7 @@ static int io_init_req(struct io_ring_ctx *ctx, struct io_kiocb *req,
 		if (state->need_plug && def->plug) {
 			state->plug_started = true;
 			state->need_plug = false;
-			blk_start_plug_nr_ios(&state->plug, state->submit_nr);
+			blk_start_plug_nr_ios(state->plug, state->submit_nr);
 		}
 	}
 
@@ -1938,13 +1938,14 @@ static void io_submit_state_end(struct io_ring_ctx *ctx)
 	/* flush only after queuing links as they can generate completions */
 	io_submit_flush_completions(ctx);
 	if (state->plug_started)
-		blk_finish_plug(&state->plug);
+		blk_finish_plug(state->plug);
 }
 
 /*
  * Start submission side cache.
  */
 static void io_submit_state_start(struct io_submit_state *state,
+				  struct blk_plug *plug,
 				  unsigned int max_ios)
 {
 	state->plug_started = false;
@@ -1952,6 +1953,8 @@ static void io_submit_state_start(struct io_submit_state *state,
 	state->submit_nr = max_ios;
 	/* set only head, no need to init link_last in advance */
 	state->link.head = NULL;
+	/* on the submitter's stack, the block layer caches current->plug */
+	state->plug = plug;
 }
 
 static void io_commit_sqring(struct io_ring_ctx *ctx)
@@ -2035,6 +2038,7 @@ int io_submit_sqes(struct io_ring_ctx *ctx, unsigned int nr)
 {
 	unsigned int entries;
 	unsigned int left;
+	struct blk_plug plug;
 
 	if (ctx->flags & IORING_SETUP_SQ_REWIND)
 		entries = ctx->sq_entries;
@@ -2047,7 +2051,7 @@ int io_submit_sqes(struct io_ring_ctx *ctx, unsigned int nr)
 
 	left = entries;
 	io_get_task_refs(left);
-	io_submit_state_start(&ctx->submit_state, left);
+	io_submit_state_start(&ctx->submit_state, &plug, left);
 
 	do {
 		const struct io_uring_sqe *sqe;
-- 
2.55.0