[PATCH 2/3] io_uring: split req prep and submit loops

[Keith Busch <kbusch@meta.com>]

From: Keith Busch <kbusch@kernel.org>

Do all the prep work up front, then dispatch all synchronous requests at
once. This will make it easier to count batches for plugging.

Signed-off-by: Keith Busch <kbusch@kernel.org>
---
 io_uring/io_uring.c | 26 ++++++++++++++++++--------
 io_uring/slist.h    |  4 ++++
 2 files changed, 22 insertions(+), 8 deletions(-)

diff --git a/io_uring/io_uring.c b/io_uring/io_uring.c
index 818b2d1661c5e..5434aef0a8ef7 100644
--- a/io_uring/io_uring.c
+++ b/io_uring/io_uring.c
@@ -1082,6 +1082,7 @@ static void io_preinit_req(struct io_kiocb *req, struct io_ring_ctx *ctx)
 	req->ctx = ctx;
 	req->link = NULL;
 	req->async_data = NULL;
+	req->comp_list.next = NULL;
 	/* not necessary, but safer to zero */
 	req->cqe.res = 0;
 }
@@ -2282,11 +2283,7 @@ static __cold int io_submit_fail_init(const struct io_uring_sqe *sqe,
 static inline void io_submit_sqe(struct io_kiocb *req)
 {
 	trace_io_uring_submit_req(req);
-
-	if (unlikely(req->flags & (REQ_F_FORCE_ASYNC | REQ_F_FAIL)))
-		io_queue_sqe_fallback(req);
-	else
-		io_queue_sqe(req);
+	io_queue_sqe(req);
 }
 
 static int io_setup_link(struct io_submit_link *link, struct io_kiocb **orig)
@@ -2409,6 +2406,9 @@ int io_submit_sqes(struct io_ring_ctx *ctx, unsigned int nr)
 {
 	struct io_submit_link *link = &ctx->submit_state.link;
 	unsigned int entries = io_sqring_entries(ctx);
+	struct io_wq_work_node *pos, *next;
+	struct io_wq_work_list req_list;
+	struct io_kiocb *req;
 	unsigned int left;
 	int ret, err;
 
@@ -2419,6 +2419,7 @@ int io_submit_sqes(struct io_ring_ctx *ctx, unsigned int nr)
 	io_get_task_refs(left);
 	io_submit_state_start(&ctx->submit_state, left);
 
+	INIT_WQ_LIST(&req_list);
 	do {
 		const struct io_uring_sqe *sqe;
 		struct io_kiocb *req;
@@ -2437,9 +2438,12 @@ int io_submit_sqes(struct io_ring_ctx *ctx, unsigned int nr)
 		err = io_setup_link(link, &req);
 		if (unlikely(err))
 			goto error;
-
-		if (likely(req))
-			io_submit_sqe(req);
+		else if (unlikely(!req))
+			continue;
+		else if (unlikely(req->flags & (REQ_F_FORCE_ASYNC | REQ_F_FAIL)))
+			io_queue_sqe_fallback(req);
+		else
+			wq_list_add_tail(&req->comp_list, &req_list);
 		continue;
 error:
 		/*
@@ -2453,6 +2457,12 @@ int io_submit_sqes(struct io_ring_ctx *ctx, unsigned int nr)
 		}
 	} while (--left);
 
+	wq_list_for_each_safe(pos, next, &req_list) {
+		req = container_of(pos, struct io_kiocb, comp_list);
+		req->comp_list.next = NULL;
+		io_submit_sqe(req);
+	}
+
 	if (unlikely(left)) {
 		ret -= left;
 		/* try again if it submitted nothing and can't allocate a req */
diff --git a/io_uring/slist.h b/io_uring/slist.h
index 0eb194817242e..93fbb715111ca 100644
--- a/io_uring/slist.h
+++ b/io_uring/slist.h
@@ -12,6 +12,10 @@
 #define wq_list_for_each_resume(pos, prv)			\
 	for (; pos; prv = pos, pos = (pos)->next)
 
+#define wq_list_for_each_safe(pos, n, head)			\
+	for (pos = (head)->first, n = pos ? pos->next : NULL;	\
+	     pos; pos = n, n = pos ? pos->next : NULL)
+
 #define wq_list_empty(list)	(READ_ONCE((list)->first) == NULL)
 
 #define INIT_WQ_LIST(list)	do {				\
-- 
2.34.1