[PATCH v2 1/2] io_uring: defer submission ref put

[Pavel Begunkov <asml.silence@gmail.com>]

Signed-off-by: Pavel Begunkov <asml.silence@gmail.com>
---
 fs/io_uring.c | 27 ++++++++++++---------------
 1 file changed, 12 insertions(+), 15 deletions(-)

diff --git a/fs/io_uring.c b/fs/io_uring.c
index 43b00077dbd3..9c8e1e773a34 100644
--- a/fs/io_uring.c
+++ b/fs/io_uring.c
@@ -2124,7 +2124,7 @@ static void io_req_free_batch(struct req_batch *rb, struct io_kiocb *req,
 static void io_submit_flush_completions(struct io_comp_state *cs,
 					struct io_ring_ctx *ctx)
 {
-	int i, nr = cs->nr;
+	int refs, i, nr = cs->nr;
 	struct io_kiocb *req;
 	struct req_batch rb;
 
@@ -2132,8 +2132,9 @@ static void io_submit_flush_completions(struct io_comp_state *cs,
 	spin_lock_irq(&ctx->completion_lock);
 	for (i = 0; i < nr; i++) {
 		req = cs->reqs[i];
-		__io_cqring_fill_event(ctx, req->user_data, req->result,
-					req->compl.cflags);
+		if (req->flags & REQ_F_COMPLETE_INLINE)
+			__io_cqring_fill_event(ctx, req->user_data, req->result,
+						req->compl.cflags);
 	}
 	io_commit_cqring(ctx);
 	spin_unlock_irq(&ctx->completion_lock);
@@ -2141,9 +2142,10 @@ static void io_submit_flush_completions(struct io_comp_state *cs,
 	io_cqring_ev_posted(ctx);
 	for (i = 0; i < nr; i++) {
 		req = cs->reqs[i];
+		refs = 1 + !!(req->flags & REQ_F_COMPLETE_INLINE);
 
 		/* submission and completion refs */
-		if (req_ref_sub_and_test(req, 2))
+		if (req_ref_sub_and_test(req, refs))
 			io_req_free_batch(&rb, req, &ctx->submit_state);
 	}
 
@@ -6417,17 +6419,12 @@ static void __io_queue_sqe(struct io_kiocb *req)
 	 * doesn't support non-blocking read/write attempts
 	 */
 	if (likely(!ret)) {
-		/* drop submission reference */
-		if (req->flags & REQ_F_COMPLETE_INLINE) {
-			struct io_ring_ctx *ctx = req->ctx;
-			struct io_comp_state *cs = &ctx->submit_state.comp;
-
-			cs->reqs[cs->nr++] = req;
-			if (cs->nr == ARRAY_SIZE(cs->reqs))
-				io_submit_flush_completions(cs, ctx);
-		} else {
-			io_put_req(req);
-		}
+		struct io_ring_ctx *ctx = req->ctx;
+		struct io_comp_state *cs = &ctx->submit_state.comp;
+
+		cs->reqs[cs->nr++] = req;
+		if (cs->nr == ARRAY_SIZE(cs->reqs))
+			io_submit_flush_completions(cs, ctx);
 	} else if (ret == -EAGAIN && !(req->flags & REQ_F_NOWAIT)) {
 		if (!io_arm_poll_handler(req)) {
 			/*
-- 
2.31.1