[PATCH 2/4] io_uring: move locking inside overflow posting

[Pavel Begunkov <asml.silence@gmail.com>]

A preparation patch moving locking protecting the overflow list into
io_cqring_event_overflow(). The locking needs to be conditional because
callers might already hold the lock. It's not the prettiest option, but
it's not much different from the current state. Hopefully, one day we'll
get rid of this nasty locking pattern.

Signed-off-by: Pavel Begunkov <asml.silence@gmail.com>
---
 io_uring/io_uring.c | 53 +++++++++++++++++++++++++++------------------
 1 file changed, 32 insertions(+), 21 deletions(-)

diff --git a/io_uring/io_uring.c b/io_uring/io_uring.c
index 068e140b6bd8..5b253e2b6c49 100644
--- a/io_uring/io_uring.c
+++ b/io_uring/io_uring.c
@@ -718,8 +718,9 @@ static __cold void io_uring_drop_tctx_refs(struct task_struct *task)
 	}
 }
 
-static bool io_cqring_event_overflow(struct io_ring_ctx *ctx, u64 user_data,
-				     s32 res, u32 cflags, u64 extra1, u64 extra2)
+static bool __io_cqring_event_overflow(struct io_ring_ctx *ctx,
+				       u64 user_data, s32 res, u32 cflags,
+				       u64 extra1, u64 extra2)
 {
 	struct io_overflow_cqe *ocqe;
 	size_t ocq_size = sizeof(struct io_overflow_cqe);
@@ -760,6 +761,24 @@ static bool io_cqring_event_overflow(struct io_ring_ctx *ctx, u64 user_data,
 	return true;
 }
 
+static bool io_cqring_event_overflow(struct io_ring_ctx *ctx, bool locked,
+				     u64 user_data, s32 res, u32 cflags,
+				     u64 extra1, u64 extra2)
+{
+	bool queued;
+
+	if (locked) {
+		queued = __io_cqring_event_overflow(ctx, user_data, res, cflags,
+						    extra1, extra2);
+	} else {
+		spin_lock(&ctx->completion_lock);
+		queued = __io_cqring_event_overflow(ctx, user_data, res, cflags,
+						    extra1, extra2);
+		spin_unlock(&ctx->completion_lock);
+	}
+	return queued;
+}
+
 /*
  * writes to the cq entry need to come after reading head; the
  * control dependency is enough as we're using WRITE_ONCE to
@@ -825,7 +844,8 @@ bool io_post_aux_cqe(struct io_ring_ctx *ctx, u64 user_data, s32 res, u32 cflags
 	io_cq_lock(ctx);
 	filled = io_fill_cqe_aux(ctx, user_data, res, cflags);
 	if (!filled)
-		filled = io_cqring_event_overflow(ctx, user_data, res, cflags, 0, 0);
+		filled = io_cqring_event_overflow(ctx, true,
+						  user_data, res, cflags, 0, 0);
 	io_cq_unlock_post(ctx);
 	return filled;
 }
@@ -839,11 +859,9 @@ void io_add_aux_cqe(struct io_ring_ctx *ctx, u64 user_data, s32 res, u32 cflags)
 	lockdep_assert_held(&ctx->uring_lock);
 	lockdep_assert(ctx->lockless_cq);
 
-	if (!io_fill_cqe_aux(ctx, user_data, res, cflags)) {
-		spin_lock(&ctx->completion_lock);
-		io_cqring_event_overflow(ctx, user_data, res, cflags, 0, 0);
-		spin_unlock(&ctx->completion_lock);
-	}
+	if (!io_fill_cqe_aux(ctx, user_data, res, cflags))
+		io_cqring_event_overflow(ctx, false, user_data, res, cflags, 0, 0);
+
 	ctx->submit_state.cq_flush = true;
 }
 
@@ -1432,20 +1450,13 @@ void __io_submit_flush_completions(struct io_ring_ctx *ctx)
 		 */
 		if (!(req->flags & (REQ_F_CQE_SKIP | REQ_F_REISSUE)) &&
 		    unlikely(!io_fill_cqe_req(ctx, req))) {
-			if (ctx->lockless_cq) {
-				spin_lock(&ctx->completion_lock);
-				io_cqring_event_overflow(req->ctx, req->cqe.user_data,
-							req->cqe.res, req->cqe.flags,
-							req->big_cqe.extra1,
-							req->big_cqe.extra2);
-				spin_unlock(&ctx->completion_lock);
-			} else {
-				io_cqring_event_overflow(req->ctx, req->cqe.user_data,
-							req->cqe.res, req->cqe.flags,
-							req->big_cqe.extra1,
-							req->big_cqe.extra2);
-			}
+			bool locked = !ctx->lockless_cq;
 
+			io_cqring_event_overflow(req->ctx, locked,
+						req->cqe.user_data,
+						req->cqe.res, req->cqe.flags,
+						req->big_cqe.extra1,
+						req->big_cqe.extra2);
 			memset(&req->big_cqe, 0, sizeof(req->big_cqe));
 		}
 	}
-- 
2.49.0