[PATCH 3/4] io_uring: alloc overflow entry before locking

[Pavel Begunkov <asml.silence@gmail.com>]

Allocate and populate struct io_overflow_cqe before taking the lock. It
simplifies __io_cqring_event_overflow(), and that allows to avoid
GFP_ATOMIC for DEFER_TASKRUN rings.

Signed-off-by: Pavel Begunkov <asml.silence@gmail.com>
---
 io_uring/io_uring.c | 49 ++++++++++++++++++++++++---------------------
 1 file changed, 26 insertions(+), 23 deletions(-)

diff --git a/io_uring/io_uring.c b/io_uring/io_uring.c
index 5b253e2b6c49..927d8e45dbdb 100644
--- a/io_uring/io_uring.c
+++ b/io_uring/io_uring.c
@@ -719,20 +719,10 @@ static __cold void io_uring_drop_tctx_refs(struct task_struct *task)
 }
 
 static bool __io_cqring_event_overflow(struct io_ring_ctx *ctx,
-				       u64 user_data, s32 res, u32 cflags,
-				       u64 extra1, u64 extra2)
+					struct io_overflow_cqe *ocqe)
 {
-	struct io_overflow_cqe *ocqe;
-	size_t ocq_size = sizeof(struct io_overflow_cqe);
-	bool is_cqe32 = (ctx->flags & IORING_SETUP_CQE32);
-
 	lockdep_assert_held(&ctx->completion_lock);
 
-	if (is_cqe32)
-		ocq_size += sizeof(struct io_uring_cqe);
-
-	ocqe = kmalloc(ocq_size, GFP_ATOMIC | __GFP_ACCOUNT);
-	trace_io_uring_cqe_overflow(ctx, user_data, res, cflags, ocqe);
 	if (!ocqe) {
 		struct io_rings *r = ctx->rings;
 
@@ -745,17 +735,10 @@ static bool __io_cqring_event_overflow(struct io_ring_ctx *ctx,
 		set_bit(IO_CHECK_CQ_DROPPED_BIT, &ctx->check_cq);
 		return false;
 	}
+
 	if (list_empty(&ctx->cq_overflow_list)) {
 		set_bit(IO_CHECK_CQ_OVERFLOW_BIT, &ctx->check_cq);
 		atomic_or(IORING_SQ_CQ_OVERFLOW, &ctx->rings->sq_flags);
-
-	}
-	ocqe->cqe.user_data = user_data;
-	ocqe->cqe.res = res;
-	ocqe->cqe.flags = cflags;
-	if (is_cqe32) {
-		ocqe->cqe.big_cqe[0] = extra1;
-		ocqe->cqe.big_cqe[1] = extra2;
 	}
 	list_add_tail(&ocqe->list, &ctx->cq_overflow_list);
 	return true;
@@ -765,15 +748,35 @@ static bool io_cqring_event_overflow(struct io_ring_ctx *ctx, bool locked,
 				     u64 user_data, s32 res, u32 cflags,
 				     u64 extra1, u64 extra2)
 {
+	struct io_overflow_cqe *ocqe;
+	size_t ocq_size = sizeof(struct io_overflow_cqe);
+	bool is_cqe32 = (ctx->flags & IORING_SETUP_CQE32);
+	gfp_t gfp = GFP_KERNEL_ACCOUNT;
 	bool queued;
 
+	if (is_cqe32)
+		ocq_size += sizeof(struct io_uring_cqe);
+	if (locked)
+		gfp = GFP_ATOMIC | __GFP_ACCOUNT;
+
+	ocqe = kmalloc(ocq_size, gfp);
+	trace_io_uring_cqe_overflow(ctx, user_data, res, cflags, ocqe);
+
+	if (ocqe) {
+		ocqe->cqe.user_data = user_data;
+		ocqe->cqe.res = res;
+		ocqe->cqe.flags = cflags;
+		if (is_cqe32) {
+			ocqe->cqe.big_cqe[0] = extra1;
+			ocqe->cqe.big_cqe[1] = extra2;
+		}
+	}
+
 	if (locked) {
-		queued = __io_cqring_event_overflow(ctx, user_data, res, cflags,
-						    extra1, extra2);
+		queued = __io_cqring_event_overflow(ctx, ocqe);
 	} else {
 		spin_lock(&ctx->completion_lock);
-		queued = __io_cqring_event_overflow(ctx, user_data, res, cflags,
-						    extra1, extra2);
+		queued = __io_cqring_event_overflow(ctx, ocqe);
 		spin_unlock(&ctx->completion_lock);
 	}
 	return queued;
-- 
2.49.0