[PATCH v2 6/7] io_uring: avoid GFP_ATOMIC for overflows if possible

[Pavel Begunkov <asml.silence@gmail.com>]

DEFER_TASKRUN enabled rings don't hold the completion lock or any other
spinlocks for CQE posting, so when an overflow happens they can do non
atomic allocations.

Signed-off-by: Pavel Begunkov <asml.silence@gmail.com>
---
 io_uring/io_uring.c | 16 +++++++++++-----
 1 file changed, 11 insertions(+), 5 deletions(-)

diff --git a/io_uring/io_uring.c b/io_uring/io_uring.c
index 86b39a01a136..0e0b3e75010c 100644
--- a/io_uring/io_uring.c
+++ b/io_uring/io_uring.c
@@ -730,7 +730,8 @@ static __cold void io_uring_drop_tctx_refs(struct task_struct *task)
 }
 
 static bool io_cqring_event_overflow(struct io_ring_ctx *ctx, u64 user_data,
-				     s32 res, u32 cflags, u64 extra1, u64 extra2)
+				     s32 res, u32 cflags, u64 extra1, u64 extra2,
+				     gfp_t gfp)
 {
 	struct io_overflow_cqe *ocqe;
 	size_t ocq_size = sizeof(struct io_overflow_cqe);
@@ -739,7 +740,7 @@ static bool io_cqring_event_overflow(struct io_ring_ctx *ctx, u64 user_data,
 	if (is_cqe32)
 		ocq_size += sizeof(struct io_uring_cqe);
 
-	ocqe = kmalloc(ocq_size, GFP_ATOMIC | __GFP_ACCOUNT);
+	ocqe = kmalloc(ocq_size, gfp | __GFP_ACCOUNT);
 	if (ocqe) {
 		ocqe->cqe.user_data = user_data;
 		ocqe->cqe.res = res;
@@ -839,7 +840,8 @@ bool io_post_aux_cqe(struct io_ring_ctx *ctx, u64 user_data, s32 res, u32 cflags
 	io_cq_lock(ctx);
 	filled = io_fill_cqe_aux(ctx, user_data, res, cflags);
 	if (!filled)
-		filled = io_cqring_event_overflow(ctx, user_data, res, cflags, 0, 0);
+		filled = io_cqring_event_overflow(ctx, user_data, res, cflags,
+						  0, 0, GFP_ATOMIC);
 	io_cq_unlock_post(ctx);
 	return filled;
 }
@@ -854,7 +856,8 @@ void io_add_aux_cqe(struct io_ring_ctx *ctx, u64 user_data, s32 res, u32 cflags)
 	lockdep_assert(ctx->lockless_cq);
 
 	if (!io_fill_cqe_aux(ctx, user_data, res, cflags))
-		io_cqring_event_overflow(ctx, user_data, res, cflags, 0, 0);
+		io_cqring_event_overflow(ctx, user_data, res, cflags, 0, 0,
+					 GFP_KERNEL);
 
 	ctx->submit_state.cq_flush = true;
 }
@@ -1444,10 +1447,13 @@ void __io_submit_flush_completions(struct io_ring_ctx *ctx)
 		 */
 		if (!(req->flags & (REQ_F_CQE_SKIP | REQ_F_REISSUE)) &&
 		    unlikely(!io_fill_cqe_req(ctx, req))) {
+			gfp_t gfp = ctx->lockless_cq ? GFP_KERNEL : GFP_ATOMIC;
+
 			io_cqring_event_overflow(req->ctx, req->cqe.user_data,
 						req->cqe.res, req->cqe.flags,
 						req->big_cqe.extra1,
-						req->big_cqe.extra2);
+						req->big_cqe.extra2,
+						gfp);
 			memset(&req->big_cqe, 0, sizeof(req->big_cqe));
 		}
 	}
-- 
2.49.0