Re: [PATCH 5/5] io_uring: add new helpers for posting overflows

[Caleb Sander Mateos <csander@purestorage.com>]

On Fri, May 16, 2025 at 1:10 PM Jens Axboe <axboe@kernel.dk> wrote:
>
> Add two helpers, one for posting overflows for lockless_cq rings, and
> one for non-lockless_cq rings. The former can allocate sanely with
> GFP_KERNEL, but needs to grab the completion lock for posting, while the
> latter must do non-sleeping allocs as it already holds the completion
> lock.
>
> While at it, mark the overflow handling functions as __cold as well, as
> they should not generally be called during normal operations of the
> ring.
>
> Signed-off-by: Jens Axboe <axboe@kernel.dk>
> ---
>  io_uring/io_uring.c | 50 ++++++++++++++++++++++++++-------------------
>  1 file changed, 29 insertions(+), 21 deletions(-)
>
> diff --git a/io_uring/io_uring.c b/io_uring/io_uring.c
> index c66fc4b7356b..52087b079a0c 100644
> --- a/io_uring/io_uring.c
> +++ b/io_uring/io_uring.c
> @@ -697,8 +697,8 @@ static __cold void io_uring_drop_tctx_refs(struct task_struct *task)
>         }
>  }
>
> -static bool io_cqring_add_overflow(struct io_ring_ctx *ctx,
> -                                  struct io_overflow_cqe *ocqe)
> +static __cold bool io_cqring_add_overflow(struct io_ring_ctx *ctx,
> +                                         struct io_overflow_cqe *ocqe)
>  {
>         lockdep_assert_held(&ctx->completion_lock);
>
> @@ -808,6 +808,27 @@ static bool io_fill_cqe_aux(struct io_ring_ctx *ctx, u64 user_data, s32 res,
>         return false;
>  }
>
> +static __cold void io_cqe_overflow_lockless(struct io_ring_ctx *ctx,
> +                                           struct io_cqe *cqe,
> +                                           struct io_big_cqe *big_cqe)

Naming nit: "lockless" seems a bit misleading since this does still
take the completion_lock. Maybe name this function "io_cqe_overflow()"
and the other "io_cqe_overflow_locked()"?

Otherwise,

Reviewed-by: Caleb Sander Mateos <csander@purestorage.com>

> +{
> +       struct io_overflow_cqe *ocqe;
> +
> +       ocqe = io_alloc_ocqe(ctx, cqe, big_cqe, GFP_KERNEL);
> +       spin_lock(&ctx->completion_lock);
> +       io_cqring_add_overflow(ctx, ocqe);
> +       spin_unlock(&ctx->completion_lock);
> +}
> +
> +static __cold bool io_cqe_overflow(struct io_ring_ctx *ctx, struct io_cqe *cqe,
> +                                  struct io_big_cqe *big_cqe)
> +{
> +       struct io_overflow_cqe *ocqe;
> +
> +       ocqe = io_alloc_ocqe(ctx, cqe, big_cqe, GFP_ATOMIC);
> +       return io_cqring_add_overflow(ctx, ocqe);
> +}
> +
>  #define io_init_cqe(user_data, res, cflags)    \
>         (struct io_cqe) { .user_data = user_data, .res = res, .flags = cflags }
>
> @@ -818,11 +839,9 @@ bool io_post_aux_cqe(struct io_ring_ctx *ctx, u64 user_data, s32 res, u32 cflags
>         io_cq_lock(ctx);
>         filled = io_fill_cqe_aux(ctx, user_data, res, cflags);
>         if (unlikely(!filled)) {
> -               struct io_overflow_cqe *ocqe;
>                 struct io_cqe cqe = io_init_cqe(user_data, res, cflags);
>
> -               ocqe = io_alloc_ocqe(ctx, &cqe, NULL, GFP_ATOMIC);
> -               filled = io_cqring_add_overflow(ctx, ocqe);
> +               filled = io_cqe_overflow(ctx, &cqe, NULL);
>         }
>         io_cq_unlock_post(ctx);
>         return filled;
> @@ -838,13 +857,9 @@ void io_add_aux_cqe(struct io_ring_ctx *ctx, u64 user_data, s32 res, u32 cflags)
>         lockdep_assert(ctx->lockless_cq);
>
>         if (!io_fill_cqe_aux(ctx, user_data, res, cflags)) {
> -               struct io_overflow_cqe *ocqe;
>                 struct io_cqe cqe = io_init_cqe(user_data, res, cflags);
>
> -               ocqe = io_alloc_ocqe(ctx, &cqe, NULL, GFP_KERNEL);
> -               spin_lock(&ctx->completion_lock);
> -               io_cqring_add_overflow(ctx, ocqe);
> -               spin_unlock(&ctx->completion_lock);
> +               io_cqe_overflow_lockless(ctx, &cqe, NULL);
>         }
>         ctx->submit_state.cq_flush = true;
>  }
> @@ -1448,17 +1463,10 @@ void __io_submit_flush_completions(struct io_ring_ctx *ctx)
>                  */
>                 if (!(req->flags & (REQ_F_CQE_SKIP | REQ_F_REISSUE)) &&
>                     unlikely(!io_fill_cqe_req(ctx, req))) {
> -                       gfp_t gfp = ctx->lockless_cq ? GFP_KERNEL : GFP_ATOMIC;
> -                       struct io_overflow_cqe *ocqe;
> -
> -                       ocqe = io_alloc_ocqe(ctx, &req->cqe, &req->big_cqe, gfp);
> -                       if (ctx->lockless_cq) {
> -                               spin_lock(&ctx->completion_lock);
> -                               io_cqring_add_overflow(ctx, ocqe);
> -                               spin_unlock(&ctx->completion_lock);
> -                       } else {
> -                               io_cqring_add_overflow(ctx, ocqe);
> -                       }
> +                       if (ctx->lockless_cq)
> +                               io_cqe_overflow_lockless(ctx, &req->cqe, &req->big_cqe);
> +                       else
> +                               io_cqe_overflow(ctx, &req->cqe, &req->big_cqe);
>                 }
>         }
>         __io_cq_unlock_post(ctx);
> --
> 2.49.0
>