From: Pavel Begunkov <[email protected]>
To: [email protected]
Cc: Jens Axboe <[email protected]>, [email protected]
Subject: [PATCH v2 04/15] io_uring: refactor __io_get_cqe()
Date: Thu, 24 Aug 2023 23:53:26 +0100 [thread overview]
Message-ID: <74dc8fdf2657e438b2e05e1d478a3596924604e9.1692916914.git.asml.silence@gmail.com> (raw)
In-Reply-To: <[email protected]>
Make __io_get_cqe simpler by not grabbing the cqe from refilled cached,
but letting io_get_cqe() do it for us. That's cleaner and removes some
duplication.
Signed-off-by: Pavel Begunkov <[email protected]>
---
io_uring/io_uring.c | 13 ++++---------
io_uring/io_uring.h | 23 ++++++++++++-----------
2 files changed, 16 insertions(+), 20 deletions(-)
diff --git a/io_uring/io_uring.c b/io_uring/io_uring.c
index 0aeb33256a6d..de05831eeca7 100644
--- a/io_uring/io_uring.c
+++ b/io_uring/io_uring.c
@@ -818,7 +818,7 @@ void io_req_cqe_overflow(struct io_kiocb *req)
* control dependency is enough as we're using WRITE_ONCE to
* fill the cq entry
*/
-struct io_uring_cqe *__io_get_cqe(struct io_ring_ctx *ctx, bool overflow)
+bool io_cqe_cache_refill(struct io_ring_ctx *ctx, bool overflow)
{
struct io_rings *rings = ctx->rings;
unsigned int off = ctx->cached_cq_tail & (ctx->cq_entries - 1);
@@ -830,7 +830,7 @@ struct io_uring_cqe *__io_get_cqe(struct io_ring_ctx *ctx, bool overflow)
* Force overflow the completion.
*/
if (!overflow && (ctx->check_cq & BIT(IO_CHECK_CQ_OVERFLOW_BIT)))
- return NULL;
+ return false;
/* userspace may cheat modifying the tail, be safe and do min */
queued = min(__io_cqring_events(ctx), ctx->cq_entries);
@@ -838,7 +838,7 @@ struct io_uring_cqe *__io_get_cqe(struct io_ring_ctx *ctx, bool overflow)
/* we need a contiguous range, limit based on the current array offset */
len = min(free, ctx->cq_entries - off);
if (!len)
- return NULL;
+ return false;
if (ctx->flags & IORING_SETUP_CQE32) {
off <<= 1;
@@ -847,12 +847,7 @@ struct io_uring_cqe *__io_get_cqe(struct io_ring_ctx *ctx, bool overflow)
ctx->cqe_cached = &rings->cqes[off];
ctx->cqe_sentinel = ctx->cqe_cached + len;
-
- ctx->cached_cq_tail++;
- ctx->cqe_cached++;
- if (ctx->flags & IORING_SETUP_CQE32)
- ctx->cqe_cached++;
- return &rings->cqes[off];
+ return true;
}
static bool io_fill_cqe_aux(struct io_ring_ctx *ctx, u64 user_data, s32 res,
diff --git a/io_uring/io_uring.h b/io_uring/io_uring.h
index 9b5dfb6ef484..9c80d20fe18f 100644
--- a/io_uring/io_uring.h
+++ b/io_uring/io_uring.h
@@ -38,7 +38,7 @@ enum {
IOU_STOP_MULTISHOT = -ECANCELED,
};
-struct io_uring_cqe *__io_get_cqe(struct io_ring_ctx *ctx, bool overflow);
+bool io_cqe_cache_refill(struct io_ring_ctx *ctx, bool overflow);
void io_req_cqe_overflow(struct io_kiocb *req);
int io_run_task_work_sig(struct io_ring_ctx *ctx);
void io_req_defer_failed(struct io_kiocb *req, s32 res);
@@ -112,19 +112,20 @@ static inline void io_req_task_work_add(struct io_kiocb *req)
static inline struct io_uring_cqe *io_get_cqe_overflow(struct io_ring_ctx *ctx,
bool overflow)
{
- io_lockdep_assert_cq_locked(ctx);
+ struct io_uring_cqe *cqe;
- if (likely(ctx->cqe_cached < ctx->cqe_sentinel)) {
- struct io_uring_cqe *cqe = ctx->cqe_cached;
+ io_lockdep_assert_cq_locked(ctx);
- ctx->cached_cq_tail++;
- ctx->cqe_cached++;
- if (ctx->flags & IORING_SETUP_CQE32)
- ctx->cqe_cached++;
- return cqe;
+ if (unlikely(ctx->cqe_cached >= ctx->cqe_sentinel)) {
+ if (unlikely(!io_cqe_cache_refill(ctx, overflow)))
+ return NULL;
}
-
- return __io_get_cqe(ctx, overflow);
+ cqe = ctx->cqe_cached;
+ ctx->cached_cq_tail++;
+ ctx->cqe_cached++;
+ if (ctx->flags & IORING_SETUP_CQE32)
+ ctx->cqe_cached++;
+ return cqe;
}
static inline struct io_uring_cqe *io_get_cqe(struct io_ring_ctx *ctx)
--
2.41.0
next prev parent reply other threads:[~2023-08-24 22:56 UTC|newest]
Thread overview: 17+ messages / expand[flat|nested] mbox.gz Atom feed top
2023-08-24 22:53 [PATCH v2 00/15] caching and SQ/CQ optimisations Pavel Begunkov
2023-08-24 22:53 ` [PATCH v2 01/15] io_uring: improve cqe !tracing hot path Pavel Begunkov
2023-08-24 22:53 ` [PATCH v2 02/15] io_uring: cqe init hardening Pavel Begunkov
2023-08-24 22:53 ` [PATCH v2 03/15] io_uring: simplify big_cqe handling Pavel Begunkov
2023-08-24 22:53 ` Pavel Begunkov [this message]
2023-08-24 22:53 ` [PATCH v2 05/15] io_uring: optimise extra io_get_cqe null check Pavel Begunkov
2023-08-24 22:53 ` [PATCH v2 06/15] io_uring: reorder cqring_flush and wakeups Pavel Begunkov
2023-08-24 22:53 ` [PATCH v2 07/15] io_uring: merge iopoll and normal completion paths Pavel Begunkov
2023-08-24 22:53 ` [PATCH v2 08/15] io_uring: force inline io_fill_cqe_req Pavel Begunkov
2023-08-24 22:53 ` [PATCH v2 09/15] io_uring: compact SQ/CQ heads/tails Pavel Begunkov
2023-08-24 22:53 ` [PATCH v2 10/15] io_uring: add option to remove SQ indirection Pavel Begunkov
2023-08-24 22:53 ` [PATCH v2 11/15] io_uring: move non aligned field to the end Pavel Begunkov
2023-08-24 22:53 ` [PATCH v2 12/15] io_uring: banish non-hot data to end of io_ring_ctx Pavel Begunkov
2023-08-24 22:53 ` [PATCH v2 13/15] io_uring: separate task_work/waiting cache line Pavel Begunkov
2023-08-24 22:53 ` [PATCH v2 14/15] io_uring: move multishot cqe cache in ctx Pavel Begunkov
2023-08-24 22:53 ` [PATCH v2 15/15] io_uring: move iopoll ctx fields around Pavel Begunkov
2023-08-24 23:16 ` [PATCH v2 00/15] caching and SQ/CQ optimisations Jens Axboe
Reply instructions:
You may reply publicly to this message via plain-text email
using any one of the following methods:
* Save the following mbox file, import it into your mail client,
and reply-to-all from there: mbox
Avoid top-posting and favor interleaved quoting:
https://en.wikipedia.org/wiki/Posting_style#Interleaved_style
* Reply using the --to, --cc, and --in-reply-to
switches of git-send-email(1):
git send-email \
--in-reply-to=74dc8fdf2657e438b2e05e1d478a3596924604e9.1692916914.git.asml.silence@gmail.com \
[email protected] \
[email protected] \
[email protected] \
/path/to/YOUR_REPLY
https://kernel.org/pub/software/scm/git/docs/git-send-email.html
* If your mail client supports setting the In-Reply-To header
via mailto: links, try the mailto: link
Be sure your reply has a Subject: header at the top and a blank line
before the message body.
This is a public inbox, see mirroring instructions
for how to clone and mirror all data and code used for this inbox