public inbox for [email protected]
 help / color / mirror / Atom feed
From: Pavel Begunkov <[email protected]>
To: Jens Axboe <[email protected]>,
	[email protected], [email protected]
Subject: [PATCH 2/5] io_uring: replace list with array for compl batch
Date: Tue, 14 Jul 2020 02:41:53 +0300	[thread overview]
Message-ID: <cde0ab12ccb5f74f82db265cd8b00bf82f0a1e39.1594683622.git.asml.silence@gmail.com> (raw)
In-Reply-To: <[email protected]>

We limit how much request we batch on the completion path, use a fixed
size array instead of lists for completion batching. That also allows
to split io_submit_flush_completions() into 2 steps: the first is
filling CQEs, and the second actually frees requests.

There are plenty of benefits:
- list head tossing is expensive + removes LIST_INIT in state prep
- doesn't do extra unlock/lock to put a linked request
- filling CQEs first gives better latency
- will be used to handle list entry aliasing and add batch free there

Signed-off-by: Pavel Begunkov <[email protected]>
---
 fs/io_uring.c | 33 +++++++++++++--------------------
 1 file changed, 13 insertions(+), 20 deletions(-)

diff --git a/fs/io_uring.c b/fs/io_uring.c
index 609c7da044d7..3277a06e2fb6 100644
--- a/fs/io_uring.c
+++ b/fs/io_uring.c
@@ -680,11 +680,12 @@ struct io_defer_entry {
 };
 
 #define IO_IOPOLL_BATCH			8
+#define IO_COMPL_BATCH			32
 
 struct io_comp_state {
-	unsigned int		nr;
-	struct list_head	list;
 	struct io_ring_ctx	*ctx;
+	unsigned int		nr;
+	struct io_kiocb		*reqs[IO_COMPL_BATCH];
 };
 
 struct io_submit_state {
@@ -1794,28 +1795,21 @@ static void io_req_free_batch(struct req_batch *rb, struct io_kiocb *req)
 
 static void io_submit_flush_completions(struct io_comp_state *cs)
 {
+	struct io_kiocb *req;
 	struct io_ring_ctx *ctx = cs->ctx;
+	int i, nr = cs->nr;
 
 	spin_lock_irq(&ctx->completion_lock);
-	while (!list_empty(&cs->list)) {
-		struct io_kiocb *req;
-
-		req = list_first_entry(&cs->list, struct io_kiocb, compl.list);
-		list_del(&req->compl.list);
+	for (i = 0; i < nr; ++i) {
+		req = cs->reqs[i];
 		__io_cqring_fill_event(req, req->result, req->compl.cflags);
-		if (!(req->flags & REQ_F_LINK_HEAD)) {
-			req->flags |= REQ_F_COMP_LOCKED;
-			io_put_req(req);
-		} else {
-			spin_unlock_irq(&ctx->completion_lock);
-			io_put_req(req);
-			spin_lock_irq(&ctx->completion_lock);
-		}
 	}
 	io_commit_cqring(ctx);
 	spin_unlock_irq(&ctx->completion_lock);
-
 	io_cqring_ev_posted(ctx);
+
+	for (i = 0; i < nr; ++i)
+		io_put_req(cs->reqs[i]);
 	cs->nr = 0;
 }
 
@@ -1829,8 +1823,8 @@ static void __io_req_complete(struct io_kiocb *req, long res, unsigned cflags,
 		io_clean_op(req);
 		req->result = res;
 		req->compl.cflags = cflags;
-		list_add_tail(&req->compl.list, &cs->list);
-		if (++cs->nr >= 32)
+		cs->reqs[cs->nr++] = req;
+		if (cs->nr == IO_COMPL_BATCH)
 			io_submit_flush_completions(cs);
 	}
 }
@@ -6118,7 +6112,7 @@ static int io_submit_sqe(struct io_kiocb *req, const struct io_uring_sqe *sqe,
  */
 static void io_submit_state_end(struct io_submit_state *state)
 {
-	if (!list_empty(&state->comp.list))
+	if (state->comp.nr)
 		io_submit_flush_completions(&state->comp);
 	blk_finish_plug(&state->plug);
 	io_state_file_put(state);
@@ -6137,7 +6131,6 @@ static void io_submit_state_start(struct io_submit_state *state,
 	state->plug.nowait = true;
 #endif
 	state->comp.nr = 0;
-	INIT_LIST_HEAD(&state->comp.list);
 	state->comp.ctx = ctx;
 	state->free_reqs = 0;
 	state->file = NULL;
-- 
2.24.0


  parent reply	other threads:[~2020-07-13 23:44 UTC|newest]

Thread overview: 8+ messages / expand[flat|nested]  mbox.gz  Atom feed  top
2020-07-13 23:41 [PATCH 0/5] batch completion + freeing improvements Pavel Begunkov
2020-07-13 23:41 ` [PATCH 1/5] io_uring: move io_req_complete() definition Pavel Begunkov
2020-07-13 23:41 ` Pavel Begunkov [this message]
2020-07-13 23:41 ` [PATCH 3/5] io_uring: batch free in batched completion Pavel Begunkov
2020-07-13 23:41 ` [PATCH 4/5] tasks: add put_task_struct_many() Pavel Begunkov
2020-07-13 23:41 ` [PATCH 5/5] io_uring: batch put_task_struct() Pavel Begunkov
2020-07-14  1:08 ` [PATCH 0/5] batch completion + freeing improvements Jens Axboe
2020-07-14  7:15   ` Pavel Begunkov

Reply instructions:

You may reply publicly to this message via plain-text email
using any one of the following methods:

* Save the following mbox file, import it into your mail client,
  and reply-to-all from there: mbox

  Avoid top-posting and favor interleaved quoting:
  https://en.wikipedia.org/wiki/Posting_style#Interleaved_style

* Reply using the --to, --cc, and --in-reply-to
  switches of git-send-email(1):

  git send-email \
    --in-reply-to=cde0ab12ccb5f74f82db265cd8b00bf82f0a1e39.1594683622.git.asml.silence@gmail.com \
    [email protected] \
    [email protected] \
    [email protected] \
    [email protected] \
    /path/to/YOUR_REPLY

  https://kernel.org/pub/software/scm/git/docs/git-send-email.html

* If your mail client supports setting the In-Reply-To header
  via mailto: links, try the mailto: link
Be sure your reply has a Subject: header at the top and a blank line before the message body.
This is a public inbox, see mirroring instructions
for how to clone and mirror all data and code used for this inbox