public inbox for [email protected]
 help / color / mirror / Atom feed
From: Pavel Begunkov <[email protected]>
To: [email protected]
Cc: Jens Axboe <[email protected]>, [email protected]
Subject: [PATCH v2 12/15] io_uring: banish non-hot data to end of io_ring_ctx
Date: Thu, 24 Aug 2023 23:53:34 +0100	[thread overview]
Message-ID: <fc471b63925a0bf90a34943c4d36163c523cfb43.1692916914.git.asml.silence@gmail.com> (raw)
In-Reply-To: <[email protected]>

Let's move all slow path, setup/init and so on fields to the end of
io_ring_ctx, that makes ctx reorganisation later easier. That includes,
page arrays used only on tear down, CQ overflow list, old provided
buffer caches and used by io-wq poll hashes.

Signed-off-by: Pavel Begunkov <[email protected]>
---
 include/linux/io_uring_types.h | 37 +++++++++++++++++-----------------
 1 file changed, 19 insertions(+), 18 deletions(-)

diff --git a/include/linux/io_uring_types.h b/include/linux/io_uring_types.h
index ad87d6074fb2..72e609752323 100644
--- a/include/linux/io_uring_types.h
+++ b/include/linux/io_uring_types.h
@@ -211,20 +211,11 @@ struct io_ring_ctx {
 		unsigned int		drain_disabled: 1;
 		unsigned int		compat: 1;
 
-		enum task_work_notify_mode	notify_method;
+		struct task_struct	*submitter_task;
+		struct io_rings		*rings;
+		struct percpu_ref	refs;
 
-		/*
-		 * If IORING_SETUP_NO_MMAP is used, then the below holds
-		 * the gup'ed pages for the two rings, and the sqes.
-		 */
-		unsigned short		n_ring_pages;
-		unsigned short		n_sqe_pages;
-		struct page		**ring_pages;
-		struct page		**sqe_pages;
-
-		struct io_rings			*rings;
-		struct task_struct		*submitter_task;
-		struct percpu_ref		refs;
+		enum task_work_notify_mode	notify_method;
 	} ____cacheline_aligned_in_smp;
 
 	/* submission data */
@@ -262,10 +253,8 @@ struct io_ring_ctx {
 
 		struct io_buffer_list	*io_bl;
 		struct xarray		io_bl_xa;
-		struct list_head	io_buffers_cache;
 
 		struct io_hash_table	cancel_table_locked;
-		struct list_head	cq_overflow_list;
 		struct io_alloc_cache	apoll_cache;
 		struct io_alloc_cache	netmsg_cache;
 	} ____cacheline_aligned_in_smp;
@@ -298,11 +287,8 @@ struct io_ring_ctx {
 		 * manipulate the list, hence no extra locking is needed there.
 		 */
 		struct io_wq_work_list	iopoll_list;
-		struct io_hash_table	cancel_table;
 
 		struct llist_head	work_llist;
-
-		struct list_head	io_buffers_comp;
 	} ____cacheline_aligned_in_smp;
 
 	/* timeouts */
@@ -318,6 +304,10 @@ struct io_ring_ctx {
 	struct io_wq_work_list	locked_free_list;
 	unsigned int		locked_free_nr;
 
+	struct list_head	io_buffers_comp;
+	struct list_head	cq_overflow_list;
+	struct io_hash_table	cancel_table;
+
 	const struct cred	*sq_creds;	/* cred used for __io_sq_thread() */
 	struct io_sq_data	*sq_data;	/* if using sq thread polling */
 
@@ -332,6 +322,8 @@ struct io_ring_ctx {
 	struct xarray		personalities;
 	u32			pers_next;
 
+	struct list_head	io_buffers_cache;
+
 	/* Keep this last, we don't need it for the fast path */
 	struct wait_queue_head		poll_wq;
 	struct io_restriction		restrictions;
@@ -375,6 +367,15 @@ struct io_ring_ctx {
 	unsigned			sq_thread_idle;
 	/* protected by ->completion_lock */
 	unsigned			evfd_last_cq_tail;
+
+	/*
+	 * If IORING_SETUP_NO_MMAP is used, then the below holds
+	 * the gup'ed pages for the two rings, and the sqes.
+	 */
+	unsigned short			n_ring_pages;
+	unsigned short			n_sqe_pages;
+	struct page			**ring_pages;
+	struct page			**sqe_pages;
 };
 
 struct io_tw_state {
-- 
2.41.0


  parent reply	other threads:[~2023-08-24 22:56 UTC|newest]

Thread overview: 17+ messages / expand[flat|nested]  mbox.gz  Atom feed  top
2023-08-24 22:53 [PATCH v2 00/15] caching and SQ/CQ optimisations Pavel Begunkov
2023-08-24 22:53 ` [PATCH v2 01/15] io_uring: improve cqe !tracing hot path Pavel Begunkov
2023-08-24 22:53 ` [PATCH v2 02/15] io_uring: cqe init hardening Pavel Begunkov
2023-08-24 22:53 ` [PATCH v2 03/15] io_uring: simplify big_cqe handling Pavel Begunkov
2023-08-24 22:53 ` [PATCH v2 04/15] io_uring: refactor __io_get_cqe() Pavel Begunkov
2023-08-24 22:53 ` [PATCH v2 05/15] io_uring: optimise extra io_get_cqe null check Pavel Begunkov
2023-08-24 22:53 ` [PATCH v2 06/15] io_uring: reorder cqring_flush and wakeups Pavel Begunkov
2023-08-24 22:53 ` [PATCH v2 07/15] io_uring: merge iopoll and normal completion paths Pavel Begunkov
2023-08-24 22:53 ` [PATCH v2 08/15] io_uring: force inline io_fill_cqe_req Pavel Begunkov
2023-08-24 22:53 ` [PATCH v2 09/15] io_uring: compact SQ/CQ heads/tails Pavel Begunkov
2023-08-24 22:53 ` [PATCH v2 10/15] io_uring: add option to remove SQ indirection Pavel Begunkov
2023-08-24 22:53 ` [PATCH v2 11/15] io_uring: move non aligned field to the end Pavel Begunkov
2023-08-24 22:53 ` Pavel Begunkov [this message]
2023-08-24 22:53 ` [PATCH v2 13/15] io_uring: separate task_work/waiting cache line Pavel Begunkov
2023-08-24 22:53 ` [PATCH v2 14/15] io_uring: move multishot cqe cache in ctx Pavel Begunkov
2023-08-24 22:53 ` [PATCH v2 15/15] io_uring: move iopoll ctx fields around Pavel Begunkov
2023-08-24 23:16 ` [PATCH v2 00/15] caching and SQ/CQ optimisations Jens Axboe

Reply instructions:

You may reply publicly to this message via plain-text email
using any one of the following methods:

* Save the following mbox file, import it into your mail client,
  and reply-to-all from there: mbox

  Avoid top-posting and favor interleaved quoting:
  https://en.wikipedia.org/wiki/Posting_style#Interleaved_style

* Reply using the --to, --cc, and --in-reply-to
  switches of git-send-email(1):

  git send-email \
    --in-reply-to=fc471b63925a0bf90a34943c4d36163c523cfb43.1692916914.git.asml.silence@gmail.com \
    [email protected] \
    [email protected] \
    [email protected] \
    /path/to/YOUR_REPLY

  https://kernel.org/pub/software/scm/git/docs/git-send-email.html

* If your mail client supports setting the In-Reply-To header
  via mailto: links, try the mailto: link
Be sure your reply has a Subject: header at the top and a blank line before the message body.
This is a public inbox, see mirroring instructions
for how to clone and mirror all data and code used for this inbox