public inbox for [email protected]
 help / color / mirror / Atom feed
From: Jens Axboe <[email protected]>
To: [email protected]
Cc: [email protected], Jens Axboe <[email protected]>
Subject: [PATCH 2/4] io_uring: add abstraction around apoll cache
Date: Thu,  7 Jul 2022 17:23:44 -0600	[thread overview]
Message-ID: <[email protected]> (raw)
In-Reply-To: <[email protected]>

In preparation for adding limits, and one more user, abstract out the
core bits of the allocation+free cache.

Signed-off-by: Jens Axboe <[email protected]>
---
 include/linux/io_uring_types.h |  6 +++++-
 io_uring/alloc_cache.h         |  4 ++++
 io_uring/io_uring.c            |  7 ++++---
 io_uring/poll.c                | 16 ++++++++--------
 io_uring/poll.h                |  5 ++++-
 5 files changed, 25 insertions(+), 13 deletions(-)
 create mode 100644 io_uring/alloc_cache.h

diff --git a/include/linux/io_uring_types.h b/include/linux/io_uring_types.h
index 26ef11e978d4..b548da03b563 100644
--- a/include/linux/io_uring_types.h
+++ b/include/linux/io_uring_types.h
@@ -158,6 +158,10 @@ struct io_ev_fd {
 	struct rcu_head		rcu;
 };
 
+struct io_alloc_cache {
+	struct hlist_head	list;
+};
+
 struct io_ring_ctx {
 	/* const or read-mostly hot data */
 	struct {
@@ -216,7 +220,7 @@ struct io_ring_ctx {
 
 		struct io_hash_table	cancel_table_locked;
 		struct list_head	cq_overflow_list;
-		struct list_head	apoll_cache;
+		struct io_alloc_cache	apoll_cache;
 		struct xarray		personalities;
 		u32			pers_next;
 	} ____cacheline_aligned_in_smp;
diff --git a/io_uring/alloc_cache.h b/io_uring/alloc_cache.h
new file mode 100644
index 000000000000..49ac6ae237ef
--- /dev/null
+++ b/io_uring/alloc_cache.h
@@ -0,0 +1,4 @@
+static inline void io_alloc_cache_init(struct io_alloc_cache *cache)
+{
+	INIT_HLIST_HEAD(&cache->list);
+}
diff --git a/io_uring/io_uring.c b/io_uring/io_uring.c
index 4d1ce58b015e..3b9033c401bf 100644
--- a/io_uring/io_uring.c
+++ b/io_uring/io_uring.c
@@ -92,6 +92,7 @@
 
 #include "timeout.h"
 #include "poll.h"
+#include "alloc_cache.h"
 
 #define IORING_MAX_ENTRIES	32768
 #define IORING_MAX_CQ_ENTRIES	(2 * IORING_MAX_ENTRIES)
@@ -295,7 +296,7 @@ static __cold struct io_ring_ctx *io_ring_ctx_alloc(struct io_uring_params *p)
 	INIT_LIST_HEAD(&ctx->sqd_list);
 	INIT_LIST_HEAD(&ctx->cq_overflow_list);
 	INIT_LIST_HEAD(&ctx->io_buffers_cache);
-	INIT_LIST_HEAD(&ctx->apoll_cache);
+	io_alloc_cache_init(&ctx->apoll_cache);
 	init_completion(&ctx->ref_comp);
 	xa_init_flags(&ctx->personalities, XA_FLAGS_ALLOC1);
 	mutex_init(&ctx->uring_lock);
@@ -1180,8 +1181,8 @@ void io_free_batch_list(struct io_ring_ctx *ctx, struct io_wq_work_node *node)
 
 				if (apoll->double_poll)
 					kfree(apoll->double_poll);
-				list_add(&apoll->poll.wait.entry,
-						&ctx->apoll_cache);
+				hlist_add_head(&apoll->cache_list,
+						&ctx->apoll_cache.list);
 				req->flags &= ~REQ_F_POLLED;
 			}
 			if (req->flags & IO_REQ_LINK_FLAGS)
diff --git a/io_uring/poll.c b/io_uring/poll.c
index f0fe209490d8..f3aae3cc6501 100644
--- a/io_uring/poll.c
+++ b/io_uring/poll.c
@@ -589,10 +589,10 @@ static struct async_poll *io_req_alloc_apoll(struct io_kiocb *req,
 		apoll = req->apoll;
 		kfree(apoll->double_poll);
 	} else if (!(issue_flags & IO_URING_F_UNLOCKED) &&
-		   !list_empty(&ctx->apoll_cache)) {
-		apoll = list_first_entry(&ctx->apoll_cache, struct async_poll,
-						poll.wait.entry);
-		list_del_init(&apoll->poll.wait.entry);
+		   !hlist_empty(&ctx->apoll_cache.list)) {
+		apoll = hlist_entry(ctx->apoll_cache.list.first,
+						struct async_poll, cache_list);
+		hlist_del(&apoll->cache_list);
 	} else {
 		apoll = kmalloc(sizeof(*apoll), GFP_ATOMIC);
 		if (unlikely(!apoll))
@@ -963,10 +963,10 @@ void io_flush_apoll_cache(struct io_ring_ctx *ctx)
 {
 	struct async_poll *apoll;
 
-	while (!list_empty(&ctx->apoll_cache)) {
-		apoll = list_first_entry(&ctx->apoll_cache, struct async_poll,
-						poll.wait.entry);
-		list_del(&apoll->poll.wait.entry);
+	while (!hlist_empty(&ctx->apoll_cache.list)) {
+		apoll = hlist_entry(ctx->apoll_cache.list.first,
+						struct async_poll, cache_list);
+		hlist_del(&apoll->cache_list);
 		kfree(apoll);
 	}
 }
diff --git a/io_uring/poll.h b/io_uring/poll.h
index 95f192c7babb..cb528f8ef203 100644
--- a/io_uring/poll.h
+++ b/io_uring/poll.h
@@ -14,7 +14,10 @@ struct io_poll {
 };
 
 struct async_poll {
-	struct io_poll		poll;
+	union {
+		struct io_poll		poll;
+		struct hlist_node	cache_list;
+	};
 	struct io_poll		*double_poll;
 };
 
-- 
2.35.1


  parent reply	other threads:[~2022-07-07 23:24 UTC|newest]

Thread overview: 8+ messages / expand[flat|nested]  mbox.gz  Atom feed  top
2022-07-07 23:23 [PATCHSET for-next] Add alloc cache for sendmsg/recvmsg Jens Axboe
2022-07-07 23:23 ` [PATCH 1/4] io_uring: move apoll cache to poll.c Jens Axboe
2022-07-07 23:23 ` Jens Axboe [this message]
2022-07-07 23:23 ` [PATCH 3/4] io_uring: impose max limit on apoll cache Jens Axboe
2022-07-07 23:23 ` [PATCH 4/4] io_uring: add netmsg cache Jens Axboe
2022-07-08  7:33   ` Dylan Yudaken
2022-07-08 12:47     ` Jens Axboe
  -- strict thread matches above, loose matches on Subject: below --
2022-07-08 13:30 [PATCHSET v2 for-next] Add alloc cache for sendmsg/recvmsg Jens Axboe
2022-07-08 13:30 ` [PATCH 2/4] io_uring: add abstraction around apoll cache Jens Axboe

Reply instructions:

You may reply publicly to this message via plain-text email
using any one of the following methods:

* Save the following mbox file, import it into your mail client,
  and reply-to-all from there: mbox

  Avoid top-posting and favor interleaved quoting:
  https://en.wikipedia.org/wiki/Posting_style#Interleaved_style

* Reply using the --to, --cc, and --in-reply-to
  switches of git-send-email(1):

  git send-email \
    [email protected] \
    [email protected] \
    [email protected] \
    [email protected] \
    /path/to/YOUR_REPLY

  https://kernel.org/pub/software/scm/git/docs/git-send-email.html

* If your mail client supports setting the In-Reply-To header
  via mailto: links, try the mailto: link
Be sure your reply has a Subject: header at the top and a blank line before the message body.
This is a public inbox, see mirroring instructions
for how to clone and mirror all data and code used for this inbox