public inbox for [email protected]
 help / color / mirror / Atom feed
From: Chenliang Li <[email protected]>
To: [email protected], [email protected]
Cc: [email protected], [email protected],
	[email protected], [email protected],
	[email protected], Chenliang Li <[email protected]>
Subject: [PATCH v3 1/5] io_uring/rsrc: add hugepage buffer coalesce helpers
Date: Mon, 13 May 2024 16:22:56 +0800	[thread overview]
Message-ID: <[email protected]> (raw)
In-Reply-To: <[email protected]>

Introduce helper functions to check whether a buffer can
be coalesced or not, and gather folio data for later use.

The coalescing optimizes time and space consumption caused
by mapping and storing multi-hugepage fixed buffers.

A coalescable multi-hugepage buffer should fully cover its folios
(except potentially the first and last one), and these folios should
have the same size. These requirements are for easier later process,
also we need same size'd chunks in io_import_fixed for fast iov_iter
adjust.

Signed-off-by: Chenliang Li <[email protected]>
---
 io_uring/rsrc.c | 78 +++++++++++++++++++++++++++++++++++++++++++++++++
 io_uring/rsrc.h | 10 +++++++
 2 files changed, 88 insertions(+)

diff --git a/io_uring/rsrc.c b/io_uring/rsrc.c
index 65417c9553b1..d08224c0c5b0 100644
--- a/io_uring/rsrc.c
+++ b/io_uring/rsrc.c
@@ -871,6 +871,84 @@ static int io_buffer_account_pin(struct io_ring_ctx *ctx, struct page **pages,
 	return ret;
 }
 
+static bool __io_sqe_buffer_try_coalesce(struct page **pages, int nr_pages,
+					 struct io_imu_folio_data *data)
+{
+	struct folio *folio = page_folio(pages[0]);
+	unsigned int count = 1;
+	int i;
+
+	data->nr_pages_mid = folio_nr_pages(folio);
+	if (data->nr_pages_mid == 1)
+		return false;
+
+	data->folio_shift = folio_shift(folio);
+	data->folio_size = folio_size(folio);
+	data->nr_folios = 1;
+	/*
+	 * Check if pages are contiguous inside a folio, and all folios have
+	 * the same page count except for the head and tail.
+	 */
+	for (i = 1; i < nr_pages; i++) {
+		if (page_folio(pages[i]) == folio &&
+			pages[i] == pages[i-1] + 1) {
+			count++;
+			continue;
+		}
+
+		if (data->nr_folios == 1)
+			data->nr_pages_head = count;
+		else if (count != data->nr_pages_mid)
+			return false;
+
+		folio = page_folio(pages[i]);
+		if (folio_size(folio) != data->folio_size)
+			return false;
+
+		count = 1;
+		data->nr_folios++;
+	}
+	if (data->nr_folios == 1)
+		data->nr_pages_head = count;
+
+	return true;
+}
+
+static bool io_sqe_buffer_try_coalesce(struct page **pages, int nr_pages,
+				       struct io_imu_folio_data *data)
+{
+	int i, j;
+
+	if (nr_pages <= 1 ||
+		!__io_sqe_buffer_try_coalesce(pages, nr_pages, data))
+		return false;
+
+	/*
+	 * The pages are bound to the folio, it doesn't
+	 * actually unpin them but drops all but one reference,
+	 * which is usually put down by io_buffer_unmap().
+	 * Note, needs a better helper.
+	 */
+	if (data->nr_pages_head > 1)
+		unpin_user_pages(&pages[1], data->nr_pages_head - 1);
+
+	j = data->nr_pages_head;
+	nr_pages -= data->nr_pages_head;
+	for (i = 1; i < data->nr_folios; i++) {
+		unsigned int nr_unpin;
+
+		nr_unpin = min_t(unsigned int, nr_pages - 1,
+					data->nr_pages_mid - 1);
+		if (nr_unpin == 0)
+			break;
+		unpin_user_pages(&pages[j+1], nr_unpin);
+		j += data->nr_pages_mid;
+		nr_pages -= data->nr_pages_mid;
+	}
+
+	return true;
+}
+
 static int io_sqe_buffer_register(struct io_ring_ctx *ctx, struct iovec *iov,
 				  struct io_mapped_ubuf **pimu,
 				  struct page **last_hpage)
diff --git a/io_uring/rsrc.h b/io_uring/rsrc.h
index c032ca3436ca..b2a9d66b76dd 100644
--- a/io_uring/rsrc.h
+++ b/io_uring/rsrc.h
@@ -50,6 +50,16 @@ struct io_mapped_ubuf {
 	struct bio_vec	bvec[] __counted_by(nr_bvecs);
 };
 
+struct io_imu_folio_data {
+	/* Head folio can be partially included in the fixed buf */
+	unsigned int	nr_pages_head;
+	/* For non-head/tail folios, has to be fully included */
+	unsigned int	nr_pages_mid;
+	unsigned int	nr_folios;
+	unsigned int	folio_shift;
+	size_t		folio_size;
+};
+
 void io_rsrc_node_ref_zero(struct io_rsrc_node *node);
 void io_rsrc_node_destroy(struct io_ring_ctx *ctx, struct io_rsrc_node *ref_node);
 struct io_rsrc_node *io_rsrc_node_alloc(struct io_ring_ctx *ctx);
-- 
2.34.1


  parent reply	other threads:[~2024-05-13  8:29 UTC|newest]

Thread overview: 12+ messages / expand[flat|nested]  mbox.gz  Atom feed  top
     [not found] <CGME20240513082306epcas5p2fd8ea6fd88b2c4ab1d17b1508fe2af97@epcas5p2.samsung.com>
2024-05-13  8:22 ` [PATCH v3 0/5] io_uring/rsrc: coalescing multi-hugepage registered buffers Chenliang Li
     [not found]   ` <CGME20240513082308epcas5p3c38ce4d44fa1613988bbae84eaec41b9@epcas5p3.samsung.com>
2024-05-13  8:22     ` Chenliang Li [this message]
     [not found]   ` <CGME20240513082310epcas5p27576a80eae3ee09e40102b179ce46fa9@epcas5p2.samsung.com>
2024-05-13  8:22     ` [PATCH v3 2/5] io_uring/rsrc: store folio shift and mask into imu Chenliang Li
     [not found]   ` <CGME20240513082311epcas5p3556d301a1f1faca0c6b613555324861e@epcas5p3.samsung.com>
2024-05-13  8:22     ` [PATCH v3 3/5] io_uring/rsrc: add init and account functions for coalesced imus Chenliang Li
     [not found]   ` <CGME20240513082313epcas5p2a781d3393e4bf92d13d04ac62bb28fb7@epcas5p2.samsung.com>
2024-05-13  8:22     ` [PATCH v3 4/5] io_uring/rsrc: enable multi-hugepage buffer coalescing Chenliang Li
2024-05-13 12:11       ` Anuj gupta
     [not found]   ` <CGME20240513082314epcas5p309fa70575596792b5c9923ce76a3778f@epcas5p3.samsung.com>
2024-05-13  8:23     ` [PATCH v3 5/5] liburing: add test cases for hugepage registered buffers Chenliang Li
2024-05-13 12:09   ` [PATCH v3 0/5] io_uring/rsrc: coalescing multi-hugepage " Anuj gupta
2024-05-13 13:40     ` Jens Axboe
     [not found]       ` <CGME20240514001813epcas5p455c8a3dd6f2164626a526c05f7fd92c4@epcas5p4.samsung.com>
2024-05-14  0:18         ` Chenliang Li
     [not found]     ` <CGME20240514001443epcas5p362c60c233ed2aecdb5e144099b44d9be@epcas5p3.samsung.com>
2024-05-14  0:14       ` [PATCH v3 4/5] io_uring/rsrc: enable multi-hugepage buffer coalescing Chenliang Li
     [not found]     ` <CGME20240514001620epcas5p10d8c08ffc3dbd746213df21e47df19f7@epcas5p1.samsung.com>
2024-05-14  0:16       ` [PATCH v3 0/5] io_uring/rsrc: coalescing multi-hugepage registered buffers Chenliang Li

Reply instructions:

You may reply publicly to this message via plain-text email
using any one of the following methods:

* Save the following mbox file, import it into your mail client,
  and reply-to-all from there: mbox

  Avoid top-posting and favor interleaved quoting:
  https://en.wikipedia.org/wiki/Posting_style#Interleaved_style

* Reply using the --to, --cc, and --in-reply-to
  switches of git-send-email(1):

  git send-email \
    [email protected] \
    [email protected] \
    [email protected] \
    [email protected] \
    [email protected] \
    [email protected] \
    [email protected] \
    [email protected] \
    [email protected] \
    /path/to/YOUR_REPLY

  https://kernel.org/pub/software/scm/git/docs/git-send-email.html

* If your mail client supports setting the In-Reply-To header
  via mailto: links, try the mailto: link
Be sure your reply has a Subject: header at the top and a blank line before the message body.
This is a public inbox, see mirroring instructions
for how to clone and mirror all data and code used for this inbox