[PATCH v4 0/6]liburing: add support for large CQE sizes

[PATCH v4 0/6]liburing: add support for large CQE sizes

[Stefan Roesch]

This adds support for large CQE sizes in the liburing layer. The large CQE
sizes double the size compared to the default CQE size.

To support larger CQE sizes the mmap call needs to be modified to map a larger
memory region for large CQE's. For default CQE's the size of the mapping stays
the same.
Also the ring size calculation needs to change.

Finally when large CQE's are indexed, they need to take into account the bigger
CQE size. The index manipulation remains unchanged, only when the CQE array is
accessed, the offset is changed for large CQE's.

The nop test has been modified to test that the new values are set correctly.

Testing:
The liburing test suite has been run with the four different configurations:
- default
- large SQE
- large CQE
- large SQE & large CQE
To do this the default setting has been changed for the test run to the above
values.:

To use these changes, also the corresponding kernel changes are required.


Changes:
  V2: the changed kernel definition of io_uring_cqe_extra has been applied to
      the first patch in this patch series.
  V3: - change definition of extra io_uring_cqe fields like in linux kernel
      - remove io_uring_cqe_extra definition
  V4: change nop test to use new field names


Stefan Roesch (6):
  liburing: Update io_uring.h with large CQE kernel changes
  liburing: increase mmap size for large CQE's
  liburing: return correct ring size for large CQE's
  liburing: index large CQE's correctly
  liburing: add large CQE tests to nop test
  liburing: Test all configurations with NOP test

 src/include/liburing.h          | 18 +++++++-
 src/include/liburing/io_uring.h |  7 ++++
 src/queue.c                     |  6 ++-
 src/setup.c                     | 13 ++++--
 test/nop.c                      | 74 ++++++++++++++++++++-------------
 test/test.h                     | 35 ++++++++++++++++
 6 files changed, 117 insertions(+), 36 deletions(-)
 create mode 100644 test/test.h


base-commit: c0ba24d1215e9f2b08266b28b35436988c6f3543
-- 
2.30.2

[PATCH v4 1/6] liburing: Update io_uring.h with large CQE kernel changes

[Stefan Roesch]

This updates the io_uring.h file with the changes in the kernel.

Signed-off-by: Stefan Roesch <shr@fb.com>
---
 src/include/liburing/io_uring.h | 7 +++++++
 1 file changed, 7 insertions(+)

diff --git a/src/include/liburing/io_uring.h b/src/include/liburing/io_uring.h
index a38a45b..95e6a43 100644
--- a/src/include/liburing/io_uring.h
+++ b/src/include/liburing/io_uring.h
@@ -113,6 +113,7 @@ enum {
 #define IORING_SETUP_R_DISABLED	(1U << 6)	/* start with ring disabled */
 #define IORING_SETUP_SUBMIT_ALL	(1U << 7)	/* continue submit on error */
 #define IORING_SETUP_SQE128	(1U << 8)	/* SQEs are 128b */
+#define IORING_SETUP_CQE32	(1U << 9)	/* CQEs are 32b */
 
 enum {
 	IORING_OP_NOP,
@@ -205,6 +206,12 @@ struct io_uring_cqe {
 	__u64	user_data;	/* sqe->data submission passed back */
 	__s32	res;		/* result code for this event */
 	__u32	flags;
+
+	/*
+	 * If the ring is initialized wit IORING_SETUP_CQE32, then this field
+	 * contains 16-bytes of padding, doubling the size fo the CQE.
+	 */
+	__u64 big_cqe[];
 };
 
 /*
-- 
2.30.2

[PATCH v4 2/6] liburing: increase mmap size for large CQE's

[Stefan Roesch]

This doubles the mmap size for large CQE's.

Signed-off-by: Stefan Roesch <shr@fb.com>
---
 src/setup.c | 6 +++++-
 1 file changed, 5 insertions(+), 1 deletion(-)

diff --git a/src/setup.c b/src/setup.c
index aec8b33..dd6a712 100644
--- a/src/setup.c
+++ b/src/setup.c
@@ -21,8 +21,12 @@ static int io_uring_mmap(int fd, struct io_uring_params *p,
 	size_t size;
 	int ret;
 
+	size = sizeof(struct io_uring_cqe);
+	if (p->flags & IORING_SETUP_CQE32)
+		size += sizeof(struct io_uring_cqe);
+
 	sq->ring_sz = p->sq_off.array + p->sq_entries * sizeof(unsigned);
-	cq->ring_sz = p->cq_off.cqes + p->cq_entries * sizeof(struct io_uring_cqe);
+	cq->ring_sz = p->cq_off.cqes + p->cq_entries * size;
 
 	if (p->features & IORING_FEAT_SINGLE_MMAP) {
 		if (cq->ring_sz > sq->ring_sz)
-- 
2.30.2

[PATCH v4 3/6] liburing: return correct ring size for large CQE's

[Stefan Roesch]

Return the correct ring_size when large CQE's are used.

Signed-off-by: Stefan Roesch <shr@fb.com>
---
 src/setup.c | 7 +++++--
 1 file changed, 5 insertions(+), 2 deletions(-)

diff --git a/src/setup.c b/src/setup.c
index dd6a712..d2adc7f 100644
--- a/src/setup.c
+++ b/src/setup.c
@@ -257,8 +257,11 @@ static size_t rings_size(struct io_uring_params *p, unsigned entries,
 {
 	size_t pages, sq_size, cq_size;
 
-	cq_size = KRING_SIZE;
-	cq_size += cq_entries * sizeof(struct io_uring_cqe);
+	cq_size = sizeof(struct io_uring_cqe);
+	if (p->flags & IORING_SETUP_CQE32)
+		cq_size += sizeof(struct io_uring_cqe);
+	cq_size *= cq_entries;
+	cq_size += KRING_SIZE;
 	cq_size = (cq_size + 63) & ~63UL;
 	pages = (size_t) 1 << npages(cq_size, page_size);
 
-- 
2.30.2

[PATCH v4 4/6] liburing: index large CQE's correctly

[Stefan Roesch]

Large CQE's need to take into account that each CQE has double the size.
When the CQE array is indexed, the offset into the array needs to be
changed accordingly.

Signed-off-by: Stefan Roesch <shr@fb.com>
---
 src/include/liburing.h | 18 ++++++++++++++++--
 src/queue.c            |  6 +++++-
 2 files changed, 21 insertions(+), 3 deletions(-)

diff --git a/src/include/liburing.h b/src/include/liburing.h
index c01c231..317963c 100644
--- a/src/include/liburing.h
+++ b/src/include/liburing.h
@@ -188,6 +188,16 @@ int __io_uring_get_cqe(struct io_uring *ring,
 
 #define LIBURING_UDATA_TIMEOUT	((__u64) -1)
 
+/*
+ * Calculates the step size for CQE iteration.
+ * 	For standard CQE's its 1, for big CQE's its two.
+ */
+#define io_uring_cqe_shift(ring)					\
+	(!!((ring)->flags & IORING_SETUP_CQE32))
+
+#define io_uring_cqe_index(ring,ptr,mask)				\
+	(((ptr) & (mask)) << io_uring_cqe_shift(ring))
+
 #define io_uring_for_each_cqe(ring, head, cqe)				\
 	/*								\
 	 * io_uring_smp_load_acquire() enforces the order of tail	\
@@ -195,7 +205,7 @@ int __io_uring_get_cqe(struct io_uring *ring,
 	 */								\
 	for (head = *(ring)->cq.khead;					\
 	     (cqe = (head != io_uring_smp_load_acquire((ring)->cq.ktail) ? \
-		&(ring)->cq.cqes[head & (*(ring)->cq.kring_mask)] : NULL)); \
+		&(ring)->cq.cqes[io_uring_cqe_index(ring, head, *(ring)->cq.kring_mask)] : NULL)); \
 	     head++)							\
 
 /*
@@ -844,6 +854,10 @@ static inline int __io_uring_peek_cqe(struct io_uring *ring,
 	int err = 0;
 	unsigned available;
 	unsigned mask = *ring->cq.kring_mask;
+	int shift = 0;
+
+	if (ring->flags & IORING_SETUP_CQE32)
+		shift = 1;
 
 	do {
 		unsigned tail = io_uring_smp_load_acquire(ring->cq.ktail);
@@ -854,7 +868,7 @@ static inline int __io_uring_peek_cqe(struct io_uring *ring,
 		if (!available)
 			break;
 
-		cqe = &ring->cq.cqes[head & mask];
+		cqe = &ring->cq.cqes[(head & mask) << shift];
 		if (!(ring->features & IORING_FEAT_EXT_ARG) &&
 				cqe->user_data == LIBURING_UDATA_TIMEOUT) {
 			if (cqe->res < 0)
diff --git a/src/queue.c b/src/queue.c
index 2f85756..4ad41fc 100644
--- a/src/queue.c
+++ b/src/queue.c
@@ -132,6 +132,10 @@ unsigned io_uring_peek_batch_cqe(struct io_uring *ring,
 {
 	unsigned ready;
 	bool overflow_checked = false;
+	int shift = 0;
+
+	if (ring->flags & IORING_SETUP_CQE32)
+		shift = 1;
 
 again:
 	ready = io_uring_cq_ready(ring);
@@ -144,7 +148,7 @@ again:
 		count = count > ready ? ready : count;
 		last = head + count;
 		for (;head != last; head++, i++)
-			cqes[i] = &ring->cq.cqes[head & mask];
+			cqes[i] = &ring->cq.cqes[(head & mask) << shift];
 
 		return count;
 	}
-- 
2.30.2

[PATCH v4 5/6] liburing: add large CQE tests to nop test

[Stefan Roesch]

This adds two test cases for large CQE's:
- Single NOP test, which checks that the new extra1 and extra2 fields
  are set.
- Multiple NOP submission test which also checks for the new fields.

Signed-off-by: Stefan Roesch <shr@fb.com>
---
 test/nop.c | 31 +++++++++++++++++++++++++++++++
 1 file changed, 31 insertions(+)

diff --git a/test/nop.c b/test/nop.c
index d477a1b..8656373 100644
--- a/test/nop.c
+++ b/test/nop.c
@@ -19,6 +19,7 @@ static int test_single_nop(struct io_uring *ring)
 	struct io_uring_cqe *cqe;
 	struct io_uring_sqe *sqe;
 	int ret;
+	bool cqe32 = (ring->flags & IORING_SETUP_CQE32);
 
 	sqe = io_uring_get_sqe(ring);
 	if (!sqe) {
@@ -27,6 +28,10 @@ static int test_single_nop(struct io_uring *ring)
 	}
 
 	io_uring_prep_nop(sqe);
+	if (cqe32) {
+		sqe->addr = 1234;
+		sqe->addr2 = 5678;
+	}
 	sqe->user_data = ++seq;
 
 	ret = io_uring_submit(ring);
@@ -44,6 +49,17 @@ static int test_single_nop(struct io_uring *ring)
 		fprintf(stderr, "Unexpected 0 user_data\n");
 		goto err;
 	}
+	if (cqe32) {
+		if (cqe->big_cqe[0] != 1234) {
+			fprintf(stderr, "Unexpected extra1\n");
+			goto err;
+
+		}
+		if (cqe->big_cqe[1] != 5678) {
+			fprintf(stderr, "Unexpected extra2\n");
+			goto err;
+		}
+	}
 	io_uring_cqe_seen(ring, cqe);
 	return 0;
 err:
@@ -55,6 +71,7 @@ static int test_barrier_nop(struct io_uring *ring)
 	struct io_uring_cqe *cqe;
 	struct io_uring_sqe *sqe;
 	int ret, i;
+	bool cqe32 = (ring->flags & IORING_SETUP_CQE32);
 
 	for (i = 0; i < 8; i++) {
 		sqe = io_uring_get_sqe(ring);
@@ -66,6 +83,10 @@ static int test_barrier_nop(struct io_uring *ring)
 		io_uring_prep_nop(sqe);
 		if (i == 4)
 			sqe->flags = IOSQE_IO_DRAIN;
+		if (cqe32) {
+			sqe->addr = 1234;
+			sqe->addr2 = 5678;
+		}
 		sqe->user_data = ++seq;
 	}
 
@@ -88,6 +109,16 @@ static int test_barrier_nop(struct io_uring *ring)
 			fprintf(stderr, "Unexpected 0 user_data\n");
 			goto err;
 		}
+		if (cqe32) {
+			if (cqe->big_cqe[0] != 1234) {
+				fprintf(stderr, "Unexpected extra1\n");
+				goto err;
+			}
+			if (cqe->big_cqe[1] != 5678) {
+				fprintf(stderr, "Unexpected extra2\n");
+				goto err;
+			}
+		}
 		io_uring_cqe_seen(ring, cqe);
 	}
 
-- 
2.30.2

[PATCH v4 6/6] liburing: Test all configurations with NOP test

[Stefan Roesch]

This runs the NOP test with all four configurations:
- default SQE and CQE size
- large SQE size
- large CQE size
- large SQE and large CQE size

Signed-off-by: Stefan Roesch <shr@fb.com>
---
 test/nop.c  | 43 +++++++++++++------------------------------
 test/test.h | 35 +++++++++++++++++++++++++++++++++++
 2 files changed, 48 insertions(+), 30 deletions(-)
 create mode 100644 test/test.h

diff --git a/test/nop.c b/test/nop.c
index 8656373..59da22c 100644
--- a/test/nop.c
+++ b/test/nop.c
@@ -11,6 +11,7 @@
 #include <fcntl.h>
 
 #include "liburing.h"
+#include "test.h"
 
 static int seq;
 
@@ -127,12 +128,14 @@ err:
 	return 1;
 }
 
-static int test_p(struct io_uring_params *p)
+static int test_ring(unsigned flags)
 {
 	struct io_uring ring;
+	struct io_uring_params p = { };
 	int ret;
 
-	ret = io_uring_queue_init_params(8, &ring, p);
+	p.flags = flags;
+	ret = io_uring_queue_init_params(8, &ring, &p);
 	if (ret) {
 		fprintf(stderr, "ring setup failed: %d\n", ret);
 		return 1;
@@ -150,29 +153,11 @@ static int test_p(struct io_uring_params *p)
 		goto err;
 	}
 
-	io_uring_queue_exit(&ring);
-	return 0;
 err:
 	io_uring_queue_exit(&ring);
 	return ret;
 }
 
-static int test_normal_ring(void)
-{
-	struct io_uring_params p = { };
-
-	return test_p(&p);
-}
-
-static int test_big_ring(void)
-{
-	struct io_uring_params p = { };
-
-	p.flags = IORING_SETUP_SQE128;
-	return test_p(&p);
-}
-
-
 int main(int argc, char *argv[])
 {
 	int ret;
@@ -180,17 +165,15 @@ int main(int argc, char *argv[])
 	if (argc > 1)
 		return 0;
 
-	ret = test_normal_ring();
-	if (ret) {
-		fprintf(stderr, "Normal ring test failed\n");
-		return ret;
-	}
-
-	ret = test_big_ring();
-	if (ret) {
-		fprintf(stderr, "Big ring test failed\n");
-		return ret;
+	FOR_ALL_TEST_CONFIGS {
+		fprintf(stderr, "Testing %s config\n", IORING_GET_TEST_CONFIG_DESCRIPTION());
+		ret = test_ring(IORING_GET_TEST_CONFIG_FLAGS());
+		if (ret) {
+			fprintf(stderr, "Normal ring test failed\n");
+			return ret;
+		}
 	}
 
+	fprintf(stderr, "PASS\n");
 	return 0;
 }
diff --git a/test/test.h b/test/test.h
new file mode 100644
index 0000000..3628163
--- /dev/null
+++ b/test/test.h
@@ -0,0 +1,35 @@
+/* SPDX-License-Identifier: MIT */
+/*
+ * Description: Test configs for tests.
+ */
+#ifndef LIBURING_TEST_H
+#define LIBURING_TEST_H
+
+#ifdef __cplusplus
+extern "C" {
+#endif
+
+typedef struct io_uring_test_config {
+	unsigned int flags;
+	const char *description;
+} io_uring_test_config;
+
+io_uring_test_config io_uring_test_configs[] = {
+	{ 0, 						"default" },
+	{ IORING_SETUP_SQE128, 				"large SQE"},
+	{ IORING_SETUP_CQE32, 				"large CQE"},
+	{ IORING_SETUP_SQE128 | IORING_SETUP_CQE32, 	"large SQE/CQE" },
+};
+
+#define FOR_ALL_TEST_CONFIGS							\
+	for (int i = 0; i < sizeof(io_uring_test_configs) / sizeof(io_uring_test_configs[0]); i++)
+
+#define IORING_GET_TEST_CONFIG_FLAGS() (io_uring_test_configs[i].flags)
+#define IORING_GET_TEST_CONFIG_DESCRIPTION() (io_uring_test_configs[i].description)
+
+
+#ifdef __cplusplus
+}
+#endif
+
+#endif
-- 
2.30.2