[PATCH liburing v2 0/3] single-issuer and poll benchmark

[PATCH liburing v2 0/3] single-issuer and poll benchmark

[Pavel Begunkov]

Add some tests to check the kernel enforcing single-issuer right, and
add a simple poll benchmark, might be useful if we do poll changes.

v2:
    remove copy-pasted test comments
    fix poll-bench 'argc' build problem

Pavel Begunkov (3):
  io_uring: update headers with IORING_SETUP_SINGLE_ISSUER
  examples: add a simple single-shot poll benchmark
  tests: test IORING_SETUP_SINGLE_ISSUER

 examples/Makefile               |   3 +-
 examples/poll-bench.c           | 101 +++++++++++++++++++
 src/include/liburing/io_uring.h |   5 +-
 test/Makefile                   |   1 +
 test/single-issuer.c            | 169 ++++++++++++++++++++++++++++++++
 5 files changed, 277 insertions(+), 2 deletions(-)
 create mode 100644 examples/poll-bench.c
 create mode 100644 test/single-issuer.c

-- 
2.36.1

[PATCH liburing v2 1/3] io_uring: update headers with IORING_SETUP_SINGLE_ISSUER

[Pavel Begunkov]

Signed-off-by: Pavel Begunkov <asml.silence@gmail.com>
---
 src/include/liburing/io_uring.h | 5 ++++-
 1 file changed, 4 insertions(+), 1 deletion(-)

diff --git a/src/include/liburing/io_uring.h b/src/include/liburing/io_uring.h
index 15d9fbd..ee6ccc9 100644
--- a/src/include/liburing/io_uring.h
+++ b/src/include/liburing/io_uring.h
@@ -137,9 +137,12 @@ enum {
  * IORING_SQ_TASKRUN in the sq ring flags. Not valid with COOP_TASKRUN.
  */
 #define IORING_SETUP_TASKRUN_FLAG	(1U << 9)
-
 #define IORING_SETUP_SQE128		(1U << 10) /* SQEs are 128 byte */
 #define IORING_SETUP_CQE32		(1U << 11) /* CQEs are 32 byte */
+/*
+ * Only one task is allowed to submit requests
+ */
+#define IORING_SETUP_SINGLE_ISSUER	(1U << 12)
 
 enum io_uring_op {
 	IORING_OP_NOP,
-- 
2.36.1

[PATCH liburing v2 2/3] examples: add a simple single-shot poll benchmark

[Pavel Begunkov]

Signed-off-by: Pavel Begunkov <asml.silence@gmail.com>
---
 examples/Makefile     |   3 +-
 examples/poll-bench.c | 101 ++++++++++++++++++++++++++++++++++++++++++
 2 files changed, 103 insertions(+), 1 deletion(-)
 create mode 100644 examples/poll-bench.c

diff --git a/examples/Makefile b/examples/Makefile
index 95a45f9..8e7067f 100644
--- a/examples/Makefile
+++ b/examples/Makefile
@@ -13,7 +13,8 @@ endif
 example_srcs := \
 	io_uring-cp.c \
 	io_uring-test.c \
-	link-cp.c
+	link-cp.c \
+	poll-bench.c
 
 all_targets :=
 
diff --git a/examples/poll-bench.c b/examples/poll-bench.c
new file mode 100644
index 0000000..66fd9e3
--- /dev/null
+++ b/examples/poll-bench.c
@@ -0,0 +1,101 @@
+/* SPDX-License-Identifier: MIT */
+#include <errno.h>
+#include <stdio.h>
+#include <unistd.h>
+#include <stdlib.h>
+#include <string.h>
+#include <signal.h>
+#include <poll.h>
+#include <sys/time.h>
+#include <sys/wait.h>
+
+#include "liburing.h"
+
+static char buf[4096];
+static unsigned long runtime_ms = 30000;
+
+static unsigned long gettimeofday_ms(void)
+{
+	struct timeval tv;
+
+	gettimeofday(&tv, NULL);
+	return (tv.tv_sec * 1000) + (tv.tv_usec / 1000);
+}
+
+int main(void)
+{
+	unsigned long tstop;
+	unsigned long nr_reqs = 0;
+	struct io_uring_cqe *cqe;
+	struct io_uring_sqe *sqe;
+	struct io_uring ring;
+	int pipe1[2];
+	int ret, i, qd = 32;
+
+	if (pipe(pipe1) != 0) {
+		perror("pipe");
+		return 1;
+	}
+
+	ret = io_uring_queue_init(1024, &ring, IORING_SETUP_SINGLE_ISSUER);
+	if (ret == -EINVAL) {
+		fprintf(stderr, "can't single\n");
+		ret = io_uring_queue_init(1024, &ring, 0);
+	}
+	if (ret) {
+		fprintf(stderr, "child: ring setup failed: %d\n", ret);
+		return 1;
+	}
+
+	ret = io_uring_register_files(&ring, pipe1, 2);
+	if (ret < 0) {
+		fprintf(stderr, "io_uring_register_files failed\n");
+		return 1;
+	}
+
+	ret = io_uring_register_ring_fd(&ring);
+	if (ret < 0) {
+		fprintf(stderr, "io_uring_register_ring_fd failed\n");
+		return 1;
+	}
+
+	tstop = gettimeofday_ms() + runtime_ms;
+	do {
+		for (i = 0; i < qd; i++) {
+			sqe = io_uring_get_sqe(&ring);
+			io_uring_prep_poll_add(sqe, 0, POLLIN);
+			sqe->flags |= IOSQE_FIXED_FILE;
+			sqe->user_data = 1;
+		}
+
+		ret = io_uring_submit(&ring);
+		if (ret != qd) {
+			fprintf(stderr, "child: sqe submit failed: %d\n", ret);
+			return 1;
+		}
+
+		ret = write(pipe1[1], buf, 1);
+		if (ret != 1) {
+			fprintf(stderr, "write failed %i\n", errno);
+			return 1;
+		}
+		ret = read(pipe1[0], buf, 1);
+		if (ret != 1) {
+			fprintf(stderr, "read failed %i\n", errno);
+			return 1;
+		}
+
+		for (i = 0; i < qd; i++) {
+			ret = io_uring_wait_cqe(&ring, &cqe);
+			if (ret < 0) {
+				fprintf(stderr, "child: wait completion %d\n", ret);
+				break;
+			}
+			io_uring_cqe_seen(&ring, cqe);
+			nr_reqs++;
+		}
+	} while (gettimeofday_ms() < tstop);
+
+	fprintf(stderr, "requests/s: %lu\n", nr_reqs * 1000UL / runtime_ms);
+	return 0;
+}
-- 
2.36.1

[PATCH liburing v2 3/3] tests: test IORING_SETUP_SINGLE_ISSUER

[Pavel Begunkov]

Signed-off-by: Pavel Begunkov <asml.silence@gmail.com>
---
 test/Makefile        |   1 +
 test/single-issuer.c | 169 +++++++++++++++++++++++++++++++++++++++++++
 2 files changed, 170 insertions(+)
 create mode 100644 test/single-issuer.c

diff --git a/test/Makefile b/test/Makefile
index ab031e0..0750996 100644
--- a/test/Makefile
+++ b/test/Makefile
@@ -169,6 +169,7 @@ test_srcs := \
 	wakeup-hang.c \
 	xattr.c \
 	skip-cqe.c \
+	single-issuer.c \
 	# EOL
 
 
diff --git a/test/single-issuer.c b/test/single-issuer.c
new file mode 100644
index 0000000..a688626
--- /dev/null
+++ b/test/single-issuer.c
@@ -0,0 +1,169 @@
+/* SPDX-License-Identifier: MIT */
+#include <errno.h>
+#include <stdio.h>
+#include <unistd.h>
+#include <stdlib.h>
+#include <string.h>
+#include <fcntl.h>
+#include <error.h>
+#include <sys/types.h>
+#include <sys/wait.h>
+
+#include "liburing.h"
+#include "test.h"
+
+static pid_t pid;
+
+static pid_t fork_t(void)
+{
+	pid = fork();
+	if (pid == -1) {
+		fprintf(stderr, "fork failed\n");
+		exit(1);
+	}
+	return pid;
+}
+
+static int wait_child_t(void)
+{
+	int wstatus;
+
+	if (waitpid(pid, &wstatus, 0) == (pid_t)-1) {
+		perror("waitpid()");
+		exit(1);
+	}
+	if (!WIFEXITED(wstatus)) {
+		fprintf(stderr, "child failed %i\n", WEXITSTATUS(wstatus));
+		exit(1);
+	}
+
+	return WEXITSTATUS(wstatus);
+}
+
+static int try_submit(struct io_uring *ring)
+{
+	struct io_uring_cqe *cqe;
+	struct io_uring_sqe *sqe;
+	int ret;
+
+	sqe = io_uring_get_sqe(ring);
+	io_uring_prep_nop(sqe);
+	sqe->user_data = 42;
+
+	ret = io_uring_submit(ring);
+	if (ret < 0)
+		return ret;
+
+	if (ret != 1)
+		error(1, ret, "submit %i", ret);
+	ret = io_uring_wait_cqe(ring, &cqe);
+	if (ret)
+		error(1, ret, "wait fail %i", ret);
+
+	if (cqe->res || cqe->user_data != 42)
+		error(1, ret, "invalid cqe");
+
+	io_uring_cqe_seen(ring, cqe);
+	return 0;
+}
+
+int main(int argc, char *argv[])
+{
+	struct io_uring ring;
+	int ret;
+
+	if (argc > 1)
+		return 0;
+
+	ret = io_uring_queue_init(8, &ring, IORING_SETUP_SINGLE_ISSUER);
+	if (ret == -EINVAL) {
+		fprintf(stderr, "SETUP_SINGLE_ISSUER is not supported, skip\n");
+		return 0;
+	} else if (ret) {
+		error(1, ret, "ring init (1) %i", ret);
+	}
+
+	/* test that the creator iw allowed to submit */
+	ret = try_submit(&ring);
+	if (ret) {
+		fprintf(stderr, "the creater can't submit %i\n", ret);
+		return 1;
+	}
+
+	/* test that a second submitter doesn't succeed */
+	if (!fork_t()) {
+		ret = try_submit(&ring);
+		if (ret != -EEXIST) {
+			fprintf(stderr, "not owner child could submit %i\n", ret);
+			exit(1);
+		}
+		exit(0);
+	} else {
+		if (wait_child_t())
+			return 1;
+	}
+	io_uring_queue_exit(&ring);
+
+
+	/* test that the first submitter but not creator can submit */
+	ret = io_uring_queue_init(8, &ring, IORING_SETUP_SINGLE_ISSUER);
+	if (ret)
+		error(1, ret, "ring init (2) %i", ret);
+
+	if (!fork_t()) {
+		ret = try_submit(&ring);
+		if (ret) {
+			fprintf(stderr, "not owner child could submit %i\n", ret);
+			exit(1);
+		}
+		exit(0);
+	} else {
+		if (wait_child_t())
+			return 1;
+	}
+	io_uring_queue_exit(&ring);
+
+	/* test that anyone can submit to a SQPOLL|SINGLE_ISSUER ring */
+	ret = io_uring_queue_init(8, &ring, IORING_SETUP_SINGLE_ISSUER|IORING_SETUP_SQPOLL);
+	if (ret)
+		error(1, ret, "ring init (3) %i", ret);
+
+	ret = try_submit(&ring);
+	if (ret) {
+		fprintf(stderr, "SQPOLL submit failed (creator) %i\n", ret);
+		exit(1);
+	}
+
+	if (!fork_t()) {
+		ret = try_submit(&ring);
+		if (ret) {
+			fprintf(stderr, "SQPOLL submit failed (child) %i\n", ret);
+			exit(1);
+		}
+		exit(0);
+	} else {
+		if (wait_child_t())
+			return 1;
+	}
+	io_uring_queue_exit(&ring);
+
+	/* test that IORING_ENTER_REGISTERED_RING doesn't break anything */
+	ret = io_uring_queue_init(8, &ring, IORING_SETUP_SINGLE_ISSUER);
+	if (ret)
+		error(1, ret, "ring init (4) %i", ret);
+
+	if (!fork_t()) {
+		ret = try_submit(&ring);
+		if (ret) {
+			fprintf(stderr, "not owner child could submit %i\n", ret);
+			exit(1);
+		}
+		exit(0);
+	} else {
+		if (wait_child_t())
+			return 1;
+	}
+	io_uring_queue_exit(&ring);
+
+	return 0;
+}
-- 
2.36.1

Re: [PATCH liburing v2 0/3] single-issuer and poll benchmark

[Jens Axboe]

On 6/14/22 9:27 AM, Pavel Begunkov wrote:
> Add some tests to check the kernel enforcing single-issuer right, and
> add a simple poll benchmark, might be useful if we do poll changes.

Should we add a benchmark/ or something directory rather than use
examples/ ?

I know Dylan was looking at that at one point. I don't feel too
strongly, as long as it doesn't go into test/.

-- 
Jens Axboe

Re: [PATCH liburing v2 0/3] single-issuer and poll benchmark

[Pavel Begunkov]

On 6/14/22 16:30, Jens Axboe wrote:
> On 6/14/22 9:27 AM, Pavel Begunkov wrote:
>> Add some tests to check the kernel enforcing single-issuer right, and
>> add a simple poll benchmark, might be useful if we do poll changes.
> 
> Should we add a benchmark/ or something directory rather than use
> examples/ ?
> 
> I know Dylan was looking at that at one point. I don't feel too
> strongly, as long as it doesn't go into test/.

I don't care much myself, I can respin it once (if) the kernel
side is queued.

-- 
Pavel Begunkov

Re: [PATCH liburing v2 0/3] single-issuer and poll benchmark

[Jens Axboe]

On 6/14/22 9:48 AM, Pavel Begunkov wrote:
> On 6/14/22 16:30, Jens Axboe wrote:
>> On 6/14/22 9:27 AM, Pavel Begunkov wrote:
>>> Add some tests to check the kernel enforcing single-issuer right, and
>>> add a simple poll benchmark, might be useful if we do poll changes.
>>
>> Should we add a benchmark/ or something directory rather than use
>> examples/ ?
>>
>> I know Dylan was looking at that at one point. I don't feel too
>> strongly, as long as it doesn't go into test/.
> 
> I don't care much myself, I can respin it once (if) the kernel
> side is queued.

I'm leaning towards just using examples/ - but maybe Dylan had some
reasoning for the new directory. CC'ed.

-- 
Jens Axboe

Re: [PATCH liburing v2 0/3] single-issuer and poll benchmark

[Dylan Yudaken]

[-- Warning: decoded text below may be mangled, UTF-8 assumed --]
[-- Attachment #1: Type: text/plain; charset="utf-8", Size: 567 bytes --]

:tî{m6ÛNµá«t÷tëM	z{\0ź\x1eº-xéúÿ^?Ûoxð\x03\x0fj÷¥\x05è.žJ/º-xéúÿ^?ÛmzßB^žÀ1n‡°®‹^:~¿×öÛÝ»\0ÃÚ½éAz\v§’‹ð®‹^\x01×l¢g­zËl¶‡!yÉ-…é\x1e®w¥zwè­È§‚ȧ‚W¢²Ëž®¸ †Ö§u§]jȦ¦W©¢Y[zw!™ªäš(!µ·®±çî–'ðyÚ)¢Y\…©àzÄ¡¢é]Áæu¦ÞÈfj¹?¢»(™ëaŠx\x1dŠ·œ¶Šò­«az»aj{¬yìZš™^³ò$žŒ\x03ÊV§Á«%¢‰"ž\x06­¶\x16­jÚ'zš"žÒ\x1d¢{_yém¢‹-®‰à—&¬–‰àjÈ­v‡¬žØ(Š{hµë-ü‡hž×\x1a­é®r\x19²±é_!Ƨ­ë)Šx­¢w\x1e‰ûazG«él‰×¢²«ž¹çHšWšž)ඌ\x1a­Û#ºËn²)à{\x16¦¦W¬ý»­™¬›x<¥jxZvÊ&z·š²‰âž\aè®Ø^ì\x1dŠ·œ¶Šò\b'#\x06§µçm¢\x16¯zÊ&yÊ&š‰ß­©žÂŠä~ŠÛzw!™ªä²Ú&jG¢µæ¬ÊÚ0®+^ì(ë\x1aØ\x1e¶Ê&zxœz{¦mêì"{aj×\x1a±è­Â‹¥vf¤zǧ±çè®Ø^šÚ^[z)èêeiǁºÒ!j÷§µø§ŠÈ^v+rzÛ(¦º^[i¹r~Šç£\a±jjezÏâ±·¬µ©Ý‰ÿð…éÈ~)â²\x18­!Ƨ¦Ší¶\x17¬z‹Þ

Re: [PATCH liburing v2 0/3] single-issuer and poll benchmark

[Hao Xu]

On 6/15/22 02:23, Dylan Yudaken wrote:
> On Tue, 2022-06-14 at 09:51 -0600, Jens Axboe wrote:
>> On 6/14/22 9:48 AM, Pavel Begunkov wrote:
>>> On 6/14/22 16:30, Jens Axboe wrote:
>>>> On 6/14/22 9:27 AM, Pavel Begunkov wrote:
>>>>> Add some tests to check the kernel enforcing single-issuer
>>>>> right, and
>>>>> add a simple poll benchmark, might be useful if we do poll
>>>>> changes.
>>>>
>>>> Should we add a benchmark/ or something directory rather than use
>>>> examples/ ?
>>>>
>>>> I know Dylan was looking at that at one point. I don't feel too
>>>> strongly, as long as it doesn't go into test/.
>>>
>>> I don't care much myself, I can respin it once (if) the kernel
>>> side is queued.
>>
>> I'm leaning towards just using examples/ - but maybe Dylan had some
>> reasoning for the new directory. CC'ed.
>>
> 
> I wanted to have some common framework for benchmarks to make it easy
> to write new ones and get some nice numbers. In that case it would make
> sense for them to be in one place.

This sounds great, it's really nice to have a series of standard
performance tests rather than temporary tests created along with new
kernel patches. A framework is something nicer. Actually there are
people doing this though just a simple one, FWIW:

https://gitee.com/anolis/perf-test-for-io_uring

> 
> But I haven't finished it yet, so probably for now examples/ is best
> and if/when I finish it I can port these over.
>