[PATCH liburing v2 0/3] single-issuer and poll benchmark

[PATCH liburing v2 0/3] single-issuer and poll benchmark

[Pavel Begunkov]

Add some tests to check the kernel enforcing single-issuer right, and
add a simple poll benchmark, might be useful if we do poll changes.

v2:
    remove copy-pasted test comments
    fix poll-bench 'argc' build problem

Pavel Begunkov (3):
  io_uring: update headers with IORING_SETUP_SINGLE_ISSUER
  examples: add a simple single-shot poll benchmark
  tests: test IORING_SETUP_SINGLE_ISSUER

 examples/Makefile               |   3 +-
 examples/poll-bench.c           | 101 +++++++++++++++++++
 src/include/liburing/io_uring.h |   5 +-
 test/Makefile                   |   1 +
 test/single-issuer.c            | 169 ++++++++++++++++++++++++++++++++
 5 files changed, 277 insertions(+), 2 deletions(-)
 create mode 100644 examples/poll-bench.c
 create mode 100644 test/single-issuer.c

-- 
2.36.1

[PATCH liburing v2 1/3] io_uring: update headers with IORING_SETUP_SINGLE_ISSUER

[Pavel Begunkov]

Signed-off-by: Pavel Begunkov <asml.silence@gmail.com>
---
 src/include/liburing/io_uring.h | 5 ++++-
 1 file changed, 4 insertions(+), 1 deletion(-)

diff --git a/src/include/liburing/io_uring.h b/src/include/liburing/io_uring.h
index 15d9fbd..ee6ccc9 100644
--- a/src/include/liburing/io_uring.h
+++ b/src/include/liburing/io_uring.h
@@ -137,9 +137,12 @@ enum {
  * IORING_SQ_TASKRUN in the sq ring flags. Not valid with COOP_TASKRUN.
  */
 #define IORING_SETUP_TASKRUN_FLAG	(1U << 9)
-
 #define IORING_SETUP_SQE128		(1U << 10) /* SQEs are 128 byte */
 #define IORING_SETUP_CQE32		(1U << 11) /* CQEs are 32 byte */
+/*
+ * Only one task is allowed to submit requests
+ */
+#define IORING_SETUP_SINGLE_ISSUER	(1U << 12)
 
 enum io_uring_op {
 	IORING_OP_NOP,
-- 
2.36.1

[PATCH liburing v2 2/3] examples: add a simple single-shot poll benchmark

[Pavel Begunkov]

Signed-off-by: Pavel Begunkov <asml.silence@gmail.com>
---
 examples/Makefile     |   3 +-
 examples/poll-bench.c | 101 ++++++++++++++++++++++++++++++++++++++++++
 2 files changed, 103 insertions(+), 1 deletion(-)
 create mode 100644 examples/poll-bench.c

diff --git a/examples/Makefile b/examples/Makefile
index 95a45f9..8e7067f 100644
--- a/examples/Makefile
+++ b/examples/Makefile
@@ -13,7 +13,8 @@ endif
 example_srcs := \
 	io_uring-cp.c \
 	io_uring-test.c \
-	link-cp.c
+	link-cp.c \
+	poll-bench.c
 
 all_targets :=
 
diff --git a/examples/poll-bench.c b/examples/poll-bench.c
new file mode 100644
index 0000000..66fd9e3
--- /dev/null
+++ b/examples/poll-bench.c
@@ -0,0 +1,101 @@
+/* SPDX-License-Identifier: MIT */
+#include <errno.h>
+#include <stdio.h>
+#include <unistd.h>
+#include <stdlib.h>
+#include <string.h>
+#include <signal.h>
+#include <poll.h>
+#include <sys/time.h>
+#include <sys/wait.h>
+
+#include "liburing.h"
+
+static char buf[4096];
+static unsigned long runtime_ms = 30000;
+
+static unsigned long gettimeofday_ms(void)
+{
+	struct timeval tv;
+
+	gettimeofday(&tv, NULL);
+	return (tv.tv_sec * 1000) + (tv.tv_usec / 1000);
+}
+
+int main(void)
+{
+	unsigned long tstop;
+	unsigned long nr_reqs = 0;
+	struct io_uring_cqe *cqe;
+	struct io_uring_sqe *sqe;
+	struct io_uring ring;
+	int pipe1[2];
+	int ret, i, qd = 32;
+
+	if (pipe(pipe1) != 0) {
+		perror("pipe");
+		return 1;
+	}
+
+	ret = io_uring_queue_init(1024, &ring, IORING_SETUP_SINGLE_ISSUER);
+	if (ret == -EINVAL) {
+		fprintf(stderr, "can't single\n");
+		ret = io_uring_queue_init(1024, &ring, 0);
+	}
+	if (ret) {
+		fprintf(stderr, "child: ring setup failed: %d\n", ret);
+		return 1;
+	}
+
+	ret = io_uring_register_files(&ring, pipe1, 2);
+	if (ret < 0) {
+		fprintf(stderr, "io_uring_register_files failed\n");
+		return 1;
+	}
+
+	ret = io_uring_register_ring_fd(&ring);
+	if (ret < 0) {
+		fprintf(stderr, "io_uring_register_ring_fd failed\n");
+		return 1;
+	}
+
+	tstop = gettimeofday_ms() + runtime_ms;
+	do {
+		for (i = 0; i < qd; i++) {
+			sqe = io_uring_get_sqe(&ring);
+			io_uring_prep_poll_add(sqe, 0, POLLIN);
+			sqe->flags |= IOSQE_FIXED_FILE;
+			sqe->user_data = 1;
+		}
+
+		ret = io_uring_submit(&ring);
+		if (ret != qd) {
+			fprintf(stderr, "child: sqe submit failed: %d\n", ret);
+			return 1;
+		}
+
+		ret = write(pipe1[1], buf, 1);
+		if (ret != 1) {
+			fprintf(stderr, "write failed %i\n", errno);
+			return 1;
+		}
+		ret = read(pipe1[0], buf, 1);
+		if (ret != 1) {
+			fprintf(stderr, "read failed %i\n", errno);
+			return 1;
+		}
+
+		for (i = 0; i < qd; i++) {
+			ret = io_uring_wait_cqe(&ring, &cqe);
+			if (ret < 0) {
+				fprintf(stderr, "child: wait completion %d\n", ret);
+				break;
+			}
+			io_uring_cqe_seen(&ring, cqe);
+			nr_reqs++;
+		}
+	} while (gettimeofday_ms() < tstop);
+
+	fprintf(stderr, "requests/s: %lu\n", nr_reqs * 1000UL / runtime_ms);
+	return 0;
+}
-- 
2.36.1

[PATCH liburing v2 3/3] tests: test IORING_SETUP_SINGLE_ISSUER

[Pavel Begunkov]

Signed-off-by: Pavel Begunkov <asml.silence@gmail.com>
---
 test/Makefile        |   1 +
 test/single-issuer.c | 169 +++++++++++++++++++++++++++++++++++++++++++
 2 files changed, 170 insertions(+)
 create mode 100644 test/single-issuer.c

diff --git a/test/Makefile b/test/Makefile
index ab031e0..0750996 100644
--- a/test/Makefile
+++ b/test/Makefile
@@ -169,6 +169,7 @@ test_srcs := \
 	wakeup-hang.c \
 	xattr.c \
 	skip-cqe.c \
+	single-issuer.c \
 	# EOL
 
 
diff --git a/test/single-issuer.c b/test/single-issuer.c
new file mode 100644
index 0000000..a688626
--- /dev/null
+++ b/test/single-issuer.c
@@ -0,0 +1,169 @@
+/* SPDX-License-Identifier: MIT */
+#include <errno.h>
+#include <stdio.h>
+#include <unistd.h>
+#include <stdlib.h>
+#include <string.h>
+#include <fcntl.h>
+#include <error.h>
+#include <sys/types.h>
+#include <sys/wait.h>
+
+#include "liburing.h"
+#include "test.h"
+
+static pid_t pid;
+
+static pid_t fork_t(void)
+{
+	pid = fork();
+	if (pid == -1) {
+		fprintf(stderr, "fork failed\n");
+		exit(1);
+	}
+	return pid;
+}
+
+static int wait_child_t(void)
+{
+	int wstatus;
+
+	if (waitpid(pid, &wstatus, 0) == (pid_t)-1) {
+		perror("waitpid()");
+		exit(1);
+	}
+	if (!WIFEXITED(wstatus)) {
+		fprintf(stderr, "child failed %i\n", WEXITSTATUS(wstatus));
+		exit(1);
+	}
+
+	return WEXITSTATUS(wstatus);
+}
+
+static int try_submit(struct io_uring *ring)
+{
+	struct io_uring_cqe *cqe;
+	struct io_uring_sqe *sqe;
+	int ret;
+
+	sqe = io_uring_get_sqe(ring);
+	io_uring_prep_nop(sqe);
+	sqe->user_data = 42;
+
+	ret = io_uring_submit(ring);
+	if (ret < 0)
+		return ret;
+
+	if (ret != 1)
+		error(1, ret, "submit %i", ret);
+	ret = io_uring_wait_cqe(ring, &cqe);
+	if (ret)
+		error(1, ret, "wait fail %i", ret);
+
+	if (cqe->res || cqe->user_data != 42)
+		error(1, ret, "invalid cqe");
+
+	io_uring_cqe_seen(ring, cqe);
+	return 0;
+}
+
+int main(int argc, char *argv[])
+{
+	struct io_uring ring;
+	int ret;
+
+	if (argc > 1)
+		return 0;
+
+	ret = io_uring_queue_init(8, &ring, IORING_SETUP_SINGLE_ISSUER);
+	if (ret == -EINVAL) {
+		fprintf(stderr, "SETUP_SINGLE_ISSUER is not supported, skip\n");
+		return 0;
+	} else if (ret) {
+		error(1, ret, "ring init (1) %i", ret);
+	}
+
+	/* test that the creator iw allowed to submit */
+	ret = try_submit(&ring);
+	if (ret) {
+		fprintf(stderr, "the creater can't submit %i\n", ret);
+		return 1;
+	}
+
+	/* test that a second submitter doesn't succeed */
+	if (!fork_t()) {
+		ret = try_submit(&ring);
+		if (ret != -EEXIST) {
+			fprintf(stderr, "not owner child could submit %i\n", ret);
+			exit(1);
+		}
+		exit(0);
+	} else {
+		if (wait_child_t())
+			return 1;
+	}
+	io_uring_queue_exit(&ring);
+
+
+	/* test that the first submitter but not creator can submit */
+	ret = io_uring_queue_init(8, &ring, IORING_SETUP_SINGLE_ISSUER);
+	if (ret)
+		error(1, ret, "ring init (2) %i", ret);
+
+	if (!fork_t()) {
+		ret = try_submit(&ring);
+		if (ret) {
+			fprintf(stderr, "not owner child could submit %i\n", ret);
+			exit(1);
+		}
+		exit(0);
+	} else {
+		if (wait_child_t())
+			return 1;
+	}
+	io_uring_queue_exit(&ring);
+
+	/* test that anyone can submit to a SQPOLL|SINGLE_ISSUER ring */
+	ret = io_uring_queue_init(8, &ring, IORING_SETUP_SINGLE_ISSUER|IORING_SETUP_SQPOLL);
+	if (ret)
+		error(1, ret, "ring init (3) %i", ret);
+
+	ret = try_submit(&ring);
+	if (ret) {
+		fprintf(stderr, "SQPOLL submit failed (creator) %i\n", ret);
+		exit(1);
+	}
+
+	if (!fork_t()) {
+		ret = try_submit(&ring);
+		if (ret) {
+			fprintf(stderr, "SQPOLL submit failed (child) %i\n", ret);
+			exit(1);
+		}
+		exit(0);
+	} else {
+		if (wait_child_t())
+			return 1;
+	}
+	io_uring_queue_exit(&ring);
+
+	/* test that IORING_ENTER_REGISTERED_RING doesn't break anything */
+	ret = io_uring_queue_init(8, &ring, IORING_SETUP_SINGLE_ISSUER);
+	if (ret)
+		error(1, ret, "ring init (4) %i", ret);
+
+	if (!fork_t()) {
+		ret = try_submit(&ring);
+		if (ret) {
+			fprintf(stderr, "not owner child could submit %i\n", ret);
+			exit(1);
+		}
+		exit(0);
+	} else {
+		if (wait_child_t())
+			return 1;
+	}
+	io_uring_queue_exit(&ring);
+
+	return 0;
+}
-- 
2.36.1

Re: [PATCH liburing v2 0/3] single-issuer and poll benchmark

[Jens Axboe]

On 6/14/22 9:27 AM, Pavel Begunkov wrote:
> Add some tests to check the kernel enforcing single-issuer right, and
> add a simple poll benchmark, might be useful if we do poll changes.

Should we add a benchmark/ or something directory rather than use
examples/ ?

I know Dylan was looking at that at one point. I don't feel too
strongly, as long as it doesn't go into test/.

-- 
Jens Axboe

Re: [PATCH liburing v2 0/3] single-issuer and poll benchmark

[Pavel Begunkov]

On 6/14/22 16:30, Jens Axboe wrote:
> On 6/14/22 9:27 AM, Pavel Begunkov wrote:
>> Add some tests to check the kernel enforcing single-issuer right, and
>> add a simple poll benchmark, might be useful if we do poll changes.
> 
> Should we add a benchmark/ or something directory rather than use
> examples/ ?
> 
> I know Dylan was looking at that at one point. I don't feel too
> strongly, as long as it doesn't go into test/.

I don't care much myself, I can respin it once (if) the kernel
side is queued.

-- 
Pavel Begunkov

Re: [PATCH liburing v2 0/3] single-issuer and poll benchmark

[Jens Axboe]

On 6/14/22 9:48 AM, Pavel Begunkov wrote:
> On 6/14/22 16:30, Jens Axboe wrote:
>> On 6/14/22 9:27 AM, Pavel Begunkov wrote:
>>> Add some tests to check the kernel enforcing single-issuer right, and
>>> add a simple poll benchmark, might be useful if we do poll changes.
>>
>> Should we add a benchmark/ or something directory rather than use
>> examples/ ?
>>
>> I know Dylan was looking at that at one point. I don't feel too
>> strongly, as long as it doesn't go into test/.
> 
> I don't care much myself, I can respin it once (if) the kernel
> side is queued.

I'm leaning towards just using examples/ - but maybe Dylan had some
reasoning for the new directory. CC'ed.

-- 
Jens Axboe

Re: [PATCH liburing v2 0/3] single-issuer and poll benchmark

[Dylan Yudaken]

On Tue, 2022-06-14 at 09:51 -0600, Jens Axboe wrote:
> On 6/14/22 9:48 AM, Pavel Begunkov wrote:
> > On 6/14/22 16:30, Jens Axboe wrote:
> > > On 6/14/22 9:27 AM, Pavel Begunkov wrote:
> > > > Add some tests to check the kernel enforcing single-issuer
> > > > right, and
> > > > add a simple poll benchmark, might be useful if we do poll
> > > > changes.
> > > 
> > > Should we add a benchmark/ or something directory rather than use
> > > examples/ ?
> > > 
> > > I know Dylan was looking at that at one point. I don't feel too
> > > strongly, as long as it doesn't go into test/.
> > 
> > I don't care much myself, I can respin it once (if) the kernel
> > side is queued.
> 
> I'm leaning towards just using examples/ - but maybe Dylan had some
> reasoning for the new directory. CC'ed.
> 

I wanted to have some common framework for benchmarks to make it easy
to write new ones and get some nice numbers. In that case it would make
sense for them to be in one place.

But I haven't finished it yet, so probably for now examples/ is best
and if/when I finish it I can port these over.

Re: [PATCH liburing v2 0/3] single-issuer and poll benchmark

[Hao Xu]

On 6/15/22 02:23, Dylan Yudaken wrote:
> On Tue, 2022-06-14 at 09:51 -0600, Jens Axboe wrote:
>> On 6/14/22 9:48 AM, Pavel Begunkov wrote:
>>> On 6/14/22 16:30, Jens Axboe wrote:
>>>> On 6/14/22 9:27 AM, Pavel Begunkov wrote:
>>>>> Add some tests to check the kernel enforcing single-issuer
>>>>> right, and
>>>>> add a simple poll benchmark, might be useful if we do poll
>>>>> changes.
>>>>
>>>> Should we add a benchmark/ or something directory rather than use
>>>> examples/ ?
>>>>
>>>> I know Dylan was looking at that at one point. I don't feel too
>>>> strongly, as long as it doesn't go into test/.
>>>
>>> I don't care much myself, I can respin it once (if) the kernel
>>> side is queued.
>>
>> I'm leaning towards just using examples/ - but maybe Dylan had some
>> reasoning for the new directory. CC'ed.
>>
> 
> I wanted to have some common framework for benchmarks to make it easy
> to write new ones and get some nice numbers. In that case it would make
> sense for them to be in one place.

This sounds great, it's really nice to have a series of standard
performance tests rather than temporary tests created along with new
kernel patches. A framework is something nicer. Actually there are
people doing this though just a simple one, FWIW:

https://gitee.com/anolis/perf-test-for-io_uring

> 
> But I haven't finished it yet, so probably for now examples/ is best
> and if/when I finish it I can port these over.
>