[PATCH liburing 0/5] multishot recvmsg docs and example

[PATCH liburing 0/5] multishot recvmsg docs and example

[Dylan Yudaken]

Some multishot recvmsg patches for liburing:

Patches 1-3  cleanup the API a little while we're doing this.
Patch 4 adds docs for the new API
Patch 5 adds an example UDP echo server that uses the API


Dylan Yudaken (5):
  more consistent multishot recvmsg API parameter names
  order like functions together in liburing.h
  change io_uring_recvmsg_payload_length return type
  add documentation for multishot recvmsg
  add an example for a UDP server

 .gitignore                            |   1 +
 examples/Makefile                     |   1 +
 examples/io_uring-udp.c               | 388 ++++++++++++++++++++++++++
 man/io_uring_prep_recvmsg.3           |  30 ++
 man/io_uring_prep_recvmsg_multishot.3 |   1 +
 man/io_uring_recvmsg_cmsg_firsthdr.3  |   1 +
 man/io_uring_recvmsg_cmsg_nexthdr.3   |   1 +
 man/io_uring_recvmsg_name.3           |   1 +
 man/io_uring_recvmsg_out.3            |  78 ++++++
 man/io_uring_recvmsg_payload.3        |   1 +
 man/io_uring_recvmsg_payload_length.3 |   1 +
 man/io_uring_recvmsg_validate.3       |   1 +
 src/include/liburing.h                |  48 ++--
 13 files changed, 529 insertions(+), 24 deletions(-)
 create mode 100644 examples/io_uring-udp.c
 create mode 120000 man/io_uring_prep_recvmsg_multishot.3
 create mode 120000 man/io_uring_recvmsg_cmsg_firsthdr.3
 create mode 120000 man/io_uring_recvmsg_cmsg_nexthdr.3
 create mode 120000 man/io_uring_recvmsg_name.3
 create mode 100644 man/io_uring_recvmsg_out.3
 create mode 120000 man/io_uring_recvmsg_payload.3
 create mode 120000 man/io_uring_recvmsg_payload_length.3
 create mode 120000 man/io_uring_recvmsg_validate.3


base-commit: 30a20795d7e4f300c606c6a2aa0a4c9492882d1d
-- 
2.30.2

[PATCH liburing 1/5] more consistent multishot recvmsg API parameter names

[Dylan Yudaken]

Make the parameters more consistent with for example cmsg(3), and with
itself (in the form of buf_len).

Signed-off-by: Dylan Yudaken <dylany@fb.com>
---
 src/include/liburing.h | 22 +++++++++++-----------
 1 file changed, 11 insertions(+), 11 deletions(-)

diff --git a/src/include/liburing.h b/src/include/liburing.h
index cffdabd71144..d3cbdb037b32 100644
--- a/src/include/liburing.h
+++ b/src/include/liburing.h
@@ -734,9 +734,9 @@ static inline void io_uring_prep_recv_multishot(struct io_uring_sqe *sqe,
 }
 
 static inline struct io_uring_recvmsg_out *
-io_uring_recvmsg_validate(void *buf, int buf_len, struct msghdr *m)
+io_uring_recvmsg_validate(void *buf, int buf_len, struct msghdr *msgh)
 {
-	int header = m->msg_controllen + m->msg_namelen +
+	int header = msgh->msg_controllen + msgh->msg_namelen +
 				sizeof(struct io_uring_recvmsg_out);
 
 	if (buf_len < header)
@@ -751,42 +751,42 @@ static inline void *io_uring_recvmsg_name(struct io_uring_recvmsg_out *o)
 
 static inline struct cmsghdr *
 io_uring_recvmsg_cmsg_firsthdr(struct io_uring_recvmsg_out *o,
-			       struct msghdr *m)
+			       struct msghdr *msgh)
 {
 	if (o->controllen < sizeof(struct cmsghdr))
 		return NULL;
 
 	return (struct cmsghdr *)((unsigned char *) io_uring_recvmsg_name(o) +
-			m->msg_namelen);
+			msgh->msg_namelen);
 }
 
 static inline void *io_uring_recvmsg_payload(struct io_uring_recvmsg_out *o,
-					     struct msghdr *m)
+					     struct msghdr *msgh)
 {
 	return (void *)((unsigned char *)io_uring_recvmsg_name(o) +
-			m->msg_namelen + m->msg_controllen);
+			msgh->msg_namelen + msgh->msg_controllen);
 }
 
 static inline size_t
 io_uring_recvmsg_payload_length(struct io_uring_recvmsg_out *o,
-				int buf_length, struct msghdr *m)
+				int buf_len, struct msghdr *msgh)
 {
 	unsigned long payload_start, payload_end;
 
-	payload_start = (unsigned long) io_uring_recvmsg_payload(o, m);
-	payload_end = (unsigned long) o + buf_length;
+	payload_start = (unsigned long) io_uring_recvmsg_payload(o, msgh);
+	payload_end = (unsigned long) o + buf_len;
 	return payload_end - payload_start;
 }
 
 static inline struct cmsghdr *
-io_uring_recvmsg_cmsg_nexthdr(struct io_uring_recvmsg_out *o, struct msghdr *m,
+io_uring_recvmsg_cmsg_nexthdr(struct io_uring_recvmsg_out *o, struct msghdr *msgh,
 			      struct cmsghdr *cmsg)
 {
 	unsigned char *end;
 
 	if (cmsg->cmsg_len < sizeof(struct cmsghdr))
 		return NULL;
-	end = (unsigned char *) io_uring_recvmsg_cmsg_firsthdr(o, m) +
+	end = (unsigned char *) io_uring_recvmsg_cmsg_firsthdr(o, msgh) +
 		o->controllen;
 	cmsg = (struct cmsghdr *)((unsigned char *) cmsg +
 			CMSG_ALIGN(cmsg->cmsg_len));
-- 
2.30.2

[PATCH liburing 2/5] order like functions together in liburing.h

[Dylan Yudaken]

the cmsg related functions were not together - which was confusing

Signed-off-by: Dylan Yudaken <dylany@fb.com>
---
 src/include/liburing.h | 36 ++++++++++++++++++------------------
 1 file changed, 18 insertions(+), 18 deletions(-)

diff --git a/src/include/liburing.h b/src/include/liburing.h
index d3cbdb037b32..9e7f06f390be 100644
--- a/src/include/liburing.h
+++ b/src/include/liburing.h
@@ -760,24 +760,6 @@ io_uring_recvmsg_cmsg_firsthdr(struct io_uring_recvmsg_out *o,
 			msgh->msg_namelen);
 }
 
-static inline void *io_uring_recvmsg_payload(struct io_uring_recvmsg_out *o,
-					     struct msghdr *msgh)
-{
-	return (void *)((unsigned char *)io_uring_recvmsg_name(o) +
-			msgh->msg_namelen + msgh->msg_controllen);
-}
-
-static inline size_t
-io_uring_recvmsg_payload_length(struct io_uring_recvmsg_out *o,
-				int buf_len, struct msghdr *msgh)
-{
-	unsigned long payload_start, payload_end;
-
-	payload_start = (unsigned long) io_uring_recvmsg_payload(o, msgh);
-	payload_end = (unsigned long) o + buf_len;
-	return payload_end - payload_start;
-}
-
 static inline struct cmsghdr *
 io_uring_recvmsg_cmsg_nexthdr(struct io_uring_recvmsg_out *o, struct msghdr *msgh,
 			      struct cmsghdr *cmsg)
@@ -799,6 +781,24 @@ io_uring_recvmsg_cmsg_nexthdr(struct io_uring_recvmsg_out *o, struct msghdr *msg
 	return cmsg;
 }
 
+static inline void *io_uring_recvmsg_payload(struct io_uring_recvmsg_out *o,
+					     struct msghdr *msgh)
+{
+	return (void *)((unsigned char *)io_uring_recvmsg_name(o) +
+			msgh->msg_namelen + msgh->msg_controllen);
+}
+
+static inline size_t
+io_uring_recvmsg_payload_length(struct io_uring_recvmsg_out *o,
+				int buf_len, struct msghdr *msgh)
+{
+	unsigned long payload_start, payload_end;
+
+	payload_start = (unsigned long) io_uring_recvmsg_payload(o, msgh);
+	payload_end = (unsigned long) o + buf_len;
+	return payload_end - payload_start;
+}
+
 static inline void io_uring_prep_openat2(struct io_uring_sqe *sqe, int dfd,
 					const char *path, struct open_how *how)
 {
-- 
2.30.2

[PATCH liburing 3/5] change io_uring_recvmsg_payload_length return type

[Dylan Yudaken]

Payloads can not be bigger than UINT_MAX, so no need to force a bigger
type on the caller.

Signed-off-by: Dylan Yudaken <dylany@fb.com>
---
 src/include/liburing.h | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)

diff --git a/src/include/liburing.h b/src/include/liburing.h
index 9e7f06f390be..06f4a50bacb1 100644
--- a/src/include/liburing.h
+++ b/src/include/liburing.h
@@ -788,7 +788,7 @@ static inline void *io_uring_recvmsg_payload(struct io_uring_recvmsg_out *o,
 			msgh->msg_namelen + msgh->msg_controllen);
 }
 
-static inline size_t
+static inline unsigned int
 io_uring_recvmsg_payload_length(struct io_uring_recvmsg_out *o,
 				int buf_len, struct msghdr *msgh)
 {
@@ -796,7 +796,7 @@ io_uring_recvmsg_payload_length(struct io_uring_recvmsg_out *o,
 
 	payload_start = (unsigned long) io_uring_recvmsg_payload(o, msgh);
 	payload_end = (unsigned long) o + buf_len;
-	return payload_end - payload_start;
+	return (unsigned int) (payload_end - payload_start);
 }
 
 static inline void io_uring_prep_openat2(struct io_uring_sqe *sqe, int dfd,
-- 
2.30.2

[PATCH liburing 4/5] add documentation for multishot recvmsg

[Dylan Yudaken]

Add missing documentation for new multishot recvmsg API

Signed-off-by: Dylan Yudaken <dylany@fb.com>
---
 man/io_uring_prep_recvmsg.3           | 30 +++++++++++
 man/io_uring_prep_recvmsg_multishot.3 |  1 +
 man/io_uring_recvmsg_cmsg_firsthdr.3  |  1 +
 man/io_uring_recvmsg_cmsg_nexthdr.3   |  1 +
 man/io_uring_recvmsg_name.3           |  1 +
 man/io_uring_recvmsg_out.3            | 78 +++++++++++++++++++++++++++
 man/io_uring_recvmsg_payload.3        |  1 +
 man/io_uring_recvmsg_payload_length.3 |  1 +
 man/io_uring_recvmsg_validate.3       |  1 +
 9 files changed, 115 insertions(+)
 create mode 120000 man/io_uring_prep_recvmsg_multishot.3
 create mode 120000 man/io_uring_recvmsg_cmsg_firsthdr.3
 create mode 120000 man/io_uring_recvmsg_cmsg_nexthdr.3
 create mode 120000 man/io_uring_recvmsg_name.3
 create mode 100644 man/io_uring_recvmsg_out.3
 create mode 120000 man/io_uring_recvmsg_payload.3
 create mode 120000 man/io_uring_recvmsg_payload_length.3
 create mode 120000 man/io_uring_recvmsg_validate.3

diff --git a/man/io_uring_prep_recvmsg.3 b/man/io_uring_prep_recvmsg.3
index f64326be0a99..07096ee4826c 100644
--- a/man/io_uring_prep_recvmsg.3
+++ b/man/io_uring_prep_recvmsg.3
@@ -15,6 +15,11 @@ io_uring_prep_recvmsg \- prepare a recvmsg request
 .BI "                           int " fd ","
 .BI "                           struct msghdr *" msg ","
 .BI "                           unsigned " flags ");"
+.PP
+.BI "void io_uring_prep_recvmsg_multishot(struct io_uring_sqe *" sqe ","
+.BI "                                     int " fd ","
+.BI "                                     struct msghdr *" msg ","
+.BI "                                     unsigned " flags ");"
 .fi
 .SH DESCRIPTION
 .PP
@@ -37,6 +42,31 @@ This function prepares an async
 request. See that man page for details on the arguments specified to this
 prep helper.
 
+The multishot version allows the application to issue a single receive request,
+which repeatedly posts a CQE when data is available. It requires the
+.B IOSQE_BUFFER_SELECT
+flag to be set and no
+.B MSG_WAITALL
+flag to be set.
+Therefore each CQE will take a buffer out of a provided buffer pool for receiving.
+The application should check the flags of each CQE, regardless of it's result.
+If a posted CQE does not have the
+.B IORING_CQE_F_MORE
+flag set then the multishot receive will be done and the application should issue a
+new request.
+
+Unlike
+.BR recvmsg (2)
+, multishot recvmsg will prepend a
+.I struct io_uring_recvmsg_out
+which describes the layout of the rest of the buffer in combination with the initial
+.I struct msghdr
+submitted with the request. See
+.B io_uring_recvmsg_out (3)
+for more information on accessing the data.
+
+Multishot variants are available since kernel 5.20.
+
 After calling this function, additional io_uring internal modifier flags
 may be set in the SQE
 .I ioprio
diff --git a/man/io_uring_prep_recvmsg_multishot.3 b/man/io_uring_prep_recvmsg_multishot.3
new file mode 120000
index 000000000000..cd9566f2c2be
--- /dev/null
+++ b/man/io_uring_prep_recvmsg_multishot.3
@@ -0,0 +1 @@
+io_uring_prep_recvmsg.3
\ No newline at end of file
diff --git a/man/io_uring_recvmsg_cmsg_firsthdr.3 b/man/io_uring_recvmsg_cmsg_firsthdr.3
new file mode 120000
index 000000000000..8eb17436288d
--- /dev/null
+++ b/man/io_uring_recvmsg_cmsg_firsthdr.3
@@ -0,0 +1 @@
+io_uring_recvmsg_out.3
\ No newline at end of file
diff --git a/man/io_uring_recvmsg_cmsg_nexthdr.3 b/man/io_uring_recvmsg_cmsg_nexthdr.3
new file mode 120000
index 000000000000..8eb17436288d
--- /dev/null
+++ b/man/io_uring_recvmsg_cmsg_nexthdr.3
@@ -0,0 +1 @@
+io_uring_recvmsg_out.3
\ No newline at end of file
diff --git a/man/io_uring_recvmsg_name.3 b/man/io_uring_recvmsg_name.3
new file mode 120000
index 000000000000..8eb17436288d
--- /dev/null
+++ b/man/io_uring_recvmsg_name.3
@@ -0,0 +1 @@
+io_uring_recvmsg_out.3
\ No newline at end of file
diff --git a/man/io_uring_recvmsg_out.3 b/man/io_uring_recvmsg_out.3
new file mode 100644
index 000000000000..60f92619d566
--- /dev/null
+++ b/man/io_uring_recvmsg_out.3
@@ -0,0 +1,78 @@
+.\" Copyright (C), 2022  Dylan Yudaken <dylany@fb.com>
+.\"
+.\" SPDX-License-Identifier: LGPL-2.0-or-later
+.\"
+.TH io_uring_recvmsg_out 3 "Julyu 26, 2022" "liburing-2.2" "liburing Manual"
+.SH NAME
+io_uring_recvmsg_out - access data from multishot recvmsg
+.SH SYNOPSIS
+.nf
+.B #include <liburing.h>
+.PP
+.BI "struct io_uring_recvmsg_out *io_uring_recvmsg_validate(void *" buf ","
+.BI "                                                       int " buf_len ","
+.BI "                                                       struct msghdr *" msgh ");"
+.PP
+.BI "void *io_uring_recvmsg_name(struct io_uring_recvmsg_out *" o ");"
+.PP
+.BI "struct cmsghdr *io_uring_recvmsg_cmsg_firsthdr(struct io_uring_recvmsg_out * " o ","
+.BI "                                                struct msghdr *" msgh ");"
+.BI "struct cmsghdr *io_uring_recvmsg_cmsg_nexthdr(struct io_uring_recvmsg_out * " o ","
+.BI "                                              struct msghdr *" msgh ","
+.BI "                                              struct cmsghdr *" cmsg ");"
+.PP
+.BI "void *io_uring_recvmsg_payload(struct io_uring_recvmsg_out * " o ","
+.BI "                               struct msghdr *" msgh ");"
+.BI "unsigned int io_uring_recvmsg_payload_length(struct io_uring_recvmsg_out *" o ","
+.BI "                                             int " buf_len ","
+.BI "                                             struct msghdr *" msgh ");"
+.PP
+.fi
+
+.SH DESCRIPTION
+
+These functions are used to access data in the payload delivered by
+.BR io_uring_prep_recv_multishot (3)
+.
+.PP
+.BR io_uring_recvmsg_validate (3)
+will validate a buffer delivered by
+.BR io_uring_prep_recv_multishot (3)
+and extract the
+.I io_uring_recvmsg_out
+if it is valid, returning a pointer to it or else NULL.
+.PP
+The structure is defined as follows:
+.PP
+.in +4n
+.EX
+
+struct io_uring_recvmsg_out {
+        __u32 namelen;    /* Name byte count as would have been populated
+                           * by recvmsg(2) */
+        __u32 controllen; /* Control byte count */
+        __u32 payloadlen; /* Payload byte count as would have been returned
+                           * by recvmsg(2) */
+        __u32 flags;      /* Flags result as would have been populated
+                           * by recvmsg(2) */
+};
+
+.IP * 3
+.BR io_uring_recvmsg_name (3)
+returns a pointer to the name in the buffer.
+.IP *
+.BR io_uring_recvmsg_cmsg_firsthdr (3)
+returns a pointer to the first cmsg in the buffer, or NULL.
+.IP *
+.BR io_uring_recvmsg_cmsg_nexthdr (3)
+returns a pointer to the next cmsg in the buffer, or NULL.
+.IP *
+.BR io_uring_recvmsg_payload (3)
+returns a pointer to the payload in the buffer.
+.IP *
+.BR io_uring_recvmsg_payload_length (3)
+Calculates the usable payload length in bytes.
+
+
+.SH "SEE ALSO"
+.BR io_uring_prep_recv_multishot (3)
diff --git a/man/io_uring_recvmsg_payload.3 b/man/io_uring_recvmsg_payload.3
new file mode 120000
index 000000000000..8eb17436288d
--- /dev/null
+++ b/man/io_uring_recvmsg_payload.3
@@ -0,0 +1 @@
+io_uring_recvmsg_out.3
\ No newline at end of file
diff --git a/man/io_uring_recvmsg_payload_length.3 b/man/io_uring_recvmsg_payload_length.3
new file mode 120000
index 000000000000..8eb17436288d
--- /dev/null
+++ b/man/io_uring_recvmsg_payload_length.3
@@ -0,0 +1 @@
+io_uring_recvmsg_out.3
\ No newline at end of file
diff --git a/man/io_uring_recvmsg_validate.3 b/man/io_uring_recvmsg_validate.3
new file mode 120000
index 000000000000..8eb17436288d
--- /dev/null
+++ b/man/io_uring_recvmsg_validate.3
@@ -0,0 +1 @@
+io_uring_recvmsg_out.3
\ No newline at end of file
-- 
2.30.2

[PATCH liburing 5/5] add an example for a UDP server

[Dylan Yudaken]

Add an example showing off multishot recvmsg and provided buffer rings.
The application is a UDP echo server that responds to all packets with the
same packet.

Signed-off-by: Dylan Yudaken <dylany@fb.com>
---
 .gitignore              |   1 +
 examples/Makefile       |   1 +
 examples/io_uring-udp.c | 388 ++++++++++++++++++++++++++++++++++++++++
 3 files changed, 390 insertions(+)
 create mode 100644 examples/io_uring-udp.c

diff --git a/.gitignore b/.gitignore
index b5acffd68c95..7853dbdf1a97 100644
--- a/.gitignore
+++ b/.gitignore
@@ -13,6 +13,7 @@
 
 /examples/io_uring-cp
 /examples/io_uring-test
+/examples/io_uring-udp
 /examples/link-cp
 /examples/ucontext-cp
 
diff --git a/examples/Makefile b/examples/Makefile
index 1997a31dcc08..e561e05c1785 100644
--- a/examples/Makefile
+++ b/examples/Makefile
@@ -13,6 +13,7 @@ endif
 example_srcs := \
 	io_uring-cp.c \
 	io_uring-test.c \
+	io_uring-udp.c \
 	link-cp.c \
 	poll-bench.c \
 	send-zerocopy.c
diff --git a/examples/io_uring-udp.c b/examples/io_uring-udp.c
new file mode 100644
index 000000000000..77472df3d224
--- /dev/null
+++ b/examples/io_uring-udp.c
@@ -0,0 +1,388 @@
+/* SPDX-License-Identifier: MIT */
+
+#include <stdio.h>
+#include <unistd.h>
+#include <errno.h>
+#include <sys/mman.h>
+#include <stdlib.h>
+#include <string.h>
+#include <netinet/udp.h>
+#include <arpa/inet.h>
+
+#include "liburing.h"
+
+#define QD 64
+#define BUF_SHIFT 12 /* 4k */
+#define CQES (QD * 16)
+#define BUFFERS CQES
+#define CONTROLLEN 0
+
+struct sendmsg_ctx {
+	struct msghdr msg;
+	struct iovec iov;
+};
+
+struct ctx {
+	struct io_uring ring;
+	struct io_uring_buf_ring *buf_ring;
+	unsigned char *buffer_base;
+	struct msghdr msg;
+	int buf_shift;
+	int af;
+	bool verbose;
+	struct sendmsg_ctx send[BUFFERS];
+	size_t buf_ring_size;
+};
+
+static size_t buffer_size(struct ctx *ctx)
+{
+	return 1U << ctx->buf_shift;
+}
+
+static unsigned char *get_buffer(struct ctx *ctx, int idx)
+{
+	return ctx->buffer_base + (idx << ctx->buf_shift);
+}
+
+static int setup_buffer_pool(struct ctx *ctx)
+{
+	int ret, i;
+	void *mapped;
+	struct io_uring_buf_reg reg = { .ring_addr = 0,
+					.ring_entries = BUFFERS,
+					.bgid = 0 };
+
+	ctx->buf_ring_size = (sizeof(struct io_uring_buf) + buffer_size(ctx)) * BUFFERS;
+	mapped = mmap(NULL, ctx->buf_ring_size, PROT_READ | PROT_WRITE,
+		      MAP_ANONYMOUS | MAP_PRIVATE, 0, 0);
+	if (mapped == MAP_FAILED) {
+		fprintf(stderr, "buf_ring mmap: %s\n", strerror(errno));
+		return -1;
+	}
+	ctx->buf_ring = (struct io_uring_buf_ring *)mapped;
+
+	io_uring_buf_ring_init(ctx->buf_ring);
+
+	reg = (struct io_uring_buf_reg) {
+		.ring_addr = (unsigned long)ctx->buf_ring,
+		.ring_entries = BUFFERS,
+		.bgid = 0
+	};
+	ctx->buffer_base = (unsigned char *)ctx->buf_ring +
+			   sizeof(struct io_uring_buf) * BUFFERS;
+
+	ret = io_uring_register_buf_ring(&ctx->ring, &reg, 0);
+	if (ret) {
+		fprintf(stderr, "buf_ring init: %s\n", strerror(-ret));
+		return ret;
+	}
+
+	for (i = 0; i < BUFFERS; i++) {
+		io_uring_buf_ring_add(ctx->buf_ring, get_buffer(ctx, i), buffer_size(ctx), i,
+				      io_uring_buf_ring_mask(BUFFERS), i);
+	}
+	io_uring_buf_ring_advance(ctx->buf_ring, BUFFERS);
+
+	return 0;
+}
+
+static int setup_context(struct ctx *ctx)
+{
+	struct io_uring_params params;
+	int ret;
+
+	memset(&params, 0, sizeof(params));
+	params.cq_entries = QD * 8;
+	params.flags = IORING_SETUP_SUBMIT_ALL | IORING_SETUP_COOP_TASKRUN |
+		       IORING_SETUP_CQSIZE;
+
+	ret = io_uring_queue_init_params(QD, &ctx->ring, &params);
+	if (ret < 0) {
+		fprintf(stderr, "queue_init: %s\n", strerror(-ret));
+		return ret;
+	}
+
+	ret = setup_buffer_pool(ctx);
+	if (ret)
+		io_uring_queue_exit(&ctx->ring);
+
+	memset(&ctx->msg, 0, sizeof(ctx->msg));
+	ctx->msg.msg_namelen = sizeof(struct sockaddr_storage);
+	ctx->msg.msg_controllen = CONTROLLEN;
+	return ret;
+}
+
+static int setup_sock(int af, int port)
+{
+	int ret;
+	int fd;
+	uint16_t nport = port <= 0 ? 0 : htons(port);
+
+	fd = socket(af, SOCK_DGRAM, 0);
+	if (fd < 0) {
+		fprintf(stderr, "sock_init: %s\n", strerror(errno));
+		return -1;
+	}
+
+	if (af == AF_INET6) {
+		struct sockaddr_in6 addr6 = {
+			.sin6_family = af,
+			.sin6_port = nport,
+			.sin6_addr = IN6ADDR_ANY_INIT
+		};
+
+		ret = bind(fd, &addr6, sizeof(addr6));
+	} else {
+		struct sockaddr_in addr = {
+			.sin_family = af,
+			.sin_port = nport,
+			.sin_addr = { INADDR_ANY }
+		};
+
+		ret = bind(fd, &addr, sizeof(addr));
+	}
+
+	if (ret) {
+		fprintf(stderr, "sock_bind: %s\n", strerror(errno));
+		close(fd);
+		return -1;
+	}
+
+	if (port <= 0) {
+		int port;
+		struct sockaddr_storage s;
+		socklen_t sz = sizeof(s);
+
+		if (getsockname(fd, (struct sockaddr *)&s, &sz)) {
+			fprintf(stderr, "getsockname failed\n");
+			close(fd);
+			return -1;
+		}
+
+		port = ntohs(((struct sockaddr_in *)&s)->sin_port);
+		fprintf(stderr, "port bound to %d\n", port);
+	}
+
+	return fd;
+}
+
+static void cleanup_context(struct ctx *ctx)
+{
+	munmap(ctx->buf_ring, ctx->buf_ring_size);
+	io_uring_queue_exit(&ctx->ring);
+}
+
+static bool get_sqe(struct ctx *ctx, struct io_uring_sqe **sqe)
+{
+	*sqe = io_uring_get_sqe(&ctx->ring);
+
+	if (!*sqe) {
+		io_uring_submit(&ctx->ring);
+		*sqe = io_uring_get_sqe(&ctx->ring);
+	}
+	if (!*sqe) {
+		fprintf(stderr, "cannot get sqe\n");
+		return true;
+	}
+	return false;
+}
+
+static int add_recv(struct ctx *ctx, int idx)
+{
+	struct io_uring_sqe *sqe;
+
+	if (get_sqe(ctx, &sqe))
+		return -1;
+
+	io_uring_prep_recvmsg_multishot(sqe, idx, &ctx->msg, MSG_TRUNC);
+	sqe->flags |= IOSQE_FIXED_FILE;
+
+	sqe->flags |= IOSQE_BUFFER_SELECT;
+	sqe->buf_group = 0;
+	io_uring_sqe_set_data64(sqe, BUFFERS + 1);
+	return 0;
+}
+
+static void recycle_buffer(struct ctx *ctx, int idx)
+{
+	io_uring_buf_ring_add(ctx->buf_ring, get_buffer(ctx, idx), buffer_size(ctx), idx,
+			      io_uring_buf_ring_mask(BUFFERS), 0);
+	io_uring_buf_ring_advance(ctx->buf_ring, 1);
+}
+
+static int process_cqe_send(struct ctx *ctx, struct io_uring_cqe *cqe)
+{
+	int idx = cqe->user_data;
+
+	if (cqe->res < 0)
+		fprintf(stderr, "bad send %s\n", strerror(-cqe->res));
+	recycle_buffer(ctx, idx);
+	return 0;
+}
+
+static int process_cqe_recv(struct ctx *ctx, struct io_uring_cqe *cqe,
+			    int fdidx)
+{
+	int ret, idx;
+	struct io_uring_recvmsg_out *o;
+	struct io_uring_sqe *sqe;
+
+	if (!(cqe->flags & IORING_CQE_F_MORE)) {
+		ret = add_recv(ctx, fdidx);
+		if (ret)
+			return ret;
+	}
+
+	if (cqe->res == -ENOBUFS)
+		return 0;
+
+	if (!(cqe->flags & IORING_CQE_F_BUFFER) || cqe->res < 0) {
+		fprintf(stderr, "bad res %d\n", cqe->res);
+		return -1;
+	}
+	idx = cqe->flags >> 16;
+
+	o = io_uring_recvmsg_validate(get_buffer(ctx, cqe->flags >> 16),
+				      cqe->res, &ctx->msg);
+	if (!o) {
+		fprintf(stderr, "bad recvmsg\n");
+		return -1;
+	}
+	if (o->namelen > ctx->msg.msg_namelen) {
+		fprintf(stderr, "truncated name\n");
+		recycle_buffer(ctx, idx);
+		return 0;
+	}
+	if (o->flags & MSG_TRUNC) {
+		unsigned int r;
+
+		r = io_uring_recvmsg_payload_length(o, cqe->res, &ctx->msg);
+		fprintf(stderr, "truncated msg need %u received %u\n",
+				o->payloadlen, r);
+		recycle_buffer(ctx, idx);
+		return 0;
+	}
+
+	if (ctx->verbose) {
+		char buff[INET6_ADDRSTRLEN + 1];
+		const char *name;
+		struct sockaddr_in *addr = io_uring_recvmsg_name(o);
+
+		name = inet_ntop(ctx->af, addr, buff, sizeof(buff));
+		if (!name)
+			name = "<INVALID>";
+		fprintf(stderr, "received %u bytes %d from %s:%d\n",
+			io_uring_recvmsg_payload_length(o, cqe->res, &ctx->msg),
+			o->namelen, name, (int)ntohs(addr->sin_port));
+	}
+
+	if (get_sqe(ctx, &sqe))
+		return -1;
+
+	ctx->send[idx].iov = (struct iovec) {
+		.iov_base = io_uring_recvmsg_payload(o, &ctx->msg),
+		.iov_len =
+			io_uring_recvmsg_payload_length(o, cqe->res, &ctx->msg)
+	};
+	ctx->send[idx].msg = (struct msghdr) {
+		.msg_namelen = o->namelen,
+		.msg_name = io_uring_recvmsg_name(o),
+		.msg_control = NULL,
+		.msg_controllen = 0,
+		.msg_iov = &ctx->send[idx].iov,
+		.msg_iovlen = 1
+	};
+
+	io_uring_prep_sendmsg(sqe, fdidx, &ctx->send[idx].msg, 0);
+	io_uring_sqe_set_data64(sqe, idx);
+	sqe->flags |= IOSQE_FIXED_FILE;
+
+	return 0;
+}
+static int process_cqe(struct ctx *ctx, struct io_uring_cqe *cqe, int fdidx)
+{
+	if (cqe->user_data < BUFFERS)
+		return process_cqe_send(ctx, cqe);
+	else
+		return process_cqe_recv(ctx, cqe, fdidx);
+}
+
+int main(int argc, char *argv[])
+{
+	struct ctx ctx;
+	int ret;
+	int port = -1;
+	int sockfd;
+	int opt;
+	struct io_uring_cqe *cqes[CQES];
+	unsigned int count, i;
+
+	memset(&ctx, 0, sizeof(ctx));
+	ctx.verbose = false;
+	ctx.af = AF_INET;
+	ctx.buf_shift = BUF_SHIFT;
+
+	while ((opt = getopt(argc, argv, "6vp:b:")) != -1) {
+		switch (opt) {
+		case '6':
+			ctx.af = AF_INET6;
+			break;
+		case 'p':
+			port = atoi(optarg);
+			break;
+		case 'b':
+			ctx.buf_shift = atoi(optarg);
+			break;
+		case 'v':
+			ctx.verbose = true;
+			break;
+		default:
+			fprintf(stderr, "Usage: %s [-p port] "
+					"[-b log2(BufferSize)] [-6] [-v]\n",
+					argv[0]);
+			exit(-1);
+		}
+	}
+
+	sockfd = setup_sock(ctx.af, port);
+	if (sockfd < 0)
+		return 1;
+
+	if (setup_context(&ctx)) {
+		close(sockfd);
+		return 1;
+	}
+
+	ret = io_uring_register_files(&ctx.ring, &sockfd, 1);
+	if (ret) {
+		fprintf(stderr, "register files: %s\n", strerror(-ret));
+		return -1;
+	}
+
+	ret = add_recv(&ctx, 0);
+	if (ret)
+		return 1;
+
+	while (true) {
+		ret = io_uring_submit_and_wait(&ctx.ring, 1);
+		if (ret == -EINTR)
+			continue;
+		if (ret < 0) {
+			fprintf(stderr, "submit and wait failed %d\n", ret);
+			break;
+		}
+
+		count = io_uring_peek_batch_cqe(&ctx.ring, &cqes[0], CQES);
+		for (i = 0; i < count; i++) {
+			ret = process_cqe(&ctx, cqes[i], 0);
+			if (ret)
+				goto cleanup;
+		}
+		io_uring_cq_advance(&ctx.ring, count);
+	}
+
+cleanup:
+	cleanup_context(&ctx);
+	close(sockfd);
+	return ret;
+}
-- 
2.30.2

Re: [PATCH liburing 0/5] multishot recvmsg docs and example

[Jens Axboe]

On Tue, 26 Jul 2022 05:14:57 -0700, Dylan Yudaken wrote:
> Some multishot recvmsg patches for liburing:
> 
> Patches 1-3  cleanup the API a little while we're doing this.
> Patch 4 adds docs for the new API
> Patch 5 adds an example UDP echo server that uses the API
> 
> 
> [...]

Applied, thanks!

[1/5] more consistent multishot recvmsg API parameter names
      commit: c025f206a8d7337109f148a738bf5df75aeed494
[2/5] order like functions together in liburing.h
      commit: b569b365c3c7d4ac78e8c2f482d5a227a980977f
[3/5] change io_uring_recvmsg_payload_length return type
      commit: 464ed8e15b3dbec2b44e087c4620dcffdc28a753
[4/5] add documentation for multishot recvmsg
      commit: 06f979baaae2c5a17f6b4233cb1f1d6720378149
[5/5] add an example for a UDP server
      commit: 61d472b51e761e61cbf46caea40aaf40d8ed1484

Best regards,
-- 
Jens Axboe

Re: [PATCH liburing 0/5] multishot recvmsg docs and example

[Ammar Faizi]

On 7/26/22 11:23 PM, Jens Axboe wrote:
> [5/5] add an example for a UDP server
>        commit: 61d472b51e761e61cbf46caea40aaf40d8ed1484

This one breaks clang-13 build, I'll send a patch.

-- 
Ammar Faizi

Re: [PATCH liburing 0/5] multishot recvmsg docs and example

[Jens Axboe]

On 7/26/22 10:32 AM, Ammar Faizi wrote:
> 
> 
> On 7/26/22 11:23 PM, Jens Axboe wrote:
>> [5/5] add an example for a UDP server
>>        commit: 61d472b51e761e61cbf46caea40aaf40d8ed1484
> 
> This one breaks clang-13 build, I'll send a patch.

Hmm, built fine with clang-13/14 here?

-- 
Jens Axboe

Re: [PATCH liburing 0/5] multishot recvmsg docs and example

[Ammar Faizi]

On 7/26/22 11:40 PM, Jens Axboe wrote:
> On 7/26/22 10:32 AM, Ammar Faizi wrote:
>> On 7/26/22 11:23 PM, Jens Axboe wrote:
>>> [5/5] add an example for a UDP server
>>>         commit: 61d472b51e761e61cbf46caea40aaf40d8ed1484
>>
>> This one breaks clang-13 build, I'll send a patch.
> 
> Hmm, built fine with clang-13/14 here?

Not sure what is going on, but clang-13 on my machine is not happy:

     io_uring-udp.c:134:18: error: incompatible pointer types passing \
     'struct sockaddr_in6 *' to parameter of type 'const struct sockaddr *' \
     [-Werror,-Wincompatible-pointer-types

     io_uring-udp.c:142:18: error: incompatible pointer types passing \
     'struct sockaddr_in *' to parameter of type 'const struct sockaddr *' \
     [-Werror,-Wincompatible-pointer-types]

Changing the compiler to GCC builds just fine. I have fixed something like
this more than once. Strange indeed.

-- 
Ammar Faizi

Re: [PATCH liburing 0/5] multishot recvmsg docs and example

[Jens Axboe]

On 7/26/22 10:48 AM, Ammar Faizi wrote:
> On 7/26/22 11:40 PM, Jens Axboe wrote:
>> On 7/26/22 10:32 AM, Ammar Faizi wrote:
>>> On 7/26/22 11:23 PM, Jens Axboe wrote:
>>>> [5/5] add an example for a UDP server
>>>>         commit: 61d472b51e761e61cbf46caea40aaf40d8ed1484
>>>
>>> This one breaks clang-13 build, I'll send a patch.
>>
>> Hmm, built fine with clang-13/14 here?
> 
> Not sure what is going on, but clang-13 on my machine is not happy:
> 
>     io_uring-udp.c:134:18: error: incompatible pointer types passing \
>     'struct sockaddr_in6 *' to parameter of type 'const struct sockaddr *' \
>     [-Werror,-Wincompatible-pointer-types
> 
>     io_uring-udp.c:142:18: error: incompatible pointer types passing \
>     'struct sockaddr_in *' to parameter of type 'const struct sockaddr *' \
>     [-Werror,-Wincompatible-pointer-types]
> 
> Changing the compiler to GCC builds just fine. I have fixed something like
> this more than once. Strange indeed.

Regardless, the patch is sane!

-- 
Jens Axboe

Re: [PATCH liburing 0/5] multishot recvmsg docs and example

[Dylan Yudaken]

On Tue, 2022-07-26 at 10:49 -0600, Jens Axboe wrote:
> 
> On 7/26/22 10:48 AM, Ammar Faizi wrote:
> > On 7/26/22 11:40 PM, Jens Axboe wrote:
> > > On 7/26/22 10:32 AM, Ammar Faizi wrote:
> > > > On 7/26/22 11:23 PM, Jens Axboe wrote:
> > > > > [5/5] add an example for a UDP server
> > > > >         commit: 61d472b51e761e61cbf46caea40aaf40d8ed1484
> > > > 
> > > > This one breaks clang-13 build, I'll send a patch.
> > > 
> > > Hmm, built fine with clang-13/14 here?
> > 
> > Not sure what is going on, but clang-13 on my machine is not happy:
> > 
> >     io_uring-udp.c:134:18: error: incompatible pointer types
> > passing \
> >     'struct sockaddr_in6 *' to parameter of type 'const struct
> > sockaddr *' \
> >     [-Werror,-Wincompatible-pointer-types
> > 
> >     io_uring-udp.c:142:18: error: incompatible pointer types
> > passing \
> >     'struct sockaddr_in *' to parameter of type 'const struct
> > sockaddr *' \
> >     [-Werror,-Wincompatible-pointer-types]
> > 
> > Changing the compiler to GCC builds just fine. I have fixed
> > something like
> > this more than once. Strange indeed.


Interestingly it did not show up on the Github CI either. What flags
are you setting for this? Maybe the CI can be expanded to include those
flags.
As you say its not the first time you've fixed this, or that I've done
this.

Dylan

Re: [PATCH liburing 0/5] multishot recvmsg docs and example

[Ammar Faizi]

On 7/27/22 2:57 PM, Dylan Yudaken wrote:
> Interestingly it did not show up on the Github CI either. What flags
> are you setting for this? Maybe the CI can be expanded to include those
> flags.
> As you say its not the first time you've fixed this, or that I've done
> this.

I use the same flag with the GitHub CI. Just a small experiment here...

I compile this with default compilation flags:

#include <arpa/inet.h>
#include <sys/socket.h>
#include <sys/types.h>

int main(void)
{
         struct sockaddr_in addr = { };
         return bind(0, &addr, sizeof(addr));
}

===============================================================================

ammarfaizi2@integral2:/tmp$ gcc test.c -o test
test.c: In function ‘main’:
test.c:9:24: warning: passing argument 2 of ‘bind’ from incompatible pointer type [-Wincompatible-pointer-types]
     9 |         return bind(0, &addr, sizeof(addr));
       |                        ^~~~~
       |                        |
       |                        struct sockaddr_in *
In file included from /usr/include/netinet/in.h:23,
                  from /usr/include/arpa/inet.h:22,
                  from test.c:2:
/usr/include/x86_64-linux-gnu/sys/socket.h:112:49: note: expected ‘const struct sockaddr *’ but argument is of type ‘struct sockaddr_in *’
   112 | extern int bind (int __fd, __CONST_SOCKADDR_ARG __addr, socklen_t __len)
       |

===============================================================================

ammarfaizi2@integral2:/tmp$ clang test.c -o test
test.c:9:17: warning: incompatible pointer types passing 'struct sockaddr_in *' to parameter of type 'const struct sockaddr *' [-Wincompatible-pointer-types]
         return bind(0, &addr, sizeof(addr));
                        ^~~~~
/usr/include/x86_64-linux-gnu/sys/socket.h:112:49: note: passing argument to parameter '__addr' here
extern int bind (int __fd, __CONST_SOCKADDR_ARG __addr, socklen_t __len)
                                                 ^
1 warning generated.

===============================================================================

Interestingly GCC also complains here, but it doesn't complain when
compiling your code. Your code only breaks my clang-13.

What is the magic behind this?

We never disable -Wincompatible-pointer-types in liburing either.
It's enabled by default.

-- 
Ammar Faizi

Re: [PATCH liburing 0/5] multishot recvmsg docs and example

[Ammar Faizi]

On 7/27/22 4:59 PM, Ammar Faizi wrote:
> Interestingly GCC also complains here, but it doesn't complain when
> compiling your code. Your code only breaks my clang-13.
> 
> What is the magic behind this?

OK, I figured it out.

This work:

    gcc -D_GNU_SOURCE test.c -o test;

These 3 break:

    clang -D_GNU_SOURCE test.c -o test;
    clang test.c -o test;
    gcc test.c -o test;


So -D_GNU_SOURCE is the culprit. It seems to be unavoidable as
the warn seems to be compiler specific or something. Maybe that
_GNU_SOURCE patches the definition of bind().

-- 
Ammar Faizi

Re: [PATCH liburing 0/5] multishot recvmsg docs and example

[Ammar Faizi]

On 7/27/22 5:11 PM, Ammar Faizi wrote:
> So -D_GNU_SOURCE is the culprit. It seems to be unavoidable as
> the warn seems to be compiler specific or something. Maybe that
> _GNU_SOURCE patches the definition of bind().

I did:

   gcc -E -D_GNU_SOURCE test.c -o xtest.c

and examined the xtest.c output.

So basically when we use _GNU_SOURCE, sometimes the declaration of
bind() is like this:

extern int bind (int __fd, __CONST_SOCKADDR_ARG __addr, socklen_t __len)
      __attribute__ ((__nothrow__ , __leaf__));

With __CONST_SOCKADDR_ARG being a union:

typedef union {
     const struct sockaddr *__restrict __sockaddr__;
     const struct sockaddr_at *__restrict __sockaddr_at__;
     const struct sockaddr_ax25 *__restrict __sockaddr_ax25__;
     const struct sockaddr_dl *__restrict __sockaddr_dl__;
     const struct sockaddr_eon *__restrict __sockaddr_eon__;
     const struct sockaddr_in *__restrict __sockaddr_in__;
     const struct sockaddr_in6 *__restrict __sockaddr_in6__;
     const struct sockaddr_inarp *__restrict __sockaddr_inarp__;
     const struct sockaddr_ipx *__restrict __sockaddr_ipx__;
     const struct sockaddr_iso *__restrict __sockaddr_iso__;
     const struct sockaddr_ns *__restrict __sockaddr_ns__;
     const struct sockaddr_un *__restrict __sockaddr_un__;
     const struct sockaddr_x25 *__restrict __sockaddr_x25__;
} __CONST_SOCKADDR_ARG __attribute__ ((__transparent_union__));

But not all header file included by the compiler has this union stuff.
When it doesn't, it will throw a warning like that.

-- 
Ammar Faizi