[PATCHSET 0.4] Reduce overhead of multishot recv/recvmsg

[PATCHSET 0.4] Reduce overhead of multishot recv/recvmsg

[Jens Axboe]

Hi,

With the way that the multishot variants of recv and recvmsg currently
work, we always end up doing two sock_recvmsg() call for each subsequent
receive outside of the initial receive. That is obviously extra overhead,
and to make matters worse, that also involves picking and recycling an
appropriate buffer for those operations.

For protocols that sanely pass back whether a socket has more data
after a receive, we can rely on that hint to avoid iterating over
recvmsg/buffers an extra time per receive.

Patches 1..3 are prep patches, patch 4 has the logic mentioned above.

-- 
Jens Axboe

[PATCH 1/4] io_uring/net: initialize struct msghdr more sanely for io_recv()

[Jens Axboe]

We only need to clear the input fields on the first invocation, not
when potentially doing a retry.

Signed-off-by: Jens Axboe <axboe@kernel.dk>
---
 io_uring/net.c | 15 ++++++++-------
 1 file changed, 8 insertions(+), 7 deletions(-)

diff --git a/io_uring/net.c b/io_uring/net.c
index 89e839013837..08fe42673b75 100644
--- a/io_uring/net.c
+++ b/io_uring/net.c
@@ -860,6 +860,14 @@ int io_recv(struct io_kiocb *req, unsigned int issue_flags)
 	if (unlikely(!sock))
 		return -ENOTSOCK;
 
+	msg.msg_name = NULL;
+	msg.msg_namelen = 0;
+	msg.msg_control = NULL;
+	msg.msg_get_inq = 1;
+	msg.msg_controllen = 0;
+	msg.msg_iocb = NULL;
+	msg.msg_ubuf = NULL;
+
 retry_multishot:
 	if (io_do_buffer_select(req)) {
 		void __user *buf;
@@ -874,14 +882,7 @@ int io_recv(struct io_kiocb *req, unsigned int issue_flags)
 	if (unlikely(ret))
 		goto out_free;
 
-	msg.msg_name = NULL;
-	msg.msg_namelen = 0;
-	msg.msg_control = NULL;
-	msg.msg_get_inq = 1;
 	msg.msg_flags = 0;
-	msg.msg_controllen = 0;
-	msg.msg_iocb = NULL;
-	msg.msg_ubuf = NULL;
 
 	flags = sr->msg_flags;
 	if (force_nonblock)
-- 
2.39.2

[PATCH 2/4] io_uring/net: initalize msghdr->msg_inq to known value

[Jens Axboe]

We can't currently tell if ->msg_inq was set when we ask for msg_get_inq,
initialize it to -1U so we can tell apart if it was set and there's
no data left, or if it just wasn't set at all by the protocol.

Signed-off-by: Jens Axboe <axboe@kernel.dk>
---
 io_uring/net.c | 6 ++++--
 1 file changed, 4 insertions(+), 2 deletions(-)

diff --git a/io_uring/net.c b/io_uring/net.c
index 08fe42673b75..45f9c3046d67 100644
--- a/io_uring/net.c
+++ b/io_uring/net.c
@@ -785,6 +785,7 @@ int io_recvmsg(struct io_kiocb *req, unsigned int issue_flags)
 		min_ret = iov_iter_count(&kmsg->msg.msg_iter);
 
 	kmsg->msg.msg_get_inq = 1;
+	kmsg->msg.msg_inq = -1U;
 	if (req->flags & REQ_F_APOLL_MULTISHOT)
 		ret = io_recvmsg_multishot(sock, sr, kmsg, flags,
 					   &mshot_finished);
@@ -821,7 +822,7 @@ int io_recvmsg(struct io_kiocb *req, unsigned int issue_flags)
 		io_kbuf_recycle(req, issue_flags);
 
 	cflags = io_put_kbuf(req, issue_flags);
-	if (kmsg->msg.msg_inq)
+	if (kmsg->msg.msg_inq && kmsg->msg.msg_inq != -1U)
 		cflags |= IORING_CQE_F_SOCK_NONEMPTY;
 
 	if (!io_recv_finish(req, &ret, cflags, mshot_finished, issue_flags))
@@ -882,6 +883,7 @@ int io_recv(struct io_kiocb *req, unsigned int issue_flags)
 	if (unlikely(ret))
 		goto out_free;
 
+	msg.msg_inq = -1U;
 	msg.msg_flags = 0;
 
 	flags = sr->msg_flags;
@@ -923,7 +925,7 @@ int io_recv(struct io_kiocb *req, unsigned int issue_flags)
 		io_kbuf_recycle(req, issue_flags);
 
 	cflags = io_put_kbuf(req, issue_flags);
-	if (msg.msg_inq)
+	if (msg.msg_inq && msg.msg_inq != -1U)
 		cflags |= IORING_CQE_F_SOCK_NONEMPTY;
 
 	if (!io_recv_finish(req, &ret, cflags, ret <= 0, issue_flags))
-- 
2.39.2

[PATCH 3/4] io_uring/net: push IORING_CQE_F_SOCK_NONEMPTY into io_recv_finish()

[Jens Axboe]

Rather than have this logic in both io_recv() and io_recvmsg_multishot(),
push it into the handler they both call when finishing a receive
operation.

Signed-off-by: Jens Axboe <axboe@kernel.dk>
---
 io_uring/net.c | 22 +++++++++-------------
 1 file changed, 9 insertions(+), 13 deletions(-)

diff --git a/io_uring/net.c b/io_uring/net.c
index 45f9c3046d67..9e0034771dbb 100644
--- a/io_uring/net.c
+++ b/io_uring/net.c
@@ -616,9 +616,15 @@ static inline void io_recv_prep_retry(struct io_kiocb *req)
  * again (for multishot).
  */
 static inline bool io_recv_finish(struct io_kiocb *req, int *ret,
-				  unsigned int cflags, bool mshot_finished,
+				  struct msghdr *msg, bool mshot_finished,
 				  unsigned issue_flags)
 {
+	unsigned int cflags;
+
+	cflags = io_put_kbuf(req, issue_flags);
+	if (msg->msg_inq && msg->msg_inq != -1U)
+		cflags |= IORING_CQE_F_SOCK_NONEMPTY;
+
 	if (!(req->flags & REQ_F_APOLL_MULTISHOT)) {
 		io_req_set_res(req, *ret, cflags);
 		*ret = IOU_OK;
@@ -732,7 +738,6 @@ int io_recvmsg(struct io_kiocb *req, unsigned int issue_flags)
 	struct io_sr_msg *sr = io_kiocb_to_cmd(req, struct io_sr_msg);
 	struct io_async_msghdr iomsg, *kmsg;
 	struct socket *sock;
-	unsigned int cflags;
 	unsigned flags;
 	int ret, min_ret = 0;
 	bool force_nonblock = issue_flags & IO_URING_F_NONBLOCK;
@@ -821,11 +826,7 @@ int io_recvmsg(struct io_kiocb *req, unsigned int issue_flags)
 	else
 		io_kbuf_recycle(req, issue_flags);
 
-	cflags = io_put_kbuf(req, issue_flags);
-	if (kmsg->msg.msg_inq && kmsg->msg.msg_inq != -1U)
-		cflags |= IORING_CQE_F_SOCK_NONEMPTY;
-
-	if (!io_recv_finish(req, &ret, cflags, mshot_finished, issue_flags))
+	if (!io_recv_finish(req, &ret, &kmsg->msg, mshot_finished, issue_flags))
 		goto retry_multishot;
 
 	if (mshot_finished) {
@@ -844,7 +845,6 @@ int io_recv(struct io_kiocb *req, unsigned int issue_flags)
 	struct io_sr_msg *sr = io_kiocb_to_cmd(req, struct io_sr_msg);
 	struct msghdr msg;
 	struct socket *sock;
-	unsigned int cflags;
 	unsigned flags;
 	int ret, min_ret = 0;
 	bool force_nonblock = issue_flags & IO_URING_F_NONBLOCK;
@@ -924,11 +924,7 @@ int io_recv(struct io_kiocb *req, unsigned int issue_flags)
 	else
 		io_kbuf_recycle(req, issue_flags);
 
-	cflags = io_put_kbuf(req, issue_flags);
-	if (msg.msg_inq && msg.msg_inq != -1U)
-		cflags |= IORING_CQE_F_SOCK_NONEMPTY;
-
-	if (!io_recv_finish(req, &ret, cflags, ret <= 0, issue_flags))
+	if (!io_recv_finish(req, &ret, &msg, ret <= 0, issue_flags))
 		goto retry_multishot;
 
 	return ret;
-- 
2.39.2

[PATCH 4/4] io_uring/net: don't retry recvmsg() unnecessarily

[Jens Axboe]

If we're doing multishot receives, then we always end up doing two trips
through sock_recvmsg(). For protocols that sanely set msghdr->msg_inq,
then we don't need to waste time picking a new buffer and attemtping a
new receive if there's nothing there.

Signed-off-by: Jens Axboe <axboe@kernel.dk>
---
 io_uring/net.c | 10 +++++++++-
 1 file changed, 9 insertions(+), 1 deletion(-)

diff --git a/io_uring/net.c b/io_uring/net.c
index 9e0034771dbb..0795f3783013 100644
--- a/io_uring/net.c
+++ b/io_uring/net.c
@@ -635,7 +635,15 @@ static inline bool io_recv_finish(struct io_kiocb *req, int *ret,
 		if (io_aux_cqe(req->ctx, issue_flags & IO_URING_F_COMPLETE_DEFER,
 			       req->cqe.user_data, *ret, cflags | IORING_CQE_F_MORE, true)) {
 			io_recv_prep_retry(req);
-			return false;
+			/* Known not-empty or unknown state, retry */
+			if (cflags & IORING_CQE_F_SOCK_NONEMPTY ||
+			    msg->msg_inq == -1U)
+				return false;
+			if (issue_flags & IO_URING_F_MULTISHOT)
+				*ret = IOU_ISSUE_SKIP_COMPLETE;
+			else
+				*ret = -EAGAIN;
+			return true;
 		}
 		/* Otherwise stop multishot but use the current result. */
 	}
-- 
2.39.2