From: Jens Axboe <[email protected]>
To: [email protected]
Cc: Jens Axboe <[email protected]>
Subject: [PATCH 08/15] io_uring/net: add iovec recycling
Date: Tue, 19 Mar 2024 19:17:36 -0600 [thread overview]
Message-ID: <[email protected]> (raw)
In-Reply-To: <[email protected]>
Right now the io_async_msghdr is recycled to avoid the overhead of
allocating+freeing it for every request. But the iovec is not included,
hence that will be allocated and freed for each transfer regardless.
This commit enables recyling of the iovec between io_async_msghdr
recycles. This avoids alloc+free for each one if we use it, and on top
of that, it extends the cache hot nature of msg to the iovec as well.
Enable KASAN for the iovec, if we're recycling it, to provide some extra
protection against it being reused while in the cache.
The io_async_msghdr also shrinks from 376 -> 272 bytes, a 104 byte
saving (or ~28% smaller), as the fast_iovec entry is dropped from 8
entries to a single entry. There's no point keeping a big fast iovec
entry, if we're keeping the iovec around.
Signed-off-by: Jens Axboe <[email protected]>
---
io_uring/net.c | 107 +++++++++++++++++++++++++++++++------------------
io_uring/net.h | 6 ++-
2 files changed, 74 insertions(+), 39 deletions(-)
diff --git a/io_uring/net.c b/io_uring/net.c
index 21624b7ead8a..bae8cab08e36 100644
--- a/io_uring/net.c
+++ b/io_uring/net.c
@@ -115,14 +115,30 @@ static bool io_net_retry(struct socket *sock, int flags)
return sock->type == SOCK_STREAM || sock->type == SOCK_SEQPACKET;
}
+static void io_netmsg_iovec_free(struct io_async_msghdr *kmsg)
+{
+ if (kmsg->free_iov) {
+ kfree(kmsg->free_iov);
+ kmsg->free_iov_nr = 0;
+ kmsg->free_iov = NULL;
+ }
+}
+
static void io_netmsg_recycle(struct io_kiocb *req, unsigned int issue_flags)
{
struct io_async_msghdr *hdr = req->async_data;
- if (!req_has_async_data(req) || issue_flags & IO_URING_F_UNLOCKED)
+ if (!req_has_async_data(req))
return;
+ /* can't recycle, ensure we free the iovec if we have one */
+ if (issue_flags & IO_URING_F_UNLOCKED) {
+ io_netmsg_iovec_free(hdr);
+ return;
+ }
/* Let normal cleanup path reap it if we fail adding to the cache */
+ if (hdr->free_iov)
+ kasan_mempool_poison_object(hdr->free_iov);
if (io_alloc_cache_put(&req->ctx->netmsg_cache, &hdr->cache)) {
req->async_data = NULL;
req->flags &= ~REQ_F_ASYNC_DATA;
@@ -138,14 +154,17 @@ static struct io_async_msghdr *io_msg_alloc_async(struct io_kiocb *req)
entry = io_alloc_cache_get(&ctx->netmsg_cache);
if (entry) {
hdr = container_of(entry, struct io_async_msghdr, cache);
- hdr->free_iov = NULL;
- req->flags |= REQ_F_ASYNC_DATA;
+ if (hdr->free_iov)
+ kasan_mempool_unpoison_object(hdr->free_iov,
+ hdr->free_iov_nr * sizeof(struct iovec));
+ req->flags |= REQ_F_ASYNC_DATA | REQ_F_NEED_CLEANUP;
req->async_data = hdr;
return hdr;
}
if (!io_alloc_async_data(req)) {
hdr = req->async_data;
+ hdr->free_iov_nr = 0;
hdr->free_iov = NULL;
return hdr;
}
@@ -159,7 +178,8 @@ static int io_compat_msg_copy_hdr(struct io_kiocb *req,
{
struct io_sr_msg *sr = io_kiocb_to_cmd(req, struct io_sr_msg);
struct compat_iovec __user *uiov;
- int ret;
+ struct iovec *iov;
+ int ret, nr_segs;
if (copy_from_user(msg, sr->umsg_compat, sizeof(*msg)))
return -EFAULT;
@@ -168,7 +188,6 @@ static int io_compat_msg_copy_hdr(struct io_kiocb *req,
if (req->flags & REQ_F_BUFFER_SELECT) {
compat_ssize_t clen;
- iomsg->free_iov = NULL;
if (msg->msg_iovlen == 0) {
sr->len = 0;
} else if (msg->msg_iovlen > 1) {
@@ -186,13 +205,22 @@ static int io_compat_msg_copy_hdr(struct io_kiocb *req,
return 0;
}
- iomsg->free_iov = iomsg->fast_iov;
+ if (iomsg->free_iov) {
+ nr_segs = iomsg->free_iov_nr;
+ iov = iomsg->free_iov;
+ } else {
+ iov = &iomsg->fast_iov;
+ nr_segs = 1;
+ }
ret = __import_iovec(ddir, (struct iovec __user *)uiov, msg->msg_iovlen,
- UIO_FASTIOV, &iomsg->free_iov,
- &iomsg->msg.msg_iter, true);
+ nr_segs, &iov, &iomsg->msg.msg_iter, true);
if (unlikely(ret < 0))
return ret;
-
+ if (iov) {
+ iomsg->free_iov_nr = iomsg->msg.msg_iter.nr_segs;
+ kfree(iomsg->free_iov);
+ iomsg->free_iov = iov;
+ }
return 0;
}
#endif
@@ -201,7 +229,8 @@ static int io_msg_copy_hdr(struct io_kiocb *req, struct io_async_msghdr *iomsg,
struct user_msghdr *msg, int ddir)
{
struct io_sr_msg *sr = io_kiocb_to_cmd(req, struct io_sr_msg);
- int ret;
+ struct iovec *iov;
+ int ret, nr_segs;
if (!user_access_begin(sr->umsg, sizeof(*sr->umsg)))
return -EFAULT;
@@ -217,9 +246,8 @@ static int io_msg_copy_hdr(struct io_kiocb *req, struct io_async_msghdr *iomsg,
if (req->flags & REQ_F_BUFFER_SELECT) {
if (msg->msg_iovlen == 0) {
- sr->len = iomsg->fast_iov[0].iov_len = 0;
- iomsg->fast_iov[0].iov_base = NULL;
- iomsg->free_iov = NULL;
+ sr->len = iomsg->fast_iov.iov_len = 0;
+ iomsg->fast_iov.iov_base = NULL;
} else if (msg->msg_iovlen > 1) {
ret = -EINVAL;
goto ua_end;
@@ -227,10 +255,9 @@ static int io_msg_copy_hdr(struct io_kiocb *req, struct io_async_msghdr *iomsg,
/* we only need the length for provided buffers */
if (!access_ok(&msg->msg_iov[0].iov_len, sizeof(__kernel_size_t)))
goto ua_end;
- unsafe_get_user(iomsg->fast_iov[0].iov_len,
+ unsafe_get_user(iomsg->fast_iov.iov_len,
&msg->msg_iov[0].iov_len, ua_end);
- sr->len = iomsg->fast_iov[0].iov_len;
- iomsg->free_iov = NULL;
+ sr->len = iomsg->fast_iov.iov_len;
}
ret = 0;
ua_end:
@@ -239,12 +266,22 @@ static int io_msg_copy_hdr(struct io_kiocb *req, struct io_async_msghdr *iomsg,
}
user_access_end();
- iomsg->free_iov = iomsg->fast_iov;
- ret = __import_iovec(ddir, msg->msg_iov, msg->msg_iovlen, UIO_FASTIOV,
- &iomsg->free_iov, &iomsg->msg.msg_iter, false);
+ if (iomsg->free_iov) {
+ nr_segs = iomsg->free_iov_nr;
+ iov = iomsg->free_iov;
+ } else {
+ iov = &iomsg->fast_iov;
+ nr_segs = 1;
+ }
+ ret = __import_iovec(ddir, msg->msg_iov, msg->msg_iovlen, nr_segs,
+ &iov, &iomsg->msg.msg_iter, false);
if (unlikely(ret < 0))
return ret;
+ if (nr_segs == 1 && iov) {
+ iomsg->free_iov_nr = iomsg->msg.msg_iter.nr_segs;
+ iomsg->free_iov = iov;
+ }
return 0;
}
@@ -285,7 +322,7 @@ void io_sendmsg_recvmsg_cleanup(struct io_kiocb *req)
{
struct io_async_msghdr *io = req->async_data;
- kfree(io->free_iov);
+ io_netmsg_iovec_free(io);
}
static int io_send_setup(struct io_kiocb *req)
@@ -374,9 +411,6 @@ static void io_req_msg_cleanup(struct io_kiocb *req,
unsigned int issue_flags)
{
req->flags &= ~REQ_F_NEED_CLEANUP;
- /* fast path, check for non-NULL to avoid function call */
- if (kmsg->free_iov)
- kfree(kmsg->free_iov);
io_netmsg_recycle(req, issue_flags);
}
@@ -625,11 +659,6 @@ static inline void io_recv_prep_retry(struct io_kiocb *req,
{
struct io_sr_msg *sr = io_kiocb_to_cmd(req, struct io_sr_msg);
- if (kmsg->free_iov) {
- kfree(kmsg->free_iov);
- kmsg->free_iov = NULL;
- }
-
req->flags &= ~REQ_F_BL_EMPTY;
sr->done_io = 0;
sr->len = 0; /* get from the provided buffer */
@@ -943,13 +972,13 @@ int io_recv(struct io_kiocb *req, unsigned int issue_flags)
void io_send_zc_cleanup(struct io_kiocb *req)
{
struct io_sr_msg *zc = io_kiocb_to_cmd(req, struct io_sr_msg);
- struct io_async_msghdr *io;
if (req_has_async_data(req)) {
- io = req->async_data;
+ struct io_async_msghdr *io = req->async_data;
+
/* might be ->fast_iov if *msg_copy_hdr failed */
- if (io->free_iov != io->fast_iov)
- kfree(io->free_iov);
+ if (io->free_iov != &io->fast_iov)
+ io_netmsg_iovec_free(io);
}
if (zc->notif) {
io_notif_flush(zc->notif);
@@ -1220,11 +1249,6 @@ int io_sendmsg_zc(struct io_kiocb *req, unsigned int issue_flags)
ret = -EINTR;
req_set_fail(req);
}
- /* fast path, check for non-NULL to avoid function call */
- if (kmsg->free_iov) {
- kfree(kmsg->free_iov);
- kmsg->free_iov = NULL;
- }
io_netmsg_recycle(req, issue_flags);
if (ret >= 0)
@@ -1482,6 +1506,13 @@ int io_connect(struct io_kiocb *req, unsigned int issue_flags)
void io_netmsg_cache_free(struct io_cache_entry *entry)
{
- kfree(container_of(entry, struct io_async_msghdr, cache));
+ struct io_async_msghdr *kmsg;
+
+ kmsg = container_of(entry, struct io_async_msghdr, cache);
+ if (kmsg->free_iov)
+ kasan_mempool_unpoison_object(kmsg->free_iov,
+ kmsg->free_iov_nr * sizeof(struct iovec));
+ io_netmsg_iovec_free(kmsg);
+ kfree(kmsg);
}
#endif
diff --git a/io_uring/net.h b/io_uring/net.h
index 2713dc90fdd4..783dd601a432 100644
--- a/io_uring/net.h
+++ b/io_uring/net.h
@@ -8,7 +8,11 @@
struct io_async_msghdr {
#if defined(CONFIG_NET)
union {
- struct iovec fast_iov[UIO_FASTIOV];
+ struct {
+ struct iovec fast_iov;
+ /* entry size of ->free_iov, if valid */
+ int free_iov_nr;
+ };
struct {
__kernel_size_t controllen;
int namelen;
--
2.43.0
next prev parent reply other threads:[~2024-03-20 1:23 UTC|newest]
Thread overview: 16+ messages / expand[flat|nested] mbox.gz Atom feed top
2024-03-20 1:17 [PATCHSET 0/15] Get rid of ->prep_async() Jens Axboe
2024-03-20 1:17 ` [PATCH 01/15] io_uring/net: switch io_send() and io_send_zc() to using io_async_msghdr Jens Axboe
2024-03-20 1:17 ` [PATCH 02/15] io_uring/net: switch io_recv() " Jens Axboe
2024-03-20 1:17 ` [PATCH 03/15] io_uring/net: unify cleanup handling Jens Axboe
2024-03-20 1:17 ` [PATCH 04/15] io_uring/net: always setup an io_async_msghdr Jens Axboe
2024-03-20 1:17 ` [PATCH 05/15] io_uring/net: get rid of ->prep_async() for receive side Jens Axboe
2024-03-20 1:17 ` [PATCH 06/15] io_uring/net: get rid of ->prep_async() for send side Jens Axboe
2024-03-20 1:17 ` [PATCH 07/15] io_uring: kill io_msg_alloc_async_prep() Jens Axboe
2024-03-20 1:17 ` Jens Axboe [this message]
2024-03-20 1:17 ` [PATCH 09/15] io_uring/net: drop 'kmsg' parameter from io_req_msg_cleanup() Jens Axboe
2024-03-20 1:17 ` [PATCH 10/15] io_uring/rw: always setup io_async_rw for read/write requests Jens Axboe
2024-03-20 1:17 ` [PATCH 11/15] io_uring: get rid of struct io_rw_state Jens Axboe
2024-03-20 1:17 ` [PATCH 12/15] io_uring/rw: add iovec recycling Jens Axboe
2024-03-20 1:17 ` [PATCH 13/15] io_uring/net: move connect to always using async data Jens Axboe
2024-03-20 1:17 ` [PATCH 14/15] io_uring/uring_cmd: switch to always allocating " Jens Axboe
2024-03-20 1:17 ` [PATCH 15/15] io_uring: drop ->prep_async() Jens Axboe
Reply instructions:
You may reply publicly to this message via plain-text email
using any one of the following methods:
* Save the following mbox file, import it into your mail client,
and reply-to-all from there: mbox
Avoid top-posting and favor interleaved quoting:
https://en.wikipedia.org/wiki/Posting_style#Interleaved_style
* Reply using the --to, --cc, and --in-reply-to
switches of git-send-email(1):
git send-email \
[email protected] \
[email protected] \
[email protected] \
/path/to/YOUR_REPLY
https://kernel.org/pub/software/scm/git/docs/git-send-email.html
* If your mail client supports setting the In-Reply-To header
via mailto: links, try the mailto: link
Be sure your reply has a Subject: header at the top and a blank line
before the message body.
This is a public inbox, see mirroring instructions
for how to clone and mirror all data and code used for this inbox