From: Jens Axboe <[email protected]>
To: [email protected]
Cc: [email protected], Jens Axboe <[email protected]>
Subject: [PATCH 3/5] io_uring: ensure async punted sendmsg/recvmsg requests copy data
Date: Mon, 2 Dec 2019 19:54:42 -0700 [thread overview]
Message-ID: <[email protected]> (raw)
In-Reply-To: <[email protected]>
Just like commit bd26dacbd5ce for read/write requests, this one ensures
that the msghdr data is fully copied if we need to punt a recvmsg or
sendmsg system call to async context.
Signed-off-by: Jens Axboe <[email protected]>
---
fs/io_uring.c | 145 +++++++++++++++++++++++++++++++++++------
include/linux/socket.h | 15 +++--
net/socket.c | 60 +++++------------
3 files changed, 152 insertions(+), 68 deletions(-)
diff --git a/fs/io_uring.c b/fs/io_uring.c
index bd8fab9277d6..11d181ed2076 100644
--- a/fs/io_uring.c
+++ b/fs/io_uring.c
@@ -308,6 +308,13 @@ struct io_timeout {
struct io_timeout_data *data;
};
+struct io_async_msghdr {
+ struct iovec fast_iov[UIO_FASTIOV];
+ struct iovec *iov;
+ struct sockaddr __user *uaddr;
+ struct msghdr msg;
+};
+
struct io_async_rw {
struct iovec fast_iov[UIO_FASTIOV];
struct iovec *iov;
@@ -319,6 +326,7 @@ struct io_async_ctx {
struct io_uring_sqe sqe;
union {
struct io_async_rw rw;
+ struct io_async_msghdr msg;
};
};
@@ -1991,12 +1999,21 @@ static int io_sync_file_range(struct io_kiocb *req,
return 0;
}
-#if defined(CONFIG_NET)
-static int io_send_recvmsg(struct io_kiocb *req, const struct io_uring_sqe *sqe,
- struct io_kiocb **nxt, bool force_nonblock,
- long (*fn)(struct socket *, struct user_msghdr __user *,
- unsigned int))
+static int io_sendmsg_prep(struct io_kiocb *req, struct io_async_ctx *io)
{
+ const struct io_uring_sqe *sqe = req->sqe;
+ struct user_msghdr __user *msg;
+ unsigned flags;
+
+ flags = READ_ONCE(sqe->msg_flags);
+ msg = (struct user_msghdr __user *)(unsigned long) READ_ONCE(sqe->addr);
+ return sendmsg_copy_msghdr(&io->msg.msg, msg, flags, &io->msg.iov);
+}
+
+static int io_sendmsg(struct io_kiocb *req, const struct io_uring_sqe *sqe,
+ struct io_kiocb **nxt, bool force_nonblock)
+{
+#if defined(CONFIG_NET)
struct socket *sock;
int ret;
@@ -2005,7 +2022,9 @@ static int io_send_recvmsg(struct io_kiocb *req, const struct io_uring_sqe *sqe,
sock = sock_from_file(req->file, &ret);
if (sock) {
- struct user_msghdr __user *msg;
+ struct io_async_ctx io, *copy;
+ struct sockaddr_storage addr;
+ struct msghdr *kmsg;
unsigned flags;
flags = READ_ONCE(sqe->msg_flags);
@@ -2014,41 +2033,119 @@ static int io_send_recvmsg(struct io_kiocb *req, const struct io_uring_sqe *sqe,
else if (force_nonblock)
flags |= MSG_DONTWAIT;
- msg = (struct user_msghdr __user *) (unsigned long)
- READ_ONCE(sqe->addr);
+ if (req->io) {
+ kmsg = &req->io->msg.msg;
+ kmsg->msg_name = &addr;
+ } else {
+ kmsg = &io.msg.msg;
+ kmsg->msg_name = &addr;
+ io.msg.iov = io.msg.fast_iov;
+ ret = io_sendmsg_prep(req, &io);
+ if (ret)
+ goto out;
+ }
- ret = fn(sock, msg, flags);
- if (force_nonblock && ret == -EAGAIN)
+ ret = __sys_sendmsg_sock(sock, kmsg, flags);
+ if (force_nonblock && ret == -EAGAIN) {
+ copy = kmalloc(sizeof(*copy), GFP_KERNEL);
+ if (!copy) {
+ ret = -ENOMEM;
+ goto out;
+ }
+ memcpy(©->msg, &io.msg, sizeof(copy->msg));
+ req->io = copy;
+ memcpy(&req->io->sqe, req->sqe, sizeof(*req->sqe));
+ req->sqe = &req->io->sqe;
return ret;
+ }
if (ret == -ERESTARTSYS)
ret = -EINTR;
}
+out:
io_cqring_add_event(req, ret);
if (ret < 0 && (req->flags & REQ_F_LINK))
req->flags |= REQ_F_FAIL_LINK;
io_put_req_find_next(req, nxt);
return 0;
-}
-#endif
-
-static int io_sendmsg(struct io_kiocb *req, const struct io_uring_sqe *sqe,
- struct io_kiocb **nxt, bool force_nonblock)
-{
-#if defined(CONFIG_NET)
- return io_send_recvmsg(req, sqe, nxt, force_nonblock,
- __sys_sendmsg_sock);
#else
return -EOPNOTSUPP;
#endif
}
+static int io_recvmsg_prep(struct io_kiocb *req, struct io_async_ctx *io)
+{
+ const struct io_uring_sqe *sqe = req->sqe;
+ struct user_msghdr __user *msg;
+ unsigned flags;
+
+ flags = READ_ONCE(sqe->msg_flags);
+ msg = (struct user_msghdr __user *)(unsigned long) READ_ONCE(sqe->addr);
+ return recvmsg_copy_msghdr(&io->msg.msg, msg, flags, &io->msg.uaddr,
+ &io->msg.iov);
+}
+
static int io_recvmsg(struct io_kiocb *req, const struct io_uring_sqe *sqe,
struct io_kiocb **nxt, bool force_nonblock)
{
#if defined(CONFIG_NET)
- return io_send_recvmsg(req, sqe, nxt, force_nonblock,
- __sys_recvmsg_sock);
+ struct socket *sock;
+ int ret;
+
+ if (unlikely(req->ctx->flags & IORING_SETUP_IOPOLL))
+ return -EINVAL;
+
+ sock = sock_from_file(req->file, &ret);
+ if (sock) {
+ struct user_msghdr __user *msg;
+ struct io_async_ctx io, *copy;
+ struct sockaddr_storage addr;
+ struct msghdr *kmsg;
+ unsigned flags;
+
+ flags = READ_ONCE(sqe->msg_flags);
+ if (flags & MSG_DONTWAIT)
+ req->flags |= REQ_F_NOWAIT;
+ else if (force_nonblock)
+ flags |= MSG_DONTWAIT;
+
+ msg = (struct user_msghdr __user *) (unsigned long)
+ READ_ONCE(sqe->addr);
+ if (req->io) {
+ kmsg = &req->io->msg.msg;
+ kmsg->msg_name = &addr;
+ } else {
+ kmsg = &io.msg.msg;
+ kmsg->msg_name = &addr;
+ io.msg.iov = io.msg.fast_iov;
+ ret = io_recvmsg_prep(req, &io);
+ if (ret)
+ goto out;
+ }
+
+ ret = __sys_recvmsg_sock(sock, kmsg, msg, io.msg.uaddr, flags);
+ if (force_nonblock && ret == -EAGAIN) {
+ copy = kmalloc(sizeof(*copy), GFP_KERNEL);
+ if (!copy) {
+ ret = -ENOMEM;
+ goto out;
+ }
+ memcpy(copy, &io, sizeof(*copy));
+ req->io = copy;
+ memcpy(&req->io->sqe, req->sqe, sizeof(*req->sqe));
+ req->sqe = &req->io->sqe;
+ return ret;
+ }
+ if (ret == -ERESTARTSYS)
+ ret = -EINTR;
+ }
+
+out:
+ io_cqring_add_event(req, ret);
+ if (ret < 0 && (req->flags & REQ_F_LINK))
+ req->flags |= REQ_F_FAIL_LINK;
+ io_put_req_find_next(req, nxt);
+ return 0;
#else
return -EOPNOTSUPP;
#endif
@@ -2719,6 +2816,12 @@ static int io_req_defer_prep(struct io_kiocb *req, struct io_async_ctx *io)
case IORING_OP_WRITEV:
ret = io_write_prep(req, &iovec, &iter, true);
break;
+ case IORING_OP_SENDMSG:
+ ret = io_sendmsg_prep(req, io);
+ break;
+ case IORING_OP_RECVMSG:
+ ret = io_recvmsg_prep(req, io);
+ break;
default:
req->io = io;
return 0;
diff --git a/include/linux/socket.h b/include/linux/socket.h
index 4bde63021c09..903507fb901f 100644
--- a/include/linux/socket.h
+++ b/include/linux/socket.h
@@ -378,12 +378,19 @@ extern int __sys_recvmmsg(int fd, struct mmsghdr __user *mmsg,
extern int __sys_sendmmsg(int fd, struct mmsghdr __user *mmsg,
unsigned int vlen, unsigned int flags,
bool forbid_cmsg_compat);
-extern long __sys_sendmsg_sock(struct socket *sock,
- struct user_msghdr __user *msg,
+extern long __sys_sendmsg_sock(struct socket *sock, struct msghdr *msg,
unsigned int flags);
-extern long __sys_recvmsg_sock(struct socket *sock,
- struct user_msghdr __user *msg,
+extern long __sys_recvmsg_sock(struct socket *sock, struct msghdr *msg,
+ struct user_msghdr __user *umsg,
+ struct sockaddr __user *uaddr,
unsigned int flags);
+extern int sendmsg_copy_msghdr(struct msghdr *msg,
+ struct user_msghdr __user *umsg, unsigned flags,
+ struct iovec **iov);
+extern int recvmsg_copy_msghdr(struct msghdr *msg,
+ struct user_msghdr __user *umsg, unsigned flags,
+ struct sockaddr __user **uaddr,
+ struct iovec **iov);
/* helpers which do the actual work for syscalls */
extern int __sys_recvfrom(int fd, void __user *ubuf, size_t size,
diff --git a/net/socket.c b/net/socket.c
index ea28cbb9e2e7..0fb0820edeec 100644
--- a/net/socket.c
+++ b/net/socket.c
@@ -2346,9 +2346,9 @@ static int ____sys_sendmsg(struct socket *sock, struct msghdr *msg_sys,
return err;
}
-static int sendmsg_copy_msghdr(struct msghdr *msg,
- struct user_msghdr __user *umsg, unsigned flags,
- struct iovec **iov)
+int sendmsg_copy_msghdr(struct msghdr *msg,
+ struct user_msghdr __user *umsg, unsigned flags,
+ struct iovec **iov)
{
int err;
@@ -2390,27 +2390,14 @@ static int ___sys_sendmsg(struct socket *sock, struct user_msghdr __user *msg,
/*
* BSD sendmsg interface
*/
-long __sys_sendmsg_sock(struct socket *sock, struct user_msghdr __user *umsg,
+long __sys_sendmsg_sock(struct socket *sock, struct msghdr *msg,
unsigned int flags)
{
- struct iovec iovstack[UIO_FASTIOV], *iov = iovstack;
- struct sockaddr_storage address;
- struct msghdr msg = { .msg_name = &address };
- ssize_t err;
-
- err = sendmsg_copy_msghdr(&msg, umsg, flags, &iov);
- if (err)
- return err;
/* disallow ancillary data requests from this path */
- if (msg.msg_control || msg.msg_controllen) {
- err = -EINVAL;
- goto out;
- }
+ if (msg->msg_control || msg->msg_controllen)
+ return -EINVAL;
- err = ____sys_sendmsg(sock, &msg, flags, NULL, 0);
-out:
- kfree(iov);
- return err;
+ return ____sys_sendmsg(sock, msg, flags, NULL, 0);
}
long __sys_sendmsg(int fd, struct user_msghdr __user *msg, unsigned int flags,
@@ -2516,10 +2503,10 @@ SYSCALL_DEFINE4(sendmmsg, int, fd, struct mmsghdr __user *, mmsg,
return __sys_sendmmsg(fd, mmsg, vlen, flags, true);
}
-static int recvmsg_copy_msghdr(struct msghdr *msg,
- struct user_msghdr __user *umsg, unsigned flags,
- struct sockaddr __user **uaddr,
- struct iovec **iov)
+int recvmsg_copy_msghdr(struct msghdr *msg,
+ struct user_msghdr __user *umsg, unsigned flags,
+ struct sockaddr __user **uaddr,
+ struct iovec **iov)
{
ssize_t err;
@@ -2609,28 +2596,15 @@ static int ___sys_recvmsg(struct socket *sock, struct user_msghdr __user *msg,
* BSD recvmsg interface
*/
-long __sys_recvmsg_sock(struct socket *sock, struct user_msghdr __user *umsg,
- unsigned int flags)
+long __sys_recvmsg_sock(struct socket *sock, struct msghdr *msg,
+ struct user_msghdr __user *umsg,
+ struct sockaddr __user *uaddr, unsigned int flags)
{
- struct iovec iovstack[UIO_FASTIOV], *iov = iovstack;
- struct sockaddr_storage address;
- struct msghdr msg = { .msg_name = &address };
- struct sockaddr __user *uaddr;
- ssize_t err;
-
- err = recvmsg_copy_msghdr(&msg, umsg, flags, &uaddr, &iov);
- if (err)
- return err;
/* disallow ancillary data requests from this path */
- if (msg.msg_control || msg.msg_controllen) {
- err = -EINVAL;
- goto out;
- }
+ if (msg->msg_control || msg->msg_controllen)
+ return -EINVAL;
- err = ____sys_recvmsg(sock, &msg, umsg, uaddr, flags, 0);
-out:
- kfree(iov);
- return err;
+ return ____sys_recvmsg(sock, msg, umsg, uaddr, flags, 0);
}
long __sys_recvmsg(int fd, struct user_msghdr __user *msg, unsigned int flags,
--
2.24.0
next prev parent reply other threads:[~2019-12-03 2:55 UTC|newest]
Thread overview: 6+ messages / expand[flat|nested] mbox.gz Atom feed top
2019-12-03 2:54 [PATCHSET v2 0/5] io_uring: ensure all needed read/write data is stable Jens Axboe
2019-12-03 2:54 ` [PATCH 1/5] io_uring: add general async offload context Jens Axboe
2019-12-03 2:54 ` [PATCH 2/5] io_uring: ensure async punted read/write requests copy iovec Jens Axboe
2019-12-03 2:54 ` Jens Axboe [this message]
2019-12-03 2:54 ` [PATCH 4/5] io_uring: ensure async punted connect requests copy data Jens Axboe
2019-12-03 2:54 ` [PATCH 5/5] io_uring: mark us with IORING_FEAT_SUBMIT_STABLE Jens Axboe
Reply instructions:
You may reply publicly to this message via plain-text email
using any one of the following methods:
* Save the following mbox file, import it into your mail client,
and reply-to-all from there: mbox
Avoid top-posting and favor interleaved quoting:
https://en.wikipedia.org/wiki/Posting_style#Interleaved_style
* Reply using the --to, --cc, and --in-reply-to
switches of git-send-email(1):
git send-email \
[email protected] \
[email protected] \
[email protected] \
[email protected] \
/path/to/YOUR_REPLY
https://kernel.org/pub/software/scm/git/docs/git-send-email.html
* If your mail client supports setting the In-Reply-To header
via mailto: links, try the mailto: link
Be sure your reply has a Subject: header at the top and a blank line
before the message body.
This is a public inbox, see mirroring instructions
for how to clone and mirror all data and code used for this inbox