From: Pavel Begunkov <[email protected]>
To: Jens Axboe <[email protected]>, [email protected]
Subject: [PATCH 21/28] io_uring: hide async dadta behind flags
Date: Mon, 9 Aug 2021 13:04:21 +0100 [thread overview]
Message-ID: <707ce8945e0247db7a585b5d1c9e8240a22e6708.1628471125.git.asml.silence@gmail.com> (raw)
In-Reply-To: <[email protected]>
Checking flags is a bit faster and can be batched, but the main reason
of controlling ->async_data with req->flags but not relying on NULL is
that we safely move it now to the end of io_kiocb, where cachelines are
rarely loaded, and use that freed space for something more hot like
io_mapped_ubuf.
Signed-off-by: Pavel Begunkov <[email protected]>
---
fs/io_uring.c | 70 ++++++++++++++++++++++++++++++++-------------------
1 file changed, 44 insertions(+), 26 deletions(-)
diff --git a/fs/io_uring.c b/fs/io_uring.c
index 0982b0dba6b0..9e359acf2f51 100644
--- a/fs/io_uring.c
+++ b/fs/io_uring.c
@@ -106,7 +106,8 @@
IOSQE_IO_HARDLINK | IOSQE_ASYNC | \
IOSQE_BUFFER_SELECT)
#define IO_REQ_CLEAN_FLAGS (REQ_F_BUFFER_SELECTED | REQ_F_NEED_CLEANUP | \
- REQ_F_POLLED | REQ_F_INFLIGHT | REQ_F_CREDS)
+ REQ_F_POLLED | REQ_F_INFLIGHT | REQ_F_CREDS | \
+ REQ_F_ASYNC_DATA)
#define IO_TCTX_REFS_CACHE_NR (1U << 10)
@@ -716,6 +717,7 @@ enum {
REQ_F_REISSUE_BIT,
REQ_F_DONT_REISSUE_BIT,
REQ_F_CREDS_BIT,
+ REQ_F_ASYNC_DATA_BIT,
/* keep async read/write and isreg together and in order */
REQ_F_NOWAIT_READ_BIT,
REQ_F_NOWAIT_WRITE_BIT,
@@ -771,6 +773,8 @@ enum {
REQ_F_ISREG = BIT(REQ_F_ISREG_BIT),
/* has creds assigned */
REQ_F_CREDS = BIT(REQ_F_CREDS_BIT),
+ /* ->async_data allocated */
+ REQ_F_ASYNC_DATA = BIT(REQ_F_ASYNC_DATA_BIT),
};
struct async_poll {
@@ -828,8 +832,6 @@ struct io_kiocb {
struct io_completion compl;
};
- /* opcode allocated if it needs to store data for async defer */
- void *async_data;
u8 opcode;
/* polled IO has completed */
u8 iopoll_completed;
@@ -845,6 +847,14 @@ struct io_kiocb {
struct io_kiocb *link;
struct percpu_ref *fixed_rsrc_refs;
+ /* store used ubuf, so we can prevent reloading */
+ struct io_mapped_ubuf *imu;
+
+ /*
+ * Opcode allocated if it needs to store data for async defer,
+ * only valid if REQ_F_ASYNC_DATA is set
+ */
+ void *async_data;
/* used with ctx->iopoll_list with reads/writes */
struct list_head inflight_entry;
@@ -853,10 +863,8 @@ struct io_kiocb {
struct hlist_node hash_node;
struct async_poll *apoll;
struct io_wq_work work;
+ /* only valid when REQ_F_CREDS is set */
const struct cred *creds;
-
- /* store used ubuf, so we can prevent reloading */
- struct io_mapped_ubuf *imu;
};
struct io_tctx_node {
@@ -1127,6 +1135,11 @@ static bool io_match_task(struct io_kiocb *head, struct task_struct *task,
return false;
}
+static inline bool req_has_async_data(struct io_kiocb *req)
+{
+ return req->flags & REQ_F_ASYNC_DATA;
+}
+
static inline void req_set_fail(struct io_kiocb *req)
{
req->flags |= REQ_F_FAIL;
@@ -1808,10 +1821,6 @@ static void io_dismantle_req(struct io_kiocb *req)
io_put_file(req->file);
if (req->fixed_rsrc_refs)
percpu_ref_put(req->fixed_rsrc_refs);
- if (req->async_data) {
- kfree(req->async_data);
- req->async_data = NULL;
- }
}
static void __io_free_req(struct io_kiocb *req)
@@ -2422,7 +2431,7 @@ static bool io_resubmit_prep(struct io_kiocb *req)
{
struct io_async_rw *rw = req->async_data;
- if (!rw)
+ if (!req_has_async_data(req))
return !io_req_prep_async(req);
if (rw->iter.truncated)
return false;
@@ -2766,7 +2775,7 @@ static void kiocb_done(struct kiocb *kiocb, ssize_t ret,
bool check_reissue = kiocb->ki_complete == io_complete_rw;
/* add previously done IO, if any */
- if (io && io->bytes_done > 0) {
+ if (req_has_async_data(req) && io->bytes_done > 0) {
if (ret < 0)
ret = io->bytes_done;
else
@@ -3141,6 +3150,8 @@ static inline int io_alloc_async_data(struct io_kiocb *req)
{
WARN_ON_ONCE(!io_op_defs[req->opcode].async_size);
req->async_data = kmalloc(io_op_defs[req->opcode].async_size, GFP_KERNEL);
+ if (req->async_data)
+ req->flags |= REQ_F_ASYNC_DATA;
return req->async_data == NULL;
}
@@ -3150,7 +3161,7 @@ static int io_setup_async_rw(struct io_kiocb *req, const struct iovec *iovec,
{
if (!force && !io_op_defs[req->opcode].needs_async_setup)
return 0;
- if (!req->async_data) {
+ if (!req_has_async_data(req)) {
if (io_alloc_async_data(req)) {
kfree(iovec);
return -ENOMEM;
@@ -3274,11 +3285,12 @@ static int io_read(struct io_kiocb *req, unsigned int issue_flags)
struct iovec inline_vecs[UIO_FASTIOV], *iovec = inline_vecs;
struct kiocb *kiocb = &req->rw.kiocb;
struct iov_iter __iter, *iter = &__iter;
- struct io_async_rw *rw = req->async_data;
+ struct io_async_rw *rw = NULL;
ssize_t io_size, ret, ret2;
bool force_nonblock = issue_flags & IO_URING_F_NONBLOCK;
- if (rw) {
+ if (req_has_async_data(req)) {
+ rw = req->async_data;
iter = &rw->iter;
iovec = NULL;
} else {
@@ -3381,11 +3393,12 @@ static int io_write(struct io_kiocb *req, unsigned int issue_flags)
struct iovec inline_vecs[UIO_FASTIOV], *iovec = inline_vecs;
struct kiocb *kiocb = &req->rw.kiocb;
struct iov_iter __iter, *iter = &__iter;
- struct io_async_rw *rw = req->async_data;
+ struct io_async_rw *rw = NULL;
ssize_t ret, ret2, io_size;
bool force_nonblock = issue_flags & IO_URING_F_NONBLOCK;
- if (rw) {
+ if (req_has_async_data(req)) {
+ rw = req->async_data;
iter = &rw->iter;
iovec = NULL;
} else {
@@ -4385,8 +4398,9 @@ static int io_sendmsg(struct io_kiocb *req, unsigned int issue_flags)
if (unlikely(!sock))
return -ENOTSOCK;
- kmsg = req->async_data;
- if (!kmsg) {
+ if (req_has_async_data(req)) {
+ kmsg = req->async_data;
+ } else {
ret = io_sendmsg_copy_hdr(req, &iomsg);
if (ret)
return ret;
@@ -4609,8 +4623,9 @@ static int io_recvmsg(struct io_kiocb *req, unsigned int issue_flags)
if (unlikely(!sock))
return -ENOTSOCK;
- kmsg = req->async_data;
- if (!kmsg) {
+ if (req_has_async_data(req)) {
+ kmsg = req->async_data;
+ } else {
ret = io_recvmsg_copy_hdr(req, &iomsg);
if (ret)
return ret;
@@ -4776,7 +4791,7 @@ static int io_connect(struct io_kiocb *req, unsigned int issue_flags)
int ret;
bool force_nonblock = issue_flags & IO_URING_F_NONBLOCK;
- if (req->async_data) {
+ if (req_has_async_data(req)) {
io = req->async_data;
} else {
ret = move_addr_to_kernel(req->connect.addr,
@@ -4792,7 +4807,7 @@ static int io_connect(struct io_kiocb *req, unsigned int issue_flags)
ret = __sys_connect_file(req->file, &io->address,
req->connect.addr_len, file_flags);
if ((ret == -EAGAIN || ret == -EINPROGRESS) && force_nonblock) {
- if (req->async_data)
+ if (req_has_async_data(req))
return -EAGAIN;
if (io_alloc_async_data(req)) {
ret = -ENOMEM;
@@ -5675,7 +5690,7 @@ static int io_timeout_prep(struct io_kiocb *req, const struct io_uring_sqe *sqe,
if (unlikely(off && !req->ctx->off_timeout_used))
req->ctx->off_timeout_used = true;
- if (!req->async_data && io_alloc_async_data(req))
+ if (!req_has_async_data(req) && io_alloc_async_data(req))
return -ENOMEM;
data = req->async_data;
@@ -5990,7 +6005,7 @@ static int io_req_prep_async(struct io_kiocb *req)
{
if (!io_op_defs[req->opcode].needs_async_setup)
return 0;
- if (WARN_ON_ONCE(req->async_data))
+ if (WARN_ON_ONCE(req_has_async_data(req)))
return -EFAULT;
if (io_alloc_async_data(req))
return -EAGAIN;
@@ -6156,7 +6171,10 @@ static void io_clean_op(struct io_kiocb *req)
}
if (req->flags & REQ_F_CREDS)
put_cred(req->creds);
-
+ if (req->flags & REQ_F_ASYNC_DATA) {
+ kfree(req->async_data);
+ req->async_data = NULL;
+ }
req->flags &= ~IO_REQ_CLEAN_FLAGS;
}
--
2.32.0
next prev parent reply other threads:[~2021-08-09 12:05 UTC|newest]
Thread overview: 32+ messages / expand[flat|nested] mbox.gz Atom feed top
2021-08-09 12:04 [PATCH v2 00/28] for-next patches Pavel Begunkov
2021-08-09 12:04 ` [PATCH 01/28] io_uring: use kvmalloc for fixed files Pavel Begunkov
2021-08-09 12:04 ` [PATCH 02/28] io_uring: inline fixed part of io_file_get() Pavel Begunkov
2021-08-09 12:04 ` [PATCH 03/28] io_uring: rename io_file_supports_async() Pavel Begunkov
2021-08-09 12:04 ` [PATCH 04/28] io_uring: avoid touching inode in rw prep Pavel Begunkov
2021-08-09 12:04 ` [PATCH 05/28] io_uring: clean io-wq callbacks Pavel Begunkov
2021-08-09 12:04 ` [PATCH 06/28] io_uring: remove unnecessary PF_EXITING check Pavel Begunkov
2021-08-09 12:04 ` [PATCH 07/28] io-wq: improve wq_list_add_tail() Pavel Begunkov
2021-08-09 12:04 ` [PATCH 08/28] io_uring: refactor io_alloc_req Pavel Begunkov
2021-08-09 12:04 ` [PATCH 09/28] io_uring: don't halt iopoll too early Pavel Begunkov
2021-08-09 12:04 ` [PATCH 10/28] io_uring: add more locking annotations for submit Pavel Begunkov
2021-08-09 12:04 ` [PATCH 11/28] io_uring: optimise io_cqring_wait() hot path Pavel Begunkov
2021-08-09 12:04 ` [PATCH 12/28] io_uring: extract a helper for ctx quiesce Pavel Begunkov
2021-08-09 12:04 ` [PATCH 13/28] io_uring: move io_put_task() definition Pavel Begunkov
2021-08-09 12:04 ` [PATCH 14/28] io_uring: move io_rsrc_node_alloc() definition Pavel Begunkov
2021-08-09 12:04 ` [PATCH 15/28] io_uring: inline io_free_req_deferred Pavel Begunkov
2021-08-09 12:04 ` [PATCH 16/28] io_uring: deduplicate open iopoll check Pavel Begunkov
2021-08-09 12:04 ` [PATCH 17/28] io_uring: improve ctx hang handling Pavel Begunkov
2021-08-09 12:04 ` [PATCH 18/28] io_uring: kill unused IO_IOPOLL_BATCH Pavel Begunkov
2021-08-09 12:04 ` [PATCH 19/28] io_uring: drop exec checks from io_req_task_submit Pavel Begunkov
2021-08-09 12:04 ` [PATCH 20/28] io_uring: optimise putting task struct Pavel Begunkov
2021-08-09 12:04 ` Pavel Begunkov [this message]
2021-08-09 17:30 ` [PATCH 21/28] io_uring: hide async dadta behind flags Jens Axboe
2021-08-09 17:44 ` Pavel Begunkov
2021-08-09 12:04 ` [PATCH 22/28] io_uring: move io_fallback_req_func() Pavel Begunkov
2021-08-09 12:04 ` [PATCH 23/28] io_uring: cache __io_free_req()'d requests Pavel Begunkov
2021-08-09 12:04 ` [PATCH 24/28] io_uring: remove redundant args from cache_free Pavel Begunkov
2021-08-09 12:04 ` [PATCH 25/28] io_uring: use inflight_entry instead of compl.list Pavel Begunkov
2021-08-09 12:04 ` [PATCH 26/28] io_uring: inline struct io_comp_state Pavel Begunkov
2021-08-09 12:04 ` [PATCH 27/28] io_uring: remove extra argument for overflow flush Pavel Begunkov
2021-08-09 12:04 ` [PATCH 28/28] io_uring: inline io_poll_remove_waitqs Pavel Begunkov
2021-08-09 17:48 ` [PATCH v2 00/28] for-next patches Jens Axboe
Reply instructions:
You may reply publicly to this message via plain-text email
using any one of the following methods:
* Save the following mbox file, import it into your mail client,
and reply-to-all from there: mbox
Avoid top-posting and favor interleaved quoting:
https://en.wikipedia.org/wiki/Posting_style#Interleaved_style
* Reply using the --to, --cc, and --in-reply-to
switches of git-send-email(1):
git send-email \
--in-reply-to=707ce8945e0247db7a585b5d1c9e8240a22e6708.1628471125.git.asml.silence@gmail.com \
[email protected] \
[email protected] \
[email protected] \
/path/to/YOUR_REPLY
https://kernel.org/pub/software/scm/git/docs/git-send-email.html
* If your mail client supports setting the In-Reply-To header
via mailto: links, try the mailto: link
Be sure your reply has a Subject: header at the top and a blank line
before the message body.
This is a public inbox, see mirroring instructions
for how to clone and mirror all data and code used for this inbox