From: Pavel Begunkov <[email protected]>
To: Jens Axboe <[email protected]>,
[email protected], [email protected]
Subject: [PATCH 4/4] io-wq: return next work from ->do_work() directly
Date: Tue, 23 Jun 2020 01:16:35 +0300 [thread overview]
Message-ID: <915321064eaf58b3b6fc2c6fed015879c769f3a1.1592863245.git.asml.silence@gmail.com> (raw)
In-Reply-To: <[email protected]>
It's easier to return next work from ->do_work() than
having an in-out argument. Looks nicer and easier to compile.
Also, merge io_wq_assign_next() into its only user.
Signed-off-by: Pavel Begunkov <[email protected]>
---
fs/io-wq.c | 8 +++-----
fs/io-wq.h | 2 +-
fs/io_uring.c | 53 ++++++++++++++++++++-------------------------------
3 files changed, 25 insertions(+), 38 deletions(-)
diff --git a/fs/io-wq.c b/fs/io-wq.c
index 47c5f3aeb460..72f759e1d6eb 100644
--- a/fs/io-wq.c
+++ b/fs/io-wq.c
@@ -523,9 +523,8 @@ static void io_worker_handle_work(struct io_worker *worker)
work->flags |= IO_WQ_WORK_CANCEL;
hash = io_get_work_hash(work);
- linked = old_work = work;
- wq->do_work(&linked);
- linked = (old_work == linked) ? NULL : linked;
+ old_work = work;
+ linked = wq->do_work(work);
work = next_hashed;
if (!work && linked && !io_wq_is_hashed(linked)) {
@@ -781,8 +780,7 @@ static void io_run_cancel(struct io_wq_work *work, struct io_wqe *wqe)
struct io_wq_work *old_work = work;
work->flags |= IO_WQ_WORK_CANCEL;
- wq->do_work(&work);
- work = (work == old_work) ? NULL : work;
+ work = wq->do_work(work);
wq->free_work(old_work);
} while (work);
}
diff --git a/fs/io-wq.h b/fs/io-wq.h
index 04239dfb12b0..114f12ec2d65 100644
--- a/fs/io-wq.h
+++ b/fs/io-wq.h
@@ -101,7 +101,7 @@ static inline struct io_wq_work *wq_next_work(struct io_wq_work *work)
}
typedef void (free_work_fn)(struct io_wq_work *);
-typedef void (io_wq_work_fn)(struct io_wq_work **);
+typedef struct io_wq_work *(io_wq_work_fn)(struct io_wq_work *);
struct io_wq_data {
struct user_struct *user;
diff --git a/fs/io_uring.c b/fs/io_uring.c
index 595d2bbb31b1..5282711eeeaf 100644
--- a/fs/io_uring.c
+++ b/fs/io_uring.c
@@ -886,7 +886,6 @@ enum io_mem_account {
static void io_complete_rw_common(struct kiocb *kiocb, long res);
static bool io_rw_reissue(struct io_kiocb *req, long res);
-static void io_wq_submit_work(struct io_wq_work **workptr);
static void io_cqring_fill_event(struct io_kiocb *req, long res);
static void io_put_req(struct io_kiocb *req);
static void __io_double_put_req(struct io_kiocb *req);
@@ -1624,20 +1623,6 @@ static void io_free_req(struct io_kiocb *req)
io_queue_async_work(nxt);
}
-static void io_wq_assign_next(struct io_wq_work **workptr, struct io_kiocb *nxt)
-{
- struct io_kiocb *link;
- const struct io_op_def *def = &io_op_defs[nxt->opcode];
-
- if ((nxt->flags & REQ_F_ISREG) && def->hash_reg_file)
- io_wq_hash_work(&nxt->work, file_inode(nxt->file));
-
- *workptr = &nxt->work;
- link = io_prep_linked_timeout(nxt);
- if (link)
- nxt->flags |= REQ_F_QUEUE_TIMEOUT;
-}
-
/*
* Drop reference to request, return next in chain (if there is one) if this
* was the last reference to this request.
@@ -1657,24 +1642,29 @@ static void io_put_req(struct io_kiocb *req)
io_free_req(req);
}
-static void io_steal_work(struct io_kiocb *req,
- struct io_wq_work **workptr)
+static struct io_wq_work *io_steal_work(struct io_kiocb *req)
{
+ struct io_kiocb *link, *nxt = NULL;
+
/*
- * It's in an io-wq worker, so there always should be at least
- * one reference, which will be dropped in io_put_work() just
- * after the current handler returns.
- *
- * It also means, that if the counter dropped to 1, then there is
- * no asynchronous users left, so it's safe to steal the next work.
+ * A ref is owned by io-wq in which context we're. So, if that's the
+ * last one, it's safe to steal next work. False negatives are Ok,
+ * it just will be re-punted async in io_put_work()
*/
- if (refcount_read(&req->refs) == 1) {
- struct io_kiocb *nxt = NULL;
+ if (refcount_read(&req->refs) != 1)
+ return NULL;
- io_req_find_next(req, &nxt);
- if (nxt)
- io_wq_assign_next(workptr, nxt);
- }
+ io_req_find_next(req, &nxt);
+ if (!nxt)
+ return NULL;
+
+ if ((nxt->flags & REQ_F_ISREG) && io_op_defs[nxt->opcode].hash_reg_file)
+ io_wq_hash_work(&nxt->work, file_inode(nxt->file));
+
+ link = io_prep_linked_timeout(nxt);
+ if (link)
+ nxt->flags |= REQ_F_QUEUE_TIMEOUT;
+ return &nxt->work;
}
/*
@@ -5626,9 +5616,8 @@ static void io_arm_async_linked_timeout(struct io_kiocb *req)
io_queue_linked_timeout(link);
}
-static void io_wq_submit_work(struct io_wq_work **workptr)
+static struct io_wq_work *io_wq_submit_work(struct io_wq_work *work)
{
- struct io_wq_work *work = *workptr;
struct io_kiocb *req = container_of(work, struct io_kiocb, work);
int ret = 0;
@@ -5660,7 +5649,7 @@ static void io_wq_submit_work(struct io_wq_work **workptr)
io_put_req(req);
}
- io_steal_work(req, workptr);
+ return io_steal_work(req);
}
static inline struct file *io_file_from_index(struct io_ring_ctx *ctx,
--
2.24.0
prev parent reply other threads:[~2020-06-22 22:18 UTC|newest]
Thread overview: 11+ messages / expand[flat|nested] mbox.gz Atom feed top
2020-06-22 22:16 [PATCH 0/4] iopoll fixes + cleanups Pavel Begunkov
2020-06-22 22:16 ` [PATCH 1/4] io_uring: fix hanging iopoll in case of -EAGAIN Pavel Begunkov
2020-06-23 2:07 ` Jens Axboe
2020-06-23 2:18 ` Jens Axboe
2020-06-23 11:57 ` Pavel Begunkov
2020-06-23 19:01 ` Jens Axboe
2020-06-24 16:56 ` Pavel Begunkov
2020-06-22 22:16 ` [PATCH 2/4] io_uring: handle EAGAIN iopoll Pavel Begunkov
2020-06-30 4:01 ` kernel test robot
2020-06-22 22:16 ` [PATCH 3/4] io-wq: compact io-wq flags numbers Pavel Begunkov
2020-06-22 22:16 ` Pavel Begunkov [this message]
Reply instructions:
You may reply publicly to this message via plain-text email
using any one of the following methods:
* Save the following mbox file, import it into your mail client,
and reply-to-all from there: mbox
Avoid top-posting and favor interleaved quoting:
https://en.wikipedia.org/wiki/Posting_style#Interleaved_style
* Reply using the --to, --cc, and --in-reply-to
switches of git-send-email(1):
git send-email \
--in-reply-to=915321064eaf58b3b6fc2c6fed015879c769f3a1.1592863245.git.asml.silence@gmail.com \
[email protected] \
[email protected] \
[email protected] \
[email protected] \
/path/to/YOUR_REPLY
https://kernel.org/pub/software/scm/git/docs/git-send-email.html
* If your mail client supports setting the In-Reply-To header
via mailto: links, try the mailto: link
Be sure your reply has a Subject: header at the top and a blank line
before the message body.
This is a public inbox, see mirroring instructions
for how to clone and mirror all data and code used for this inbox