From: Jens Axboe <[email protected]>
To: [email protected]
Cc: Jens Axboe <[email protected]>
Subject: [PATCH 1/2] io_uring: wrap multi-req freeing in struct req_batch
Date: Sat, 28 Dec 2019 12:18:56 -0700 [thread overview]
Message-ID: <[email protected]> (raw)
In-Reply-To: <[email protected]>
This cleans up the code a bit, and it allows us to build on top of the
multi-req freeing.
Signed-off-by: Jens Axboe <[email protected]>
---
fs/io_uring.c | 65 ++++++++++++++++++++++++---------------------------
1 file changed, 31 insertions(+), 34 deletions(-)
diff --git a/fs/io_uring.c b/fs/io_uring.c
index 0e6ed31cdb12..2de1e8409ab6 100644
--- a/fs/io_uring.c
+++ b/fs/io_uring.c
@@ -1129,14 +1129,19 @@ static struct io_kiocb *io_get_req(struct io_ring_ctx *ctx,
return NULL;
}
-static void io_free_req_many(struct io_ring_ctx *ctx, void **reqs, int *nr)
+struct req_batch {
+ void *reqs[IO_IOPOLL_BATCH];
+ int to_free;
+};
+
+static void io_free_req_many(struct io_ring_ctx *ctx, struct req_batch *rb)
{
- if (*nr) {
- kmem_cache_free_bulk(req_cachep, *nr, reqs);
- percpu_ref_put_many(&ctx->refs, *nr);
- percpu_ref_put_many(&ctx->file_data->refs, *nr);
- *nr = 0;
- }
+ if (!rb->to_free)
+ return;
+ kmem_cache_free_bulk(req_cachep, rb->to_free, rb->reqs);
+ percpu_ref_put_many(&ctx->refs, rb->to_free);
+ percpu_ref_put_many(&ctx->file_data->refs, rb->to_free);
+ rb->to_free = 0;
}
static void __io_req_do_free(struct io_kiocb *req)
@@ -1368,7 +1373,7 @@ static inline unsigned int io_sqring_entries(struct io_ring_ctx *ctx)
return smp_load_acquire(&rings->sq.tail) - ctx->cached_sq_head;
}
-static inline bool io_req_multi_free(struct io_kiocb *req)
+static inline bool io_req_multi_free(struct req_batch *rb, struct io_kiocb *req)
{
/*
* If we're not using fixed files, we have to pair the completion part
@@ -1376,8 +1381,12 @@ static inline bool io_req_multi_free(struct io_kiocb *req)
* free for fixed file and non-linked commands.
*/
if (((req->flags & (REQ_F_FIXED_FILE|REQ_F_LINK)) == REQ_F_FIXED_FILE)
- && !io_is_fallback_req(req) && !req->io)
+ && !io_is_fallback_req(req) && !req->io) {
+ rb->reqs[rb->to_free++] = req;
+ if (unlikely(rb->to_free == ARRAY_SIZE(rb->reqs)))
+ io_free_req_many(req->ctx, rb);
return true;
+ }
return false;
}
@@ -1388,11 +1397,10 @@ static inline bool io_req_multi_free(struct io_kiocb *req)
static void io_iopoll_complete(struct io_ring_ctx *ctx, unsigned int *nr_events,
struct list_head *done)
{
- void *reqs[IO_IOPOLL_BATCH];
+ struct req_batch rb;
struct io_kiocb *req;
- int to_free;
- to_free = 0;
+ rb.to_free = 0;
while (!list_empty(done)) {
req = list_first_entry(done, struct io_kiocb, list);
list_del(&req->list);
@@ -1400,19 +1408,13 @@ static void io_iopoll_complete(struct io_ring_ctx *ctx, unsigned int *nr_events,
io_cqring_fill_event(req, req->result);
(*nr_events)++;
- if (refcount_dec_and_test(&req->refs)) {
- if (io_req_multi_free(req)) {
- reqs[to_free++] = req;
- if (to_free == ARRAY_SIZE(reqs))
- io_free_req_many(ctx, reqs, &to_free);
- } else {
- io_free_req(req);
- }
- }
+ if (refcount_dec_and_test(&req->refs) &&
+ !io_req_multi_free(&rb, req))
+ io_free_req(req);
}
io_commit_cqring(ctx);
- io_free_req_many(ctx, reqs, &to_free);
+ io_free_req_many(ctx, &rb);
}
static int io_do_iopoll(struct io_ring_ctx *ctx, unsigned int *nr_events,
@@ -3225,30 +3227,25 @@ static void io_poll_complete_work(struct io_wq_work **workptr)
static void __io_poll_flush(struct io_ring_ctx *ctx, struct llist_node *nodes)
{
- void *reqs[IO_IOPOLL_BATCH];
struct io_kiocb *req, *tmp;
- int to_free = 0;
+ struct req_batch rb;
+ rb.to_free = 0;
spin_lock_irq(&ctx->completion_lock);
llist_for_each_entry_safe(req, tmp, nodes, llist_node) {
hash_del(&req->hash_node);
io_poll_complete(req, req->result, 0);
- if (refcount_dec_and_test(&req->refs)) {
- if (io_req_multi_free(req)) {
- reqs[to_free++] = req;
- if (to_free == ARRAY_SIZE(reqs))
- io_free_req_many(ctx, reqs, &to_free);
- } else {
- req->flags |= REQ_F_COMP_LOCKED;
- io_free_req(req);
- }
+ if (refcount_dec_and_test(&req->refs) &&
+ !io_req_multi_free(&rb, req)) {
+ req->flags |= REQ_F_COMP_LOCKED;
+ io_free_req(req);
}
}
spin_unlock_irq(&ctx->completion_lock);
io_cqring_ev_posted(ctx);
- io_free_req_many(ctx, reqs, &to_free);
+ io_free_req_many(ctx, &rb);
}
static void io_poll_flush(struct io_wq_work **workptr)
--
2.24.1
next prev parent reply other threads:[~2019-12-28 19:19 UTC|newest]
Thread overview: 3+ messages / expand[flat|nested] mbox.gz Atom feed top
2019-12-28 19:18 [PATCHSET 0/2] Extend io_uring batch freeing Jens Axboe
2019-12-28 19:18 ` Jens Axboe [this message]
2019-12-28 19:18 ` [PATCH 2/2] io_uring: extend batch freeing to cover more cases Jens Axboe
Reply instructions:
You may reply publicly to this message via plain-text email
using any one of the following methods:
* Save the following mbox file, import it into your mail client,
and reply-to-all from there: mbox
Avoid top-posting and favor interleaved quoting:
https://en.wikipedia.org/wiki/Posting_style#Interleaved_style
* Reply using the --to, --cc, and --in-reply-to
switches of git-send-email(1):
git send-email \
[email protected] \
[email protected] \
[email protected] \
/path/to/YOUR_REPLY
https://kernel.org/pub/software/scm/git/docs/git-send-email.html
* If your mail client supports setting the In-Reply-To header
via mailto: links, try the mailto: link
Be sure your reply has a Subject: header at the top and a blank line
before the message body.
This is a public inbox, see mirroring instructions
for how to clone and mirror all data and code used for this inbox