[PATCH 04/10] io_uring: put request files before posting the completions

[Jens Axboe <axboe@kernel.dk>]

The batch flush posts the CQEs first and drops the files afterwards,
with a plain fput() that defers the release to task_work. For the
submitting task that runs on the way back to userspace, but with SQPOLL
the sqpoll thread only gets to it after the io_uring task_work is done.
Userspace sees the CQE in between, and closing the file and exec'ing
it fails with ETXTBSY as the request still holds it.

Drop the files synchronously before the CQEs are posted, for requests
that the flush is about to free. Ring files stay deferred, releasing
one takes its uring_lock and may wait on its requests.

Signed-off-by: Jens Axboe <axboe@kernel.dk>
---
 io_uring/io_uring.c | 29 ++++++++++++++++++++---------
 1 file changed, 20 insertions(+), 9 deletions(-)

diff --git a/io_uring/io_uring.c b/io_uring/io_uring.c
index 69f1b3f4c4a1..9ad1962c8dd8 100644
--- a/io_uring/io_uring.c
+++ b/io_uring/io_uring.c
@@ -907,11 +907,8 @@ bool io_req_post_cqe32(struct io_kiocb *req, struct io_uring_cqe cqe[2])
 	return posted;
 }
 
-/*
- * Drop any io-wq request with a file upfront, otherwise it gets deferred to
- * much later post CQE posting.
- */
-static void io_req_put_file_iowq(struct io_kiocb *req, bool sync)
+/* drop the request file before the CQE is posted, not deferred after it */
+static void io_req_put_file(struct io_kiocb *req, bool sync)
 {
 	struct file *file = req->file;
 
@@ -919,7 +916,8 @@ static void io_req_put_file_iowq(struct io_kiocb *req, bool sync)
 		return;
 
 	WRITE_ONCE(req->file, NULL);
-	if (sync)
+	/* releasing a ring may wait on other rings, keep that deferred */
+	if (sync && !io_is_uring_fops(file))
 		__fput_sync(file);
 	else
 		fput(file);
@@ -937,7 +935,7 @@ static void io_req_complete_post(struct io_kiocb *req, unsigned issue_flags)
 	if (WARN_ON_ONCE(!(issue_flags & IO_URING_F_IOWQ)))
 		return;
 
-	io_req_put_file_iowq(req, true);
+	io_req_put_file(req, true);
 
 	/*
 	 * Handle special CQ sync cases via task_work. DEFER_TASKRUN requires
@@ -1152,6 +1150,19 @@ void __io_submit_flush_completions(struct io_ring_ctx *ctx)
 	struct io_submit_state *state = &ctx->submit_state;
 	struct io_wq_work_node *node;
 
+	/*
+	 * Drop the files before the CQEs are posted, so they're released by
+	 * the time the completions are visible. Not for requests that are
+	 * requeued or still referenced, those aren't freed below.
+	 */
+	__wq_list_for_each(node, &state->compl_reqs) {
+		struct io_kiocb *req = container_of(node, struct io_kiocb,
+					    comp_list);
+
+		if (!io_req_shared(req))
+			io_req_put_file(req, true);
+	}
+
 	__io_cq_lock(ctx);
 	__wq_list_for_each(node, &state->compl_reqs) {
 		struct io_kiocb *req = container_of(node, struct io_kiocb,
@@ -1505,7 +1516,7 @@ void io_wq_submit_work(struct io_wq_work *work)
 	/* either cancelled or io-wq is dying, so don't touch tctx->iowq */
 	if (atomic_read(&work->flags) & IO_WQ_WORK_CANCEL) {
 fail:
-		io_req_put_file_iowq(req, false);
+		io_req_put_file(req, false);
 		io_req_task_queue_fail(req, err);
 		return;
 	}
@@ -1582,7 +1593,7 @@ void io_wq_submit_work(struct io_wq_work *work)
 
 	/* avoid locking problems by failing it from a clean context */
 	if (ret) {
-		io_req_put_file_iowq(req, true);
+		io_req_put_file(req, true);
 		io_req_task_queue_fail(req, ret);
 	}
 }
-- 
2.55.0