[PATCH 03/11] io_uring: optimise out task_work checks on enter

[Pavel Begunkov <asml.silence@gmail.com>]

io_run_task_work() does some extra work for safety, that isn't actually
needed in many cases, particularly when it's known that current is not
exiting and in the TASK_RUNNING state, like in the beginning of a
syscall.

Signed-off-by: Pavel Begunkov <asml.silence@gmail.com>
---
 fs/io_uring.c | 16 +++++++++++++---
 1 file changed, 13 insertions(+), 3 deletions(-)

diff --git a/fs/io_uring.c b/fs/io_uring.c
index 080fac4d4543..45b49273df8b 100644
--- a/fs/io_uring.c
+++ b/fs/io_uring.c
@@ -2227,6 +2227,16 @@ static inline unsigned int io_put_rw_kbuf(struct io_kiocb *req)
 	return io_put_kbuf(req, kbuf);
 }
 
+/* only safe when TASK_RUNNING and it's not PF_EXITING */
+static inline bool __io_run_task_work(void)
+{
+	if (current->task_works) {
+		task_work_run();
+		return true;
+	}
+	return false;
+}
+
 static inline bool io_run_task_work(void)
 {
 	/*
@@ -6729,7 +6739,7 @@ static int io_sq_thread(void *data)
 		}
 
 		if (sqt_spin || !time_after(jiffies, timeout)) {
-			io_run_task_work();
+			__io_run_task_work();
 			cond_resched();
 			if (sqt_spin)
 				timeout = jiffies + sqd->sq_thread_idle;
@@ -6870,7 +6880,7 @@ static int io_cqring_wait(struct io_ring_ctx *ctx, int min_events,
 		io_cqring_overflow_flush(ctx, false, NULL, NULL);
 		if (io_cqring_events(ctx) >= min_events)
 			return 0;
-		if (!io_run_task_work())
+		if (!__io_run_task_work())
 			break;
 	} while (1);
 
@@ -9117,7 +9127,7 @@ SYSCALL_DEFINE6(io_uring_enter, unsigned int, fd, u32, to_submit,
 	struct fd f;
 	long ret;
 
-	io_run_task_work();
+	__io_run_task_work();
 
 	if (unlikely(flags & ~(IORING_ENTER_GETEVENTS | IORING_ENTER_SQ_WAKEUP |
 			       IORING_ENTER_SQ_WAIT | IORING_ENTER_EXT_ARG)))
-- 
2.24.0