[PATCH V3 6/9] io_uring: support sqe group with members depending on leader

[Ming Lei <ming.lei@redhat.com>]

Generic sqe group provides flexible way for supporting N:M dependency
between groups.

However, some resource can't cross OPs, such as kernel buffer, otherwise
the buffer may be leaked easily in case that any OP failure or application
panic.

Add flag REQ_F_SQE_GROUP_DEP for allowing members to depend on group leader,
so that group members won't be queued until the leader request is completed,
and we still commit leader's CQE after all members CQE are posted. With this
way, the kernel resource lifetime can be aligned with group leader or group,
one typical use case is to support zero copy for device internal buffer.

This use case may not be generic enough, so set it only for specific OP which
can serve as group leader, meantime we have run out of sqe flags.

Signed-off-by: Ming Lei <ming.lei@redhat.com>
---
 include/linux/io_uring_types.h |  3 +++
 io_uring/io_uring.c            | 16 +++++++++++++++-
 io_uring/io_uring.h            |  5 ++++-
 3 files changed, 22 insertions(+), 2 deletions(-)

diff --git a/include/linux/io_uring_types.h b/include/linux/io_uring_types.h
index 62311b0f0e0b..5cbc9d3346a7 100644
--- a/include/linux/io_uring_types.h
+++ b/include/linux/io_uring_types.h
@@ -477,6 +477,7 @@ enum {
 	REQ_F_BL_NO_RECYCLE_BIT,
 	REQ_F_BUFFERS_COMMIT_BIT,
 	REQ_F_SQE_GROUP_LEADER_BIT,
+	REQ_F_SQE_GROUP_DEP_BIT,
 
 	/* not a real bit, just to check we're not overflowing the space */
 	__REQ_F_LAST_BIT,
@@ -561,6 +562,8 @@ enum {
 	REQ_F_BUFFERS_COMMIT	= IO_REQ_FLAG(REQ_F_BUFFERS_COMMIT_BIT),
 	/* sqe group lead */
 	REQ_F_SQE_GROUP_LEADER	= IO_REQ_FLAG(REQ_F_SQE_GROUP_LEADER_BIT),
+	/* sqe group with members depending on leader */
+	REQ_F_SQE_GROUP_DEP	= IO_REQ_FLAG(REQ_F_SQE_GROUP_DEP_BIT),
 };
 
 typedef void (*io_req_tw_func_t)(struct io_kiocb *req, struct io_tw_state *ts);
diff --git a/io_uring/io_uring.c b/io_uring/io_uring.c
index b87c5452de43..5d94629c01b8 100644
--- a/io_uring/io_uring.c
+++ b/io_uring/io_uring.c
@@ -936,7 +936,14 @@ static __always_inline void io_req_commit_cqe(struct io_kiocb *req,
 
 static inline bool need_queue_group_members(struct io_kiocb *req)
 {
-	return req_is_group_leader(req) && req->grp_link;
+	if (likely(!(req->flags & REQ_F_SQE_GROUP)))
+		return false;
+
+	if (!(req->flags & REQ_F_SQE_GROUP_LEADER) ||
+			(req->flags & REQ_F_SQE_GROUP_DEP))
+		return false;
+
+	return !!req->grp_link;
 }
 
 /* Can only be called after this request is issued */
@@ -983,6 +990,9 @@ void io_queue_group_members(struct io_kiocb *req, bool async)
 
 		if (unlikely(member->flags & REQ_F_FAIL)) {
 			io_req_task_queue_fail(member, member->cqe.res);
+		} else if (unlikely((req->flags & REQ_F_FAIL) &&
+			     (req->flags & REQ_F_SQE_GROUP_DEP))) {
+			io_req_task_queue_fail(member, -ECANCELED);
 		} else if (member->flags & REQ_F_FORCE_ASYNC) {
 			io_req_task_queue(member);
 		} else {
@@ -1065,6 +1075,10 @@ static void io_req_complete_post(struct io_kiocb *req, unsigned issue_flags)
 		return;
 	}
 
+	/* queue members which may depend on leader */
+	if (req_is_group_leader(req) && (req->flags & REQ_F_SQE_GROUP_DEP))
+		io_queue_group_members(req, true);
+
 	io_cq_lock(ctx);
 	if (!(req->flags & REQ_F_CQE_SKIP))
 		io_req_commit_cqe(req, false);
diff --git a/io_uring/io_uring.h b/io_uring/io_uring.h
index b11db3bdd8d8..f593ff8b2deb 100644
--- a/io_uring/io_uring.h
+++ b/io_uring/io_uring.h
@@ -368,7 +368,10 @@ static inline void io_req_complete_defer(struct io_kiocb *req)
 
 	wq_list_add_tail(&req->comp_list, &state->compl_reqs);
 
-	/* members may not be issued when leader is completed */
+	/*
+	 * Members may not be issued when leader is completed, or members
+	 * depend on leader in case of REQ_F_SQE_GROUP_DEP
+	 */
 	if (unlikely(req_is_group_leader(req) && req->grp_link))
 		io_queue_group_members(req, false);
 }
-- 
2.42.0