[PATCH v2 0/2] cleanup of submission path

[PATCH v2 0/2] cleanup of submission path

[Pavel Begunkov]

A minor cleanup of io_submit_sqes() and io_ring_submit().

v2: rebased
    move io_queue_link_head()

Pavel Begunkov (2):
  io_uring: Merge io_submit_sqes and io_ring_submit
  io_uring: io_queue_link*() right after submit

 fs/io_uring.c | 110 +++++++++++++-------------------------------------
 1 file changed, 28 insertions(+), 82 deletions(-)

-- 
2.23.0

[PATCH v2 1/2] io_uring: Merge io_submit_sqes and io_ring_submit

[Pavel Begunkov]

io_submit_sqes() and io_ring_submit() are doing the same stuff with
a little difference. Deduplicate them.

Signed-off-by: Pavel Begunkov <asml.silence@gmail.com>
---
 fs/io_uring.c | 88 +++++++++++----------------------------------------
 1 file changed, 18 insertions(+), 70 deletions(-)

diff --git a/fs/io_uring.c b/fs/io_uring.c
index 7813bc7d5b61..ebe2a4edd644 100644
--- a/fs/io_uring.c
+++ b/fs/io_uring.c
@@ -2681,7 +2681,8 @@ static bool io_get_sqring(struct io_ring_ctx *ctx, struct sqe_submit *s)
 }
 
 static int io_submit_sqes(struct io_ring_ctx *ctx, unsigned int nr,
-			  struct mm_struct **mm)
+			  struct file *ring_file, int ring_fd,
+			  struct mm_struct **mm, bool async)
 {
 	struct io_submit_state state, *statep = NULL;
 	struct io_kiocb *link = NULL;
@@ -2732,10 +2733,12 @@ static int io_submit_sqes(struct io_ring_ctx *ctx, unsigned int nr,
 		}
 
 out:
+		s.ring_file = ring_file;
+		s.ring_fd = ring_fd;
 		s.has_user = *mm != NULL;
-		s.in_async = true;
-		s.needs_fixed_file = true;
-		trace_io_uring_submit_sqe(ctx, s.sqe->user_data, true, true);
+		s.in_async = async;
+		s.needs_fixed_file = async;
+		trace_io_uring_submit_sqe(ctx, s.sqe->user_data, true, async);
 		io_submit_sqe(ctx, &s, statep, &link);
 		submitted++;
 	}
@@ -2745,6 +2748,9 @@ static int io_submit_sqes(struct io_ring_ctx *ctx, unsigned int nr,
 	if (statep)
 		io_submit_state_end(&state);
 
+	 /* Commit SQ ring head once we've consumed and submitted all SQEs */
+	io_commit_sqring(ctx);
+
 	return submitted;
 }
 
@@ -2849,7 +2855,8 @@ static int io_sq_thread(void *data)
 		}
 
 		to_submit = min(to_submit, ctx->sq_entries);
-		inflight += io_submit_sqes(ctx, to_submit, &cur_mm);
+		inflight += io_submit_sqes(ctx, to_submit, NULL, -1, &cur_mm,
+					   true);
 
 		/* Commit SQ ring head once we've consumed all SQEs */
 		io_commit_sqring(ctx);
@@ -2866,69 +2873,6 @@ static int io_sq_thread(void *data)
 	return 0;
 }
 
-static int io_ring_submit(struct io_ring_ctx *ctx, unsigned int to_submit,
-			  struct file *ring_file, int ring_fd)
-{
-	struct io_submit_state state, *statep = NULL;
-	struct io_kiocb *link = NULL;
-	struct io_kiocb *shadow_req = NULL;
-	bool prev_was_link = false;
-	int i, submit = 0;
-
-	if (to_submit > IO_PLUG_THRESHOLD) {
-		io_submit_state_start(&state, ctx, to_submit);
-		statep = &state;
-	}
-
-	for (i = 0; i < to_submit; i++) {
-		struct sqe_submit s;
-
-		if (!io_get_sqring(ctx, &s))
-			break;
-
-		/*
-		 * If previous wasn't linked and we have a linked command,
-		 * that's the end of the chain. Submit the previous link.
-		 */
-		if (!prev_was_link && link) {
-			io_queue_link_head(ctx, link, &link->submit, shadow_req);
-			link = NULL;
-			shadow_req = NULL;
-		}
-		prev_was_link = (s.sqe->flags & IOSQE_IO_LINK) != 0;
-
-		if (link && (s.sqe->flags & IOSQE_IO_DRAIN)) {
-			if (!shadow_req) {
-				shadow_req = io_get_req(ctx, NULL);
-				if (unlikely(!shadow_req))
-					goto out;
-				shadow_req->flags |= (REQ_F_IO_DRAIN | REQ_F_SHADOW_DRAIN);
-				refcount_dec(&shadow_req->refs);
-			}
-			shadow_req->sequence = s.sequence;
-		}
-
-out:
-		s.ring_file = ring_file;
-		s.has_user = true;
-		s.in_async = false;
-		s.needs_fixed_file = false;
-		s.ring_fd = ring_fd;
-		submit++;
-		trace_io_uring_submit_sqe(ctx, s.sqe->user_data, true, false);
-		io_submit_sqe(ctx, &s, statep, &link);
-	}
-
-	if (link)
-		io_queue_link_head(ctx, link, &link->submit, shadow_req);
-	if (statep)
-		io_submit_state_end(statep);
-
-	io_commit_sqring(ctx);
-
-	return submit;
-}
-
 struct io_wait_queue {
 	struct wait_queue_entry wq;
 	struct io_ring_ctx *ctx;
@@ -4049,10 +3993,14 @@ SYSCALL_DEFINE6(io_uring_enter, unsigned int, fd, u32, to_submit,
 			wake_up(&ctx->sqo_wait);
 		submitted = to_submit;
 	} else if (to_submit) {
-		to_submit = min(to_submit, ctx->sq_entries);
+		struct mm_struct *cur_mm;
 
+		to_submit = min(to_submit, ctx->sq_entries);
 		mutex_lock(&ctx->uring_lock);
-		submitted = io_ring_submit(ctx, to_submit, f.file, fd);
+		/* already have mm, so io_submit_sqes() won't try to grab it */
+		cur_mm = ctx->sqo_mm;
+		submitted = io_submit_sqes(ctx, to_submit, f.file, fd,
+					   &cur_mm, false);
 		mutex_unlock(&ctx->uring_lock);
 	}
 	if (flags & IORING_ENTER_GETEVENTS) {
-- 
2.23.0

[PATCH v2 2/2] io_uring: io_queue_link*() right after submit

[Pavel Begunkov]

After a call to io_submit_sqe(), it's already known whether it needs
to queue a link or not. Do it there, as it's simplier and doesn't keep
an extra variable across the loop.

Signed-off-by: Pavel Begunkov <asml.silence@gmail.com>
---
 fs/io_uring.c | 22 ++++++++++------------
 1 file changed, 10 insertions(+), 12 deletions(-)

diff --git a/fs/io_uring.c b/fs/io_uring.c
index ebe2a4edd644..82c2da99cb5c 100644
--- a/fs/io_uring.c
+++ b/fs/io_uring.c
@@ -2687,7 +2687,6 @@ static int io_submit_sqes(struct io_ring_ctx *ctx, unsigned int nr,
 	struct io_submit_state state, *statep = NULL;
 	struct io_kiocb *link = NULL;
 	struct io_kiocb *shadow_req = NULL;
-	bool prev_was_link = false;
 	int i, submitted = 0;
 	bool mm_fault = false;
 
@@ -2710,17 +2709,6 @@ static int io_submit_sqes(struct io_ring_ctx *ctx, unsigned int nr,
 			}
 		}
 
-		/*
-		 * If previous wasn't linked and we have a linked command,
-		 * that's the end of the chain. Submit the previous link.
-		 */
-		if (!prev_was_link && link) {
-			io_queue_link_head(ctx, link, &link->submit, shadow_req);
-			link = NULL;
-			shadow_req = NULL;
-		}
-		prev_was_link = (s.sqe->flags & IOSQE_IO_LINK) != 0;
-
 		if (link && (s.sqe->flags & IOSQE_IO_DRAIN)) {
 			if (!shadow_req) {
 				shadow_req = io_get_req(ctx, NULL);
@@ -2741,6 +2729,16 @@ static int io_submit_sqes(struct io_ring_ctx *ctx, unsigned int nr,
 		trace_io_uring_submit_sqe(ctx, s.sqe->user_data, true, async);
 		io_submit_sqe(ctx, &s, statep, &link);
 		submitted++;
+
+		/*
+		 * If previous wasn't linked and we have a linked command,
+		 * that's the end of the chain. Submit the previous link.
+		 */
+		if (!(s.sqe->flags & IOSQE_IO_LINK) && link) {
+			io_queue_link_head(ctx, link, &link->submit, shadow_req);
+			link = NULL;
+			shadow_req = NULL;
+		}
 	}
 
 	if (link)
-- 
2.23.0

Re: [PATCH v2 2/2] io_uring: io_queue_link*() right after submit

[Bob Liu]

On 11/6/19 5:22 AM, Pavel Begunkov wrote:
> After a call to io_submit_sqe(), it's already known whether it needs
> to queue a link or not. Do it there, as it's simplier and doesn't keep
> an extra variable across the loop.
> 
> Signed-off-by: Pavel Begunkov <asml.silence@gmail.com>
> ---
>  fs/io_uring.c | 22 ++++++++++------------
>  1 file changed, 10 insertions(+), 12 deletions(-)
> 
> diff --git a/fs/io_uring.c b/fs/io_uring.c
> index ebe2a4edd644..82c2da99cb5c 100644
> --- a/fs/io_uring.c
> +++ b/fs/io_uring.c
> @@ -2687,7 +2687,6 @@ static int io_submit_sqes(struct io_ring_ctx *ctx, unsigned int nr,
>  	struct io_submit_state state, *statep = NULL;
>  	struct io_kiocb *link = NULL;
>  	struct io_kiocb *shadow_req = NULL;
> -	bool prev_was_link = false;
>  	int i, submitted = 0;
>  	bool mm_fault = false;
>  
> @@ -2710,17 +2709,6 @@ static int io_submit_sqes(struct io_ring_ctx *ctx, unsigned int nr,
>  			}
>  		}
>  
> -		/*
> -		 * If previous wasn't linked and we have a linked command,
> -		 * that's the end of the chain. Submit the previous link.
> -		 */
> -		if (!prev_was_link && link) {
> -			io_queue_link_head(ctx, link, &link->submit, shadow_req);
> -			link = NULL;
> -			shadow_req = NULL;
> -		}
> -		prev_was_link = (s.sqe->flags & IOSQE_IO_LINK) != 0;
> -
>  		if (link && (s.sqe->flags & IOSQE_IO_DRAIN)) {
>  			if (!shadow_req) {
>  				shadow_req = io_get_req(ctx, NULL);
> @@ -2741,6 +2729,16 @@ static int io_submit_sqes(struct io_ring_ctx *ctx, unsigned int nr,
>  		trace_io_uring_submit_sqe(ctx, s.sqe->user_data, true, async);
>  		io_submit_sqe(ctx, &s, statep, &link);
>  		submitted++;
> +
> +		/*
> +		 * If previous wasn't linked and we have a linked command,
> +		 * that's the end of the chain. Submit the previous link.
> +		 */
> +		if (!(s.sqe->flags & IOSQE_IO_LINK) && link) 
The behavior changed to 'current seq' instead of previous after dropping prev_was_link?

> +			io_queue_link_head(ctx, link, &link->submit, shadow_req);
> +			link = NULL;
> +			shadow_req = NULL;
> +		}
>  	}
>  
>  	if (link)
> 

Re: [PATCH v2 1/2] io_uring: Merge io_submit_sqes and io_ring_submit

[Bob Liu]

On 11/6/19 5:22 AM, Pavel Begunkov wrote:
> io_submit_sqes() and io_ring_submit() are doing the same stuff with
> a little difference. Deduplicate them.
> 
> Signed-off-by: Pavel Begunkov <asml.silence@gmail.com>
> ---
>  fs/io_uring.c | 88 +++++++++++----------------------------------------
>  1 file changed, 18 insertions(+), 70 deletions(-)
> 
> diff --git a/fs/io_uring.c b/fs/io_uring.c
> index 7813bc7d5b61..ebe2a4edd644 100644
> --- a/fs/io_uring.c
> +++ b/fs/io_uring.c
> @@ -2681,7 +2681,8 @@ static bool io_get_sqring(struct io_ring_ctx *ctx, struct sqe_submit *s)
>  }
>  
>  static int io_submit_sqes(struct io_ring_ctx *ctx, unsigned int nr,
> -			  struct mm_struct **mm)
> +			  struct file *ring_file, int ring_fd,
> +			  struct mm_struct **mm, bool async)
>  {
>  	struct io_submit_state state, *statep = NULL;
>  	struct io_kiocb *link = NULL;
> @@ -2732,10 +2733,12 @@ static int io_submit_sqes(struct io_ring_ctx *ctx, unsigned int nr,
>  		}
>  
>  out:
> +		s.ring_file = ring_file;
> +		s.ring_fd = ring_fd;
>  		s.has_user = *mm != NULL;
> -		s.in_async = true;
> -		s.needs_fixed_file = true;
> -		trace_io_uring_submit_sqe(ctx, s.sqe->user_data, true, true);
> +		s.in_async = async;
> +		s.needs_fixed_file = async;
> +		trace_io_uring_submit_sqe(ctx, s.sqe->user_data, true, async);
>  		io_submit_sqe(ctx, &s, statep, &link);
>  		submitted++;
>  	}
> @@ -2745,6 +2748,9 @@ static int io_submit_sqes(struct io_ring_ctx *ctx, unsigned int nr,
>  	if (statep)
>  		io_submit_state_end(&state);
>  
> +	 /* Commit SQ ring head once we've consumed and submitted all SQEs */
> +	io_commit_sqring(ctx);
> +

Then don't need io_commit_sqring() in io_sq_thread any more?

Anyway, looks good to me.
Reviewed-by: Bob Liu <bob.liu@oracle.com>

>  	return submitted;
>  }
>  
> @@ -2849,7 +2855,8 @@ static int io_sq_thread(void *data)
>  		}
>  
>  		to_submit = min(to_submit, ctx->sq_entries);
> -		inflight += io_submit_sqes(ctx, to_submit, &cur_mm);
> +		inflight += io_submit_sqes(ctx, to_submit, NULL, -1, &cur_mm,
> +					   true);
>  
>  		/* Commit SQ ring head once we've consumed all SQEs */
>  		io_commit_sqring(ctx);
> @@ -2866,69 +2873,6 @@ static int io_sq_thread(void *data)
>  	return 0;
>  }
>  
> -static int io_ring_submit(struct io_ring_ctx *ctx, unsigned int to_submit,
> -			  struct file *ring_file, int ring_fd)
> -{
> -	struct io_submit_state state, *statep = NULL;
> -	struct io_kiocb *link = NULL;
> -	struct io_kiocb *shadow_req = NULL;
> -	bool prev_was_link = false;
> -	int i, submit = 0;
> -
> -	if (to_submit > IO_PLUG_THRESHOLD) {
> -		io_submit_state_start(&state, ctx, to_submit);
> -		statep = &state;
> -	}
> -
> -	for (i = 0; i < to_submit; i++) {
> -		struct sqe_submit s;
> -
> -		if (!io_get_sqring(ctx, &s))
> -			break;
> -
> -		/*
> -		 * If previous wasn't linked and we have a linked command,
> -		 * that's the end of the chain. Submit the previous link.
> -		 */
> -		if (!prev_was_link && link) {
> -			io_queue_link_head(ctx, link, &link->submit, shadow_req);
> -			link = NULL;
> -			shadow_req = NULL;
> -		}
> -		prev_was_link = (s.sqe->flags & IOSQE_IO_LINK) != 0;
> -
> -		if (link && (s.sqe->flags & IOSQE_IO_DRAIN)) {
> -			if (!shadow_req) {
> -				shadow_req = io_get_req(ctx, NULL);
> -				if (unlikely(!shadow_req))
> -					goto out;
> -				shadow_req->flags |= (REQ_F_IO_DRAIN | REQ_F_SHADOW_DRAIN);
> -				refcount_dec(&shadow_req->refs);
> -			}
> -			shadow_req->sequence = s.sequence;
> -		}
> -
> -out:
> -		s.ring_file = ring_file;
> -		s.has_user = true;
> -		s.in_async = false;
> -		s.needs_fixed_file = false;
> -		s.ring_fd = ring_fd;
> -		submit++;
> -		trace_io_uring_submit_sqe(ctx, s.sqe->user_data, true, false);
> -		io_submit_sqe(ctx, &s, statep, &link);
> -	}
> -
> -	if (link)
> -		io_queue_link_head(ctx, link, &link->submit, shadow_req);
> -	if (statep)
> -		io_submit_state_end(statep);
> -
> -	io_commit_sqring(ctx);
> -
> -	return submit;
> -}
> -
>  struct io_wait_queue {
>  	struct wait_queue_entry wq;
>  	struct io_ring_ctx *ctx;
> @@ -4049,10 +3993,14 @@ SYSCALL_DEFINE6(io_uring_enter, unsigned int, fd, u32, to_submit,
>  			wake_up(&ctx->sqo_wait);
>  		submitted = to_submit;
>  	} else if (to_submit) {
> -		to_submit = min(to_submit, ctx->sq_entries);
> +		struct mm_struct *cur_mm;
>  
> +		to_submit = min(to_submit, ctx->sq_entries);
>  		mutex_lock(&ctx->uring_lock);
> -		submitted = io_ring_submit(ctx, to_submit, f.file, fd);
> +		/* already have mm, so io_submit_sqes() won't try to grab it */
> +		cur_mm = ctx->sqo_mm;
> +		submitted = io_submit_sqes(ctx, to_submit, f.file, fd,
> +					   &cur_mm, false);
>  		mutex_unlock(&ctx->uring_lock);
>  	}
>  	if (flags & IORING_ENTER_GETEVENTS) {
> 

Re: [PATCH v2 2/2] io_uring: io_queue_link*() right after submit

[Pavel Begunkov]

On 11/6/2019 11:36 AM, Bob Liu wrote:
> On 11/6/19 5:22 AM, Pavel Begunkov wrote:
>> After a call to io_submit_sqe(), it's already known whether it needs
>> to queue a link or not. Do it there, as it's simplier and doesn't keep
>> an extra variable across the loop.
>>
>> Signed-off-by: Pavel Begunkov <asml.silence@gmail.com>
>> ---
>>  fs/io_uring.c | 22 ++++++++++------------
>>  1 file changed, 10 insertions(+), 12 deletions(-)
>>
>> diff --git a/fs/io_uring.c b/fs/io_uring.c
>> index ebe2a4edd644..82c2da99cb5c 100644
>> --- a/fs/io_uring.c
>> +++ b/fs/io_uring.c
>> @@ -2687,7 +2687,6 @@ static int io_submit_sqes(struct io_ring_ctx *ctx, unsigned int nr,
>>  	struct io_submit_state state, *statep = NULL;
>>  	struct io_kiocb *link = NULL;
>>  	struct io_kiocb *shadow_req = NULL;
>> -	bool prev_was_link = false;
>>  	int i, submitted = 0;
>>  	bool mm_fault = false;
>>  
>> @@ -2710,17 +2709,6 @@ static int io_submit_sqes(struct io_ring_ctx *ctx, unsigned int nr,
>>  			}
>>  		}
>>  
>> -		/*
>> -		 * If previous wasn't linked and we have a linked command,
>> -		 * that's the end of the chain. Submit the previous link.
>> -		 */
>> -		if (!prev_was_link && link) {
>> -			io_queue_link_head(ctx, link, &link->submit, shadow_req);
>> -			link = NULL;
>> -			shadow_req = NULL;
>> -		}
>> -		prev_was_link = (s.sqe->flags & IOSQE_IO_LINK) != 0;
>> -
>>  		if (link && (s.sqe->flags & IOSQE_IO_DRAIN)) {
>>  			if (!shadow_req) {
>>  				shadow_req = io_get_req(ctx, NULL);
>> @@ -2741,6 +2729,16 @@ static int io_submit_sqes(struct io_ring_ctx *ctx, unsigned int nr,
>>  		trace_io_uring_submit_sqe(ctx, s.sqe->user_data, true, async);
>>  		io_submit_sqe(ctx, &s, statep, &link);
>>  		submitted++;
>> +
>> +		/*
>> +		 * If previous wasn't linked and we have a linked command,
>> +		 * that's the end of the chain. Submit the previous link.
>> +		 */
>> +		if (!(s.sqe->flags & IOSQE_IO_LINK) && link) 
> The behavior changed to 'current seq' instead of previous after dropping prev_was_link?
> 
The old behaviour was to remember @prev_was_link for current sqe, and
use at the beginning of the next iteration, where it becomes
"previous/last sqe". See, prev_was_link was set after io_queue_link_head.

If @i is iteration idx, then timeline was:
i:   sqe[i-1].is_link -> io_queue_link_head() # if (prev_was_link)
i:   sqe[i].is_link = prev_was_link = (sqe[i].flags & LINK)
i+1: sqe[i].is_link -> io_queue_link_head() # if (prev_was_link)
i+1: sqe[i+1].is_link = ...


After the change, it's done at the same loop iteration by swapping order
of checking @prev_was_link and io_queue_link_head().

i:   sqe[i].is_link = ...
i:   sqe[i].is_link -> io_queue_link_head()
i+1: sqe[i+1].is_link = ...
i+1: sqe[i+1].is_link -> io_queue_link_head()

Shouldn't change the behavior, if I'm not missing something.


>> +			io_queue_link_head(ctx, link, &link->submit, shadow_req);
>> +			link = NULL;
>> +			shadow_req = NULL;
>> +		}
>>  	}
>>  
>>  	if (link)
>>
> 

Re: [PATCH v2 1/2] io_uring: Merge io_submit_sqes and io_ring_submit

[Pavel Begunkov]

On 11/6/2019 11:57 AM, Bob Liu wrote:
> On 11/6/19 5:22 AM, Pavel Begunkov wrote:
>> io_submit_sqes() and io_ring_submit() are doing the same stuff with
>> a little difference. Deduplicate them.
>>
>> Signed-off-by: Pavel Begunkov <asml.silence@gmail.com>
>> ---
>>  fs/io_uring.c | 88 +++++++++++----------------------------------------
>>  1 file changed, 18 insertions(+), 70 deletions(-)
>>
>> diff --git a/fs/io_uring.c b/fs/io_uring.c
>> index 7813bc7d5b61..ebe2a4edd644 100644
>> --- a/fs/io_uring.c
>> +++ b/fs/io_uring.c
>> @@ -2681,7 +2681,8 @@ static bool io_get_sqring(struct io_ring_ctx *ctx, struct sqe_submit *s)
>>  }
>>  
>>  static int io_submit_sqes(struct io_ring_ctx *ctx, unsigned int nr,
>> -			  struct mm_struct **mm)
>> +			  struct file *ring_file, int ring_fd,
>> +			  struct mm_struct **mm, bool async)
>>  {
>>  	struct io_submit_state state, *statep = NULL;
>>  	struct io_kiocb *link = NULL;
>> @@ -2732,10 +2733,12 @@ static int io_submit_sqes(struct io_ring_ctx *ctx, unsigned int nr,
>>  		}
>>  
>>  out:
>> +		s.ring_file = ring_file;
>> +		s.ring_fd = ring_fd;
>>  		s.has_user = *mm != NULL;
>> -		s.in_async = true;
>> -		s.needs_fixed_file = true;
>> -		trace_io_uring_submit_sqe(ctx, s.sqe->user_data, true, true);
>> +		s.in_async = async;
>> +		s.needs_fixed_file = async;
>> +		trace_io_uring_submit_sqe(ctx, s.sqe->user_data, true, async);
>>  		io_submit_sqe(ctx, &s, statep, &link);
>>  		submitted++;
>>  	}
>> @@ -2745,6 +2748,9 @@ static int io_submit_sqes(struct io_ring_ctx *ctx, unsigned int nr,
>>  	if (statep)
>>  		io_submit_state_end(&state);
>>  
>> +	 /* Commit SQ ring head once we've consumed and submitted all SQEs */
>> +	io_commit_sqring(ctx);
>> +
> 
> Then don't need io_commit_sqring() in io_sq_thread any more?
>
Right, thanks! I'll resend with the change.


> Anyway, looks good to me.
> Reviewed-by: Bob Liu <bob.liu@oracle.com>
> 
>>  	return submitted;
>>  }
>>  
>> @@ -2849,7 +2855,8 @@ static int io_sq_thread(void *data)
>>  		}
>>  
>>  		to_submit = min(to_submit, ctx->sq_entries);
>> -		inflight += io_submit_sqes(ctx, to_submit, &cur_mm);
>> +		inflight += io_submit_sqes(ctx, to_submit, NULL, -1, &cur_mm,
>> +					   true);
>>  
>>  		/* Commit SQ ring head once we've consumed all SQEs */
>>  		io_commit_sqring(ctx);
>> @@ -2866,69 +2873,6 @@ static int io_sq_thread(void *data)
>>  	return 0;
>>  }
>>  
>> -static int io_ring_submit(struct io_ring_ctx *ctx, unsigned int to_submit,
>> -			  struct file *ring_file, int ring_fd)
>> -{
>> -	struct io_submit_state state, *statep = NULL;
>> -	struct io_kiocb *link = NULL;
>> -	struct io_kiocb *shadow_req = NULL;
>> -	bool prev_was_link = false;
>> -	int i, submit = 0;
>> -
>> -	if (to_submit > IO_PLUG_THRESHOLD) {
>> -		io_submit_state_start(&state, ctx, to_submit);
>> -		statep = &state;
>> -	}
>> -
>> -	for (i = 0; i < to_submit; i++) {
>> -		struct sqe_submit s;
>> -
>> -		if (!io_get_sqring(ctx, &s))
>> -			break;
>> -
>> -		/*
>> -		 * If previous wasn't linked and we have a linked command,
>> -		 * that's the end of the chain. Submit the previous link.
>> -		 */
>> -		if (!prev_was_link && link) {
>> -			io_queue_link_head(ctx, link, &link->submit, shadow_req);
>> -			link = NULL;
>> -			shadow_req = NULL;
>> -		}
>> -		prev_was_link = (s.sqe->flags & IOSQE_IO_LINK) != 0;
>> -
>> -		if (link && (s.sqe->flags & IOSQE_IO_DRAIN)) {
>> -			if (!shadow_req) {
>> -				shadow_req = io_get_req(ctx, NULL);
>> -				if (unlikely(!shadow_req))
>> -					goto out;
>> -				shadow_req->flags |= (REQ_F_IO_DRAIN | REQ_F_SHADOW_DRAIN);
>> -				refcount_dec(&shadow_req->refs);
>> -			}
>> -			shadow_req->sequence = s.sequence;
>> -		}
>> -
>> -out:
>> -		s.ring_file = ring_file;
>> -		s.has_user = true;
>> -		s.in_async = false;
>> -		s.needs_fixed_file = false;
>> -		s.ring_fd = ring_fd;
>> -		submit++;
>> -		trace_io_uring_submit_sqe(ctx, s.sqe->user_data, true, false);
>> -		io_submit_sqe(ctx, &s, statep, &link);
>> -	}
>> -
>> -	if (link)
>> -		io_queue_link_head(ctx, link, &link->submit, shadow_req);
>> -	if (statep)
>> -		io_submit_state_end(statep);
>> -
>> -	io_commit_sqring(ctx);
>> -
>> -	return submit;
>> -}
>> -
>>  struct io_wait_queue {
>>  	struct wait_queue_entry wq;
>>  	struct io_ring_ctx *ctx;
>> @@ -4049,10 +3993,14 @@ SYSCALL_DEFINE6(io_uring_enter, unsigned int, fd, u32, to_submit,
>>  			wake_up(&ctx->sqo_wait);
>>  		submitted = to_submit;
>>  	} else if (to_submit) {
>> -		to_submit = min(to_submit, ctx->sq_entries);
>> +		struct mm_struct *cur_mm;
>>  
>> +		to_submit = min(to_submit, ctx->sq_entries);
>>  		mutex_lock(&ctx->uring_lock);
>> -		submitted = io_ring_submit(ctx, to_submit, f.file, fd);
>> +		/* already have mm, so io_submit_sqes() won't try to grab it */
>> +		cur_mm = ctx->sqo_mm;
>> +		submitted = io_submit_sqes(ctx, to_submit, f.file, fd,
>> +					   &cur_mm, false);
>>  		mutex_unlock(&ctx->uring_lock);
>>  	}
>>  	if (flags & IORING_ENTER_GETEVENTS) {
>>
> 

Re: [PATCH v2 2/2] io_uring: io_queue_link*() right after submit

[Pavel Begunkov]

On 11/6/2019 12:06 PM, Pavel Begunkov wrote:
> On 11/6/2019 11:36 AM, Bob Liu wrote:
>> On 11/6/19 5:22 AM, Pavel Begunkov wrote:
>>> After a call to io_submit_sqe(), it's already known whether it needs
>>> to queue a link or not. Do it there, as it's simplier and doesn't keep
>>> an extra variable across the loop.
>>>
>>> Signed-off-by: Pavel Begunkov <asml.silence@gmail.com>
>>> ---
>>>  fs/io_uring.c | 22 ++++++++++------------
>>>  1 file changed, 10 insertions(+), 12 deletions(-)
>>>
>>> diff --git a/fs/io_uring.c b/fs/io_uring.c
>>> index ebe2a4edd644..82c2da99cb5c 100644
>>> --- a/fs/io_uring.c
>>> +++ b/fs/io_uring.c
>>> @@ -2687,7 +2687,6 @@ static int io_submit_sqes(struct io_ring_ctx *ctx, unsigned int nr,
>>>  	struct io_submit_state state, *statep = NULL;
>>>  	struct io_kiocb *link = NULL;
>>>  	struct io_kiocb *shadow_req = NULL;
>>> -	bool prev_was_link = false;
>>>  	int i, submitted = 0;
>>>  	bool mm_fault = false;
>>>  
>>> @@ -2710,17 +2709,6 @@ static int io_submit_sqes(struct io_ring_ctx *ctx, unsigned int nr,
>>>  			}
>>>  		}
>>>  
>>> -		/*
>>> -		 * If previous wasn't linked and we have a linked command,
>>> -		 * that's the end of the chain. Submit the previous link.
>>> -		 */
>>> -		if (!prev_was_link && link) {
>>> -			io_queue_link_head(ctx, link, &link->submit, shadow_req);
>>> -			link = NULL;
>>> -			shadow_req = NULL;
>>> -		}
>>> -		prev_was_link = (s.sqe->flags & IOSQE_IO_LINK) != 0;
>>> -
>>>  		if (link && (s.sqe->flags & IOSQE_IO_DRAIN)) {
>>>  			if (!shadow_req) {
>>>  				shadow_req = io_get_req(ctx, NULL);
>>> @@ -2741,6 +2729,16 @@ static int io_submit_sqes(struct io_ring_ctx *ctx, unsigned int nr,
>>>  		trace_io_uring_submit_sqe(ctx, s.sqe->user_data, true, async);
>>>  		io_submit_sqe(ctx, &s, statep, &link);
>>>  		submitted++;
>>> +
>>> +		/*
>>> +		 * If previous wasn't linked and we have a linked command,
>>> +		 * that's the end of the chain. Submit the previous link.
>>> +		 */
>>> +		if (!(s.sqe->flags & IOSQE_IO_LINK) && link) 
>> The behavior changed to 'current seq' instead of previous after dropping prev_was_link?
>>
> The old behaviour was to remember @prev_was_link for current sqe, and
> use at the beginning of the next iteration, where it becomes
> "previous/last sqe". See, prev_was_link was set after io_queue_link_head.
> 
> If @i is iteration idx, then timeline was:
> i:   sqe[i-1].is_link -> io_queue_link_head() # if (prev_was_link)
> i:   sqe[i].is_link = prev_was_link = (sqe[i].flags & LINK)
> i+1: sqe[i].is_link -> io_queue_link_head() # if (prev_was_link)
> i+1: sqe[i+1].is_link = ...
> 
> 
> After the change, it's done at the same loop iteration by swapping order
> of checking @prev_was_link and io_queue_link_head().
> 
> i:   sqe[i].is_link = ...
> i:   sqe[i].is_link -> io_queue_link_head()
> i+1: sqe[i+1].is_link = ...
> i+1: sqe[i+1].is_link -> io_queue_link_head()
> 
> Shouldn't change the behavior, if I'm not missing something.
> 
And the same goes for ordering with io_submit_sqe(), which assembles a link.

i:   prev_was_link = ... # for sqe[i]
i:   io_submit_sqe() # for sqe[i]
i+1: prev_was_link -> io_queue_link_head # for sqe[i]

after:
i:   io_submit_sqe() # for sqe[i]
i:   is_link = ... # for sqe[i]
i:   is_link -> io_queue_link_head # for sqe[i]

> 
>>> +			io_queue_link_head(ctx, link, &link->submit, shadow_req);
>>> +			link = NULL;
>>> +			shadow_req = NULL;
>>> +		}
>>>  	}
>>>  
>>>  	if (link)
>>>
>>

Re: [PATCH v2 2/2] io_uring: io_queue_link*() right after submit

[Bob Liu]

On 11/6/19 5:31 PM, Pavel Begunkov wrote:
> On 11/6/2019 12:06 PM, Pavel Begunkov wrote:
>> On 11/6/2019 11:36 AM, Bob Liu wrote:
>>> On 11/6/19 5:22 AM, Pavel Begunkov wrote:
>>>> After a call to io_submit_sqe(), it's already known whether it needs
>>>> to queue a link or not. Do it there, as it's simplier and doesn't keep
>>>> an extra variable across the loop.
>>>>
>>>> Signed-off-by: Pavel Begunkov <asml.silence@gmail.com>
>>>> ---
>>>>  fs/io_uring.c | 22 ++++++++++------------
>>>>  1 file changed, 10 insertions(+), 12 deletions(-)
>>>>
>>>> diff --git a/fs/io_uring.c b/fs/io_uring.c
>>>> index ebe2a4edd644..82c2da99cb5c 100644
>>>> --- a/fs/io_uring.c
>>>> +++ b/fs/io_uring.c
>>>> @@ -2687,7 +2687,6 @@ static int io_submit_sqes(struct io_ring_ctx *ctx, unsigned int nr,
>>>>  	struct io_submit_state state, *statep = NULL;
>>>>  	struct io_kiocb *link = NULL;
>>>>  	struct io_kiocb *shadow_req = NULL;
>>>> -	bool prev_was_link = false;
>>>>  	int i, submitted = 0;
>>>>  	bool mm_fault = false;
>>>>  
>>>> @@ -2710,17 +2709,6 @@ static int io_submit_sqes(struct io_ring_ctx *ctx, unsigned int nr,
>>>>  			}
>>>>  		}
>>>>  
>>>> -		/*
>>>> -		 * If previous wasn't linked and we have a linked command,
>>>> -		 * that's the end of the chain. Submit the previous link.
>>>> -		 */
>>>> -		if (!prev_was_link && link) {
>>>> -			io_queue_link_head(ctx, link, &link->submit, shadow_req);
>>>> -			link = NULL;
>>>> -			shadow_req = NULL;
>>>> -		}
>>>> -		prev_was_link = (s.sqe->flags & IOSQE_IO_LINK) != 0;
>>>> -
>>>>  		if (link && (s.sqe->flags & IOSQE_IO_DRAIN)) {
>>>>  			if (!shadow_req) {
>>>>  				shadow_req = io_get_req(ctx, NULL);
>>>> @@ -2741,6 +2729,16 @@ static int io_submit_sqes(struct io_ring_ctx *ctx, unsigned int nr,
>>>>  		trace_io_uring_submit_sqe(ctx, s.sqe->user_data, true, async);
>>>>  		io_submit_sqe(ctx, &s, statep, &link);
>>>>  		submitted++;
>>>> +
>>>> +		/*
>>>> +		 * If previous wasn't linked and we have a linked command,
>>>> +		 * that's the end of the chain. Submit the previous link.
>>>> +		 */
>>>> +		if (!(s.sqe->flags & IOSQE_IO_LINK) && link) 
>>> The behavior changed to 'current seq' instead of previous after dropping prev_was_link?
>>>
>> The old behaviour was to remember @prev_was_link for current sqe, and
>> use at the beginning of the next iteration, where it becomes
>> "previous/last sqe". See, prev_was_link was set after io_queue_link_head.
>>
>> If @i is iteration idx, then timeline was:
>> i:   sqe[i-1].is_link -> io_queue_link_head() # if (prev_was_link)
>> i:   sqe[i].is_link = prev_was_link = (sqe[i].flags & LINK)
>> i+1: sqe[i].is_link -> io_queue_link_head() # if (prev_was_link)
>> i+1: sqe[i+1].is_link = ...
>>
>>
>> After the change, it's done at the same loop iteration by swapping order
>> of checking @prev_was_link and io_queue_link_head().
>>
>> i:   sqe[i].is_link = ...
>> i:   sqe[i].is_link -> io_queue_link_head()
>> i+1: sqe[i+1].is_link = ...
>> i+1: sqe[i+1].is_link -> io_queue_link_head()
>>
>> Shouldn't change the behavior, if I'm not missing something.
>>
> And the same goes for ordering with io_submit_sqe(), which assembles a link.
> 
> i:   prev_was_link = ... # for sqe[i]
> i:   io_submit_sqe() # for sqe[i]
> i+1: prev_was_link -> io_queue_link_head # for sqe[i]
> 
> after:
> i:   io_submit_sqe() # for sqe[i]
> i:   is_link = ... # for sqe[i]
> i:   is_link -> io_queue_link_head # for sqe[i]
> 

I see, sorry for the noise.
Reviewed-by： Bob Liu <bob.liu@oracle.com>

>>
>>>> +			io_queue_link_head(ctx, link, &link->submit, shadow_req);
>>>> +			link = NULL;
>>>> +			shadow_req = NULL;
>>>> +		}
>>>>  	}
>>>>  
>>>>  	if (link)
>>>>
>>>

Re: [PATCH v2 0/2] cleanup of submission path

[Jens Axboe]

On 11/5/19 2:22 PM, Pavel Begunkov wrote:
> A minor cleanup of io_submit_sqes() and io_ring_submit().
> 
> v2: rebased
>      move io_queue_link_head()
> 
> Pavel Begunkov (2):
>    io_uring: Merge io_submit_sqes and io_ring_submit
>    io_uring: io_queue_link*() right after submit
> 
>   fs/io_uring.c | 110 +++++++++++++-------------------------------------
>   1 file changed, 28 insertions(+), 82 deletions(-)

Applied this series, thanks Pavel.

-- 
Jens Axboe