[PATCH] fuse: add zero-copy to fuse-over-io_uring

[Xiaobing Li <xiaobing.li@samsung.com>]

Joanne has submitted a patch for adding a zero-copy solution.

We have also done some research and testing before, and
the test data shows improved performance. This patch is
submitted for discussion.

libfuse section:
https://github.com/lreeze123/libfuse/tree/zero-copy

Signed-off-by: Xiaobing Li <xiaobing.li@samsung.com>
---
 fs/fuse/dev_uring.c   | 44 +++++++++++++++++++++++++++++++------------
 fs/fuse/dev_uring_i.h |  4 ++++
 2 files changed, 36 insertions(+), 12 deletions(-)

diff --git a/fs/fuse/dev_uring.c b/fs/fuse/dev_uring.c
index f6b12aebb8bb..23790ae78853 100644
--- a/fs/fuse/dev_uring.c
+++ b/fs/fuse/dev_uring.c
@@ -584,15 +584,20 @@ static int fuse_uring_copy_from_ring(struct fuse_ring *ring,
 	int err;
 	struct fuse_uring_ent_in_out ring_in_out;
 
-	err = copy_from_user(&ring_in_out, &ent->headers->ring_ent_in_out,
-			     sizeof(ring_in_out));
-	if (err)
-		return -EFAULT;
+	if (ent->zero_copy) {
+		iter = ent->payload_iter;
+		ring_in_out.payload_sz = ent->cmd->sqe->len;
+	} else {
+		err = copy_from_user(&ring_in_out, &ent->headers->ring_ent_in_out,
+					sizeof(ring_in_out));
+		if (err)
+			return -EFAULT;
 
-	err = import_ubuf(ITER_SOURCE, ent->payload, ring->max_payload_sz,
-			  &iter);
-	if (err)
-		return err;
+		err = import_ubuf(ITER_SOURCE, ent->payload, ring->max_payload_sz,
+				 &iter);
+		if (err)
+			return err;
+	}
 
 	fuse_copy_init(&cs, false, &iter);
 	cs.is_uring = true;
@@ -618,10 +623,14 @@ static int fuse_uring_args_to_ring(struct fuse_ring *ring, struct fuse_req *req,
 		.commit_id = req->in.h.unique,
 	};
 
-	err = import_ubuf(ITER_DEST, ent->payload, ring->max_payload_sz, &iter);
-	if (err) {
-		pr_info_ratelimited("fuse: Import of user buffer failed\n");
-		return err;
+	if (ent->zero_copy) {
+		iter = ent->payload_iter;
+	} else {
+		err = import_ubuf(ITER_DEST, ent->payload, ring->max_payload_sz, &iter);
+		if (err) {
+			pr_info_ratelimited("fuse: Import of user buffer failed\n");
+			return err;
+		}
 	}
 
 	fuse_copy_init(&cs, true, &iter);
@@ -1068,6 +1077,17 @@ fuse_uring_create_ring_ent(struct io_uring_cmd *cmd,
 	ent->headers = iov[0].iov_base;
 	ent->payload = iov[1].iov_base;
 
+	if (READ_ONCE(cmd->sqe->uring_cmd_flags) & IORING_URING_CMD_FIXED) {
+		ent->zero_copy = true;
+		err = io_uring_cmd_import_fixed((u64)ent->payload, payload_size, ITER_DEST,
+						&ent->payload_iter, cmd, 0);
+
+		if (err) {
+			kfree(ent);
+			return ERR_PTR(err);
+		}
+	}
+
 	atomic_inc(&ring->queue_refs);
 	return ent;
 }
diff --git a/fs/fuse/dev_uring_i.h b/fs/fuse/dev_uring_i.h
index 51a563922ce1..cb5de6e7a262 100644
--- a/fs/fuse/dev_uring_i.h
+++ b/fs/fuse/dev_uring_i.h
@@ -38,6 +38,10 @@ enum fuse_ring_req_state {
 
 /** A fuse ring entry, part of the ring queue */
 struct fuse_ring_ent {
+	bool zero_copy;
+
+	struct iov_iter payload_iter;
+
 	/* userspace buffer */
 	struct fuse_uring_req_header __user *headers;
 	void __user *payload;
-- 
2.34.1