[PATCH v1 2/2] fuse: support io-uring registered buffers

[Joanne Koong <joannelkoong@gmail.com>]

Add support for io-uring registered buffers for fuse daemons
communicating through the io-uring interface. Daemons may register
buffers ahead of time, which will eliminate the overhead of
pinning/unpinning user pages and translating virtual addresses for every
server-kernel interaction.

To support page-aligned payloads, the buffer is structured such that the
payload is at the front of the buffer and the fuse_uring_req_header is
offset from the end of the buffer.

To be backwards compatible, fuse uring still needs to support non-registered
buffers as well.

Signed-off-by: Joanne Koong <joannelkoong@gmail.com>
---
 fs/fuse/dev_uring.c   | 216 ++++++++++++++++++++++++++++++++++++++----
 fs/fuse/dev_uring_i.h |  17 +++-
 2 files changed, 213 insertions(+), 20 deletions(-)

diff --git a/fs/fuse/dev_uring.c b/fs/fuse/dev_uring.c
index f6b12aebb8bb..c4dd4d168b61 100644
--- a/fs/fuse/dev_uring.c
+++ b/fs/fuse/dev_uring.c
@@ -574,6 +574,37 @@ static int fuse_uring_out_header_has_err(struct fuse_out_header *oh,
 	return err;
 }
 
+static int fuse_uring_copy_from_ring_fixed_buf(struct fuse_ring *ring,
+					       struct fuse_req *req,
+					       struct fuse_ring_ent *ent)
+{
+	struct fuse_copy_state cs;
+	struct fuse_args *args = req->args;
+	struct iov_iter payload_iter;
+	struct iov_iter headers_iter;
+	struct fuse_uring_ent_in_out ring_in_out;
+	size_t copied;
+
+	payload_iter = ent->fixed_buffer.payload_iter;
+	payload_iter.data_source = ITER_SOURCE;
+	headers_iter = ent->fixed_buffer.headers_iter;
+	headers_iter.data_source = ITER_SOURCE;
+
+	iov_iter_advance(&headers_iter, offsetof(struct fuse_uring_req_header,
+						 ring_ent_in_out));
+
+	copied = copy_from_iter(&ring_in_out, sizeof(ring_in_out),
+				&headers_iter);
+	if (copied != sizeof(ring_in_out))
+		return -EFAULT;
+
+	fuse_copy_init(&cs, false, &payload_iter);
+	cs.is_uring = true;
+	cs.req = req;
+
+	return fuse_copy_out_args(&cs, args, ring_in_out.payload_sz);
+}
+
 static int fuse_uring_copy_from_ring(struct fuse_ring *ring,
 				     struct fuse_req *req,
 				     struct fuse_ring_ent *ent)
@@ -584,12 +615,12 @@ static int fuse_uring_copy_from_ring(struct fuse_ring *ring,
 	int err;
 	struct fuse_uring_ent_in_out ring_in_out;
 
-	err = copy_from_user(&ring_in_out, &ent->headers->ring_ent_in_out,
+	err = copy_from_user(&ring_in_out, &ent->user.headers->ring_ent_in_out,
 			     sizeof(ring_in_out));
 	if (err)
 		return -EFAULT;
 
-	err = import_ubuf(ITER_SOURCE, ent->payload, ring->max_payload_sz,
+	err = import_ubuf(ITER_SOURCE, ent->user.payload, ring->max_payload_sz,
 			  &iter);
 	if (err)
 		return err;
@@ -601,6 +632,79 @@ static int fuse_uring_copy_from_ring(struct fuse_ring *ring,
 	return fuse_copy_out_args(&cs, args, ring_in_out.payload_sz);
 }
 
+static int fuse_uring_args_to_ring_fixed_buf(struct fuse_ring *ring,
+					     struct fuse_req *req,
+					     struct fuse_ring_ent *ent)
+{
+	struct fuse_copy_state cs;
+	struct fuse_args *args = req->args;
+	struct fuse_in_arg *in_args = args->in_args;
+	int num_args = args->in_numargs;
+	struct iov_iter payload_iter;
+	struct iov_iter headers_iter;
+	struct fuse_uring_ent_in_out ent_in_out = {
+		.flags = 0,
+		.commit_id = req->in.h.unique,
+	};
+	size_t copied;
+	bool advanced_headers = false;
+	int err;
+
+	payload_iter = ent->fixed_buffer.payload_iter;
+	payload_iter.data_source = ITER_DEST;
+
+	headers_iter = ent->fixed_buffer.headers_iter;
+	headers_iter.data_source = ITER_DEST;
+
+	fuse_copy_init(&cs, true, &payload_iter);
+	cs.is_uring = true;
+	cs.req = req;
+
+	if (num_args > 0) {
+		/*
+		 * Expectation is that the first argument is the per op header.
+		 * Some op code have that as zero size.
+		 */
+		if (args->in_args[0].size > 0) {
+			iov_iter_advance(&headers_iter,
+					 offsetof(struct fuse_uring_req_header,
+						  op_in));
+			copied = copy_to_iter(in_args->value, in_args->size,
+					      &headers_iter);
+			if (copied != in_args->size) {
+				pr_info_ratelimited(
+					"Copying the header failed.\n");
+				return -EFAULT;
+			}
+
+			iov_iter_advance(&headers_iter,
+					 FUSE_URING_OP_IN_OUT_SZ - in_args->size);
+			advanced_headers = true;
+		}
+		in_args++;
+		num_args--;
+	}
+	if (!advanced_headers)
+		iov_iter_advance(&headers_iter,
+				 offsetof(struct fuse_uring_req_header,
+					  ring_ent_in_out));
+
+	/* copy the payload */
+	err = fuse_copy_args(&cs, num_args, args->in_pages,
+			     (struct fuse_arg *)in_args, 0);
+	if (err) {
+		pr_info_ratelimited("%s fuse_copy_args failed\n", __func__);
+		return err;
+	}
+
+	ent_in_out.payload_sz = cs.ring.copied_sz;
+	copied = copy_to_iter(&ent_in_out, sizeof(ent_in_out), &headers_iter);
+	if (copied != sizeof(ent_in_out))
+		return -EFAULT;
+
+	return 0;
+}
+
  /*
   * Copy data from the req to the ring buffer
   */
@@ -618,7 +722,7 @@ static int fuse_uring_args_to_ring(struct fuse_ring *ring, struct fuse_req *req,
 		.commit_id = req->in.h.unique,
 	};
 
-	err = import_ubuf(ITER_DEST, ent->payload, ring->max_payload_sz, &iter);
+	err = import_ubuf(ITER_DEST, ent->user.payload, ring->max_payload_sz, &iter);
 	if (err) {
 		pr_info_ratelimited("fuse: Import of user buffer failed\n");
 		return err;
@@ -634,7 +738,7 @@ static int fuse_uring_args_to_ring(struct fuse_ring *ring, struct fuse_req *req,
 		 * Some op code have that as zero size.
 		 */
 		if (args->in_args[0].size > 0) {
-			err = copy_to_user(&ent->headers->op_in, in_args->value,
+			err = copy_to_user(&ent->user.headers->op_in, in_args->value,
 					   in_args->size);
 			if (err) {
 				pr_info_ratelimited(
@@ -655,7 +759,7 @@ static int fuse_uring_args_to_ring(struct fuse_ring *ring, struct fuse_req *req,
 	}
 
 	ent_in_out.payload_sz = cs.ring.copied_sz;
-	err = copy_to_user(&ent->headers->ring_ent_in_out, &ent_in_out,
+	err = copy_to_user(&ent->user.headers->ring_ent_in_out, &ent_in_out,
 			   sizeof(ent_in_out));
 	return err ? -EFAULT : 0;
 }
@@ -679,18 +783,31 @@ static int fuse_uring_copy_to_ring(struct fuse_ring_ent *ent,
 		return err;
 
 	/* copy the request */
-	err = fuse_uring_args_to_ring(ring, req, ent);
+	if (ent->is_fixed_buffer)
+		err = fuse_uring_args_to_ring_fixed_buf(ring, req, ent);
+	else
+		err = fuse_uring_args_to_ring(ring, req, ent);
 	if (unlikely(err)) {
 		pr_info_ratelimited("Copy to ring failed: %d\n", err);
 		return err;
 	}
 
 	/* copy fuse_in_header */
-	err = copy_to_user(&ent->headers->in_out, &req->in.h,
-			   sizeof(req->in.h));
-	if (err) {
-		err = -EFAULT;
-		return err;
+	if (ent->is_fixed_buffer) {
+		struct iov_iter headers_iter = ent->fixed_buffer.headers_iter;
+		size_t copied;
+
+		headers_iter.data_source = ITER_DEST;
+		copied = copy_to_iter(&req->in.h, sizeof(req->in.h),
+				      &headers_iter);
+
+		if (copied != sizeof(req->in.h))
+			return -EFAULT;
+	} else {
+		err = copy_to_user(&ent->user.headers->in_out, &req->in.h,
+				   sizeof(req->in.h));
+		if (err)
+			return -EFAULT;
 	}
 
 	return 0;
@@ -815,8 +932,18 @@ static void fuse_uring_commit(struct fuse_ring_ent *ent, struct fuse_req *req,
 	struct fuse_conn *fc = ring->fc;
 	ssize_t err = 0;
 
-	err = copy_from_user(&req->out.h, &ent->headers->in_out,
-			     sizeof(req->out.h));
+	if (ent->is_fixed_buffer) {
+		struct iov_iter headers_iter = ent->fixed_buffer.headers_iter;
+		size_t copied;
+
+		headers_iter.data_source = ITER_SOURCE;
+		copied = copy_from_iter(&req->out.h, sizeof(req->out.h), &headers_iter);
+		if (copied != sizeof(req->out.h))
+			err = -EFAULT;
+	} else {
+		err = copy_from_user(&req->out.h, &ent->user.headers->in_out,
+				     sizeof(req->out.h));
+	}
 	if (err) {
 		req->out.h.error = -EFAULT;
 		goto out;
@@ -828,7 +955,11 @@ static void fuse_uring_commit(struct fuse_ring_ent *ent, struct fuse_req *req,
 		goto out;
 	}
 
-	err = fuse_uring_copy_from_ring(ring, req, ent);
+	if (ent->is_fixed_buffer)
+		err = fuse_uring_copy_from_ring_fixed_buf(ring, req, ent);
+	else
+		err = fuse_uring_copy_from_ring(ring, req, ent);
+
 out:
 	fuse_uring_req_end(ent, req, err);
 }
@@ -1027,6 +1158,52 @@ static int fuse_uring_get_iovec_from_sqe(const struct io_uring_sqe *sqe,
 	return 0;
 }
 
+static struct fuse_ring_ent *
+fuse_uring_create_ring_ent_fixed_buf(struct io_uring_cmd *cmd,
+				     struct fuse_ring_queue *queue)
+{
+	struct fuse_ring *ring = queue->ring;
+	struct fuse_ring_ent *ent;
+	unsigned payload_size, len;
+	u64 ubuf;
+	int err;
+
+	err = -ENOMEM;
+	ent = kzalloc(sizeof(*ent), GFP_KERNEL_ACCOUNT);
+	if (!ent)
+		return ERR_PTR(err);
+
+	INIT_LIST_HEAD(&ent->list);
+
+	ent->queue = queue;
+	ent->is_fixed_buffer = true;
+
+	err = io_uring_cmd_get_buffer_info(cmd, &ubuf, &len);
+	if (err)
+		goto error;
+
+	payload_size = len - sizeof(struct fuse_uring_req_header);
+	err = io_uring_cmd_import_fixed(ubuf, payload_size, ITER_DEST,
+					&ent->fixed_buffer.payload_iter, cmd, 0);
+	if (err)
+		goto error;
+
+	err = io_uring_cmd_import_fixed(ubuf + payload_size,
+					sizeof(struct fuse_uring_req_header),
+					ITER_DEST,
+					&ent->fixed_buffer.headers_iter, cmd, 0);
+	if (err)
+		goto error;
+
+	atomic_inc(&ring->queue_refs);
+
+	return ent;
+
+error:
+	kfree(ent);
+	return ERR_PTR(err);
+}
+
 static struct fuse_ring_ent *
 fuse_uring_create_ring_ent(struct io_uring_cmd *cmd,
 			   struct fuse_ring_queue *queue)
@@ -1065,8 +1242,8 @@ fuse_uring_create_ring_ent(struct io_uring_cmd *cmd,
 	INIT_LIST_HEAD(&ent->list);
 
 	ent->queue = queue;
-	ent->headers = iov[0].iov_base;
-	ent->payload = iov[1].iov_base;
+	ent->user.headers = iov[0].iov_base;
+	ent->user.payload = iov[1].iov_base;
 
 	atomic_inc(&ring->queue_refs);
 	return ent;
@@ -1085,6 +1262,8 @@ static int fuse_uring_register(struct io_uring_cmd *cmd,
 	struct fuse_ring_ent *ent;
 	int err;
 	unsigned int qid = READ_ONCE(cmd_req->qid);
+	bool is_fixed_buffer =
+		cmd->sqe->uring_cmd_flags & IORING_URING_CMD_FIXED;
 
 	err = -ENOMEM;
 	if (!ring) {
@@ -1110,7 +1289,10 @@ static int fuse_uring_register(struct io_uring_cmd *cmd,
 	 * case of entry errors below, will be done at ring destruction time.
 	 */
 
-	ent = fuse_uring_create_ring_ent(cmd, queue);
+	if (is_fixed_buffer)
+		ent = fuse_uring_create_ring_ent_fixed_buf(cmd, queue);
+	else
+		ent = fuse_uring_create_ring_ent(cmd, queue);
 	if (IS_ERR(ent))
 		return PTR_ERR(ent);
 
diff --git a/fs/fuse/dev_uring_i.h b/fs/fuse/dev_uring_i.h
index 51a563922ce1..748c87e325f5 100644
--- a/fs/fuse/dev_uring_i.h
+++ b/fs/fuse/dev_uring_i.h
@@ -38,9 +38,20 @@ enum fuse_ring_req_state {
 
 /** A fuse ring entry, part of the ring queue */
 struct fuse_ring_ent {
-	/* userspace buffer */
-	struct fuse_uring_req_header __user *headers;
-	void __user *payload;
+	/* True if daemon has registered its buffers ahead of time */
+	bool is_fixed_buffer;
+	union {
+		/* userspace buffer */
+		struct {
+			struct fuse_uring_req_header __user *headers;
+			void __user *payload;
+		} user;
+
+		struct {
+			struct iov_iter payload_iter;
+			struct iov_iter headers_iter;
+		} fixed_buffer;
+	};
 
 	/* the ring queue that owns the request */
 	struct fuse_ring_queue *queue;
-- 
2.47.3