[PATCH v2 0/13] io_uring: add IORING_OP_BPF for extending io_uring

[PATCH v2 0/13] io_uring: add IORING_OP_BPF for extending io_uring

[Ming Lei]

Hello,

Add IORING_OP_BPF for extending io_uring operations, follows typical cases:

- buffer registered zero copy [1]

Also there are some RAID like ublk servers which needs to generate data
parity in case of ublk zero copy

- extend io_uring operations from application

Easy to add one new syscall with IORING_OP_BPF

- extend 64 byte SQE

bpf map can store IO data conveniently

- communicate in IO chain

IORING_OP_BPF can be used for communicate among IOs seamlessly without requiring
extra syscall

- pretty handy to inject error for test purpose

Any comments & feedback are welcome!


[1] lpc2024: ublk based zero copy I/O - use case in Android

https://lpc.events/event/18/contributions/1710/attachments/1440/3070/LPC2024_ublk_zero_copy.pdf


V2:
	- per-ring struct ops (Stefan Metzmacher, Caleb Sander Mateos)
	- refactor io_import_fixed()/io_prep_reg_iovec()/io_import_reg_vec()
	  for allowing to handle multiple buffers for single request
	- kernel selftests
	- all kinds of comments from Caleb Sander Mateos
	- support vectored and registered vector buffer


Ming Lei (12):
  io_uring: make io_import_fixed() global
  io_uring: refactor io_prep_reg_iovec() for BPF kfunc use
  io_uring: refactor io_import_reg_vec() for BPF kfunc use
  io_uring: prepare for extending io_uring with bpf
  io_uring: bpf: extend io_uring with bpf struct_ops
  io_uring: bpf: implement struct_ops registration
  io_uring: bpf: add BPF buffer descriptor for IORING_OP_BPF
  io_uring: bpf: add uring_bpf_memcpy() kfunc
  selftests/io_uring: add BPF struct_ops and kfunc tests
  selftests/io_uring: add bpf_memcpy selftest for uring_bpf_memcpy()
    kfunc
  selftests/io_uring: add copy_user_to_fixed() and copy_fixed_to_user()
    bpf_memcpy tests
  selftests/io_uring: add copy_user_to_reg_vec() and
    copy_reg_vec_to_user() bpf_memcpy tests

Pavel Begunkov (1):
  selftests/io_uring: update mini liburing

 include/linux/io_uring_types.h                |   5 +
 include/uapi/linux/io_uring.h                 |  40 ++
 init/Kconfig                                  |   7 +
 io_uring/Makefile                             |   1 +
 io_uring/bpf_op.c                             | 669 ++++++++++++++++++
 io_uring/bpf_op.h                             |  61 ++
 io_uring/io_uring.c                           |   5 +
 io_uring/io_uring.h                           |   6 +-
 io_uring/opdef.c                              |  16 +
 io_uring/rsrc.c                               |  46 +-
 io_uring/rsrc.h                               |  68 +-
 tools/include/io_uring/mini_liburing.h        |  67 +-
 tools/testing/selftests/Makefile              |   3 +-
 tools/testing/selftests/io_uring/.gitignore   |   2 +
 tools/testing/selftests/io_uring/Makefile     | 173 +++++
 .../selftests/io_uring/basic_bpf_ops.bpf.c    |  94 +++
 .../selftests/io_uring/basic_bpf_ops.c        | 215 ++++++
 .../selftests/io_uring/bpf_memcpy.bpf.c       |  98 +++
 tools/testing/selftests/io_uring/bpf_memcpy.c | 517 ++++++++++++++
 .../selftests/io_uring/include/iou_test.h     |  98 +++
 tools/testing/selftests/io_uring/runner.c     | 206 ++++++
 21 files changed, 2339 insertions(+), 58 deletions(-)
 create mode 100644 io_uring/bpf_op.c
 create mode 100644 io_uring/bpf_op.h
 create mode 100644 tools/testing/selftests/io_uring/.gitignore
 create mode 100644 tools/testing/selftests/io_uring/Makefile
 create mode 100644 tools/testing/selftests/io_uring/basic_bpf_ops.bpf.c
 create mode 100644 tools/testing/selftests/io_uring/basic_bpf_ops.c
 create mode 100644 tools/testing/selftests/io_uring/bpf_memcpy.bpf.c
 create mode 100644 tools/testing/selftests/io_uring/bpf_memcpy.c
 create mode 100644 tools/testing/selftests/io_uring/include/iou_test.h
 create mode 100644 tools/testing/selftests/io_uring/runner.c

-- 
2.47.0

[PATCH V2 01/13] io_uring: make io_import_fixed() global

[Ming Lei]

Refactor buffer import functions:
- Make io_import_fixed() global so BPF kfuncs can use it directly
- Make io_import_reg_buf() static inline in rsrc.h

This allows BPF kfuncs to import buffers without associating them
with a request, useful when one request has multiple buffers.

Signed-off-by: Ming Lei <ming.lei@redhat.com>
---
 io_uring/rsrc.c | 17 +++--------------
 io_uring/rsrc.h | 18 +++++++++++++++---
 2 files changed, 18 insertions(+), 17 deletions(-)

diff --git a/io_uring/rsrc.c b/io_uring/rsrc.c
index 41c89f5c616d..8aa2f7473c89 100644
--- a/io_uring/rsrc.c
+++ b/io_uring/rsrc.c
@@ -1069,9 +1069,9 @@ static int io_import_kbuf(int ddir, struct iov_iter *iter,
 	return 0;
 }
 
-static int io_import_fixed(int ddir, struct iov_iter *iter,
-			   struct io_mapped_ubuf *imu,
-			   u64 buf_addr, size_t len)
+int io_import_fixed(int ddir, struct iov_iter *iter,
+		    struct io_mapped_ubuf *imu,
+		    u64 buf_addr, size_t len)
 {
 	const struct bio_vec *bvec;
 	size_t folio_mask;
@@ -1140,17 +1140,6 @@ inline struct io_rsrc_node *io_find_buf_node(struct io_kiocb *req,
 	return NULL;
 }
 
-int io_import_reg_buf(struct io_kiocb *req, struct iov_iter *iter,
-			u64 buf_addr, size_t len, int ddir,
-			unsigned issue_flags)
-{
-	struct io_rsrc_node *node;
-
-	node = io_find_buf_node(req, issue_flags);
-	if (!node)
-		return -EFAULT;
-	return io_import_fixed(ddir, iter, node->buf, buf_addr, len);
-}
 
 /* Lock two rings at once. The rings must be different! */
 static void lock_two_rings(struct io_ring_ctx *ctx1, struct io_ring_ctx *ctx2)
diff --git a/io_uring/rsrc.h b/io_uring/rsrc.h
index d603f6a47f5e..bf77bc618fb5 100644
--- a/io_uring/rsrc.h
+++ b/io_uring/rsrc.h
@@ -61,9 +61,21 @@ int io_rsrc_data_alloc(struct io_rsrc_data *data, unsigned nr);
 
 struct io_rsrc_node *io_find_buf_node(struct io_kiocb *req,
 				      unsigned issue_flags);
-int io_import_reg_buf(struct io_kiocb *req, struct iov_iter *iter,
-			u64 buf_addr, size_t len, int ddir,
-			unsigned issue_flags);
+int io_import_fixed(int ddir, struct iov_iter *iter,
+		    struct io_mapped_ubuf *imu,
+		    u64 buf_addr, size_t len);
+
+static inline int io_import_reg_buf(struct io_kiocb *req, struct iov_iter *iter,
+				    u64 buf_addr, size_t len, int ddir,
+				    unsigned issue_flags)
+{
+	struct io_rsrc_node *node;
+
+	node = io_find_buf_node(req, issue_flags);
+	if (!node)
+		return -EFAULT;
+	return io_import_fixed(ddir, iter, node->buf, buf_addr, len);
+}
 int io_import_reg_vec(int ddir, struct iov_iter *iter,
 			struct io_kiocb *req, struct iou_vec *vec,
 			unsigned nr_iovs, unsigned issue_flags);
-- 
2.47.0

[PATCH V2 02/13] io_uring: refactor io_prep_reg_iovec() for BPF kfunc use

[Ming Lei]

Split io_prep_reg_iovec() into:
- __io_prep_reg_iovec(): core logic without request association
- io_prep_reg_iovec(): inline wrapper handling request flags

The core function takes explicit 'compat' and 'need_clean' parameters
instead of accessing req directly. This allows BPF kfuncs to prepare
vectored buffers without request association, enabling support for
multiple buffers per request.

Signed-off-by: Ming Lei <ming.lei@redhat.com>
---
 io_uring/rsrc.c | 11 +++++------
 io_uring/rsrc.h | 21 +++++++++++++++++++--
 2 files changed, 24 insertions(+), 8 deletions(-)

diff --git a/io_uring/rsrc.c b/io_uring/rsrc.c
index 8aa2f7473c89..ec716e14d467 100644
--- a/io_uring/rsrc.c
+++ b/io_uring/rsrc.c
@@ -1540,8 +1540,8 @@ int io_import_reg_vec(int ddir, struct iov_iter *iter,
 	return io_vec_fill_bvec(ddir, iter, imu, iov, nr_iovs, vec);
 }
 
-int io_prep_reg_iovec(struct io_kiocb *req, struct iou_vec *iv,
-		      const struct iovec __user *uvec, size_t uvec_segs)
+int __io_prep_reg_iovec(struct iou_vec *iv, const struct iovec __user *uvec,
+			size_t uvec_segs, bool compat, bool *need_clean)
 {
 	struct iovec *iov;
 	int iovec_off, ret;
@@ -1551,17 +1551,16 @@ int io_prep_reg_iovec(struct io_kiocb *req, struct iou_vec *iv,
 		ret = io_vec_realloc(iv, uvec_segs);
 		if (ret)
 			return ret;
-		req->flags |= REQ_F_NEED_CLEANUP;
+		if (need_clean)
+			*need_clean = true;
 	}
 
 	/* pad iovec to the right */
 	iovec_off = iv->nr - uvec_segs;
 	iov = iv->iovec + iovec_off;
-	res = iovec_from_user(uvec, uvec_segs, uvec_segs, iov,
-			      io_is_compat(req->ctx));
+	res = iovec_from_user(uvec, uvec_segs, uvec_segs, iov, compat);
 	if (IS_ERR(res))
 		return PTR_ERR(res);
 
-	req->flags |= REQ_F_IMPORT_BUFFER;
 	return 0;
 }
diff --git a/io_uring/rsrc.h b/io_uring/rsrc.h
index bf77bc618fb5..2a29da350727 100644
--- a/io_uring/rsrc.h
+++ b/io_uring/rsrc.h
@@ -4,6 +4,7 @@
 
 #include <linux/io_uring_types.h>
 #include <linux/lockdep.h>
+#include "io_uring.h"
 
 #define IO_VEC_CACHE_SOFT_CAP		256
 
@@ -79,8 +80,24 @@ static inline int io_import_reg_buf(struct io_kiocb *req, struct iov_iter *iter,
 int io_import_reg_vec(int ddir, struct iov_iter *iter,
 			struct io_kiocb *req, struct iou_vec *vec,
 			unsigned nr_iovs, unsigned issue_flags);
-int io_prep_reg_iovec(struct io_kiocb *req, struct iou_vec *iv,
-			const struct iovec __user *uvec, size_t uvec_segs);
+int __io_prep_reg_iovec(struct iou_vec *iv, const struct iovec __user *uvec,
+			size_t uvec_segs, bool compat, bool *need_clean);
+
+static inline int io_prep_reg_iovec(struct io_kiocb *req, struct iou_vec *iv,
+				    const struct iovec __user *uvec,
+				    size_t uvec_segs)
+{
+	bool need_clean = false;
+	int ret;
+
+	ret = __io_prep_reg_iovec(iv, uvec, uvec_segs,
+				  io_is_compat(req->ctx), &need_clean);
+	if (need_clean)
+		req->flags |= REQ_F_NEED_CLEANUP;
+	if (ret >= 0)
+		req->flags |= REQ_F_IMPORT_BUFFER;
+	return ret;
+}
 
 int io_register_clone_buffers(struct io_ring_ctx *ctx, void __user *arg);
 int io_sqe_buffers_unregister(struct io_ring_ctx *ctx);
-- 
2.47.0

[PATCH V2 03/13] io_uring: refactor io_import_reg_vec() for BPF kfunc use

[Ming Lei]

Split io_import_reg_vec() into:
- __io_import_reg_vec(): core logic taking io_mapped_ubuf directly
- io_import_reg_vec(): inline wrapper handling buffer lookup and
  request flags

The core function takes 'imu' and 'need_clean' parameters instead of
accessing req directly. This allows BPF kfuncs to import vectored
buffers without request association, enabling support for multiple
buffers per request.

Signed-off-by: Ming Lei <ming.lei@redhat.com>
---
 io_uring/rsrc.c | 18 +++++-------------
 io_uring/rsrc.h | 29 ++++++++++++++++++++++++++---
 2 files changed, 31 insertions(+), 16 deletions(-)

diff --git a/io_uring/rsrc.c b/io_uring/rsrc.c
index ec716e14d467..7eb2c70185e4 100644
--- a/io_uring/rsrc.c
+++ b/io_uring/rsrc.c
@@ -1476,23 +1476,14 @@ static int io_kern_bvec_size(struct iovec *iov, unsigned nr_iovs,
 	return 0;
 }
 
-int io_import_reg_vec(int ddir, struct iov_iter *iter,
-			struct io_kiocb *req, struct iou_vec *vec,
-			unsigned nr_iovs, unsigned issue_flags)
+int __io_import_reg_vec(int ddir, struct iov_iter *iter,
+			struct io_mapped_ubuf *imu, struct iou_vec *vec,
+			unsigned nr_iovs, bool *need_clean)
 {
-	struct io_rsrc_node *node;
-	struct io_mapped_ubuf *imu;
 	unsigned iovec_off;
 	struct iovec *iov;
 	unsigned nr_segs;
 
-	node = io_find_buf_node(req, issue_flags);
-	if (!node)
-		return -EFAULT;
-	imu = node->buf;
-	if (!(imu->dir & (1 << ddir)))
-		return -EFAULT;
-
 	iovec_off = vec->nr - nr_iovs;
 	iov = vec->iovec + iovec_off;
 
@@ -1531,7 +1522,8 @@ int io_import_reg_vec(int ddir, struct iov_iter *iter,
 
 		*vec = tmp_vec;
 		iov = vec->iovec + iovec_off;
-		req->flags |= REQ_F_NEED_CLEANUP;
+		if (need_clean)
+			*need_clean = true;
 	}
 
 	if (imu->is_kbuf)
diff --git a/io_uring/rsrc.h b/io_uring/rsrc.h
index 2a29da350727..3203277ac289 100644
--- a/io_uring/rsrc.h
+++ b/io_uring/rsrc.h
@@ -77,9 +77,32 @@ static inline int io_import_reg_buf(struct io_kiocb *req, struct iov_iter *iter,
 		return -EFAULT;
 	return io_import_fixed(ddir, iter, node->buf, buf_addr, len);
 }
-int io_import_reg_vec(int ddir, struct iov_iter *iter,
-			struct io_kiocb *req, struct iou_vec *vec,
-			unsigned nr_iovs, unsigned issue_flags);
+int __io_import_reg_vec(int ddir, struct iov_iter *iter,
+			struct io_mapped_ubuf *imu, struct iou_vec *vec,
+			unsigned nr_iovs, bool *need_clean);
+
+static inline int io_import_reg_vec(int ddir, struct iov_iter *iter,
+				    struct io_kiocb *req, struct iou_vec *vec,
+				    unsigned nr_iovs, unsigned issue_flags)
+{
+	struct io_rsrc_node *node;
+	struct io_mapped_ubuf *imu;
+	bool need_clean = false;
+	int ret;
+
+	node = io_find_buf_node(req, issue_flags);
+	if (!node)
+		return -EFAULT;
+	imu = node->buf;
+	if (!(imu->dir & (1 << ddir)))
+		return -EFAULT;
+
+	ret = __io_import_reg_vec(ddir, iter, imu, vec, nr_iovs, &need_clean);
+	if (need_clean)
+		req->flags |= REQ_F_NEED_CLEANUP;
+	return ret;
+}
+
 int __io_prep_reg_iovec(struct iou_vec *iv, const struct iovec __user *uvec,
 			size_t uvec_segs, bool compat, bool *need_clean);
 
-- 
2.47.0

[PATCH V2 04/13] io_uring: prepare for extending io_uring with bpf

[Ming Lei]

Add one bpf operation & related framework and prepare for extending io_uring
with bpf.

Signed-off-by: Ming Lei <ming.lei@redhat.com>
---
 include/uapi/linux/io_uring.h |  1 +
 init/Kconfig                  |  7 +++++++
 io_uring/Makefile             |  1 +
 io_uring/bpf_op.c             | 26 ++++++++++++++++++++++++++
 io_uring/bpf_op.h             | 15 +++++++++++++++
 io_uring/opdef.c              | 16 ++++++++++++++++
 6 files changed, 66 insertions(+)
 create mode 100644 io_uring/bpf_op.c
 create mode 100644 io_uring/bpf_op.h

diff --git a/include/uapi/linux/io_uring.h b/include/uapi/linux/io_uring.h
index b5b23c0d5283..30406cfb2e21 100644
--- a/include/uapi/linux/io_uring.h
+++ b/include/uapi/linux/io_uring.h
@@ -303,6 +303,7 @@ enum io_uring_op {
 	IORING_OP_PIPE,
 	IORING_OP_NOP128,
 	IORING_OP_URING_CMD128,
+	IORING_OP_BPF,
 
 	/* this goes last, obviously */
 	IORING_OP_LAST,
diff --git a/init/Kconfig b/init/Kconfig
index fa79feb8fe57..b2f2a5473538 100644
--- a/init/Kconfig
+++ b/init/Kconfig
@@ -1868,6 +1868,13 @@ config IO_URING
 	  applications to submit and complete IO through submission and
 	  completion rings that are shared between the kernel and application.
 
+config IO_URING_BPF_OP
+	bool "Enable IO uring bpf extension" if EXPERT
+	depends on IO_URING && BPF
+	help
+	  This option enables bpf extension for the io_uring interface, so
+	  application can define its own io_uring operation by bpf program.
+
 config GCOV_PROFILE_URING
 	bool "Enable GCOV profiling on the io_uring subsystem"
 	depends on IO_URING && GCOV_KERNEL
diff --git a/io_uring/Makefile b/io_uring/Makefile
index bc4e4a3fa0a5..b2db2b334cee 100644
--- a/io_uring/Makefile
+++ b/io_uring/Makefile
@@ -22,3 +22,4 @@ obj-$(CONFIG_NET_RX_BUSY_POLL)	+= napi.o
 obj-$(CONFIG_NET) += net.o cmd_net.o
 obj-$(CONFIG_PROC_FS) += fdinfo.o
 obj-$(CONFIG_IO_URING_MOCK_FILE) += mock_file.o
+obj-$(CONFIG_IO_URING_BPF_OP)	+= bpf_op.o
diff --git a/io_uring/bpf_op.c b/io_uring/bpf_op.c
new file mode 100644
index 000000000000..2ab6f93bbad8
--- /dev/null
+++ b/io_uring/bpf_op.c
@@ -0,0 +1,26 @@
+// SPDX-License-Identifier: GPL-2.0
+/* Copyright (c) 2025 Red Hat */
+
+#include <linux/kernel.h>
+#include <linux/errno.h>
+#include <uapi/linux/io_uring.h>
+#include "io_uring.h"
+#include "bpf_op.h"
+
+int io_uring_bpf_issue(struct io_kiocb *req, unsigned int issue_flags)
+{
+	return -EOPNOTSUPP;
+}
+
+int io_uring_bpf_prep(struct io_kiocb *req, const struct io_uring_sqe *sqe)
+{
+	return -EOPNOTSUPP;
+}
+
+void io_uring_bpf_fail(struct io_kiocb *req)
+{
+}
+
+void io_uring_bpf_cleanup(struct io_kiocb *req)
+{
+}
diff --git a/io_uring/bpf_op.h b/io_uring/bpf_op.h
new file mode 100644
index 000000000000..7b61612c28c4
--- /dev/null
+++ b/io_uring/bpf_op.h
@@ -0,0 +1,15 @@
+// SPDX-License-Identifier: GPL-2.0
+#ifndef IOU_BPF_OP_H
+#define IOU_BPF_OP_H
+
+struct io_kiocb;
+struct io_uring_sqe;
+
+#ifdef CONFIG_IO_URING_BPF_OP
+int io_uring_bpf_issue(struct io_kiocb *req, unsigned int issue_flags);
+int io_uring_bpf_prep(struct io_kiocb *req, const struct io_uring_sqe *sqe);
+void io_uring_bpf_fail(struct io_kiocb *req);
+void io_uring_bpf_cleanup(struct io_kiocb *req);
+#endif
+
+#endif
diff --git a/io_uring/opdef.c b/io_uring/opdef.c
index df52d760240e..289107f3c00a 100644
--- a/io_uring/opdef.c
+++ b/io_uring/opdef.c
@@ -38,6 +38,7 @@
 #include "futex.h"
 #include "truncate.h"
 #include "zcrx.h"
+#include "bpf_op.h"
 
 static int io_no_issue(struct io_kiocb *req, unsigned int issue_flags)
 {
@@ -593,6 +594,14 @@ const struct io_issue_def io_issue_defs[] = {
 		.prep			= io_uring_cmd_prep,
 		.issue			= io_uring_cmd,
 	},
+	[IORING_OP_BPF] = {
+#if defined(CONFIG_IO_URING_BPF_OP)
+		.prep			= io_uring_bpf_prep,
+		.issue			= io_uring_bpf_issue,
+#else
+		.prep			= io_eopnotsupp_prep,
+#endif
+	},
 };
 
 const struct io_cold_def io_cold_defs[] = {
@@ -851,6 +860,13 @@ const struct io_cold_def io_cold_defs[] = {
 		.sqe_copy		= io_uring_cmd_sqe_copy,
 		.cleanup		= io_uring_cmd_cleanup,
 	},
+	[IORING_OP_BPF] = {
+		.name			= "BPF",
+#if defined(CONFIG_IO_URING_BPF_OP)
+		.cleanup		= io_uring_bpf_cleanup,
+		.fail			= io_uring_bpf_fail,
+#endif
+	},
 };
 
 const char *io_uring_get_opcode(u8 opcode)
-- 
2.47.0

[PATCH V2 05/13] io_uring: bpf: extend io_uring with bpf struct_ops

[Ming Lei]

io_uring can be extended with bpf struct_ops in the following ways:

1) add new io_uring operation from application
- one typical use case is for operating device zero-copy buffer, which
belongs to kernel, and not visible or too expensive to export to
userspace, such as supporting copy data from this buffer to userspace,
decompressing data to zero-copy buffer in Android case[1][2], or
checksum/decrypting.

[1] https://lpc.events/event/18/contributions/1710/attachments/1440/3070/LPC2024_ublk_zero_copy.pdf

2) extend 64 byte SQE, since bpf map can be used to store IO data
   conveniently

3) communicate in IO chain, since bpf map can be shared among IOs,
when one bpf IO is completed, data can be written to IO chain wide
bpf map, then the following bpf IO can retrieve the data from this bpf
map, this way is more flexible than io_uring built-in buffer

4) pretty handy to inject error for test purpose

bpf struct_ops is one very handy way to attach bpf prog with kernel, and
this patch simply wires existed io_uring operation callbacks with added
uring bpf struct_ops, so application can define its own uring bpf
operations.

Signed-off-by: Ming Lei <ming.lei@redhat.com>
---
 include/linux/io_uring_types.h |   3 +
 include/uapi/linux/io_uring.h  |  12 ++
 io_uring/bpf_op.c              | 231 ++++++++++++++++++++++++++++++++-
 io_uring/bpf_op.h              |  42 ++++++
 io_uring/io_uring.c            |   5 +
 io_uring/io_uring.h            |   6 +-
 6 files changed, 295 insertions(+), 4 deletions(-)

diff --git a/include/linux/io_uring_types.h b/include/linux/io_uring_types.h
index 9ad389f0715b..62ff38b3ce1e 100644
--- a/include/linux/io_uring_types.h
+++ b/include/linux/io_uring_types.h
@@ -8,6 +8,8 @@
 #include <linux/llist.h>
 #include <uapi/linux/io_uring.h>
 
+struct uring_bpf_ops_kern;
+
 enum {
 	/*
 	 * A hint to not wake right away but delay until there are enough of
@@ -276,6 +278,7 @@ struct io_ring_ctx {
 		struct io_rings		*rings;
 		struct percpu_ref	refs;
 
+		struct uring_bpf_ops_kern	*bpf_ops;
 		clockid_t		clockid;
 		enum tk_offsets		clock_offset;
 
diff --git a/include/uapi/linux/io_uring.h b/include/uapi/linux/io_uring.h
index 30406cfb2e21..441a1038a58a 100644
--- a/include/uapi/linux/io_uring.h
+++ b/include/uapi/linux/io_uring.h
@@ -74,6 +74,7 @@ struct io_uring_sqe {
 		__u32		install_fd_flags;
 		__u32		nop_flags;
 		__u32		pipe_flags;
+		__u32		bpf_op_flags;
 	};
 	__u64	user_data;	/* data to be passed back at completion time */
 	/* pack this to avoid bogus arm OABI complaints */
@@ -237,6 +238,9 @@ enum io_uring_sqe_flags_bit {
  */
 #define IORING_SETUP_SQE_MIXED		(1U << 19)
 
+/* Allow userspace to define io_uring operation by BPF prog */
+#define IORING_SETUP_BPF_OP		(1U << 20)
+
 enum io_uring_op {
 	IORING_OP_NOP,
 	IORING_OP_READV,
@@ -422,6 +426,13 @@ enum io_uring_op {
 #define IORING_RECVSEND_BUNDLE		(1U << 4)
 #define IORING_SEND_VECTORIZED		(1U << 5)
 
+/*
+ * sqe->bpf_op_flags		top 8bits is for storing bpf prog sub op
+ *				The other 24bits are used for bpf prog
+ */
+#define IORING_BPF_OP_BITS	8
+#define IORING_BPF_OP_SHIFT	24
+
 /*
  * cqe.res for IORING_CQE_F_NOTIF if
  * IORING_SEND_ZC_REPORT_USAGE was requested
@@ -626,6 +637,7 @@ struct io_uring_params {
 #define IORING_FEAT_MIN_TIMEOUT		(1U << 15)
 #define IORING_FEAT_RW_ATTR		(1U << 16)
 #define IORING_FEAT_NO_IOWAIT		(1U << 17)
+#define IORING_FEAT_BPF			(1U << 18)
 
 /*
  * io_uring_register(2) opcodes and arguments
diff --git a/io_uring/bpf_op.c b/io_uring/bpf_op.c
index 2ab6f93bbad8..f616416652e9 100644
--- a/io_uring/bpf_op.c
+++ b/io_uring/bpf_op.c
@@ -3,24 +3,251 @@
 
 #include <linux/kernel.h>
 #include <linux/errno.h>
+#include <linux/init.h>
+#include <linux/types.h>
+#include <linux/bpf_verifier.h>
+#include <linux/bpf.h>
+#include <linux/btf.h>
+#include <linux/btf_ids.h>
+#include <linux/filter.h>
 #include <uapi/linux/io_uring.h>
 #include "io_uring.h"
 #include "bpf_op.h"
 
-int io_uring_bpf_issue(struct io_kiocb *req, unsigned int issue_flags)
+static inline unsigned char uring_bpf_get_op(u32 op_flags)
 {
-	return -EOPNOTSUPP;
+	return (unsigned char)(op_flags >> IORING_BPF_OP_SHIFT);
+}
+
+static inline unsigned int uring_bpf_get_flags(u32 op_flags)
+{
+	return op_flags & ((1U << IORING_BPF_OP_SHIFT) - 1);
 }
 
 int io_uring_bpf_prep(struct io_kiocb *req, const struct io_uring_sqe *sqe)
 {
+	struct uring_bpf_data *data = io_kiocb_to_cmd(req, struct uring_bpf_data);
+	u32 opf = READ_ONCE(sqe->bpf_op_flags);
+	unsigned char bpf_op = uring_bpf_get_op(opf);
+	const struct uring_bpf_ops *ops;
+
+	if (unlikely(!(req->ctx->flags & IORING_SETUP_BPF_OP)))
+		goto fail;
+
+	if (bpf_op >= IO_RING_MAX_BPF_OPS)
+		return -EINVAL;
+
+	ops = req->ctx->bpf_ops[bpf_op].ops;
+	data->opf = opf;
+	data->ops = ops;
+	if (ops && ops->prep_fn)
+		return ops->prep_fn(data, sqe);
+fail:
 	return -EOPNOTSUPP;
 }
 
+static int __io_uring_bpf_issue(struct io_kiocb *req)
+{
+	struct uring_bpf_data *data = io_kiocb_to_cmd(req, struct uring_bpf_data);
+	const struct uring_bpf_ops *ops = data->ops;
+	int ret = 0;
+
+	if (ops && ops->issue_fn) {
+		ret = ops->issue_fn(data);
+		if (ret == IOU_ISSUE_SKIP_COMPLETE)
+			return -EINVAL;
+	}
+	return ret;
+}
+
+int io_uring_bpf_issue(struct io_kiocb *req, unsigned int issue_flags)
+{
+	return __io_uring_bpf_issue(req);
+}
+
 void io_uring_bpf_fail(struct io_kiocb *req)
 {
+	struct uring_bpf_data *data = io_kiocb_to_cmd(req, struct uring_bpf_data);
+	const struct uring_bpf_ops *ops = data->ops;
+
+	if (ops && ops->fail_fn)
+		ops->fail_fn(data);
 }
 
 void io_uring_bpf_cleanup(struct io_kiocb *req)
 {
+	struct uring_bpf_data *data = io_kiocb_to_cmd(req, struct uring_bpf_data);
+	const struct uring_bpf_ops *ops = data->ops;
+
+	if (ops && ops->cleanup_fn)
+		ops->cleanup_fn(data);
+}
+
+static const struct btf_type *uring_bpf_data_type;
+
+static int uring_bpf_ops_btf_struct_access(struct bpf_verifier_log *log,
+					const struct bpf_reg_state *reg,
+					int off, int size)
+{
+	const struct btf_type *t;
+
+	t = btf_type_by_id(reg->btf, reg->btf_id);
+	if (t != uring_bpf_data_type) {
+		bpf_log(log, "only read is supported\n");
+		return -EACCES;
+	}
+
+	if (off < offsetof(struct uring_bpf_data, pdu) ||
+			off + size > sizeof(struct uring_bpf_data))
+		return -EACCES;
+
+	return NOT_INIT;
+}
+
+static const struct bpf_verifier_ops io_bpf_verifier_ops = {
+	.get_func_proto = bpf_base_func_proto,
+	.is_valid_access = bpf_tracing_btf_ctx_access,
+	.btf_struct_access = uring_bpf_ops_btf_struct_access,
+};
+
+static int uring_bpf_ops_init(struct btf *btf)
+{
+	s32 type_id;
+
+	type_id = btf_find_by_name_kind(btf, "uring_bpf_data", BTF_KIND_STRUCT);
+	if (type_id < 0)
+		return -EINVAL;
+	uring_bpf_data_type = btf_type_by_id(btf, type_id);
+	return 0;
+}
+
+static int uring_bpf_ops_check_member(const struct btf_type *t,
+				   const struct btf_member *member,
+				   const struct bpf_prog *prog)
+{
+	/*
+	 * All io_uring BPF ops callbacks are called in non-sleepable
+	 * context, so reject sleepable BPF programs.
+	 */
+	if (prog->sleepable)
+		return -EINVAL;
+
+	return 0;
+}
+
+static int uring_bpf_ops_init_member(const struct btf_type *t,
+				 const struct btf_member *member,
+				 void *kdata, const void *udata)
+{
+	const struct uring_bpf_ops *uuring_bpf_ops;
+	struct uring_bpf_ops *kuring_bpf_ops;
+	u32 moff;
+
+	uuring_bpf_ops = udata;
+	kuring_bpf_ops = kdata;
+
+	moff = __btf_member_bit_offset(t, member) / 8;
+
+	switch (moff) {
+	case offsetof(struct uring_bpf_ops, id):
+		/* For id, this function has to copy it and return 1 to
+		 * indicate that the data has been handled by the struct_ops
+		 * type, or the verifier will reject the map if the value of
+		 * those fields is not zero.
+		 */
+		kuring_bpf_ops->id = uuring_bpf_ops->id;
+		return 1;
+	}
+	return 0;
+}
+
+static int io_bpf_prep_io(struct uring_bpf_data *data, const struct io_uring_sqe *sqe)
+{
+	return 0;
+}
+
+static int io_bpf_issue_io(struct uring_bpf_data *data)
+{
+	return 0;
+}
+
+static void io_bpf_fail_io(struct uring_bpf_data *data)
+{
+}
+
+static void io_bpf_cleanup_io(struct uring_bpf_data *data)
+{
+}
+
+static struct uring_bpf_ops __bpf_uring_bpf_ops = {
+	.prep_fn	= io_bpf_prep_io,
+	.issue_fn	= io_bpf_issue_io,
+	.fail_fn	= io_bpf_fail_io,
+	.cleanup_fn	= io_bpf_cleanup_io,
+};
+
+static struct bpf_struct_ops bpf_uring_bpf_ops = {
+	.verifier_ops = &io_bpf_verifier_ops,
+	.init = uring_bpf_ops_init,
+	.check_member = uring_bpf_ops_check_member,
+	.init_member = uring_bpf_ops_init_member,
+	.name = "uring_bpf_ops",
+	.cfi_stubs = &__bpf_uring_bpf_ops,
+	.owner = THIS_MODULE,
+};
+
+__bpf_kfunc_start_defs();
+__bpf_kfunc void uring_bpf_set_result(struct uring_bpf_data *data, int res)
+{
+	struct io_kiocb *req = cmd_to_io_kiocb(data);
+
+	if (res < 0)
+		req_set_fail(req);
+	io_req_set_res(req, res, 0);
+}
+__bpf_kfunc_end_defs();
+
+BTF_KFUNCS_START(uring_bpf_kfuncs)
+BTF_ID_FLAGS(func, uring_bpf_set_result)
+BTF_KFUNCS_END(uring_bpf_kfuncs)
+
+static const struct btf_kfunc_id_set uring_kfunc_set = {
+	.owner = THIS_MODULE,
+	.set   = &uring_bpf_kfuncs,
+};
+
+int io_bpf_alloc(struct io_ring_ctx *ctx)
+{
+	if (!(ctx->flags & IORING_SETUP_BPF_OP))
+		return 0;
+
+	ctx->bpf_ops = kcalloc(IO_RING_MAX_BPF_OPS,
+			sizeof(struct uring_bpf_ops_kern), GFP_KERNEL);
+	if (!ctx->bpf_ops)
+		return -ENOMEM;
+	return 0;
+}
+
+void io_bpf_free(struct io_ring_ctx *ctx)
+{
+	kfree(ctx->bpf_ops);
+	ctx->bpf_ops = NULL;
+}
+
+static int __init io_bpf_init(void)
+{
+	int err;
+
+	err = register_btf_kfunc_id_set(BPF_PROG_TYPE_STRUCT_OPS, &uring_kfunc_set);
+	if (err) {
+		pr_warn("error while setting io_uring BPF kfuncs: %d\n", err);
+		return err;
+	}
+
+	err = register_bpf_struct_ops(&bpf_uring_bpf_ops, uring_bpf_ops);
+	if (err)
+		pr_warn("error while registering io_uring BPF struct ops: %d\n", err);
+
+	return err;
 }
+__initcall(io_bpf_init);
diff --git a/io_uring/bpf_op.h b/io_uring/bpf_op.h
index 7b61612c28c4..99708140992f 100644
--- a/io_uring/bpf_op.h
+++ b/io_uring/bpf_op.h
@@ -4,12 +4,54 @@
 
 struct io_kiocb;
 struct io_uring_sqe;
+struct uring_bpf_ops;
+
+/* Arbitrary limit, can be raised if need be */
+#define IO_RING_MAX_BPF_OPS 16
+
+struct uring_bpf_data {
+	void				*req_data;  /* not for bpf prog */
+	const struct uring_bpf_ops	*ops;
+	u32				opf;
+
+	/* writeable for bpf prog */
+	u8              pdu[64 - sizeof(void *) -
+		sizeof(struct uring_bpf_ops *) - sizeof(u32)];
+};
+
+typedef int (*uring_bpf_prep_t)(struct uring_bpf_data *data,
+				const struct io_uring_sqe *sqe);
+typedef int (*uring_bpf_issue_t)(struct uring_bpf_data *data);
+typedef void (*uring_bpf_fail_t)(struct uring_bpf_data *data);
+typedef void (*uring_bpf_cleanup_t)(struct uring_bpf_data *data);
+
+struct uring_bpf_ops {
+	unsigned short		id;
+	uring_bpf_prep_t	prep_fn;
+	uring_bpf_issue_t	issue_fn;
+	uring_bpf_fail_t	fail_fn;
+	uring_bpf_cleanup_t	cleanup_fn;
+};
+
+struct uring_bpf_ops_kern {
+	const struct uring_bpf_ops *ops;
+};
 
 #ifdef CONFIG_IO_URING_BPF_OP
 int io_uring_bpf_issue(struct io_kiocb *req, unsigned int issue_flags);
 int io_uring_bpf_prep(struct io_kiocb *req, const struct io_uring_sqe *sqe);
 void io_uring_bpf_fail(struct io_kiocb *req);
 void io_uring_bpf_cleanup(struct io_kiocb *req);
+int io_bpf_alloc(struct io_ring_ctx *ctx);
+void io_bpf_free(struct io_ring_ctx *ctx);
+#else
+static inline int io_bpf_alloc(struct io_ring_ctx *ctx)
+{
+	return 0;
+}
+static inline void io_bpf_free(struct io_ring_ctx *ctx)
+{
+}
 #endif
 
 #endif
diff --git a/io_uring/io_uring.c b/io_uring/io_uring.c
index 1ddd7a2c53a6..d6ac43bc0e63 100644
--- a/io_uring/io_uring.c
+++ b/io_uring/io_uring.c
@@ -105,6 +105,7 @@
 #include "rw.h"
 #include "alloc_cache.h"
 #include "eventfd.h"
+#include "bpf_op.h"
 
 #define SQE_COMMON_FLAGS (IOSQE_FIXED_FILE | IOSQE_IO_LINK | \
 			  IOSQE_IO_HARDLINK | IOSQE_ASYNC)
@@ -353,6 +354,9 @@ static __cold struct io_ring_ctx *io_ring_ctx_alloc(struct io_uring_params *p)
 	io_napi_init(ctx);
 	mutex_init(&ctx->mmap_lock);
 
+	if (io_bpf_alloc(ctx))
+		goto free_ref;
+
 	return ctx;
 
 free_ref:
@@ -2867,6 +2871,7 @@ static __cold void io_ring_ctx_free(struct io_ring_ctx *ctx)
 	if (ctx->hash_map)
 		io_wq_put_hash(ctx->hash_map);
 	io_napi_free(ctx);
+	io_bpf_free(ctx);
 	kvfree(ctx->cancel_table.hbs);
 	xa_destroy(&ctx->io_bl_xa);
 	kfree(ctx);
diff --git a/io_uring/io_uring.h b/io_uring/io_uring.h
index a790c16854d3..71b4a44dba49 100644
--- a/io_uring/io_uring.h
+++ b/io_uring/io_uring.h
@@ -48,7 +48,8 @@ struct io_ctx_config {
 			IORING_FEAT_RECVSEND_BUNDLE |\
 			IORING_FEAT_MIN_TIMEOUT |\
 			IORING_FEAT_RW_ATTR |\
-			IORING_FEAT_NO_IOWAIT)
+			IORING_FEAT_NO_IOWAIT |\
+			IORING_FEAT_BPF)
 
 #define IORING_SETUP_FLAGS (IORING_SETUP_IOPOLL |\
 			IORING_SETUP_SQPOLL |\
@@ -69,7 +70,8 @@ struct io_ctx_config {
 			IORING_SETUP_NO_SQARRAY |\
 			IORING_SETUP_HYBRID_IOPOLL |\
 			IORING_SETUP_CQE_MIXED |\
-			IORING_SETUP_SQE_MIXED)
+			IORING_SETUP_SQE_MIXED |\
+			IORING_SETUP_BPF_OP)
 
 #define IORING_ENTER_FLAGS (IORING_ENTER_GETEVENTS |\
 			IORING_ENTER_SQ_WAKEUP |\
-- 
2.47.0

[PATCH V2 06/13] io_uring: bpf: implement struct_ops registration

[Ming Lei]

Complete the BPF struct_ops registration mechanism by implementing
refcount-based lifecycle management:

- Add refcount field to struct uring_bpf_ops_kern for tracking active
  requests
- Add wait_queue_head_t bpf_wq to struct io_ring_ctx for synchronizing
  unregistration with in-flight requests
- Implement io_bpf_reg_unreg() to handle registration (refcount=1) and
  unregistration (wait for in-flight requests to complete)
- Update io_uring_bpf_prep() to increment refcount on success and reject
  new requests when refcount is zero (unregistration in progress)
- Update io_uring_bpf_cleanup() to decrement refcount and wake up waiters
  when it reaches zero

Signed-off-by: Ming Lei <ming.lei@redhat.com>
---
 include/linux/io_uring_types.h |   2 +
 io_uring/bpf_op.c              | 104 ++++++++++++++++++++++++++++++++-
 io_uring/bpf_op.h              |   3 +
 3 files changed, 106 insertions(+), 3 deletions(-)

diff --git a/include/linux/io_uring_types.h b/include/linux/io_uring_types.h
index 62ff38b3ce1e..b8eb9d8ba4ce 100644
--- a/include/linux/io_uring_types.h
+++ b/include/linux/io_uring_types.h
@@ -474,6 +474,8 @@ struct io_ring_ctx {
 	struct io_mapped_region		ring_region;
 	/* used for optimised request parameter and wait argument passing  */
 	struct io_mapped_region		param_region;
+
+	wait_queue_head_t		bpf_wq;
 };
 
 /*
diff --git a/io_uring/bpf_op.c b/io_uring/bpf_op.c
index f616416652e9..d6f146abe304 100644
--- a/io_uring/bpf_op.c
+++ b/io_uring/bpf_op.c
@@ -12,6 +12,7 @@
 #include <linux/filter.h>
 #include <uapi/linux/io_uring.h>
 #include "io_uring.h"
+#include "register.h"
 #include "bpf_op.h"
 
 static inline unsigned char uring_bpf_get_op(u32 op_flags)
@@ -29,7 +30,9 @@ int io_uring_bpf_prep(struct io_kiocb *req, const struct io_uring_sqe *sqe)
 	struct uring_bpf_data *data = io_kiocb_to_cmd(req, struct uring_bpf_data);
 	u32 opf = READ_ONCE(sqe->bpf_op_flags);
 	unsigned char bpf_op = uring_bpf_get_op(opf);
+	struct uring_bpf_ops_kern *ops_kern;
 	const struct uring_bpf_ops *ops;
+	int ret;
 
 	if (unlikely(!(req->ctx->flags & IORING_SETUP_BPF_OP)))
 		goto fail;
@@ -37,11 +40,20 @@ int io_uring_bpf_prep(struct io_kiocb *req, const struct io_uring_sqe *sqe)
 	if (bpf_op >= IO_RING_MAX_BPF_OPS)
 		return -EINVAL;
 
-	ops = req->ctx->bpf_ops[bpf_op].ops;
+	ops_kern = &req->ctx->bpf_ops[bpf_op];
+	ops = ops_kern->ops;
+	if (!ops || !ops->prep_fn || !ops_kern->refcount)
+		goto fail;
+
 	data->opf = opf;
 	data->ops = ops;
-	if (ops && ops->prep_fn)
-		return ops->prep_fn(data, sqe);
+	ret = ops->prep_fn(data, sqe);
+	if (!ret) {
+		/* Only increment refcount on success (uring_lock already held) */
+		req->flags |= REQ_F_NEED_CLEANUP;
+		ops_kern->refcount++;
+	}
+	return ret;
 fail:
 	return -EOPNOTSUPP;
 }
@@ -78,9 +90,18 @@ void io_uring_bpf_cleanup(struct io_kiocb *req)
 {
 	struct uring_bpf_data *data = io_kiocb_to_cmd(req, struct uring_bpf_data);
 	const struct uring_bpf_ops *ops = data->ops;
+	struct uring_bpf_ops_kern *ops_kern;
+	unsigned char bpf_op;
 
 	if (ops && ops->cleanup_fn)
 		ops->cleanup_fn(data);
+
+	bpf_op = uring_bpf_get_op(data->opf);
+	ops_kern = &req->ctx->bpf_ops[bpf_op];
+
+	/* Decrement refcount after cleanup (uring_lock already held) */
+	if (--ops_kern->refcount == 0)
+		wake_up(&req->ctx->bpf_wq);
 }
 
 static const struct btf_type *uring_bpf_data_type;
@@ -157,10 +178,82 @@ static int uring_bpf_ops_init_member(const struct btf_type *t,
 		 */
 		kuring_bpf_ops->id = uuring_bpf_ops->id;
 		return 1;
+	case offsetof(struct uring_bpf_ops, ring_fd):
+		kuring_bpf_ops->ring_fd = uuring_bpf_ops->ring_fd;
+		return 1;
 	}
 	return 0;
 }
 
+static int io_bpf_reg_unreg(struct uring_bpf_ops *ops, bool reg)
+{
+	struct uring_bpf_ops_kern *ops_kern;
+	struct io_ring_ctx *ctx;
+	struct file *file;
+	int ret = -EINVAL;
+
+	if (ops->id >= IO_RING_MAX_BPF_OPS)
+		return -EINVAL;
+
+	file = io_uring_register_get_file(ops->ring_fd, false);
+	if (IS_ERR(file))
+		return PTR_ERR(file);
+
+	ctx = file->private_data;
+	if (!(ctx->flags & IORING_SETUP_BPF_OP))
+		goto out;
+
+	ops_kern = &ctx->bpf_ops[ops->id];
+
+	mutex_lock(&ctx->uring_lock);
+	if (reg) {
+		/* Registration: set refcount to 1 and store ops */
+		if (ops_kern->ops) {
+			ret = -EBUSY;
+		} else {
+			ops_kern->ops = ops;
+			ops_kern->refcount = 1;
+			ret = 0;
+		}
+	} else {
+		/* Unregistration */
+		if (!ops_kern->ops) {
+			ret = -EINVAL;
+		} else {
+			ops_kern->refcount--;
+retry:
+			if (ops_kern->refcount == 0) {
+				ops_kern->ops = NULL;
+				ret = 0;
+			} else {
+				mutex_unlock(&ctx->uring_lock);
+				wait_event(ctx->bpf_wq, ops_kern->refcount == 0);
+				mutex_lock(&ctx->uring_lock);
+				goto retry;
+			}
+		}
+	}
+	mutex_unlock(&ctx->uring_lock);
+
+out:
+	fput(file);
+	return ret;
+}
+
+static int io_bpf_reg(void *kdata, struct bpf_link *link)
+{
+	struct uring_bpf_ops *ops = kdata;
+
+	return io_bpf_reg_unreg(ops, true);
+}
+
+static void io_bpf_unreg(void *kdata, struct bpf_link *link)
+{
+	struct uring_bpf_ops *ops = kdata;
+
+	io_bpf_reg_unreg(ops, false);
+}
+
 static int io_bpf_prep_io(struct uring_bpf_data *data, const struct io_uring_sqe *sqe)
 {
 	return 0;
@@ -191,6 +284,8 @@ static struct bpf_struct_ops bpf_uring_bpf_ops = {
 	.init = uring_bpf_ops_init,
 	.check_member = uring_bpf_ops_check_member,
 	.init_member = uring_bpf_ops_init_member,
+	.reg = io_bpf_reg,
+	.unreg = io_bpf_unreg,
 	.name = "uring_bpf_ops",
 	.cfi_stubs = &__bpf_uring_bpf_ops,
 	.owner = THIS_MODULE,
@@ -218,6 +313,8 @@ static const struct btf_kfunc_id_set uring_kfunc_set = {
 
 int io_bpf_alloc(struct io_ring_ctx *ctx)
 {
+	init_waitqueue_head(&ctx->bpf_wq);
+
 	if (!(ctx->flags & IORING_SETUP_BPF_OP))
 		return 0;
 
@@ -225,6 +322,7 @@ int io_bpf_alloc(struct io_ring_ctx *ctx)
 			sizeof(struct uring_bpf_ops_kern), GFP_KERNEL);
 	if (!ctx->bpf_ops)
 		return -ENOMEM;
+
 	return 0;
 }
 
diff --git a/io_uring/bpf_op.h b/io_uring/bpf_op.h
index 99708140992f..9de0606f5d25 100644
--- a/io_uring/bpf_op.h
+++ b/io_uring/bpf_op.h
@@ -27,14 +27,17 @@ typedef void (*uring_bpf_cleanup_t)(struct uring_bpf_data *data);
 
 struct uring_bpf_ops {
 	unsigned short		id;
+	int			ring_fd;
 	uring_bpf_prep_t	prep_fn;
 	uring_bpf_issue_t	issue_fn;
 	uring_bpf_fail_t	fail_fn;
 	uring_bpf_cleanup_t	cleanup_fn;
 };
 
+/* TODO: manage it via `io_rsrc_node` */
 struct uring_bpf_ops_kern {
 	const struct uring_bpf_ops *ops;
+	int refcount;
 };
 
 #ifdef CONFIG_IO_URING_BPF_OP
-- 
2.47.0

[PATCH V2 07/13] io_uring: bpf: add BPF buffer descriptor for IORING_OP_BPF

[Ming Lei]

Add io_bpf_buf_desc struct and io_bpf_buf_type enum to describe
buffer parameters for IORING_OP_BPF kfuncs. Supports plain userspace,
registered, vectored, and registered vectored buffer types.

Registered buffers (FIXED, KFIXED, REG_VEC) refer to buffers
pre-registered with io_uring and can be either userspace or kernel
buffers depending on how they were registered.

Signed-off-by: Ming Lei <ming.lei@redhat.com>
---
 include/uapi/linux/io_uring.h | 27 +++++++++++++++++++++++++++
 1 file changed, 27 insertions(+)

diff --git a/include/uapi/linux/io_uring.h b/include/uapi/linux/io_uring.h
index 441a1038a58a..113d8c7b8e05 100644
--- a/include/uapi/linux/io_uring.h
+++ b/include/uapi/linux/io_uring.h
@@ -433,6 +433,33 @@ enum io_uring_op {
 #define IORING_BPF_OP_BITS	8
 #define IORING_BPF_OP_SHIFT	24
 
+/*
+ * BPF buffer descriptor types.
+ *
+ * Registered buffers (FIXED, KFIXED, REG_VEC) refer to buffers pre-registered
+ * with io_uring. These can be either userspace or kernel buffers depending on
+ * how they were registered.
+ *
+ * For KFIXED, addr is an offset from the registered buffer start.
+ * For REG_VEC with kernel buffers, each iov.iov_base is offset-based.
+ */
+enum io_bpf_buf_type {
+	IO_BPF_BUF_USER		= 0,	/* plain userspace buffer */
+	IO_BPF_BUF_FIXED	= 1,	/* registered buffer (absolute address) */
+	IO_BPF_BUF_VEC		= 2,	/* vectored buffer (iovec array) */
+	IO_BPF_BUF_KFIXED	= 3,	/* registered buffer (offset-based) */
+	IO_BPF_BUF_REG_VEC	= 4,	/* registered vectored buffer */
+};
+
+/* BPF buffer descriptor for IORING_OP_BPF */
+struct io_bpf_buf_desc {
+	__u8  type;		/* IO_BPF_BUF_* */
+	__u8  reserved;
+	__u16 buf_index;	/* registered buffer index (FIXED/KFIXED/REG_VEC) */
+	__u32 len;		/* length (non-vec) or nr_vecs (vec types) */
+	__u64 addr;		/* userspace address, iovec ptr, or offset (KFIXED) */
+};
+
 /*
  * cqe.res for IORING_CQE_F_NOTIF if
  * IORING_SEND_ZC_REPORT_USAGE was requested
-- 
2.47.0

[PATCH V2 08/13] io_uring: bpf: add uring_bpf_memcpy() kfunc

[Ming Lei]

Add uring_bpf_memcpy() kfunc that copies data between io_uring BPF
buffers. This kfunc supports all 5 buffer types defined in
io_bpf_buf_desc:

- IO_BPF_BUF_USER: plain userspace buffer
- IO_BPF_BUF_FIXED: fixed buffer (absolute address within buffer)
- IO_BPF_BUF_VEC: vectored userspace buffer (iovec array)
- IO_BPF_BUF_KFIXED: kernel fixed buffer (offset-based addressing)
- IO_BPF_BUF_KVEC: kernel vectored buffer

Add helper functions for buffer import:
- io_bpf_import_fixed_buf(): handles FIXED/KFIXED types with proper
  node reference counting
- io_bpf_import_kvec_buf(): handles KVEC using __io_prep_reg_iovec()
  and __io_import_reg_vec()
- io_bpf_import_buffer(): unified dispatcher for all buffer types
- io_bpf_copy_iters(): page-based copy between iov_iters

The kfunc properly manages buffer node references and submit lock
for registered buffer access.

Signed-off-by: Ming Lei <ming.lei@redhat.com>
---
 io_uring/bpf_op.c | 320 +++++++++++++++++++++++++++++++++++++++++++++-
 io_uring/bpf_op.h |   3 +-
 2 files changed, 321 insertions(+), 2 deletions(-)

diff --git a/io_uring/bpf_op.c b/io_uring/bpf_op.c
index d6f146abe304..3c577aa3dfc4 100644
--- a/io_uring/bpf_op.c
+++ b/io_uring/bpf_op.c
@@ -10,9 +10,11 @@
 #include <linux/btf.h>
 #include <linux/btf_ids.h>
 #include <linux/filter.h>
+#include <linux/uio.h>
 #include <uapi/linux/io_uring.h>
 #include "io_uring.h"
 #include "register.h"
+#include "rsrc.h"
 #include "bpf_op.h"
 
 static inline unsigned char uring_bpf_get_op(u32 op_flags)
@@ -47,6 +49,7 @@ int io_uring_bpf_prep(struct io_kiocb *req, const struct io_uring_sqe *sqe)
 
 	data->opf = opf;
 	data->ops = ops;
+	data->issue_flags = 0;
 	ret = ops->prep_fn(data, sqe);
 	if (!ret) {
 		/* Only increment refcount on success (uring_lock already held) */
@@ -74,7 +77,13 @@ static int __io_uring_bpf_issue(struct io_kiocb *req)
 
 int io_uring_bpf_issue(struct io_kiocb *req, unsigned int issue_flags)
 {
-	return __io_uring_bpf_issue(req);
+	struct uring_bpf_data *data = io_kiocb_to_cmd(req, struct uring_bpf_data);
+	int ret;
+
+	data->issue_flags = issue_flags;
+	ret = __io_uring_bpf_issue(req);
+	data->issue_flags = 0;
+	return ret;
 }
 
 void io_uring_bpf_fail(struct io_kiocb *req)
@@ -291,6 +300,235 @@ static struct bpf_struct_ops bpf_uring_bpf_ops = {
 	.owner = THIS_MODULE,
 };
 
+/*
+ * Helper to copy data between two iov_iters using page extraction.
+ * Extracts pages from source iterator and copies them to destination.
+ * Returns number of bytes copied or negative error code.
+ */
+static ssize_t io_bpf_copy_iters(struct iov_iter *src, struct iov_iter *dst,
+				 size_t len)
+{
+#define MAX_PAGES_PER_LOOP 32
+	struct page *pages[MAX_PAGES_PER_LOOP];
+	size_t total_copied = 0;
+	bool need_unpin;
+
+	need_unpin = iov_iter_extract_will_pin(src);
+
+	while (len > 0) {
+		struct page **page_array = pages;
+		size_t offset, copied = 0;
+		ssize_t extracted;
+		unsigned int nr_pages;
+		size_t chunk_len;
+		int i;
+
+		chunk_len = min_t(size_t, len, MAX_PAGES_PER_LOOP * PAGE_SIZE);
+		extracted = iov_iter_extract_pages(src, &page_array, chunk_len,
+						   MAX_PAGES_PER_LOOP, 0, &offset);
+		if (extracted <= 0) {
+			if (total_copied > 0)
+				break;
+			return extracted < 0 ? extracted : -EFAULT;
+		}
+
+		nr_pages = DIV_ROUND_UP(offset + extracted, PAGE_SIZE);
+
+		for (i = 0; i < nr_pages && copied < extracted; i++) {
+			size_t page_offset = (i == 0) ? offset : 0;
+			size_t page_len = min_t(size_t, extracted - copied,
+						PAGE_SIZE - page_offset);
+			size_t n;
+
+			n = copy_page_to_iter(page_array[i], page_offset, page_len, dst);
+			copied += n;
+			if (n < page_len)
+				break;
+		}
+
+		if (need_unpin)
+			unpin_user_pages(page_array, nr_pages);
+
+		total_copied += copied;
+		len -= copied;
+
+		if (copied < extracted)
+			break;
+	}
+
+	return total_copied;
+#undef MAX_PAGES_PER_LOOP
+}
+
+/*
+ * Helper to import fixed buffer (FIXED or KFIXED).
+ * Must be called with submit lock held.
+ *
+ * FIXED: addr is absolute userspace address within buffer
+ * KFIXED: addr is offset from buffer start
+ *
+ * Returns node with incremented refcount on success, ERR_PTR on failure.
+ */
+static struct io_rsrc_node *io_bpf_import_fixed_buf(struct io_ring_ctx *ctx,
+						    struct iov_iter *iter,
+						    const struct io_bpf_buf_desc *desc,
+						    int ddir)
+{
+	struct io_rsrc_node *node;
+	struct io_mapped_ubuf *imu;
+	int ret;
+
+	node = io_rsrc_node_lookup(&ctx->buf_table, desc->buf_index);
+	if (!node)
+		return ERR_PTR(-EFAULT);
+
+	imu = node->buf;
+	if (!(imu->dir & (1 << ddir)))
+		return ERR_PTR(-EFAULT);
+
+	node->refs++;
+
+	ret = io_import_fixed(ddir, iter, imu, desc->addr, desc->len);
+	if (ret) {
+		node->refs--;
+		return ERR_PTR(ret);
+	}
+
+	return node;
+}
+
+/*
+ * Helper to import registered vectored buffer (KVEC).
+ * Must be called with submit lock held.
+ *
+ * addr: userspace iovec pointer
+ * len: number of iovecs
+ * buf_index: registered buffer index
+ *
+ * Returns node with incremented refcount on success, ERR_PTR on failure.
+ * Caller must call io_vec_free(vec) after use.
+ */
+static struct io_rsrc_node *io_bpf_import_reg_vec(struct io_ring_ctx *ctx,
+						   struct iov_iter *iter,
+						   const struct io_bpf_buf_desc *desc,
+						   int ddir, struct iou_vec *vec)
+{
+	struct io_rsrc_node *node;
+	struct io_mapped_ubuf *imu;
+	int ret;
+
+	node = io_rsrc_node_lookup(&ctx->buf_table, desc->buf_index);
+	if (!node)
+		return ERR_PTR(-EFAULT);
+
+	imu = node->buf;
+	if (!(imu->dir & (1 << ddir)))
+		return ERR_PTR(-EFAULT);
+
+	node->refs++;
+
+	/* Prepare iovec from userspace */
+	ret = __io_prep_reg_iovec(vec, u64_to_user_ptr(desc->addr),
+				  desc->len, ctx->compat, NULL);
+	if (ret)
+		goto err;
+
+	/* Import vectored buffer from registered buffer */
+	ret = __io_import_reg_vec(ddir, iter, imu, vec, desc->len, NULL);
+	if (ret)
+		goto err;
+
+	return node;
+err:
+	node->refs--;
+	return ERR_PTR(ret);
+}
+
+/*
+ * Helper to import a vectored user buffer (VEC) into iou_vec.
+ * Allocates space in vec and copies iovec from userspace.
+ *
+ * Returns 0 on success, negative error code on failure.
+ * Caller must call io_vec_free(vec) after use.
+ */
+static int io_bpf_import_vec_buf(struct io_ring_ctx *ctx,
+				 struct iov_iter *iter,
+				 const struct io_bpf_buf_desc *desc,
+				 int ddir, struct iou_vec *vec)
+{
+	unsigned nr_vecs = desc->len;
+	struct iovec *iov;
+	size_t total_len = 0;
+	void *res;
+	int ret, i;
+
+	if (nr_vecs > vec->nr) {
+		ret = io_vec_realloc(vec, nr_vecs);
+		if (ret)
+			return ret;
+	}
+
+	iov = vec->iovec;
+	res = iovec_from_user(u64_to_user_ptr(desc->addr), nr_vecs,
+			      nr_vecs, iov, ctx->compat);
+	if (IS_ERR(res))
+		return PTR_ERR(res);
+
+	for (i = 0; i < nr_vecs; i++)
+		total_len += iov[i].iov_len;
+
+	iov_iter_init(iter, ddir, iov, nr_vecs, total_len);
+	return 0;
+}
+
+/*
+ * Helper to import a buffer into an iov_iter based on io_bpf_buf_desc.
+ * Supports all 5 buffer types: USER, FIXED, VEC, KFIXED, KVEC.
+ * Must be called with submit lock held for FIXED/KFIXED/KVEC types.
+ *
+ * @ctx: ring context
+ * @iter: output iterator
+ * @desc: buffer descriptor
+ * @ddir: direction (ITER_SOURCE for source, ITER_DEST for destination)
+ * @vec: iou_vec for VEC/KVEC types (caller must call io_vec_free after use)
+ *
+ * Returns node pointer (may be NULL for USER/VEC), or ERR_PTR on failure.
+ * Caller must drop node reference when done if non-NULL.
+ */
+static struct io_rsrc_node *io_bpf_import_buffer(struct io_ring_ctx *ctx,
+						 struct iov_iter *iter,
+						 const struct io_bpf_buf_desc *desc,
+						 int ddir, struct iou_vec *vec)
+{
+	int ret;
+
+	switch (desc->type) {
+	case IO_BPF_BUF_USER:
+		/* Plain user buffer */
+		ret = import_ubuf(ddir, u64_to_user_ptr(desc->addr),
+				  desc->len, iter);
+		return ret ? ERR_PTR(ret) : NULL;
+
+	case IO_BPF_BUF_FIXED:
+	case IO_BPF_BUF_KFIXED:
+		/* FIXED: addr is absolute address within buffer */
+		/* KFIXED: addr is offset from buffer start */
+		return io_bpf_import_fixed_buf(ctx, iter, desc, ddir);
+
+	case IO_BPF_BUF_VEC:
+		/* Vectored user buffer - addr is iovec ptr, len is nr_vecs */
+		ret = io_bpf_import_vec_buf(ctx, iter, desc, ddir, vec);
+		return ret ? ERR_PTR(ret) : NULL;
+
+	case IO_BPF_BUF_REG_VEC:
+		/* Registered vectored buffer */
+		return io_bpf_import_reg_vec(ctx, iter, desc, ddir, vec);
+
+	default:
+		return ERR_PTR(-EINVAL);
+	}
+}
+
 __bpf_kfunc_start_defs();
 __bpf_kfunc void uring_bpf_set_result(struct uring_bpf_data *data, int res)
 {
@@ -300,10 +538,90 @@ __bpf_kfunc void uring_bpf_set_result(struct uring_bpf_data *data, int res)
 		req_set_fail(req);
 	io_req_set_res(req, res, 0);
 }
+
+/**
+ * uring_bpf_memcpy - Copy data between io_uring BPF buffers
+ * @data: BPF request data containing request context
+ * @dst: Destination buffer descriptor
+ * @src: Source buffer descriptor
+ *
+ * Copies data from source buffer to destination buffer.
+ * Supports all 5 buffer types: USER, FIXED, VEC, KFIXED, REG_VEC.
+ * The copy length is min of actual buffer sizes (for VEC types,
+ * total bytes across all vectors, not nr_vecs).
+ *
+ * Returns: Number of bytes copied on success, negative error code on failure
+ */
+__bpf_kfunc ssize_t uring_bpf_memcpy(const struct uring_bpf_data *data,
+				     struct io_bpf_buf_desc *dst,
+				     struct io_bpf_buf_desc *src)
+{
+	struct io_kiocb *req = cmd_to_io_kiocb((void *)data);
+	struct io_ring_ctx *ctx = req->ctx;
+	unsigned int issue_flags = data->issue_flags;
+	struct io_rsrc_node *src_node, *dst_node;
+	struct iov_iter src_iter, dst_iter;
+	struct iou_vec src_vec = {};
+	struct iou_vec dst_vec = {};
+	ssize_t ret;
+	size_t len;
+
+	/* Validate buffer types */
+	if (src->type > IO_BPF_BUF_REG_VEC || dst->type > IO_BPF_BUF_REG_VEC)
+		return -EINVAL;
+
+	io_ring_submit_lock(ctx, issue_flags);
+
+	/* Import source buffer */
+	src_node = io_bpf_import_buffer(ctx, &src_iter, src, ITER_SOURCE,
+					&src_vec);
+	if (IS_ERR(src_node)) {
+		ret = PTR_ERR(src_node);
+		goto unlock;
+	}
+
+	/* Import destination buffer */
+	dst_node = io_bpf_import_buffer(ctx, &dst_iter, dst, ITER_DEST,
+					&dst_vec);
+	if (IS_ERR(dst_node)) {
+		ret = PTR_ERR(dst_node);
+		goto put_src;
+	}
+
+	/*
+	 * Calculate copy length from actual iterator sizes.
+	 * For VEC types, desc->len is nr_vecs, not total bytes.
+	 */
+	len = min(iov_iter_count(&src_iter), iov_iter_count(&dst_iter));
+	if (!len) {
+		ret = 0;
+		goto put_dst;
+	}
+	if (len > MAX_RW_COUNT) {
+		ret = -EINVAL;
+		goto put_dst;
+	}
+
+	/* Copy data between iterators */
+	ret = io_bpf_copy_iters(&src_iter, &dst_iter, len);
+
+put_dst:
+	io_vec_free(&dst_vec);
+	if (dst_node)
+		io_put_rsrc_node(ctx, dst_node);
+put_src:
+	io_vec_free(&src_vec);
+	if (src_node)
+		io_put_rsrc_node(ctx, src_node);
+unlock:
+	io_ring_submit_unlock(ctx, issue_flags);
+	return ret;
+}
 __bpf_kfunc_end_defs();
 
 BTF_KFUNCS_START(uring_bpf_kfuncs)
 BTF_ID_FLAGS(func, uring_bpf_set_result)
+BTF_ID_FLAGS(func, uring_bpf_memcpy)
 BTF_KFUNCS_END(uring_bpf_kfuncs)
 
 static const struct btf_kfunc_id_set uring_kfunc_set = {
diff --git a/io_uring/bpf_op.h b/io_uring/bpf_op.h
index 9de0606f5d25..6004fb906983 100644
--- a/io_uring/bpf_op.h
+++ b/io_uring/bpf_op.h
@@ -13,10 +13,11 @@ struct uring_bpf_data {
 	void				*req_data;  /* not for bpf prog */
 	const struct uring_bpf_ops	*ops;
 	u32				opf;
+	u32				issue_flags; /* io_uring issue flags */
 
 	/* writeable for bpf prog */
 	u8              pdu[64 - sizeof(void *) -
-		sizeof(struct uring_bpf_ops *) - sizeof(u32)];
+		sizeof(struct uring_bpf_ops *) - 2 * sizeof(u32)];
 };
 
 typedef int (*uring_bpf_prep_t)(struct uring_bpf_data *data,
-- 
2.47.0

[PATCH V2 09/13] selftests/io_uring: update mini liburing

[Ming Lei]

From: Pavel Begunkov <asml.silence@gmail.com>

Add a helper for creating a ring from parameters and add support for
IORING_SETUP_NO_SQARRAY.

Add io_uring_unregister_buffers().

Signed-off-by: Pavel Begunkov <asml.silence@gmail.com>
Signed-off-by: Ming Lei <ming.lei@redhat.com>
---
 tools/include/io_uring/mini_liburing.h | 67 ++++++++++++++++++++------
 1 file changed, 53 insertions(+), 14 deletions(-)

diff --git a/tools/include/io_uring/mini_liburing.h b/tools/include/io_uring/mini_liburing.h
index 9ccb16074eb5..8070e63437d3 100644
--- a/tools/include/io_uring/mini_liburing.h
+++ b/tools/include/io_uring/mini_liburing.h
@@ -1,11 +1,13 @@
 /* SPDX-License-Identifier: MIT */
 
 #include <linux/io_uring.h>
+#include <signal.h>
 #include <sys/mman.h>
 #include <sys/syscall.h>
 #include <stdio.h>
 #include <string.h>
 #include <unistd.h>
+#include <sys/uio.h>
 
 struct io_sq_ring {
 	unsigned int *head;
@@ -55,6 +57,7 @@ struct io_uring {
 	struct io_uring_sq sq;
 	struct io_uring_cq cq;
 	int ring_fd;
+	unsigned flags;
 };
 
 #if defined(__x86_64) || defined(__i386__)
@@ -72,7 +75,14 @@ static inline int io_uring_mmap(int fd, struct io_uring_params *p,
 	void *ptr;
 	int ret;
 
-	sq->ring_sz = p->sq_off.array + p->sq_entries * sizeof(unsigned int);
+	if (p->flags & IORING_SETUP_NO_SQARRAY) {
+		sq->ring_sz = p->cq_off.cqes;
+		sq->ring_sz += p->cq_entries * sizeof(struct io_uring_cqe);
+	} else {
+		sq->ring_sz = p->sq_off.array;
+		sq->ring_sz += p->sq_entries * sizeof(unsigned int);
+	}
+
 	ptr = mmap(0, sq->ring_sz, PROT_READ | PROT_WRITE,
 		   MAP_SHARED | MAP_POPULATE, fd, IORING_OFF_SQ_RING);
 	if (ptr == MAP_FAILED)
@@ -83,7 +93,8 @@ static inline int io_uring_mmap(int fd, struct io_uring_params *p,
 	sq->kring_entries = ptr + p->sq_off.ring_entries;
 	sq->kflags = ptr + p->sq_off.flags;
 	sq->kdropped = ptr + p->sq_off.dropped;
-	sq->array = ptr + p->sq_off.array;
+	if (!(p->flags & IORING_SETUP_NO_SQARRAY))
+		sq->array = ptr + p->sq_off.array;
 
 	size = p->sq_entries * sizeof(struct io_uring_sqe);
 	sq->sqes = mmap(0, size, PROT_READ | PROT_WRITE,
@@ -126,28 +137,39 @@ static inline int io_uring_enter(int fd, unsigned int to_submit,
 		       flags, sig, _NSIG / 8);
 }
 
-static inline int io_uring_queue_init(unsigned int entries,
+static inline int io_uring_queue_init_params(unsigned int entries,
 				      struct io_uring *ring,
-				      unsigned int flags)
+				      struct io_uring_params *p)
 {
-	struct io_uring_params p;
 	int fd, ret;
 
 	memset(ring, 0, sizeof(*ring));
-	memset(&p, 0, sizeof(p));
-	p.flags = flags;
 
-	fd = io_uring_setup(entries, &p);
+	fd = io_uring_setup(entries, p);
 	if (fd < 0)
 		return fd;
-	ret = io_uring_mmap(fd, &p, &ring->sq, &ring->cq);
-	if (!ret)
+	ret = io_uring_mmap(fd, p, &ring->sq, &ring->cq);
+	if (!ret) {
 		ring->ring_fd = fd;
-	else
+		ring->flags = p->flags;
+	} else {
 		close(fd);
+	}
 	return ret;
 }
 
+static inline int io_uring_queue_init(unsigned int entries,
+				      struct io_uring *ring,
+				      unsigned int flags)
+{
+	struct io_uring_params p;
+
+	memset(&p, 0, sizeof(p));
+	p.flags = flags;
+
+	return io_uring_queue_init_params(entries, ring, &p);
+}
+
 /* Get a sqe */
 static inline struct io_uring_sqe *io_uring_get_sqe(struct io_uring *ring)
 {
@@ -199,10 +221,18 @@ static inline int io_uring_submit(struct io_uring *ring)
 
 	ktail = *sq->ktail;
 	to_submit = sq->sqe_tail - sq->sqe_head;
-	for (submitted = 0; submitted < to_submit; submitted++) {
-		read_barrier();
-		sq->array[ktail++ & mask] = sq->sqe_head++ & mask;
+
+	if (!(ring->flags & IORING_SETUP_NO_SQARRAY)) {
+		for (submitted = 0; submitted < to_submit; submitted++) {
+			read_barrier();
+			sq->array[ktail++ & mask] = sq->sqe_head++ & mask;
+		}
+	} else {
+		ktail += to_submit;
+		sq->sqe_head += to_submit;
+		submitted = to_submit;
 	}
+
 	if (!submitted)
 		return 0;
 
@@ -255,6 +285,15 @@ static inline int io_uring_register_buffers(struct io_uring *ring,
 	return (ret < 0) ? -errno : ret;
 }
 
+static inline int io_uring_unregister_buffers(struct io_uring *ring)
+{
+	int ret;
+
+	ret = syscall(__NR_io_uring_register, ring->ring_fd,
+		      IORING_UNREGISTER_BUFFERS, NULL, 0);
+	return (ret < 0) ? -errno : ret;
+}
+
 static inline void io_uring_prep_send(struct io_uring_sqe *sqe, int sockfd,
 				      const void *buf, size_t len, int flags)
 {
-- 
2.47.0

[PATCH V2 10/13] selftests/io_uring: add BPF struct_ops and kfunc tests

[Ming Lei]

Add selftests for io_uring BPF struct_ops and kfunc functionality:

- basic_bpf_ops: Tests IORING_OP_BPF struct_ops registration and execution
  with multiple struct_ops support

The test framework includes:
- runner.c: Main test runner with auto-discovery
- iou_test.h: Common test infrastructure
- Makefile: Build system with BPF skeleton generation

Signed-off-by: Ming Lei <ming.lei@redhat.com>
---
 tools/testing/selftests/Makefile              |   3 +-
 tools/testing/selftests/io_uring/.gitignore   |   2 +
 tools/testing/selftests/io_uring/Makefile     | 172 ++++++++++++++
 .../selftests/io_uring/basic_bpf_ops.bpf.c    |  94 ++++++++
 .../selftests/io_uring/basic_bpf_ops.c        | 215 ++++++++++++++++++
 .../selftests/io_uring/include/iou_test.h     |  98 ++++++++
 tools/testing/selftests/io_uring/runner.c     | 206 +++++++++++++++++
 7 files changed, 789 insertions(+), 1 deletion(-)
 create mode 100644 tools/testing/selftests/io_uring/.gitignore
 create mode 100644 tools/testing/selftests/io_uring/Makefile
 create mode 100644 tools/testing/selftests/io_uring/basic_bpf_ops.bpf.c
 create mode 100644 tools/testing/selftests/io_uring/basic_bpf_ops.c
 create mode 100644 tools/testing/selftests/io_uring/include/iou_test.h
 create mode 100644 tools/testing/selftests/io_uring/runner.c

diff --git a/tools/testing/selftests/Makefile b/tools/testing/selftests/Makefile
index 56e44a98d6a5..c742af56ec51 100644
--- a/tools/testing/selftests/Makefile
+++ b/tools/testing/selftests/Makefile
@@ -44,6 +44,7 @@ TARGETS += futex
 TARGETS += gpio
 TARGETS += hid
 TARGETS += intel_pstate
+TARGETS += io_uring
 TARGETS += iommu
 TARGETS += ipc
 TARGETS += ir
@@ -147,7 +148,7 @@ endif
 # User can optionally provide a TARGETS skiplist. By default we skip
 # targets using BPF since it has cutting edge build time dependencies
 # which require more effort to install.
-SKIP_TARGETS ?= bpf sched_ext
+SKIP_TARGETS ?= bpf io_uring sched_ext
 ifneq ($(SKIP_TARGETS),)
 	TMP := $(filter-out $(SKIP_TARGETS), $(TARGETS))
 	override TARGETS := $(TMP)
diff --git a/tools/testing/selftests/io_uring/.gitignore b/tools/testing/selftests/io_uring/.gitignore
new file mode 100644
index 000000000000..c0e488dc0622
--- /dev/null
+++ b/tools/testing/selftests/io_uring/.gitignore
@@ -0,0 +1,2 @@
+/build/
+/runner
diff --git a/tools/testing/selftests/io_uring/Makefile b/tools/testing/selftests/io_uring/Makefile
new file mode 100644
index 000000000000..f88a6a749484
--- /dev/null
+++ b/tools/testing/selftests/io_uring/Makefile
@@ -0,0 +1,172 @@
+# SPDX-License-Identifier: GPL-2.0
+# Copyright (c) 2025 Red Hat, Inc.
+include ../../../build/Build.include
+include ../../../scripts/Makefile.arch
+include ../../../scripts/Makefile.include
+
+TEST_GEN_PROGS := runner
+
+# override lib.mk's default rules
+OVERRIDE_TARGETS := 1
+include ../lib.mk
+
+CURDIR := $(abspath .)
+REPOROOT := $(abspath ../../../..)
+TOOLSDIR := $(REPOROOT)/tools
+LIBDIR := $(TOOLSDIR)/lib
+BPFDIR := $(LIBDIR)/bpf
+TOOLSINCDIR := $(TOOLSDIR)/include
+BPFTOOLDIR := $(TOOLSDIR)/bpf/bpftool
+APIDIR := $(TOOLSINCDIR)/uapi
+GENDIR := $(REPOROOT)/include/generated
+GENHDR := $(GENDIR)/autoconf.h
+
+OUTPUT_DIR := $(OUTPUT)/build
+OBJ_DIR := $(OUTPUT_DIR)/obj
+INCLUDE_DIR := $(OUTPUT_DIR)/include
+BPFOBJ_DIR := $(OBJ_DIR)/libbpf
+IOUOBJ_DIR := $(OBJ_DIR)/io_uring
+BPFOBJ := $(BPFOBJ_DIR)/libbpf.a
+LIBBPF_OUTPUT := $(OBJ_DIR)/libbpf/libbpf.a
+
+DEFAULT_BPFTOOL := $(OUTPUT_DIR)/host/sbin/bpftool
+HOST_OBJ_DIR := $(OBJ_DIR)/host/bpftool
+HOST_LIBBPF_OUTPUT := $(OBJ_DIR)/host/libbpf/
+HOST_LIBBPF_DESTDIR := $(OUTPUT_DIR)/host/
+HOST_DESTDIR := $(OUTPUT_DIR)/host/
+
+VMLINUX_BTF_PATHS ?= $(if $(O),$(O)/vmlinux)					\
+		     $(if $(KBUILD_OUTPUT),$(KBUILD_OUTPUT)/vmlinux)		\
+		     ../../../../vmlinux					\
+		     /sys/kernel/btf/vmlinux					\
+		     /boot/vmlinux-$(shell uname -r)
+VMLINUX_BTF ?= $(abspath $(firstword $(wildcard $(VMLINUX_BTF_PATHS))))
+ifeq ($(VMLINUX_BTF),)
+$(error Cannot find a vmlinux for VMLINUX_BTF at any of "$(VMLINUX_BTF_PATHS)")
+endif
+
+BPFTOOL ?= $(DEFAULT_BPFTOOL)
+
+ifneq ($(wildcard $(GENHDR)),)
+  GENFLAGS := -DHAVE_GENHDR
+endif
+
+CFLAGS += -g -O2 -rdynamic -pthread -Wall -Werror $(GENFLAGS)			\
+	  -I$(INCLUDE_DIR) -I$(GENDIR) -I$(LIBDIR)				\
+	  -I$(REPOROOT)/usr/include						\
+	  -I$(TOOLSINCDIR) -I$(APIDIR) -I$(CURDIR)/include
+
+# Silence some warnings when compiled with clang
+ifneq ($(LLVM),)
+CFLAGS += -Wno-unused-command-line-argument
+endif
+
+LDFLAGS = -lelf -lz -lpthread -lzstd
+
+IS_LITTLE_ENDIAN = $(shell $(CC) -dM -E - </dev/null |				\
+			grep 'define __BYTE_ORDER__ __ORDER_LITTLE_ENDIAN__')
+
+# Get Clang's default includes on this system
+define get_sys_includes
+$(shell $(1) $(2) -v -E - </dev/null 2>&1 \
+	| sed -n '/<...> search starts here:/,/End of search list./{ s| \(/.*\)|-idirafter \1|p }') \
+$(shell $(1) $(2) -dM -E - </dev/null | grep '__riscv_xlen ' | awk '{printf("-D__riscv_xlen=%d -D__BITS_PER_LONG=%d", $$3, $$3)}')
+endef
+
+ifneq ($(CROSS_COMPILE),)
+CLANG_TARGET_ARCH = --target=$(notdir $(CROSS_COMPILE:%-=%))
+endif
+
+CLANG_SYS_INCLUDES = $(call get_sys_includes,$(CLANG),$(CLANG_TARGET_ARCH))
+
+BPF_CFLAGS = -g -D__TARGET_ARCH_$(SRCARCH)					\
+	     $(if $(IS_LITTLE_ENDIAN),-mlittle-endian,-mbig-endian)		\
+	     -I$(CURDIR)/include -I$(CURDIR)/include/bpf-compat			\
+	     -I$(INCLUDE_DIR) -I$(APIDIR)					\
+	     -I$(REPOROOT)/include						\
+	     $(CLANG_SYS_INCLUDES)						\
+	     -Wall -Wno-compare-distinct-pointer-types				\
+	     -Wno-incompatible-function-pointer-types				\
+	     -Wno-missing-declarations						\
+	     -O2 -mcpu=v3
+
+# sort removes libbpf duplicates when not cross-building
+MAKE_DIRS := $(sort $(OBJ_DIR)/libbpf $(OBJ_DIR)/libbpf				\
+	       $(OBJ_DIR)/bpftool $(OBJ_DIR)/resolve_btfids			\
+	       $(HOST_OBJ_DIR) $(INCLUDE_DIR) $(IOUOBJ_DIR))
+
+$(MAKE_DIRS):
+	$(call msg,MKDIR,,$@)
+	$(Q)mkdir -p $@
+
+$(BPFOBJ): $(wildcard $(BPFDIR)/*.[ch] $(BPFDIR)/Makefile)			\
+	   $(APIDIR)/linux/bpf.h						\
+	   | $(OBJ_DIR)/libbpf
+	$(Q)$(MAKE) $(submake_extras) -C $(BPFDIR) OUTPUT=$(OBJ_DIR)/libbpf/	\
+		    ARCH=$(ARCH) CC="$(CC)" CROSS_COMPILE=$(CROSS_COMPILE)	\
+		    EXTRA_CFLAGS='-g -O0 -fPIC'					\
+		    DESTDIR=$(OUTPUT_DIR) prefix= all install_headers
+
+$(DEFAULT_BPFTOOL): $(wildcard $(BPFTOOLDIR)/*.[ch] $(BPFTOOLDIR)/Makefile)	\
+		    $(LIBBPF_OUTPUT) | $(HOST_OBJ_DIR)
+	$(Q)$(MAKE) $(submake_extras)  -C $(BPFTOOLDIR)				\
+		    ARCH= CROSS_COMPILE= CC=$(HOSTCC) LD=$(HOSTLD)		\
+		    EXTRA_CFLAGS='-g -O0'					\
+		    OUTPUT=$(HOST_OBJ_DIR)/					\
+		    LIBBPF_OUTPUT=$(HOST_LIBBPF_OUTPUT)				\
+		    LIBBPF_DESTDIR=$(HOST_LIBBPF_DESTDIR)			\
+		    prefix= DESTDIR=$(HOST_DESTDIR) install-bin
+
+$(INCLUDE_DIR)/vmlinux.h: $(VMLINUX_BTF) $(BPFTOOL) | $(INCLUDE_DIR)
+ifeq ($(VMLINUX_H),)
+	$(call msg,GEN,,$@)
+	$(Q)$(BPFTOOL) btf dump file $(VMLINUX_BTF) format c > $@
+else
+	$(call msg,CP,,$@)
+	$(Q)cp "$(VMLINUX_H)" $@
+endif
+
+$(IOUOBJ_DIR)/%.bpf.o: %.bpf.c $(INCLUDE_DIR)/vmlinux.h	| $(BPFOBJ) $(IOUOBJ_DIR)
+	$(call msg,CLNG-BPF,,$(notdir $@))
+	$(Q)$(CLANG) $(BPF_CFLAGS) -target bpf -c $< -o $@
+
+$(INCLUDE_DIR)/%.bpf.skel.h: $(IOUOBJ_DIR)/%.bpf.o $(INCLUDE_DIR)/vmlinux.h $(BPFTOOL) | $(INCLUDE_DIR)
+	$(eval sched=$(notdir $@))
+	$(call msg,GEN-SKEL,,$(sched))
+	$(Q)$(BPFTOOL) gen object $(<:.o=.linked1.o) $<
+	$(Q)$(BPFTOOL) gen object $(<:.o=.linked2.o) $(<:.o=.linked1.o)
+	$(Q)$(BPFTOOL) gen object $(<:.o=.linked3.o) $(<:.o=.linked2.o)
+	$(Q)diff $(<:.o=.linked2.o) $(<:.o=.linked3.o)
+	$(Q)$(BPFTOOL) gen skeleton $(<:.o=.linked3.o) name $(subst .bpf.skel.h,,$(sched)) > $@
+	$(Q)$(BPFTOOL) gen subskeleton $(<:.o=.linked3.o) name $(subst .bpf.skel.h,,$(sched)) > $(@:.skel.h=.subskel.h)
+
+override define CLEAN
+	rm -rf $(OUTPUT_DIR)
+	rm -f $(TEST_GEN_PROGS)
+endef
+
+# Every testcase takes all of the BPF progs as dependencies by default.
+all_test_bpfprogs := $(foreach prog,$(wildcard *.bpf.c),$(INCLUDE_DIR)/$(patsubst %.c,%.skel.h,$(prog)))
+
+auto-test-targets :=			\
+	basic_bpf_ops			\
+
+testcase-targets := $(addsuffix .o,$(addprefix $(IOUOBJ_DIR)/,$(auto-test-targets)))
+
+$(IOUOBJ_DIR)/runner.o: runner.c | $(IOUOBJ_DIR) $(BPFOBJ)
+	$(call msg,CC,,$@)
+	$(Q)$(CC) $(CFLAGS) -c $< -o $@
+
+$(testcase-targets): $(IOUOBJ_DIR)/%.o: %.c $(IOUOBJ_DIR)/runner.o $(all_test_bpfprogs) | $(IOUOBJ_DIR)
+	$(call msg,CC,,$@)
+	$(Q)$(CC) $(CFLAGS) -c $< -o $@
+
+$(OUTPUT)/runner: $(IOUOBJ_DIR)/runner.o $(BPFOBJ) $(testcase-targets)
+	$(call msg,LINK,,$@)
+	$(Q)$(CC) $(CFLAGS) -o $@ $^ $(LDFLAGS)
+
+.DEFAULT_GOAL := all
+
+.DELETE_ON_ERROR:
+
+.SECONDARY:
diff --git a/tools/testing/selftests/io_uring/basic_bpf_ops.bpf.c b/tools/testing/selftests/io_uring/basic_bpf_ops.bpf.c
new file mode 100644
index 000000000000..2343c647575b
--- /dev/null
+++ b/tools/testing/selftests/io_uring/basic_bpf_ops.bpf.c
@@ -0,0 +1,94 @@
+/* SPDX-License-Identifier: GPL-2.0 */
+/*
+ * Copyright (c) 2025 Red Hat, Inc.
+ * Basic io_uring BPF struct_ops test.
+ *
+ * This tests registering a minimal uring_bpf_ops struct_ops
+ * with prep/issue/cleanup callbacks.
+ */
+
+#include "vmlinux.h"
+#include <bpf/bpf_helpers.h>
+#include <bpf/bpf_tracing.h>
+
+char LICENSE[] SEC("license") = "GPL";
+
+/* Counters to verify callbacks are invoked */
+int prep_count = 0;
+int issue_count = 0;
+int cleanup_count = 0;
+
+/* Test result stored in pdu */
+#define PDU_MAGIC 0xdeadbeef
+
+SEC("struct_ops/basic_prep")
+int BPF_PROG(basic_prep, struct uring_bpf_data *data,
+	     const struct io_uring_sqe *sqe)
+{
+	__u32 *magic;
+
+	prep_count++;
+
+	/* Store magic value in pdu to verify data flow */
+	magic = (__u32 *)data->pdu;
+	*magic = PDU_MAGIC;
+
+	bpf_printk("basic_prep: count=%d", prep_count);
+	return 0;
+}
+
+extern void uring_bpf_set_result(struct uring_bpf_data *data, int res) __ksym;
+
+SEC("struct_ops/basic_issue")
+int BPF_PROG(basic_issue, struct uring_bpf_data *data)
+{
+	__u32 *magic;
+
+	issue_count++;
+
+	/* Verify pdu contains the magic value from prep */
+	magic = (__u32 *)data->pdu;
+	if (*magic != PDU_MAGIC) {
+		bpf_printk("basic_issue: pdu magic mismatch!");
+		uring_bpf_set_result(data, -22); /* -EINVAL */
+		return 0;
+	}
+
+	bpf_printk("basic_issue: count=%d, pdu_magic=0x%x", issue_count, *magic);
+
+	/* Set successful result */
+	uring_bpf_set_result(data, 42);
+	return 0;
+}
+
+SEC("struct_ops/basic_fail")
+void BPF_PROG(basic_fail, struct uring_bpf_data *data)
+{
+	bpf_printk("basic_fail: invoked");
+}
+
+SEC("struct_ops/basic_cleanup")
+void BPF_PROG(basic_cleanup, struct uring_bpf_data *data)
+{
+	cleanup_count++;
+	bpf_printk("basic_cleanup: count=%d", cleanup_count);
+}
+
+SEC(".struct_ops.link")
+struct uring_bpf_ops basic_bpf_ops = {
+	.id		= 0,
+	.prep_fn	= (void *)basic_prep,
+	.issue_fn	= (void *)basic_issue,
+	.fail_fn	= (void *)basic_fail,
+	.cleanup_fn	= (void *)basic_cleanup,
+};
+
+/* Second struct_ops to verify multiple registrations work */
+SEC(".struct_ops.link")
+struct uring_bpf_ops basic_bpf_ops_2 = {
+	.id		= 1,
+	.prep_fn	= (void *)basic_prep,
+	.issue_fn	= (void *)basic_issue,
+	.fail_fn	= (void *)basic_fail,
+	.cleanup_fn	= (void *)basic_cleanup,
+};
diff --git a/tools/testing/selftests/io_uring/basic_bpf_ops.c b/tools/testing/selftests/io_uring/basic_bpf_ops.c
new file mode 100644
index 000000000000..c68aea0b5ed7
--- /dev/null
+++ b/tools/testing/selftests/io_uring/basic_bpf_ops.c
@@ -0,0 +1,215 @@
+/* SPDX-License-Identifier: GPL-2.0 */
+/*
+ * Copyright (c) 2025 Red Hat, Inc.
+ * Basic io_uring BPF struct_ops test - userspace part.
+ */
+#include <bpf/bpf.h>
+#include <bpf/libbpf.h>
+#include <errno.h>
+#include <linux/io_uring.h>
+#include <io_uring/mini_liburing.h>
+
+#include "iou_test.h"
+#include "basic_bpf_ops.bpf.skel.h"
+
+struct test_ctx {
+	struct basic_bpf_ops *skel;
+	struct bpf_link *link;
+	struct bpf_link *link_2;
+	struct io_uring ring;
+	int nr_ops;
+};
+
+static enum iou_test_status bpf_setup(struct test_ctx *ctx)
+{
+	int ret;
+
+	/* Load BPF skeleton */
+	ctx->skel = basic_bpf_ops__open();
+	if (!ctx->skel) {
+		IOU_ERR("Failed to open BPF skeleton");
+		return IOU_TEST_FAIL;
+	}
+
+	/* Set ring_fd in struct_ops before loading (id is hardcoded in BPF) */
+	ctx->skel->struct_ops.basic_bpf_ops->ring_fd = ctx->ring.ring_fd;
+	ctx->skel->struct_ops.basic_bpf_ops_2->ring_fd = ctx->ring.ring_fd;
+
+	ret = basic_bpf_ops__load(ctx->skel);
+	if (ret) {
+		IOU_ERR("Failed to load BPF skeleton: %d", ret);
+		basic_bpf_ops__destroy(ctx->skel);
+		ctx->skel = NULL;
+		return IOU_TEST_FAIL;
+	}
+
+	/* Attach first struct_ops */
+	ctx->link = bpf_map__attach_struct_ops(ctx->skel->maps.basic_bpf_ops);
+	if (!ctx->link) {
+		IOU_ERR("Failed to attach struct_ops");
+		basic_bpf_ops__destroy(ctx->skel);
+		ctx->skel = NULL;
+		return IOU_TEST_FAIL;
+	}
+	ctx->nr_ops++;
+
+	/* Attach second struct_ops */
+	ctx->link_2 = bpf_map__attach_struct_ops(ctx->skel->maps.basic_bpf_ops_2);
+	if (!ctx->link_2) {
+		IOU_ERR("Failed to attach struct_ops_2");
+		bpf_link__destroy(ctx->link);
+		ctx->link = NULL;
+		basic_bpf_ops__destroy(ctx->skel);
+		ctx->skel = NULL;
+		return IOU_TEST_FAIL;
+	}
+	ctx->nr_ops++;
+
+	return IOU_TEST_PASS;
+}
+
+static enum iou_test_status setup(void **ctx_out)
+{
+	struct io_uring_params p;
+	struct test_ctx *ctx;
+	enum iou_test_status status;
+	int ret;
+
+	ctx = calloc(1, sizeof(*ctx));
+	if (!ctx) {
+		IOU_ERR("Failed to allocate context");
+		return IOU_TEST_FAIL;
+	}
+
+	/* Setup io_uring ring with BPF_OP flag */
+	memset(&p, 0, sizeof(p));
+	p.flags = IORING_SETUP_BPF_OP | IORING_SETUP_NO_SQARRAY;
+
+	ret = io_uring_queue_init_params(8, &ctx->ring, &p);
+	if (ret < 0) {
+		IOU_ERR("io_uring_queue_init_params failed: %s (flags=0x%x)",
+			strerror(-ret), p.flags);
+		free(ctx);
+		return IOU_TEST_SKIP;
+	}
+
+	status = bpf_setup(ctx);
+	if (status != IOU_TEST_PASS) {
+		io_uring_queue_exit(&ctx->ring);
+		free(ctx);
+		return status;
+	}
+
+	*ctx_out = ctx;
+	return IOU_TEST_PASS;
+}
+
+static enum iou_test_status test_bpf_op(struct test_ctx *ctx, int op_id)
+{
+	struct io_uring_sqe *sqe;
+	struct io_uring_cqe *cqe;
+	__u64 user_data = 0x12345678 + op_id;
+	int ret;
+
+	sqe = io_uring_get_sqe(&ctx->ring);
+	if (!sqe) {
+		IOU_ERR("Failed to get SQE for op %d", op_id);
+		return IOU_TEST_FAIL;
+	}
+
+	memset(sqe, 0, sizeof(*sqe));
+	sqe->opcode = IORING_OP_BPF;
+	sqe->fd = -1;
+	sqe->bpf_op_flags = (op_id << IORING_BPF_OP_SHIFT);
+	sqe->user_data = user_data;
+
+	ret = io_uring_submit(&ctx->ring);
+	if (ret < 0) {
+		IOU_ERR("io_uring_submit for op %d failed: %d", op_id, ret);
+		return IOU_TEST_FAIL;
+	}
+
+	ret = io_uring_wait_cqe(&ctx->ring, &cqe);
+	if (ret < 0) {
+		IOU_ERR("io_uring_wait_cqe for op %d failed: %d", op_id, ret);
+		return IOU_TEST_FAIL;
+	}
+
+	if (cqe->user_data != user_data) {
+		IOU_ERR("CQE user_data mismatch for op %d: 0x%llx", op_id, cqe->user_data);
+		return IOU_TEST_FAIL;
+	}
+
+	if (cqe->res != 42) {
+		IOU_ERR("CQE result mismatch for op %d: %d (expected 42)", op_id, cqe->res);
+		return IOU_TEST_FAIL;
+	}
+
+	io_uring_cqe_seen(&ctx->ring);
+	return IOU_TEST_PASS;
+}
+
+static enum iou_test_status verify_counters(struct test_ctx *ctx, int expected)
+{
+	if (ctx->skel->bss->prep_count != expected) {
+		IOU_ERR("prep_count mismatch: %d (expected %d)",
+			ctx->skel->bss->prep_count, expected);
+		return IOU_TEST_FAIL;
+	}
+	if (ctx->skel->bss->issue_count != expected) {
+		IOU_ERR("issue_count mismatch: %d (expected %d)",
+			ctx->skel->bss->issue_count, expected);
+		return IOU_TEST_FAIL;
+	}
+	if (ctx->skel->bss->cleanup_count != expected) {
+		IOU_ERR("cleanup_count mismatch: %d (expected %d)",
+			ctx->skel->bss->cleanup_count, expected);
+		return IOU_TEST_FAIL;
+	}
+	return IOU_TEST_PASS;
+}
+
+static enum iou_test_status run(void *ctx_ptr)
+{
+	struct test_ctx *ctx = ctx_ptr;
+	enum iou_test_status status;
+	int i;
+
+	/* Test all registered struct_ops */
+	for (i = 0; i < ctx->nr_ops; i++) {
+		status = test_bpf_op(ctx, i);
+		if (status != IOU_TEST_PASS)
+			return status;
+
+		/* Verify counters after each op */
+		status = verify_counters(ctx, i + 1);
+		if (status != IOU_TEST_PASS)
+			return status;
+	}
+
+	IOU_INFO("IORING_OP_BPF multiple struct_ops test passed");
+	return IOU_TEST_PASS;
+}
+
+static void cleanup(void *ctx_ptr)
+{
+	struct test_ctx *ctx = ctx_ptr;
+
+	if (ctx->link_2)
+		bpf_link__destroy(ctx->link_2);
+	if (ctx->link)
+		bpf_link__destroy(ctx->link);
+	if (ctx->skel)
+		basic_bpf_ops__destroy(ctx->skel);
+	io_uring_queue_exit(&ctx->ring);
+	free(ctx);
+}
+
+struct iou_test basic_bpf_ops_test = {
+	.name = "basic_bpf_ops",
+	.description = "Test IORING_OP_BPF struct_ops registration and execution",
+	.setup = setup,
+	.run = run,
+	.cleanup = cleanup,
+};
+REGISTER_IOU_TEST(basic_bpf_ops_test)
diff --git a/tools/testing/selftests/io_uring/include/iou_test.h b/tools/testing/selftests/io_uring/include/iou_test.h
new file mode 100644
index 000000000000..8e7880e81314
--- /dev/null
+++ b/tools/testing/selftests/io_uring/include/iou_test.h
@@ -0,0 +1,98 @@
+/* SPDX-License-Identifier: GPL-2.0 */
+/*
+ * Copyright (c) 2025 Red Hat, Inc.
+ */
+
+#ifndef __IOU_TEST_H__
+#define __IOU_TEST_H__
+
+#include <errno.h>
+#include <stdio.h>
+#include <stdlib.h>
+#include <unistd.h>
+
+enum iou_test_status {
+	IOU_TEST_PASS = 0,
+	IOU_TEST_SKIP,
+	IOU_TEST_FAIL,
+};
+
+struct iou_test {
+	/**
+	 * name - The name of the testcase.
+	 */
+	const char *name;
+
+	/**
+	 * description - A description of the testcase.
+	 */
+	const char *description;
+
+	/**
+	 * setup - Setup callback to initialize the test.
+	 * @ctx: A pointer to a context object that will be passed to run
+	 *       and cleanup.
+	 *
+	 * Return: IOU_TEST_PASS if setup was successful, IOU_TEST_SKIP
+	 *         if the test should be skipped, or IOU_TEST_FAIL if the
+	 *         test should be marked as failed.
+	 */
+	enum iou_test_status (*setup)(void **ctx);
+
+	/**
+	 * run - The main test function.
+	 * @ctx: Context object returned from setup().
+	 *
+	 * Return: IOU_TEST_PASS if the test passed, or IOU_TEST_FAIL
+	 *         if it failed.
+	 */
+	enum iou_test_status (*run)(void *ctx);
+
+	/**
+	 * cleanup - Cleanup callback.
+	 * @ctx: Context object returned from setup().
+	 */
+	void (*cleanup)(void *ctx);
+};
+
+void iou_test_register(struct iou_test *test);
+
+#define REGISTER_IOU_TEST(__test)					\
+	__attribute__((constructor))					\
+	static void __test##_register(void)				\
+	{								\
+		iou_test_register(&(__test));				\
+	}
+
+#define IOU_BUG(__cond, __fmt, ...)					\
+	do {								\
+		if (__cond) {						\
+			fprintf(stderr, "FATAL (%s:%d): " __fmt "\n",	\
+				__FILE__, __LINE__,			\
+				##__VA_ARGS__);				\
+			exit(1);					\
+		}							\
+	} while (0)
+
+#define IOU_BUG_ON(__cond) IOU_BUG(__cond, "BUG: %s", #__cond)
+
+#define IOU_FAIL(__fmt, ...)						\
+	do {								\
+		fprintf(stderr, "FAIL (%s:%d): " __fmt "\n",		\
+			__FILE__, __LINE__, ##__VA_ARGS__);		\
+		return IOU_TEST_FAIL;					\
+	} while (0)
+
+#define IOU_FAIL_IF(__cond, __fmt, ...)					\
+	do {								\
+		if (__cond)						\
+			IOU_FAIL(__fmt, ##__VA_ARGS__);			\
+	} while (0)
+
+#define IOU_ERR(__fmt, ...)						\
+	fprintf(stderr, "ERR: " __fmt "\n", ##__VA_ARGS__)
+
+#define IOU_INFO(__fmt, ...)						\
+	fprintf(stdout, "INFO: " __fmt "\n", ##__VA_ARGS__)
+
+#endif /* __IOU_TEST_H__ */
diff --git a/tools/testing/selftests/io_uring/runner.c b/tools/testing/selftests/io_uring/runner.c
new file mode 100644
index 000000000000..09ac1ac2d633
--- /dev/null
+++ b/tools/testing/selftests/io_uring/runner.c
@@ -0,0 +1,206 @@
+/* SPDX-License-Identifier: GPL-2.0 */
+/*
+ * Copyright (c) 2025 Red Hat, Inc.
+ * Test runner for io_uring BPF selftests.
+ */
+#include <stdio.h>
+#include <unistd.h>
+#include <signal.h>
+#include <libgen.h>
+#include <bpf/bpf.h>
+#include "iou_test.h"
+
+const char help_fmt[] =
+"The runner for io_uring BPF tests.\n"
+"\n"
+"The runner is statically linked against all testcases, and runs them all serially.\n"
+"\n"
+"Usage: %s [-t TEST] [-h]\n"
+"\n"
+"  -t TEST       Only run tests whose name includes this string\n"
+"  -s            Include print output for skipped tests\n"
+"  -l            List all available tests\n"
+"  -q            Don't print the test descriptions during run\n"
+"  -h            Display this help and exit\n";
+
+static volatile int exit_req;
+static bool quiet, print_skipped, list;
+
+#define MAX_IOU_TESTS 2048
+
+static struct iou_test __iou_tests[MAX_IOU_TESTS];
+static unsigned __iou_num_tests = 0;
+
+static void sigint_handler(int simple)
+{
+	exit_req = 1;
+}
+
+static void print_test_preamble(const struct iou_test *test, bool quiet)
+{
+	printf("===== START =====\n");
+	printf("TEST: %s\n", test->name);
+	if (!quiet)
+		printf("DESCRIPTION: %s\n", test->description);
+	printf("OUTPUT:\n");
+
+	fflush(stdout);
+	fflush(stderr);
+}
+
+static const char *status_to_result(enum iou_test_status status)
+{
+	switch (status) {
+	case IOU_TEST_PASS:
+	case IOU_TEST_SKIP:
+		return "ok";
+	case IOU_TEST_FAIL:
+		return "not ok";
+	default:
+		return "<UNKNOWN>";
+	}
+}
+
+static void print_test_result(const struct iou_test *test,
+			      enum iou_test_status status,
+			      unsigned int testnum)
+{
+	const char *result = status_to_result(status);
+	const char *directive = status == IOU_TEST_SKIP ? "SKIP " : "";
+
+	printf("%s %u %s # %s\n", result, testnum, test->name, directive);
+	printf("=====  END  =====\n");
+}
+
+static bool should_skip_test(const struct iou_test *test, const char *filter)
+{
+	return !strstr(test->name, filter);
+}
+
+static enum iou_test_status run_test(const struct iou_test *test)
+{
+	enum iou_test_status status;
+	void *context = NULL;
+
+	if (test->setup) {
+		status = test->setup(&context);
+		if (status != IOU_TEST_PASS)
+			return status;
+	}
+
+	status = test->run(context);
+
+	if (test->cleanup)
+		test->cleanup(context);
+
+	return status;
+}
+
+static bool test_valid(const struct iou_test *test)
+{
+	if (!test) {
+		fprintf(stderr, "NULL test detected\n");
+		return false;
+	}
+
+	if (!test->name) {
+		fprintf(stderr,
+			"Test with no name found. Must specify test name.\n");
+		return false;
+	}
+
+	if (!test->description) {
+		fprintf(stderr, "Test %s requires description.\n", test->name);
+		return false;
+	}
+
+	if (!test->run) {
+		fprintf(stderr, "Test %s has no run() callback\n", test->name);
+		return false;
+	}
+
+	return true;
+}
+
+int main(int argc, char **argv)
+{
+	const char *filter = NULL;
+	unsigned testnum = 0, i;
+	unsigned passed = 0, skipped = 0, failed = 0;
+	int opt;
+
+	signal(SIGINT, sigint_handler);
+	signal(SIGTERM, sigint_handler);
+
+	libbpf_set_strict_mode(LIBBPF_STRICT_ALL);
+
+	while ((opt = getopt(argc, argv, "qslt:h")) != -1) {
+		switch (opt) {
+		case 'q':
+			quiet = true;
+			break;
+		case 's':
+			print_skipped = true;
+			break;
+		case 'l':
+			list = true;
+			break;
+		case 't':
+			filter = optarg;
+			break;
+		default:
+			fprintf(stderr, help_fmt, basename(argv[0]));
+			return opt != 'h';
+		}
+	}
+
+	for (i = 0; i < __iou_num_tests; i++) {
+		enum iou_test_status status;
+		struct iou_test *test = &__iou_tests[i];
+
+		if (list) {
+			printf("%s\n", test->name);
+			if (i == (__iou_num_tests - 1))
+				return 0;
+			continue;
+		}
+
+		if (filter && should_skip_test(test, filter)) {
+			if (print_skipped) {
+				print_test_preamble(test, quiet);
+				print_test_result(test, IOU_TEST_SKIP, ++testnum);
+			}
+			continue;
+		}
+
+		print_test_preamble(test, quiet);
+		status = run_test(test);
+		print_test_result(test, status, ++testnum);
+		switch (status) {
+		case IOU_TEST_PASS:
+			passed++;
+			break;
+		case IOU_TEST_SKIP:
+			skipped++;
+			break;
+		case IOU_TEST_FAIL:
+			failed++;
+			break;
+		}
+	}
+	printf("\n\n=============================\n\n");
+	printf("RESULTS:\n\n");
+	printf("PASSED:  %u\n", passed);
+	printf("SKIPPED: %u\n", skipped);
+	printf("FAILED:  %u\n", failed);
+
+	return failed ? 1 : 0;
+}
+
+void iou_test_register(struct iou_test *test)
+{
+	IOU_BUG_ON(!test_valid(test));
+	IOU_BUG_ON(__iou_num_tests >= MAX_IOU_TESTS);
+
+	__iou_tests[__iou_num_tests++] = *test;
+}
-- 
2.47.0

[PATCH V2 11/13] selftests/io_uring: add bpf_memcpy selftest for uring_bpf_memcpy() kfunc

[Ming Lei]

Add a selftest to verify the uring_bpf_memcpy() kfunc works correctly
with different buffer types. The test uses BPF struct_ops to implement
prep/issue/fail/cleanup operations for the IORING_OP_BPF opcode.

Three test cases are included:
- copy_user_to_user(): Tests IO_BPF_BUF_USER buffer type (flat userspace
  buffer) for both source and destination
- copy_vec_to_vec(): Tests IO_BPF_BUF_VEC buffer type (iovec array split
  into multiple chunks) for both source and destination
- copy_user_to_vec(): Tests mixed buffer types with USER source and VEC
  destination

All tests allocate source/destination buffers, fill the source with a
pattern, invoke uring_bpf_memcpy() via io_uring submission, and verify
the copy succeeded by checking CQE result and destination buffer contents.

Signed-off-by: Ming Lei <ming.lei@redhat.com>
---
 tools/testing/selftests/io_uring/Makefile     |   1 +
 .../selftests/io_uring/bpf_memcpy.bpf.c       |  98 +++++
 tools/testing/selftests/io_uring/bpf_memcpy.c | 374 ++++++++++++++++++
 3 files changed, 473 insertions(+)
 create mode 100644 tools/testing/selftests/io_uring/bpf_memcpy.bpf.c
 create mode 100644 tools/testing/selftests/io_uring/bpf_memcpy.c

diff --git a/tools/testing/selftests/io_uring/Makefile b/tools/testing/selftests/io_uring/Makefile
index f88a6a749484..e1fa77b0a000 100644
--- a/tools/testing/selftests/io_uring/Makefile
+++ b/tools/testing/selftests/io_uring/Makefile
@@ -150,6 +150,7 @@ all_test_bpfprogs := $(foreach prog,$(wildcard *.bpf.c),$(INCLUDE_DIR)/$(patsubs
 
 auto-test-targets :=			\
 	basic_bpf_ops			\
+	bpf_memcpy			\
 
 testcase-targets := $(addsuffix .o,$(addprefix $(IOUOBJ_DIR)/,$(auto-test-targets)))
 
diff --git a/tools/testing/selftests/io_uring/bpf_memcpy.bpf.c b/tools/testing/selftests/io_uring/bpf_memcpy.bpf.c
new file mode 100644
index 000000000000..d8056de639c1
--- /dev/null
+++ b/tools/testing/selftests/io_uring/bpf_memcpy.bpf.c
@@ -0,0 +1,98 @@
+/* SPDX-License-Identifier: GPL-2.0 */
+/*
+ * Copyright (c) 2025 Red Hat, Inc.
+ * Test for uring_bpf_memcpy() kfunc.
+ *
+ * This tests the uring_bpf_memcpy() kfunc with USER buffer type,
+ * copying data between two userspace buffers.
+ *
+ * Buffer descriptors are passed via sqe->addr as an array of two
+ * io_bpf_buf_desc structures:
+ *   [0] = source buffer descriptor
+ *   [1] = destination buffer descriptor
+ * sqe->len contains the number of descriptors (2).
+ */
+
+#include "vmlinux.h"
+#include <bpf/bpf_helpers.h>
+#include <bpf/bpf_tracing.h>
+#include <asm-generic/errno.h>
+
+char LICENSE[] SEC("license") = "GPL";
+
+/* PDU layout for storing buffer descriptors between prep and issue */
+struct memcpy_pdu {
+	struct io_bpf_buf_desc descs[2];  /* [0]=src, [1]=dst */
+};
+
+/* kfunc declarations */
+extern void uring_bpf_set_result(struct uring_bpf_data *data, int res) __ksym;
+extern int uring_bpf_read_buf_descs(struct io_bpf_buf_desc *descs,
+				    __u64 user_addr, int nr_descs) __ksym;
+extern __s64 uring_bpf_memcpy(const struct uring_bpf_data *data,
+			      struct io_bpf_buf_desc *dst,
+			      struct io_bpf_buf_desc *src) __ksym;
+
+SEC("struct_ops/memcpy_prep")
+int BPF_PROG(memcpy_prep, struct uring_bpf_data *data,
+	     const struct io_uring_sqe *sqe)
+{
+	struct memcpy_pdu *pdu = (struct memcpy_pdu *)data->pdu;
+	struct io_bpf_buf_desc descs[2];
+	int ret;
+
+	/* Validate descriptor count */
+	if (sqe->len != 2)
+		return -EINVAL;
+
+	ret = bpf_probe_read_user(descs, sizeof(descs), (void *)sqe->addr);
+	if (ret) {
+		bpf_printk("memcpy_prep: uring_bpf_read_buf_descs failed: %d", ret);
+		return ret;
+	}
+
+	__builtin_memcpy(&pdu->descs, &descs, sizeof(descs));
+	bpf_printk("memcpy_prep: src=0x%llx dst=0x%llx len=%u",
+		   pdu->descs[0].addr, pdu->descs[1].addr, pdu->descs[0].len);
+	return 0;
+}
+
+SEC("struct_ops/memcpy_issue")
+int BPF_PROG(memcpy_issue, struct uring_bpf_data *data)
+{
+	struct memcpy_pdu *pdu = (struct memcpy_pdu *)data->pdu;
+	struct io_bpf_buf_desc dst_desc, src_desc;
+	__s64 ret;
+
+	/* Copy descriptors to stack to satisfy verifier type checking */
+	src_desc = pdu->descs[0];
+	dst_desc = pdu->descs[1];
+
+	/* Call uring_bpf_memcpy() kfunc using stack-based descriptors */
+	ret = uring_bpf_memcpy(data, &dst_desc, &src_desc);
+
+	bpf_printk("memcpy_issue: uring_bpf_memcpy returned %lld", ret);
+
+	uring_bpf_set_result(data, (int)ret);
+	return 0;
+}
+
+SEC("struct_ops/memcpy_fail")
+void BPF_PROG(memcpy_fail, struct uring_bpf_data *data)
+{
+	bpf_printk("memcpy_fail: invoked");
+}
+
+SEC("struct_ops/memcpy_cleanup")
+void BPF_PROG(memcpy_cleanup, struct uring_bpf_data *data)
+{
+	bpf_printk("memcpy_cleanup: invoked");
+}
+
+SEC(".struct_ops.link")
+struct uring_bpf_ops bpf_memcpy_ops = {
+	.prep_fn	= (void *)memcpy_prep,
+	.issue_fn	= (void *)memcpy_issue,
+	.fail_fn	= (void *)memcpy_fail,
+	.cleanup_fn	= (void *)memcpy_cleanup,
+};
diff --git a/tools/testing/selftests/io_uring/bpf_memcpy.c b/tools/testing/selftests/io_uring/bpf_memcpy.c
new file mode 100644
index 000000000000..0fad6d0583c3
--- /dev/null
+++ b/tools/testing/selftests/io_uring/bpf_memcpy.c
@@ -0,0 +1,374 @@
+/* SPDX-License-Identifier: GPL-2.0 */
+/*
+ * Copyright (c) 2025 Red Hat, Inc.
+ * Test for uring_bpf_memcpy() kfunc - userspace part.
+ */
+#include <bpf/bpf.h>
+#include <bpf/libbpf.h>
+#include <errno.h>
+#include <linux/io_uring.h>
+#include <sys/uio.h>
+#include <io_uring/mini_liburing.h>
+
+#include "iou_test.h"
+#include "bpf_memcpy.bpf.skel.h"
+
+#define TEST_BUF_SIZE		(4096 * 4 + 1024 + 511)
+#define TEST_PATTERN		0xAB
+#define MAX_VECS		32
+
+struct test_ctx {
+	struct bpf_memcpy *skel;
+	struct bpf_link *link;
+	struct io_uring ring;
+
+	/* Buffer descriptors and buffers */
+	struct io_bpf_buf_desc descs[2];
+	char *src_buf;
+	char *dst_buf;
+	size_t src_buf_size;
+	size_t dst_buf_size;
+	__u8 src_type;
+	__u8 dst_type;
+	const char *desc;
+
+	/* Vectored buffer support */
+	struct iovec src_vec[MAX_VECS];
+	struct iovec dst_vec[MAX_VECS];
+	int src_nr_vec;
+	int dst_nr_vec;
+};
+
+static enum iou_test_status bpf_setup(struct test_ctx *ctx)
+{
+	int ret;
+
+	/* Load BPF skeleton */
+	ctx->skel = bpf_memcpy__open();
+	if (!ctx->skel) {
+		IOU_ERR("Failed to open BPF skeleton");
+		return IOU_TEST_FAIL;
+	}
+
+	/* Set ring_fd in struct_ops before loading */
+	ctx->skel->struct_ops.bpf_memcpy_ops->ring_fd = ctx->ring.ring_fd;
+	ctx->skel->struct_ops.bpf_memcpy_ops->id = 0;
+
+	ret = bpf_memcpy__load(ctx->skel);
+	if (ret) {
+		IOU_ERR("Failed to load BPF skeleton: %d", ret);
+		bpf_memcpy__destroy(ctx->skel);
+		ctx->skel = NULL;
+		return IOU_TEST_FAIL;
+	}
+
+	/* Attach struct_ops */
+	ctx->link = bpf_map__attach_struct_ops(ctx->skel->maps.bpf_memcpy_ops);
+	if (!ctx->link) {
+		IOU_ERR("Failed to attach struct_ops");
+		bpf_memcpy__destroy(ctx->skel);
+		ctx->skel = NULL;
+		return IOU_TEST_FAIL;
+	}
+
+	return IOU_TEST_PASS;
+}
+
+static enum iou_test_status setup(void **ctx_out)
+{
+	struct io_uring_params p;
+	struct test_ctx *ctx;
+	enum iou_test_status status;
+	int ret;
+
+	ctx = calloc(1, sizeof(*ctx));
+	if (!ctx) {
+		IOU_ERR("Failed to allocate context");
+		return IOU_TEST_FAIL;
+	}
+
+	/* Setup io_uring ring with BPF_OP flag */
+	memset(&p, 0, sizeof(p));
+	p.flags = IORING_SETUP_BPF_OP | IORING_SETUP_NO_SQARRAY;
+
+	ret = io_uring_queue_init_params(8, &ctx->ring, &p);
+	if (ret < 0) {
+		IOU_ERR("io_uring_queue_init_params failed: %s (flags=0x%x)",
+			strerror(-ret), p.flags);
+		free(ctx);
+		return IOU_TEST_SKIP;
+	}
+
+	status = bpf_setup(ctx);
+	if (status != IOU_TEST_PASS) {
+		io_uring_queue_exit(&ctx->ring);
+		free(ctx);
+		return status;
+	}
+
+	*ctx_out = ctx;
+	return IOU_TEST_PASS;
+}
+
+static int allocate_buf(char **buf, size_t size, __u8 buf_type,
+			struct iovec *vec, int nr_vec)
+{
+	char *p;
+	size_t chunk_size;
+	int i;
+
+	switch (buf_type) {
+	case IO_BPF_BUF_USER:
+		p = aligned_alloc(4096, size);
+		if (!p)
+			return -ENOMEM;
+		*buf = p;
+		return 0;
+	case IO_BPF_BUF_VEC:
+		if (nr_vec <= 0 || nr_vec > MAX_VECS)
+			return -EINVAL;
+		p = aligned_alloc(4096, size);
+		if (!p)
+			return -ENOMEM;
+		*buf = p;
+		/* Split buffer into nr_vec pieces */
+		chunk_size = size / nr_vec;
+		for (i = 0; i < nr_vec; i++) {
+			vec[i].iov_base = p + i * chunk_size;
+			vec[i].iov_len = chunk_size;
+		}
+		/* Last chunk gets remainder */
+		vec[nr_vec - 1].iov_len += size % nr_vec;
+		return 0;
+	default:
+		return -EINVAL;
+	}
+}
+
+static void free_buf(char *buf, __u8 buf_type)
+{
+	switch (buf_type) {
+	case IO_BPF_BUF_USER:
+	case IO_BPF_BUF_VEC:
+		free(buf);
+		break;
+	default:
+		break;
+	}
+}
+
+static enum iou_test_status allocate_bufs(struct test_ctx *ctx)
+{
+	int ret;
+
+	ret = allocate_buf(&ctx->src_buf, ctx->src_buf_size, ctx->src_type,
+			   ctx->src_vec, ctx->src_nr_vec);
+	if (ret) {
+		IOU_ERR("Failed to allocate source buffer: %d", ret);
+		return IOU_TEST_FAIL;
+	}
+
+	ret = allocate_buf(&ctx->dst_buf, ctx->dst_buf_size, ctx->dst_type,
+			   ctx->dst_vec, ctx->dst_nr_vec);
+	if (ret) {
+		IOU_ERR("Failed to allocate destination buffer: %d", ret);
+		free_buf(ctx->src_buf, ctx->src_type);
+		ctx->src_buf = NULL;
+		return IOU_TEST_FAIL;
+	}
+
+	/* Initialize source buffer with pattern, destination with zeros */
+	memset(ctx->src_buf, TEST_PATTERN, ctx->src_buf_size);
+	memset(ctx->dst_buf, 0, ctx->dst_buf_size);
+
+	/* Build buffer descriptors */
+	memset(ctx->descs, 0, sizeof(ctx->descs));
+	ctx->descs[0].type = ctx->src_type;
+	ctx->descs[1].type = ctx->dst_type;
+
+	if (ctx->src_type == IO_BPF_BUF_VEC) {
+		ctx->descs[0].addr = (__u64)(uintptr_t)ctx->src_vec;
+		ctx->descs[0].len = ctx->src_nr_vec;
+	} else {
+		ctx->descs[0].addr = (__u64)(uintptr_t)ctx->src_buf;
+		ctx->descs[0].len = ctx->src_buf_size;
+	}
+
+	if (ctx->dst_type == IO_BPF_BUF_VEC) {
+		ctx->descs[1].addr = (__u64)(uintptr_t)ctx->dst_vec;
+		ctx->descs[1].len = ctx->dst_nr_vec;
+	} else {
+		ctx->descs[1].addr = (__u64)(uintptr_t)ctx->dst_buf;
+		ctx->descs[1].len = ctx->dst_buf_size;
+	}
+
+	return IOU_TEST_PASS;
+}
+
+static void free_bufs(struct test_ctx *ctx)
+{
+	if (ctx->src_buf) {
+		free_buf(ctx->src_buf, ctx->src_type);
+		ctx->src_buf = NULL;
+	}
+	if (ctx->dst_buf) {
+		free_buf(ctx->dst_buf, ctx->dst_type);
+		ctx->dst_buf = NULL;
+	}
+}
+
+static enum iou_test_status submit_and_wait(struct test_ctx *ctx)
+{
+	struct io_uring_sqe *sqe;
+	struct io_uring_cqe *cqe;
+	int ret;
+
+	/* Get an SQE and prepare BPF op request */
+	sqe = io_uring_get_sqe(&ctx->ring);
+	if (!sqe) {
+		IOU_ERR("Failed to get SQE");
+		return IOU_TEST_FAIL;
+	}
+
+	memset(sqe, 0, sizeof(*sqe));
+	sqe->opcode = IORING_OP_BPF;
+	sqe->fd = -1;
+	sqe->bpf_op_flags = (0 << IORING_BPF_OP_SHIFT); /* BPF op id = 0 */
+	sqe->addr = (__u64)(uintptr_t)ctx->descs;
+	sqe->len = 2;  /* number of descriptors */
+	sqe->user_data = 0xCAFEBABE;
+
+	/* Submit and wait for completion */
+	ret = io_uring_submit(&ctx->ring);
+	if (ret < 0) {
+		IOU_ERR("io_uring_submit failed: %d", ret);
+		return IOU_TEST_FAIL;
+	}
+
+	ret = io_uring_wait_cqe(&ctx->ring, &cqe);
+	if (ret < 0) {
+		IOU_ERR("io_uring_wait_cqe failed: %d", ret);
+		return IOU_TEST_FAIL;
+	}
+
+	/* Verify CQE */
+	if (cqe->user_data != 0xCAFEBABE) {
+		IOU_ERR("CQE user_data mismatch: 0x%llx", cqe->user_data);
+		return IOU_TEST_FAIL;
+	}
+
+	if (cqe->res != (int)ctx->src_buf_size) {
+		IOU_ERR("CQE result mismatch: %d (expected %zu)",
+			cqe->res, ctx->src_buf_size);
+		if (cqe->res < 0)
+			IOU_ERR("Error from uring_bpf_memcpy: %s", strerror(-cqe->res));
+		return IOU_TEST_FAIL;
+	}
+
+	io_uring_cqe_seen(&ctx->ring);
+
+	/* Verify destination buffer contains the pattern */
+	for (size_t i = 0; i < ctx->dst_buf_size; i++) {
+		if ((unsigned char)ctx->dst_buf[i] != TEST_PATTERN) {
+			IOU_ERR("Data mismatch at offset %zu: 0x%02x (expected 0x%02x)",
+				i, (unsigned char)ctx->dst_buf[i], TEST_PATTERN);
+			return IOU_TEST_FAIL;
+		}
+	}
+
+	return IOU_TEST_PASS;
+}
+
+static enum iou_test_status test_copy(struct test_ctx *ctx)
+{
+	enum iou_test_status status;
+
+	status = allocate_bufs(ctx);
+	if (status != IOU_TEST_PASS)
+		return status;
+
+	status = submit_and_wait(ctx);
+	free_bufs(ctx);
+
+	if (status == IOU_TEST_PASS)
+		IOU_INFO("%s: copied %zu bytes", ctx->desc, ctx->src_buf_size);
+
+	return status;
+}
+
+static enum iou_test_status copy_user_to_user(struct test_ctx *ctx)
+{
+	ctx->src_type = IO_BPF_BUF_USER;
+	ctx->dst_type = IO_BPF_BUF_USER;
+	ctx->src_buf_size = TEST_BUF_SIZE;
+	ctx->dst_buf_size = TEST_BUF_SIZE;
+	ctx->desc = "USER -> USER";
+
+	return test_copy(ctx);
+}
+
+static enum iou_test_status copy_vec_to_vec(struct test_ctx *ctx)
+{
+	ctx->src_type = IO_BPF_BUF_VEC;
+	ctx->dst_type = IO_BPF_BUF_VEC;
+	ctx->src_buf_size = TEST_BUF_SIZE;
+	ctx->dst_buf_size = TEST_BUF_SIZE;
+	ctx->src_nr_vec = 4;
+	ctx->dst_nr_vec = 4;
+	ctx->desc = "VEC -> VEC";
+
+	return test_copy(ctx);
+}
+
+static enum iou_test_status copy_user_to_vec(struct test_ctx *ctx)
+{
+	ctx->src_type = IO_BPF_BUF_USER;
+	ctx->dst_type = IO_BPF_BUF_VEC;
+	ctx->src_buf_size = TEST_BUF_SIZE;
+	ctx->dst_buf_size = TEST_BUF_SIZE;
+	ctx->dst_nr_vec = 4;
+	ctx->desc = "USER -> VEC";
+
+	return test_copy(ctx);
+}
+
+static enum iou_test_status run(void *ctx_ptr)
+{
+	struct test_ctx *ctx = ctx_ptr;
+	enum iou_test_status status;
+
+	status = copy_user_to_user(ctx);
+	if (status != IOU_TEST_PASS)
+		return status;
+
+	status = copy_vec_to_vec(ctx);
+	if (status != IOU_TEST_PASS)
+		return status;
+
+	status = copy_user_to_vec(ctx);
+	if (status != IOU_TEST_PASS)
+		return status;
+
+	return IOU_TEST_PASS;
+}
+
+static void cleanup(void *ctx_ptr)
+{
+	struct test_ctx *ctx = ctx_ptr;
+
+	if (ctx->link)
+		bpf_link__destroy(ctx->link);
+	if (ctx->skel)
+		bpf_memcpy__destroy(ctx->skel);
+	io_uring_queue_exit(&ctx->ring);
+	free(ctx);
+}
+
+struct iou_test bpf_memcpy_test = {
+	.name = "bpf_memcpy",
+	.description = "Test uring_bpf_memcpy() kfunc with USER, VEC, and mixed buffer types",
+	.setup = setup,
+	.run = run,
+	.cleanup = cleanup,
+};
+REGISTER_IOU_TEST(bpf_memcpy_test)
-- 
2.47.0

[PATCH V2 12/13] selftests/io_uring: add copy_user_to_fixed() and copy_fixed_to_user() bpf_memcpy tests

[Ming Lei]

Add tests for IO_BPF_BUF_FIXED buffer type to verify uring_bpf_memcpy()
kfunc works correctly with registered fixed buffers.

Changes:
- Add io_uring_unregister_buffers() to mini_liburing.h
- Add fixed buffer index tracking (src_buf_index/dst_buf_index) to test_ctx
- Add register_fixed_bufs()/unregister_fixed_bufs() helpers to manage
  buffer registration with the io_uring ring
- Update allocate_buf()/free_buf() to handle IO_BPF_BUF_FIXED type
- Add copy_user_to_fixed(): Tests USER source to FIXED destination
- Add copy_fixed_to_user(): Tests FIXED source to USER destination

Signed-off-by: Ming Lei <ming.lei@redhat.com>
---
 tools/testing/selftests/io_uring/bpf_memcpy.c | 98 ++++++++++++++++++-
 1 file changed, 97 insertions(+), 1 deletion(-)

diff --git a/tools/testing/selftests/io_uring/bpf_memcpy.c b/tools/testing/selftests/io_uring/bpf_memcpy.c
index 0fad6d0583c3..923b9d81b508 100644
--- a/tools/testing/selftests/io_uring/bpf_memcpy.c
+++ b/tools/testing/selftests/io_uring/bpf_memcpy.c
@@ -37,6 +37,10 @@ struct test_ctx {
 	struct iovec dst_vec[MAX_VECS];
 	int src_nr_vec;
 	int dst_nr_vec;
+
+	/* Fixed buffer support */
+	__u16 src_buf_index;
+	__u16 dst_buf_index;
 };
 
 static enum iou_test_status bpf_setup(struct test_ctx *ctx)
@@ -119,6 +123,7 @@ static int allocate_buf(char **buf, size_t size, __u8 buf_type,
 
 	switch (buf_type) {
 	case IO_BPF_BUF_USER:
+	case IO_BPF_BUF_FIXED:
 		p = aligned_alloc(4096, size);
 		if (!p)
 			return -ENOMEM;
@@ -150,6 +155,7 @@ static void free_buf(char *buf, __u8 buf_type)
 	switch (buf_type) {
 	case IO_BPF_BUF_USER:
 	case IO_BPF_BUF_VEC:
+	case IO_BPF_BUF_FIXED:
 		free(buf);
 		break;
 	default:
@@ -157,8 +163,48 @@ static void free_buf(char *buf, __u8 buf_type)
 	}
 }
 
+static enum iou_test_status register_fixed_bufs(struct test_ctx *ctx)
+{
+	struct iovec iovecs[2];
+	int nr_iovecs = 0;
+	int ret;
+
+	if (ctx->src_type == IO_BPF_BUF_FIXED) {
+		ctx->src_buf_index = nr_iovecs;
+		iovecs[nr_iovecs].iov_base = ctx->src_buf;
+		iovecs[nr_iovecs].iov_len = ctx->src_buf_size;
+		nr_iovecs++;
+	}
+
+	if (ctx->dst_type == IO_BPF_BUF_FIXED) {
+		ctx->dst_buf_index = nr_iovecs;
+		iovecs[nr_iovecs].iov_base = ctx->dst_buf;
+		iovecs[nr_iovecs].iov_len = ctx->dst_buf_size;
+		nr_iovecs++;
+	}
+
+	if (nr_iovecs == 0)
+		return IOU_TEST_PASS;
+
+	ret = io_uring_register_buffers(&ctx->ring, iovecs, nr_iovecs);
+	if (ret) {
+		IOU_ERR("Failed to register buffers: %d", ret);
+		return IOU_TEST_FAIL;
+	}
+
+	return IOU_TEST_PASS;
+}
+
+static void unregister_fixed_bufs(struct test_ctx *ctx)
+{
+	if (ctx->src_type == IO_BPF_BUF_FIXED ||
+	    ctx->dst_type == IO_BPF_BUF_FIXED)
+		io_uring_unregister_buffers(&ctx->ring);
+}
+
 static enum iou_test_status allocate_bufs(struct test_ctx *ctx)
 {
+	enum iou_test_status status;
 	int ret;
 
 	ret = allocate_buf(&ctx->src_buf, ctx->src_buf_size, ctx->src_type,
@@ -181,6 +227,16 @@ static enum iou_test_status allocate_bufs(struct test_ctx *ctx)
 	memset(ctx->src_buf, TEST_PATTERN, ctx->src_buf_size);
 	memset(ctx->dst_buf, 0, ctx->dst_buf_size);
 
+	/* Register fixed buffers if needed */
+	status = register_fixed_bufs(ctx);
+	if (status != IOU_TEST_PASS) {
+		free_buf(ctx->dst_buf, ctx->dst_type);
+		ctx->dst_buf = NULL;
+		free_buf(ctx->src_buf, ctx->src_type);
+		ctx->src_buf = NULL;
+		return status;
+	}
+
 	/* Build buffer descriptors */
 	memset(ctx->descs, 0, sizeof(ctx->descs));
 	ctx->descs[0].type = ctx->src_type;
@@ -189,6 +245,10 @@ static enum iou_test_status allocate_bufs(struct test_ctx *ctx)
 	if (ctx->src_type == IO_BPF_BUF_VEC) {
 		ctx->descs[0].addr = (__u64)(uintptr_t)ctx->src_vec;
 		ctx->descs[0].len = ctx->src_nr_vec;
+	} else if (ctx->src_type == IO_BPF_BUF_FIXED) {
+		ctx->descs[0].addr = (__u64)(uintptr_t)ctx->src_buf;
+		ctx->descs[0].len = ctx->src_buf_size;
+		ctx->descs[0].buf_index = ctx->src_buf_index;
 	} else {
 		ctx->descs[0].addr = (__u64)(uintptr_t)ctx->src_buf;
 		ctx->descs[0].len = ctx->src_buf_size;
@@ -197,6 +257,10 @@ static enum iou_test_status allocate_bufs(struct test_ctx *ctx)
 	if (ctx->dst_type == IO_BPF_BUF_VEC) {
 		ctx->descs[1].addr = (__u64)(uintptr_t)ctx->dst_vec;
 		ctx->descs[1].len = ctx->dst_nr_vec;
+	} else if (ctx->dst_type == IO_BPF_BUF_FIXED) {
+		ctx->descs[1].addr = (__u64)(uintptr_t)ctx->dst_buf;
+		ctx->descs[1].len = ctx->dst_buf_size;
+		ctx->descs[1].buf_index = ctx->dst_buf_index;
 	} else {
 		ctx->descs[1].addr = (__u64)(uintptr_t)ctx->dst_buf;
 		ctx->descs[1].len = ctx->dst_buf_size;
@@ -207,6 +271,8 @@ static enum iou_test_status allocate_bufs(struct test_ctx *ctx)
 
 static void free_bufs(struct test_ctx *ctx)
 {
+	unregister_fixed_bufs(ctx);
+
 	if (ctx->src_buf) {
 		free_buf(ctx->src_buf, ctx->src_type);
 		ctx->src_buf = NULL;
@@ -332,6 +398,28 @@ static enum iou_test_status copy_user_to_vec(struct test_ctx *ctx)
 	return test_copy(ctx);
 }
 
+static enum iou_test_status copy_user_to_fixed(struct test_ctx *ctx)
+{
+	ctx->src_type = IO_BPF_BUF_USER;
+	ctx->dst_type = IO_BPF_BUF_FIXED;
+	ctx->src_buf_size = TEST_BUF_SIZE;
+	ctx->dst_buf_size = TEST_BUF_SIZE;
+	ctx->desc = "USER -> FIXED";
+
+	return test_copy(ctx);
+}
+
+static enum iou_test_status copy_fixed_to_user(struct test_ctx *ctx)
+{
+	ctx->src_type = IO_BPF_BUF_FIXED;
+	ctx->dst_type = IO_BPF_BUF_USER;
+	ctx->src_buf_size = TEST_BUF_SIZE;
+	ctx->dst_buf_size = TEST_BUF_SIZE;
+	ctx->desc = "FIXED -> USER";
+
+	return test_copy(ctx);
+}
+
 static enum iou_test_status run(void *ctx_ptr)
 {
 	struct test_ctx *ctx = ctx_ptr;
@@ -349,6 +437,14 @@ static enum iou_test_status run(void *ctx_ptr)
 	if (status != IOU_TEST_PASS)
 		return status;
 
+	status = copy_user_to_fixed(ctx);
+	if (status != IOU_TEST_PASS)
+		return status;
+
+	status = copy_fixed_to_user(ctx);
+	if (status != IOU_TEST_PASS)
+		return status;
+
 	return IOU_TEST_PASS;
 }
 
@@ -366,7 +462,7 @@ static void cleanup(void *ctx_ptr)
 
 struct iou_test bpf_memcpy_test = {
 	.name = "bpf_memcpy",
-	.description = "Test uring_bpf_memcpy() kfunc with USER, VEC, and mixed buffer types",
+	.description = "Test uring_bpf_memcpy() kfunc with USER, VEC, FIXED buffer types",
 	.setup = setup,
 	.run = run,
 	.cleanup = cleanup,
-- 
2.47.0

[PATCH V2 13/13] selftests/io_uring: add copy_user_to_reg_vec() and copy_reg_vec_to_user() bpf_memcpy tests

[Ming Lei]

Add tests for IO_BPF_BUF_REG_VEC buffer type to verify uring_bpf_memcpy()
kfunc works correctly with registered vectored buffers.

IO_BPF_BUF_REG_VEC combines the vectored buffer layout (iovec array) with
registered buffer optimization - the underlying buffer is registered with
io_uring while the descriptor points to an iovec array for flexible
scatter-gather access.

Changes:
- Add is_registered_buf() helper to check if buffer type needs registration
- Update allocate_buf()/free_buf() to handle IO_BPF_BUF_REG_VEC type
- Update register_fixed_bufs()/unregister_fixed_bufs() to handle REG_VEC
- Update allocate_bufs() to set up REG_VEC descriptors with buf_index
- Add copy_user_to_reg_vec(): Tests USER source to REG_VEC destination
- Add copy_reg_vec_to_user(): Tests REG_VEC source to USER destination

Signed-off-by: Ming Lei <ming.lei@redhat.com>
---
 tools/testing/selftests/io_uring/bpf_memcpy.c | 57 +++++++++++++++++--
 1 file changed, 52 insertions(+), 5 deletions(-)

diff --git a/tools/testing/selftests/io_uring/bpf_memcpy.c b/tools/testing/selftests/io_uring/bpf_memcpy.c
index 923b9d81b508..a221b84839bd 100644
--- a/tools/testing/selftests/io_uring/bpf_memcpy.c
+++ b/tools/testing/selftests/io_uring/bpf_memcpy.c
@@ -130,6 +130,7 @@ static int allocate_buf(char **buf, size_t size, __u8 buf_type,
 		*buf = p;
 		return 0;
 	case IO_BPF_BUF_VEC:
+	case IO_BPF_BUF_REG_VEC:
 		if (nr_vec <= 0 || nr_vec > MAX_VECS)
 			return -EINVAL;
 		p = aligned_alloc(4096, size);
@@ -156,6 +157,7 @@ static void free_buf(char *buf, __u8 buf_type)
 	case IO_BPF_BUF_USER:
 	case IO_BPF_BUF_VEC:
 	case IO_BPF_BUF_FIXED:
+	case IO_BPF_BUF_REG_VEC:
 		free(buf);
 		break;
 	default:
@@ -163,20 +165,25 @@ static void free_buf(char *buf, __u8 buf_type)
 	}
 }
 
+static inline bool is_registered_buf(__u8 type)
+{
+	return type == IO_BPF_BUF_FIXED || type == IO_BPF_BUF_REG_VEC;
+}
+
 static enum iou_test_status register_fixed_bufs(struct test_ctx *ctx)
 {
 	struct iovec iovecs[2];
 	int nr_iovecs = 0;
 	int ret;
 
-	if (ctx->src_type == IO_BPF_BUF_FIXED) {
+	if (is_registered_buf(ctx->src_type)) {
 		ctx->src_buf_index = nr_iovecs;
 		iovecs[nr_iovecs].iov_base = ctx->src_buf;
 		iovecs[nr_iovecs].iov_len = ctx->src_buf_size;
 		nr_iovecs++;
 	}
 
-	if (ctx->dst_type == IO_BPF_BUF_FIXED) {
+	if (is_registered_buf(ctx->dst_type)) {
 		ctx->dst_buf_index = nr_iovecs;
 		iovecs[nr_iovecs].iov_base = ctx->dst_buf;
 		iovecs[nr_iovecs].iov_len = ctx->dst_buf_size;
@@ -197,8 +204,8 @@ static enum iou_test_status register_fixed_bufs(struct test_ctx *ctx)
 
 static void unregister_fixed_bufs(struct test_ctx *ctx)
 {
-	if (ctx->src_type == IO_BPF_BUF_FIXED ||
-	    ctx->dst_type == IO_BPF_BUF_FIXED)
+	if (is_registered_buf(ctx->src_type) ||
+	    is_registered_buf(ctx->dst_type))
 		io_uring_unregister_buffers(&ctx->ring);
 }
 
@@ -249,6 +256,10 @@ static enum iou_test_status allocate_bufs(struct test_ctx *ctx)
 		ctx->descs[0].addr = (__u64)(uintptr_t)ctx->src_buf;
 		ctx->descs[0].len = ctx->src_buf_size;
 		ctx->descs[0].buf_index = ctx->src_buf_index;
+	} else if (ctx->src_type == IO_BPF_BUF_REG_VEC) {
+		ctx->descs[0].addr = (__u64)(uintptr_t)ctx->src_vec;
+		ctx->descs[0].len = ctx->src_nr_vec;
+		ctx->descs[0].buf_index = ctx->src_buf_index;
 	} else {
 		ctx->descs[0].addr = (__u64)(uintptr_t)ctx->src_buf;
 		ctx->descs[0].len = ctx->src_buf_size;
@@ -261,6 +272,10 @@ static enum iou_test_status allocate_bufs(struct test_ctx *ctx)
 		ctx->descs[1].addr = (__u64)(uintptr_t)ctx->dst_buf;
 		ctx->descs[1].len = ctx->dst_buf_size;
 		ctx->descs[1].buf_index = ctx->dst_buf_index;
+	} else if (ctx->dst_type == IO_BPF_BUF_REG_VEC) {
+		ctx->descs[1].addr = (__u64)(uintptr_t)ctx->dst_vec;
+		ctx->descs[1].len = ctx->dst_nr_vec;
+		ctx->descs[1].buf_index = ctx->dst_buf_index;
 	} else {
 		ctx->descs[1].addr = (__u64)(uintptr_t)ctx->dst_buf;
 		ctx->descs[1].len = ctx->dst_buf_size;
@@ -420,6 +435,30 @@ static enum iou_test_status copy_fixed_to_user(struct test_ctx *ctx)
 	return test_copy(ctx);
 }
 
+static enum iou_test_status copy_user_to_reg_vec(struct test_ctx *ctx)
+{
+	ctx->src_type = IO_BPF_BUF_USER;
+	ctx->dst_type = IO_BPF_BUF_REG_VEC;
+	ctx->src_buf_size = TEST_BUF_SIZE;
+	ctx->dst_buf_size = TEST_BUF_SIZE;
+	ctx->dst_nr_vec = 4;
+	ctx->desc = "USER -> REG_VEC";
+
+	return test_copy(ctx);
+}
+
+static enum iou_test_status copy_reg_vec_to_user(struct test_ctx *ctx)
+{
+	ctx->src_type = IO_BPF_BUF_REG_VEC;
+	ctx->dst_type = IO_BPF_BUF_USER;
+	ctx->src_buf_size = TEST_BUF_SIZE;
+	ctx->dst_buf_size = TEST_BUF_SIZE;
+	ctx->src_nr_vec = 4;
+	ctx->desc = "REG_VEC -> USER";
+
+	return test_copy(ctx);
+}
+
 static enum iou_test_status run(void *ctx_ptr)
 {
 	struct test_ctx *ctx = ctx_ptr;
@@ -445,6 +484,14 @@ static enum iou_test_status run(void *ctx_ptr)
 	if (status != IOU_TEST_PASS)
 		return status;
 
+	status = copy_user_to_reg_vec(ctx);
+	if (status != IOU_TEST_PASS)
+		return status;
+
+	status = copy_reg_vec_to_user(ctx);
+	if (status != IOU_TEST_PASS)
+		return status;
+
 	return IOU_TEST_PASS;
 }
 
@@ -462,7 +509,7 @@ static void cleanup(void *ctx_ptr)
 
 struct iou_test bpf_memcpy_test = {
 	.name = "bpf_memcpy",
-	.description = "Test uring_bpf_memcpy() kfunc with USER, VEC, FIXED buffer types",
+	.description = "Test uring_bpf_memcpy() kfunc with USER, VEC, FIXED, REG_VEC buffer types",
 	.setup = setup,
 	.run = run,
 	.cleanup = cleanup,
-- 
2.47.0