[PATCH 17/23] io_uring: enable bpf to reap CQEs

[Pavel Begunkov <asml.silence@gmail.com>]

Signed-off-by: Pavel Begunkov <asml.silence@gmail.com>
---
 fs/io_uring.c            | 48 ++++++++++++++++++++++++++++++++++++++++
 include/uapi/linux/bpf.h |  1 +
 2 files changed, 49 insertions(+)

diff --git a/fs/io_uring.c b/fs/io_uring.c
index 464d630904e2..7c165b2ce8e4 100644
--- a/fs/io_uring.c
+++ b/fs/io_uring.c
@@ -10394,6 +10394,42 @@ BPF_CALL_5(io_bpf_emit_cqe, struct io_bpf_ctx *,		bpf_ctx,
 	return submitted ? 0 : -ENOMEM;
 }
 
+BPF_CALL_4(io_bpf_reap_cqe, struct io_bpf_ctx *,		bpf_ctx,
+			    u32,				cq_idx,
+			    struct io_uring_cqe *,		cqe_out,
+			    u32,				cqe_len)
+{
+	struct io_ring_ctx *ctx = bpf_ctx->ctx;
+	struct io_uring_cqe *cqe;
+	struct io_cqring *cq;
+	struct io_rings *r;
+	unsigned tail, head, mask;
+	int ret = -EINVAL;
+
+	if (unlikely(cqe_len != sizeof(*cqe_out)))
+		goto err;
+	if (unlikely(cq_idx >= ctx->cq_nr))
+		goto err;
+
+	cq = &ctx->cqs[cq_idx];
+	r = cq->rings;
+	tail = READ_ONCE(r->cq.tail);
+	head = smp_load_acquire(&r->cq.head);
+
+	ret = -ENOENT;
+	if (unlikely(tail == head))
+		goto err;
+
+	mask = cq->entries - 1;
+	cqe = &r->cqes[head & mask];
+	memcpy(cqe_out, cqe, sizeof(*cqe_out));
+	WRITE_ONCE(r->cq.head, head + 1);
+	return 0;
+err:
+	memset(cqe_out, 0, sizeof(*cqe_out));
+	return ret;
+}
+
 const struct bpf_func_proto io_bpf_queue_sqe_proto = {
 	.func = io_bpf_queue_sqe,
 	.gpl_only = false,
@@ -10414,6 +10450,16 @@ const struct bpf_func_proto io_bpf_emit_cqe_proto = {
 	.arg5_type = ARG_ANYTHING,
 };
 
+const struct bpf_func_proto io_bpf_reap_cqe_proto = {
+	.func = io_bpf_reap_cqe,
+	.gpl_only = false,
+	.ret_type = RET_INTEGER,
+	.arg1_type = ARG_PTR_TO_CTX,
+	.arg2_type = ARG_ANYTHING,
+	.arg3_type = ARG_PTR_TO_UNINIT_MEM,
+	.arg4_type = ARG_CONST_SIZE,
+};
+
 static const struct bpf_func_proto *
 io_bpf_func_proto(enum bpf_func_id func_id, const struct bpf_prog *prog)
 {
@@ -10424,6 +10470,8 @@ io_bpf_func_proto(enum bpf_func_id func_id, const struct bpf_prog *prog)
 		return prog->aux->sleepable ? &io_bpf_queue_sqe_proto : NULL;
 	case BPF_FUNC_iouring_emit_cqe:
 		return &io_bpf_emit_cqe_proto;
+	case BPF_FUNC_iouring_reap_cqe:
+		return &io_bpf_reap_cqe_proto;
 	default:
 		return bpf_base_func_proto(func_id);
 	}
diff --git a/include/uapi/linux/bpf.h b/include/uapi/linux/bpf.h
index c6b023be7848..7719ec4a33e7 100644
--- a/include/uapi/linux/bpf.h
+++ b/include/uapi/linux/bpf.h
@@ -4084,6 +4084,7 @@ union bpf_attr {
 	FN(check_mtu),			\
 	FN(iouring_queue_sqe),		\
 	FN(iouring_emit_cqe),		\
+	FN(iouring_reap_cqe),		\
 	/* */
 
 /* integer value in 'imm' field of BPF_CALL instruction selects which helper
-- 
2.31.1