[PATCH liburing] tests: test io_uring bpf ops

[Pavel Begunkov <asml.silence@gmail.com>]

Add a BPF struct ops io_uring example issuing nop requests in a loop. It
needs clang, llvm, bpftool, libbpf, etc. to compile so it's gated on
passing BPF_TESTS to make. The Makefile change looks sane but maybe
there are better ways, starting with probing all the toolchain
availability.

Signed-off-by: Pavel Begunkov <asml.silence@gmail.com>
---
 test/Makefile                  | 44 ++++++++++++++-
 test/bpf-nops.c                | 99 ++++++++++++++++++++++++++++++++++
 test/bpf-progs/nops_loop.bpf.c | 99 ++++++++++++++++++++++++++++++++++
 3 files changed, 240 insertions(+), 2 deletions(-)
 create mode 100644 test/bpf-nops.c
 create mode 100644 test/bpf-progs/nops_loop.bpf.c

diff --git a/test/Makefile b/test/Makefile
index 1bbae1cc..0a4f275a 100644
--- a/test/Makefile
+++ b/test/Makefile
@@ -1,6 +1,12 @@
 prefix ?= /usr
 datadir ?= $(prefix)/share
 
+CLANG ?= clang
+BPFTOOL ?= bpftool
+BPF_PROGS_DIR = bpf-progs
+BPF_OUTPUT = output/bpf/
+BPF_VMLINUX ?= /sys/kernel/btf/vmlinux
+
 INSTALL=install
 
 ifneq ($(MAKECMDGOALS),clean)
@@ -298,6 +304,17 @@ ifdef CONFIG_HAVE_CXX
 endif
 all_targets += sq-full-cpp.t
 
+bpf_tests := bpf-nops.c
+opt_deps :=
+
+ifdef BPF_TESTS
+	override CFLAGS += -I$(BPF_OUTPUT)
+	override LDFLAGS += -lbpf
+	test_srcs += $(bpf_tests)
+	opt_deps += bpf-progs
+endif
+
+$(info flags $(CFLAGS))
 
 test_targets := $(patsubst %.c,%,$(test_srcs))
 test_targets := $(patsubst %.cc,%,$(test_targets))
@@ -321,7 +338,7 @@ helpers.o: helpers.c
 
 LIBURING := $(shell if [ -e ../src/liburing.a ]; then echo ../src/liburing.a; fi)
 
-%.t: %.c $(helpers) helpers.h $(LIBURING)
+%.t: %.c $(helpers) helpers.h $(LIBURING) $(opt_deps)
 	$(QUIET_CC)$(CC) $(CPPFLAGS) $(CFLAGS) -o $@ $< $(helpers) $(LDFLAGS)
 
 #
@@ -336,6 +353,29 @@ LIBURING := $(shell if [ -e ../src/liburing.a ]; then echo ../src/liburing.a; fi
 	$(patsubst -Wmissing-prototypes,,$(CXXFLAGS)) \
 	-o $@ $< $(helpers) $(LDFLAGS)
 
+BPF_PROGS = nops_loop.bpf.c
+
+CLANG_BPF_SYS_INCLUDES ?= $(shell $(CLANG) -v -E - </dev/null 2>&1 \
+	| sed -n '/<...> search starts here:/,/End of search list./{ s| \(/.*\)|-idirafter \1|p }')
+
+$(BPF_OUTPUT)/vmlinux.h:
+	mkdir -p $(BPF_OUTPUT)
+	$(BPFTOOL) btf dump file $(BPF_VMLINUX) format c > $@
+
+# Build BPF code
+$(BPF_OUTPUT)/%.bpf.o: $(BPF_PROGS_DIR)/%.bpf.c $(wildcard %.h) $(BPF_OUTPUT)/vmlinux.h
+	mkdir -p ${BPF_OUTPUT}
+	$(CLANG) -g -O2 -target bpf \
+		     -I$(BPF_OUTPUT) $(CLANG_BPF_SYS_INCLUDES) \
+		     -Wno-missing-declarations \
+		     -c $(filter %.c,$^) -o $(patsubst %.bpf.o,%.tmp.bpf.o,$@) -mcpu=v4
+	$(BPFTOOL) gen object $@ $(patsubst %.bpf.o,%.tmp.bpf.o,$@)
+
+$(BPF_OUTPUT)/%.skel.h: $(BPF_OUTPUT)/%.bpf.o $(BPF_OUTPUT)/vmlinux.h
+	$(BPFTOOL) gen skeleton $< > $@
+
+bpf-progs: $(patsubst %.bpf.c,$(BPF_OUTPUT)/%.bpf.o,$(BPF_PROGS)) \
+			$(patsubst %.bpf.c,$(BPF_OUTPUT)/%.skel.h,$(BPF_PROGS))
 
 install: $(test_targets) runtests.sh runtests-loop.sh
 	$(INSTALL) -D -d -m 755 $(datadir)/liburing-test/
@@ -362,4 +402,4 @@ runtests-loop: all
 runtests-parallel: $(run_test_targets)
 	@echo "All tests passed"
 
-.PHONY: all install clean runtests runtests-loop runtests-parallel
+.PHONY: all install clean runtests runtests-loop runtests-parallel bpf-progs
diff --git a/test/bpf-nops.c b/test/bpf-nops.c
new file mode 100644
index 00000000..3fe6befe
--- /dev/null
+++ b/test/bpf-nops.c
@@ -0,0 +1,99 @@
+/* SPDX-License-Identifier: GPL-2.0 */
+#include <linux/stddef.h>
+#include <errno.h>
+#include <signal.h>
+#include <stdlib.h>
+#include <bpf/libbpf.h>
+
+#include "liburing.h"
+#include "nops_loop.skel.h"
+#include "helpers.h"
+
+static struct nops_loop_bpf *skel;
+static struct bpf_link *nops_loop_bpf_link;
+
+#define CQ_ENTRIES 8
+#define SQ_ENTRIES 8
+#define NR_ITERS 1000
+
+static int setup_ring_ops(struct io_uring *ring)
+{
+	struct io_uring_params params;
+	int ret;
+
+	memset(&params, 0, sizeof(params));
+	params.cq_entries = CQ_ENTRIES;
+	params.flags = IORING_SETUP_SINGLE_ISSUER |
+			IORING_SETUP_DEFER_TASKRUN |
+			IORING_SETUP_NO_SQARRAY |
+			IORING_SETUP_CQSIZE |
+			IORING_SETUP_SQ_REWIND;
+
+	ret = t_create_ring_params(SQ_ENTRIES, ring, &params);
+	if (ret == T_SETUP_SKIP) {
+		printf("Can't setup a ring, skip\n");
+		return T_EXIT_SKIP;
+	}
+	if (ret != T_SETUP_OK)
+		return T_EXIT_FAIL;
+
+	skel = nops_loop_bpf__open();
+	if (!skel) {
+		fprintf(stderr, "can't generate skeleton\n");
+		return T_EXIT_FAIL;
+	}
+
+	skel->struct_ops.nops_ops->ring_fd = ring->ring_fd;
+	skel->bss->reqs_to_run = NR_ITERS;
+	skel->rodata->sq_hdr_offset = params.sq_off.head;
+	skel->rodata->cq_hdr_offset = params.cq_off.head;
+	skel->rodata->cqes_offset = params.cq_off.cqes;
+	skel->rodata->cq_entries = CQ_ENTRIES;
+	skel->rodata->sq_entries = SQ_ENTRIES;
+
+	ret = nops_loop_bpf__load(skel);
+	if (ret) {
+		if (ret == -ESRCH) {
+			printf("io_uring BPF ops are not supported\n");
+			return T_EXIT_SKIP;
+		}
+		fprintf(stderr, "failed to load skeleton\n");
+		return T_EXIT_FAIL;
+	}
+
+	nops_loop_bpf_link = bpf_map__attach_struct_ops(skel->maps.nops_ops);
+	if (!nops_loop_bpf_link) {
+		fprintf(stderr, "failed to attach ops\n");
+		return T_EXIT_FAIL;
+	}
+	return T_EXIT_PASS;
+}
+
+int main()
+{
+	struct io_uring ring;
+	unsigned left;
+	int ret;
+
+	ret = setup_ring_ops(&ring);
+	if (ret != T_EXIT_PASS)
+		return ret;
+
+	ret = io_uring_enter(ring.ring_fd, 0, 0, IORING_ENTER_GETEVENTS, NULL);
+	if (ret) {
+		fprintf(stderr, "run failed\n");
+		return T_EXIT_FAIL;
+	}
+
+	left = skel->bss->reqs_to_run;
+	if (left) {
+		fprintf(stderr, "Run failed, couldn't submit all nops %i / %i\n",
+			NR_ITERS - left, NR_ITERS);
+		return T_EXIT_FAIL;
+	}
+
+	bpf_link__destroy(nops_loop_bpf_link);
+	nops_loop_bpf__destroy(skel);
+	io_uring_queue_exit(&ring);
+	return T_EXIT_PASS;
+}
diff --git a/test/bpf-progs/nops_loop.bpf.c b/test/bpf-progs/nops_loop.bpf.c
new file mode 100644
index 00000000..00075bb6
--- /dev/null
+++ b/test/bpf-progs/nops_loop.bpf.c
@@ -0,0 +1,99 @@
+/* SPDX-License-Identifier: GPL-2.0 */
+#include "vmlinux.h"
+#include <bpf/bpf_helpers.h>
+#include <bpf/bpf_tracing.h>
+#include <linux/errno.h>
+
+char LICENSE[] SEC("license") = "Dual BSD/GPL";
+
+#define REQ_TOKEN 0xabba1741
+
+const unsigned max_inflight = 8;
+const volatile unsigned cq_hdr_offset;
+const volatile unsigned sq_hdr_offset;
+const volatile unsigned cqes_offset;
+const volatile unsigned cq_entries;
+const volatile unsigned sq_entries;
+
+unsigned reqs_inflight = 0;
+int reqs_to_run;
+
+#define t_min(a, b) ((a) < (b) ? (a) : (b))
+
+static unsigned nr_to_submit(void)
+{
+	unsigned to_submit = 0;
+	unsigned inflight = reqs_inflight;
+
+	if (inflight < max_inflight) {
+		to_submit = max_inflight - inflight;
+		to_submit = t_min(to_submit, reqs_to_run - inflight);
+	}
+	return to_submit;
+}
+
+SEC("struct_ops.s/nops_loop_step")
+int BPF_PROG(nops_loop_step, struct io_ring_ctx *ring, struct iou_loop_params *ls)
+{
+	struct io_uring_sqe *sqes;
+	struct io_uring_cqe *cqes;
+	struct io_uring *cq_hdr;
+	unsigned to_submit;
+	unsigned to_wait;
+	unsigned nr_cqes;
+	void *rings;
+	int ret, i;
+
+	sqes = (void *)bpf_io_uring_get_region(ring, IOU_REGION_SQ,
+				sq_entries * sizeof(struct io_uring_sqe));
+	rings = (void *)bpf_io_uring_get_region(ring, IOU_REGION_CQ,
+				cqes_offset + cq_entries * sizeof(struct io_uring_cqe));
+	if (!rings || !sqes)
+		return IOU_LOOP_STOP;
+	cq_hdr = rings + cq_hdr_offset;
+	cqes = rings + cqes_offset;
+
+	to_submit = nr_to_submit();
+	if (to_submit) {
+		for (i = 0; i < to_submit; i++) {
+			struct io_uring_sqe *sqe = &sqes[i];
+
+			*sqe = (struct io_uring_sqe){};
+			sqe->opcode = IORING_OP_NOP;
+			sqe->user_data = REQ_TOKEN;
+		}
+
+		ret = bpf_io_uring_submit_sqes(ring, to_submit);
+		if (ret != to_submit)
+			return IOU_LOOP_STOP;
+		reqs_inflight += to_submit;
+	}
+
+	nr_cqes = cq_hdr->tail - cq_hdr->head;
+	nr_cqes = t_min(nr_cqes, max_inflight);
+	for (i = 0; i < nr_cqes; i++) {
+		struct io_uring_cqe *cqe = &cqes[cq_hdr->head & (cq_entries - 1)];
+
+		if (cqe->user_data != REQ_TOKEN)
+			return IOU_LOOP_STOP;
+		cq_hdr->head++;
+	}
+
+	reqs_inflight -= nr_cqes;
+	reqs_to_run -= nr_cqes;
+
+	if (reqs_to_run <= 0 && !reqs_inflight)
+		return IOU_LOOP_STOP;
+
+	to_wait = reqs_inflight;
+	/* Don't sleep if there are still CQEs left */
+	if (cq_hdr->tail != cq_hdr->head)
+		to_wait = 0;
+	ls->cq_wait_idx = cq_hdr->head + to_wait;
+	return IOU_LOOP_CONTINUE;
+}
+
+SEC(".struct_ops.link")
+struct io_uring_bpf_ops nops_ops = {
+	.loop_step = (void *)nops_loop_step,
+};
-- 
2.53.0