[PATCH liburing 0/4] zerocopy send API changes

[PATCH liburing 0/4] zerocopy send API changes

[Pavel Begunkov]

Fix up helpers and tests to match API changes and also add some more tests.

Pavel Begunkov (4):
  tests: verify that send addr is copied when async
  zc: adjust sendzc to the simpler uapi
  test: test iowq zc sends
  examples: adjust zc bench to the new uapi

 examples/send-zerocopy.c        |  72 ++--
 src/include/liburing.h          |  39 +-
 src/include/liburing/io_uring.h |  28 +-
 src/register.c                  |  20 -
 test/send-zerocopy.c            | 667 +++++++-------------------------
 5 files changed, 178 insertions(+), 648 deletions(-)

-- 
2.37.2

[PATCH liburing 1/4] tests: verify that send addr is copied when async

[Pavel Begunkov]

The kernel should not be touching timespec after returning from submit
syscall, make sure it's not reloaded after a request goes async.

Signed-off-by: Pavel Begunkov <asml.silence@gmail.com>
---
 test/send-zerocopy.c | 69 ++++++++++++++++++++++++++++++++++++++++++++
 1 file changed, 69 insertions(+)

diff --git a/test/send-zerocopy.c b/test/send-zerocopy.c
index 307c114..97727e3 100644
--- a/test/send-zerocopy.c
+++ b/test/send-zerocopy.c
@@ -836,6 +836,69 @@ static int test_inet_send(struct io_uring *ring)
 	return 0;
 }
 
+static int test_async_addr(struct io_uring *ring)
+{
+	struct io_uring_sqe *sqe;
+	struct io_uring_cqe *cqe;
+	struct sockaddr_storage addr;
+	int sock_tx = -1, sock_rx = -1;
+	struct __kernel_timespec ts;
+	int ret;
+
+	ret = prepare_ip(&addr, &sock_tx, &sock_rx, true, false, false, false);
+	if (ret) {
+		fprintf(stderr, "sock prep failed %d\n", ret);
+		return 1;
+	}
+
+	sqe = io_uring_get_sqe(ring);
+	ts.tv_sec = 1;
+	ts.tv_nsec = 0;
+	io_uring_prep_timeout(sqe, &ts, 0, IORING_TIMEOUT_ETIME_SUCCESS);
+	sqe->user_data = 1;
+	sqe->flags |= IOSQE_IO_LINK;
+
+	sqe = io_uring_get_sqe(ring);
+	io_uring_prep_sendzc(sqe, sock_tx, tx_buffer, 1, 0, 0, 0);
+	sqe->user_data = 2;
+	io_uring_prep_sendzc_set_addr(sqe, (const struct sockaddr *)&addr,
+				      sizeof(struct sockaddr_in6));
+
+	ret = io_uring_submit(ring);
+	assert(ret == 2);
+	memset(&addr, 0, sizeof(addr));
+
+	ret = io_uring_wait_cqe(ring, &cqe);
+	if (ret) {
+		fprintf(stderr, "io_uring_wait_cqe failed %i\n", ret);
+		return 1;
+	}
+	if (cqe->user_data != 1 || cqe->res != -ETIME) {
+		fprintf(stderr, "invalid timeout res %i %i\n",
+			(int)cqe->user_data, cqe->res);
+		return 1;
+	}
+	io_uring_cqe_seen(ring, cqe);
+
+	ret = io_uring_wait_cqe(ring, &cqe);
+	if (ret) {
+		fprintf(stderr, "io_uring_wait_cqe failed %i\n", ret);
+		return 1;
+	}
+	if (cqe->user_data != 2 || cqe->res != 1) {
+		fprintf(stderr, "invalid send %i %i\n",
+			(int)cqe->user_data, cqe->res);
+		return 1;
+	}
+	io_uring_cqe_seen(ring, cqe);
+	ret = recv(sock_rx, rx_buffer, 1, MSG_TRUNC);
+	assert(ret == 1);
+
+	close(sock_tx);
+	close(sock_rx);
+	return 0;
+}
+
 int main(int argc, char *argv[])
 {
 	struct io_uring ring;
@@ -973,6 +1036,12 @@ int main(int argc, char *argv[])
 		fprintf(stderr, "test_inet_send() failed\n");
 		return ret;
 	}
+
+	ret = test_async_addr(&ring);
+	if (ret) {
+		fprintf(stderr, "test_async_addr() failed\n");
+		return ret;
+	}
 out:
 	io_uring_queue_exit(&ring);
 	close(sp[0]);
-- 
2.37.2

[PATCH liburing 2/4] zc: adjust sendzc to the simpler uapi

[Pavel Begunkov]

We update the zerocopy API, reflect it in liburing, this includes
removing the whole notification story, updating headers and fixing
up tests.

Signed-off-by: Pavel Begunkov <asml.silence@gmail.com>
---
 src/include/liburing.h          |  39 +-
 src/include/liburing/io_uring.h |  28 +-
 src/register.c                  |  20 --
 test/send-zerocopy.c            | 611 ++++----------------------------
 4 files changed, 91 insertions(+), 607 deletions(-)

diff --git a/src/include/liburing.h b/src/include/liburing.h
index 68a1646..c672440 100644
--- a/src/include/liburing.h
+++ b/src/include/liburing.h
@@ -199,10 +199,6 @@ int io_uring_register_sync_cancel(struct io_uring *ring,
 int io_uring_register_file_alloc_range(struct io_uring *ring,
 					unsigned off, unsigned len);
 
-int io_uring_register_notifications(struct io_uring *ring, unsigned nr,
-				    struct io_uring_notification_slot *slots);
-int io_uring_unregister_notifications(struct io_uring *ring);
-
 /*
  * io_uring syscalls.
  */
@@ -702,44 +698,23 @@ static inline void io_uring_prep_send(struct io_uring_sqe *sqe, int sockfd,
 	sqe->msg_flags = (__u32) flags;
 }
 
-static inline void io_uring_prep_sendzc(struct io_uring_sqe *sqe, int sockfd,
-				        const void *buf, size_t len, int flags,
-				        unsigned slot_idx, unsigned zc_flags)
+static inline void io_uring_prep_send_zc(struct io_uring_sqe *sqe, int sockfd,
+					 const void *buf, size_t len, int flags,
+					 unsigned zc_flags)
 {
-	io_uring_prep_rw(IORING_OP_SENDZC_NOTIF, sqe, sockfd, buf, (__u32) len, 0);
+	io_uring_prep_rw(IORING_OP_SEND_ZC, sqe, sockfd, buf, (__u32) len, 0);
 	sqe->msg_flags = (__u32) flags;
-	sqe->notification_idx = slot_idx;
 	sqe->ioprio = zc_flags;
 }
 
-static inline void io_uring_prep_sendzc_fixed(struct io_uring_sqe *sqe, int sockfd,
-					      const void *buf, size_t len,
-					      int flags, unsigned slot_idx,
-					      unsigned zc_flags, unsigned buf_idx)
-{
-	io_uring_prep_sendzc(sqe, sockfd, buf, len, flags, slot_idx, zc_flags);
-	sqe->ioprio |= IORING_RECVSEND_FIXED_BUF;
-	sqe->buf_index = buf_idx;
-}
-
-static inline void io_uring_prep_sendzc_set_addr(struct io_uring_sqe *sqe,
-						 const struct sockaddr *dest_addr,
-						 __u16 addr_len)
+static inline void io_uring_prep_send_set_addr(struct io_uring_sqe *sqe,
+						const struct sockaddr *dest_addr,
+						__u16 addr_len)
 {
 	sqe->addr2 = (unsigned long)(void *)dest_addr;
 	sqe->addr_len = addr_len;
 }
 
-static inline void io_uring_prep_notif_update(struct io_uring_sqe *sqe,
-					      __u64 new_tag, /* 0 to ignore */
-					      unsigned offset, unsigned nr)
-{
-	io_uring_prep_rw(IORING_OP_FILES_UPDATE, sqe, -1, 0, nr,
-			 (__u64)offset);
-	sqe->addr = new_tag;
-	sqe->ioprio = IORING_RSRC_UPDATE_NOTIF;
-}
-
 static inline void io_uring_prep_recv(struct io_uring_sqe *sqe, int sockfd,
 				      void *buf, size_t len, int flags)
 {
diff --git a/src/include/liburing/io_uring.h b/src/include/liburing/io_uring.h
index 9e0b5c8..6b83177 100644
--- a/src/include/liburing/io_uring.h
+++ b/src/include/liburing/io_uring.h
@@ -71,8 +71,8 @@ struct io_uring_sqe {
 		__s32	splice_fd_in;
 		__u32	file_index;
 		struct {
-			__u16	notification_idx;
 			__u16	addr_len;
+			__u16	__pad3[1];
 		};
 	};
 	union {
@@ -178,8 +178,7 @@ enum io_uring_op {
 	IORING_OP_FALLOCATE,
 	IORING_OP_OPENAT,
 	IORING_OP_CLOSE,
-	IORING_OP_RSRC_UPDATE,
-	IORING_OP_FILES_UPDATE = IORING_OP_RSRC_UPDATE,
+	IORING_OP_FILES_UPDATE,
 	IORING_OP_STATX,
 	IORING_OP_READ,
 	IORING_OP_WRITE,
@@ -206,7 +205,7 @@ enum io_uring_op {
 	IORING_OP_GETXATTR,
 	IORING_OP_SOCKET,
 	IORING_OP_URING_CMD,
-	IORING_OP_SENDZC_NOTIF,
+	IORING_OP_SEND_ZC,
 
 	/* this goes last, obviously */
 	IORING_OP_LAST,
@@ -228,7 +227,6 @@ enum io_uring_op {
 #define IORING_TIMEOUT_ETIME_SUCCESS	(1U << 5)
 #define IORING_TIMEOUT_CLOCK_MASK	(IORING_TIMEOUT_BOOTTIME | IORING_TIMEOUT_REALTIME)
 #define IORING_TIMEOUT_UPDATE_MASK	(IORING_TIMEOUT_UPDATE | IORING_LINK_TIMEOUT_UPDATE)
-
 /*
  * sqe->splice_flags
  * extends splice(2) flags
@@ -281,29 +279,16 @@ enum io_uring_op {
  *
  * IORING_RECVSEND_FIXED_BUF	Use registered buffers, the index is stored in
  *				the buf_index field.
- *
- * IORING_RECVSEND_NOTIF_FLUSH	Flush a notification after a successful
- *				successful. Only for zerocopy sends.
  */
 #define IORING_RECVSEND_POLL_FIRST	(1U << 0)
 #define IORING_RECV_MULTISHOT		(1U << 1)
 #define IORING_RECVSEND_FIXED_BUF	(1U << 2)
-#define IORING_RECVSEND_NOTIF_FLUSH	(1U << 3)
 
 /*
  * accept flags stored in sqe->ioprio
  */
 #define IORING_ACCEPT_MULTISHOT	(1U << 0)
 
-
-/*
- * IORING_OP_RSRC_UPDATE flags
- */
-enum {
-	IORING_RSRC_UPDATE_FILES,
-	IORING_RSRC_UPDATE_NOTIF,
-};
-
 /*
  * IORING_OP_MSG_RING command types, stored in sqe->addr
  */
@@ -341,10 +326,13 @@ struct io_uring_cqe {
  * IORING_CQE_F_BUFFER	If set, the upper 16 bits are the buffer ID
  * IORING_CQE_F_MORE	If set, parent SQE will generate more CQE entries
  * IORING_CQE_F_SOCK_NONEMPTY	If set, more data to read after socket recv
+ * IORING_CQE_F_NOTIF	Set for notification CQEs. Can be used to distinct
+ * 			them from sends.
  */
 #define IORING_CQE_F_BUFFER		(1U << 0)
 #define IORING_CQE_F_MORE		(1U << 1)
 #define IORING_CQE_F_SOCK_NONEMPTY	(1U << 2)
+#define IORING_CQE_F_NOTIF		(1U << 3)
 
 enum {
 	IORING_CQE_BUFFER_SHIFT		= 16,
@@ -485,10 +473,6 @@ enum {
 	/* register a range of fixed file slots for automatic slot allocation */
 	IORING_REGISTER_FILE_ALLOC_RANGE	= 25,
 
-	/* zerocopy notification API */
-	IORING_REGISTER_NOTIFIERS		= 26,
-	IORING_UNREGISTER_NOTIFIERS		= 27,
-
 	/* this goes last */
 	IORING_REGISTER_LAST
 };
diff --git a/src/register.c b/src/register.c
index 7482112..2b37e5f 100644
--- a/src/register.c
+++ b/src/register.c
@@ -364,23 +364,3 @@ int io_uring_register_file_alloc_range(struct io_uring *ring,
 				       IORING_REGISTER_FILE_ALLOC_RANGE, &range,
 				       0);
 }
-
-int io_uring_register_notifications(struct io_uring *ring, unsigned nr,
-				    struct io_uring_notification_slot *slots)
-{
-	struct io_uring_notification_register r = {
-		.nr_slots = nr,
-		.data = (unsigned long)slots,
-	};
-
-	return __sys_io_uring_register(ring->ring_fd,
-				       IORING_REGISTER_NOTIFIERS,
-				       &r, sizeof(r));
-}
-
-int io_uring_unregister_notifications(struct io_uring *ring)
-{
-	return __sys_io_uring_register(ring->ring_fd,
-				       IORING_UNREGISTER_NOTIFIERS,
-				       NULL, 0);
-}
diff --git a/test/send-zerocopy.c b/test/send-zerocopy.c
index 97727e3..7b58ae7 100644
--- a/test/send-zerocopy.c
+++ b/test/send-zerocopy.c
@@ -44,7 +44,6 @@
 #define HOST	"127.0.0.1"
 #define HOSTV6	"::1"
 
-#define NR_SLOTS 5
 #define ZC_TAG 10000
 #define BUFFER_OFFSET 41
 
@@ -52,15 +51,9 @@
 	#define ARRAY_SIZE(a) (sizeof(a)/sizeof((a)[0]))
 #endif
 
-static int seqs[NR_SLOTS];
 static char *tx_buffer, *rx_buffer;
 static struct iovec buffers_iov[3];
 
-static inline bool tag_userdata(__u64 user_data)
-{
-	return ZC_TAG <= user_data && user_data < ZC_TAG + NR_SLOTS;
-}
-
 static bool check_cq_empty(struct io_uring *ring)
 {
 	struct io_uring_cqe *cqe = NULL;
@@ -70,41 +63,18 @@ static bool check_cq_empty(struct io_uring *ring)
 	return ret == -EAGAIN;
 }
 
-static int register_notifications(struct io_uring *ring)
-{
-	struct io_uring_notification_slot slots[NR_SLOTS] = {};
-	int i;
-
-	memset(seqs, 0, sizeof(seqs));
-	for (i = 0; i < NR_SLOTS; i++)
-		slots[i].tag = ZC_TAG + i;
-	return io_uring_register_notifications(ring, NR_SLOTS, slots);
-}
-
-static int reregister_notifications(struct io_uring *ring)
-{
-	int ret;
-
-	ret = io_uring_unregister_notifications(ring);
-	if (ret) {
-		fprintf(stderr, "unreg notifiers failed %i\n", ret);
-		return ret;
-	}
-
-	return register_notifications(ring);
-}
-
-static int do_one(struct io_uring *ring, int sock_tx, int slot_idx)
+static int test_basic_send(struct io_uring *ring, int sock_tx, int sock_rx)
 {
 	struct io_uring_sqe *sqe;
 	struct io_uring_cqe *cqe;
 	int msg_flags = 0;
 	unsigned zc_flags = 0;
+	int payload_size = 100;
 	int ret;
 
 	sqe = io_uring_get_sqe(ring);
-	io_uring_prep_sendzc(sqe, sock_tx, tx_buffer, 1, msg_flags,
-			     slot_idx, zc_flags);
+	io_uring_prep_send_zc(sqe, sock_tx, tx_buffer, payload_size,
+			      msg_flags, zc_flags);
 	sqe->user_data = 1;
 
 	ret = io_uring_submit(ring);
@@ -112,434 +82,61 @@ static int do_one(struct io_uring *ring, int sock_tx, int slot_idx)
 	ret = io_uring_wait_cqe(ring, &cqe);
 	assert(!ret);
 	assert(cqe->user_data == 1);
-	ret = cqe->res;
+	if (ret == -EINVAL) {
+		assert(!(cqe->flags & IORING_CQE_F_MORE));
+		return T_EXIT_SKIP;
+	}
+	assert(cqe->res == payload_size);
+	assert(cqe->flags & IORING_CQE_F_MORE);
 	io_uring_cqe_seen(ring, cqe);
-	assert(check_cq_empty(ring));
-	return ret;
-}
-
-static int test_invalid_slot(struct io_uring *ring, int sock_tx, int sock_rx)
-{
-	int ret;
-
-	ret = do_one(ring, sock_tx, NR_SLOTS);
-	assert(ret == -EINVAL);
-	return 0;
-}
-
-static int test_basic_send(struct io_uring *ring, int sock_tx, int sock_rx)
-{
-	struct io_uring_sqe *sqe;
-	struct io_uring_cqe *cqe;
-	int msg_flags = 0;
-	int slot_idx = 0;
-	unsigned zc_flags = 0;
-	int payload_size = 100;
-	int ret;
-
-	sqe = io_uring_get_sqe(ring);
-	io_uring_prep_sendzc(sqe, sock_tx, tx_buffer, payload_size, msg_flags,
-			     slot_idx, zc_flags);
-	sqe->user_data = 1;
 
-	ret = io_uring_submit(ring);
-	assert(ret == 1);
 	ret = io_uring_wait_cqe(ring, &cqe);
 	assert(!ret);
-	assert(cqe->user_data == 1 && cqe->res >= 0);
+	assert(cqe->user_data == 1);
+	assert(cqe->flags & IORING_CQE_F_NOTIF);
+	assert(!(cqe->flags & IORING_CQE_F_MORE));
 	io_uring_cqe_seen(ring, cqe);
 	assert(check_cq_empty(ring));
 
 	ret = recv(sock_rx, rx_buffer, payload_size, MSG_TRUNC);
 	assert(ret == payload_size);
-	return 0;
+	return T_EXIT_PASS;
 }
 
-static int test_send_flush(struct io_uring *ring, int sock_tx, int sock_rx)
+static int test_send_faults(struct io_uring *ring, int sock_tx, int sock_rx)
 {
 	struct io_uring_sqe *sqe;
 	struct io_uring_cqe *cqe;
 	int msg_flags = 0;
-	int slot_idx = 0;
 	unsigned zc_flags = 0;
 	int payload_size = 100;
-	int ret, i, j;
-	int req_cqes, notif_cqes;
-
-	/* now do send+flush, do many times to verify seqs */
-	for (j = 0; j < NR_SLOTS * 5; j++) {
-		zc_flags = IORING_RECVSEND_NOTIF_FLUSH;
-		slot_idx = rand() % NR_SLOTS;
-		sqe = io_uring_get_sqe(ring);
-		io_uring_prep_sendzc(sqe, sock_tx, tx_buffer, payload_size,
-				     msg_flags, slot_idx, zc_flags);
-		sqe->user_data = 1;
-
-		ret = io_uring_submit(ring);
-		assert(ret == 1);
-
-		req_cqes = notif_cqes = 1;
-		for (i = 0; i < 2; i ++) {
-			ret = io_uring_wait_cqe(ring, &cqe);
-			assert(!ret);
-
-			if (cqe->user_data == 1) {
-				assert(req_cqes > 0);
-				req_cqes--;
-				assert(cqe->res == payload_size);
-			} else if (cqe->user_data == ZC_TAG + slot_idx) {
-				assert(notif_cqes > 0);
-				notif_cqes--;
-				assert(cqe->res == 0 && cqe->flags == seqs[slot_idx]);
-				seqs[slot_idx]++;
-			} else {
-				fprintf(stderr, "invalid cqe %lu %i\n",
-					(unsigned long)cqe->user_data, cqe->res);
-				return -1;
-			}
-			io_uring_cqe_seen(ring, cqe);
-		}
-		assert(check_cq_empty(ring));
-
-		ret = recv(sock_rx, rx_buffer, payload_size, MSG_TRUNC);
-		assert(ret == payload_size);
-	}
-	return 0;
-}
-
-static int test_multireq_notif(struct io_uring *ring, int sock_tx, int sock_rx)
-{
-	bool slot_seen[NR_SLOTS] = {};
-	struct io_uring_sqe *sqe;
-	struct io_uring_cqe *cqe;
-	int msg_flags = 0;
-	int slot_idx = 0;
-	unsigned zc_flags = 0;
-	int payload_size = 1;
-	int ret, j, i = 0;
-	int nr = NR_SLOTS * 21;
-
-	while (i < nr) {
-		int nr_per_wave = 23;
-
-		for (j = 0; j < nr_per_wave && i < nr; j++, i++) {
-			slot_idx = rand() % NR_SLOTS;
-			sqe = io_uring_get_sqe(ring);
-			io_uring_prep_sendzc(sqe, sock_tx, tx_buffer, payload_size,
-					     msg_flags, slot_idx, zc_flags);
-			sqe->user_data = i;
-		}
-		ret = io_uring_submit(ring);
-		assert(ret == j);
-	}
-
-	for (i = 0; i < nr; i++) {
-		ret = io_uring_wait_cqe(ring, &cqe);
-		assert(!ret);
-		assert(cqe->user_data < nr && cqe->res == payload_size);
-		io_uring_cqe_seen(ring, cqe);
-
-		ret = recv(sock_rx, rx_buffer, payload_size, MSG_TRUNC);
-		assert(ret == payload_size);
-	}
-	assert(check_cq_empty(ring));
-
-	zc_flags = IORING_RECVSEND_NOTIF_FLUSH;
-	for (slot_idx = 0; slot_idx < NR_SLOTS; slot_idx++) {
-		sqe = io_uring_get_sqe(ring);
-		io_uring_prep_sendzc(sqe, sock_tx, tx_buffer, payload_size,
-				     msg_flags, slot_idx, zc_flags);
-		sqe->user_data = slot_idx;
-		/* just to simplify cqe handling */
-		sqe->flags |= IOSQE_CQE_SKIP_SUCCESS;
-	}
-	ret = io_uring_submit(ring);
-	assert(ret == NR_SLOTS);
-
-	for (i = 0; i < NR_SLOTS; i++) {
-		int slot_idx;
-
-		ret = io_uring_wait_cqe(ring, &cqe);
-		assert(!ret);
-		assert(tag_userdata(cqe->user_data));
-
-		slot_idx = cqe->user_data - ZC_TAG;
-		assert(!slot_seen[slot_idx]);
-		slot_seen[slot_idx] = true;
-
-		assert(cqe->res == 0 && cqe->flags == seqs[slot_idx]);
-		seqs[slot_idx]++;
-		io_uring_cqe_seen(ring, cqe);
-
-		ret = recv(sock_rx, rx_buffer, payload_size, MSG_TRUNC);
-		assert(ret == payload_size);
-	}
-	assert(check_cq_empty(ring));
-
-	for (i = 0; i < NR_SLOTS; i++)
-		assert(slot_seen[i]);
-	return 0;
-}
-
-static int test_multi_send_flushing(struct io_uring *ring, int sock_tx, int sock_rx)
-{
-	struct io_uring_sqe *sqe;
-	struct io_uring_cqe *cqe;
-	unsigned zc_flags = IORING_RECVSEND_NOTIF_FLUSH;
-	int msg_flags = 0, slot_idx = 0;
-	int payload_size = 1;
-	int ret, j, i = 0;
-	int nr = NR_SLOTS * 30;
-	unsigned long long check = 0, expected = 0;
-
-	while (i < nr) {
-		int nr_per_wave = 25;
-
-		for (j = 0; j < nr_per_wave && i < nr; j++, i++) {
-			sqe = io_uring_get_sqe(ring);
-			io_uring_prep_sendzc(sqe, sock_tx, tx_buffer, payload_size,
-					     msg_flags, slot_idx, zc_flags);
-			sqe->user_data = 1;
-			sqe->flags |= IOSQE_CQE_SKIP_SUCCESS;
-		}
-		ret = io_uring_submit(ring);
-		assert(ret == j);
-	}
-
-	for (i = 0; i < nr; i++) {
-		int seq;
-
-		ret = io_uring_wait_cqe(ring, &cqe);
-		assert(!ret);
-		assert(!cqe->res);
-		assert(tag_userdata(cqe->user_data));
-
-		seq = cqe->flags;
-		check += seq * 100007UL;
-		io_uring_cqe_seen(ring, cqe);
-
-		ret = recv(sock_rx, rx_buffer, payload_size, MSG_TRUNC);
-		assert(ret == payload_size);
-	}
-	assert(check_cq_empty(ring));
-
-	for (i = 0; i < nr; i++)
-		expected += (i + seqs[slot_idx]) * 100007UL;
-	assert(check == expected);
-	seqs[slot_idx] += nr;
-	return 0;
-}
-
-static int do_one_fail_notif_flush(struct io_uring *ring, int off, int nr)
-{
-	struct io_uring_cqe *cqe;
-	struct io_uring_sqe *sqe;
-	int ret;
-
-	/* single out-of-bounds slot */
-	sqe = io_uring_get_sqe(ring);
-	io_uring_prep_notif_update(sqe, 0, off, nr);
-	sqe->user_data = 1;
-	ret = io_uring_submit(ring);
-	assert(ret == 1);
-	ret = io_uring_wait_cqe(ring, &cqe);
-	assert(!ret && cqe->user_data == 1);
-	ret = cqe->res;
-	io_uring_cqe_seen(ring, cqe);
-	return ret;
-}
-
-static int test_update_flush_fail(struct io_uring *ring)
-{
-	int ret;
-
-	/* single out-of-bounds slot */
-	ret = do_one_fail_notif_flush(ring, NR_SLOTS, 1);
-	assert(ret == -EINVAL);
-
-	/* out-of-bounds range */
-	ret = do_one_fail_notif_flush(ring, 0, NR_SLOTS + 3);
-	assert(ret == -EINVAL);
-	ret = do_one_fail_notif_flush(ring, NR_SLOTS - 1, 2);
-	assert(ret == -EINVAL);
-
-	/* overflow checks, note it's u32 internally */
-	ret = do_one_fail_notif_flush(ring, ~(__u32)0, 1);
-	assert(ret == -EOVERFLOW);
-	ret = do_one_fail_notif_flush(ring, NR_SLOTS - 1, ~(__u32)0);
-	assert(ret == -EOVERFLOW);
-	return 0;
-}
-
-static void do_one_consume(struct io_uring *ring, int sock_tx, int sock_rx,
-			  int slot_idx)
-{
-	int ret;
-
-	ret = do_one(ring, sock_tx, slot_idx);
-	assert(ret == 1);
-
-	ret = recv(sock_rx, rx_buffer, 1, MSG_TRUNC);
-	assert(ret == 1);
-}
-
-static int test_update_flush(struct io_uring *ring, int sock_tx, int sock_rx)
-{
-	struct io_uring_sqe *sqe;
-	struct io_uring_cqe *cqe;
-	int offset = 1, nr_to_flush = 3;
-	int ret, i, slot_idx;
-
-	/*
-	 * Flush will be skipped for unused slots, so attached at least 1 req
-	 * to each active notifier / slot
-	 */
-	for (slot_idx = 0; slot_idx < NR_SLOTS; slot_idx++)
-		do_one_consume(ring, sock_tx, sock_rx, slot_idx);
-
-	assert(check_cq_empty(ring));
-
-	/* flush first */
-	sqe = io_uring_get_sqe(ring);
-	io_uring_prep_notif_update(sqe, 0, 0, 1);
-	sqe->user_data = 1;
-	sqe->flags |= IOSQE_CQE_SKIP_SUCCESS;
-	ret = io_uring_submit(ring);
-	assert(ret == 1);
-
-	ret = io_uring_wait_cqe(ring, &cqe);
-	assert(!ret && !cqe->res && cqe->user_data == ZC_TAG);
-	assert(cqe->flags == seqs[0]);
-	seqs[0]++;
-	io_uring_cqe_seen(ring, cqe);
-	do_one_consume(ring, sock_tx, sock_rx, 0);
-	assert(check_cq_empty(ring));
+	int ret, i;
 
-	/* flush last */
 	sqe = io_uring_get_sqe(ring);
-	io_uring_prep_notif_update(sqe, 0, NR_SLOTS - 1, 1);
+	io_uring_prep_send_zc(sqe, sock_tx, (void *)1UL, payload_size,
+			      msg_flags, zc_flags);
 	sqe->user_data = 1;
-	sqe->flags |= IOSQE_CQE_SKIP_SUCCESS;
-	ret = io_uring_submit(ring);
-	assert(ret == 1);
-
-	ret = io_uring_wait_cqe(ring, &cqe);
-	assert(!ret && !cqe->res && cqe->user_data == ZC_TAG + NR_SLOTS - 1);
-	assert(cqe->flags == seqs[NR_SLOTS - 1]);
-	seqs[NR_SLOTS - 1]++;
-	io_uring_cqe_seen(ring, cqe);
-	assert(check_cq_empty(ring));
 
-	/* we left the last slot without attached requests, flush should ignore it */
 	sqe = io_uring_get_sqe(ring);
-	io_uring_prep_notif_update(sqe, 0, NR_SLOTS - 1, 1);
-	sqe->user_data = 1;
-	ret = io_uring_submit(ring);
-	assert(ret == 1);
-
-	ret = io_uring_wait_cqe(ring, &cqe);
-	assert(!ret && !cqe->res && cqe->user_data == 1);
-	io_uring_cqe_seen(ring, cqe);
-	assert(check_cq_empty(ring));
+	io_uring_prep_send_zc(sqe, sock_tx, (void *)1UL, payload_size,
+			      msg_flags, zc_flags);
+	sqe->user_data = 2;
+	io_uring_prep_send_set_addr(sqe, (const struct sockaddr *)1UL,
+				    sizeof(struct sockaddr_in6));
 
-	/* flush range */
-	sqe = io_uring_get_sqe(ring);
-	io_uring_prep_notif_update(sqe, 0, offset, nr_to_flush);
-	sqe->user_data = 1;
-	sqe->flags |= IOSQE_CQE_SKIP_SUCCESS;
 	ret = io_uring_submit(ring);
-	assert(ret == 1);
-
-	for (i = 0; i < nr_to_flush; i++) {
-		int slot_idx;
+	assert(ret == 2);
 
+	for (i = 0; i < 2; i++) {
 		ret = io_uring_wait_cqe(ring, &cqe);
-		assert(!ret && !cqe->res);
-		assert(ZC_TAG + offset <= cqe->user_data &&
-		       cqe->user_data < ZC_TAG + offset + nr_to_flush);
-		slot_idx = cqe->user_data - ZC_TAG;
-		assert(cqe->flags == seqs[slot_idx]);
-		seqs[slot_idx]++;
+		assert(!ret);
+		assert(cqe->user_data <= 2);
+		assert(cqe->res == -EFAULT);
+		assert(!(cqe->flags & IORING_CQE_F_MORE));
 		io_uring_cqe_seen(ring, cqe);
 	}
 	assert(check_cq_empty(ring));
-	return 0;
-}
-
-static int test_registration(int sock_tx, int sock_rx)
-{
-	struct io_uring_notification_slot slots[2] = {
-		{.tag = 1}, {.tag = 2},
-	};
-	void *invalid_slots = (void *)1UL;
-	struct io_uring ring;
-	int ret, i;
-
-	ret = io_uring_queue_init(4, &ring, 0);
-	if (ret) {
-		fprintf(stderr, "queue init failed: %d\n", ret);
-		return 1;
-	}
-
-	ret = io_uring_unregister_notifications(&ring);
-	if (ret != -ENXIO) {
-		fprintf(stderr, "unregister nothing: %d\n", ret);
-		return 1;
-	}
-
-	ret = io_uring_register_notifications(&ring, 2, slots);
-	if (ret) {
-		fprintf(stderr, "io_uring_register_notifications failed: %d\n", ret);
-		return 1;
-	}
-
-	ret = io_uring_register_notifications(&ring, 2, slots);
-	if (ret != -EBUSY) {
-		fprintf(stderr, "double register: %d\n", ret);
-		return 1;
-	}
-
-	ret = io_uring_unregister_notifications(&ring);
-	if (ret) {
-		fprintf(stderr, "unregister failed: %d\n", ret);
-		return 1;
-	}
-
-	ret = io_uring_register_notifications(&ring, 2, slots);
-	if (ret) {
-		fprintf(stderr, "second register failed: %d\n", ret);
-		return 1;
-	}
-
-	ret = test_invalid_slot(&ring, sock_tx, sock_rx);
-	if (ret) {
-		fprintf(stderr, "test_invalid_slot() failed\n");
-		return ret;
-	}
-
-	for (i = 0; i < 2; i++) {
-		ret = do_one(&ring, sock_tx, 0);
-		assert(ret == 1);
-
-		ret = recv(sock_rx, rx_buffer, 1, MSG_TRUNC);
-		assert(ret == 1);
-	}
-
-	io_uring_queue_exit(&ring);
-	ret = io_uring_queue_init(4, &ring, 0);
-	if (ret) {
-		fprintf(stderr, "queue init failed: %d\n", ret);
-		return 1;
-	}
-
-	ret = io_uring_register_notifications(&ring, 4, invalid_slots);
-	if (ret != -EFAULT) {
-		fprintf(stderr, "io_uring_register_notifications with invalid ptr: %d\n", ret);
-		return 1;
-	}
-
-	io_uring_queue_exit(&ring);
-	return 0;
+	return T_EXIT_PASS;
 }
 
 static int prepare_ip(struct sockaddr_storage *addr, int *sock_client, int *sock_server,
@@ -644,12 +241,11 @@ static int do_test_inet_send(struct io_uring *ring, int sock_client, int sock_se
 			     size_t send_size, bool cork, bool mix_register,
 			     int buf_idx)
 {
-	const unsigned slot_idx = 0;
 	const unsigned zc_flags = 0;
 	struct io_uring_sqe *sqe;
 	struct io_uring_cqe *cqe;
 	int nr_reqs = cork ? 5 : 1;
-	int i, ret;
+	int i, ret, nr_cqes;
 	size_t chunk_size = send_size / nr_reqs;
 	size_t chunk_size_last = send_size - chunk_size * (nr_reqs - 1);
 	char *buf = buffers_iov[buf_idx].iov_base;
@@ -660,39 +256,35 @@ static int do_test_inet_send(struct io_uring *ring, int sock_client, int sock_se
 	memset(rx_buffer, 0, send_size);
 
 	for (i = 0; i < nr_reqs; i++) {
-		bool cur_fixed_buf = fixed_buf;
+		bool real_fixed_buf = fixed_buf;
 		size_t cur_size = chunk_size;
 		int msg_flags = MSG_WAITALL;
 
 		if (mix_register)
-			cur_fixed_buf = rand() & 1;
+			real_fixed_buf = rand() & 1;
 
 		if (cork && i != nr_reqs - 1)
-			msg_flags = MSG_MORE;
+			msg_flags |= MSG_MORE;
 		if (i == nr_reqs - 1)
 			cur_size = chunk_size_last;
 
 		sqe = io_uring_get_sqe(ring);
-		if (cur_fixed_buf)
-			io_uring_prep_sendzc_fixed(sqe, sock_client,
-					     buf + i * chunk_size,
-					     cur_size, msg_flags, slot_idx,
-					     zc_flags, buf_idx);
-		else
-			io_uring_prep_sendzc(sqe, sock_client,
-					     buf + i * chunk_size,
-					     cur_size, msg_flags, slot_idx,
-					     zc_flags);
+		io_uring_prep_send_zc(sqe, sock_client, buf + i * chunk_size,
+				      cur_size, msg_flags, zc_flags);
+		sqe->user_data = i;
 
+		if (real_fixed_buf) {
+			sqe->ioprio |= IORING_RECVSEND_FIXED_BUF;
+			sqe->buf_index = buf_idx;
+		}
 		if (addr) {
 			sa_family_t fam = ((struct sockaddr_in *)addr)->sin_family;
 			int addr_len = fam == AF_INET ? sizeof(struct sockaddr_in) :
 							sizeof(struct sockaddr_in6);
 
-			io_uring_prep_sendzc_set_addr(sqe, (const struct sockaddr *)addr,
-						      addr_len);
+			io_uring_prep_send_set_addr(sqe, (const struct sockaddr *)addr,
+						    addr_len);
 		}
-		sqe->user_data = i;
 	}
 
 	ret = io_uring_submit(ring);
@@ -705,9 +297,7 @@ static int do_test_inet_send(struct io_uring *ring, int sock_client, int sock_se
 	if (p == -1) {
 		fprintf(stderr, "fork() failed\n");
 		return 1;
-	}
-
-	if (p == 0) {
+	} else if (p == 0) {
 		size_t bytes_received = 0;
 
 		while (bytes_received != send_size) {
@@ -732,7 +322,8 @@ static int do_test_inet_send(struct io_uring *ring, int sock_client, int sock_se
 		exit(0);
 	}
 
-	for (i = 0; i < nr_reqs; i++) {
+	nr_cqes = 2 * nr_reqs;
+	for (i = 0; i < nr_cqes; i++) {
 		int expected = chunk_size;
 
 		ret = io_uring_wait_cqe(ring, &cqe);
@@ -744,11 +335,18 @@ static int do_test_inet_send(struct io_uring *ring, int sock_client, int sock_se
 			fprintf(stderr, "invalid user_data\n");
 			return 1;
 		}
-		if (cqe->user_data == nr_reqs - 1)
-			expected = chunk_size_last;
-		if (cqe->res != expected) {
-			fprintf(stderr, "invalid cqe->res %d expected %d\n",
-					 cqe->res, expected);
+		if (!(cqe->flags & IORING_CQE_F_NOTIF)) {
+			if (cqe->user_data == nr_reqs - 1)
+				expected = chunk_size_last;
+			if (cqe->res != expected) {
+				fprintf(stderr, "invalid cqe->res %d expected %d\n",
+						 cqe->res, expected);
+				return 1;
+			}
+		}
+		if ((cqe->flags & IORING_CQE_F_MORE) ==
+		    (cqe->flags & IORING_CQE_F_NOTIF)) {
+			fprintf(stderr, "unexpected cflags %i\n", cqe->flags);
 			return 1;
 		}
 		io_uring_cqe_seen(ring, cqe);
@@ -845,6 +443,8 @@ static int test_async_addr(struct io_uring *ring)
 	struct __kernel_timespec ts;
 	int ret;
 
+	ts.tv_sec = 1;
+	ts.tv_nsec = 0;
 	ret = prepare_ip(&addr, &sock_tx, &sock_rx, true, false, false, false);
 	if (ret) {
 		fprintf(stderr, "sock prep failed %d\n", ret);
@@ -852,17 +452,15 @@ static int test_async_addr(struct io_uring *ring)
 	}
 
 	sqe = io_uring_get_sqe(ring);
-	ts.tv_sec = 1;
-	ts.tv_nsec = 0;
 	io_uring_prep_timeout(sqe, &ts, 0, IORING_TIMEOUT_ETIME_SUCCESS);
 	sqe->user_data = 1;
 	sqe->flags |= IOSQE_IO_LINK;
 
 	sqe = io_uring_get_sqe(ring);
-	io_uring_prep_sendzc(sqe, sock_tx, tx_buffer, 1, 0, 0, 0);
+	io_uring_prep_send_zc(sqe, sock_tx, tx_buffer, 1, 0, 0);
 	sqe->user_data = 2;
-	io_uring_prep_sendzc_set_addr(sqe, (const struct sockaddr *)&addr,
-				      sizeof(struct sockaddr_in6));
+	io_uring_prep_send_set_addr(sqe, (const struct sockaddr *)&addr,
+				    sizeof(struct sockaddr_in6));
 
 	ret = io_uring_submit(ring);
 	assert(ret == 2);
@@ -894,6 +492,14 @@ static int test_async_addr(struct io_uring *ring)
 	ret = recv(sock_rx, rx_buffer, 1, MSG_TRUNC);
 	assert(ret == 1);
 
+	ret = io_uring_wait_cqe(ring, &cqe);
+	if (ret) {
+		fprintf(stderr, "io_uring_wait_cqe failed %i\n", ret);
+		return 1;
+	}
+	assert(cqe->flags & IORING_CQE_F_NOTIF);
+	io_uring_cqe_seen(ring, cqe);
+
 	close(sock_tx);
 	close(sock_rx);
 	return 0;
@@ -937,15 +543,6 @@ int main(int argc, char *argv[])
 		return T_EXIT_FAIL;
 	}
 
-	ret = register_notifications(&ring);
-	if (ret == -EINVAL) {
-		printf("sendzc is not supported, skip\n");
-		return T_EXIT_SKIP;
-	} else if (ret) {
-		fprintf(stderr, "register notif failed %i\n", ret);
-		return T_EXIT_FAIL;
-	}
-
 	srand((unsigned)time(NULL));
 	for (i = 0; i < len; i++)
 		tx_buffer[i] = i;
@@ -956,70 +553,24 @@ int main(int argc, char *argv[])
 		return T_EXIT_FAIL;
 	}
 
-	ret = test_registration(sp[0], sp[1]);
-	if (ret) {
-		fprintf(stderr, "test_registration() failed\n");
-		return ret;
-	}
-
-	ret = test_invalid_slot(&ring, sp[0], sp[1]);
-	if (ret) {
-		fprintf(stderr, "test_invalid_slot() failed\n");
-		return T_EXIT_FAIL;
-	}
-
 	ret = test_basic_send(&ring, sp[0], sp[1]);
+	if (ret == T_EXIT_SKIP)
+		return ret;
 	if (ret) {
 		fprintf(stderr, "test_basic_send() failed\n");
 		return T_EXIT_FAIL;
 	}
 
-	ret = test_send_flush(&ring, sp[0], sp[1]);
-	if (ret) {
-		fprintf(stderr, "test_send_flush() failed\n");
-		return T_EXIT_FAIL;
-	}
-
-	ret = test_multireq_notif(&ring, sp[0], sp[1]);
-	if (ret) {
-		fprintf(stderr, "test_multireq_notif() failed\n");
-		return T_EXIT_FAIL;
-	}
-
-	ret = reregister_notifications(&ring);
+	ret = test_send_faults(&ring, sp[0], sp[1]);
 	if (ret) {
-		fprintf(stderr, "reregister notifiers failed %i\n", ret);
-		return T_EXIT_FAIL;
-	}
-	/* retry a few tests after registering notifs */
-	ret = test_invalid_slot(&ring, sp[0], sp[1]);
-	if (ret) {
-		fprintf(stderr, "test_invalid_slot() failed\n");
+		fprintf(stderr, "test_send_faults() failed\n");
 		return T_EXIT_FAIL;
 	}
 
-	ret = test_multireq_notif(&ring, sp[0], sp[1]);
-	if (ret) {
-		fprintf(stderr, "test_multireq_notif2() failed\n");
-		return T_EXIT_FAIL;
-	}
-
-	ret = test_multi_send_flushing(&ring, sp[0], sp[1]);
-	if (ret) {
-		fprintf(stderr, "test_multi_send_flushing() failed\n");
-		return T_EXIT_FAIL;
-	}
-
-	ret = test_update_flush_fail(&ring);
-	if (ret) {
-		fprintf(stderr, "test_update_flush_fail() failed\n");
-		return T_EXIT_FAIL;
-	}
-
-	ret = test_update_flush(&ring, sp[0], sp[1]);
+	ret = test_async_addr(&ring);
 	if (ret) {
-		fprintf(stderr, "test_update_flush() failed\n");
-		return T_EXIT_FAIL;
+		fprintf(stderr, "test_async_addr() failed\n");
+		return ret;
 	}
 
 	ret = t_register_buffers(&ring, buffers_iov, ARRAY_SIZE(buffers_iov));
@@ -1036,12 +587,6 @@ int main(int argc, char *argv[])
 		fprintf(stderr, "test_inet_send() failed\n");
 		return ret;
 	}
-
-	ret = test_async_addr(&ring);
-	if (ret) {
-		fprintf(stderr, "test_async_addr() failed\n");
-		return ret;
-	}
 out:
 	io_uring_queue_exit(&ring);
 	close(sp[0]);
-- 
2.37.2

[PATCH liburing 3/4] test: test iowq zc sends

[Pavel Begunkov]

Add tests exercising io-wq paths with zc by setting IOSQE_ASYNC.

Signed-off-by: Pavel Begunkov <asml.silence@gmail.com>
---
 test/send-zerocopy.c | 9 ++++++---
 1 file changed, 6 insertions(+), 3 deletions(-)

diff --git a/test/send-zerocopy.c b/test/send-zerocopy.c
index 7b58ae7..8714b6f 100644
--- a/test/send-zerocopy.c
+++ b/test/send-zerocopy.c
@@ -239,7 +239,7 @@ static int prepare_ip(struct sockaddr_storage *addr, int *sock_client, int *sock
 static int do_test_inet_send(struct io_uring *ring, int sock_client, int sock_server,
 			     bool fixed_buf, struct sockaddr_storage *addr,
 			     size_t send_size, bool cork, bool mix_register,
-			     int buf_idx)
+			     int buf_idx, bool force_async)
 {
 	const unsigned zc_flags = 0;
 	struct io_uring_sqe *sqe;
@@ -285,6 +285,8 @@ static int do_test_inet_send(struct io_uring *ring, int sock_client, int sock_se
 			io_uring_prep_send_set_addr(sqe, (const struct sockaddr *)addr,
 						    addr_len);
 		}
+		if (force_async)
+			sqe->flags |= IOSQE_ASYNC;
 	}
 
 	ret = io_uring_submit(ring);
@@ -389,7 +391,7 @@ static int test_inet_send(struct io_uring *ring)
 			return 1;
 		}
 
-		for (i = 0; i < 128; i++) {
+		for (i = 0; i < 256; i++) {
 			bool fixed_buf = i & 1;
 			struct sockaddr_storage *addr_arg = (i & 2) ? &addr : NULL;
 			size_t size = (i & 4) ? 137 : 4096;
@@ -398,6 +400,7 @@ static int test_inet_send(struct io_uring *ring)
 			bool aligned = i & 32;
 			bool large_buf = i & 64;
 			int buf_idx = aligned ? 0 : 1;
+			bool force_async = i & 128;
 
 			if (!tcp || !large_buf)
 				continue;
@@ -418,7 +421,7 @@ static int test_inet_send(struct io_uring *ring)
 
 			ret = do_test_inet_send(ring, sock_client, sock_server, fixed_buf,
 						addr_arg, size, cork, mix_register,
-						buf_idx);
+						buf_idx, force_async);
 			if (ret) {
 				fprintf(stderr, "send failed fixed buf %i, conn %i, addr %i, "
 					"cork %i\n",
-- 
2.37.2

[PATCH liburing 4/4] examples: adjust zc bench to the new uapi

[Pavel Begunkov]

Signed-off-by: Pavel Begunkov <asml.silence@gmail.com>
---
 examples/send-zerocopy.c | 72 +++++++++++++---------------------------
 1 file changed, 23 insertions(+), 49 deletions(-)

diff --git a/examples/send-zerocopy.c b/examples/send-zerocopy.c
index e42aa71..4ed0f67 100644
--- a/examples/send-zerocopy.c
+++ b/examples/send-zerocopy.c
@@ -44,7 +44,6 @@
 static bool cfg_reg_ringfd = true;
 static bool cfg_fixed_files = 1;
 static bool cfg_zc = 1;
-static bool cfg_flush = 0;
 static int  cfg_nr_reqs = 8;
 static bool cfg_fixed_buf = 1;
 
@@ -146,13 +145,6 @@ static void do_tx(int domain, int type, int protocol)
 	if (ret)
 		error(1, ret, "io_uring: queue init");
 
-	if (cfg_zc) {
-		struct io_uring_notification_slot b[1] = {{.tag = ZC_TAG}};
-
-		ret = io_uring_register_notifications(&ring, 1, b);
-		if (ret)
-			error(1, ret, "io_uring: tx ctx registration");
-	}
 	if (cfg_fixed_files) {
 		ret = io_uring_register_files(&ring, &fd, 1);
 		if (ret < 0)
@@ -175,14 +167,8 @@ static void do_tx(int domain, int type, int protocol)
 	do {
 		struct io_uring_sqe *sqe;
 		struct io_uring_cqe *cqe;
-		unsigned zc_flags = 0;
 		unsigned buf_idx = 0;
-		unsigned slot_idx = 0;
-		unsigned msg_flags = 0;
-
-		compl_cqes += cfg_flush ? cfg_nr_reqs : 0;
-		if (cfg_flush)
-			zc_flags |= IORING_RECVSEND_NOTIF_FLUSH;
+		unsigned msg_flags = MSG_WAITALL;
 
 		for (i = 0; i < cfg_nr_reqs; i++) {
 			sqe = io_uring_get_sqe(&ring);
@@ -190,22 +176,22 @@ static void do_tx(int domain, int type, int protocol)
 			if (!cfg_zc)
 				io_uring_prep_send(sqe, fd, payload,
 						   cfg_payload_len, 0);
-			else if (cfg_fixed_buf)
-				io_uring_prep_sendzc_fixed(sqe, fd, payload,
-							   cfg_payload_len,
-							   msg_flags, slot_idx,
-							   zc_flags, buf_idx);
-			else
-				io_uring_prep_sendzc(sqe, fd, payload,
-						     cfg_payload_len, msg_flags,
-						     slot_idx, zc_flags);
-
+			else {
+				io_uring_prep_send_zc(sqe, fd, payload,
+						     cfg_payload_len, msg_flags, 0);
+				if (cfg_fixed_buf) {
+					sqe->ioprio |= IORING_RECVSEND_FIXED_BUF;
+					sqe->buf_index = buf_idx;
+				}
+			}
 			sqe->user_data = 1;
 			if (cfg_fixed_files) {
 				sqe->fd = 0;
 				sqe->flags |= IOSQE_FIXED_FILE;
 			}
 		}
+		if (cfg_zc)
+			compl_cqes += cfg_nr_reqs;
 
 		ret = io_uring_submit(&ring);
 		if (ret != cfg_nr_reqs)
@@ -214,27 +200,26 @@ static void do_tx(int domain, int type, int protocol)
 		for (i = 0; i < cfg_nr_reqs; i++) {
 			cqe = wait_cqe_fast(&ring);
 
-			if (cqe->user_data == ZC_TAG) {
+			if (cqe->flags & IORING_CQE_F_NOTIF) {
+				if (cqe->flags & IORING_CQE_F_MORE)
+					error(1, -EINVAL, "F_MORE notif");
 				compl_cqes--;
 				i--;
-			} else if (cqe->user_data != 1) {
-				error(1, cqe->user_data, "invalid user_data");
-			} else if (cqe->res > 0) {
+			} else if (cqe->res >= 0) {
+				if (!(cqe->flags & IORING_CQE_F_MORE) && cfg_zc)
+					error(1, -EINVAL, "no F_MORE");
 				packets++;
 				bytes += cqe->res;
 			} else if (cqe->res == -EAGAIN) {
-				/* request failed, don't flush */
-				if (cfg_flush)
+				if (cfg_zc)
 					compl_cqes--;
-			} else if (cqe->res == -ECONNREFUSED ||
-				   cqe->res == -ECONNRESET ||
-				   cqe->res == -EPIPE) {
-				fprintf(stderr, "Connection failure\n");
+			} else if (cqe->res == -ECONNREFUSED || cqe->res == -EPIPE ||
+				   cqe->res == -ECONNRESET) {
+				fprintf(stderr, "Connection failure");
 				goto out_fail;
 			} else {
 				error(1, cqe->res, "send failed");
 			}
-
 			io_uring_cqe_seen(&ring, cqe);
 		}
 	} while (gettimeofday_ms() < tstop);
@@ -255,12 +240,6 @@ out_fail:
 		io_uring_cqe_seen(&ring, cqe);
 		compl_cqes--;
 	}
-
-	if (cfg_zc) {
-		ret = io_uring_unregister_notifications(&ring);
-		if (ret)
-			error(1, ret, "io_uring: tx ctx unregistration");
-	}
 	io_uring_queue_exit(&ring);
 }
 
@@ -276,7 +255,7 @@ static void do_test(int domain, int type, int protocol)
 
 static void usage(const char *filepath)
 {
-	error(1, 0, "Usage: %s [-f] [-n<N>] [-z0] [-s<payload size>] "
+	error(1, 0, "Usage: %s [-n<N>] [-z<val>] [-s<payload size>] "
 		    "(-4|-6) [-t<time s>] -D<dst_ip> udp", filepath);
 }
 
@@ -294,7 +273,7 @@ static void parse_opts(int argc, char **argv)
 
 	cfg_payload_len = max_payload_len;
 
-	while ((c = getopt(argc, argv, "46D:p:s:t:n:fz:b:k")) != -1) {
+	while ((c = getopt(argc, argv, "46D:p:s:t:n:z:b:k")) != -1) {
 		switch (c) {
 		case '4':
 			if (cfg_family != PF_UNSPEC)
@@ -323,9 +302,6 @@ static void parse_opts(int argc, char **argv)
 		case 'n':
 			cfg_nr_reqs = strtoul(optarg, NULL, 0);
 			break;
-		case 'f':
-			cfg_flush = 1;
-			break;
 		case 'z':
 			cfg_zc = strtoul(optarg, NULL, 0);
 			break;
@@ -337,8 +313,6 @@ static void parse_opts(int argc, char **argv)
 
 	if (cfg_nr_reqs > MAX_SUBMIT_NR)
 		error(1, 0, "-n: submit batch nr exceeds max (%d)", MAX_SUBMIT_NR);
-	if (cfg_flush && !cfg_zc)
-		error(1, 0, "cfg_flush should be used with zc only");
 	if (cfg_payload_len > max_payload_len)
 		error(1, 0, "-s: payload exceeds max (%d)", max_payload_len);
 
-- 
2.37.2

Re: [PATCH liburing 0/4] zerocopy send API changes

[Ammar Faizi]

On 9/2/22 6:12 PM, Pavel Begunkov wrote:
> Fix up helpers and tests to match API changes and also add some more tests.
> 
> Pavel Begunkov (4):
>    tests: verify that send addr is copied when async
>    zc: adjust sendzc to the simpler uapi
>    test: test iowq zc sends
>    examples: adjust zc bench to the new uapi

Hi Pavel,

Patch #2 and #3 are broken, but after applying patch #4, everything builds
just fine. Please resend and avoid breakage in the middle.

Thanks!

--------------------------------------------------------------------------------
Patch #2

   send-zerocopy.c:152:9: error: call to undeclared function 'io_uring_register_notifications'; ISO C99 and later do not support implicit function declarations [-Wimplicit-function-declaration]
                   ret = io_uring_register_notifications(&ring, 1, b);
                         ^
   send-zerocopy.c:152:9: note: did you mean 'io_uring_register_restrictions'?
   ../src/include/liburing.h:181:5: note: 'io_uring_register_restrictions' declared here
   int io_uring_register_restrictions(struct io_uring *ring,
       ^
   send-zerocopy.c:185:16: error: use of undeclared identifier 'IORING_RECVSEND_NOTIF_FLUSH'
                           zc_flags |= IORING_RECVSEND_NOTIF_FLUSH;
                                       ^
   send-zerocopy.c:194:5: error: call to undeclared function 'io_uring_prep_sendzc_fixed'; ISO C99 and later do not support implicit function declarations [-Wimplicit-function-declaration]
                                   io_uring_prep_sendzc_fixed(sqe, fd, payload,
                                   ^
   send-zerocopy.c:194:5: note: did you mean 'io_uring_prep_read_fixed'?
   ../src/include/liburing.h:405:20: note: 'io_uring_prep_read_fixed' declared here
   static inline void io_uring_prep_read_fixed(struct io_uring_sqe *sqe, int fd,
                      ^
   send-zerocopy.c:199:5: error: call to undeclared function 'io_uring_prep_sendzc'; ISO C99 and later do not support implicit function declarations [-Wimplicit-function-declaration]
                                   io_uring_prep_sendzc(sqe, fd, payload,
                                   ^
   send-zerocopy.c:199:5: note: did you mean 'io_uring_prep_send_zc'?
   ../src/include/liburing.h:701:20: note: 'io_uring_prep_send_zc' declared here
   static inline void io_uring_prep_send_zc(struct io_uring_sqe *sqe, int sockfd,
                      ^
   send-zerocopy.c:260:9: error: call to undeclared function 'io_uring_unregister_notifications'; ISO C99 and later do not support implicit function declarations [-Wimplicit-function-declaration]
                   ret = io_uring_unregister_notifications(&ring);
                         ^
   send-zerocopy.c:260:9: note: did you mean 'io_uring_register_restrictions'?
   ../src/include/liburing.h:181:5: note: 'io_uring_register_restrictions' declared here
   int io_uring_register_restrictions(struct io_uring *ring,
       ^
   5 errors generated.
   make[1]: *** [Makefile:36: send-zerocopy] Error 1
   make[1]: *** Waiting for unfinished jobs....
   make[1]: Leaving directory '/home/runner/work/liburing/liburing/examples'
   make: *** [Makefile:12: all] Error 2
   Error: Process completed with exit code 2.


--------------------------------------------------------------------------------
Patch #3:

   send-zerocopy.c: In function ‘do_tx’:
   send-zerocopy.c:152:23: error: implicit declaration of function ‘io_uring_register_notifications’; did you mean ‘io_uring_register_restrictions’? [-Werror=implicit-function-declaration]
     152 |                 ret = io_uring_register_notifications(&ring, 1, b);
         |                       ^~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
         |                       io_uring_register_restrictions
   send-zerocopy.c:185:37: error: ‘IORING_RECVSEND_NOTIF_FLUSH’ undeclared (first use in this function); did you mean ‘IORING_RECVSEND_POLL_FIRST’?
     185 |                         zc_flags |= IORING_RECVSEND_NOTIF_FLUSH;
         |                                     ^~~~~~~~~~~~~~~~~~~~~~~~~~~
         |                                     IORING_RECVSEND_POLL_FIRST
   send-zerocopy.c:185:37: note: each undeclared identifier is reported only once for each function it appears in
   send-zerocopy.c:194:33: error: implicit declaration of function ‘io_uring_prep_sendzc_fixed’; did you mean ‘io_uring_prep_read_fixed’? [-Werror=implicit-function-declaration]
     194 |                                 io_uring_prep_sendzc_fixed(sqe, fd, payload,
         |                                 ^~~~~~~~~~~~~~~~~~~~~~~~~~
         |                                 io_uring_prep_read_fixed
   send-zerocopy.c:199:33: error: implicit declaration of function ‘io_uring_prep_sendzc’; did you mean ‘io_uring_prep_send_zc’? [-Werror=implicit-function-declaration]
     199 |                                 io_uring_prep_sendzc(sqe, fd, payload,
         |                                 ^~~~~~~~~~~~~~~~~~~~
         |                                 io_uring_prep_send_zc
   send-zerocopy.c:260:23: error: implicit declaration of function ‘io_uring_unregister_notifications’; did you mean ‘io_uring_register_restrictions’? [-Werror=implicit-function-declaration]
     260 |                 ret = io_uring_unregister_notifications(&ring);
         |                       ^~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
         |                       io_uring_register_restrictions
   cc1: all warnings being treated as errors
   make[1]: *** [Makefile:36: send-zerocopy] Error 1
   make[1]: *** Waiting for unfinished jobs....
   make[1]: Leaving directory '/home/runner/work/liburing/liburing/examples'
   make: *** [Makefile:12: all] Error 2
   Error: Process completed with exit code 2.

-- 
Ammar Faizi

Re: [PATCH liburing 0/4] zerocopy send API changes

[Ammar Faizi]

On 9/2/22 6:56 PM, Ammar Faizi wrote:
> On 9/2/22 6:12 PM, Pavel Begunkov wrote:
>> Fix up helpers and tests to match API changes and also add some more tests.
>>
>> Pavel Begunkov (4):
>>    tests: verify that send addr is copied when async
>>    zc: adjust sendzc to the simpler uapi
>>    test: test iowq zc sends
>>    examples: adjust zc bench to the new uapi
> 
> Hi Pavel,
> 
> Patch #2 and #3 are broken, but after applying patch #4, everything builds
> just fine. Please resend and avoid breakage in the middle.
> 
> Thanks!

Nevermind. It's already upstream now.

-- 
Ammar Faizi

Re: [PATCH liburing 0/4] zerocopy send API changes

[Jens Axboe]

On 9/2/22 6:01 AM, Ammar Faizi wrote:
> On 9/2/22 6:56 PM, Ammar Faizi wrote:
>> On 9/2/22 6:12 PM, Pavel Begunkov wrote:
>>> Fix up helpers and tests to match API changes and also add some more tests.
>>>
>>> Pavel Begunkov (4):
>>>    tests: verify that send addr is copied when async
>>>    zc: adjust sendzc to the simpler uapi
>>>    test: test iowq zc sends
>>>    examples: adjust zc bench to the new uapi
>>
>> Hi Pavel,
>>
>> Patch #2 and #3 are broken, but after applying patch #4, everything builds
>> just fine. Please resend and avoid breakage in the middle.
>>
>> Thanks!
> 
> Nevermind. It's already upstream now.

Ah shoot, how did I miss that... That's annoying.

-- 
Jens Axboe

Re: [PATCH liburing 0/4] zerocopy send API changes

[Pavel Begunkov]

On 9/2/22 13:03, Jens Axboe wrote:
> On 9/2/22 6:01 AM, Ammar Faizi wrote:
>> On 9/2/22 6:56 PM, Ammar Faizi wrote:
>>> On 9/2/22 6:12 PM, Pavel Begunkov wrote:
>>>> Fix up helpers and tests to match API changes and also add some more tests.
>>>>
>>>> Pavel Begunkov (4):
>>>>     tests: verify that send addr is copied when async
>>>>     zc: adjust sendzc to the simpler uapi
>>>>     test: test iowq zc sends
>>>>     examples: adjust zc bench to the new uapi
>>>
>>> Hi Pavel,
>>>
>>> Patch #2 and #3 are broken, but after applying patch #4, everything builds
>>> just fine. Please resend and avoid breakage in the middle.
>>>
>>> Thanks!
>>
>> Nevermind. It's already upstream now.
> 
> Ah shoot, how did I miss that... That's annoying.

We can squash them into a single commit if we care about it.
Don't really want do the disable + fix +e nable dancing here.

-- 
Pavel Begunkov

Re: [PATCH liburing 0/4] zerocopy send API changes

[Jens Axboe]

On 9/2/22 6:07 AM, Pavel Begunkov wrote:
> On 9/2/22 13:03, Jens Axboe wrote:
>> On 9/2/22 6:01 AM, Ammar Faizi wrote:
>>> On 9/2/22 6:56 PM, Ammar Faizi wrote:
>>>> On 9/2/22 6:12 PM, Pavel Begunkov wrote:
>>>>> Fix up helpers and tests to match API changes and also add some more tests.
>>>>>
>>>>> Pavel Begunkov (4):
>>>>> ??? tests: verify that send addr is copied when async
>>>>> ??? zc: adjust sendzc to the simpler uapi
>>>>> ??? test: test iowq zc sends
>>>>> ??? examples: adjust zc bench to the new uapi
>>>>
>>>> Hi Pavel,
>>>>
>>>> Patch #2 and #3 are broken, but after applying patch #4, everything builds
>>>> just fine. Please resend and avoid breakage in the middle.
>>>>
>>>> Thanks!
>>>
>>> Nevermind. It's already upstream now.
>>
>> Ah shoot, how did I miss that... That's annoying.
> 
> We can squash them into a single commit if we care about it.
> Don't really want do the disable + fix +e nable dancing here.

It's already pushed out, so whatever is there is set in stone... Not a
huge deal, but would've been nice to avoid. It's problematic when
someone needs to bisect and issue and runs into a non-compiling step.
Makes that process a lot more annoying, so yes we definitely do care
about not introducing build breakage in a series of patches.

-- 
Jens Axboe

Re: [PATCH liburing 0/4] zerocopy send API changes

[Jens Axboe]

On Fri, 2 Sep 2022 12:12:35 +0100, Pavel Begunkov wrote:
> Fix up helpers and tests to match API changes and also add some more tests.
> 
> Pavel Begunkov (4):
>   tests: verify that send addr is copied when async
>   zc: adjust sendzc to the simpler uapi
>   test: test iowq zc sends
>   examples: adjust zc bench to the new uapi
> 
> [...]

Applied, thanks!

[1/4] tests: verify that send addr is copied when async
      commit: 54b16e6cfa489edd1f4f538ae245d98d65d42db7
[2/4] zc: adjust sendzc to the simpler uapi
      commit: 880a932c8ae36506b4d5040b9258a91251164589
[3/4] test: test iowq zc sends
      commit: 713ecf1cf9ad58ceb19893eead2b704b27367a8a
[4/4] examples: adjust zc bench to the new uapi
      commit: 860db521db4c86a1cb5d0b672a8fba83a89f01f0

Best regards,
-- 
Jens Axboe