[PATCH 0/5] liburing: Add {g,s}etsockopt command support

[PATCH 0/5] liburing: Add {g,s}etsockopt command support

[Breno Leitao]

These are liburing patches that add support for the new
SOCKET_URING_OP_GETSOCKOPT and SOCKET_URING_OP_SETSOCKOPT socket
commands.

This patchset basically synchronize the UAPI bits, teach
io_uring_prep_cmd(3) how to use the new fields, create a unit test and
add the proper man page documentation.

Breno Leitao (5):
  io_uring: uapi: Sync the {g,s}etsockopt fields
  liburing.h: Populate SQE for {s,g} etsockopt
  tests/socket-getsetsock-cmd: New test for {g,s}etsockopt
  man/io_uring_prep_cmd: Fix argument name
  man/io_uring_prep_cmd: Add the new sockopt commands

 CHANGELOG                       |   4 +
 man/io_uring_prep_cmd.3         |  34 +++-
 src/include/liburing.h          |  11 +-
 src/include/liburing/io_uring.h |   8 +
 test/Makefile                   |   1 +
 test/socket-getsetsock-cmd.c    | 328 ++++++++++++++++++++++++++++++++
 6 files changed, 378 insertions(+), 8 deletions(-)
 create mode 100644 test/socket-getsetsock-cmd.c

-- 
2.34.1

[PATCH 1/5] io_uring: uapi: Sync the {g,s}etsockopt fields

[Breno Leitao]

Copy the UAPI fields necessary to support the new
SOCKET_URING_OP_GETSOCKOPT and SOCKET_URING_OP_SETSOCKOPT socket
commands.

Sync the necessary bits for io_uring_sqe struct from the kernel, and
also adds the SOCKET_URING_OP_GETSOCKOPT and SOCKET_URING_OP_SETSOCKOPT
to the enum that defines the commands.

Signed-off-by: Breno Leitao <leitao@debian.org>
---
 src/include/liburing/io_uring.h | 8 ++++++++
 1 file changed, 8 insertions(+)

diff --git a/src/include/liburing/io_uring.h b/src/include/liburing/io_uring.h
index 10a6ef0..a7a95a3 100644
--- a/src/include/liburing/io_uring.h
+++ b/src/include/liburing/io_uring.h
@@ -43,6 +43,10 @@ struct io_uring_sqe {
 	union {
 		__u64	addr;	/* pointer to buffer or iovecs */
 		__u64	splice_off_in;
+		struct {
+			__u32	level;
+			__u32	optname;
+		};
 	};
 	__u32	len;		/* buffer size or number of iovecs */
 	union {
@@ -79,6 +83,7 @@ struct io_uring_sqe {
 	union {
 		__s32	splice_fd_in;
 		__u32	file_index;
+		__u32	optlen;
 		struct {
 			__u16	addr_len;
 			__u16	__pad3[1];
@@ -89,6 +94,7 @@ struct io_uring_sqe {
 			__u64	addr3;
 			__u64	__pad2[1];
 		};
+		__u64	optval;
 		/*
 		 * If the ring is initialized with IORING_SETUP_SQE128, then
 		 * this field is used for 80 bytes of arbitrary command data
@@ -721,6 +727,8 @@ struct io_uring_recvmsg_out {
 enum {
 	SOCKET_URING_OP_SIOCINQ		= 0,
 	SOCKET_URING_OP_SIOCOUTQ,
+	SOCKET_URING_OP_GETSOCKOPT,
+	SOCKET_URING_OP_SETSOCKOPT,
 };
 
 #ifdef __cplusplus
-- 
2.34.1

[PATCH 2/5] liburing.h: Populate SQE for {s,g} etsockopt

[Breno Leitao]

The current io_uring_prep_cmd_sock() liburing function is not populating
the SQE will all the fields because the kernel implementation was not
ready.

With the integration of the kernel part[1], populate the SQE with the
missing fields (optlen, optval, level and optname). This enables
the usage of this function to prepare commands that executes setsockopt
and getsockopt operations.

[1] Link:  https://lore.kernel.org/all/20231016134750.1381153-1-leitao@debian.org/
Signed-off-by: Breno Leitao <leitao@debian.org>
---
 CHANGELOG              |  4 ++++
 src/include/liburing.h | 11 ++++-------
 2 files changed, 8 insertions(+), 7 deletions(-)

diff --git a/CHANGELOG b/CHANGELOG
index 42a7fc1..61199e2 100644
--- a/CHANGELOG
+++ b/CHANGELOG
@@ -1,3 +1,7 @@
+liburing-2.5 release
+
+- Add getsockopt and setsockopt socket commands
+
 liburing-2.4 release
 
 - Add io_uring_{major,minor,check}_version() functions.
diff --git a/src/include/liburing.h b/src/include/liburing.h
index 1008544..84400ea 100644
--- a/src/include/liburing.h
+++ b/src/include/liburing.h
@@ -1129,8 +1129,6 @@ IOURINGINLINE void io_uring_prep_socket_direct_alloc(struct io_uring_sqe *sqe,
 }
 
 
-#define UNUSED(x) (void)(x)
-
 /*
  * Prepare commands for sockets
  */
@@ -1142,13 +1140,12 @@ IOURINGINLINE void io_uring_prep_cmd_sock(struct io_uring_sqe *sqe,
 					  void *optval,
 					  int optlen)
 {
-	/* This will be removed once the get/setsockopt() patches land */
-	UNUSED(optlen);
-	UNUSED(optval);
-	UNUSED(level);
-	UNUSED(optname);
 	io_uring_prep_rw(IORING_OP_URING_CMD, sqe, fd, NULL, 0, 0);
+	sqe->optval = (long long unsigned int)optval;
+	sqe->optname = optname;
+	sqe->optlen = optlen;
 	sqe->cmd_op = cmd_op;
+	sqe->level = level;
 }
 
 /*
-- 
2.34.1

[PATCH 3/5] tests/socket-getsetsock-cmd: New test for {g,s}etsockopt

[Breno Leitao]

Introduce a new test to exercise the io_uring getsockopt and setsockopt
commands.

On the liburing side, use the io_uring_prep_cmd_sock() function to
prepare the SQE.

The test executes the same operation using a regular systemcall, such as
getsockopt(2), and the io_uring getsockopt command and compare the
results. The same for the setsockopt counterpart.

Signed-off-by: Breno Leitao <leitao@debian.org>
---
 test/Makefile                |   1 +
 test/socket-getsetsock-cmd.c | 328 +++++++++++++++++++++++++++++++++++
 2 files changed, 329 insertions(+)
 create mode 100644 test/socket-getsetsock-cmd.c

diff --git a/test/Makefile b/test/Makefile
index c15b2a3..6b25e58 100644
--- a/test/Makefile
+++ b/test/Makefile
@@ -167,6 +167,7 @@ test_srcs := \
 	skip-cqe.c \
 	socket.c \
 	socket-io-cmd.c \
+	socket-getsetsock-cmd.c \
 	socket-rw.c \
 	socket-rw-eagain.c \
 	socket-rw-offset.c \
diff --git a/test/socket-getsetsock-cmd.c b/test/socket-getsetsock-cmd.c
new file mode 100644
index 0000000..5b6aa48
--- /dev/null
+++ b/test/socket-getsetsock-cmd.c
@@ -0,0 +1,328 @@
+/* SPDX-License-Identifier: MIT */
+/*
+ * Description: Check that {g,s}etsockopt CMD operations on sockets are
+ * consistent.
+ *
+ * The tests basically do the same socket operation using regular system calls
+ * and io_uring commands, and then compare the results.
+ */
+
+#include <stdio.h>
+#include <assert.h>
+#include <string.h>
+#include <unistd.h>
+#include <linux/tcp.h>
+
+#include "liburing.h"
+#include "helpers.h"
+
+#define USERDATA 0xff42ff
+#define MSG "foobarbaz"
+
+struct fds {
+	int tx;
+	int rx;
+};
+
+static struct fds create_sockets(void)
+{
+	struct fds retval;
+	int fd[2];
+
+	t_create_socket_pair(fd, true);
+
+	retval.tx = fd[0];
+	retval.rx = fd[1];
+
+	return retval;
+}
+
+static struct io_uring create_ring(void)
+{
+	struct io_uring ring;
+	int ring_flags = 0;
+	int err;
+
+	err = io_uring_queue_init(32, &ring, ring_flags);
+	assert(err == 0);
+
+	return ring;
+}
+
+static int submit_cmd_sqe(struct io_uring *ring, int32_t fd,
+			  int op, int level, int optname,
+			  void *optval, int optlen)
+{
+	struct io_uring_sqe *sqe;
+	int err;
+
+	assert(fd > 0);
+
+	sqe = io_uring_get_sqe(ring);
+	assert(sqe != NULL);
+
+	io_uring_prep_cmd_sock(sqe, op, fd, level, optname, optval, optlen);
+	sqe->user_data = USERDATA;
+
+	/* Submitting SQE */
+	err = io_uring_submit_and_wait(ring, 1);
+	if (err != 1)
+		fprintf(stderr, "Failure: io_uring_submit_and_wait returned %d\n", err);
+
+	return err;
+}
+
+static int receive_cqe(struct io_uring *ring)
+{
+	struct io_uring_cqe *cqe;
+	int err;
+
+	err = io_uring_wait_cqe(ring, &cqe);
+	assert(err ==  0);
+	assert(cqe->user_data == USERDATA);
+	io_uring_cqe_seen(ring, cqe);
+
+	/* Return the result of the operation */
+	return cqe->res;
+}
+
+/*
+ * Run getsock operation using SO_RCVBUF using io_uring cmd operation and
+ * getsockopt(2) and compare the results.
+ */
+static int run_get_rcvbuf(struct io_uring *ring, struct fds *sockfds)
+{
+	int sval, uval, ulen, err;
+	unsigned int slen;
+
+	/* System call values */
+	slen = sizeof(sval);
+	/* io_uring values */
+	ulen = sizeof(uval);
+
+	/* get through io_uring cmd */
+	err = submit_cmd_sqe(ring, sockfds->rx, SOCKET_URING_OP_GETSOCKOPT,
+			     SOL_SOCKET, SO_RCVBUF, &uval, ulen);
+	assert(err == 1);
+
+	/* Wait for the CQE */
+	err = receive_cqe(ring);
+	if (err == -EOPNOTSUPP)
+		return T_EXIT_SKIP;
+	if (err < 0) {
+		fprintf(stderr, "Error received. %d\n", err);
+		return T_EXIT_FAIL;
+	}
+	/* The output of CQE->res contains the length */
+	ulen = err;
+
+	/* Executes the same operation using system call */
+	err = getsockopt(sockfds->rx, SOL_SOCKET, SO_RCVBUF, &sval, &slen);
+	assert(err == 0);
+
+	/* Make sure that io_uring operation returns the same value as the systemcall */
+	assert(ulen == slen);
+	assert(uval == sval);
+
+	return T_EXIT_PASS;
+}
+
+/*
+ * Run getsock operation using SO_PEERNAME using io_uring cmd operation
+ * and getsockopt(2) and compare the results.
+ */
+static int run_get_peername(struct io_uring *ring, struct fds *sockfds)
+{
+	struct sockaddr sval, uval = {};
+	socklen_t slen = sizeof(sval);
+	socklen_t ulen = sizeof(uval);
+	int err;
+
+	/* Get values from the systemcall */
+	err = getsockopt(sockfds->tx, SOL_SOCKET, SO_PEERNAME, &sval, &slen);
+	assert(err == 0);
+
+	/* Getting SO_PEERNAME */
+	err = submit_cmd_sqe(ring, sockfds->rx, SOCKET_URING_OP_GETSOCKOPT,
+			     SOL_SOCKET, SO_PEERNAME, &uval, ulen);
+	assert(err == 1);
+
+	/* Wait for the CQE */
+	err = receive_cqe(ring);
+	if (err == -EOPNOTSUPP)
+		return T_EXIT_SKIP;
+
+	if (err < 0) {
+		fprintf(stderr, "%s: Error in the CQE: %d\n", __func__, err);
+		return T_EXIT_FAIL;
+	}
+
+	/* The length comes from cqe->res, which is returned from receive_cqe() */
+	ulen = err;
+
+	/* Make sure that io_uring operation returns the same values as the systemcall */
+	assert(sval.sa_family == uval.sa_family);
+	assert(slen == ulen);
+
+	return T_EXIT_PASS;
+}
+
+/*
+ * Run getsockopt tests. Basically comparing io_uring output and systemcall results
+ */
+static int run_getsockopt_test(struct io_uring *ring, struct fds *sockfds)
+{
+	int err;
+
+	fprintf(stderr, "Testing getsockopt SO_PEERNAME\n");
+	err = run_get_peername(ring, sockfds);
+	if (err)
+		return err;
+
+	fprintf(stderr, "Testing getsockopt SO_RCVBUF\n");
+	err = run_get_rcvbuf(ring, sockfds);
+
+	return err;
+}
+
+/*
+ * Given a `val` value, set it in SO_REUSEPORT using io_uring cmd, and read using
+ * getsockopt(2), and make sure they match.
+ */
+static int run_setsockopt_reuseport(struct io_uring *ring, struct fds *sockfds, int val)
+{
+	unsigned int slen, ulen;
+	int sval, uval = val;
+	int err;
+
+	slen = sizeof(sval);
+	ulen = sizeof(uval);
+
+	/* Setting SO_REUSEPORT */
+	err = submit_cmd_sqe(ring, sockfds->rx, SOCKET_URING_OP_SETSOCKOPT,
+			     SOL_SOCKET, SO_REUSEPORT, &uval, ulen);
+	assert(err == 1);
+
+	err = receive_cqe(ring);
+	if (err == -EOPNOTSUPP)
+		return T_EXIT_SKIP;
+
+	/* Get values from the systemcall */
+	err = getsockopt(sockfds->rx, SOL_SOCKET, SO_REUSEPORT, &sval, &slen);
+	assert(err == 0);
+
+	/* Make sure the set using io_uring cmd matches what systemcall returns */
+	assert(uval == sval);
+	assert(ulen == slen);
+
+	return T_EXIT_PASS;
+}
+
+/*
+ * Given a `val` value, set the TCP_USER_TIMEOUT using io_uring and read using
+ * getsockopt(2). Make sure they match
+ */
+static int run_setsockopt_usertimeout(struct io_uring *ring, struct fds *sockfds, int val)
+{
+	int optname = TCP_USER_TIMEOUT;
+	int level = IPPROTO_TCP;
+	unsigned int slen, ulen;
+	int sval, uval, err;
+
+	slen = sizeof(uval);
+	ulen = sizeof(uval);
+
+	uval = val;
+
+	/* Setting timeout */
+	err = submit_cmd_sqe(ring, sockfds->rx, SOCKET_URING_OP_SETSOCKOPT,
+			     level, optname, &uval, ulen);
+	assert(err == 1);
+
+	err = receive_cqe(ring);
+	if (err == -EOPNOTSUPP)
+		return T_EXIT_SKIP;
+	if (err < 0) {
+		fprintf(stderr, "%s: Got an error: %d\n", __func__, err);
+		return T_EXIT_FAIL;
+	}
+
+	/* Get the value from the systemcall, to make sure it was set */
+	err = getsockopt(sockfds->rx, level, optname, &sval, &slen);
+	assert(err == 0);
+	assert(uval == sval);
+
+	return T_EXIT_PASS;
+}
+
+/* Test setsockopt() for SOL_SOCKET */
+static int run_setsockopt_test(struct io_uring *ring, struct fds *sockfds)
+{
+	int err, i;
+
+	fprintf(stderr, "Testing setsockopt SOL_SOCKET/SO_REUSEPORT\n");
+	for (i = 0; i <= 1; i++) {
+		err = run_setsockopt_reuseport(ring, sockfds, i);
+		if (err)
+			return err;
+	}
+
+	fprintf(stderr, "Testing setsockopt IPPROTO_TCP/TCP_FASTOPEN\n");
+	for (i = 1; i <= 10; i++) {
+		err = run_setsockopt_usertimeout(ring, sockfds, i);
+		if (err)
+			return err;
+	}
+
+	return err;
+}
+
+/* Send data throughts the sockets */
+void send_data(struct fds *s)
+{
+	int written_bytes;
+	/* Send data sing the sockstruct->send */
+	written_bytes = write(s->tx, MSG, strlen(MSG));
+	assert(written_bytes == strlen(MSG));
+}
+
+int main(int argc, char *argv[])
+{
+	struct fds sockfds;
+	struct io_uring ring;
+	int err;
+
+	if (argc > 1)
+		return 0;
+
+	/* Simply io_uring ring creation */
+	ring = create_ring();
+
+	/* Create sockets */
+	sockfds = create_sockets();
+
+	send_data(&sockfds);
+
+	err = run_getsockopt_test(&ring, &sockfds);
+	if (err) {
+		if (err == T_EXIT_SKIP) {
+			fprintf(stderr, "Skipping tests.\n");
+			return T_EXIT_SKIP;
+		}
+		fprintf(stderr, "Failed to run test: %d\n", err);
+		return err;
+	}
+
+	err = run_setsockopt_test(&ring, &sockfds);
+	if (err) {
+		if (err == T_EXIT_SKIP) {
+			fprintf(stderr, "Skipping tests.\n");
+			return T_EXIT_SKIP;
+		}
+		fprintf(stderr, "Failed to run test: %d\n", err);
+		return err;
+	}
+
+	io_uring_queue_exit(&ring);
+	return err;
+}
-- 
2.34.1

[PATCH 4/5] man/io_uring_prep_cmd: Fix argument name

[Breno Leitao]

io_uring_prep_cmd(3) documents that one of the argument is SIOCOUTQ,
while, in fact, it is called `SOCKET_URING_OP_SIOCOUTQ`.

Fix the argument name, by replacing SIOCOUTQ to
SOCKET_URING_OP_SIOCOUTQ.

Fixes: 2459fef09411 ("io_uring_prep_cmd: Create a new helper for command ops")
Signed-off-by: Breno Leitao <leitao@debian.org>
---
 man/io_uring_prep_cmd.3 | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/man/io_uring_prep_cmd.3 b/man/io_uring_prep_cmd.3
index 01b48da..d6ec909 100644
--- a/man/io_uring_prep_cmd.3
+++ b/man/io_uring_prep_cmd.3
@@ -55,7 +55,7 @@ For more information about this command, please check
 
 
 .TP
-.B SIOCOUTQ
+.B SOCKET_URING_OP_SIOCOUTQ
 Returns the amount of unsent data in the socket send queue.
 The socket must not be in LISTEN state, otherwise an error
 .B -EINVAL
-- 
2.34.1

[PATCH 5/5] man/io_uring_prep_cmd: Add the new sockopt commands

[Breno Leitao]

io_uring now supports getsockopt and setsockopt socket commands.
Document these two new commands in the io_uring_prep_cmd man page.

Signed-off-by: Breno Leitao <leitao@debian.org>
---
 man/io_uring_prep_cmd.3 | 32 ++++++++++++++++++++++++++++++++
 1 file changed, 32 insertions(+)

diff --git a/man/io_uring_prep_cmd.3 b/man/io_uring_prep_cmd.3
index d6ec909..12f607e 100644
--- a/man/io_uring_prep_cmd.3
+++ b/man/io_uring_prep_cmd.3
@@ -72,6 +72,38 @@ Negative return value means an error.
 For more information about this command, please check
 .BR unix(7).
 
+.TP
+.B SOCKET_URING_OP_GETSOCKOPT
+Command to get options for the socket referred to by the socket file descriptor
+.I fd.
+The arguments are similar to the
+.BR getsockopt(2)
+system call.
+
+The
+.BR SOCKET_URING_OP_GETSOCKOPT
+command is limited to
+.BR SOL_SOCKET
+.I level.
+
+Differently from the
+.BR getsockopt(2)
+system call, the updated
+.I optlen
+value is returned in the CQE
+.I res
+field, on success. On failure, the CQE
+.I res
+contains a negative error number.
+
+.TP
+.B SOCKET_URING_OP_SETSOCKOPT
+Command to set options for the socket referred to by the socket file descriptor
+.I fd.
+The arguments are similar to the
+.BR setsockopt(2)
+system call.
+
 .SH NOTES
 The memory block pointed by
 .I optval
-- 
2.34.1

Re: [PATCH 0/5] liburing: Add {g,s}etsockopt command support

[Jens Axboe]

On 10/20/23 7:39 AM, Breno Leitao wrote:
> These are liburing patches that add support for the new
> SOCKET_URING_OP_GETSOCKOPT and SOCKET_URING_OP_SETSOCKOPT socket
> commands.
> 
> This patchset basically synchronize the UAPI bits, teach
> io_uring_prep_cmd(3) how to use the new fields, create a unit test and
> add the proper man page documentation.

Applied to the 'next' branch. Had to hand apply patch 2, but the rest
applied just fine.

Thanks!

-- 
Jens Axboe