[PATCH 0/7] further ring init cleanups

[PATCH 0/7] further ring init cleanups

[Pavel Begunkov]

There are several goals for this patch set. It deduplicates ring size and
offset calculations between ring setup and resizing. It moves most of
verification earlier before any allocations, which usually means simpler
error handling. And it keeps the logic localised instead of spreading it
across the file.

Pavel Begunkov (7):
  io_uring: refactor rings_size nosqarray handling
  io_uring: use size_add helpers for ring offsets
  io_uring: convert params to pointer in ring reisze
  io_uring: introduce struct io_ctx_config
  io_uring: keep ring laoyut in a structure
  io_uring: pre-calculate scq layout
  io_uring: move cq/sq user offset init around

 io_uring/io_uring.c | 137 ++++++++++++++++++++++++--------------------
 io_uring/io_uring.h |  19 +++++-
 io_uring/register.c |  65 +++++++++------------
 3 files changed, 119 insertions(+), 102 deletions(-)

-- 
2.49.0

[PATCH 1/7] io_uring: refactor rings_size nosqarray handling

[Pavel Begunkov]

A preparation patch inversing the IORING_SETUP_NO_SQARRAY check, this
way there is only one successful return path from the function, which
will be helpful later.

Signed-off-by: Pavel Begunkov <asml.silence@gmail.com>
---
 io_uring/io_uring.c | 24 ++++++++++++------------
 1 file changed, 12 insertions(+), 12 deletions(-)

diff --git a/io_uring/io_uring.c b/io_uring/io_uring.c
index 7e069d56b8a1..b37beb1cfefe 100644
--- a/io_uring/io_uring.c
+++ b/io_uring/io_uring.c
@@ -2761,7 +2761,9 @@ unsigned long rings_size(unsigned int flags, unsigned int sq_entries,
 			 unsigned int cq_entries, size_t *sq_offset)
 {
 	struct io_rings *rings;
-	size_t off, sq_array_size;
+	size_t off;
+
+	*sq_offset = SIZE_MAX;
 
 	off = struct_size(rings, cqes, cq_entries);
 	if (off == SIZE_MAX)
@@ -2785,19 +2787,17 @@ unsigned long rings_size(unsigned int flags, unsigned int sq_entries,
 		return SIZE_MAX;
 #endif
 
-	if (flags & IORING_SETUP_NO_SQARRAY) {
-		*sq_offset = SIZE_MAX;
-		return off;
-	}
-
-	*sq_offset = off;
+	if (!(flags & IORING_SETUP_NO_SQARRAY)) {
+		size_t sq_array_size;
 
-	sq_array_size = array_size(sizeof(u32), sq_entries);
-	if (sq_array_size == SIZE_MAX)
-		return SIZE_MAX;
+		*sq_offset = off;
 
-	if (check_add_overflow(off, sq_array_size, &off))
-		return SIZE_MAX;
+		sq_array_size = array_size(sizeof(u32), sq_entries);
+		if (sq_array_size == SIZE_MAX)
+			return SIZE_MAX;
+		if (check_add_overflow(off, sq_array_size, &off))
+			return SIZE_MAX;
+	}
 
 	return off;
 }
-- 
2.49.0

[PATCH 2/7] io_uring: use size_add helpers for ring offsets

[Pavel Begunkov]

Use size_add / size_mul set of functions for rings_size() calculations.
It's more consistent with struct_size(), and errors are preserved across
a series of calculations, so intermediate result checks can be omitted.

Signed-off-by: Pavel Begunkov <asml.silence@gmail.com>
---
 io_uring/io_uring.c | 18 ++++++++----------
 1 file changed, 8 insertions(+), 10 deletions(-)

diff --git a/io_uring/io_uring.c b/io_uring/io_uring.c
index b37beb1cfefe..57ebba8ba46c 100644
--- a/io_uring/io_uring.c
+++ b/io_uring/io_uring.c
@@ -2765,13 +2765,6 @@ unsigned long rings_size(unsigned int flags, unsigned int sq_entries,
 
 	*sq_offset = SIZE_MAX;
 
-	off = struct_size(rings, cqes, cq_entries);
-	if (off == SIZE_MAX)
-		return SIZE_MAX;
-	if (flags & IORING_SETUP_CQE32) {
-		if (check_shl_overflow(off, 1, &off))
-			return SIZE_MAX;
-	}
 	if (flags & IORING_SETUP_CQE_MIXED) {
 		if (cq_entries < 2)
 			return SIZE_MAX;
@@ -2781,6 +2774,12 @@ unsigned long rings_size(unsigned int flags, unsigned int sq_entries,
 			return SIZE_MAX;
 	}
 
+	off = struct_size(rings, cqes, cq_entries);
+	if (flags & IORING_SETUP_CQE32)
+		off = size_mul(off, 2);
+	if (off == SIZE_MAX)
+		return SIZE_MAX;
+
 #ifdef CONFIG_SMP
 	off = ALIGN(off, SMP_CACHE_BYTES);
 	if (off == 0)
@@ -2793,9 +2792,8 @@ unsigned long rings_size(unsigned int flags, unsigned int sq_entries,
 		*sq_offset = off;
 
 		sq_array_size = array_size(sizeof(u32), sq_entries);
-		if (sq_array_size == SIZE_MAX)
-			return SIZE_MAX;
-		if (check_add_overflow(off, sq_array_size, &off))
+		off = size_add(off, sq_array_size);
+		if (off == SIZE_MAX)
 			return SIZE_MAX;
 	}
 
-- 
2.49.0

[PATCH 3/7] io_uring: convert params to pointer in ring reisze

[Pavel Begunkov]

The parameters in io_register_resize_rings() will be moved into another
structure in a later patch. In preparation to that, convert the params
variable it to a pointer, but still store the data on stack.

Signed-off-by: Pavel Begunkov <asml.silence@gmail.com>
---
 io_uring/register.c | 48 ++++++++++++++++++++++-----------------------
 1 file changed, 24 insertions(+), 24 deletions(-)

diff --git a/io_uring/register.c b/io_uring/register.c
index afb924ceb9b6..f6b7b1c1be48 100644
--- a/io_uring/register.c
+++ b/io_uring/register.c
@@ -402,33 +402,33 @@ static int io_register_resize_rings(struct io_ring_ctx *ctx, void __user *arg)
 	struct io_ring_ctx_rings o = { }, n = { }, *to_free = NULL;
 	size_t size, sq_array_offset;
 	unsigned i, tail, old_head;
-	struct io_uring_params p;
+	struct io_uring_params __p, *p = &__p;
 	int ret;
 
 	/* limited to DEFER_TASKRUN for now */
 	if (!(ctx->flags & IORING_SETUP_DEFER_TASKRUN))
 		return -EINVAL;
-	if (copy_from_user(&p, arg, sizeof(p)))
+	if (copy_from_user(p, arg, sizeof(*p)))
 		return -EFAULT;
-	if (p.flags & ~RESIZE_FLAGS)
+	if (p->flags & ~RESIZE_FLAGS)
 		return -EINVAL;
 
 	/* properties that are always inherited */
-	p.flags |= (ctx->flags & COPY_FLAGS);
+	p->flags |= (ctx->flags & COPY_FLAGS);
 
-	ret = io_uring_fill_params(&p);
+	ret = io_uring_fill_params(p);
 	if (unlikely(ret))
 		return ret;
 
-	size = rings_size(p.flags, p.sq_entries, p.cq_entries,
+	size = rings_size(p->flags, p->sq_entries, p->cq_entries,
 				&sq_array_offset);
 	if (size == SIZE_MAX)
 		return -EOVERFLOW;
 
 	memset(&rd, 0, sizeof(rd));
 	rd.size = PAGE_ALIGN(size);
-	if (p.flags & IORING_SETUP_NO_MMAP) {
-		rd.user_addr = p.cq_off.user_addr;
+	if (p->flags & IORING_SETUP_NO_MMAP) {
+		rd.user_addr = p->cq_off.user_addr;
 		rd.flags |= IORING_MEM_REGION_TYPE_USER;
 	}
 	ret = io_create_region(ctx, &n.ring_region, &rd, IORING_OFF_CQ_RING);
@@ -445,20 +445,20 @@ static int io_register_resize_rings(struct io_ring_ctx *ctx, void __user *arg)
 	 * intent... Use read/write once helpers from here on to indicate the
 	 * shared nature of it.
 	 */
-	WRITE_ONCE(n.rings->sq_ring_mask, p.sq_entries - 1);
-	WRITE_ONCE(n.rings->cq_ring_mask, p.cq_entries - 1);
-	WRITE_ONCE(n.rings->sq_ring_entries, p.sq_entries);
-	WRITE_ONCE(n.rings->cq_ring_entries, p.cq_entries);
+	WRITE_ONCE(n.rings->sq_ring_mask, p->sq_entries - 1);
+	WRITE_ONCE(n.rings->cq_ring_mask, p->cq_entries - 1);
+	WRITE_ONCE(n.rings->sq_ring_entries, p->sq_entries);
+	WRITE_ONCE(n.rings->cq_ring_entries, p->cq_entries);
 
-	if (copy_to_user(arg, &p, sizeof(p))) {
+	if (copy_to_user(arg, p, sizeof(*p))) {
 		io_register_free_rings(ctx, &n);
 		return -EFAULT;
 	}
 
-	if (p.flags & IORING_SETUP_SQE128)
-		size = array_size(2 * sizeof(struct io_uring_sqe), p.sq_entries);
+	if (p->flags & IORING_SETUP_SQE128)
+		size = array_size(2 * sizeof(struct io_uring_sqe), p->sq_entries);
 	else
-		size = array_size(sizeof(struct io_uring_sqe), p.sq_entries);
+		size = array_size(sizeof(struct io_uring_sqe), p->sq_entries);
 	if (size == SIZE_MAX) {
 		io_register_free_rings(ctx, &n);
 		return -EOVERFLOW;
@@ -466,8 +466,8 @@ static int io_register_resize_rings(struct io_ring_ctx *ctx, void __user *arg)
 
 	memset(&rd, 0, sizeof(rd));
 	rd.size = PAGE_ALIGN(size);
-	if (p.flags & IORING_SETUP_NO_MMAP) {
-		rd.user_addr = p.sq_off.user_addr;
+	if (p->flags & IORING_SETUP_NO_MMAP) {
+		rd.user_addr = p->sq_off.user_addr;
 		rd.flags |= IORING_MEM_REGION_TYPE_USER;
 	}
 	ret = io_create_region(ctx, &n.sq_region, &rd, IORING_OFF_SQES);
@@ -508,11 +508,11 @@ static int io_register_resize_rings(struct io_ring_ctx *ctx, void __user *arg)
 	 */
 	tail = READ_ONCE(o.rings->sq.tail);
 	old_head = READ_ONCE(o.rings->sq.head);
-	if (tail - old_head > p.sq_entries)
+	if (tail - old_head > p->sq_entries)
 		goto overflow;
 	for (i = old_head; i < tail; i++) {
 		unsigned src_head = i & (ctx->sq_entries - 1);
-		unsigned dst_head = i & (p.sq_entries - 1);
+		unsigned dst_head = i & (p->sq_entries - 1);
 
 		n.sq_sqes[dst_head] = o.sq_sqes[src_head];
 	}
@@ -521,7 +521,7 @@ static int io_register_resize_rings(struct io_ring_ctx *ctx, void __user *arg)
 
 	tail = READ_ONCE(o.rings->cq.tail);
 	old_head = READ_ONCE(o.rings->cq.head);
-	if (tail - old_head > p.cq_entries) {
+	if (tail - old_head > p->cq_entries) {
 overflow:
 		/* restore old rings, and return -EOVERFLOW via cleanup path */
 		ctx->rings = o.rings;
@@ -532,7 +532,7 @@ static int io_register_resize_rings(struct io_ring_ctx *ctx, void __user *arg)
 	}
 	for (i = old_head; i < tail; i++) {
 		unsigned src_head = i & (ctx->cq_entries - 1);
-		unsigned dst_head = i & (p.cq_entries - 1);
+		unsigned dst_head = i & (p->cq_entries - 1);
 
 		n.rings->cqes[dst_head] = o.rings->cqes[src_head];
 	}
@@ -550,8 +550,8 @@ static int io_register_resize_rings(struct io_ring_ctx *ctx, void __user *arg)
 	if (!(ctx->flags & IORING_SETUP_NO_SQARRAY))
 		ctx->sq_array = (u32 *)((char *)n.rings + sq_array_offset);
 
-	ctx->sq_entries = p.sq_entries;
-	ctx->cq_entries = p.cq_entries;
+	ctx->sq_entries = p->sq_entries;
+	ctx->cq_entries = p->cq_entries;
 
 	ctx->rings = n.rings;
 	ctx->sq_sqes = n.sq_sqes;
-- 
2.49.0

[PATCH 4/7] io_uring: introduce struct io_ctx_config

[Pavel Begunkov]

There will be more information needed during ctx setup, and instead of
passing a handful of pointers around, wrap them all into a new
structure. Add a helper for encapsulating all configuration checks and
preparation, that's also reused for ring resizing.

Note, it indirectly adds a io_uring_sanitise_params() check to ring
resizing, which is a good thing.

Signed-off-by: Pavel Begunkov <asml.silence@gmail.com>
---
 io_uring/io_uring.c | 41 ++++++++++++++++++++++++++++-------------
 io_uring/io_uring.h |  8 +++++++-
 io_uring/register.c |  7 +++++--
 3 files changed, 40 insertions(+), 16 deletions(-)

diff --git a/io_uring/io_uring.c b/io_uring/io_uring.c
index 57ebba8ba46c..f039e293582a 100644
--- a/io_uring/io_uring.c
+++ b/io_uring/io_uring.c
@@ -3480,7 +3480,7 @@ static int io_uring_sanitise_params(struct io_uring_params *p)
 	return 0;
 }
 
-int io_uring_fill_params(struct io_uring_params *p)
+static int io_uring_fill_params(struct io_uring_params *p)
 {
 	unsigned entries = p->sq_entries;
 
@@ -3545,12 +3545,9 @@ int io_uring_fill_params(struct io_uring_params *p)
 	return 0;
 }
 
-static __cold int io_uring_create(struct io_uring_params *p,
-				  struct io_uring_params __user *params)
+int io_prepare_config(struct io_ctx_config *config)
 {
-	struct io_ring_ctx *ctx;
-	struct io_uring_task *tctx;
-	struct file *file;
+	struct io_uring_params *p = &config->p;
 	int ret;
 
 	ret = io_uring_sanitise_params(p);
@@ -3558,7 +3555,22 @@ static __cold int io_uring_create(struct io_uring_params *p,
 		return ret;
 
 	ret = io_uring_fill_params(p);
-	if (unlikely(ret))
+	if (ret)
+		return ret;
+
+	return 0;
+}
+
+static __cold int io_uring_create(struct io_ctx_config *config)
+{
+	struct io_uring_params *p = &config->p;
+	struct io_ring_ctx *ctx;
+	struct io_uring_task *tctx;
+	struct file *file;
+	int ret;
+
+	ret = io_prepare_config(config);
+	if (ret)
 		return ret;
 
 	ctx = io_ring_ctx_alloc(p);
@@ -3631,7 +3643,7 @@ static __cold int io_uring_create(struct io_uring_params *p,
 
 	p->features = IORING_FEAT_FLAGS;
 
-	if (copy_to_user(params, p, sizeof(*p))) {
+	if (copy_to_user(config->uptr, p, sizeof(*p))) {
 		ret = -EFAULT;
 		goto err;
 	}
@@ -3684,16 +3696,19 @@ static __cold int io_uring_create(struct io_uring_params *p,
  */
 static long io_uring_setup(u32 entries, struct io_uring_params __user *params)
 {
-	struct io_uring_params p;
+	struct io_ctx_config config;
+
+	memset(&config, 0, sizeof(config));
 
-	if (copy_from_user(&p, params, sizeof(p)))
+	if (copy_from_user(&config.p, params, sizeof(config.p)))
 		return -EFAULT;
 
-	if (!mem_is_zero(&p.resv, sizeof(p.resv)))
+	if (!mem_is_zero(&config.p.resv, sizeof(config.p.resv)))
 		return -EINVAL;
 
-	p.sq_entries = entries;
-	return io_uring_create(&p, params);
+	config.p.sq_entries = entries;
+	config.uptr = params;
+	return io_uring_create(&config);
 }
 
 static inline int io_uring_allowed(void)
diff --git a/io_uring/io_uring.h b/io_uring/io_uring.h
index b2251446497a..d8bc44acb9fa 100644
--- a/io_uring/io_uring.h
+++ b/io_uring/io_uring.h
@@ -17,6 +17,11 @@
 #include <trace/events/io_uring.h>
 #endif
 
+struct io_ctx_config {
+	struct io_uring_params p;
+	struct io_uring_params __user *uptr;
+};
+
 #define IORING_FEAT_FLAGS (IORING_FEAT_SINGLE_MMAP |\
 			IORING_FEAT_NODROP |\
 			IORING_FEAT_SUBMIT_STABLE |\
@@ -136,7 +141,8 @@ static inline bool io_should_wake(struct io_wait_queue *iowq)
 
 unsigned long rings_size(unsigned int flags, unsigned int sq_entries,
 			 unsigned int cq_entries, size_t *sq_offset);
-int io_uring_fill_params(struct io_uring_params *p);
+int io_prepare_config(struct io_ctx_config *config);
+
 bool io_cqe_cache_refill(struct io_ring_ctx *ctx, bool overflow, bool cqe32);
 int io_run_task_work_sig(struct io_ring_ctx *ctx);
 int io_run_local_work(struct io_ring_ctx *ctx, int min_events, int max_events);
diff --git a/io_uring/register.c b/io_uring/register.c
index f6b7b1c1be48..13385ac0f85a 100644
--- a/io_uring/register.c
+++ b/io_uring/register.c
@@ -398,13 +398,16 @@ static void io_register_free_rings(struct io_ring_ctx *ctx,
 
 static int io_register_resize_rings(struct io_ring_ctx *ctx, void __user *arg)
 {
+	struct io_ctx_config config;
 	struct io_uring_region_desc rd;
 	struct io_ring_ctx_rings o = { }, n = { }, *to_free = NULL;
 	size_t size, sq_array_offset;
 	unsigned i, tail, old_head;
-	struct io_uring_params __p, *p = &__p;
+	struct io_uring_params *p = &config.p;
 	int ret;
 
+	memset(&config, 0, sizeof(config));
+
 	/* limited to DEFER_TASKRUN for now */
 	if (!(ctx->flags & IORING_SETUP_DEFER_TASKRUN))
 		return -EINVAL;
@@ -416,7 +419,7 @@ static int io_register_resize_rings(struct io_ring_ctx *ctx, void __user *arg)
 	/* properties that are always inherited */
 	p->flags |= (ctx->flags & COPY_FLAGS);
 
-	ret = io_uring_fill_params(p);
+	ret = io_prepare_config(&config);
 	if (unlikely(ret))
 		return ret;
 
-- 
2.49.0

[PATCH 5/7] io_uring: keep ring laoyut in a structure

[Pavel Begunkov]

Add a structure keeping SQ/CQ sizes and offsets. For now it only records
data previously returned from rings_size and the SQ size.

Signed-off-by: Pavel Begunkov <asml.silence@gmail.com>
---
 io_uring/io_uring.c | 54 +++++++++++++++++++++++----------------------
 io_uring/io_uring.h | 12 ++++++++--
 io_uring/register.c | 24 ++++++--------------
 3 files changed, 45 insertions(+), 45 deletions(-)

diff --git a/io_uring/io_uring.c b/io_uring/io_uring.c
index f039e293582a..1dfd0a8a7270 100644
--- a/io_uring/io_uring.c
+++ b/io_uring/io_uring.c
@@ -2757,47 +2757,57 @@ static void io_rings_free(struct io_ring_ctx *ctx)
 	ctx->sq_sqes = NULL;
 }
 
-unsigned long rings_size(unsigned int flags, unsigned int sq_entries,
-			 unsigned int cq_entries, size_t *sq_offset)
+int rings_size(unsigned int flags, unsigned int sq_entries,
+		unsigned int cq_entries, struct io_rings_layout *rl)
 {
 	struct io_rings *rings;
+	size_t sqe_size;
 	size_t off;
 
-	*sq_offset = SIZE_MAX;
-
 	if (flags & IORING_SETUP_CQE_MIXED) {
 		if (cq_entries < 2)
-			return SIZE_MAX;
+			return -EOVERFLOW;
 	}
 	if (flags & IORING_SETUP_SQE_MIXED) {
 		if (sq_entries < 2)
-			return SIZE_MAX;
+			return -EOVERFLOW;
 	}
 
+	rl->sq_array_offset = SIZE_MAX;
+
+	sqe_size = sizeof(struct io_uring_sqe);
+	if (flags & IORING_SETUP_SQE128)
+		sqe_size *= 2;
+
+	rl->sq_size = array_size(sqe_size, sq_entries);
+	if (rl->sq_size == SIZE_MAX)
+		return -EOVERFLOW;
+
 	off = struct_size(rings, cqes, cq_entries);
 	if (flags & IORING_SETUP_CQE32)
 		off = size_mul(off, 2);
 	if (off == SIZE_MAX)
-		return SIZE_MAX;
+		return -EOVERFLOW;
 
 #ifdef CONFIG_SMP
 	off = ALIGN(off, SMP_CACHE_BYTES);
 	if (off == 0)
-		return SIZE_MAX;
+		return -EOVERFLOW;
 #endif
 
 	if (!(flags & IORING_SETUP_NO_SQARRAY)) {
 		size_t sq_array_size;
 
-		*sq_offset = off;
+		rl->sq_array_offset = off;
 
 		sq_array_size = array_size(sizeof(u32), sq_entries);
 		off = size_add(off, sq_array_size);
 		if (off == SIZE_MAX)
-			return SIZE_MAX;
+			return -EOVERFLOW;
 	}
 
-	return off;
+	rl->rings_size = off;
+	return 0;
 }
 
 static __cold void __io_req_caches_free(struct io_ring_ctx *ctx)
@@ -3346,28 +3356,20 @@ static __cold int io_allocate_scq_urings(struct io_ring_ctx *ctx,
 					 struct io_uring_params *p)
 {
 	struct io_uring_region_desc rd;
+	struct io_rings_layout __rl, *rl = &__rl;
 	struct io_rings *rings;
-	size_t sq_array_offset;
-	size_t sq_size, cq_size, sqe_size;
 	int ret;
 
 	/* make sure these are sane, as we already accounted them */
 	ctx->sq_entries = p->sq_entries;
 	ctx->cq_entries = p->cq_entries;
 
-	sqe_size = sizeof(struct io_uring_sqe);
-	if (p->flags & IORING_SETUP_SQE128)
-		sqe_size *= 2;
-	sq_size = array_size(sqe_size, p->sq_entries);
-	if (sq_size == SIZE_MAX)
-		return -EOVERFLOW;
-	cq_size = rings_size(ctx->flags, p->sq_entries, p->cq_entries,
-			  &sq_array_offset);
-	if (cq_size == SIZE_MAX)
-		return -EOVERFLOW;
+	ret = rings_size(ctx->flags, p->sq_entries, p->cq_entries, rl);
+	if (ret)
+		return ret;
 
 	memset(&rd, 0, sizeof(rd));
-	rd.size = PAGE_ALIGN(cq_size);
+	rd.size = PAGE_ALIGN(rl->rings_size);
 	if (ctx->flags & IORING_SETUP_NO_MMAP) {
 		rd.user_addr = p->cq_off.user_addr;
 		rd.flags |= IORING_MEM_REGION_TYPE_USER;
@@ -3378,10 +3380,10 @@ static __cold int io_allocate_scq_urings(struct io_ring_ctx *ctx,
 	ctx->rings = rings = io_region_get_ptr(&ctx->ring_region);
 
 	if (!(ctx->flags & IORING_SETUP_NO_SQARRAY))
-		ctx->sq_array = (u32 *)((char *)rings + sq_array_offset);
+		ctx->sq_array = (u32 *)((char *)rings + rl->sq_array_offset);
 
 	memset(&rd, 0, sizeof(rd));
-	rd.size = PAGE_ALIGN(sq_size);
+	rd.size = PAGE_ALIGN(rl->sq_size);
 	if (ctx->flags & IORING_SETUP_NO_MMAP) {
 		rd.user_addr = p->sq_off.user_addr;
 		rd.flags |= IORING_MEM_REGION_TYPE_USER;
diff --git a/io_uring/io_uring.h b/io_uring/io_uring.h
index d8bc44acb9fa..5e544c2d27c8 100644
--- a/io_uring/io_uring.h
+++ b/io_uring/io_uring.h
@@ -17,6 +17,14 @@
 #include <trace/events/io_uring.h>
 #endif
 
+struct io_rings_layout {
+	/* size of CQ + headers + SQ offset array */
+	size_t rings_size;
+	size_t sq_size;
+
+	size_t sq_array_offset;
+};
+
 struct io_ctx_config {
 	struct io_uring_params p;
 	struct io_uring_params __user *uptr;
@@ -139,8 +147,8 @@ static inline bool io_should_wake(struct io_wait_queue *iowq)
 #define IORING_MAX_ENTRIES	32768
 #define IORING_MAX_CQ_ENTRIES	(2 * IORING_MAX_ENTRIES)
 
-unsigned long rings_size(unsigned int flags, unsigned int sq_entries,
-			 unsigned int cq_entries, size_t *sq_offset);
+int rings_size(unsigned int flags, unsigned int sq_entries,
+		unsigned int cq_entries, struct io_rings_layout *rl);
 int io_prepare_config(struct io_ctx_config *config);
 
 bool io_cqe_cache_refill(struct io_ring_ctx *ctx, bool overflow, bool cqe32);
diff --git a/io_uring/register.c b/io_uring/register.c
index 13385ac0f85a..98693021edbe 100644
--- a/io_uring/register.c
+++ b/io_uring/register.c
@@ -401,9 +401,9 @@ static int io_register_resize_rings(struct io_ring_ctx *ctx, void __user *arg)
 	struct io_ctx_config config;
 	struct io_uring_region_desc rd;
 	struct io_ring_ctx_rings o = { }, n = { }, *to_free = NULL;
-	size_t size, sq_array_offset;
 	unsigned i, tail, old_head;
 	struct io_uring_params *p = &config.p;
+	struct io_rings_layout __rl, *rl = &__rl;
 	int ret;
 
 	memset(&config, 0, sizeof(config));
@@ -423,13 +423,12 @@ static int io_register_resize_rings(struct io_ring_ctx *ctx, void __user *arg)
 	if (unlikely(ret))
 		return ret;
 
-	size = rings_size(p->flags, p->sq_entries, p->cq_entries,
-				&sq_array_offset);
-	if (size == SIZE_MAX)
-		return -EOVERFLOW;
+	ret = rings_size(p->flags, p->sq_entries, p->cq_entries, rl);
+	if (ret)
+		return ret;
 
 	memset(&rd, 0, sizeof(rd));
-	rd.size = PAGE_ALIGN(size);
+	rd.size = PAGE_ALIGN(rl->rings_size);
 	if (p->flags & IORING_SETUP_NO_MMAP) {
 		rd.user_addr = p->cq_off.user_addr;
 		rd.flags |= IORING_MEM_REGION_TYPE_USER;
@@ -458,17 +457,8 @@ static int io_register_resize_rings(struct io_ring_ctx *ctx, void __user *arg)
 		return -EFAULT;
 	}
 
-	if (p->flags & IORING_SETUP_SQE128)
-		size = array_size(2 * sizeof(struct io_uring_sqe), p->sq_entries);
-	else
-		size = array_size(sizeof(struct io_uring_sqe), p->sq_entries);
-	if (size == SIZE_MAX) {
-		io_register_free_rings(ctx, &n);
-		return -EOVERFLOW;
-	}
-
 	memset(&rd, 0, sizeof(rd));
-	rd.size = PAGE_ALIGN(size);
+	rd.size = PAGE_ALIGN(rl->sq_size);
 	if (p->flags & IORING_SETUP_NO_MMAP) {
 		rd.user_addr = p->sq_off.user_addr;
 		rd.flags |= IORING_MEM_REGION_TYPE_USER;
@@ -551,7 +541,7 @@ static int io_register_resize_rings(struct io_ring_ctx *ctx, void __user *arg)
 
 	/* all done, store old pointers and assign new ones */
 	if (!(ctx->flags & IORING_SETUP_NO_SQARRAY))
-		ctx->sq_array = (u32 *)((char *)n.rings + sq_array_offset);
+		ctx->sq_array = (u32 *)((char *)n.rings + rl->sq_array_offset);
 
 	ctx->sq_entries = p->sq_entries;
 	ctx->cq_entries = p->cq_entries;
-- 
2.49.0

[PATCH 6/7] io_uring: pre-calculate scq layout

[Pavel Begunkov]

Move ring layouts calculations into io_prepare_config(), so that more
misconfiguration checking can be done earlier before creating a ctx.
It also deduplicates some code with ring resizing. And as a bonus, now
it initialises params->sq_off.array, which is closer to all other user
offset init, and also applies it to ring resizing, which was previously
missing it.

Signed-off-by: Pavel Begunkov <asml.silence@gmail.com>
---
 io_uring/io_uring.c | 27 ++++++++++++++-------------
 io_uring/io_uring.h |  3 +--
 io_uring/register.c |  4 ----
 3 files changed, 15 insertions(+), 19 deletions(-)

diff --git a/io_uring/io_uring.c b/io_uring/io_uring.c
index 1dfd0a8a7270..d286118dcd9d 100644
--- a/io_uring/io_uring.c
+++ b/io_uring/io_uring.c
@@ -2757,8 +2757,8 @@ static void io_rings_free(struct io_ring_ctx *ctx)
 	ctx->sq_sqes = NULL;
 }
 
-int rings_size(unsigned int flags, unsigned int sq_entries,
-		unsigned int cq_entries, struct io_rings_layout *rl)
+static int rings_size(unsigned int flags, unsigned int sq_entries,
+		      unsigned int cq_entries, struct io_rings_layout *rl)
 {
 	struct io_rings *rings;
 	size_t sqe_size;
@@ -3353,10 +3353,11 @@ bool io_is_uring_fops(struct file *file)
 }
 
 static __cold int io_allocate_scq_urings(struct io_ring_ctx *ctx,
-					 struct io_uring_params *p)
+					 struct io_ctx_config *config)
 {
+	struct io_uring_params *p = &config->p;
+	struct io_rings_layout *rl = &config->layout;
 	struct io_uring_region_desc rd;
-	struct io_rings_layout __rl, *rl = &__rl;
 	struct io_rings *rings;
 	int ret;
 
@@ -3364,10 +3365,6 @@ static __cold int io_allocate_scq_urings(struct io_ring_ctx *ctx,
 	ctx->sq_entries = p->sq_entries;
 	ctx->cq_entries = p->cq_entries;
 
-	ret = rings_size(ctx->flags, p->sq_entries, p->cq_entries, rl);
-	if (ret)
-		return ret;
-
 	memset(&rd, 0, sizeof(rd));
 	rd.size = PAGE_ALIGN(rl->rings_size);
 	if (ctx->flags & IORING_SETUP_NO_MMAP) {
@@ -3378,7 +3375,6 @@ static __cold int io_allocate_scq_urings(struct io_ring_ctx *ctx,
 	if (ret)
 		return ret;
 	ctx->rings = rings = io_region_get_ptr(&ctx->ring_region);
-
 	if (!(ctx->flags & IORING_SETUP_NO_SQARRAY))
 		ctx->sq_array = (u32 *)((char *)rings + rl->sq_array_offset);
 
@@ -3560,6 +3556,14 @@ int io_prepare_config(struct io_ctx_config *config)
 	if (ret)
 		return ret;
 
+	ret = rings_size(p->flags, p->sq_entries, p->cq_entries,
+			 &config->layout);
+	if (ret)
+		return ret;
+
+	if (!(p->flags & IORING_SETUP_NO_SQARRAY))
+		p->sq_off.array = config->layout.sq_array_offset;
+
 	return 0;
 }
 
@@ -3632,13 +3636,10 @@ static __cold int io_uring_create(struct io_ctx_config *config)
 	mmgrab(current->mm);
 	ctx->mm_account = current->mm;
 
-	ret = io_allocate_scq_urings(ctx, p);
+	ret = io_allocate_scq_urings(ctx, config);
 	if (ret)
 		goto err;
 
-	if (!(p->flags & IORING_SETUP_NO_SQARRAY))
-		p->sq_off.array = (char *)ctx->sq_array - (char *)ctx->rings;
-
 	ret = io_sq_offload_create(ctx, p);
 	if (ret)
 		goto err;
diff --git a/io_uring/io_uring.h b/io_uring/io_uring.h
index 5e544c2d27c8..a790c16854d3 100644
--- a/io_uring/io_uring.h
+++ b/io_uring/io_uring.h
@@ -27,6 +27,7 @@ struct io_rings_layout {
 
 struct io_ctx_config {
 	struct io_uring_params p;
+	struct io_rings_layout layout;
 	struct io_uring_params __user *uptr;
 };
 
@@ -147,8 +148,6 @@ static inline bool io_should_wake(struct io_wait_queue *iowq)
 #define IORING_MAX_ENTRIES	32768
 #define IORING_MAX_CQ_ENTRIES	(2 * IORING_MAX_ENTRIES)
 
-int rings_size(unsigned int flags, unsigned int sq_entries,
-		unsigned int cq_entries, struct io_rings_layout *rl);
 int io_prepare_config(struct io_ctx_config *config);
 
 bool io_cqe_cache_refill(struct io_ring_ctx *ctx, bool overflow, bool cqe32);
diff --git a/io_uring/register.c b/io_uring/register.c
index 98693021edbe..110e978b872d 100644
--- a/io_uring/register.c
+++ b/io_uring/register.c
@@ -423,10 +423,6 @@ static int io_register_resize_rings(struct io_ring_ctx *ctx, void __user *arg)
 	if (unlikely(ret))
 		return ret;
 
-	ret = rings_size(p->flags, p->sq_entries, p->cq_entries, rl);
-	if (ret)
-		return ret;
-
 	memset(&rd, 0, sizeof(rd));
 	rd.size = PAGE_ALIGN(rl->rings_size);
 	if (p->flags & IORING_SETUP_NO_MMAP) {
-- 
2.49.0

[PATCH 7/7] io_uring: move cq/sq user offset init around

[Pavel Begunkov]

Move user SQ/CQ offset initialisation at the end of io_prepare_config()
where it already calculated all information to set it properly.

Signed-off-by: Pavel Begunkov <asml.silence@gmail.com>
---
 io_uring/io_uring.c | 41 ++++++++++++++++++++---------------------
 1 file changed, 20 insertions(+), 21 deletions(-)

diff --git a/io_uring/io_uring.c b/io_uring/io_uring.c
index d286118dcd9d..c1c923d19cc3 100644
--- a/io_uring/io_uring.c
+++ b/io_uring/io_uring.c
@@ -3519,27 +3519,6 @@ static int io_uring_fill_params(struct io_uring_params *p)
 		p->cq_entries = 2 * p->sq_entries;
 	}
 
-	p->sq_off.head = offsetof(struct io_rings, sq.head);
-	p->sq_off.tail = offsetof(struct io_rings, sq.tail);
-	p->sq_off.ring_mask = offsetof(struct io_rings, sq_ring_mask);
-	p->sq_off.ring_entries = offsetof(struct io_rings, sq_ring_entries);
-	p->sq_off.flags = offsetof(struct io_rings, sq_flags);
-	p->sq_off.dropped = offsetof(struct io_rings, sq_dropped);
-	p->sq_off.resv1 = 0;
-	if (!(p->flags & IORING_SETUP_NO_MMAP))
-		p->sq_off.user_addr = 0;
-
-	p->cq_off.head = offsetof(struct io_rings, cq.head);
-	p->cq_off.tail = offsetof(struct io_rings, cq.tail);
-	p->cq_off.ring_mask = offsetof(struct io_rings, cq_ring_mask);
-	p->cq_off.ring_entries = offsetof(struct io_rings, cq_ring_entries);
-	p->cq_off.overflow = offsetof(struct io_rings, cq_overflow);
-	p->cq_off.cqes = offsetof(struct io_rings, cqes);
-	p->cq_off.flags = offsetof(struct io_rings, cq_flags);
-	p->cq_off.resv1 = 0;
-	if (!(p->flags & IORING_SETUP_NO_MMAP))
-		p->cq_off.user_addr = 0;
-
 	return 0;
 }
 
@@ -3561,6 +3540,26 @@ int io_prepare_config(struct io_ctx_config *config)
 	if (ret)
 		return ret;
 
+	p->sq_off.head = offsetof(struct io_rings, sq.head);
+	p->sq_off.tail = offsetof(struct io_rings, sq.tail);
+	p->sq_off.ring_mask = offsetof(struct io_rings, sq_ring_mask);
+	p->sq_off.ring_entries = offsetof(struct io_rings, sq_ring_entries);
+	p->sq_off.flags = offsetof(struct io_rings, sq_flags);
+	p->sq_off.dropped = offsetof(struct io_rings, sq_dropped);
+	p->sq_off.resv1 = 0;
+	if (!(p->flags & IORING_SETUP_NO_MMAP))
+		p->sq_off.user_addr = 0;
+
+	p->cq_off.head = offsetof(struct io_rings, cq.head);
+	p->cq_off.tail = offsetof(struct io_rings, cq.tail);
+	p->cq_off.ring_mask = offsetof(struct io_rings, cq_ring_mask);
+	p->cq_off.ring_entries = offsetof(struct io_rings, cq_ring_entries);
+	p->cq_off.overflow = offsetof(struct io_rings, cq_overflow);
+	p->cq_off.cqes = offsetof(struct io_rings, cqes);
+	p->cq_off.flags = offsetof(struct io_rings, cq_flags);
+	p->cq_off.resv1 = 0;
+	if (!(p->flags & IORING_SETUP_NO_MMAP))
+		p->cq_off.user_addr = 0;
 	if (!(p->flags & IORING_SETUP_NO_SQARRAY))
 		p->sq_off.array = config->layout.sq_array_offset;
 
-- 
2.49.0

Re: [PATCH 0/7] further ring init cleanups

[Jens Axboe]

On 11/12/25 5:45 AM, Pavel Begunkov wrote:
> There are several goals for this patch set. It deduplicates ring size and
> offset calculations between ring setup and resizing. It moves most of
> verification earlier before any allocations, which usually means simpler
> error handling. And it keeps the logic localised instead of spreading it
> across the file.
> 
> Pavel Begunkov (7):
>   io_uring: refactor rings_size nosqarray handling
>   io_uring: use size_add helpers for ring offsets
>   io_uring: convert params to pointer in ring reisze
>   io_uring: introduce struct io_ctx_config
>   io_uring: keep ring laoyut in a structure
>   io_uring: pre-calculate scq layout
>   io_uring: move cq/sq user offset init around
> 
>  io_uring/io_uring.c | 137 ++++++++++++++++++++++++--------------------
>  io_uring/io_uring.h |  19 +++++-
>  io_uring/register.c |  65 +++++++++------------
>  3 files changed, 119 insertions(+), 102 deletions(-)

All looks pretty straightforward to me. I'm going to let it simmer for a
day or so to give other folks a chance to take a look too.

-- 
Jens Axboe

Re: [PATCH 0/7] further ring init cleanups

[Jens Axboe]

On Wed, 12 Nov 2025 12:45:52 +0000, Pavel Begunkov wrote:
> There are several goals for this patch set. It deduplicates ring size and
> offset calculations between ring setup and resizing. It moves most of
> verification earlier before any allocations, which usually means simpler
> error handling. And it keeps the logic localised instead of spreading it
> across the file.
> 
> Pavel Begunkov (7):
>   io_uring: refactor rings_size nosqarray handling
>   io_uring: use size_add helpers for ring offsets
>   io_uring: convert params to pointer in ring reisze
>   io_uring: introduce struct io_ctx_config
>   io_uring: keep ring laoyut in a structure
>   io_uring: pre-calculate scq layout
>   io_uring: move cq/sq user offset init around
> 
> [...]

Applied, thanks!

[1/7] io_uring: refactor rings_size nosqarray handling
      commit: e279bb4b4c4d012808fb21ff41183a2e76c26679
[2/7] io_uring: use size_add helpers for ring offsets
      commit: 94cd832916521d8d51b25b40691354c24831c655
[3/7] io_uring: convert params to pointer in ring reisze
      commit: 929dbbb699110c9377da721ed7b44a660bb4ee01
[4/7] io_uring: introduce struct io_ctx_config
      commit: 0f4b537363cb66c78e97bb58c26986af62856356
[5/7] io_uring: keep ring laoyut in a structure
      commit: 001b76b7e755767d847e9aebf1fd6e525f1e58c8
[6/7] io_uring: pre-calculate scq layout
      commit: eb76ff6a6829a9a54a385804cc9dbe4460f156d6
[7/7] io_uring: move cq/sq user offset init around
      commit: d741c6255524f0691aea53381219fadcd2b38408

Best regards,
-- 
Jens Axboe