From: Jens Axboe <axboe@kernel.dk>
To: io-uring@vger.kernel.org
Cc: linux-arm-kernel@lists.infradead.org,
linux-kernel@vger.kernel.org, tglx@kernel.org, mingo@redhat.com,
peterz@infradead.org, Jens Axboe <axboe@kernel.dk>
Subject: [PATCH 07/15] io_uring: add uring_lock section depth tracking and blockable opdef flag
Date: Fri, 11 Sep 2026 09:40:57 -0600 [thread overview]
Message-ID: <20260911154148.644489-8-axboe@kernel.dk> (raw)
In-Reply-To: <20260911154148.644489-1-axboe@kernel.dk>
Prep patch for issuing requests inline in blocking mode and catching the
sleep when it happens, rather than punting to io-wq upfront because an
operation may block.
If a request blocks inline, uring_lock must be dropped on behalf of the
sleeping task, which is only safe outside the sections that rely on the
lock being held. Track those with a depth counter in
io_ring_submit_lock() and io_ring_submit_unlock().
Add a "blockable" flag to io_issue_def for opcodes whose issue path can
cope with blocking inline: read/write, the forced async fs ops, open,
close and splice/tee. uring_cmd is excluded for now, drivers may bind
state to the submitting task.
No functional changes in this patch.
Signed-off-by: Jens Axboe <axboe@kernel.dk>
---
include/linux/io_uring_types.h | 5 +++++
io_uring/io_uring.h | 3 +++
io_uring/opdef.c | 29 +++++++++++++++++++++++++++++
io_uring/opdef.h | 2 ++
4 files changed, 39 insertions(+)
diff --git a/include/linux/io_uring_types.h b/include/linux/io_uring_types.h
index 4af3d579ead6..50a4a0ad222f 100644
--- a/include/linux/io_uring_types.h
+++ b/include/linux/io_uring_types.h
@@ -352,6 +352,11 @@ struct io_ring_ctx {
/* submission data */
struct {
struct mutex uring_lock;
+ /*
+ * io_ring_submit_lock() nesting depth, non-zero means the
+ * issue path relies on the lock being held.
+ */
+ unsigned int submit_lock_depth;
/*
* Ring buffer of indices into array of io_uring_sqe, which is
diff --git a/io_uring/io_uring.h b/io_uring/io_uring.h
index 896aab1ed026..870bb4dcc415 100644
--- a/io_uring/io_uring.h
+++ b/io_uring/io_uring.h
@@ -393,6 +393,8 @@ static inline void io_ring_submit_unlock(struct io_ring_ctx *ctx,
unsigned issue_flags)
{
lockdep_assert_held(&ctx->uring_lock);
+ lockdep_assert(ctx->submit_lock_depth > 0);
+ ctx->submit_lock_depth--;
if (unlikely(issue_flags & IO_URING_F_UNLOCKED))
mutex_unlock(&ctx->uring_lock);
}
@@ -409,6 +411,7 @@ static inline void io_ring_submit_lock(struct io_ring_ctx *ctx,
if (unlikely(issue_flags & IO_URING_F_UNLOCKED))
mutex_lock(&ctx->uring_lock);
lockdep_assert_held(&ctx->uring_lock);
+ ctx->submit_lock_depth++;
}
static inline void io_commit_cqring(struct io_ring_ctx *ctx)
diff --git a/io_uring/opdef.c b/io_uring/opdef.c
index cf3aa2242cd7..fa07a2b94536 100644
--- a/io_uring/opdef.c
+++ b/io_uring/opdef.c
@@ -69,6 +69,7 @@ const struct io_issue_def io_issue_defs[] = {
.iopoll = 1,
.vectored = 1,
.async_size = sizeof(struct io_async_rw),
+ .blockable = 1,
.prep = io_prep_readv,
.issue = io_read,
},
@@ -83,12 +84,14 @@ const struct io_issue_def io_issue_defs[] = {
.iopoll = 1,
.vectored = 1,
.async_size = sizeof(struct io_async_rw),
+ .blockable = 1,
.prep = io_prep_writev,
.issue = io_write,
},
[IORING_OP_FSYNC] = {
.needs_file = 1,
.audit_skip = 1,
+ .blockable = 1,
.prep = io_fsync_prep,
.issue = io_fsync,
},
@@ -101,6 +104,7 @@ const struct io_issue_def io_issue_defs[] = {
.ioprio = 1,
.iopoll = 1,
.async_size = sizeof(struct io_async_rw),
+ .blockable = 1,
.prep = io_prep_read_fixed,
.issue = io_read_fixed,
},
@@ -114,6 +118,7 @@ const struct io_issue_def io_issue_defs[] = {
.ioprio = 1,
.iopoll = 1,
.async_size = sizeof(struct io_async_rw),
+ .blockable = 1,
.prep = io_prep_write_fixed,
.issue = io_write_fixed,
},
@@ -132,6 +137,7 @@ const struct io_issue_def io_issue_defs[] = {
[IORING_OP_SYNC_FILE_RANGE] = {
.needs_file = 1,
.audit_skip = 1,
+ .blockable = 1,
.prep = io_sfr_prep,
.issue = io_sync_file_range,
},
@@ -215,16 +221,19 @@ const struct io_issue_def io_issue_defs[] = {
[IORING_OP_FALLOCATE] = {
.needs_file = 1,
.hash_reg_file = 1,
+ .blockable = 1,
.prep = io_fallocate_prep,
.issue = io_fallocate,
},
[IORING_OP_OPENAT] = {
.filter_pdu_size = sizeof_field(struct io_uring_bpf_ctx, open),
+ .blockable = 1,
.prep = io_openat_prep,
.issue = io_openat,
.filter_populate = io_openat_bpf_populate,
},
[IORING_OP_CLOSE] = {
+ .blockable = 1,
.prep = io_close_prep,
.issue = io_close,
},
@@ -236,6 +245,7 @@ const struct io_issue_def io_issue_defs[] = {
},
[IORING_OP_STATX] = {
.audit_skip = 1,
+ .blockable = 1,
.prep = io_statx_prep,
.issue = io_statx,
},
@@ -249,6 +259,7 @@ const struct io_issue_def io_issue_defs[] = {
.ioprio = 1,
.iopoll = 1,
.async_size = sizeof(struct io_async_rw),
+ .blockable = 1,
.prep = io_prep_read,
.issue = io_read,
},
@@ -262,17 +273,20 @@ const struct io_issue_def io_issue_defs[] = {
.ioprio = 1,
.iopoll = 1,
.async_size = sizeof(struct io_async_rw),
+ .blockable = 1,
.prep = io_prep_write,
.issue = io_write,
},
[IORING_OP_FADVISE] = {
.needs_file = 1,
.audit_skip = 1,
+ .blockable = 1,
.prep = io_fadvise_prep,
.issue = io_fadvise,
},
[IORING_OP_MADVISE] = {
.audit_skip = 1,
+ .blockable = 1,
.prep = io_madvise_prep,
.issue = io_madvise,
},
@@ -308,6 +322,7 @@ const struct io_issue_def io_issue_defs[] = {
},
[IORING_OP_OPENAT2] = {
.filter_pdu_size = sizeof_field(struct io_uring_bpf_ctx, open),
+ .blockable = 1,
.prep = io_openat2_prep,
.issue = io_openat2,
.filter_populate = io_openat_bpf_populate,
@@ -327,6 +342,7 @@ const struct io_issue_def io_issue_defs[] = {
.hash_reg_file = 1,
.unbound_nonreg_file = 1,
.audit_skip = 1,
+ .blockable = 1,
.prep = io_splice_prep,
.issue = io_splice,
},
@@ -347,6 +363,7 @@ const struct io_issue_def io_issue_defs[] = {
.hash_reg_file = 1,
.unbound_nonreg_file = 1,
.audit_skip = 1,
+ .blockable = 1,
.prep = io_tee_prep,
.issue = io_tee,
},
@@ -360,22 +377,27 @@ const struct io_issue_def io_issue_defs[] = {
#endif
},
[IORING_OP_RENAMEAT] = {
+ .blockable = 1,
.prep = io_renameat_prep,
.issue = io_renameat,
},
[IORING_OP_UNLINKAT] = {
+ .blockable = 1,
.prep = io_unlinkat_prep,
.issue = io_unlinkat,
},
[IORING_OP_MKDIRAT] = {
+ .blockable = 1,
.prep = io_mkdirat_prep,
.issue = io_mkdirat,
},
[IORING_OP_SYMLINKAT] = {
+ .blockable = 1,
.prep = io_symlinkat_prep,
.issue = io_symlinkat,
},
[IORING_OP_LINKAT] = {
+ .blockable = 1,
.prep = io_linkat_prep,
.issue = io_linkat,
},
@@ -387,19 +409,23 @@ const struct io_issue_def io_issue_defs[] = {
},
[IORING_OP_FSETXATTR] = {
.needs_file = 1,
+ .blockable = 1,
.prep = io_fsetxattr_prep,
.issue = io_fsetxattr,
},
[IORING_OP_SETXATTR] = {
+ .blockable = 1,
.prep = io_setxattr_prep,
.issue = io_setxattr,
},
[IORING_OP_FGETXATTR] = {
.needs_file = 1,
+ .blockable = 1,
.prep = io_fgetxattr_prep,
.issue = io_fgetxattr,
},
[IORING_OP_GETXATTR] = {
+ .blockable = 1,
.prep = io_getxattr_prep,
.issue = io_getxattr,
},
@@ -497,6 +523,7 @@ const struct io_issue_def io_issue_defs[] = {
[IORING_OP_FTRUNCATE] = {
.needs_file = 1,
.hash_reg_file = 1,
+ .blockable = 1,
.prep = io_ftruncate_prep,
.issue = io_ftruncate,
},
@@ -553,6 +580,7 @@ const struct io_issue_def io_issue_defs[] = {
.iopoll = 1,
.vectored = 1,
.async_size = sizeof(struct io_async_rw),
+ .blockable = 1,
.prep = io_prep_readv_fixed,
.issue = io_read,
},
@@ -567,6 +595,7 @@ const struct io_issue_def io_issue_defs[] = {
.iopoll = 1,
.vectored = 1,
.async_size = sizeof(struct io_async_rw),
+ .blockable = 1,
.prep = io_prep_writev_fixed,
.issue = io_write,
},
diff --git a/io_uring/opdef.h b/io_uring/opdef.h
index 667f981e63b0..45c2f77cf782 100644
--- a/io_uring/opdef.h
+++ b/io_uring/opdef.h
@@ -29,6 +29,8 @@ struct io_issue_def {
unsigned vectored : 1;
/* set to 1 if this opcode uses 128b sqes in a mixed sq */
unsigned is_128 : 1;
+ /* issue path is safe to run inline in blocking mode */
+ unsigned blockable : 1;
/* size of async data needed, if any */
unsigned short async_size;
--
2.55.0
next prev parent reply other threads:[~2026-09-11 15:42 UTC|newest]
Thread overview: 18+ messages / expand[flat|nested] mbox.gz Atom feed top
2026-09-11 15:40 [RFC PATCH 00/15] io_uring: thread identity handoff for blocking inline issue Jens Axboe
2026-09-11 15:40 ` [PATCH 01/15] kernel: add thread identity handoff Jens Axboe
2026-09-11 15:40 ` [PATCH 02/15] sched: call into io_uring when a PF_IO_HANDOFF task blocks Jens Axboe
2026-09-11 15:40 ` [PATCH 03/15] arm64: implement thread identity handoff Jens Axboe
2026-09-11 15:40 ` [PATCH 04/15] x86: " Jens Axboe
2026-09-11 15:40 ` [PATCH 05/15] io_uring/kbuf: use io_ring_submit_unlock() helper Jens Axboe
2026-09-11 15:40 ` [PATCH 06/15] io_uring: keep the tctx nodes on a list Jens Axboe
2026-09-11 15:40 ` Jens Axboe [this message]
2026-09-11 15:40 ` [PATCH 08/15] io_uring: split io_uring_enter() and io_submit_sqes() into helpers Jens Axboe
2026-09-11 15:40 ` [PATCH 09/15] io_uring: keep the submission plug on the io_submit_sqes() stack Jens Axboe
2026-09-11 15:41 ` [PATCH 10/15] io-wq: support handing a task identity to an idle worker Jens Axboe
2026-09-11 15:41 ` [PATCH 11/15] io_uring: enable handing submitter identity to an io-wq worker Jens Axboe
2026-09-11 15:41 ` [PATCH 12/15] io_uring: defer the identity migration to the end of the submission Jens Axboe
2026-09-11 15:41 ` [PATCH 13/15] io_uring: issue blockable requests inline in blocking mode Jens Axboe
2026-09-11 15:41 ` [PATCH 14/15] io_uring: add tracepoints for the handoff operation Jens Axboe
2026-09-11 15:41 ` [PATCH 15/15] io_uring: issue IOSQE_ASYNC requests inline when a handoff is possible Jens Axboe
2026-09-11 17:33 ` [RFC PATCH 00/15] io_uring: thread identity handoff for blocking inline issue Gabriel Krisman Bertazi
2026-09-11 17:51 ` Jens Axboe
Reply instructions:
You may reply publicly to this message via plain-text email
using any one of the following methods:
* Save the following mbox file, import it into your mail client,
and reply-to-all from there: mbox
Avoid top-posting and favor interleaved quoting:
https://en.wikipedia.org/wiki/Posting_style#Interleaved_style
* Reply using the --to, --cc, and --in-reply-to
switches of git-send-email(1):
git send-email \
--in-reply-to=20260911154148.644489-8-axboe@kernel.dk \
--to=axboe@kernel.dk \
--cc=io-uring@vger.kernel.org \
--cc=linux-arm-kernel@lists.infradead.org \
--cc=linux-kernel@vger.kernel.org \
--cc=mingo@redhat.com \
--cc=peterz@infradead.org \
--cc=tglx@kernel.org \
/path/to/YOUR_REPLY
https://kernel.org/pub/software/scm/git/docs/git-send-email.html
* If your mail client supports setting the In-Reply-To header
via mailto: links, try the mailto: link
Be sure your reply has a Subject: header at the top and a blank line
before the message body.
This is a public inbox, see mirroring instructions
for how to clone and mirror all data and code used for this inbox