Re: [RFC v2 05/11] block: add infra to handle dmabuf tokens

[Nitesh Shetty <nj.shetty@samsung.com>]

[-- Attachment #1: Type: text/plain, Size: 10234 bytes --]

On 23/11/25 10:51PM, Pavel Begunkov wrote:
>Add blk-mq infrastructure to handle dmabuf tokens. There are two main
>objects. The first is struct blk_mq_dma_token, which is an extension of
>struct dma_token and passed in an iterator. The second is struct
>blk_mq_dma_map, which keeps the actual mapping and unlike the token, can
>be ejected (e.g. by move_notify) and recreated.
>
>The token keeps an rcu protected pointer to the mapping, so when it
>resolves a token into a mapping to pass it to a request, it'll do an rcu
>protected lookup and get a percpu reference to the mapping.
>
>If there is no current mapping attached to a token, it'll need to be
>created by calling the driver (e.g. nvme) via a new callback. It
>requires waiting, thefore can't be done for nowait requests and couldn't
>happen deeper in the stack, e.g. during nvme request submission.
>
>The structure split is needed because move_notify can request to
>invalidate the dma mapping at any moment, and we need a way to
>concurrently remove it and wait for the inflight requests using the
>previous mapping to complete.
>
>Signed-off-by: Pavel Begunkov <asml.silence@gmail.com>
>---
> block/Makefile                   |   1 +
> block/bdev.c                     |  14 ++
> block/blk-mq-dma-token.c         | 236 +++++++++++++++++++++++++++++++
> block/blk-mq.c                   |  20 +++
> block/fops.c                     |   1 +
> include/linux/blk-mq-dma-token.h |  60 ++++++++
> include/linux/blk-mq.h           |  21 +++
> include/linux/blkdev.h           |   3 +
> 8 files changed, 356 insertions(+)
> create mode 100644 block/blk-mq-dma-token.c
> create mode 100644 include/linux/blk-mq-dma-token.h
>
>diff --git a/block/Makefile b/block/Makefile
>index c65f4da93702..0190e5aa9f00 100644
>--- a/block/Makefile
>+++ b/block/Makefile
>@@ -36,3 +36,4 @@ obj-$(CONFIG_BLK_INLINE_ENCRYPTION)	+= blk-crypto.o blk-crypto-profile.o \
> 					   blk-crypto-sysfs.o
> obj-$(CONFIG_BLK_INLINE_ENCRYPTION_FALLBACK)	+= blk-crypto-fallback.o
> obj-$(CONFIG_BLOCK_HOLDER_DEPRECATED)	+= holder.o
>+obj-$(CONFIG_DMA_SHARED_BUFFER) += blk-mq-dma-token.o
>diff --git a/block/bdev.c b/block/bdev.c
>index 810707cca970..da89d20f33f3 100644
>--- a/block/bdev.c
>+++ b/block/bdev.c
>@@ -28,6 +28,7 @@
> #include <linux/part_stat.h>
> #include <linux/uaccess.h>
> #include <linux/stat.h>
>+#include <linux/blk-mq-dma-token.h>
> #include "../fs/internal.h"
> #include "blk.h"
>
>@@ -61,6 +62,19 @@ struct block_device *file_bdev(struct file *bdev_file)
> }
> EXPORT_SYMBOL(file_bdev);
>
>+struct dma_token *blkdev_dma_map(struct file *file,
>+				 struct dma_token_params *params)
>+{
>+	struct request_queue *q = bdev_get_queue(file_bdev(file));
>+
>+	if (!(file->f_flags & O_DIRECT))
>+		return ERR_PTR(-EINVAL);
>+	if (!q->mq_ops)
>+		return ERR_PTR(-EINVAL);
>+
>+	return blk_mq_dma_map(q, params);
>+}
>+
> static void bdev_write_inode(struct block_device *bdev)
> {
> 	struct inode *inode = BD_INODE(bdev);
>diff --git a/block/blk-mq-dma-token.c b/block/blk-mq-dma-token.c
>new file mode 100644
>index 000000000000..cd62c4d09422
>--- /dev/null
>+++ b/block/blk-mq-dma-token.c
>@@ -0,0 +1,236 @@
>+#include <linux/blk-mq-dma-token.h>
>+#include <linux/dma-resv.h>
>+
>+struct blk_mq_dma_fence {
>+	struct dma_fence base;
>+	spinlock_t lock;
>+};
>+
>+static const char *blk_mq_fence_drv_name(struct dma_fence *fence)
>+{
>+	return "blk-mq";
>+}
>+
>+const struct dma_fence_ops blk_mq_dma_fence_ops = {
>+	.get_driver_name = blk_mq_fence_drv_name,
>+	.get_timeline_name = blk_mq_fence_drv_name,
>+};
>+
>+static void blk_mq_dma_token_free(struct blk_mq_dma_token *token)
>+{
>+	token->q->mq_ops->clean_dma_token(token->q, token);
>+	dma_buf_put(token->dmabuf);
>+	kfree(token);
>+}
>+
>+static inline void blk_mq_dma_token_put(struct blk_mq_dma_token *token)
>+{
>+	if (refcount_dec_and_test(&token->refs))
>+		blk_mq_dma_token_free(token);
>+}
>+
>+static void blk_mq_dma_mapping_free(struct blk_mq_dma_map *map)
>+{
>+	struct blk_mq_dma_token *token = map->token;
>+
>+	if (map->sgt)
>+		token->q->mq_ops->dma_unmap(token->q, map);
>+
>+	dma_fence_put(&map->fence->base);
>+	percpu_ref_exit(&map->refs);
>+	kfree(map);
>+	blk_mq_dma_token_put(token);
>+}
>+
>+static void blk_mq_dma_map_work_free(struct work_struct *work)
>+{
>+	struct blk_mq_dma_map *map = container_of(work, struct blk_mq_dma_map,
>+						free_work);
>+
>+	dma_fence_signal(&map->fence->base);
>+	blk_mq_dma_mapping_free(map);
>+}
>+
>+static void blk_mq_dma_map_refs_free(struct percpu_ref *ref)
>+{
>+	struct blk_mq_dma_map *map = container_of(ref, struct blk_mq_dma_map, refs);
>+
>+	INIT_WORK(&map->free_work, blk_mq_dma_map_work_free);
>+	queue_work(system_wq, &map->free_work);
>+}
>+
>+static struct blk_mq_dma_map *blk_mq_alloc_dma_mapping(struct blk_mq_dma_token *token)
>+{
>+	struct blk_mq_dma_fence *fence = NULL;
>+	struct blk_mq_dma_map *map;
>+	int ret = -ENOMEM;
>+
>+	map = kzalloc(sizeof(*map), GFP_KERNEL);
>+	if (!map)
>+		return ERR_PTR(-ENOMEM);
>+
>+	fence = kzalloc(sizeof(*fence), GFP_KERNEL);
>+	if (!fence)
>+		goto err;
>+
>+	ret = percpu_ref_init(&map->refs, blk_mq_dma_map_refs_free, 0,
>+			      GFP_KERNEL);
>+	if (ret)
>+		goto err;
>+
>+	dma_fence_init(&fence->base, &blk_mq_dma_fence_ops, &fence->lock,
>+			token->fence_ctx, atomic_inc_return(&token->fence_seq));
>+	spin_lock_init(&fence->lock);
>+	map->fence = fence;
>+	map->token = token;
>+	refcount_inc(&token->refs);
>+	return map;
>+err:
>+	kfree(map);
>+	kfree(fence);
>+	return ERR_PTR(ret);
>+}
>+
>+static inline
>+struct blk_mq_dma_map *blk_mq_get_token_map(struct blk_mq_dma_token *token)
>+{
>+	struct blk_mq_dma_map *map;
>+
>+	guard(rcu)();
>+
>+	map = rcu_dereference(token->map);
>+	if (unlikely(!map || !percpu_ref_tryget_live_rcu(&map->refs)))
>+		return NULL;
>+	return map;
>+}
>+
>+static struct blk_mq_dma_map *
>+blk_mq_create_dma_map(struct blk_mq_dma_token *token)
>+{
>+	struct dma_buf *dmabuf = token->dmabuf;
>+	struct blk_mq_dma_map *map;
>+	long ret;
>+
>+	guard(mutex)(&token->mapping_lock);
>+
>+	map = blk_mq_get_token_map(token);
>+	if (map)
>+		return map;
>+
>+	map = blk_mq_alloc_dma_mapping(token);
>+	if (IS_ERR(map))
>+		return NULL;
>+
>+	dma_resv_lock(dmabuf->resv, NULL);
>+	ret = dma_resv_wait_timeout(dmabuf->resv, DMA_RESV_USAGE_BOOKKEEP,
>+				    true, MAX_SCHEDULE_TIMEOUT);
>+	ret = ret ? ret : -ETIME;
>+	if (ret > 0)
>+		ret = token->q->mq_ops->dma_map(token->q, map);
>+	dma_resv_unlock(dmabuf->resv);
>+
>+	if (ret)
>+		return ERR_PTR(ret);
>+
>+	percpu_ref_get(&map->refs);
>+	rcu_assign_pointer(token->map, map);
>+	return map;
>+}
>+
>+static void blk_mq_dma_map_remove(struct blk_mq_dma_token *token)
>+{
>+	struct dma_buf *dmabuf = token->dmabuf;
>+	struct blk_mq_dma_map *map;
>+	int ret;
>+
>+	dma_resv_assert_held(dmabuf->resv);
>+
>+	ret = dma_resv_reserve_fences(dmabuf->resv, 1);
>+	if (WARN_ON_ONCE(ret))
>+		return;
>+
>+	map = rcu_dereference_protected(token->map,
>+					dma_resv_held(dmabuf->resv));
>+	if (!map)
>+		return;
>+	rcu_assign_pointer(token->map, NULL);
>+
>+	dma_resv_add_fence(dmabuf->resv, &map->fence->base,
>+			   DMA_RESV_USAGE_KERNEL);
>+	percpu_ref_kill(&map->refs);
>+}
>+
>+blk_status_t blk_rq_assign_dma_map(struct request *rq,
>+				   struct blk_mq_dma_token *token)
>+{
>+	struct blk_mq_dma_map *map;
>+
>+	map = blk_mq_get_token_map(token);
>+	if (map)
>+		goto complete;
>+
>+	if (rq->cmd_flags & REQ_NOWAIT)
>+		return BLK_STS_AGAIN;
>+
>+	map = blk_mq_create_dma_map(token);
>+	if (IS_ERR(map))
>+		return BLK_STS_RESOURCE;
>+complete:
>+	rq->dma_map = map;
>+	return BLK_STS_OK;
>+}
>+
>+void blk_mq_dma_map_move_notify(struct blk_mq_dma_token *token)
>+{
>+	blk_mq_dma_map_remove(token);
>+}
>+
>+static void blk_mq_release_dma_mapping(struct dma_token *base_token)
>+{
>+	struct blk_mq_dma_token *token = dma_token_to_blk_mq(base_token);
>+	struct dma_buf *dmabuf = token->dmabuf;
>+
>+	dma_resv_lock(dmabuf->resv, NULL);
>+	blk_mq_dma_map_remove(token);
>+	dma_resv_unlock(dmabuf->resv);
>+
>+	blk_mq_dma_token_put(token);
>+}
>+
>+struct dma_token *blk_mq_dma_map(struct request_queue *q,
>+				  struct dma_token_params *params)
>+{
>+	struct dma_buf *dmabuf = params->dmabuf;
>+	struct blk_mq_dma_token *token;
>+	int ret;
>+
>+	if (!q->mq_ops->dma_map || !q->mq_ops->dma_unmap ||
>+	    !q->mq_ops->init_dma_token || !q->mq_ops->clean_dma_token)
>+		return ERR_PTR(-EINVAL);
>+
>+	token = kzalloc(sizeof(*token), GFP_KERNEL);
>+	if (!token)
>+		return ERR_PTR(-ENOMEM);
>+
>+	get_dma_buf(dmabuf);
>+	token->fence_ctx = dma_fence_context_alloc(1);
>+	token->dmabuf = dmabuf;
>+	token->dir = params->dir;
>+	token->base.release = blk_mq_release_dma_mapping;
>+	token->q = q;
>+	refcount_set(&token->refs, 1);
>+	mutex_init(&token->mapping_lock);
>+
>+	if (!blk_get_queue(q)) {
>+		kfree(token);
>+		return ERR_PTR(-EFAULT);
>+	}
>+
>+	ret = token->q->mq_ops->init_dma_token(token->q, token);
>+	if (ret) {
>+		kfree(token);
>+		blk_put_queue(q);
>+		return ERR_PTR(ret);
>+	}
>+	return &token->base;
>+}
>diff --git a/block/blk-mq.c b/block/blk-mq.c
>index f2650c97a75e..1ff3a7e3191b 100644
>--- a/block/blk-mq.c
>+++ b/block/blk-mq.c
>@@ -29,6 +29,7 @@
> #include <linux/blk-crypto.h>
> #include <linux/part_stat.h>
> #include <linux/sched/isolation.h>
>+#include <linux/blk-mq-dma-token.h>
>
> #include <trace/events/block.h>
>
>@@ -439,6 +440,7 @@ static struct request *blk_mq_rq_ctx_init(struct blk_mq_alloc_data *data,
> 	rq->nr_integrity_segments = 0;
> 	rq->end_io = NULL;
> 	rq->end_io_data = NULL;
>+	rq->dma_map = NULL;
>
> 	blk_crypto_rq_set_defaults(rq);
> 	INIT_LIST_HEAD(&rq->queuelist);
>@@ -794,6 +796,7 @@ static void __blk_mq_free_request(struct request *rq)
> 	blk_pm_mark_last_busy(rq);
> 	rq->mq_hctx = NULL;
>
>+	blk_rq_drop_dma_map(rq);
blk_rq_drop_dma_map(rq), needs to be added in blk_mq_end_request_batch
as well[1], otherwise I am seeing we leave with increased reference
count in dma-buf exporter side.

Thanks,
Nitesh

[1]
--- a/block/blk-mq.c
+++ b/block/blk-mq.c
@@ -1214,6 +1214,7 @@ void blk_mq_end_request_batch(struct io_comp_batch *iob)

                  blk_crypto_free_request(rq);
                  blk_pm_mark_last_busy(rq);
+               blk_rq_drop_dma_map(rq);

[-- Attachment #2: Type: text/plain, Size: 0 bytes --]