From 0bc78c843b8636dcdfe45dd07328ca826fa67f9b Mon Sep 17 00:00:00 2001 From: Jens Axboe Date: Fri, 2 Sep 2022 15:17:30 -0600 Subject: [PATCH 2/3] nvme: use separate end IO handler for IOPOLL Don't need to rely on the cookie or request type, set the right handler based on how we're handling the IO. Signed-off-by: Jens Axboe --- drivers/nvme/host/ioctl.c | 30 ++++++++++++++++++++++-------- 1 file changed, 22 insertions(+), 8 deletions(-) diff --git a/drivers/nvme/host/ioctl.c b/drivers/nvme/host/ioctl.c index 7756b439a688..f34abe95821e 100644 --- a/drivers/nvme/host/ioctl.c +++ b/drivers/nvme/host/ioctl.c @@ -385,25 +385,36 @@ static void nvme_uring_task_cb(struct io_uring_cmd *ioucmd) io_uring_cmd_done(ioucmd, status, result); } -static void nvme_uring_cmd_end_io(struct request *req, blk_status_t err) +static void nvme_uring_iopoll_cmd_end_io(struct request *req, blk_status_t err) { struct io_uring_cmd *ioucmd = req->end_io_data; struct nvme_uring_cmd_pdu *pdu = nvme_uring_cmd_pdu(ioucmd); /* extract bio before reusing the same field for request */ struct bio *bio = pdu->bio; - void *cookie = READ_ONCE(ioucmd->cookie); pdu->req = req; req->bio = bio; /* * For iopoll, complete it directly. - * Otherwise, move the completion to task work. */ - if (cookie != NULL && blk_rq_is_poll(req)) - nvme_uring_task_cb(ioucmd); - else - io_uring_cmd_complete_in_task(ioucmd, nvme_uring_task_cb); + nvme_uring_task_cb(ioucmd); +} + +static void nvme_uring_cmd_end_io(struct request *req, blk_status_t err) +{ + struct io_uring_cmd *ioucmd = req->end_io_data; + struct nvme_uring_cmd_pdu *pdu = nvme_uring_cmd_pdu(ioucmd); + /* extract bio before reusing the same field for request */ + struct bio *bio = pdu->bio; + + pdu->req = req; + req->bio = bio; + + /* + * Move the completion to task work. + */ + io_uring_cmd_complete_in_task(ioucmd, nvme_uring_task_cb); } static int nvme_uring_cmd_io(struct nvme_ctrl *ctrl, struct nvme_ns *ns, @@ -464,7 +475,10 @@ static int nvme_uring_cmd_io(struct nvme_ctrl *ctrl, struct nvme_ns *ns, blk_flags); if (IS_ERR(req)) return PTR_ERR(req); - req->end_io = nvme_uring_cmd_end_io; + if (issue_flags & IO_URING_F_IOPOLL) + req->end_io = nvme_uring_iopoll_cmd_end_io; + else + req->end_io = nvme_uring_cmd_end_io; req->end_io_data = ioucmd; if (issue_flags & IO_URING_F_IOPOLL && rq_flags & REQ_POLLED) { -- 2.35.1