From 0de2836ce21df6801db580e154296544a741b6c4 Mon Sep 17 00:00:00 2001 From: Max Gurtovoy Date: Thu, 2 Dec 2021 19:59:00 +0200 Subject: [PATCH 1/2] nvme-rdma: prepare for queue_rqs implementation Signed-off-by: Max Gurtovoy --- drivers/nvme/host/rdma.c | 127 ++++++++++++++++++++++++++------------- 1 file changed, 84 insertions(+), 43 deletions(-) diff --git a/drivers/nvme/host/rdma.c b/drivers/nvme/host/rdma.c index 850f84d204d0..2d608cb48392 100644 --- a/drivers/nvme/host/rdma.c +++ b/drivers/nvme/host/rdma.c @@ -70,6 +70,7 @@ struct nvme_rdma_request { struct ib_sge sge[1 + NVME_RDMA_MAX_INLINE_SEGMENTS]; u32 num_sge; struct ib_reg_wr reg_wr; + struct ib_send_wr send_wr; struct ib_cqe reg_cqe; struct nvme_rdma_queue *queue; struct nvme_rdma_sgl data_sgl; @@ -1635,33 +1636,31 @@ static void nvme_rdma_send_done(struct ib_cq *cq, struct ib_wc *wc) nvme_rdma_end_request(req); } -static int nvme_rdma_post_send(struct nvme_rdma_queue *queue, +static void nvme_rdma_prep_send(struct nvme_rdma_queue *queue, struct nvme_rdma_qe *qe, struct ib_sge *sge, u32 num_sge, - struct ib_send_wr *first) + struct ib_send_wr *wr) { - struct ib_send_wr wr; - int ret; - - sge->addr = qe->dma; + sge->addr = qe->dma; sge->length = sizeof(struct nvme_command); - sge->lkey = queue->device->pd->local_dma_lkey; + sge->lkey = queue->device->pd->local_dma_lkey; - wr.next = NULL; - wr.wr_cqe = &qe->cqe; - wr.sg_list = sge; - wr.num_sge = num_sge; - wr.opcode = IB_WR_SEND; - wr.send_flags = IB_SEND_SIGNALED; + wr->next = NULL; + wr->wr_cqe = &qe->cqe; + wr->sg_list = sge; + wr->num_sge = num_sge; + wr->opcode = IB_WR_SEND; + wr->send_flags = IB_SEND_SIGNALED; +} - if (first) - first->next = ≀ - else - first = ≀ +static int nvme_rdma_post_send(struct nvme_rdma_queue *queue, + struct ib_send_wr *wr) +{ + int ret; - ret = ib_post_send(queue->qp, first, NULL); + ret = ib_post_send(queue->qp, wr, NULL); if (unlikely(ret)) { dev_err(queue->ctrl->ctrl.device, - "%s failed with error code %d\n", __func__, ret); + "%s failed with error code %d\n", __func__, ret); } return ret; } @@ -1715,6 +1714,7 @@ static void nvme_rdma_submit_async_event(struct nvme_ctrl *arg) struct nvme_rdma_qe *sqe = &ctrl->async_event_sqe; struct nvme_command *cmd = sqe->data; struct ib_sge sge; + struct ib_send_wr wr; int ret; ib_dma_sync_single_for_cpu(dev, sqe->dma, sizeof(*cmd), DMA_TO_DEVICE); @@ -1730,7 +1730,8 @@ static void nvme_rdma_submit_async_event(struct nvme_ctrl *arg) ib_dma_sync_single_for_device(dev, sqe->dma, sizeof(*cmd), DMA_TO_DEVICE); - ret = nvme_rdma_post_send(queue, sqe, &sge, 1, NULL); + nvme_rdma_prep_send(queue, sqe, &sge, 1, &wr); + ret = nvme_rdma_post_send(queue, &wr); WARN_ON_ONCE(ret); } @@ -2034,27 +2035,35 @@ nvme_rdma_timeout(struct request *rq, bool reserved) return BLK_EH_RESET_TIMER; } -static blk_status_t nvme_rdma_queue_rq(struct blk_mq_hw_ctx *hctx, - const struct blk_mq_queue_data *bd) +static blk_status_t nvme_rdma_cleanup_rq(struct nvme_rdma_queue *queue, + struct request *rq, int err) { - struct nvme_ns *ns = hctx->queue->queuedata; - struct nvme_rdma_queue *queue = hctx->driver_data; - struct request *rq = bd->rq; + struct nvme_rdma_request *req = blk_mq_rq_to_pdu(rq); + blk_status_t ret; + + nvme_rdma_unmap_data(queue, rq); + if (err == -EIO) + ret = nvme_host_path_error(rq); + else if (err == -ENOMEM || err == -EAGAIN) + ret = BLK_STS_RESOURCE; + else + ret = BLK_STS_IOERR; + nvme_cleanup_cmd(rq); + ib_dma_unmap_single(queue->device->dev, req->sqe.dma, + sizeof(struct nvme_command), DMA_TO_DEVICE); + return ret; +} + +static blk_status_t nvme_rdma_prep_rq(struct nvme_rdma_queue *queue, + struct request *rq, struct nvme_ns *ns) +{ + struct ib_device *dev = queue->device->dev; struct nvme_rdma_request *req = blk_mq_rq_to_pdu(rq); struct nvme_rdma_qe *sqe = &req->sqe; struct nvme_command *c = nvme_req(rq)->cmd; - struct ib_device *dev; - bool queue_ready = test_bit(NVME_RDMA_Q_LIVE, &queue->flags); blk_status_t ret; int err; - WARN_ON_ONCE(rq->tag < 0); - - if (!nvme_check_ready(&queue->ctrl->ctrl, rq, queue_ready)) - return nvme_fail_nonready_command(&queue->ctrl->ctrl, rq); - - dev = queue->device->dev; - req->sqe.dma = ib_dma_map_single(dev, req->sqe.data, sizeof(struct nvme_command), DMA_TO_DEVICE); @@ -2083,8 +2092,8 @@ static blk_status_t nvme_rdma_queue_rq(struct blk_mq_hw_ctx *hctx, err = nvme_rdma_map_data(queue, rq, c); if (unlikely(err < 0)) { dev_err(queue->ctrl->ctrl.device, - "Failed to map data (%d)\n", err); - goto err; + "Failed to map data (%d)\n", err); + goto out_err; } sqe->cqe.done = nvme_rdma_send_done; @@ -2092,16 +2101,13 @@ static blk_status_t nvme_rdma_queue_rq(struct blk_mq_hw_ctx *hctx, ib_dma_sync_single_for_device(dev, sqe->dma, sizeof(struct nvme_command), DMA_TO_DEVICE); - err = nvme_rdma_post_send(queue, sqe, req->sge, req->num_sge, - req->mr ? &req->reg_wr.wr : NULL); - if (unlikely(err)) - goto err_unmap; + nvme_rdma_prep_send(queue, sqe, req->sge, req->num_sge, &req->send_wr); + if (req->mr) + req->reg_wr.wr.next = &req->send_wr; return BLK_STS_OK; -err_unmap: - nvme_rdma_unmap_data(queue, rq); -err: +out_err: if (err == -EIO) ret = nvme_host_path_error(rq); else if (err == -ENOMEM || err == -EAGAIN) @@ -2115,6 +2121,41 @@ static blk_status_t nvme_rdma_queue_rq(struct blk_mq_hw_ctx *hctx, return ret; } +static blk_status_t nvme_rdma_queue_rq(struct blk_mq_hw_ctx *hctx, + const struct blk_mq_queue_data *bd) +{ + struct nvme_ns *ns = hctx->queue->queuedata; + struct nvme_rdma_queue *queue = hctx->driver_data; + struct request *rq = bd->rq; + struct nvme_rdma_request *req = blk_mq_rq_to_pdu(rq); + bool queue_ready = test_bit(NVME_RDMA_Q_LIVE, &queue->flags); + struct ib_send_wr *wr; + int err; + + WARN_ON_ONCE(rq->tag < 0); + + if (!nvme_check_ready(&queue->ctrl->ctrl, rq, queue_ready)) + return nvme_fail_nonready_command(&queue->ctrl->ctrl, rq); + + err = nvme_rdma_prep_rq(queue, rq, ns); + if (unlikely(err)) + return err; + + if (req->mr) + wr = &req->reg_wr.wr; + else + wr = &req->send_wr; + + err = nvme_rdma_post_send(queue, wr); + if (unlikely(err)) + goto out_cleanup_rq; + + return BLK_STS_OK; + +out_cleanup_rq: + return nvme_rdma_cleanup_rq(queue, rq, err); +} + static int nvme_rdma_poll(struct blk_mq_hw_ctx *hctx, struct io_comp_batch *iob) { struct nvme_rdma_queue *queue = hctx->driver_data; -- 2.18.1