diff --git a/drivers/block/nvme-core.c b/drivers/block/nvme-core.c index 22d8013cd4ff..fd3f11d837bb 100644 --- a/drivers/block/nvme-core.c +++ b/drivers/block/nvme-core.c @@ -357,17 +357,22 @@ static void req_completion(struct nvme_queue *nvmeq, void *ctx, u16 status = le16_to_cpup(&cqe->status) >> 1; + if (unlikely(status)) { + if (!(status & NVME_SC_DNR || blk_noretry_request(req)) + && (jiffies - req->start_time) < req->timeout) { + blk_mq_requeue_request(req); + return; + } + req->errors = -EIO; + } else + req->errors = 0; + if (iod->nents) { dma_unmap_sg(&nvmeq->dev->pci_dev->dev, iod->sg, iod->nents, rq_data_dir(req) ? DMA_TO_DEVICE : DMA_FROM_DEVICE); } nvme_free_iod(nvmeq->dev, iod); - if (unlikely(status)) - req->errors = -EIO; - else - req->errors = 0; - blk_mq_complete_request(req); } @@ -569,11 +574,19 @@ static int nvme_submit_req_queue(struct nvme_queue *nvmeq, struct nvme_ns *ns, enum dma_data_direction dma_dir; int psegs = req->nr_phys_segments; + /* + * Requeued IO has already been prepped + */ + iod = req->special; + if (iod) + goto submit_iod; + iod = nvme_alloc_iod(psegs, blk_rq_bytes(req), GFP_ATOMIC); if (!iod) return BLK_MQ_RQ_QUEUE_BUSY; iod->private = req; + req->special = iod; nvme_set_info(cmd, iod, req_completion); @@ -602,6 +615,7 @@ static int nvme_submit_req_queue(struct nvme_queue *nvmeq, struct nvme_ns *ns, goto finish_cmd; } + submit_iod: if (!nvme_submit_iod(nvmeq, iod, ns)) return 0;