| lists.openwall.net | lists / announce owl-users owl-dev john-users john-dev passwdqc-users yescrypt popa3d-users / oss-security kernel-hardening musl sabotage tlsify passwords / crypt-dev xvendor / Bugtraq Full-Disclosure linux-kernel linux-netdev linux-ext4 linux-hardening linux-cve-announce PHC | |
|
Open Source and information security mailing list archives
| ||
|
Message-ID: <20251020-block-with-mmio-v2-2-147e9f93d8d4@nvidia.com>
Date: Mon, 20 Oct 2025 20:00:21 +0300
From: Leon Romanovsky <leon@...nel.org>
To: Jens Axboe <axboe@...nel.dk>,
Keith Busch <kbusch@...nel.org>,
Christoph Hellwig <hch@....de>,
Sagi Grimberg <sagi@...mberg.me>
Cc: linux-block@...r.kernel.org,
linux-kernel@...r.kernel.org,
linux-nvme@...ts.infradead.org
Subject: [PATCH v2 2/2] block-dma: properly take MMIO path
From: Leon Romanovsky <leonro@...dia.com>
In commit eadaa8b255f3 ("dma-mapping: introduce new DMA attribute to
indicate MMIO memory"), DMA_ATTR_MMIO attribute was added to describe
MMIO addresses, which require to avoid any memory cache flushing, as
an outcome of the discussion pointed in Link tag below.
In case of PCI_P2PDMA_MAP_THRU_HOST_BRIDGE transfer, blk-mq-dm logic
treated this as regular page and relied on "struct page" DMA flow.
That flow performs CPU cache flushing, which shouldn't be done here,
and doesn't set IOMMU_MMIO flag in DMA-IOMMU case.
Link: https://lore.kernel.org/all/f912c446-1ae9-4390-9c11-00dce7bf0fd3@arm.com/
Signed-off-by: Leon Romanovsky <leonro@...dia.com>
---
block/blk-mq-dma.c | 6 ++++--
drivers/nvme/host/pci.c | 23 +++++++++++++++++++++--
include/linux/blk-integrity.h | 7 ++++---
include/linux/blk-mq-dma.h | 11 +++++++----
4 files changed, 36 insertions(+), 11 deletions(-)
diff --git a/block/blk-mq-dma.c b/block/blk-mq-dma.c
index 4ba7b0323da4..3ede8022b41c 100644
--- a/block/blk-mq-dma.c
+++ b/block/blk-mq-dma.c
@@ -94,7 +94,7 @@ static bool blk_dma_map_direct(struct request *req, struct device *dma_dev,
struct blk_dma_iter *iter, struct phys_vec *vec)
{
iter->addr = dma_map_phys(dma_dev, vec->paddr, vec->len,
- rq_dma_dir(req), 0);
+ rq_dma_dir(req), iter->attrs);
if (dma_mapping_error(dma_dev, iter->addr)) {
iter->status = BLK_STS_RESOURCE;
return false;
@@ -116,7 +116,7 @@ static bool blk_rq_dma_map_iova(struct request *req, struct device *dma_dev,
do {
error = dma_iova_link(dma_dev, state, vec->paddr, mapped,
- vec->len, dir, 0);
+ vec->len, dir, iter->attrs);
if (error)
break;
mapped += vec->len;
@@ -184,6 +184,8 @@ static bool blk_dma_map_iter_start(struct request *req, struct device *dma_dev,
* P2P transfers through the host bridge are treated the
* same as non-P2P transfers below and during unmap.
*/
+ iter->attrs |= DMA_ATTR_MMIO;
+ fallthrough;
case PCI_P2PDMA_MAP_NONE:
break;
default:
diff --git a/drivers/nvme/host/pci.c b/drivers/nvme/host/pci.c
index 91a8965754f0..f45d1968611d 100644
--- a/drivers/nvme/host/pci.c
+++ b/drivers/nvme/host/pci.c
@@ -260,6 +260,12 @@ enum nvme_iod_flags {
/* single segment dma mapping */
IOD_SINGLE_SEGMENT = 1U << 2,
+ /* Data payload contains MMIO memory */
+ IOD_DATA_MMIO = 1U << 3,
+
+ /* Metadata contains MMIO memory */
+ IOD_META_MMIO = 1U << 4,
+
/* Metadata using non-coalesced MPTR */
IOD_SINGLE_META_SEGMENT = 1U << 5,
};
@@ -733,8 +739,11 @@ static void nvme_unmap_metadata(struct request *req)
return;
}
+ if (iod->flags & IOD_META_MMIO)
+ attrs |= DMA_ATTR_MMIO;
+
if (!blk_rq_integrity_dma_unmap(req, dma_dev, &iod->meta_dma_state,
- iod->meta_total_len)) {
+ iod->meta_total_len, attrs)) {
if (nvme_pci_cmd_use_meta_sgl(&iod->cmd))
nvme_free_sgls(req, sge, &sge[1], attrs);
else
@@ -762,7 +771,11 @@ static void nvme_unmap_data(struct request *req)
return;
}
- if (!blk_rq_dma_unmap(req, dma_dev, &iod->dma_state, iod->total_len)) {
+ if (iod->flags & IOD_DATA_MMIO)
+ attrs |= DMA_ATTR_MMIO;
+
+ if (!blk_rq_dma_unmap(req, dma_dev, &iod->dma_state, iod->total_len,
+ attrs)) {
if (nvme_pci_cmd_use_sgl(&iod->cmd))
nvme_free_sgls(req, iod->descriptors[0],
&iod->cmd.common.dptr.sgl, attrs);
@@ -1038,6 +1051,9 @@ static blk_status_t nvme_map_data(struct request *req)
if (!blk_rq_dma_map_iter_start(req, dev->dev, &iod->dma_state, &iter))
return iter.status;
+ if (iter.attrs & DMA_ATTR_MMIO)
+ iod->flags |= IOD_DATA_MMIO;
+
if (use_sgl == SGL_FORCED ||
(use_sgl == SGL_SUPPORTED &&
(sgl_threshold && nvme_pci_avg_seg_size(req) >= sgl_threshold)))
@@ -1060,6 +1076,9 @@ static blk_status_t nvme_pci_setup_meta_sgls(struct request *req)
&iod->meta_dma_state, &iter))
return iter.status;
+ if (iter.attrs & DMA_ATTR_MMIO)
+ iod->flags |= IOD_META_MMIO;
+
if (blk_rq_dma_map_coalesce(&iod->meta_dma_state))
entries = 1;
diff --git a/include/linux/blk-integrity.h b/include/linux/blk-integrity.h
index b659373788f6..aa42172f5cc9 100644
--- a/include/linux/blk-integrity.h
+++ b/include/linux/blk-integrity.h
@@ -30,10 +30,11 @@ int blk_rq_map_integrity_sg(struct request *, struct scatterlist *);
static inline bool blk_rq_integrity_dma_unmap(struct request *req,
struct device *dma_dev, struct dma_iova_state *state,
- size_t mapped_len)
+ size_t mapped_len, unsigned int attrs)
{
return blk_dma_unmap(req, dma_dev, state, mapped_len,
- bio_integrity(req->bio)->bip_flags & BIP_P2P_DMA);
+ bio_integrity(req->bio)->bip_flags & BIP_P2P_DMA,
+ attrs);
}
int blk_rq_count_integrity_sg(struct request_queue *, struct bio *);
@@ -126,7 +127,7 @@ static inline int blk_rq_map_integrity_sg(struct request *q,
}
static inline bool blk_rq_integrity_dma_unmap(struct request *req,
struct device *dma_dev, struct dma_iova_state *state,
- size_t mapped_len)
+ size_t mapped_len, unsigned int attrs)
{
return false;
}
diff --git a/include/linux/blk-mq-dma.h b/include/linux/blk-mq-dma.h
index faf4dd574c62..aab4d04e6c69 100644
--- a/include/linux/blk-mq-dma.h
+++ b/include/linux/blk-mq-dma.h
@@ -50,19 +50,21 @@ static inline bool blk_rq_dma_map_coalesce(struct dma_iova_state *state)
* @state: DMA IOVA state
* @mapped_len: number of bytes to unmap
* @is_p2p: true if mapped with PCI_P2PDMA_MAP_BUS_ADDR
+ * @attrs: DMA attributes
*
* Returns %false if the callers need to manually unmap every DMA segment
* mapped using @iter or %true if no work is left to be done.
*/
static inline bool blk_dma_unmap(struct request *req, struct device *dma_dev,
- struct dma_iova_state *state, size_t mapped_len, bool is_p2p)
+ struct dma_iova_state *state, size_t mapped_len, bool is_p2p,
+ unsigned int attrs)
{
if (is_p2p)
return true;
if (dma_use_iova(state)) {
dma_iova_destroy(dma_dev, state, mapped_len, rq_dma_dir(req),
- 0);
+ attrs);
return true;
}
@@ -70,10 +72,11 @@ static inline bool blk_dma_unmap(struct request *req, struct device *dma_dev,
}
static inline bool blk_rq_dma_unmap(struct request *req, struct device *dma_dev,
- struct dma_iova_state *state, size_t mapped_len)
+ struct dma_iova_state *state, size_t mapped_len,
+ unsigned int attrs)
{
return blk_dma_unmap(req, dma_dev, state, mapped_len,
- req->cmd_flags & REQ_P2PDMA);
+ req->cmd_flags & REQ_P2PDMA, attrs);
}
#endif /* BLK_MQ_DMA_H */
--
2.51.0
Powered by blists - more mailing lists