| lists.openwall.net | lists / announce owl-users owl-dev john-users john-dev passwdqc-users yescrypt popa3d-users / oss-security kernel-hardening musl sabotage tlsify passwords / crypt-dev xvendor / Bugtraq Full-Disclosure linux-kernel linux-netdev linux-ext4 linux-hardening linux-cve-announce PHC | |
|
Open Source and information security mailing list archives
| ||
|
Message-Id: <1265834250-29170-3-git-send-email-adharmap@codeaurora.org>
Date: Wed, 10 Feb 2010 12:37:30 -0800
From: adharmap@...eaurora.org
To: linux-arm-kernel@...ts.infradead.org
Cc: linux-arm-msm@...r.kernel.org,
Russell King <linux@....linux.org.uk>,
linux-kernel@...r.kernel.org,
Catalin Marinas <catalin.marinas@....com>,
Abhijeet Dharmapurikar <adharmap@...cinc.com>
Subject: [PATCH 2/2] dma: fix scatter-gather api to use barrierless map/unmap functions
From: Abhijeet Dharmapurikar <adharmap@...cinc.com>
dma_map/unmap_sg need to execute barrier only after the last buffer has been
mapped/unmapped. This imporves performance in situations where multiple
buffers need to be mapped for a single DMA operation.
Signed-off-by: Abhijeet Dharmapurikar <adharmap@...cinc.com>
---
arch/arm/include/asm/dma-mapping.h | 87 ++++++++++++++++++++++++++++++++++++
arch/arm/mm/dma-mapping.c | 59 +++++++++++++++++++++---
2 files changed, 139 insertions(+), 7 deletions(-)
diff --git a/arch/arm/include/asm/dma-mapping.h b/arch/arm/include/asm/dma-mapping.h
index 256ee1c..06b528d 100644
--- a/arch/arm/include/asm/dma-mapping.h
+++ b/arch/arm/include/asm/dma-mapping.h
@@ -110,6 +110,26 @@ static inline void __dma_page_dev_to_cpu(struct page *page, unsigned long off,
___dma_page_dev_to_cpu(page, off, size, dir);
}
+static inline void __dma_page_cpu_to_dev_nobarrier(struct page *page,
+ unsigned long off, size_t size, enum dma_data_direction dir)
+{
+ extern void ___dma_page_cpu_to_dev_nobarrier(struct page *,
+ unsigned long, size_t, enum dma_data_direction);
+
+ if (!arch_is_coherent())
+ ___dma_page_cpu_to_dev_nobarrier(page, off, size, dir);
+}
+
+static inline void __dma_page_dev_to_cpu_nobarrier(struct page *page,
+ unsigned long off, size_t size, enum dma_data_direction dir)
+{
+ extern void ___dma_page_dev_to_cpu_nobarrier(struct page *,
+ unsigned long, size_t, enum dma_data_direction);
+
+ if (!arch_is_coherent())
+ ___dma_page_dev_to_cpu_nobarrier(page, off, size, dir);
+}
+
/*
* Return whether the given device DMA address mask can be supported
* properly. For example, if your device can only drive the low 24-bits
@@ -305,6 +325,23 @@ extern void dma_unmap_page(struct device *, dma_addr_t, size_t,
enum dma_data_direction);
/*
+ * for DMA_BOUNCE we keep the nobarrier version same as their barriered
+ * counterpart
+ */
+static inline dma_addr_t dma_map_page_nobarrier(struct device *dev,
+ struct page *page, unsigned long offset, size_t size,
+ enum dma_data_direction dir)
+{
+ return dma_map_page(dev, page, offset, size, dir);
+}
+
+static inline void dma_unmap_page_nobarrier(struct device *dev,
+ dma_addr_t handle, size_t size, enum dma_data_direction dir)
+{
+ return dma_unmap_page(dev, handle, size, dir);
+}
+
+/*
* Private functions
*/
int dmabounce_sync_for_cpu(struct device *, dma_addr_t, unsigned long,
@@ -374,6 +411,34 @@ static inline dma_addr_t dma_map_page(struct device *dev, struct page *page,
}
/**
+ * dma_map_page_nobarrier - map a portion of a page for streaming DMA without a
+ * barrier
+ * @dev: valid struct device pointer, or NULL for ISA and EISA-like devices
+ * @page: page that buffer resides in
+ * @offset: offset into page for start of buffer
+ * @size: size of buffer to map
+ * @dir: DMA transfer direction
+ *
+ * Once this call is followed by a barrier it is ensured that any data held
+ * in the cache is appropriately discarded or written back.
+ *
+ * The device owns this memory once this call has completed and a barrier is
+ * executed. The CPU
+ * can regain ownership by calling dma_unmap_page() or
+ * dma_unmap_page_nobarrier() followed by a barrier.
+ */
+static inline dma_addr_t dma_map_page_nobarrier(struct device *dev,
+ struct page *page, unsigned long offset, size_t size,
+ enum dma_data_direction dir)
+{
+ BUG_ON(!valid_dma_direction(dir));
+
+ __dma_page_cpu_to_dev_nobarrier(page, offset, size, dir);
+
+ return page_to_dma(dev, page) + offset;
+}
+
+/**
* dma_unmap_single - unmap a single buffer previously mapped
* @dev: valid struct device pointer, or NULL for ISA and EISA-like devices
* @handle: DMA address of buffer
@@ -413,6 +478,28 @@ static inline void dma_unmap_page(struct device *dev, dma_addr_t handle,
__dma_page_dev_to_cpu(dma_to_page(dev, handle), handle & ~PAGE_MASK,
size, dir);
}
+
+/**
+ * dma_unmap_page_nobarrier - unmap a buffer previously mapped through dma_map_page()
+ * or dma_map_page_nobarrier() followed by a barrier
+ * @dev: valid struct device pointer, or NULL for ISA and EISA-like devices
+ * @handle: DMA address of buffer
+ * @size: size of buffer (same as passed to dma_map_page)
+ * @dir: DMA transfer direction (same as passed to dma_map_page)
+ *
+ * Unmap a page streaming mode DMA translation. The handle and size
+ * must match what was provided in the previous dma_map_page() call.
+ * All other usages are undefined.
+ *
+ * After this call, followed by a barrier(dsb/dmb), reads by the CPU to the
+ * buffer are guaranteed to see whatever the device wrote there.
+ */
+static inline void dma_unmap_page_nobarrier(struct device *dev,
+ dma_addr_t handle, size_t size, enum dma_data_direction dir)
+{
+ __dma_page_dev_to_cpu_nobarrier(dma_to_page(dev, handle),
+ handle & ~PAGE_MASK, size, dir);
+}
#endif /* CONFIG_DMABOUNCE */
/**
diff --git a/arch/arm/mm/dma-mapping.c b/arch/arm/mm/dma-mapping.c
index 64daef2..23556ab 100644
--- a/arch/arm/mm/dma-mapping.c
+++ b/arch/arm/mm/dma-mapping.c
@@ -509,6 +509,37 @@ void ___dma_page_dev_to_cpu(struct page *page, unsigned long off,
}
EXPORT_SYMBOL(___dma_page_dev_to_cpu);
+
+void ___dma_page_cpu_to_dev_nobarrier(struct page *page, unsigned long off,
+ size_t size, enum dma_data_direction dir)
+{
+ unsigned long paddr;
+
+ dma_cache_maint_page(page, off, size, dir, dmac_map_area_nobarrier);
+
+ paddr = page_to_phys(page) + off;
+ if (dir == DMA_FROM_DEVICE) {
+ outer_inv_range(paddr, paddr + size);
+ } else {
+ outer_clean_range(paddr, paddr + size);
+ }
+ /* FIXME: non-speculating: flush on bidirectional mappings? */
+}
+EXPORT_SYMBOL(___dma_page_cpu_to_dev_nobarrier);
+
+void ___dma_page_dev_to_cpu_nobarrier(struct page *page, unsigned long off,
+ size_t size, enum dma_data_direction dir)
+{
+ unsigned long paddr = page_to_phys(page) + off;
+
+ /* FIXME: non-speculating: not required */
+ /* don't bother invalidating if DMA to device */
+ if (dir != DMA_TO_DEVICE)
+ outer_inv_range(paddr, paddr + size);
+
+ dma_cache_maint_page(page, off, size, dir, dmac_unmap_area_nobarrier);
+}
+EXPORT_SYMBOL(___dma_page_dev_to_cpu_nobarrier);
/**
* dma_map_sg - map a set of SG buffers for streaming mode DMA
* @dev: valid struct device pointer, or NULL for ISA and EISA-like devices
@@ -531,17 +562,28 @@ int dma_map_sg(struct device *dev, struct scatterlist *sg, int nents,
struct scatterlist *s;
int i, j;
- for_each_sg(sg, s, nents, i) {
- s->dma_address = dma_map_page(dev, sg_page(s), s->offset,
- s->length, dir);
+ for_each_sg(sg, s, nents - 1 , i) {
+ s->dma_address = dma_map_page_nobarrier(dev, sg_page(s),
+ s->offset, s->length, dir);
if (dma_mapping_error(dev, s->dma_address))
goto bad_mapping;
}
+
+ s = sg_next(s);
+ i++;
+ s->dma_address = dma_map_page(dev, sg_page(s), s->offset,
+ s->length, dir);
+ if (dma_mapping_error(dev, s->dma_address))
+ goto bad_mapping;
+
return nents;
bad_mapping:
- for_each_sg(sg, s, i, j)
- dma_unmap_page(dev, sg_dma_address(s), sg_dma_len(s), dir);
+ for_each_sg(sg, s, i - 1, j)
+ dma_unmap_page_nobarrier(dev, sg_dma_address(s),
+ sg_dma_len(s), dir);
+ s = sg_next(s);
+ dma_unmap_page(dev, sg_dma_address(s), sg_dma_len(s), dir);
return 0;
}
EXPORT_SYMBOL(dma_map_sg);
@@ -562,8 +604,11 @@ void dma_unmap_sg(struct device *dev, struct scatterlist *sg, int nents,
struct scatterlist *s;
int i;
- for_each_sg(sg, s, nents, i)
- dma_unmap_page(dev, sg_dma_address(s), sg_dma_len(s), dir);
+ for_each_sg(sg, s, nents - 1, i)
+ dma_unmap_page_nobarrier(dev, sg_dma_address(s),
+ sg_dma_len(s), dir);
+ s = sg_next(s);
+ dma_unmap_page(dev, sg_dma_address(s), sg_dma_len(s), dir);
}
EXPORT_SYMBOL(dma_unmap_sg);
--
1.5.6.3
--
To unsubscribe from this list: send the line "unsubscribe linux-kernel" in
the body of a message to majordomo@...r.kernel.org
More majordomo info at http://vger.kernel.org/majordomo-info.html
Please read the FAQ at http://www.tux.org/lkml/
Powered by blists - more mailing lists