| lists.openwall.net | lists / announce owl-users owl-dev john-users john-dev passwdqc-users yescrypt popa3d-users / oss-security kernel-hardening musl sabotage tlsify passwords / crypt-dev xvendor / Bugtraq Full-Disclosure linux-kernel linux-netdev linux-ext4 linux-hardening linux-cve-announce PHC | |
|
Open Source and information security mailing list archives
| ||
|
Message-Id: <1470302117-32296-3-git-send-email-mbrugger@suse.com>
Date: Thu, 4 Aug 2016 11:15:15 +0200
From: Matthias Brugger <mbrugger@...e.com>
To: pbonzini@...hat.com, rkrcmar@...hat.com,
christoffer.dall@...aro.org, marc.zyngier@....com,
linux@...linux.org.uk, catalin.marinas@....com, will.deacon@....com
Cc: suzuki.poulose@....com, james.morse@....com,
david.daney@...ium.com, rrichter@...ium.com, agraf@...e.de,
mbrugger@...e.com, mark.rutland@....com, lorenzo.pieralisi@....com,
dave.long@...aro.org, ard.biesheuvel@...aro.org,
zlim.lnx@...il.com, kvm@...r.kernel.org,
linux-arm-kernel@...ts.infradead.org, kvmarm@...ts.cs.columbia.edu,
linux-kernel@...r.kernel.org
Subject: [PATCH 2/4] arm64: Implement IPI based TLB invalidation
Hardware may lack a sane implementation of TLB invalidation
using broadcast TLBI command. Add a capability to enable TLB
invalidation using IPI.
Signed-off-by: David Daney <david.daney@...ium.com>
Signed-off-by: Robert Richter <rrichter@...ium.com>
Signed-off-by: Alexander Graf <agraf@...e.de>
Signed-off-by: Matthias Brugger <mbrugger@...e.com>
---
arch/arm64/include/asm/cpufeature.h | 3 +-
arch/arm64/include/asm/tlbflush.h | 94 ++++++++++++++++++++++++++++++++-----
arch/arm64/mm/flush.c | 46 ++++++++++++++++++
3 files changed, 129 insertions(+), 14 deletions(-)
diff --git a/arch/arm64/include/asm/cpufeature.h b/arch/arm64/include/asm/cpufeature.h
index 49dd1bd..c4bf72b 100644
--- a/arch/arm64/include/asm/cpufeature.h
+++ b/arch/arm64/include/asm/cpufeature.h
@@ -36,8 +36,9 @@
#define ARM64_HAS_VIRT_HOST_EXTN 11
#define ARM64_WORKAROUND_CAVIUM_27456 12
#define ARM64_HAS_32BIT_EL0 13
+#define ARM64_HAS_NO_BCAST_TLBI 14
-#define ARM64_NCAPS 14
+#define ARM64_NCAPS 15
#ifndef __ASSEMBLY__
diff --git a/arch/arm64/include/asm/tlbflush.h b/arch/arm64/include/asm/tlbflush.h
index b460ae2..edc5495 100644
--- a/arch/arm64/include/asm/tlbflush.h
+++ b/arch/arm64/include/asm/tlbflush.h
@@ -71,7 +71,10 @@ static inline void local_flush_tlb_all(void)
isb();
}
-static inline void flush_tlb_all(void)
+void __flush_tlb_all_ipi(void);
+void __flush_tlb_mm_ipi(struct mm_struct *mm);
+
+static inline void __flush_tlb_all_tlbi(void)
{
dsb(ishst);
asm("tlbi vmalle1is");
@@ -79,7 +82,17 @@ static inline void flush_tlb_all(void)
isb();
}
-static inline void flush_tlb_mm(struct mm_struct *mm)
+static inline void flush_tlb_all(void)
+{
+ if (cpus_have_cap(ARM64_HAS_NO_BCAST_TLBI)) {
+ __flush_tlb_all_ipi();
+ return;
+ }
+
+ __flush_tlb_all_tlbi();
+}
+
+static inline void __flush_tlb_mm_tlbi(struct mm_struct *mm)
{
unsigned long asid = ASID(mm) << 48;
@@ -88,8 +101,18 @@ static inline void flush_tlb_mm(struct mm_struct *mm)
dsb(ish);
}
-static inline void flush_tlb_page(struct vm_area_struct *vma,
- unsigned long uaddr)
+static inline void flush_tlb_mm(struct mm_struct *mm)
+{
+ if (cpus_have_cap(ARM64_HAS_NO_BCAST_TLBI)) {
+ __flush_tlb_mm_ipi(mm);
+ return;
+ }
+
+ __flush_tlb_mm_tlbi(mm);
+}
+
+static inline void __flush_tlb_page_tlbi(struct vm_area_struct *vma,
+ unsigned long uaddr)
{
unsigned long addr = uaddr >> 12 | (ASID(vma->vm_mm) << 48);
@@ -98,15 +121,26 @@ static inline void flush_tlb_page(struct vm_area_struct *vma,
dsb(ish);
}
+static inline void flush_tlb_page(struct vm_area_struct *vma,
+ unsigned long uaddr)
+{
+ if (cpus_have_cap(ARM64_HAS_NO_BCAST_TLBI)) {
+ __flush_tlb_mm_ipi(vma->vm_mm);
+ return;
+ }
+
+ __flush_tlb_page_tlbi(vma, uaddr);
+}
+
/*
* This is meant to avoid soft lock-ups on large TLB flushing ranges and not
* necessarily a performance improvement.
*/
#define MAX_TLB_RANGE (1024UL << PAGE_SHIFT)
-static inline void __flush_tlb_range(struct vm_area_struct *vma,
- unsigned long start, unsigned long end,
- bool last_level)
+static inline void __flush_tlb_range_tlbi(struct vm_area_struct *vma,
+ unsigned long start, unsigned long end,
+ bool last_level)
{
unsigned long asid = ASID(vma->vm_mm) << 48;
unsigned long addr;
@@ -129,13 +163,26 @@ static inline void __flush_tlb_range(struct vm_area_struct *vma,
dsb(ish);
}
+static inline void __flush_tlb_range(struct vm_area_struct *vma,
+ unsigned long start, unsigned long end,
+ bool last_level)
+{
+ if (cpus_have_cap(ARM64_HAS_NO_BCAST_TLBI)) {
+ __flush_tlb_mm_ipi(vma->vm_mm);
+ return;
+ }
+
+ __flush_tlb_range_tlbi(vma, start, end, last_level);
+}
+
static inline void flush_tlb_range(struct vm_area_struct *vma,
- unsigned long start, unsigned long end)
+ unsigned long start, unsigned long end)
{
__flush_tlb_range(vma, start, end, false);
}
-static inline void flush_tlb_kernel_range(unsigned long start, unsigned long end)
+static inline void __flush_tlb_kernel_range_tlbi(unsigned long start,
+ unsigned long end)
{
unsigned long addr;
@@ -154,17 +201,38 @@ static inline void flush_tlb_kernel_range(unsigned long start, unsigned long end
isb();
}
+static inline void flush_tlb_kernel_range(unsigned long start, unsigned long end)
+{
+ if (cpus_have_cap(ARM64_HAS_NO_BCAST_TLBI)) {
+ __flush_tlb_all_ipi();
+ return;
+ }
+
+ __flush_tlb_kernel_range_tlbi(start, end);
+}
+
+static inline void __flush_tlb_pgtable_tlbi(struct mm_struct *mm,
+ unsigned long uaddr)
+{
+ unsigned long addr = uaddr >> 12 | (ASID(mm) << 48);
+
+ asm("tlbi vae1is, %0" : : "r" (addr));
+ dsb(ish);
+}
+
/*
* Used to invalidate the TLB (walk caches) corresponding to intermediate page
* table levels (pgd/pud/pmd).
*/
static inline void __flush_tlb_pgtable(struct mm_struct *mm,
- unsigned long uaddr)
+ unsigned long uaddr)
{
- unsigned long addr = uaddr >> 12 | (ASID(mm) << 48);
+ if (cpus_have_cap(ARM64_HAS_NO_BCAST_TLBI)) {
+ __flush_tlb_mm_ipi(mm);
+ return;
+ }
- asm("tlbi vae1is, %0" : : "r" (addr));
- dsb(ish);
+ __flush_tlb_pgtable_tlbi(mm, uaddr);
}
#endif
diff --git a/arch/arm64/mm/flush.c b/arch/arm64/mm/flush.c
index 43a76b0..402036a 100644
--- a/arch/arm64/mm/flush.c
+++ b/arch/arm64/mm/flush.c
@@ -27,6 +27,24 @@
#include "mm.h"
+static void flush_tlb_local(void *info)
+{
+ local_flush_tlb_all();
+}
+
+static void flush_tlb_mm_local(void *info)
+{
+ unsigned long asid = (unsigned long)info;
+
+ asm volatile("\n"
+ " dsb nshst\n"
+ " tlbi aside1, %0\n"
+ " dsb nsh\n"
+ " isb sy"
+ : : "r" (asid)
+ );
+}
+
void flush_cache_range(struct vm_area_struct *vma, unsigned long start,
unsigned long end)
{
@@ -90,6 +108,34 @@ void flush_dcache_page(struct page *page)
}
EXPORT_SYMBOL(flush_dcache_page);
+void __flush_tlb_mm_ipi(struct mm_struct *mm)
+{
+ unsigned long asid;
+
+ if (!mm) {
+ flush_tlb_all();
+ } else {
+ asid = ASID(mm) << 48;
+ /* Make sure page table modifications are visible. */
+ dsb(ishst);
+ /* IPI to all CPUs to do local flush. */
+ on_each_cpu(flush_tlb_mm_local, (void *)asid, 1);
+ }
+}
+EXPORT_SYMBOL(__flush_tlb_mm_ipi);
+
+void __flush_tlb_all_ipi(void)
+{
+ /* Make sure page table modifications are visible. */
+ dsb(ishst);
+ if (num_online_cpus() <= 1)
+ local_flush_tlb_all();
+ else
+ /* IPI to all CPUs to do local flush. */
+ on_each_cpu(flush_tlb_local, NULL, 1);
+}
+EXPORT_SYMBOL(__flush_tlb_all_ipi);
+
/*
* Additional functions defined in assembly.
*/
--
2.6.6
Powered by blists - more mailing lists