[<prev] [next>] [<thread-prev] [thread-next>] [day] [month] [year] [list]
Message-Id: <20230801085402.1168351-4-alexghiti@rivosinc.com>
Date: Tue, 1 Aug 2023 10:54:01 +0200
From: Alexandre Ghiti <alexghiti@...osinc.com>
To: Will Deacon <will@...nel.org>,
"Aneesh Kumar K . V" <aneesh.kumar@...ux.ibm.com>,
Andrew Morton <akpm@...ux-foundation.org>,
Nick Piggin <npiggin@...il.com>,
Peter Zijlstra <peterz@...radead.org>,
Mayuresh Chitale <mchitale@...tanamicro.com>,
Vincent Chen <vincent.chen@...ive.com>,
Paul Walmsley <paul.walmsley@...ive.com>,
Palmer Dabbelt <palmer@...belt.com>,
Albert Ou <aou@...s.berkeley.edu>, linux-arch@...r.kernel.org,
linux-mm@...ck.org, linux-riscv@...ts.infradead.org,
linux-kernel@...r.kernel.org
Cc: Alexandre Ghiti <alexghiti@...osinc.com>,
Andrew Jones <ajones@...tanamicro.com>
Subject: [PATCH v3 3/4] riscv: Make __flush_tlb_range() loop over pte instead of flushing the whole tlb
Currently, when the range to flush covers more than one page (a 4K page or
a hugepage), __flush_tlb_range() flushes the whole tlb. Flushing the whole
tlb comes with a greater cost than flushing a single entry so we should
flush single entries up to a certain threshold so that:
threshold * cost of flushing a single entry < cost of flushing the whole
tlb.
Co-developed-by: Mayuresh Chitale <mchitale@...tanamicro.com>
Signed-off-by: Mayuresh Chitale <mchitale@...tanamicro.com>
Signed-off-by: Alexandre Ghiti <alexghiti@...osinc.com>
Reviewed-by: Andrew Jones <ajones@...tanamicro.com>
---
arch/riscv/mm/tlbflush.c | 48 ++++++++++++++++++++++++++++++++++++----
1 file changed, 44 insertions(+), 4 deletions(-)
diff --git a/arch/riscv/mm/tlbflush.c b/arch/riscv/mm/tlbflush.c
index d883df0dee4a..0c955c474f3a 100644
--- a/arch/riscv/mm/tlbflush.c
+++ b/arch/riscv/mm/tlbflush.c
@@ -7,6 +7,9 @@
#include <asm/sbi.h>
#include <asm/mmu_context.h>
+#define FLUSH_TLB_MAX_SIZE ((unsigned long)-1)
+#define FLUSH_TLB_NO_ASID ((unsigned long)-1)
+
static inline void local_flush_tlb_all_asid(unsigned long asid)
{
__asm__ __volatile__ ("sfence.vma x0, %0"
@@ -24,13 +27,48 @@ static inline void local_flush_tlb_page_asid(unsigned long addr,
: "memory");
}
+/*
+ * Flush entire TLB if number of entries to be flushed is greater
+ * than the threshold below.
+ */
+static unsigned long tlb_flush_all_threshold __read_mostly = 64;
+
+static void local_flush_tlb_range_threshold_asid(unsigned long start,
+ unsigned long size,
+ unsigned long stride,
+ unsigned long asid)
+{
+ u16 nr_ptes_in_range = DIV_ROUND_UP(size, stride);
+ int i;
+
+ if (nr_ptes_in_range > tlb_flush_all_threshold) {
+ if (asid != FLUSH_TLB_NO_ASID)
+ local_flush_tlb_all_asid(asid);
+ else
+ local_flush_tlb_all();
+ return;
+ }
+
+ for (i = 0; i < nr_ptes_in_range; ++i) {
+ if (asid != FLUSH_TLB_NO_ASID)
+ local_flush_tlb_page_asid(start, asid);
+ else
+ local_flush_tlb_page(start);
+ start += stride;
+ }
+}
+
static inline void local_flush_tlb_range(unsigned long start,
unsigned long size, unsigned long stride)
{
if (size <= stride)
local_flush_tlb_page(start);
- else
+ else if (size == FLUSH_TLB_MAX_SIZE)
local_flush_tlb_all();
+ else
+ local_flush_tlb_range_threshold_asid(start, size, stride,
+ FLUSH_TLB_NO_ASID);
+
}
static inline void local_flush_tlb_range_asid(unsigned long start,
@@ -38,8 +76,10 @@ static inline void local_flush_tlb_range_asid(unsigned long start,
{
if (size <= stride)
local_flush_tlb_page_asid(start, asid);
- else
+ else if (size == FLUSH_TLB_MAX_SIZE)
local_flush_tlb_all_asid(asid);
+ else
+ local_flush_tlb_range_threshold_asid(start, size, stride, asid);
}
static void __ipi_flush_tlb_all(void *info)
@@ -52,7 +92,7 @@ void flush_tlb_all(void)
if (riscv_use_ipi_for_rfence())
on_each_cpu(__ipi_flush_tlb_all, NULL, 1);
else
- sbi_remote_sfence_vma(NULL, 0, -1);
+ sbi_remote_sfence_vma(NULL, 0, FLUSH_TLB_MAX_SIZE);
}
struct flush_tlb_range_data {
@@ -130,7 +170,7 @@ static void __flush_tlb_range(struct mm_struct *mm, unsigned long start,
void flush_tlb_mm(struct mm_struct *mm)
{
- __flush_tlb_range(mm, 0, -1, PAGE_SIZE);
+ __flush_tlb_range(mm, 0, FLUSH_TLB_MAX_SIZE, PAGE_SIZE);
}
void flush_tlb_mm_range(struct mm_struct *mm,
--
2.39.2
Powered by blists - more mailing lists