[<prev] [next>] [<thread-prev] [thread-next>] [day] [month] [year] [list]
Message-ID: <20250506003811.92405-9-riel@surriel.com>
Date: Mon, 5 May 2025 20:37:46 -0400
From: Rik van Riel <riel@...riel.com>
To: linux-kernel@...r.kernel.org
Cc: linux-mm@...ck.org,
x86@...nel.org,
kernel-team@...a.com,
dave.hansen@...ux.intel.com,
luto@...nel.org,
peterz@...radead.org,
tglx@...utronix.de,
mingo@...hat.com,
bp@...en8.de,
hpa@...or.com,
Rik van Riel <riel@...com>,
Rik van Riel <riel@...riel.com>
Subject: [RFC PATCH 8/9] x86/mm: use RAR for kernel TLB flushes
From: Rik van Riel <riel@...com>
Use Intel RAR for kernel TLB flushes, when enabled.
Pass in PCID 0 to smp_call_rar_many() to flush kernel memory.
Unfortunately RAR_INVPCID_ADDR excludes global PTE mappings, so only
full flushes with RAR_INVPCID_ALL will flush kernel mappings.
Signed-off-by: Rik van Riel <riel@...riel.com>
---
arch/x86/mm/tlb.c | 38 ++++++++++++++++++++++++++++++++++++++
1 file changed, 38 insertions(+)
diff --git a/arch/x86/mm/tlb.c b/arch/x86/mm/tlb.c
index 7c61bf11d472..a4f3941281b6 100644
--- a/arch/x86/mm/tlb.c
+++ b/arch/x86/mm/tlb.c
@@ -21,6 +21,7 @@
#include <asm/apic.h>
#include <asm/msr.h>
#include <asm/perf_event.h>
+#include <asm/rar.h>
#include <asm/tlb.h>
#include "mm_internal.h"
@@ -1451,6 +1452,18 @@ static void do_flush_tlb_all(void *info)
__flush_tlb_all();
}
+static void rar_full_flush(const cpumask_t *cpumask)
+{
+ guard(preempt)();
+ smp_call_rar_many(cpumask, 0, 0, TLB_FLUSH_ALL);
+ invpcid_flush_all();
+}
+
+static void rar_flush_all(void)
+{
+ rar_full_flush(cpu_online_mask);
+}
+
void flush_tlb_all(void)
{
count_vm_tlb_event(NR_TLB_REMOTE_FLUSH);
@@ -1458,6 +1471,8 @@ void flush_tlb_all(void)
/* First try (faster) hardware-assisted TLB invalidation. */
if (cpu_feature_enabled(X86_FEATURE_INVLPGB))
invlpgb_flush_all();
+ else if (cpu_feature_enabled(X86_FEATURE_RAR))
+ rar_flush_all();
else
/* Fall back to the IPI-based invalidation. */
on_each_cpu(do_flush_tlb_all, NULL, 1);
@@ -1487,15 +1502,36 @@ static void do_kernel_range_flush(void *info)
struct flush_tlb_info *f = info;
unsigned long addr;
+ /*
+ * With PTI kernel TLB entries in all PCIDs need to be flushed.
+ * With RAR the PCID space becomes so large, we might as well flush it all.
+ *
+ * Either of the two by itself works with targeted flushes.
+ */
+ if (cpu_feature_enabled(X86_FEATURE_RAR) &&
+ cpu_feature_enabled(X86_FEATURE_PTI)) {
+ invpcid_flush_all();
+ return;
+ }
+
/* flush range by one by one 'invlpg' */
for (addr = f->start; addr < f->end; addr += PAGE_SIZE)
flush_tlb_one_kernel(addr);
}
+static void rar_kernel_range_flush(struct flush_tlb_info *info)
+{
+ guard(preempt)();
+ smp_call_rar_many(cpu_online_mask, 0, info->start, info->end);
+ do_kernel_range_flush(info);
+}
+
static void kernel_tlb_flush_all(struct flush_tlb_info *info)
{
if (cpu_feature_enabled(X86_FEATURE_INVLPGB))
invlpgb_flush_all();
+ else if (cpu_feature_enabled(X86_FEATURE_RAR))
+ rar_flush_all();
else
on_each_cpu(do_flush_tlb_all, NULL, 1);
}
@@ -1504,6 +1540,8 @@ static void kernel_tlb_flush_range(struct flush_tlb_info *info)
{
if (cpu_feature_enabled(X86_FEATURE_INVLPGB))
invlpgb_kernel_range_flush(info);
+ else if (cpu_feature_enabled(X86_FEATURE_RAR))
+ rar_kernel_range_flush(info);
else
on_each_cpu(do_kernel_range_flush, info, 1);
}
--
2.49.0
Powered by blists - more mailing lists