linux-kernel - [PATCH 04/10] x86,mm: use INVLPGB for kernel TLB flushes

lists.openwall.net		lists / announce owl-users owl-dev john-users john-dev passwdqc-users yescrypt popa3d-users / oss-security kernel-hardening musl sabotage tlsify passwords / crypt-dev xvendor / Bugtraq Full-Disclosure linux-kernel linux-netdev linux-ext4 linux-hardening linux-cve-announce PHC
Open Source and information security mailing list archives

Hash Suite: Windows password security audit tool. GUI, reports in PDF.

[<prev] [next>] [<thread-prev] [thread-next>] [day] [month] [year] [list]

Message-ID: <20241222040717.3096835-5-riel@surriel.com>
Date: Sat, 21 Dec 2024 23:06:36 -0500
From: Rik van Riel <riel@...riel.com>
To: x86@...nel.org
Cc: linux-kernel@...r.kernel.org,
	kernel-team@...a.com,
	dave.hansen@...ux.intel.com,
	luto@...nel.org,
	peterz@...radead.org,
	tglx@...utronix.de,
	mingo@...hat.com,
	bp@...en8.de,
	hpa@...or.com,
	akpm@...ux-foundation.org,
	Rik van Riel <riel@...riel.com>
Subject: [PATCH 04/10] x86,mm: use INVLPGB for kernel TLB flushes

Use broadcast TLB invalidation for kernel addresses when available.

This stops us from having to send IPIs for kernel TLB flushes.

Signed-off-by: Rik van Riel <riel@...riel.com>
---
 arch/x86/mm/tlb.c | 32 ++++++++++++++++++++++++++++++++
 1 file changed, 32 insertions(+)

diff --git a/arch/x86/mm/tlb.c b/arch/x86/mm/tlb.c
index 6cf881a942bb..09980fb17907 100644
--- a/arch/x86/mm/tlb.c
+++ b/arch/x86/mm/tlb.c
@@ -1077,6 +1077,33 @@ void flush_tlb_all(void)
 	on_each_cpu(do_flush_tlb_all, NULL, 1);
 }
 
+static void broadcast_kernel_range_flush(unsigned long start, unsigned long end)
+{
+	unsigned long addr;
+	unsigned long maxnr = boot_cpu_data.invlpgb_count_max;
+	unsigned long threshold = tlb_single_page_flush_ceiling * maxnr;
+
+	/*
+	 * TLBSYNC only waits for flushes originating on the same CPU.
+	 * Disabling migration allows us to wait on all flushes.
+	 */
+	migrate_disable();
+
+	if (end == TLB_FLUSH_ALL ||
+	    (end - start) > threshold << PAGE_SHIFT) {
+		invlpgb_flush_all();
+	} else {
+		unsigned long nr;
+		for (addr = start; addr < end; addr += nr << PAGE_SHIFT) {
+			nr = min((end - addr) >> PAGE_SHIFT, maxnr);
+			invlpgb_flush_addr(addr, nr);
+		}
+	}
+
+	tlbsync();
+	migrate_enable();
+}
+
 static void do_kernel_range_flush(void *info)
 {
 	struct flush_tlb_info *f = info;
@@ -1089,6 +1116,11 @@ static void do_kernel_range_flush(void *info)
 
 void flush_tlb_kernel_range(unsigned long start, unsigned long end)
 {
+	if (static_cpu_has(X86_FEATURE_INVLPGB)) {
+		broadcast_kernel_range_flush(start, end);
+		return;
+	}
+
 	/* Balance as user space task's flush, a bit conservative */
 	if (end == TLB_FLUSH_ALL ||
 	    (end - start) > tlb_single_page_flush_ceiling << PAGE_SHIFT) {
-- 
2.47.1