linux-kernel - [PATCH] x86/mm: Pre-allocate p4d/pud pages for vmalloc area

lists.openwall.net		lists / announce owl-users owl-dev john-users john-dev passwdqc-users yescrypt popa3d-users / oss-security kernel-hardening musl sabotage tlsify passwords / crypt-dev xvendor / Bugtraq Full-Disclosure linux-kernel linux-netdev linux-ext4 linux-hardening linux-cve-announce PHC
Open Source and information security mailing list archives

Hash Suite: Windows password security audit tool. GUI, reports in PDF.

[<prev] [next>] [thread-next>] [day] [month] [year] [list]

Date:   Fri, 26 Jun 2020 11:34:50 +0200
From:   Joerg Roedel <joro@...tes.org>
To:     x86@...nel.org
Cc:     hpa@...or.com, Dave Hansen <dave.hansen@...ux.intel.com>,
        Andy Lutomirski <luto@...nel.org>,
        Peter Zijlstra <peterz@...radead.org>,
        Andrew Morton <akpm@...ux-foundation.org>,
        Steven Rostedt <rostedt@...dmis.org>, joro@...tes.org,
        linux-kernel@...r.kernel.org, linux-mm@...ck.org,
        Joerg Roedel <jroedel@...e.de>
Subject: [PATCH] x86/mm: Pre-allocate p4d/pud pages for vmalloc area

From: Joerg Roedel <jroedel@...e.de>

Pre-allocate the page-table pages for the vmalloc area at the level
which needs synchronization on x86. This is P4D for 5-level and PUD
for 4-level paging.

Doing this at boot makes sure all page-tables in the system have these
pages already and do not need to be synchronized at runtime. The
runtime synchronizatin takes the pgd_lock and iterates over all
page-tables in the system, so it can take quite long and is better
avoided.

Signed-off-by: Joerg Roedel <jroedel@...e.de>
---
 arch/x86/mm/init_64.c | 55 +++++++++++++++++++++++++++++++++++++++++++
 1 file changed, 55 insertions(+)

diff --git a/arch/x86/mm/init_64.c b/arch/x86/mm/init_64.c
index dbae185511cd..475a4008445b 100644
--- a/arch/x86/mm/init_64.c
+++ b/arch/x86/mm/init_64.c
@@ -1238,6 +1238,59 @@ static void __init register_page_bootmem_info(void)
 #endif
 }
 
+/*
+ * Pre-allocates page-table pages for the vmalloc area in the kernel page-table.
+ * Only the level which needs to be synchronized between all page-tables is
+ * allocated because the synchronization can be expensive.
+ */
+static void __init preallocate_vmalloc_pages(void)
+{
+	unsigned long addr;
+	const char *lvl;
+	int count = 0;
+
+	for (addr = VMALLOC_START; addr <= VMALLOC_END; addr = ALIGN(addr + 1, PGDIR_SIZE)) {
+		pgd_t *pgd = pgd_offset_k(addr);
+		p4d_t *p4d;
+		pud_t *pud;
+
+		p4d = p4d_offset(pgd, addr);
+		if (p4d_none(*p4d)) {
+			/* Can only happen with 5-level paging */
+			p4d = p4d_alloc(&init_mm, pgd, addr);
+			if (!p4d) {
+				lvl = "p4d";
+				goto failed;
+			}
+			count += 1;
+		}
+
+		if (pgtable_l5_enabled())
+			continue;
+
+		pud = pud_offset(p4d, addr);
+		if (pud_none(*pud)) {
+			/* Ends up here only with 4-level paging */
+			pud = pud_alloc(&init_mm, p4d, addr);
+			if (!pud) {
+				lvl = "pud";
+				goto failed;
+			}
+			count += 1;
+		}
+	}
+
+	return;
+
+failed:
+
+	/*
+	 * A failure here is not fatal - If the pages can be allocated later it
+	 * will be synchronized to other page-tables.
+	 */
+	pr_err("Failed to pre-allocate %s pages for vmalloc area\n", lvl);
+}
+
 void __init mem_init(void)
 {
 	pci_iommu_alloc();
@@ -1261,6 +1314,8 @@ void __init mem_init(void)
 	if (get_gate_vma(&init_mm))
 		kclist_add(&kcore_vsyscall, (void *)VSYSCALL_ADDR, PAGE_SIZE, KCORE_USER);
 
+	preallocate_vmalloc_pages();
+
 	mem_init_print_info(NULL);
 }
 
-- 
2.27.0