From: Andy Lutomirski <luto@kernel.org>

Map the ESPFIX pages into user space when PTI is enabled. Move
init_espfix_bsp() before pti_init() so it can be correcly populated.

Signed-off-by: Andy Lutomirski <luto@kernel.org>
Signed-off-by: Thomas Gleixner <tglx@linutronix.de>
Cc: Kees Cook <keescook@chromium.org>
Cc: Peter Zijlstra <peterz@infradead.org>
Cc: Brian Gerst <brgerst@gmail.com>
Cc: David Laight <David.Laight@aculab.com>
Cc: Borislav Petkov <bp@alien8.de>
---
 arch/x86/mm/pti.c |   20 ++++++++++++++++++++
 init/main.c       |   11 +++++++----
 2 files changed, 27 insertions(+), 4 deletions(-)

--- a/arch/x86/mm/pti.c
+++ b/arch/x86/mm/pti.c
@@ -186,6 +186,25 @@ pti_clone_pmds(unsigned long start, unsi
 	}
 }
 
+static void __init pti_setup_espfix64(void)
+{
+#ifdef CONFIG_X86_ESPFIX64
+	/*
+	 * ESPFIX64 uses a single p4d (i.e. a top-level entry on 4-level
+	 * systems and a next-level entry on 5-level systems.  Share that
+	 * entry between the user and kernel pagetables.
+	 */
+	pgd_t *kernel_pgd;
+	p4d_t *kernel_p4d, *user_p4d;
+
+	pr_err("espfix64 base = %lx\n", ESPFIX_BASE_ADDR);
+	user_p4d = pti_user_pagetable_walk_p4d(ESPFIX_BASE_ADDR);
+	kernel_pgd = pgd_offset_k(ESPFIX_BASE_ADDR);
+	kernel_p4d = p4d_offset(kernel_pgd, ESPFIX_BASE_ADDR);
+	*user_p4d = *kernel_p4d;
+#endif
+}
+
 /*
  * Clone the populated PMDs of the user shared fixmaps into the user space
  * visible page table.
@@ -224,4 +243,5 @@ void __init pti_init(void)
 
 	pti_clone_user_shared();
 	pti_clone_entry_text();
+	pti_setup_espfix64();
 }
--- a/init/main.c
+++ b/init/main.c
@@ -505,6 +505,13 @@ static void __init mm_init(void)
 	pgtable_init();
 	vmalloc_init();
 	ioremap_huge_init();
+
+#ifdef CONFIG_X86_ESPFIX64
+	/* Should be run before the first non-init thread is created */
+	init_espfix_bsp();
+#endif
+
+	/* Should be run after espfix64 is set up. */
 	pti_init();
 }
 
@@ -681,10 +688,6 @@ asmlinkage __visible void __init start_k
 	if (efi_enabled(EFI_RUNTIME_SERVICES))
 		efi_enter_virtual_mode();
 #endif
-#ifdef CONFIG_X86_ESPFIX64
-	/* Should be run before the first non-init thread is created */
-	init_espfix_bsp();
-#endif
 	thread_stack_cache_init();
 	cred_init();
 	fork_init();