Instead of relying on INVPCID to shoot down user PCID, delay the invalidate until we switch to the user page-tables. This gets rid of the INVPCID dependence for KAISER PCID. XXX we could do a much larger ALTERNATIVE, there is no point in testing the mask if we don't have PCID support. Suggested-by: Andy Lutomirski Signed-off-by: Peter Zijlstra (Intel) --- arch/x86/entry/calling.h | 29 +++++++++++++++++------ arch/x86/include/asm/tlbflush.h | 39 +++++++++++++++++++++++-------- arch/x86/mm/init.c | 13 ---------- arch/x86/mm/tlb.c | 49 +--------------------------------------- 4 files changed, 53 insertions(+), 77 deletions(-) --- a/arch/x86/entry/calling.h +++ b/arch/x86/entry/calling.h @@ -4,6 +4,7 @@ #include #include #include +#include /* @@ -203,12 +204,6 @@ For 32-bit we have the following convent andq $(~KAISER_SWITCH_MASK), \reg .endm -.macro ADJUST_USER_CR3 reg:req - ALTERNATIVE "", "bts $63, \reg", X86_FEATURE_PCID - /* Set user PCID bit, and move CR3 up a page to the user page tables: */ - orq $(KAISER_SWITCH_MASK), \reg -.endm - .macro SWITCH_TO_KERNEL_CR3 scratch_reg:req STATIC_JUMP_IF_FALSE .Lend_\@, kaiser_enabled_key, def=1 mov %cr3, \scratch_reg @@ -220,7 +215,27 @@ For 32-bit we have the following convent .macro SWITCH_TO_USER_CR3 scratch_reg:req STATIC_JUMP_IF_FALSE .Lend_\@, kaiser_enabled_key, def=1 mov %cr3, \scratch_reg - ADJUST_USER_CR3 \scratch_reg + + /* + * Test if the ASID needs a flush. + */ + push \scratch_reg /* preserve CR3 */ + andq $(0x7FF), \scratch_reg /* mask ASID */ + bt \scratch_reg, PER_CPU_VAR(user_asid_flush_mask) + jnc .Lnoflush_\@ + + /* Flush needed, clear the bit */ + btr \scratch_reg, PER_CPU_VAR(user_asid_flush_mask) + pop \scratch_reg /* original CR3 */ + jmp .Ldo_\@ + +.Lnoflush_\@: + pop \scratch_reg /* original CR3 */ + ALTERNATIVE "", "bts $63, \scratch_reg", X86_FEATURE_PCID + +.Ldo_\@: + /* Flip the PGD and ASID to the user version */ + orq $(KAISER_SWITCH_MASK), \scratch_reg mov \scratch_reg, %cr3 .Lend_\@: .endm --- a/arch/x86/include/asm/tlbflush.h +++ b/arch/x86/include/asm/tlbflush.h @@ -348,19 +348,37 @@ static inline void cr4_set_bits_and_upda extern void initialize_tlbstate_and_flush(void); +DECLARE_PER_CPU(unsigned long, user_asid_flush_mask); + +/* + * Given an ASID, flush the corresponding user ASID. + * We can delay this until the next time we switch to it. + * + * See SWITCH_TO_USER_CR3. + */ +static inline void flush_user_asid(u16 asid) +{ + /* There is no user ASID if KAISER is off */ + if (!IS_ENABLED(CONFIG_KAISER)) + return; + + /* + * We only have a single ASID if PCID is off and the CR3 + * write will have flushed it. + */ + if (!cpu_feature_enabled(X86_FEATURE_PCID)) + return; + + if (!kaiser_enabled) + return; + + __set_bit(kern_asid(asid), this_cpu_ptr(&user_asid_flush_mask)); +} + static inline void __native_flush_tlb(void) { if (!cpu_feature_enabled(X86_FEATURE_INVPCID)) { - /* - * native_write_cr3() only clears the current PCID if - * CR4 has X86_CR4_PCIDE set. In other words, this does - * not fully flush the TLB if PCIDs are in use. - * - * With KAISER and PCIDs, that means that we did not - * flush the user PCID. Warn if it gets called. - */ - if (IS_ENABLED(CONFIG_KAISER) && kaiser_enabled) - WARN_ON_ONCE(this_cpu_read(cpu_tlbstate.cr4) & X86_CR4_PCIDE); + flush_user_asid(this_cpu_read(cpu_tlbstate.loaded_mm_asid)); /* * If current->mm == NULL then we borrow a mm @@ -436,6 +454,7 @@ static inline void __native_flush_tlb_si * early. */ if (!this_cpu_has(X86_FEATURE_INVPCID_SINGLE)) { + flush_user_asid(loaded_mm_asid); asm volatile("invlpg (%0)" ::"r" (addr) : "memory"); return; } --- a/arch/x86/mm/init.c +++ b/arch/x86/mm/init.c @@ -211,19 +211,6 @@ static void setup_pcid(void) if (boot_cpu_has(X86_FEATURE_PGE)) { /* - * KAISER uses a PCID for the kernel and another - * for userspace. Both PCIDs need to be flushed - * when the TLB flush functions are called. But, - * flushing *another* PCID is insane without - * INVPCID. Just avoid using PCIDs at all if we - * have KAISER and do not have INVPCID. - */ - if (!IS_ENABLED(CONFIG_X86_GLOBAL_PAGES) && - kaiser_enabled && !boot_cpu_has(X86_FEATURE_INVPCID)) { - setup_clear_cpu_cap(X86_FEATURE_PCID); - return; - } - /* * This can't be cr4_set_bits_and_update_boot() -- * the trampoline code can't handle CR4.PCIDE and * it wouldn't do any good anyway. Despite the name, --- a/arch/x86/mm/tlb.c +++ b/arch/x86/mm/tlb.c @@ -101,59 +101,14 @@ static void choose_new_asid(struct mm_st *need_flush = true; } -/* - * Given a kernel asid, flush the corresponding KAISER - * user ASID. - */ -static void flush_user_asid(pgd_t *pgd, u16 kern_asid) -{ - /* There is no user ASID if KAISER is off */ - if (!IS_ENABLED(CONFIG_KAISER)) - return; - /* - * We only have a single ASID if PCID is off and the CR3 - * write will have flushed it. - */ - if (!cpu_feature_enabled(X86_FEATURE_PCID)) - return; - - if (!kaiser_enabled) - return; - - /* - * With PCIDs enabled, write_cr3() only flushes TLB - * entries for the current (kernel) ASID. This leaves - * old TLB entries for the user ASID in place and we must - * flush that context separately. We can theoretically - * delay doing this until we actually load up the - * userspace CR3, but do it here for simplicity. - */ - if (cpu_feature_enabled(X86_FEATURE_INVPCID)) { - invpcid_flush_single_context(user_asid(kern_asid)); - } else { - /* - * On systems with PCIDs, but no INVPCID, the only - * way to flush a PCID is a CR3 write. Note that - * we use the kernel page tables with the *user* - * ASID here. - */ - unsigned long user_asid_flush_cr3; - user_asid_flush_cr3 = build_cr3(pgd, user_asid(kern_asid)); - write_cr3(user_asid_flush_cr3); - /* - * We do not use PCIDs with KAISER unless we also - * have INVPCID. Getting here is unexpected. - */ - WARN_ON_ONCE(1); - } -} +__visible DEFINE_PER_CPU(unsigned long, user_asid_flush_mask); static void load_new_mm_cr3(pgd_t *pgdir, u16 new_asid, bool need_flush) { unsigned long new_mm_cr3; if (need_flush) { - flush_user_asid(pgdir, new_asid); + flush_user_asid(new_asid); new_mm_cr3 = build_cr3(pgdir, new_asid); } else { new_mm_cr3 = build_cr3_noflush(pgdir, new_asid);