From: Dave Hansen If changing the page tables in such a way that an invalidation of all contexts (aka. PCIDs / ASIDs) is required, they can be actively invalidated by: 1. INVPCID for each PCID (works for single pages too). 2. Load CR3 with each PCID without the NOFLUSH bit set 3. Load CR3 with the NOFLUSH bit set for each and do INVLPG for each address. But, none of these are really feasible since there are ~6 ASIDs (12 with KERNEL_PAGE_TABLE_ISOLATION) at the time that invalidation is required. Instead of actively invalidating them, invalidate the *current* context and also mark the cpu_tlbstate _quickly_ to indicate future invalidation to be required. At the next context-switch, look for this indicator ('invalidate_other' being set) invalidate all of the cpu_tlbstate.ctxs[] entries. This ensures that any future context switches will do a full flush of the TLB, picking up the previous changes. Signed-off-by: Dave Hansen Signed-off-by: Ingo Molnar Signed-off-by: Thomas Gleixner Signed-off-by: Peter Zijlstra (Intel) Cc: Rik van Riel Cc: Denys Vlasenko Cc: Andy Lutomirski Cc: michael.schwarz@iaik.tugraz.at Cc: daniel.gruss@iaik.tugraz.at Cc: Brian Gerst Cc: Josh Poimboeuf Cc: hughd@google.com Cc: Borislav Petkov Cc: moritz.lipp@iaik.tugraz.at Cc: keescook@google.com Cc: Linus Torvalds Cc: richard.fellner@student.tugraz.at Link: https://lkml.kernel.org/r/20171123003507.E8C327F5@viggo.jf.intel.com --- arch/x86/include/asm/tlbflush.h | 42 ++++++++++++++++++++++++++++++---------- arch/x86/mm/tlb.c | 37 +++++++++++++++++++++++++++++++++++ 2 files changed, 69 insertions(+), 10 deletions(-) --- a/arch/x86/include/asm/tlbflush.h +++ b/arch/x86/include/asm/tlbflush.h @@ -188,6 +188,17 @@ struct tlb_state { bool is_lazy; /* + * If set we changed the page tables in such a way that we + * needed an invalidation of all contexts (aka. PCIDs / ASIDs). + * This tells us to go invalidate all the non-loaded ctxs[] + * on the next context switch. + * + * The current ctx was kept up-to-date as it ran and does not + * need to be invalidated. + */ + bool invalidate_other; + + /* * Access to this CR4 shadow and to H/W CR4 is protected by * disabling interrupts when modifying either one. */ @@ -267,6 +278,19 @@ static inline unsigned long cr4_read_sha return this_cpu_read(cpu_tlbstate.cr4); } +static inline void invalidate_pcid_other(void) +{ + /* + * With global pages, all of the shared kenel page tables + * are set as _PAGE_GLOBAL. We have no shared nonglobals + * and nothing to do here. + */ + if (!static_cpu_has_bug(X86_BUG_CPU_SECURE_MODE_KPTI)) + return; + + this_cpu_write(cpu_tlbstate.invalidate_other, true); +} + /* * Save some of cr4 feature set we're using (e.g. Pentium 4MB * enable and PPro Global page enable), so that any CPU's that boot @@ -341,24 +365,22 @@ static inline void __native_flush_tlb_si static inline void __flush_tlb_all(void) { - if (boot_cpu_has(X86_FEATURE_PGE)) + if (boot_cpu_has(X86_FEATURE_PGE)) { __flush_tlb_global(); - else + } else { __flush_tlb(); - - /* - * Note: if we somehow had PCID but not PGE, then this wouldn't work -- - * we'd end up flushing kernel translations for the current ASID but - * we might fail to flush kernel translations for other cached ASIDs. - * - * To avoid this issue, we force PCID off if PGE is off. - */ + } } static inline void __flush_tlb_one(unsigned long addr) { count_vm_tlb_event(NR_TLB_LOCAL_FLUSH_ONE); __flush_tlb_single(addr); + /* + * Invalidate other address spaces inaccessible to single-page + * invalidation: + */ + invalidate_pcid_other(); } #define TLB_FLUSH_ALL -1UL --- a/arch/x86/mm/tlb.c +++ b/arch/x86/mm/tlb.c @@ -28,6 +28,38 @@ * Implement flush IPI by CALL_FUNCTION_VECTOR, Alex Shi */ +/* + * We get here when we do something requiring a TLB invalidation + * but could not go invalidate all of the contexts. We do the + * necessary invalidation by clearing out the 'ctx_id' which + * forces a TLB flush when the context is loaded. + */ +void clear_asid_other(void) +{ + u16 asid; + + /* + * This is only expected to be set if we have disabled + * kernel _PAGE_GLOBAL pages. + */ + if (!static_cpu_has_bug(X86_BUG_CPU_SECURE_MODE_KPTI)) { + WARN_ON_ONCE(1); + return; + } + + for (asid = 0; asid < TLB_NR_DYN_ASIDS; asid++) { + /* Do not need to flush the current asid */ + if (asid == this_cpu_read(cpu_tlbstate.loaded_mm_asid)) + continue; + /* + * Make sure the next time we go to switch to + * this asid, we do a flush: + */ + this_cpu_write(cpu_tlbstate.ctxs[asid].ctx_id, 0); + } + this_cpu_write(cpu_tlbstate.invalidate_other, false); +} + atomic64_t last_mm_ctx_id = ATOMIC64_INIT(1); @@ -42,6 +74,9 @@ static void choose_new_asid(struct mm_st return; } + if (this_cpu_read(cpu_tlbstate.invalidate_other)) + clear_asid_other(); + for (asid = 0; asid < TLB_NR_DYN_ASIDS; asid++) { if (this_cpu_read(cpu_tlbstate.ctxs[asid].ctx_id) != next->context.ctx_id) @@ -552,6 +587,8 @@ static void do_kernel_range_flush(void * /* flush range by one by one 'invlpg' */ for (addr = f->start; addr < f->end; addr += PAGE_SIZE) __flush_tlb_single(addr); + + invalidate_pcid_other(); } void flush_tlb_kernel_range(unsigned long start, unsigned long end)