lists.openwall.net | lists / announce owl-users owl-dev john-users john-dev passwdqc-users yescrypt popa3d-users / oss-security kernel-hardening musl sabotage tlsify passwords / crypt-dev xvendor / Bugtraq Full-Disclosure linux-kernel linux-netdev linux-ext4 linux-hardening linux-cve-announce PHC | |
Open Source and information security mailing list archives
| ||
|
Date: Sat, 25 May 2019 01:22:01 -0700 From: Nadav Amit <namit@...are.com> To: Ingo Molnar <mingo@...hat.com>, Peter Zijlstra <peterz@...radead.org>, Andy Lutomirski <luto@...nel.org> Cc: Borislav Petkov <bp@...en8.de>, linux-kernel@...r.kernel.org, Nadav Amit <namit@...are.com>, Dave Hansen <dave.hansen@...ux.intel.com>, Thomas Gleixner <tglx@...utronix.de>, "H. Peter Anvin" <hpa@...or.com>, x86@...nel.org Subject: [RFC PATCH 4/6] x86/mm/tlb: Refactor common code into flush_tlb_on_cpus() arch_tlbbatch_flush() and flush_tlb_mm_range() have very similar code, which is effectively the same. Extract the mutual code into a new function flush_tlb_on_cpus(). There is one functional change, which should not affect correctness: flush_tlb_mm_range compared loaded_mm and the mm to figure out if local flush is needed. Instead, the common code would look at the mm_cpumask() which should give the same result. Performance should not be affected, since this cpumask should not change in such a frequency that would introduce cache contention. Cc: Dave Hansen <dave.hansen@...ux.intel.com> Cc: Andy Lutomirski <luto@...nel.org> Cc: Peter Zijlstra <peterz@...radead.org> Cc: Thomas Gleixner <tglx@...utronix.de> Cc: Ingo Molnar <mingo@...hat.com> Cc: Borislav Petkov <bp@...en8.de> Cc: "H. Peter Anvin" <hpa@...or.com> Cc: x86@...nel.org Signed-off-by: Nadav Amit <namit@...are.com> --- arch/x86/mm/tlb.c | 55 ++++++++++++++++++++++------------------------- 1 file changed, 26 insertions(+), 29 deletions(-) diff --git a/arch/x86/mm/tlb.c b/arch/x86/mm/tlb.c index 7f61431c75fb..afd2859a6966 100644 --- a/arch/x86/mm/tlb.c +++ b/arch/x86/mm/tlb.c @@ -733,7 +733,11 @@ static inline struct flush_tlb_info *get_flush_tlb_info(struct mm_struct *mm, unsigned int stride_shift, bool freed_tables, u64 new_tlb_gen) { - struct flush_tlb_info *info = this_cpu_ptr(&flush_tlb_info); + struct flush_tlb_info *info; + + preempt_disable(); + + info = this_cpu_ptr(&flush_tlb_info); #ifdef CONFIG_DEBUG_VM /* @@ -761,6 +765,23 @@ static inline void put_flush_tlb_info(void) barrier(); this_cpu_dec(flush_tlb_info_idx); #endif + preempt_enable(); +} + +static void flush_tlb_on_cpus(const cpumask_t *cpumask, + const struct flush_tlb_info *info) +{ + int this_cpu = smp_processor_id(); + + if (cpumask_test_cpu(this_cpu, cpumask)) { + lockdep_assert_irqs_enabled(); + local_irq_disable(); + flush_tlb_func_local(info, TLB_LOCAL_MM_SHOOTDOWN); + local_irq_enable(); + } + + if (cpumask_any_but(cpumask, this_cpu) < nr_cpu_ids) + flush_tlb_others(cpumask, info); } void flush_tlb_mm_range(struct mm_struct *mm, unsigned long start, @@ -769,9 +790,6 @@ void flush_tlb_mm_range(struct mm_struct *mm, unsigned long start, { struct flush_tlb_info *info; u64 new_tlb_gen; - int cpu; - - cpu = get_cpu(); /* Should we flush just the requested range? */ if ((end == TLB_FLUSH_ALL) || @@ -786,18 +804,9 @@ void flush_tlb_mm_range(struct mm_struct *mm, unsigned long start, info = get_flush_tlb_info(mm, start, end, stride_shift, freed_tables, new_tlb_gen); - if (mm == this_cpu_read(cpu_tlbstate.loaded_mm)) { - lockdep_assert_irqs_enabled(); - local_irq_disable(); - flush_tlb_func_local(info, TLB_LOCAL_MM_SHOOTDOWN); - local_irq_enable(); - } - - if (cpumask_any_but(mm_cpumask(mm), cpu) < nr_cpu_ids) - flush_tlb_others(mm_cpumask(mm), info); + flush_tlb_on_cpus(mm_cpumask(mm), info); put_flush_tlb_info(); - put_cpu(); } @@ -832,13 +841,11 @@ void flush_tlb_kernel_range(unsigned long start, unsigned long end) } else { struct flush_tlb_info *info; - preempt_disable(); info = get_flush_tlb_info(NULL, start, end, 0, false, 0); on_each_cpu(do_kernel_range_flush, info, 1); put_flush_tlb_info(); - preempt_enable(); } } @@ -856,21 +863,11 @@ static const struct flush_tlb_info full_flush_tlb_info = { void arch_tlbbatch_flush(struct arch_tlbflush_unmap_batch *batch) { - int cpu = get_cpu(); - - if (cpumask_test_cpu(cpu, &batch->cpumask)) { - lockdep_assert_irqs_enabled(); - local_irq_disable(); - flush_tlb_func_local(&full_flush_tlb_info, TLB_LOCAL_SHOOTDOWN); - local_irq_enable(); - } - - if (cpumask_any_but(&batch->cpumask, cpu) < nr_cpu_ids) - flush_tlb_others(&batch->cpumask, &full_flush_tlb_info); + preempt_disable(); + flush_tlb_on_cpus(&batch->cpumask, &full_flush_tlb_info); + preempt_enable(); cpumask_clear(&batch->cpumask); - - put_cpu(); } static ssize_t tlbflush_read_file(struct file *file, char __user *user_buf, -- 2.20.1
Powered by blists - more mailing lists