From fa80e9691202183a2879a013e497f221b22305a6 Mon Sep 17 00:00:00 2001 From: Balbir Singh Date: Wed, 25 Mar 2020 16:41:18 +1100 Subject: [RFC PATCH v2 4/4] arch/x86: L1D flush, optimize the context switch Use a static branch/jump label to optimize the code. Right now we don't ref count the users, but that could be done if needed in the future. Signed-off-by: Balbir Singh --- arch/x86/include/asm/nospec-branch.h | 2 ++ arch/x86/mm/tlb.c | 52 +++++++++++++++++----------- 2 files changed, 34 insertions(+), 20 deletions(-) diff --git a/arch/x86/include/asm/nospec-branch.h b/arch/x86/include/asm/nospec-branch.h index 07e95dcb40ad..371e28cea1f4 100644 --- a/arch/x86/include/asm/nospec-branch.h +++ b/arch/x86/include/asm/nospec-branch.h @@ -310,6 +310,8 @@ DECLARE_STATIC_KEY_FALSE(switch_mm_always_ibpb); DECLARE_STATIC_KEY_FALSE(mds_user_clear); DECLARE_STATIC_KEY_FALSE(mds_idle_clear); +DECLARE_STATIC_KEY_FALSE(switch_mm_l1d_flush); + #include /** diff --git a/arch/x86/mm/tlb.c b/arch/x86/mm/tlb.c index 22f96c5f74f0..8f272e5921ce 100644 --- a/arch/x86/mm/tlb.c +++ b/arch/x86/mm/tlb.c @@ -155,6 +155,11 @@ EXPORT_SYMBOL_GPL(leave_mm); static void *l1d_flush_pages; static DEFINE_MUTEX(l1d_flush_mutex); +/* Flush L1D on switch_mm() */ +DEFINE_STATIC_KEY_FALSE(switch_mm_l1d_flush); +EXPORT_SYMBOL_GPL(switch_mm_l1d_flush); + + int enable_l1d_flush_for_task(struct task_struct *tsk) { struct page *page; @@ -170,6 +175,11 @@ int enable_l1d_flush_for_task(struct task_struct *tsk) l1d_flush_pages = alloc_l1d_flush_pages(); if (!l1d_flush_pages) return -ENOMEM; + /* + * We could do more accurate ref counting + * if needed + */ + static_branch_enable(&switch_mm_l1d_flush); } mutex_unlock(&l1d_flush_mutex); } @@ -195,29 +205,31 @@ static void l1d_flush(struct mm_struct *next, struct task_struct *tsk) { struct mm_struct *real_prev = this_cpu_read(cpu_tlbstate.loaded_mm); - /* - * If we are not really switching mm's, we can just return - */ - if (real_prev == next) - return; + if (static_branch_unlikely(&switch_mm_l1d_flush)) { + /* + * If we are not really switching mm's, we can just return + */ + if (real_prev == next) + return; - /* - * Do we need flushing for by the previous task - */ - if (this_cpu_read(cpu_tlbstate.last_user_mm_l1d_flush) != 0) { - if (!flush_l1d_cache_hw()) - flush_l1d_cache_sw(l1d_flush_pages); - this_cpu_write(cpu_tlbstate.last_user_mm_l1d_flush, 0); - /* Make sure we clear the values before we set it again */ - barrier(); - } + /* + * Do we need flushing for by the previous task + */ + if (this_cpu_read(cpu_tlbstate.last_user_mm_l1d_flush) != 0) { + if (!flush_l1d_cache_hw()) + flush_l1d_cache_sw(l1d_flush_pages); + this_cpu_write(cpu_tlbstate.last_user_mm_l1d_flush, 0); + /* Make sure we clear the values before we set it again */ + barrier(); + } - if (tsk == NULL) - return; + if (tsk == NULL) + return; - /* We don't need stringent checks as we opt-in/opt-out */ - if (test_ti_thread_flag(&tsk->thread_info, TIF_L1D_FLUSH)) - this_cpu_write(cpu_tlbstate.last_user_mm_l1d_flush, 1); + /* We don't need stringent checks as we opt-in/opt-out */ + if (test_ti_thread_flag(&tsk->thread_info, TIF_L1D_FLUSH)) + this_cpu_write(cpu_tlbstate.last_user_mm_l1d_flush, 1); + } } void switch_mm(struct mm_struct *prev, struct mm_struct *next, -- 2.17.1