[<prev] [next>] [<thread-prev] [thread-next>] [day] [month] [year] [list]
Message-ID: <ZiZEcBEvK8NOQvwU@localhost.localdomain>
Date: Mon, 22 Apr 2024 13:05:20 +0200
From: Frederic Weisbecker <frederic@...nel.org>
To: Lai Jiangshan <jiangshanlai@...il.com>
Cc: linux-kernel@...r.kernel.org, rcu@...r.kernel.org, x86@...nel.org,
Lai Jiangshan <jiangshan.ljs@...group.com>,
"Paul E. McKenney" <paulmck@...nel.org>,
Peter Zijlstra <peterz@...radead.org>,
Thomas Gleixner <tglx@...utronix.de>,
Ingo Molnar <mingo@...hat.com>, Borislav Petkov <bp@...en8.de>,
Dave Hansen <dave.hansen@...ux.intel.com>,
"H. Peter Anvin" <hpa@...or.com>, Uros Bizjak <ubizjak@...il.com>,
Josh Poimboeuf <jpoimboe@...nel.org>, Nadav Amit <namit@...are.com>,
Breno Leitao <leitao@...ian.org>,
Kent Overstreet <kent.overstreet@...ux.dev>,
Pawan Gupta <pawan.kumar.gupta@...ux.intel.com>,
Rick Edgecombe <rick.p.edgecombe@...el.com>,
Vegard Nossum <vegard.nossum@...cle.com>,
Daniel Sneddon <daniel.sneddon@...ux.intel.com>,
Nikolay Borisov <nik.borisov@...e.com>
Subject: Re: [PATCH 09/10] x86/rcu: Add rcu_preempt_count
Le Thu, Mar 28, 2024 at 03:53:17PM +0800, Lai Jiangshan a écrit :
> From: Lai Jiangshan <jiangshan.ljs@...group.com>
>
> Implement PCPU_RCU_PREEMPT_COUNT for x86.
> Mainly copied from asm/preempt.h
>
> Make rcu_read_[un]lock() inlined for rcu-preempt.
> Make rcu_read_lock() only one instruction.
> Make rcu_read_unlock() only two instructions in the fast path.
>
> Cc: "Paul E. McKenney" <paulmck@...nel.org>
> Cc: Peter Zijlstra <peterz@...radead.org>
> Cc: Frederic Weisbecker <frederic@...nel.org>
> Signed-off-by: Lai Jiangshan <jiangshan.ljs@...group.com>
> ---
> arch/x86/Kconfig | 1 +
> arch/x86/include/asm/current.h | 3 +
> arch/x86/include/asm/rcu_preempt.h | 107 +++++++++++++++++++++++++++++
> arch/x86/kernel/cpu/common.c | 7 +-
> 4 files changed, 115 insertions(+), 3 deletions(-)
> create mode 100644 arch/x86/include/asm/rcu_preempt.h
>
> diff --git a/arch/x86/Kconfig b/arch/x86/Kconfig
> index 78050d5d7fac..7eb17c12f7b7 100644
> --- a/arch/x86/Kconfig
> +++ b/arch/x86/Kconfig
> @@ -257,6 +257,7 @@ config X86
> select HAVE_OBJTOOL if X86_64
> select HAVE_OPTPROBES
> select HAVE_PAGE_SIZE_4KB
> + select HAVE_PCPU_RCU_PREEMPT_COUNT
> select HAVE_PCSPKR_PLATFORM
> select HAVE_PERF_EVENTS
> select HAVE_PERF_EVENTS_NMI
> diff --git a/arch/x86/include/asm/current.h b/arch/x86/include/asm/current.h
> index bf5953883ec3..dcc2ef784120 100644
> --- a/arch/x86/include/asm/current.h
> +++ b/arch/x86/include/asm/current.h
> @@ -24,6 +24,9 @@ struct pcpu_hot {
> unsigned long top_of_stack;
> void *hardirq_stack_ptr;
> u16 softirq_pending;
> +#ifdef CONFIG_PCPU_RCU_PREEMPT_COUNT
> + int rcu_preempt_count;
> +#endif // #ifdef CONFIG_PCPU_RCU_PREEMPT_COUNT
> #ifdef CONFIG_X86_64
> bool hardirq_stack_inuse;
> #else
> diff --git a/arch/x86/include/asm/rcu_preempt.h b/arch/x86/include/asm/rcu_preempt.h
> new file mode 100644
> index 000000000000..cb25ebe038a5
> --- /dev/null
> +++ b/arch/x86/include/asm/rcu_preempt.h
> @@ -0,0 +1,107 @@
> +/* SPDX-License-Identifier: GPL-2.0 */
> +#ifndef __ASM_RCU_PREEMPT_H
> +#define __ASM_RCU_PREEMPT_H
> +
> +#include <asm/rmwcc.h>
> +#include <asm/percpu.h>
> +#include <asm/current.h>
> +
> +#ifdef CONFIG_PCPU_RCU_PREEMPT_COUNT
> +
> +/* We use the MSB mostly because its available */
I think you can safely remove the "We " from all the comments :-)
> +#define RCU_PREEMPT_UNLOCK_SPECIAL_INVERTED 0x80000000
How about RCU_PREEMPT_UNLOCK_FASTPATH ?
> +
> +/*
> + * We use the RCU_PREEMPT_UNLOCK_SPECIAL_INVERTED bit as an inverted
> + * current->rcu_read_unlock_special.s such that a decrement hitting 0
> + * means we can and should call rcu_read_unlock_special().
> + */
> +#define RCU_PREEMPT_INIT (0 + RCU_PREEMPT_UNLOCK_SPECIAL_INVERTED)
Or simply:
#define RCU_PREEMPT_INIT RCU_PREEMPT_UNLOCK_FASTPATH
Or you can even remove RCU_PREEMPT_INIT and use RCU_PREEMPT_UNLOCK_FASTPATH directly.
> +/*
> + * Because we keep RCU_PREEMPT_UNLOCK_SPECIAL_INVERTED set when we do
> + * _not_ need to handle unlock-special for a fast-path decrement.
> + */
> +static __always_inline bool pcpu_rcu_preempt_count_dec_and_test(void)
> +{
> + return GEN_UNARY_RMWcc("decl", __my_cpu_var(pcpu_hot.rcu_preempt_count), e,
> + __percpu_arg([var]));
> +}
> +
> +#define pcpu_rcu_read_unlock_special() \
> +do { \
> + rcu_read_unlock_special(); \
> +} while (0)
Why not just call that directly?
> +
> +#endif // #ifdef CONFIG_PCPU_RCU_PREEMPT_COUNT
> +
> +#endif /* __ASM_RCU_PREEMPT_H */
> diff --git a/arch/x86/kernel/cpu/common.c b/arch/x86/kernel/cpu/common.c
> index ba8cf5e9ce56..0b204a649442 100644
> --- a/arch/x86/kernel/cpu/common.c
> +++ b/arch/x86/kernel/cpu/common.c
> @@ -1992,9 +1992,10 @@ static __init int setup_clearcpuid(char *arg)
> __setup("clearcpuid=", setup_clearcpuid);
>
> DEFINE_PER_CPU_ALIGNED(struct pcpu_hot, pcpu_hot) = {
> - .current_task = &init_task,
> - .preempt_count = INIT_PREEMPT_COUNT,
> - .top_of_stack = TOP_OF_INIT_STACK,
> + .current_task = &init_task,
> + .preempt_count = INIT_PREEMPT_COUNT,
> + .top_of_stack = TOP_OF_INIT_STACK,
> + .rcu_preempt_count = RCU_PREEMPT_INIT,
#ifdef CONFIG_PCPU_RCU_PREEMPT_COUNT ?
Thanks.
> };
> EXPORT_PER_CPU_SYMBOL(pcpu_hot);
> EXPORT_PER_CPU_SYMBOL(const_pcpu_hot);
> --
> 2.19.1.6.gb485710b
>
Powered by blists - more mailing lists