linux-kernel - Re: [PATCH 09/10] x86/rcu: Add rcu_preempt

lists.openwall.net		lists / announce owl-users owl-dev john-users john-dev passwdqc-users yescrypt popa3d-users / oss-security kernel-hardening musl sabotage tlsify passwords / crypt-dev xvendor / Bugtraq Full-Disclosure linux-kernel linux-netdev linux-ext4 linux-hardening linux-cve-announce PHC
Open Source and information security mailing list archives
Hash Suite: Windows password security audit tool. GUI, reports in PDF.
[<prev] [next>] [<thread-prev] [thread-next>] [day] [month] [year] [list]
Date: Mon, 22 Apr 2024 13:05:20 +0200
From: Frederic Weisbecker <frederic@...nel.org>
To: Lai Jiangshan <jiangshanlai@...il.com>
Cc: linux-kernel@...r.kernel.org, rcu@...r.kernel.org, x86@...nel.org,
	Lai Jiangshan <jiangshan.ljs@...group.com>,
	"Paul E. McKenney" <paulmck@...nel.org>,
	Peter Zijlstra <peterz@...radead.org>,
	Thomas Gleixner <tglx@...utronix.de>,
	Ingo Molnar <mingo@...hat.com>, Borislav Petkov <bp@...en8.de>,
	Dave Hansen <dave.hansen@...ux.intel.com>,
	"H. Peter Anvin" <hpa@...or.com>, Uros Bizjak <ubizjak@...il.com>,
	Josh Poimboeuf <jpoimboe@...nel.org>, Nadav Amit <namit@...are.com>,
	Breno Leitao <leitao@...ian.org>,
	Kent Overstreet <kent.overstreet@...ux.dev>,
	Pawan Gupta <pawan.kumar.gupta@...ux.intel.com>,
	Rick Edgecombe <rick.p.edgecombe@...el.com>,
	Vegard Nossum <vegard.nossum@...cle.com>,
	Daniel Sneddon <daniel.sneddon@...ux.intel.com>,
	Nikolay Borisov <nik.borisov@...e.com>
Subject: Re: [PATCH 09/10] x86/rcu: Add rcu_preempt_count

Le Thu, Mar 28, 2024 at 03:53:17PM +0800, Lai Jiangshan a écrit :
> From: Lai Jiangshan <jiangshan.ljs@...group.com>
> 
> Implement PCPU_RCU_PREEMPT_COUNT for x86.
> Mainly copied from asm/preempt.h
> 
> Make rcu_read_[un]lock() inlined for rcu-preempt.
> Make rcu_read_lock() only one instruction.
> Make rcu_read_unlock() only two instructions in the fast path.
> 
> Cc: "Paul E. McKenney" <paulmck@...nel.org>
> Cc: Peter Zijlstra <peterz@...radead.org>
> Cc: Frederic Weisbecker <frederic@...nel.org>
> Signed-off-by: Lai Jiangshan <jiangshan.ljs@...group.com>
> ---
>  arch/x86/Kconfig                   |   1 +
>  arch/x86/include/asm/current.h     |   3 +
>  arch/x86/include/asm/rcu_preempt.h | 107 +++++++++++++++++++++++++++++
>  arch/x86/kernel/cpu/common.c       |   7 +-
>  4 files changed, 115 insertions(+), 3 deletions(-)
>  create mode 100644 arch/x86/include/asm/rcu_preempt.h
> 
> diff --git a/arch/x86/Kconfig b/arch/x86/Kconfig
> index 78050d5d7fac..7eb17c12f7b7 100644
> --- a/arch/x86/Kconfig
> +++ b/arch/x86/Kconfig
> @@ -257,6 +257,7 @@ config X86
>  	select HAVE_OBJTOOL			if X86_64
>  	select HAVE_OPTPROBES
>  	select HAVE_PAGE_SIZE_4KB
> +	select HAVE_PCPU_RCU_PREEMPT_COUNT
>  	select HAVE_PCSPKR_PLATFORM
>  	select HAVE_PERF_EVENTS
>  	select HAVE_PERF_EVENTS_NMI
> diff --git a/arch/x86/include/asm/current.h b/arch/x86/include/asm/current.h
> index bf5953883ec3..dcc2ef784120 100644
> --- a/arch/x86/include/asm/current.h
> +++ b/arch/x86/include/asm/current.h
> @@ -24,6 +24,9 @@ struct pcpu_hot {
>  			unsigned long		top_of_stack;
>  			void			*hardirq_stack_ptr;
>  			u16			softirq_pending;
> +#ifdef CONFIG_PCPU_RCU_PREEMPT_COUNT
> +			int			rcu_preempt_count;
> +#endif // #ifdef CONFIG_PCPU_RCU_PREEMPT_COUNT
>  #ifdef CONFIG_X86_64
>  			bool			hardirq_stack_inuse;
>  #else
> diff --git a/arch/x86/include/asm/rcu_preempt.h b/arch/x86/include/asm/rcu_preempt.h
> new file mode 100644
> index 000000000000..cb25ebe038a5
> --- /dev/null
> +++ b/arch/x86/include/asm/rcu_preempt.h
> @@ -0,0 +1,107 @@
> +/* SPDX-License-Identifier: GPL-2.0 */
> +#ifndef __ASM_RCU_PREEMPT_H
> +#define __ASM_RCU_PREEMPT_H
> +
> +#include <asm/rmwcc.h>
> +#include <asm/percpu.h>
> +#include <asm/current.h>
> +
> +#ifdef CONFIG_PCPU_RCU_PREEMPT_COUNT
> +
> +/* We use the MSB mostly because its available */

I think you can safely remove the "We " from all the comments :-)

> +#define RCU_PREEMPT_UNLOCK_SPECIAL_INVERTED	0x80000000

How about RCU_PREEMPT_UNLOCK_FASTPATH ?

> +
> +/*
> + * We use the RCU_PREEMPT_UNLOCK_SPECIAL_INVERTED bit as an inverted
> + * current->rcu_read_unlock_special.s such that a decrement hitting 0
> + * means we can and should call rcu_read_unlock_special().
> + */
> +#define RCU_PREEMPT_INIT	(0 + RCU_PREEMPT_UNLOCK_SPECIAL_INVERTED)

Or simply:

#define RCU_PREEMPT_INIT     RCU_PREEMPT_UNLOCK_FASTPATH

Or you can even remove RCU_PREEMPT_INIT and use RCU_PREEMPT_UNLOCK_FASTPATH directly.

> +/*
> + * Because we keep RCU_PREEMPT_UNLOCK_SPECIAL_INVERTED set when we do
> + * _not_ need to handle unlock-special for a fast-path decrement.
> + */
> +static __always_inline bool pcpu_rcu_preempt_count_dec_and_test(void)
> +{
> +	return GEN_UNARY_RMWcc("decl", __my_cpu_var(pcpu_hot.rcu_preempt_count), e,
> +			       __percpu_arg([var]));
> +}
> +
> +#define pcpu_rcu_read_unlock_special()						\
> +do {										\
> +	rcu_read_unlock_special();						\
> +} while (0)

Why not just call that directly?

> +
> +#endif // #ifdef CONFIG_PCPU_RCU_PREEMPT_COUNT
> +
> +#endif /* __ASM_RCU_PREEMPT_H */
> diff --git a/arch/x86/kernel/cpu/common.c b/arch/x86/kernel/cpu/common.c
> index ba8cf5e9ce56..0b204a649442 100644
> --- a/arch/x86/kernel/cpu/common.c
> +++ b/arch/x86/kernel/cpu/common.c
> @@ -1992,9 +1992,10 @@ static __init int setup_clearcpuid(char *arg)
>  __setup("clearcpuid=", setup_clearcpuid);
>  
>  DEFINE_PER_CPU_ALIGNED(struct pcpu_hot, pcpu_hot) = {
> -	.current_task	= &init_task,
> -	.preempt_count	= INIT_PREEMPT_COUNT,
> -	.top_of_stack	= TOP_OF_INIT_STACK,
> +	.current_task		= &init_task,
> +	.preempt_count		= INIT_PREEMPT_COUNT,
> +	.top_of_stack		= TOP_OF_INIT_STACK,
> +	.rcu_preempt_count	= RCU_PREEMPT_INIT,

#ifdef CONFIG_PCPU_RCU_PREEMPT_COUNT ?

Thanks.


>  };
>  EXPORT_PER_CPU_SYMBOL(pcpu_hot);
>  EXPORT_PER_CPU_SYMBOL(const_pcpu_hot);
> -- 
> 2.19.1.6.gb485710b
>