linux-kernel - [RFC PATCH V3 12/43] rv64ilp32_abi: riscv: Introduce cmpxchg

lists.openwall.net		lists / announce owl-users owl-dev john-users john-dev passwdqc-users yescrypt popa3d-users / oss-security kernel-hardening musl sabotage tlsify passwords / crypt-dev xvendor / Bugtraq Full-Disclosure linux-kernel linux-netdev linux-ext4 linux-hardening linux-cve-announce PHC
Open Source and information security mailing list archives
Hash Suite: Windows password security audit tool. GUI, reports in PDF.
[<prev] [next>] [<thread-prev] [thread-next>] [day] [month] [year] [list]
Message-Id: <20250325121624.523258-13-guoren@kernel.org>
Date: Tue, 25 Mar 2025 08:15:53 -0400
From: guoren@...nel.org
To: arnd@...db.de,
	gregkh@...uxfoundation.org,
	torvalds@...ux-foundation.org,
	paul.walmsley@...ive.com,
	palmer@...belt.com,
	anup@...infault.org,
	atishp@...shpatra.org,
	oleg@...hat.com,
	kees@...nel.org,
	tglx@...utronix.de,
	will@...nel.org,
	mark.rutland@....com,
	brauner@...nel.org,
	akpm@...ux-foundation.org,
	rostedt@...dmis.org,
	edumazet@...gle.com,
	unicorn_wang@...look.com,
	inochiama@...look.com,
	gaohan@...as.ac.cn,
	shihua@...as.ac.cn,
	jiawei@...as.ac.cn,
	wuwei2016@...as.ac.cn,
	drew@...7.com,
	prabhakar.mahadev-lad.rj@...renesas.com,
	ctsai390@...estech.com,
	wefu@...hat.com,
	kuba@...nel.org,
	pabeni@...hat.com,
	josef@...icpanda.com,
	dsterba@...e.com,
	mingo@...hat.com,
	peterz@...radead.org,
	boqun.feng@...il.com,
	guoren@...nel.org,
	xiao.w.wang@...el.com,
	qingfang.deng@...lower.com.cn,
	leobras@...hat.com,
	jszhang@...nel.org,
	conor.dooley@...rochip.com,
	samuel.holland@...ive.com,
	yongxuan.wang@...ive.com,
	luxu.kernel@...edance.com,
	david@...hat.com,
	ruanjinjie@...wei.com,
	cuiyunhui@...edance.com,
	wangkefeng.wang@...wei.com,
	qiaozhe@...as.ac.cn
Cc: ardb@...nel.org,
	ast@...nel.org,
	linux-kernel@...r.kernel.org,
	linux-riscv@...ts.infradead.org,
	kvm@...r.kernel.org,
	kvm-riscv@...ts.infradead.org,
	linux-mm@...ck.org,
	linux-crypto@...r.kernel.org,
	bpf@...r.kernel.org,
	linux-input@...r.kernel.org,
	linux-perf-users@...r.kernel.org,
	linux-serial@...r.kernel.org,
	linux-fsdevel@...r.kernel.org,
	linux-arch@...r.kernel.org,
	maple-tree@...ts.infradead.org,
	linux-trace-kernel@...r.kernel.org,
	netdev@...r.kernel.org,
	linux-atm-general@...ts.sourceforge.net,
	linux-btrfs@...r.kernel.org,
	netfilter-devel@...r.kernel.org,
	coreteam@...filter.org,
	linux-nfs@...r.kernel.org,
	linux-sctp@...r.kernel.org,
	linux-usb@...r.kernel.org,
	linux-media@...r.kernel.org
Subject: [RFC PATCH V3 12/43] rv64ilp32_abi: riscv: Introduce cmpxchg_double

From: "Guo Ren (Alibaba DAMO Academy)" <guoren@...nel.org>

The rv64ilp32 abi has the ability to exclusively load and store
(ld/sd) a pair of words from an address. Then the SLUB can take
advantage of a cmpxchg_double implementation to avoid taking some
locks.

This patch provides an implementation of cmpxchg_double for 32-bit
pairs, and activates the logic required for the SLUB to use these
functions (HAVE_ALIGNED_STRUCT_PAGE and HAVE_CMPXCHG_DOUBLE).

Inspired from the commit: 5284e1b4bc8a ("arm64: xchg: Implement
cmpxchg_double")

Signed-off-by: Guo Ren (Alibaba DAMO Academy) <guoren@...nel.org>
---
 arch/riscv/Kconfig               |  1 +
 arch/riscv/include/asm/cmpxchg.h | 53 ++++++++++++++++++++++++++++++++
 2 files changed, 54 insertions(+)

diff --git a/arch/riscv/Kconfig b/arch/riscv/Kconfig
index da2111b0111c..884235cf4092 100644
--- a/arch/riscv/Kconfig
+++ b/arch/riscv/Kconfig
@@ -141,6 +141,7 @@ config RISCV
 	select HAVE_ARCH_USERFAULTFD_MINOR if 64BIT && USERFAULTFD
 	select HAVE_ARCH_VMAP_STACK if MMU && 64BIT
 	select HAVE_ASM_MODVERSIONS
+	select HAVE_CMPXCHG_DOUBLE if ABI_RV64ILP32
 	select HAVE_CONTEXT_TRACKING_USER
 	select HAVE_DEBUG_KMEMLEAK
 	select HAVE_DMA_CONTIGUOUS if MMU
diff --git a/arch/riscv/include/asm/cmpxchg.h b/arch/riscv/include/asm/cmpxchg.h
index 938d50194dba..944f6d825f78 100644
--- a/arch/riscv/include/asm/cmpxchg.h
+++ b/arch/riscv/include/asm/cmpxchg.h
@@ -7,6 +7,7 @@
 #define _ASM_RISCV_CMPXCHG_H
 
 #include <linux/bug.h>
+#include <linux/mmdebug.h>
 
 #include <asm/alternative-macros.h>
 #include <asm/fence.h>
@@ -409,6 +410,58 @@ static __always_inline void __cmpwait(volatile void *ptr,
 
 #define __cmpwait_relaxed(ptr, val) \
 	__cmpwait((ptr), (unsigned long)(val), sizeof(*(ptr)))
+
+#ifdef CONFIG_HAVE_CMPXCHG_DOUBLE
+#define system_has_cmpxchg_double()	1
+
+#define __cmpxchg_double_check(ptr1, ptr2)				\
+({									\
+	if (sizeof(*(ptr1)) != 4)					\
+		BUILD_BUG();						\
+	if (sizeof(*(ptr2)) != 4)					\
+		BUILD_BUG();						\
+	VM_BUG_ON((ulong *)(ptr2) - (ulong *)(ptr1) != 1);		\
+	VM_BUG_ON(((ulong)ptr1 & 0x7) != 0);				\
+})
+
+#define __cmpxchg_double(old1, old2, new1, new2, ptr)			\
+({									\
+	__typeof__(ptr) __ptr = (ptr);					\
+	register unsigned int __ret;					\
+	u64 __old;							\
+	u64 __new;							\
+	u64 __tmp;							\
+	switch (sizeof(*(ptr))) {					\
+	case 4:								\
+		__old = ((u64)old2 << 32) | (u64)old1;			\
+		__new = ((u64)new2 << 32) | (u64)new1;			\
+		__asm__ __volatile__ (					\
+			"0:	lr.d %0, %2\n"				\
+			"	bne %0, %z3, 1f\n"			\
+			"	sc.d %1, %z4, %2\n"			\
+			"	bnez %1, 0b\n"				\
+			"1:\n"						\
+			: "=&r" (__tmp), "=&r" (__ret), "+A" (*__ptr)	\
+			: "rJ" (__old), "rJ" (__new)			\
+			: "memory");					\
+		__ret = (__old == __tmp);				\
+		break;							\
+	default:							\
+		BUILD_BUG();						\
+	}								\
+	__ret;								\
+})
+
+#define arch_cmpxchg_double(ptr1, ptr2, o1, o2, n1, n2)			\
+({									\
+	int __ret;							\
+	__cmpxchg_double_check(ptr1, ptr2);				\
+	__ret = __cmpxchg_double((ulong)(o1), (ulong)(o2),		\
+				 (ulong)(n1), (ulong)(n2),		\
+				  ptr1);				\
+	__ret;								\
+})
+#endif
 #endif
 
 #endif /* _ASM_RISCV_CMPXCHG_H */
-- 
2.40.1