| lists.openwall.net | lists / announce owl-users owl-dev john-users john-dev passwdqc-users yescrypt popa3d-users / oss-security kernel-hardening musl sabotage tlsify passwords / crypt-dev xvendor / Bugtraq Full-Disclosure linux-kernel linux-netdev linux-ext4 linux-hardening linux-cve-announce PHC | |
|
Open Source and information security mailing list archives
| ||
|
Message-ID: <CAAhV-H5av6GOMjXNbkaT=PoS-X=ccQYS41Tqq2cs3ubus6phrA@mail.gmail.com>
Date: Sun, 15 May 2022 22:47:05 +0800
From: Huacai Chen <chenhuacai@...il.com>
To: WANG Xuerui <kernel@...0n.name>
Cc: Huacai Chen <chenhuacai@...ngson.cn>,
Arnd Bergmann <arnd@...db.de>,
Andy Lutomirski <luto@...nel.org>,
Thomas Gleixner <tglx@...utronix.de>,
Peter Zijlstra <peterz@...radead.org>,
Andrew Morton <akpm@...ux-foundation.org>,
David Airlie <airlied@...ux.ie>,
Jonathan Corbet <corbet@....net>,
Linus Torvalds <torvalds@...ux-foundation.org>,
linux-arch <linux-arch@...r.kernel.org>,
"open list:DOCUMENTATION" <linux-doc@...r.kernel.org>,
LKML <linux-kernel@...r.kernel.org>,
Xuefeng Li <lixuefeng@...ngson.cn>,
Yanteng Si <siyanteng@...ngson.cn>,
Guo Ren <guoren@...nel.org>,
Jiaxun Yang <jiaxun.yang@...goat.com>,
Stephen Rothwell <sfr@...b.auug.org.au>
Subject: Re: [PATCH V10 20/22] LoongArch: Add multi-processor (SMP) support
Hi, Xuerui,
On Sun, May 15, 2022 at 10:16 PM WANG Xuerui <kernel@...0n.name> wrote:
>
> Hi,
>
> On 5/14/22 16:04, Huacai Chen wrote:
> > LoongArch-based procesors have 4, 8 or 16 cores per package. This patch
> > adds multi-processor (SMP) support for LoongArch.
> >
> > Signed-off-by: Huacai Chen <chenhuacai@...ngson.cn>
> > ---
> > arch/loongarch/Kconfig | 44 +-
> > arch/loongarch/include/asm/atomic.h | 4 +
> > arch/loongarch/include/asm/barrier.h | 108 ++++
> > arch/loongarch/include/asm/cmpxchg.h | 1 +
> > arch/loongarch/include/asm/futex.h | 1 +
> > arch/loongarch/include/asm/hardirq.h | 2 +
> > arch/loongarch/include/asm/irq.h | 3 +
> > arch/loongarch/include/asm/percpu.h | 194 ++++++
> > arch/loongarch/include/asm/pgtable.h | 21 +
> > arch/loongarch/include/asm/smp.h | 124 ++++
> > arch/loongarch/include/asm/stackframe.h | 17 +-
> > arch/loongarch/include/asm/tlbflush.h | 13 +
> > arch/loongarch/include/asm/topology.h | 7 +-
> > arch/loongarch/kernel/Makefile | 2 +
> > arch/loongarch/kernel/acpi.c | 70 ++-
> > arch/loongarch/kernel/asm-offsets.c | 10 +
> > arch/loongarch/kernel/head.S | 30 +
> > arch/loongarch/kernel/irq.c | 13 +-
> > arch/loongarch/kernel/proc.c | 5 +
> > arch/loongarch/kernel/process.c | 7 +
> > arch/loongarch/kernel/reset.c | 12 +
> > arch/loongarch/kernel/setup.c | 26 +
> > arch/loongarch/kernel/smp.c | 744 ++++++++++++++++++++++++
> > arch/loongarch/kernel/topology.c | 43 +-
> > arch/loongarch/kernel/vmlinux.lds.S | 4 +
> > arch/loongarch/mm/tlbex.S | 69 +++
> > include/linux/cpuhotplug.h | 1 +
> > 27 files changed, 1564 insertions(+), 11 deletions(-)
> > create mode 100644 arch/loongarch/include/asm/smp.h
> > create mode 100644 arch/loongarch/kernel/smp.c
> >
> > diff --git a/arch/loongarch/Kconfig b/arch/loongarch/Kconfig
> > index 564bd6d2c2b9..f1053d77f06e 100644
> > --- a/arch/loongarch/Kconfig
> > +++ b/arch/loongarch/Kconfig
> > @@ -65,6 +65,7 @@ config LOONGARCH
> > select GENERIC_LIB_UCMPDI2
> > select GENERIC_PCI_IOMAP
> > select GENERIC_SCHED_CLOCK
> > + select GENERIC_SMP_IDLE_THREAD
> > select GENERIC_TIME_VSYSCALL
> > select GPIOLIB
> > select HAVE_ARCH_AUDITSYSCALL
> > @@ -95,7 +96,7 @@ config LOONGARCH
> > select HAVE_RSEQ
> > select HAVE_SYSCALL_TRACEPOINTS
> > select HAVE_TIF_NOHZ
> > - select HAVE_VIRT_CPU_ACCOUNTING_GEN
> > + select HAVE_VIRT_CPU_ACCOUNTING_GEN if !SMP
> > select IRQ_FORCED_THREADING
> > select IRQ_LOONGARCH_CPU
> > select MODULES_USE_ELF_RELA if MODULES
> > @@ -312,6 +313,47 @@ config EFI_STUB
> > This kernel feature allows the kernel to be loaded directly by
> > EFI firmware without the use of a bootloader.
> >
> > +config SMP
> > + bool "Multi-Processing support"
> > + help
> > + This enables support for systems with more than one CPU. If you have
> > + a system with only one CPU, say N. If you have a system with more
> > + than one CPU, say Y.
> > +
> > + If you say N here, the kernel will run on uni- and multiprocessor
> > + machines, but will use only one CPU of a multiprocessor machine. If
> > + you say Y here, the kernel will run on many, but not all,
> > + uniprocessor machines. On a uniprocessor machine, the kernel
> > + will run faster if you say N here.
> > +
> > + People using multiprocessor machines who say Y here should also say
> > + Y to "Enhanced Real Time Clock Support", below.
> This paragraph could be removed as it's not applicable to us. (We don't
> have an "Enhanced RTC support" config option.)
OK, thanks.
> > +
> > + See also the SMP-HOWTO available at
> > + <http://www.tldp.org/docs.html#howto>.
> > +
> > + If you don't know what to do here, say N.
> > +
> > +config HOTPLUG_CPU
> > + bool "Support for hot-pluggable CPUs"
> > + depends on SMP
> > + select GENERIC_IRQ_MIGRATION
> > + help
> > + Say Y here to allow turning CPUs off and on. CPUs can be
> > + controlled through /sys/devices/system/cpu.
> > + (Note: power management support will enable this option
> > + automatically on SMP systems. )
> > + Say N if you want to disable CPU hotplug.
> > +
> > +config NR_CPUS
> > + int "Maximum number of CPUs (2-256)"
> > + range 2 256
> > + depends on SMP
> > + default "16"
> > + help
> > + This allows you to specify the maximum number of CPUs which this
> > + kernel will support.
> > +
> > config FORCE_MAX_ZONEORDER
> > int "Maximum zone order"
> > range 14 64 if PAGE_SIZE_64KB
> > diff --git a/arch/loongarch/include/asm/atomic.h b/arch/loongarch/include/asm/atomic.h
> > index 932352342b12..979367ad4e2c 100644
> > --- a/arch/loongarch/include/asm/atomic.h
> > +++ b/arch/loongarch/include/asm/atomic.h
> > @@ -162,6 +162,7 @@ static inline int arch_atomic_sub_if_positive(int i, atomic_t *v)
> > " sc.w %1, %2 \n"
> > " beq $zero, %1, 1b \n"
> > "2: \n"
> > + __WEAK_LLSC_MB
> > : "=&r" (result), "=&r" (temp),
> > "+" GCC_OFF_SMALL_ASM() (v->counter)
> > : "I" (-i));
> > @@ -174,6 +175,7 @@ static inline int arch_atomic_sub_if_positive(int i, atomic_t *v)
> > " sc.w %1, %2 \n"
> > " beq $zero, %1, 1b \n"
> > "2: \n"
> > + __WEAK_LLSC_MB
> > : "=&r" (result), "=&r" (temp),
> > "+" GCC_OFF_SMALL_ASM() (v->counter)
> > : "r" (i));
> > @@ -323,6 +325,7 @@ static inline long arch_atomic64_sub_if_positive(long i, atomic64_t *v)
> > " sc.d %1, %2 \n"
> > " beq %1, $zero, 1b \n"
> > "2: \n"
> > + __WEAK_LLSC_MB
> > : "=&r" (result), "=&r" (temp),
> > "+" GCC_OFF_SMALL_ASM() (v->counter)
> > : "I" (-i));
> > @@ -335,6 +338,7 @@ static inline long arch_atomic64_sub_if_positive(long i, atomic64_t *v)
> > " sc.d %1, %2 \n"
> > " beq %1, $zero, 1b \n"
> > "2: \n"
> > + __WEAK_LLSC_MB
> > : "=&r" (result), "=&r" (temp),
> > "+" GCC_OFF_SMALL_ASM() (v->counter)
> > : "r" (i));
> > diff --git a/arch/loongarch/include/asm/barrier.h b/arch/loongarch/include/asm/barrier.h
> > index cc6c7e3f5ce6..6c567c750d04 100644
> > --- a/arch/loongarch/include/asm/barrier.h
> > +++ b/arch/loongarch/include/asm/barrier.h
> > @@ -18,6 +18,19 @@
> > #define mb() fast_mb()
> > #define iob() fast_iob()
> >
> > +#define __smp_mb() __asm__ __volatile__("dbar 0" : : : "memory")
> > +#define __smp_rmb() __asm__ __volatile__("dbar 0" : : : "memory")
> > +#define __smp_wmb() __asm__ __volatile__("dbar 0" : : : "memory")
>
> It's always unfortunate to see full barriers used in place of more
> fine-grained barriers, but that's all we have in LoongArch v1.00... This
> is life, I think, but make sure to tell the hardware engineers, that
> users will be thankful for the additional barriers. ;-)
OK, Loongson-3A6000 will have fine-grained barriers.
>
> > +
> > +#ifdef CONFIG_SMP
> > +#define __WEAK_LLSC_MB " dbar 0 \n"
> > +#else
> > +#define __WEAK_LLSC_MB " \n"
> > +#endif
> > +
> > +#define __smp_mb__before_atomic() barrier()
> > +#define __smp_mb__after_atomic() barrier()
> > +
> > /**
> > * array_index_mask_nospec() - generate a ~0 mask when index < size, 0 otherwise
> > * @index: array element index
> > @@ -46,6 +59,101 @@ static inline unsigned long array_index_mask_nospec(unsigned long index,
> > return mask;
> > }
> >
> > +#define __smp_load_acquire(p) \
> > +({ \
> > + union { typeof(*p) __val; char __c[1]; } __u; \
> > + unsigned long __tmp = 0; \
> > + compiletime_assert_atomic_type(*p); \
> > + switch (sizeof(*p)) { \
> > + case 1: \
> > + *(__u8 *)__u.__c = *(volatile __u8 *)p; \
> > + __smp_mb(); \
> > + break; \
> > + case 2: \
> > + *(__u16 *)__u.__c = *(volatile __u16 *)p; \
> > + __smp_mb(); \
> > + break; \
> > + case 4: \
> > + __asm__ __volatile__( \
> > + "amor_db.w %[val], %[tmp], %[mem] \n" \
> > + : [val] "=&r" (*(__u32 *)__u.__c) \
> > + : [mem] "ZB" (*(u32 *) p), [tmp] "r" (__tmp) \
> > + : "memory"); \
> > + break; \
> > + case 8: \
> > + __asm__ __volatile__( \
> > + "amor_db.d %[val], %[tmp], %[mem] \n" \
> > + : [val] "=&r" (*(__u64 *)__u.__c) \
> > + : [mem] "ZB" (*(u64 *) p), [tmp] "r" (__tmp) \
> > + : "memory"); \
> > + break; \
> This is a little bit scary; while it's working right now it's perhaps
> better to, if possible, rewrite these with generic helpers such as
> READ_ONCE/WRITE_ONCE, like what arch/riscv has done. Cleanups can always
> come later...
> > + } \
> > + (typeof(*p))__u.__val; \
> > +})
> > +
> > +#define __smp_store_release(p, v) \
> > +do { \
> > + union { typeof(*p) __val; char __c[1]; } __u = \
> > + { .__val = (__force typeof(*p)) (v) }; \
> > + unsigned long __tmp; \
> > + compiletime_assert_atomic_type(*p); \
> > + switch (sizeof(*p)) { \
> > + case 1: \
> > + __smp_mb(); \
> > + *(volatile __u8 *)p = *(__u8 *)__u.__c; \
> > + break; \
> > + case 2: \
> > + __smp_mb(); \
> > + *(volatile __u16 *)p = *(__u16 *)__u.__c; \
> > + break; \
> > + case 4: \
> > + __asm__ __volatile__( \
> > + "amswap_db.w %[tmp], %[val], %[mem] \n" \
> > + : [mem] "+ZB" (*(u32 *)p), [tmp] "=&r" (__tmp) \
> > + : [val] "r" (*(__u32 *)__u.__c) \
> > + : ); \
> > + break; \
> > + case 8: \
> > + __asm__ __volatile__( \
> > + "amswap_db.d %[tmp], %[val], %[mem] \n" \
> > + : [mem] "+ZB" (*(u64 *)p), [tmp] "=&r" (__tmp) \
> > + : [val] "r" (*(__u64 *)__u.__c) \
> > + : ); \
> > + break; \
> > + } \
> > +} while (0)
> > +
> > +#define __smp_store_mb(p, v) \
> > +do { \
> > + union { typeof(p) __val; char __c[1]; } __u = \
> > + { .__val = (__force typeof(p)) (v) }; \
> > + unsigned long __tmp; \
> > + switch (sizeof(p)) { \
> > + case 1: \
> > + *(volatile __u8 *)&p = *(__u8 *)__u.__c; \
> > + __smp_mb(); \
> > + break; \
> > + case 2: \
> > + *(volatile __u16 *)&p = *(__u16 *)__u.__c; \
> > + __smp_mb(); \
> > + break; \
> > + case 4: \
> > + __asm__ __volatile__( \
> > + "amswap_db.w %[tmp], %[val], %[mem] \n" \
> > + : [mem] "+ZB" (*(u32 *)&p), [tmp] "=&r" (__tmp) \
> > + : [val] "r" (*(__u32 *)__u.__c) \
> > + : ); \
> > + break; \
> > + case 8: \
> > + __asm__ __volatile__( \
> > + "amswap_db.d %[tmp], %[val], %[mem] \n" \
> > + : [mem] "+ZB" (*(u64 *)&p), [tmp] "=&r" (__tmp) \
> > + : [val] "r" (*(__u64 *)__u.__c) \
> > + : ); \
> > + break; \
> > + } \
> > +} while (0)
> > +
> > #include <asm-generic/barrier.h>
> >
> > #endif /* __ASM_BARRIER_H */
> > diff --git a/arch/loongarch/include/asm/cmpxchg.h b/arch/loongarch/include/asm/cmpxchg.h
> > index 699925b8ff27..7a880651161b 100644
> > --- a/arch/loongarch/include/asm/cmpxchg.h
> > +++ b/arch/loongarch/include/asm/cmpxchg.h
> > @@ -58,6 +58,7 @@ static inline unsigned long __xchg(volatile void *ptr, unsigned long x,
> > " " st " $t0, %1 \n" \
> > " beq $zero, $t0, 1b \n" \
> > "2: \n" \
> > + __WEAK_LLSC_MB \
> > : "=&r" (__ret), "=ZB"(*m) \
> > : "ZB"(*m), "Jr" (old), "Jr" (new) \
> > : "t0", "memory"); \
> > diff --git a/arch/loongarch/include/asm/futex.h b/arch/loongarch/include/asm/futex.h
> > index b27d55f92db7..9de8231694ec 100644
> > --- a/arch/loongarch/include/asm/futex.h
> > +++ b/arch/loongarch/include/asm/futex.h
> > @@ -86,6 +86,7 @@ futex_atomic_cmpxchg_inatomic(u32 *uval, u32 __user *uaddr, u32 oldval, u32 newv
> > "2: sc.w $t0, %2 \n"
> > " beq $zero, $t0, 1b \n"
> > "3: \n"
> > + __WEAK_LLSC_MB
> > " .section .fixup,\"ax\" \n"
> > "4: li.d %0, %6 \n"
> > " b 3b \n"
> > diff --git a/arch/loongarch/include/asm/hardirq.h b/arch/loongarch/include/asm/hardirq.h
> > index d32f83938880..befe8184aa08 100644
> > --- a/arch/loongarch/include/asm/hardirq.h
> > +++ b/arch/loongarch/include/asm/hardirq.h
> > @@ -21,4 +21,6 @@ typedef struct {
> >
> > DECLARE_PER_CPU_ALIGNED(irq_cpustat_t, irq_stat);
> >
> > +#define __ARCH_IRQ_STAT
> > +
> > #endif /* _ASM_HARDIRQ_H */
> > diff --git a/arch/loongarch/include/asm/irq.h b/arch/loongarch/include/asm/irq.h
> > index cd95d0d4e10f..ca3a68767b54 100644
> > --- a/arch/loongarch/include/asm/irq.h
> > +++ b/arch/loongarch/include/asm/irq.h
> > @@ -125,6 +125,9 @@ extern struct irq_domain *pch_lpc_domain;
> > extern struct irq_domain *pch_msi_domain[MAX_IO_PICS];
> > extern struct irq_domain *pch_pic_domain[MAX_IO_PICS];
> >
> > +extern void fixup_irqs(void);
> > +extern irqreturn_t loongson3_ipi_interrupt(int irq, void *dev);
> > +
> > #include <asm-generic/irq.h>
> >
> > #endif /* _ASM_IRQ_H */
> > diff --git a/arch/loongarch/include/asm/percpu.h b/arch/loongarch/include/asm/percpu.h
> > index 7d5b22ebd834..6f0d6e6fde10 100644
> > --- a/arch/loongarch/include/asm/percpu.h
> > +++ b/arch/loongarch/include/asm/percpu.h
> > @@ -5,6 +5,8 @@
> > #ifndef __ASM_PERCPU_H
> > #define __ASM_PERCPU_H
> >
> > +#include <asm/cmpxchg.h>
> > +
> > /* Use r21 for fast access */
> > register unsigned long __my_cpu_offset __asm__("$r21");
> >
> > @@ -15,6 +17,198 @@ static inline void set_my_cpu_offset(unsigned long off)
> > }
> > #define __my_cpu_offset __my_cpu_offset
> >
> > +#define PERCPU_OP(op, asm_op, c_op) \
> > +static inline unsigned long __percpu_##op(void *ptr, \
> > + unsigned long val, int size) \
> > +{ \
> > + unsigned long ret; \
> > + \
> > + switch (size) { \
> > + case 4: \
> > + __asm__ __volatile__( \
> > + "am"#asm_op".w" " %[ret], %[val], %[ptr] \n" \
> > + : [ret] "=&r" (ret), [ptr] "+ZB"(*(u32 *)ptr) \
> > + : [val] "r" (val)); \
> > + break; \
> > + case 8: \
> > + __asm__ __volatile__( \
> > + "am"#asm_op".d" " %[ret], %[val], %[ptr] \n" \
> > + : [ret] "=&r" (ret), [ptr] "+ZB"(*(u64 *)ptr) \
> > + : [val] "r" (val)); \
> > + break; \
> > + default: \
> > + ret = 0; \
> > + BUILD_BUG(); \
> > + } \
> > + \
> > + return ret c_op val; \
> > +}
> > +
> > +PERCPU_OP(add, add, +)
> > +PERCPU_OP(and, and, &)
> > +PERCPU_OP(or, or, |)
> > +#undef PERCPU_OP
> > +
> > +static inline unsigned long __percpu_read(void *ptr, int size)
> > +{
> > + unsigned long ret;
> > +
> > + switch (size) {
> > + case 1:
> > + __asm__ __volatile__ ("ldx.b %[ret], $r21, %[ptr] \n"
> > + : [ret] "=&r"(ret)
> > + : [ptr] "r"(ptr)
> > + : "memory");
> > + break;
> > + case 2:
> > + __asm__ __volatile__ ("ldx.h %[ret], $r21, %[ptr] \n"
> > + : [ret] "=&r"(ret)
> > + : [ptr] "r"(ptr)
> > + : "memory");
> > + break;
> > + case 4:
> > + __asm__ __volatile__ ("ldx.w %[ret], $r21, %[ptr] \n"
> > + : [ret] "=&r"(ret)
> > + : [ptr] "r"(ptr)
> > + : "memory");
> > + break;
> > + case 8:
> > + __asm__ __volatile__ ("ldx.d %[ret], $r21, %[ptr] \n"
> > + : [ret] "=&r"(ret)
> > + : [ptr] "r"(ptr)
> > + : "memory");
> > + break;
> > + default:
> > + ret = 0;
> > + BUILD_BUG();
> > + }
> > +
> > + return ret;
> > +}
> > +
> > +static inline void __percpu_write(void *ptr, unsigned long val, int size)
> > +{
> > + switch (size) {
> > + case 1:
> > + __asm__ __volatile__("stx.b %[val], $r21, %[ptr] \n"
> > + :
> > + : [val] "r" (val), [ptr] "r" (ptr)
> > + : "memory");
> > + break;
> > + case 2:
> > + __asm__ __volatile__("stx.h %[val], $r21, %[ptr] \n"
> > + :
> > + : [val] "r" (val), [ptr] "r" (ptr)
> > + : "memory");
> > + break;
> > + case 4:
> > + __asm__ __volatile__("stx.w %[val], $r21, %[ptr] \n"
> > + :
> > + : [val] "r" (val), [ptr] "r" (ptr)
> > + : "memory");
> > + break;
> > + case 8:
> > + __asm__ __volatile__("stx.d %[val], $r21, %[ptr] \n"
> > + :
> > + : [val] "r" (val), [ptr] "r" (ptr)
> > + : "memory");
> > + break;
> > + default:
> > + BUILD_BUG();
> > + }
> > +}
> > +
> > +static inline unsigned long __percpu_xchg(void *ptr, unsigned long val,
> > + int size)
> > +{
> > + switch (size) {
> > + case 4:
> > + return __xchg_asm("amswap.w", (volatile u32 *)ptr, (u32)val);
> > +
> > + case 8:
> > + return __xchg_asm("amswap.d", (volatile u64 *)ptr, (u64)val);
> > +
> > + default:
> > + BUILD_BUG();
> > + }
> > +
> > + return 0;
> > +}
> > +
> > +/* this_cpu_cmpxchg */
> > +#define _protect_cmpxchg_local(pcp, o, n) \
> > +({ \
> > + typeof(*raw_cpu_ptr(&(pcp))) __ret; \
> > + preempt_disable_notrace(); \
> > + __ret = cmpxchg_local(raw_cpu_ptr(&(pcp)), o, n); \
> > + preempt_enable_notrace(); \
> > + __ret; \
> > +})
> > +
> > +#define _percpu_read(pcp) \
> > +({ \
> > + typeof(pcp) __retval; \
> > + __retval = (typeof(pcp))__percpu_read(&(pcp), sizeof(pcp)); \
> > + __retval; \
> > +})
> > +
> > +#define _percpu_write(pcp, val) \
> > +do { \
> > + __percpu_write(&(pcp), (unsigned long)(val), sizeof(pcp)); \
> > +} while (0) \
> > +
> > +#define _pcp_protect(operation, pcp, val) \
> > +({ \
> > + typeof(pcp) __retval; \
> > + preempt_disable_notrace(); \
> > + __retval = (typeof(pcp))operation(raw_cpu_ptr(&(pcp)), \
> > + (val), sizeof(pcp)); \
> > + preempt_enable_notrace(); \
> > + __retval; \
> > +})
> > +
> > +#define _percpu_add(pcp, val) \
> > + _pcp_protect(__percpu_add, pcp, val)
> > +
> > +#define _percpu_add_return(pcp, val) _percpu_add(pcp, val)
> > +
> > +#define _percpu_and(pcp, val) \
> > + _pcp_protect(__percpu_and, pcp, val)
> > +
> > +#define _percpu_or(pcp, val) \
> > + _pcp_protect(__percpu_or, pcp, val)
> > +
> > +#define _percpu_xchg(pcp, val) ((typeof(pcp)) \
> > + _pcp_protect(__percpu_xchg, pcp, (unsigned long)(val)))
> > +
> > +#define this_cpu_add_4(pcp, val) _percpu_add(pcp, val)
> > +#define this_cpu_add_8(pcp, val) _percpu_add(pcp, val)
> > +
> > +#define this_cpu_add_return_4(pcp, val) _percpu_add_return(pcp, val)
> > +#define this_cpu_add_return_8(pcp, val) _percpu_add_return(pcp, val)
> > +
> > +#define this_cpu_and_4(pcp, val) _percpu_and(pcp, val)
> > +#define this_cpu_and_8(pcp, val) _percpu_and(pcp, val)
> > +
> > +#define this_cpu_or_4(pcp, val) _percpu_or(pcp, val)
> > +#define this_cpu_or_8(pcp, val) _percpu_or(pcp, val)
> > +
> > +#define this_cpu_read_1(pcp) _percpu_read(pcp)
> > +#define this_cpu_read_2(pcp) _percpu_read(pcp)
> > +#define this_cpu_read_4(pcp) _percpu_read(pcp)
> > +#define this_cpu_read_8(pcp) _percpu_read(pcp)
> > +
> > +#define this_cpu_write_1(pcp, val) _percpu_write(pcp, val)
> > +#define this_cpu_write_2(pcp, val) _percpu_write(pcp, val)
> > +#define this_cpu_write_4(pcp, val) _percpu_write(pcp, val)
> > +#define this_cpu_write_8(pcp, val) _percpu_write(pcp, val)
> > +
> > +#define this_cpu_xchg_4(pcp, val) _percpu_xchg(pcp, val)
> > +#define this_cpu_xchg_8(pcp, val) _percpu_xchg(pcp, val)
> > +
> > +#define this_cpu_cmpxchg_4(ptr, o, n) _protect_cmpxchg_local(ptr, o, n)
> > +#define this_cpu_cmpxchg_8(ptr, o, n) _protect_cmpxchg_local(ptr, o, n)
> > +
> > #include <asm-generic/percpu.h>
> >
> > #endif /* __ASM_PERCPU_H */
> > diff --git a/arch/loongarch/include/asm/pgtable.h b/arch/loongarch/include/asm/pgtable.h
> > index 8920dd8b297b..5e33987d0a13 100644
> > --- a/arch/loongarch/include/asm/pgtable.h
> > +++ b/arch/loongarch/include/asm/pgtable.h
> > @@ -279,8 +279,29 @@ static inline void set_pte(pte_t *ptep, pte_t pteval)
> > * Make sure the buddy is global too (if it's !none,
> > * it better already be global)
> > */
> > +#ifdef CONFIG_SMP
> > + /*
> > + * For SMP, multiple CPUs can race, so we need to do
> > + * this atomically.
> > + */
> > + unsigned long page_global = _PAGE_GLOBAL;
> > + unsigned long tmp;
> > +
> > + __asm__ __volatile__ (
> > + "1:" __LL "%[tmp], %[buddy] \n"
> > + " bnez %[tmp], 2f \n"
> > + " or %[tmp], %[tmp], %[global] \n"
> > + __SC "%[tmp], %[buddy] \n"
> > + " beqz %[tmp], 1b \n"
> > + " nop \n"
> > + "2: \n"
> > + __WEAK_LLSC_MB
> > + : [buddy] "+m" (buddy->pte), [tmp] "=&r" (tmp)
> > + : [global] "r" (page_global));
> > +#else /* !CONFIG_SMP */
> > if (pte_none(*buddy))
> > pte_val(*buddy) = pte_val(*buddy) | _PAGE_GLOBAL;
> > +#endif /* CONFIG_SMP */
> > }
> > }
> >
> > diff --git a/arch/loongarch/include/asm/smp.h b/arch/loongarch/include/asm/smp.h
> > new file mode 100644
> > index 000000000000..551e1f37c705
> > --- /dev/null
> > +++ b/arch/loongarch/include/asm/smp.h
> > @@ -0,0 +1,124 @@
> > +/* SPDX-License-Identifier: GPL-2.0 */
> > +/*
> > + * Author: Huacai Chen <chenhuacai@...ngson.cn>
> > + * Copyright (C) 2020-2022 Loongson Technology Corporation Limited
> > + */
> > +#ifndef __ASM_SMP_H
> > +#define __ASM_SMP_H
> > +
> > +#include <linux/atomic.h>
> > +#include <linux/bitops.h>
> > +#include <linux/linkage.h>
> > +#include <linux/smp.h>
> > +#include <linux/threads.h>
> > +#include <linux/cpumask.h>
> > +
> > +void loongson3_smp_setup(void);
> > +void loongson3_prepare_cpus(unsigned int max_cpus);
> > +void loongson3_boot_secondary(int cpu, struct task_struct *idle);
> > +void loongson3_init_secondary(void);
> > +void loongson3_smp_finish(void);
> > +void loongson3_send_ipi_single(int cpu, unsigned int action);
> > +void loongson3_send_ipi_mask(const struct cpumask *mask, unsigned int action);
> > +#ifdef CONFIG_HOTPLUG_CPU
> > +int loongson3_cpu_disable(void);
> > +void loongson3_cpu_die(unsigned int cpu);
> > +#endif
> > +
> > +#ifdef CONFIG_SMP
> > +
> > +static inline void plat_smp_setup(void)
> > +{
> > + loongson3_smp_setup();
> > +}
> > +
> > +#else /* !CONFIG_SMP */
> > +
> > +static inline void plat_smp_setup(void) { }
> > +
> > +#endif /* !CONFIG_SMP */
> > +
> > +extern int smp_num_siblings;
> > +extern int num_processors;
> > +extern int disabled_cpus;
> > +extern cpumask_t cpu_sibling_map[];
> > +extern cpumask_t cpu_core_map[];
> > +extern cpumask_t cpu_foreign_map[];
> > +
> > +static inline int raw_smp_processor_id(void)
> > +{
> > +#if defined(__VDSO__)
> > + extern int vdso_smp_processor_id(void)
> > + __compiletime_error("VDSO should not call smp_processor_id()");
> > + return vdso_smp_processor_id();
> > +#else
> > + return current_thread_info()->cpu;
> > +#endif
> > +}
> > +#define raw_smp_processor_id raw_smp_processor_id
> > +
> > +/* Map from cpu id to sequential logical cpu number. This will only
> > + * not be idempotent when cpus failed to come on-line. */
> > +extern int __cpu_number_map[NR_CPUS];
> > +#define cpu_number_map(cpu) __cpu_number_map[cpu]
> > +
> > +/* The reverse map from sequential logical cpu number to cpu id. */
> > +extern int __cpu_logical_map[NR_CPUS];
> > +#define cpu_logical_map(cpu) __cpu_logical_map[cpu]
> > +
> > +#define cpu_physical_id(cpu) cpu_logical_map(cpu)
> > +
> > +#define SMP_BOOT_CPU 0x1
> > +#define SMP_RESCHEDULE 0x2
> > +#define SMP_CALL_FUNCTION 0x4
> > +
> > +struct secondary_data {
> > + unsigned long stack;
> > + unsigned long thread_info;
> > +};
> > +extern struct secondary_data cpuboot_data;
> > +
> > +extern asmlinkage void smpboot_entry(void);
> > +
> > +extern void calculate_cpu_foreign_map(void);
> > +
> > +/*
> > + * Generate IPI list text
> > + */
> > +extern void show_ipi_list(struct seq_file *p, int prec);
> > +
> > +/*
> > + * This function sends a 'reschedule' IPI to another CPU.
> > + * it goes straight through and wastes no time serializing
> > + * anything. Worst case is that we lose a reschedule ...
> > + */
> > +static inline void smp_send_reschedule(int cpu)
> > +{
> > + loongson3_send_ipi_single(cpu, SMP_RESCHEDULE);
> > +}
> > +
> > +static inline void arch_send_call_function_single_ipi(int cpu)
> > +{
> > + loongson3_send_ipi_single(cpu, SMP_CALL_FUNCTION);
> > +}
> > +
> > +static inline void arch_send_call_function_ipi_mask(const struct cpumask *mask)
> > +{
> > + loongson3_send_ipi_mask(mask, SMP_CALL_FUNCTION);
> > +}
> > +
> > +#ifdef CONFIG_HOTPLUG_CPU
> > +static inline int __cpu_disable(void)
> > +{
> > + return loongson3_cpu_disable();
> > +}
> > +
> > +static inline void __cpu_die(unsigned int cpu)
> > +{
> > + loongson3_cpu_die(cpu);
> > +}
> > +
> > +extern void play_dead(void);
> > +#endif
> > +
> > +#endif /* __ASM_SMP_H */
> > diff --git a/arch/loongarch/include/asm/stackframe.h b/arch/loongarch/include/asm/stackframe.h
> > index 44151b878d00..4ca953062b5b 100644
> > --- a/arch/loongarch/include/asm/stackframe.h
> > +++ b/arch/loongarch/include/asm/stackframe.h
> > @@ -77,17 +77,24 @@
> > * new value in sp.
> > */
> > .macro get_saved_sp docfi=0
> > - la.abs t1, kernelsp
> > - move t0, sp
> > + la.abs t1, kernelsp
> > +#ifdef CONFIG_SMP
> > + csrrd t0, PERCPU_BASE_KS
> > + LONG_ADD t1, t1, t0
> > +#endif
> > + move t0, sp
> > .if \docfi
> > .cfi_register sp, t0
> > .endif
> > - LONG_L sp, t1, 0
> > + LONG_L sp, t1, 0
> > .endm
> >
> > .macro set_saved_sp stackp temp temp2
> > - la.abs \temp, kernelsp
> > - LONG_S \stackp, \temp, 0
> > + la.abs \temp, kernelsp
> > +#ifdef CONFIG_SMP
> > + LONG_ADD \temp, \temp, u0
> > +#endif
> > + LONG_S \stackp, \temp, 0
> > .endm
> >
> > .macro SAVE_SOME docfi=0
> > diff --git a/arch/loongarch/include/asm/tlbflush.h b/arch/loongarch/include/asm/tlbflush.h
> > index 36bd6d11dc2d..a0785e590681 100644
> > --- a/arch/loongarch/include/asm/tlbflush.h
> > +++ b/arch/loongarch/include/asm/tlbflush.h
> > @@ -25,6 +25,17 @@ extern void local_flush_tlb_kernel_range(unsigned long start, unsigned long end)
> > extern void local_flush_tlb_page(struct vm_area_struct *vma, unsigned long page);
> > extern void local_flush_tlb_one(unsigned long vaddr);
> >
> > +#ifdef CONFIG_SMP
> > +
> > +extern void flush_tlb_all(void);
> > +extern void flush_tlb_mm(struct mm_struct *);
> > +extern void flush_tlb_range(struct vm_area_struct *vma, unsigned long, unsigned long);
> > +extern void flush_tlb_kernel_range(unsigned long, unsigned long);
> > +extern void flush_tlb_page(struct vm_area_struct *, unsigned long);
> > +extern void flush_tlb_one(unsigned long vaddr);
> > +
> > +#else /* CONFIG_SMP */
> > +
> > #define flush_tlb_all() local_flush_tlb_all()
> > #define flush_tlb_mm(mm) local_flush_tlb_mm(mm)
> > #define flush_tlb_range(vma, vmaddr, end) local_flush_tlb_range(vma, vmaddr, end)
> > @@ -32,4 +43,6 @@ extern void local_flush_tlb_one(unsigned long vaddr);
> > #define flush_tlb_page(vma, page) local_flush_tlb_page(vma, page)
> > #define flush_tlb_one(vaddr) local_flush_tlb_one(vaddr)
> >
> > +#endif /* CONFIG_SMP */
> > +
> > #endif /* __ASM_TLBFLUSH_H */
> > diff --git a/arch/loongarch/include/asm/topology.h b/arch/loongarch/include/asm/topology.h
> > index 9ac71a25207a..da135841e5b1 100644
> > --- a/arch/loongarch/include/asm/topology.h
> > +++ b/arch/loongarch/include/asm/topology.h
> > @@ -7,7 +7,12 @@
> >
> > #include <linux/smp.h>
> >
> > -#define cpu_logical_map(cpu) 0
> > +#ifdef CONFIG_SMP
> > +#define topology_physical_package_id(cpu) (cpu_data[cpu].package)
> > +#define topology_core_id(cpu) (cpu_data[cpu].core)
> > +#define topology_core_cpumask(cpu) (&cpu_core_map[cpu])
> > +#define topology_sibling_cpumask(cpu) (&cpu_sibling_map[cpu])
> > +#endif
> >
> > #include <asm-generic/topology.h>
> >
> > diff --git a/arch/loongarch/kernel/Makefile b/arch/loongarch/kernel/Makefile
> > index 211f12b3781c..08ddd8c90eae 100644
> > --- a/arch/loongarch/kernel/Makefile
> > +++ b/arch/loongarch/kernel/Makefile
> > @@ -19,4 +19,6 @@ obj-$(CONFIG_MODULES) += module.o module-sections.o
> >
> > obj-$(CONFIG_PROC_FS) += proc.o
> >
> > +obj-$(CONFIG_SMP) += smp.o
> > +
> > CPPFLAGS_vmlinux.lds := $(KBUILD_CFLAGS)
> > diff --git a/arch/loongarch/kernel/acpi.c b/arch/loongarch/kernel/acpi.c
> > index 506ab9912c51..0c7f2d1077a1 100644
> > --- a/arch/loongarch/kernel/acpi.c
> > +++ b/arch/loongarch/kernel/acpi.c
> > @@ -139,6 +139,35 @@ void __init acpi_boot_table_init(void)
> > }
> > }
> >
> > +static int set_processor_mask(u32 id, u32 flags)
> > +{
> > +
> > + int cpu, cpuid = id;
> > +
> > + if (num_processors >= nr_cpu_ids) {
> > + pr_warn("acpi: nr_cpus/possible_cpus limit of %i reached."
> pr_warn(PREFIX "nr_cpus ..."); -- make use of the common PREFIX declaration.
> > + " processor 0x%x ignored.\n", nr_cpu_ids, cpuid);
> > +
> > + return -ENODEV;
> > +
> > + }
> > + if (cpuid == loongson_sysconf.boot_cpu_id)
> > + cpu = 0;
> > + else
> > + cpu = cpumask_next_zero(-1, cpu_present_mask);
> > +
> > + if (flags & ACPI_MADT_ENABLED) {
> > + num_processors++;
> > + set_cpu_possible(cpu, true);
> > + set_cpu_present(cpu, true);
> > + __cpu_number_map[cpuid] = cpu;
> > + __cpu_logical_map[cpu] = cpuid;
> > + } else
> > + disabled_cpus++;
> > +
> > + return cpu;
> > +}
> > +
> > static int __init
> > acpi_parse_cpuintc(union acpi_subtable_headers *header, const unsigned long end)
> > {
> > @@ -149,6 +178,7 @@ acpi_parse_cpuintc(union acpi_subtable_headers *header, const unsigned long end)
> > return -EINVAL;
> >
> > acpi_table_print_madt_entry(&header->common);
> > + set_processor_mask(processor->core_id, processor->flags);
> >
> > return 0;
> > }
> > @@ -250,7 +280,12 @@ acpi_parse_pch_lpc(union acpi_subtable_headers *header, const unsigned long end)
> >
> > static void __init acpi_process_madt(void)
> > {
> > - int error;
> > + int i, error;
> > +
> > + for (i = 0; i < NR_CPUS; i++) {
> > + __cpu_number_map[i] = -1;
> > + __cpu_logical_map[i] = -1;
> > + }
> >
> > /* Parse MADT CPUINTC entries */
> > error = acpi_table_parse_madt(ACPI_MADT_TYPE_CORE_PIC, acpi_parse_cpuintc, MAX_CORE_PIC);
> > @@ -336,3 +371,36 @@ void __init arch_reserve_mem_area(acpi_physical_address addr, size_t size)
> > {
> > memblock_reserve(addr, size);
> > }
> > +
> > +#ifdef CONFIG_ACPI_HOTPLUG_CPU
> > +
> > +#include <acpi/processor.h>
> > +
> > +int acpi_map_cpu(acpi_handle handle, phys_cpuid_t physid, u32 acpi_id, int *pcpu)
> > +{
> > + int cpu;
> > +
> > + cpu = set_processor_mask(physid, ACPI_MADT_ENABLED);
> > + if (cpu < 0) {
> > + pr_info(PREFIX "Unable to map lapic to logical cpu number\n");
> > + return cpu;
> > + }
> > +
> > + *pcpu = cpu;
> > +
> > + return 0;
> > +}
> > +EXPORT_SYMBOL(acpi_map_cpu);
> > +
> > +int acpi_unmap_cpu(int cpu)
> > +{
> > + set_cpu_present(cpu, false);
> > + num_processors--;
> > +
> > + pr_info("cpu%d hot remove!\n", cpu);
> > +
> > + return 0;
> > +}
> > +EXPORT_SYMBOL(acpi_unmap_cpu);
> > +
> > +#endif /* CONFIG_ACPI_HOTPLUG_CPU */
> > diff --git a/arch/loongarch/kernel/asm-offsets.c b/arch/loongarch/kernel/asm-offsets.c
> > index 3531e3c60a6e..bfb65eb2844f 100644
> > --- a/arch/loongarch/kernel/asm-offsets.c
> > +++ b/arch/loongarch/kernel/asm-offsets.c
> > @@ -252,3 +252,13 @@ void output_signal_defines(void)
> > DEFINE(_SIGXFSZ, SIGXFSZ);
> > BLANK();
> > }
> > +
> > +#ifdef CONFIG_SMP
> > +void output_smpboot_defines(void)
> > +{
> > + COMMENT("Linux smp cpu boot offsets.");
> > + OFFSET(CPU_BOOT_STACK, secondary_data, stack);
> > + OFFSET(CPU_BOOT_TINFO, secondary_data, thread_info);
> > + BLANK();
> > +}
> > +#endif
> > diff --git a/arch/loongarch/kernel/head.S b/arch/loongarch/kernel/head.S
> > index f0b3e76bb762..9f6c9f3182a5 100644
> > --- a/arch/loongarch/kernel/head.S
> > +++ b/arch/loongarch/kernel/head.S
> > @@ -94,4 +94,34 @@ SYM_CODE_START(kernel_entry) # kernel entry point
> >
> > SYM_CODE_END(kernel_entry)
> >
> > +#ifdef CONFIG_SMP
> > +
> > +/*
> > + * SMP slave cpus entry point. Board specific code for bootstrap calls this
> > + * function after setting up the stack and tp registers.
> > + */
> > +SYM_CODE_START(smpboot_entry)
> > + li.d t0, CSR_DMW0_INIT # UC, PLV0
> > + csrwr t0, LOONGARCH_CSR_DMWIN0
> > + li.d t0, CSR_DMW1_INIT # CA, PLV0
> > + csrwr t0, LOONGARCH_CSR_DMWIN1
> > + li.w t0, 0xb0 # PLV=0, IE=0, PG=1
> > + csrwr t0, LOONGARCH_CSR_CRMD
> > + li.w t0, 0x04 # PLV=0, PIE=1, PWE=0
> > + csrwr t0, LOONGARCH_CSR_PRMD
> > + li.w t0, 0x00 # FPE=0, SXE=0, ASXE=0, BTE=0
> > + csrwr t0, LOONGARCH_CSR_EUEN
> > +
> > + la.abs t0, cpuboot_data
> > + ld.d sp, t0, CPU_BOOT_STACK
> > + ld.d tp, t0, CPU_BOOT_TINFO
> > +
> > + la.abs t0, 0f
> > + jirl zero, t0, 0
> > +0:
> > + bl start_secondary
> > +SYM_CODE_END(smpboot_entry)
> > +
> > +#endif /* CONFIG_SMP */
> > +
> > SYM_ENTRY(kernel_entry_end, SYM_L_GLOBAL, SYM_A_NONE)
> > diff --git a/arch/loongarch/kernel/irq.c b/arch/loongarch/kernel/irq.c
> > index 48032ffd9331..fbb7ad1d1ac0 100644
> > --- a/arch/loongarch/kernel/irq.c
> > +++ b/arch/loongarch/kernel/irq.c
> > @@ -73,6 +73,9 @@ asmlinkage void spurious_interrupt(void)
> >
> > int arch_show_interrupts(struct seq_file *p, int prec)
> > {
> > +#ifdef CONFIG_SMP
> > + show_ipi_list(p, prec);
> > +#endif
> > seq_printf(p, "%*s: %10u\n", prec, "ERR", atomic_read(&irq_err_count));
> > return 0;
> > }
> > @@ -108,13 +111,21 @@ void __init setup_IRQ(void)
> >
> > void __init init_IRQ(void)
> > {
> > - int i;
> > + int i, r, ipi_irq;
> > + static int ipi_dummy_dev;
> > unsigned int order = get_order(IRQ_STACK_SIZE);
> >
> > clear_csr_ecfg(ECFG0_IM);
> > clear_csr_estat(ESTATF_IP);
> >
> > setup_IRQ();
> > +#ifdef CONFIG_SMP
> > + ipi_irq = get_ipi_irq();
> > + irq_set_percpu_devid(ipi_irq);
> > + r = request_percpu_irq(ipi_irq, loongson3_ipi_interrupt, "IPI", &ipi_dummy_dev);
> > + if (r < 0)
> > + panic("IPI IRQ request failed\n");
> > +#endif
> >
> > for (i = 0; i < NR_IRQS; i++)
> > irq_set_noprobe(i);
> > diff --git a/arch/loongarch/kernel/proc.c b/arch/loongarch/kernel/proc.c
> > index d1f36b510297..e5772b65f956 100644
> > --- a/arch/loongarch/kernel/proc.c
> > +++ b/arch/loongarch/kernel/proc.c
> > @@ -35,6 +35,11 @@ static int show_cpuinfo(struct seq_file *m, void *v)
> > unsigned int fp_version = cpu_data[n].fpu_vers;
> > struct proc_cpuinfo_notifier_args proc_cpuinfo_notifier_args;
> >
> > +#ifdef CONFIG_SMP
> > + if (!cpu_online(n))
> > + return 0;
> > +#endif
> > +
> > /*
> > * For the first processor also print the system type
> > */
> > diff --git a/arch/loongarch/kernel/process.c b/arch/loongarch/kernel/process.c
> > index 667b365d909d..8f74985f8782 100644
> > --- a/arch/loongarch/kernel/process.c
> > +++ b/arch/loongarch/kernel/process.c
> > @@ -53,6 +53,13 @@
> > unsigned long boot_option_idle_override = IDLE_NO_OVERRIDE;
> > EXPORT_SYMBOL(boot_option_idle_override);
> >
> > +#ifdef CONFIG_HOTPLUG_CPU
> > +void arch_cpu_idle_dead(void)
> > +{
> > + play_dead();
> > +}
> > +#endif
> > +
> > asmlinkage void ret_from_fork(void);
> > asmlinkage void ret_from_kernel_thread(void);
> >
> > diff --git a/arch/loongarch/kernel/reset.c b/arch/loongarch/kernel/reset.c
> > index ef484ce43c5c..2b86469e4718 100644
> > --- a/arch/loongarch/kernel/reset.c
> > +++ b/arch/loongarch/kernel/reset.c
> > @@ -65,16 +65,28 @@ EXPORT_SYMBOL(pm_power_off);
> >
> > void machine_halt(void)
> > {
> > +#ifdef CONFIG_SMP
> > + preempt_disable();
> > + smp_send_stop();
> > +#endif
> > default_halt();
> > }
> >
> > void machine_power_off(void)
> > {
> > +#ifdef CONFIG_SMP
> > + preempt_disable();
> > + smp_send_stop();
> > +#endif
> > pm_power_off();
> > }
> >
> > void machine_restart(char *command)
> > {
> > +#ifdef CONFIG_SMP
> > + preempt_disable();
> > + smp_send_stop();
> > +#endif
> > do_kernel_restart(command);
> > pm_restart();
> > }
> > diff --git a/arch/loongarch/kernel/setup.c b/arch/loongarch/kernel/setup.c
> > index 827d07204e62..c314c22cdb22 100644
> > --- a/arch/loongarch/kernel/setup.c
> > +++ b/arch/loongarch/kernel/setup.c
> > @@ -38,6 +38,7 @@
> > #include <asm/pgalloc.h>
> > #include <asm/sections.h>
> > #include <asm/setup.h>
> > +#include <asm/smp.h>
> > #include <asm/time.h>
> >
> > #define SMBIOS_BIOSSIZE_OFFSET 0x09
> > @@ -317,6 +318,29 @@ static int __init reserve_memblock_reserved_regions(void)
> > }
> > arch_initcall(reserve_memblock_reserved_regions);
> >
> > +#ifdef CONFIG_SMP
> > +static void __init prefill_possible_map(void)
> > +{
> > + int i, possible;
> > +
> > + possible = num_processors + disabled_cpus;
> > + if (possible > nr_cpu_ids)
> > + possible = nr_cpu_ids;
> > +
> > + pr_info("SMP: Allowing %d CPUs, %d hotplug CPUs\n",
> > + possible, max((possible - num_processors), 0));
> > +
> > + for (i = 0; i < possible; i++)
> > + set_cpu_possible(i, true);
> > + for (; i < NR_CPUS; i++)
> > + set_cpu_possible(i, false);
> > +
> > + nr_cpu_ids = possible;
> > +}
> > +#else
> > +static inline void prefill_possible_map(void) {}
> > +#endif
> > +
> > void __init setup_arch(char **cmdline_p)
> > {
> > cpu_probe();
> > @@ -331,6 +355,8 @@ void __init setup_arch(char **cmdline_p)
> > arch_mem_init(cmdline_p);
> >
> > resource_init();
> > + plat_smp_setup();
> > + prefill_possible_map();
> >
> > paging_init();
> > }
> > diff --git a/arch/loongarch/kernel/smp.c b/arch/loongarch/kernel/smp.c
> > new file mode 100644
> > index 000000000000..4211025c79d3
> > --- /dev/null
> > +++ b/arch/loongarch/kernel/smp.c
> > @@ -0,0 +1,744 @@
> > +// SPDX-License-Identifier: GPL-2.0-or-later
> > +/*
> > + * Copyright (C) 2020-2022 Loongson Technology Corporation Limited
> > + *
> > + * Derived from MIPS:
> > + * Copyright (C) 2000, 2001 Kanoj Sarcar
> > + * Copyright (C) 2000, 2001 Ralf Baechle
> > + * Copyright (C) 2000, 2001 Silicon Graphics, Inc.
> > + * Copyright (C) 2000, 2001, 2003 Broadcom Corporation
> > + */
> > +#include <linux/cpu.h>
> > +#include <linux/cpumask.h>
> > +#include <linux/init.h>
> > +#include <linux/interrupt.h>
> > +#include <linux/seq_file.h>
> > +#include <linux/smp.h>
> > +#include <linux/threads.h>
> > +#include <linux/export.h>
> > +#include <linux/time.h>
> > +#include <linux/tracepoint.h>
> > +#include <linux/sched/hotplug.h>
> > +#include <linux/sched/task_stack.h>
> > +
> > +#include <asm/cpu.h>
> > +#include <asm/idle.h>
> > +#include <asm/loongson.h>
> > +#include <asm/mmu_context.h>
> > +#include <asm/processor.h>
> > +#include <asm/setup.h>
> > +#include <asm/time.h>
> > +
> > +int __cpu_number_map[NR_CPUS]; /* Map physical to logical */
> > +EXPORT_SYMBOL(__cpu_number_map);
> > +
> > +int __cpu_logical_map[NR_CPUS]; /* Map logical to physical */
> > +EXPORT_SYMBOL(__cpu_logical_map);
> > +
> > +/* Number of threads (siblings) per CPU core */
> > +int smp_num_siblings = 1;
> > +EXPORT_SYMBOL(smp_num_siblings);
> > +
> > +/* Representing the threads (siblings) of each logical CPU */
> > +cpumask_t cpu_sibling_map[NR_CPUS] __read_mostly;
> > +EXPORT_SYMBOL(cpu_sibling_map);
> > +
> > +/* Representing the core map of multi-core chips of each logical CPU */
> > +cpumask_t cpu_core_map[NR_CPUS] __read_mostly;
> > +EXPORT_SYMBOL(cpu_core_map);
> > +
> > +static DECLARE_COMPLETION(cpu_starting);
> > +static DECLARE_COMPLETION(cpu_running);
> > +
> > +/*
> > + * A logcal cpu mask containing only one VPE per core to
> > + * reduce the number of IPIs on large MT systems.
> > + */
> > +cpumask_t cpu_foreign_map[NR_CPUS] __read_mostly;
> > +EXPORT_SYMBOL(cpu_foreign_map);
> > +
> > +/* representing cpus for which sibling maps can be computed */
> > +static cpumask_t cpu_sibling_setup_map;
> > +
> > +/* representing cpus for which core maps can be computed */
> > +static cpumask_t cpu_core_setup_map;
> > +
> > +struct secondary_data cpuboot_data;
> > +static DEFINE_PER_CPU(int, cpu_state);
> > +DEFINE_PER_CPU_SHARED_ALIGNED(irq_cpustat_t, irq_stat);
> > +EXPORT_PER_CPU_SYMBOL(irq_stat);
> > +
> > +enum ipi_msg_type {
> > + IPI_RESCHEDULE,
> > + IPI_CALL_FUNCTION,
> > +};
> > +
> > +static const char *ipi_types[NR_IPI] __tracepoint_string = {
> > + [IPI_RESCHEDULE] = "Rescheduling interrupts",
> > + [IPI_CALL_FUNCTION] = "Call Function interrupts",
> "Function call interrupts" would sound more natural. Plus it's what all
> other arches call it.
OK, thanks.
Huacai
> > +};
> > +
> > +void show_ipi_list(struct seq_file *p, int prec)
> > +{
> > + unsigned int cpu, i;
> > +
> > + for (i = 0; i < NR_IPI; i++) {
> > + seq_printf(p, "%*s%u:%s", prec - 1, "IPI", i, prec >= 4 ? " " : "");
> > + for_each_online_cpu(cpu)
> > + seq_printf(p, "%10u ", per_cpu(irq_stat, cpu).ipi_irqs[i]);
> > + seq_printf(p, " LoongArch %d %s\n", i + 1, ipi_types[i]);
> > + }
> > +}
> > +
> > +/* Send mailbox buffer via Mail_Send */
> > +static void csr_mail_send(uint64_t data, int cpu, int mailbox)
> > +{
> > + uint64_t val;
> > +
> > + /* Send high 32 bits */
> > + val = IOCSR_MBUF_SEND_BLOCKING;
> > + val |= (IOCSR_MBUF_SEND_BOX_HI(mailbox) << IOCSR_MBUF_SEND_BOX_SHIFT);
> > + val |= (cpu << IOCSR_MBUF_SEND_CPU_SHIFT);
> > + val |= (data & IOCSR_MBUF_SEND_H32_MASK);
> > + iocsr_writeq(val, LOONGARCH_IOCSR_MBUF_SEND);
> > +
> > + /* Send low 32 bits */
> > + val = IOCSR_MBUF_SEND_BLOCKING;
> > + val |= (IOCSR_MBUF_SEND_BOX_LO(mailbox) << IOCSR_MBUF_SEND_BOX_SHIFT);
> > + val |= (cpu << IOCSR_MBUF_SEND_CPU_SHIFT);
> > + val |= (data << IOCSR_MBUF_SEND_BUF_SHIFT);
> > + iocsr_writeq(val, LOONGARCH_IOCSR_MBUF_SEND);
> > +};
> > +
> > +static u32 ipi_read_clear(int cpu)
> > +{
> > + u32 action;
> > +
> > + /* Load the ipi register to figure out what we're supposed to do */
> > + action = iocsr_readl(LOONGARCH_IOCSR_IPI_STATUS);
> > + /* Clear the ipi register to clear the interrupt */
> > + iocsr_writel(action, LOONGARCH_IOCSR_IPI_CLEAR);
> > + smp_mb();
> > +
> > + return action;
> > +}
> > +
> > +static void ipi_write_action(int cpu, u32 action)
> > +{
> > + unsigned int irq = 0;
> > +
> > + while ((irq = ffs(action))) {
> > + uint32_t val = IOCSR_IPI_SEND_BLOCKING;
> > +
> > + val |= (irq - 1);
> > + val |= (cpu << IOCSR_IPI_SEND_CPU_SHIFT);
> > + iocsr_writel(val, LOONGARCH_IOCSR_IPI_SEND);
> > + action &= ~BIT(irq - 1);
> > + }
> > +}
> > +
> > +void loongson3_send_ipi_single(int cpu, unsigned int action)
> > +{
> > + ipi_write_action(cpu_logical_map(cpu), (u32)action);
> > +}
> > +
> > +void loongson3_send_ipi_mask(const struct cpumask *mask, unsigned int action)
> > +{
> > + unsigned int i;
> > +
> > + for_each_cpu(i, mask)
> > + ipi_write_action(cpu_logical_map(i), (u32)action);
> > +}
> > +
> > +irqreturn_t loongson3_ipi_interrupt(int irq, void *dev)
> > +{
> > + unsigned int action;
> > + unsigned int cpu = smp_processor_id();
> > +
> > + action = ipi_read_clear(cpu_logical_map(cpu));
> > +
> > + if (action & SMP_RESCHEDULE) {
> > + scheduler_ipi();
> > + per_cpu(irq_stat, cpu).ipi_irqs[IPI_RESCHEDULE]++;
> > + }
> > +
> > + if (action & SMP_CALL_FUNCTION) {
> > + generic_smp_call_function_interrupt();
> > + per_cpu(irq_stat, cpu).ipi_irqs[IPI_CALL_FUNCTION]++;
> > + }
> > +
> > + return IRQ_HANDLED;
> > +}
> > +
> > +void __init loongson3_smp_setup(void)
> > +{
> > + cpu_data[0].core = cpu_logical_map(0) % loongson_sysconf.cores_per_package;
> > + cpu_data[0].package = cpu_logical_map(0) / loongson_sysconf.cores_per_package;
> > +
> > + iocsr_writel(0xffffffff, LOONGARCH_IOCSR_IPI_EN);
> > + pr_info("Detected %i available CPU(s)\n", loongson_sysconf.nr_cpus);
> > +}
> > +
> > +void __init loongson3_prepare_cpus(unsigned int max_cpus)
> > +{
> > + int i = 0;
> > +
> > + for (i = 0; i < loongson_sysconf.nr_cpus; i++) {
> > + set_cpu_present(i, true);
> > + csr_mail_send(0, __cpu_logical_map[i], 0);
> > + }
> > +
> > + per_cpu(cpu_state, smp_processor_id()) = CPU_ONLINE;
> > +}
> > +
> > +/*
> > + * Setup the PC, SP, and TP of a secondary processor and start it running!
> > + */
> > +void loongson3_boot_secondary(int cpu, struct task_struct *idle)
> > +{
> > + unsigned long entry;
> > +
> > + pr_info("Booting CPU#%d...\n", cpu);
> > +
> > + entry = __pa_symbol((unsigned long)&smpboot_entry);
> > + cpuboot_data.stack = (unsigned long)__KSTK_TOS(idle);
> > + cpuboot_data.thread_info = (unsigned long)task_thread_info(idle);
> > +
> > + csr_mail_send(entry, cpu_logical_map(cpu), 0);
> > +
> > + loongson3_send_ipi_single(cpu, SMP_BOOT_CPU);
> > +}
> > +
> > +/*
> > + * SMP init and finish on secondary CPUs
> > + */
> > +void loongson3_init_secondary(void)
> > +{
> > + unsigned int cpu = smp_processor_id();
> > + unsigned int imask = ECFGF_IP0 | ECFGF_IP1 | ECFGF_IP2 |
> > + ECFGF_IPI | ECFGF_PMC | ECFGF_TIMER;
> > +
> > + change_csr_ecfg(ECFG0_IM, imask);
> > +
> > + iocsr_writel(0xffffffff, LOONGARCH_IOCSR_IPI_EN);
> > +
> > + per_cpu(cpu_state, cpu) = CPU_ONLINE;
> > + cpu_data[cpu].core =
> > + cpu_logical_map(cpu) % loongson_sysconf.cores_per_package;
> > + cpu_data[cpu].package =
> > + cpu_logical_map(cpu) / loongson_sysconf.cores_per_package;
> > +}
> > +
> > +void loongson3_smp_finish(void)
> > +{
> > + local_irq_enable();
> > + iocsr_writeq(0, LOONGARCH_IOCSR_MBUF0);
> > + pr_info("CPU#%d finished\n", smp_processor_id());
> > +}
> > +
> > +#ifdef CONFIG_HOTPLUG_CPU
> > +
> > +static bool io_master(int cpu)
> > +{
> > + int i, node, master;
> > +
> > + if (cpu == 0)
> > + return true;
> > +
> > + for (i = 1; i < loongson_sysconf.nr_io_pics; i++) {
> > + node = eiointc_get_node(i);
> > + master = cpu_number_map(node * CORES_PER_EIO_NODE);
> > + if (cpu == master)
> > + return true;
> > + }
> > +
> > + return false;
> > +}
> > +
> > +int loongson3_cpu_disable(void)
> > +{
> > + unsigned long flags;
> > + unsigned int cpu = smp_processor_id();
> > +
> > + if (io_master(cpu))
> > + return -EBUSY;
> > +
> > + set_cpu_online(cpu, false);
> > + calculate_cpu_foreign_map();
> > + local_irq_save(flags);
> > + irq_migrate_all_off_this_cpu();
> > + clear_csr_ecfg(ECFG0_IM);
> > + local_irq_restore(flags);
> > + local_flush_tlb_all();
> > +
> > + return 0;
> > +}
> > +
> > +void loongson3_cpu_die(unsigned int cpu)
> > +{
> > + while (per_cpu(cpu_state, cpu) != CPU_DEAD)
> > + cpu_relax();
> > +
> > + mb();
> > +}
> > +
> > +/*
> > + * The target CPU should go to XKPRANGE (uncached area) and flush
> > + * ICache/DCache/VCache before the control CPU can safely disable its clock.
> > + */
> > +static void loongson3_play_dead(int *state_addr)
> > +{
> > + register int val;
> > + register void *addr;
> > + register void (*init_fn)(void);
> > +
> > + __asm__ __volatile__(
> > + " li.d %[addr], 0x8000000000000000\n"
> > + "1: cacop 0x8, %[addr], 0 \n" /* flush ICache */
> > + " cacop 0x8, %[addr], 1 \n"
> > + " cacop 0x8, %[addr], 2 \n"
> > + " cacop 0x8, %[addr], 3 \n"
> > + " cacop 0x9, %[addr], 0 \n" /* flush DCache */
> > + " cacop 0x9, %[addr], 1 \n"
> > + " cacop 0x9, %[addr], 2 \n"
> > + " cacop 0x9, %[addr], 3 \n"
> > + " addi.w %[sets], %[sets], -1 \n"
> > + " addi.d %[addr], %[addr], 0x40 \n"
> > + " bnez %[sets], 1b \n"
> > + " li.d %[addr], 0x8000000000000000\n"
> > + "2: cacop 0xa, %[addr], 0 \n" /* flush VCache */
> > + " cacop 0xa, %[addr], 1 \n"
> > + " cacop 0xa, %[addr], 2 \n"
> > + " cacop 0xa, %[addr], 3 \n"
> > + " cacop 0xa, %[addr], 4 \n"
> > + " cacop 0xa, %[addr], 5 \n"
> > + " cacop 0xa, %[addr], 6 \n"
> > + " cacop 0xa, %[addr], 7 \n"
> > + " cacop 0xa, %[addr], 8 \n"
> > + " cacop 0xa, %[addr], 9 \n"
> > + " cacop 0xa, %[addr], 10 \n"
> > + " cacop 0xa, %[addr], 11 \n"
> > + " cacop 0xa, %[addr], 12 \n"
> > + " cacop 0xa, %[addr], 13 \n"
> > + " cacop 0xa, %[addr], 14 \n"
> > + " cacop 0xa, %[addr], 15 \n"
> > + " addi.w %[vsets], %[vsets], -1 \n"
> > + " addi.d %[addr], %[addr], 0x40 \n"
> > + " bnez %[vsets], 2b \n"
> > + " li.w %[val], 0x7 \n" /* *state_addr = CPU_DEAD; */
> > + " st.w %[val], %[state_addr], 0 \n"
> > + " dbar 0 \n"
> > + " cacop 0x11, %[state_addr], 0 \n" /* flush entry of *state_addr */
> > + : [addr] "=&r" (addr), [val] "=&r" (val)
> > + : [state_addr] "r" (state_addr),
> > + [sets] "r" (cpu_data[smp_processor_id()].dcache.sets),
> > + [vsets] "r" (cpu_data[smp_processor_id()].vcache.sets));
> > +
> > + local_irq_enable();
> > + change_csr_ecfg(ECFG0_IM, ECFGF_IPI);
> > +
> > + __asm__ __volatile__(
> > + " idle 0 \n"
> > + " li.w $t0, 0x1020 \n"
> > + " iocsrrd.d %[init_fn], $t0 \n" /* Get init PC */
> > + : [init_fn] "=&r" (addr)
> > + : /* No Input */
> > + : "a0");
> > + init_fn = __va(addr);
> > +
> > + init_fn();
> > + unreachable();
> > +}
> > +
> > +void play_dead(void)
> > +{
> > + int *state_addr;
> > + unsigned int cpu = smp_processor_id();
> > + void (*play_dead_uncached)(int *s);
> > +
> > + idle_task_exit();
> > + play_dead_uncached = (void *)TO_UNCACHE(__pa((unsigned long)loongson3_play_dead));
> > + state_addr = &per_cpu(cpu_state, cpu);
> > + mb();
> > + play_dead_uncached(state_addr);
> > +}
> > +
> > +static int loongson3_enable_clock(unsigned int cpu)
> > +{
> > + uint64_t core_id = cpu_data[cpu].core;
> > + uint64_t package_id = cpu_data[cpu].package;
> > +
> > + LOONGSON_FREQCTRL(package_id) |= 1 << (core_id * 4 + 3);
> > +
> > + return 0;
> > +}
> > +
> > +static int loongson3_disable_clock(unsigned int cpu)
> > +{
> > + uint64_t core_id = cpu_data[cpu].core;
> > + uint64_t package_id = cpu_data[cpu].package;
> > +
> > + LOONGSON_FREQCTRL(package_id) &= ~(1 << (core_id * 4 + 3));
> > +
> > + return 0;
> > +}
> > +
> > +static int register_loongson3_notifier(void)
> > +{
> > + return cpuhp_setup_state_nocalls(CPUHP_LOONGARCH_SOC_PREPARE,
> > + "loongarch/loongson:prepare",
> > + loongson3_enable_clock,
> > + loongson3_disable_clock);
> > +}
> > +early_initcall(register_loongson3_notifier);
> > +
> > +#endif
> > +
> > +/*
> > + * Power management
> > + */
> > +#ifdef CONFIG_PM
> > +
> > +static int loongson3_ipi_suspend(void)
> > +{
> > + return 0;
> > +}
> > +
> > +static void loongson3_ipi_resume(void)
> > +{
> > + iocsr_writel(0xffffffff, LOONGARCH_IOCSR_IPI_EN);
> > +}
> > +
> > +static struct syscore_ops loongson3_ipi_syscore_ops = {
> > + .resume = loongson3_ipi_resume,
> > + .suspend = loongson3_ipi_suspend,
> > +};
> > +
> > +/*
> > + * Enable boot cpu ipi before enabling nonboot cpus
> > + * during syscore_resume.
> > + */
> > +static int __init ipi_pm_init(void)
> > +{
> > + register_syscore_ops(&loongson3_ipi_syscore_ops);
> > + return 0;
> > +}
> > +
> > +core_initcall(ipi_pm_init);
> > +#endif
> > +
> > +static inline void set_cpu_sibling_map(int cpu)
> > +{
> > + int i;
> > +
> > + cpumask_set_cpu(cpu, &cpu_sibling_setup_map);
> > +
> > + if (smp_num_siblings <= 1)
> > + cpumask_set_cpu(cpu, &cpu_sibling_map[cpu]);
> > + else {
> > + for_each_cpu(i, &cpu_sibling_setup_map) {
> > + if (cpus_are_siblings(cpu, i)) {
> > + cpumask_set_cpu(i, &cpu_sibling_map[cpu]);
> > + cpumask_set_cpu(cpu, &cpu_sibling_map[i]);
> > + }
> > + }
> > + }
> > +}
> > +
> > +static inline void set_cpu_core_map(int cpu)
> > +{
> > + int i;
> > +
> > + cpumask_set_cpu(cpu, &cpu_core_setup_map);
> > +
> > + for_each_cpu(i, &cpu_core_setup_map) {
> > + if (cpu_data[cpu].package == cpu_data[i].package) {
> > + cpumask_set_cpu(i, &cpu_core_map[cpu]);
> > + cpumask_set_cpu(cpu, &cpu_core_map[i]);
> > + }
> > + }
> > +}
> > +
> > +/*
> > + * Calculate a new cpu_foreign_map mask whenever a
> > + * new cpu appears or disappears.
> > + */
> > +void calculate_cpu_foreign_map(void)
> > +{
> > + int i, k, core_present;
> > + cpumask_t temp_foreign_map;
> > +
> > + /* Re-calculate the mask */
> > + cpumask_clear(&temp_foreign_map);
> > + for_each_online_cpu(i) {
> > + core_present = 0;
> > + for_each_cpu(k, &temp_foreign_map)
> > + if (cpus_are_siblings(i, k))
> > + core_present = 1;
> > + if (!core_present)
> > + cpumask_set_cpu(i, &temp_foreign_map);
> > + }
> > +
> > + for_each_online_cpu(i)
> > + cpumask_andnot(&cpu_foreign_map[i],
> > + &temp_foreign_map, &cpu_sibling_map[i]);
> > +}
> > +
> > +/* Preload SMP state for boot cpu */
> > +void smp_prepare_boot_cpu(void)
> > +{
> > + unsigned int cpu;
> > +
> > + set_cpu_possible(0, true);
> > + set_cpu_online(0, true);
> > + set_my_cpu_offset(per_cpu_offset(0));
> > +
> > + for_each_possible_cpu(cpu)
> > + set_cpu_numa_node(cpu, 0);
> > +}
> > +
> > +/* called from main before smp_init() */
> > +void __init smp_prepare_cpus(unsigned int max_cpus)
> > +{
> > + init_new_context(current, &init_mm);
> > + current_thread_info()->cpu = 0;
> > + loongson3_prepare_cpus(max_cpus);
> > + set_cpu_sibling_map(0);
> > + set_cpu_core_map(0);
> > + calculate_cpu_foreign_map();
> > +#ifndef CONFIG_HOTPLUG_CPU
> > + init_cpu_present(cpu_possible_mask);
> > +#endif
> > +}
> > +
> > +int __cpu_up(unsigned int cpu, struct task_struct *tidle)
> > +{
> > + loongson3_boot_secondary(cpu, tidle);
> > +
> > + /* Wait for CPU to start and be ready to sync counters */
> > + if (!wait_for_completion_timeout(&cpu_starting,
> > + msecs_to_jiffies(5000))) {
> > + pr_crit("CPU%u: failed to start\n", cpu);
> > + return -EIO;
> > + }
> > +
> > + /* Wait for CPU to finish startup & mark itself online before return */
> > + wait_for_completion(&cpu_running);
> > +
> > + return 0;
> > +}
> > +
> > +/*
> > + * First C code run on the secondary CPUs after being started up by
> > + * the master.
> > + */
> > +asmlinkage void start_secondary(void)
> > +{
> > + unsigned int cpu;
> > +
> > + sync_counter();
> > + cpu = smp_processor_id();
> > + set_my_cpu_offset(per_cpu_offset(cpu));
> > +
> > + cpu_probe();
> > + constant_clockevent_init();
> > + loongson3_init_secondary();
> > +
> > + set_cpu_sibling_map(cpu);
> > + set_cpu_core_map(cpu);
> > +
> > + notify_cpu_starting(cpu);
> > +
> > + /* Notify boot CPU that we're starting */
> > + complete(&cpu_starting);
> > +
> > + /* The CPU is running, now mark it online */
> > + set_cpu_online(cpu, true);
> > +
> > + calculate_cpu_foreign_map();
> > +
> > + /*
> > + * Notify boot CPU that we're up & online and it can safely return
> > + * from __cpu_up()
> > + */
> > + complete(&cpu_running);
> > +
> > + /*
> > + * irq will be enabled in loongson3_smp_finish(), enabling it too
> > + * early is dangerous.
> > + */
> > + WARN_ON_ONCE(!irqs_disabled());
> > + loongson3_smp_finish();
> > +
> > + cpu_startup_entry(CPUHP_AP_ONLINE_IDLE);
> > +}
> > +
> > +void __init smp_cpus_done(unsigned int max_cpus)
> > +{
> > +}
> > +
> > +static void stop_this_cpu(void *dummy)
> > +{
> > + set_cpu_online(smp_processor_id(), false);
> > + calculate_cpu_foreign_map();
> > + local_irq_disable();
> > + while (true);
> > +}
> > +
> > +void smp_send_stop(void)
> > +{
> > + smp_call_function(stop_this_cpu, NULL, 0);
> > +}
> > +
> > +int setup_profiling_timer(unsigned int multiplier)
> > +{
> > + return 0;
> > +}
> > +
> > +static void flush_tlb_all_ipi(void *info)
> > +{
> > + local_flush_tlb_all();
> > +}
> > +
> > +void flush_tlb_all(void)
> > +{
> > + on_each_cpu(flush_tlb_all_ipi, NULL, 1);
> > +}
> > +
> > +static void flush_tlb_mm_ipi(void *mm)
> > +{
> > + local_flush_tlb_mm((struct mm_struct *)mm);
> > +}
> > +
> > +void flush_tlb_mm(struct mm_struct *mm)
> > +{
> > + if (atomic_read(&mm->mm_users) == 0)
> > + return; /* happens as a result of exit_mmap() */
> > +
> > + preempt_disable();
> > +
> > + if ((atomic_read(&mm->mm_users) != 1) || (current->mm != mm)) {
> > + on_each_cpu_mask(mm_cpumask(mm), flush_tlb_mm_ipi, mm, 1);
> > + } else {
> > + unsigned int cpu;
> > +
> > + for_each_online_cpu(cpu) {
> > + if (cpu != smp_processor_id() && cpu_context(cpu, mm))
> > + cpu_context(cpu, mm) = 0;
> > + }
> > + local_flush_tlb_mm(mm);
> > + }
> > +
> > + preempt_enable();
> > +}
> > +
> > +struct flush_tlb_data {
> > + struct vm_area_struct *vma;
> > + unsigned long addr1;
> > + unsigned long addr2;
> > +};
> > +
> > +static void flush_tlb_range_ipi(void *info)
> > +{
> > + struct flush_tlb_data *fd = info;
> > +
> > + local_flush_tlb_range(fd->vma, fd->addr1, fd->addr2);
> > +}
> > +
> > +void flush_tlb_range(struct vm_area_struct *vma, unsigned long start, unsigned long end)
> > +{
> > + struct mm_struct *mm = vma->vm_mm;
> > +
> > + preempt_disable();
> > + if ((atomic_read(&mm->mm_users) != 1) || (current->mm != mm)) {
> > + struct flush_tlb_data fd = {
> > + .vma = vma,
> > + .addr1 = start,
> > + .addr2 = end,
> > + };
> > +
> > + on_each_cpu_mask(mm_cpumask(mm), flush_tlb_range_ipi, &fd, 1);
> > + } else {
> > + unsigned int cpu;
> > + int exec = vma->vm_flags & VM_EXEC;
> > +
> > + for_each_online_cpu(cpu) {
> > + /*
> > + * flush_cache_range() will only fully flush icache if
> > + * the VMA is executable, otherwise we must invalidate
> > + * ASID without it appearing to has_valid_asid() as if
> > + * mm has been completely unused by that CPU.
> > + */
> > + if (cpu != smp_processor_id() && cpu_context(cpu, mm))
> > + cpu_context(cpu, mm) = !exec;
> > + }
> > + local_flush_tlb_range(vma, start, end);
> > + }
> > + preempt_enable();
> > +}
> > +
> > +static void flush_tlb_kernel_range_ipi(void *info)
> > +{
> > + struct flush_tlb_data *fd = info;
> > +
> > + local_flush_tlb_kernel_range(fd->addr1, fd->addr2);
> > +}
> > +
> > +void flush_tlb_kernel_range(unsigned long start, unsigned long end)
> > +{
> > + struct flush_tlb_data fd = {
> > + .addr1 = start,
> > + .addr2 = end,
> > + };
> > +
> > + on_each_cpu(flush_tlb_kernel_range_ipi, &fd, 1);
> > +}
> > +
> > +static void flush_tlb_page_ipi(void *info)
> > +{
> > + struct flush_tlb_data *fd = info;
> > +
> > + local_flush_tlb_page(fd->vma, fd->addr1);
> > +}
> > +
> > +void flush_tlb_page(struct vm_area_struct *vma, unsigned long page)
> > +{
> > + preempt_disable();
> > + if ((atomic_read(&vma->vm_mm->mm_users) != 1) || (current->mm != vma->vm_mm)) {
> > + struct flush_tlb_data fd = {
> > + .vma = vma,
> > + .addr1 = page,
> > + };
> > +
> > + on_each_cpu_mask(mm_cpumask(vma->vm_mm), flush_tlb_page_ipi, &fd, 1);
> > + } else {
> > + unsigned int cpu;
> > +
> > + for_each_online_cpu(cpu) {
> > + /*
> > + * flush_cache_page() only does partial flushes, so
> > + * invalidate ASID without it appearing to
> > + * has_valid_asid() as if mm has been completely unused
> > + * by that CPU.
> > + */
> > + if (cpu != smp_processor_id() && cpu_context(cpu, vma->vm_mm))
> > + cpu_context(cpu, vma->vm_mm) = 1;
> > + }
> > + local_flush_tlb_page(vma, page);
> > + }
> > + preempt_enable();
> > +}
> > +EXPORT_SYMBOL(flush_tlb_page);
> > +
> > +static void flush_tlb_one_ipi(void *info)
> > +{
> > + unsigned long vaddr = (unsigned long) info;
> > +
> > + local_flush_tlb_one(vaddr);
> > +}
> > +
> > +void flush_tlb_one(unsigned long vaddr)
> > +{
> > + on_each_cpu(flush_tlb_one_ipi, (void *)vaddr, 1);
> > +}
> > +EXPORT_SYMBOL(flush_tlb_one);
> > diff --git a/arch/loongarch/kernel/topology.c b/arch/loongarch/kernel/topology.c
> > index 3b2cbb95875b..ab1a75c0b5a6 100644
> > --- a/arch/loongarch/kernel/topology.c
> > +++ b/arch/loongarch/kernel/topology.c
> > @@ -1,13 +1,52 @@
> > // SPDX-License-Identifier: GPL-2.0
> > #include <linux/cpu.h>
> > +#include <linux/cpumask.h>
> > #include <linux/init.h>
> > +#include <linux/node.h>
> > +#include <linux/nodemask.h>
> > #include <linux/percpu.h>
> >
> > -static struct cpu cpu_device;
> > +static DEFINE_PER_CPU(struct cpu, cpu_devices);
> > +
> > +#ifdef CONFIG_HOTPLUG_CPU
> > +int arch_register_cpu(int cpu)
> > +{
> > + int ret;
> > + struct cpu *c = &per_cpu(cpu_devices, cpu);
> > +
> > + c->hotpluggable = 1;
> > + ret = register_cpu(c, cpu);
> > + if (ret < 0)
> > + pr_warn("register_cpu %d failed (%d)\n", cpu, ret);
> > +
> > + return ret;
> > +}
> > +EXPORT_SYMBOL(arch_register_cpu);
> > +
> > +void arch_unregister_cpu(int cpu)
> > +{
> > + struct cpu *c = &per_cpu(cpu_devices, cpu);
> > +
> > + c->hotpluggable = 0;
> > + unregister_cpu(c);
> > +}
> > +EXPORT_SYMBOL(arch_unregister_cpu);
> > +#endif
> >
> > static int __init topology_init(void)
> > {
> > - return register_cpu(&cpu_device, 0);
> > + int i, ret;
> > +
> > + for_each_present_cpu(i) {
> > + struct cpu *c = &per_cpu(cpu_devices, i);
> > +
> > + c->hotpluggable = !!i;
> > + ret = register_cpu(c, i);
> > + if (ret < 0)
> > + pr_warn("topology_init: register_cpu %d failed (%d)\n", i, ret);
> > + }
> > +
> > + return 0;
> > }
> >
> > subsys_initcall(topology_init);
> > diff --git a/arch/loongarch/kernel/vmlinux.lds.S b/arch/loongarch/kernel/vmlinux.lds.S
> > index 7da4c4d7c50d..006cbb1bd5c6 100644
> > --- a/arch/loongarch/kernel/vmlinux.lds.S
> > +++ b/arch/loongarch/kernel/vmlinux.lds.S
> > @@ -73,6 +73,10 @@ SECTIONS
> > EXIT_DATA
> > }
> >
> > +#ifdef CONFIG_SMP
> > + PERCPU_SECTION(1 << CONFIG_L1_CACHE_SHIFT)
> > +#endif
> > +
> > .init.bss : {
> > *(.init.bss)
> > }
> > diff --git a/arch/loongarch/mm/tlbex.S b/arch/loongarch/mm/tlbex.S
> > index a4ca4e507ee8..606b7800edc6 100644
> > --- a/arch/loongarch/mm/tlbex.S
> > +++ b/arch/loongarch/mm/tlbex.S
> > @@ -88,7 +88,14 @@ vmalloc_done_load:
> > slli.d t0, t0, _PTE_T_LOG2
> > add.d t1, ra, t0
> >
> > +#ifdef CONFIG_SMP
> > +smp_pgtable_change_load:
> > +#endif
> > +#ifdef CONFIG_SMP
> > + ll.d t0, t1, 0
> > +#else
> > ld.d t0, t1, 0
> > +#endif
> > tlbsrch
> >
> > srli.d ra, t0, _PAGE_PRESENT_SHIFT
> > @@ -96,7 +103,12 @@ vmalloc_done_load:
> > beq ra, $r0, nopage_tlb_load
> >
> > ori t0, t0, _PAGE_VALID
> > +#ifdef CONFIG_SMP
> > + sc.d t0, t1, 0
> > + beq t0, $r0, smp_pgtable_change_load
> > +#else
> > st.d t0, t1, 0
> > +#endif
> > ori t1, t1, 8
> > xori t1, t1, 8
> > ld.d t0, t1, 0
> > @@ -120,14 +132,24 @@ vmalloc_load:
> > * spots a huge page.
> > */
> > tlb_huge_update_load:
> > +#ifdef CONFIG_SMP
> > + ll.d t0, t1, 0
> > +#else
> > ld.d t0, t1, 0
> > +#endif
> > srli.d ra, t0, _PAGE_PRESENT_SHIFT
> > andi ra, ra, 1
> > beq ra, $r0, nopage_tlb_load
> > tlbsrch
> >
> > ori t0, t0, _PAGE_VALID
> > +#ifdef CONFIG_SMP
> > + sc.d t0, t1, 0
> > + beq t0, $r0, tlb_huge_update_load
> > + ld.d t0, t1, 0
> > +#else
> > st.d t0, t1, 0
> > +#endif
> > addu16i.d t1, $r0, -(CSR_TLBIDX_EHINV >> 16)
> > addi.d ra, t1, 0
> > csrxchg ra, t1, LOONGARCH_CSR_TLBIDX
> > @@ -173,6 +195,7 @@ tlb_huge_update_load:
> > csrxchg t1, t0, LOONGARCH_CSR_TLBIDX
> >
> > nopage_tlb_load:
> > + dbar 0
> > csrrd ra, EXCEPTION_KS2
> > la.abs t0, tlb_do_page_fault_0
> > jirl $r0, t0, 0
> > @@ -229,7 +252,14 @@ vmalloc_done_store:
> > slli.d t0, t0, _PTE_T_LOG2
> > add.d t1, ra, t0
> >
> > +#ifdef CONFIG_SMP
> > +smp_pgtable_change_store:
> > +#endif
> > +#ifdef CONFIG_SMP
> > + ll.d t0, t1, 0
> > +#else
> > ld.d t0, t1, 0
> > +#endif
> > tlbsrch
> >
> > srli.d ra, t0, _PAGE_PRESENT_SHIFT
> > @@ -238,7 +268,12 @@ vmalloc_done_store:
> > bne ra, $r0, nopage_tlb_store
> >
> > ori t0, t0, (_PAGE_VALID | _PAGE_DIRTY | _PAGE_MODIFIED)
> > +#ifdef CONFIG_SMP
> > + sc.d t0, t1, 0
> > + beq t0, $r0, smp_pgtable_change_store
> > +#else
> > st.d t0, t1, 0
> > +#endif
> >
> > ori t1, t1, 8
> > xori t1, t1, 8
> > @@ -263,7 +298,11 @@ vmalloc_store:
> > * spots a huge page.
> > */
> > tlb_huge_update_store:
> > +#ifdef CONFIG_SMP
> > + ll.d t0, t1, 0
> > +#else
> > ld.d t0, t1, 0
> > +#endif
> > srli.d ra, t0, _PAGE_PRESENT_SHIFT
> > andi ra, ra, ((_PAGE_PRESENT | _PAGE_WRITE) >> _PAGE_PRESENT_SHIFT)
> > xori ra, ra, ((_PAGE_PRESENT | _PAGE_WRITE) >> _PAGE_PRESENT_SHIFT)
> > @@ -272,7 +311,13 @@ tlb_huge_update_store:
> > tlbsrch
> > ori t0, t0, (_PAGE_VALID | _PAGE_DIRTY | _PAGE_MODIFIED)
> >
> > +#ifdef CONFIG_SMP
> > + sc.d t0, t1, 0
> > + beq t0, $r0, tlb_huge_update_store
> > + ld.d t0, t1, 0
> > +#else
> > st.d t0, t1, 0
> > +#endif
> > addu16i.d t1, $r0, -(CSR_TLBIDX_EHINV >> 16)
> > addi.d ra, t1, 0
> > csrxchg ra, t1, LOONGARCH_CSR_TLBIDX
> > @@ -318,6 +363,7 @@ tlb_huge_update_store:
> > csrxchg t1, t0, LOONGARCH_CSR_TLBIDX
> >
> > nopage_tlb_store:
> > + dbar 0
> > csrrd ra, EXCEPTION_KS2
> > la.abs t0, tlb_do_page_fault_1
> > jirl $r0, t0, 0
> > @@ -373,7 +419,14 @@ vmalloc_done_modify:
> > slli.d t0, t0, _PTE_T_LOG2
> > add.d t1, ra, t0
> >
> > +#ifdef CONFIG_SMP
> > +smp_pgtable_change_modify:
> > +#endif
> > +#ifdef CONFIG_SMP
> > + ll.d t0, t1, 0
> > +#else
> > ld.d t0, t1, 0
> > +#endif
> > tlbsrch
> >
> > srli.d ra, t0, _PAGE_WRITE_SHIFT
> > @@ -381,7 +434,12 @@ vmalloc_done_modify:
> > beq ra, $r0, nopage_tlb_modify
> >
> > ori t0, t0, (_PAGE_VALID | _PAGE_DIRTY | _PAGE_MODIFIED)
> > +#ifdef CONFIG_SMP
> > + sc.d t0, t1, 0
> > + beq t0, $r0, smp_pgtable_change_modify
> > +#else
> > st.d t0, t1, 0
> > +#endif
> > ori t1, t1, 8
> > xori t1, t1, 8
> > ld.d t0, t1, 0
> > @@ -405,7 +463,11 @@ vmalloc_modify:
> > * build_tlbchange_handler_head spots a huge page.
> > */
> > tlb_huge_update_modify:
> > +#ifdef CONFIG_SMP
> > + ll.d t0, t1, 0
> > +#else
> > ld.d t0, t1, 0
> > +#endif
> >
> > srli.d ra, t0, _PAGE_WRITE_SHIFT
> > andi ra, ra, 1
> > @@ -414,7 +476,13 @@ tlb_huge_update_modify:
> > tlbsrch
> > ori t0, t0, (_PAGE_VALID | _PAGE_DIRTY | _PAGE_MODIFIED)
> >
> > +#ifdef CONFIG_SMP
> > + sc.d t0, t1, 0
> > + beq t0, $r0, tlb_huge_update_modify
> > + ld.d t0, t1, 0
> > +#else
> > st.d t0, t1, 0
> > +#endif
> > /*
> > * A huge PTE describes an area the size of the
> > * configured huge page size. This is twice the
> > @@ -454,6 +522,7 @@ tlb_huge_update_modify:
> > csrxchg t1, t0, LOONGARCH_CSR_TLBIDX
> >
> > nopage_tlb_modify:
> > + dbar 0
> > csrrd ra, EXCEPTION_KS2
> > la.abs t0, tlb_do_page_fault_1
> > jirl $r0, t0, 0
> > diff --git a/include/linux/cpuhotplug.h b/include/linux/cpuhotplug.h
> > index 2af7c6587875..8abd28c4f32b 100644
> > --- a/include/linux/cpuhotplug.h
> > +++ b/include/linux/cpuhotplug.h
> > @@ -130,6 +130,7 @@ enum cpuhp_state {
> > CPUHP_ZCOMP_PREPARE,
> > CPUHP_TIMERS_PREPARE,
> > CPUHP_MIPS_SOC_PREPARE,
> > + CPUHP_LOONGARCH_SOC_PREPARE,
> > CPUHP_BP_PREPARE_DYN,
> > CPUHP_BP_PREPARE_DYN_END = CPUHP_BP_PREPARE_DYN + 20,
> > CPUHP_BRINGUP_CPU,
>
> All of this looks reasonable, not that I write kernel code day and
> night, but maybe I just read too much arch/mips code in the past years
> and the changes here look natural to me. And the names and code flows
> are sensible enough that people moderately familiar with the kernel
> (such as me) could easily follow.
>
> So, like with some earlier patches, a rather weak:
>
> Reviewed-by: WANG Xuerui <git@...0n.name>
>
Powered by blists - more mailing lists