linux-kernel - [PATCH v8 6/6] powerpc/pv-qspinlock: Optimise native unlock path

lists.openwall.net		lists / announce owl-users owl-dev john-users john-dev passwdqc-users yescrypt popa3d-users / oss-security kernel-hardening musl sabotage tlsify passwords / crypt-dev xvendor / Bugtraq Full-Disclosure linux-kernel linux-netdev linux-ext4 linux-hardening linux-cve-announce PHC
Open Source and information security mailing list archives

Hash Suite: Windows password security audit tool. GUI, reports in PDF.

[<prev] [next>] [<thread-prev] [thread-next>] [day] [month] [year] [list]

Message-Id: <1480951166-44830-7-git-send-email-xinhui.pan@linux.vnet.ibm.com>
Date:   Mon,  5 Dec 2016 10:19:26 -0500
From:   Pan Xinhui <xinhui.pan@...ux.vnet.ibm.com>
To:     linux-kernel@...r.kernel.org
Cc:     linuxppc-dev@...ts.ozlabs.org, benh@...nel.crashing.org,
        paulus@...ba.org, mpe@...erman.id.au, peterz@...radead.org,
        mingo@...hat.com, paulmck@...ux.vnet.ibm.com, waiman.long@....com,
        xinhui.pan@...ux.vnet.ibm.com,
        virtualization@...ts.linux-foundation.org, boqun.feng@...il.com
Subject: [PATCH v8 6/6] powerpc/pv-qspinlock: Optimise native unlock path

Avoid a function call under native version of qspinlock. On powerNV,
bafore applying this patch, every unlock is expensive. This small
optimizes enhance the performance.

We use static_key with jump_label which removes unnecessary loads of
lppaca and its stuff.

Signed-off-by: Pan Xinhui <xinhui.pan@...ux.vnet.ibm.com>
---
 arch/powerpc/include/asm/qspinlock_paravirt.h | 18 +++++++++++++++++-
 arch/powerpc/kernel/paravirt.c                |  4 ++++
 2 files changed, 21 insertions(+), 1 deletion(-)

diff --git a/arch/powerpc/include/asm/qspinlock_paravirt.h b/arch/powerpc/include/asm/qspinlock_paravirt.h
index d87cda0..8d39446 100644
--- a/arch/powerpc/include/asm/qspinlock_paravirt.h
+++ b/arch/powerpc/include/asm/qspinlock_paravirt.h
@@ -6,12 +6,14 @@
 #define _ASM_QSPINLOCK_PARAVIRT_H
 
 #include  <asm/qspinlock_paravirt_types.h>
+#include  <linux/jump_label.h>
 
 extern void pv_lock_init(void);
 extern void native_queued_spin_lock_slowpath(struct qspinlock *lock, u32 val);
 extern void __pv_init_lock_hash(void);
 extern void __pv_queued_spin_lock_slowpath(struct qspinlock *lock, u32 val);
 extern void __pv_queued_spin_unlock(struct qspinlock *lock);
+extern struct static_key_true sharedprocessor_key;
 
 static inline void pv_queued_spin_lock(struct qspinlock *lock, u32 val)
 {
@@ -20,7 +22,21 @@ static inline void pv_queued_spin_lock(struct qspinlock *lock, u32 val)
 
 static inline void pv_queued_spin_unlock(struct qspinlock *lock)
 {
-	pv_lock_op.unlock(lock);
+	/*
+	 * on powerNV and pSeries with jump_label, code will be
+	 *	PowerNV:		pSeries:
+	 *	nop;			b 2f;
+	 *	native unlock		2:
+	 *				pv unlock;
+	 * In this way, we can do unlock quick in native case.
+	 *
+	 * IF jump_label is not enabled, we fall back into
+	 * if condition, IOW, ld && cmp && bne.
+	 */
+	if (static_branch_likely(&sharedprocessor_key))
+		native_queued_spin_unlock(lock);
+	else
+		pv_lock_op.unlock(lock);
 }
 
 static inline void pv_wait(u8 *ptr, u8 val)
diff --git a/arch/powerpc/kernel/paravirt.c b/arch/powerpc/kernel/paravirt.c
index e697b17..a0a000e 100644
--- a/arch/powerpc/kernel/paravirt.c
+++ b/arch/powerpc/kernel/paravirt.c
@@ -140,6 +140,9 @@ struct pv_lock_ops pv_lock_op = {
 };
 EXPORT_SYMBOL(pv_lock_op);
 
+struct static_key_true sharedprocessor_key = STATIC_KEY_TRUE_INIT;
+EXPORT_SYMBOL(sharedprocessor_key);
+
 void __init pv_lock_init(void)
 {
 	if (SHARED_PROCESSOR) {
@@ -149,5 +152,6 @@ void __init pv_lock_init(void)
 		pv_lock_op.unlock = __pv_queued_spin_unlock;
 		pv_lock_op.wait = __pv_wait;
 		pv_lock_op.kick = __pv_kick;
+		static_branch_disable(&sharedprocessor_key);
 	}
 }
-- 
2.4.11