linux-kernel - [RFC PATCH-tip v4 09/10] locking/rwsem: Enable reactivation of reader spinning

lists.openwall.net		lists / announce owl-users owl-dev john-users john-dev passwdqc-users yescrypt popa3d-users / oss-security kernel-hardening musl sabotage tlsify passwords / crypt-dev xvendor / Bugtraq Full-Disclosure linux-kernel linux-netdev linux-ext4 linux-hardening linux-cve-announce PHC
Open Source and information security mailing list archives

Hash Suite: Windows password security audit tool. GUI, reports in PDF.

[<prev] [next>] [<thread-prev] [thread-next>] [day] [month] [year] [list]

Date:   Thu, 18 Aug 2016 17:11:11 -0400
From:   Waiman Long <Waiman.Long@....com>
To:     Peter Zijlstra <peterz@...radead.org>,
        Ingo Molnar <mingo@...hat.com>
Cc:     linux-kernel@...r.kernel.org, x86@...nel.org,
        linux-alpha@...r.kernel.org, linux-ia64@...r.kernel.org,
        linux-s390@...r.kernel.org, linux-arch@...r.kernel.org,
        linux-doc@...r.kernel.org, Davidlohr Bueso <dave@...olabs.net>,
        Jason Low <jason.low2@...com>,
        Dave Chinner <david@...morbit.com>,
        Jonathan Corbet <corbet@....net>,
        Scott J Norton <scott.norton@....com>,
        Douglas Hatch <doug.hatch@....com>,
        Waiman Long <Waiman.Long@....com>
Subject: [RFC PATCH-tip v4 09/10] locking/rwsem: Enable reactivation of reader spinning

Reader optimistic spinning will be disabled once the rspin_enabled
count reaches 0. After that, it cannot be re-enabled. This may cause
an eligible rwsem locked out of reader spinning because of a series
of unfortunate events.

This patch looks at the regular writer-on-writer spinning history. If
there are sufficient more successful spin attempts than failed ones,
it will try to reactivate reader spinning.

Signed-off-by: Waiman Long <Waiman.Long@....com>
---
 include/linux/rwsem.h       |   12 ++++++++----
 kernel/locking/rwsem-xadd.c |   27 +++++++++++++++++++++++++--
 2 files changed, 33 insertions(+), 6 deletions(-)

diff --git a/include/linux/rwsem.h b/include/linux/rwsem.h
index 8978f87..98284b4 100644
--- a/include/linux/rwsem.h
+++ b/include/linux/rwsem.h
@@ -32,7 +32,11 @@ struct rw_semaphore {
 	raw_spinlock_t wait_lock;
 #ifdef CONFIG_RWSEM_SPIN_ON_OWNER
 	struct optimistic_spin_queue osq; /* spinner MCS lock */
-	int rspin_enabled;	/* protected by osq lock */
+	/*
+	 * Reader optimistic spinning fields protected by osq lock
+	 */
+	uint16_t rspin_enabled;
+	int16_t  wspin_cnt;
 
 	/*
 	 * Write owner. Used as a speculative check to see
@@ -74,10 +78,10 @@ static inline int rwsem_is_locked(struct rw_semaphore *sem)
 /*
  * Each successful reader spin will increment the rspin_enabled by 1.
  * Each unsuccessful spin, on the other hand, will decrement it by 2.
- * Reader spinning will be permanently disabled when it reaches 0.
+ * Reader spinning will be disabled when it reaches 0.
  */
 #ifndef RWSEM_RSPIN_ENABLED_DEFAULT
-# define RWSEM_RSPIN_ENABLED_DEFAULT	40
+# define RWSEM_RSPIN_ENABLED_DEFAULT	30
 #endif
 #define RWSEM_RSPIN_ENABLED_MAX		1024
 
@@ -87,7 +91,7 @@ static inline int rwsem_is_locked(struct rw_semaphore *sem)
 
 #ifdef CONFIG_RWSEM_SPIN_ON_OWNER
 #define __RWSEM_OPT_INIT(lockname) , .osq = OSQ_LOCK_UNLOCKED, .owner = NULL, \
-		.rspin_enabled = RWSEM_RSPIN_ENABLED_DEFAULT
+		.rspin_enabled = RWSEM_RSPIN_ENABLED_DEFAULT, .wspin_cnt = 0
 #else
 #define __RWSEM_OPT_INIT(lockname)
 #endif
diff --git a/kernel/locking/rwsem-xadd.c b/kernel/locking/rwsem-xadd.c
index d850214..9978159 100644
--- a/kernel/locking/rwsem-xadd.c
+++ b/kernel/locking/rwsem-xadd.c
@@ -108,6 +108,7 @@ void __init_rwsem(struct rw_semaphore *sem, const char *name,
 #ifdef CONFIG_RWSEM_SPIN_ON_OWNER
 	sem->owner = NULL;
 	sem->rspin_enabled = RWSEM_RSPIN_ENABLED_DEFAULT;
+	sem->wspin_cnt = 0;
 	osq_lock_init(&sem->osq);
 #endif
 }
@@ -458,10 +459,32 @@ static bool rwsem_optimistic_spin(struct rw_semaphore *sem,
 		if (taken && (sem->rspin_enabled < RWSEM_RSPIN_ENABLED_MAX)) {
 			sem->rspin_enabled++;
 		} else if (!taken) {
-			if  (sem->rspin_enabled > 2)
+			if  (sem->rspin_enabled > 2) {
 				sem->rspin_enabled -= 2;
-			else
+			} else if (sem->rspin_enabled) {
 				sem->rspin_enabled = 0;
+				/*
+				 * Reset wspin_cnt so that it won't get
+				 * re-enabled too soon.
+				 */
+				if (sem->wspin_cnt > -30)
+					sem->wspin_cnt = -30;
+			}
+		}
+	} else if (type == RWSEM_WAITING_FOR_WRITE) {
+		/*
+		 * Every 10 successful writer-on-writer spins more than failed
+		 * spins will increment rspin_enabled to encourage more
+		 * writer-on-reader spinning attempts.
+		 */
+		if (taken) {
+			if ((++sem->wspin_cnt >= 10) &&
+			    (sem->rspin_enabled < RWSEM_RSPIN_ENABLED_MAX)) {
+				sem->wspin_cnt = 0;
+				sem->rspin_enabled++;
+			}
+		} else if (sem->wspin_cnt > -100) {
+			sem->wspin_cnt--;
 		}
 	}
 	osq_unlock(&sem->osq);
-- 
1.7.1