linux-kernel - [PATCH 4/5] locking/rwsem: Enable timeout check when staying in the OSQ

lists.openwall.net		lists / announce owl-users owl-dev john-users john-dev passwdqc-users yescrypt popa3d-users / oss-security kernel-hardening musl sabotage tlsify passwords / crypt-dev xvendor / Bugtraq Full-Disclosure linux-kernel linux-netdev linux-ext4 linux-hardening linux-cve-announce PHC
Open Source and information security mailing list archives

Hash Suite: Windows password security audit tool. GUI, reports in PDF.

[<prev] [next>] [<thread-prev] [thread-next>] [day] [month] [year] [list]

Message-Id: <20190911150537.19527-5-longman@redhat.com>
Date:   Wed, 11 Sep 2019 16:05:36 +0100
From:   Waiman Long <longman@...hat.com>
To:     Peter Zijlstra <peterz@...radead.org>,
        Ingo Molnar <mingo@...hat.com>,
        Will Deacon <will.deacon@....com>,
        Alexander Viro <viro@...iv.linux.org.uk>,
        Mike Kravetz <mike.kravetz@...cle.com>
Cc:     linux-kernel@...r.kernel.org, linux-fsdevel@...r.kernel.org,
        linux-mm@...ck.org, Davidlohr Bueso <dave@...olabs.net>,
        Waiman Long <longman@...hat.com>
Subject: [PATCH 4/5] locking/rwsem: Enable timeout check when staying in the OSQ

Use the break function allowed by the new osq_lock() to enable early
break from the OSQ when a timeout value is specified and expiration
time has been reached.

Signed-off-by: Waiman Long <longman@...hat.com>
---
 kernel/locking/rwsem.c | 35 +++++++++++++++++++++++++++++++----
 1 file changed, 31 insertions(+), 4 deletions(-)

diff --git a/kernel/locking/rwsem.c b/kernel/locking/rwsem.c
index c15926ecb21e..78708097162a 100644
--- a/kernel/locking/rwsem.c
+++ b/kernel/locking/rwsem.c
@@ -794,23 +794,50 @@ static inline u64 rwsem_rspin_threshold(struct rw_semaphore *sem)
 	return sched_clock() + delta;
 }
 
+struct rwsem_break_arg {
+	u64 timeout;
+	int loopcnt;
+};
+
+static bool rwsem_osq_break(void *brk_arg)
+{
+	struct rwsem_break_arg *arg = brk_arg;
+
+	arg->loopcnt++;
+	/*
+	 * Check sched_clock() only once every 256 iterations.
+	 */
+	if (!(arg->loopcnt++ & 0xff) && (sched_clock() >= arg->timeout))
+		return true;
+	return false;
+}
+
 static bool rwsem_optimistic_spin(struct rw_semaphore *sem, bool wlock,
 				  ktime_t timeout)
 {
-	bool taken = false;
+	bool taken = false, locked;
 	int prev_owner_state = OWNER_NULL;
 	int loop = 0;
 	u64 rspin_threshold = 0, curtime;
+	struct rwsem_break_arg break_arg;
 	unsigned long nonspinnable = wlock ? RWSEM_WR_NONSPINNABLE
 					   : RWSEM_RD_NONSPINNABLE;
 
 	preempt_disable();
 
 	/* sem->wait_lock should not be held when doing optimistic spinning */
-	if (!osq_lock(&sem->osq, NULL, NULL))
-		goto done;
+	if (timeout) {
+		break_arg.timeout = ktime_to_ns(timeout);
+		break_arg.loopcnt = 0;
+		locked = osq_lock(&sem->osq, rwsem_osq_break, &break_arg);
+		curtime = sched_clock();
+	} else {
+		locked = osq_lock(&sem->osq, NULL, NULL);
+		curtime = 0;
+	}
 
-	curtime = timeout ? sched_clock() : 0;
+	if (!locked)
+		goto done;
 
 	/*
 	 * Optimistically spin on the owner field and attempt to acquire the
-- 
2.18.1