| lists.openwall.net | lists / announce owl-users owl-dev john-users john-dev passwdqc-users yescrypt popa3d-users / oss-security kernel-hardening musl sabotage tlsify passwords / crypt-dev xvendor / Bugtraq Full-Disclosure linux-kernel linux-netdev linux-ext4 linux-hardening linux-cve-announce PHC | |
|
Open Source and information security mailing list archives
| ||
|
Message-Id: <20220208184208.79303-12-namhyung@kernel.org>
Date: Tue, 8 Feb 2022 10:42:07 -0800
From: Namhyung Kim <namhyung@...nel.org>
To: Peter Zijlstra <peterz@...radead.org>,
Ingo Molnar <mingo@...nel.org>, Will Deacon <will@...nel.org>,
Waiman Long <longman@...hat.com>,
Boqun Feng <boqun.feng@...il.com>
Cc: LKML <linux-kernel@...r.kernel.org>,
Thomas Gleixner <tglx@...utronix.de>,
Steven Rostedt <rostedt@...dmis.org>,
Byungchul Park <byungchul.park@....com>,
"Paul E. McKenney" <paul.mckenney@...aro.org>,
Mathieu Desnoyers <mathieu.desnoyers@...icios.com>,
Radoslaw Burny <rburny@...gle.com>
Subject: [PATCH 11/12] locking/mutex: Revive fast functions for CONFIG_LOCK_TRACEPOINTS
The CONFIG_LOCK_TRACEPOINTS used the same path as CONFIG_DEBUG_ALLOC
or CONFIG_LOCKDEP but it caused performance impact on mutex as it
removes _fast variants at the beginning.
I'm not entirely sure why it removed the fast versions when lockdep is
on. It seems easy to add the required annotation when the fast
version is succeeded as far as the tracpoints are concerned.
It reduces around 2% of elapsed time when I ran a micro-benchmark. It
was `perf bench sched messaging -p -l 1000000` and it had a pretty low
lock contention rate (under 1% of total lock acquisition). So the
improvement should come from the fast path.
Signed-off-by: Namhyung Kim <namhyung@...nel.org>
---
include/linux/mutex.h | 2 +-
kernel/locking/mutex.c | 30 ++++++++++++++++++++----------
2 files changed, 21 insertions(+), 11 deletions(-)
diff --git a/include/linux/mutex.h b/include/linux/mutex.h
index b2d018250a41..49bde305453e 100644
--- a/include/linux/mutex.h
+++ b/include/linux/mutex.h
@@ -172,7 +172,7 @@ do { \
* See kernel/locking/mutex.c for detailed documentation of these APIs.
* Also see Documentation/locking/mutex-design.rst.
*/
-#ifdef CONFIG_LOCK_INFO
+#ifdef CONFIG_DEBUG_LOCK_ALLOC
extern void mutex_lock_nested(struct mutex *lock, unsigned int subclass);
extern void _mutex_lock_nest_lock(struct mutex *lock, struct lockdep_map *nest_lock);
diff --git a/kernel/locking/mutex.c b/kernel/locking/mutex.c
index 8733b96ce20a..f8bc4ae312a0 100644
--- a/kernel/locking/mutex.c
+++ b/kernel/locking/mutex.c
@@ -149,7 +149,7 @@ static inline bool __mutex_trylock(struct mutex *lock)
return !__mutex_trylock_common(lock, false);
}
-#ifndef CONFIG_LOCK_INFO
+#ifndef CONFIG_DEBUG_LOCK_ALLOC
/*
* Lockdep annotations are contained to the slow paths for simplicity.
* There is nothing that would stop spreading the lockdep annotations outwards
@@ -245,7 +245,7 @@ static void __mutex_handoff(struct mutex *lock, struct task_struct *task)
}
}
-#ifndef CONFIG_LOCK_INFO
+#ifndef CONFIG_DEBUG_LOCK_ALLOC
/*
* We split the mutex lock/unlock logic into separate fastpath and
* slowpath functions, to reduce the register pressure on the fastpath.
@@ -281,6 +281,8 @@ void __sched mutex_lock(struct mutex *lock)
if (!__mutex_trylock_fast(lock))
__mutex_lock_slowpath(lock);
+ else
+ mutex_acquire(&lock->dep_map, 0, 0, _RET_IP_);
}
EXPORT_SYMBOL(mutex_lock);
#endif
@@ -533,9 +535,11 @@ static noinline void __sched __mutex_unlock_slowpath(struct mutex *lock, unsigne
*/
void __sched mutex_unlock(struct mutex *lock)
{
-#ifndef CONFIG_LOCK_INFO
- if (__mutex_unlock_fast(lock))
+#ifndef CONFIG_DEBUG_LOCK_ALLOC
+ if (__mutex_unlock_fast(lock)) {
+ mutex_release(&lock->dep_map, _RET_IP_);
return;
+ }
#endif
__mutex_unlock_slowpath(lock, _RET_IP_);
}
@@ -591,7 +595,7 @@ __mutex_lock_common(struct mutex *lock, unsigned int state, unsigned int subclas
if (ww_ctx->acquired == 0)
ww_ctx->wounded = 0;
-#ifdef CONFIG_LOCK_INFO
+#ifdef CONFIG_DEBUG_LOCK_ALLOC
nest_lock = &ww_ctx->dep_map;
#endif
}
@@ -778,7 +782,7 @@ int ww_mutex_trylock(struct ww_mutex *ww, struct ww_acquire_ctx *ww_ctx)
}
EXPORT_SYMBOL(ww_mutex_trylock);
-#ifdef CONFIG_LOCK_INFO
+#ifdef CONFIG_DEBUG_LOCK_ALLOC
void __sched
mutex_lock_nested(struct mutex *lock, unsigned int subclass)
{
@@ -937,7 +941,7 @@ static noinline void __sched __mutex_unlock_slowpath(struct mutex *lock, unsigne
wake_up_q(&wake_q);
}
-#ifndef CONFIG_LOCK_INFO
+#ifndef CONFIG_DEBUG_LOCK_ALLOC
/*
* Here come the less common (and hence less performance-critical) APIs:
* mutex_lock_interruptible() and mutex_trylock().
@@ -964,8 +968,10 @@ int __sched mutex_lock_interruptible(struct mutex *lock)
{
might_sleep();
- if (__mutex_trylock_fast(lock))
+ if (__mutex_trylock_fast(lock)) {
+ mutex_acquire(&lock->dep_map, 0, 0, _RET_IP_);
return 0;
+ }
return __mutex_lock_interruptible_slowpath(lock);
}
@@ -988,8 +994,10 @@ int __sched mutex_lock_killable(struct mutex *lock)
{
might_sleep();
- if (__mutex_trylock_fast(lock))
+ if (__mutex_trylock_fast(lock)) {
+ mutex_acquire(&lock->dep_map, 0, 0, _RET_IP_);
return 0;
+ }
return __mutex_lock_killable_slowpath(lock);
}
@@ -1078,7 +1086,7 @@ int __sched mutex_trylock(struct mutex *lock)
}
EXPORT_SYMBOL(mutex_trylock);
-#ifndef CONFIG_LOCK_INFO
+#ifndef CONFIG_DEBUG_LOCK_ALLOC
int __sched
ww_mutex_lock(struct ww_mutex *lock, struct ww_acquire_ctx *ctx)
{
@@ -1087,6 +1095,7 @@ ww_mutex_lock(struct ww_mutex *lock, struct ww_acquire_ctx *ctx)
if (__mutex_trylock_fast(&lock->base)) {
if (ctx)
ww_mutex_set_context_fastpath(lock, ctx);
+ mutex_acquire(&lock->base.dep_map, 0, 0, _RET_IP_);
return 0;
}
@@ -1102,6 +1111,7 @@ ww_mutex_lock_interruptible(struct ww_mutex *lock, struct ww_acquire_ctx *ctx)
if (__mutex_trylock_fast(&lock->base)) {
if (ctx)
ww_mutex_set_context_fastpath(lock, ctx);
+ mutex_acquire(&lock->base.dep_map, 0, 0, _RET_IP_);
return 0;
}
--
2.35.0.263.gb82422642f-goog
Powered by blists - more mailing lists