| lists.openwall.net | lists / announce owl-users owl-dev john-users john-dev passwdqc-users yescrypt popa3d-users / oss-security kernel-hardening musl sabotage tlsify passwords / crypt-dev xvendor / Bugtraq Full-Disclosure linux-kernel linux-netdev linux-ext4 linux-hardening linux-cve-announce PHC | |
|
Open Source and information security mailing list archives
| ||
|
Date: Thu, 26 Jun 2014 19:01:43 +0200
From: Oleg Nesterov <oleg@...hat.com>
To: "Paul E. McKenney" <paulmck@...ux.vnet.ibm.com>
Cc: Josh Triplett <josh@...htriplett.org>,
Lai Jiangshan <laijs@...fujitsu.com>,
Peter Zijlstra <peterz@...radead.org>,
linux-kernel@...r.kernel.org
Subject: [PATCH 1/1] rcu: uninline rcu_lock_acquire() and rcu_lock_release()
Uninline rcu_lock_acquire() and rcu_lock_release() to shrink .text/data.
The difference in "size vmlinux" looks good,
with CONFIG_DEBUG_LOCK_ALLOC
- 5377829 3018320 14757888 23154037
+ 5352229 3010160 14757888 23120277
33760 bytes saved.
with CONFIG_DEBUG_LOCK_ALLOC + CONFIG_PROVE_RCU + CONFIG_TREE_RCU_TRACE
- 5682283 3022936 14757888 23463107
+ 5578667 3026776 14757888 23363331
saves 99776 bytes.
However, this obviously means that the "warn once" logic is moved from
the current callers of rcu_lockdep_assert(rcu_is_watching()) to update.c.
Also, with this patch we do not bother to report which function abused
rcu_is_watching(), this should be clear from dump_stack().
Signed-off-by: Oleg Nesterov <oleg@...hat.com>
---
include/linux/rcupdate.h | 55 ++++++++++++++++++++++---------------------
include/linux/srcu.h | 4 +-
kernel/rcu/rcu.h | 6 ++--
kernel/rcu/update.c | 57 ++++++++++++++++++++++++++++++++++++++++++++++
4 files changed, 90 insertions(+), 32 deletions(-)
diff --git a/include/linux/rcupdate.h b/include/linux/rcupdate.h
index 5a75d19..a5b1631 100644
--- a/include/linux/rcupdate.h
+++ b/include/linux/rcupdate.h
@@ -379,18 +379,34 @@ static inline bool rcu_lockdep_current_cpu_online(void)
}
#endif /* #else #if defined(CONFIG_HOTPLUG_CPU) && defined(CONFIG_PROVE_RCU) */
-#ifdef CONFIG_DEBUG_LOCK_ALLOC
-
-static inline void rcu_lock_acquire(struct lockdep_map *map)
+static inline void __rcu_lock_acquire(struct lockdep_map *map, unsigned long ip)
{
- lock_acquire(map, 0, 0, 2, 0, NULL, _THIS_IP_);
+ lock_acquire(map, 0, 0, 2, 0, NULL, ip);
}
-static inline void rcu_lock_release(struct lockdep_map *map)
+static inline void __rcu_lock_release(struct lockdep_map *map, unsigned long ip)
{
- lock_release(map, 1, _THIS_IP_);
+ lock_release(map, 1, ip);
}
+#if defined(CONFIG_DEBUG_LOCK_ALLOC) || defined(CONFIG_PROVE_RCU)
+extern void rcu_lock_acquire(void);
+extern void rcu_lock_release(void);
+extern void rcu_lock_acquire_bh(void);
+extern void rcu_lock_release_bh(void);
+extern void rcu_lock_acquire_sched(void);
+extern void rcu_lock_release_sched(void);
+#else
+#define rcu_lock_acquire() do { } while (0)
+#define rcu_lock_release() do { } while (0)
+#define rcu_lock_acquire_bh() do { } while (0)
+#define rcu_lock_release_bh() do { } while (0)
+#define rcu_lock_acquire_sched() do { } while (0)
+#define rcu_lock_release_sched() do { } while (0)
+#endif
+
+#ifdef CONFIG_DEBUG_LOCK_ALLOC
+
extern struct lockdep_map rcu_lock_map;
extern struct lockdep_map rcu_bh_lock_map;
extern struct lockdep_map rcu_sched_lock_map;
@@ -489,9 +505,6 @@ static inline int rcu_read_lock_sched_held(void)
#else /* #ifdef CONFIG_DEBUG_LOCK_ALLOC */
-# define rcu_lock_acquire(a) do { } while (0)
-# define rcu_lock_release(a) do { } while (0)
-
static inline int rcu_read_lock_held(void)
{
return 1;
@@ -866,9 +879,7 @@ static inline void rcu_read_lock(void)
{
__rcu_read_lock();
__acquire(RCU);
- rcu_lock_acquire(&rcu_lock_map);
- rcu_lockdep_assert(rcu_is_watching(),
- "rcu_read_lock() used illegally while idle");
+ rcu_lock_acquire();
}
/*
@@ -888,9 +899,7 @@ static inline void rcu_read_lock(void)
*/
static inline void rcu_read_unlock(void)
{
- rcu_lockdep_assert(rcu_is_watching(),
- "rcu_read_unlock() used illegally while idle");
- rcu_lock_release(&rcu_lock_map);
+ rcu_lock_release();
__release(RCU);
__rcu_read_unlock();
}
@@ -916,9 +925,7 @@ static inline void rcu_read_lock_bh(void)
{
local_bh_disable();
__acquire(RCU_BH);
- rcu_lock_acquire(&rcu_bh_lock_map);
- rcu_lockdep_assert(rcu_is_watching(),
- "rcu_read_lock_bh() used illegally while idle");
+ rcu_lock_acquire_bh();
}
/*
@@ -928,9 +935,7 @@ static inline void rcu_read_lock_bh(void)
*/
static inline void rcu_read_unlock_bh(void)
{
- rcu_lockdep_assert(rcu_is_watching(),
- "rcu_read_unlock_bh() used illegally while idle");
- rcu_lock_release(&rcu_bh_lock_map);
+ rcu_lock_release_bh();
__release(RCU_BH);
local_bh_enable();
}
@@ -952,9 +957,7 @@ static inline void rcu_read_lock_sched(void)
{
preempt_disable();
__acquire(RCU_SCHED);
- rcu_lock_acquire(&rcu_sched_lock_map);
- rcu_lockdep_assert(rcu_is_watching(),
- "rcu_read_lock_sched() used illegally while idle");
+ rcu_lock_acquire_sched();
}
/* Used by lockdep and tracing: cannot be traced, cannot call lockdep. */
@@ -971,9 +974,7 @@ static inline notrace void rcu_read_lock_sched_notrace(void)
*/
static inline void rcu_read_unlock_sched(void)
{
- rcu_lockdep_assert(rcu_is_watching(),
- "rcu_read_unlock_sched() used illegally while idle");
- rcu_lock_release(&rcu_sched_lock_map);
+ rcu_lock_release_sched();
__release(RCU_SCHED);
preempt_enable();
}
diff --git a/include/linux/srcu.h b/include/linux/srcu.h
index a2783cb..5c06289 100644
--- a/include/linux/srcu.h
+++ b/include/linux/srcu.h
@@ -219,7 +219,7 @@ static inline int srcu_read_lock(struct srcu_struct *sp) __acquires(sp)
{
int retval = __srcu_read_lock(sp);
- rcu_lock_acquire(&(sp)->dep_map);
+ __rcu_lock_acquire(&(sp)->dep_map, _THIS_IP_);
return retval;
}
@@ -233,7 +233,7 @@ static inline int srcu_read_lock(struct srcu_struct *sp) __acquires(sp)
static inline void srcu_read_unlock(struct srcu_struct *sp, int idx)
__releases(sp)
{
- rcu_lock_release(&(sp)->dep_map);
+ __rcu_lock_release(&(sp)->dep_map, _THIS_IP_);
__srcu_read_unlock(sp, idx);
}
diff --git a/kernel/rcu/rcu.h b/kernel/rcu/rcu.h
index bfda272..5702910 100644
--- a/kernel/rcu/rcu.h
+++ b/kernel/rcu/rcu.h
@@ -103,16 +103,16 @@ static inline bool __rcu_reclaim(const char *rn, struct rcu_head *head)
{
unsigned long offset = (unsigned long)head->func;
- rcu_lock_acquire(&rcu_callback_map);
+ __rcu_lock_acquire(&rcu_callback_map, _THIS_IP_);
if (__is_kfree_rcu_offset(offset)) {
RCU_TRACE(trace_rcu_invoke_kfree_callback(rn, head, offset));
kfree((void *)head - offset);
- rcu_lock_release(&rcu_callback_map);
+ __rcu_lock_release(&rcu_callback_map, _THIS_IP_);
return 1;
} else {
RCU_TRACE(trace_rcu_invoke_callback(rn, head));
head->func(head);
- rcu_lock_release(&rcu_callback_map);
+ __rcu_lock_release(&rcu_callback_map, _THIS_IP_);
return 0;
}
}
diff --git a/kernel/rcu/update.c b/kernel/rcu/update.c
index a2aeb4d..bf98a37 100644
--- a/kernel/rcu/update.c
+++ b/kernel/rcu/update.c
@@ -168,6 +168,63 @@ EXPORT_SYMBOL_GPL(rcu_read_lock_bh_held);
#endif /* #ifdef CONFIG_DEBUG_LOCK_ALLOC */
+#if defined(CONFIG_DEBUG_LOCK_ALLOC) || defined(CONFIG_PROVE_RCU)
+
+static void rcu_lockdep_assert_watching(void)
+{
+ rcu_lockdep_assert(rcu_is_watching(), "RCU used illegally while idle");
+}
+
+static void rcu_acquire_map(struct lockdep_map *map, unsigned long ip)
+{
+ __rcu_lock_acquire(map, ip);
+ rcu_lockdep_assert_watching();
+}
+
+static void rcu_release_map(struct lockdep_map *map, unsigned long ip)
+{
+ rcu_lockdep_assert_watching();
+ __rcu_lock_release(&rcu_lock_map, ip);
+}
+
+void rcu_lock_acquire(void)
+{
+ rcu_acquire_map(&rcu_lock_map, _RET_IP_);
+}
+EXPORT_SYMBOL(rcu_lock_acquire);
+
+void rcu_lock_release(void)
+{
+ rcu_release_map(&rcu_lock_map, _RET_IP_);
+}
+EXPORT_SYMBOL(rcu_lock_release);
+
+void rcu_lock_acquire_bh(void)
+{
+ rcu_acquire_map(&rcu_bh_lock_map, _RET_IP_);
+}
+EXPORT_SYMBOL(rcu_lock_acquire_bh);
+
+void rcu_lock_release_bh(void)
+{
+ rcu_release_map(&rcu_bh_lock_map, _RET_IP_);
+}
+EXPORT_SYMBOL(rcu_lock_release_bh);
+
+void rcu_lock_acquire_sched(void)
+{
+ rcu_acquire_map(&rcu_sched_lock_map, _RET_IP_);
+}
+EXPORT_SYMBOL(rcu_lock_acquire_sched);
+
+void rcu_lock_release_sched(void)
+{
+ rcu_release_map(&rcu_sched_lock_map, _RET_IP_);
+}
+EXPORT_SYMBOL(rcu_lock_release_sched);
+
+#endif
+
struct rcu_synchronize {
struct rcu_head head;
struct completion completion;
--
1.5.5.1
--
To unsubscribe from this list: send the line "unsubscribe linux-kernel" in
the body of a message to majordomo@...r.kernel.org
More majordomo info at http://vger.kernel.org/majordomo-info.html
Please read the FAQ at http://www.tux.org/lkml/
Powered by blists - more mailing lists