| lists.openwall.net | lists / announce owl-users owl-dev john-users john-dev passwdqc-users yescrypt popa3d-users / oss-security kernel-hardening musl sabotage tlsify passwords / crypt-dev xvendor / Bugtraq Full-Disclosure linux-kernel linux-netdev linux-ext4 linux-hardening linux-cve-announce PHC | |
|
Open Source and information security mailing list archives
| ||
|
Date: Tue, 23 Jan 2018 15:59:33 +0800
From: <lianglihao@...wei.com>
To: <paulmck@...ux.vnet.ibm.com>
CC: <guohanjun@...wei.com>, <heng.z@...wei.com>, <hb.chen@...wei.com>,
<lihao.liang@...il.com>, <linux-kernel@...r.kernel.org>
Subject: [PATCH RFC 08/16] prcu: Implement PRCU callback processing
From: Lihao Liang <lianglihao@...wei.com>
Currently, PRCU core processing only consists of callback processing
in prcu_process_callbacks(), which is triggered by the scheduling-clock
interrupt.
Reviewed-by: Heng Zhang <heng.z@...wei.com>
Signed-off-by: Lihao Liang <lianglihao@...wei.com>
---
include/linux/interrupt.h | 3 ++
include/linux/prcu.h | 8 +++++
kernel/rcu/prcu.c | 86 +++++++++++++++++++++++++++++++++++++++++++++++
kernel/rcu/tree.c | 1 +
kernel/time/timer.c | 2 ++
5 files changed, 100 insertions(+)
diff --git a/include/linux/interrupt.h b/include/linux/interrupt.h
index 0991f973..f05ef62a 100644
--- a/include/linux/interrupt.h
+++ b/include/linux/interrupt.h
@@ -456,6 +456,9 @@ enum
SCHED_SOFTIRQ,
HRTIMER_SOFTIRQ, /* Unused, but kept as tools rely on the
numbering. Sigh! */
+#ifdef CONFIG_PRCU
+ PRCU_SOFTIRQ,
+#endif
RCU_SOFTIRQ, /* Preferable RCU should always be the last softirq */
NR_SOFTIRQS
diff --git a/include/linux/prcu.h b/include/linux/prcu.h
index e5e09c9b..4e7d5d65 100644
--- a/include/linux/prcu.h
+++ b/include/linux/prcu.h
@@ -31,11 +31,13 @@ struct prcu_local_struct {
unsigned int locked;
unsigned int online;
unsigned long long version;
+ unsigned long long cb_version;
struct prcu_cblist cblist;
};
struct prcu_struct {
atomic64_t global_version;
+ atomic64_t cb_version;
atomic_t active_ctr;
struct mutex mtx;
wait_queue_head_t wait_q;
@@ -48,6 +50,9 @@ void synchronize_prcu(void);
void call_prcu(struct rcu_head *head, rcu_callback_t func);
void prcu_init(void);
void prcu_note_context_switch(void);
+int prcu_pending(void);
+void invoke_prcu_core(void);
+void prcu_check_callbacks(void);
#else /* #ifdef CONFIG_PRCU */
@@ -57,6 +62,9 @@ void prcu_note_context_switch(void);
#define call_prcu() do {} while (0)
#define prcu_init() do {} while (0)
#define prcu_note_context_switch() do {} while (0)
+#define prcu_pending() 0
+#define invoke_prcu_core() do {} while (0)
+#define prcu_check_callbacks() do {} while (0)
#endif /* #ifdef CONFIG_PRCU */
#endif /* __LINUX_PRCU_H */
diff --git a/kernel/rcu/prcu.c b/kernel/rcu/prcu.c
index f198285c..373039c5 100644
--- a/kernel/rcu/prcu.c
+++ b/kernel/rcu/prcu.c
@@ -1,6 +1,7 @@
#include <linux/smp.h>
#include <linux/percpu.h>
#include <linux/prcu.h>
+#include <linux/interrupt.h>
#include <linux/sched.h>
#include <linux/slab.h>
#include <asm/barrier.h>
@@ -11,6 +12,7 @@ DEFINE_PER_CPU_SHARED_ALIGNED(struct prcu_local_struct, prcu_local);
struct prcu_struct global_prcu = {
.global_version = ATOMIC64_INIT(0),
+ .cb_version = ATOMIC64_INIT(0),
.active_ctr = ATOMIC_INIT(0),
.mtx = __MUTEX_INITIALIZER(global_prcu.mtx),
.wait_q = __WAIT_QUEUE_HEAD_INITIALIZER(global_prcu.wait_q)
@@ -27,6 +29,35 @@ static void prcu_cblist_init(struct prcu_cblist *rclp)
rclp->len = 0;
}
+/*
+ * Dequeue the oldest rcu_head structure from the specified callback list;
+ * store the callback grace period version number into the version pointer.
+ */
+static struct rcu_head *prcu_cblist_dequeue(struct prcu_cblist *rclp)
+{
+ struct rcu_head *rhp;
+ struct prcu_version_head *vhp;
+
+ rhp = rclp->head;
+ if (!rhp) {
+ WARN_ON(vhp);
+ WARN_ON(rclp->len);
+ return NULL;
+ }
+
+ vhp = rclp->version_head;
+ rclp->version_head = vhp->next;
+ rclp->head = rhp->next;
+ rclp->len--;
+
+ if (!rclp->head) {
+ rclp->tail = &rclp->head;
+ rclp->version_tail = &rclp->version_head;
+ }
+
+ return rhp;
+}
+
static inline void prcu_report(struct prcu_local_struct *local)
{
unsigned long long global_version;
@@ -117,6 +148,7 @@ void synchronize_prcu(void)
if (atomic_read(&prcu->active_ctr))
wait_event(prcu->wait_q, !atomic_read(&prcu->active_ctr));
+ atomic64_set(&prcu->cb_version, version);
mutex_unlock(&prcu->mtx);
}
EXPORT_SYMBOL(synchronize_prcu);
@@ -166,6 +198,58 @@ void call_prcu(struct rcu_head *head, rcu_callback_t func)
}
EXPORT_SYMBOL(call_prcu);
+int prcu_pending(void)
+{
+ struct prcu_local_struct *local = get_cpu_ptr(&prcu_local);
+ unsigned long long cb_version = local->cb_version;
+ struct prcu_cblist *rclp = &local->cblist;
+
+ put_cpu_ptr(&prcu_local);
+ return cb_version < atomic64_read(&prcu->cb_version) && rclp->head;
+}
+
+void invoke_prcu_core(void)
+{
+ if (cpu_online(smp_processor_id()))
+ raise_softirq(PRCU_SOFTIRQ);
+}
+
+void prcu_check_callbacks(void)
+{
+ if (prcu_pending())
+ invoke_prcu_core();
+}
+
+static __latent_entropy void prcu_process_callbacks(struct softirq_action *unused)
+{
+ unsigned long flags;
+ unsigned long long cb_version;
+ struct prcu_local_struct *local;
+ struct prcu_cblist *rclp;
+ struct rcu_head *rhp;
+ struct prcu_version_head *vhp;
+
+ if (cpu_is_offline(smp_processor_id()))
+ return;
+
+ cb_version = atomic64_read(&prcu->cb_version);
+
+ /* Disable interrupts to prevent races with call_prcu() */
+ local_irq_save(flags);
+ local = this_cpu_ptr(&prcu_local);
+ rclp = &local->cblist;
+ rhp = rclp->head;
+ vhp = rclp->version_head;
+ for (; rhp && vhp && vhp->version < cb_version;
+ rhp = rclp->head, vhp = rclp->version_head) {
+ rhp = prcu_cblist_dequeue(rclp);
+ debug_rcu_head_unqueue(rhp);
+ rhp->func(rhp);
+ }
+ local->cb_version = cb_version;
+ local_irq_restore(flags);
+}
+
void prcu_init_local_struct(int cpu)
{
struct prcu_local_struct *local;
@@ -174,6 +258,7 @@ void prcu_init_local_struct(int cpu)
local->locked = 0;
local->online = 0;
local->version = 0;
+ local->cb_version = 0;
prcu_cblist_init(&local->cblist);
}
@@ -181,6 +266,7 @@ void __init prcu_init(void)
{
int cpu;
+ open_softirq(PRCU_SOFTIRQ, prcu_process_callbacks);
for_each_possible_cpu(cpu)
prcu_init_local_struct(cpu);
}
diff --git a/kernel/rcu/tree.c b/kernel/rcu/tree.c
index e354e475..46910114 100644
--- a/kernel/rcu/tree.c
+++ b/kernel/rcu/tree.c
@@ -2852,6 +2852,7 @@ void rcu_check_callbacks(int user)
{
trace_rcu_utilization(TPS("Start scheduler-tick"));
increment_cpu_stall_ticks();
+
if (user || rcu_is_cpu_rrupt_from_idle()) {
/*
diff --git a/kernel/time/timer.c b/kernel/time/timer.c
index d3f33020..ed863e63 100644
--- a/kernel/time/timer.c
+++ b/kernel/time/timer.c
@@ -44,6 +44,7 @@
#include <linux/sched/debug.h>
#include <linux/slab.h>
#include <linux/compat.h>
+#include <linux/prcu.h>
#include <linux/uaccess.h>
#include <asm/unistd.h>
@@ -1568,6 +1569,7 @@ void update_process_times(int user_tick)
/* Note: this timer irq context must be accounted for as well. */
account_process_tick(p, user_tick);
run_local_timers();
+ prcu_check_callbacks();
rcu_check_callbacks(user_tick);
#ifdef CONFIG_IRQ_WORK
if (in_irq())
--
2.14.1.729.g59c0ea183
Powered by blists - more mailing lists