| lists.openwall.net | lists / announce owl-users owl-dev john-users john-dev passwdqc-users yescrypt popa3d-users / oss-security kernel-hardening musl sabotage tlsify passwords / crypt-dev xvendor / Bugtraq Full-Disclosure linux-kernel linux-netdev linux-ext4 linux-hardening linux-cve-announce PHC | |
|
Open Source and information security mailing list archives
| ||
|
Message-ID: <48A70192.7020002@goop.org>
Date: Sat, 16 Aug 2008 09:34:26 -0700
From: Jeremy Fitzhardinge <jeremy@...p.org>
To: Ingo Molnar <mingo@...e.hu>
CC: Jens Axboe <jens.axboe@...cle.com>,
Peter Zijlstra <a.p.zijlstra@...llo.nl>,
Christian Borntraeger <borntraeger@...ibm.com>,
Linux Kernel Mailing List <linux-kernel@...r.kernel.org>,
Rusty Russell <rusty@...tcorp.com.au>,
Arjan van de Ven <arjan@...radead.org>
Subject: [PATCH RFC 3/3] x86: use mwait for trigger API
If the cpu implements monitor/mwait, use it for the trigger API.
TODO: work out if any mwait hints are going to be useful here.
Signed-off-by: Jeremy Fitzhardinge <jeremy.fitzhardinge@...rix.com>
Cc: Ingo Molnar <mingo@...e.hu>
Cc: Peter Zijlstra <a.p.zijlstra@...llo.nl>
Cc: Christian Borntraeger <borntraeger@...ibm.com>
Cc: Rusty Russell <rusty@...tcorp.com.au>
---
arch/x86/kernel/paravirt-spinlocks.c | 22 ++++++++++++
arch/x86/kernel/trigger.c | 59 ++++++++++++++++++++++++++++++++++
include/asm-x86/trigger.h | 27 +++++++++++++--
3 files changed, 104 insertions(+), 4 deletions(-)
===================================================================
--- a/arch/x86/kernel/paravirt-spinlocks.c
+++ b/arch/x86/kernel/paravirt-spinlocks.c
@@ -7,6 +7,7 @@
#include <linux/module.h>
#include <asm/paravirt.h>
+#include <asm/cpufeature.h>
static void default_spin_lock_flags(struct raw_spinlock *lock, unsigned long flags)
{
@@ -41,3 +42,24 @@
pv_lock_ops.spin_unlock = __byte_spin_unlock;
#endif
}
+
+static int __init use_mwait_trigger(void)
+{
+ /*
+ * If we're using normal native trigger operations and the cpu
+ * supports mwait, then switch to using it.
+ */
+ if (pv_lock_ops.trigger_wait == native_trigger_wait &&
+ pv_lock_ops.trigger_reset == native_trigger_reset &&
+ pv_lock_ops.trigger_kick == native_trigger_kick &&
+ pv_lock_ops.trigger_finish == paravirt_nop &&
+ boot_cpu_has(X86_FEATURE_MWAIT)) {
+ pv_lock_ops.trigger_reset = mwait_trigger_reset;
+ pv_lock_ops.trigger_wait = mwait_trigger_wait;
+ pv_lock_ops.trigger_kick = mwait_trigger_kick;
+ pv_lock_ops.trigger_finish = mwait_trigger_finish;
+ }
+
+ return 0;
+}
+early_initcall(use_mwait_trigger);
===================================================================
--- a/arch/x86/kernel/trigger.c
+++ b/arch/x86/kernel/trigger.c
@@ -1,5 +1,9 @@
#include <linux/trigger.h>
#include <linux/smp.h>
+#include <linux/percpu.h>
+#include <linux/irqflags.h>
+
+#include <asm/processor.h>
void native_trigger_reset(trigger_t *t)
{
@@ -20,4 +24,59 @@
{
smp_wmb();
cpus_setall(t->cpus);
+ smp_wmb();
}
+
+/*
+ * mwait requires interrupts to be disabled between monitor and mwait.
+ * On monitor, we disable interrupts and store the previous state
+ * here. When mwaiting, we enable interrupts if they were originally
+ * enabled.
+ */
+static DEFINE_PER_CPU(unsigned long, mwait_saved_flags);
+
+void mwait_trigger_reset(trigger_t *t)
+{
+ unsigned long flags;
+
+ t->trigger = 0;
+
+ local_save_flags(flags);
+ __get_cpu_var(mwait_saved_flags) = flags;
+
+ __monitor(&t->trigger, 0, 0);
+}
+
+void mwait_trigger_wait(trigger_t *t)
+{
+ unsigned long flags = __get_cpu_var(mwait_saved_flags);
+ int cpu = smp_processor_id();
+
+ if (irqs_disabled_flags(flags)) {
+ while(!cpu_isset(cpu, t->cpus)) {
+ __mwait(0, 0);
+ barrier();
+ __monitor(&t->trigger, 0, 0);
+ }
+ } else {
+ while(!cpu_isset(cpu, t->cpus)) {
+ __sti_mwait(0, 0);
+ barrier();
+ local_irq_disable();
+ __monitor(&t->trigger, 0, 0);
+ }
+ }
+}
+
+void mwait_trigger_finish(trigger_t *t)
+{
+ local_irq_restore(__get_cpu_var(mwait_saved_flags));
+}
+
+void mwait_trigger_kick(trigger_t *t)
+{
+ cpus_setall(t->cpus);
+ smp_wmb();
+ t->trigger = 1;
+ smp_wmb();
+}
===================================================================
--- a/include/asm-x86/trigger.h
+++ b/include/asm-x86/trigger.h
@@ -1,5 +1,7 @@
#ifndef _ASM_X86_TRIGGER_H
#define _ASM_X86_TRIGGER_H
+
+#include <asm/cpufeature.h>
#include <linux/cpumask.h>
@@ -30,27 +32,44 @@
extern void native_trigger_wait(trigger_t *t);
extern void native_trigger_kick(trigger_t *t);
+extern void mwait_trigger_reset(trigger_t *);
+extern void mwait_trigger_wait(trigger_t *);
+extern void mwait_trigger_finish(trigger_t *);
+extern void mwait_trigger_kick(trigger_t *);
+
#ifdef CONFIG_PARAVIRT
#include <asm/paravirt.h>
#else /* CONFIG_PARAVIRT */
static inline void __raw_trigger_reset(trigger_t *t)
{
- native_trigger_reset(t);
+ if (boot_cpu_has(X86_FEATURE_MWAIT))
+ mwait_trigger_reset(t);
+ else
+ native_trigger_reset(t);
}
static inline void __raw_trigger_wait(trigger_t *t)
{
- native_trigger_wait(t);
+ if (boot_cpu_has(X86_FEATURE_MWAIT))
+ mwait_trigger_wait(t);
+ else
+ native_trigger_wait(t);
}
static inline void __raw_trigger_finish(trigger_t *t)
{
- native_trigger_finish(t);
+ if (boot_cpu_has(X86_FEATURE_MWAIT))
+ mwait_trigger_finish(t);
+ else
+ native_trigger_finish(t);
}
static inline void __raw_trigger_kick(trigger_t *t)
{
- native_trigger_kick(t);
+ if (boot_cpu_has(X86_FEATURE_MWAIT))
+ mwait_trigger_finish(t);
+ else
+ native_trigger_finish(t);
}
#endif /* CONFIG_PARAVIRT */
--
To unsubscribe from this list: send the line "unsubscribe linux-kernel" in
the body of a message to majordomo@...r.kernel.org
More majordomo info at http://vger.kernel.org/majordomo-info.html
Please read the FAQ at http://www.tux.org/lkml/
Powered by blists - more mailing lists