linux-kernel - [PATCH] intel_idle: use static_key to optimize idle enter/exit paths

lists.openwall.net		lists / announce owl-users owl-dev john-users john-dev passwdqc-users yescrypt popa3d-users / oss-security kernel-hardening musl sabotage tlsify passwords / crypt-dev xvendor / Bugtraq Full-Disclosure linux-kernel linux-netdev linux-ext4 linux-hardening linux-cve-announce PHC
Open Source and information security mailing list archives

Hash Suite: Windows password security audit tool. GUI, reports in PDF.

[<prev] [next>] [thread-next>] [day] [month] [year] [list]

Message-Id: <20140711175458.6793B2029@prod-mail-relay06.akamai.com>
Date:	Fri, 11 Jul 2014 17:54:58 +0000 (GMT)
From:	Jason Baron <jbaron@...mai.com>
To:	lenb@...nel.org
Cc:	linux-pm@...r.kernel.org, linux-kernel@...r.kernel.org
Subject: [PATCH] intel_idle: use static_key to optimize idle enter/exit paths

If 'arat' is set in the cpuflags, we can avoid the checks for entering/exiting
the tick broadcast code entirely. It would seem that this is a hot enough code
path to make this worthwhile. I ran a few hackbench runs, and consistenly see
reduced branches and cycles.

Signed-off-by: Jason Baron <jbaron@...mai.com>
---
 drivers/idle/intel_idle.c | 29 ++++++++++++++++++++---------
 1 file changed, 20 insertions(+), 9 deletions(-)

diff --git a/drivers/idle/intel_idle.c b/drivers/idle/intel_idle.c
index 4d140bb..61e965c 100644
--- a/drivers/idle/intel_idle.c
+++ b/drivers/idle/intel_idle.c
@@ -80,6 +80,8 @@ static unsigned int mwait_substates;
 #define LAPIC_TIMER_ALWAYS_RELIABLE 0xFFFFFFFF
 /* Reliable LAPIC Timer States, bit 1 for C1 etc.  */
 static unsigned int lapic_timer_reliable_states = (1 << 1);	 /* Default to only C1 */
+/* if arat is set no sense in checking on each c-state transition */
+static struct static_key lapic_timer_unreliable __read_mostly;
 
 struct idle_cpu {
 	struct cpuidle_state *state_table;
@@ -507,12 +509,10 @@ static int intel_idle(struct cpuidle_device *dev,
 {
 	unsigned long ecx = 1; /* break on interrupt flag */
 	struct cpuidle_state *state = &drv->states[index];
-	unsigned long eax = flg2MWAIT(state->flags);
-	unsigned int cstate;
+	unsigned long uninitialized_var(eax);
+	unsigned int uninitialized_var(cstate);
 	int cpu = smp_processor_id();
 
-	cstate = (((eax) >> MWAIT_SUBSTATE_SIZE) & MWAIT_CSTATE_MASK) + 1;
-
 	/*
 	 * leave_mm() to avoid costly and often unnecessary wakeups
 	 * for flushing the user TLB's associated with the active mm.
@@ -520,13 +520,22 @@ static int intel_idle(struct cpuidle_device *dev,
 	if (state->flags & CPUIDLE_FLAG_TLB_FLUSHED)
 		leave_mm(cpu);
 
-	if (!(lapic_timer_reliable_states & (1 << (cstate))))
-		clockevents_notify(CLOCK_EVT_NOTIFY_BROADCAST_ENTER, &cpu);
+	if (static_key_false(&lapic_timer_unreliable)) {
+		eax = flg2MWAIT(state->flags);
+		cstate = (((eax) >> MWAIT_SUBSTATE_SIZE) &
+					MWAIT_CSTATE_MASK) + 1;
+		if (!(lapic_timer_reliable_states & (1 << (cstate))))
+			clockevents_notify(CLOCK_EVT_NOTIFY_BROADCAST_ENTER,
+					   &cpu);
+	}
 
 	mwait_idle_with_hints(eax, ecx);
 
-	if (!(lapic_timer_reliable_states & (1 << (cstate))))
-		clockevents_notify(CLOCK_EVT_NOTIFY_BROADCAST_EXIT, &cpu);
+	if (static_key_false(&lapic_timer_unreliable)) {
+		if (!(lapic_timer_reliable_states & (1 << (cstate))))
+			clockevents_notify(CLOCK_EVT_NOTIFY_BROADCAST_EXIT,
+					   &cpu);
+	}
 
 	return index;
 }
@@ -702,8 +711,10 @@ static int __init intel_idle_probe(void)
 
 	if (boot_cpu_has(X86_FEATURE_ARAT))	/* Always Reliable APIC Timer */
 		lapic_timer_reliable_states = LAPIC_TIMER_ALWAYS_RELIABLE;
-	else
+	else {
+		static_key_slow_inc(&lapic_timer_unreliable);
 		on_each_cpu(__setup_broadcast_timer, (void *)true, 1);
+	}
 
 	pr_debug(PREFIX "v" INTEL_IDLE_VERSION
 		" model 0x%X\n", boot_cpu_data.x86_model);
-- 
1.8.2.rc2

--
To unsubscribe from this list: send the line "unsubscribe linux-kernel" in
the body of a message to majordomo@...r.kernel.org
More majordomo info at  http://vger.kernel.org/majordomo-info.html
Please read the FAQ at  http://www.tux.org/lkml/