linux-kernel - Re: [PATCH 1/2] x86,sched: Add support for frequency invariance

lists.openwall.net		lists / announce owl-users owl-dev john-users john-dev passwdqc-users yescrypt popa3d-users / oss-security kernel-hardening musl sabotage tlsify passwords / crypt-dev xvendor / Bugtraq Full-Disclosure linux-kernel linux-netdev linux-ext4 linux-hardening linux-cve-announce PHC
Open Source and information security mailing list archives

Hash Suite: Windows password security audit tool. GUI, reports in PDF.

[<prev] [next>] [<thread-prev] [thread-next>] [day] [month] [year] [list]

Date:   Tue, 24 Sep 2019 18:00:22 +0200
From:   Peter Zijlstra <peterz@...radead.org>
To:     Giovanni Gherdovich <ggherdovich@...e.cz>
Cc:     Quentin Perret <qperret@...rret.net>,
        srinivas.pandruvada@...ux.intel.com, tglx@...utronix.de,
        mingo@...hat.com, bp@...e.de, lenb@...nel.org, rjw@...ysocki.net,
        x86@...nel.org, linux-pm@...r.kernel.org,
        linux-kernel@...r.kernel.org, mgorman@...hsingularity.net,
        matt@...eblueprint.co.uk, viresh.kumar@...aro.org,
        juri.lelli@...hat.com, pjt@...gle.com, vincent.guittot@...aro.org,
        dietmar.eggemann@....com
Subject: Re: [PATCH 1/2] x86,sched: Add support for frequency invariance

On Tue, Sep 24, 2019 at 04:03:32PM +0200, Peter Zijlstra wrote:

> > I'll check what's the cost of static_cpu_has() and if it's non-negligible I'll
> > do what you suggest (x86-specific version of arch_scale_freq_invariant().
> 
> static_cpu_has() is an alternative and ends up being a static branch
> (similar to static_key) once the alternative patching runs.

That said; I think you want a static key anyway, because if we can't
tell the max_freq we don't want to use the invariant stuff.

Something a little like so on top perhaps.

Also, the below fixes that silly tick_disable stuff.

---
--- a/arch/x86/include/asm/topology.h
+++ b/arch/x86/include/asm/topology.h
@@ -196,20 +196,24 @@ static inline void sched_clear_itmt_supp
 #ifdef CONFIG_SMP
 #include <asm/cpufeature.h>
 
-#define arch_scale_freq_tick arch_scale_freq_tick
-#define arch_scale_freq_capacity arch_scale_freq_capacity
+DECLARE_STATIC_KEY_FALSE(arch_scale_freq_key);
+
+#define arch_scale_freq_invariant() static_branch_likely(&arch_scale_freq_key)
 
 DECLARE_PER_CPU(unsigned long, arch_cpu_freq);
 
 static inline long arch_scale_freq_capacity(int cpu)
 {
-	if (static_cpu_has(X86_FEATURE_APERFMPERF))
+	if (arch_scale_freq_invariant())
 		return per_cpu(arch_cpu_freq, cpu);
 
 	return 1024 /* SCHED_CAPACITY_SCALE */;
 }
+#define arch_scale_freq_capacity arch_scale_freq_capacity
 
 extern void arch_scale_freq_tick(void);
+#define arch_scale_freq_tick arch_scale_freq_tick
+
 extern void x86_arch_scale_freq_tick_enable(void);
 extern void x86_arch_scale_freq_tick_disable(void);
 #else
--- a/arch/x86/kernel/smpboot.c
+++ b/arch/x86/kernel/smpboot.c
@@ -1799,6 +1799,8 @@ void native_play_dead(void)
  * freq_curr / freq_max to eventually grow >1, in which case we clip it to 1.
  */
 
+DEFINE_STATIC_KEY_FALSE(arch_scale_freq_key);
+
 static DEFINE_PER_CPU(u64, arch_prev_aperf);
 static DEFINE_PER_CPU(u64, arch_prev_mperf);
 static u64 arch_max_freq = SCHED_CAPACITY_SCALE;
@@ -1860,6 +1862,8 @@ static void core_set_cpu_max_freq(void)
 	turbo_ratio = (turbo_ratio >> 24) & 0xFF;   /* 4C turbo ratio */
 
 	arch_max_freq = div_u64(turbo_ratio * SCHED_CAPACITY_SCALE, ratio);
+
+	static_key_enable(&arch_scale_freq_key);
 }
 
 static void intel_set_cpu_max_freq(void)
@@ -1876,10 +1880,19 @@ static void intel_set_cpu_max_freq(void)
 	core_set_cpu_max_freq();
 }
 
-static void set_cpu_max_freq(void)
+static void init_scale_freq(void *arg)
 {
 	u64 aperf, mperf;
 
+	rdmsrl(MSR_IA32_APERF, aperf);
+	rdmsrl(MSR_IA32_MPERF, mperf);
+
+	this_cpu_write(arch_prev_aperf, aperf);
+	this_cpu_write(arch_prev_mperf, mperf);
+}
+
+static void set_cpu_max_freq(void)
+{
 	if (!boot_cpu_has(X86_FEATURE_APERFMPERF))
 		return;
 
@@ -1891,11 +1904,7 @@ static void set_cpu_max_freq(void)
 		break;
 	}
 
-	rdmsrl(MSR_IA32_APERF, aperf);
-	rdmsrl(MSR_IA32_MPERF, mperf);
-
-	this_cpu_write(arch_prev_aperf, aperf);
-	this_cpu_write(arch_prev_mperf, mperf);
+	init_scale_freq(NULL);
 }
 
 DEFINE_PER_CPU(unsigned long, arch_cpu_freq);
@@ -1908,7 +1917,7 @@ void arch_scale_freq_tick(void)
 	u64 aperf, mperf;
 	u64 acnt, mcnt;
 
-	if (!static_cpu_has(X86_FEATURE_APERFMPERF) || tick_disable)
+	if (!arch_scale_freq_invariant() || tick_disable)
 		return;
 
 	rdmsrl(MSR_IA32_APERF, aperf);
@@ -1940,5 +1949,6 @@ void x86_arch_scale_freq_tick_enable(voi
 
 void x86_arch_scale_freq_tick_disable(void)
 {
+	on_each_cpu(init_scale_freq, NULL, 1);
 	tick_disable = true;
 }