| lists.openwall.net | lists / announce owl-users owl-dev john-users john-dev passwdqc-users yescrypt popa3d-users / oss-security kernel-hardening musl sabotage tlsify passwords / crypt-dev xvendor / Bugtraq Full-Disclosure linux-kernel linux-netdev linux-ext4 linux-hardening linux-cve-announce PHC | |
|
Open Source and information security mailing list archives
| ||
|
Message-Id: <1477787923-61185-28-git-send-email-davidcc@google.com>
Date: Sat, 29 Oct 2016 17:38:24 -0700
From: David Carrillo-Cisneros <davidcc@...gle.com>
To: linux-kernel@...r.kernel.org
Cc: "x86@...nel.org" <x86@...nel.org>, Ingo Molnar <mingo@...hat.com>,
Thomas Gleixner <tglx@...utronix.de>,
Andi Kleen <ak@...ux.intel.com>,
Kan Liang <kan.liang@...el.com>,
Peter Zijlstra <peterz@...radead.org>,
Vegard Nossum <vegard.nossum@...il.com>,
Marcelo Tosatti <mtosatti@...hat.com>,
Nilay Vaish <nilayvaish@...il.com>,
Borislav Petkov <bp@...e.de>,
Vikas Shivappa <vikas.shivappa@...ux.intel.com>,
Ravi V Shankar <ravi.v.shankar@...el.com>,
Fenghua Yu <fenghua.yu@...el.com>,
Paul Turner <pjt@...gle.com>,
Stephane Eranian <eranian@...gle.com>,
David Carrillo-Cisneros <davidcc@...gle.com>
Subject: [PATCH v3 27/46] perf/x86/intel: add pqr cache flags and intel_pqr_ctx_switch
In order to support intel_cmt for cgroups with no perf_event, this driver
calls intel_pqr_ctx_switch during a task context switch, after perf_event
has add/del all events (if any), using the newly introduced
finish_arch_pre_lock_switch() .
If pmu->add sets a rmids, the next rmid is "marked" using the flag
PQR_RMID_FLAG_EVENT. Removing an event sets the root rmid with the flag
PQR_RMID_FLAG_EVENT to indicate that it can be over-written by an actively
monitored cgroup.
If no event added to the pmu writes a rmid into PQR msr
(and PQR_RMID_FLAG_EVENT is not set), then __pqr_ctx_switch will
call __intel_cmt_no_event_sched_in to update PQR cache with the rmid of
the active perf cgroup's monr.
In some cases, a change to PQR msr software cache must be write-through
immediately, using the flag PQR_FLAG_WT. These are:
1) updating PQR_ASSOC when a cpu goes offline.
2) first execution slot of a newly exec'd task, since new exec'd tasks
runs before a context switch.
Signed-off-by: David Carrillo-Cisneros <davidcc@...gle.com>
---
arch/x86/events/intel/cmt.c | 39 ++++++++++++++++++++++++++++-----
arch/x86/include/asm/intel_rdt_common.h | 38 ++++++++++++++++++++++++++++----
arch/x86/include/asm/processor.h | 4 ++++
arch/x86/kernel/cpu/intel_rdt_common.c | 29 ++++++++++++++++++++++++
4 files changed, 100 insertions(+), 10 deletions(-)
diff --git a/arch/x86/events/intel/cmt.c b/arch/x86/events/intel/cmt.c
index 830ce29..bd903ae 100644
--- a/arch/x86/events/intel/cmt.c
+++ b/arch/x86/events/intel/cmt.c
@@ -1323,12 +1323,13 @@ static void intel_cmt_event_read(struct perf_event *event)
}
static inline void __intel_cmt_event_start(struct perf_event *event,
- union pmonr_rmids rmids)
+ union pmonr_rmids rmids, bool wt)
{
if (!(event->hw.state & PERF_HES_STOPPED))
return;
event->hw.state &= ~PERF_HES_STOPPED;
- pqr_cache_update_rmid(rmids.sched_rmid);
+ pqr_cache_update_rmid(rmids.sched_rmid,
+ PQR_RMID_FLAG_EVENT | (wt ? PQR_FLAG_WT : 0));
}
static void intel_cmt_event_start(struct perf_event *event, int mode)
@@ -1336,7 +1337,7 @@ static void intel_cmt_event_start(struct perf_event *event, int mode)
union pmonr_rmids rmids;
rmids = monr_get_sched_in_rmids(monr_from_event(event));
- __intel_cmt_event_start(event, rmids);
+ __intel_cmt_event_start(event, rmids, false);
}
static void intel_cmt_event_stop(struct perf_event *event, int mode)
@@ -1352,18 +1353,25 @@ static void intel_cmt_event_stop(struct perf_event *event, int mode)
* reads occur even if event is Inactive. Therefore there is no need to
* read when event is stopped.
*/
- pqr_cache_update_rmid(rmids.sched_rmid);
+ pqr_cache_update_rmid(rmids.sched_rmid, 0);
}
static int intel_cmt_event_add(struct perf_event *event, int mode)
{
union pmonr_rmids rmids;
+ bool wt;
event->hw.state = PERF_HES_STOPPED;
rmids = monr_get_sched_in_rmids(monr_from_event(event));
+ /*
+ * A newly exec'd task may run from the file loader without a context
+ * switch, if so, the PQR sw cache will not update the rmid in hw.
+ * Avoid that by issuing a write-through mode to the PQR sw cache.
+ */
+ wt = event->total_time_running == 0;
if (mode & PERF_EF_START)
- __intel_cmt_event_start(event, rmids);
+ __intel_cmt_event_start(event, rmids, wt);
return 0;
}
@@ -1697,7 +1705,7 @@ static int intel_cmt_hp_online_exit(unsigned int cpu)
struct pkg_data *pkgd;
u16 pkgid = topology_logical_package_id(cpu);
- pqr_cache_update_rmid(0);
+ pqr_cache_update_rmid(0, PQR_FLAG_WT);
memset(state, 0, sizeof(*state));
rcu_read_lock();
@@ -1932,6 +1940,8 @@ static int __init intel_cmt_init(void)
rcu_read_unlock();
pr_cont("and l3 scale of %d KBs.\n", cmt_l3_scale);
+ static_branch_inc(&pqr_common_enable_key);
+
return err;
err_stop:
@@ -1943,4 +1953,21 @@ static int __init intel_cmt_init(void)
return err;
}
+/* Schedule task without a CMT perf_event. */
+inline void __intel_cmt_no_event_sched_in(void)
+{
+#ifdef CONFIG_CGROUP_PERF
+ struct monr *monr;
+ union pmonr_rmids rmids;
+
+ /* Assume CMT enabled is likely given that PQR is enabled. */
+ if (!static_branch_likely(&cmt_initialized_key))
+ return;
+ /* Safe to call from_task since we are in scheduler lock. */
+ monr = monr_from_perf_cgroup(perf_cgroup_from_task(current, NULL));
+ rmids = monr_get_sched_in_rmids(monr);
+ pqr_cache_update_rmid(rmids.sched_rmid, 0);
+#endif
+}
+
device_initcall(intel_cmt_init);
diff --git a/arch/x86/include/asm/intel_rdt_common.h b/arch/x86/include/asm/intel_rdt_common.h
index 1d5e691..d8d0dc3 100644
--- a/arch/x86/include/asm/intel_rdt_common.h
+++ b/arch/x86/include/asm/intel_rdt_common.h
@@ -3,6 +3,7 @@
#if defined(CONFIG_INTEL_RDT_A) || defined(CONFIG_INTEL_RDT_M)
+#include <linux/jump_label.h>
#include <linux/types.h>
#include <asm/percpu.h>
#include <asm/msr.h>
@@ -10,35 +11,49 @@
#define MSR_IA32_PQR_ASSOC 0x0c8f
+extern struct static_key_false pqr_common_enable_key;
+
/**
* struct intel_pqr_state - State cache for the PQR MSR
* @rmid: The cached Resource Monitoring ID
* @next_rmid: Next rmid to write to hw
* @closid: The cached Class Of Service ID
* @next_closid: Next closid to write to hw
+ * @next_rmid_flags: Next rmid's flags
*
* The upper 32 bits of MSR_IA32_PQR_ASSOC contain closid and the
* lower 10 bits rmid. The update to MSR_IA32_PQR_ASSOC always
* contains both parts, so we need to cache them.
*
* The cache also helps to avoid pointless updates if the value does
- * not change.
+ * not change. It also keeps track of the type of RMID set (event vs
+ * no event) used to determine when a cgroup RMID is required.
*/
struct intel_pqr_state {
u32 rmid;
u32 next_rmid;
u32 closid;
u32 next_closid;
+ int next_rmid_flags;
};
+#define PQR_FLAG_WT BIT(0) /* write-through. */
+#define PQR_RMID_FLAG_EVENT BIT(1) /* associated to a perf_event. */
+
DECLARE_PER_CPU(struct intel_pqr_state, pqr_state);
-static inline void pqr_cache_update_rmid(u32 rmid)
+static inline void pqr_cache_update_rmid(u32 rmid, int next_rmid_flags)
{
struct intel_pqr_state *state = this_cpu_ptr(&pqr_state);
state->next_rmid = rmid;
- wrmsr(MSR_IA32_PQR_ASSOC, state->rmid, state->closid);
+ state->next_rmid_flags |= next_rmid_flags;
+
+ if (next_rmid_flags & PQR_FLAG_WT) {
+ state->rmid = rmid;
+ wrmsr(MSR_IA32_PQR_ASSOC, state->rmid, state->closid);
+ state->next_rmid_flags &= ~PQR_FLAG_WT;
+ }
}
static inline void pqr_cache_update_closid(u32 closid)
@@ -46,7 +61,22 @@ static inline void pqr_cache_update_closid(u32 closid)
struct intel_pqr_state *state = this_cpu_ptr(&pqr_state);
state->next_closid = closid;
- wrmsr(MSR_IA32_PQR_ASSOC, state->rmid, state->closid);
+}
+
+void __pqr_ctx_switch(void);
+
+inline void __intel_cmt_no_event_sched_in(void);
+
+static inline void intel_pqr_ctx_switch(void)
+{
+ if (static_branch_unlikely(&pqr_common_enable_key))
+ __pqr_ctx_switch();
+}
+
+#else
+
+static inline void intel_pqr_ctx_switch(void)
+{
}
#endif
diff --git a/arch/x86/include/asm/processor.h b/arch/x86/include/asm/processor.h
index 984a7bf..967ea93 100644
--- a/arch/x86/include/asm/processor.h
+++ b/arch/x86/include/asm/processor.h
@@ -22,6 +22,7 @@ struct vm86;
#include <asm/nops.h>
#include <asm/special_insns.h>
#include <asm/fpu/types.h>
+#include <asm/intel_rdt_common.h>
#include <linux/personality.h>
#include <linux/cache.h>
@@ -856,4 +857,7 @@ bool xen_set_default_idle(void);
void stop_this_cpu(void *dummy);
void df_debug(struct pt_regs *regs, long error_code);
+
+#define finish_arch_pre_lock_switch intel_pqr_ctx_switch
+
#endif /* _ASM_X86_PROCESSOR_H */
diff --git a/arch/x86/kernel/cpu/intel_rdt_common.c b/arch/x86/kernel/cpu/intel_rdt_common.c
index 7fd5b20..6e45287 100644
--- a/arch/x86/kernel/cpu/intel_rdt_common.c
+++ b/arch/x86/kernel/cpu/intel_rdt_common.c
@@ -6,3 +6,32 @@
* must ensure interruptions are handled properly.
*/
DEFINE_PER_CPU(struct intel_pqr_state, pqr_state);
+
+DEFINE_STATIC_KEY_FALSE(pqr_common_enable_key);
+
+/*
+ * Update hw's RMID using cgroup's if perf_event did not.
+ * Sync pqr cache with MSR.
+ */
+inline void __pqr_ctx_switch(void)
+{
+ struct intel_pqr_state *state = this_cpu_ptr(&pqr_state);
+
+ /*
+ * Obtain a rmid from current task's cgroup if no perf event
+ * set a rmid.
+ */
+ if (likely(!(state->next_rmid_flags & PQR_RMID_FLAG_EVENT)))
+ __intel_cmt_no_event_sched_in();
+
+ state->next_rmid_flags = 0;
+
+ /* __intel_cmt_no_event_sched_in might have changed next_rmid. */
+ if (likely(state->rmid == state->next_rmid &&
+ state->closid == state->next_closid))
+ return;
+
+ state->rmid = state->next_rmid;
+ state->closid = state->next_closid;
+ wrmsr(MSR_IA32_PQR_ASSOC, state->rmid, state->closid);
+}
--
2.8.0.rc3.226.g39d4020
Powered by blists - more mailing lists