| lists.openwall.net | lists / announce owl-users owl-dev john-users john-dev passwdqc-users yescrypt popa3d-users / oss-security kernel-hardening musl sabotage tlsify passwords / crypt-dev xvendor / Bugtraq Full-Disclosure linux-kernel linux-netdev linux-ext4 linux-hardening linux-cve-announce PHC | |
|
Open Source and information security mailing list archives
| ||
|
Message-Id: <1470105895-71379-3-git-send-email-davidcc@google.com>
Date: Mon, 1 Aug 2016 19:44:54 -0700
From: David Carrillo-Cisneros <davidcc@...gle.com>
To: linux-kernel@...r.kernel.org
Cc: "x86@...nel.org" <x86@...nel.org>, Ingo Molnar <mingo@...hat.com>,
Thomas Gleixner <tglx@...utronix.de>,
Andi Kleen <ak@...ux.intel.com>,
Kan Liang <kan.liang@...el.com>,
Peter Zijlstra <peterz@...radead.org>,
Paul Turner <pjt@...gle.com>,
Stephane Eranian <eranian@...gle.com>,
David Carrillo-Cisneros <davidcc@...gle.com>
Subject: [PATCH 2/3] perf/core: introduce pmu_event_flags and PMUEF_READ_CPU_PKG
Introduce the flag PMUEF_READ_CPU_PKG, useful for uncore events, that
allows a PMU to signal the generic perf code that an event is readable
on the current CPU if the event is:
- active in a CPU in the same package as the current CPU (local CPU)
- not active but is attached to a CPU (i.e. event->cpu != -1) in the
same package as the current CPU.
This is an optimization that avoids a unnecessary IPI for the common case
where uncore events are run and read in the same package but in
different CPUs.
As an example, the IPI removal speeds up perf_read in my Haswell system
as follows:
- For event UNC_C_LLC_LOOKUP: From 260 us to 31 us.
- For event RAPL's power/energy-cores/: From to 255 us to 27 us.
To store the flag PMUEF_READ_CPU_PKG, a new field, pmu_event_flags is
added to perf_event struct. The flags in this field are intended to be set
by the PMU in a per-event basis, allowing different events in the same PMU
to have different flags on. This patch does not demonstrate that events in
a PMU can have distinct flags, but the Intel CQM/CMT patch series does
makes use of this and uses the PMUEF_READ_CPU_PKG in a per-event basis
Signed-off-by: David Carrillo-Cisneros <davidcc@...gle.com>
Reviewed-by: Stephane Eranian <eranian@...gle.com>
---
include/linux/perf_event.h | 12 ++++++++++
kernel/events/core.c | 55 +++++++++++++++++++++++++++++++++++++---------
2 files changed, 57 insertions(+), 10 deletions(-)
diff --git a/include/linux/perf_event.h b/include/linux/perf_event.h
index 7921f4f..d026166 100644
--- a/include/linux/perf_event.h
+++ b/include/linux/perf_event.h
@@ -681,9 +681,21 @@ struct perf_event {
#endif
struct list_head sb_list;
+
+ /* Per-event flags to generic code set by PMU. */
+ int pmu_event_flags;
+
#endif /* CONFIG_PERF_EVENTS */
};
+/*
+ * Flags for pmu_event_flags.
+ *
+ * PMUEF_READ_CPU_PKG: A CPU event (or cgroup event) that can be read in
+ * any CPU in event->cpu's package, even if inactive.
+ */
+#define PMUEF_READ_CPU_PKG BIT(0)
+
/**
* struct perf_event_context - event context structure
*
diff --git a/kernel/events/core.c b/kernel/events/core.c
index 1c30f52..c2c6ad0 100644
--- a/kernel/events/core.c
+++ b/kernel/events/core.c
@@ -3336,6 +3336,39 @@ struct perf_read_data {
int ret;
};
+static inline bool can_read_inactive(struct perf_event *event)
+{
+ return event->cpu >= 0 &&
+ (event->pmu_event_flags & PMUEF_READ_CPU_PKG);
+}
+
+static int find_cpu_to_read(struct perf_event *event)
+{
+ bool active = event->state == PERF_EVENT_STATE_ACTIVE;
+ int event_cpu = active ? event->oncpu : event->cpu;
+ int local_cpu = smp_processor_id();
+ u16 local_pkg, event_pkg;
+
+ if (event_cpu < 0)
+ return -1;
+
+ if (event->pmu_event_flags & PMUEF_READ_CPU_PKG) {
+ /*
+ * Event with PMUEF_READ_CPU_PKG can be read in local CPU if:
+ * - CPU where event runs is on local CPU's pkg, or
+ * - event is bound to a CPU on local CPU's pkg
+ * (even if event is not in active state).
+ */
+ event_pkg = topology_physical_package_id(event_cpu);
+ local_pkg = topology_physical_package_id(local_cpu);
+
+ if (event_pkg == local_pkg)
+ return local_cpu;
+ }
+
+ return event_cpu;
+}
+
/*
* Cross CPU call to read the hardware event
*/
@@ -3346,6 +3379,7 @@ static void __perf_event_read(void *info)
struct perf_event_context *ctx = event->ctx;
struct perf_cpu_context *cpuctx = __get_cpu_context(ctx);
struct pmu *pmu = event->pmu;
+ bool active, read_inactive = can_read_inactive(event);
/*
* If this is a task context, we need to check whether it is
@@ -3354,7 +3388,7 @@ static void __perf_event_read(void *info)
* event->count would have been updated to a recent sample
* when the event was scheduled out.
*/
- if (ctx->task && cpuctx->task_ctx != ctx)
+ if (ctx->task && cpuctx->task_ctx != ctx && !read_inactive)
return;
raw_spin_lock(&ctx->lock);
@@ -3364,7 +3398,9 @@ static void __perf_event_read(void *info)
}
update_event_times(event);
- if (event->state != PERF_EVENT_STATE_ACTIVE)
+
+ active = event->state == PERF_EVENT_STATE_ACTIVE;
+ if (!active && !read_inactive)
goto unlock;
if (!data->group) {
@@ -3379,7 +3415,8 @@ static void __perf_event_read(void *info)
list_for_each_entry(sub, &event->sibling_list, group_entry) {
update_event_times(sub);
- if (sub->state == PERF_EVENT_STATE_ACTIVE) {
+ active = sub->state == PERF_EVENT_STATE_ACTIVE;
+ if (active || can_read_inactive(sub)) {
/*
* Use sibling's PMU rather than @event's since
* sibling could be on different (eg: software) PMU.
@@ -3457,19 +3494,17 @@ u64 perf_event_read_local(struct perf_event *event)
static int perf_event_read(struct perf_event *event, bool group)
{
- int ret = 0;
+ int ret = 0, cpu_to_read;
- /*
- * If event is enabled and currently active on a CPU, update the
- * value in the event structure:
- */
- if (event->state == PERF_EVENT_STATE_ACTIVE) {
+ cpu_to_read = find_cpu_to_read(event);
+
+ if (cpu_to_read >= 0) {
struct perf_read_data data = {
.event = event,
.group = group,
.ret = 0,
};
- ret = smp_call_function_single(event->oncpu,
+ ret = smp_call_function_single(cpu_to_read,
__perf_event_read, &data, 1);
ret = ret ? : data.ret;
} else if (event->state == PERF_EVENT_STATE_INACTIVE) {
--
2.8.0.rc3.226.g39d4020
Powered by blists - more mailing lists