Message-Id: <20170905133026.13689-11-alexander.shishkin@linux.intel.com>
Date:   Tue,  5 Sep 2017 16:30:19 +0300
From:   Alexander Shishkin <alexander.shishkin@...ux.intel.com>
To:     Peter Zijlstra <a.p.zijlstra@...llo.nl>
Cc:     Ingo Molnar <mingo@...hat.com>, linux-kernel@...r.kernel.org,
        acme@...hat.com, kirill.shutemov@...ux.intel.com,
        Borislav Petkov <bp@...en8.de>, rric@...nel.org,
        Alexander Shishkin <alexander.shishkin@...ux.intel.com>
Subject: [RFC PATCH 10/17] perf: Implement pinning and scheduling for SHMEM events

A SHMEM buffer is only pinned in while its task is scheduled in and the
pinning is done in a task work, which also implies that the corresponding
event can only be started from that task work.

Pinning is done on a per-cpu basis: if an event has previously been
pinned on the local cpu, its unpinned and its pin count is dropped and a
new event is pinned on this cpu. When the pin count goes to zero, we
unpin the pages, when it goes to one, we pin them.

Signed-off-by: Alexander Shishkin <alexander.shishkin@...ux.intel.com>
---
 include/linux/perf_event.h  |  10 +++
 kernel/events/core.c        | 134 ++++++++++++++++++++++++++++-
 kernel/events/internal.h    |   5 ++
 kernel/events/ring_buffer.c | 202 +++++++++++++++++++++++++++++++++++++++++++-
 4 files changed, 347 insertions(+), 4 deletions(-)

diff --git a/include/linux/perf_event.h b/include/linux/perf_event.h
index 341e9960bc..4b966dd0d8 100644
--- a/include/linux/perf_event.h
+++ b/include/linux/perf_event.h
@@ -703,6 +703,13 @@ struct perf_event {
 	unsigned long			rcu_batches;
 	int				rcu_pending;
 
+	/*
+	 * Number of times (CPUs) this event's been pinned (on):
+	 *  xpinned -> 0: unpin the pages,
+	 *  xpinned -> 1: pin the pages. See get_pages_work().
+	 */
+	atomic_t			xpinned;
+
 	/* poll related */
 	wait_queue_head_t		waitq;
 	struct fasync_struct		*fasync;
@@ -735,6 +742,9 @@ struct perf_event {
 	struct bpf_prog			*prog;
 #endif
 
+	/* Task work to pin event's rb pages if needed */
+	struct callback_head		get_pages_work;
+
 #ifdef CONFIG_EVENT_TRACING
 	struct trace_event_call		*tp_event;
 	struct event_filter		*filter;
diff --git a/kernel/events/core.c b/kernel/events/core.c
index feff812e30..c80ffcdb5c 100644
--- a/kernel/events/core.c
+++ b/kernel/events/core.c
@@ -50,6 +50,7 @@
 #include <linux/sched/mm.h>
 #include <linux/proc_ns.h>
 #include <linux/mount.h>
+#include <linux/task_work.h>
 #include <linux/tracefs.h>
 
 #include "internal.h"
@@ -383,6 +384,7 @@ static atomic_t perf_sched_count;
 static DEFINE_PER_CPU(atomic_t, perf_cgroup_events);
 static DEFINE_PER_CPU(int, perf_sched_cb_usages);
 static DEFINE_PER_CPU(struct pmu_event_list, pmu_sb_events);
+static DEFINE_PER_CPU(struct perf_event *, shmem_events);
 
 static atomic_t nr_mmap_events __read_mostly;
 static atomic_t nr_comm_events __read_mostly;
@@ -2058,6 +2060,94 @@ static void perf_set_shadow_time(struct perf_event *event,
 		event->shadow_ctx_time = tstamp - ctx->timestamp;
 }
 
+static void __unpin_event_pages(struct perf_event *event,
+				struct perf_cpu_context *cpuctx,
+				struct perf_event_context *ctx,
+				void *info)
+{
+	if (!atomic_dec_and_test(&event->xpinned))
+		return;
+
+	/*
+	 * If this event happens to be running, we need to stop it before we
+	 * can pull the pages. Note that this will be happening if we allow
+	 * concurrent shmem events, which seems like a bad idea.
+	 */
+	if (READ_ONCE(event->state) == PERF_EVENT_STATE_ACTIVE)
+		event->pmu->stop(event, PERF_EF_UPDATE);
+
+	rb_put_kernel_pages(event->rb, false);
+}
+
+enum pin_event_t {
+	PIN_IN = 0,
+	PIN_NOP,
+};
+
+static enum pin_event_t pin_event_pages(struct perf_event *event)
+{
+	struct perf_event **pinned_event = this_cpu_ptr(&shmem_events);
+	struct perf_event *old_event = *pinned_event;
+
+	if (old_event == event)
+		return PIN_NOP;
+
+	if (old_event && old_event->state > PERF_EVENT_STATE_DEAD)
+		event_function_call(old_event, __unpin_event_pages, NULL);
+
+	*pinned_event = event;
+	if (atomic_inc_return(&event->xpinned) != 1)
+		return PIN_NOP;
+
+	return PIN_IN;
+}
+
+static int perf_event_stop(struct perf_event *event, int restart);
+
+static void get_pages_work(struct callback_head *work)
+{
+	struct perf_event *event = container_of(work, struct perf_event, get_pages_work);
+	int ret;
+	struct ring_buffer *rb = event->rb;
+	int (*get_fn)(struct perf_event *event) = rb_get_kernel_pages;
+
+	work->func = NULL;
+
+	if (!rb || current->flags & PF_EXITING)
+		return;
+
+	if (!rb->shmem_file_addr) {
+		get_fn = rb_inject;
+		if (atomic_cmpxchg(&event->xpinned, 1, 0))
+			rb_put_kernel_pages(rb, false);
+	}
+
+	if (pin_event_pages(event) == PIN_IN) {
+		ret = get_fn(event);
+	} else {
+		ret = 0;
+	}
+
+	if (!ret)
+		perf_event_stop(event, 1);
+}
+
+static int perf_event_queue_work(struct perf_event *event,
+				 struct task_struct *task)
+{
+	int ret;
+
+	if (event->get_pages_work.func)
+		return 0;
+
+	init_task_work(&event->get_pages_work, get_pages_work);
+	ret = task_work_add(task, &event->get_pages_work, true);
+	if (ret)
+		return ret;
+
+	return 0;
+}
+
 #define MAX_INTERRUPTS (~0ULL)
 
 static void perf_log_throttle(struct perf_event *event, int enable);
@@ -2069,7 +2159,7 @@ event_sched_in(struct perf_event *event,
 		 struct perf_event_context *ctx)
 {
 	u64 tstamp = perf_event_time(event);
-	int ret = 0;
+	int ret = 0, shmem =  event->attach_state & PERF_ATTACH_SHMEM;
 
 	lockdep_assert_held(&ctx->lock);
 
@@ -2105,13 +2195,21 @@ event_sched_in(struct perf_event *event,
 
 	perf_log_itrace_start(event);
 
-	if (event->pmu->add(event, PERF_EF_START)) {
+	/*
+	 * For shmem events pmu::start will fail because of
+	 * rb::aux_mmap_count==0, so skip the PERF_EF_START, but
+	 * queue the task work that will actually start it.
+	 */
+	if (event->pmu->add(event, shmem ? 0 : PERF_EF_START)) {
 		event->state = PERF_EVENT_STATE_INACTIVE;
 		event->oncpu = -1;
 		ret = -EAGAIN;
 		goto out;
 	}
 
+	if (shmem)
+		perf_event_queue_work(event, ctx->task);
+
 	event->tstamp_running += tstamp - event->tstamp_stopped;
 
 	if (!is_software_event(event))
@@ -4182,6 +4280,30 @@ static void _free_event(struct perf_event *event)
 
 	unaccount_event(event);
 
+	if (event->attach_state & PERF_ATTACH_SHMEM) {
+		struct perf_event_context *ctx = event->ctx;
+		int cpu;
+
+		atomic_set(&event->xpinned, 0);
+		for_each_possible_cpu(cpu) {
+			struct perf_event **pinned_event =
+				per_cpu_ptr(&shmem_events, cpu);
+
+			cmpxchg(pinned_event, event, NULL);
+		}
+
+		event->attach_state &= ~PERF_ATTACH_SHMEM;
+
+		/*
+		 * XXX: !ctx means event is still being created;
+		 * we can get here via tracefs file though
+		 */
+		if (ctx && ctx->task && ctx->task != TASK_TOMBSTONE)
+			task_work_cancel(ctx->task, get_pages_work);
+
+		rb_put_kernel_pages(event->rb, false);
+	}
+
 	if (event->dent) {
 		tracefs_remove(event->dent);
 
@@ -4948,6 +5070,10 @@ void perf_event_update_userpage(struct perf_event *event)
 	if (!rb)
 		goto unlock;
 
+	/* Don't bother with the file backed rb when it's inactive */
+	if (rb->shmem_file && rb->paused)
+		goto unlock;
+
 	/*
 	 * compute total_time_enabled, total_time_running
 	 * based on snapshot values taken when the event
@@ -10684,6 +10810,8 @@ void perf_event_exit_task(struct task_struct *child)
 	}
 	mutex_unlock(&child->perf_event_mutex);
 
+	task_work_cancel(child, get_pages_work);
+
 	for_each_task_context_nr(ctxn)
 		perf_event_exit_task_context(child, ctxn);
 
@@ -10881,6 +11009,8 @@ inherit_event(struct perf_event *parent_event,
 	 * For per-task detached events with ring buffers, set_output doesn't
 	 * make sense, but we can allocate a new buffer here. CPU-wide events
 	 * don't have inheritance.
+	 * If we have to allocate a ring buffer, it must be shmem backed,
+	 * otherwise inheritance is disallowed in rb_alloc_detached().
 	 */
 	if (detached) {
 		int err;
diff --git a/kernel/events/internal.h b/kernel/events/internal.h
index 8de9e9cb6a..80d36a7277 100644
--- a/kernel/events/internal.h
+++ b/kernel/events/internal.h
@@ -55,11 +55,16 @@ struct ring_buffer {
 
 	/* tmpfs file for kernel-owned ring buffers */
 	struct file			*shmem_file;
+	unsigned long			shmem_file_addr;
+	int				shmem_pages_in;
 
 	struct perf_event_mmap_page	*user_page;
 	void				*data_pages[0];
 };
 
+extern int rb_inject(struct perf_event *event);
+extern int rb_get_kernel_pages(struct perf_event *event);
+extern void rb_put_kernel_pages(struct ring_buffer *rb, bool final);
 extern void rb_free(struct ring_buffer *rb);
 extern void ring_buffer_unaccount(struct ring_buffer *rb, bool aux);
 
diff --git a/kernel/events/ring_buffer.c b/kernel/events/ring_buffer.c
index 25159fe038..771dfdb71f 100644
--- a/kernel/events/ring_buffer.c
+++ b/kernel/events/ring_buffer.c
@@ -15,6 +15,8 @@
 #include <linux/circ_buf.h>
 #include <linux/poll.h>
 #include <linux/shmem_fs.h>
+#include <linux/mman.h>
+#include <linux/sched/mm.h>
 
 #include "internal.h"
 
@@ -384,8 +386,11 @@ void *perf_aux_output_begin(struct perf_output_handle *handle,
 	unsigned long aux_head, aux_tail;
 	struct ring_buffer *rb;
 
-	if (output_event->parent)
+	if (output_event->parent) {
+		WARN_ON_ONCE(is_detached_event(event));
+		WARN_ON_ONCE(event->attach_state & PERF_ATTACH_SHMEM);
 		output_event = output_event->parent;
+	}
 
 	/*
 	 * Since this will typically be open across pmu::add/pmu::del, we
@@ -851,6 +856,64 @@ void rb_free_aux(struct ring_buffer *rb)
 	}
 }
 
+static unsigned long perf_rb_size(struct ring_buffer *rb)
+{
+	return perf_data_size(rb) + perf_aux_size(rb) + PAGE_SIZE;
+}
+
+int rb_inject(struct perf_event *event)
+{
+	struct ring_buffer *rb = event->rb;
+	struct mm_struct *mm;
+	unsigned long addr;
+	int err = -ENOMEM;
+
+	mm = get_task_mm(current);
+	if (!mm)
+		return -ESRCH;
+
+	err = rb_get_kernel_pages(event);
+	if (err)
+		goto err_mmput;
+
+	addr = vm_mmap(rb->shmem_file, 0, perf_rb_size(rb), PROT_READ,
+		       MAP_SHARED | MAP_POPULATE, 0);
+
+	mmput(mm);
+	rb->mmap_mapping = mm;
+	rb->shmem_file_addr = addr;
+
+	return 0;
+
+err_mmput:
+	mmput(mm);
+
+	return err;
+}
+
+static void rb_shmem_unmap(struct perf_event *event)
+{
+	struct ring_buffer *rb = event->rb;
+	struct mm_struct *mm = rb->mmap_mapping;
+
+	rb_toggle_paused(rb, true);
+
+	if (!rb->shmem_file_addr)
+		return;
+
+	/*
+	 * EXIT state means the task is past exit_mm(),
+	 * no need to unmap anything
+	 */
+	if (event->state == PERF_EVENT_STATE_EXIT)
+		return;
+
+	down_write(&mm->mmap_sem);
+	(void)do_munmap(mm, rb->shmem_file_addr, perf_rb_size(rb), NULL);
+	up_write(&mm->mmap_sem);
+	rb->shmem_file_addr = 0;
+}
+
 static int rb_shmem_setup(struct perf_event *event,
 			  struct task_struct *task,
 			  struct ring_buffer *rb)
@@ -892,6 +955,138 @@ static int rb_shmem_setup(struct perf_event *event,
 	return 0;
 }
 
+/*
+ * Pin ring_buffer's pages to memory while the task is scheduled in;
+ * populate its page arrays (data_pages, aux_pages, user_page).
+ */
+int rb_get_kernel_pages(struct perf_event *event)
+{
+	struct ring_buffer *rb = event->rb;
+	struct address_space *mapping;
+	int nr_pages, i = 0, err = -EINVAL, changed = 0, mc = 0;
+	struct page *page;
+
+	/*
+	 * The mmap_count rules for SHMEM buffers:
+	 *  - they are always taken together
+	 *  - except for perf_mmap(), which doesn't work for shmem buffers:
+	 *    mmaping will force-pin more user's pages than is allowed
+	 *  - if either of them was taken before us, the pages are there
+	 */
+	if (atomic_inc_return(&rb->mmap_count) == 1)
+		mc++;
+
+	if (atomic_inc_return(&rb->aux_mmap_count) == 1)
+		mc++;
+
+	if (mc < 2)
+		goto done;
+
+	if (WARN_ON_ONCE(!rb->shmem_file))
+		goto err_put;
+
+	nr_pages = perf_rb_size(rb) >> PAGE_SHIFT;
+
+	mapping = rb->shmem_file->f_mapping;
+
+restart:
+	for (i = 0; i < nr_pages; i++) {
+		WRITE_ONCE(rb->shmem_pages_in, i);
+		err = shmem_getpage(mapping->host, i, &page, SGP_NOHUGE);
+		if (err)
+			goto err_put;
+
+		unlock_page(page);
+
+		if (READ_ONCE(rb->shmem_pages_in) != i) {
+			put_page(page);
+			goto restart;
+		}
+
+		mark_page_accessed(page);
+		set_page_dirty(page);
+		page->mapping = mapping;
+
+		if (page == perf_mmap_to_page(rb, i))
+			continue;
+
+		changed++;
+		if (!i) {
+			bool init = !rb->user_page;
+
+			rb->user_page = page_address(page);
+			if (init)
+				perf_event_init_userpage(event, rb);
+		} else if (i <= rb->nr_pages) {
+			rb->data_pages[i - 1] = page_address(page);
+		} else {
+			rb->aux_pages[i - rb->nr_pages - 1] = page_address(page);
+		}
+	}
+
+	/* rebuild SG tables: pages may have changed */
+	if (changed) {
+		if (rb->aux_priv)
+			rb->free_aux(rb->aux_priv);
+
+		rb->aux_priv = event->pmu->setup_aux(smp_processor_id(),
+						     rb->aux_pages,
+						     rb->aux_nr_pages, true);
+	}
+
+	if (!rb->aux_priv) {
+		err = -ENOMEM;
+		goto err_put;
+	}
+
+done:
+	rb_toggle_paused(rb, false);
+	if (changed)
+		perf_event_update_userpage(event);
+
+	return 0;
+
+err_put:
+	for (i--; i >= 0; i--) {
+		page = perf_mmap_to_page(rb, i);
+		put_page(page);
+	}
+
+	atomic_dec(&rb->aux_mmap_count);
+	atomic_dec(&rb->mmap_count);
+
+	return err;
+}
+
+void rb_put_kernel_pages(struct ring_buffer *rb, bool final)
+{
+	struct page *page;
+	int i;
+
+	if (!rb || !rb->shmem_file)
+		return;
+
+	rb_toggle_paused(rb, true);
+
+	/*
+	 * If both mmap_counts go to zero, put the pages, otherwise
+	 * do nothing.
+	 */
+	if (!atomic_dec_and_test(&rb->aux_mmap_count) ||
+	    !atomic_dec_and_test(&rb->mmap_count))
+		return;
+
+	for (i = 0; i < READ_ONCE(rb->shmem_pages_in); i++) {
+		page = perf_mmap_to_page(rb, i);
+		set_page_dirty(page);
+		if (final)
+			page->mapping = NULL;
+		put_page(page);
+	}
+
+	WRITE_ONCE(rb->shmem_pages_in, 0);
+}
+
 /*
  * Allocate a ring_buffer for a detached event and attach it to this event.
  * There's one ring_buffer per detached event and vice versa, so
@@ -962,8 +1157,11 @@ void rb_free_detached(struct ring_buffer *rb, struct perf_event *event)
 	/* Must be the last one */
 	WARN_ON_ONCE(atomic_read(&rb->refcount) != 1);
 
-	if (rb->shmem_file)
+	if (rb->shmem_file) {
+		rb_shmem_unmap(event);
 		shmem_truncate_range(rb->shmem_file->f_inode, 0, (loff_t)-1);
+		rb_put_kernel_pages(rb, true);
+	}
 
 	atomic_set(&rb->aux_mmap_count, 0);
 	rcu_assign_pointer(event->rb, NULL);
-- 
2.14.1

Powered by blists - more mailing lists