For better cacheline performance, this patch creates a separate struct for each CPU with the percpu data grouped together. Signed-off-by: Steven Rostedt --- lib/tracing/tracer.c | 42 +++++++++++++++++++++++------------------- lib/tracing/tracer.h | 12 ++++++++---- 2 files changed, 31 insertions(+), 23 deletions(-) Index: linux-compile-i386.git/lib/tracing/tracer.c =================================================================== --- linux-compile-i386.git.orig/lib/tracing/tracer.c 2008-01-09 14:14:43.000000000 -0500 +++ linux-compile-i386.git/lib/tracing/tracer.c 2008-01-09 15:17:28.000000000 -0500 @@ -16,6 +16,7 @@ #include #include #include +#include #include #include #include @@ -25,6 +26,7 @@ #include "tracer_interface.h" static struct mctracer_trace mctracer_trace; +static DEFINE_PER_CPU(struct mctracer_trace_cpu, mctracer_trace_cpu); static inline notrace void mctracer_add_trace_entry(struct mctracer_trace *tr, @@ -35,21 +37,22 @@ mctracer_add_trace_entry(struct mctracer unsigned long idx, idx_next; struct mctracer_entry *entry; struct task_struct *tsk = current; + struct mctracer_trace_cpu *data = tr->data[cpu]; - idx = tr->trace_idx[cpu]; + idx = data->trace_idx; idx_next = idx + 1; if (unlikely(idx_next >= tr->entries)) { - atomic_inc(&tr->underrun[cpu]); + atomic_inc(&data->underrun); idx_next = 0; } - tr->trace_idx[cpu] = idx_next; + data->trace_idx = idx_next; - if (unlikely(idx_next != 0 && atomic_read(&tr->underrun[cpu]))) - atomic_inc(&tr->underrun[cpu]); + if (unlikely(idx_next != 0 && atomic_read(&data->underrun))) + atomic_inc(&data->underrun); - entry = tr->trace[cpu] + idx * MCTRACER_ENTRY_SIZE; + entry = data->trace + idx * MCTRACER_ENTRY_SIZE; entry->idx = atomic_inc_return(&tr->cnt); entry->ip = ip; entry->parent_ip = parent_ip; @@ -69,11 +72,11 @@ static notrace void trace_function(const tr = &mctracer_trace; - atomic_inc(&tr->disabled[cpu]); - if (likely(atomic_read(&tr->disabled[cpu]) == 1)) + atomic_inc(&tr->data[cpu]->disabled); + if (likely(atomic_read(&tr->data[cpu]->disabled) == 1)) mctracer_add_trace_entry(tr, cpu, ip, parent_ip); - atomic_dec(&tr->disabled[cpu]); + atomic_dec(&tr->data[cpu]->disabled); raw_local_irq_restore(flags); } @@ -83,8 +86,8 @@ static notrace void mctracer_reset(struc int cpu; for_each_online_cpu(cpu) { - tr->trace_idx[cpu] = 0; - atomic_set(&tr->underrun[cpu], 0); + tr->data[cpu]->trace_idx = 0; + atomic_set(&tr->data[cpu]->underrun, 0); } } @@ -105,16 +108,16 @@ static struct mctracer_entry *mctracer_e unsigned long idx, int cpu) { - struct mctracer_entry *array = tr->trace[cpu]; + struct mctracer_entry *array = tr->data[cpu]->trace; unsigned long underrun; if (idx >= tr->entries) return NULL; - underrun = atomic_read(&tr->underrun[cpu]); + underrun = atomic_read(&tr->data[cpu]->underrun); if (underrun) idx = ((underrun - 1) + idx) % tr->entries; - else if (idx >= tr->trace_idx[cpu]) + else if (idx >= tr->data[cpu]->trace_idx) return NULL; return &array[idx]; @@ -129,7 +132,7 @@ static void *find_next_entry(struct mctr int i; for_each_possible_cpu(i) { - if (!tr->trace[i]) + if (!tr->data[i]->trace) continue; ent = mctracer_entry_idx(tr, iter->next_idx[i], i); if (ent && (!next || next->idx > ent->idx)) { @@ -452,6 +455,7 @@ static notrace int mctracer_alloc_buffer int i; for_each_possible_cpu(i) { + mctracer_trace.data[i] = &per_cpu(mctracer_trace_cpu, i); array = (struct mctracer_entry *) __get_free_pages(GFP_KERNEL, order); if (array == NULL) { @@ -459,7 +463,7 @@ static notrace int mctracer_alloc_buffer " %ld bytes for trace buffer!\n", size); goto free_buffers; } - mctracer_trace.trace[i] = array; + mctracer_trace.data[i]->trace = array; } /* @@ -478,10 +482,10 @@ static notrace int mctracer_alloc_buffer free_buffers: for (i-- ; i >= 0; i--) { - if (mctracer_trace.trace[i]) { - free_pages((unsigned long)mctracer_trace.trace[i], + if (mctracer_trace.data[i] && mctracer_trace.data[i]->trace) { + free_pages((unsigned long)mctracer_trace.data[i]->trace, order); - mctracer_trace.trace[i] = NULL; + mctracer_trace.data[i]->trace = NULL; } } return -ENOMEM; Index: linux-compile-i386.git/lib/tracing/tracer.h =================================================================== --- linux-compile-i386.git.orig/lib/tracing/tracer.h 2008-01-09 14:14:02.000000000 -0500 +++ linux-compile-i386.git/lib/tracing/tracer.h 2008-01-09 15:17:28.000000000 -0500 @@ -12,15 +12,19 @@ struct mctracer_entry { pid_t pid; }; +struct mctracer_trace_cpu { + void *trace; + unsigned long trace_idx; + atomic_t disabled; + atomic_t underrun; +}; + struct mctracer_trace { - void *trace[NR_CPUS]; - unsigned long trace_idx[NR_CPUS]; unsigned long entries; long ctrl; unsigned long iter_flags; atomic_t cnt; - atomic_t disabled[NR_CPUS]; - atomic_t underrun[NR_CPUS]; + struct mctracer_trace_cpu *data[NR_CPUS]; }; #endif /* _LINUX_MCOUNT_TRACER_H */ -- -- To unsubscribe from this list: send the line "unsubscribe linux-kernel" in the body of a message to majordomo@vger.kernel.org More majordomo info at http://vger.kernel.org/majordomo-info.html Please read the FAQ at http://www.tux.org/lkml/