From: Christoph Lameter Make percpu_counter_add hot path preemptless. Signed-off-by: Christoph Lameter Signed-off-by: Shaohua Li --- lib/percpu_counter.c | 37 +++++++++++++++++++++++++------------ 1 file changed, 25 insertions(+), 12 deletions(-) Index: linux/lib/percpu_counter.c =================================================================== --- linux.orig/lib/percpu_counter.c 2011-05-10 16:23:01.000000000 +0800 +++ linux/lib/percpu_counter.c 2011-05-10 16:23:01.000000000 +0800 @@ -67,21 +67,34 @@ void percpu_counter_set(struct percpu_co } EXPORT_SYMBOL(percpu_counter_set); +static bool __percpu_counter_add_cmpxchg(struct percpu_counter *fbc, + s64 count, s64 new) +{ +#ifdef CONFIG_PREEMPT + return this_cpu_cmpxchg(*fbc->counters, count, new) == count; +#else + this_cpu_write(*fbc->counters, new); + return true; +#endif +} + void __percpu_counter_add(struct percpu_counter *fbc, s64 amount, s32 batch) { - s64 count; + s64 count, new; + + do { + count = this_cpu_read(*fbc->counters); + new = count + amount; - preempt_disable(); - count = __this_cpu_read(*fbc->counters) + amount; - if (count >= batch || count <= -batch) { - lg_local_lock(fbc->lglock); - atomic64_add(count, &fbc->count); - __this_cpu_write(*fbc->counters, 0); - lg_local_unlock(fbc->lglock); - } else { - __this_cpu_write(*fbc->counters, count); - } - preempt_enable(); + if (new >= batch || new <= -batch) { + lg_local_lock(fbc->lglock); + count = __this_cpu_read(*fbc->counters) + amount; + atomic64_add(count, &fbc->count); + __this_cpu_write(*fbc->counters, 0); + lg_local_unlock(fbc->lglock); + return; + } + } while (!__percpu_counter_add_cmpxchg(fbc, count, new)); } EXPORT_SYMBOL(__percpu_counter_add); -- To unsubscribe from this list: send the line "unsubscribe linux-kernel" in the body of a message to majordomo@vger.kernel.org More majordomo info at http://vger.kernel.org/majordomo-info.html Please read the FAQ at http://www.tux.org/lkml/