Break out the WAIT and LOCK bit operations into functions and provide some extra comments. Signed-off-by: Peter Zijlstra --- kernel/smp.c | 100 +++++++++++++++++++++++++++++++++++------------------------ 1 file changed, 60 insertions(+), 40 deletions(-) Index: linux-2.6/kernel/smp.c =================================================================== --- linux-2.6.orig/kernel/smp.c +++ linux-2.6/kernel/smp.c @@ -100,14 +100,50 @@ static int __cpuinit init_call_single_da } early_initcall(init_call_single_data); -static void csd_flag_wait(struct call_single_data *data) +/* + * csd_wait/csd_complete are used for synchronous ipi calls + */ +static void csd_wait_prepare(struct call_single_data *data) +{ + data->flags |= CSD_FLAG_WAIT; +} + +static void csd_complete(struct call_single_data *data) +{ + /* + * Serialize stores to data with the flag clear and wakeup. + */ + smp_wmb(); + data->flags &= ~CSD_FLAG_WAIT; +} + +static void csd_wait(struct call_single_data *data) +{ + while (data->flags & CSD_FLAG_WAIT) + cpu_relax(); +} + +/* + * csd_lock/csd_unlock used to serialize access to per-cpu csd resources + * + * For non-synchronous ipi calls the csd can still be in use by the previous + * function call. For multi-cpu calls its even more interesting as we'll have + * to ensure no other cpu is observing our csd. + */ +static void csd_lock(struct call_single_data *data) { - /* Wait for response */ - do { - if (!(data->flags & CSD_FLAG_WAIT)) - break; + while (data->flags & CSD_FLAG_LOCK) cpu_relax(); - } while (1); + data->flags = CSD_FLAG_LOCK; +} + +static void csd_unlock(struct call_single_data *data) +{ + /* + * Serialize stores to data with the flags clear. + */ + smp_wmb(); + data->flags &= ~CSD_FLAG_LOCK; } /* @@ -134,7 +170,7 @@ static void generic_exec_single(int cpu, arch_send_call_function_single_ipi(cpu); if (wait) - csd_flag_wait(data); + csd_wait(data); } /* @@ -172,18 +208,17 @@ void generic_smp_call_function_interrupt spin_lock(&call_function.lock); list_add(&data->free_list, &call_function.free_list); list_del_rcu(&data->csd.list); + /* + * When the global queue is empty, its guaranteed that no cpu + * is still observing any entry on the free_list, therefore + * we can go ahead and unlock them. + */ if (!--call_function.counter) list_splice_init(&call_function.free_list, &free_list); spin_unlock(&call_function.lock); - if (data->csd.flags & CSD_FLAG_WAIT) { - /* - * serialize stores to data with the flag clear - * and wakeup - */ - smp_wmb(); - data->csd.flags &= ~CSD_FLAG_WAIT; - } + if (data->csd.flags & CSD_FLAG_WAIT) + csd_complete(&data->csd); while (!list_empty(&free_list)) { struct call_function_data *free; @@ -192,12 +227,7 @@ void generic_smp_call_function_interrupt struct call_function_data, free_list); list_del(&data->free_list); - /* - * serialize stores to data with the flags - * clear - */ - smp_wmb(); - free->csd.flags &= ~CSD_FLAG_LOCK; + csd_unlock(&free->csd); } } @@ -242,13 +272,10 @@ void generic_smp_call_function_single_in data->func(data->info); - if (data_flags & CSD_FLAG_WAIT) { - smp_wmb(); - data->flags &= ~CSD_FLAG_WAIT; - } else if (data_flags & CSD_FLAG_LOCK) { - smp_wmb(); - data->flags &= ~CSD_FLAG_LOCK; - } + if (data_flags & CSD_FLAG_WAIT) + csd_complete(data); + else if (data_flags & CSD_FLAG_LOCK) + csd_unlock(data); } /* * See comment on outer loop @@ -308,12 +335,10 @@ int smp_call_function_single(int cpu, vo * data before a new caller will use it. */ data = &per_cpu(csd_data, me); - while (data->flags & CSD_FLAG_LOCK) - cpu_relax(); - data->flags = CSD_FLAG_LOCK; + csd_lock(data); } else { data = &d; - data->flags = CSD_FLAG_WAIT; + csd_wait_prepare(data); } data->func = func; @@ -398,16 +423,11 @@ void smp_call_function_many(const struct } data = &per_cpu(cfd_data, me); - /* - * We need to wait for all previous users to go away. - */ - while (data->csd.flags & CSD_FLAG_LOCK) - cpu_relax(); - data->csd.flags = CSD_FLAG_LOCK; + csd_lock(&data->csd); spin_lock_init(&data->lock); if (wait) - data->csd.flags |= CSD_FLAG_WAIT; + csd_wait_prepare(&data->csd); data->csd.func = func; data->csd.info = info; cpumask_and(data->cpumask, mask, cpu_online_mask); @@ -429,7 +449,7 @@ void smp_call_function_many(const struct /* optionally wait for the CPUs to complete */ if (wait) - csd_flag_wait(&data->csd); + csd_wait(&data->csd); } EXPORT_SYMBOL(smp_call_function_many); -- -- To unsubscribe from this list: send the line "unsubscribe linux-kernel" in the body of a message to majordomo@vger.kernel.org More majordomo info at http://vger.kernel.org/majordomo-info.html Please read the FAQ at http://www.tux.org/lkml/