-allow stop_mahcine_run() to call a function on all cpus. Calling stop_machine_run() with a 'ALL_CPUS' invokes this new behavior. stop_machine_run() proceeds as normal until the calling cpu has invoked 'fn'. Then, we tell all the other cpus to call 'fn'. Signed-off-by: Jason Baron Signed-off-by: Mathieu Desnoyers CC: Rusty Russell CC: Adrian Bunk CC: Andi Kleen CC: Christoph Hellwig CC: mingo@elte.hu CC: akpm@osdl.org --- include/linux/stop_machine.h | 8 +++++++- kernel/stop_machine.c | 32 +++++++++++++++++++++++++------- 2 files changed, 32 insertions(+), 8 deletions(-) Index: linux-2.6-sched-devel/include/linux/stop_machine.h =================================================================== --- linux-2.6-sched-devel.orig/include/linux/stop_machine.h 2008-04-19 17:41:24.000000000 -0400 +++ linux-2.6-sched-devel/include/linux/stop_machine.h 2008-04-22 20:19:41.000000000 -0400 @@ -8,11 +8,17 @@ #include #if defined(CONFIG_STOP_MACHINE) && defined(CONFIG_SMP) + +#define ALL_CPUS ~0U + /** * stop_machine_run: freeze the machine on all CPUs and run this function * @fn: the function to run * @data: the data ptr for the @fn() - * @cpu: the cpu to run @fn() on (or any, if @cpu == NR_CPUS. + * @cpu: if @cpu == n, run @fn() on cpu n + * if @cpu == NR_CPUS, run @fn() on any cpu + * if @cpu == ALL_CPUS, run @fn() first on the calling cpu, and then + * concurrently on all the other cpus * * Description: This causes a thread to be scheduled on every other cpu, * each of which disables interrupts, and finally interrupts are disabled Index: linux-2.6-sched-devel/kernel/stop_machine.c =================================================================== --- linux-2.6-sched-devel.orig/kernel/stop_machine.c 2008-04-22 20:04:13.000000000 -0400 +++ linux-2.6-sched-devel/kernel/stop_machine.c 2008-04-22 20:21:38.000000000 -0400 @@ -22,9 +22,17 @@ enum stopmachine_state { STOPMACHINE_WAIT, STOPMACHINE_PREPARE, STOPMACHINE_DISABLE_IRQ, + STOPMACHINE_RUN, STOPMACHINE_EXIT, }; +struct stop_machine_data { + int (*fn)(void *); + void *data; + struct completion done; + int run_all; +} smdata; + static enum stopmachine_state stopmachine_state; static unsigned int stopmachine_num_threads; static atomic_t stopmachine_thread_ack; @@ -33,6 +41,7 @@ static int stopmachine(void *cpu) { int irqs_disabled = 0; int prepared = 0; + int ran = 0; set_cpus_allowed_ptr(current, &cpumask_of_cpu((int)(long)cpu)); @@ -57,6 +66,11 @@ static int stopmachine(void *cpu) prepared = 1; smp_mb(); /* Must read state first. */ atomic_inc(&stopmachine_thread_ack); + } else if (stopmachine_state == STOPMACHINE_RUN && !ran) { + smdata.fn(smdata.data); + ran = 1; + smp_mb(); /* Must read state first. */ + atomic_inc(&stopmachine_thread_ack); } /* Yield in first stage: migration threads need to * help our sisters onto their CPUs. */ @@ -134,11 +148,10 @@ static void restart_machine(void) preempt_enable_no_resched(); } -struct stop_machine_data { - int (*fn)(void *); - void *data; - struct completion done; -}; +static void run_other_cpus(void) +{ + stopmachine_set_state(STOPMACHINE_RUN); +} static int do_stop(void *_smdata) { @@ -148,6 +161,8 @@ static int do_stop(void *_smdata) ret = stop_machine(); if (ret == 0) { ret = smdata->fn(smdata->data); + if (smdata->run_all) + run_other_cpus(); restart_machine(); } @@ -171,14 +186,17 @@ struct task_struct *__stop_machine_run(i struct stop_machine_data smdata; struct task_struct *p; + mutex_lock(&stopmachine_mutex); + smdata.fn = fn; smdata.data = data; + smdata.run_all = (cpu == ALL_CPUS) ? 1 : 0; init_completion(&smdata.done); - mutex_lock(&stopmachine_mutex); + smp_wmb(); /* make sure other cpus see smdata updates */ /* If they don't care which CPU fn runs on, bind to any online one. */ - if (cpu == NR_CPUS) + if (cpu == NR_CPUS || cpu == ALL_CPUS) cpu = raw_smp_processor_id(); p = kthread_create(do_stop, &smdata, "kstopmachine"); -- Mathieu Desnoyers Computer Engineering Ph.D. Student, Ecole Polytechnique de Montreal OpenPGP key fingerprint: 8CD5 52C3 8E3C 4140 715F BA06 3F25 A8FE 3BAE 9A68 -- To unsubscribe from this list: send the line "unsubscribe linux-kernel" in the body of a message to majordomo@vger.kernel.org More majordomo info at http://vger.kernel.org/majordomo-info.html Please read the FAQ at http://www.tux.org/lkml/