Since we now serialize ttwu() using p->pi_lock, we also need to serialize ttwu_local() using that, otherwise, once we drop the rq->lock from ttwu() it can race with ttwu_local(). Reviewed-by: Frank Rowand Signed-off-by: Peter Zijlstra --- kernel/sched.c | 28 +++++++++++++++++----------- 1 file changed, 17 insertions(+), 11 deletions(-) Index: linux-2.6/kernel/sched.c =================================================================== --- linux-2.6.orig/kernel/sched.c +++ linux-2.6/kernel/sched.c @@ -2560,9 +2560,9 @@ static int try_to_wake_up(struct task_st * try_to_wake_up_local - try to wake up a local task with rq lock held * @p: the thread to be awakened * - * Put @p on the run-queue if it's not already there. The caller must + * Put @p on the run-queue if it's not already there. The caller must * ensure that this_rq() is locked, @p is bound to this_rq() and not - * the current task. this_rq() stays locked over invocation. + * the current task. */ static void try_to_wake_up_local(struct task_struct *p) { @@ -2570,16 +2570,21 @@ static void try_to_wake_up_local(struct BUG_ON(rq != this_rq()); BUG_ON(p == current); - lockdep_assert_held(&rq->lock); + + raw_spin_unlock(&rq->lock); + raw_spin_lock(&p->pi_lock); + raw_spin_lock(&rq->lock); if (!(p->state & TASK_NORMAL)) - return; + goto out; if (!p->on_rq) activate_task(rq, p, ENQUEUE_WAKEUP); ttwu_post_activation(p, rq, 0); ttwu_stat(rq, p, smp_processor_id(), 0); +out: + raw_spin_unlock(&p->pi_lock); } /** @@ -4084,6 +4089,7 @@ pick_next_task(struct rq *rq) */ asmlinkage void __sched schedule(void) { + struct task_struct *to_wakeup = NULL; struct task_struct *prev, *next; unsigned long *switch_count; struct rq *rq; @@ -4114,13 +4120,8 @@ asmlinkage void __sched schedule(void) * task to maintain concurrency. If so, wake * up the task. */ - if (prev->flags & PF_WQ_WORKER) { - struct task_struct *to_wakeup; - + if (prev->flags & PF_WQ_WORKER) to_wakeup = wq_worker_sleeping(prev, cpu); - if (to_wakeup) - try_to_wake_up_local(to_wakeup); - } deactivate_task(rq, prev, DEQUEUE_SLEEP); prev->on_rq = 0; } @@ -4137,8 +4138,13 @@ asmlinkage void __sched schedule(void) raw_spin_lock(&rq->lock); } + /* + * All three: try_to_wake_up_local(), pre_schedule() and idle_balance() + * can drop rq->lock. + */ + if (to_wakeup) + try_to_wake_up_local(to_wakeup); pre_schedule(rq, prev); - if (unlikely(!rq->nr_running)) idle_balance(cpu, rq); -- To unsubscribe from this list: send the line "unsubscribe linux-kernel" in the body of a message to majordomo@vger.kernel.org More majordomo info at http://vger.kernel.org/majordomo-info.html Please read the FAQ at http://www.tux.org/lkml/