Doesn't really matter yet, but pull the HANDOFF and trylock out from under the wait_lock. The intention is to add an optimistic spin loop here, which requires we do not hold the wait_lock, so shuffle code around in preparation. Also clarify the purpose of taking the wait_lock in the wait loop, its tempting to want to avoid it altogether, but the cancellation cases need to to avoid loosing wakeups. Suggested-by: Waiman Long Signed-off-by: Peter Zijlstra (Intel) --- kernel/locking/mutex.c | 18 ++++++++++++++---- 1 file changed, 14 insertions(+), 4 deletions(-) --- a/kernel/locking/mutex.c +++ b/kernel/locking/mutex.c @@ -627,10 +627,12 @@ __mutex_lock_common(struct mutex *lock, lock_contended(&lock->dep_map, ip); + set_task_state(task, state); for (;;) { /* - * got a signal? (This code gets eliminated in the - * TASK_UNINTERRUPTIBLE case.) + * Check for signals and wound conditions while holding + * wait_lock. This ensures the lock cancellation is ordered + * against mutex_unlock() and wake-ups do not go missing. */ if (unlikely(signal_pending_state(state, task))) { ret = -EINTR; @@ -643,20 +645,27 @@ __mutex_lock_common(struct mutex *lock, goto err; } - __set_task_state(task, state); spin_unlock_mutex(&lock->wait_lock, flags); schedule_preempt_disabled(); - spin_lock_mutex(&lock->wait_lock, flags); if (__mutex_waiter_is_first(lock, &waiter)) { first = true; __mutex_set_flag(lock, MUTEX_FLAG_HANDOFF); } + set_task_state(task, state); + /* + * Here we order against unlock; we must either see it change + * state back to RUNNING and fall through the next schedule(), + * or we must see its unlock and acquire. + */ if (__mutex_trylock(lock, true)) break; + + spin_lock_mutex(&lock->wait_lock, flags); } __set_task_state(task, TASK_RUNNING); + spin_lock_mutex(&lock->wait_lock, flags); remove_waiter: mutex_remove_waiter(lock, &waiter, task); @@ -681,6 +690,7 @@ __mutex_lock_common(struct mutex *lock, return 0; err: + __set_task_state(task, TASK_RUNNING); mutex_remove_waiter(lock, &waiter, task); spin_unlock_mutex(&lock->wait_lock, flags); debug_mutex_free_waiter(&waiter);