Message-ID: <20170613003336.GI3623@X58A-UD3R>
Date:   Tue, 13 Jun 2017 09:33:36 +0900
From:   Byungchul Park <byungchul.park@....com>
To:     peterz@...radead.org, mingo@...nel.org
Cc:     tglx@...utronix.de, walken@...gle.com, boqun.feng@...il.com,
        kirill@...temov.name, linux-kernel@...r.kernel.org,
        linux-mm@...ck.org, akpm@...ux-foundation.org, willy@...radead.org,
        npiggin@...il.com, kernel-team@....com
Subject: Re: [PATCH v7 05/16] lockdep: Implement crossrelease feature

On Wed, May 24, 2017 at 05:59:38PM +0900, Byungchul Park wrote:
> Lockdep is a runtime locking correctness validator that detects and
> reports a deadlock or its possibility by checking dependencies between
> locks. It's useful since it does not report just an actual deadlock but
> also the possibility of a deadlock that has not actually happened yet.
> That enables problems to be fixed before they affect real systems.
> 
> However, this facility is only applicable to typical locks, such as
> spinlocks and mutexes, which are normally released within the context in
> which they were acquired. However, synchronization primitives like page
> locks or completions, which are allowed to be released in any context,
> also create dependencies and can cause a deadlock. So lockdep should
> track these locks to do a better job. The 'crossrelease' implementation
> makes these primitives also be tracked.

Hello,

I think you need to spend much time to review the patches, but 3 weeks
has passed since I submited. It would be appriciated if you spend more
time on it.

Thank you,
Byungchul

> 
> Signed-off-by: Byungchul Park <byungchul.park@....com>
> ---
>  include/linux/irqflags.h |  24 ++-
>  include/linux/lockdep.h  | 111 ++++++++++-
>  include/linux/sched.h    |   8 +
>  kernel/exit.c            |   1 +
>  kernel/fork.c            |   3 +
>  kernel/locking/lockdep.c | 474 ++++++++++++++++++++++++++++++++++++++++++++---
>  kernel/workqueue.c       |   2 +
>  lib/Kconfig.debug        |  12 ++
>  8 files changed, 601 insertions(+), 34 deletions(-)
> 
> diff --git a/include/linux/irqflags.h b/include/linux/irqflags.h
> index 5dd1272..c40af8a 100644
> --- a/include/linux/irqflags.h
> +++ b/include/linux/irqflags.h
> @@ -23,10 +23,26 @@
>  # define trace_softirq_context(p)	((p)->softirq_context)
>  # define trace_hardirqs_enabled(p)	((p)->hardirqs_enabled)
>  # define trace_softirqs_enabled(p)	((p)->softirqs_enabled)
> -# define trace_hardirq_enter()	do { current->hardirq_context++; } while (0)
> -# define trace_hardirq_exit()	do { current->hardirq_context--; } while (0)
> -# define lockdep_softirq_enter()	do { current->softirq_context++; } while (0)
> -# define lockdep_softirq_exit()	do { current->softirq_context--; } while (0)
> +# define trace_hardirq_enter()		\
> +do {					\
> +	current->hardirq_context++;	\
> +	crossrelease_hardirq_start();	\
> +} while (0)
> +# define trace_hardirq_exit()		\
> +do {					\
> +	current->hardirq_context--;	\
> +	crossrelease_hardirq_end();	\
> +} while (0)
> +# define lockdep_softirq_enter()	\
> +do {					\
> +	current->softirq_context++;	\
> +	crossrelease_softirq_start();	\
> +} while (0)
> +# define lockdep_softirq_exit()		\
> +do {					\
> +	current->softirq_context--;	\
> +	crossrelease_softirq_end();	\
> +} while (0)
>  # define INIT_TRACE_IRQFLAGS	.softirqs_enabled = 1,
>  #else
>  # define trace_hardirqs_on()		do { } while (0)
> diff --git a/include/linux/lockdep.h b/include/linux/lockdep.h
> index c1458fe..d531097 100644
> --- a/include/linux/lockdep.h
> +++ b/include/linux/lockdep.h
> @@ -155,6 +155,12 @@ struct lockdep_map {
>  	int				cpu;
>  	unsigned long			ip;
>  #endif
> +#ifdef CONFIG_LOCKDEP_CROSSRELEASE
> +	/*
> +	 * Whether it's a crosslock.
> +	 */
> +	int				cross;
> +#endif
>  };
>  
>  static inline void lockdep_copy_map(struct lockdep_map *to,
> @@ -258,7 +264,61 @@ struct held_lock {
>  	unsigned int hardirqs_off:1;
>  	unsigned int references:12;					/* 32 bits */
>  	unsigned int pin_count;
> +#ifdef CONFIG_LOCKDEP_CROSSRELEASE
> +	/*
> +	 * Generation id.
> +	 *
> +	 * A value of cross_gen_id will be stored when holding this,
> +	 * which is globally increased whenever each crosslock is held.
> +	 */
> +	unsigned int gen_id;
> +#endif
> +};
> +
> +#ifdef CONFIG_LOCKDEP_CROSSRELEASE
> +#define MAX_XHLOCK_TRACE_ENTRIES 5
> +
> +/*
> + * This is for keeping locks waiting for commit so that true dependencies
> + * can be added at commit step.
> + */
> +struct hist_lock {
> +	/*
> +	 * Seperate stack_trace data. This will be used at commit step.
> +	 */
> +	struct stack_trace	trace;
> +	unsigned long		trace_entries[MAX_XHLOCK_TRACE_ENTRIES];
> +
> +	/*
> +	 * Seperate hlock instance. This will be used at commit step.
> +	 *
> +	 * TODO: Use a smaller data structure containing only necessary
> +	 * data. However, we should make lockdep code able to handle the
> +	 * smaller one first.
> +	 */
> +	struct held_lock	hlock;
> +};
> +
> +/*
> + * To initialize a lock as crosslock, lockdep_init_map_crosslock() should
> + * be called instead of lockdep_init_map().
> + */
> +struct cross_lock {
> +	/*
> +	 * Seperate hlock instance. This will be used at commit step.
> +	 *
> +	 * TODO: Use a smaller data structure containing only necessary
> +	 * data. However, we should make lockdep code able to handle the
> +	 * smaller one first.
> +	 */
> +	struct held_lock	hlock;
> +};
> +
> +struct lockdep_map_cross {
> +	struct lockdep_map map;
> +	struct cross_lock xlock;
>  };
> +#endif
>  
>  /*
>   * Initialization, self-test and debugging-output methods:
> @@ -282,13 +342,6 @@ extern void lockdep_init_map(struct lockdep_map *lock, const char *name,
>  			     struct lock_class_key *key, int subclass);
>  
>  /*
> - * To initialize a lockdep_map statically use this macro.
> - * Note that _name must not be NULL.
> - */
> -#define STATIC_LOCKDEP_MAP_INIT(_name, _key) \
> -	{ .name = (_name), .key = (void *)(_key), }
> -
> -/*
>   * Reinitialize a lock key - for cases where there is special locking or
>   * special initialization of locks so that the validator gets the scope
>   * of dependencies wrong: they are either too broad (they need a class-split)
> @@ -443,6 +496,50 @@ static inline void lockdep_on(void)
>  
>  #endif /* !LOCKDEP */
>  
> +#ifdef CONFIG_LOCKDEP_CROSSRELEASE
> +extern void lockdep_init_map_crosslock(struct lockdep_map *lock,
> +				       const char *name,
> +				       struct lock_class_key *key,
> +				       int subclass);
> +extern void lock_commit_crosslock(struct lockdep_map *lock);
> +
> +#define STATIC_CROSS_LOCKDEP_MAP_INIT(_name, _key) \
> +	{ .map.name = (_name), .map.key = (void *)(_key), \
> +	  .map.cross = 1, }
> +
> +/*
> + * To initialize a lockdep_map statically use this macro.
> + * Note that _name must not be NULL.
> + */
> +#define STATIC_LOCKDEP_MAP_INIT(_name, _key) \
> +	{ .name = (_name), .key = (void *)(_key), .cross = 0, }
> +
> +extern void crossrelease_hardirq_start(void);
> +extern void crossrelease_hardirq_end(void);
> +extern void crossrelease_softirq_start(void);
> +extern void crossrelease_softirq_end(void);
> +extern void crossrelease_work_start(void);
> +extern void crossrelease_work_end(void);
> +extern void init_crossrelease_task(struct task_struct *task);
> +extern void free_crossrelease_task(struct task_struct *task);
> +#else
> +/*
> + * To initialize a lockdep_map statically use this macro.
> + * Note that _name must not be NULL.
> + */
> +#define STATIC_LOCKDEP_MAP_INIT(_name, _key) \
> +	{ .name = (_name), .key = (void *)(_key), }
> +
> +static inline void crossrelease_hardirq_start(void) {}
> +static inline void crossrelease_hardirq_end(void) {}
> +static inline void crossrelease_softirq_start(void) {}
> +static inline void crossrelease_softirq_end(void) {}
> +static inline void crossrelease_work_start(void) {}
> +static inline void crossrelease_work_end(void) {}
> +static inline void init_crossrelease_task(struct task_struct *task) {}
> +static inline void free_crossrelease_task(struct task_struct *task) {}
> +#endif
> +
>  #ifdef CONFIG_LOCK_STAT
>  
>  extern void lock_contended(struct lockdep_map *lock, unsigned long ip);
> diff --git a/include/linux/sched.h b/include/linux/sched.h
> index e9c009d..5f6d6f4 100644
> --- a/include/linux/sched.h
> +++ b/include/linux/sched.h
> @@ -1749,6 +1749,14 @@ struct task_struct {
>  	struct held_lock held_locks[MAX_LOCK_DEPTH];
>  	gfp_t lockdep_reclaim_gfp;
>  #endif
> +#ifdef CONFIG_LOCKDEP_CROSSRELEASE
> +#define MAX_XHLOCKS_NR 64UL
> +	struct hist_lock *xhlocks; /* Crossrelease history locks */
> +	unsigned int xhlock_idx;
> +	unsigned int xhlock_idx_soft; /* For restoring at softirq exit */
> +	unsigned int xhlock_idx_hard; /* For restoring at hardirq exit */
> +	unsigned int xhlock_idx_work; /* For restoring at work exit */
> +#endif
>  #ifdef CONFIG_UBSAN
>  	unsigned int in_ubsan;
>  #endif
> diff --git a/kernel/exit.c b/kernel/exit.c
> index 3076f30..cc56aad 100644
> --- a/kernel/exit.c
> +++ b/kernel/exit.c
> @@ -883,6 +883,7 @@ void __noreturn do_exit(long code)
>  	exit_rcu();
>  	TASKS_RCU(__srcu_read_unlock(&tasks_rcu_exit_srcu, tasks_rcu_i));
>  
> +	free_crossrelease_task(tsk);
>  	do_task_dead();
>  }
>  EXPORT_SYMBOL_GPL(do_exit);
> diff --git a/kernel/fork.c b/kernel/fork.c
> index 997ac1d..f9623a0 100644
> --- a/kernel/fork.c
> +++ b/kernel/fork.c
> @@ -451,6 +451,7 @@ void __init fork_init(void)
>  	for (i = 0; i < UCOUNT_COUNTS; i++) {
>  		init_user_ns.ucount_max[i] = max_threads/2;
>  	}
> +	init_crossrelease_task(&init_task);
>  }
>  
>  int __weak arch_dup_task_struct(struct task_struct *dst,
> @@ -1611,6 +1612,7 @@ static __latent_entropy struct task_struct *copy_process(
>  	p->lockdep_depth = 0; /* no locks held yet */
>  	p->curr_chain_key = 0;
>  	p->lockdep_recursion = 0;
> +	init_crossrelease_task(p);
>  #endif
>  
>  #ifdef CONFIG_DEBUG_MUTEXES
> @@ -1856,6 +1858,7 @@ static __latent_entropy struct task_struct *copy_process(
>  bad_fork_cleanup_perf:
>  	perf_event_free_task(p);
>  bad_fork_cleanup_policy:
> +	free_crossrelease_task(p);
>  #ifdef CONFIG_NUMA
>  	mpol_put(p->mempolicy);
>  bad_fork_cleanup_threadgroup_lock:
> diff --git a/kernel/locking/lockdep.c b/kernel/locking/lockdep.c
> index 2847356..63eb04a 100644
> --- a/kernel/locking/lockdep.c
> +++ b/kernel/locking/lockdep.c
> @@ -55,6 +55,10 @@
>  #define CREATE_TRACE_POINTS
>  #include <trace/events/lock.h>
>  
> +#ifdef CONFIG_LOCKDEP_CROSSRELEASE
> +#include <linux/slab.h>
> +#endif
> +
>  #ifdef CONFIG_PROVE_LOCKING
>  int prove_locking = 1;
>  module_param(prove_locking, int, 0644);
> @@ -709,6 +713,18 @@ static int count_matching_names(struct lock_class *new_class)
>  	return NULL;
>  }
>  
> +#ifdef CONFIG_LOCKDEP_CROSSRELEASE
> +static void cross_init(struct lockdep_map *lock, int cross);
> +static int cross_lock(struct lockdep_map *lock);
> +static int lock_acquire_crosslock(struct held_lock *hlock);
> +static int lock_release_crosslock(struct lockdep_map *lock);
> +#else
> +static inline void cross_init(struct lockdep_map *lock, int cross) {}
> +static inline int cross_lock(struct lockdep_map *lock) { return 0; }
> +static inline int lock_acquire_crosslock(struct held_lock *hlock) { return 2; }
> +static inline int lock_release_crosslock(struct lockdep_map *lock) { return 2; }
> +#endif
> +
>  /*
>   * Register a lock's class in the hash-table, if the class is not present
>   * yet. Otherwise we look it up. We cache the result in the lock object
> @@ -1768,6 +1784,9 @@ static inline void inc_chains(void)
>  		if (nest)
>  			return 2;
>  
> +		if (cross_lock(prev->instance))
> +			continue;
> +
>  		return print_deadlock_bug(curr, prev, next);
>  	}
>  	return 1;
> @@ -1921,30 +1940,36 @@ static inline void inc_chains(void)
>  		int distance = curr->lockdep_depth - depth + 1;
>  		hlock = curr->held_locks + depth - 1;
>  		/*
> -		 * Only non-recursive-read entries get new dependencies
> -		 * added:
> +		 * Only non-crosslock entries get new dependencies added.
> +		 * Crosslock entries will be added by commit later:
>  		 */
> -		if (hlock->read != 2 && hlock->check) {
> -			int ret = check_prev_add(curr, hlock, next,
> -						distance, &trace, save);
> -			if (!ret)
> -				return 0;
> -
> +		if (!cross_lock(hlock->instance)) {
>  			/*
> -			 * Stop saving stack_trace if save_trace() was
> -			 * called at least once:
> +			 * Only non-recursive-read entries get new dependencies
> +			 * added:
>  			 */
> -			if (save && ret == 2)
> -				save = NULL;
> +			if (hlock->read != 2 && hlock->check) {
> +				int ret = check_prev_add(curr, hlock, next,
> +							 distance, &trace, save);
> +				if (!ret)
> +					return 0;
>  
> -			/*
> -			 * Stop after the first non-trylock entry,
> -			 * as non-trylock entries have added their
> -			 * own direct dependencies already, so this
> -			 * lock is connected to them indirectly:
> -			 */
> -			if (!hlock->trylock)
> -				break;
> +				/*
> +				 * Stop saving stack_trace if save_trace() was
> +				 * called at least once:
> +				 */
> +				if (save && ret == 2)
> +					save = NULL;
> +
> +				/*
> +				 * Stop after the first non-trylock entry,
> +				 * as non-trylock entries have added their
> +				 * own direct dependencies already, so this
> +				 * lock is connected to them indirectly:
> +				 */
> +				if (!hlock->trylock)
> +					break;
> +			}
>  		}
>  		depth--;
>  		/*
> @@ -3203,7 +3228,7 @@ static int mark_lock(struct task_struct *curr, struct held_lock *this,
>  /*
>   * Initialize a lock instance's lock-class mapping info:
>   */
> -void lockdep_init_map(struct lockdep_map *lock, const char *name,
> +static void __lockdep_init_map(struct lockdep_map *lock, const char *name,
>  		      struct lock_class_key *key, int subclass)
>  {
>  	int i;
> @@ -3261,8 +3286,25 @@ void lockdep_init_map(struct lockdep_map *lock, const char *name,
>  		raw_local_irq_restore(flags);
>  	}
>  }
> +
> +void lockdep_init_map(struct lockdep_map *lock, const char *name,
> +		      struct lock_class_key *key, int subclass)
> +{
> +	cross_init(lock, 0);
> +	__lockdep_init_map(lock, name, key, subclass);
> +}
>  EXPORT_SYMBOL_GPL(lockdep_init_map);
>  
> +#ifdef CONFIG_LOCKDEP_CROSSRELEASE
> +void lockdep_init_map_crosslock(struct lockdep_map *lock, const char *name,
> +		      struct lock_class_key *key, int subclass)
> +{
> +	cross_init(lock, 1);
> +	__lockdep_init_map(lock, name, key, subclass);
> +}
> +EXPORT_SYMBOL_GPL(lockdep_init_map_crosslock);
> +#endif
> +
>  struct lock_class_key __lockdep_no_validate__;
>  EXPORT_SYMBOL_GPL(__lockdep_no_validate__);
>  
> @@ -3317,6 +3359,7 @@ static int __lock_acquire(struct lockdep_map *lock, unsigned int subclass,
>  	unsigned int depth;
>  	int chain_head = 0;
>  	int class_idx;
> +	int ret;
>  	u64 chain_key;
>  
>  	if (unlikely(!debug_locks))
> @@ -3366,7 +3409,8 @@ static int __lock_acquire(struct lockdep_map *lock, unsigned int subclass,
>  
>  	class_idx = class - lock_classes + 1;
>  
> -	if (depth) {
> +	/* TODO: nest_lock is not implemented for crosslock yet. */
> +	if (depth && !cross_lock(lock)) {
>  		hlock = curr->held_locks + depth - 1;
>  		if (hlock->class_idx == class_idx && nest_lock) {
>  			if (hlock->references)
> @@ -3447,6 +3491,14 @@ static int __lock_acquire(struct lockdep_map *lock, unsigned int subclass,
>  	if (!validate_chain(curr, lock, hlock, chain_head, chain_key))
>  		return 0;
>  
> +	ret = lock_acquire_crosslock(hlock);
> +	/*
> +	 * 2 means normal acquire operations are needed. Otherwise, it's
> +	 * ok just to return with '0:fail, 1:success'.
> +	 */
> +	if (ret != 2)
> +		return ret;
> +
>  	curr->curr_chain_key = chain_key;
>  	curr->lockdep_depth++;
>  	check_chain_key(curr);
> @@ -3610,11 +3662,19 @@ static int match_held_lock(struct held_lock *hlock, struct lockdep_map *lock)
>  	struct task_struct *curr = current;
>  	struct held_lock *hlock, *prev_hlock;
>  	unsigned int depth;
> -	int i;
> +	int ret, i;
>  
>  	if (unlikely(!debug_locks))
>  		return 0;
>  
> +	ret = lock_release_crosslock(lock);
> +	/*
> +	 * 2 means normal release operations are needed. Otherwise, it's
> +	 * ok just to return with '0:fail, 1:success'.
> +	 */
> +	if (ret != 2)
> +		return ret;
> +
>  	depth = curr->lockdep_depth;
>  	/*
>  	 * So we're all set to release this lock.. wait what lock? We don't
> @@ -4557,3 +4617,371 @@ void lockdep_rcu_suspicious(const char *file, const int line, const char *s)
>  	dump_stack();
>  }
>  EXPORT_SYMBOL_GPL(lockdep_rcu_suspicious);
> +
> +#ifdef CONFIG_LOCKDEP_CROSSRELEASE
> +
> +#define xhlock(i)         (current->xhlocks[(i) % MAX_XHLOCKS_NR])
> +
> +/*
> + * Whenever a crosslock is held, cross_gen_id will be increased.
> + */
> +static atomic_t cross_gen_id; /* Can be wrapped */
> +
> +void crossrelease_hardirq_start(void)
> +{
> +	if (current->xhlocks)
> +		current->xhlock_idx_hard = current->xhlock_idx;
> +}
> +
> +void crossrelease_hardirq_end(void)
> +{
> +	if (current->xhlocks)
> +		current->xhlock_idx = current->xhlock_idx_hard;
> +}
> +
> +void crossrelease_softirq_start(void)
> +{
> +	if (current->xhlocks)
> +		current->xhlock_idx_soft = current->xhlock_idx;
> +}
> +
> +void crossrelease_softirq_end(void)
> +{
> +	if (current->xhlocks)
> +		current->xhlock_idx = current->xhlock_idx_soft;
> +}
> +
> +/*
> + * Each work of workqueue might run in a different context,
> + * thanks to concurrency support of workqueue. So we have to
> + * distinguish each work to avoid false positive.
> + */
> +void crossrelease_work_start(void)
> +{
> +	if (current->xhlocks)
> +		current->xhlock_idx_work = current->xhlock_idx;
> +}
> +
> +void crossrelease_work_end(void)
> +{
> +	if (current->xhlocks)
> +		current->xhlock_idx = current->xhlock_idx_work;
> +}
> +
> +static int cross_lock(struct lockdep_map *lock)
> +{
> +	return lock ? lock->cross : 0;
> +}
> +
> +/*
> + * This is needed to decide the relationship between wrapable variables.
> + */
> +static inline int before(unsigned int a, unsigned int b)
> +{
> +	return (int)(a - b) < 0;
> +}
> +
> +static inline struct lock_class *xhlock_class(struct hist_lock *xhlock)
> +{
> +	return hlock_class(&xhlock->hlock);
> +}
> +
> +static inline struct lock_class *xlock_class(struct cross_lock *xlock)
> +{
> +	return hlock_class(&xlock->hlock);
> +}
> +
> +/*
> + * Should we check a dependency with previous one?
> + */
> +static inline int depend_before(struct held_lock *hlock)
> +{
> +	return hlock->read != 2 && hlock->check && !hlock->trylock;
> +}
> +
> +/*
> + * Should we check a dependency with next one?
> + */
> +static inline int depend_after(struct held_lock *hlock)
> +{
> +	return hlock->read != 2 && hlock->check;
> +}
> +
> +/*
> + * Check if the xhlock is valid, which would be false if,
> + *
> + *    1. Has not used after initializaion yet.
> + *
> + * Remind hist_lock is implemented as a ring buffer.
> + */
> +static inline int xhlock_valid(struct hist_lock *xhlock)
> +{
> +	/*
> +	 * xhlock->hlock.instance must be !NULL.
> +	 */
> +	return !!xhlock->hlock.instance;
> +}
> +
> +/*
> + * Record a hist_lock entry.
> + *
> + * Irq disable is only required.
> + */
> +static void add_xhlock(struct held_lock *hlock)
> +{
> +	unsigned int idx = ++current->xhlock_idx;
> +	struct hist_lock *xhlock = &xhlock(idx);
> +
> +#ifdef CONFIG_DEBUG_LOCKDEP
> +	/*
> +	 * This can be done locklessly because they are all task-local
> +	 * state, we must however ensure IRQs are disabled.
> +	 */
> +	WARN_ON_ONCE(!irqs_disabled());
> +#endif
> +
> +	/* Initialize hist_lock's members */
> +	xhlock->hlock = *hlock;
> +
> +	xhlock->trace.nr_entries = 0;
> +	xhlock->trace.max_entries = MAX_XHLOCK_TRACE_ENTRIES;
> +	xhlock->trace.entries = xhlock->trace_entries;
> +	xhlock->trace.skip = 3;
> +	save_stack_trace(&xhlock->trace);
> +}
> +
> +static inline int same_context_xhlock(struct hist_lock *xhlock)
> +{
> +	return xhlock->hlock.irq_context == task_irq_context(current);
> +}
> +
> +/*
> + * This should be lockless as far as possible because this would be
> + * called very frequently.
> + */
> +static void check_add_xhlock(struct held_lock *hlock)
> +{
> +	/*
> +	 * Record a hist_lock, only in case that acquisitions ahead
> +	 * could depend on the held_lock. For example, if the held_lock
> +	 * is trylock then acquisitions ahead never depends on that.
> +	 * In that case, we don't need to record it. Just return.
> +	 */
> +	if (!current->xhlocks || !depend_before(hlock))
> +		return;
> +
> +	add_xhlock(hlock);
> +}
> +
> +/*
> + * For crosslock.
> + */
> +static int add_xlock(struct held_lock *hlock)
> +{
> +	struct cross_lock *xlock;
> +	unsigned int gen_id;
> +
> +	if (!graph_lock())
> +		return 0;
> +
> +	xlock = &((struct lockdep_map_cross *)hlock->instance)->xlock;
> +
> +	gen_id = (unsigned int)atomic_inc_return(&cross_gen_id);
> +	xlock->hlock = *hlock;
> +	xlock->hlock.gen_id = gen_id;
> +	graph_unlock();
> +
> +	return 1;
> +}
> +
> +/*
> + * Called for both normal and crosslock acquires. Normal locks will be
> + * pushed on the hist_lock queue. Cross locks will record state and
> + * stop regular lock_acquire() to avoid being placed on the held_lock
> + * stack.
> + *
> + * Return: 0 - failure;
> + *         1 - crosslock, done;
> + *         2 - normal lock, continue to held_lock[] ops.
> + */
> +static int lock_acquire_crosslock(struct held_lock *hlock)
> +{
> +	/*
> +	 *	CONTEXT 1		CONTEXT 2
> +	 *	---------		---------
> +	 *	lock A (cross)
> +	 *	X = atomic_inc_return(&cross_gen_id)
> +	 *	~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
> +	 *				Y = atomic_read_acquire(&cross_gen_id)
> +	 *				lock B
> +	 *
> +	 * atomic_read_acquire() is for ordering between A and B,
> +	 * IOW, A happens before B, when CONTEXT 2 see Y >= X.
> +	 *
> +	 * Pairs with atomic_inc_return() in add_xlock().
> +	 */
> +	hlock->gen_id = (unsigned int)atomic_read_acquire(&cross_gen_id);
> +
> +	if (cross_lock(hlock->instance))
> +		return add_xlock(hlock);
> +
> +	check_add_xhlock(hlock);
> +	return 2;
> +}
> +
> +static int copy_trace(struct stack_trace *trace)
> +{
> +	unsigned long *buf = stack_trace + nr_stack_trace_entries;
> +	unsigned int max_nr = MAX_STACK_TRACE_ENTRIES - nr_stack_trace_entries;
> +	unsigned int nr = min(max_nr, trace->nr_entries);
> +
> +	trace->nr_entries = nr;
> +	memcpy(buf, trace->entries, nr * sizeof(trace->entries[0]));
> +	trace->entries = buf;
> +	nr_stack_trace_entries += nr;
> +
> +	if (nr_stack_trace_entries >= MAX_STACK_TRACE_ENTRIES-1) {
> +		if (!debug_locks_off_graph_unlock())
> +			return 0;
> +
> +		print_lockdep_off("BUG: MAX_STACK_TRACE_ENTRIES too low!");
> +		dump_stack();
> +
> +		return 0;
> +	}
> +
> +	return 1;
> +}
> +
> +static int commit_xhlock(struct cross_lock *xlock, struct hist_lock *xhlock)
> +{
> +	unsigned int xid, pid;
> +	u64 chain_key;
> +
> +	xid = xlock_class(xlock) - lock_classes;
> +	chain_key = iterate_chain_key((u64)0, xid);
> +	pid = xhlock_class(xhlock) - lock_classes;
> +	chain_key = iterate_chain_key(chain_key, pid);
> +
> +	if (lookup_chain_cache(chain_key))
> +		return 1;
> +
> +	if (!add_chain_cache_classes(xid, pid, xhlock->hlock.irq_context,
> +				chain_key))
> +		return 0;
> +
> +	if (!check_prev_add(current, &xlock->hlock, &xhlock->hlock, 1,
> +			    &xhlock->trace, copy_trace))
> +		return 0;
> +
> +	return 1;
> +}
> +
> +static void commit_xhlocks(struct cross_lock *xlock)
> +{
> +	unsigned int cur = current->xhlock_idx;
> +	unsigned int i;
> +
> +	if (!graph_lock())
> +		return;
> +
> +	for (i = 0; i < MAX_XHLOCKS_NR; i++) {
> +		struct hist_lock *xhlock = &xhlock(cur - i);
> +
> +		if (!xhlock_valid(xhlock))
> +			break;
> +
> +		if (before(xhlock->hlock.gen_id, xlock->hlock.gen_id))
> +			break;
> +
> +		if (!same_context_xhlock(xhlock))
> +			break;
> +
> +		/*
> +		 * commit_xhlock() returns 0 with graph_lock already
> +		 * released if fail.
> +		 */
> +		if (!commit_xhlock(xlock, xhlock))
> +			return;
> +	}
> +
> +	graph_unlock();
> +}
> +
> +void lock_commit_crosslock(struct lockdep_map *lock)
> +{
> +	struct cross_lock *xlock;
> +	unsigned long flags;
> +
> +	if (unlikely(!debug_locks || current->lockdep_recursion))
> +		return;
> +
> +	if (!current->xhlocks)
> +		return;
> +
> +	/*
> +	 * Do commit hist_locks with the cross_lock, only in case that
> +	 * the cross_lock could depend on acquisitions after that.
> +	 *
> +	 * For example, if the cross_lock does not have the 'check' flag
> +	 * then we don't need to check dependencies and commit for that.
> +	 * Just skip it. In that case, of course, the cross_lock does
> +	 * not depend on acquisitions ahead, either.
> +	 *
> +	 * WARNING: Don't do that in add_xlock() in advance. When an
> +	 * acquisition context is different from the commit context,
> +	 * invalid(skipped) cross_lock might be accessed.
> +	 */
> +	if (!depend_after(&((struct lockdep_map_cross *)lock)->xlock.hlock))
> +		return;
> +
> +	raw_local_irq_save(flags);
> +	check_flags(flags);
> +	current->lockdep_recursion = 1;
> +	xlock = &((struct lockdep_map_cross *)lock)->xlock;
> +	commit_xhlocks(xlock);
> +	current->lockdep_recursion = 0;
> +	raw_local_irq_restore(flags);
> +}
> +EXPORT_SYMBOL_GPL(lock_commit_crosslock);
> +
> +/*
> + * Return: 1 - crosslock, done;
> + *         2 - normal lock, continue to held_lock[] ops.
> + */
> +static int lock_release_crosslock(struct lockdep_map *lock)
> +{
> +	return cross_lock(lock) ? 1 : 2;
> +}
> +
> +static void cross_init(struct lockdep_map *lock, int cross)
> +{
> +	lock->cross = cross;
> +
> +	/*
> +	 * Crossrelease assumes that the ring buffer size of xhlocks
> +	 * is aligned with power of 2. So force it on build.
> +	 */
> +	BUILD_BUG_ON(MAX_XHLOCKS_NR & (MAX_XHLOCKS_NR - 1));
> +}
> +
> +void init_crossrelease_task(struct task_struct *task)
> +{
> +	task->xhlock_idx = UINT_MAX;
> +	task->xhlock_idx_soft = UINT_MAX;
> +	task->xhlock_idx_hard = UINT_MAX;
> +	task->xhlock_idx_work = UINT_MAX;
> +	task->xhlocks = kzalloc(sizeof(struct hist_lock) * MAX_XHLOCKS_NR,
> +				GFP_KERNEL);
> +}
> +
> +void free_crossrelease_task(struct task_struct *task)
> +{
> +	if (task->xhlocks) {
> +		void *tmp = task->xhlocks;
> +		/* Diable crossrelease for current */
> +		task->xhlocks = NULL;
> +		kfree(tmp);
> +	}
> +}
> +#endif
> diff --git a/kernel/workqueue.c b/kernel/workqueue.c
> index 479d840..2f43ac1 100644
> --- a/kernel/workqueue.c
> +++ b/kernel/workqueue.c
> @@ -2092,6 +2092,7 @@ static void process_one_work(struct worker *worker, struct work_struct *work)
>  
>  	lock_map_acquire_read(&pwq->wq->lockdep_map);
>  	lock_map_acquire(&lockdep_map);
> +	crossrelease_work_start();
>  	trace_workqueue_execute_start(work);
>  	worker->current_func(work);
>  	/*
> @@ -2099,6 +2100,7 @@ static void process_one_work(struct worker *worker, struct work_struct *work)
>  	 * point will only record its address.
>  	 */
>  	trace_workqueue_execute_end(work);
> +	crossrelease_work_end();
>  	lock_map_release(&lockdep_map);
>  	lock_map_release(&pwq->wq->lockdep_map);
>  
> diff --git a/lib/Kconfig.debug b/lib/Kconfig.debug
> index a6c8db1..e584431 100644
> --- a/lib/Kconfig.debug
> +++ b/lib/Kconfig.debug
> @@ -1042,6 +1042,18 @@ config DEBUG_LOCK_ALLOC
>  	 spin_lock_init()/mutex_init()/etc., or whether there is any lock
>  	 held during task exit.
>  
> +config LOCKDEP_CROSSRELEASE
> +	bool "Lock debugging: make lockdep work for crosslocks"
> +	depends on PROVE_LOCKING
> +	default n
> +	help
> +	 This makes lockdep work for crosslock which is a lock allowed to
> +	 be released in a different context from the acquisition context.
> +	 Normally a lock must be released in the context acquiring the lock.
> +	 However, relexing this constraint helps synchronization primitives
> +	 such as page locks or completions can use the lock correctness
> +	 detector, lockdep.
> +
>  config PROVE_LOCKING
>  	bool "Lock debugging: prove locking correctness"
>  	depends on DEBUG_KERNEL && TRACE_IRQFLAGS_SUPPORT && STACKTRACE_SUPPORT && LOCKDEP_SUPPORT
> -- 
> 1.9.1

Powered by blists - more mailing lists