Message-ID: <20170622232745.GA20323@X58A-UD3R>
Date:   Fri, 23 Jun 2017 08:27:45 +0900
From:   Byungchul Park <byungchul.park@....com>
To:     peterz@...radead.org, mingo@...nel.org
Cc:     tglx@...utronix.de, walken@...gle.com, boqun.feng@...il.com,
        kirill@...temov.name, linux-kernel@...r.kernel.org,
        linux-mm@...ck.org, akpm@...ux-foundation.org, willy@...radead.org,
        npiggin@...il.com, kernel-team@....com
Subject: Re: [PATCH v7 05/16] lockdep: Implement crossrelease feature

On Tue, Jun 13, 2017 at 09:33:36AM +0900, Byungchul Park wrote:
> On Wed, May 24, 2017 at 05:59:38PM +0900, Byungchul Park wrote:
> > Lockdep is a runtime locking correctness validator that detects and
> > reports a deadlock or its possibility by checking dependencies between
> > locks. It's useful since it does not report just an actual deadlock but
> > also the possibility of a deadlock that has not actually happened yet.
> > That enables problems to be fixed before they affect real systems.
> > 
> > However, this facility is only applicable to typical locks, such as
> > spinlocks and mutexes, which are normally released within the context in
> > which they were acquired. However, synchronization primitives like page
> > locks or completions, which are allowed to be released in any context,
> > also create dependencies and can cause a deadlock. So lockdep should
> > track these locks to do a better job. The 'crossrelease' implementation
> > makes these primitives also be tracked.
> 
> Hello,
> 
> I think you need to spend much time to review the patches, but 3 weeks
> has passed since I submited. It would be appriciated if you spend more
> time on it.

Hello, Peter

I meant you might need much time to review it.

But more than one month passed. It would be appriciated if you check it.

> 
> Thank you,
> Byungchul
> 
> > 
> > Signed-off-by: Byungchul Park <byungchul.park@....com>
> > ---
> >  include/linux/irqflags.h |  24 ++-
> >  include/linux/lockdep.h  | 111 ++++++++++-
> >  include/linux/sched.h    |   8 +
> >  kernel/exit.c            |   1 +
> >  kernel/fork.c            |   3 +
> >  kernel/locking/lockdep.c | 474 ++++++++++++++++++++++++++++++++++++++++++++---
> >  kernel/workqueue.c       |   2 +
> >  lib/Kconfig.debug        |  12 ++
> >  8 files changed, 601 insertions(+), 34 deletions(-)
> > 
> > diff --git a/include/linux/irqflags.h b/include/linux/irqflags.h
> > index 5dd1272..c40af8a 100644
> > --- a/include/linux/irqflags.h
> > +++ b/include/linux/irqflags.h
> > @@ -23,10 +23,26 @@
> >  # define trace_softirq_context(p)	((p)->softirq_context)
> >  # define trace_hardirqs_enabled(p)	((p)->hardirqs_enabled)
> >  # define trace_softirqs_enabled(p)	((p)->softirqs_enabled)
> > -# define trace_hardirq_enter()	do { current->hardirq_context++; } while (0)
> > -# define trace_hardirq_exit()	do { current->hardirq_context--; } while (0)
> > -# define lockdep_softirq_enter()	do { current->softirq_context++; } while (0)
> > -# define lockdep_softirq_exit()	do { current->softirq_context--; } while (0)
> > +# define trace_hardirq_enter()		\
> > +do {					\
> > +	current->hardirq_context++;	\
> > +	crossrelease_hardirq_start();	\
> > +} while (0)
> > +# define trace_hardirq_exit()		\
> > +do {					\
> > +	current->hardirq_context--;	\
> > +	crossrelease_hardirq_end();	\
> > +} while (0)
> > +# define lockdep_softirq_enter()	\
> > +do {					\
> > +	current->softirq_context++;	\
> > +	crossrelease_softirq_start();	\
> > +} while (0)
> > +# define lockdep_softirq_exit()		\
> > +do {					\
> > +	current->softirq_context--;	\
> > +	crossrelease_softirq_end();	\
> > +} while (0)
> >  # define INIT_TRACE_IRQFLAGS	.softirqs_enabled = 1,
> >  #else
> >  # define trace_hardirqs_on()		do { } while (0)
> > diff --git a/include/linux/lockdep.h b/include/linux/lockdep.h
> > index c1458fe..d531097 100644
> > --- a/include/linux/lockdep.h
> > +++ b/include/linux/lockdep.h
> > @@ -155,6 +155,12 @@ struct lockdep_map {
> >  	int				cpu;
> >  	unsigned long			ip;
> >  #endif
> > +#ifdef CONFIG_LOCKDEP_CROSSRELEASE
> > +	/*
> > +	 * Whether it's a crosslock.
> > +	 */
> > +	int				cross;
> > +#endif
> >  };
> >  
> >  static inline void lockdep_copy_map(struct lockdep_map *to,
> > @@ -258,7 +264,61 @@ struct held_lock {
> >  	unsigned int hardirqs_off:1;
> >  	unsigned int references:12;					/* 32 bits */
> >  	unsigned int pin_count;
> > +#ifdef CONFIG_LOCKDEP_CROSSRELEASE
> > +	/*
> > +	 * Generation id.
> > +	 *
> > +	 * A value of cross_gen_id will be stored when holding this,
> > +	 * which is globally increased whenever each crosslock is held.
> > +	 */
> > +	unsigned int gen_id;
> > +#endif
> > +};
> > +
> > +#ifdef CONFIG_LOCKDEP_CROSSRELEASE
> > +#define MAX_XHLOCK_TRACE_ENTRIES 5
> > +
> > +/*
> > + * This is for keeping locks waiting for commit so that true dependencies
> > + * can be added at commit step.
> > + */
> > +struct hist_lock {
> > +	/*
> > +	 * Seperate stack_trace data. This will be used at commit step.
> > +	 */
> > +	struct stack_trace	trace;
> > +	unsigned long		trace_entries[MAX_XHLOCK_TRACE_ENTRIES];
> > +
> > +	/*
> > +	 * Seperate hlock instance. This will be used at commit step.
> > +	 *
> > +	 * TODO: Use a smaller data structure containing only necessary
> > +	 * data. However, we should make lockdep code able to handle the
> > +	 * smaller one first.
> > +	 */
> > +	struct held_lock	hlock;
> > +};
> > +
> > +/*
> > + * To initialize a lock as crosslock, lockdep_init_map_crosslock() should
> > + * be called instead of lockdep_init_map().
> > + */
> > +struct cross_lock {
> > +	/*
> > +	 * Seperate hlock instance. This will be used at commit step.
> > +	 *
> > +	 * TODO: Use a smaller data structure containing only necessary
> > +	 * data. However, we should make lockdep code able to handle the
> > +	 * smaller one first.
> > +	 */
> > +	struct held_lock	hlock;
> > +};
> > +
> > +struct lockdep_map_cross {
> > +	struct lockdep_map map;
> > +	struct cross_lock xlock;
> >  };
> > +#endif
> >  
> >  /*
> >   * Initialization, self-test and debugging-output methods:
> > @@ -282,13 +342,6 @@ extern void lockdep_init_map(struct lockdep_map *lock, const char *name,
> >  			     struct lock_class_key *key, int subclass);
> >  
> >  /*
> > - * To initialize a lockdep_map statically use this macro.
> > - * Note that _name must not be NULL.
> > - */
> > -#define STATIC_LOCKDEP_MAP_INIT(_name, _key) \
> > -	{ .name = (_name), .key = (void *)(_key), }
> > -
> > -/*
> >   * Reinitialize a lock key - for cases where there is special locking or
> >   * special initialization of locks so that the validator gets the scope
> >   * of dependencies wrong: they are either too broad (they need a class-split)
> > @@ -443,6 +496,50 @@ static inline void lockdep_on(void)
> >  
> >  #endif /* !LOCKDEP */
> >  
> > +#ifdef CONFIG_LOCKDEP_CROSSRELEASE
> > +extern void lockdep_init_map_crosslock(struct lockdep_map *lock,
> > +				       const char *name,
> > +				       struct lock_class_key *key,
> > +				       int subclass);
> > +extern void lock_commit_crosslock(struct lockdep_map *lock);
> > +
> > +#define STATIC_CROSS_LOCKDEP_MAP_INIT(_name, _key) \
> > +	{ .map.name = (_name), .map.key = (void *)(_key), \
> > +	  .map.cross = 1, }
> > +
> > +/*
> > + * To initialize a lockdep_map statically use this macro.
> > + * Note that _name must not be NULL.
> > + */
> > +#define STATIC_LOCKDEP_MAP_INIT(_name, _key) \
> > +	{ .name = (_name), .key = (void *)(_key), .cross = 0, }
> > +
> > +extern void crossrelease_hardirq_start(void);
> > +extern void crossrelease_hardirq_end(void);
> > +extern void crossrelease_softirq_start(void);
> > +extern void crossrelease_softirq_end(void);
> > +extern void crossrelease_work_start(void);
> > +extern void crossrelease_work_end(void);
> > +extern void init_crossrelease_task(struct task_struct *task);
> > +extern void free_crossrelease_task(struct task_struct *task);
> > +#else
> > +/*
> > + * To initialize a lockdep_map statically use this macro.
> > + * Note that _name must not be NULL.
> > + */
> > +#define STATIC_LOCKDEP_MAP_INIT(_name, _key) \
> > +	{ .name = (_name), .key = (void *)(_key), }
> > +
> > +static inline void crossrelease_hardirq_start(void) {}
> > +static inline void crossrelease_hardirq_end(void) {}
> > +static inline void crossrelease_softirq_start(void) {}
> > +static inline void crossrelease_softirq_end(void) {}
> > +static inline void crossrelease_work_start(void) {}
> > +static inline void crossrelease_work_end(void) {}
> > +static inline void init_crossrelease_task(struct task_struct *task) {}
> > +static inline void free_crossrelease_task(struct task_struct *task) {}
> > +#endif
> > +
> >  #ifdef CONFIG_LOCK_STAT
> >  
> >  extern void lock_contended(struct lockdep_map *lock, unsigned long ip);
> > diff --git a/include/linux/sched.h b/include/linux/sched.h
> > index e9c009d..5f6d6f4 100644
> > --- a/include/linux/sched.h
> > +++ b/include/linux/sched.h
> > @@ -1749,6 +1749,14 @@ struct task_struct {
> >  	struct held_lock held_locks[MAX_LOCK_DEPTH];
> >  	gfp_t lockdep_reclaim_gfp;
> >  #endif
> > +#ifdef CONFIG_LOCKDEP_CROSSRELEASE
> > +#define MAX_XHLOCKS_NR 64UL
> > +	struct hist_lock *xhlocks; /* Crossrelease history locks */
> > +	unsigned int xhlock_idx;
> > +	unsigned int xhlock_idx_soft; /* For restoring at softirq exit */
> > +	unsigned int xhlock_idx_hard; /* For restoring at hardirq exit */
> > +	unsigned int xhlock_idx_work; /* For restoring at work exit */
> > +#endif
> >  #ifdef CONFIG_UBSAN
> >  	unsigned int in_ubsan;
> >  #endif
> > diff --git a/kernel/exit.c b/kernel/exit.c
> > index 3076f30..cc56aad 100644
> > --- a/kernel/exit.c
> > +++ b/kernel/exit.c
> > @@ -883,6 +883,7 @@ void __noreturn do_exit(long code)
> >  	exit_rcu();
> >  	TASKS_RCU(__srcu_read_unlock(&tasks_rcu_exit_srcu, tasks_rcu_i));
> >  
> > +	free_crossrelease_task(tsk);
> >  	do_task_dead();
> >  }
> >  EXPORT_SYMBOL_GPL(do_exit);
> > diff --git a/kernel/fork.c b/kernel/fork.c
> > index 997ac1d..f9623a0 100644
> > --- a/kernel/fork.c
> > +++ b/kernel/fork.c
> > @@ -451,6 +451,7 @@ void __init fork_init(void)
> >  	for (i = 0; i < UCOUNT_COUNTS; i++) {
> >  		init_user_ns.ucount_max[i] = max_threads/2;
> >  	}
> > +	init_crossrelease_task(&init_task);
> >  }
> >  
> >  int __weak arch_dup_task_struct(struct task_struct *dst,
> > @@ -1611,6 +1612,7 @@ static __latent_entropy struct task_struct *copy_process(
> >  	p->lockdep_depth = 0; /* no locks held yet */
> >  	p->curr_chain_key = 0;
> >  	p->lockdep_recursion = 0;
> > +	init_crossrelease_task(p);
> >  #endif
> >  
> >  #ifdef CONFIG_DEBUG_MUTEXES
> > @@ -1856,6 +1858,7 @@ static __latent_entropy struct task_struct *copy_process(
> >  bad_fork_cleanup_perf:
> >  	perf_event_free_task(p);
> >  bad_fork_cleanup_policy:
> > +	free_crossrelease_task(p);
> >  #ifdef CONFIG_NUMA
> >  	mpol_put(p->mempolicy);
> >  bad_fork_cleanup_threadgroup_lock:
> > diff --git a/kernel/locking/lockdep.c b/kernel/locking/lockdep.c
> > index 2847356..63eb04a 100644
> > --- a/kernel/locking/lockdep.c
> > +++ b/kernel/locking/lockdep.c
> > @@ -55,6 +55,10 @@
> >  #define CREATE_TRACE_POINTS
> >  #include <trace/events/lock.h>
> >  
> > +#ifdef CONFIG_LOCKDEP_CROSSRELEASE
> > +#include <linux/slab.h>
> > +#endif
> > +
> >  #ifdef CONFIG_PROVE_LOCKING
> >  int prove_locking = 1;
> >  module_param(prove_locking, int, 0644);
> > @@ -709,6 +713,18 @@ static int count_matching_names(struct lock_class *new_class)
> >  	return NULL;
> >  }
> >  
> > +#ifdef CONFIG_LOCKDEP_CROSSRELEASE
> > +static void cross_init(struct lockdep_map *lock, int cross);
> > +static int cross_lock(struct lockdep_map *lock);
> > +static int lock_acquire_crosslock(struct held_lock *hlock);
> > +static int lock_release_crosslock(struct lockdep_map *lock);
> > +#else
> > +static inline void cross_init(struct lockdep_map *lock, int cross) {}
> > +static inline int cross_lock(struct lockdep_map *lock) { return 0; }
> > +static inline int lock_acquire_crosslock(struct held_lock *hlock) { return 2; }
> > +static inline int lock_release_crosslock(struct lockdep_map *lock) { return 2; }
> > +#endif
> > +
> >  /*
> >   * Register a lock's class in the hash-table, if the class is not present
> >   * yet. Otherwise we look it up. We cache the result in the lock object
> > @@ -1768,6 +1784,9 @@ static inline void inc_chains(void)
> >  		if (nest)
> >  			return 2;
> >  
> > +		if (cross_lock(prev->instance))
> > +			continue;
> > +
> >  		return print_deadlock_bug(curr, prev, next);
> >  	}
> >  	return 1;
> > @@ -1921,30 +1940,36 @@ static inline void inc_chains(void)
> >  		int distance = curr->lockdep_depth - depth + 1;
> >  		hlock = curr->held_locks + depth - 1;
> >  		/*
> > -		 * Only non-recursive-read entries get new dependencies
> > -		 * added:
> > +		 * Only non-crosslock entries get new dependencies added.
> > +		 * Crosslock entries will be added by commit later:
> >  		 */
> > -		if (hlock->read != 2 && hlock->check) {
> > -			int ret = check_prev_add(curr, hlock, next,
> > -						distance, &trace, save);
> > -			if (!ret)
> > -				return 0;
> > -
> > +		if (!cross_lock(hlock->instance)) {
> >  			/*
> > -			 * Stop saving stack_trace if save_trace() was
> > -			 * called at least once:
> > +			 * Only non-recursive-read entries get new dependencies
> > +			 * added:
> >  			 */
> > -			if (save && ret == 2)
> > -				save = NULL;
> > +			if (hlock->read != 2 && hlock->check) {
> > +				int ret = check_prev_add(curr, hlock, next,
> > +							 distance, &trace, save);
> > +				if (!ret)
> > +					return 0;
> >  
> > -			/*
> > -			 * Stop after the first non-trylock entry,
> > -			 * as non-trylock entries have added their
> > -			 * own direct dependencies already, so this
> > -			 * lock is connected to them indirectly:
> > -			 */
> > -			if (!hlock->trylock)
> > -				break;
> > +				/*
> > +				 * Stop saving stack_trace if save_trace() was
> > +				 * called at least once:
> > +				 */
> > +				if (save && ret == 2)
> > +					save = NULL;
> > +
> > +				/*
> > +				 * Stop after the first non-trylock entry,
> > +				 * as non-trylock entries have added their
> > +				 * own direct dependencies already, so this
> > +				 * lock is connected to them indirectly:
> > +				 */
> > +				if (!hlock->trylock)
> > +					break;
> > +			}
> >  		}
> >  		depth--;
> >  		/*
> > @@ -3203,7 +3228,7 @@ static int mark_lock(struct task_struct *curr, struct held_lock *this,
> >  /*
> >   * Initialize a lock instance's lock-class mapping info:
> >   */
> > -void lockdep_init_map(struct lockdep_map *lock, const char *name,
> > +static void __lockdep_init_map(struct lockdep_map *lock, const char *name,
> >  		      struct lock_class_key *key, int subclass)
> >  {
> >  	int i;
> > @@ -3261,8 +3286,25 @@ void lockdep_init_map(struct lockdep_map *lock, const char *name,
> >  		raw_local_irq_restore(flags);
> >  	}
> >  }
> > +
> > +void lockdep_init_map(struct lockdep_map *lock, const char *name,
> > +		      struct lock_class_key *key, int subclass)
> > +{
> > +	cross_init(lock, 0);
> > +	__lockdep_init_map(lock, name, key, subclass);
> > +}
> >  EXPORT_SYMBOL_GPL(lockdep_init_map);
> >  
> > +#ifdef CONFIG_LOCKDEP_CROSSRELEASE
> > +void lockdep_init_map_crosslock(struct lockdep_map *lock, const char *name,
> > +		      struct lock_class_key *key, int subclass)
> > +{
> > +	cross_init(lock, 1);
> > +	__lockdep_init_map(lock, name, key, subclass);
> > +}
> > +EXPORT_SYMBOL_GPL(lockdep_init_map_crosslock);
> > +#endif
> > +
> >  struct lock_class_key __lockdep_no_validate__;
> >  EXPORT_SYMBOL_GPL(__lockdep_no_validate__);
> >  
> > @@ -3317,6 +3359,7 @@ static int __lock_acquire(struct lockdep_map *lock, unsigned int subclass,
> >  	unsigned int depth;
> >  	int chain_head = 0;
> >  	int class_idx;
> > +	int ret;
> >  	u64 chain_key;
> >  
> >  	if (unlikely(!debug_locks))
> > @@ -3366,7 +3409,8 @@ static int __lock_acquire(struct lockdep_map *lock, unsigned int subclass,
> >  
> >  	class_idx = class - lock_classes + 1;
> >  
> > -	if (depth) {
> > +	/* TODO: nest_lock is not implemented for crosslock yet. */
> > +	if (depth && !cross_lock(lock)) {
> >  		hlock = curr->held_locks + depth - 1;
> >  		if (hlock->class_idx == class_idx && nest_lock) {
> >  			if (hlock->references)
> > @@ -3447,6 +3491,14 @@ static int __lock_acquire(struct lockdep_map *lock, unsigned int subclass,
> >  	if (!validate_chain(curr, lock, hlock, chain_head, chain_key))
> >  		return 0;
> >  
> > +	ret = lock_acquire_crosslock(hlock);
> > +	/*
> > +	 * 2 means normal acquire operations are needed. Otherwise, it's
> > +	 * ok just to return with '0:fail, 1:success'.
> > +	 */
> > +	if (ret != 2)
> > +		return ret;
> > +
> >  	curr->curr_chain_key = chain_key;
> >  	curr->lockdep_depth++;
> >  	check_chain_key(curr);
> > @@ -3610,11 +3662,19 @@ static int match_held_lock(struct held_lock *hlock, struct lockdep_map *lock)
> >  	struct task_struct *curr = current;
> >  	struct held_lock *hlock, *prev_hlock;
> >  	unsigned int depth;
> > -	int i;
> > +	int ret, i;
> >  
> >  	if (unlikely(!debug_locks))
> >  		return 0;
> >  
> > +	ret = lock_release_crosslock(lock);
> > +	/*
> > +	 * 2 means normal release operations are needed. Otherwise, it's
> > +	 * ok just to return with '0:fail, 1:success'.
> > +	 */
> > +	if (ret != 2)
> > +		return ret;
> > +
> >  	depth = curr->lockdep_depth;
> >  	/*
> >  	 * So we're all set to release this lock.. wait what lock? We don't
> > @@ -4557,3 +4617,371 @@ void lockdep_rcu_suspicious(const char *file, const int line, const char *s)
> >  	dump_stack();
> >  }
> >  EXPORT_SYMBOL_GPL(lockdep_rcu_suspicious);
> > +
> > +#ifdef CONFIG_LOCKDEP_CROSSRELEASE
> > +
> > +#define xhlock(i)         (current->xhlocks[(i) % MAX_XHLOCKS_NR])
> > +
> > +/*
> > + * Whenever a crosslock is held, cross_gen_id will be increased.
> > + */
> > +static atomic_t cross_gen_id; /* Can be wrapped */
> > +
> > +void crossrelease_hardirq_start(void)
> > +{
> > +	if (current->xhlocks)
> > +		current->xhlock_idx_hard = current->xhlock_idx;
> > +}
> > +
> > +void crossrelease_hardirq_end(void)
> > +{
> > +	if (current->xhlocks)
> > +		current->xhlock_idx = current->xhlock_idx_hard;
> > +}
> > +
> > +void crossrelease_softirq_start(void)
> > +{
> > +	if (current->xhlocks)
> > +		current->xhlock_idx_soft = current->xhlock_idx;
> > +}
> > +
> > +void crossrelease_softirq_end(void)
> > +{
> > +	if (current->xhlocks)
> > +		current->xhlock_idx = current->xhlock_idx_soft;
> > +}
> > +
> > +/*
> > + * Each work of workqueue might run in a different context,
> > + * thanks to concurrency support of workqueue. So we have to
> > + * distinguish each work to avoid false positive.
> > + */
> > +void crossrelease_work_start(void)
> > +{
> > +	if (current->xhlocks)
> > +		current->xhlock_idx_work = current->xhlock_idx;
> > +}
> > +
> > +void crossrelease_work_end(void)
> > +{
> > +	if (current->xhlocks)
> > +		current->xhlock_idx = current->xhlock_idx_work;
> > +}
> > +
> > +static int cross_lock(struct lockdep_map *lock)
> > +{
> > +	return lock ? lock->cross : 0;
> > +}
> > +
> > +/*
> > + * This is needed to decide the relationship between wrapable variables.
> > + */
> > +static inline int before(unsigned int a, unsigned int b)
> > +{
> > +	return (int)(a - b) < 0;
> > +}
> > +
> > +static inline struct lock_class *xhlock_class(struct hist_lock *xhlock)
> > +{
> > +	return hlock_class(&xhlock->hlock);
> > +}
> > +
> > +static inline struct lock_class *xlock_class(struct cross_lock *xlock)
> > +{
> > +	return hlock_class(&xlock->hlock);
> > +}
> > +
> > +/*
> > + * Should we check a dependency with previous one?
> > + */
> > +static inline int depend_before(struct held_lock *hlock)
> > +{
> > +	return hlock->read != 2 && hlock->check && !hlock->trylock;
> > +}
> > +
> > +/*
> > + * Should we check a dependency with next one?
> > + */
> > +static inline int depend_after(struct held_lock *hlock)
> > +{
> > +	return hlock->read != 2 && hlock->check;
> > +}
> > +
> > +/*
> > + * Check if the xhlock is valid, which would be false if,
> > + *
> > + *    1. Has not used after initializaion yet.
> > + *
> > + * Remind hist_lock is implemented as a ring buffer.
> > + */
> > +static inline int xhlock_valid(struct hist_lock *xhlock)
> > +{
> > +	/*
> > +	 * xhlock->hlock.instance must be !NULL.
> > +	 */
> > +	return !!xhlock->hlock.instance;
> > +}
> > +
> > +/*
> > + * Record a hist_lock entry.
> > + *
> > + * Irq disable is only required.
> > + */
> > +static void add_xhlock(struct held_lock *hlock)
> > +{
> > +	unsigned int idx = ++current->xhlock_idx;
> > +	struct hist_lock *xhlock = &xhlock(idx);
> > +
> > +#ifdef CONFIG_DEBUG_LOCKDEP
> > +	/*
> > +	 * This can be done locklessly because they are all task-local
> > +	 * state, we must however ensure IRQs are disabled.
> > +	 */
> > +	WARN_ON_ONCE(!irqs_disabled());
> > +#endif
> > +
> > +	/* Initialize hist_lock's members */
> > +	xhlock->hlock = *hlock;
> > +
> > +	xhlock->trace.nr_entries = 0;
> > +	xhlock->trace.max_entries = MAX_XHLOCK_TRACE_ENTRIES;
> > +	xhlock->trace.entries = xhlock->trace_entries;
> > +	xhlock->trace.skip = 3;
> > +	save_stack_trace(&xhlock->trace);
> > +}
> > +
> > +static inline int same_context_xhlock(struct hist_lock *xhlock)
> > +{
> > +	return xhlock->hlock.irq_context == task_irq_context(current);
> > +}
> > +
> > +/*
> > + * This should be lockless as far as possible because this would be
> > + * called very frequently.
> > + */
> > +static void check_add_xhlock(struct held_lock *hlock)
> > +{
> > +	/*
> > +	 * Record a hist_lock, only in case that acquisitions ahead
> > +	 * could depend on the held_lock. For example, if the held_lock
> > +	 * is trylock then acquisitions ahead never depends on that.
> > +	 * In that case, we don't need to record it. Just return.
> > +	 */
> > +	if (!current->xhlocks || !depend_before(hlock))
> > +		return;
> > +
> > +	add_xhlock(hlock);
> > +}
> > +
> > +/*
> > + * For crosslock.
> > + */
> > +static int add_xlock(struct held_lock *hlock)
> > +{
> > +	struct cross_lock *xlock;
> > +	unsigned int gen_id;
> > +
> > +	if (!graph_lock())
> > +		return 0;
> > +
> > +	xlock = &((struct lockdep_map_cross *)hlock->instance)->xlock;
> > +
> > +	gen_id = (unsigned int)atomic_inc_return(&cross_gen_id);
> > +	xlock->hlock = *hlock;
> > +	xlock->hlock.gen_id = gen_id;
> > +	graph_unlock();
> > +
> > +	return 1;
> > +}
> > +
> > +/*
> > + * Called for both normal and crosslock acquires. Normal locks will be
> > + * pushed on the hist_lock queue. Cross locks will record state and
> > + * stop regular lock_acquire() to avoid being placed on the held_lock
> > + * stack.
> > + *
> > + * Return: 0 - failure;
> > + *         1 - crosslock, done;
> > + *         2 - normal lock, continue to held_lock[] ops.
> > + */
> > +static int lock_acquire_crosslock(struct held_lock *hlock)
> > +{
> > +	/*
> > +	 *	CONTEXT 1		CONTEXT 2
> > +	 *	---------		---------
> > +	 *	lock A (cross)
> > +	 *	X = atomic_inc_return(&cross_gen_id)
> > +	 *	~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
> > +	 *				Y = atomic_read_acquire(&cross_gen_id)
> > +	 *				lock B
> > +	 *
> > +	 * atomic_read_acquire() is for ordering between A and B,
> > +	 * IOW, A happens before B, when CONTEXT 2 see Y >= X.
> > +	 *
> > +	 * Pairs with atomic_inc_return() in add_xlock().
> > +	 */
> > +	hlock->gen_id = (unsigned int)atomic_read_acquire(&cross_gen_id);
> > +
> > +	if (cross_lock(hlock->instance))
> > +		return add_xlock(hlock);
> > +
> > +	check_add_xhlock(hlock);
> > +	return 2;
> > +}
> > +
> > +static int copy_trace(struct stack_trace *trace)
> > +{
> > +	unsigned long *buf = stack_trace + nr_stack_trace_entries;
> > +	unsigned int max_nr = MAX_STACK_TRACE_ENTRIES - nr_stack_trace_entries;
> > +	unsigned int nr = min(max_nr, trace->nr_entries);
> > +
> > +	trace->nr_entries = nr;
> > +	memcpy(buf, trace->entries, nr * sizeof(trace->entries[0]));
> > +	trace->entries = buf;
> > +	nr_stack_trace_entries += nr;
> > +
> > +	if (nr_stack_trace_entries >= MAX_STACK_TRACE_ENTRIES-1) {
> > +		if (!debug_locks_off_graph_unlock())
> > +			return 0;
> > +
> > +		print_lockdep_off("BUG: MAX_STACK_TRACE_ENTRIES too low!");
> > +		dump_stack();
> > +
> > +		return 0;
> > +	}
> > +
> > +	return 1;
> > +}
> > +
> > +static int commit_xhlock(struct cross_lock *xlock, struct hist_lock *xhlock)
> > +{
> > +	unsigned int xid, pid;
> > +	u64 chain_key;
> > +
> > +	xid = xlock_class(xlock) - lock_classes;
> > +	chain_key = iterate_chain_key((u64)0, xid);
> > +	pid = xhlock_class(xhlock) - lock_classes;
> > +	chain_key = iterate_chain_key(chain_key, pid);
> > +
> > +	if (lookup_chain_cache(chain_key))
> > +		return 1;
> > +
> > +	if (!add_chain_cache_classes(xid, pid, xhlock->hlock.irq_context,
> > +				chain_key))
> > +		return 0;
> > +
> > +	if (!check_prev_add(current, &xlock->hlock, &xhlock->hlock, 1,
> > +			    &xhlock->trace, copy_trace))
> > +		return 0;
> > +
> > +	return 1;
> > +}
> > +
> > +static void commit_xhlocks(struct cross_lock *xlock)
> > +{
> > +	unsigned int cur = current->xhlock_idx;
> > +	unsigned int i;
> > +
> > +	if (!graph_lock())
> > +		return;
> > +
> > +	for (i = 0; i < MAX_XHLOCKS_NR; i++) {
> > +		struct hist_lock *xhlock = &xhlock(cur - i);
> > +
> > +		if (!xhlock_valid(xhlock))
> > +			break;
> > +
> > +		if (before(xhlock->hlock.gen_id, xlock->hlock.gen_id))
> > +			break;
> > +
> > +		if (!same_context_xhlock(xhlock))
> > +			break;
> > +
> > +		/*
> > +		 * commit_xhlock() returns 0 with graph_lock already
> > +		 * released if fail.
> > +		 */
> > +		if (!commit_xhlock(xlock, xhlock))
> > +			return;
> > +	}
> > +
> > +	graph_unlock();
> > +}
> > +
> > +void lock_commit_crosslock(struct lockdep_map *lock)
> > +{
> > +	struct cross_lock *xlock;
> > +	unsigned long flags;
> > +
> > +	if (unlikely(!debug_locks || current->lockdep_recursion))
> > +		return;
> > +
> > +	if (!current->xhlocks)
> > +		return;
> > +
> > +	/*
> > +	 * Do commit hist_locks with the cross_lock, only in case that
> > +	 * the cross_lock could depend on acquisitions after that.
> > +	 *
> > +	 * For example, if the cross_lock does not have the 'check' flag
> > +	 * then we don't need to check dependencies and commit for that.
> > +	 * Just skip it. In that case, of course, the cross_lock does
> > +	 * not depend on acquisitions ahead, either.
> > +	 *
> > +	 * WARNING: Don't do that in add_xlock() in advance. When an
> > +	 * acquisition context is different from the commit context,
> > +	 * invalid(skipped) cross_lock might be accessed.
> > +	 */
> > +	if (!depend_after(&((struct lockdep_map_cross *)lock)->xlock.hlock))
> > +		return;
> > +
> > +	raw_local_irq_save(flags);
> > +	check_flags(flags);
> > +	current->lockdep_recursion = 1;
> > +	xlock = &((struct lockdep_map_cross *)lock)->xlock;
> > +	commit_xhlocks(xlock);
> > +	current->lockdep_recursion = 0;
> > +	raw_local_irq_restore(flags);
> > +}
> > +EXPORT_SYMBOL_GPL(lock_commit_crosslock);
> > +
> > +/*
> > + * Return: 1 - crosslock, done;
> > + *         2 - normal lock, continue to held_lock[] ops.
> > + */
> > +static int lock_release_crosslock(struct lockdep_map *lock)
> > +{
> > +	return cross_lock(lock) ? 1 : 2;
> > +}
> > +
> > +static void cross_init(struct lockdep_map *lock, int cross)
> > +{
> > +	lock->cross = cross;
> > +
> > +	/*
> > +	 * Crossrelease assumes that the ring buffer size of xhlocks
> > +	 * is aligned with power of 2. So force it on build.
> > +	 */
> > +	BUILD_BUG_ON(MAX_XHLOCKS_NR & (MAX_XHLOCKS_NR - 1));
> > +}
> > +
> > +void init_crossrelease_task(struct task_struct *task)
> > +{
> > +	task->xhlock_idx = UINT_MAX;
> > +	task->xhlock_idx_soft = UINT_MAX;
> > +	task->xhlock_idx_hard = UINT_MAX;
> > +	task->xhlock_idx_work = UINT_MAX;
> > +	task->xhlocks = kzalloc(sizeof(struct hist_lock) * MAX_XHLOCKS_NR,
> > +				GFP_KERNEL);
> > +}
> > +
> > +void free_crossrelease_task(struct task_struct *task)
> > +{
> > +	if (task->xhlocks) {
> > +		void *tmp = task->xhlocks;
> > +		/* Diable crossrelease for current */
> > +		task->xhlocks = NULL;
> > +		kfree(tmp);
> > +	}
> > +}
> > +#endif
> > diff --git a/kernel/workqueue.c b/kernel/workqueue.c
> > index 479d840..2f43ac1 100644
> > --- a/kernel/workqueue.c
> > +++ b/kernel/workqueue.c
> > @@ -2092,6 +2092,7 @@ static void process_one_work(struct worker *worker, struct work_struct *work)
> >  
> >  	lock_map_acquire_read(&pwq->wq->lockdep_map);
> >  	lock_map_acquire(&lockdep_map);
> > +	crossrelease_work_start();
> >  	trace_workqueue_execute_start(work);
> >  	worker->current_func(work);
> >  	/*
> > @@ -2099,6 +2100,7 @@ static void process_one_work(struct worker *worker, struct work_struct *work)
> >  	 * point will only record its address.
> >  	 */
> >  	trace_workqueue_execute_end(work);
> > +	crossrelease_work_end();
> >  	lock_map_release(&lockdep_map);
> >  	lock_map_release(&pwq->wq->lockdep_map);
> >  
> > diff --git a/lib/Kconfig.debug b/lib/Kconfig.debug
> > index a6c8db1..e584431 100644
> > --- a/lib/Kconfig.debug
> > +++ b/lib/Kconfig.debug
> > @@ -1042,6 +1042,18 @@ config DEBUG_LOCK_ALLOC
> >  	 spin_lock_init()/mutex_init()/etc., or whether there is any lock
> >  	 held during task exit.
> >  
> > +config LOCKDEP_CROSSRELEASE
> > +	bool "Lock debugging: make lockdep work for crosslocks"
> > +	depends on PROVE_LOCKING
> > +	default n
> > +	help
> > +	 This makes lockdep work for crosslock which is a lock allowed to
> > +	 be released in a different context from the acquisition context.
> > +	 Normally a lock must be released in the context acquiring the lock.
> > +	 However, relexing this constraint helps synchronization primitives
> > +	 such as page locks or completions can use the lock correctness
> > +	 detector, lockdep.
> > +
> >  config PROVE_LOCKING
> >  	bool "Lock debugging: prove locking correctness"
> >  	depends on DEBUG_KERNEL && TRACE_IRQFLAGS_SUPPORT && STACKTRACE_SUPPORT && LOCKDEP_SUPPORT
> > -- 
> > 1.9.1

Powered by blists - more mailing lists