As Oleg suggested, replace file_lock_list with a structure containing the hlist head and a spinlock. This completely removes the lglock from fs/locks. Cc: Al Viro Cc: oleg@redhat.com Cc: paulmck@linux.vnet.ibm.com Cc: tj@kernel.org Cc: mingo@redhat.com Cc: der.herr@hofr.at Cc: dave@stgolabs.net Cc: riel@redhat.com Cc: torvalds@linux-foundation.org Suggested-by: Oleg Nesterov Signed-off-by: Peter Zijlstra (Intel) --- fs/Kconfig | 1 + fs/locks.c | 47 +++++++++++++++++++++++++++++------------------ 2 files changed, 30 insertions(+), 18 deletions(-) --- a/fs/Kconfig +++ b/fs/Kconfig @@ -62,6 +62,7 @@ config EXPORTFS config FILE_LOCKING bool "Enable POSIX file locking API" if EXPERT default y + select PERCPU_RWSEM help This option enables standard file locking support, required for filesystems like NFS and for the flock() system --- a/fs/locks.c +++ b/fs/locks.c @@ -128,7 +128,6 @@ #include #include #include -#include #define CREATE_TRACE_POINTS #include @@ -159,12 +158,17 @@ int lease_break_time = 45; /* * The global file_lock_list is only used for displaying /proc/locks, so we - * keep a list on each CPU, with each list protected by its own spinlock via - * the file_lock_lglock. Note that alterations to the list also require that - * the relevant flc_lock is held. + * keep a list on each CPU, with each list protected by its own spinlock. + * Global serialization is done using file_rwsem. + * + * Note that alterations to the list also require that the relevant flc_lock is + * held. */ -DEFINE_STATIC_LGLOCK(file_lock_lglock); -static DEFINE_PER_CPU(struct hlist_head, file_lock_list); +struct file_lock_list_struct { + spinlock_t lock; + struct hlist_head hlist; +}; +static DEFINE_PER_CPU(struct file_lock_list_struct, file_lock_list); DEFINE_STATIC_PERCPU_RWSEM(file_rwsem); /* @@ -557,17 +561,21 @@ static int posix_same_owner(struct file_ /* Must be called with the flc_lock held! */ static void locks_insert_global_locks(struct file_lock *fl) { + struct file_lock_list_struct *fll = this_cpu_ptr(&file_lock_list); + percpu_rwsem_assert_held(&file_rwsem); - lg_local_lock(&file_lock_lglock); + spin_lock(&fll->lock); fl->fl_link_cpu = smp_processor_id(); - hlist_add_head(&fl->fl_link, this_cpu_ptr(&file_lock_list)); - lg_local_unlock(&file_lock_lglock); + hlist_add_head(&fl->fl_link, &fll->hlist); + spin_unlock(&fll->lock); } /* Must be called with the flc_lock held! */ static void locks_delete_global_locks(struct file_lock *fl) { + struct file_lock_list_struct *fll; + percpu_rwsem_assert_held(&file_rwsem); /* @@ -577,9 +585,11 @@ static void locks_delete_global_locks(st */ if (hlist_unhashed(&fl->fl_link)) return; - lg_local_lock_cpu(&file_lock_lglock, fl->fl_link_cpu); + + fll = per_cpu_ptr(&file_lock_list, fl->fl_link_cpu); + spin_lock(&fll->lock); hlist_del_init(&fl->fl_link); - lg_local_unlock_cpu(&file_lock_lglock, fl->fl_link_cpu); + spin_unlock(&fll->lock); } static unsigned long @@ -2653,9 +2663,8 @@ static void *locks_start(struct seq_file iter->li_pos = *pos + 1; percpu_down_write(&file_rwsem); - lg_global_lock(&file_lock_lglock); spin_lock(&blocked_lock_lock); - return seq_hlist_start_percpu(&file_lock_list, &iter->li_cpu, *pos); + return seq_hlist_start_percpu(&file_lock_list.hlist, &iter->li_cpu, *pos); } static void *locks_next(struct seq_file *f, void *v, loff_t *pos) @@ -2663,14 +2672,13 @@ static void *locks_next(struct seq_file struct locks_iterator *iter = f->private; ++iter->li_pos; - return seq_hlist_next_percpu(v, &file_lock_list, &iter->li_cpu, pos); + return seq_hlist_next_percpu(v, &file_lock_list.hlist, &iter->li_cpu, pos); } static void locks_stop(struct seq_file *f, void *v) __releases(&blocked_lock_lock) { spin_unlock(&blocked_lock_lock); - lg_global_unlock(&file_lock_lglock); percpu_up_write(&file_rwsem); } @@ -2712,10 +2720,13 @@ static int __init filelock_init(void) filelock_cache = kmem_cache_create("file_lock_cache", sizeof(struct file_lock), 0, SLAB_PANIC, NULL); - lg_lock_init(&file_lock_lglock, "file_lock_lglock"); - for_each_possible_cpu(i) - INIT_HLIST_HEAD(per_cpu_ptr(&file_lock_list, i)); + for_each_possible_cpu(i) { + struct file_lock_list_struct *fll = per_cpu_ptr(&file_lock_list, i); + + spin_lock_init(&fll->lock); + INIT_HLIST_HEAD(&fll->hlist); + } return 0; }