Put inode counters under locks to reduce atomics. Signed-off-by: Nick Piggin --- fs/fs-writeback.c | 6 ++---- fs/inode.c | 30 ++++++++++++------------------ include/linux/fs.h | 4 ++-- 3 files changed, 16 insertions(+), 24 deletions(-) Index: linux-2.6/fs/inode.c =================================================================== --- linux-2.6.orig/fs/inode.c +++ linux-2.6/fs/inode.c @@ -141,8 +141,8 @@ static DECLARE_RWSEM(iprune_sem); * Statistics gathering.. */ struct inodes_stat_t inodes_stat = { - .nr_inodes = ATOMIC_INIT(0), - .nr_unused = ATOMIC_INIT(0), + .nr_inodes = 0, + .nr_unused = 0, }; static struct kmem_cache *inode_cachep __read_mostly; @@ -390,7 +390,6 @@ static void dispose_list(struct list_hea destroy_inode(inode); nr_disposed++; } - atomic_sub(nr_disposed, &inodes_stat.nr_inodes); } /* @@ -399,7 +398,7 @@ static void dispose_list(struct list_hea static int invalidate_list(struct list_head *head, struct list_head *dispose) { struct list_head *next; - int busy = 0, count = 0; + int busy = 0; next = head->next; for (;;) { @@ -419,19 +418,17 @@ static int invalidate_list(struct list_h if (!inode->i_count) { spin_lock(&wb_inode_list_lock); list_del(&inode->i_list); + inodes_stat.nr_unused--; spin_unlock(&wb_inode_list_lock); WARN_ON(inode->i_state & I_NEW); inode->i_state |= I_FREEING; spin_unlock(&inode->i_lock); list_add(&inode->i_list, dispose); - count++; continue; } spin_unlock(&inode->i_lock); busy = 1; } - /* only unused inodes may be cached with i_count zero */ - atomic_sub(count, &inodes_stat.nr_unused); return busy; } @@ -483,7 +480,6 @@ EXPORT_SYMBOL(invalidate_inodes); static void prune_icache(int nr_to_scan) { LIST_HEAD(freeable); - int nr_pruned = 0; unsigned long reap = 0; down_read(&iprune_sem); @@ -504,7 +500,7 @@ again: if (inode->i_count || (inode->i_state & ~I_REFERENCED)) { list_del_init(&inode->i_list); spin_unlock(&inode->i_lock); - atomic_dec(&inodes_stat.nr_unused); + inodes_stat.nr_unused--; continue; } if (inode->i_state) { @@ -530,9 +526,8 @@ again: WARN_ON(inode->i_state & I_NEW); inode->i_state |= I_FREEING; spin_unlock(&inode->i_lock); - nr_pruned++; + inodes_stat.nr_unused--; } - atomic_sub(nr_pruned, &inodes_stat.nr_unused); if (current_is_kswapd()) __count_vm_events(KSWAPD_INODESTEAL, reap); else @@ -564,8 +559,7 @@ static int shrink_icache_memory(int nr, return -1; prune_icache(nr); } - return (atomic_read(&inodes_stat.nr_unused) / 100) * - sysctl_vfs_cache_pressure; + return inodes_stat.nr_unused / 100 * sysctl_vfs_cache_pressure; } static struct shrinker icache_shrinker = { @@ -661,9 +655,9 @@ static inline void __inode_add_to_lists(struct super_block *sb, struct inode_hash_bucket *b, struct inode *inode) { - atomic_inc(&inodes_stat.nr_inodes); spin_lock(&sb_inode_list_lock); list_add_rcu(&inode->i_sb_list, &sb->s_inodes); + inodes_stat.nr_inodes++; spin_unlock(&sb_inode_list_lock); if (b) { spin_lock_bucket(b); @@ -1309,17 +1303,17 @@ void generic_delete_inode(struct inode * if (!list_empty(&inode->i_list)) { spin_lock(&wb_inode_list_lock); list_del_init(&inode->i_list); - spin_unlock(&wb_inode_list_lock); if (!inode->i_state) - atomic_dec(&inodes_stat.nr_unused); + inodes_stat.nr_unused--; + spin_unlock(&wb_inode_list_lock); } spin_lock(&sb_inode_list_lock); list_del_rcu(&inode->i_sb_list); + inodes_stat.nr_inodes--; spin_unlock(&sb_inode_list_lock); WARN_ON(inode->i_state & I_NEW); inode->i_state |= I_FREEING; spin_unlock(&inode->i_lock); - atomic_dec(&inodes_stat.nr_inodes); if (op->delete_inode) { void (*delete)(struct inode *) = op->delete_inode; @@ -1367,8 +1361,8 @@ int generic_detach_inode(struct inode *i list_empty(&inode->i_list)) { spin_lock(&wb_inode_list_lock); list_add(&inode->i_list, &inode_unused); + inodes_stat.nr_unused++; spin_unlock(&wb_inode_list_lock); - atomic_inc(&inodes_stat.nr_unused); } spin_unlock(&inode->i_lock); return 0; @@ -1385,17 +1379,17 @@ int generic_detach_inode(struct inode *i if (!list_empty(&inode->i_list)) { spin_lock(&wb_inode_list_lock); list_del_init(&inode->i_list); - spin_unlock(&wb_inode_list_lock); if (!inode->i_state) - atomic_dec(&inodes_stat.nr_unused); + inodes_stat.nr_unused--; + spin_unlock(&wb_inode_list_lock); } spin_lock(&sb_inode_list_lock); list_del_rcu(&inode->i_sb_list); + inodes_stat.nr_inodes--; spin_unlock(&sb_inode_list_lock); WARN_ON(inode->i_state & I_NEW); inode->i_state |= I_FREEING; spin_unlock(&inode->i_lock); - atomic_dec(&inodes_stat.nr_inodes); return 1; } EXPORT_SYMBOL_GPL(generic_detach_inode); Index: linux-2.6/include/linux/fs.h =================================================================== --- linux-2.6.orig/include/linux/fs.h +++ linux-2.6/include/linux/fs.h @@ -31,6 +31,12 @@ #define SEEK_END 2 /* seek relative to end of file */ #define SEEK_MAX SEEK_END +struct inodes_stat_t { + int nr_inodes; + int nr_unused; + int dummy[5]; /* padding for sysctl ABI compatibility */ +}; + /* And dynamically-tunable limits and defaults: */ struct files_stat_struct { int nr_files; /* read only */ @@ -411,12 +417,6 @@ typedef int (get_block_t)(struct inode * typedef void (dio_iodone_t)(struct kiocb *iocb, loff_t offset, ssize_t bytes, void *private); -struct inodes_stat_t { - atomic_t nr_inodes; - atomic_t nr_unused; - int dummy[5]; /* padding for sysctl ABI compatibility */ -}; - /* * Attribute flags. These should be or-ed together to figure out what * has been changed! Index: linux-2.6/fs/fs-writeback.c =================================================================== --- linux-2.6.orig/fs/fs-writeback.c +++ linux-2.6/fs/fs-writeback.c @@ -904,8 +904,7 @@ static long wb_check_old_data_flush(stru wb->last_old_flush = jiffies; nr_pages = global_page_state(NR_FILE_DIRTY) + global_page_state(NR_UNSTABLE_NFS) + - (atomic_read(&inodes_stat.nr_inodes) - - atomic_read(&inodes_stat.nr_unused)); + inodes_stat.nr_inodes - inodes_stat.nr_unused; if (nr_pages) { struct wb_writeback_args args = { @@ -1258,8 +1257,7 @@ void writeback_inodes_sb(struct super_bl long nr_to_write; nr_to_write = nr_dirty + nr_unstable + - (atomic_read(&inodes_stat.nr_inodes) - - atomic_read(&inodes_stat.nr_unused)); + inodes_stat.nr_inodes - inodes_stat.nr_unused; bdi_start_writeback(sb->s_bdi, sb, nr_to_write); } -- To unsubscribe from this list: send the line "unsubscribe linux-kernel" in the body of a message to majordomo@vger.kernel.org More majordomo info at http://vger.kernel.org/majordomo-info.html Please read the FAQ at http://www.tux.org/lkml/