Split inode reclaim and writeback lists in preparation to scale them up (per-bdi locking for i_io and per-zone locking for i_lru) Signed-off-by: Nick Piggin -- Index: linux-2.6/fs/fs-writeback.c =================================================================== --- linux-2.6.orig/fs/fs-writeback.c +++ linux-2.6/fs/fs-writeback.c @@ -291,11 +291,11 @@ static void redirty_tail(struct inode *i if (!list_empty(&wb->b_dirty)) { struct inode *tail; - tail = list_entry(wb->b_dirty.next, struct inode, i_list); + tail = list_entry(wb->b_dirty.next, struct inode, i_io); if (time_before(inode->dirtied_when, tail->dirtied_when)) inode->dirtied_when = jiffies; } - list_move(&inode->i_list, &wb->b_dirty); + list_move(&inode->i_io, &wb->b_dirty); } /* @@ -306,7 +306,7 @@ static void requeue_io(struct inode *ino struct bdi_writeback *wb = &inode_to_bdi(inode)->wb; assert_spin_locked(&wb_inode_list_lock); - list_move(&inode->i_list, &wb->b_more_io); + list_move(&inode->i_io, &wb->b_more_io); } static void inode_sync_complete(struct inode *inode) @@ -348,14 +348,14 @@ static void move_expired_inodes(struct l assert_spin_locked(&wb_inode_list_lock); while (!list_empty(delaying_queue)) { - inode = list_entry(delaying_queue->prev, struct inode, i_list); + inode = list_entry(delaying_queue->prev, struct inode, i_io); if (older_than_this && inode_dirtied_after(inode, *older_than_this)) break; if (sb && sb != inode->i_sb) do_sb_sort = 1; sb = inode->i_sb; - list_move(&inode->i_list, &tmp); + list_move(&inode->i_io, &tmp); } /* just one sb in list, splice to dispatch_queue and we're done */ @@ -366,12 +366,12 @@ static void move_expired_inodes(struct l /* Move inodes from one superblock together */ while (!list_empty(&tmp)) { - inode = list_entry(tmp.prev, struct inode, i_list); + inode = list_entry(tmp.prev, struct inode, i_io); sb = inode->i_sb; list_for_each_prev_safe(pos, node, &tmp) { - inode = list_entry(pos, struct inode, i_list); + inode = list_entry(pos, struct inode, i_io); if (inode->i_sb == sb) - list_move(&inode->i_list, dispatch_queue); + list_move(&inode->i_io, dispatch_queue); } } } @@ -556,7 +556,11 @@ select_queue: } } else { /* The inode is clean */ - list_move(&inode->i_list, &inode_unused); + list_del_init(&inode->i_io); + if (list_empty(&inode->i_lru)) { + list_add(&inode->i_lru, &inode_unused); + inodes_stat.nr_unused++; + } } } inode_sync_complete(inode); @@ -623,7 +627,7 @@ again: while (!list_empty(&wb->b_io)) { long pages_skipped; struct inode *inode = list_entry(wb->b_io.prev, - struct inode, i_list); + struct inode, i_io); if (!spin_trylock(&inode->i_lock)) { spin_unlock(&wb_inode_list_lock); spin_lock(&wb_inode_list_lock); @@ -696,7 +700,7 @@ again: while (!list_empty(&wb->b_io)) { struct inode *inode = list_entry(wb->b_io.prev, - struct inode, i_list); + struct inode, i_io); struct super_block *sb = inode->i_sb; enum sb_pin_state state; @@ -845,7 +849,7 @@ retry: spin_lock(&wb_inode_list_lock); if (!list_empty(&wb->b_more_io)) { inode = list_entry(wb->b_more_io.prev, - struct inode, i_list); + struct inode, i_io); if (!spin_trylock(&inode->i_lock)) { spin_unlock(&wb_inode_list_lock); goto retry; @@ -1164,7 +1168,7 @@ void __mark_inode_dirty(struct inode *in inode->dirtied_when = jiffies; spin_lock(&wb_inode_list_lock); - list_move(&inode->i_list, &wb->b_dirty); + list_move(&inode->i_io, &wb->b_dirty); spin_unlock(&wb_inode_list_lock); } } Index: linux-2.6/include/linux/fs.h =================================================================== --- linux-2.6.orig/include/linux/fs.h +++ linux-2.6/include/linux/fs.h @@ -726,7 +726,8 @@ struct posix_acl; struct inode { struct hlist_bl_node i_hash; - struct list_head i_list; /* backing dev IO list */ + struct list_head i_io; /* backing dev IO list */ + struct list_head i_lru; struct list_head i_sb_list; union { struct list_head i_dentry; Index: linux-2.6/mm/backing-dev.c =================================================================== --- linux-2.6.orig/mm/backing-dev.c +++ linux-2.6/mm/backing-dev.c @@ -80,11 +80,11 @@ static int bdi_debug_stats_show(struct s spin_lock(&wb_inode_list_lock); list_for_each_entry(wb, &bdi->wb_list, list) { nr_wb++; - list_for_each_entry(inode, &wb->b_dirty, i_list) + list_for_each_entry(inode, &wb->b_dirty, i_io) nr_dirty++; - list_for_each_entry(inode, &wb->b_io, i_list) + list_for_each_entry(inode, &wb->b_io, i_io) nr_io++; - list_for_each_entry(inode, &wb->b_more_io, i_list) + list_for_each_entry(inode, &wb->b_more_io, i_io) nr_more_io++; } spin_unlock(&wb_inode_list_lock); Index: linux-2.6/fs/inode.c =================================================================== --- linux-2.6.orig/fs/inode.c +++ linux-2.6/fs/inode.c @@ -35,12 +35,13 @@ * inode_hash_bucket lock protects: * inode hash table, i_hash * wb_inode_list_lock protects: - * inode_in_use, inode_unused, b_io, b_more_io, b_dirty, i_list + * inode_in_use, inode_unused, b_io, b_more_io, b_dirty, i_io, i_lru * inode->i_lock protects: * i_state * i_count * i_hash - * i_list + * i_io + * i_lru * i_sb_list * * Ordering: @@ -313,6 +314,7 @@ static void i_callback(struct rcu_head * void destroy_inode(struct inode *inode) { + BUG_ON(!list_empty(&inode->i_io)); __destroy_inode(inode); if (inode->i_sb->s_op->destroy_inode) inode->i_sb->s_op->destroy_inode(inode); @@ -331,7 +333,8 @@ void inode_init_once(struct inode *inode INIT_HLIST_BL_NODE(&inode->i_hash); INIT_LIST_HEAD(&inode->i_dentry); INIT_LIST_HEAD(&inode->i_devices); - INIT_LIST_HEAD(&inode->i_list); + INIT_LIST_HEAD(&inode->i_io); + INIT_LIST_HEAD(&inode->i_lru); INIT_RADIX_TREE(&inode->i_data.page_tree, GFP_ATOMIC); spin_lock_init(&inode->i_data.tree_lock); spin_lock_init(&inode->i_data.i_mmap_lock); @@ -401,8 +404,8 @@ static void dispose_list(struct list_hea while (!list_empty(head)) { struct inode *inode; - inode = list_first_entry(head, struct inode, i_list); - list_del_init(&inode->i_list); + inode = list_first_entry(head, struct inode, i_lru); + list_del_init(&inode->i_lru); if (inode->i_data.nrpages) truncate_inode_pages(&inode->i_data, 0); @@ -436,13 +439,14 @@ static int invalidate_sb_inodes(struct s invalidate_inode_buffers(inode); if (!inode->i_count) { spin_lock(&wb_inode_list_lock); - list_del(&inode->i_list); + list_del_init(&inode->i_io); + list_del(&inode->i_lru); inodes_stat.nr_unused--; spin_unlock(&wb_inode_list_lock); WARN_ON(inode->i_state & I_NEW); inode->i_state |= I_FREEING; spin_unlock(&inode->i_lock); - list_add(&inode->i_list, dispose); + list_add(&inode->i_lru, dispose); continue; } spin_unlock(&inode->i_lock); @@ -511,26 +515,26 @@ again: if (list_empty(&inode_unused)) break; - inode = list_entry(inode_unused.prev, struct inode, i_list); + inode = list_entry(inode_unused.prev, struct inode, i_lru); if (!spin_trylock(&inode->i_lock)) { spin_unlock(&wb_inode_list_lock); goto again; } if (inode->i_count || (inode->i_state & ~I_REFERENCED)) { - list_del_init(&inode->i_list); + list_del_init(&inode->i_lru); spin_unlock(&inode->i_lock); inodes_stat.nr_unused--; continue; } if (inode->i_state) { - list_move(&inode->i_list, &inode_unused); + list_move(&inode->i_lru, &inode_unused); inode->i_state &= ~I_REFERENCED; spin_unlock(&inode->i_lock); continue; } if (inode_has_buffers(inode) || inode->i_data.nrpages) { - list_move(&inode->i_list, &inode_unused); + list_move(&inode->i_lru, &inode_unused); spin_unlock(&wb_inode_list_lock); __iget(inode); spin_unlock(&inode->i_lock); @@ -542,7 +546,7 @@ again: spin_lock(&wb_inode_list_lock); continue; } - list_move(&inode->i_list, &freeable); + list_move(&inode->i_lru, &freeable); WARN_ON(inode->i_state & I_NEW); inode->i_state |= I_FREEING; spin_unlock(&inode->i_lock); @@ -1395,11 +1399,15 @@ void generic_delete_inode(struct inode * { const struct super_operations *op = inode->i_sb->s_op; - if (!list_empty(&inode->i_list)) { + if (!list_empty(&inode->i_lru)) { spin_lock(&wb_inode_list_lock); - list_del_init(&inode->i_list); - if (!inode->i_state) - inodes_stat.nr_unused--; + list_del_init(&inode->i_lru); + inodes_stat.nr_unused--; + spin_unlock(&wb_inode_list_lock); + } + if (!list_empty(&inode->i_io)) { + spin_lock(&wb_inode_list_lock); + list_del_init(&inode->i_io); spin_unlock(&wb_inode_list_lock); } inode_sb_list_del(inode); @@ -1451,9 +1459,9 @@ int generic_detach_inode(struct inode *i if (sb->s_flags & MS_ACTIVE) { inode->i_state |= I_REFERENCED; if (!(inode->i_state & (I_DIRTY|I_SYNC)) && - list_empty(&inode->i_list)) { + list_empty(&inode->i_lru)) { spin_lock(&wb_inode_list_lock); - list_add(&inode->i_list, &inode_unused); + list_add(&inode->i_lru, &inode_unused); inodes_stat.nr_unused++; spin_unlock(&wb_inode_list_lock); } @@ -1469,11 +1477,15 @@ int generic_detach_inode(struct inode *i inode->i_state &= ~I_WILL_FREE; __remove_inode_hash(inode); } - if (!list_empty(&inode->i_list)) { + if (!list_empty(&inode->i_lru)) { spin_lock(&wb_inode_list_lock); - list_del_init(&inode->i_list); - if (!inode->i_state) - inodes_stat.nr_unused--; + list_del_init(&inode->i_lru); + inodes_stat.nr_unused--; + spin_unlock(&wb_inode_list_lock); + } + if (!list_empty(&inode->i_io)) { + spin_lock(&wb_inode_list_lock); + list_del_init(&inode->i_io); spin_unlock(&wb_inode_list_lock); } inode_sb_list_del(inode); -- To unsubscribe from this list: send the line "unsubscribe linux-kernel" in the body of a message to majordomo@vger.kernel.org More majordomo info at http://vger.kernel.org/majordomo-info.html Please read the FAQ at http://www.tux.org/lkml/