Add migrate_misplaced_page() which deals with migrating pages from faults. This includes adding a new MIGRATE_FAULT migration mode to deal with the extra page reference required due to having to look up the page. Based-on-work-by: Lee Schermerhorn Cc: Paul Turner Signed-off-by: Peter Zijlstra --- include/linux/migrate.h | 7 +++ include/linux/migrate_mode.h | 3 + mm/migrate.c | 85 ++++++++++++++++++++++++++++++++++++++----- 3 files changed, 87 insertions(+), 8 deletions(-) --- a/include/linux/migrate.h +++ b/include/linux/migrate.h @@ -30,6 +30,7 @@ extern int migrate_vmas(struct mm_struct extern void migrate_page_copy(struct page *newpage, struct page *page); extern int migrate_huge_page_move_mapping(struct address_space *mapping, struct page *newpage, struct page *page); +extern int migrate_misplaced_page(struct mm_struct *, struct page *, int); #else static inline void putback_lru_pages(struct list_head *l) {} @@ -63,5 +64,11 @@ static inline int migrate_huge_page_move #define migrate_page NULL #define fail_migrate_page NULL +static inline +int migrate_misplaced_page(struct mm_struct *mm, struct page *page, int node) +{ + return -EAGAIN; /* can't migrate now */ +} #endif /* CONFIG_MIGRATION */ + #endif /* _LINUX_MIGRATE_H */ --- a/include/linux/migrate_mode.h +++ b/include/linux/migrate_mode.h @@ -6,11 +6,14 @@ * on most operations but not ->writepage as the potential stall time * is too significant * MIGRATE_SYNC will block when migrating pages + * MIGRATE_FAULT called from the fault path to migrate-on-fault for mempolicy + * this path has an extra reference count */ enum migrate_mode { MIGRATE_ASYNC, MIGRATE_SYNC_LIGHT, MIGRATE_SYNC, + MIGRATE_FAULT, }; #endif /* MIGRATE_MODE_H_INCLUDED */ --- a/mm/migrate.c +++ b/mm/migrate.c @@ -224,7 +224,7 @@ static bool buffer_migrate_lock_buffers( struct buffer_head *bh = head; /* Simple case, sync compaction */ - if (mode != MIGRATE_ASYNC) { + if (mode != MIGRATE_ASYNC && mode != MIGRATE_FAULT) { do { get_bh(bh); lock_buffer(bh); @@ -278,12 +278,22 @@ static int migrate_page_move_mapping(str struct page *newpage, struct page *page, struct buffer_head *head, enum migrate_mode mode) { - int expected_count; + int expected_count = 0; void **pslot; + if (mode == MIGRATE_FAULT) { + /* + * MIGRATE_FAULT has an extra reference on the page and + * otherwise acts like ASYNC, no point in delaying the + * fault, we'll try again next time. + */ + expected_count++; + } + if (!mapping) { /* Anonymous page without mapping */ - if (page_count(page) != 1) + expected_count += 1; + if (page_count(page) != expected_count) return -EAGAIN; return 0; } @@ -293,7 +303,7 @@ static int migrate_page_move_mapping(str pslot = radix_tree_lookup_slot(&mapping->page_tree, page_index(page)); - expected_count = 2 + page_has_private(page); + expected_count += 2 + page_has_private(page); if (page_count(page) != expected_count || radix_tree_deref_slot_protected(pslot, &mapping->tree_lock) != page) { spin_unlock_irq(&mapping->tree_lock); @@ -312,7 +322,7 @@ static int migrate_page_move_mapping(str * the mapping back due to an elevated page count, we would have to * block waiting on other references to be dropped. */ - if (mode == MIGRATE_ASYNC && head && + if ((mode == MIGRATE_ASYNC || mode == MIGRATE_FAULT) && head && !buffer_migrate_lock_buffers(head, mode)) { page_unfreeze_refs(page, expected_count); spin_unlock_irq(&mapping->tree_lock); @@ -520,7 +530,7 @@ int buffer_migrate_page(struct address_s * with an IRQ-safe spinlock held. In the sync case, the buffers * need to be locked now */ - if (mode != MIGRATE_ASYNC) + if (mode != MIGRATE_ASYNC && mode != MIGRATE_FAULT) BUG_ON(!buffer_migrate_lock_buffers(head, mode)); ClearPagePrivate(page); @@ -687,7 +697,7 @@ static int __unmap_and_move(struct page struct anon_vma *anon_vma = NULL; if (!trylock_page(page)) { - if (!force || mode == MIGRATE_ASYNC) + if (!force || mode == MIGRATE_ASYNC || mode == MIGRATE_FAULT) goto out; /* @@ -1428,4 +1438,63 @@ int migrate_vmas(struct mm_struct *mm, c } return err; } -#endif + +/* + * Attempt to migrate a misplaced page to the specified destination + * node. + */ +int migrate_misplaced_page(struct mm_struct *mm, struct page *page, int node) +{ + struct address_space *mapping = page_mapping(page); + int page_lru = page_is_file_cache(page); + struct page *newpage; + int ret = -EAGAIN; + gfp_t gfp = GFP_HIGHUSER_MOVABLE; + + /* + * Don't migrate pages that are mapped in multiple processes. + */ + if (page_mapcount(page) != 1) + goto out; + + /* + * Never wait for allocations just to migrate on fault, but don't dip + * into reserves. And, only accept pages from the specified node. No + * sense migrating to a different "misplaced" page! + */ + if (mapping) + gfp = mapping_gfp_mask(mapping); + gfp &= ~__GFP_WAIT; + gfp |= __GFP_NOMEMALLOC | GFP_THISNODE; + + newpage = alloc_pages_node(node, gfp, 0); + if (!newpage) { + ret = -ENOMEM; + goto out; + } + + if (isolate_lru_page(page)) { + ret = -EBUSY; + goto put_new; + } + + inc_zone_page_state(page, NR_ISOLATED_ANON + page_lru); + ret = __unmap_and_move(page, newpage, 0, 0, MIGRATE_FAULT); + /* + * A page that has been migrated has all references removed and will be + * freed. A page that has not been migrated will have kepts its + * references and be restored. + */ + dec_zone_page_state(page, NR_ISOLATED_ANON + page_lru); + putback_lru_page(page); +put_new: + /* + * Move the new page to the LRU. If migration was not successful + * then this will free the page. + */ + putback_lru_page(newpage); +out: + return ret; +} + +#endif /* CONFIG_NUMA */ -- To unsubscribe from this list: send the line "unsubscribe linux-kernel" in the body of a message to majordomo@vger.kernel.org More majordomo info at http://vger.kernel.org/majordomo-info.html Please read the FAQ at http://www.tux.org/lkml/