diff --git a/mm/page_alloc.c b/mm/page_alloc.c index 4403009..d2a515d 100644 --- a/mm/page_alloc.c +++ b/mm/page_alloc.c @@ -676,6 +676,24 @@ static void free_pcppages_bulk(struct zone *zone, int count, spin_unlock(&zone->lock); } +/* + * This function is almost same with free_one_page except that it + * doesn't increase NR_FREE_PAGES and free_area[order].nr_free. + * Because page allocator can't allocate MIGRATE_ISOLATE type page. + * + * Caller should hold zone->lock. + */ +static void free_one_isolated_page(struct zone *zone, struct page *page, + int order) +{ + zone->all_unreclaimable = 0; + zone->pages_scanned = 0; + + __free_one_page(page, zone, order, MIGRATE_ISOLATE); + /* rollback nr_free increased by __free_one_page */ + zone->free_area[order].nr_free--; +} + static void free_one_page(struct zone *zone, struct page *page, int order, int migratetype) { @@ -683,6 +701,13 @@ static void free_one_page(struct zone *zone, struct page *page, int order, zone->all_unreclaimable = 0; zone->pages_scanned = 0; + /* + * Freed MIGRATE_ISOLATE page should be free_one_isolated_page path + * because page allocator don't want to increase NR_FREE_PAGES and + * free_area[order].nr_free. + */ + VM_BUG_ON(migratetype == MIGRATE_ISOLATE); + __free_one_page(page, zone, order, migratetype); __mod_zone_page_state(zone, NR_FREE_PAGES, 1 << order); spin_unlock(&zone->lock); @@ -718,6 +743,7 @@ static void __free_pages_ok(struct page *page, unsigned int order) { unsigned long flags; int wasMlocked = __TestClearPageMlocked(page); + int migratetype; if (!free_pages_prepare(page, order)) return; @@ -726,8 +752,21 @@ static void __free_pages_ok(struct page *page, unsigned int order) if (unlikely(wasMlocked)) free_page_mlock(page); __count_vm_events(PGFREE, 1 << order); - free_one_page(page_zone(page), page, order, - get_pageblock_migratetype(page)); + migratetype = get_pageblock_migratetype(page); + /* + * High order page alloc/free is rare compared to + * order-0. So this condition check should be not + * critical about performance. + */ + if (unlikely(migratetype == MIGRATE_ISOLATE)) { + struct zone *zone = page_zone(page); + spin_lock(&zone->lock); + free_one_isolated_page(zone, page, order); + spin_unlock(&zone->lock); + } + else { + free_one_page(page_zone(page), page, order, migratetype); + } local_irq_restore(flags); } @@ -906,6 +945,55 @@ static int fallbacks[MIGRATE_TYPES][4] = { [MIGRATE_ISOLATE] = { MIGRATE_RESERVE }, /* Never used */ }; +static int hotplug_move_freepages(struct zone *zone, + struct page *start_page, struct page *end_page, + int from_migratetype, int to_migratetype) +{ + struct page *page; + unsigned long order; + int pages_moved = 0; + +#ifndef CONFIG_HOLES_IN_ZONE + /* + * page_zone is not safe to call in this context when + * CONFIG_HOLES_IN_ZONE is set. This bug check is probably redundant + * anyway as we check zone boundaries in move_freepages_block(). + * Remove at a later date when no bug reports exist related to + * grouping pages by mobility + */ + BUG_ON(page_zone(start_page) != page_zone(end_page)); +#endif + + BUG_ON(from_migratetype == to_migratetype); + + for (page = start_page; page <= end_page;) { + /* Make sure we are not inadvertently changing nodes */ + VM_BUG_ON(page_to_nid(page) != zone_to_nid(zone)); + + if (!pfn_valid_within(page_to_pfn(page))) { + page++; + continue; + } + + if (!PageBuddy(page)) { + page++; + continue; + } + + order = page_order(page); + list_move(&page->lru, + &zone->free_area[order].free_list[to_migratetype]); + if (to_migratetype == MIGRATE_ISOLATE) + zone->free_area[order].nr_free--; + else if (from_migratetype == MIGRATE_ISOLATE) + zone->free_area[order].nr_free++; + page += 1 << order; + pages_moved += 1 << order; + } + + return pages_moved; +} + /* * Move the free pages in a range to the free lists of the requested type. * Note that start_page and end_pages are not aligned on a pageblock @@ -954,6 +1042,32 @@ static int move_freepages(struct zone *zone, return pages_moved; } +/* + * It's almost same with move_freepages_block except [from, to] migratetype. + * We need it for accounting zone->free_area[order].nr_free exactly. + */ +static int hotplug_move_freepages_block(struct zone *zone, struct page *page, + int from_migratetype, int to_migratetype) +{ + unsigned long start_pfn, end_pfn; + struct page *start_page, *end_page; + + start_pfn = page_to_pfn(page); + start_pfn = start_pfn & ~(pageblock_nr_pages-1); + start_page = pfn_to_page(start_pfn); + end_page = start_page + pageblock_nr_pages - 1; + end_pfn = start_pfn + pageblock_nr_pages - 1; + + /* Do not cross zone boundaries */ + if (start_pfn < zone->zone_start_pfn) + start_page = page; + if (end_pfn >= zone->zone_start_pfn + zone->spanned_pages) + return 0; + + return hotplug_move_freepages(zone, start_page, end_page, + from_migratetype, to_migratetype); +} + static int move_freepages_block(struct zone *zone, struct page *page, int migratetype) { @@ -1311,7 +1425,9 @@ void free_hot_cold_page(struct page *page, int cold) */ if (migratetype >= MIGRATE_PCPTYPES) { if (unlikely(migratetype == MIGRATE_ISOLATE)) { - free_one_page(zone, page, 0, migratetype); + spin_lock(&zone->lock); + free_one_isolated_page(zone, page, 0); + spin_unlock(&zone->lock); goto out; } migratetype = MIGRATE_MOVABLE; @@ -1388,6 +1504,7 @@ int split_free_page(struct page *page) unsigned int order; unsigned long watermark; struct zone *zone; + int migratetype; BUG_ON(!PageBuddy(page)); @@ -1400,10 +1517,17 @@ int split_free_page(struct page *page) return 0; /* Remove page from free list */ + migratetype = get_pageblock_migratetype(page); list_del(&page->lru); - zone->free_area[order].nr_free--; + /* + * Page allocator didn't increase nr_free and NR_FREE_PAGES on pages + * which are in free_area[order].free_list[MIGRATE_ISOLATE] pages. + */ + if (migratetype != MIGRATE_ISOLATE) { + zone->free_area[order].nr_free--; + __mod_zone_page_state(zone, NR_FREE_PAGES, -(1UL << order)); + } rmv_page_order(page); - __mod_zone_page_state(zone, NR_FREE_PAGES, -(1UL << order)); /* Split into individual pages */ set_page_refcounted(page); @@ -5593,8 +5717,11 @@ int set_migratetype_isolate(struct page *page) out: if (!ret) { + int pages_moved; set_pageblock_migratetype(page, MIGRATE_ISOLATE); - move_freepages_block(zone, page, MIGRATE_ISOLATE); + pages_moved = hotplug_move_freepages_block(zone, page, + MIGRATE_MOVABLE, MIGRATE_ISOLATE); + __mod_zone_page_state(zone, NR_FREE_PAGES, -pages_moved); } spin_unlock_irqrestore(&zone->lock, flags); @@ -5607,12 +5734,15 @@ void unset_migratetype_isolate(struct page *page, unsigned migratetype) { struct zone *zone; unsigned long flags; + int pages_moved; zone = page_zone(page); spin_lock_irqsave(&zone->lock, flags); if (get_pageblock_migratetype(page) != MIGRATE_ISOLATE) goto out; set_pageblock_migratetype(page, migratetype); - move_freepages_block(zone, page, migratetype); + pages_moved = hotplug_move_freepages_block(zone, page, + MIGRATE_ISOLATE, migratetype); + __mod_zone_page_state(zone, NR_FREE_PAGES, pages_moved); out: spin_unlock_irqrestore(&zone->lock, flags); } @@ -5900,9 +6030,6 @@ __offline_isolated_pages(unsigned long start_pfn, unsigned long end_pfn) #endif list_del(&page->lru); rmv_page_order(page); - zone->free_area[order].nr_free--; - __mod_zone_page_state(zone, NR_FREE_PAGES, - - (1UL << order)); for (i = 0; i < (1 << order); i++) SetPageReserved((page+i)); pfn += (1 << order);