| lists.openwall.net | lists / announce owl-users owl-dev john-users john-dev passwdqc-users yescrypt popa3d-users / oss-security kernel-hardening musl sabotage tlsify passwords / crypt-dev xvendor / Bugtraq Full-Disclosure linux-kernel linux-netdev linux-ext4 linux-hardening linux-cve-announce PHC | |
|
Open Source and information security mailing list archives
| ||
|
Date: Tue, 28 Feb 2012 10:38:37 +0900
From: KAMEZAWA Hiroyuki <kamezawa.hiroyu@...fujitsu.com>
To: Konstantin Khlebnikov <khlebnikov@...nvz.org>
Cc: Hugh Dickins <hughd@...gle.com>, linux-kernel@...r.kernel.org,
linux-mm@...ck.org, Johannes Weiner <hannes@...xchg.org>,
Andrew Morton <akpm@...ux-foundation.org>,
Andi Kleen <andi@...stfloor.org>
Subject: Re: [PATCH v3 18/21] mm: add to lruvec isolated pages counters
On Thu, 23 Feb 2012 17:53:14 +0400
Konstantin Khlebnikov <khlebnikov@...nvz.org> wrote:
> This patch adds into struct lruvec counter of isolated pages.
> It is required for keeping lruvec alive till the isolated page putback.
> We cannot rely on resource counter in memory controller, because it
> does not account uncharged memory. And it much better to have common engine
> for dynamical lruvec management, than to tie all logic with memory cgroup magic.
> Plus this is useful information for memory reclaimer balancing.
>
> There also appears per-cpu page-vectors for putting isolated pages back,
> and function add_page_to_evictable_list(). It is similar to lru_cache_add_lru()
> but it reuse page reference from caller and can adjust isolated pages counters.
> There also new function free_isolated_page_list() which is used at the end of
> shrink_page_list() for freeing pages and adjusting counters of isolated pages.
>
> Memory cgroups code can shuffle pages between lruvecs without isolation
> if page is already isolated with someone else. Thus page lruvec reference is
> unstable even if page is isolated. It is stable only under lru_lock or if page
> reference count is zero. That's why we must always recheck page_lruvec() even
> on non-lumpy 0-order reclaim, where all pages are isolated from one lruvec.
>
> Memory controller at moving pege between cgroups now adjust isolated pages
> counter for old lruvec before inserting page to new lruvec.
> Locking lruvec->lru_lock in mem_cgroup_adjust_isolated() also effectively
> stabilizes PageLRU() sign, so nobody will see PageLRU() under old lru_lock
> while page is already moved into other lruvec.
>
> [ BTW, all lru-id arithmetic can be simplified if we devide unevictable list
> into file and anon parts. After that we can swap bits in page->flags and
> calculate lru-id with single instruction ]
>
> Signed-off-by: Konstantin Khlebnikov <khlebnikov@...nvz.org>
Could you split the part showing statistics of isolated pages into another patch ?
> ---
> include/linux/mmzone.h | 11 ++++-
> include/linux/swap.h | 4 +-
> mm/compaction.c | 1
> mm/huge_memory.c | 4 ++
> mm/internal.h | 6 ++
> mm/ksm.c | 2 -
> mm/memcontrol.c | 39 ++++++++++++++--
> mm/migrate.c | 2 -
> mm/rmap.c | 2 -
> mm/swap.c | 76 +++++++++++++++++++++++++++++++
> mm/vmscan.c | 116 +++++++++++++++++++++++++++++++++++-------------
> 11 files changed, 218 insertions(+), 45 deletions(-)
>
> diff --git a/include/linux/mmzone.h b/include/linux/mmzone.h
> index 82d5ff3..2e3a298 100644
> --- a/include/linux/mmzone.h
> +++ b/include/linux/mmzone.h
> @@ -137,13 +137,20 @@ enum lru_list {
> LRU_INACTIVE_FILE = LRU_BASE + LRU_FILE,
> LRU_ACTIVE_FILE = LRU_BASE + LRU_FILE + LRU_ACTIVE,
> LRU_UNEVICTABLE,
> - NR_LRU_LISTS
> + NR_EVICTABLE_LRU_LISTS = LRU_UNEVICTABLE,
> + NR_LRU_LISTS,
> + LRU_ISOLATED = NR_LRU_LISTS,
> + LRU_ISOLATED_ANON = LRU_ISOLATED,
> + LRU_ISOLATED_FILE,
> + NR_LRU_COUNTERS,
> };
>
> #define for_each_lru(lru) for (lru = 0; lru < NR_LRU_LISTS; lru++)
>
> #define for_each_evictable_lru(lru) for (lru = 0; lru <= LRU_ACTIVE_FILE; lru++)
>
> +#define for_each_lru_counter(cnt) for (cnt = 0; cnt < NR_LRU_COUNTERS; cnt++)
> +
> static inline int is_file_lru(enum lru_list lru)
> {
> return (lru == LRU_INACTIVE_FILE || lru == LRU_ACTIVE_FILE);
> @@ -298,7 +305,7 @@ struct zone_reclaim_stat {
>
> struct lruvec {
> struct list_head pages_lru[NR_LRU_LISTS];
> - unsigned long pages_count[NR_LRU_LISTS];
> + unsigned long pages_count[NR_LRU_COUNTERS];
>
> struct zone_reclaim_stat reclaim_stat;
>
> diff --git a/include/linux/swap.h b/include/linux/swap.h
> index 8630354..3a3ff2c 100644
> --- a/include/linux/swap.h
> +++ b/include/linux/swap.h
> @@ -234,7 +234,9 @@ extern void rotate_reclaimable_page(struct page *page);
> extern void deactivate_page(struct page *page);
> extern void swap_setup(void);
>
> -extern void add_page_to_unevictable_list(struct page *page);
> +extern void add_page_to_unevictable_list(struct page *page, bool isolated);
> +extern void add_page_to_evictable_list(struct page *page,
> + enum lru_list lru, bool isolated);
>
> /**
> * lru_cache_add: add a page to the page lists
> diff --git a/mm/compaction.c b/mm/compaction.c
> index 54340e4..fa74cbe 100644
> --- a/mm/compaction.c
> +++ b/mm/compaction.c
> @@ -384,6 +384,7 @@ static isolate_migrate_t isolate_migratepages(struct zone *zone,
>
> /* Successfully isolated */
> del_page_from_lru_list(lruvec, page, page_lru(page));
> + lruvec->pages_count[LRU_ISOLATED + page_is_file_cache(page)]++;
> list_add(&page->lru, migratelist);
> cc->nr_migratepages++;
> nr_isolated++;
> diff --git a/mm/huge_memory.c b/mm/huge_memory.c
> index 74996b8..46d9f44 100644
> --- a/mm/huge_memory.c
> +++ b/mm/huge_memory.c
> @@ -1316,6 +1316,10 @@ static void __split_huge_page_refcount(struct page *page)
> __dec_zone_page_state(page, NR_ANON_TRANSPARENT_HUGEPAGES);
> __mod_zone_page_state(lruvec_zone(lruvec), NR_ANON_PAGES, HPAGE_PMD_NR);
>
> + /* Fixup isolated pages counter if head page currently isolated */
> + if (!PageLRU(page))
> + lruvec->pages_count[LRU_ISOLATED_ANON] -= HPAGE_PMD_NR-1;
> +
> ClearPageCompound(page);
> compound_unlock(page);
> unlock_lruvec_irq(lruvec);
> diff --git a/mm/internal.h b/mm/internal.h
> index 9454752..6dd2e70 100644
> --- a/mm/internal.h
> +++ b/mm/internal.h
> @@ -265,7 +265,11 @@ extern unsigned long highest_memmap_pfn;
> * in mm/vmscan.c:
> */
> extern int isolate_lru_page(struct page *page);
> -extern void putback_lru_page(struct page *page);
> +extern void __putback_lru_page(struct page *page, bool isolated);
> +static inline void putback_lru_page(struct page *page)
> +{
> + __putback_lru_page(page, true);
> +}
>
> /*
> * in mm/page_alloc.c
> diff --git a/mm/ksm.c b/mm/ksm.c
> index e20de58..109e6ec 100644
> --- a/mm/ksm.c
> +++ b/mm/ksm.c
> @@ -1592,7 +1592,7 @@ struct page *ksm_does_need_to_copy(struct page *page,
> if (page_evictable(new_page, vma))
> lru_cache_add_lru(new_page, LRU_ACTIVE_ANON);
> else
> - add_page_to_unevictable_list(new_page);
> + add_page_to_unevictable_list(new_page, false);
> }
>
> return new_page;
> diff --git a/mm/memcontrol.c b/mm/memcontrol.c
> index 230f434..4de8044 100644
> --- a/mm/memcontrol.c
> +++ b/mm/memcontrol.c
> @@ -697,7 +697,7 @@ mem_cgroup_zone_nr_lru_pages(struct mem_cgroup *memcg, int nid, int zid,
>
> mz = mem_cgroup_zoneinfo(memcg, nid, zid);
>
> - for_each_lru(lru) {
> + for_each_lru_counter(lru) {
> if (BIT(lru) & lru_mask)
> ret += mz->lruvec.pages_count[lru];
> }
> @@ -2354,6 +2354,17 @@ void mem_cgroup_split_huge_fixup(struct page *head)
> }
> #endif /* CONFIG_TRANSPARENT_HUGEPAGE */
>
> +static void mem_cgroup_adjust_isolated(struct lruvec *lruvec,
> + struct page *page, int delta)
> +{
> + int file = page_is_file_cache(page);
> + unsigned long flags;
> +
> + lock_lruvec(lruvec, &flags);
> + lruvec->pages_count[LRU_ISOLATED + file] += delta;
> + unlock_lruvec(lruvec, &flags);
> +}
> +
> /**
> * mem_cgroup_move_account - move account of the page
> * @page: the page
> @@ -2452,6 +2463,7 @@ static int mem_cgroup_move_parent(struct page *page,
> struct mem_cgroup *parent;
> unsigned int nr_pages;
> unsigned long uninitialized_var(flags);
> + struct lruvec *lruvec;
> int ret;
>
> /* Is ROOT ? */
> @@ -2471,6 +2483,8 @@ static int mem_cgroup_move_parent(struct page *page,
> if (ret)
> goto put_back;
>
> + lruvec = page_lruvec(page);
> +
> if (nr_pages > 1)
> flags = compound_lock_irqsave(page);
>
> @@ -2480,8 +2494,11 @@ static int mem_cgroup_move_parent(struct page *page,
>
> if (nr_pages > 1)
> compound_unlock_irqrestore(page, flags);
> + if (!ret)
> + /* This also stabilize PageLRU() sign for lruvec lock holder. */
> + mem_cgroup_adjust_isolated(lruvec, page, -nr_pages);
> put_back:
> - putback_lru_page(page);
> + __putback_lru_page(page, !ret);
> put:
> put_page(page);
> out:
> @@ -3879,6 +3896,8 @@ enum {
> MCS_INACTIVE_FILE,
> MCS_ACTIVE_FILE,
> MCS_UNEVICTABLE,
> + MCS_ISOLATED_ANON,
> + MCS_ISOLATED_FILE,
> NR_MCS_STAT,
> };
>
> @@ -3902,7 +3921,9 @@ struct {
> {"active_anon", "total_active_anon"},
> {"inactive_file", "total_inactive_file"},
> {"active_file", "total_active_file"},
> - {"unevictable", "total_unevictable"}
> + {"unevictable", "total_unevictable"},
> + {"isolated_anon", "total_isolated_anon"},
> + {"isolated_file", "total_isolated_file"},
> };
>
>
> @@ -3942,6 +3963,10 @@ mem_cgroup_get_local_stat(struct mem_cgroup *memcg, struct mcs_total_stat *s)
> s->stat[MCS_ACTIVE_FILE] += val * PAGE_SIZE;
> val = mem_cgroup_nr_lru_pages(memcg, BIT(LRU_UNEVICTABLE));
> s->stat[MCS_UNEVICTABLE] += val * PAGE_SIZE;
> + val = mem_cgroup_nr_lru_pages(memcg, BIT(LRU_ISOLATED_ANON));
> + s->stat[MCS_ISOLATED_ANON] += val * PAGE_SIZE;
> + val = mem_cgroup_nr_lru_pages(memcg, BIT(LRU_ISOLATED_FILE));
> + s->stat[MCS_ISOLATED_FILE] += val * PAGE_SIZE;
> }
>
> static void
> @@ -5243,6 +5268,7 @@ retry:
> struct page *page;
> struct page_cgroup *pc;
> swp_entry_t ent;
> + struct lruvec *lruvec;
>
> if (!mc.precharge)
> break;
> @@ -5253,14 +5279,17 @@ retry:
> page = target.page;
> if (isolate_lru_page(page))
> goto put;
> + lruvec = page_lruvec(page);
> pc = lookup_page_cgroup(page);
> if (!mem_cgroup_move_account(page, 1, pc,
> mc.from, mc.to, false)) {
> mc.precharge--;
> /* we uncharge from mc.from later. */
> mc.moved_charge++;
> - }
> - putback_lru_page(page);
> + mem_cgroup_adjust_isolated(lruvec, page, -1);
> + __putback_lru_page(page, false);
> + } else
> + __putback_lru_page(page, true);
> put: /* is_target_pte_for_mc() gets the page */
> put_page(page);
> break;
> diff --git a/mm/migrate.c b/mm/migrate.c
> index df141f6..de13a0e 100644
> --- a/mm/migrate.c
> +++ b/mm/migrate.c
> @@ -868,7 +868,7 @@ out:
> * Move the new page to the LRU. If migration was not successful
> * then this will free the page.
> */
> - putback_lru_page(newpage);
> + __putback_lru_page(newpage, false);
> if (result) {
> if (rc)
> *result = rc;
> diff --git a/mm/rmap.c b/mm/rmap.c
> index aa547d4..06b5def9 100644
> --- a/mm/rmap.c
> +++ b/mm/rmap.c
> @@ -1139,7 +1139,7 @@ void page_add_new_anon_rmap(struct page *page,
> if (page_evictable(page, vma))
> lru_cache_add_lru(page, LRU_ACTIVE_ANON);
> else
> - add_page_to_unevictable_list(page);
> + add_page_to_unevictable_list(page, false);
> }
>
> /**
> diff --git a/mm/swap.c b/mm/swap.c
> index 3689e3d..998c71c 100644
> --- a/mm/swap.c
> +++ b/mm/swap.c
> @@ -37,6 +37,8 @@
> int page_cluster;
>
> static DEFINE_PER_CPU(struct pagevec[NR_LRU_LISTS], lru_add_pvecs);
> +static DEFINE_PER_CPU(struct pagevec[NR_EVICTABLE_LRU_LISTS],
> + lru_add_isolated_pvecs);
> static DEFINE_PER_CPU(struct pagevec, lru_rotate_pvecs);
> static DEFINE_PER_CPU(struct pagevec, lru_deactivate_pvecs);
>
> @@ -381,6 +383,67 @@ void lru_cache_add_lru(struct page *page, enum lru_list lru)
> __lru_cache_add(page, lru);
> }
>
> +static void __lru_add_isolated_fn(struct lruvec *lruvec,
> + struct page *page, void *arg)
> +{
> + enum lru_list lru = (enum lru_list)arg;
> +
> + VM_BUG_ON(PageActive(page));
> + VM_BUG_ON(PageUnevictable(page));
> + VM_BUG_ON(PageLRU(page));
> +
> + SetPageLRU(page);
> + if (is_active_lru(lru))
> + SetPageActive(page);
> + update_page_reclaim_stat(lruvec, lru);
> + add_page_to_lru_list(lruvec, page, lru);
> + lruvec->pages_count[LRU_ISOLATED + is_file_lru(lru)] -=
> + hpage_nr_pages(page);
> +}
> +
> +static void __lru_add_isolated(struct pagevec *pvec, enum lru_list lru)
> +{
> + VM_BUG_ON(is_unevictable_lru(lru));
> + pagevec_lru_move_fn(pvec, __lru_add_isolated_fn, (void *)lru);
> +}
> +
> +/**
> + * add_page_to_evictable_list - add page to lru list
> + * @page the page to be added into the lru list
> + * @lru lru list id
> + * @isolated need to adjust isolated pages counter
> + *
> + * Like lru_cache_add_lru() but reuses caller's reference to page and
> + * taking care about isolated pages counter on lruvec if isolated = true.
> + */
> +void add_page_to_evictable_list(struct page *page,
> + enum lru_list lru, bool isolated)
> +{
> + struct pagevec *pvec;
> +
> + if (PageActive(page)) {
> + VM_BUG_ON(PageUnevictable(page));
> + ClearPageActive(page);
> + } else if (PageUnevictable(page)) {
> + VM_BUG_ON(PageActive(page));
> + ClearPageUnevictable(page);
> + }
> +
> + VM_BUG_ON(PageLRU(page) || PageActive(page) || PageUnevictable(page));
> +
> + preempt_disable();
> + if (isolated) {
> + pvec = __this_cpu_ptr(lru_add_isolated_pvecs + lru);
> + if (!pagevec_add(pvec, page))
> + __lru_add_isolated(pvec, lru);
> + } else {
> + pvec = __this_cpu_ptr(lru_add_pvecs + lru);
> + if (!pagevec_add(pvec, page))
> + __pagevec_lru_add(pvec, lru);
> + }
> + preempt_enable();
> +}
> +
> /**
> * add_page_to_unevictable_list - add a page to the unevictable list
> * @page: the page to be added to the unevictable list
> @@ -391,7 +454,7 @@ void lru_cache_add_lru(struct page *page, enum lru_list lru)
> * while it's locked or otherwise "invisible" to other tasks. This is
> * difficult to do when using the pagevec cache, so bypass that.
> */
> -void add_page_to_unevictable_list(struct page *page)
> +void add_page_to_unevictable_list(struct page *page, bool isolated)
> {
> struct lruvec *lruvec;
>
> @@ -399,6 +462,10 @@ void add_page_to_unevictable_list(struct page *page)
> SetPageUnevictable(page);
> SetPageLRU(page);
> add_page_to_lru_list(lruvec, page, LRU_UNEVICTABLE);
> + if (isolated) {
> + int type = LRU_ISOLATED + page_is_file_cache(page);
> + lruvec->pages_count[type] -= hpage_nr_pages(page);
> + }
> unlock_lruvec_irq(lruvec);
> }
>
> @@ -485,6 +552,13 @@ static void drain_cpu_pagevecs(int cpu)
> __pagevec_lru_add(pvec, lru);
> }
>
> + pvecs = per_cpu(lru_add_isolated_pvecs, cpu);
> + for_each_evictable_lru(lru) {
> + pvec = &pvecs[lru];
> + if (pagevec_count(pvec))
> + __lru_add_isolated(pvec, lru);
> + }
> +
> pvec = &per_cpu(lru_rotate_pvecs, cpu);
> if (pagevec_count(pvec)) {
> unsigned long flags;
> diff --git a/mm/vmscan.c b/mm/vmscan.c
> index 6eeeb4b..a1ff010 100644
> --- a/mm/vmscan.c
> +++ b/mm/vmscan.c
> @@ -42,6 +42,7 @@
> #include <linux/sysctl.h>
> #include <linux/oom.h>
> #include <linux/prefetch.h>
> +#include <trace/events/kmem.h>
>
> #include <asm/tlbflush.h>
> #include <asm/div64.h>
> @@ -585,15 +586,16 @@ int remove_mapping(struct address_space *mapping, struct page *page)
> }
>
> /**
> - * putback_lru_page - put previously isolated page onto appropriate LRU list
> + * __putback_lru_page - put previously isolated page onto appropriate LRU list
> * @page: page to be put back to appropriate lru list
> + * @isolated: isolated pages counter update required
> *
> * Add previously isolated @page to appropriate LRU list.
> * Page may still be unevictable for other reasons.
> *
> * lru_lock must not be held, interrupts must be enabled.
> */
> -void putback_lru_page(struct page *page)
> +void __putback_lru_page(struct page *page, bool isolated)
> {
> int lru;
> int active = !!TestClearPageActive(page);
> @@ -612,14 +614,16 @@ redo:
> * We know how to handle that.
> */
> lru = active + page_lru_base_type(page);
> - lru_cache_add_lru(page, lru);
> + add_page_to_evictable_list(page, lru, isolated);
> + if (was_unevictable)
> + count_vm_event(UNEVICTABLE_PGRESCUED);
> } else {
> /*
> * Put unevictable pages directly on zone's unevictable
> * list.
> */
> lru = LRU_UNEVICTABLE;
> - add_page_to_unevictable_list(page);
> + add_page_to_unevictable_list(page, isolated);
> /*
> * When racing with an mlock or AS_UNEVICTABLE clearing
> * (page is unlocked) make sure that if the other thread
> @@ -631,30 +635,26 @@ redo:
> * The other side is TestClearPageMlocked() or shmem_lock().
> */
> smp_mb();
> - }
> -
> - /*
> - * page's status can change while we move it among lru. If an evictable
> - * page is on unevictable list, it never be freed. To avoid that,
> - * check after we added it to the list, again.
> - */
> - if (lru == LRU_UNEVICTABLE && page_evictable(page, NULL)) {
> - if (!isolate_lru_page(page)) {
> - put_page(page);
> - goto redo;
> - }
> - /* This means someone else dropped this page from LRU
> - * So, it will be freed or putback to LRU again. There is
> - * nothing to do here.
> + /*
> + * page's status can change while we move it among lru.
> + * If an evictable page is on unevictable list, it never be freed.
> + * To avoid that, check after we added it to the list, again.
> */
> + if (page_evictable(page, NULL)) {
> + if (!isolate_lru_page(page)) {
> + isolated = true;
> + put_page(page);
> + goto redo;
> + }
> + /* This means someone else dropped this page from LRU
> + * So, it will be freed or putback to LRU again. There is
> + * nothing to do here.
> + */
> + }
> + put_page(page); /* drop ref from isolate */
> + if (!was_unevictable)
> + count_vm_event(UNEVICTABLE_PGCULLED);
> }
> -
> - if (was_unevictable && lru != LRU_UNEVICTABLE)
> - count_vm_event(UNEVICTABLE_PGRESCUED);
> - else if (!was_unevictable && lru == LRU_UNEVICTABLE)
> - count_vm_event(UNEVICTABLE_PGCULLED);
> -
> - put_page(page); /* drop ref from isolate */
> }
>
> enum page_references {
> @@ -724,6 +724,48 @@ static enum page_references page_check_references(struct page *page,
> }
>
> /*
> + * Free a list of isolated 0-order pages
> + */
> +static void free_isolated_page_list(struct lruvec *lruvec,
> + struct list_head *list, int cold)
> +{
> + struct page *page, *next;
> + unsigned long nr_pages[2];
> + struct list_head queue;
> +
> +again:
> + INIT_LIST_HEAD(&queue);
> + nr_pages[0] = nr_pages[1] = 0;
> +
> + list_for_each_entry_safe(page, next, list, lru) {
> + if (unlikely(lruvec != page_lruvec(page))) {
> + list_add_tail(&page->lru, &queue);
> + continue;
> + }
> + nr_pages[page_is_file_cache(page)]++;
> + trace_mm_page_free_batched(page, cold);
> + free_hot_cold_page(page, cold);
> + }
> +
> + lock_lruvec_irq(lruvec);
> + lruvec->pages_count[LRU_ISOLATED_ANON] -= nr_pages[0];
> + lruvec->pages_count[LRU_ISOLATED_FILE] -= nr_pages[1];
> + unlock_lruvec_irq(lruvec);
Is it guaranteed that all pages are from the same lruvec ?
Thanks,
-Kame
> +
> + /*
> + * Usually there will be only one iteration, because
> + * at 0-order reclaim all pages are from one lruvec
> + * if we didn't raced with memory cgroup shuffling.
> + */
> + if (unlikely(!list_empty(&queue))) {
> + list_replace(&queue, list);
> + lruvec = page_lruvec(list_first_entry(list,
> + struct page, lru));
> + goto again;
> + }
> +}
> +
> +/*
> * shrink_page_list() returns the number of reclaimed pages
> */
> static unsigned long shrink_page_list(struct list_head *page_list,
> @@ -986,7 +1028,7 @@ keep_lumpy:
> if (nr_dirty && nr_dirty == nr_congested && global_reclaim(sc))
> zone_set_flag(lruvec_zone(lruvec), ZONE_CONGESTED);
>
> - free_hot_cold_page_list(&free_pages, 1);
> + free_isolated_page_list(lruvec, &free_pages, 1);
>
> list_splice(&ret_pages, page_list);
> count_vm_events(PGACTIVATE, pgactivate);
> @@ -1206,11 +1248,14 @@ static unsigned long isolate_lru_pages(unsigned long nr_to_scan,
> if (__isolate_lru_page(cursor_page, mode, file) == 0) {
> unsigned int isolated_pages;
> int cursor_lru = page_lru(cursor_page);
> + int cur_file = page_is_file_cache(cursor_page);
>
> list_move(&cursor_page->lru, dst);
> isolated_pages = hpage_nr_pages(cursor_page);
> cursor_lruvec->pages_count[cursor_lru] -=
> isolated_pages;
> + cursor_lruvec->pages_count[LRU_ISOLATED +
> + cur_file] += isolated_pages;
> VM_BUG_ON((long)cursor_lruvec->
> pages_count[cursor_lru] < 0);
> nr_taken += isolated_pages;
> @@ -1248,6 +1293,7 @@ static unsigned long isolate_lru_pages(unsigned long nr_to_scan,
> }
>
> lruvec->pages_count[lru] -= nr_taken - nr_lumpy_taken;
> + lruvec->pages_count[LRU_ISOLATED + file] += nr_taken - nr_lumpy_taken;
> VM_BUG_ON((long)lruvec->pages_count[lru] < 0);
>
> *nr_scanned = scan;
> @@ -1296,11 +1342,14 @@ int isolate_lru_page(struct page *page)
>
> lruvec = lock_page_lruvec_irq(page);
> if (PageLRU(page)) {
> + int file = page_is_file_cache(page);
> int lru = page_lru(page);
> ret = 0;
> get_page(page);
> ClearPageLRU(page);
> del_page_from_lru_list(lruvec, page, lru);
> + lruvec->pages_count[LRU_ISOLATED + file] +=
> + hpage_nr_pages(page);
> }
> unlock_lruvec_irq(lruvec);
> }
> @@ -1347,7 +1396,7 @@ putback_inactive_pages(struct lruvec *lruvec,
> */
> while (!list_empty(page_list)) {
> struct page *page = lru_to_page(page_list);
> - int lru;
> + int numpages, lru, file;
>
> VM_BUG_ON(PageLRU(page));
> list_del(&page->lru);
> @@ -1363,13 +1412,13 @@ putback_inactive_pages(struct lruvec *lruvec,
>
> SetPageLRU(page);
> lru = page_lru(page);
> + file = is_file_lru(lru);
> + numpages = hpage_nr_pages(page);
>
> add_page_to_lru_list(lruvec, page, lru);
> - if (is_active_lru(lru)) {
> - int file = is_file_lru(lru);
> - int numpages = hpage_nr_pages(page);
> + lruvec->pages_count[LRU_ISOLATED + file] -= numpages;
> + if (is_active_lru(lru))
> reclaim_stat->recent_rotated[file] += numpages;
> - }
> if (put_page_testzero(page)) {
> __ClearPageLRU(page);
> __ClearPageActive(page);
> @@ -1656,6 +1705,9 @@ move_active_pages_to_lru(struct lruvec *lruvec,
> list_move(&page->lru, &lruvec->pages_lru[lru]);
> numpages = hpage_nr_pages(page);
> lruvec->pages_count[lru] += numpages;
> + /* There should be no mess between file and anon pages */
> + lruvec->pages_count[LRU_ISOLATED +
> + is_file_lru(lru)] -= numpages;
> pgmoved += numpages;
>
> if (put_page_testzero(page)) {
>
> --
> To unsubscribe from this list: send the line "unsubscribe linux-kernel" in
> the body of a message to majordomo@...r.kernel.org
> More majordomo info at http://vger.kernel.org/majordomo-info.html
> Please read the FAQ at http://www.tux.org/lkml/
>
--
To unsubscribe from this list: send the line "unsubscribe linux-kernel" in
the body of a message to majordomo@...r.kernel.org
More majordomo info at http://vger.kernel.org/majordomo-info.html
Please read the FAQ at http://www.tux.org/lkml/
Powered by blists - more mailing lists