>From 50c81e697f68aba7362dbf77b9017f8bba666e17 Mon Sep 17 00:00:00 2001
From: Matthew Wilcox <willy@linux.intel.com>
Date: Fri, 25 Jul 2014 15:10:12 -0400
Subject: [PATCH 4/4] mm: Introduce zap_pte_single()

zap_pte_single() is a new wrapper around zap_pte() that does all the
necessary setup for insert_pte() and insert_page().
---
 mm/memory.c | 44 ++++++++++++++++++++++++++++++++++----------
 1 file changed, 34 insertions(+), 10 deletions(-)

diff --git a/mm/memory.c b/mm/memory.c
index 6a35f98..a8e17ce 100644
--- a/mm/memory.c
+++ b/mm/memory.c
@@ -1077,7 +1077,8 @@ static bool zap_pte(struct mmu_gather *tlb, struct vm_area_struct *vma,
 				struct zap_details *details, int *rss,
 				int *force_flush)
 {
-	struct mm_struct *mm = tlb->mm;
+	struct mm_struct *mm = tlb ? tlb->mm : vma->vm_mm;
+	bool fullmm = tlb ? tlb->fullmm : false;
 	pte_t ptent = *pte;
 
 	if (pte_none(ptent))
@@ -1105,9 +1106,9 @@ static bool zap_pte(struct mmu_gather *tlb, struct vm_area_struct *vma,
 			     page->index > details->last_index))
 				return false;
 		}
-		ptent = ptep_get_and_clear_full(mm, addr, pte,
-						tlb->fullmm);
-		tlb_remove_tlb_entry(tlb, pte, addr);
+		ptent = ptep_get_and_clear_full(mm, addr, pte, fullmm);
+		if (tlb)
+			tlb_remove_tlb_entry(tlb, pte, addr);
 		if (unlikely(!page))
 			return false;
 		if (unlikely(details) && details->nonlinear_vma
@@ -1133,7 +1134,7 @@ static bool zap_pte(struct mmu_gather *tlb, struct vm_area_struct *vma,
 		page_remove_rmap(page);
 		if (unlikely(page_mapcount(page) < 0))
 			print_bad_pte(vma, addr, ptent, page);
-		if (unlikely(!__tlb_remove_page(tlb, page))) {
+		if (unlikely(tlb && !__tlb_remove_page(tlb, page))) {
 			*force_flush = 1;
 			return true;
 		}
@@ -1166,10 +1167,27 @@ static bool zap_pte(struct mmu_gather *tlb, struct vm_area_struct *vma,
 		if (unlikely(!free_swap_and_cache(entry)))
 			print_bad_pte(vma, addr, ptent, NULL);
 	}
-	pte_clear_not_present_full(mm, addr, pte, tlb->fullmm);
+	pte_clear_not_present_full(mm, addr, pte, fullmm);
 	return false;
 }
 
+static void zap_pte_single(struct vm_area_struct *vma, pte_t *pte,
+				unsigned long addr)
+{
+	struct mm_struct *mm = vma->vm_mm;
+	int force_flush = 0;
+	int rss[NR_MM_COUNTERS];
+
+	VM_BUG_ON(!mutex_is_locked(&vma->vm_file->f_mapping->i_mmap_mutex));
+
+	init_rss_vec(rss);
+	update_hiwater_rss(mm);
+	flush_cache_page(vma, addr, pte_pfn(*pte));
+	zap_pte(NULL, vma, pte, addr, NULL, rss, &force_flush);
+	flush_tlb_page(vma, addr);
+	add_mm_rss_vec(mm, rss);
+}
+
 static unsigned long zap_pte_range(struct mmu_gather *tlb,
 				struct vm_area_struct *vma, pmd_t *pmd,
 				unsigned long addr, unsigned long end,
@@ -1494,6 +1512,7 @@ static int insert_page(struct vm_area_struct *vma, unsigned long addr,
 	int retval;
 	pte_t *pte;
 	spinlock_t *ptl;
+	bool replaced = false;
 
 	retval = -EINVAL;
 	if (PageAnon(page))
@@ -1507,8 +1526,8 @@ static int insert_page(struct vm_area_struct *vma, unsigned long addr,
 	if (!pte_none(*pte)) {
 		if (!replace)
 			goto out_unlock;
-		VM_BUG_ON(!mutex_is_locked(&vma->vm_file->f_mapping->i_mmap_mutex));
-		zap_page_range_single(vma, addr, PAGE_SIZE, NULL);
+		zap_pte_single(vma, pte, addr);
+		replaced = true;
 	}
 
 	/* Ok, finally just insert the thing.. */
@@ -1519,6 +1538,8 @@ static int insert_page(struct vm_area_struct *vma, unsigned long addr,
 
 	retval = 0;
 	pte_unmap_unlock(pte, ptl);
+	if (replaced)
+		mmu_notifier_invalidate_page(mm, addr);
 	return retval;
 out_unlock:
 	pte_unmap_unlock(pte, ptl);
@@ -1576,6 +1597,7 @@ static int insert_pfn(struct vm_area_struct *vma, unsigned long addr,
 	int retval;
 	pte_t *pte, entry;
 	spinlock_t *ptl;
+	bool replaced = false;
 
 	retval = -ENOMEM;
 	pte = get_locked_pte(mm, addr, &ptl);
@@ -1585,8 +1607,8 @@ static int insert_pfn(struct vm_area_struct *vma, unsigned long addr,
 	if (!pte_none(*pte)) {
 		if (!replace)
 			goto out_unlock;
-		VM_BUG_ON(!mutex_is_locked(&vma->vm_file->f_mapping->i_mmap_mutex));
-		zap_page_range_single(vma, addr, PAGE_SIZE, NULL);
+		zap_pte_single(vma, pte, addr);
+		replaced = true;
 	}
 
 	/* Ok, finally just insert the thing.. */
@@ -1597,6 +1619,8 @@ static int insert_pfn(struct vm_area_struct *vma, unsigned long addr,
 	retval = 0;
 out_unlock:
 	pte_unmap_unlock(pte, ptl);
+	if (replaced)
+		mmu_notifier_invalidate_page(mm, addr);
 out:
 	return retval;
 }
-- 
2.0.1