| lists.openwall.net | lists / announce owl-users owl-dev john-users john-dev passwdqc-users yescrypt popa3d-users / oss-security kernel-hardening musl sabotage tlsify passwords / crypt-dev xvendor / Bugtraq Full-Disclosure linux-kernel linux-netdev linux-ext4 linux-hardening linux-cve-announce PHC | |
|
Open Source and information security mailing list archives
| ||
|
Message-Id: <20250211111326.14295-7-dev.jain@arm.com>
Date: Tue, 11 Feb 2025 16:43:15 +0530
From: Dev Jain <dev.jain@....com>
To: akpm@...ux-foundation.org,
david@...hat.com,
willy@...radead.org,
kirill.shutemov@...ux.intel.com
Cc: npache@...hat.com,
ryan.roberts@....com,
anshuman.khandual@....com,
catalin.marinas@....com,
cl@...two.org,
vbabka@...e.cz,
mhocko@...e.com,
apopple@...dia.com,
dave.hansen@...ux.intel.com,
will@...nel.org,
baohua@...nel.org,
jack@...e.cz,
srivatsa@...il.mit.edu,
haowenchao22@...il.com,
hughd@...gle.com,
aneesh.kumar@...nel.org,
yang@...amperecomputing.com,
peterx@...hat.com,
ioworker0@...il.com,
wangkefeng.wang@...wei.com,
ziy@...dia.com,
jglisse@...gle.com,
surenb@...gle.com,
vishal.moola@...il.com,
zokeefe@...gle.com,
zhengqi.arch@...edance.com,
jhubbard@...dia.com,
21cnbao@...il.com,
linux-mm@...ck.org,
linux-kernel@...r.kernel.org,
Dev Jain <dev.jain@....com>
Subject: [PATCH v2 06/17] khugepaged: Abstract PMD-THP collapse
Abstract away copying page contents, and setting the PMD, into
vma_collapse_anon_folio_pmd().
Signed-off-by: Dev Jain <dev.jain@....com>
---
mm/khugepaged.c | 140 +++++++++++++++++++++++++++---------------------
1 file changed, 78 insertions(+), 62 deletions(-)
diff --git a/mm/khugepaged.c b/mm/khugepaged.c
index 99eb1f72a508..498cb5ad9ff1 100644
--- a/mm/khugepaged.c
+++ b/mm/khugepaged.c
@@ -1109,76 +1109,27 @@ static int alloc_charge_folio(struct folio **foliop, struct mm_struct *mm,
return SCAN_SUCCEED;
}
-static int collapse_huge_page(struct mm_struct *mm, unsigned long address,
- int referenced, int unmapped,
- struct collapse_control *cc)
+static int vma_collapse_anon_folio_pmd(struct mm_struct *mm, unsigned long address,
+ struct vm_area_struct *vma, struct collapse_control *cc, pmd_t *pmd,
+ struct folio *folio)
{
LIST_HEAD(compound_pagelist);
- pmd_t *pmd, _pmd;
- pte_t *pte;
pgtable_t pgtable;
- struct folio *folio;
spinlock_t *pmd_ptl, *pte_ptl;
int result = SCAN_FAIL;
- struct vm_area_struct *vma;
struct mmu_notifier_range range;
+ pmd_t _pmd;
+ pte_t *pte;
VM_BUG_ON(address & ~HPAGE_PMD_MASK);
- /*
- * Before allocating the hugepage, release the mmap_lock read lock.
- * The allocation can take potentially a long time if it involves
- * sync compaction, and we do not need to hold the mmap_lock during
- * that. We will recheck the vma after taking it again in write mode.
- */
- mmap_read_unlock(mm);
-
- result = alloc_charge_folio(&folio, mm, HPAGE_PMD_ORDER, cc);
- if (result != SCAN_SUCCEED)
- goto out_nolock;
-
- mmap_read_lock(mm);
- result = hugepage_vma_revalidate(mm, address, true, &vma, HPAGE_PMD_ORDER, cc);
- if (result != SCAN_SUCCEED) {
- mmap_read_unlock(mm);
- goto out_nolock;
- }
-
- result = find_pmd_or_thp_or_none(mm, address, &pmd);
- if (result != SCAN_SUCCEED) {
- mmap_read_unlock(mm);
- goto out_nolock;
- }
-
- if (unmapped) {
- /*
- * __collapse_huge_page_swapin will return with mmap_lock
- * released when it fails. So we jump out_nolock directly in
- * that case. Continuing to collapse causes inconsistency.
- */
- result = __collapse_huge_page_swapin(mm, vma, address, pmd,
- referenced, HPAGE_PMD_ORDER);
- if (result != SCAN_SUCCEED)
- goto out_nolock;
- }
-
- mmap_read_unlock(mm);
- /*
- * Prevent all access to pagetables with the exception of
- * gup_fast later handled by the ptep_clear_flush and the VM
- * handled by the anon_vma lock + PG_lock.
- *
- * UFFDIO_MOVE is prevented to race as well thanks to the
- * mmap_lock.
- */
- mmap_write_lock(mm);
result = hugepage_vma_revalidate(mm, address, true, &vma, HPAGE_PMD_ORDER, cc);
if (result != SCAN_SUCCEED)
- goto out_up_write;
+ goto out;
/* check if the pmd is still valid */
result = check_pmd_still_valid(mm, address, pmd);
if (result != SCAN_SUCCEED)
- goto out_up_write;
+ goto out;
vma_start_write(vma);
anon_vma_lock_write(vma->anon_vma);
@@ -1223,7 +1174,7 @@ static int collapse_huge_page(struct mm_struct *mm, unsigned long address,
pmd_populate(mm, pmd, pmd_pgtable(_pmd));
spin_unlock(pmd_ptl);
anon_vma_unlock_write(vma->anon_vma);
- goto out_up_write;
+ goto out;
}
/*
@@ -1237,7 +1188,7 @@ static int collapse_huge_page(struct mm_struct *mm, unsigned long address,
&compound_pagelist, HPAGE_PMD_ORDER);
pte_unmap(pte);
if (unlikely(result != SCAN_SUCCEED))
- goto out_up_write;
+ goto out;
/*
* The smp_wmb() inside __folio_mark_uptodate() ensures the
@@ -1260,11 +1211,76 @@ static int collapse_huge_page(struct mm_struct *mm, unsigned long address,
deferred_split_folio(folio, false);
spin_unlock(pmd_ptl);
- folio = NULL;
-
result = SCAN_SUCCEED;
-out_up_write:
+out:
+ return result;
+}
+
+static int collapse_huge_page(struct mm_struct *mm, unsigned long address,
+ int referenced, int unmapped, int order,
+ struct collapse_control *cc)
+{
+ struct vm_area_struct *vma;
+ int result = SCAN_FAIL;
+ struct folio *folio;
+ pmd_t *pmd;
+
+ /*
+ * Before allocating the hugepage, release the mmap_lock read lock.
+ * The allocation can take potentially a long time if it involves
+ * sync compaction, and we do not need to hold the mmap_lock during
+ * that. We will recheck the vma after taking it again in write mode.
+ */
+ mmap_read_unlock(mm);
+
+ result = alloc_charge_folio(&folio, mm, order, cc);
+ if (result != SCAN_SUCCEED)
+ goto out_nolock;
+
+ mmap_read_lock(mm);
+ result = hugepage_vma_revalidate(mm, address, true, &vma, order, cc);
+ if (result != SCAN_SUCCEED) {
+ mmap_read_unlock(mm);
+ goto out_nolock;
+ }
+
+ result = find_pmd_or_thp_or_none(mm, address, &pmd);
+ if (result != SCAN_SUCCEED) {
+ mmap_read_unlock(mm);
+ goto out_nolock;
+ }
+
+ if (unmapped) {
+ /*
+ * __collapse_huge_page_swapin will return with mmap_lock
+ * released when it fails. So we jump out_nolock directly in
+ * that case. Continuing to collapse causes inconsistency.
+ */
+ result = __collapse_huge_page_swapin(mm, vma, address, pmd,
+ referenced, order);
+ if (result != SCAN_SUCCEED)
+ goto out_nolock;
+ }
+
+ mmap_read_unlock(mm);
+ /*
+ * Prevent all access to pagetables with the exception of
+ * gup_fast later handled by the ptep_clear_flush and the VM
+ * handled by the anon_vma lock + PG_lock.
+ *
+ * UFFDIO_MOVE is prevented to race as well thanks to the
+ * mmap_lock.
+ */
+ mmap_write_lock(mm);
+
+ if (order == HPAGE_PMD_ORDER)
+ result = vma_collapse_anon_folio_pmd(mm, address, vma, cc, pmd, folio);
+
mmap_write_unlock(mm);
+
+ if (result == SCAN_SUCCEED)
+ folio = NULL;
+
out_nolock:
if (folio)
folio_put(folio);
@@ -1440,7 +1456,7 @@ static int hpage_collapse_scan_pmd(struct mm_struct *mm,
pte_unmap_unlock(pte, ptl);
if (result == SCAN_SUCCEED) {
result = collapse_huge_page(mm, address, referenced,
- unmapped, cc);
+ unmapped, HPAGE_PMD_ORDER, cc);
/* collapse_huge_page will return with the mmap_lock released */
*mmap_locked = false;
}
--
2.30.2
Powered by blists - more mailing lists