| lists.openwall.net | lists / announce owl-users owl-dev john-users john-dev passwdqc-users yescrypt popa3d-users / oss-security kernel-hardening musl sabotage tlsify passwords / crypt-dev xvendor / Bugtraq Full-Disclosure linux-kernel linux-netdev linux-ext4 linux-hardening linux-cve-announce PHC | |
|
Open Source and information security mailing list archives
| ||
|
Message-ID: <20230218002819.1486479-21-jthoughton@google.com>
Date: Sat, 18 Feb 2023 00:27:53 +0000
From: James Houghton <jthoughton@...gle.com>
To: Mike Kravetz <mike.kravetz@...cle.com>,
Muchun Song <songmuchun@...edance.com>,
Peter Xu <peterx@...hat.com>,
Andrew Morton <akpm@...ux-foundation.org>
Cc: David Hildenbrand <david@...hat.com>,
David Rientjes <rientjes@...gle.com>,
Axel Rasmussen <axelrasmussen@...gle.com>,
Mina Almasry <almasrymina@...gle.com>,
"Zach O'Keefe" <zokeefe@...gle.com>,
Manish Mishra <manish.mishra@...anix.com>,
Naoya Horiguchi <naoya.horiguchi@....com>,
"Dr . David Alan Gilbert" <dgilbert@...hat.com>,
"Matthew Wilcox (Oracle)" <willy@...radead.org>,
Vlastimil Babka <vbabka@...e.cz>,
Baolin Wang <baolin.wang@...ux.alibaba.com>,
Miaohe Lin <linmiaohe@...wei.com>,
Yang Shi <shy828301@...il.com>,
Frank van der Linden <fvdl@...gle.com>,
Jiaqi Yan <jiaqiyan@...gle.com>, linux-mm@...ck.org,
linux-kernel@...r.kernel.org,
James Houghton <jthoughton@...gle.com>
Subject: [PATCH v2 20/46] hugetlb: add HGM support to follow_hugetlb_page
Enable high-granularity mapping support in GUP.
In case it is confusing, pfn_offset is the offset (in PAGE_SIZE units)
that vaddr points to within the subpage that hpte points to.
Signed-off-by: James Houghton <jthoughton@...gle.com>
diff --git a/mm/hugetlb.c b/mm/hugetlb.c
index 7321c6602d6f..c26b040f4fb5 100644
--- a/mm/hugetlb.c
+++ b/mm/hugetlb.c
@@ -6634,11 +6634,9 @@ static void record_subpages_vmas(struct page *page, struct vm_area_struct *vma,
}
static inline bool __follow_hugetlb_must_fault(struct vm_area_struct *vma,
- unsigned int flags, pte_t *pte,
+ unsigned int flags, pte_t pteval,
bool *unshare)
{
- pte_t pteval = huge_ptep_get(pte);
-
*unshare = false;
if (is_swap_pte(pteval))
return true;
@@ -6713,11 +6711,13 @@ long follow_hugetlb_page(struct mm_struct *mm, struct vm_area_struct *vma,
int err = -EFAULT, refs;
while (vaddr < vma->vm_end && remainder) {
- pte_t *pte;
+ pte_t *ptep, pte;
spinlock_t *ptl = NULL;
bool unshare = false;
int absent;
- struct page *page;
+ unsigned long pages_per_hpte;
+ struct page *page, *subpage;
+ struct hugetlb_pte hpte;
/*
* If we have a pending SIGKILL, don't keep faulting pages and
@@ -6734,13 +6734,19 @@ long follow_hugetlb_page(struct mm_struct *mm, struct vm_area_struct *vma,
* each hugepage. We have to make sure we get the
* first, for the page indexing below to work.
*
- * Note that page table lock is not held when pte is null.
+ * hugetlb_full_walk will mask the address appropriately.
+ *
+ * Note that page table lock is not held when ptep is null.
*/
- pte = hugetlb_walk(vma, vaddr & huge_page_mask(h),
- huge_page_size(h));
- if (pte)
- ptl = huge_pte_lock(h, mm, pte);
- absent = !pte || huge_pte_none(huge_ptep_get(pte));
+ if (hugetlb_full_walk(&hpte, vma, vaddr)) {
+ ptep = NULL;
+ absent = true;
+ } else {
+ ptl = hugetlb_pte_lock(&hpte);
+ ptep = hpte.ptep;
+ pte = huge_ptep_get(ptep);
+ absent = huge_pte_none(pte);
+ }
/*
* When coredumping, it suits get_dump_page if we just return
@@ -6751,13 +6757,21 @@ long follow_hugetlb_page(struct mm_struct *mm, struct vm_area_struct *vma,
*/
if (absent && (flags & FOLL_DUMP) &&
!hugetlbfs_pagecache_present(h, vma, vaddr)) {
- if (pte)
+ if (ptep)
spin_unlock(ptl);
hugetlb_vma_unlock_read(vma);
remainder = 0;
break;
}
+ if (!absent && pte_present(pte) &&
+ !hugetlb_pte_present_leaf(&hpte, pte)) {
+ /* We raced with someone splitting the PTE, so retry. */
+ spin_unlock(ptl);
+ hugetlb_vma_unlock_read(vma);
+ continue;
+ }
+
/*
* We need call hugetlb_fault for both hugepages under migration
* (in which case hugetlb_fault waits for the migration,) and
@@ -6773,7 +6787,7 @@ long follow_hugetlb_page(struct mm_struct *mm, struct vm_area_struct *vma,
vm_fault_t ret;
unsigned int fault_flags = 0;
- if (pte)
+ if (ptep)
spin_unlock(ptl);
hugetlb_vma_unlock_read(vma);
@@ -6822,8 +6836,10 @@ long follow_hugetlb_page(struct mm_struct *mm, struct vm_area_struct *vma,
continue;
}
- pfn_offset = (vaddr & ~huge_page_mask(h)) >> PAGE_SHIFT;
- page = pte_page(huge_ptep_get(pte));
+ pfn_offset = (vaddr & ~hugetlb_pte_mask(&hpte)) >> PAGE_SHIFT;
+ subpage = pte_page(pte);
+ pages_per_hpte = hugetlb_pte_size(&hpte) / PAGE_SIZE;
+ page = compound_head(subpage);
VM_BUG_ON_PAGE((flags & FOLL_PIN) && PageAnon(page) &&
!PageAnonExclusive(page), page);
@@ -6833,22 +6849,22 @@ long follow_hugetlb_page(struct mm_struct *mm, struct vm_area_struct *vma,
* and skip the same_page loop below.
*/
if (!pages && !vmas && !pfn_offset &&
- (vaddr + huge_page_size(h) < vma->vm_end) &&
- (remainder >= pages_per_huge_page(h))) {
- vaddr += huge_page_size(h);
- remainder -= pages_per_huge_page(h);
- i += pages_per_huge_page(h);
+ (vaddr + hugetlb_pte_size(&hpte) < vma->vm_end) &&
+ (remainder >= pages_per_hpte)) {
+ vaddr += hugetlb_pte_size(&hpte);
+ remainder -= pages_per_hpte;
+ i += pages_per_hpte;
spin_unlock(ptl);
hugetlb_vma_unlock_read(vma);
continue;
}
/* vaddr may not be aligned to PAGE_SIZE */
- refs = min3(pages_per_huge_page(h) - pfn_offset, remainder,
+ refs = min3(pages_per_hpte - pfn_offset, remainder,
(vma->vm_end - ALIGN_DOWN(vaddr, PAGE_SIZE)) >> PAGE_SHIFT);
if (pages || vmas)
- record_subpages_vmas(nth_page(page, pfn_offset),
+ record_subpages_vmas(nth_page(subpage, pfn_offset),
vma, refs,
likely(pages) ? pages + i : NULL,
vmas ? vmas + i : NULL);
--
2.39.2.637.g21b0678d19-goog
Powered by blists - more mailing lists