| lists.openwall.net | lists / announce owl-users owl-dev john-users john-dev passwdqc-users yescrypt popa3d-users / oss-security kernel-hardening musl sabotage tlsify passwords / crypt-dev xvendor / Bugtraq Full-Disclosure linux-kernel linux-netdev linux-ext4 linux-hardening linux-cve-announce PHC | |
|
Open Source and information security mailing list archives
| ||
|
Message-Id: <20200928175428.4110504-5-zi.yan@sent.com>
Date: Mon, 28 Sep 2020 13:54:02 -0400
From: Zi Yan <zi.yan@...t.com>
To: linux-mm@...ck.org
Cc: "Kirill A . Shutemov" <kirill.shutemov@...ux.intel.com>,
Roman Gushchin <guro@...com>, Rik van Riel <riel@...riel.com>,
Matthew Wilcox <willy@...radead.org>,
Shakeel Butt <shakeelb@...gle.com>,
Yang Shi <shy828301@...il.com>,
Jason Gunthorpe <jgg@...dia.com>,
Mike Kravetz <mike.kravetz@...cle.com>,
Michal Hocko <mhocko@...e.com>,
David Hildenbrand <david@...hat.com>,
William Kucharski <william.kucharski@...cle.com>,
Andrea Arcangeli <aarcange@...hat.com>,
John Hubbard <jhubbard@...dia.com>,
David Nellans <dnellans@...dia.com>,
linux-kernel@...r.kernel.org, Zi Yan <ziy@...dia.com>
Subject: [RFC PATCH v2 04/30] mm: add new helper functions to allocate one PMD page with 512 PTE pages.
From: Zi Yan <ziy@...dia.com>
This prepares for PUD THP support, which allocates 512 of such PMD pages
when creating a PUD THP. These page table pages will be withdrawn during
THP split.
Signed-off-by: Zi Yan <ziy@...dia.com>
---
arch/x86/include/asm/pgalloc.h | 60 ++++++++++++++++++++++++++++++++++
arch/x86/mm/pgtable.c | 25 ++++++++++++++
include/linux/huge_mm.h | 3 ++
3 files changed, 88 insertions(+)
diff --git a/arch/x86/include/asm/pgalloc.h b/arch/x86/include/asm/pgalloc.h
index 62ad61d6fefc..b24284522973 100644
--- a/arch/x86/include/asm/pgalloc.h
+++ b/arch/x86/include/asm/pgalloc.h
@@ -52,6 +52,19 @@ extern pgd_t *pgd_alloc(struct mm_struct *);
extern void pgd_free(struct mm_struct *mm, pgd_t *pgd);
extern pgtable_t pte_alloc_one(struct mm_struct *);
+extern pgtable_t pte_alloc_order(struct mm_struct *mm, unsigned long address,
+ int order);
+
+static inline void pte_free_order(struct mm_struct *mm, struct page *pte,
+ int order)
+{
+ int i;
+
+ for (i = 0; i < (1<<order); i++) {
+ pgtable_pte_page_dtor(&pte[i]);
+ __free_page(&pte[i]);
+ }
+}
extern void ___pte_free_tlb(struct mmu_gather *tlb, struct page *pte);
@@ -87,6 +100,53 @@ static inline void pmd_populate(struct mm_struct *mm, pmd_t *pmd,
#define pmd_pgtable(pmd) pmd_page(pmd)
#if CONFIG_PGTABLE_LEVELS > 2
+static inline pmd_t *pmd_alloc_one_page_with_ptes(struct mm_struct *mm, unsigned long addr)
+{
+ pgtable_t pte_pgtables;
+ pmd_t *pmd;
+ spinlock_t *pmd_ptl;
+ int i;
+
+ pte_pgtables = pte_alloc_order(mm, addr,
+ HPAGE_PUD_ORDER - HPAGE_PMD_ORDER);
+ if (!pte_pgtables)
+ return NULL;
+
+ pmd = pmd_alloc_one(mm, addr);
+ if (unlikely(!pmd)) {
+ pte_free_order(mm, pte_pgtables,
+ HPAGE_PUD_ORDER - HPAGE_PMD_ORDER);
+ return NULL;
+ }
+ pmd_ptl = pmd_lock(mm, pmd);
+
+ for (i = 0; i < (1<<(HPAGE_PUD_ORDER - HPAGE_PMD_ORDER)); i++)
+ pgtable_trans_huge_deposit(mm, pmd, pte_pgtables + i);
+
+ spin_unlock(pmd_ptl);
+
+ return pmd;
+}
+
+static inline void pmd_free_page_with_ptes(struct mm_struct *mm, pmd_t *pmd)
+{
+ spinlock_t *pmd_ptl;
+ int i;
+
+ BUG_ON((unsigned long)pmd & (PAGE_SIZE-1));
+ pmd_ptl = pmd_lock(mm, pmd);
+
+ for (i = 0; i < (1<<(HPAGE_PUD_ORDER - HPAGE_PMD_ORDER)); i++) {
+ pgtable_t pte_pgtable;
+
+ pte_pgtable = pgtable_trans_huge_withdraw(mm, pmd);
+ pte_free(mm, pte_pgtable);
+ }
+
+ spin_unlock(pmd_ptl);
+ pmd_free(mm, pmd);
+}
+
extern void ___pmd_free_tlb(struct mmu_gather *tlb, pmd_t *pmd);
static inline void __pmd_free_tlb(struct mmu_gather *tlb, pmd_t *pmd,
diff --git a/arch/x86/mm/pgtable.c b/arch/x86/mm/pgtable.c
index dfd82f51ba66..7be73aee6183 100644
--- a/arch/x86/mm/pgtable.c
+++ b/arch/x86/mm/pgtable.c
@@ -33,6 +33,31 @@ pgtable_t pte_alloc_one(struct mm_struct *mm)
return __pte_alloc_one(mm, __userpte_alloc_gfp);
}
+pgtable_t pte_alloc_order(struct mm_struct *mm, unsigned long address, int order)
+{
+ struct page *pte;
+ int i;
+
+ pte = alloc_pages(__userpte_alloc_gfp, order);
+ if (!pte)
+ return NULL;
+ split_page(pte, order);
+ for (i = 1; i < (1 << order); i++)
+ set_page_private(pte + i, 0);
+
+ for (i = 0; i < (1<<order); i++) {
+ if (!pgtable_pte_page_ctor(&pte[i])) {
+ __free_page(&pte[i]);
+ while (--i >= 0) {
+ pgtable_pte_page_dtor(&pte[i]);
+ __free_page(&pte[i]);
+ }
+ return NULL;
+ }
+ }
+ return pte;
+}
+
static int __init setup_userpte(char *arg)
{
if (!arg)
diff --git a/include/linux/huge_mm.h b/include/linux/huge_mm.h
index 8a8bc46a2432..e9d228d4fc69 100644
--- a/include/linux/huge_mm.h
+++ b/include/linux/huge_mm.h
@@ -115,6 +115,9 @@ extern struct kobj_attribute shmem_enabled_attr;
#define HPAGE_PMD_ORDER (HPAGE_PMD_SHIFT-PAGE_SHIFT)
#define HPAGE_PMD_NR (1<<HPAGE_PMD_ORDER)
+#define HPAGE_PUD_ORDER (HPAGE_PUD_SHIFT-PAGE_SHIFT)
+#define HPAGE_PUD_NR (1<<HPAGE_PUD_ORDER)
+
#ifdef CONFIG_TRANSPARENT_HUGEPAGE
#define HPAGE_PMD_SHIFT PMD_SHIFT
#define HPAGE_PMD_SIZE ((1UL) << HPAGE_PMD_SHIFT)
--
2.28.0
Powered by blists - more mailing lists