| lists.openwall.net | lists / announce owl-users owl-dev john-users john-dev passwdqc-users yescrypt popa3d-users / oss-security kernel-hardening musl sabotage tlsify passwords / crypt-dev xvendor / Bugtraq Full-Disclosure linux-kernel linux-netdev linux-ext4 linux-hardening linux-cve-announce PHC | |
|
Open Source and information security mailing list archives
| ||
|
Message-Id: <20241205103729.14798-5-luxu.kernel@bytedance.com>
Date: Thu, 5 Dec 2024 18:37:12 +0800
From: Xu Lu <luxu.kernel@...edance.com>
To: paul.walmsley@...ive.com,
palmer@...belt.com,
aou@...s.berkeley.edu,
ardb@...nel.org,
anup@...infault.org,
atishp@...shpatra.org
Cc: xieyongji@...edance.com,
lihangjing@...edance.com,
punit.agrawal@...edance.com,
linux-kernel@...r.kernel.org,
linux-riscv@...ts.infradead.org,
Xu Lu <luxu.kernel@...edance.com>
Subject: [RFC PATCH v2 04/21] riscv: mm: Reimplement page table entry constructor function
This commit reimplements the page table entry constructor. As each page
can contains several hardware pages now, the pte constructor need to
initialize all mapping entries of these hardware pages. Note that the
step path between mapping entries differs in different page table entry
levels. For example, in PTE level, the step path between hardware
mapping entries is hardware page size (aka 4K). In PMD level, the step
path is (2 ^ 9) * hardware page size (aka 2M), etc.
Signed-off-by: Xu Lu <luxu.kernel@...edance.com>
---
arch/riscv/include/asm/pgtable-32.h | 5 +++
arch/riscv/include/asm/pgtable-64.h | 41 +++++++++++++++++++---
arch/riscv/include/asm/pgtable.h | 54 ++++++++++++++++++++++++-----
arch/riscv/mm/pgtable.c | 47 +++++++++++++++++++++++++
4 files changed, 133 insertions(+), 14 deletions(-)
diff --git a/arch/riscv/include/asm/pgtable-32.h b/arch/riscv/include/asm/pgtable-32.h
index 159a668c3dd8..2959ab72f926 100644
--- a/arch/riscv/include/asm/pgtable-32.h
+++ b/arch/riscv/include/asm/pgtable-32.h
@@ -37,4 +37,9 @@
static const __maybe_unused int pgtable_l4_enabled;
static const __maybe_unused int pgtable_l5_enabled;
+static inline int __pgd_present(unsigned long pgdval)
+{
+ return pgdval & _PAGE_PRESENT;
+}
+
#endif /* _ASM_RISCV_PGTABLE_32_H */
diff --git a/arch/riscv/include/asm/pgtable-64.h b/arch/riscv/include/asm/pgtable-64.h
index e736873d7768..efcf63667f93 100644
--- a/arch/riscv/include/asm/pgtable-64.h
+++ b/arch/riscv/include/asm/pgtable-64.h
@@ -204,9 +204,14 @@ static inline u64 riscv_page_io(void)
_PAGE_USER | _PAGE_GLOBAL | \
_PAGE_MTMASK))
+static inline int __pud_present(unsigned long pudval)
+{
+ return pudval & _PAGE_PRESENT;
+}
+
static inline int pud_present(pud_t pud)
{
- return (pud_val(pud) & _PAGE_PRESENT);
+ return __pud_present(pud_val(pud));
}
static inline int pud_none(pud_t pud)
@@ -219,11 +224,16 @@ static inline int pud_bad(pud_t pud)
return !pud_present(pud);
}
-#define pud_leaf pud_leaf
+static inline bool __pud_leaf(unsigned long pudval)
+{
+ return __pud_present(pudval) && (pudval & _PAGE_LEAF);
+}
+
static inline bool pud_leaf(pud_t pud)
{
- return pud_present(pud) && (pud_val(pud) & _PAGE_LEAF);
+ return __pud_leaf(pud_val(pud));
}
+#define pud_leaf pud_leaf
static inline int pud_user(pud_t pud)
{
@@ -321,14 +331,30 @@ static inline int p4d_none(p4d_t p4d)
return 0;
}
+static inline int __p4d_present(unsigned long p4dval)
+{
+ return p4dval & _PAGE_PRESENT;
+}
+
static inline int p4d_present(p4d_t p4d)
{
if (pgtable_l4_enabled)
- return (p4d_val(p4d) & _PAGE_PRESENT);
+ return __p4d_present(p4d_val(p4d));
return 1;
}
+static inline int __p4d_leaf(unsigned long p4dval)
+{
+ return 0;
+}
+
+static inline int p4d_leaf(p4d_t p4d)
+{
+ return __p4d_leaf(p4d_val(p4d));
+}
+#define p4d_leaf p4d_leaf
+
static inline int p4d_bad(p4d_t p4d)
{
if (pgtable_l4_enabled)
@@ -388,10 +414,15 @@ static inline int pgd_none(pgd_t pgd)
return 0;
}
+static inline int __pgd_present(unsigned long pgdval)
+{
+ return pgdval & _PAGE_PRESENT;
+}
+
static inline int pgd_present(pgd_t pgd)
{
if (pgtable_l5_enabled)
- return (pgd_val(pgd) & _PAGE_PRESENT);
+ return __pgd_present(pgd_val(pgd));
return 1;
}
diff --git a/arch/riscv/include/asm/pgtable.h b/arch/riscv/include/asm/pgtable.h
index f9aed43809b3..1d5f533edbd5 100644
--- a/arch/riscv/include/asm/pgtable.h
+++ b/arch/riscv/include/asm/pgtable.h
@@ -220,8 +220,19 @@ static inline unsigned long satp_pfn(unsigned long satp)
return hwpfn_to_pfn(hwpfn);
}
+static inline int __pgd_leaf(unsigned long pgdval)
+{
+ return __pgd_present(pgdval) && (pgdval & _PAGE_LEAF);
+}
+
+static inline int pgd_leaf(pgd_t pgd)
+{
+ return __pgd_leaf(pgd_val(pgd));
+}
+#define pgd_leaf pgd_leaf
+
#ifdef CONFIG_TRANSPARENT_HUGEPAGE
-static inline int pmd_present(pmd_t pmd)
+static inline int __pmd_present(unsigned long pmdval)
{
/*
* Checking for _PAGE_LEAF is needed too because:
@@ -229,15 +240,20 @@ static inline int pmd_present(pmd_t pmd)
* the present bit, in this situation, pmd_present() and
* pmd_trans_huge() still needs to return true.
*/
- return (pmd_val(pmd) & (_PAGE_PRESENT | _PAGE_PROT_NONE | _PAGE_LEAF));
+ return (pmdval & (_PAGE_PRESENT | _PAGE_PROT_NONE | _PAGE_LEAF));
}
#else
-static inline int pmd_present(pmd_t pmd)
+static inline int __pmd_present(unsigned long pmdval)
{
- return (pmd_val(pmd) & (_PAGE_PRESENT | _PAGE_PROT_NONE));
+ return (pmdval & (_PAGE_PRESENT | _PAGE_PROT_NONE));
}
#endif
+static inline int pmd_present(pmd_t pmd)
+{
+ return __pmd_present(pmd_val(pmd));
+}
+
static inline int pmd_none(pmd_t pmd)
{
return (pmd_val(pmd) == 0);
@@ -248,11 +264,16 @@ static inline int pmd_bad(pmd_t pmd)
return !pmd_present(pmd) || (pmd_val(pmd) & _PAGE_LEAF);
}
-#define pmd_leaf pmd_leaf
+static inline bool __pmd_leaf(unsigned long pmdval)
+{
+ return __pmd_present(pmdval) && (pmdval & _PAGE_LEAF);
+}
+
static inline bool pmd_leaf(pmd_t pmd)
{
- return pmd_present(pmd) && (pmd_val(pmd) & _PAGE_LEAF);
+ return __pmd_leaf(pmd_val(pmd));
}
+#define pmd_leaf pmd_leaf
static inline void set_pmd(pmd_t *pmdp, pmd_t pmd)
{
@@ -306,9 +327,14 @@ static __always_inline bool has_svnapot(void)
return riscv_has_extension_likely(RISCV_ISA_EXT_SVNAPOT);
}
+static inline unsigned long __pte_napot(unsigned long val)
+{
+ return val & _PAGE_NAPOT;
+}
+
static inline unsigned long pte_napot(pte_t pte)
{
- return pte_val(pte) & _PAGE_NAPOT;
+ return __pte_napot(pte_val(pte));
}
static inline pte_t pte_mknapot(pte_t pte, unsigned int order)
@@ -324,11 +350,16 @@ static inline pte_t pte_mknapot(pte_t pte, unsigned int order)
static __always_inline bool has_svnapot(void) { return false; }
-static inline unsigned long pte_napot(pte_t pte)
+static inline unsigned long __pte_napot(unsigned long pteval)
{
return 0;
}
+static inline unsigned long pte_napot(pte_t pte)
+{
+ return __pte_napot(pte_val(pte));
+}
+
#endif /* CONFIG_RISCV_ISA_SVNAPOT */
/* Yields the page frame number (PFN) of a page table entry */
@@ -356,9 +387,14 @@ static inline pte_t pfn_pte(unsigned long pfn, pgprot_t prot)
#define mk_pte(page, prot) pfn_pte(page_to_pfn(page), prot)
+static inline int __pte_present(unsigned long pteval)
+{
+ return (pteval & (_PAGE_PRESENT | _PAGE_PROT_NONE));
+}
+
static inline int pte_present(pte_t pte)
{
- return (pte_val(pte) & (_PAGE_PRESENT | _PAGE_PROT_NONE));
+ return __pte_present(pte_val(pte));
}
#define pte_accessible pte_accessible
diff --git a/arch/riscv/mm/pgtable.c b/arch/riscv/mm/pgtable.c
index 0c6b2fc6be58..f57ada26a183 100644
--- a/arch/riscv/mm/pgtable.c
+++ b/arch/riscv/mm/pgtable.c
@@ -10,6 +10,13 @@
pte_t __pte(unsigned long pteval)
{
pte_t pte;
+ unsigned int i;
+
+ for (i = 0; i < HW_PAGES_PER_PAGE; i++) {
+ pte.ptes[i] = pteval;
+ if (__pte_present(pteval) && !__pte_napot(pteval))
+ pteval += 1 << _PAGE_HWPFN_SHIFT;
+ }
return pte;
}
@@ -18,6 +25,16 @@ EXPORT_SYMBOL(__pte);
pgd_t __pgd(unsigned long pgdval)
{
pgd_t pgd;
+ unsigned int i;
+
+ for (i = 0; i < HW_PAGES_PER_PAGE; i++) {
+ pgd.pgds[i] = pgdval;
+ if (__pgd_leaf(pgdval))
+ pgdval += (1 << (PGDIR_SHIFT - PAGE_SHIFT)) <<
+ _PAGE_HWPFN_SHIFT;
+ else if (__pgd_present(pgdval))
+ pgdval += 1 << _PAGE_HWPFN_SHIFT;
+ }
return pgd;
}
@@ -27,6 +44,16 @@ EXPORT_SYMBOL(__pgd);
p4d_t __p4d(unsigned long p4dval)
{
p4d_t p4d;
+ unsigned int i;
+
+ for (i = 0; i < HW_PAGES_PER_PAGE; i++) {
+ p4d.p4ds[i] = p4dval;
+ if (__p4d_leaf(p4dval))
+ p4dval += (1 << (P4D_SHIFT - PAGE_SHIFT)) <<
+ _PAGE_HWPFN_SHIFT;
+ else if (__p4d_present(p4dval))
+ p4dval += 1 << _PAGE_HWPFN_SHIFT;
+ }
return p4d;
}
@@ -35,6 +62,16 @@ EXPORT_SYMBOL(__p4d);
pud_t __pud(unsigned long pudval)
{
pud_t pud;
+ unsigned int i;
+
+ for (i = 0; i < HW_PAGES_PER_PAGE; i++) {
+ pud.puds[i] = pudval;
+ if (__pud_leaf(pudval))
+ pudval += (1 << (PUD_SHIFT - PAGE_SHIFT)) <<
+ _PAGE_HWPFN_SHIFT;
+ else if (__pud_present(pudval))
+ pudval += 1 << _PAGE_HWPFN_SHIFT;
+ }
return pud;
}
@@ -43,6 +80,16 @@ EXPORT_SYMBOL(__pud);
pmd_t __pmd(unsigned long pmdval)
{
pmd_t pmd;
+ unsigned int i;
+
+ for (i = 0; i < HW_PAGES_PER_PAGE; i++) {
+ pmd.pmds[i] = pmdval;
+ if (__pmd_leaf(pmdval))
+ pmdval += (1 << (PMD_SHIFT - PAGE_SHIFT)) <<
+ _PAGE_HWPFN_SHIFT;
+ else if (__pmd_present(pmdval))
+ pmdval += 1 << _PAGE_HWPFN_SHIFT;
+ }
return pmd;
}
--
2.20.1
Powered by blists - more mailing lists