Xen requires all active pagetables to be marked read-only. When the base of the pagetable is loaded into %cr3, the hypervisor validates the entire pagetable and only allows the load to proceed if it all checks out. This is pretty slow, so to mitigate this cost Xen has a notion of pinned pagetables. Pinned pagetables are pagetables which are considered to be active even if no processor's cr3 is pointing to is. This means that it must remain read-only and all updates are validated by the hypervisor. This makes context switches much cheaper, because the hypervisor doesn't need to revalidate the pagetable each time. This also adds a new paravirt hook which is called during setup once the zones and memory allocator have been initialized. When the init_mm pagetable is first built, the struct page array does not yet exist, and so there's nowhere to put he init_mm pagetable's PG_pinned flags. Once the zones are initialized and the struct page array exists, we can set the PG_pinned flags for those pages. This patch also adds the Xen support for pte pages allocated out of highmem (highpte) by implementing xen_kmap_atomic_pte. Signed-off-by: Jeremy Fitzhardinge Signed-off-by: Chris Wright Cc: Zach Amsden --- arch/i386/xen/enlighten.c | 107 ++++++++++++---- arch/i386/xen/mmu.c | 289 ++++++++++++++++++++++++++++----------------- arch/i386/xen/mmu.h | 2 arch/i386/xen/xen-ops.h | 2 4 files changed, 266 insertions(+), 134 deletions(-) =================================================================== --- a/arch/i386/xen/enlighten.c +++ b/arch/i386/xen/enlighten.c @@ -21,6 +21,9 @@ #include #include #include +#include +#include +#include #include #include @@ -496,32 +499,59 @@ static void xen_write_cr3(unsigned long } } +/* Early in boot, while setting up the initial pagetable, assume + everything is pinned. */ +static void xen_alloc_pt_init(struct mm_struct *mm, u32 pfn) +{ + BUG_ON(mem_map); /* should only be used early */ + make_lowmem_page_readonly(__va(PFN_PHYS(pfn))); +} + +/* This needs to make sure the new pte page is pinned iff its being + attached to a pinned pagetable. */ static void xen_alloc_pt(struct mm_struct *mm, u32 pfn) { - /* XXX pfn isn't necessarily a lowmem page */ - make_lowmem_page_readonly(__va(PFN_PHYS(pfn))); -} - -static void xen_alloc_pd(u32 pfn) -{ - make_lowmem_page_readonly(__va(PFN_PHYS(pfn))); -} - -static void xen_release_pd(u32 pfn) -{ - make_lowmem_page_readwrite(__va(PFN_PHYS(pfn))); -} - + struct page *page = pfn_to_page(pfn); + + if (PagePinned(virt_to_page(mm->pgd))) { + SetPagePinned(page); + + if (!PageHighMem(page)) + make_lowmem_page_readonly(__va(PFN_PHYS(pfn))); + else + /* make sure there are no stray mappings of + this page */ + kmap_flush_unused(); + } +} + +/* This should never happen until we're OK to use struct page */ static void xen_release_pt(u32 pfn) { - make_lowmem_page_readwrite(__va(PFN_PHYS(pfn))); -} - -static void xen_alloc_pd_clone(u32 pfn, u32 clonepfn, - u32 start, u32 count) -{ - xen_alloc_pd(pfn); -} + struct page *page = pfn_to_page(pfn); + + if (PagePinned(page)) { + if (!PageHighMem(page)) + make_lowmem_page_readwrite(__va(PFN_PHYS(pfn))); + } +} + +#ifdef CONFIG_HIGHPTE +static void *xen_kmap_atomic_pte(struct page *page, enum km_type type) +{ + pgprot_t prot = PAGE_KERNEL; + + if (PagePinned(page)) + prot = PAGE_KERNEL_RO; + + if (0 && PageHighMem(page)) + printk("mapping highpte %lx type %d prot %s\n", + page_to_pfn(page), type, + (unsigned long)pgprot_val(prot) & _PAGE_RW ? "WRITE" : "READ"); + + return kmap_atomic_prot(page, type, prot); +} +#endif static __init void xen_pagetable_setup_start(pgd_t *base) { @@ -549,7 +579,7 @@ static __init void xen_pagetable_setup_s memcpy(pmd, (void *)pgd_page_vaddr(xen_pgd[i]), PAGE_SIZE); - xen_alloc_pd(PFN_DOWN(__pa(pmd))); + make_lowmem_page_readonly(pmd); set_pgd(&base[i], __pgd(1 + __pa(pmd))); } else @@ -570,6 +600,10 @@ static __init void xen_pagetable_setup_s static __init void xen_pagetable_setup_done(pgd_t *base) { + /* This will work as long as patching hasn't happened yet + (which it hasn't) */ + paravirt_ops.alloc_pt = xen_alloc_pt; + if (!xen_feature(XENFEAT_auto_translated_physmap)) { /* * Create a mapping for the shared info page. @@ -587,7 +621,19 @@ static __init void xen_pagetable_setup_d HYPERVISOR_shared_info = (struct shared_info *)__va(xen_start_info->shared_info); - xen_pgd_pin(base); + /* Actually pin the pagetable down, but we can't set PG_pinned + yet because the page structures don't exist yet. */ + { + struct mmuext_op op; +#ifdef CONFIG_X86_PAE + op.cmd = MMUEXT_PIN_L3_TABLE; +#else + op.cmd = MMUEXT_PIN_L3_TABLE; +#endif + op.arg1.mfn = pfn_to_mfn(PFN_DOWN(__pa(base))); + if (HYPERVISOR_mmuext_op(&op, 1, NULL, DOMID_SELF)) + BUG(); + } xen_vcpu_setup(smp_processor_id()); } @@ -604,6 +650,7 @@ static const struct paravirt_ops xen_par .memory_setup = xen_memory_setup, .arch_setup = xen_arch_setup, .init_IRQ = xen_init_IRQ, + .post_allocator_init = xen_mark_init_mm_pinned, .time_init = xen_time_init, .set_wallclock = xen_set_wallclock, @@ -684,11 +731,15 @@ static const struct paravirt_ops xen_par .pagetable_setup_start = xen_pagetable_setup_start, .pagetable_setup_done = xen_pagetable_setup_done, - .alloc_pt = xen_alloc_pt, - .alloc_pd = xen_alloc_pd, - .alloc_pd_clone = xen_alloc_pd_clone, - .release_pd = xen_release_pd, + .alloc_pt = xen_alloc_pt_init, .release_pt = xen_release_pt, + .alloc_pd = paravirt_nop, + .alloc_pd_clone = paravirt_nop, + .release_pd = paravirt_nop, + +#ifdef CONFIG_HIGHPTE + .kmap_atomic_pte = xen_kmap_atomic_pte, +#endif .set_pte = xen_set_pte, .set_pte_at = xen_set_pte_at, =================================================================== --- a/arch/i386/xen/mmu.c +++ b/arch/i386/xen/mmu.c @@ -38,18 +38,21 @@ * * Jeremy Fitzhardinge , XenSource Inc, 2007 */ +#include + #include - #include #include #include +#include #include -#include +#include #include #include +#include "multicalls.h" #include "mmu.h" xmaddr_t arbitrary_virt_to_machine(unsigned long address) @@ -87,17 +90,6 @@ void make_lowmem_page_readwrite(void *va ptev = pte_mkwrite(*pte); if(HYPERVISOR_update_va_mapping(address, ptev, 0)) - BUG(); -} - - -void xen_set_pte(pte_t *ptep, pte_t pte) -{ - struct mmu_update u; - - u.ptr = virt_to_machine(ptep).maddr; - u.val = pte_val_ma(pte); - if (HYPERVISOR_mmu_update(&u, 1, NULL, DOMID_SELF) < 0) BUG(); } @@ -110,18 +102,6 @@ void xen_set_pmd(pmd_t *ptr, pmd_t val) if (HYPERVISOR_mmu_update(&u, 1, NULL, DOMID_SELF) < 0) BUG(); } - -#ifdef CONFIG_X86_PAE -void xen_set_pud(pud_t *ptr, pud_t val) -{ - struct mmu_update u; - - u.ptr = virt_to_machine(ptr).maddr; - u.val = pud_val_ma(val); - if (HYPERVISOR_mmu_update(&u, 1, NULL, DOMID_SELF) < 0) - BUG(); -} -#endif /* * Associate a virtual page frame with a given physical page frame @@ -169,12 +149,29 @@ void xen_set_pte_at(struct mm_struct *mm } #ifdef CONFIG_X86_PAE +void xen_set_pud(pud_t *ptr, pud_t val) +{ + struct mmu_update u; + + u.ptr = virt_to_machine(ptr).maddr; + u.val = pud_val_ma(val); + if (HYPERVISOR_mmu_update(&u, 1, NULL, DOMID_SELF) < 0) + BUG(); +} + +void xen_set_pte(pte_t *ptep, pte_t pte) +{ + ptep->pte_high = pte.pte_high; + smp_wmb(); + ptep->pte_low = pte.pte_low; +} + void xen_set_pte_atomic(pte_t *ptep, pte_t pte) { set_64bit((u64 *)ptep, pte_val_ma(pte)); } -void xen_pte_clear(struct mm_struct *mm, unsigned long addr,pte_t *ptep) +void xen_pte_clear(struct mm_struct *mm, unsigned long addr, pte_t *ptep) { ptep->pte_low = 0; smp_wmb(); @@ -238,6 +235,11 @@ pgd_t xen_make_pgd(unsigned long long pg return (pgd_t){ pgd }; } #else /* !PAE */ +void xen_set_pte(pte_t *ptep, pte_t pte) +{ + *ptep = pte; +} + unsigned long xen_pte_val(pte_t pte) { unsigned long ret = pte.pte_low; @@ -246,13 +248,6 @@ unsigned long xen_pte_val(pte_t pte) ret = machine_to_phys(XMADDR(ret)).paddr; return ret; -} - -unsigned long xen_pmd_val(pmd_t pmd) -{ - /* a BUG here is a lot easier to track down than a NULL eip */ - BUG(); - return 0; } unsigned long xen_pgd_val(pgd_t pgd) @@ -271,13 +266,6 @@ pte_t xen_make_pte(unsigned long pte) return (pte_t){ pte }; } -pmd_t xen_make_pmd(unsigned long pmd) -{ - /* a BUG here is a lot easier to track down than a NULL eip */ - BUG(); - return __pmd(0); -} - pgd_t xen_make_pgd(unsigned long pgd) { if (pgd & _PAGE_PRESENT) @@ -288,109 +276,200 @@ pgd_t xen_make_pgd(unsigned long pgd) #endif /* CONFIG_X86_PAE */ - -static void pgd_walk_set_prot(void *pt, pgprot_t flags) -{ - unsigned long pfn = PFN_DOWN(__pa(pt)); - - if (HYPERVISOR_update_va_mapping((unsigned long)pt, - pfn_pte(pfn, flags), 0) < 0) - BUG(); -} - -static void pgd_walk(pgd_t *pgd_base, pgprot_t flags) +/* + (Yet another) pagetable walker. This one is intended for pinning a + pagetable. This means that it walks a pagetable and calls the + callback function on each page it finds making up the page table, + at every level. It walks the entire pagetable, but it only bothers + pinning pte pages which are below pte_limit. In the normal case + this will be TASK_SIZE, but at boot we need to pin up to + FIXADDR_TOP. But the important bit is that we don't pin beyond + there, because then we start getting into Xen's ptes. +*/ +static int pgd_walk(pgd_t *pgd_base, int (*func)(struct page *, unsigned), + unsigned long limit) { pgd_t *pgd = pgd_base; - pud_t *pud; - pmd_t *pmd; - pte_t *pte; - int g, u, m; + int flush = 0; + unsigned long addr = 0; + unsigned long pgd_next; + + BUG_ON(limit > FIXADDR_TOP); if (xen_feature(XENFEAT_auto_translated_physmap)) - return; - - for (g = 0; g < USER_PTRS_PER_PGD; g++, pgd++) { - if (pgd_none(*pgd)) + return 0; + + for (; addr != FIXADDR_TOP; pgd++, addr = pgd_next) { + pud_t *pud; + unsigned long pud_limit, pud_next; + + pgd_next = pud_limit = pgd_addr_end(addr, FIXADDR_TOP); + + if (!pgd_val(*pgd)) continue; + pud = pud_offset(pgd, 0); if (PTRS_PER_PUD > 1) /* not folded */ - pgd_walk_set_prot(pud,flags); - - for (u = 0; u < PTRS_PER_PUD; u++, pud++) { + flush |= (*func)(virt_to_page(pud), 0); + + for (; addr != pud_limit; pud++, addr = pud_next) { + pmd_t *pmd; + unsigned long pmd_limit; + + pud_next = pud_addr_end(addr, pud_limit); + + if (pud_next < limit) + pmd_limit = pud_next; + else + pmd_limit = limit; + if (pud_none(*pud)) continue; + pmd = pmd_offset(pud, 0); if (PTRS_PER_PMD > 1) /* not folded */ - pgd_walk_set_prot(pmd,flags); - - for (m = 0; m < PTRS_PER_PMD; m++, pmd++) { + flush |= (*func)(virt_to_page(pmd), 0); + + for (; addr != pmd_limit; pmd++) { + addr += (PAGE_SIZE * PTRS_PER_PTE); + if ((pmd_limit-1) < (addr-1)) { + addr = pmd_limit; + break; + } + if (pmd_none(*pmd)) continue; - /* This can get called before mem_map - is set up, so we assume nothing is - highmem at that point. */ - if (mem_map == NULL || - !PageHighMem(pmd_page(*pmd))) { - pte = pte_offset_kernel(pmd,0); - pgd_walk_set_prot(pte,flags); - } + flush |= (*func)(pmd_page(*pmd), 0); } } } - if (HYPERVISOR_update_va_mapping((unsigned long)pgd_base, - pfn_pte(PFN_DOWN(__pa(pgd_base)), - flags), - UVMF_TLB_FLUSH) < 0) - BUG(); -} - - -/* This is called just after a mm has been duplicated from its parent, - but it has not been used yet. We need to make sure that its - pagetable is all read-only, and can be pinned. */ + flush |= (*func)(virt_to_page(pgd_base), UVMF_TLB_FLUSH); + + return flush; +} + +static int pin_page(struct page *page, unsigned flags) +{ + unsigned pgfl = test_and_set_bit(PG_pinned, &page->flags); + int flush; + + if (pgfl) + flush = 0; /* already pinned */ + else if (PageHighMem(page)) + /* kmaps need flushing if we found an unpinned + highpage */ + flush = 1; + else { + void *pt = lowmem_page_address(page); + unsigned long pfn = page_to_pfn(page); + struct multicall_space mcs = __xen_mc_entry(0); + + flush = 0; + + MULTI_update_va_mapping(mcs.mc, (unsigned long)pt, + pfn_pte(pfn, PAGE_KERNEL_RO), + flags); + } + + return flush; +} + +/* This is called just after a mm has been created, but it has not + been used yet. We need to make sure that its pagetable is all + read-only, and can be pinned. */ void xen_pgd_pin(pgd_t *pgd) { - struct mmuext_op op; - - pgd_walk(pgd, PAGE_KERNEL_RO); - -#if defined(CONFIG_X86_PAE) - op.cmd = MMUEXT_PIN_L3_TABLE; + struct multicall_space mcs; + struct mmuext_op *op; + + xen_mc_batch(); + + if (pgd_walk(pgd, pin_page, TASK_SIZE)) + kmap_flush_unused(); + + mcs = __xen_mc_entry(sizeof(*op)); + op = mcs.args; + +#ifdef CONFIG_X86_PAE + op->cmd = MMUEXT_PIN_L3_TABLE; #else - op.cmd = MMUEXT_PIN_L2_TABLE; + op->cmd = MMUEXT_PIN_L2_TABLE; #endif - op.arg1.mfn = pfn_to_mfn(PFN_DOWN(__pa(pgd))); - if (HYPERVISOR_mmuext_op(&op, 1, NULL, DOMID_SELF) < 0) - BUG(); + op->arg1.mfn = pfn_to_mfn(PFN_DOWN(__pa(pgd))); + MULTI_mmuext_op(mcs.mc, op, 1, NULL, DOMID_SELF); + + xen_mc_issue(0); +} + +/* The init_mm pagetable is really pinned as soon as its created, but + that's before we have page structures to store the bits. So do all + the book-keeping now. */ +static __init int mark_pinned(struct page *page, unsigned flags) +{ + SetPagePinned(page); + return 0; +} + +void __init xen_mark_init_mm_pinned(void) +{ + pgd_walk(init_mm.pgd, mark_pinned, FIXADDR_TOP); +} + +static int unpin_page(struct page *page, unsigned flags) +{ + unsigned pgfl = test_and_clear_bit(PG_pinned, &page->flags); + + if (pgfl && !PageHighMem(page)) { + void *pt = lowmem_page_address(page); + unsigned long pfn = page_to_pfn(page); + struct multicall_space mcs = __xen_mc_entry(0); + + MULTI_update_va_mapping(mcs.mc, (unsigned long)pt, + pfn_pte(pfn, PAGE_KERNEL), + flags); + } + + return 0; /* never need to flush on unpin */ } /* Release a pagetables pages back as normal RW */ -void xen_pgd_unpin(pgd_t *pgd) -{ - struct mmuext_op op; - - op.cmd = MMUEXT_UNPIN_TABLE; - op.arg1.mfn = pfn_to_mfn(PFN_DOWN(__pa(pgd))); - - if (HYPERVISOR_mmuext_op(&op, 1, NULL, DOMID_SELF) < 0) - BUG(); - - pgd_walk(pgd, PAGE_KERNEL); +static void xen_pgd_unpin(pgd_t *pgd) +{ + struct mmuext_op *op; + struct multicall_space mcs; + + xen_mc_batch(); + + mcs = __xen_mc_entry(sizeof(*op)); + + op = mcs.args; + op->cmd = MMUEXT_UNPIN_TABLE; + op->arg1.mfn = pfn_to_mfn(PFN_DOWN(__pa(pgd))); + + MULTI_mmuext_op(mcs.mc, op, 1, NULL, DOMID_SELF); + + pgd_walk(pgd, unpin_page, TASK_SIZE); + + xen_mc_issue(0); } void xen_activate_mm(struct mm_struct *prev, struct mm_struct *next) { + spin_lock(&next->page_table_lock); xen_pgd_pin(next->pgd); + spin_unlock(&next->page_table_lock); } void xen_dup_mmap(struct mm_struct *oldmm, struct mm_struct *mm) { + spin_lock(&mm->page_table_lock); xen_pgd_pin(mm->pgd); + spin_unlock(&mm->page_table_lock); } void xen_exit_mmap(struct mm_struct *mm) =================================================================== --- a/arch/i386/xen/mmu.h +++ b/arch/i386/xen/mmu.h @@ -15,7 +15,7 @@ void xen_exit_mmap(struct mm_struct *mm) void xen_exit_mmap(struct mm_struct *mm); void xen_pgd_pin(pgd_t *pgd); -void xen_pgd_unpin(pgd_t *pgd); +//void xen_pgd_unpin(pgd_t *pgd); #ifdef CONFIG_X86_PAE unsigned long long xen_pte_val(pte_t); =================================================================== --- a/arch/i386/xen/xen-ops.h +++ b/arch/i386/xen/xen-ops.h @@ -20,6 +20,8 @@ int xen_set_wallclock(unsigned long time int xen_set_wallclock(unsigned long time); cycle_t xen_clocksource_read(void); +void xen_mark_init_mm_pinned(void); + DECLARE_PER_CPU(enum paravirt_lazy_mode, xen_lazy_mode); static inline unsigned xen_get_lazy_mode(void) -- - To unsubscribe from this list: send the line "unsubscribe linux-kernel" in the body of a message to majordomo@vger.kernel.org More majordomo info at http://vger.kernel.org/majordomo-info.html Please read the FAQ at http://www.tux.org/lkml/