Implement the x86 architecture support for speculative pagefaults. Signed-off-by: Peter Zijlstra --- arch/x86/mm/fault.c | 31 +++++++++++-------------------- arch/x86/mm/gup.c | 10 ++++++++++ 2 files changed, 21 insertions(+), 20 deletions(-) Index: linux-2.6/arch/x86/mm/fault.c =================================================================== --- linux-2.6.orig/arch/x86/mm/fault.c +++ linux-2.6/arch/x86/mm/fault.c @@ -777,30 +777,13 @@ bad_area_access_error(struct pt_regs *re __bad_area(regs, error_code, address, SEGV_ACCERR); } -/* TODO: fixup for "mm-invoke-oom-killer-from-page-fault.patch" */ -static void -out_of_memory(struct pt_regs *regs, unsigned long error_code, - unsigned long address) -{ - /* - * We ran out of memory, call the OOM killer, and return the userspace - * (which will retry the fault, or kill us if we got oom-killed): - */ - up_read(¤t->mm->mmap_sem); - - pagefault_out_of_memory(); -} - static void do_sigbus(struct pt_regs *regs, unsigned long error_code, unsigned long address, unsigned int fault) { struct task_struct *tsk = current; - struct mm_struct *mm = tsk->mm; int code = BUS_ADRERR; - up_read(&mm->mmap_sem); - /* Kernel mode? Handle exceptions or die: */ if (!(error_code & PF_USER)) no_context(regs, error_code, address); @@ -829,7 +812,7 @@ mm_fault_error(struct pt_regs *regs, uns unsigned long address, unsigned int fault) { if (fault & VM_FAULT_OOM) { - out_of_memory(regs, error_code, address); + pagefault_out_of_memory(); } else { if (fault & (VM_FAULT_SIGBUS|VM_FAULT_HWPOISON)) do_sigbus(regs, error_code, address, fault); @@ -1040,6 +1023,14 @@ do_page_fault(struct pt_regs *regs, unsi return; } + if (error_code & PF_USER) { + fault = handle_speculative_fault(mm, address, + error_code & PF_WRITE ? FAULT_FLAG_WRITE : 0); + + if (!(fault & VM_FAULT_RETRY)) + goto done; + } + /* * When running in the kernel we expect faults to occur only to * addresses in user space. All other faults represent errors in @@ -1119,6 +1110,8 @@ good_area: */ fault = handle_mm_fault(mm, vma, address, write ? FAULT_FLAG_WRITE : 0); + up_read(&mm->mmap_sem); +done: if (unlikely(fault & VM_FAULT_ERROR)) { mm_fault_error(regs, error_code, address, fault); return; @@ -1135,6 +1128,4 @@ good_area: } check_v8086_mode(regs, address, tsk); - - up_read(&mm->mmap_sem); } Index: linux-2.6/arch/x86/mm/gup.c =================================================================== --- linux-2.6.orig/arch/x86/mm/gup.c +++ linux-2.6/arch/x86/mm/gup.c @@ -373,3 +373,13 @@ slow_irqon: return ret; } } + +void pin_page_tables(void) +{ + local_irq_disable(); +} + +void unpin_page_tables(void) +{ + local_irq_enable(); +} -- -- To unsubscribe from this list: send the line "unsubscribe linux-kernel" in the body of a message to majordomo@vger.kernel.org More majordomo info at http://vger.kernel.org/majordomo-info.html Please read the FAQ at http://www.tux.org/lkml/