>From afce8c6e41e215c7e3ba2272c8431a9acf71305a Mon Sep 17 00:00:00 2001 From: KOSAKI Motohiro Date: Thu, 9 Jun 2011 16:19:33 +0900 Subject: [PATCH 2/2] make slowpath Signed-off-by: KOSAKI Motohiro --- kernel/futex.c | 60 ++++++++++++++++++++++++++++++++++++++++++++++++++++--- 1 files changed, 56 insertions(+), 4 deletions(-) diff --git a/kernel/futex.c b/kernel/futex.c index 1a91ea2..5382c05 100644 --- a/kernel/futex.c +++ b/kernel/futex.c @@ -213,6 +213,58 @@ static void drop_futex_key_refs(union futex_key *key) } } +static noinline int +get_futex_key_slowpath(unsigned long address, union futex_key *key, int rw) +{ + int err; + struct mm_struct *mm = current->mm; + struct page *page; + struct vm_area_struct *vma; + + down_read(&mm->mmap_sem); + + vma = find_extend_vma(mm, address); + err = -EFAULT; + if (unlikely(!vma)) + goto err_out; + + err = -EACCES; + if (!(vma->vm_flags & (rw==VERIFY_READ ? VM_READ : VM_WRITE))) + goto err_out; + + err = 0; + if (likely(!(vma->vm_flags & VM_MAYSHARE))) { + key->both.offset |= FUT_OFF_MMSHARED; + key->private.mm = mm; + key->private.address = address; + goto out; + } + + if (likely(!(vma->vm_flags & VM_NONLINEAR))) { + key->shared.inode = vma->vm_file->f_path.dentry->d_inode; + key->both.offset |= FUT_OFF_INODE; + key->shared.pgoff = (((address - vma->vm_start) >> PAGE_SHIFT) + + vma->vm_pgoff); + goto out; + } + + err = get_user_pages(current, mm, address, 1, 0, 0, &page, NULL); + if (err < 0) + goto err_out; + + key->shared.inode = vma->vm_file->f_path.dentry->d_inode; + key->both.offset |= FUT_OFF_INODE; + key->shared.pgoff = page->index << (PAGE_CACHE_SHIFT - PAGE_SHIFT); + put_page(page); + + out: + get_futex_key_refs(key); + err_out: + up_read(¤t->mm->mmap_sem); + + return err; +} + /** * get_futex_key() - Get parameters which are the keys for a futex * @uaddr: virtual address of the futex @@ -236,7 +288,7 @@ get_futex_key(u32 __user *uaddr, int fshared, union futex_key *key, int rw) unsigned long address = (unsigned long)uaddr; struct mm_struct *mm = current->mm; struct page *page, *page_head; - int err; + int ret; /* * The futex address must be "naturally" aligned. @@ -263,9 +315,9 @@ get_futex_key(u32 __user *uaddr, int fshared, union futex_key *key, int rw) } again: - err = get_user_pages_fast(address, 1, rw == VERIFY_WRITE, &page); - if (err < 0) - return err; + ret = __get_user_pages_fast(address, 1, 1, &page); + if (unlikely(!ret)) + return get_futex_key_slowpath(address, key, rw); #ifdef CONFIG_TRANSPARENT_HUGEPAGE page_head = page; -- 1.7.3.1