| lists.openwall.net | lists / announce owl-users owl-dev john-users john-dev passwdqc-users yescrypt popa3d-users / oss-security kernel-hardening musl sabotage tlsify passwords / crypt-dev xvendor / Bugtraq Full-Disclosure linux-kernel linux-netdev linux-ext4 linux-hardening linux-cve-announce PHC | |
|
Open Source and information security mailing list archives
| ||
|
Date: Mon, 5 Feb 2018 17:09:56 +0100
From: Laurent Dufour <ldufour@...ux.vnet.ibm.com>
To: Davidlohr Bueso <dbueso@...e.de>, akpm@...ux-foundation.org,
mingo@...nel.org
Cc: peterz@...radead.org, jack@...e.cz, mhocko@...nel.org,
kirill.shutemov@...ux.intel.com, mawilcox@...rosoft.com,
mgorman@...hsingularity.net, dave@...olabs.net, linux-mm@...ck.org,
linux-kernel@...r.kernel.org
Subject: Re: [PATCH 06/64] mm: teach pagefault paths about range locking
On 05/02/2018 02:26, Davidlohr Bueso wrote:
> From: Davidlohr Bueso <dave@...olabs.net>
>
> In handle_mm_fault() we need to remember the range lock specified
> when the mmap_sem was first taken as pf paths can drop the lock.
> Although this patch may seem far too big at first, it is so due to
> bisectability, and later conversion patches become quite easy to
> follow. Furthermore, most of what this patch does is pass a pointer
> to an 'mmrange' stack allocated parameter that is later used by the
> vm_fault structure. The new interfaces are pretty much all in the
> following areas:
>
> - vma handling (vma_merge(), vma_adjust(), split_vma(), copy_vma())
> - gup family (all except get_user_pages_unlocked(), which internally
> passes the mmrange).
> - mm walking (walk_page_vma())
> - mmap/unmap (do_mmap(), do_munmap())
> - handle_mm_fault(), fixup_user_fault()
>
> Most of the pain of the patch is updating all callers in the kernel
> for this. While tedious, it is not that hard to review, I hope.
> The idea is to use a local variable (no concurrency) whenever the
> mmap_sem is taken and we end up in pf paths that end up retaking
> the lock. Ie:
>
> DEFINE_RANGE_LOCK_FULL(mmrange);
>
> down_write(&mm->mmap_sem);
> some_fn(a, b, c, &mmrange);
> ....
> ....
> ...
> handle_mm_fault(vma, addr, flags, mmrange);
> ...
> up_write(&mm->mmap_sem);
>
> Semantically nothing changes at all, and the 'mmrange' ends up
> being unused for now. Later patches will use the variable when
> the mmap_sem wrappers replace straightforward down/up.
>
> Compile tested defconfigs on various non-x86 archs without breaking.
>
> Signed-off-by: Davidlohr Bueso <dbueso@...e.de>
> ---
> arch/alpha/mm/fault.c | 3 +-
> arch/arc/mm/fault.c | 3 +-
> arch/arm/mm/fault.c | 8 ++-
> arch/arm/probes/uprobes/core.c | 5 +-
> arch/arm64/mm/fault.c | 7 ++-
> arch/cris/mm/fault.c | 3 +-
> arch/frv/mm/fault.c | 3 +-
> arch/hexagon/mm/vm_fault.c | 3 +-
> arch/ia64/mm/fault.c | 3 +-
> arch/m32r/mm/fault.c | 3 +-
> arch/m68k/mm/fault.c | 3 +-
> arch/metag/mm/fault.c | 3 +-
> arch/microblaze/mm/fault.c | 3 +-
> arch/mips/kernel/vdso.c | 3 +-
> arch/mips/mm/fault.c | 3 +-
> arch/mn10300/mm/fault.c | 3 +-
> arch/nios2/mm/fault.c | 3 +-
> arch/openrisc/mm/fault.c | 3 +-
> arch/parisc/mm/fault.c | 3 +-
> arch/powerpc/include/asm/mmu_context.h | 3 +-
> arch/powerpc/include/asm/powernv.h | 5 +-
> arch/powerpc/mm/copro_fault.c | 4 +-
> arch/powerpc/mm/fault.c | 3 +-
> arch/powerpc/platforms/powernv/npu-dma.c | 5 +-
> arch/riscv/mm/fault.c | 3 +-
> arch/s390/include/asm/gmap.h | 14 +++--
> arch/s390/kvm/gaccess.c | 31 ++++++----
> arch/s390/mm/fault.c | 3 +-
> arch/s390/mm/gmap.c | 80 +++++++++++++++---------
> arch/score/mm/fault.c | 3 +-
> arch/sh/mm/fault.c | 3 +-
> arch/sparc/mm/fault_32.c | 6 +-
> arch/sparc/mm/fault_64.c | 3 +-
> arch/tile/mm/fault.c | 3 +-
> arch/um/include/asm/mmu_context.h | 3 +-
> arch/um/kernel/trap.c | 3 +-
> arch/unicore32/mm/fault.c | 8 ++-
> arch/x86/entry/vdso/vma.c | 3 +-
> arch/x86/include/asm/mmu_context.h | 5 +-
> arch/x86/include/asm/mpx.h | 6 +-
> arch/x86/mm/fault.c | 3 +-
> arch/x86/mm/mpx.c | 41 ++++++++-----
> arch/xtensa/mm/fault.c | 3 +-
> drivers/gpu/drm/amd/amdgpu/amdgpu_ttm.c | 3 +-
> drivers/gpu/drm/i915/i915_gem_userptr.c | 4 +-
> drivers/gpu/drm/radeon/radeon_ttm.c | 4 +-
> drivers/infiniband/core/umem.c | 3 +-
> drivers/infiniband/core/umem_odp.c | 3 +-
> drivers/infiniband/hw/qib/qib_user_pages.c | 7 ++-
> drivers/infiniband/hw/usnic/usnic_uiom.c | 3 +-
> drivers/iommu/amd_iommu_v2.c | 5 +-
> drivers/iommu/intel-svm.c | 5 +-
> drivers/media/v4l2-core/videobuf-dma-sg.c | 18 ++++--
> drivers/misc/mic/scif/scif_rma.c | 3 +-
> drivers/misc/sgi-gru/grufault.c | 43 ++++++++-----
> drivers/vfio/vfio_iommu_type1.c | 3 +-
> fs/aio.c | 3 +-
> fs/binfmt_elf.c | 3 +-
> fs/exec.c | 20 ++++--
> fs/proc/internal.h | 3 +
> fs/proc/task_mmu.c | 29 ++++++---
> fs/proc/vmcore.c | 14 ++++-
> fs/userfaultfd.c | 18 +++---
> include/asm-generic/mm_hooks.h | 3 +-
> include/linux/hmm.h | 4 +-
> include/linux/ksm.h | 6 +-
> include/linux/migrate.h | 4 +-
> include/linux/mm.h | 73 +++++++++++++---------
> include/linux/uprobes.h | 15 +++--
> ipc/shm.c | 14 +++--
> kernel/events/uprobes.c | 49 +++++++++------
> kernel/futex.c | 3 +-
> mm/frame_vector.c | 4 +-
> mm/gup.c | 60 ++++++++++--------
> mm/hmm.c | 37 ++++++-----
> mm/internal.h | 3 +-
> mm/ksm.c | 24 +++++---
> mm/madvise.c | 58 ++++++++++-------
> mm/memcontrol.c | 13 ++--
> mm/memory.c | 10 +--
> mm/mempolicy.c | 35 ++++++-----
> mm/migrate.c | 20 +++---
> mm/mincore.c | 24 +++++---
> mm/mlock.c | 33 ++++++----
> mm/mmap.c | 99 +++++++++++++++++-------------
> mm/mprotect.c | 14 +++--
> mm/mremap.c | 30 +++++----
> mm/nommu.c | 32 ++++++----
> mm/pagewalk.c | 56 +++++++++--------
> mm/process_vm_access.c | 4 +-
> mm/util.c | 3 +-
> security/tomoyo/domain.c | 3 +-
> virt/kvm/async_pf.c | 3 +-
> virt/kvm/kvm_main.c | 16 +++--
> 94 files changed, 784 insertions(+), 474 deletions(-)
>
> diff --git a/arch/alpha/mm/fault.c b/arch/alpha/mm/fault.c
> index cd3c572ee912..690d86a00a20 100644
> --- a/arch/alpha/mm/fault.c
> +++ b/arch/alpha/mm/fault.c
> @@ -90,6 +90,7 @@ do_page_fault(unsigned long address, unsigned long mmcsr,
> int fault, si_code = SEGV_MAPERR;
> siginfo_t info;
> unsigned int flags = FAULT_FLAG_ALLOW_RETRY | FAULT_FLAG_KILLABLE;
> + DEFINE_RANGE_LOCK_FULL(mmrange);
>
> /* As of EV6, a load into $31/$f31 is a prefetch, and never faults
> (or is suppressed by the PALcode). Support that for older CPUs
> @@ -148,7 +149,7 @@ do_page_fault(unsigned long address, unsigned long mmcsr,
> /* If for any reason at all we couldn't handle the fault,
> make sure we exit gracefully rather than endlessly redo
> the fault. */
> - fault = handle_mm_fault(vma, address, flags);
> + fault = handle_mm_fault(vma, address, flags, &mmrange);
>
> if ((fault & VM_FAULT_RETRY) && fatal_signal_pending(current))
> return;
> diff --git a/arch/arc/mm/fault.c b/arch/arc/mm/fault.c
> index a0b7bd6d030d..e423f764f159 100644
> --- a/arch/arc/mm/fault.c
> +++ b/arch/arc/mm/fault.c
> @@ -69,6 +69,7 @@ void do_page_fault(unsigned long address, struct pt_regs *regs)
> int fault, ret;
> int write = regs->ecr_cause & ECR_C_PROTV_STORE; /* ST/EX */
> unsigned int flags = FAULT_FLAG_ALLOW_RETRY | FAULT_FLAG_KILLABLE;
> + DEFINE_RANGE_LOCK_FULL(mmrange);
>
> /*
> * We fault-in kernel-space virtual memory on-demand. The
> @@ -137,7 +138,7 @@ void do_page_fault(unsigned long address, struct pt_regs *regs)
> * make sure we exit gracefully rather than endlessly redo
> * the fault.
> */
> - fault = handle_mm_fault(vma, address, flags);
> + fault = handle_mm_fault(vma, address, flags, &mmrange);
>
> /* If Pagefault was interrupted by SIGKILL, exit page fault "early" */
> if (unlikely(fatal_signal_pending(current))) {
> diff --git a/arch/arm/mm/fault.c b/arch/arm/mm/fault.c
> index b75eada23d0a..99ae40b5851a 100644
> --- a/arch/arm/mm/fault.c
> +++ b/arch/arm/mm/fault.c
> @@ -221,7 +221,8 @@ static inline bool access_error(unsigned int fsr, struct vm_area_struct *vma)
>
> static int __kprobes
> __do_page_fault(struct mm_struct *mm, unsigned long addr, unsigned int fsr,
> - unsigned int flags, struct task_struct *tsk)
> + unsigned int flags, struct task_struct *tsk,
> + struct range_lock *mmrange)
> {
> struct vm_area_struct *vma;
> int fault;
> @@ -243,7 +244,7 @@ __do_page_fault(struct mm_struct *mm, unsigned long addr, unsigned int fsr,
> goto out;
> }
>
> - return handle_mm_fault(vma, addr & PAGE_MASK, flags);
> + return handle_mm_fault(vma, addr & PAGE_MASK, flags, mmrange);
>
> check_stack:
> /* Don't allow expansion below FIRST_USER_ADDRESS */
> @@ -261,6 +262,7 @@ do_page_fault(unsigned long addr, unsigned int fsr, struct pt_regs *regs)
> struct mm_struct *mm;
> int fault, sig, code;
> unsigned int flags = FAULT_FLAG_ALLOW_RETRY | FAULT_FLAG_KILLABLE;
> + DEFINE_RANGE_LOCK_FULL(mmrange);
>
> if (notify_page_fault(regs, fsr))
> return 0;
> @@ -308,7 +310,7 @@ do_page_fault(unsigned long addr, unsigned int fsr, struct pt_regs *regs)
> #endif
> }
>
> - fault = __do_page_fault(mm, addr, fsr, flags, tsk);
> + fault = __do_page_fault(mm, addr, fsr, flags, tsk, &mmrange);
>
> /* If we need to retry but a fatal signal is pending, handle the
> * signal first. We do not need to release the mmap_sem because
> diff --git a/arch/arm/probes/uprobes/core.c b/arch/arm/probes/uprobes/core.c
> index d1329f1ba4e4..e8b893eaebcf 100644
> --- a/arch/arm/probes/uprobes/core.c
> +++ b/arch/arm/probes/uprobes/core.c
> @@ -30,10 +30,11 @@ bool is_swbp_insn(uprobe_opcode_t *insn)
> }
>
> int set_swbp(struct arch_uprobe *auprobe, struct mm_struct *mm,
> - unsigned long vaddr)
> + unsigned long vaddr, struct range_lock *mmrange)
> {
> return uprobe_write_opcode(mm, vaddr,
> - __opcode_to_mem_arm(auprobe->bpinsn));
> + __opcode_to_mem_arm(auprobe->bpinsn),
> + mmrange);
> }
>
> bool arch_uprobe_ignore(struct arch_uprobe *auprobe, struct pt_regs *regs)
> diff --git a/arch/arm64/mm/fault.c b/arch/arm64/mm/fault.c
> index ce441d29e7f6..1f3ad9e4f214 100644
> --- a/arch/arm64/mm/fault.c
> +++ b/arch/arm64/mm/fault.c
> @@ -342,7 +342,7 @@ static void do_bad_area(unsigned long addr, unsigned int esr, struct pt_regs *re
>
> static int __do_page_fault(struct mm_struct *mm, unsigned long addr,
> unsigned int mm_flags, unsigned long vm_flags,
> - struct task_struct *tsk)
> + struct task_struct *tsk, struct range_lock *mmrange)
> {
> struct vm_area_struct *vma;
> int fault;
> @@ -368,7 +368,7 @@ static int __do_page_fault(struct mm_struct *mm, unsigned long addr,
> goto out;
> }
>
> - return handle_mm_fault(vma, addr & PAGE_MASK, mm_flags);
> + return handle_mm_fault(vma, addr & PAGE_MASK, mm_flags, mmrange);
>
> check_stack:
> if (vma->vm_flags & VM_GROWSDOWN && !expand_stack(vma, addr))
> @@ -390,6 +390,7 @@ static int __kprobes do_page_fault(unsigned long addr, unsigned int esr,
> int fault, sig, code, major = 0;
> unsigned long vm_flags = VM_READ | VM_WRITE;
> unsigned int mm_flags = FAULT_FLAG_ALLOW_RETRY | FAULT_FLAG_KILLABLE;
> + DEFINE_RANGE_LOCK_FULL(mmrange);
>
> if (notify_page_fault(regs, esr))
> return 0;
> @@ -450,7 +451,7 @@ static int __kprobes do_page_fault(unsigned long addr, unsigned int esr,
> #endif
> }
>
> - fault = __do_page_fault(mm, addr, mm_flags, vm_flags, tsk);
> + fault = __do_page_fault(mm, addr, mm_flags, vm_flags, tsk, &mmrange);
> major |= fault & VM_FAULT_MAJOR;
>
> if (fault & VM_FAULT_RETRY) {
> diff --git a/arch/cris/mm/fault.c b/arch/cris/mm/fault.c
> index 29cc58038b98..16af16d77269 100644
> --- a/arch/cris/mm/fault.c
> +++ b/arch/cris/mm/fault.c
> @@ -61,6 +61,7 @@ do_page_fault(unsigned long address, struct pt_regs *regs,
> siginfo_t info;
> int fault;
> unsigned int flags = FAULT_FLAG_ALLOW_RETRY | FAULT_FLAG_KILLABLE;
> + DEFINE_RANGE_LOCK_FULL(mmrange);
>
> D(printk(KERN_DEBUG
> "Page fault for %lX on %X at %lX, prot %d write %d\n",
> @@ -170,7 +171,7 @@ do_page_fault(unsigned long address, struct pt_regs *regs,
> * the fault.
> */
>
> - fault = handle_mm_fault(vma, address, flags);
> + fault = handle_mm_fault(vma, address, flags, &mmrange);
>
> if ((fault & VM_FAULT_RETRY) && fatal_signal_pending(current))
> return;
> diff --git a/arch/frv/mm/fault.c b/arch/frv/mm/fault.c
> index cbe7aec863e3..494d33b628fc 100644
> --- a/arch/frv/mm/fault.c
> +++ b/arch/frv/mm/fault.c
> @@ -41,6 +41,7 @@ asmlinkage void do_page_fault(int datammu, unsigned long esr0, unsigned long ear
> pud_t *pue;
> pte_t *pte;
> int fault;
> + DEFINE_RANGE_LOCK_FULL(mmrange);
>
> #if 0
> const char *atxc[16] = {
> @@ -165,7 +166,7 @@ asmlinkage void do_page_fault(int datammu, unsigned long esr0, unsigned long ear
> * make sure we exit gracefully rather than endlessly redo
> * the fault.
> */
> - fault = handle_mm_fault(vma, ear0, flags);
> + fault = handle_mm_fault(vma, ear0, flags, &mmrange);
> if (unlikely(fault & VM_FAULT_ERROR)) {
> if (fault & VM_FAULT_OOM)
> goto out_of_memory;
> diff --git a/arch/hexagon/mm/vm_fault.c b/arch/hexagon/mm/vm_fault.c
> index 3eec33c5cfd7..7d6ada2c2230 100644
> --- a/arch/hexagon/mm/vm_fault.c
> +++ b/arch/hexagon/mm/vm_fault.c
> @@ -55,6 +55,7 @@ void do_page_fault(unsigned long address, long cause, struct pt_regs *regs)
> int fault;
> const struct exception_table_entry *fixup;
> unsigned int flags = FAULT_FLAG_ALLOW_RETRY | FAULT_FLAG_KILLABLE;
> + DEFINE_RANGE_LOCK_FULL(mmrange);
>
> /*
> * If we're in an interrupt or have no user context,
> @@ -102,7 +103,7 @@ void do_page_fault(unsigned long address, long cause, struct pt_regs *regs)
> break;
> }
>
> - fault = handle_mm_fault(vma, address, flags);
> + fault = handle_mm_fault(vma, address, flags, &mmrange);
>
> if ((fault & VM_FAULT_RETRY) && fatal_signal_pending(current))
> return;
> diff --git a/arch/ia64/mm/fault.c b/arch/ia64/mm/fault.c
> index dfdc152d6737..44f0ec5f77c2 100644
> --- a/arch/ia64/mm/fault.c
> +++ b/arch/ia64/mm/fault.c
> @@ -89,6 +89,7 @@ ia64_do_page_fault (unsigned long address, unsigned long isr, struct pt_regs *re
> unsigned long mask;
> int fault;
> unsigned int flags = FAULT_FLAG_ALLOW_RETRY | FAULT_FLAG_KILLABLE;
> + DEFINE_RANGE_LOCK_FULL(mmrange);
>
> mask = ((((isr >> IA64_ISR_X_BIT) & 1UL) << VM_EXEC_BIT)
> | (((isr >> IA64_ISR_W_BIT) & 1UL) << VM_WRITE_BIT));
> @@ -162,7 +163,7 @@ ia64_do_page_fault (unsigned long address, unsigned long isr, struct pt_regs *re
> * sure we exit gracefully rather than endlessly redo the
> * fault.
> */
> - fault = handle_mm_fault(vma, address, flags);
> + fault = handle_mm_fault(vma, address, flags, &mmrange);
>
> if ((fault & VM_FAULT_RETRY) && fatal_signal_pending(current))
> return;
> diff --git a/arch/m32r/mm/fault.c b/arch/m32r/mm/fault.c
> index 46d9a5ca0e3a..0129aea46729 100644
> --- a/arch/m32r/mm/fault.c
> +++ b/arch/m32r/mm/fault.c
> @@ -82,6 +82,7 @@ asmlinkage void do_page_fault(struct pt_regs *regs, unsigned long error_code,
> unsigned long flags = 0;
> int fault;
> siginfo_t info;
> + DEFINE_RANGE_LOCK_FULL(mmrange);
>
> /*
> * If BPSW IE bit enable --> set PSW IE bit
> @@ -197,7 +198,7 @@ asmlinkage void do_page_fault(struct pt_regs *regs, unsigned long error_code,
> */
> addr = (address & PAGE_MASK);
> set_thread_fault_code(error_code);
> - fault = handle_mm_fault(vma, addr, flags);
> + fault = handle_mm_fault(vma, addr, flags, &mmrange);
> if (unlikely(fault & VM_FAULT_ERROR)) {
> if (fault & VM_FAULT_OOM)
> goto out_of_memory;
> diff --git a/arch/m68k/mm/fault.c b/arch/m68k/mm/fault.c
> index 03253c4f8e6a..ec32a193726f 100644
> --- a/arch/m68k/mm/fault.c
> +++ b/arch/m68k/mm/fault.c
> @@ -75,6 +75,7 @@ int do_page_fault(struct pt_regs *regs, unsigned long address,
> struct vm_area_struct * vma;
> int fault;
> unsigned int flags = FAULT_FLAG_ALLOW_RETRY | FAULT_FLAG_KILLABLE;
> + DEFINE_RANGE_LOCK_FULL(mmrange);
>
> pr_debug("do page fault:\nregs->sr=%#x, regs->pc=%#lx, address=%#lx, %ld, %p\n",
> regs->sr, regs->pc, address, error_code, mm ? mm->pgd : NULL);
> @@ -138,7 +139,7 @@ int do_page_fault(struct pt_regs *regs, unsigned long address,
> * the fault.
> */
>
> - fault = handle_mm_fault(vma, address, flags);
> + fault = handle_mm_fault(vma, address, flags, &mmrange);
> pr_debug("handle_mm_fault returns %d\n", fault);
>
> if ((fault & VM_FAULT_RETRY) && fatal_signal_pending(current))
> diff --git a/arch/metag/mm/fault.c b/arch/metag/mm/fault.c
> index de54fe686080..e16ba0ea7ea1 100644
> --- a/arch/metag/mm/fault.c
> +++ b/arch/metag/mm/fault.c
> @@ -56,6 +56,7 @@ int do_page_fault(struct pt_regs *regs, unsigned long address,
> siginfo_t info;
> int fault;
> unsigned int flags = FAULT_FLAG_ALLOW_RETRY | FAULT_FLAG_KILLABLE;
> + DEFINE_RANGE_LOCK_FULL(mmrange);
>
> tsk = current;
>
> @@ -135,7 +136,7 @@ int do_page_fault(struct pt_regs *regs, unsigned long address,
> * make sure we exit gracefully rather than endlessly redo
> * the fault.
> */
> - fault = handle_mm_fault(vma, address, flags);
> + fault = handle_mm_fault(vma, address, flags, &mmrange);
>
> if ((fault & VM_FAULT_RETRY) && fatal_signal_pending(current))
> return 0;
> diff --git a/arch/microblaze/mm/fault.c b/arch/microblaze/mm/fault.c
> index f91b30f8aaa8..fd49efbdfbf4 100644
> --- a/arch/microblaze/mm/fault.c
> +++ b/arch/microblaze/mm/fault.c
> @@ -93,6 +93,7 @@ void do_page_fault(struct pt_regs *regs, unsigned long address,
> int is_write = error_code & ESR_S;
> int fault;
> unsigned int flags = FAULT_FLAG_ALLOW_RETRY | FAULT_FLAG_KILLABLE;
> + DEFINE_RANGE_LOCK_FULL(mmrange);
>
> regs->ear = address;
> regs->esr = error_code;
> @@ -216,7 +217,7 @@ void do_page_fault(struct pt_regs *regs, unsigned long address,
> * make sure we exit gracefully rather than endlessly redo
> * the fault.
> */
> - fault = handle_mm_fault(vma, address, flags);
> + fault = handle_mm_fault(vma, address, flags, &mmrange);
>
> if ((fault & VM_FAULT_RETRY) && fatal_signal_pending(current))
> return;
> diff --git a/arch/mips/kernel/vdso.c b/arch/mips/kernel/vdso.c
> index 019035d7225c..56b7c29991db 100644
> --- a/arch/mips/kernel/vdso.c
> +++ b/arch/mips/kernel/vdso.c
> @@ -102,6 +102,7 @@ int arch_setup_additional_pages(struct linux_binprm *bprm, int uses_interp)
> unsigned long gic_size, vvar_size, size, base, data_addr, vdso_addr, gic_pfn;
> struct vm_area_struct *vma;
> int ret;
> + DEFINE_RANGE_LOCK_FULL(mmrange);
>
> if (down_write_killable(&mm->mmap_sem))
> return -EINTR;
> @@ -110,7 +111,7 @@ int arch_setup_additional_pages(struct linux_binprm *bprm, int uses_interp)
> base = mmap_region(NULL, STACK_TOP, PAGE_SIZE,
> VM_READ|VM_WRITE|VM_EXEC|
> VM_MAYREAD|VM_MAYWRITE|VM_MAYEXEC,
> - 0, NULL);
> + 0, NULL, &mmrange);
> if (IS_ERR_VALUE(base)) {
> ret = base;
> goto out;
> diff --git a/arch/mips/mm/fault.c b/arch/mips/mm/fault.c
> index 4f8f5bf46977..1433edd01d09 100644
> --- a/arch/mips/mm/fault.c
> +++ b/arch/mips/mm/fault.c
> @@ -47,6 +47,7 @@ static void __kprobes __do_page_fault(struct pt_regs *regs, unsigned long write,
> unsigned int flags = FAULT_FLAG_ALLOW_RETRY | FAULT_FLAG_KILLABLE;
>
> static DEFINE_RATELIMIT_STATE(ratelimit_state, 5 * HZ, 10);
> + DEFINE_RANGE_LOCK_FULL(mmrange);
>
> #if 0
> printk("Cpu%d[%s:%d:%0*lx:%ld:%0*lx]\n", raw_smp_processor_id(),
> @@ -152,7 +153,7 @@ static void __kprobes __do_page_fault(struct pt_regs *regs, unsigned long write,
> * make sure we exit gracefully rather than endlessly redo
> * the fault.
> */
> - fault = handle_mm_fault(vma, address, flags);
> + fault = handle_mm_fault(vma, address, flags, &mmrange);
>
> if ((fault & VM_FAULT_RETRY) && fatal_signal_pending(current))
> return;
> diff --git a/arch/mn10300/mm/fault.c b/arch/mn10300/mm/fault.c
> index f0bfa1448744..71c38f0c8702 100644
> --- a/arch/mn10300/mm/fault.c
> +++ b/arch/mn10300/mm/fault.c
> @@ -125,6 +125,7 @@ asmlinkage void do_page_fault(struct pt_regs *regs, unsigned long fault_code,
> siginfo_t info;
> int fault;
> unsigned int flags = FAULT_FLAG_ALLOW_RETRY | FAULT_FLAG_KILLABLE;
> + DEFINE_RANGE_LOCK_FULL(mmrange);
>
> #ifdef CONFIG_GDBSTUB
> /* handle GDB stub causing a fault */
> @@ -254,7 +255,7 @@ asmlinkage void do_page_fault(struct pt_regs *regs, unsigned long fault_code,
> * make sure we exit gracefully rather than endlessly redo
> * the fault.
> */
> - fault = handle_mm_fault(vma, address, flags);
> + fault = handle_mm_fault(vma, address, flags, &mmrange);
>
> if ((fault & VM_FAULT_RETRY) && fatal_signal_pending(current))
> return;
> diff --git a/arch/nios2/mm/fault.c b/arch/nios2/mm/fault.c
> index b804dd06ea1c..768678b685af 100644
> --- a/arch/nios2/mm/fault.c
> +++ b/arch/nios2/mm/fault.c
> @@ -49,6 +49,7 @@ asmlinkage void do_page_fault(struct pt_regs *regs, unsigned long cause,
> int code = SEGV_MAPERR;
> int fault;
> unsigned int flags = FAULT_FLAG_ALLOW_RETRY | FAULT_FLAG_KILLABLE;
> + DEFINE_RANGE_LOCK_FULL(mmrange);
>
> cause >>= 2;
>
> @@ -132,7 +133,7 @@ asmlinkage void do_page_fault(struct pt_regs *regs, unsigned long cause,
> * make sure we exit gracefully rather than endlessly redo
> * the fault.
> */
> - fault = handle_mm_fault(vma, address, flags);
> + fault = handle_mm_fault(vma, address, flags, mmrange);
>
> if ((fault & VM_FAULT_RETRY) && fatal_signal_pending(current))
> return;
> diff --git a/arch/openrisc/mm/fault.c b/arch/openrisc/mm/fault.c
> index d0021dfae20a..75ddb1e8e7e7 100644
> --- a/arch/openrisc/mm/fault.c
> +++ b/arch/openrisc/mm/fault.c
> @@ -55,6 +55,7 @@ asmlinkage void do_page_fault(struct pt_regs *regs, unsigned long address,
> siginfo_t info;
> int fault;
> unsigned int flags = FAULT_FLAG_ALLOW_RETRY | FAULT_FLAG_KILLABLE;
> + DEFINE_RANGE_LOCK_FULL(mmrange);
>
> tsk = current;
>
> @@ -163,7 +164,7 @@ asmlinkage void do_page_fault(struct pt_regs *regs, unsigned long address,
> * the fault.
> */
>
> - fault = handle_mm_fault(vma, address, flags);
> + fault = handle_mm_fault(vma, address, flags, &mmrange);
>
> if ((fault & VM_FAULT_RETRY) && fatal_signal_pending(current))
> return;
> diff --git a/arch/parisc/mm/fault.c b/arch/parisc/mm/fault.c
> index e247edbca68e..79db33a0cb0c 100644
> --- a/arch/parisc/mm/fault.c
> +++ b/arch/parisc/mm/fault.c
> @@ -264,6 +264,7 @@ void do_page_fault(struct pt_regs *regs, unsigned long code,
> unsigned long acc_type;
> int fault = 0;
> unsigned int flags;
> + DEFINE_RANGE_LOCK_FULL(mmrange);
>
> if (faulthandler_disabled())
> goto no_context;
> @@ -301,7 +302,7 @@ void do_page_fault(struct pt_regs *regs, unsigned long code,
> * fault.
> */
>
> - fault = handle_mm_fault(vma, address, flags);
> + fault = handle_mm_fault(vma, address, flags, &mmrange);
>
> if ((fault & VM_FAULT_RETRY) && fatal_signal_pending(current))
> return;
> diff --git a/arch/powerpc/include/asm/mmu_context.h b/arch/powerpc/include/asm/mmu_context.h
> index 051b3d63afe3..089b3cf948eb 100644
> --- a/arch/powerpc/include/asm/mmu_context.h
> +++ b/arch/powerpc/include/asm/mmu_context.h
> @@ -176,7 +176,8 @@ extern void arch_exit_mmap(struct mm_struct *mm);
>
> static inline void arch_unmap(struct mm_struct *mm,
> struct vm_area_struct *vma,
> - unsigned long start, unsigned long end)
> + unsigned long start, unsigned long end,
> + struct range_lock *mmrange)
> {
> if (start <= mm->context.vdso_base && mm->context.vdso_base < end)
> mm->context.vdso_base = 0;
> diff --git a/arch/powerpc/include/asm/powernv.h b/arch/powerpc/include/asm/powernv.h
> index dc5f6a5d4575..805ff3ba94e1 100644
> --- a/arch/powerpc/include/asm/powernv.h
> +++ b/arch/powerpc/include/asm/powernv.h
> @@ -21,7 +21,7 @@ extern void pnv_npu2_destroy_context(struct npu_context *context,
> struct pci_dev *gpdev);
> extern int pnv_npu2_handle_fault(struct npu_context *context, uintptr_t *ea,
> unsigned long *flags, unsigned long *status,
> - int count);
> + int count, struct range_lock *mmrange);
>
> void pnv_tm_init(void);
> #else
> @@ -35,7 +35,8 @@ static inline void pnv_npu2_destroy_context(struct npu_context *context,
>
> static inline int pnv_npu2_handle_fault(struct npu_context *context,
> uintptr_t *ea, unsigned long *flags,
> - unsigned long *status, int count) {
> + unsigned long *status, int count,
> + struct range_lock *mmrange) {
> return -ENODEV;
> }
>
> diff --git a/arch/powerpc/mm/copro_fault.c b/arch/powerpc/mm/copro_fault.c
> index 697b70ad1195..8f5e604828a1 100644
> --- a/arch/powerpc/mm/copro_fault.c
> +++ b/arch/powerpc/mm/copro_fault.c
> @@ -39,6 +39,7 @@ int copro_handle_mm_fault(struct mm_struct *mm, unsigned long ea,
> struct vm_area_struct *vma;
> unsigned long is_write;
> int ret;
> + DEFINE_RANGE_LOCK_FULL(mmrange);
>
> if (mm == NULL)
> return -EFAULT;
> @@ -77,7 +78,8 @@ int copro_handle_mm_fault(struct mm_struct *mm, unsigned long ea,
> }
>
> ret = 0;
> - *flt = handle_mm_fault(vma, ea, is_write ? FAULT_FLAG_WRITE : 0);
> + *flt = handle_mm_fault(vma, ea, is_write ? FAULT_FLAG_WRITE : 0,
> + &mmrange);
> if (unlikely(*flt & VM_FAULT_ERROR)) {
> if (*flt & VM_FAULT_OOM) {
> ret = -ENOMEM;
> diff --git a/arch/powerpc/mm/fault.c b/arch/powerpc/mm/fault.c
> index 866446cf2d9a..d562dc88687d 100644
> --- a/arch/powerpc/mm/fault.c
> +++ b/arch/powerpc/mm/fault.c
> @@ -399,6 +399,7 @@ static int __do_page_fault(struct pt_regs *regs, unsigned long address,
> int is_write = page_fault_is_write(error_code);
> int fault, major = 0;
> bool store_update_sp = false;
> + DEFINE_RANGE_LOCK_FULL(mmrange);
>
> if (notify_page_fault(regs))
> return 0;
> @@ -514,7 +515,7 @@ static int __do_page_fault(struct pt_regs *regs, unsigned long address,
> * make sure we exit gracefully rather than endlessly redo
> * the fault.
> */
> - fault = handle_mm_fault(vma, address, flags);
> + fault = handle_mm_fault(vma, address, flags, &mmrange);
>
> #ifdef CONFIG_PPC_MEM_KEYS
> /*
> diff --git a/arch/powerpc/platforms/powernv/npu-dma.c b/arch/powerpc/platforms/powernv/npu-dma.c
> index 0a253b64ac5f..759e9a4c7479 100644
> --- a/arch/powerpc/platforms/powernv/npu-dma.c
> +++ b/arch/powerpc/platforms/powernv/npu-dma.c
> @@ -789,7 +789,8 @@ EXPORT_SYMBOL(pnv_npu2_destroy_context);
> * Assumes mmap_sem is held for the contexts associated mm.
> */
> int pnv_npu2_handle_fault(struct npu_context *context, uintptr_t *ea,
> - unsigned long *flags, unsigned long *status, int count)
> + unsigned long *flags, unsigned long *status,
> + int count, struct range_lock *mmrange)
> {
> u64 rc = 0, result = 0;
> int i, is_write;
> @@ -807,7 +808,7 @@ int pnv_npu2_handle_fault(struct npu_context *context, uintptr_t *ea,
> is_write = flags[i] & NPU2_WRITE;
> rc = get_user_pages_remote(NULL, mm, ea[i], 1,
> is_write ? FOLL_WRITE : 0,
> - page, NULL, NULL);
> + page, NULL, NULL, mmrange);
>
> /*
> * To support virtualised environments we will have to do an
> diff --git a/arch/riscv/mm/fault.c b/arch/riscv/mm/fault.c
> index 148c98ca9b45..75d15e73ba39 100644
> --- a/arch/riscv/mm/fault.c
> +++ b/arch/riscv/mm/fault.c
> @@ -42,6 +42,7 @@ asmlinkage void do_page_fault(struct pt_regs *regs)
> unsigned long addr, cause;
> unsigned int flags = FAULT_FLAG_ALLOW_RETRY | FAULT_FLAG_KILLABLE;
> int fault, code = SEGV_MAPERR;
> + DEFINE_RANGE_LOCK_FULL(mmrange);
>
> cause = regs->scause;
> addr = regs->sbadaddr;
> @@ -119,7 +120,7 @@ asmlinkage void do_page_fault(struct pt_regs *regs)
> * make sure we exit gracefully rather than endlessly redo
> * the fault.
> */
> - fault = handle_mm_fault(vma, addr, flags);
> + fault = handle_mm_fault(vma, addr, flags, &mmrange);
>
> /*
> * If we need to retry but a fatal signal is pending, handle the
> diff --git a/arch/s390/include/asm/gmap.h b/arch/s390/include/asm/gmap.h
> index e07cce88dfb0..117c19a947c9 100644
> --- a/arch/s390/include/asm/gmap.h
> +++ b/arch/s390/include/asm/gmap.h
> @@ -107,22 +107,24 @@ void gmap_discard(struct gmap *, unsigned long from, unsigned long to);
> void __gmap_zap(struct gmap *, unsigned long gaddr);
> void gmap_unlink(struct mm_struct *, unsigned long *table, unsigned long vmaddr);
>
> -int gmap_read_table(struct gmap *gmap, unsigned long gaddr, unsigned long *val);
> +int gmap_read_table(struct gmap *gmap, unsigned long gaddr, unsigned long *val,
> + struct range_lock *mmrange);
>
> struct gmap *gmap_shadow(struct gmap *parent, unsigned long asce,
> int edat_level);
> int gmap_shadow_valid(struct gmap *sg, unsigned long asce, int edat_level);
> int gmap_shadow_r2t(struct gmap *sg, unsigned long saddr, unsigned long r2t,
> - int fake);
> + int fake, struct range_lock *mmrange);
> int gmap_shadow_r3t(struct gmap *sg, unsigned long saddr, unsigned long r3t,
> - int fake);
> + int fake, struct range_lock *mmrange);
> int gmap_shadow_sgt(struct gmap *sg, unsigned long saddr, unsigned long sgt,
> - int fake);
> + int fake, struct range_lock *mmrange);
> int gmap_shadow_pgt(struct gmap *sg, unsigned long saddr, unsigned long pgt,
> - int fake);
> + int fake, struct range_lock *mmrange);
> int gmap_shadow_pgt_lookup(struct gmap *sg, unsigned long saddr,
> unsigned long *pgt, int *dat_protection, int *fake);
> -int gmap_shadow_page(struct gmap *sg, unsigned long saddr, pte_t pte);
> +int gmap_shadow_page(struct gmap *sg, unsigned long saddr, pte_t pte,
> + struct range_lock *mmrange);
>
> void gmap_register_pte_notifier(struct gmap_notifier *);
> void gmap_unregister_pte_notifier(struct gmap_notifier *);
> diff --git a/arch/s390/kvm/gaccess.c b/arch/s390/kvm/gaccess.c
> index c24bfa72baf7..ff739b86df36 100644
> --- a/arch/s390/kvm/gaccess.c
> +++ b/arch/s390/kvm/gaccess.c
> @@ -978,10 +978,11 @@ int kvm_s390_check_low_addr_prot_real(struct kvm_vcpu *vcpu, unsigned long gra)
> * @saddr: faulting address in the shadow gmap
> * @pgt: pointer to the page table address result
> * @fake: pgt references contiguous guest memory block, not a pgtable
> + * @mmrange: address space range locking
> */
> static int kvm_s390_shadow_tables(struct gmap *sg, unsigned long saddr,
> unsigned long *pgt, int *dat_protection,
> - int *fake)
> + int *fake, struct range_lock *mmrange)
> {
> struct gmap *parent;
> union asce asce;
> @@ -1034,7 +1035,8 @@ static int kvm_s390_shadow_tables(struct gmap *sg, unsigned long saddr,
> rfte.val = ptr;
> goto shadow_r2t;
> }
> - rc = gmap_read_table(parent, ptr + vaddr.rfx * 8, &rfte.val);
> + rc = gmap_read_table(parent, ptr + vaddr.rfx * 8, &rfte.val,
> + mmrange);
> if (rc)
> return rc;
> if (rfte.i)
> @@ -1047,7 +1049,7 @@ static int kvm_s390_shadow_tables(struct gmap *sg, unsigned long saddr,
> *dat_protection |= rfte.p;
> ptr = rfte.rto * PAGE_SIZE;
> shadow_r2t:
> - rc = gmap_shadow_r2t(sg, saddr, rfte.val, *fake);
> + rc = gmap_shadow_r2t(sg, saddr, rfte.val, *fake, mmrange);
> if (rc)
> return rc;
> /* fallthrough */
> @@ -1060,7 +1062,8 @@ static int kvm_s390_shadow_tables(struct gmap *sg, unsigned long saddr,
> rste.val = ptr;
> goto shadow_r3t;
> }
> - rc = gmap_read_table(parent, ptr + vaddr.rsx * 8, &rste.val);
> + rc = gmap_read_table(parent, ptr + vaddr.rsx * 8, &rste.val,
> + mmrange);
> if (rc)
> return rc;
> if (rste.i)
> @@ -1074,7 +1077,7 @@ static int kvm_s390_shadow_tables(struct gmap *sg, unsigned long saddr,
> ptr = rste.rto * PAGE_SIZE;
> shadow_r3t:
> rste.p |= *dat_protection;
> - rc = gmap_shadow_r3t(sg, saddr, rste.val, *fake);
> + rc = gmap_shadow_r3t(sg, saddr, rste.val, *fake, mmrange);
> if (rc)
> return rc;
> /* fallthrough */
> @@ -1087,7 +1090,8 @@ static int kvm_s390_shadow_tables(struct gmap *sg, unsigned long saddr,
> rtte.val = ptr;
> goto shadow_sgt;
> }
> - rc = gmap_read_table(parent, ptr + vaddr.rtx * 8, &rtte.val);
> + rc = gmap_read_table(parent, ptr + vaddr.rtx * 8, &rtte.val,
> + mmrange);
> if (rc)
> return rc;
> if (rtte.i)
> @@ -1110,7 +1114,7 @@ static int kvm_s390_shadow_tables(struct gmap *sg, unsigned long saddr,
> ptr = rtte.fc0.sto * PAGE_SIZE;
> shadow_sgt:
> rtte.fc0.p |= *dat_protection;
> - rc = gmap_shadow_sgt(sg, saddr, rtte.val, *fake);
> + rc = gmap_shadow_sgt(sg, saddr, rtte.val, *fake, mmrange);
> if (rc)
> return rc;
> /* fallthrough */
> @@ -1123,7 +1127,8 @@ static int kvm_s390_shadow_tables(struct gmap *sg, unsigned long saddr,
> ste.val = ptr;
> goto shadow_pgt;
> }
> - rc = gmap_read_table(parent, ptr + vaddr.sx * 8, &ste.val);
> + rc = gmap_read_table(parent, ptr + vaddr.sx * 8, &ste.val,
> + mmrange);
> if (rc)
> return rc;
> if (ste.i)
> @@ -1142,7 +1147,7 @@ static int kvm_s390_shadow_tables(struct gmap *sg, unsigned long saddr,
> ptr = ste.fc0.pto * (PAGE_SIZE / 2);
> shadow_pgt:
> ste.fc0.p |= *dat_protection;
> - rc = gmap_shadow_pgt(sg, saddr, ste.val, *fake);
> + rc = gmap_shadow_pgt(sg, saddr, ste.val, *fake, mmrange);
> if (rc)
> return rc;
> }
> @@ -1172,6 +1177,7 @@ int kvm_s390_shadow_fault(struct kvm_vcpu *vcpu, struct gmap *sg,
> unsigned long pgt;
> int dat_protection, fake;
> int rc;
> + DEFINE_RANGE_LOCK_FULL(mmrange);
>
> down_read(&sg->mm->mmap_sem);
> /*
> @@ -1184,7 +1190,7 @@ int kvm_s390_shadow_fault(struct kvm_vcpu *vcpu, struct gmap *sg,
> rc = gmap_shadow_pgt_lookup(sg, saddr, &pgt, &dat_protection, &fake);
> if (rc)
> rc = kvm_s390_shadow_tables(sg, saddr, &pgt, &dat_protection,
> - &fake);
> + &fake, &mmrange);
>
> vaddr.addr = saddr;
> if (fake) {
> @@ -1192,7 +1198,8 @@ int kvm_s390_shadow_fault(struct kvm_vcpu *vcpu, struct gmap *sg,
> goto shadow_page;
> }
> if (!rc)
> - rc = gmap_read_table(sg->parent, pgt + vaddr.px * 8, &pte.val);
> + rc = gmap_read_table(sg->parent, pgt + vaddr.px * 8,
> + &pte.val, &mmrange);
> if (!rc && pte.i)
> rc = PGM_PAGE_TRANSLATION;
> if (!rc && pte.z)
> @@ -1200,7 +1207,7 @@ int kvm_s390_shadow_fault(struct kvm_vcpu *vcpu, struct gmap *sg,
> shadow_page:
> pte.p |= dat_protection;
> if (!rc)
> - rc = gmap_shadow_page(sg, saddr, __pte(pte.val));
> + rc = gmap_shadow_page(sg, saddr, __pte(pte.val), &mmrange);
> ipte_unlock(vcpu);
> up_read(&sg->mm->mmap_sem);
> return rc;
> diff --git a/arch/s390/mm/fault.c b/arch/s390/mm/fault.c
> index 93faeca52284..17ba3c402f9d 100644
> --- a/arch/s390/mm/fault.c
> +++ b/arch/s390/mm/fault.c
> @@ -421,6 +421,7 @@ static inline int do_exception(struct pt_regs *regs, int access)
> unsigned long address;
> unsigned int flags;
> int fault;
> + DEFINE_RANGE_LOCK_FULL(mmrange);
>
> tsk = current;
> /*
> @@ -507,7 +508,7 @@ static inline int do_exception(struct pt_regs *regs, int access)
> * make sure we exit gracefully rather than endlessly redo
> * the fault.
> */
> - fault = handle_mm_fault(vma, address, flags);
> + fault = handle_mm_fault(vma, address, flags, &mmrange);
> /* No reason to continue if interrupted by SIGKILL. */
> if ((fault & VM_FAULT_RETRY) && fatal_signal_pending(current)) {
> fault = VM_FAULT_SIGNAL;
> diff --git a/arch/s390/mm/gmap.c b/arch/s390/mm/gmap.c
> index 2c55a2b9d6c6..b12a44813022 100644
> --- a/arch/s390/mm/gmap.c
> +++ b/arch/s390/mm/gmap.c
> @@ -621,6 +621,7 @@ int gmap_fault(struct gmap *gmap, unsigned long gaddr,
> unsigned long vmaddr;
> int rc;
> bool unlocked;
> + DEFINE_RANGE_LOCK_FULL(mmrange);
>
> down_read(&gmap->mm->mmap_sem);
>
> @@ -632,7 +633,7 @@ int gmap_fault(struct gmap *gmap, unsigned long gaddr,
> goto out_up;
> }
> if (fixup_user_fault(current, gmap->mm, vmaddr, fault_flags,
> - &unlocked)) {
> + &unlocked, &mmrange)) {
> rc = -EFAULT;
> goto out_up;
> }
> @@ -835,13 +836,15 @@ static pte_t *gmap_pte_op_walk(struct gmap *gmap, unsigned long gaddr,
> * @gaddr: virtual address in the guest address space
> * @vmaddr: address in the host process address space
> * @prot: indicates access rights: PROT_NONE, PROT_READ or PROT_WRITE
> + * @mmrange: address space range locking
> *
> * Returns 0 if the caller can retry __gmap_translate (might fail again),
> * -ENOMEM if out of memory and -EFAULT if anything goes wrong while fixing
> * up or connecting the gmap page table.
> */
> static int gmap_pte_op_fixup(struct gmap *gmap, unsigned long gaddr,
> - unsigned long vmaddr, int prot)
> + unsigned long vmaddr, int prot,
> + struct range_lock *mmrange)
> {
> struct mm_struct *mm = gmap->mm;
> unsigned int fault_flags;
> @@ -849,7 +852,8 @@ static int gmap_pte_op_fixup(struct gmap *gmap, unsigned long gaddr,
>
> BUG_ON(gmap_is_shadow(gmap));
> fault_flags = (prot == PROT_WRITE) ? FAULT_FLAG_WRITE : 0;
> - if (fixup_user_fault(current, mm, vmaddr, fault_flags, &unlocked))
> + if (fixup_user_fault(current, mm, vmaddr, fault_flags, &unlocked,
> + mmrange))
> return -EFAULT;
> if (unlocked)
> /* lost mmap_sem, caller has to retry __gmap_translate */
> @@ -874,6 +878,7 @@ static void gmap_pte_op_end(spinlock_t *ptl)
> * @len: size of area
> * @prot: indicates access rights: PROT_NONE, PROT_READ or PROT_WRITE
> * @bits: pgste notification bits to set
> + * @mmrange: address space range locking
> *
> * Returns 0 if successfully protected, -ENOMEM if out of memory and
> * -EFAULT if gaddr is invalid (or mapping for shadows is missing).
> @@ -881,7 +886,8 @@ static void gmap_pte_op_end(spinlock_t *ptl)
> * Called with sg->mm->mmap_sem in read.
> */
> static int gmap_protect_range(struct gmap *gmap, unsigned long gaddr,
> - unsigned long len, int prot, unsigned long bits)
> + unsigned long len, int prot, unsigned long bits,
> + struct range_lock *mmrange)
> {
> unsigned long vmaddr;
> spinlock_t *ptl;
> @@ -900,7 +906,8 @@ static int gmap_protect_range(struct gmap *gmap, unsigned long gaddr,
> vmaddr = __gmap_translate(gmap, gaddr);
> if (IS_ERR_VALUE(vmaddr))
> return vmaddr;
> - rc = gmap_pte_op_fixup(gmap, gaddr, vmaddr, prot);
> + rc = gmap_pte_op_fixup(gmap, gaddr, vmaddr, prot,
> + mmrange);
> if (rc)
> return rc;
> continue;
> @@ -929,13 +936,14 @@ int gmap_mprotect_notify(struct gmap *gmap, unsigned long gaddr,
> unsigned long len, int prot)
> {
> int rc;
> + DEFINE_RANGE_LOCK_FULL(mmrange);
>
> if ((gaddr & ~PAGE_MASK) || (len & ~PAGE_MASK) || gmap_is_shadow(gmap))
> return -EINVAL;
> if (!MACHINE_HAS_ESOP && prot == PROT_READ)
> return -EINVAL;
> down_read(&gmap->mm->mmap_sem);
> - rc = gmap_protect_range(gmap, gaddr, len, prot, PGSTE_IN_BIT);
> + rc = gmap_protect_range(gmap, gaddr, len, prot, PGSTE_IN_BIT, &mmrange);
> up_read(&gmap->mm->mmap_sem);
> return rc;
> }
> @@ -947,6 +955,7 @@ EXPORT_SYMBOL_GPL(gmap_mprotect_notify);
> * @gmap: pointer to guest mapping meta data structure
> * @gaddr: virtual address in the guest address space
> * @val: pointer to the unsigned long value to return
> + * @mmrange: address space range locking
> *
> * Returns 0 if the value was read, -ENOMEM if out of memory and -EFAULT
> * if reading using the virtual address failed. -EINVAL if called on a gmap
> @@ -954,7 +963,8 @@ EXPORT_SYMBOL_GPL(gmap_mprotect_notify);
> *
> * Called with gmap->mm->mmap_sem in read.
> */
> -int gmap_read_table(struct gmap *gmap, unsigned long gaddr, unsigned long *val)
> +int gmap_read_table(struct gmap *gmap, unsigned long gaddr, unsigned long *val,
> + struct range_lock *mmrange)
> {
> unsigned long address, vmaddr;
> spinlock_t *ptl;
> @@ -986,7 +996,7 @@ int gmap_read_table(struct gmap *gmap, unsigned long gaddr, unsigned long *val)
> rc = vmaddr;
> break;
> }
> - rc = gmap_pte_op_fixup(gmap, gaddr, vmaddr, PROT_READ);
> + rc = gmap_pte_op_fixup(gmap, gaddr, vmaddr, PROT_READ, mmrange);
> if (rc)
> break;
> }
> @@ -1026,12 +1036,14 @@ static inline void gmap_insert_rmap(struct gmap *sg, unsigned long vmaddr,
> * @raddr: rmap address in the shadow gmap
> * @paddr: address in the parent guest address space
> * @len: length of the memory area to protect
> + * @mmrange: address space range locking
> *
> * Returns 0 if successfully protected and the rmap was created, -ENOMEM
> * if out of memory and -EFAULT if paddr is invalid.
> */
> static int gmap_protect_rmap(struct gmap *sg, unsigned long raddr,
> - unsigned long paddr, unsigned long len)
> + unsigned long paddr, unsigned long len,
> + struct range_lock *mmrange)
> {
> struct gmap *parent;
> struct gmap_rmap *rmap;
> @@ -1069,7 +1081,7 @@ static int gmap_protect_rmap(struct gmap *sg, unsigned long raddr,
> radix_tree_preload_end();
> if (rc) {
> kfree(rmap);
> - rc = gmap_pte_op_fixup(parent, paddr, vmaddr, PROT_READ);
> + rc = gmap_pte_op_fixup(parent, paddr, vmaddr, PROT_READ, mmrange);
> if (rc)
> return rc;
> continue;
> @@ -1473,6 +1485,7 @@ struct gmap *gmap_shadow(struct gmap *parent, unsigned long asce,
> struct gmap *sg, *new;
> unsigned long limit;
> int rc;
> + DEFINE_RANGE_LOCK_FULL(mmrange);
>
> BUG_ON(gmap_is_shadow(parent));
> spin_lock(&parent->shadow_lock);
> @@ -1526,7 +1539,7 @@ struct gmap *gmap_shadow(struct gmap *parent, unsigned long asce,
> down_read(&parent->mm->mmap_sem);
> rc = gmap_protect_range(parent, asce & _ASCE_ORIGIN,
> ((asce & _ASCE_TABLE_LENGTH) + 1) * PAGE_SIZE,
> - PROT_READ, PGSTE_VSIE_BIT);
> + PROT_READ, PGSTE_VSIE_BIT, &mmrange);
> up_read(&parent->mm->mmap_sem);
> spin_lock(&parent->shadow_lock);
> new->initialized = true;
> @@ -1546,6 +1559,7 @@ EXPORT_SYMBOL_GPL(gmap_shadow);
> * @saddr: faulting address in the shadow gmap
> * @r2t: parent gmap address of the region 2 table to get shadowed
> * @fake: r2t references contiguous guest memory block, not a r2t
> + * @mmrange: address space range locking
> *
> * The r2t parameter specifies the address of the source table. The
> * four pages of the source table are made read-only in the parent gmap
> @@ -1559,7 +1573,7 @@ EXPORT_SYMBOL_GPL(gmap_shadow);
> * Called with sg->mm->mmap_sem in read.
> */
> int gmap_shadow_r2t(struct gmap *sg, unsigned long saddr, unsigned long r2t,
> - int fake)
> + int fake, struct range_lock *mmrange)
> {
> unsigned long raddr, origin, offset, len;
> unsigned long *s_r2t, *table;
> @@ -1608,7 +1622,7 @@ int gmap_shadow_r2t(struct gmap *sg, unsigned long saddr, unsigned long r2t,
> origin = r2t & _REGION_ENTRY_ORIGIN;
> offset = ((r2t & _REGION_ENTRY_OFFSET) >> 6) * PAGE_SIZE;
> len = ((r2t & _REGION_ENTRY_LENGTH) + 1) * PAGE_SIZE - offset;
> - rc = gmap_protect_rmap(sg, raddr, origin + offset, len);
> + rc = gmap_protect_rmap(sg, raddr, origin + offset, len, mmrange);
> spin_lock(&sg->guest_table_lock);
> if (!rc) {
> table = gmap_table_walk(sg, saddr, 4);
> @@ -1635,6 +1649,7 @@ EXPORT_SYMBOL_GPL(gmap_shadow_r2t);
> * @saddr: faulting address in the shadow gmap
> * @r3t: parent gmap address of the region 3 table to get shadowed
> * @fake: r3t references contiguous guest memory block, not a r3t
> + * @mmrange: address space range locking
> *
> * Returns 0 if successfully shadowed or already shadowed, -EAGAIN if the
> * shadow table structure is incomplete, -ENOMEM if out of memory and
> @@ -1643,7 +1658,7 @@ EXPORT_SYMBOL_GPL(gmap_shadow_r2t);
> * Called with sg->mm->mmap_sem in read.
> */
> int gmap_shadow_r3t(struct gmap *sg, unsigned long saddr, unsigned long r3t,
> - int fake)
> + int fake, struct range_lock *mmrange)
> {
> unsigned long raddr, origin, offset, len;
> unsigned long *s_r3t, *table;
> @@ -1691,7 +1706,7 @@ int gmap_shadow_r3t(struct gmap *sg, unsigned long saddr, unsigned long r3t,
> origin = r3t & _REGION_ENTRY_ORIGIN;
> offset = ((r3t & _REGION_ENTRY_OFFSET) >> 6) * PAGE_SIZE;
> len = ((r3t & _REGION_ENTRY_LENGTH) + 1) * PAGE_SIZE - offset;
> - rc = gmap_protect_rmap(sg, raddr, origin + offset, len);
> + rc = gmap_protect_rmap(sg, raddr, origin + offset, len, mmrange);
> spin_lock(&sg->guest_table_lock);
> if (!rc) {
> table = gmap_table_walk(sg, saddr, 3);
> @@ -1718,6 +1733,7 @@ EXPORT_SYMBOL_GPL(gmap_shadow_r3t);
> * @saddr: faulting address in the shadow gmap
> * @sgt: parent gmap address of the segment table to get shadowed
> * @fake: sgt references contiguous guest memory block, not a sgt
> + * @mmrange: address space range locking
> *
> * Returns: 0 if successfully shadowed or already shadowed, -EAGAIN if the
> * shadow table structure is incomplete, -ENOMEM if out of memory and
> @@ -1726,7 +1742,7 @@ EXPORT_SYMBOL_GPL(gmap_shadow_r3t);
> * Called with sg->mm->mmap_sem in read.
> */
> int gmap_shadow_sgt(struct gmap *sg, unsigned long saddr, unsigned long sgt,
> - int fake)
> + int fake, struct range_lock *mmrange)
> {
> unsigned long raddr, origin, offset, len;
> unsigned long *s_sgt, *table;
> @@ -1775,7 +1791,7 @@ int gmap_shadow_sgt(struct gmap *sg, unsigned long saddr, unsigned long sgt,
> origin = sgt & _REGION_ENTRY_ORIGIN;
> offset = ((sgt & _REGION_ENTRY_OFFSET) >> 6) * PAGE_SIZE;
> len = ((sgt & _REGION_ENTRY_LENGTH) + 1) * PAGE_SIZE - offset;
> - rc = gmap_protect_rmap(sg, raddr, origin + offset, len);
> + rc = gmap_protect_rmap(sg, raddr, origin + offset, len, mmrange);
> spin_lock(&sg->guest_table_lock);
> if (!rc) {
> table = gmap_table_walk(sg, saddr, 2);
> @@ -1842,6 +1858,7 @@ EXPORT_SYMBOL_GPL(gmap_shadow_pgt_lookup);
> * @saddr: faulting address in the shadow gmap
> * @pgt: parent gmap address of the page table to get shadowed
> * @fake: pgt references contiguous guest memory block, not a pgtable
> + * @mmrange: address space range locking
> *
> * Returns 0 if successfully shadowed or already shadowed, -EAGAIN if the
> * shadow table structure is incomplete, -ENOMEM if out of memory,
> @@ -1850,7 +1867,7 @@ EXPORT_SYMBOL_GPL(gmap_shadow_pgt_lookup);
> * Called with gmap->mm->mmap_sem in read
> */
> int gmap_shadow_pgt(struct gmap *sg, unsigned long saddr, unsigned long pgt,
> - int fake)
> + int fake, struct range_lock *mmrange)
> {
> unsigned long raddr, origin;
> unsigned long *s_pgt, *table;
> @@ -1894,7 +1911,7 @@ int gmap_shadow_pgt(struct gmap *sg, unsigned long saddr, unsigned long pgt,
> /* Make pgt read-only in parent gmap page table (not the pgste) */
> raddr = (saddr & _SEGMENT_MASK) | _SHADOW_RMAP_SEGMENT;
> origin = pgt & _SEGMENT_ENTRY_ORIGIN & PAGE_MASK;
> - rc = gmap_protect_rmap(sg, raddr, origin, PAGE_SIZE);
> + rc = gmap_protect_rmap(sg, raddr, origin, PAGE_SIZE, mmrange);
> spin_lock(&sg->guest_table_lock);
> if (!rc) {
> table = gmap_table_walk(sg, saddr, 1);
> @@ -1921,6 +1938,7 @@ EXPORT_SYMBOL_GPL(gmap_shadow_pgt);
> * @sg: pointer to the shadow guest address space structure
> * @saddr: faulting address in the shadow gmap
> * @pte: pte in parent gmap address space to get shadowed
> + * @mmrange: address space range locking
> *
> * Returns 0 if successfully shadowed or already shadowed, -EAGAIN if the
> * shadow table structure is incomplete, -ENOMEM if out of memory and
> @@ -1928,7 +1946,8 @@ EXPORT_SYMBOL_GPL(gmap_shadow_pgt);
> *
> * Called with sg->mm->mmap_sem in read.
> */
> -int gmap_shadow_page(struct gmap *sg, unsigned long saddr, pte_t pte)
> +int gmap_shadow_page(struct gmap *sg, unsigned long saddr, pte_t pte,
> + struct range_lock *mmrange)
> {
> struct gmap *parent;
> struct gmap_rmap *rmap;
> @@ -1982,7 +2001,7 @@ int gmap_shadow_page(struct gmap *sg, unsigned long saddr, pte_t pte)
> radix_tree_preload_end();
> if (!rc)
> break;
> - rc = gmap_pte_op_fixup(parent, paddr, vmaddr, prot);
> + rc = gmap_pte_op_fixup(parent, paddr, vmaddr, prot, mmrange);
> if (rc)
> break;
> }
> @@ -2117,7 +2136,8 @@ static inline void thp_split_mm(struct mm_struct *mm)
> * - This must be called after THP was enabled
> */
> static int __zap_zero_pages(pmd_t *pmd, unsigned long start,
> - unsigned long end, struct mm_walk *walk)
> + unsigned long end, struct mm_walk *walk,
> + struct range_lock *mmrange)
> {
> unsigned long addr;
>
> @@ -2133,12 +2153,13 @@ static int __zap_zero_pages(pmd_t *pmd, unsigned long start,
> return 0;
> }
>
> -static inline void zap_zero_pages(struct mm_struct *mm)
> +static inline void zap_zero_pages(struct mm_struct *mm,
> + struct range_lock *mmrange)
> {
> struct mm_walk walk = { .pmd_entry = __zap_zero_pages };
>
> walk.mm = mm;
> - walk_page_range(0, TASK_SIZE, &walk);
> + walk_page_range(0, TASK_SIZE, &walk, mmrange);
> }
>
> /*
> @@ -2147,6 +2168,7 @@ static inline void zap_zero_pages(struct mm_struct *mm)
> int s390_enable_sie(void)
> {
> struct mm_struct *mm = current->mm;
> + DEFINE_RANGE_LOCK_FULL(mmrange);
>
> /* Do we have pgstes? if yes, we are done */
> if (mm_has_pgste(mm))
> @@ -2158,7 +2180,7 @@ int s390_enable_sie(void)
> mm->context.has_pgste = 1;
> /* split thp mappings and disable thp for future mappings */
> thp_split_mm(mm);
> - zap_zero_pages(mm);
> + zap_zero_pages(mm, &mmrange);
> up_write(&mm->mmap_sem);
> return 0;
> }
> @@ -2182,6 +2204,7 @@ int s390_enable_skey(void)
> struct mm_struct *mm = current->mm;
> struct vm_area_struct *vma;
> int rc = 0;
> + DEFINE_RANGE_LOCK_FULL(mmrange);
>
> down_write(&mm->mmap_sem);
> if (mm_use_skey(mm))
> @@ -2190,7 +2213,7 @@ int s390_enable_skey(void)
> mm->context.use_skey = 1;
> for (vma = mm->mmap; vma; vma = vma->vm_next) {
> if (ksm_madvise(vma, vma->vm_start, vma->vm_end,
> - MADV_UNMERGEABLE, &vma->vm_flags)) {
> + MADV_UNMERGEABLE, &vma->vm_flags, &mmrange)) {
> mm->context.use_skey = 0;
> rc = -ENOMEM;
> goto out_up;
> @@ -2199,7 +2222,7 @@ int s390_enable_skey(void)
> mm->def_flags &= ~VM_MERGEABLE;
>
> walk.mm = mm;
> - walk_page_range(0, TASK_SIZE, &walk);
> + walk_page_range(0, TASK_SIZE, &walk, &mmrange);
>
> out_up:
> up_write(&mm->mmap_sem);
> @@ -2220,10 +2243,11 @@ static int __s390_reset_cmma(pte_t *pte, unsigned long addr,
> void s390_reset_cmma(struct mm_struct *mm)
> {
> struct mm_walk walk = { .pte_entry = __s390_reset_cmma };
> + DEFINE_RANGE_LOCK_FULL(mmrange);
>
> down_write(&mm->mmap_sem);
> walk.mm = mm;
> - walk_page_range(0, TASK_SIZE, &walk);
> + walk_page_range(0, TASK_SIZE, &walk, &mmrange);
> up_write(&mm->mmap_sem);
> }
> EXPORT_SYMBOL_GPL(s390_reset_cmma);
> diff --git a/arch/score/mm/fault.c b/arch/score/mm/fault.c
> index b85fad4f0874..07a8637ad142 100644
> --- a/arch/score/mm/fault.c
> +++ b/arch/score/mm/fault.c
> @@ -51,6 +51,7 @@ asmlinkage void do_page_fault(struct pt_regs *regs, unsigned long write,
> unsigned long flags = 0;
> siginfo_t info;
> int fault;
> + DEFINE_RANGE_LOCK_FULL(mmrange);
>
> info.si_code = SEGV_MAPERR;
>
> @@ -111,7 +112,7 @@ asmlinkage void do_page_fault(struct pt_regs *regs, unsigned long write,
> * make sure we exit gracefully rather than endlessly redo
> * the fault.
> */
> - fault = handle_mm_fault(vma, address, flags);
> + fault = handle_mm_fault(vma, address, flags, mmrange);
> if (unlikely(fault & VM_FAULT_ERROR)) {
> if (fault & VM_FAULT_OOM)
> goto out_of_memory;
> diff --git a/arch/sh/mm/fault.c b/arch/sh/mm/fault.c
> index 6fd1bf7481c7..d36106564728 100644
> --- a/arch/sh/mm/fault.c
> +++ b/arch/sh/mm/fault.c
> @@ -405,6 +405,7 @@ asmlinkage void __kprobes do_page_fault(struct pt_regs *regs,
> struct vm_area_struct * vma;
> int fault;
> unsigned int flags = FAULT_FLAG_ALLOW_RETRY | FAULT_FLAG_KILLABLE;
> + DEFINE_RANGE_LOCK_FULL(mmrange);
>
> tsk = current;
> mm = tsk->mm;
> @@ -488,7 +489,7 @@ asmlinkage void __kprobes do_page_fault(struct pt_regs *regs,
> * make sure we exit gracefully rather than endlessly redo
> * the fault.
> */
> - fault = handle_mm_fault(vma, address, flags);
> + fault = handle_mm_fault(vma, address, flags, &mmrange);
>
> if (unlikely(fault & (VM_FAULT_RETRY | VM_FAULT_ERROR)))
> if (mm_fault_error(regs, error_code, address, fault))
> diff --git a/arch/sparc/mm/fault_32.c b/arch/sparc/mm/fault_32.c
> index a8103a84b4ac..ebb2406dbe7c 100644
> --- a/arch/sparc/mm/fault_32.c
> +++ b/arch/sparc/mm/fault_32.c
> @@ -176,6 +176,7 @@ asmlinkage void do_sparc_fault(struct pt_regs *regs, int text_fault, int write,
> int from_user = !(regs->psr & PSR_PS);
> int fault, code;
> unsigned int flags = FAULT_FLAG_ALLOW_RETRY | FAULT_FLAG_KILLABLE;
> + DEFINE_RANGE_LOCK_FULL(mmrange);
>
> if (text_fault)
> address = regs->pc;
> @@ -242,7 +243,7 @@ asmlinkage void do_sparc_fault(struct pt_regs *regs, int text_fault, int write,
> * make sure we exit gracefully rather than endlessly redo
> * the fault.
> */
> - fault = handle_mm_fault(vma, address, flags);
> + fault = handle_mm_fault(vma, address, flags, &mmrange);
>
> if ((fault & VM_FAULT_RETRY) && fatal_signal_pending(current))
> return;
> @@ -389,6 +390,7 @@ static void force_user_fault(unsigned long address, int write)
> struct mm_struct *mm = tsk->mm;
> unsigned int flags = FAULT_FLAG_USER;
> int code;
> + DEFINE_RANGE_LOCK_FULL(mmrange);
>
> code = SEGV_MAPERR;
>
> @@ -412,7 +414,7 @@ static void force_user_fault(unsigned long address, int write)
> if (!(vma->vm_flags & (VM_READ | VM_EXEC)))
> goto bad_area;
> }
> - switch (handle_mm_fault(vma, address, flags)) {
> + switch (handle_mm_fault(vma, address, flags, &mmrange)) {
> case VM_FAULT_SIGBUS:
> case VM_FAULT_OOM:
> goto do_sigbus;
> diff --git a/arch/sparc/mm/fault_64.c b/arch/sparc/mm/fault_64.c
> index 41363f46797b..e0a3c36b0fa1 100644
> --- a/arch/sparc/mm/fault_64.c
> +++ b/arch/sparc/mm/fault_64.c
> @@ -287,6 +287,7 @@ asmlinkage void __kprobes do_sparc64_fault(struct pt_regs *regs)
> int si_code, fault_code, fault;
> unsigned long address, mm_rss;
> unsigned int flags = FAULT_FLAG_ALLOW_RETRY | FAULT_FLAG_KILLABLE;
> + DEFINE_RANGE_LOCK_FULL(mmrange);
>
> fault_code = get_thread_fault_code();
>
> @@ -438,7 +439,7 @@ asmlinkage void __kprobes do_sparc64_fault(struct pt_regs *regs)
> goto bad_area;
> }
>
> - fault = handle_mm_fault(vma, address, flags);
> + fault = handle_mm_fault(vma, address, flags, &mmrange);
>
> if ((fault & VM_FAULT_RETRY) && fatal_signal_pending(current))
> goto exit_exception;
> diff --git a/arch/tile/mm/fault.c b/arch/tile/mm/fault.c
> index f58fa06a2214..09f053eb146f 100644
> --- a/arch/tile/mm/fault.c
> +++ b/arch/tile/mm/fault.c
> @@ -275,6 +275,7 @@ static int handle_page_fault(struct pt_regs *regs,
> int is_kernel_mode;
> pgd_t *pgd;
> unsigned int flags;
> + DEFINE_RANGE_LOCK_FULL(mmrange);
>
> /* on TILE, protection faults are always writes */
> if (!is_page_fault)
> @@ -437,7 +438,7 @@ static int handle_page_fault(struct pt_regs *regs,
> * make sure we exit gracefully rather than endlessly redo
> * the fault.
> */
> - fault = handle_mm_fault(vma, address, flags);
> + fault = handle_mm_fault(vma, address, flags, &mmrange);
>
> if ((fault & VM_FAULT_RETRY) && fatal_signal_pending(current))
> return 0;
> diff --git a/arch/um/include/asm/mmu_context.h b/arch/um/include/asm/mmu_context.h
> index fca34b2177e2..98cc3e36385a 100644
> --- a/arch/um/include/asm/mmu_context.h
> +++ b/arch/um/include/asm/mmu_context.h
> @@ -23,7 +23,8 @@ static inline int arch_dup_mmap(struct mm_struct *oldmm, struct mm_struct *mm)
> extern void arch_exit_mmap(struct mm_struct *mm);
> static inline void arch_unmap(struct mm_struct *mm,
> struct vm_area_struct *vma,
> - unsigned long start, unsigned long end)
> + unsigned long start, unsigned long end,
> + struct range_lock *mmrange)
> {
> }
> static inline void arch_bprm_mm_init(struct mm_struct *mm,
> diff --git a/arch/um/kernel/trap.c b/arch/um/kernel/trap.c
> index b2b02df9896e..e632a14e896e 100644
> --- a/arch/um/kernel/trap.c
> +++ b/arch/um/kernel/trap.c
> @@ -33,6 +33,7 @@ int handle_page_fault(unsigned long address, unsigned long ip,
> pte_t *pte;
> int err = -EFAULT;
> unsigned int flags = FAULT_FLAG_ALLOW_RETRY | FAULT_FLAG_KILLABLE;
> + DEFINE_RANGE_LOCK_FULL(mmrange);
>
> *code_out = SEGV_MAPERR;
>
> @@ -74,7 +75,7 @@ int handle_page_fault(unsigned long address, unsigned long ip,
> do {
> int fault;
>
> - fault = handle_mm_fault(vma, address, flags);
> + fault = handle_mm_fault(vma, address, flags, &mmrange);
>
> if ((fault & VM_FAULT_RETRY) && fatal_signal_pending(current))
> goto out_nosemaphore;
> diff --git a/arch/unicore32/mm/fault.c b/arch/unicore32/mm/fault.c
> index bbefcc46a45e..dd35b6191798 100644
> --- a/arch/unicore32/mm/fault.c
> +++ b/arch/unicore32/mm/fault.c
> @@ -168,7 +168,8 @@ static inline bool access_error(unsigned int fsr, struct vm_area_struct *vma)
> }
>
> static int __do_pf(struct mm_struct *mm, unsigned long addr, unsigned int fsr,
> - unsigned int flags, struct task_struct *tsk)
> + unsigned int flags, struct task_struct *tsk,
> + struct range_lock *mmrange)
> {
> struct vm_area_struct *vma;
> int fault;
> @@ -194,7 +195,7 @@ static int __do_pf(struct mm_struct *mm, unsigned long addr, unsigned int fsr,
> * If for any reason at all we couldn't handle the fault, make
> * sure we exit gracefully rather than endlessly redo the fault.
> */
> - fault = handle_mm_fault(vma, addr & PAGE_MASK, flags);
> + fault = handle_mm_fault(vma, addr & PAGE_MASK, flags, mmrange);
> return fault;
>
> check_stack:
> @@ -210,6 +211,7 @@ static int do_pf(unsigned long addr, unsigned int fsr, struct pt_regs *regs)
> struct mm_struct *mm;
> int fault, sig, code;
> unsigned int flags = FAULT_FLAG_ALLOW_RETRY | FAULT_FLAG_KILLABLE;
> + DEFINE_RANGE_LOCK_FULL(mmrange);
>
> tsk = current;
> mm = tsk->mm;
> @@ -251,7 +253,7 @@ static int do_pf(unsigned long addr, unsigned int fsr, struct pt_regs *regs)
> #endif
> }
>
> - fault = __do_pf(mm, addr, fsr, flags, tsk);
> + fault = __do_pf(mm, addr, fsr, flags, tsk, &mmrange);
>
> /* If we need to retry but a fatal signal is pending, handle the
> * signal first. We do not need to release the mmap_sem because
> diff --git a/arch/x86/entry/vdso/vma.c b/arch/x86/entry/vdso/vma.c
> index 5b8b556dbb12..2e0bdf6a3aaf 100644
> --- a/arch/x86/entry/vdso/vma.c
> +++ b/arch/x86/entry/vdso/vma.c
> @@ -155,6 +155,7 @@ static int map_vdso(const struct vdso_image *image, unsigned long addr)
> struct vm_area_struct *vma;
> unsigned long text_start;
> int ret = 0;
> + DEFINE_RANGE_LOCK_FULL(mmrange);
>
> if (down_write_killable(&mm->mmap_sem))
> return -EINTR;
> @@ -192,7 +193,7 @@ static int map_vdso(const struct vdso_image *image, unsigned long addr)
>
> if (IS_ERR(vma)) {
> ret = PTR_ERR(vma);
> - do_munmap(mm, text_start, image->size, NULL);
> + do_munmap(mm, text_start, image->size, NULL, &mmrange);
> } else {
> current->mm->context.vdso = (void __user *)text_start;
> current->mm->context.vdso_image = image;
> diff --git a/arch/x86/include/asm/mmu_context.h b/arch/x86/include/asm/mmu_context.h
> index c931b88982a0..31fb02ed4770 100644
> --- a/arch/x86/include/asm/mmu_context.h
> +++ b/arch/x86/include/asm/mmu_context.h
> @@ -263,7 +263,8 @@ static inline void arch_bprm_mm_init(struct mm_struct *mm,
> }
>
> static inline void arch_unmap(struct mm_struct *mm, struct vm_area_struct *vma,
> - unsigned long start, unsigned long end)
> + unsigned long start, unsigned long end,
> + struct range_lock *mmrange)
> {
> /*
> * mpx_notify_unmap() goes and reads a rarely-hot
> @@ -283,7 +284,7 @@ static inline void arch_unmap(struct mm_struct *mm, struct vm_area_struct *vma,
> * consistently wrong.
> */
> if (unlikely(cpu_feature_enabled(X86_FEATURE_MPX)))
> - mpx_notify_unmap(mm, vma, start, end);
> + mpx_notify_unmap(mm, vma, start, end, mmrange);
> }
>
> #ifdef CONFIG_X86_INTEL_MEMORY_PROTECTION_KEYS
> diff --git a/arch/x86/include/asm/mpx.h b/arch/x86/include/asm/mpx.h
> index 61eb4b63c5ec..c26099224a17 100644
> --- a/arch/x86/include/asm/mpx.h
> +++ b/arch/x86/include/asm/mpx.h
> @@ -73,7 +73,8 @@ static inline void mpx_mm_init(struct mm_struct *mm)
> mm->context.bd_addr = MPX_INVALID_BOUNDS_DIR;
> }
> void mpx_notify_unmap(struct mm_struct *mm, struct vm_area_struct *vma,
> - unsigned long start, unsigned long end);
> + unsigned long start, unsigned long end,
> + struct range_lock *mmrange);
>
> unsigned long mpx_unmapped_area_check(unsigned long addr, unsigned long len,
> unsigned long flags);
> @@ -95,7 +96,8 @@ static inline void mpx_mm_init(struct mm_struct *mm)
> }
> static inline void mpx_notify_unmap(struct mm_struct *mm,
> struct vm_area_struct *vma,
> - unsigned long start, unsigned long end)
> + unsigned long start, unsigned long end,
> + struct range_lock *mmrange)
> {
> }
>
> diff --git a/arch/x86/mm/fault.c b/arch/x86/mm/fault.c
> index 800de815519c..93f1b8d4c88e 100644
> --- a/arch/x86/mm/fault.c
> +++ b/arch/x86/mm/fault.c
> @@ -1244,6 +1244,7 @@ __do_page_fault(struct pt_regs *regs, unsigned long error_code,
> int fault, major = 0;
> unsigned int flags = FAULT_FLAG_ALLOW_RETRY | FAULT_FLAG_KILLABLE;
> u32 pkey;
> + DEFINE_RANGE_LOCK_FULL(mmrange);
>
> tsk = current;
> mm = tsk->mm;
> @@ -1423,7 +1424,7 @@ __do_page_fault(struct pt_regs *regs, unsigned long error_code,
> * fault, so we read the pkey beforehand.
> */
> pkey = vma_pkey(vma);
> - fault = handle_mm_fault(vma, address, flags);
> + fault = handle_mm_fault(vma, address, flags, &mmrange);
> major |= fault & VM_FAULT_MAJOR;
>
> /*
> diff --git a/arch/x86/mm/mpx.c b/arch/x86/mm/mpx.c
> index e500949bae24..51c3e1f7e6be 100644
> --- a/arch/x86/mm/mpx.c
> +++ b/arch/x86/mm/mpx.c
> @@ -47,6 +47,7 @@ static unsigned long mpx_mmap(unsigned long len)
> {
> struct mm_struct *mm = current->mm;
> unsigned long addr, populate;
> + DEFINE_RANGE_LOCK_FULL(mmrange);
>
> /* Only bounds table can be allocated here */
> if (len != mpx_bt_size_bytes(mm))
> @@ -54,7 +55,8 @@ static unsigned long mpx_mmap(unsigned long len)
>
> down_write(&mm->mmap_sem);
> addr = do_mmap(NULL, 0, len, PROT_READ | PROT_WRITE,
> - MAP_ANONYMOUS | MAP_PRIVATE, VM_MPX, 0, &populate, NULL);
> + MAP_ANONYMOUS | MAP_PRIVATE, VM_MPX, 0, &populate, NULL,
> + &mmrange);
> up_write(&mm->mmap_sem);
> if (populate)
> mm_populate(addr, populate);
> @@ -427,13 +429,15 @@ int mpx_handle_bd_fault(void)
> * A thin wrapper around get_user_pages(). Returns 0 if the
> * fault was resolved or -errno if not.
> */
> -static int mpx_resolve_fault(long __user *addr, int write)
> +static int mpx_resolve_fault(long __user *addr, int write,
> + struct range_lock *mmrange)
> {
> long gup_ret;
> int nr_pages = 1;
>
> gup_ret = get_user_pages((unsigned long)addr, nr_pages,
> - write ? FOLL_WRITE : 0, NULL, NULL);
> + write ? FOLL_WRITE : 0, NULL, NULL,
> + mmrange);
> /*
> * get_user_pages() returns number of pages gotten.
> * 0 means we failed to fault in and get anything,
> @@ -500,7 +504,8 @@ static int get_user_bd_entry(struct mm_struct *mm, unsigned long *bd_entry_ret,
> */
> static int get_bt_addr(struct mm_struct *mm,
> long __user *bd_entry_ptr,
> - unsigned long *bt_addr_result)
> + unsigned long *bt_addr_result,
> + struct range_lock *mmrange)
> {
> int ret;
> int valid_bit;
> @@ -519,7 +524,8 @@ static int get_bt_addr(struct mm_struct *mm,
> if (!ret)
> break;
> if (ret == -EFAULT)
> - ret = mpx_resolve_fault(bd_entry_ptr, need_write);
> + ret = mpx_resolve_fault(bd_entry_ptr,
> + need_write, mmrange);
> /*
> * If we could not resolve the fault, consider it
> * userspace's fault and error out.
> @@ -730,7 +736,8 @@ static unsigned long mpx_get_bd_entry_offset(struct mm_struct *mm,
> }
>
> static int unmap_entire_bt(struct mm_struct *mm,
> - long __user *bd_entry, unsigned long bt_addr)
> + long __user *bd_entry, unsigned long bt_addr,
> + struct range_lock *mmrange)
> {
> unsigned long expected_old_val = bt_addr | MPX_BD_ENTRY_VALID_FLAG;
> unsigned long uninitialized_var(actual_old_val);
> @@ -747,7 +754,7 @@ static int unmap_entire_bt(struct mm_struct *mm,
> if (!ret)
> break;
> if (ret == -EFAULT)
> - ret = mpx_resolve_fault(bd_entry, need_write);
> + ret = mpx_resolve_fault(bd_entry, need_write, mmrange);
> /*
> * If we could not resolve the fault, consider it
> * userspace's fault and error out.
> @@ -780,11 +787,12 @@ static int unmap_entire_bt(struct mm_struct *mm,
> * avoid recursion, do_munmap() will check whether it comes
> * from one bounds table through VM_MPX flag.
> */
> - return do_munmap(mm, bt_addr, mpx_bt_size_bytes(mm), NULL);
> + return do_munmap(mm, bt_addr, mpx_bt_size_bytes(mm), NULL, mmrange);
> }
>
> static int try_unmap_single_bt(struct mm_struct *mm,
> - unsigned long start, unsigned long end)
> + unsigned long start, unsigned long end,
> + struct range_lock *mmrange)
> {
> struct vm_area_struct *next;
> struct vm_area_struct *prev;
> @@ -835,7 +843,7 @@ static int try_unmap_single_bt(struct mm_struct *mm,
> }
>
> bde_vaddr = mm->context.bd_addr + mpx_get_bd_entry_offset(mm, start);
> - ret = get_bt_addr(mm, bde_vaddr, &bt_addr);
> + ret = get_bt_addr(mm, bde_vaddr, &bt_addr, mmrange);
> /*
> * No bounds table there, so nothing to unmap.
> */
> @@ -853,12 +861,13 @@ static int try_unmap_single_bt(struct mm_struct *mm,
> */
> if ((start == bta_start_vaddr) &&
> (end == bta_end_vaddr))
> - return unmap_entire_bt(mm, bde_vaddr, bt_addr);
> + return unmap_entire_bt(mm, bde_vaddr, bt_addr, mmrange);
> return zap_bt_entries_mapping(mm, bt_addr, start, end);
> }
>
> static int mpx_unmap_tables(struct mm_struct *mm,
> - unsigned long start, unsigned long end)
> + unsigned long start, unsigned long end,
> + struct range_lock *mmrange)
> {
> unsigned long one_unmap_start;
> trace_mpx_unmap_search(start, end);
> @@ -876,7 +885,8 @@ static int mpx_unmap_tables(struct mm_struct *mm,
> */
> if (one_unmap_end > next_unmap_start)
> one_unmap_end = next_unmap_start;
> - ret = try_unmap_single_bt(mm, one_unmap_start, one_unmap_end);
> + ret = try_unmap_single_bt(mm, one_unmap_start, one_unmap_end,
> + mmrange);
> if (ret)
> return ret;
>
> @@ -894,7 +904,8 @@ static int mpx_unmap_tables(struct mm_struct *mm,
> * necessary, and the 'vma' is the first vma in this range (start -> end).
> */
> void mpx_notify_unmap(struct mm_struct *mm, struct vm_area_struct *vma,
> - unsigned long start, unsigned long end)
> + unsigned long start, unsigned long end,
> + struct range_lock *mmrange)
> {
> int ret;
>
> @@ -920,7 +931,7 @@ void mpx_notify_unmap(struct mm_struct *mm, struct vm_area_struct *vma,
> vma = vma->vm_next;
> } while (vma && vma->vm_start < end);
>
> - ret = mpx_unmap_tables(mm, start, end);
> + ret = mpx_unmap_tables(mm, start, end, mmrange);
> if (ret)
> force_sig(SIGSEGV, current);
> }
> diff --git a/arch/xtensa/mm/fault.c b/arch/xtensa/mm/fault.c
> index 8b9b6f44bb06..6f8e3e7cccb5 100644
> --- a/arch/xtensa/mm/fault.c
> +++ b/arch/xtensa/mm/fault.c
> @@ -44,6 +44,7 @@ void do_page_fault(struct pt_regs *regs)
> int is_write, is_exec;
> int fault;
> unsigned int flags = FAULT_FLAG_ALLOW_RETRY | FAULT_FLAG_KILLABLE;
> + DEFINE_RANGE_LOCK_FULL(mmrange);
>
> info.si_code = SEGV_MAPERR;
>
> @@ -108,7 +109,7 @@ void do_page_fault(struct pt_regs *regs)
> * make sure we exit gracefully rather than endlessly redo
> * the fault.
> */
> - fault = handle_mm_fault(vma, address, flags);
> + fault = handle_mm_fault(vma, address, flags, &mmrange);
>
> if ((fault & VM_FAULT_RETRY) && fatal_signal_pending(current))
> return;
> diff --git a/drivers/gpu/drm/amd/amdgpu/amdgpu_ttm.c b/drivers/gpu/drm/amd/amdgpu/amdgpu_ttm.c
> index e4bb435e614b..bd464a599341 100644
> --- a/drivers/gpu/drm/amd/amdgpu/amdgpu_ttm.c
> +++ b/drivers/gpu/drm/amd/amdgpu/amdgpu_ttm.c
> @@ -691,6 +691,7 @@ int amdgpu_ttm_tt_get_user_pages(struct ttm_tt *ttm, struct page **pages)
> unsigned int flags = 0;
> unsigned pinned = 0;
> int r;
> + DEFINE_RANGE_LOCK_FULL(mmrange);
>
> if (!(gtt->userflags & AMDGPU_GEM_USERPTR_READONLY))
> flags |= FOLL_WRITE;
> @@ -721,7 +722,7 @@ int amdgpu_ttm_tt_get_user_pages(struct ttm_tt *ttm, struct page **pages)
> list_add(&guptask.list, >t->guptasks);
> spin_unlock(>t->guptasklock);
>
> - r = get_user_pages(userptr, num_pages, flags, p, NULL);
> + r = get_user_pages(userptr, num_pages, flags, p, NULL, &mmrange);
>
> spin_lock(>t->guptasklock);
> list_del(&guptask.list);
> diff --git a/drivers/gpu/drm/i915/i915_gem_userptr.c b/drivers/gpu/drm/i915/i915_gem_userptr.c
> index 382a77a1097e..881bcc7d663a 100644
> --- a/drivers/gpu/drm/i915/i915_gem_userptr.c
> +++ b/drivers/gpu/drm/i915/i915_gem_userptr.c
> @@ -512,6 +512,8 @@ __i915_gem_userptr_get_pages_worker(struct work_struct *_work)
>
> ret = -EFAULT;
> if (mmget_not_zero(mm)) {
> + DEFINE_RANGE_LOCK_FULL(mmrange);
> +
> down_read(&mm->mmap_sem);
> while (pinned < npages) {
> ret = get_user_pages_remote
> @@ -519,7 +521,7 @@ __i915_gem_userptr_get_pages_worker(struct work_struct *_work)
> obj->userptr.ptr + pinned * PAGE_SIZE,
> npages - pinned,
> flags,
> - pvec + pinned, NULL, NULL);
> + pvec + pinned, NULL, NULL, &mmrange);
> if (ret < 0)
> break;
>
> diff --git a/drivers/gpu/drm/radeon/radeon_ttm.c b/drivers/gpu/drm/radeon/radeon_ttm.c
> index a0a839bc39bf..9fc3a4f86945 100644
> --- a/drivers/gpu/drm/radeon/radeon_ttm.c
> +++ b/drivers/gpu/drm/radeon/radeon_ttm.c
> @@ -545,6 +545,8 @@ static int radeon_ttm_tt_pin_userptr(struct ttm_tt *ttm)
> struct radeon_ttm_tt *gtt = (void *)ttm;
> unsigned pinned = 0, nents;
> int r;
> + // XXX: this is wrong!!
> + DEFINE_RANGE_LOCK_FULL(mmrange);
>
> int write = !(gtt->userflags & RADEON_GEM_USERPTR_READONLY);
> enum dma_data_direction direction = write ?
> @@ -569,7 +571,7 @@ static int radeon_ttm_tt_pin_userptr(struct ttm_tt *ttm)
> struct page **pages = ttm->pages + pinned;
>
> r = get_user_pages(userptr, num_pages, write ? FOLL_WRITE : 0,
> - pages, NULL);
> + pages, NULL, &mmrange);
> if (r < 0)
> goto release_pages;
>
> diff --git a/drivers/infiniband/core/umem.c b/drivers/infiniband/core/umem.c
> index 9a4e899d94b3..fd9601ed5b84 100644
> --- a/drivers/infiniband/core/umem.c
> +++ b/drivers/infiniband/core/umem.c
> @@ -96,6 +96,7 @@ struct ib_umem *ib_umem_get(struct ib_ucontext *context, unsigned long addr,
> struct scatterlist *sg, *sg_list_start;
> int need_release = 0;
> unsigned int gup_flags = FOLL_WRITE;
> + DEFINE_RANGE_LOCK_FULL(mmrange);
>
> if (dmasync)
> dma_attrs |= DMA_ATTR_WRITE_BARRIER;
> @@ -194,7 +195,7 @@ struct ib_umem *ib_umem_get(struct ib_ucontext *context, unsigned long addr,
> ret = get_user_pages_longterm(cur_base,
> min_t(unsigned long, npages,
> PAGE_SIZE / sizeof (struct page *)),
> - gup_flags, page_list, vma_list);
> + gup_flags, page_list, vma_list, &mmrange);
>
> if (ret < 0)
> goto out;
> diff --git a/drivers/infiniband/core/umem_odp.c b/drivers/infiniband/core/umem_odp.c
> index 2aadf5813a40..0572953260e8 100644
> --- a/drivers/infiniband/core/umem_odp.c
> +++ b/drivers/infiniband/core/umem_odp.c
> @@ -632,6 +632,7 @@ int ib_umem_odp_map_dma_pages(struct ib_umem *umem, u64 user_virt, u64 bcnt,
> int j, k, ret = 0, start_idx, npages = 0, page_shift;
> unsigned int flags = 0;
> phys_addr_t p = 0;
> + DEFINE_RANGE_LOCK_FULL(mmrange);
>
> if (access_mask == 0)
> return -EINVAL;
> @@ -683,7 +684,7 @@ int ib_umem_odp_map_dma_pages(struct ib_umem *umem, u64 user_virt, u64 bcnt,
> */
> npages = get_user_pages_remote(owning_process, owning_mm,
> user_virt, gup_num_pages,
> - flags, local_page_list, NULL, NULL);
> + flags, local_page_list, NULL, NULL, &mmrange);
> up_read(&owning_mm->mmap_sem);
>
> if (npages < 0)
> diff --git a/drivers/infiniband/hw/qib/qib_user_pages.c b/drivers/infiniband/hw/qib/qib_user_pages.c
> index ce83ba9a12ef..6bcb4f9f9b30 100644
> --- a/drivers/infiniband/hw/qib/qib_user_pages.c
> +++ b/drivers/infiniband/hw/qib/qib_user_pages.c
> @@ -53,7 +53,7 @@ static void __qib_release_user_pages(struct page **p, size_t num_pages,
> * Call with current->mm->mmap_sem held.
> */
> static int __qib_get_user_pages(unsigned long start_page, size_t num_pages,
> - struct page **p)
> + struct page **p, struct range_lock *mmrange)
> {
> unsigned long lock_limit;
> size_t got;
> @@ -70,7 +70,7 @@ static int __qib_get_user_pages(unsigned long start_page, size_t num_pages,
> ret = get_user_pages(start_page + got * PAGE_SIZE,
> num_pages - got,
> FOLL_WRITE | FOLL_FORCE,
> - p + got, NULL);
> + p + got, NULL, mmrange);
> if (ret < 0)
> goto bail_release;
> }
> @@ -134,10 +134,11 @@ int qib_get_user_pages(unsigned long start_page, size_t num_pages,
> struct page **p)
> {
> int ret;
> + DEFINE_RANGE_LOCK_FULL(mmrange);
>
> down_write(¤t->mm->mmap_sem);
>
> - ret = __qib_get_user_pages(start_page, num_pages, p);
> + ret = __qib_get_user_pages(start_page, num_pages, p, &mmrange);
>
> up_write(¤t->mm->mmap_sem);
>
> diff --git a/drivers/infiniband/hw/usnic/usnic_uiom.c b/drivers/infiniband/hw/usnic/usnic_uiom.c
> index 4381c0a9a873..5f36c6d2e21b 100644
> --- a/drivers/infiniband/hw/usnic/usnic_uiom.c
> +++ b/drivers/infiniband/hw/usnic/usnic_uiom.c
> @@ -113,6 +113,7 @@ static int usnic_uiom_get_pages(unsigned long addr, size_t size, int writable,
> int flags;
> dma_addr_t pa;
> unsigned int gup_flags;
> + DEFINE_RANGE_LOCK_FULL(mmrange);
>
> if (!can_do_mlock())
> return -EPERM;
> @@ -146,7 +147,7 @@ static int usnic_uiom_get_pages(unsigned long addr, size_t size, int writable,
> ret = get_user_pages(cur_base,
> min_t(unsigned long, npages,
> PAGE_SIZE / sizeof(struct page *)),
> - gup_flags, page_list, NULL);
> + gup_flags, page_list, NULL, &mmrange);
>
> if (ret < 0)
> goto out;
> diff --git a/drivers/iommu/amd_iommu_v2.c b/drivers/iommu/amd_iommu_v2.c
> index 1d0b53a04a08..15a7103fd84c 100644
> --- a/drivers/iommu/amd_iommu_v2.c
> +++ b/drivers/iommu/amd_iommu_v2.c
> @@ -512,6 +512,7 @@ static void do_fault(struct work_struct *work)
> unsigned int flags = 0;
> struct mm_struct *mm;
> u64 address;
> + DEFINE_RANGE_LOCK_FULL(mmrange);
>
> mm = fault->state->mm;
> address = fault->address;
> @@ -523,7 +524,7 @@ static void do_fault(struct work_struct *work)
> flags |= FAULT_FLAG_REMOTE;
>
> down_read(&mm->mmap_sem);
> - vma = find_extend_vma(mm, address);
> + vma = find_extend_vma(mm, address, &mmrange);
> if (!vma || address < vma->vm_start)
> /* failed to get a vma in the right range */
> goto out;
> @@ -532,7 +533,7 @@ static void do_fault(struct work_struct *work)
> if (access_error(vma, fault))
> goto out;
>
> - ret = handle_mm_fault(vma, address, flags);
> + ret = handle_mm_fault(vma, address, flags, &mmrange);
> out:
> up_read(&mm->mmap_sem);
>
> diff --git a/drivers/iommu/intel-svm.c b/drivers/iommu/intel-svm.c
> index 35a408d0ae4f..6a74386ee83f 100644
> --- a/drivers/iommu/intel-svm.c
> +++ b/drivers/iommu/intel-svm.c
> @@ -585,6 +585,7 @@ static irqreturn_t prq_event_thread(int irq, void *d)
> struct intel_iommu *iommu = d;
> struct intel_svm *svm = NULL;
> int head, tail, handled = 0;
> + DEFINE_RANGE_LOCK_FULL(mmrange);
>
> /* Clear PPR bit before reading head/tail registers, to
> * ensure that we get a new interrupt if needed. */
> @@ -643,7 +644,7 @@ static irqreturn_t prq_event_thread(int irq, void *d)
> goto bad_req;
>
> down_read(&svm->mm->mmap_sem);
> - vma = find_extend_vma(svm->mm, address);
> + vma = find_extend_vma(svm->mm, address, &mmrange);
> if (!vma || address < vma->vm_start)
> goto invalid;
>
> @@ -651,7 +652,7 @@ static irqreturn_t prq_event_thread(int irq, void *d)
> goto invalid;
>
> ret = handle_mm_fault(vma, address,
> - req->wr_req ? FAULT_FLAG_WRITE : 0);
> + req->wr_req ? FAULT_FLAG_WRITE : 0, &mmrange);
> if (ret & VM_FAULT_ERROR)
> goto invalid;
>
> diff --git a/drivers/media/v4l2-core/videobuf-dma-sg.c b/drivers/media/v4l2-core/videobuf-dma-sg.c
> index f412429cf5ba..64a4cd62eeb3 100644
> --- a/drivers/media/v4l2-core/videobuf-dma-sg.c
> +++ b/drivers/media/v4l2-core/videobuf-dma-sg.c
> @@ -152,7 +152,8 @@ static void videobuf_dma_init(struct videobuf_dmabuf *dma)
> }
>
> static int videobuf_dma_init_user_locked(struct videobuf_dmabuf *dma,
> - int direction, unsigned long data, unsigned long size)
> + int direction, unsigned long data, unsigned long size,
> + struct range_lock *mmrange)
> {
> unsigned long first, last;
> int err, rw = 0;
> @@ -186,7 +187,7 @@ static int videobuf_dma_init_user_locked(struct videobuf_dmabuf *dma,
> data, size, dma->nr_pages);
>
> err = get_user_pages_longterm(data & PAGE_MASK, dma->nr_pages,
> - flags, dma->pages, NULL);
> + flags, dma->pages, NULL, mmrange);
>
> if (err != dma->nr_pages) {
> dma->nr_pages = (err >= 0) ? err : 0;
> @@ -201,9 +202,10 @@ static int videobuf_dma_init_user(struct videobuf_dmabuf *dma, int direction,
> unsigned long data, unsigned long size)
> {
> int ret;
> + DEFINE_RANGE_LOCK_FULL(mmrange);
>
> down_read(¤t->mm->mmap_sem);
> - ret = videobuf_dma_init_user_locked(dma, direction, data, size);
> + ret = videobuf_dma_init_user_locked(dma, direction, data, size, &mmrange);
> up_read(¤t->mm->mmap_sem);
>
> return ret;
> @@ -539,9 +541,14 @@ static int __videobuf_iolock(struct videobuf_queue *q,
> we take current->mm->mmap_sem there, to prevent
> locking inversion, so don't take it here */
>
> + /* XXX: can we use a local mmrange here? */
> + DEFINE_RANGE_LOCK_FULL(mmrange);
> +
> err = videobuf_dma_init_user_locked(&mem->dma,
> - DMA_FROM_DEVICE,
> - vb->baddr, vb->bsize);
> + DMA_FROM_DEVICE,
> + vb->baddr,
> + vb->bsize,
> + &mmrange);
> if (0 != err)
> return err;
> }
> @@ -555,6 +562,7 @@ static int __videobuf_iolock(struct videobuf_queue *q,
> * building for PAE. Compiler doesn't like direct casting
> * of a 32 bit ptr to 64 bit integer.
> */
> +
> bus = (dma_addr_t)(unsigned long)fbuf->base + vb->boff;
> pages = PAGE_ALIGN(vb->size) >> PAGE_SHIFT;
> err = videobuf_dma_init_overlay(&mem->dma, DMA_FROM_DEVICE,
> diff --git a/drivers/misc/mic/scif/scif_rma.c b/drivers/misc/mic/scif/scif_rma.c
> index c824329f7012..6ecac843e5f3 100644
> --- a/drivers/misc/mic/scif/scif_rma.c
> +++ b/drivers/misc/mic/scif/scif_rma.c
> @@ -1332,6 +1332,7 @@ int __scif_pin_pages(void *addr, size_t len, int *out_prot,
> int prot = *out_prot;
> int ulimit = 0;
> struct mm_struct *mm = NULL;
> + DEFINE_RANGE_LOCK_FULL(mmrange);
>
> /* Unsupported flags */
> if (map_flags & ~(SCIF_MAP_KERNEL | SCIF_MAP_ULIMIT))
> @@ -1400,7 +1401,7 @@ int __scif_pin_pages(void *addr, size_t len, int *out_prot,
> nr_pages,
> (prot & SCIF_PROT_WRITE) ? FOLL_WRITE : 0,
> pinned_pages->pages,
> - NULL);
> + NULL, &mmrange);
> up_write(&mm->mmap_sem);
> if (nr_pages != pinned_pages->nr_pages) {
> if (try_upgrade) {
> diff --git a/drivers/misc/sgi-gru/grufault.c b/drivers/misc/sgi-gru/grufault.c
> index 93be82fc338a..b35d60bb2197 100644
> --- a/drivers/misc/sgi-gru/grufault.c
> +++ b/drivers/misc/sgi-gru/grufault.c
> @@ -189,7 +189,8 @@ static void get_clear_fault_map(struct gru_state *gru,
> */
> static int non_atomic_pte_lookup(struct vm_area_struct *vma,
> unsigned long vaddr, int write,
> - unsigned long *paddr, int *pageshift)
> + unsigned long *paddr, int *pageshift,
> + struct range_lock *mmrange)
> {
> struct page *page;
>
> @@ -198,7 +199,8 @@ static int non_atomic_pte_lookup(struct vm_area_struct *vma,
> #else
> *pageshift = PAGE_SHIFT;
> #endif
> - if (get_user_pages(vaddr, 1, write ? FOLL_WRITE : 0, &page, NULL) <= 0)
> + if (get_user_pages(vaddr, 1, write ? FOLL_WRITE : 0,
> + &page, NULL, mmrange) <= 0)
There is no need to pass down the range here since underlying called
__get_user_pages_locked() is told to not unlock the mmap_sem.
In general get_user_pages() doesn't need a range parameter.
> return -EFAULT;
> *paddr = page_to_phys(page);
> put_page(page);
> @@ -263,7 +265,8 @@ static int atomic_pte_lookup(struct vm_area_struct *vma, unsigned long vaddr,
> }
>
> static int gru_vtop(struct gru_thread_state *gts, unsigned long vaddr,
> - int write, int atomic, unsigned long *gpa, int *pageshift)
> + int write, int atomic, unsigned long *gpa, int *pageshift,
> + struct range_lock *mmrange)
> {
> struct mm_struct *mm = gts->ts_mm;
> struct vm_area_struct *vma;
> @@ -283,7 +286,8 @@ static int gru_vtop(struct gru_thread_state *gts, unsigned long vaddr,
> if (ret) {
> if (atomic)
> goto upm;
> - if (non_atomic_pte_lookup(vma, vaddr, write, &paddr, &ps))
> + if (non_atomic_pte_lookup(vma, vaddr, write, &paddr,
> + &ps, mmrange))
> goto inval;
> }
> if (is_gru_paddr(paddr))
> @@ -324,7 +328,8 @@ static void gru_preload_tlb(struct gru_state *gru,
> unsigned long fault_vaddr, int asid, int write,
> unsigned char tlb_preload_count,
> struct gru_tlb_fault_handle *tfh,
> - struct gru_control_block_extended *cbe)
> + struct gru_control_block_extended *cbe,
> + struct range_lock *mmrange)
> {
> unsigned long vaddr = 0, gpa;
> int ret, pageshift;
> @@ -342,7 +347,7 @@ static void gru_preload_tlb(struct gru_state *gru,
> vaddr = min(vaddr, fault_vaddr + tlb_preload_count * PAGE_SIZE);
>
> while (vaddr > fault_vaddr) {
> - ret = gru_vtop(gts, vaddr, write, atomic, &gpa, &pageshift);
> + ret = gru_vtop(gts, vaddr, write, atomic, &gpa, &pageshift, mmrange);
> if (ret || tfh_write_only(tfh, gpa, GAA_RAM, vaddr, asid, write,
> GRU_PAGESIZE(pageshift)))
> return;
> @@ -368,7 +373,8 @@ static void gru_preload_tlb(struct gru_state *gru,
> static int gru_try_dropin(struct gru_state *gru,
> struct gru_thread_state *gts,
> struct gru_tlb_fault_handle *tfh,
> - struct gru_instruction_bits *cbk)
> + struct gru_instruction_bits *cbk,
> + struct range_lock *mmrange)
> {
> struct gru_control_block_extended *cbe = NULL;
> unsigned char tlb_preload_count = gts->ts_tlb_preload_count;
> @@ -423,7 +429,7 @@ static int gru_try_dropin(struct gru_state *gru,
> if (atomic_read(>s->ts_gms->ms_range_active))
> goto failactive;
>
> - ret = gru_vtop(gts, vaddr, write, atomic, &gpa, &pageshift);
> + ret = gru_vtop(gts, vaddr, write, atomic, &gpa, &pageshift, mmrange);
> if (ret == VTOP_INVALID)
> goto failinval;
> if (ret == VTOP_RETRY)
> @@ -438,7 +444,8 @@ static int gru_try_dropin(struct gru_state *gru,
> }
>
> if (unlikely(cbe) && pageshift == PAGE_SHIFT) {
> - gru_preload_tlb(gru, gts, atomic, vaddr, asid, write, tlb_preload_count, tfh, cbe);
> + gru_preload_tlb(gru, gts, atomic, vaddr, asid, write,
> + tlb_preload_count, tfh, cbe, mmrange);
> gru_flush_cache_cbe(cbe);
> }
>
> @@ -587,10 +594,13 @@ static irqreturn_t gru_intr(int chiplet, int blade)
> * If it fails, retry the fault in user context.
> */
> gts->ustats.fmm_tlbmiss++;
> - if (!gts->ts_force_cch_reload &&
> - down_read_trylock(>s->ts_mm->mmap_sem)) {
> - gru_try_dropin(gru, gts, tfh, NULL);
> - up_read(>s->ts_mm->mmap_sem);
> + if (!gts->ts_force_cch_reload) {
> + DEFINE_RANGE_LOCK_FULL(mmrange);
> +
> + if (down_read_trylock(>s->ts_mm->mmap_sem)) {
> + gru_try_dropin(gru, gts, tfh, NULL, &mmrange);
> + up_read(>s->ts_mm->mmap_sem);
> + }
> } else {
> tfh_user_polling_mode(tfh);
> STAT(intr_mm_lock_failed);
> @@ -625,7 +635,7 @@ irqreturn_t gru_intr_mblade(int irq, void *dev_id)
>
> static int gru_user_dropin(struct gru_thread_state *gts,
> struct gru_tlb_fault_handle *tfh,
> - void *cb)
> + void *cb, struct range_lock *mmrange)
> {
> struct gru_mm_struct *gms = gts->ts_gms;
> int ret;
> @@ -635,7 +645,7 @@ static int gru_user_dropin(struct gru_thread_state *gts,
> wait_event(gms->ms_wait_queue,
> atomic_read(&gms->ms_range_active) == 0);
> prefetchw(tfh); /* Helps on hdw, required for emulator */
> - ret = gru_try_dropin(gts->ts_gru, gts, tfh, cb);
> + ret = gru_try_dropin(gts->ts_gru, gts, tfh, cb, mmrange);
> if (ret <= 0)
> return ret;
> STAT(call_os_wait_queue);
> @@ -653,6 +663,7 @@ int gru_handle_user_call_os(unsigned long cb)
> struct gru_thread_state *gts;
> void *cbk;
> int ucbnum, cbrnum, ret = -EINVAL;
> + DEFINE_RANGE_LOCK_FULL(mmrange);
>
> STAT(call_os);
>
> @@ -685,7 +696,7 @@ int gru_handle_user_call_os(unsigned long cb)
> tfh = get_tfh_by_index(gts->ts_gru, cbrnum);
> cbk = get_gseg_base_address_cb(gts->ts_gru->gs_gru_base_vaddr,
> gts->ts_ctxnum, ucbnum);
> - ret = gru_user_dropin(gts, tfh, cbk);
> + ret = gru_user_dropin(gts, tfh, cbk, &mmrange);
> }
> exit:
> gru_unlock_gts(gts);
> diff --git a/drivers/vfio/vfio_iommu_type1.c b/drivers/vfio/vfio_iommu_type1.c
> index e30e29ae4819..1b3b103da637 100644
> --- a/drivers/vfio/vfio_iommu_type1.c
> +++ b/drivers/vfio/vfio_iommu_type1.c
> @@ -345,13 +345,14 @@ static int vaddr_get_pfn(struct mm_struct *mm, unsigned long vaddr,
> page);
> } else {
> unsigned int flags = 0;
> + DEFINE_RANGE_LOCK_FULL(mmrange);
>
> if (prot & IOMMU_WRITE)
> flags |= FOLL_WRITE;
>
> down_read(&mm->mmap_sem);
> ret = get_user_pages_remote(NULL, mm, vaddr, 1, flags, page,
> - NULL, NULL);
> + NULL, NULL, &mmrange);
> up_read(&mm->mmap_sem);
> }
>
> diff --git a/fs/aio.c b/fs/aio.c
> index a062d75109cb..31774b75c372 100644
> --- a/fs/aio.c
> +++ b/fs/aio.c
> @@ -457,6 +457,7 @@ static int aio_setup_ring(struct kioctx *ctx, unsigned int nr_events)
> int nr_pages;
> int i;
> struct file *file;
> + DEFINE_RANGE_LOCK_FULL(mmrange);
>
> /* Compensate for the ring buffer's head/tail overlap entry */
> nr_events += 2; /* 1 is required, 2 for good luck */
> @@ -519,7 +520,7 @@ static int aio_setup_ring(struct kioctx *ctx, unsigned int nr_events)
>
> ctx->mmap_base = do_mmap_pgoff(ctx->aio_ring_file, 0, ctx->mmap_size,
> PROT_READ | PROT_WRITE,
> - MAP_SHARED, 0, &unused, NULL);
> + MAP_SHARED, 0, &unused, NULL, &mmrange);
> up_write(&mm->mmap_sem);
> if (IS_ERR((void *)ctx->mmap_base)) {
> ctx->mmap_size = 0;
> diff --git a/fs/binfmt_elf.c b/fs/binfmt_elf.c
> index 2f492dfcabde..9aea808d55d7 100644
> --- a/fs/binfmt_elf.c
> +++ b/fs/binfmt_elf.c
> @@ -180,6 +180,7 @@ create_elf_tables(struct linux_binprm *bprm, struct elfhdr *exec,
> int ei_index = 0;
> const struct cred *cred = current_cred();
> struct vm_area_struct *vma;
> + DEFINE_RANGE_LOCK_FULL(mmrange);
>
> /*
> * In some cases (e.g. Hyper-Threading), we want to avoid L1
> @@ -300,7 +301,7 @@ create_elf_tables(struct linux_binprm *bprm, struct elfhdr *exec,
> * Grow the stack manually; some architectures have a limit on how
> * far ahead a user-space access may be in order to grow the stack.
> */
> - vma = find_extend_vma(current->mm, bprm->p);
> + vma = find_extend_vma(current->mm, bprm->p, &mmrange);
> if (!vma)
> return -EFAULT;
>
> diff --git a/fs/exec.c b/fs/exec.c
> index e7b69e14649f..e46752874b47 100644
> --- a/fs/exec.c
> +++ b/fs/exec.c
> @@ -197,6 +197,11 @@ static struct page *get_arg_page(struct linux_binprm *bprm, unsigned long pos,
> struct page *page;
> int ret;
> unsigned int gup_flags = FOLL_FORCE;
> + /*
> + * No concurrency for the bprm->mm yet -- this is exec path;
> + * but gup needs an mmrange.
> + */
> + DEFINE_RANGE_LOCK_FULL(mmrange);
>
> #ifdef CONFIG_STACK_GROWSUP
> if (write) {
> @@ -214,7 +219,7 @@ static struct page *get_arg_page(struct linux_binprm *bprm, unsigned long pos,
> * doing the exec and bprm->mm is the new process's mm.
> */
> ret = get_user_pages_remote(current, bprm->mm, pos, 1, gup_flags,
> - &page, NULL, NULL);
> + &page, NULL, NULL, &mmrange);
> if (ret <= 0)
> return NULL;
>
> @@ -615,7 +620,8 @@ EXPORT_SYMBOL(copy_strings_kernel);
> * 4) Free up any cleared pgd range.
> * 5) Shrink the vma to cover only the new range.
> */
> -static int shift_arg_pages(struct vm_area_struct *vma, unsigned long shift)
> +static int shift_arg_pages(struct vm_area_struct *vma, unsigned long shift,
> + struct range_lock *mmrange)
> {
> struct mm_struct *mm = vma->vm_mm;
> unsigned long old_start = vma->vm_start;
> @@ -637,7 +643,8 @@ static int shift_arg_pages(struct vm_area_struct *vma, unsigned long shift)
> /*
> * cover the whole range: [new_start, old_end)
> */
> - if (vma_adjust(vma, new_start, old_end, vma->vm_pgoff, NULL))
> + if (vma_adjust(vma, new_start, old_end, vma->vm_pgoff, NULL,
> + mmrange))
> return -ENOMEM;
>
> /*
> @@ -671,7 +678,7 @@ static int shift_arg_pages(struct vm_area_struct *vma, unsigned long shift)
> /*
> * Shrink the vma to just the new range. Always succeeds.
> */
> - vma_adjust(vma, new_start, new_end, vma->vm_pgoff, NULL);
> + vma_adjust(vma, new_start, new_end, vma->vm_pgoff, NULL, mmrange);
>
> return 0;
> }
> @@ -694,6 +701,7 @@ int setup_arg_pages(struct linux_binprm *bprm,
> unsigned long stack_size;
> unsigned long stack_expand;
> unsigned long rlim_stack;
> + DEFINE_RANGE_LOCK_FULL(mmrange);
>
> #ifdef CONFIG_STACK_GROWSUP
> /* Limit stack size */
> @@ -749,14 +757,14 @@ int setup_arg_pages(struct linux_binprm *bprm,
> vm_flags |= VM_STACK_INCOMPLETE_SETUP;
>
> ret = mprotect_fixup(vma, &prev, vma->vm_start, vma->vm_end,
> - vm_flags);
> + vm_flags, &mmrange);
> if (ret)
> goto out_unlock;
> BUG_ON(prev != vma);
>
> /* Move stack pages down in memory. */
> if (stack_shift) {
> - ret = shift_arg_pages(vma, stack_shift);
> + ret = shift_arg_pages(vma, stack_shift, &mmrange);
> if (ret)
> goto out_unlock;
> }
> diff --git a/fs/proc/internal.h b/fs/proc/internal.h
> index d697c8ab0a14..791f9f93643c 100644
> --- a/fs/proc/internal.h
> +++ b/fs/proc/internal.h
> @@ -16,6 +16,7 @@
> #include <linux/binfmts.h>
> #include <linux/sched/coredump.h>
> #include <linux/sched/task.h>
> +#include <linux/range_lock.h>
>
> struct ctl_table_header;
> struct mempolicy;
> @@ -263,6 +264,8 @@ struct proc_maps_private {
> #ifdef CONFIG_NUMA
> struct mempolicy *task_mempolicy;
> #endif
> + /* mmap_sem is held across all stages of seqfile */
> + struct range_lock mmrange;
> } __randomize_layout;
>
> struct mm_struct *proc_mem_open(struct inode *inode, unsigned int mode);
> diff --git a/fs/proc/task_mmu.c b/fs/proc/task_mmu.c
> index b66fc8de7d34..7c0a79a937b5 100644
> --- a/fs/proc/task_mmu.c
> +++ b/fs/proc/task_mmu.c
> @@ -174,6 +174,7 @@ static void *m_start(struct seq_file *m, loff_t *ppos)
> if (!mm || !mmget_not_zero(mm))
> return NULL;
>
> + range_lock_init_full(&priv->mmrange);
> down_read(&mm->mmap_sem);
> hold_task_mempolicy(priv);
> priv->tail_vma = get_gate_vma(mm);
> @@ -514,7 +515,7 @@ static void smaps_account(struct mem_size_stats *mss, struct page *page,
>
> #ifdef CONFIG_SHMEM
> static int smaps_pte_hole(unsigned long addr, unsigned long end,
> - struct mm_walk *walk)
> + struct mm_walk *walk, struct range_lock *mmrange)
> {
> struct mem_size_stats *mss = walk->private;
>
> @@ -605,7 +606,7 @@ static void smaps_pmd_entry(pmd_t *pmd, unsigned long addr,
> #endif
>
> static int smaps_pte_range(pmd_t *pmd, unsigned long addr, unsigned long end,
> - struct mm_walk *walk)
> + struct mm_walk *walk, struct range_lock *mmrange)
> {
> struct vm_area_struct *vma = walk->vma;
> pte_t *pte;
> @@ -797,7 +798,7 @@ static int show_smap(struct seq_file *m, void *v, int is_pid)
> #endif
>
> /* mmap_sem is held in m_start */
> - walk_page_vma(vma, &smaps_walk);
> + walk_page_vma(vma, &smaps_walk, &priv->mmrange);
> if (vma->vm_flags & VM_LOCKED)
> mss->pss_locked += mss->pss;
>
> @@ -1012,7 +1013,8 @@ static inline void clear_soft_dirty_pmd(struct vm_area_struct *vma,
> #endif
>
> static int clear_refs_pte_range(pmd_t *pmd, unsigned long addr,
> - unsigned long end, struct mm_walk *walk)
> + unsigned long end, struct mm_walk *walk,
> + struct range_lock *mmrange)
> {
> struct clear_refs_private *cp = walk->private;
> struct vm_area_struct *vma = walk->vma;
> @@ -1103,6 +1105,7 @@ static ssize_t clear_refs_write(struct file *file, const char __user *buf,
> struct mmu_gather tlb;
> int itype;
> int rv;
> + DEFINE_RANGE_LOCK_FULL(mmrange);
>
> memset(buffer, 0, sizeof(buffer));
> if (count > sizeof(buffer) - 1)
> @@ -1166,7 +1169,8 @@ static ssize_t clear_refs_write(struct file *file, const char __user *buf,
> }
> mmu_notifier_invalidate_range_start(mm, 0, -1);
> }
> - walk_page_range(0, mm->highest_vm_end, &clear_refs_walk);
> + walk_page_range(0, mm->highest_vm_end, &clear_refs_walk,
> + &mmrange);
> if (type == CLEAR_REFS_SOFT_DIRTY)
> mmu_notifier_invalidate_range_end(mm, 0, -1);
> tlb_finish_mmu(&tlb, 0, -1);
> @@ -1223,7 +1227,7 @@ static int add_to_pagemap(unsigned long addr, pagemap_entry_t *pme,
> }
>
> static int pagemap_pte_hole(unsigned long start, unsigned long end,
> - struct mm_walk *walk)
> + struct mm_walk *walk, struct range_lock *mmrange)
> {
> struct pagemapread *pm = walk->private;
> unsigned long addr = start;
> @@ -1301,7 +1305,7 @@ static pagemap_entry_t pte_to_pagemap_entry(struct pagemapread *pm,
> }
>
> static int pagemap_pmd_range(pmd_t *pmdp, unsigned long addr, unsigned long end,
> - struct mm_walk *walk)
> + struct mm_walk *walk, struct range_lock *mmrange)
> {
> struct vm_area_struct *vma = walk->vma;
> struct pagemapread *pm = walk->private;
> @@ -1467,6 +1471,8 @@ static ssize_t pagemap_read(struct file *file, char __user *buf,
> unsigned long start_vaddr;
> unsigned long end_vaddr;
> int ret = 0, copied = 0;
> + DEFINE_RANGE_LOCK_FULL(tmprange);
> + struct range_lock *mmrange = &tmprange;
>
> if (!mm || !mmget_not_zero(mm))
> goto out;
> @@ -1523,7 +1529,8 @@ static ssize_t pagemap_read(struct file *file, char __user *buf,
> if (end < start_vaddr || end > end_vaddr)
> end = end_vaddr;
> down_read(&mm->mmap_sem);
> - ret = walk_page_range(start_vaddr, end, &pagemap_walk);
> + ret = walk_page_range(start_vaddr, end, &pagemap_walk,
> + mmrange);
> up_read(&mm->mmap_sem);
> start_vaddr = end;
>
> @@ -1671,7 +1678,8 @@ static struct page *can_gather_numa_stats_pmd(pmd_t pmd,
> #endif
>
> static int gather_pte_stats(pmd_t *pmd, unsigned long addr,
> - unsigned long end, struct mm_walk *walk)
> + unsigned long end, struct mm_walk *walk,
> + struct range_lock *mmrange)
> {
> struct numa_maps *md = walk->private;
> struct vm_area_struct *vma = walk->vma;
> @@ -1740,6 +1748,7 @@ static int gather_hugetlb_stats(pte_t *pte, unsigned long hmask,
> */
> static int show_numa_map(struct seq_file *m, void *v, int is_pid)
> {
> + struct proc_maps_private *priv = m->private;
> struct numa_maps_private *numa_priv = m->private;
> struct proc_maps_private *proc_priv = &numa_priv->proc_maps;
> struct vm_area_struct *vma = v;
> @@ -1785,7 +1794,7 @@ static int show_numa_map(struct seq_file *m, void *v, int is_pid)
> seq_puts(m, " huge");
>
> /* mmap_sem is held by m_start */
> - walk_page_vma(vma, &walk);
> + walk_page_vma(vma, &walk, &priv->mmrange);
>
> if (!md->pages)
> goto out;
> diff --git a/fs/proc/vmcore.c b/fs/proc/vmcore.c
> index a45f0af22a60..3768955c10bc 100644
> --- a/fs/proc/vmcore.c
> +++ b/fs/proc/vmcore.c
> @@ -350,6 +350,11 @@ static int remap_oldmem_pfn_checked(struct vm_area_struct *vma,
> unsigned long pos_start, pos_end, pos;
> unsigned long zeropage_pfn = my_zero_pfn(0);
> size_t len = 0;
> + /*
> + * No concurrency for the bprm->mm yet -- this is a vmcore path,
> + * but do_munmap() needs an mmrange.
> + */
> + DEFINE_RANGE_LOCK_FULL(mmrange);
>
> pos_start = pfn;
> pos_end = pfn + (size >> PAGE_SHIFT);
> @@ -388,7 +393,7 @@ static int remap_oldmem_pfn_checked(struct vm_area_struct *vma,
> }
> return 0;
> fail:
> - do_munmap(vma->vm_mm, from, len, NULL);
> + do_munmap(vma->vm_mm, from, len, NULL, &mmrange);
> return -EAGAIN;
> }
>
> @@ -411,6 +416,11 @@ static int mmap_vmcore(struct file *file, struct vm_area_struct *vma)
> size_t size = vma->vm_end - vma->vm_start;
> u64 start, end, len, tsz;
> struct vmcore *m;
> + /*
> + * No concurrency for the bprm->mm yet -- this is a vmcore path,
> + * but do_munmap() needs an mmrange.
> + */
> + DEFINE_RANGE_LOCK_FULL(mmrange);
>
> start = (u64)vma->vm_pgoff << PAGE_SHIFT;
> end = start + size;
> @@ -481,7 +491,7 @@ static int mmap_vmcore(struct file *file, struct vm_area_struct *vma)
>
> return 0;
> fail:
> - do_munmap(vma->vm_mm, vma->vm_start, len, NULL);
> + do_munmap(vma->vm_mm, vma->vm_start, len, NULL, &mmrange);
> return -EAGAIN;
> }
> #else
> diff --git a/fs/userfaultfd.c b/fs/userfaultfd.c
> index 87a13a7c8270..e3089865fd52 100644
> --- a/fs/userfaultfd.c
> +++ b/fs/userfaultfd.c
> @@ -851,6 +851,7 @@ static int userfaultfd_release(struct inode *inode, struct file *file)
> /* len == 0 means wake all */
> struct userfaultfd_wake_range range = { .len = 0, };
> unsigned long new_flags;
> + DEFINE_RANGE_LOCK_FULL(mmrange);
>
> WRITE_ONCE(ctx->released, true);
>
> @@ -880,7 +881,7 @@ static int userfaultfd_release(struct inode *inode, struct file *file)
> new_flags, vma->anon_vma,
> vma->vm_file, vma->vm_pgoff,
> vma_policy(vma),
> - NULL_VM_UFFD_CTX);
> + NULL_VM_UFFD_CTX, &mmrange);
> if (prev)
> vma = prev;
> else
> @@ -1276,6 +1277,7 @@ static int userfaultfd_register(struct userfaultfd_ctx *ctx,
> bool found;
> bool basic_ioctls;
> unsigned long start, end, vma_end;
> + DEFINE_RANGE_LOCK_FULL(mmrange);
>
> user_uffdio_register = (struct uffdio_register __user *) arg;
>
> @@ -1413,18 +1415,19 @@ static int userfaultfd_register(struct userfaultfd_ctx *ctx,
> prev = vma_merge(mm, prev, start, vma_end, new_flags,
> vma->anon_vma, vma->vm_file, vma->vm_pgoff,
> vma_policy(vma),
> - ((struct vm_userfaultfd_ctx){ ctx }));
> + ((struct vm_userfaultfd_ctx){ ctx }),
> + &mmrange);
> if (prev) {
> vma = prev;
> goto next;
> }
> if (vma->vm_start < start) {
> - ret = split_vma(mm, vma, start, 1);
> + ret = split_vma(mm, vma, start, 1, &mmrange);
> if (ret)
> break;
> }
> if (vma->vm_end > end) {
> - ret = split_vma(mm, vma, end, 0);
> + ret = split_vma(mm, vma, end, 0, &mmrange);
> if (ret)
> break;
> }
> @@ -1471,6 +1474,7 @@ static int userfaultfd_unregister(struct userfaultfd_ctx *ctx,
> bool found;
> unsigned long start, end, vma_end;
> const void __user *buf = (void __user *)arg;
> + DEFINE_RANGE_LOCK_FULL(mmrange);
>
> ret = -EFAULT;
> if (copy_from_user(&uffdio_unregister, buf, sizeof(uffdio_unregister)))
> @@ -1571,18 +1575,18 @@ static int userfaultfd_unregister(struct userfaultfd_ctx *ctx,
> prev = vma_merge(mm, prev, start, vma_end, new_flags,
> vma->anon_vma, vma->vm_file, vma->vm_pgoff,
> vma_policy(vma),
> - NULL_VM_UFFD_CTX);
> + NULL_VM_UFFD_CTX, &mmrange);
> if (prev) {
> vma = prev;
> goto next;
> }
> if (vma->vm_start < start) {
> - ret = split_vma(mm, vma, start, 1);
> + ret = split_vma(mm, vma, start, 1, &mmrange);
> if (ret)
> break;
> }
> if (vma->vm_end > end) {
> - ret = split_vma(mm, vma, end, 0);
> + ret = split_vma(mm, vma, end, 0, &mmrange);
> if (ret)
> break;
> }
> diff --git a/include/asm-generic/mm_hooks.h b/include/asm-generic/mm_hooks.h
> index 8ac4e68a12f0..2115deceded1 100644
> --- a/include/asm-generic/mm_hooks.h
> +++ b/include/asm-generic/mm_hooks.h
> @@ -19,7 +19,8 @@ static inline void arch_exit_mmap(struct mm_struct *mm)
>
> static inline void arch_unmap(struct mm_struct *mm,
> struct vm_area_struct *vma,
> - unsigned long start, unsigned long end)
> + unsigned long start, unsigned long end,
> + struct range_lock *mmrange)
> {
> }
>
> diff --git a/include/linux/hmm.h b/include/linux/hmm.h
> index 325017ad9311..da004594d831 100644
> --- a/include/linux/hmm.h
> +++ b/include/linux/hmm.h
> @@ -295,7 +295,7 @@ int hmm_vma_get_pfns(struct vm_area_struct *vma,
> struct hmm_range *range,
> unsigned long start,
> unsigned long end,
> - hmm_pfn_t *pfns);
> + hmm_pfn_t *pfns, struct range_lock *mmrange);
> bool hmm_vma_range_done(struct vm_area_struct *vma, struct hmm_range *range);
>
>
> @@ -323,7 +323,7 @@ int hmm_vma_fault(struct vm_area_struct *vma,
> unsigned long end,
> hmm_pfn_t *pfns,
> bool write,
> - bool block);
> + bool block, struct range_lock *mmrange);
> #endif /* IS_ENABLED(CONFIG_HMM_MIRROR) */
>
>
> diff --git a/include/linux/ksm.h b/include/linux/ksm.h
> index 44368b19b27e..19667b75f73c 100644
> --- a/include/linux/ksm.h
> +++ b/include/linux/ksm.h
> @@ -20,7 +20,8 @@ struct mem_cgroup;
>
> #ifdef CONFIG_KSM
> int ksm_madvise(struct vm_area_struct *vma, unsigned long start,
> - unsigned long end, int advice, unsigned long *vm_flags);
> + unsigned long end, int advice, unsigned long *vm_flags,
> + struct range_lock *mmrange);
> int __ksm_enter(struct mm_struct *mm);
> void __ksm_exit(struct mm_struct *mm);
>
> @@ -78,7 +79,8 @@ static inline void ksm_exit(struct mm_struct *mm)
>
> #ifdef CONFIG_MMU
> static inline int ksm_madvise(struct vm_area_struct *vma, unsigned long start,
> - unsigned long end, int advice, unsigned long *vm_flags)
> + unsigned long end, int advice, unsigned long *vm_flags,
> + struct range_lock *mmrange)
> {
> return 0;
> }
> diff --git a/include/linux/migrate.h b/include/linux/migrate.h
> index 0c6fe904bc97..fa08e348a295 100644
> --- a/include/linux/migrate.h
> +++ b/include/linux/migrate.h
> @@ -272,7 +272,7 @@ int migrate_vma(const struct migrate_vma_ops *ops,
> unsigned long end,
> unsigned long *src,
> unsigned long *dst,
> - void *private);
> + void *private, struct range_lock *mmrange);
> #else
> static inline int migrate_vma(const struct migrate_vma_ops *ops,
> struct vm_area_struct *vma,
> @@ -280,7 +280,7 @@ static inline int migrate_vma(const struct migrate_vma_ops *ops,
> unsigned long end,
> unsigned long *src,
> unsigned long *dst,
> - void *private)
> + void *private, struct range_lock *mmrange)
> {
> return -EINVAL;
> }
> diff --git a/include/linux/mm.h b/include/linux/mm.h
> index bcf2509d448d..fc4e7fdc3e76 100644
> --- a/include/linux/mm.h
> +++ b/include/linux/mm.h
> @@ -1295,11 +1295,12 @@ struct mm_walk {
> int (*pud_entry)(pud_t *pud, unsigned long addr,
> unsigned long next, struct mm_walk *walk);
> int (*pmd_entry)(pmd_t *pmd, unsigned long addr,
> - unsigned long next, struct mm_walk *walk);
> + unsigned long next, struct mm_walk *walk,
> + struct range_lock *mmrange);
> int (*pte_entry)(pte_t *pte, unsigned long addr,
> unsigned long next, struct mm_walk *walk);
> int (*pte_hole)(unsigned long addr, unsigned long next,
> - struct mm_walk *walk);
> + struct mm_walk *walk, struct range_lock *mmrange);
> int (*hugetlb_entry)(pte_t *pte, unsigned long hmask,
> unsigned long addr, unsigned long next,
> struct mm_walk *walk);
> @@ -1311,8 +1312,9 @@ struct mm_walk {
> };
>
> int walk_page_range(unsigned long addr, unsigned long end,
> - struct mm_walk *walk);
> -int walk_page_vma(struct vm_area_struct *vma, struct mm_walk *walk);
> + struct mm_walk *walk, struct range_lock *mmrange);
> +int walk_page_vma(struct vm_area_struct *vma, struct mm_walk *walk,
> + struct range_lock *mmrange);
> void free_pgd_range(struct mmu_gather *tlb, unsigned long addr,
> unsigned long end, unsigned long floor, unsigned long ceiling);
> int copy_page_range(struct mm_struct *dst, struct mm_struct *src,
> @@ -1337,17 +1339,18 @@ int invalidate_inode_page(struct page *page);
>
> #ifdef CONFIG_MMU
> extern int handle_mm_fault(struct vm_area_struct *vma, unsigned long address,
> - unsigned int flags);
> + unsigned int flags, struct range_lock *mmrange);
> extern int fixup_user_fault(struct task_struct *tsk, struct mm_struct *mm,
> unsigned long address, unsigned int fault_flags,
> - bool *unlocked);
> + bool *unlocked, struct range_lock *mmrange);
> void unmap_mapping_pages(struct address_space *mapping,
> pgoff_t start, pgoff_t nr, bool even_cows);
> void unmap_mapping_range(struct address_space *mapping,
> loff_t const holebegin, loff_t const holelen, int even_cows);
> #else
> static inline int handle_mm_fault(struct vm_area_struct *vma,
> - unsigned long address, unsigned int flags)
> + unsigned long address, unsigned int flags,
> + struct range_lock *mmrange)
> {
> /* should never happen if there's no MMU */
> BUG();
> @@ -1355,7 +1358,8 @@ static inline int handle_mm_fault(struct vm_area_struct *vma,
> }
> static inline int fixup_user_fault(struct task_struct *tsk,
> struct mm_struct *mm, unsigned long address,
> - unsigned int fault_flags, bool *unlocked)
> + unsigned int fault_flags, bool *unlocked,
> + struct range_lock *mmrange)
> {
> /* should never happen if there's no MMU */
> BUG();
> @@ -1383,24 +1387,28 @@ extern int __access_remote_vm(struct task_struct *tsk, struct mm_struct *mm,
> long get_user_pages_remote(struct task_struct *tsk, struct mm_struct *mm,
> unsigned long start, unsigned long nr_pages,
> unsigned int gup_flags, struct page **pages,
> - struct vm_area_struct **vmas, int *locked);
> + struct vm_area_struct **vmas, int *locked,
> + struct range_lock *mmrange);
> long get_user_pages(unsigned long start, unsigned long nr_pages,
> - unsigned int gup_flags, struct page **pages,
> - struct vm_area_struct **vmas);
> + unsigned int gup_flags, struct page **pages,
> + struct vm_area_struct **vmas, struct range_lock *mmrange);
> long get_user_pages_locked(unsigned long start, unsigned long nr_pages,
> - unsigned int gup_flags, struct page **pages, int *locked);
> + unsigned int gup_flags, struct page **pages,
> + int *locked, struct range_lock *mmrange);
> long get_user_pages_unlocked(unsigned long start, unsigned long nr_pages,
> struct page **pages, unsigned int gup_flags);
> #ifdef CONFIG_FS_DAX
> long get_user_pages_longterm(unsigned long start, unsigned long nr_pages,
> - unsigned int gup_flags, struct page **pages,
> - struct vm_area_struct **vmas);
> + unsigned int gup_flags, struct page **pages,
> + struct vm_area_struct **vmas,
> + struct range_lock *mmrange);
> #else
> static inline long get_user_pages_longterm(unsigned long start,
> unsigned long nr_pages, unsigned int gup_flags,
> - struct page **pages, struct vm_area_struct **vmas)
> + struct page **pages, struct vm_area_struct **vmas,
> + struct range_lock *mmrange)
> {
> - return get_user_pages(start, nr_pages, gup_flags, pages, vmas);
> + return get_user_pages(start, nr_pages, gup_flags, pages, vmas, mmrange);
> }
> #endif /* CONFIG_FS_DAX */
>
> @@ -1505,7 +1513,8 @@ extern unsigned long change_protection(struct vm_area_struct *vma, unsigned long
> int dirty_accountable, int prot_numa);
> extern int mprotect_fixup(struct vm_area_struct *vma,
> struct vm_area_struct **pprev, unsigned long start,
> - unsigned long end, unsigned long newflags);
> + unsigned long end, unsigned long newflags,
> + struct range_lock *mmrange);
>
> /*
> * doesn't attempt to fault and will return short.
> @@ -2149,28 +2158,30 @@ void anon_vma_interval_tree_verify(struct anon_vma_chain *node);
> extern int __vm_enough_memory(struct mm_struct *mm, long pages, int cap_sys_admin);
> extern int __vma_adjust(struct vm_area_struct *vma, unsigned long start,
> unsigned long end, pgoff_t pgoff, struct vm_area_struct *insert,
> - struct vm_area_struct *expand);
> + struct vm_area_struct *expand, struct range_lock *mmrange);
> static inline int vma_adjust(struct vm_area_struct *vma, unsigned long start,
> - unsigned long end, pgoff_t pgoff, struct vm_area_struct *insert)
> + unsigned long end, pgoff_t pgoff, struct vm_area_struct *insert,
> + struct range_lock *mmrange)
> {
> - return __vma_adjust(vma, start, end, pgoff, insert, NULL);
> + return __vma_adjust(vma, start, end, pgoff, insert, NULL, mmrange);
> }
> extern struct vm_area_struct *vma_merge(struct mm_struct *,
> struct vm_area_struct *prev, unsigned long addr, unsigned long end,
> unsigned long vm_flags, struct anon_vma *, struct file *, pgoff_t,
> - struct mempolicy *, struct vm_userfaultfd_ctx);
> + struct mempolicy *, struct vm_userfaultfd_ctx,
> + struct range_lock *mmrange);
> extern struct anon_vma *find_mergeable_anon_vma(struct vm_area_struct *);
> extern int __split_vma(struct mm_struct *, struct vm_area_struct *,
> - unsigned long addr, int new_below);
> + unsigned long addr, int new_below, struct range_lock *mmrange);
> extern int split_vma(struct mm_struct *, struct vm_area_struct *,
> - unsigned long addr, int new_below);
> + unsigned long addr, int new_below, struct range_lock *mmrange);
> extern int insert_vm_struct(struct mm_struct *, struct vm_area_struct *);
> extern void __vma_link_rb(struct mm_struct *, struct vm_area_struct *,
> struct rb_node **, struct rb_node *);
> extern void unlink_file_vma(struct vm_area_struct *);
> extern struct vm_area_struct *copy_vma(struct vm_area_struct **,
> unsigned long addr, unsigned long len, pgoff_t pgoff,
> - bool *need_rmap_locks);
> + bool *need_rmap_locks, struct range_lock *mmrange);
> extern void exit_mmap(struct mm_struct *);
>
> static inline int check_data_rlimit(unsigned long rlim,
> @@ -2212,21 +2223,22 @@ extern unsigned long get_unmapped_area(struct file *, unsigned long, unsigned lo
>
> extern unsigned long mmap_region(struct file *file, unsigned long addr,
> unsigned long len, vm_flags_t vm_flags, unsigned long pgoff,
> - struct list_head *uf);
> + struct list_head *uf, struct range_lock *mmrange);
> extern unsigned long do_mmap(struct file *file, unsigned long addr,
> unsigned long len, unsigned long prot, unsigned long flags,
> vm_flags_t vm_flags, unsigned long pgoff, unsigned long *populate,
> - struct list_head *uf);
> + struct list_head *uf, struct range_lock *mmrange);
> extern int do_munmap(struct mm_struct *, unsigned long, size_t,
> - struct list_head *uf);
> + struct list_head *uf, struct range_lock *mmrange);
>
> static inline unsigned long
> do_mmap_pgoff(struct file *file, unsigned long addr,
> unsigned long len, unsigned long prot, unsigned long flags,
> unsigned long pgoff, unsigned long *populate,
> - struct list_head *uf)
> + struct list_head *uf, struct range_lock *mmrange)
> {
> - return do_mmap(file, addr, len, prot, flags, 0, pgoff, populate, uf);
> + return do_mmap(file, addr, len, prot, flags, 0, pgoff, populate,
> + uf, mmrange);
> }
>
> #ifdef CONFIG_MMU
> @@ -2405,7 +2417,8 @@ unsigned long change_prot_numa(struct vm_area_struct *vma,
> unsigned long start, unsigned long end);
> #endif
>
> -struct vm_area_struct *find_extend_vma(struct mm_struct *, unsigned long addr);
> +struct vm_area_struct *find_extend_vma(struct mm_struct *, unsigned long addr,
> + struct range_lock *);
> int remap_pfn_range(struct vm_area_struct *, unsigned long addr,
> unsigned long pfn, unsigned long size, pgprot_t);
> int vm_insert_page(struct vm_area_struct *, unsigned long addr, struct page *);
> diff --git a/include/linux/uprobes.h b/include/linux/uprobes.h
> index 0a294e950df8..79eb735e7c95 100644
> --- a/include/linux/uprobes.h
> +++ b/include/linux/uprobes.h
> @@ -34,6 +34,7 @@ struct mm_struct;
> struct inode;
> struct notifier_block;
> struct page;
> +struct range_lock;
>
> #define UPROBE_HANDLER_REMOVE 1
> #define UPROBE_HANDLER_MASK 1
> @@ -115,17 +116,20 @@ struct uprobes_state {
> struct xol_area *xol_area;
> };
>
> -extern int set_swbp(struct arch_uprobe *aup, struct mm_struct *mm, unsigned long vaddr);
> -extern int set_orig_insn(struct arch_uprobe *aup, struct mm_struct *mm, unsigned long vaddr);
> +extern int set_swbp(struct arch_uprobe *aup, struct mm_struct *mm,
> + unsigned long vaddr, struct range_lock *mmrange);
> +extern int set_orig_insn(struct arch_uprobe *aup, struct mm_struct *mm,
> + unsigned long vaddr, struct range_lock *mmrange);
> extern bool is_swbp_insn(uprobe_opcode_t *insn);
> extern bool is_trap_insn(uprobe_opcode_t *insn);
> extern unsigned long uprobe_get_swbp_addr(struct pt_regs *regs);
> extern unsigned long uprobe_get_trap_addr(struct pt_regs *regs);
> -extern int uprobe_write_opcode(struct mm_struct *mm, unsigned long vaddr, uprobe_opcode_t);
> +extern int uprobe_write_opcode(struct mm_struct *mm, unsigned long vaddr,
> + uprobe_opcode_t, struct range_lock *mmrange);
> extern int uprobe_register(struct inode *inode, loff_t offset, struct uprobe_consumer *uc);
> extern int uprobe_apply(struct inode *inode, loff_t offset, struct uprobe_consumer *uc, bool);
> extern void uprobe_unregister(struct inode *inode, loff_t offset, struct uprobe_consumer *uc);
> -extern int uprobe_mmap(struct vm_area_struct *vma);
> +extern int uprobe_mmap(struct vm_area_struct *vma, struct range_lock *mmrange);;
> extern void uprobe_munmap(struct vm_area_struct *vma, unsigned long start, unsigned long end);
> extern void uprobe_start_dup_mmap(void);
> extern void uprobe_end_dup_mmap(void);
> @@ -169,7 +173,8 @@ static inline void
> uprobe_unregister(struct inode *inode, loff_t offset, struct uprobe_consumer *uc)
> {
> }
> -static inline int uprobe_mmap(struct vm_area_struct *vma)
> +static inline int uprobe_mmap(struct vm_area_struct *vma,
> + struct range_lock *mmrange)
> {
> return 0;
> }
> diff --git a/ipc/shm.c b/ipc/shm.c
> index 4643865e9171..6c29c791c7f2 100644
> --- a/ipc/shm.c
> +++ b/ipc/shm.c
> @@ -1293,6 +1293,7 @@ long do_shmat(int shmid, char __user *shmaddr, int shmflg,
> struct path path;
> fmode_t f_mode;
> unsigned long populate = 0;
> + DEFINE_RANGE_LOCK_FULL(mmrange);
>
> err = -EINVAL;
> if (shmid < 0)
> @@ -1411,7 +1412,8 @@ long do_shmat(int shmid, char __user *shmaddr, int shmflg,
> goto invalid;
> }
>
> - addr = do_mmap_pgoff(file, addr, size, prot, flags, 0, &populate, NULL);
> + addr = do_mmap_pgoff(file, addr, size, prot, flags, 0, &populate, NULL,
> + &mmrange);
> *raddr = addr;
> err = 0;
> if (IS_ERR_VALUE(addr))
> @@ -1487,6 +1489,7 @@ SYSCALL_DEFINE1(shmdt, char __user *, shmaddr)
> struct file *file;
> struct vm_area_struct *next;
> #endif
> + DEFINE_RANGE_LOCK_FULL(mmrange);
>
> if (addr & ~PAGE_MASK)
> return retval;
> @@ -1537,7 +1540,8 @@ SYSCALL_DEFINE1(shmdt, char __user *, shmaddr)
> */
> file = vma->vm_file;
> size = i_size_read(file_inode(vma->vm_file));
> - do_munmap(mm, vma->vm_start, vma->vm_end - vma->vm_start, NULL);
> + do_munmap(mm, vma->vm_start, vma->vm_end - vma->vm_start,
> + NULL, &mmrange);
> /*
> * We discovered the size of the shm segment, so
> * break out of here and fall through to the next
> @@ -1564,7 +1568,8 @@ SYSCALL_DEFINE1(shmdt, char __user *, shmaddr)
> if ((vma->vm_ops == &shm_vm_ops) &&
> ((vma->vm_start - addr)/PAGE_SIZE == vma->vm_pgoff) &&
> (vma->vm_file == file))
> - do_munmap(mm, vma->vm_start, vma->vm_end - vma->vm_start, NULL);
> + do_munmap(mm, vma->vm_start, vma->vm_end - vma->vm_start,
> + NULL, &mmrange);
> vma = next;
> }
>
> @@ -1573,7 +1578,8 @@ SYSCALL_DEFINE1(shmdt, char __user *, shmaddr)
> * given
> */
> if (vma && vma->vm_start == addr && vma->vm_ops == &shm_vm_ops) {
> - do_munmap(mm, vma->vm_start, vma->vm_end - vma->vm_start, NULL);
> + do_munmap(mm, vma->vm_start, vma->vm_end - vma->vm_start,
> + NULL, &mmrange);
> retval = 0;
> }
>
> diff --git a/kernel/events/uprobes.c b/kernel/events/uprobes.c
> index ce6848e46e94..60e12b39182c 100644
> --- a/kernel/events/uprobes.c
> +++ b/kernel/events/uprobes.c
> @@ -300,7 +300,7 @@ static int verify_opcode(struct page *page, unsigned long vaddr, uprobe_opcode_t
> * Return 0 (success) or a negative errno.
> */
> int uprobe_write_opcode(struct mm_struct *mm, unsigned long vaddr,
> - uprobe_opcode_t opcode)
> + uprobe_opcode_t opcode, struct range_lock *mmrange)
> {
> struct page *old_page, *new_page;
> struct vm_area_struct *vma;
> @@ -309,7 +309,8 @@ int uprobe_write_opcode(struct mm_struct *mm, unsigned long vaddr,
> retry:
> /* Read the page with vaddr into memory */
> ret = get_user_pages_remote(NULL, mm, vaddr, 1,
> - FOLL_FORCE | FOLL_SPLIT, &old_page, &vma, NULL);
> + FOLL_FORCE | FOLL_SPLIT, &old_page, &vma, NULL,
> + mmrange);
There is no need to pass down the range here as get_user_pages_remote() is
told to not unlock the mmap_sem.
There are other places where passing range parameter down is not necessary
and is making this series bigger than needed, adding extra parameter to a
lot of functions which doesn't need it.
Laurent.
> if (ret <= 0)
> return ret;
>
> @@ -349,9 +350,10 @@ int uprobe_write_opcode(struct mm_struct *mm, unsigned long vaddr,
> * For mm @mm, store the breakpoint instruction at @vaddr.
> * Return 0 (success) or a negative errno.
> */
> -int __weak set_swbp(struct arch_uprobe *auprobe, struct mm_struct *mm, unsigned long vaddr)
> +int __weak set_swbp(struct arch_uprobe *auprobe, struct mm_struct *mm,
> + unsigned long vaddr, struct range_lock *mmrange)
> {
> - return uprobe_write_opcode(mm, vaddr, UPROBE_SWBP_INSN);
> + return uprobe_write_opcode(mm, vaddr, UPROBE_SWBP_INSN, mmrange);
> }
>
> /**
> @@ -364,9 +366,12 @@ int __weak set_swbp(struct arch_uprobe *auprobe, struct mm_struct *mm, unsigned
> * Return 0 (success) or a negative errno.
> */
> int __weak
> -set_orig_insn(struct arch_uprobe *auprobe, struct mm_struct *mm, unsigned long vaddr)
> +set_orig_insn(struct arch_uprobe *auprobe, struct mm_struct *mm,
> + unsigned long vaddr, struct range_lock *mmrange)
> {
> - return uprobe_write_opcode(mm, vaddr, *(uprobe_opcode_t *)&auprobe->insn);
> + return uprobe_write_opcode(mm, vaddr,
> + *(uprobe_opcode_t *)&auprobe->insn,
> + mmrange);
> }
>
> static struct uprobe *get_uprobe(struct uprobe *uprobe)
> @@ -650,7 +655,8 @@ static bool filter_chain(struct uprobe *uprobe,
>
> static int
> install_breakpoint(struct uprobe *uprobe, struct mm_struct *mm,
> - struct vm_area_struct *vma, unsigned long vaddr)
> + struct vm_area_struct *vma, unsigned long vaddr,
> + struct range_lock *mmrange)
> {
> bool first_uprobe;
> int ret;
> @@ -667,7 +673,7 @@ install_breakpoint(struct uprobe *uprobe, struct mm_struct *mm,
> if (first_uprobe)
> set_bit(MMF_HAS_UPROBES, &mm->flags);
>
> - ret = set_swbp(&uprobe->arch, mm, vaddr);
> + ret = set_swbp(&uprobe->arch, mm, vaddr, mmrange);
> if (!ret)
> clear_bit(MMF_RECALC_UPROBES, &mm->flags);
> else if (first_uprobe)
> @@ -677,10 +683,11 @@ install_breakpoint(struct uprobe *uprobe, struct mm_struct *mm,
> }
>
> static int
> -remove_breakpoint(struct uprobe *uprobe, struct mm_struct *mm, unsigned long vaddr)
> +remove_breakpoint(struct uprobe *uprobe, struct mm_struct *mm,
> + unsigned long vaddr, struct range_lock *mmrange)
> {
> set_bit(MMF_RECALC_UPROBES, &mm->flags);
> - return set_orig_insn(&uprobe->arch, mm, vaddr);
> + return set_orig_insn(&uprobe->arch, mm, vaddr, mmrange);
> }
>
> static inline bool uprobe_is_active(struct uprobe *uprobe)
> @@ -794,6 +801,7 @@ register_for_each_vma(struct uprobe *uprobe, struct uprobe_consumer *new)
> bool is_register = !!new;
> struct map_info *info;
> int err = 0;
> + DEFINE_RANGE_LOCK_FULL(mmrange);
>
> percpu_down_write(&dup_mmap_sem);
> info = build_map_info(uprobe->inode->i_mapping,
> @@ -824,11 +832,13 @@ register_for_each_vma(struct uprobe *uprobe, struct uprobe_consumer *new)
> /* consult only the "caller", new consumer. */
> if (consumer_filter(new,
> UPROBE_FILTER_REGISTER, mm))
> - err = install_breakpoint(uprobe, mm, vma, info->vaddr);
> + err = install_breakpoint(uprobe, mm, vma,
> + info->vaddr, &mmrange);
> } else if (test_bit(MMF_HAS_UPROBES, &mm->flags)) {
> if (!filter_chain(uprobe,
> UPROBE_FILTER_UNREGISTER, mm))
> - err |= remove_breakpoint(uprobe, mm, info->vaddr);
> + err |= remove_breakpoint(uprobe, mm,
> + info->vaddr, &mmrange);
> }
>
> unlock:
> @@ -972,6 +982,7 @@ static int unapply_uprobe(struct uprobe *uprobe, struct mm_struct *mm)
> {
> struct vm_area_struct *vma;
> int err = 0;
> + DEFINE_RANGE_LOCK_FULL(mmrange);
>
> down_read(&mm->mmap_sem);
> for (vma = mm->mmap; vma; vma = vma->vm_next) {
> @@ -988,7 +999,7 @@ static int unapply_uprobe(struct uprobe *uprobe, struct mm_struct *mm)
> continue;
>
> vaddr = offset_to_vaddr(vma, uprobe->offset);
> - err |= remove_breakpoint(uprobe, mm, vaddr);
> + err |= remove_breakpoint(uprobe, mm, vaddr, &mmrange);
> }
> up_read(&mm->mmap_sem);
>
> @@ -1063,7 +1074,7 @@ static void build_probe_list(struct inode *inode,
> * Currently we ignore all errors and always return 0, the callers
> * can't handle the failure anyway.
> */
> -int uprobe_mmap(struct vm_area_struct *vma)
> +int uprobe_mmap(struct vm_area_struct *vma, struct range_lock *mmrange)
> {
> struct list_head tmp_list;
> struct uprobe *uprobe, *u;
> @@ -1087,7 +1098,7 @@ int uprobe_mmap(struct vm_area_struct *vma)
> if (!fatal_signal_pending(current) &&
> filter_chain(uprobe, UPROBE_FILTER_MMAP, vma->vm_mm)) {
> unsigned long vaddr = offset_to_vaddr(vma, uprobe->offset);
> - install_breakpoint(uprobe, vma->vm_mm, vma, vaddr);
> + install_breakpoint(uprobe, vma->vm_mm, vma, vaddr, mmrange);
> }
> put_uprobe(uprobe);
> }
> @@ -1698,7 +1709,8 @@ static void mmf_recalc_uprobes(struct mm_struct *mm)
> clear_bit(MMF_HAS_UPROBES, &mm->flags);
> }
>
> -static int is_trap_at_addr(struct mm_struct *mm, unsigned long vaddr)
> +static int is_trap_at_addr(struct mm_struct *mm, unsigned long vaddr,
> + struct range_lock *mmrange)
> {
> struct page *page;
> uprobe_opcode_t opcode;
> @@ -1718,7 +1730,7 @@ static int is_trap_at_addr(struct mm_struct *mm, unsigned long vaddr)
> * essentially a kernel access to the memory.
> */
> result = get_user_pages_remote(NULL, mm, vaddr, 1, FOLL_FORCE, &page,
> - NULL, NULL);
> + NULL, NULL, mmrange);
> if (result < 0)
> return result;
>
> @@ -1734,6 +1746,7 @@ static struct uprobe *find_active_uprobe(unsigned long bp_vaddr, int *is_swbp)
> struct mm_struct *mm = current->mm;
> struct uprobe *uprobe = NULL;
> struct vm_area_struct *vma;
> + DEFINE_RANGE_LOCK_FULL(mmrange);
>
> down_read(&mm->mmap_sem);
> vma = find_vma(mm, bp_vaddr);
> @@ -1746,7 +1759,7 @@ static struct uprobe *find_active_uprobe(unsigned long bp_vaddr, int *is_swbp)
> }
>
> if (!uprobe)
> - *is_swbp = is_trap_at_addr(mm, bp_vaddr);
> + *is_swbp = is_trap_at_addr(mm, bp_vaddr, &mmrange);
> } else {
> *is_swbp = -EFAULT;
> }
> diff --git a/kernel/futex.c b/kernel/futex.c
> index 1f450e092c74..09a0d86f80a0 100644
> --- a/kernel/futex.c
> +++ b/kernel/futex.c
> @@ -725,10 +725,11 @@ static int fault_in_user_writeable(u32 __user *uaddr)
> {
> struct mm_struct *mm = current->mm;
> int ret;
> + DEFINE_RANGE_LOCK_FULL(mmrange);
>
> down_read(&mm->mmap_sem);
> ret = fixup_user_fault(current, mm, (unsigned long)uaddr,
> - FAULT_FLAG_WRITE, NULL);
> + FAULT_FLAG_WRITE, NULL, &mmrange);
> up_read(&mm->mmap_sem);
>
> return ret < 0 ? ret : 0;
> diff --git a/mm/frame_vector.c b/mm/frame_vector.c
> index c64dca6e27c2..d3dccd80c6ee 100644
> --- a/mm/frame_vector.c
> +++ b/mm/frame_vector.c
> @@ -39,6 +39,7 @@ int get_vaddr_frames(unsigned long start, unsigned int nr_frames,
> int ret = 0;
> int err;
> int locked;
> + DEFINE_RANGE_LOCK_FULL(mmrange);
>
> if (nr_frames == 0)
> return 0;
> @@ -71,7 +72,8 @@ int get_vaddr_frames(unsigned long start, unsigned int nr_frames,
> vec->got_ref = true;
> vec->is_pfns = false;
> ret = get_user_pages_locked(start, nr_frames,
> - gup_flags, (struct page **)(vec->ptrs), &locked);
> + gup_flags, (struct page **)(vec->ptrs), &locked,
> + &mmrange);
> goto out;
> }
>
> diff --git a/mm/gup.c b/mm/gup.c
> index 1b46e6e74881..01983a7b3750 100644
> --- a/mm/gup.c
> +++ b/mm/gup.c
> @@ -478,7 +478,8 @@ static int get_gate_page(struct mm_struct *mm, unsigned long address,
> * If it is, *@...blocking will be set to 0 and -EBUSY returned.
> */
> static int faultin_page(struct task_struct *tsk, struct vm_area_struct *vma,
> - unsigned long address, unsigned int *flags, int *nonblocking)
> + unsigned long address, unsigned int *flags, int *nonblocking,
> + struct range_lock *mmrange)
> {
> unsigned int fault_flags = 0;
> int ret;
> @@ -499,7 +500,7 @@ static int faultin_page(struct task_struct *tsk, struct vm_area_struct *vma,
> fault_flags |= FAULT_FLAG_TRIED;
> }
>
> - ret = handle_mm_fault(vma, address, fault_flags);
> + ret = handle_mm_fault(vma, address, fault_flags, mmrange);
> if (ret & VM_FAULT_ERROR) {
> int err = vm_fault_to_errno(ret, *flags);
>
> @@ -592,6 +593,7 @@ static int check_vma_flags(struct vm_area_struct *vma, unsigned long gup_flags)
> * @vmas: array of pointers to vmas corresponding to each page.
> * Or NULL if the caller does not require them.
> * @nonblocking: whether waiting for disk IO or mmap_sem contention
> + * @mmrange: mm address space range locking
> *
> * Returns number of pages pinned. This may be fewer than the number
> * requested. If nr_pages is 0 or negative, returns 0. If no pages
> @@ -638,7 +640,8 @@ static int check_vma_flags(struct vm_area_struct *vma, unsigned long gup_flags)
> static long __get_user_pages(struct task_struct *tsk, struct mm_struct *mm,
> unsigned long start, unsigned long nr_pages,
> unsigned int gup_flags, struct page **pages,
> - struct vm_area_struct **vmas, int *nonblocking)
> + struct vm_area_struct **vmas, int *nonblocking,
> + struct range_lock *mmrange)
> {
> long i = 0;
> unsigned int page_mask;
> @@ -664,7 +667,7 @@ static long __get_user_pages(struct task_struct *tsk, struct mm_struct *mm,
>
> /* first iteration or cross vma bound */
> if (!vma || start >= vma->vm_end) {
> - vma = find_extend_vma(mm, start);
> + vma = find_extend_vma(mm, start, mmrange);
> if (!vma && in_gate_area(mm, start)) {
> int ret;
> ret = get_gate_page(mm, start & PAGE_MASK,
> @@ -697,7 +700,7 @@ static long __get_user_pages(struct task_struct *tsk, struct mm_struct *mm,
> if (!page) {
> int ret;
> ret = faultin_page(tsk, vma, start, &foll_flags,
> - nonblocking);
> + nonblocking, mmrange);
> switch (ret) {
> case 0:
> goto retry;
> @@ -796,7 +799,7 @@ static bool vma_permits_fault(struct vm_area_struct *vma,
> */
> int fixup_user_fault(struct task_struct *tsk, struct mm_struct *mm,
> unsigned long address, unsigned int fault_flags,
> - bool *unlocked)
> + bool *unlocked, struct range_lock *mmrange)
> {
> struct vm_area_struct *vma;
> int ret, major = 0;
> @@ -805,14 +808,14 @@ int fixup_user_fault(struct task_struct *tsk, struct mm_struct *mm,
> fault_flags |= FAULT_FLAG_ALLOW_RETRY;
>
> retry:
> - vma = find_extend_vma(mm, address);
> + vma = find_extend_vma(mm, address, mmrange);
> if (!vma || address < vma->vm_start)
> return -EFAULT;
>
> if (!vma_permits_fault(vma, fault_flags))
> return -EFAULT;
>
> - ret = handle_mm_fault(vma, address, fault_flags);
> + ret = handle_mm_fault(vma, address, fault_flags, mmrange);
> major |= ret & VM_FAULT_MAJOR;
> if (ret & VM_FAULT_ERROR) {
> int err = vm_fault_to_errno(ret, 0);
> @@ -849,7 +852,8 @@ static __always_inline long __get_user_pages_locked(struct task_struct *tsk,
> struct page **pages,
> struct vm_area_struct **vmas,
> int *locked,
> - unsigned int flags)
> + unsigned int flags,
> + struct range_lock *mmrange)
> {
> long ret, pages_done;
> bool lock_dropped;
> @@ -868,7 +872,7 @@ static __always_inline long __get_user_pages_locked(struct task_struct *tsk,
> lock_dropped = false;
> for (;;) {
> ret = __get_user_pages(tsk, mm, start, nr_pages, flags, pages,
> - vmas, locked);
> + vmas, locked, mmrange);
> if (!locked)
> /* VM_FAULT_RETRY couldn't trigger, bypass */
> return ret;
> @@ -908,7 +912,7 @@ static __always_inline long __get_user_pages_locked(struct task_struct *tsk,
> lock_dropped = true;
> down_read(&mm->mmap_sem);
> ret = __get_user_pages(tsk, mm, start, 1, flags | FOLL_TRIED,
> - pages, NULL, NULL);
> + pages, NULL, NULL, mmrange);
> if (ret != 1) {
> BUG_ON(ret > 1);
> if (!pages_done)
> @@ -956,11 +960,11 @@ static __always_inline long __get_user_pages_locked(struct task_struct *tsk,
> */
> long get_user_pages_locked(unsigned long start, unsigned long nr_pages,
> unsigned int gup_flags, struct page **pages,
> - int *locked)
> + int *locked, struct range_lock *mmrange)
> {
> return __get_user_pages_locked(current, current->mm, start, nr_pages,
> pages, NULL, locked,
> - gup_flags | FOLL_TOUCH);
> + gup_flags | FOLL_TOUCH, mmrange);
> }
> EXPORT_SYMBOL(get_user_pages_locked);
>
> @@ -985,10 +989,11 @@ long get_user_pages_unlocked(unsigned long start, unsigned long nr_pages,
> struct mm_struct *mm = current->mm;
> int locked = 1;
> long ret;
> + DEFINE_RANGE_LOCK_FULL(mmrange);
>
> down_read(&mm->mmap_sem);
> ret = __get_user_pages_locked(current, mm, start, nr_pages, pages, NULL,
> - &locked, gup_flags | FOLL_TOUCH);
> + &locked, gup_flags | FOLL_TOUCH, &mmrange);
> if (locked)
> up_read(&mm->mmap_sem);
> return ret;
> @@ -1054,11 +1059,13 @@ EXPORT_SYMBOL(get_user_pages_unlocked);
> long get_user_pages_remote(struct task_struct *tsk, struct mm_struct *mm,
> unsigned long start, unsigned long nr_pages,
> unsigned int gup_flags, struct page **pages,
> - struct vm_area_struct **vmas, int *locked)
> + struct vm_area_struct **vmas, int *locked,
> + struct range_lock *mmrange)
> {
> return __get_user_pages_locked(tsk, mm, start, nr_pages, pages, vmas,
> locked,
> - gup_flags | FOLL_TOUCH | FOLL_REMOTE);
> + gup_flags | FOLL_TOUCH | FOLL_REMOTE,
> + mmrange);
> }
> EXPORT_SYMBOL(get_user_pages_remote);
>
> @@ -1071,11 +1078,11 @@ EXPORT_SYMBOL(get_user_pages_remote);
> */
> long get_user_pages(unsigned long start, unsigned long nr_pages,
> unsigned int gup_flags, struct page **pages,
> - struct vm_area_struct **vmas)
> + struct vm_area_struct **vmas, struct range_lock *mmrange)
> {
> return __get_user_pages_locked(current, current->mm, start, nr_pages,
> pages, vmas, NULL,
> - gup_flags | FOLL_TOUCH);
> + gup_flags | FOLL_TOUCH, mmrange);
> }
> EXPORT_SYMBOL(get_user_pages);
>
> @@ -1094,7 +1101,8 @@ EXPORT_SYMBOL(get_user_pages);
> */
> long get_user_pages_longterm(unsigned long start, unsigned long nr_pages,
> unsigned int gup_flags, struct page **pages,
> - struct vm_area_struct **vmas_arg)
> + struct vm_area_struct **vmas_arg,
> + struct range_lock *mmrange)
> {
> struct vm_area_struct **vmas = vmas_arg;
> struct vm_area_struct *vma_prev = NULL;
> @@ -1110,7 +1118,7 @@ long get_user_pages_longterm(unsigned long start, unsigned long nr_pages,
> return -ENOMEM;
> }
>
> - rc = get_user_pages(start, nr_pages, gup_flags, pages, vmas);
> + rc = get_user_pages(start, nr_pages, gup_flags, pages, vmas, mmrange);
>
> for (i = 0; i < rc; i++) {
> struct vm_area_struct *vma = vmas[i];
> @@ -1149,6 +1157,7 @@ EXPORT_SYMBOL(get_user_pages_longterm);
> * @start: start address
> * @end: end address
> * @nonblocking:
> + * @mmrange: mm address space range locking
> *
> * This takes care of mlocking the pages too if VM_LOCKED is set.
> *
> @@ -1163,7 +1172,8 @@ EXPORT_SYMBOL(get_user_pages_longterm);
> * released. If it's released, *@...blocking will be set to 0.
> */
> long populate_vma_page_range(struct vm_area_struct *vma,
> - unsigned long start, unsigned long end, int *nonblocking)
> + unsigned long start, unsigned long end, int *nonblocking,
> + struct range_lock *mmrange)
> {
> struct mm_struct *mm = vma->vm_mm;
> unsigned long nr_pages = (end - start) / PAGE_SIZE;
> @@ -1198,7 +1208,7 @@ long populate_vma_page_range(struct vm_area_struct *vma,
> * not result in a stack expansion that recurses back here.
> */
> return __get_user_pages(current, mm, start, nr_pages, gup_flags,
> - NULL, NULL, nonblocking);
> + NULL, NULL, nonblocking, mmrange);
> }
>
> /*
> @@ -1215,6 +1225,7 @@ int __mm_populate(unsigned long start, unsigned long len, int ignore_errors)
> struct vm_area_struct *vma = NULL;
> int locked = 0;
> long ret = 0;
> + DEFINE_RANGE_LOCK_FULL(mmrange);
>
> VM_BUG_ON(start & ~PAGE_MASK);
> VM_BUG_ON(len != PAGE_ALIGN(len));
> @@ -1247,7 +1258,7 @@ int __mm_populate(unsigned long start, unsigned long len, int ignore_errors)
> * double checks the vma flags, so that it won't mlock pages
> * if the vma was already munlocked.
> */
> - ret = populate_vma_page_range(vma, nstart, nend, &locked);
> + ret = populate_vma_page_range(vma, nstart, nend, &locked, &mmrange);
> if (ret < 0) {
> if (ignore_errors) {
> ret = 0;
> @@ -1282,10 +1293,11 @@ struct page *get_dump_page(unsigned long addr)
> {
> struct vm_area_struct *vma;
> struct page *page;
> + DEFINE_RANGE_LOCK_FULL(mmrange);
>
> if (__get_user_pages(current, current->mm, addr, 1,
> FOLL_FORCE | FOLL_DUMP | FOLL_GET, &page, &vma,
> - NULL) < 1)
> + NULL, &mmrange) < 1)
> return NULL;
> flush_cache_page(vma, addr, page_to_pfn(page));
> return page;
> diff --git a/mm/hmm.c b/mm/hmm.c
> index 320545b98ff5..b14e6869689e 100644
> --- a/mm/hmm.c
> +++ b/mm/hmm.c
> @@ -245,7 +245,8 @@ struct hmm_vma_walk {
>
> static int hmm_vma_do_fault(struct mm_walk *walk,
> unsigned long addr,
> - hmm_pfn_t *pfn)
> + hmm_pfn_t *pfn,
> + struct range_lock *mmrange)
> {
> unsigned int flags = FAULT_FLAG_ALLOW_RETRY | FAULT_FLAG_REMOTE;
> struct hmm_vma_walk *hmm_vma_walk = walk->private;
> @@ -254,7 +255,7 @@ static int hmm_vma_do_fault(struct mm_walk *walk,
>
> flags |= hmm_vma_walk->block ? 0 : FAULT_FLAG_ALLOW_RETRY;
> flags |= hmm_vma_walk->write ? FAULT_FLAG_WRITE : 0;
> - r = handle_mm_fault(vma, addr, flags);
> + r = handle_mm_fault(vma, addr, flags, mmrange);
> if (r & VM_FAULT_RETRY)
> return -EBUSY;
> if (r & VM_FAULT_ERROR) {
> @@ -298,7 +299,9 @@ static void hmm_pfns_clear(hmm_pfn_t *pfns,
>
> static int hmm_vma_walk_hole(unsigned long addr,
> unsigned long end,
> - struct mm_walk *walk)
> + struct mm_walk *walk,
> + struct range_lock *mmrange)
> +
> {
> struct hmm_vma_walk *hmm_vma_walk = walk->private;
> struct hmm_range *range = hmm_vma_walk->range;
> @@ -312,7 +315,7 @@ static int hmm_vma_walk_hole(unsigned long addr,
> if (hmm_vma_walk->fault) {
> int ret;
>
> - ret = hmm_vma_do_fault(walk, addr, &pfns[i]);
> + ret = hmm_vma_do_fault(walk, addr, &pfns[i], mmrange);
> if (ret != -EAGAIN)
> return ret;
> }
> @@ -323,7 +326,8 @@ static int hmm_vma_walk_hole(unsigned long addr,
>
> static int hmm_vma_walk_clear(unsigned long addr,
> unsigned long end,
> - struct mm_walk *walk)
> + struct mm_walk *walk,
> + struct range_lock *mmrange)
> {
> struct hmm_vma_walk *hmm_vma_walk = walk->private;
> struct hmm_range *range = hmm_vma_walk->range;
> @@ -337,7 +341,7 @@ static int hmm_vma_walk_clear(unsigned long addr,
> if (hmm_vma_walk->fault) {
> int ret;
>
> - ret = hmm_vma_do_fault(walk, addr, &pfns[i]);
> + ret = hmm_vma_do_fault(walk, addr, &pfns[i], mmrange);
> if (ret != -EAGAIN)
> return ret;
> }
> @@ -349,7 +353,8 @@ static int hmm_vma_walk_clear(unsigned long addr,
> static int hmm_vma_walk_pmd(pmd_t *pmdp,
> unsigned long start,
> unsigned long end,
> - struct mm_walk *walk)
> + struct mm_walk *walk,
> + struct range_lock *mmrange)
> {
> struct hmm_vma_walk *hmm_vma_walk = walk->private;
> struct hmm_range *range = hmm_vma_walk->range;
> @@ -366,7 +371,7 @@ static int hmm_vma_walk_pmd(pmd_t *pmdp,
>
> again:
> if (pmd_none(*pmdp))
> - return hmm_vma_walk_hole(start, end, walk);
> + return hmm_vma_walk_hole(start, end, walk, mmrange);
>
> if (pmd_huge(*pmdp) && vma->vm_flags & VM_HUGETLB)
> return hmm_pfns_bad(start, end, walk);
> @@ -389,10 +394,10 @@ static int hmm_vma_walk_pmd(pmd_t *pmdp,
> if (!pmd_devmap(pmd) && !pmd_trans_huge(pmd))
> goto again;
> if (pmd_protnone(pmd))
> - return hmm_vma_walk_clear(start, end, walk);
> + return hmm_vma_walk_clear(start, end, walk, mmrange);
>
> if (write_fault && !pmd_write(pmd))
> - return hmm_vma_walk_clear(start, end, walk);
> + return hmm_vma_walk_clear(start, end, walk, mmrange);
>
> pfn = pmd_pfn(pmd) + pte_index(addr);
> flag |= pmd_write(pmd) ? HMM_PFN_WRITE : 0;
> @@ -464,7 +469,7 @@ static int hmm_vma_walk_pmd(pmd_t *pmdp,
> fault:
> pte_unmap(ptep);
> /* Fault all pages in range */
> - return hmm_vma_walk_clear(start, end, walk);
> + return hmm_vma_walk_clear(start, end, walk, mmrange);
> }
> pte_unmap(ptep - 1);
>
> @@ -495,7 +500,8 @@ int hmm_vma_get_pfns(struct vm_area_struct *vma,
> struct hmm_range *range,
> unsigned long start,
> unsigned long end,
> - hmm_pfn_t *pfns)
> + hmm_pfn_t *pfns,
> + struct range_lock *mmrange)
> {
> struct hmm_vma_walk hmm_vma_walk;
> struct mm_walk mm_walk;
> @@ -541,7 +547,7 @@ int hmm_vma_get_pfns(struct vm_area_struct *vma,
> mm_walk.pmd_entry = hmm_vma_walk_pmd;
> mm_walk.pte_hole = hmm_vma_walk_hole;
>
> - walk_page_range(start, end, &mm_walk);
> + walk_page_range(start, end, &mm_walk, mmrange);
> return 0;
> }
> EXPORT_SYMBOL(hmm_vma_get_pfns);
> @@ -664,7 +670,8 @@ int hmm_vma_fault(struct vm_area_struct *vma,
> unsigned long end,
> hmm_pfn_t *pfns,
> bool write,
> - bool block)
> + bool block,
> + struct range_lock *mmrange)
> {
> struct hmm_vma_walk hmm_vma_walk;
> struct mm_walk mm_walk;
> @@ -717,7 +724,7 @@ int hmm_vma_fault(struct vm_area_struct *vma,
> mm_walk.pte_hole = hmm_vma_walk_hole;
>
> do {
> - ret = walk_page_range(start, end, &mm_walk);
> + ret = walk_page_range(start, end, &mm_walk, mmrange);
> start = hmm_vma_walk.last;
> } while (ret == -EAGAIN);
>
> diff --git a/mm/internal.h b/mm/internal.h
> index 62d8c34e63d5..abf1de31e524 100644
> --- a/mm/internal.h
> +++ b/mm/internal.h
> @@ -289,7 +289,8 @@ void __vma_link_list(struct mm_struct *mm, struct vm_area_struct *vma,
>
> #ifdef CONFIG_MMU
> extern long populate_vma_page_range(struct vm_area_struct *vma,
> - unsigned long start, unsigned long end, int *nonblocking);
> + unsigned long start, unsigned long end, int *nonblocking,
> + struct range_lock *mmrange);
> extern void munlock_vma_pages_range(struct vm_area_struct *vma,
> unsigned long start, unsigned long end);
> static inline void munlock_vma_pages_all(struct vm_area_struct *vma)
> diff --git a/mm/ksm.c b/mm/ksm.c
> index 293721f5da70..66c350cd9799 100644
> --- a/mm/ksm.c
> +++ b/mm/ksm.c
> @@ -448,7 +448,8 @@ static inline bool ksm_test_exit(struct mm_struct *mm)
> * of the process that owns 'vma'. We also do not want to enforce
> * protection keys here anyway.
> */
> -static int break_ksm(struct vm_area_struct *vma, unsigned long addr)
> +static int break_ksm(struct vm_area_struct *vma, unsigned long addr,
> + struct range_lock *mmrange)
> {
> struct page *page;
> int ret = 0;
> @@ -461,7 +462,8 @@ static int break_ksm(struct vm_area_struct *vma, unsigned long addr)
> break;
> if (PageKsm(page))
> ret = handle_mm_fault(vma, addr,
> - FAULT_FLAG_WRITE | FAULT_FLAG_REMOTE);
> + FAULT_FLAG_WRITE | FAULT_FLAG_REMOTE,
> + mmrange);
> else
> ret = VM_FAULT_WRITE;
> put_page(page);
> @@ -516,6 +518,7 @@ static void break_cow(struct rmap_item *rmap_item)
> struct mm_struct *mm = rmap_item->mm;
> unsigned long addr = rmap_item->address;
> struct vm_area_struct *vma;
> + DEFINE_RANGE_LOCK_FULL(mmrange);
>
> /*
> * It is not an accident that whenever we want to break COW
> @@ -526,7 +529,7 @@ static void break_cow(struct rmap_item *rmap_item)
> down_read(&mm->mmap_sem);
> vma = find_mergeable_vma(mm, addr);
> if (vma)
> - break_ksm(vma, addr);
> + break_ksm(vma, addr, &mmrange);
> up_read(&mm->mmap_sem);
> }
>
> @@ -807,7 +810,8 @@ static void remove_trailing_rmap_items(struct mm_slot *mm_slot,
> * in cmp_and_merge_page on one of the rmap_items we would be removing.
> */
> static int unmerge_ksm_pages(struct vm_area_struct *vma,
> - unsigned long start, unsigned long end)
> + unsigned long start, unsigned long end,
> + struct range_lock *mmrange)
> {
> unsigned long addr;
> int err = 0;
> @@ -818,7 +822,7 @@ static int unmerge_ksm_pages(struct vm_area_struct *vma,
> if (signal_pending(current))
> err = -ERESTARTSYS;
> else
> - err = break_ksm(vma, addr);
> + err = break_ksm(vma, addr, mmrange);
> }
> return err;
> }
> @@ -922,6 +926,7 @@ static int unmerge_and_remove_all_rmap_items(void)
> struct mm_struct *mm;
> struct vm_area_struct *vma;
> int err = 0;
> + DEFINE_RANGE_LOCK_FULL(mmrange);
>
> spin_lock(&ksm_mmlist_lock);
> ksm_scan.mm_slot = list_entry(ksm_mm_head.mm_list.next,
> @@ -937,8 +942,8 @@ static int unmerge_and_remove_all_rmap_items(void)
> break;
> if (!(vma->vm_flags & VM_MERGEABLE) || !vma->anon_vma)
> continue;
> - err = unmerge_ksm_pages(vma,
> - vma->vm_start, vma->vm_end);
> + err = unmerge_ksm_pages(vma, vma->vm_start,
> + vma->vm_end, &mmrange);
> if (err)
> goto error;
> }
> @@ -2350,7 +2355,8 @@ static int ksm_scan_thread(void *nothing)
> }
>
> int ksm_madvise(struct vm_area_struct *vma, unsigned long start,
> - unsigned long end, int advice, unsigned long *vm_flags)
> + unsigned long end, int advice, unsigned long *vm_flags,
> + struct range_lock *mmrange)
> {
> struct mm_struct *mm = vma->vm_mm;
> int err;
> @@ -2384,7 +2390,7 @@ int ksm_madvise(struct vm_area_struct *vma, unsigned long start,
> return 0; /* just ignore the advice */
>
> if (vma->anon_vma) {
> - err = unmerge_ksm_pages(vma, start, end);
> + err = unmerge_ksm_pages(vma, start, end, mmrange);
> if (err)
> return err;
> }
> diff --git a/mm/madvise.c b/mm/madvise.c
> index 4d3c922ea1a1..eaec6bfc2b08 100644
> --- a/mm/madvise.c
> +++ b/mm/madvise.c
> @@ -54,7 +54,8 @@ static int madvise_need_mmap_write(int behavior)
> */
> static long madvise_behavior(struct vm_area_struct *vma,
> struct vm_area_struct **prev,
> - unsigned long start, unsigned long end, int behavior)
> + unsigned long start, unsigned long end, int behavior,
> + struct range_lock *mmrange)
> {
> struct mm_struct *mm = vma->vm_mm;
> int error = 0;
> @@ -104,7 +105,8 @@ static long madvise_behavior(struct vm_area_struct *vma,
> break;
> case MADV_MERGEABLE:
> case MADV_UNMERGEABLE:
> - error = ksm_madvise(vma, start, end, behavior, &new_flags);
> + error = ksm_madvise(vma, start, end, behavior,
> + &new_flags, mmrange);
> if (error) {
> /*
> * madvise() returns EAGAIN if kernel resources, such as
> @@ -138,7 +140,7 @@ static long madvise_behavior(struct vm_area_struct *vma,
> pgoff = vma->vm_pgoff + ((start - vma->vm_start) >> PAGE_SHIFT);
> *prev = vma_merge(mm, *prev, start, end, new_flags, vma->anon_vma,
> vma->vm_file, pgoff, vma_policy(vma),
> - vma->vm_userfaultfd_ctx);
> + vma->vm_userfaultfd_ctx, mmrange);
> if (*prev) {
> vma = *prev;
> goto success;
> @@ -151,7 +153,7 @@ static long madvise_behavior(struct vm_area_struct *vma,
> error = -ENOMEM;
> goto out;
> }
> - error = __split_vma(mm, vma, start, 1);
> + error = __split_vma(mm, vma, start, 1, mmrange);
> if (error) {
> /*
> * madvise() returns EAGAIN if kernel resources, such as
> @@ -168,7 +170,7 @@ static long madvise_behavior(struct vm_area_struct *vma,
> error = -ENOMEM;
> goto out;
> }
> - error = __split_vma(mm, vma, end, 0);
> + error = __split_vma(mm, vma, end, 0, mmrange);
> if (error) {
> /*
> * madvise() returns EAGAIN if kernel resources, such as
> @@ -191,7 +193,8 @@ static long madvise_behavior(struct vm_area_struct *vma,
>
> #ifdef CONFIG_SWAP
> static int swapin_walk_pmd_entry(pmd_t *pmd, unsigned long start,
> - unsigned long end, struct mm_walk *walk)
> + unsigned long end, struct mm_walk *walk,
> + struct range_lock *mmrange)
> {
> pte_t *orig_pte;
> struct vm_area_struct *vma = walk->private;
> @@ -226,7 +229,8 @@ static int swapin_walk_pmd_entry(pmd_t *pmd, unsigned long start,
> }
>
> static void force_swapin_readahead(struct vm_area_struct *vma,
> - unsigned long start, unsigned long end)
> + unsigned long start, unsigned long end,
> + struct range_lock *mmrange)
> {
> struct mm_walk walk = {
> .mm = vma->vm_mm,
> @@ -234,7 +238,7 @@ static void force_swapin_readahead(struct vm_area_struct *vma,
> .private = vma,
> };
>
> - walk_page_range(start, end, &walk);
> + walk_page_range(start, end, &walk, mmrange);
>
> lru_add_drain(); /* Push any new pages onto the LRU now */
> }
> @@ -272,14 +276,15 @@ static void force_shm_swapin_readahead(struct vm_area_struct *vma,
> */
> static long madvise_willneed(struct vm_area_struct *vma,
> struct vm_area_struct **prev,
> - unsigned long start, unsigned long end)
> + unsigned long start, unsigned long end,
> + struct range_lock *mmrange)
> {
> struct file *file = vma->vm_file;
>
> *prev = vma;
> #ifdef CONFIG_SWAP
> if (!file) {
> - force_swapin_readahead(vma, start, end);
> + force_swapin_readahead(vma, start, end, mmrange);
> return 0;
> }
>
> @@ -308,7 +313,8 @@ static long madvise_willneed(struct vm_area_struct *vma,
> }
>
> static int madvise_free_pte_range(pmd_t *pmd, unsigned long addr,
> - unsigned long end, struct mm_walk *walk)
> + unsigned long end, struct mm_walk *walk,
> + struct range_lock *mmrange)
>
> {
> struct mmu_gather *tlb = walk->private;
> @@ -442,7 +448,8 @@ static int madvise_free_pte_range(pmd_t *pmd, unsigned long addr,
>
> static void madvise_free_page_range(struct mmu_gather *tlb,
> struct vm_area_struct *vma,
> - unsigned long addr, unsigned long end)
> + unsigned long addr, unsigned long end,
> + struct range_lock *mmrange)
> {
> struct mm_walk free_walk = {
> .pmd_entry = madvise_free_pte_range,
> @@ -451,12 +458,14 @@ static void madvise_free_page_range(struct mmu_gather *tlb,
> };
>
> tlb_start_vma(tlb, vma);
> - walk_page_range(addr, end, &free_walk);
> + walk_page_range(addr, end, &free_walk, mmrange);
> tlb_end_vma(tlb, vma);
> }
>
> static int madvise_free_single_vma(struct vm_area_struct *vma,
> - unsigned long start_addr, unsigned long end_addr)
> + unsigned long start_addr,
> + unsigned long end_addr,
> + struct range_lock *mmrange)
> {
> unsigned long start, end;
> struct mm_struct *mm = vma->vm_mm;
> @@ -478,7 +487,7 @@ static int madvise_free_single_vma(struct vm_area_struct *vma,
> update_hiwater_rss(mm);
>
> mmu_notifier_invalidate_range_start(mm, start, end);
> - madvise_free_page_range(&tlb, vma, start, end);
> + madvise_free_page_range(&tlb, vma, start, end, mmrange);
> mmu_notifier_invalidate_range_end(mm, start, end);
> tlb_finish_mmu(&tlb, start, end);
>
> @@ -514,7 +523,7 @@ static long madvise_dontneed_single_vma(struct vm_area_struct *vma,
> static long madvise_dontneed_free(struct vm_area_struct *vma,
> struct vm_area_struct **prev,
> unsigned long start, unsigned long end,
> - int behavior)
> + int behavior, struct range_lock *mmrange)
> {
> *prev = vma;
> if (!can_madv_dontneed_vma(vma))
> @@ -562,7 +571,7 @@ static long madvise_dontneed_free(struct vm_area_struct *vma,
> if (behavior == MADV_DONTNEED)
> return madvise_dontneed_single_vma(vma, start, end);
> else if (behavior == MADV_FREE)
> - return madvise_free_single_vma(vma, start, end);
> + return madvise_free_single_vma(vma, start, end, mmrange);
> else
> return -EINVAL;
> }
> @@ -676,18 +685,21 @@ static int madvise_inject_error(int behavior,
>
> static long
> madvise_vma(struct vm_area_struct *vma, struct vm_area_struct **prev,
> - unsigned long start, unsigned long end, int behavior)
> + unsigned long start, unsigned long end, int behavior,
> + struct range_lock *mmrange)
> {
> switch (behavior) {
> case MADV_REMOVE:
> return madvise_remove(vma, prev, start, end);
> case MADV_WILLNEED:
> - return madvise_willneed(vma, prev, start, end);
> + return madvise_willneed(vma, prev, start, end, mmrange);
> case MADV_FREE:
> case MADV_DONTNEED:
> - return madvise_dontneed_free(vma, prev, start, end, behavior);
> + return madvise_dontneed_free(vma, prev, start, end, behavior,
> + mmrange);
> default:
> - return madvise_behavior(vma, prev, start, end, behavior);
> + return madvise_behavior(vma, prev, start, end, behavior,
> + mmrange);
> }
> }
>
> @@ -797,7 +809,7 @@ SYSCALL_DEFINE3(madvise, unsigned long, start, size_t, len_in, int, behavior)
> int write;
> size_t len;
> struct blk_plug plug;
> -
> + DEFINE_RANGE_LOCK_FULL(mmrange);
> if (!madvise_behavior_valid(behavior))
> return error;
>
> @@ -860,7 +872,7 @@ SYSCALL_DEFINE3(madvise, unsigned long, start, size_t, len_in, int, behavior)
> tmp = end;
>
> /* Here vma->vm_start <= start < tmp <= (end|vma->vm_end). */
> - error = madvise_vma(vma, &prev, start, tmp, behavior);
> + error = madvise_vma(vma, &prev, start, tmp, behavior, &mmrange);
> if (error)
> goto out;
> start = tmp;
> diff --git a/mm/memcontrol.c b/mm/memcontrol.c
> index 88c1af32fd67..a7ac5a14b22e 100644
> --- a/mm/memcontrol.c
> +++ b/mm/memcontrol.c
> @@ -4881,7 +4881,8 @@ static inline enum mc_target_type get_mctgt_type_thp(struct vm_area_struct *vma,
>
> static int mem_cgroup_count_precharge_pte_range(pmd_t *pmd,
> unsigned long addr, unsigned long end,
> - struct mm_walk *walk)
> + struct mm_walk *walk,
> + struct range_lock *mmrange)
> {
> struct vm_area_struct *vma = walk->vma;
> pte_t *pte;
> @@ -4915,6 +4916,7 @@ static int mem_cgroup_count_precharge_pte_range(pmd_t *pmd,
> static unsigned long mem_cgroup_count_precharge(struct mm_struct *mm)
> {
> unsigned long precharge;
> + DEFINE_RANGE_LOCK_FULL(mmrange);
>
> struct mm_walk mem_cgroup_count_precharge_walk = {
> .pmd_entry = mem_cgroup_count_precharge_pte_range,
> @@ -4922,7 +4924,7 @@ static unsigned long mem_cgroup_count_precharge(struct mm_struct *mm)
> };
> down_read(&mm->mmap_sem);
> walk_page_range(0, mm->highest_vm_end,
> - &mem_cgroup_count_precharge_walk);
> + &mem_cgroup_count_precharge_walk, &mmrange);
> up_read(&mm->mmap_sem);
>
> precharge = mc.precharge;
> @@ -5081,7 +5083,8 @@ static void mem_cgroup_cancel_attach(struct cgroup_taskset *tset)
>
> static int mem_cgroup_move_charge_pte_range(pmd_t *pmd,
> unsigned long addr, unsigned long end,
> - struct mm_walk *walk)
> + struct mm_walk *walk,
> + struct range_lock *mmrange)
> {
> int ret = 0;
> struct vm_area_struct *vma = walk->vma;
> @@ -5197,6 +5200,7 @@ static void mem_cgroup_move_charge(void)
> .pmd_entry = mem_cgroup_move_charge_pte_range,
> .mm = mc.mm,
> };
> + DEFINE_RANGE_LOCK_FULL(mmrange);
>
> lru_add_drain_all();
> /*
> @@ -5223,7 +5227,8 @@ static void mem_cgroup_move_charge(void)
> * When we have consumed all precharges and failed in doing
> * additional charge, the page walk just aborts.
> */
> - walk_page_range(0, mc.mm->highest_vm_end, &mem_cgroup_move_charge_walk);
> + walk_page_range(0, mc.mm->highest_vm_end, &mem_cgroup_move_charge_walk,
> + &mmrange);
>
> up_read(&mc.mm->mmap_sem);
> atomic_dec(&mc.from->moving_account);
> diff --git a/mm/memory.c b/mm/memory.c
> index 5ec6433d6a5c..b3561a052939 100644
> --- a/mm/memory.c
> +++ b/mm/memory.c
> @@ -4021,7 +4021,7 @@ static int handle_pte_fault(struct vm_fault *vmf)
> * return value. See filemap_fault() and __lock_page_or_retry().
> */
> static int __handle_mm_fault(struct vm_area_struct *vma, unsigned long address,
> - unsigned int flags)
> + unsigned int flags, struct range_lock *mmrange)
> {
> struct vm_fault vmf = {
> .vma = vma,
> @@ -4029,6 +4029,7 @@ static int __handle_mm_fault(struct vm_area_struct *vma, unsigned long address,
> .flags = flags,
> .pgoff = linear_page_index(vma, address),
> .gfp_mask = __get_fault_gfp_mask(vma),
> + .lockrange = mmrange,
> };
> unsigned int dirty = flags & FAULT_FLAG_WRITE;
> struct mm_struct *mm = vma->vm_mm;
> @@ -4110,7 +4111,7 @@ static int __handle_mm_fault(struct vm_area_struct *vma, unsigned long address,
> * return value. See filemap_fault() and __lock_page_or_retry().
> */
> int handle_mm_fault(struct vm_area_struct *vma, unsigned long address,
> - unsigned int flags)
> + unsigned int flags, struct range_lock *mmrange)
> {
> int ret;
>
> @@ -4137,7 +4138,7 @@ int handle_mm_fault(struct vm_area_struct *vma, unsigned long address,
> if (unlikely(is_vm_hugetlb_page(vma)))
> ret = hugetlb_fault(vma->vm_mm, vma, address, flags);
> else
> - ret = __handle_mm_fault(vma, address, flags);
> + ret = __handle_mm_fault(vma, address, flags, mmrange);
>
> if (flags & FAULT_FLAG_USER) {
> mem_cgroup_oom_disable();
> @@ -4425,6 +4426,7 @@ int __access_remote_vm(struct task_struct *tsk, struct mm_struct *mm,
> struct vm_area_struct *vma;
> void *old_buf = buf;
> int write = gup_flags & FOLL_WRITE;
> + DEFINE_RANGE_LOCK_FULL(mmrange);
>
> down_read(&mm->mmap_sem);
> /* ignore errors, just check how much was successfully transferred */
> @@ -4434,7 +4436,7 @@ int __access_remote_vm(struct task_struct *tsk, struct mm_struct *mm,
> struct page *page = NULL;
>
> ret = get_user_pages_remote(tsk, mm, addr, 1,
> - gup_flags, &page, &vma, NULL);
> + gup_flags, &page, &vma, NULL, &mmrange);
> if (ret <= 0) {
> #ifndef CONFIG_HAVE_IOREMAP_PROT
> break;
> diff --git a/mm/mempolicy.c b/mm/mempolicy.c
> index a8b7d59002e8..001dc176abc1 100644
> --- a/mm/mempolicy.c
> +++ b/mm/mempolicy.c
> @@ -467,7 +467,8 @@ static int queue_pages_pmd(pmd_t *pmd, spinlock_t *ptl, unsigned long addr,
> * and move them to the pagelist if they do.
> */
> static int queue_pages_pte_range(pmd_t *pmd, unsigned long addr,
> - unsigned long end, struct mm_walk *walk)
> + unsigned long end, struct mm_walk *walk,
> + struct range_lock *mmrange)
> {
> struct vm_area_struct *vma = walk->vma;
> struct page *page;
> @@ -618,7 +619,7 @@ static int queue_pages_test_walk(unsigned long start, unsigned long end,
> static int
> queue_pages_range(struct mm_struct *mm, unsigned long start, unsigned long end,
> nodemask_t *nodes, unsigned long flags,
> - struct list_head *pagelist)
> + struct list_head *pagelist, struct range_lock *mmrange)
> {
> struct queue_pages qp = {
> .pagelist = pagelist,
> @@ -634,7 +635,7 @@ queue_pages_range(struct mm_struct *mm, unsigned long start, unsigned long end,
> .private = &qp,
> };
>
> - return walk_page_range(start, end, &queue_pages_walk);
> + return walk_page_range(start, end, &queue_pages_walk, mmrange);
> }
>
> /*
> @@ -675,7 +676,8 @@ static int vma_replace_policy(struct vm_area_struct *vma,
>
> /* Step 2: apply policy to a range and do splits. */
> static int mbind_range(struct mm_struct *mm, unsigned long start,
> - unsigned long end, struct mempolicy *new_pol)
> + unsigned long end, struct mempolicy *new_pol,
> + struct range_lock *mmrange)
> {
> struct vm_area_struct *next;
> struct vm_area_struct *prev;
> @@ -705,7 +707,7 @@ static int mbind_range(struct mm_struct *mm, unsigned long start,
> ((vmstart - vma->vm_start) >> PAGE_SHIFT);
> prev = vma_merge(mm, prev, vmstart, vmend, vma->vm_flags,
> vma->anon_vma, vma->vm_file, pgoff,
> - new_pol, vma->vm_userfaultfd_ctx);
> + new_pol, vma->vm_userfaultfd_ctx, mmrange);
> if (prev) {
> vma = prev;
> next = vma->vm_next;
> @@ -715,12 +717,12 @@ static int mbind_range(struct mm_struct *mm, unsigned long start,
> goto replace;
> }
> if (vma->vm_start != vmstart) {
> - err = split_vma(vma->vm_mm, vma, vmstart, 1);
> + err = split_vma(vma->vm_mm, vma, vmstart, 1, mmrange);
> if (err)
> goto out;
> }
> if (vma->vm_end != vmend) {
> - err = split_vma(vma->vm_mm, vma, vmend, 0);
> + err = split_vma(vma->vm_mm, vma, vmend, 0, mmrange);
> if (err)
> goto out;
> }
> @@ -797,12 +799,12 @@ static void get_policy_nodemask(struct mempolicy *p, nodemask_t *nodes)
> }
> }
>
> -static int lookup_node(unsigned long addr)
> +static int lookup_node(unsigned long addr, struct range_lock *mmrange)
> {
> struct page *p;
> int err;
>
> - err = get_user_pages(addr & PAGE_MASK, 1, 0, &p, NULL);
> + err = get_user_pages(addr & PAGE_MASK, 1, 0, &p, NULL, mmrange);
> if (err >= 0) {
> err = page_to_nid(p);
> put_page(p);
> @@ -818,6 +820,7 @@ static long do_get_mempolicy(int *policy, nodemask_t *nmask,
> struct mm_struct *mm = current->mm;
> struct vm_area_struct *vma = NULL;
> struct mempolicy *pol = current->mempolicy;
> + DEFINE_RANGE_LOCK_FULL(mmrange);
>
> if (flags &
> ~(unsigned long)(MPOL_F_NODE|MPOL_F_ADDR|MPOL_F_MEMS_ALLOWED))
> @@ -857,7 +860,7 @@ static long do_get_mempolicy(int *policy, nodemask_t *nmask,
>
> if (flags & MPOL_F_NODE) {
> if (flags & MPOL_F_ADDR) {
> - err = lookup_node(addr);
> + err = lookup_node(addr, &mmrange);
> if (err < 0)
> goto out;
> *policy = err;
> @@ -943,7 +946,7 @@ struct page *alloc_new_node_page(struct page *page, unsigned long node)
> * Returns error or the number of pages not migrated.
> */
> static int migrate_to_node(struct mm_struct *mm, int source, int dest,
> - int flags)
> + int flags, struct range_lock *mmrange)
> {
> nodemask_t nmask;
> LIST_HEAD(pagelist);
> @@ -959,7 +962,7 @@ static int migrate_to_node(struct mm_struct *mm, int source, int dest,
> */
> VM_BUG_ON(!(flags & (MPOL_MF_MOVE | MPOL_MF_MOVE_ALL)));
> queue_pages_range(mm, mm->mmap->vm_start, mm->task_size, &nmask,
> - flags | MPOL_MF_DISCONTIG_OK, &pagelist);
> + flags | MPOL_MF_DISCONTIG_OK, &pagelist, mmrange);
>
> if (!list_empty(&pagelist)) {
> err = migrate_pages(&pagelist, alloc_new_node_page, NULL, dest,
> @@ -983,6 +986,7 @@ int do_migrate_pages(struct mm_struct *mm, const nodemask_t *from,
> int busy = 0;
> int err;
> nodemask_t tmp;
> + DEFINE_RANGE_LOCK_FULL(mmrange);
>
> err = migrate_prep();
> if (err)
> @@ -1063,7 +1067,7 @@ int do_migrate_pages(struct mm_struct *mm, const nodemask_t *from,
> break;
>
> node_clear(source, tmp);
> - err = migrate_to_node(mm, source, dest, flags);
> + err = migrate_to_node(mm, source, dest, flags, &mmrange);
> if (err > 0)
> busy += err;
> if (err < 0)
> @@ -1143,6 +1147,7 @@ static long do_mbind(unsigned long start, unsigned long len,
> unsigned long end;
> int err;
> LIST_HEAD(pagelist);
> + DEFINE_RANGE_LOCK_FULL(mmrange);
>
> if (flags & ~(unsigned long)MPOL_MF_VALID)
> return -EINVAL;
> @@ -1204,9 +1209,9 @@ static long do_mbind(unsigned long start, unsigned long len,
> goto mpol_out;
>
> err = queue_pages_range(mm, start, end, nmask,
> - flags | MPOL_MF_INVERT, &pagelist);
> + flags | MPOL_MF_INVERT, &pagelist, &mmrange);
> if (!err)
> - err = mbind_range(mm, start, end, new);
> + err = mbind_range(mm, start, end, new, &mmrange);
>
> if (!err) {
> int nr_failed = 0;
> diff --git a/mm/migrate.c b/mm/migrate.c
> index 5d0dc7b85f90..7a6afc34dd54 100644
> --- a/mm/migrate.c
> +++ b/mm/migrate.c
> @@ -2105,7 +2105,8 @@ struct migrate_vma {
>
> static int migrate_vma_collect_hole(unsigned long start,
> unsigned long end,
> - struct mm_walk *walk)
> + struct mm_walk *walk,
> + struct range_lock *mmrange)
> {
> struct migrate_vma *migrate = walk->private;
> unsigned long addr;
> @@ -2138,7 +2139,8 @@ static int migrate_vma_collect_skip(unsigned long start,
> static int migrate_vma_collect_pmd(pmd_t *pmdp,
> unsigned long start,
> unsigned long end,
> - struct mm_walk *walk)
> + struct mm_walk *walk,
> + struct range_lock *mmrange)
> {
> struct migrate_vma *migrate = walk->private;
> struct vm_area_struct *vma = walk->vma;
> @@ -2149,7 +2151,7 @@ static int migrate_vma_collect_pmd(pmd_t *pmdp,
>
> again:
> if (pmd_none(*pmdp))
> - return migrate_vma_collect_hole(start, end, walk);
> + return migrate_vma_collect_hole(start, end, walk, mmrange);
>
> if (pmd_trans_huge(*pmdp)) {
> struct page *page;
> @@ -2183,7 +2185,7 @@ static int migrate_vma_collect_pmd(pmd_t *pmdp,
> walk);
> if (pmd_none(*pmdp))
> return migrate_vma_collect_hole(start, end,
> - walk);
> + walk, mmrange);
> }
> }
>
> @@ -2309,7 +2311,8 @@ static int migrate_vma_collect_pmd(pmd_t *pmdp,
> * valid page, it updates the src array and takes a reference on the page, in
> * order to pin the page until we lock it and unmap it.
> */
> -static void migrate_vma_collect(struct migrate_vma *migrate)
> +static void migrate_vma_collect(struct migrate_vma *migrate,
> + struct range_lock *mmrange)
> {
> struct mm_walk mm_walk;
>
> @@ -2325,7 +2328,7 @@ static void migrate_vma_collect(struct migrate_vma *migrate)
> mmu_notifier_invalidate_range_start(mm_walk.mm,
> migrate->start,
> migrate->end);
> - walk_page_range(migrate->start, migrate->end, &mm_walk);
> + walk_page_range(migrate->start, migrate->end, &mm_walk, mmrange);
> mmu_notifier_invalidate_range_end(mm_walk.mm,
> migrate->start,
> migrate->end);
> @@ -2891,7 +2894,8 @@ int migrate_vma(const struct migrate_vma_ops *ops,
> unsigned long end,
> unsigned long *src,
> unsigned long *dst,
> - void *private)
> + void *private,
> + struct range_lock *mmrange)
> {
> struct migrate_vma migrate;
>
> @@ -2917,7 +2921,7 @@ int migrate_vma(const struct migrate_vma_ops *ops,
> migrate.vma = vma;
>
> /* Collect, and try to unmap source pages */
> - migrate_vma_collect(&migrate);
> + migrate_vma_collect(&migrate, mmrange);
> if (!migrate.cpages)
> return 0;
>
> diff --git a/mm/mincore.c b/mm/mincore.c
> index fc37afe226e6..a6875a34aac0 100644
> --- a/mm/mincore.c
> +++ b/mm/mincore.c
> @@ -85,7 +85,9 @@ static unsigned char mincore_page(struct address_space *mapping, pgoff_t pgoff)
> }
>
> static int __mincore_unmapped_range(unsigned long addr, unsigned long end,
> - struct vm_area_struct *vma, unsigned char *vec)
> + struct vm_area_struct *vma,
> + unsigned char *vec,
> + struct range_lock *mmrange)
> {
> unsigned long nr = (end - addr) >> PAGE_SHIFT;
> int i;
> @@ -104,15 +106,17 @@ static int __mincore_unmapped_range(unsigned long addr, unsigned long end,
> }
>
> static int mincore_unmapped_range(unsigned long addr, unsigned long end,
> - struct mm_walk *walk)
> + struct mm_walk *walk,
> + struct range_lock *mmrange)
> {
> walk->private += __mincore_unmapped_range(addr, end,
> - walk->vma, walk->private);
> + walk->vma,
> + walk->private, mmrange);
> return 0;
> }
>
> static int mincore_pte_range(pmd_t *pmd, unsigned long addr, unsigned long end,
> - struct mm_walk *walk)
> + struct mm_walk *walk, struct range_lock *mmrange)
> {
> spinlock_t *ptl;
> struct vm_area_struct *vma = walk->vma;
> @@ -128,7 +132,7 @@ static int mincore_pte_range(pmd_t *pmd, unsigned long addr, unsigned long end,
> }
>
> if (pmd_trans_unstable(pmd)) {
> - __mincore_unmapped_range(addr, end, vma, vec);
> + __mincore_unmapped_range(addr, end, vma, vec, mmrange);
> goto out;
> }
>
> @@ -138,7 +142,7 @@ static int mincore_pte_range(pmd_t *pmd, unsigned long addr, unsigned long end,
>
> if (pte_none(pte))
> __mincore_unmapped_range(addr, addr + PAGE_SIZE,
> - vma, vec);
> + vma, vec, mmrange);
> else if (pte_present(pte))
> *vec = 1;
> else { /* pte is a swap entry */
> @@ -174,7 +178,8 @@ static int mincore_pte_range(pmd_t *pmd, unsigned long addr, unsigned long end,
> * all the arguments, we hold the mmap semaphore: we should
> * just return the amount of info we're asked for.
> */
> -static long do_mincore(unsigned long addr, unsigned long pages, unsigned char *vec)
> +static long do_mincore(unsigned long addr, unsigned long pages,
> + unsigned char *vec, struct range_lock *mmrange)
> {
> struct vm_area_struct *vma;
> unsigned long end;
> @@ -191,7 +196,7 @@ static long do_mincore(unsigned long addr, unsigned long pages, unsigned char *v
> return -ENOMEM;
> mincore_walk.mm = vma->vm_mm;
> end = min(vma->vm_end, addr + (pages << PAGE_SHIFT));
> - err = walk_page_range(addr, end, &mincore_walk);
> + err = walk_page_range(addr, end, &mincore_walk, mmrange);
> if (err < 0)
> return err;
> return (end - addr) >> PAGE_SHIFT;
> @@ -227,6 +232,7 @@ SYSCALL_DEFINE3(mincore, unsigned long, start, size_t, len,
> long retval;
> unsigned long pages;
> unsigned char *tmp;
> + DEFINE_RANGE_LOCK_FULL(mmrange);
>
> /* Check the start address: needs to be page-aligned.. */
> if (start & ~PAGE_MASK)
> @@ -254,7 +260,7 @@ SYSCALL_DEFINE3(mincore, unsigned long, start, size_t, len,
> * the temporary buffer size.
> */
> down_read(¤t->mm->mmap_sem);
> - retval = do_mincore(start, min(pages, PAGE_SIZE), tmp);
> + retval = do_mincore(start, min(pages, PAGE_SIZE), tmp, &mmrange);
> up_read(¤t->mm->mmap_sem);
>
> if (retval <= 0)
> diff --git a/mm/mlock.c b/mm/mlock.c
> index 74e5a6547c3d..3f6bd953e8b0 100644
> --- a/mm/mlock.c
> +++ b/mm/mlock.c
> @@ -517,7 +517,8 @@ void munlock_vma_pages_range(struct vm_area_struct *vma,
> * For vmas that pass the filters, merge/split as appropriate.
> */
> static int mlock_fixup(struct vm_area_struct *vma, struct vm_area_struct **prev,
> - unsigned long start, unsigned long end, vm_flags_t newflags)
> + unsigned long start, unsigned long end, vm_flags_t newflags,
> + struct range_lock *mmrange)
> {
> struct mm_struct *mm = vma->vm_mm;
> pgoff_t pgoff;
> @@ -534,20 +535,20 @@ static int mlock_fixup(struct vm_area_struct *vma, struct vm_area_struct **prev,
> pgoff = vma->vm_pgoff + ((start - vma->vm_start) >> PAGE_SHIFT);
> *prev = vma_merge(mm, *prev, start, end, newflags, vma->anon_vma,
> vma->vm_file, pgoff, vma_policy(vma),
> - vma->vm_userfaultfd_ctx);
> + vma->vm_userfaultfd_ctx, mmrange);
> if (*prev) {
> vma = *prev;
> goto success;
> }
>
> if (start != vma->vm_start) {
> - ret = split_vma(mm, vma, start, 1);
> + ret = split_vma(mm, vma, start, 1, mmrange);
> if (ret)
> goto out;
> }
>
> if (end != vma->vm_end) {
> - ret = split_vma(mm, vma, end, 0);
> + ret = split_vma(mm, vma, end, 0, mmrange);
> if (ret)
> goto out;
> }
> @@ -580,7 +581,7 @@ static int mlock_fixup(struct vm_area_struct *vma, struct vm_area_struct **prev,
> }
>
> static int apply_vma_lock_flags(unsigned long start, size_t len,
> - vm_flags_t flags)
> + vm_flags_t flags, struct range_lock *mmrange)
> {
> unsigned long nstart, end, tmp;
> struct vm_area_struct * vma, * prev;
> @@ -610,7 +611,7 @@ static int apply_vma_lock_flags(unsigned long start, size_t len,
> tmp = vma->vm_end;
> if (tmp > end)
> tmp = end;
> - error = mlock_fixup(vma, &prev, nstart, tmp, newflags);
> + error = mlock_fixup(vma, &prev, nstart, tmp, newflags, mmrange);
> if (error)
> break;
> nstart = tmp;
> @@ -667,11 +668,13 @@ static int count_mm_mlocked_page_nr(struct mm_struct *mm,
> return count >> PAGE_SHIFT;
> }
>
> -static __must_check int do_mlock(unsigned long start, size_t len, vm_flags_t flags)
> +static __must_check int do_mlock(unsigned long start, size_t len,
> + vm_flags_t flags)
> {
> unsigned long locked;
> unsigned long lock_limit;
> int error = -ENOMEM;
> + DEFINE_RANGE_LOCK_FULL(mmrange);
>
> if (!can_do_mlock())
> return -EPERM;
> @@ -700,7 +703,7 @@ static __must_check int do_mlock(unsigned long start, size_t len, vm_flags_t fla
>
> /* check against resource limits */
> if ((locked <= lock_limit) || capable(CAP_IPC_LOCK))
> - error = apply_vma_lock_flags(start, len, flags);
> + error = apply_vma_lock_flags(start, len, flags, &mmrange);
>
> up_write(¤t->mm->mmap_sem);
> if (error)
> @@ -733,13 +736,14 @@ SYSCALL_DEFINE3(mlock2, unsigned long, start, size_t, len, int, flags)
> SYSCALL_DEFINE2(munlock, unsigned long, start, size_t, len)
> {
> int ret;
> + DEFINE_RANGE_LOCK_FULL(mmrange);
>
> len = PAGE_ALIGN(len + (offset_in_page(start)));
> start &= PAGE_MASK;
>
> if (down_write_killable(¤t->mm->mmap_sem))
> return -EINTR;
> - ret = apply_vma_lock_flags(start, len, 0);
> + ret = apply_vma_lock_flags(start, len, 0, &mmrange);
> up_write(¤t->mm->mmap_sem);
>
> return ret;
> @@ -755,7 +759,7 @@ SYSCALL_DEFINE2(munlock, unsigned long, start, size_t, len)
> * is called once including the MCL_FUTURE flag and then a second time without
> * it, VM_LOCKED and VM_LOCKONFAULT will be cleared from mm->def_flags.
> */
> -static int apply_mlockall_flags(int flags)
> +static int apply_mlockall_flags(int flags, struct range_lock *mmrange)
> {
> struct vm_area_struct * vma, * prev = NULL;
> vm_flags_t to_add = 0;
> @@ -784,7 +788,8 @@ static int apply_mlockall_flags(int flags)
> newflags |= to_add;
>
> /* Ignore errors */
> - mlock_fixup(vma, &prev, vma->vm_start, vma->vm_end, newflags);
> + mlock_fixup(vma, &prev, vma->vm_start, vma->vm_end, newflags,
> + mmrange);
> cond_resched();
> }
> out:
> @@ -795,6 +800,7 @@ SYSCALL_DEFINE1(mlockall, int, flags)
> {
> unsigned long lock_limit;
> int ret;
> + DEFINE_RANGE_LOCK_FULL(mmrange);
>
> if (!flags || (flags & ~(MCL_CURRENT | MCL_FUTURE | MCL_ONFAULT)))
> return -EINVAL;
> @@ -811,7 +817,7 @@ SYSCALL_DEFINE1(mlockall, int, flags)
> ret = -ENOMEM;
> if (!(flags & MCL_CURRENT) || (current->mm->total_vm <= lock_limit) ||
> capable(CAP_IPC_LOCK))
> - ret = apply_mlockall_flags(flags);
> + ret = apply_mlockall_flags(flags, &mmrange);
> up_write(¤t->mm->mmap_sem);
> if (!ret && (flags & MCL_CURRENT))
> mm_populate(0, TASK_SIZE);
> @@ -822,10 +828,11 @@ SYSCALL_DEFINE1(mlockall, int, flags)
> SYSCALL_DEFINE0(munlockall)
> {
> int ret;
> + DEFINE_RANGE_LOCK_FULL(mmrange);
>
> if (down_write_killable(¤t->mm->mmap_sem))
> return -EINTR;
> - ret = apply_mlockall_flags(0);
> + ret = apply_mlockall_flags(0, &mmrange);
> up_write(¤t->mm->mmap_sem);
> return ret;
> }
> diff --git a/mm/mmap.c b/mm/mmap.c
> index 4bb038e7984b..f61d49cb791e 100644
> --- a/mm/mmap.c
> +++ b/mm/mmap.c
> @@ -177,7 +177,8 @@ static struct vm_area_struct *remove_vma(struct vm_area_struct *vma)
> return next;
> }
>
> -static int do_brk(unsigned long addr, unsigned long len, struct list_head *uf);
> +static int do_brk(unsigned long addr, unsigned long len, struct list_head *uf,
> + struct range_lock *mmrange);
>
> SYSCALL_DEFINE1(brk, unsigned long, brk)
> {
> @@ -188,6 +189,7 @@ SYSCALL_DEFINE1(brk, unsigned long, brk)
> unsigned long min_brk;
> bool populate;
> LIST_HEAD(uf);
> + DEFINE_RANGE_LOCK_FULL(mmrange);
>
> if (down_write_killable(&mm->mmap_sem))
> return -EINTR;
> @@ -225,7 +227,7 @@ SYSCALL_DEFINE1(brk, unsigned long, brk)
>
> /* Always allow shrinking brk. */
> if (brk <= mm->brk) {
> - if (!do_munmap(mm, newbrk, oldbrk-newbrk, &uf))
> + if (!do_munmap(mm, newbrk, oldbrk-newbrk, &uf, &mmrange))
> goto set_brk;
> goto out;
> }
> @@ -236,7 +238,7 @@ SYSCALL_DEFINE1(brk, unsigned long, brk)
> goto out;
>
> /* Ok, looks good - let it rip. */
> - if (do_brk(oldbrk, newbrk-oldbrk, &uf) < 0)
> + if (do_brk(oldbrk, newbrk-oldbrk, &uf, &mmrange) < 0)
> goto out;
>
> set_brk:
> @@ -680,7 +682,7 @@ static inline void __vma_unlink_prev(struct mm_struct *mm,
> */
> int __vma_adjust(struct vm_area_struct *vma, unsigned long start,
> unsigned long end, pgoff_t pgoff, struct vm_area_struct *insert,
> - struct vm_area_struct *expand)
> + struct vm_area_struct *expand, struct range_lock *mmrange)
> {
> struct mm_struct *mm = vma->vm_mm;
> struct vm_area_struct *next = vma->vm_next, *orig_vma = vma;
> @@ -887,10 +889,10 @@ int __vma_adjust(struct vm_area_struct *vma, unsigned long start,
> i_mmap_unlock_write(mapping);
>
> if (root) {
> - uprobe_mmap(vma);
> + uprobe_mmap(vma, mmrange);
>
> if (adjust_next)
> - uprobe_mmap(next);
> + uprobe_mmap(next, mmrange);
> }
>
> if (remove_next) {
> @@ -960,7 +962,7 @@ int __vma_adjust(struct vm_area_struct *vma, unsigned long start,
> }
> }
> if (insert && file)
> - uprobe_mmap(insert);
> + uprobe_mmap(insert, mmrange);
>
> validate_mm(mm);
>
> @@ -1101,7 +1103,8 @@ struct vm_area_struct *vma_merge(struct mm_struct *mm,
> unsigned long end, unsigned long vm_flags,
> struct anon_vma *anon_vma, struct file *file,
> pgoff_t pgoff, struct mempolicy *policy,
> - struct vm_userfaultfd_ctx vm_userfaultfd_ctx)
> + struct vm_userfaultfd_ctx vm_userfaultfd_ctx,
> + struct range_lock *mmrange)
> {
> pgoff_t pglen = (end - addr) >> PAGE_SHIFT;
> struct vm_area_struct *area, *next;
> @@ -1149,10 +1152,11 @@ struct vm_area_struct *vma_merge(struct mm_struct *mm,
> /* cases 1, 6 */
> err = __vma_adjust(prev, prev->vm_start,
> next->vm_end, prev->vm_pgoff, NULL,
> - prev);
> + prev, mmrange);
> } else /* cases 2, 5, 7 */
> err = __vma_adjust(prev, prev->vm_start,
> - end, prev->vm_pgoff, NULL, prev);
> + end, prev->vm_pgoff, NULL,
> + prev, mmrange);
> if (err)
> return NULL;
> khugepaged_enter_vma_merge(prev, vm_flags);
> @@ -1169,10 +1173,12 @@ struct vm_area_struct *vma_merge(struct mm_struct *mm,
> vm_userfaultfd_ctx)) {
> if (prev && addr < prev->vm_end) /* case 4 */
> err = __vma_adjust(prev, prev->vm_start,
> - addr, prev->vm_pgoff, NULL, next);
> + addr, prev->vm_pgoff, NULL,
> + next, mmrange);
> else { /* cases 3, 8 */
> err = __vma_adjust(area, addr, next->vm_end,
> - next->vm_pgoff - pglen, NULL, next);
> + next->vm_pgoff - pglen, NULL,
> + next, mmrange);
> /*
> * In case 3 area is already equal to next and
> * this is a noop, but in case 8 "area" has
> @@ -1322,7 +1328,7 @@ unsigned long do_mmap(struct file *file, unsigned long addr,
> unsigned long len, unsigned long prot,
> unsigned long flags, vm_flags_t vm_flags,
> unsigned long pgoff, unsigned long *populate,
> - struct list_head *uf)
> + struct list_head *uf, struct range_lock *mmrange)
> {
> struct mm_struct *mm = current->mm;
> int pkey = 0;
> @@ -1491,7 +1497,7 @@ unsigned long do_mmap(struct file *file, unsigned long addr,
> vm_flags |= VM_NORESERVE;
> }
>
> - addr = mmap_region(file, addr, len, vm_flags, pgoff, uf);
> + addr = mmap_region(file, addr, len, vm_flags, pgoff, uf, mmrange);
> if (!IS_ERR_VALUE(addr) &&
> ((vm_flags & VM_LOCKED) ||
> (flags & (MAP_POPULATE | MAP_NONBLOCK)) == MAP_POPULATE))
> @@ -1628,7 +1634,7 @@ static inline int accountable_mapping(struct file *file, vm_flags_t vm_flags)
>
> unsigned long mmap_region(struct file *file, unsigned long addr,
> unsigned long len, vm_flags_t vm_flags, unsigned long pgoff,
> - struct list_head *uf)
> + struct list_head *uf, struct range_lock *mmrange)
> {
> struct mm_struct *mm = current->mm;
> struct vm_area_struct *vma, *prev;
> @@ -1654,7 +1660,7 @@ unsigned long mmap_region(struct file *file, unsigned long addr,
> /* Clear old maps */
> while (find_vma_links(mm, addr, addr + len, &prev, &rb_link,
> &rb_parent)) {
> - if (do_munmap(mm, addr, len, uf))
> + if (do_munmap(mm, addr, len, uf, mmrange))
> return -ENOMEM;
> }
>
> @@ -1672,7 +1678,7 @@ unsigned long mmap_region(struct file *file, unsigned long addr,
> * Can we just expand an old mapping?
> */
> vma = vma_merge(mm, prev, addr, addr + len, vm_flags,
> - NULL, file, pgoff, NULL, NULL_VM_UFFD_CTX);
> + NULL, file, pgoff, NULL, NULL_VM_UFFD_CTX, mmrange);
> if (vma)
> goto out;
>
> @@ -1756,7 +1762,7 @@ unsigned long mmap_region(struct file *file, unsigned long addr,
> }
>
> if (file)
> - uprobe_mmap(vma);
> + uprobe_mmap(vma, mmrange);
>
> /*
> * New (or expanded) vma always get soft dirty status.
> @@ -2435,7 +2441,8 @@ int expand_stack(struct vm_area_struct *vma, unsigned long address)
> }
>
> struct vm_area_struct *
> -find_extend_vma(struct mm_struct *mm, unsigned long addr)
> +find_extend_vma(struct mm_struct *mm, unsigned long addr,
> + struct range_lock *mmrange)
> {
> struct vm_area_struct *vma, *prev;
>
> @@ -2446,7 +2453,8 @@ find_extend_vma(struct mm_struct *mm, unsigned long addr)
> if (!prev || expand_stack(prev, addr))
> return NULL;
> if (prev->vm_flags & VM_LOCKED)
> - populate_vma_page_range(prev, addr, prev->vm_end, NULL);
> + populate_vma_page_range(prev, addr, prev->vm_end,
> + NULL, mmrange);
> return prev;
> }
> #else
> @@ -2456,7 +2464,8 @@ int expand_stack(struct vm_area_struct *vma, unsigned long address)
> }
>
> struct vm_area_struct *
> -find_extend_vma(struct mm_struct *mm, unsigned long addr)
> +find_extend_vma(struct mm_struct *mm, unsigned long addr,
> + struct range_lock *mmrange)
> {
> struct vm_area_struct *vma;
> unsigned long start;
> @@ -2473,7 +2482,7 @@ find_extend_vma(struct mm_struct *mm, unsigned long addr)
> if (expand_stack(vma, addr))
> return NULL;
> if (vma->vm_flags & VM_LOCKED)
> - populate_vma_page_range(vma, addr, start, NULL);
> + populate_vma_page_range(vma, addr, start, NULL, mmrange);
> return vma;
> }
> #endif
> @@ -2561,7 +2570,7 @@ detach_vmas_to_be_unmapped(struct mm_struct *mm, struct vm_area_struct *vma,
> * has already been checked or doesn't make sense to fail.
> */
> int __split_vma(struct mm_struct *mm, struct vm_area_struct *vma,
> - unsigned long addr, int new_below)
> + unsigned long addr, int new_below, struct range_lock *mmrange)
> {
> struct vm_area_struct *new;
> int err;
> @@ -2604,9 +2613,11 @@ int __split_vma(struct mm_struct *mm, struct vm_area_struct *vma,
>
> if (new_below)
> err = vma_adjust(vma, addr, vma->vm_end, vma->vm_pgoff +
> - ((addr - new->vm_start) >> PAGE_SHIFT), new);
> + ((addr - new->vm_start) >> PAGE_SHIFT), new,
> + mmrange);
> else
> - err = vma_adjust(vma, vma->vm_start, addr, vma->vm_pgoff, new);
> + err = vma_adjust(vma, vma->vm_start, addr, vma->vm_pgoff, new,
> + mmrange);
>
> /* Success. */
> if (!err)
> @@ -2630,12 +2641,12 @@ int __split_vma(struct mm_struct *mm, struct vm_area_struct *vma,
> * either for the first part or the tail.
> */
> int split_vma(struct mm_struct *mm, struct vm_area_struct *vma,
> - unsigned long addr, int new_below)
> + unsigned long addr, int new_below, struct range_lock *mmrange)
> {
> if (mm->map_count >= sysctl_max_map_count)
> return -ENOMEM;
>
> - return __split_vma(mm, vma, addr, new_below);
> + return __split_vma(mm, vma, addr, new_below, mmrange);
> }
>
> /* Munmap is split into 2 main parts -- this part which finds
> @@ -2644,7 +2655,7 @@ int split_vma(struct mm_struct *mm, struct vm_area_struct *vma,
> * Jeremy Fitzhardinge <jeremy@...p.org>
> */
> int do_munmap(struct mm_struct *mm, unsigned long start, size_t len,
> - struct list_head *uf)
> + struct list_head *uf, struct range_lock *mmrange)
> {
> unsigned long end;
> struct vm_area_struct *vma, *prev, *last;
> @@ -2686,7 +2697,7 @@ int do_munmap(struct mm_struct *mm, unsigned long start, size_t len,
> if (end < vma->vm_end && mm->map_count >= sysctl_max_map_count)
> return -ENOMEM;
>
> - error = __split_vma(mm, vma, start, 0);
> + error = __split_vma(mm, vma, start, 0, mmrange);
> if (error)
> return error;
> prev = vma;
> @@ -2695,7 +2706,7 @@ int do_munmap(struct mm_struct *mm, unsigned long start, size_t len,
> /* Does it split the last one? */
> last = find_vma(mm, end);
> if (last && end > last->vm_start) {
> - int error = __split_vma(mm, last, end, 1);
> + int error = __split_vma(mm, last, end, 1, mmrange);
> if (error)
> return error;
> }
> @@ -2736,7 +2747,7 @@ int do_munmap(struct mm_struct *mm, unsigned long start, size_t len,
> detach_vmas_to_be_unmapped(mm, vma, prev, end);
> unmap_region(mm, vma, prev, start, end);
>
> - arch_unmap(mm, vma, start, end);
> + arch_unmap(mm, vma, start, end, mmrange);
>
> /* Fix up all other VM information */
> remove_vma_list(mm, vma);
> @@ -2749,11 +2760,12 @@ int vm_munmap(unsigned long start, size_t len)
> int ret;
> struct mm_struct *mm = current->mm;
> LIST_HEAD(uf);
> + DEFINE_RANGE_LOCK_FULL(mmrange);
>
> if (down_write_killable(&mm->mmap_sem))
> return -EINTR;
>
> - ret = do_munmap(mm, start, len, &uf);
> + ret = do_munmap(mm, start, len, &uf, &mmrange);
> up_write(&mm->mmap_sem);
> userfaultfd_unmap_complete(mm, &uf);
> return ret;
> @@ -2779,6 +2791,7 @@ SYSCALL_DEFINE5(remap_file_pages, unsigned long, start, unsigned long, size,
> unsigned long populate = 0;
> unsigned long ret = -EINVAL;
> struct file *file;
> + DEFINE_RANGE_LOCK_FULL(mmrange);
>
> pr_warn_once("%s (%d) uses deprecated remap_file_pages() syscall. See Documentation/vm/remap_file_pages.txt.\n",
> current->comm, current->pid);
> @@ -2855,7 +2868,7 @@ SYSCALL_DEFINE5(remap_file_pages, unsigned long, start, unsigned long, size,
>
> file = get_file(vma->vm_file);
> ret = do_mmap_pgoff(vma->vm_file, start, size,
> - prot, flags, pgoff, &populate, NULL);
> + prot, flags, pgoff, &populate, NULL, &mmrange);
> fput(file);
> out:
> up_write(&mm->mmap_sem);
> @@ -2881,7 +2894,9 @@ static inline void verify_mm_writelocked(struct mm_struct *mm)
> * anonymous maps. eventually we may be able to do some
> * brk-specific accounting here.
> */
> -static int do_brk_flags(unsigned long addr, unsigned long request, unsigned long flags, struct list_head *uf)
> +static int do_brk_flags(unsigned long addr, unsigned long request,
> + unsigned long flags, struct list_head *uf,
> + struct range_lock *mmrange)
> {
> struct mm_struct *mm = current->mm;
> struct vm_area_struct *vma, *prev;
> @@ -2920,7 +2935,7 @@ static int do_brk_flags(unsigned long addr, unsigned long request, unsigned long
> */
> while (find_vma_links(mm, addr, addr + len, &prev, &rb_link,
> &rb_parent)) {
> - if (do_munmap(mm, addr, len, uf))
> + if (do_munmap(mm, addr, len, uf, mmrange))
> return -ENOMEM;
> }
>
> @@ -2936,7 +2951,7 @@ static int do_brk_flags(unsigned long addr, unsigned long request, unsigned long
>
> /* Can we just expand an old private anonymous mapping? */
> vma = vma_merge(mm, prev, addr, addr + len, flags,
> - NULL, NULL, pgoff, NULL, NULL_VM_UFFD_CTX);
> + NULL, NULL, pgoff, NULL, NULL_VM_UFFD_CTX, mmrange);
> if (vma)
> goto out;
>
> @@ -2967,9 +2982,10 @@ static int do_brk_flags(unsigned long addr, unsigned long request, unsigned long
> return 0;
> }
>
> -static int do_brk(unsigned long addr, unsigned long len, struct list_head *uf)
> +static int do_brk(unsigned long addr, unsigned long len, struct list_head *uf,
> + struct range_lock *mmrange)
> {
> - return do_brk_flags(addr, len, 0, uf);
> + return do_brk_flags(addr, len, 0, uf, mmrange);
> }
>
> int vm_brk_flags(unsigned long addr, unsigned long len, unsigned long flags)
> @@ -2978,11 +2994,12 @@ int vm_brk_flags(unsigned long addr, unsigned long len, unsigned long flags)
> int ret;
> bool populate;
> LIST_HEAD(uf);
> + DEFINE_RANGE_LOCK_FULL(mmrange);
>
> if (down_write_killable(&mm->mmap_sem))
> return -EINTR;
>
> - ret = do_brk_flags(addr, len, flags, &uf);
> + ret = do_brk_flags(addr, len, flags, &uf, &mmrange);
> populate = ((mm->def_flags & VM_LOCKED) != 0);
> up_write(&mm->mmap_sem);
> userfaultfd_unmap_complete(mm, &uf);
> @@ -3105,7 +3122,7 @@ int insert_vm_struct(struct mm_struct *mm, struct vm_area_struct *vma)
> */
> struct vm_area_struct *copy_vma(struct vm_area_struct **vmap,
> unsigned long addr, unsigned long len, pgoff_t pgoff,
> - bool *need_rmap_locks)
> + bool *need_rmap_locks, struct range_lock *mmrange)
> {
> struct vm_area_struct *vma = *vmap;
> unsigned long vma_start = vma->vm_start;
> @@ -3127,7 +3144,7 @@ struct vm_area_struct *copy_vma(struct vm_area_struct **vmap,
> return NULL; /* should never get here */
> new_vma = vma_merge(mm, prev, addr, addr + len, vma->vm_flags,
> vma->anon_vma, vma->vm_file, pgoff, vma_policy(vma),
> - vma->vm_userfaultfd_ctx);
> + vma->vm_userfaultfd_ctx, mmrange);
> if (new_vma) {
> /*
> * Source vma may have been merged into new_vma
> diff --git a/mm/mprotect.c b/mm/mprotect.c
> index e3309fcf586b..b84a70720319 100644
> --- a/mm/mprotect.c
> +++ b/mm/mprotect.c
> @@ -299,7 +299,8 @@ unsigned long change_protection(struct vm_area_struct *vma, unsigned long start,
>
> int
> mprotect_fixup(struct vm_area_struct *vma, struct vm_area_struct **pprev,
> - unsigned long start, unsigned long end, unsigned long newflags)
> + unsigned long start, unsigned long end, unsigned long newflags,
> + struct range_lock *mmrange)
> {
> struct mm_struct *mm = vma->vm_mm;
> unsigned long oldflags = vma->vm_flags;
> @@ -340,7 +341,7 @@ mprotect_fixup(struct vm_area_struct *vma, struct vm_area_struct **pprev,
> pgoff = vma->vm_pgoff + ((start - vma->vm_start) >> PAGE_SHIFT);
> *pprev = vma_merge(mm, *pprev, start, end, newflags,
> vma->anon_vma, vma->vm_file, pgoff, vma_policy(vma),
> - vma->vm_userfaultfd_ctx);
> + vma->vm_userfaultfd_ctx, mmrange);
> if (*pprev) {
> vma = *pprev;
> VM_WARN_ON((vma->vm_flags ^ newflags) & ~VM_SOFTDIRTY);
> @@ -350,13 +351,13 @@ mprotect_fixup(struct vm_area_struct *vma, struct vm_area_struct **pprev,
> *pprev = vma;
>
> if (start != vma->vm_start) {
> - error = split_vma(mm, vma, start, 1);
> + error = split_vma(mm, vma, start, 1, mmrange);
> if (error)
> goto fail;
> }
>
> if (end != vma->vm_end) {
> - error = split_vma(mm, vma, end, 0);
> + error = split_vma(mm, vma, end, 0, mmrange);
> if (error)
> goto fail;
> }
> @@ -379,7 +380,7 @@ mprotect_fixup(struct vm_area_struct *vma, struct vm_area_struct **pprev,
> */
> if ((oldflags & (VM_WRITE | VM_SHARED | VM_LOCKED)) == VM_LOCKED &&
> (newflags & VM_WRITE)) {
> - populate_vma_page_range(vma, start, end, NULL);
> + populate_vma_page_range(vma, start, end, NULL, mmrange);
> }
>
> vm_stat_account(mm, oldflags, -nrpages);
> @@ -404,6 +405,7 @@ static int do_mprotect_pkey(unsigned long start, size_t len,
> const int grows = prot & (PROT_GROWSDOWN|PROT_GROWSUP);
> const bool rier = (current->personality & READ_IMPLIES_EXEC) &&
> (prot & PROT_READ);
> + DEFINE_RANGE_LOCK_FULL(mmrange);
>
> prot &= ~(PROT_GROWSDOWN|PROT_GROWSUP);
> if (grows == (PROT_GROWSDOWN|PROT_GROWSUP)) /* can't be both */
> @@ -494,7 +496,7 @@ static int do_mprotect_pkey(unsigned long start, size_t len,
> tmp = vma->vm_end;
> if (tmp > end)
> tmp = end;
> - error = mprotect_fixup(vma, &prev, nstart, tmp, newflags);
> + error = mprotect_fixup(vma, &prev, nstart, tmp, newflags, &mmrange);
> if (error)
> goto out;
> nstart = tmp;
> diff --git a/mm/mremap.c b/mm/mremap.c
> index 049470aa1e3e..21a9e2a2baa2 100644
> --- a/mm/mremap.c
> +++ b/mm/mremap.c
> @@ -264,7 +264,8 @@ static unsigned long move_vma(struct vm_area_struct *vma,
> unsigned long old_addr, unsigned long old_len,
> unsigned long new_len, unsigned long new_addr,
> bool *locked, struct vm_userfaultfd_ctx *uf,
> - struct list_head *uf_unmap)
> + struct list_head *uf_unmap,
> + struct range_lock *mmrange)
> {
> struct mm_struct *mm = vma->vm_mm;
> struct vm_area_struct *new_vma;
> @@ -292,13 +293,13 @@ static unsigned long move_vma(struct vm_area_struct *vma,
> * so KSM can come around to merge on vma and new_vma afterwards.
> */
> err = ksm_madvise(vma, old_addr, old_addr + old_len,
> - MADV_UNMERGEABLE, &vm_flags);
> + MADV_UNMERGEABLE, &vm_flags, mmrange);
> if (err)
> return err;
>
> new_pgoff = vma->vm_pgoff + ((old_addr - vma->vm_start) >> PAGE_SHIFT);
> new_vma = copy_vma(&vma, new_addr, new_len, new_pgoff,
> - &need_rmap_locks);
> + &need_rmap_locks, mmrange);
> if (!new_vma)
> return -ENOMEM;
>
> @@ -353,7 +354,7 @@ static unsigned long move_vma(struct vm_area_struct *vma,
> if (unlikely(vma->vm_flags & VM_PFNMAP))
> untrack_pfn_moved(vma);
>
> - if (do_munmap(mm, old_addr, old_len, uf_unmap) < 0) {
> + if (do_munmap(mm, old_addr, old_len, uf_unmap, mmrange) < 0) {
> /* OOM: unable to split vma, just get accounts right */
> vm_unacct_memory(excess >> PAGE_SHIFT);
> excess = 0;
> @@ -444,7 +445,8 @@ static unsigned long mremap_to(unsigned long addr, unsigned long old_len,
> unsigned long new_addr, unsigned long new_len, bool *locked,
> struct vm_userfaultfd_ctx *uf,
> struct list_head *uf_unmap_early,
> - struct list_head *uf_unmap)
> + struct list_head *uf_unmap,
> + struct range_lock *mmrange)
> {
> struct mm_struct *mm = current->mm;
> struct vm_area_struct *vma;
> @@ -462,12 +464,13 @@ static unsigned long mremap_to(unsigned long addr, unsigned long old_len,
> if (addr + old_len > new_addr && new_addr + new_len > addr)
> goto out;
>
> - ret = do_munmap(mm, new_addr, new_len, uf_unmap_early);
> + ret = do_munmap(mm, new_addr, new_len, uf_unmap_early, mmrange);
> if (ret)
> goto out;
>
> if (old_len >= new_len) {
> - ret = do_munmap(mm, addr+new_len, old_len - new_len, uf_unmap);
> + ret = do_munmap(mm, addr+new_len, old_len - new_len,
> + uf_unmap, mmrange);
> if (ret && old_len != new_len)
> goto out;
> old_len = new_len;
> @@ -490,7 +493,7 @@ static unsigned long mremap_to(unsigned long addr, unsigned long old_len,
> goto out1;
>
> ret = move_vma(vma, addr, old_len, new_len, new_addr, locked, uf,
> - uf_unmap);
> + uf_unmap, mmrange);
> if (!(offset_in_page(ret)))
> goto out;
> out1:
> @@ -532,6 +535,7 @@ SYSCALL_DEFINE5(mremap, unsigned long, addr, unsigned long, old_len,
> struct vm_userfaultfd_ctx uf = NULL_VM_UFFD_CTX;
> LIST_HEAD(uf_unmap_early);
> LIST_HEAD(uf_unmap);
> + DEFINE_RANGE_LOCK_FULL(mmrange);
>
> if (flags & ~(MREMAP_FIXED | MREMAP_MAYMOVE))
> return ret;
> @@ -558,7 +562,8 @@ SYSCALL_DEFINE5(mremap, unsigned long, addr, unsigned long, old_len,
>
> if (flags & MREMAP_FIXED) {
> ret = mremap_to(addr, old_len, new_addr, new_len,
> - &locked, &uf, &uf_unmap_early, &uf_unmap);
> + &locked, &uf, &uf_unmap_early,
> + &uf_unmap, &mmrange);
> goto out;
> }
>
> @@ -568,7 +573,8 @@ SYSCALL_DEFINE5(mremap, unsigned long, addr, unsigned long, old_len,
> * do_munmap does all the needed commit accounting
> */
> if (old_len >= new_len) {
> - ret = do_munmap(mm, addr+new_len, old_len - new_len, &uf_unmap);
> + ret = do_munmap(mm, addr+new_len, old_len - new_len,
> + &uf_unmap, &mmrange);
> if (ret && old_len != new_len)
> goto out;
> ret = addr;
> @@ -592,7 +598,7 @@ SYSCALL_DEFINE5(mremap, unsigned long, addr, unsigned long, old_len,
> int pages = (new_len - old_len) >> PAGE_SHIFT;
>
> if (vma_adjust(vma, vma->vm_start, addr + new_len,
> - vma->vm_pgoff, NULL)) {
> + vma->vm_pgoff, NULL, &mmrange)) {
> ret = -ENOMEM;
> goto out;
> }
> @@ -628,7 +634,7 @@ SYSCALL_DEFINE5(mremap, unsigned long, addr, unsigned long, old_len,
> }
>
> ret = move_vma(vma, addr, old_len, new_len, new_addr,
> - &locked, &uf, &uf_unmap);
> + &locked, &uf, &uf_unmap, &mmrange);
> }
> out:
> if (offset_in_page(ret)) {
> diff --git a/mm/nommu.c b/mm/nommu.c
> index ebb6e618dade..1805f0a788b3 100644
> --- a/mm/nommu.c
> +++ b/mm/nommu.c
> @@ -113,7 +113,8 @@ unsigned int kobjsize(const void *objp)
> static long __get_user_pages(struct task_struct *tsk, struct mm_struct *mm,
> unsigned long start, unsigned long nr_pages,
> unsigned int foll_flags, struct page **pages,
> - struct vm_area_struct **vmas, int *nonblocking)
> + struct vm_area_struct **vmas, int *nonblocking,
> + struct range_lock *mmrange)
> {
> struct vm_area_struct *vma;
> unsigned long vm_flags;
> @@ -162,18 +163,19 @@ static long __get_user_pages(struct task_struct *tsk, struct mm_struct *mm,
> */
> long get_user_pages(unsigned long start, unsigned long nr_pages,
> unsigned int gup_flags, struct page **pages,
> - struct vm_area_struct **vmas)
> + struct vm_area_struct **vmas,
> + struct range_lock *mmrange)
> {
> return __get_user_pages(current, current->mm, start, nr_pages,
> - gup_flags, pages, vmas, NULL);
> + gup_flags, pages, vmas, NULL, mmrange);
> }
> EXPORT_SYMBOL(get_user_pages);
>
> long get_user_pages_locked(unsigned long start, unsigned long nr_pages,
> unsigned int gup_flags, struct page **pages,
> - int *locked)
> + int *locked, struct range_lock *mmrange)
> {
> - return get_user_pages(start, nr_pages, gup_flags, pages, NULL);
> + return get_user_pages(start, nr_pages, gup_flags, pages, NULL, mmrange);
> }
> EXPORT_SYMBOL(get_user_pages_locked);
>
> @@ -183,9 +185,11 @@ static long __get_user_pages_unlocked(struct task_struct *tsk,
> unsigned int gup_flags)
> {
> long ret;
> + DEFINE_RANGE_LOCK_FULL(mmrange);
> +
> down_read(&mm->mmap_sem);
> ret = __get_user_pages(tsk, mm, start, nr_pages, gup_flags, pages,
> - NULL, NULL);
> + NULL, NULL, &mmrange);
> up_read(&mm->mmap_sem);
> return ret;
> }
> @@ -836,7 +840,8 @@ EXPORT_SYMBOL(find_vma);
> * find a VMA
> * - we don't extend stack VMAs under NOMMU conditions
> */
> -struct vm_area_struct *find_extend_vma(struct mm_struct *mm, unsigned long addr)
> +struct vm_area_struct *find_extend_vma(struct mm_struct *mm, unsigned long addr,
> + struct range_lock *mmrange)
> {
> return find_vma(mm, addr);
> }
> @@ -1206,7 +1211,8 @@ unsigned long do_mmap(struct file *file,
> vm_flags_t vm_flags,
> unsigned long pgoff,
> unsigned long *populate,
> - struct list_head *uf)
> + struct list_head *uf,
> + struct range_lock *mmrange)
> {
> struct vm_area_struct *vma;
> struct vm_region *region;
> @@ -1476,7 +1482,7 @@ SYSCALL_DEFINE1(old_mmap, struct mmap_arg_struct __user *, arg)
> * for the first part or the tail.
> */
> int split_vma(struct mm_struct *mm, struct vm_area_struct *vma,
> - unsigned long addr, int new_below)
> + unsigned long addr, int new_below, struct range_lock *mmrange)
> {
> struct vm_area_struct *new;
> struct vm_region *region;
> @@ -1578,7 +1584,8 @@ static int shrink_vma(struct mm_struct *mm,
> * - under NOMMU conditions the chunk to be unmapped must be backed by a single
> * VMA, though it need not cover the whole VMA
> */
> -int do_munmap(struct mm_struct *mm, unsigned long start, size_t len, struct list_head *uf)
> +int do_munmap(struct mm_struct *mm, unsigned long start, size_t len,
> + struct list_head *uf, struct range_lock *mmrange)
> {
> struct vm_area_struct *vma;
> unsigned long end;
> @@ -1624,7 +1631,7 @@ int do_munmap(struct mm_struct *mm, unsigned long start, size_t len, struct list
> if (end != vma->vm_end && offset_in_page(end))
> return -EINVAL;
> if (start != vma->vm_start && end != vma->vm_end) {
> - ret = split_vma(mm, vma, start, 1);
> + ret = split_vma(mm, vma, start, 1, mmrange);
> if (ret < 0)
> return ret;
> }
> @@ -1642,9 +1649,10 @@ int vm_munmap(unsigned long addr, size_t len)
> {
> struct mm_struct *mm = current->mm;
> int ret;
> + DEFINE_RANGE_LOCK_FULL(mmrange);
>
> down_write(&mm->mmap_sem);
> - ret = do_munmap(mm, addr, len, NULL);
> + ret = do_munmap(mm, addr, len, NULL, &mmrange);
> up_write(&mm->mmap_sem);
> return ret;
> }
> diff --git a/mm/pagewalk.c b/mm/pagewalk.c
> index 8d2da5dec1e0..44a2507c94fd 100644
> --- a/mm/pagewalk.c
> +++ b/mm/pagewalk.c
> @@ -26,7 +26,7 @@ static int walk_pte_range(pmd_t *pmd, unsigned long addr, unsigned long end,
> }
>
> static int walk_pmd_range(pud_t *pud, unsigned long addr, unsigned long end,
> - struct mm_walk *walk)
> + struct mm_walk *walk, struct range_lock *mmrange)
> {
> pmd_t *pmd;
> unsigned long next;
> @@ -38,7 +38,7 @@ static int walk_pmd_range(pud_t *pud, unsigned long addr, unsigned long end,
> next = pmd_addr_end(addr, end);
> if (pmd_none(*pmd) || !walk->vma) {
> if (walk->pte_hole)
> - err = walk->pte_hole(addr, next, walk);
> + err = walk->pte_hole(addr, next, walk, mmrange);
> if (err)
> break;
> continue;
> @@ -48,7 +48,7 @@ static int walk_pmd_range(pud_t *pud, unsigned long addr, unsigned long end,
> * needs to know about pmd_trans_huge() pmds
> */
> if (walk->pmd_entry)
> - err = walk->pmd_entry(pmd, addr, next, walk);
> + err = walk->pmd_entry(pmd, addr, next, walk, mmrange);
> if (err)
> break;
>
> @@ -71,7 +71,7 @@ static int walk_pmd_range(pud_t *pud, unsigned long addr, unsigned long end,
> }
>
> static int walk_pud_range(p4d_t *p4d, unsigned long addr, unsigned long end,
> - struct mm_walk *walk)
> + struct mm_walk *walk, struct range_lock *mmrange)
> {
> pud_t *pud;
> unsigned long next;
> @@ -83,7 +83,7 @@ static int walk_pud_range(p4d_t *p4d, unsigned long addr, unsigned long end,
> next = pud_addr_end(addr, end);
> if (pud_none(*pud) || !walk->vma) {
> if (walk->pte_hole)
> - err = walk->pte_hole(addr, next, walk);
> + err = walk->pte_hole(addr, next, walk, mmrange);
> if (err)
> break;
> continue;
> @@ -106,7 +106,7 @@ static int walk_pud_range(p4d_t *p4d, unsigned long addr, unsigned long end,
> goto again;
>
> if (walk->pmd_entry || walk->pte_entry)
> - err = walk_pmd_range(pud, addr, next, walk);
> + err = walk_pmd_range(pud, addr, next, walk, mmrange);
> if (err)
> break;
> } while (pud++, addr = next, addr != end);
> @@ -115,7 +115,7 @@ static int walk_pud_range(p4d_t *p4d, unsigned long addr, unsigned long end,
> }
>
> static int walk_p4d_range(pgd_t *pgd, unsigned long addr, unsigned long end,
> - struct mm_walk *walk)
> + struct mm_walk *walk, struct range_lock *mmrange)
> {
> p4d_t *p4d;
> unsigned long next;
> @@ -126,13 +126,13 @@ static int walk_p4d_range(pgd_t *pgd, unsigned long addr, unsigned long end,
> next = p4d_addr_end(addr, end);
> if (p4d_none_or_clear_bad(p4d)) {
> if (walk->pte_hole)
> - err = walk->pte_hole(addr, next, walk);
> + err = walk->pte_hole(addr, next, walk, mmrange);
> if (err)
> break;
> continue;
> }
> if (walk->pmd_entry || walk->pte_entry)
> - err = walk_pud_range(p4d, addr, next, walk);
> + err = walk_pud_range(p4d, addr, next, walk, mmrange);
> if (err)
> break;
> } while (p4d++, addr = next, addr != end);
> @@ -141,7 +141,7 @@ static int walk_p4d_range(pgd_t *pgd, unsigned long addr, unsigned long end,
> }
>
> static int walk_pgd_range(unsigned long addr, unsigned long end,
> - struct mm_walk *walk)
> + struct mm_walk *walk, struct range_lock *mmrange)
> {
> pgd_t *pgd;
> unsigned long next;
> @@ -152,13 +152,13 @@ static int walk_pgd_range(unsigned long addr, unsigned long end,
> next = pgd_addr_end(addr, end);
> if (pgd_none_or_clear_bad(pgd)) {
> if (walk->pte_hole)
> - err = walk->pte_hole(addr, next, walk);
> + err = walk->pte_hole(addr, next, walk, mmrange);
> if (err)
> break;
> continue;
> }
> if (walk->pmd_entry || walk->pte_entry)
> - err = walk_p4d_range(pgd, addr, next, walk);
> + err = walk_p4d_range(pgd, addr, next, walk, mmrange);
> if (err)
> break;
> } while (pgd++, addr = next, addr != end);
> @@ -175,7 +175,7 @@ static unsigned long hugetlb_entry_end(struct hstate *h, unsigned long addr,
> }
>
> static int walk_hugetlb_range(unsigned long addr, unsigned long end,
> - struct mm_walk *walk)
> + struct mm_walk *walk, struct range_lock *mmrange)
> {
> struct vm_area_struct *vma = walk->vma;
> struct hstate *h = hstate_vma(vma);
> @@ -192,7 +192,7 @@ static int walk_hugetlb_range(unsigned long addr, unsigned long end,
> if (pte)
> err = walk->hugetlb_entry(pte, hmask, addr, next, walk);
> else if (walk->pte_hole)
> - err = walk->pte_hole(addr, next, walk);
> + err = walk->pte_hole(addr, next, walk, mmrange);
>
> if (err)
> break;
> @@ -203,7 +203,7 @@ static int walk_hugetlb_range(unsigned long addr, unsigned long end,
>
> #else /* CONFIG_HUGETLB_PAGE */
> static int walk_hugetlb_range(unsigned long addr, unsigned long end,
> - struct mm_walk *walk)
> + struct mm_walk *walk, struct range_lock *mmrange)
> {
> return 0;
> }
> @@ -217,7 +217,7 @@ static int walk_hugetlb_range(unsigned long addr, unsigned long end,
> * error, where we abort the current walk.
> */
> static int walk_page_test(unsigned long start, unsigned long end,
> - struct mm_walk *walk)
> + struct mm_walk *walk, struct range_lock *mmrange)
> {
> struct vm_area_struct *vma = walk->vma;
>
> @@ -235,23 +235,23 @@ static int walk_page_test(unsigned long start, unsigned long end,
> if (vma->vm_flags & VM_PFNMAP) {
> int err = 1;
> if (walk->pte_hole)
> - err = walk->pte_hole(start, end, walk);
> + err = walk->pte_hole(start, end, walk, mmrange);
> return err ? err : 1;
> }
> return 0;
> }
>
> static int __walk_page_range(unsigned long start, unsigned long end,
> - struct mm_walk *walk)
> + struct mm_walk *walk, struct range_lock *mmrange)
> {
> int err = 0;
> struct vm_area_struct *vma = walk->vma;
>
> if (vma && is_vm_hugetlb_page(vma)) {
> if (walk->hugetlb_entry)
> - err = walk_hugetlb_range(start, end, walk);
> + err = walk_hugetlb_range(start, end, walk, mmrange);
> } else
> - err = walk_pgd_range(start, end, walk);
> + err = walk_pgd_range(start, end, walk, mmrange);
>
> return err;
> }
> @@ -285,10 +285,11 @@ static int __walk_page_range(unsigned long start, unsigned long end,
> * Locking:
> * Callers of walk_page_range() and walk_page_vma() should hold
> * @walk->mm->mmap_sem, because these function traverse vma list and/or
> - * access to vma's data.
> + * access to vma's data. As such, the @mmrange will represent the
> + * address space range.
> */
> int walk_page_range(unsigned long start, unsigned long end,
> - struct mm_walk *walk)
> + struct mm_walk *walk, struct range_lock *mmrange)
> {
> int err = 0;
> unsigned long next;
> @@ -315,7 +316,7 @@ int walk_page_range(unsigned long start, unsigned long end,
> next = min(end, vma->vm_end);
> vma = vma->vm_next;
>
> - err = walk_page_test(start, next, walk);
> + err = walk_page_test(start, next, walk, mmrange);
> if (err > 0) {
> /*
> * positive return values are purely for
> @@ -329,14 +330,15 @@ int walk_page_range(unsigned long start, unsigned long end,
> break;
> }
> if (walk->vma || walk->pte_hole)
> - err = __walk_page_range(start, next, walk);
> + err = __walk_page_range(start, next, walk, mmrange);
> if (err)
> break;
> } while (start = next, start < end);
> return err;
> }
>
> -int walk_page_vma(struct vm_area_struct *vma, struct mm_walk *walk)
> +int walk_page_vma(struct vm_area_struct *vma, struct mm_walk *walk,
> + struct range_lock *mmrange)
> {
> int err;
>
> @@ -346,10 +348,10 @@ int walk_page_vma(struct vm_area_struct *vma, struct mm_walk *walk)
> VM_BUG_ON(!rwsem_is_locked(&walk->mm->mmap_sem));
> VM_BUG_ON(!vma);
> walk->vma = vma;
> - err = walk_page_test(vma->vm_start, vma->vm_end, walk);
> + err = walk_page_test(vma->vm_start, vma->vm_end, walk, mmrange);
> if (err > 0)
> return 0;
> if (err < 0)
> return err;
> - return __walk_page_range(vma->vm_start, vma->vm_end, walk);
> + return __walk_page_range(vma->vm_start, vma->vm_end, walk, mmrange);
> }
> diff --git a/mm/process_vm_access.c b/mm/process_vm_access.c
> index a447092d4635..ff6772b86195 100644
> --- a/mm/process_vm_access.c
> +++ b/mm/process_vm_access.c
> @@ -90,6 +90,7 @@ static int process_vm_rw_single_vec(unsigned long addr,
> unsigned long max_pages_per_loop = PVM_MAX_KMALLOC_PAGES
> / sizeof(struct pages *);
> unsigned int flags = 0;
> + DEFINE_RANGE_LOCK_FULL(mmrange);
>
> /* Work out address and page range required */
> if (len == 0)
> @@ -111,7 +112,8 @@ static int process_vm_rw_single_vec(unsigned long addr,
> */
> down_read(&mm->mmap_sem);
> pages = get_user_pages_remote(task, mm, pa, pages, flags,
> - process_pages, NULL, &locked);
> + process_pages, NULL, &locked,
> + &mmrange);
> if (locked)
> up_read(&mm->mmap_sem);
> if (pages <= 0)
> diff --git a/mm/util.c b/mm/util.c
> index c1250501364f..b0ec1d88bb71 100644
> --- a/mm/util.c
> +++ b/mm/util.c
> @@ -347,13 +347,14 @@ unsigned long vm_mmap_pgoff(struct file *file, unsigned long addr,
> struct mm_struct *mm = current->mm;
> unsigned long populate;
> LIST_HEAD(uf);
> + DEFINE_RANGE_LOCK_FULL(mmrange);
>
> ret = security_mmap_file(file, prot, flag);
> if (!ret) {
> if (down_write_killable(&mm->mmap_sem))
> return -EINTR;
> ret = do_mmap_pgoff(file, addr, len, prot, flag, pgoff,
> - &populate, &uf);
> + &populate, &uf, &mmrange);
> up_write(&mm->mmap_sem);
> userfaultfd_unmap_complete(mm, &uf);
> if (populate)
> diff --git a/security/tomoyo/domain.c b/security/tomoyo/domain.c
> index f6758dad981f..c1e36ea2c6fc 100644
> --- a/security/tomoyo/domain.c
> +++ b/security/tomoyo/domain.c
> @@ -868,6 +868,7 @@ bool tomoyo_dump_page(struct linux_binprm *bprm, unsigned long pos,
> struct tomoyo_page_dump *dump)
> {
> struct page *page;
> + DEFINE_RANGE_LOCK_FULL(mmrange); /* see get_page_arg() in fs/exec.c */
>
> /* dump->data is released by tomoyo_find_next_domain(). */
> if (!dump->data) {
> @@ -884,7 +885,7 @@ bool tomoyo_dump_page(struct linux_binprm *bprm, unsigned long pos,
> * the execve().
> */
> if (get_user_pages_remote(current, bprm->mm, pos, 1,
> - FOLL_FORCE, &page, NULL, NULL) <= 0)
> + FOLL_FORCE, &page, NULL, NULL, &mmrange) <= 0)
> return false;
> #else
> page = bprm->page[pos / PAGE_SIZE];
> diff --git a/virt/kvm/async_pf.c b/virt/kvm/async_pf.c
> index 57bcb27dcf30..4cd2b93bb20c 100644
> --- a/virt/kvm/async_pf.c
> +++ b/virt/kvm/async_pf.c
> @@ -78,6 +78,7 @@ static void async_pf_execute(struct work_struct *work)
> unsigned long addr = apf->addr;
> gva_t gva = apf->gva;
> int locked = 1;
> + DEFINE_RANGE_LOCK_FULL(mmrange);
>
> might_sleep();
>
> @@ -88,7 +89,7 @@ static void async_pf_execute(struct work_struct *work)
> */
> down_read(&mm->mmap_sem);
> get_user_pages_remote(NULL, mm, addr, 1, FOLL_WRITE, NULL, NULL,
> - &locked);
> + &locked, &mmrange);
> if (locked)
> up_read(&mm->mmap_sem);
>
> diff --git a/virt/kvm/kvm_main.c b/virt/kvm/kvm_main.c
> index 4501e658e8d6..86ec078f4c3b 100644
> --- a/virt/kvm/kvm_main.c
> +++ b/virt/kvm/kvm_main.c
> @@ -1317,11 +1317,12 @@ unsigned long kvm_vcpu_gfn_to_hva_prot(struct kvm_vcpu *vcpu, gfn_t gfn, bool *w
> return gfn_to_hva_memslot_prot(slot, gfn, writable);
> }
>
> -static inline int check_user_page_hwpoison(unsigned long addr)
> +static inline int check_user_page_hwpoison(unsigned long addr,
> + struct range_lock *mmrange)
> {
> int rc, flags = FOLL_HWPOISON | FOLL_WRITE;
>
> - rc = get_user_pages(addr, 1, flags, NULL, NULL);
> + rc = get_user_pages(addr, 1, flags, NULL, NULL, mmrange);
> return rc == -EHWPOISON;
> }
>
> @@ -1411,7 +1412,8 @@ static bool vma_is_valid(struct vm_area_struct *vma, bool write_fault)
> static int hva_to_pfn_remapped(struct vm_area_struct *vma,
> unsigned long addr, bool *async,
> bool write_fault, bool *writable,
> - kvm_pfn_t *p_pfn)
> + kvm_pfn_t *p_pfn,
> + struct range_lock *mmrange)
> {
> unsigned long pfn;
> int r;
> @@ -1425,7 +1427,7 @@ static int hva_to_pfn_remapped(struct vm_area_struct *vma,
> bool unlocked = false;
> r = fixup_user_fault(current, current->mm, addr,
> (write_fault ? FAULT_FLAG_WRITE : 0),
> - &unlocked);
> + &unlocked, mmrange);
> if (unlocked)
> return -EAGAIN;
> if (r)
> @@ -1477,6 +1479,7 @@ static kvm_pfn_t hva_to_pfn(unsigned long addr, bool atomic, bool *async,
> struct vm_area_struct *vma;
> kvm_pfn_t pfn = 0;
> int npages, r;
> + DEFINE_RANGE_LOCK_FULL(mmrange);
>
> /* we can do it either atomically or asynchronously, not both */
> BUG_ON(atomic && async);
> @@ -1493,7 +1496,7 @@ static kvm_pfn_t hva_to_pfn(unsigned long addr, bool atomic, bool *async,
>
> down_read(¤t->mm->mmap_sem);
> if (npages == -EHWPOISON ||
> - (!async && check_user_page_hwpoison(addr))) {
> + (!async && check_user_page_hwpoison(addr, &mmrange))) {
> pfn = KVM_PFN_ERR_HWPOISON;
> goto exit;
> }
> @@ -1504,7 +1507,8 @@ static kvm_pfn_t hva_to_pfn(unsigned long addr, bool atomic, bool *async,
> if (vma == NULL)
> pfn = KVM_PFN_ERR_FAULT;
> else if (vma->vm_flags & (VM_IO | VM_PFNMAP)) {
> - r = hva_to_pfn_remapped(vma, addr, async, write_fault, writable, &pfn);
> + r = hva_to_pfn_remapped(vma, addr, async, write_fault, writable,
> + &pfn, &mmrange);
> if (r == -EAGAIN)
> goto retry;
> if (r < 0)
>
Powered by blists - more mailing lists