--- linux-2.6.22-rc3/fs/proc/task_mmu.c.orig 2007-05-29 21:05:31.000000000 -0700 +++ linux-2.6.22-rc3/fs/proc/task_mmu.c 2007-05-30 08:17:44.000000000 -0700 @@ -31,7 +31,7 @@ if (hiwater_rss < mm->hiwater_rss) hiwater_rss = mm->hiwater_rss; - data = mm->total_vm - mm->shared_vm - mm->stack_vm; + data = mm->total_vm - mm->shared_vm - mm->stack_vm - mm->exec_vm; text = (PAGE_ALIGN(mm->end_code) - (mm->start_code & PAGE_MASK)) >> 10; lib = (mm->exec_vm << (PAGE_SHIFT-10)) - text; buffer += sprintf(buffer, --- linux-2.6.22-rc3/mm/mprotect.c.orig 2007-04-25 20:08:32.000000000 -0700 +++ linux-2.6.22-rc3/mm/mprotect.c 2007-05-31 14:20:59.000000000 -0700 @@ -162,6 +162,22 @@ } } + /* + * Check against private data size limit. When execute permission + * is removed, the mapping is counted toward the private data size. + */ + if (!(newflags & VM_EXEC) && (vma->vm_flags & VM_EXEC)) { + unsigned long rlim; + rlim = current->signal->rlim[RLIMIT_DATA].rlim_cur; + if (rlim < RLIM_INFINITY) { + unsigned long priv_vm; + priv_vm = mm->total_vm - mm->shared_vm - mm->exec_vm; + priv_vm <<= PAGE_SHIFT; + if (priv_vm + (end - start) > rlim) + return -ENOMEM; + } + } + /* * First try to merge with previous and/or next vma. */ --- linux-2.6.22-rc3/mm/mremap.c.orig 2007-04-25 20:08:32.000000000 -0700 +++ linux-2.6.22-rc3/mm/mremap.c 2007-05-31 14:20:42.000000000 -0700 @@ -343,6 +343,23 @@ goto out; } + /* + * Check against private data size limit. Count memory allocations + * which are not shared and not executable code as private data. + */ + if (!(vma->vm_flags & (VM_EXEC|VM_SHARED))) { + unsigned long rlim; + rlim = current->signal->rlim[RLIMIT_DATA].rlim_cur; + if (rlim < RLIM_INFINITY) { + unsigned long priv_vm; + struct mm_struct *mm = current->mm; + priv_vm = mm->total_vm - mm->shared_vm - mm->exec_vm; + priv_vm <<= PAGE_SHIFT; + if (priv_vm + (new_len - old_len) > rlim) + goto out; + } + } + if (vma->vm_flags & VM_ACCOUNT) { charged = (new_len - old_len) >> PAGE_SHIFT; if (security_vm_enough_memory(charged)) --- linux-2.6.22-rc3/mm/mmap.c.orig 2007-05-29 21:05:58.000000000 -0700 +++ linux-2.6.22-rc3/mm/mmap.c 2007-05-31 14:20:10.000000000 -0700 @@ -245,16 +245,6 @@ if (brk < mm->end_code) goto out; - /* - * Check against rlimit here. If this check is done later after the test - * of oldbrk with newbrk then it can escape the test and let the data - * segment grow beyond its set limit the in case where the limit is - * not page aligned -Ram Gupta - */ - rlim = current->signal->rlim[RLIMIT_DATA].rlim_cur; - if (rlim < RLIM_INFINITY && brk - mm->start_data > rlim) - goto out; - newbrk = PAGE_ALIGN(brk); oldbrk = PAGE_ALIGN(mm->brk); if (oldbrk == newbrk) @@ -267,6 +257,19 @@ goto out; } + /* + * Check against private data size limit. Count memory allocations + * which are not shared and not executable code as private data. + */ + rlim = current->signal->rlim[RLIMIT_DATA].rlim_cur; + if (rlim < RLIM_INFINITY) { + unsigned long priv_vm; + priv_vm = mm->total_vm - mm->shared_vm - mm->exec_vm; + priv_vm <<= PAGE_SHIFT; + if (priv_vm + (newbrk-oldbrk) > rlim) + goto out; + } + /* Check against existing mmap mappings. */ if (find_vma_intersection(mm, oldbrk, newbrk+PAGE_SIZE)) goto out; @@ -875,7 +878,8 @@ = VM_STACK_FLAGS & (VM_GROWSUP|VM_GROWSDOWN); if (file) { - mm->shared_vm += pages; + if (flags & VM_SHARED) + mm->shared_vm += pages; if ((flags & (VM_EXEC|VM_WRITE)) == VM_EXEC) mm->exec_vm += pages; } else if (flags & stack_flags) @@ -1041,6 +1045,21 @@ if (!may_expand_vm(mm, len >> PAGE_SHIFT)) return -ENOMEM; + /* + * Check against private data size limit. Count memory allocations + * which are not shared and not executable code as private data. + */ + if (!(vm_flags & (VM_SHARED|VM_EXEC))) { + unsigned long rlim, priv_vm; + rlim = current->signal->rlim[RLIMIT_DATA].rlim_cur; + if (rlim < RLIM_INFINITY) { + priv_vm = mm->total_vm - mm->shared_vm - mm->exec_vm; + priv_vm <<= PAGE_SHIFT; + if (priv_vm + len > rlim) + return -ENOMEM; + } + } + if (accountable && (!(flags & MAP_NORESERVE) || sysctl_overcommit_memory == OVERCOMMIT_NEVER)) { if (vm_flags & VM_SHARED) {