| lists.openwall.net | lists / announce owl-users owl-dev john-users john-dev passwdqc-users yescrypt popa3d-users / oss-security kernel-hardening musl sabotage tlsify passwords / crypt-dev xvendor / Bugtraq Full-Disclosure linux-kernel linux-netdev linux-ext4 linux-hardening linux-cve-announce PHC | |
|
Open Source and information security mailing list archives
| ||
|
Message-ID: <20210428153542.2814175-68-Liam.Howlett@Oracle.com>
Date: Wed, 28 Apr 2021 15:36:21 +0000
From: Liam Howlett <liam.howlett@...cle.com>
To: "maple-tree@...ts.infradead.org" <maple-tree@...ts.infradead.org>,
"linux-mm@...ck.org" <linux-mm@...ck.org>,
"linux-kernel@...r.kernel.org" <linux-kernel@...r.kernel.org>,
Andrew Morton <akpm@...ux-foundation.org>
CC: Song Liu <songliubraving@...com>,
Davidlohr Bueso <dave@...olabs.net>,
"Paul E . McKenney" <paulmck@...nel.org>,
Matthew Wilcox <willy@...radead.org>,
Laurent Dufour <ldufour@...ux.ibm.com>,
David Rientjes <rientjes@...gle.com>,
Axel Rasmussen <axelrasmussen@...gle.com>,
Suren Baghdasaryan <surenb@...gle.com>,
Vlastimil Babka <vbabka@...e.cz>,
Rik van Riel <riel@...riel.com>,
Peter Zijlstra <peterz@...radead.org>,
Michel Lespinasse <walken.cr@...il.com>,
Liam Howlett <liam.howlett@...cle.com>
Subject: [PATCH 67/94] fs/userfaultfd: Stop using vma linked list.
Don't use the mm_struct linked list or the vma->vm_next in prep for removal
Signed-off-by: Liam R. Howlett <Liam.Howlett@...cle.com>
---
fs/userfaultfd.c | 43 ++++++++++++++++++++++++++++++++++++-------
1 file changed, 36 insertions(+), 7 deletions(-)
diff --git a/fs/userfaultfd.c b/fs/userfaultfd.c
index 14f92285d04f..1fd0f5b5c934 100644
--- a/fs/userfaultfd.c
+++ b/fs/userfaultfd.c
@@ -605,14 +605,18 @@ static void userfaultfd_event_wait_completion(struct userfaultfd_ctx *ctx,
if (release_new_ctx) {
struct vm_area_struct *vma;
struct mm_struct *mm = release_new_ctx->mm;
+ MA_STATE(mas, &mm->mm_mt, 0, 0);
/* the various vma->vm_userfaultfd_ctx still points to it */
mmap_write_lock(mm);
- for (vma = mm->mmap; vma; vma = vma->vm_next)
+ mas_lock(&mas);
+ mas_for_each(&mas, vma, ULONG_MAX) {
if (vma->vm_userfaultfd_ctx.ctx == release_new_ctx) {
vma->vm_userfaultfd_ctx = NULL_VM_UFFD_CTX;
vma->vm_flags &= ~__VM_UFFD_FLAGS;
}
+ }
+ mas_unlock(&mas);
mmap_write_unlock(mm);
userfaultfd_ctx_put(release_new_ctx);
@@ -797,7 +801,10 @@ int userfaultfd_unmap_prep(struct vm_area_struct *vma,
unsigned long start, unsigned long end,
struct list_head *unmaps)
{
- for ( ; vma && vma->vm_start < end; vma = vma->vm_next) {
+ MA_STATE(mas, &vma->vm_mm->mm_mt, vma->vm_start, vma->vm_start);
+
+ rcu_read_lock();
+ mas_for_each(&mas, vma, end) {
struct userfaultfd_unmap_ctx *unmap_ctx;
struct userfaultfd_ctx *ctx = vma->vm_userfaultfd_ctx.ctx;
@@ -816,6 +823,7 @@ int userfaultfd_unmap_prep(struct vm_area_struct *vma,
unmap_ctx->end = end;
list_add_tail(&unmap_ctx->list, unmaps);
}
+ rcu_read_unlock();
return 0;
}
@@ -847,6 +855,7 @@ static int userfaultfd_release(struct inode *inode, struct file *file)
/* len == 0 means wake all */
struct userfaultfd_wake_range range = { .len = 0, };
unsigned long new_flags;
+ MA_STATE(mas, &mm->mm_mt, 0, 0);
WRITE_ONCE(ctx->released, true);
@@ -862,9 +871,14 @@ static int userfaultfd_release(struct inode *inode, struct file *file)
* taking the mmap_lock for writing.
*/
mmap_write_lock(mm);
+ mas_lock(&mas);
prev = NULL;
- for (vma = mm->mmap; vma; vma = vma->vm_next) {
+ mas_for_each(&mas, vma, ULONG_MAX) {
+ mas_unlock(&mas);
+ mas_pause(&mas);
cond_resched();
+ mas_lock(&mas);
+
BUG_ON(!!vma->vm_userfaultfd_ctx.ctx ^
!!(vma->vm_flags & __VM_UFFD_FLAGS));
if (vma->vm_userfaultfd_ctx.ctx != ctx) {
@@ -884,6 +898,7 @@ static int userfaultfd_release(struct inode *inode, struct file *file)
vma->vm_flags = new_flags;
vma->vm_userfaultfd_ctx = NULL_VM_UFFD_CTX;
}
+ mas_unlock(&mas);
mmap_write_unlock(mm);
mmput(mm);
wakeup:
@@ -1288,6 +1303,7 @@ static int userfaultfd_register(struct userfaultfd_ctx *ctx,
bool found;
bool basic_ioctls;
unsigned long start, end, vma_end;
+ MA_STATE(mas, &mm->mm_mt, 0, 0);
user_uffdio_register = (struct uffdio_register __user *) arg;
@@ -1326,6 +1342,7 @@ static int userfaultfd_register(struct userfaultfd_ctx *ctx,
goto out;
mmap_write_lock(mm);
+ rcu_read_lock();
vma = find_vma_prev(mm, start, &prev);
if (!vma)
goto out_unlock;
@@ -1351,8 +1368,12 @@ static int userfaultfd_register(struct userfaultfd_ctx *ctx,
*/
found = false;
basic_ioctls = false;
- for (cur = vma; cur && cur->vm_start < end; cur = cur->vm_next) {
+ mas_set(&mas, vma->vm_start);
+ mas_for_each(&mas, cur, end) {
+ rcu_read_unlock();
+ mas_pause(&mas);
cond_resched();
+ rcu_read_lock();
BUG_ON(!!cur->vm_userfaultfd_ctx.ctx ^
!!(cur->vm_flags & __VM_UFFD_FLAGS));
@@ -1469,9 +1490,10 @@ static int userfaultfd_register(struct userfaultfd_ctx *ctx,
skip:
prev = vma;
start = vma->vm_end;
- vma = vma->vm_next;
+ vma = vma_next(mm, vma);
} while (vma && vma->vm_start < end);
out_unlock:
+ rcu_read_unlock();
mmap_write_unlock(mm);
mmput(mm);
if (!ret) {
@@ -1514,6 +1536,7 @@ static int userfaultfd_unregister(struct userfaultfd_ctx *ctx,
bool found;
unsigned long start, end, vma_end;
const void __user *buf = (void __user *)arg;
+ MA_STATE(mas, &mm->mm_mt, 0, 0);
ret = -EFAULT;
if (copy_from_user(&uffdio_unregister, buf, sizeof(uffdio_unregister)))
@@ -1557,8 +1580,13 @@ static int userfaultfd_unregister(struct userfaultfd_ctx *ctx,
*/
found = false;
ret = -EINVAL;
- for (cur = vma; cur && cur->vm_start < end; cur = cur->vm_next) {
+ rcu_read_lock();
+ mas_set(&mas, vma->vm_start);
+ mas_for_each(&mas, cur, end) {
+ rcu_read_unlock();
+ mas_pause(&mas);
cond_resched();
+ rcu_read_lock();
BUG_ON(!!cur->vm_userfaultfd_ctx.ctx ^
!!(cur->vm_flags & __VM_UFFD_FLAGS));
@@ -1575,6 +1603,7 @@ static int userfaultfd_unregister(struct userfaultfd_ctx *ctx,
found = true;
}
+ rcu_read_unlock();
BUG_ON(!found);
if (vma->vm_start < start)
@@ -1643,7 +1672,7 @@ static int userfaultfd_unregister(struct userfaultfd_ctx *ctx,
skip:
prev = vma;
start = vma->vm_end;
- vma = vma->vm_next;
+ vma = vma_next(mm, vma);
} while (vma && vma->vm_start < end);
out_unlock:
mmap_write_unlock(mm);
--
2.30.2
Powered by blists - more mailing lists