lists.openwall.net   lists  /  announce  owl-users  owl-dev  john-users  john-dev  passwdqc-users  yescrypt  popa3d-users  /  oss-security  kernel-hardening  musl  sabotage  tlsify  passwords  /  crypt-dev  xvendor  /  Bugtraq  Full-Disclosure  linux-kernel  linux-netdev  linux-ext4  linux-hardening  linux-cve-announce  PHC 
Open Source and information security mailing list archives
 
Hash Suite: Windows password security audit tool. GUI, reports in PDF.
[<prev] [next>] [<thread-prev] [thread-next>] [day] [month] [year] [list]
Message-Id: <20250317-slub-percpu-caches-v3-5-9d9884d8b643@suse.cz>
Date: Mon, 17 Mar 2025 15:33:06 +0100
From: Vlastimil Babka <vbabka@...e.cz>
To: Suren Baghdasaryan <surenb@...gle.com>, 
 "Liam R. Howlett" <Liam.Howlett@...cle.com>, 
 Christoph Lameter <cl@...ux.com>, David Rientjes <rientjes@...gle.com>
Cc: Roman Gushchin <roman.gushchin@...ux.dev>, 
 Harry Yoo <harry.yoo@...cle.com>, Uladzislau Rezki <urezki@...il.com>, 
 linux-mm@...ck.org, linux-kernel@...r.kernel.org, rcu@...r.kernel.org, 
 maple-tree@...ts.infradead.org, vbabka@...e.cz
Subject: [PATCH RFC v3 5/8] slab: determine barn status racily outside of
 lock

The possibility of many barn operations is determined by the current
number of full or empty sheaves. Taking the barn->lock just to find out
that e.g. there are no empty sheaves results in unnecessary overhead and
lock contention. Thus perform these checks outside of the lock with a
data_race() annotated variable read and fail quickly without taking the
lock.

Checks for sheaf availability that racily succeed have to be obviously
repeated under the lock for correctness, but we can skip repeating
checks if there are too many sheaves on the given list as the limits
don't need to be strict.

Signed-off-by: Vlastimil Babka <vbabka@...e.cz>
Reviewed-by: Suren Baghdasaryan <surenb@...gle.com>
Reviewed-by: Harry Yoo <harry.yoo@...cle.com>
---
 mm/slub.c | 57 ++++++++++++++++++++++++++++++++++-----------------------
 1 file changed, 34 insertions(+), 23 deletions(-)

diff --git a/mm/slub.c b/mm/slub.c
index ab3532d5f41045d8268b12ad774541dcd066c4c4..2c7b2a85c628d01fa4811dd6cd3e4fd198d80381 100644
--- a/mm/slub.c
+++ b/mm/slub.c
@@ -2698,9 +2698,12 @@ static struct slab_sheaf *barn_get_empty_sheaf(struct node_barn *barn)
 	struct slab_sheaf *empty = NULL;
 	unsigned long flags;
 
+	if (!data_race(barn->nr_empty))
+		return NULL;
+
 	spin_lock_irqsave(&barn->lock, flags);
 
-	if (barn->nr_empty) {
+	if (likely(barn->nr_empty)) {
 		empty = list_first_entry(&barn->sheaves_empty,
 					 struct slab_sheaf, barn_list);
 		list_del(&empty->barn_list);
@@ -2716,38 +2719,36 @@ static int barn_put_empty_sheaf(struct node_barn *barn,
 				struct slab_sheaf *sheaf, bool ignore_limit)
 {
 	unsigned long flags;
-	int ret = 0;
+
+	/* we don't repeat the check under barn->lock as it's not critical */
+	if (!ignore_limit && data_race(barn->nr_empty) >= MAX_EMPTY_SHEAVES)
+		return -E2BIG;
 
 	spin_lock_irqsave(&barn->lock, flags);
 
-	if (!ignore_limit && barn->nr_empty >= MAX_EMPTY_SHEAVES) {
-		ret = -E2BIG;
-	} else {
-		list_add(&sheaf->barn_list, &barn->sheaves_empty);
-		barn->nr_empty++;
-	}
+	list_add(&sheaf->barn_list, &barn->sheaves_empty);
+	barn->nr_empty++;
 
 	spin_unlock_irqrestore(&barn->lock, flags);
-	return ret;
+	return 0;
 }
 
 static int barn_put_full_sheaf(struct node_barn *barn, struct slab_sheaf *sheaf,
 			       bool ignore_limit)
 {
 	unsigned long flags;
-	int ret = 0;
+
+	/* we don't repeat the check under barn->lock as it's not critical */
+	if (!ignore_limit && data_race(barn->nr_full) >= MAX_FULL_SHEAVES)
+		return -E2BIG;
 
 	spin_lock_irqsave(&barn->lock, flags);
 
-	if (!ignore_limit && barn->nr_full >= MAX_FULL_SHEAVES) {
-		ret = -E2BIG;
-	} else {
-		list_add(&sheaf->barn_list, &barn->sheaves_full);
-		barn->nr_full++;
-	}
+	list_add(&sheaf->barn_list, &barn->sheaves_full);
+	barn->nr_full++;
 
 	spin_unlock_irqrestore(&barn->lock, flags);
-	return ret;
+	return 0;
 }
 
 static struct slab_sheaf *barn_get_full_or_empty_sheaf(struct node_barn *barn)
@@ -2755,6 +2756,9 @@ static struct slab_sheaf *barn_get_full_or_empty_sheaf(struct node_barn *barn)
 	struct slab_sheaf *sheaf = NULL;
 	unsigned long flags;
 
+	if (!data_race(barn->nr_full) && !data_race(barn->nr_empty))
+		return NULL;
+
 	spin_lock_irqsave(&barn->lock, flags);
 
 	if (barn->nr_full) {
@@ -2785,9 +2789,12 @@ barn_replace_empty_sheaf(struct node_barn *barn, struct slab_sheaf *empty)
 	struct slab_sheaf *full = NULL;
 	unsigned long flags;
 
+	if (!data_race(barn->nr_full))
+		return NULL;
+
 	spin_lock_irqsave(&barn->lock, flags);
 
-	if (barn->nr_full) {
+	if (likely(barn->nr_full)) {
 		full = list_first_entry(&barn->sheaves_full, struct slab_sheaf,
 					barn_list);
 		list_del(&full->barn_list);
@@ -2810,19 +2817,23 @@ barn_replace_full_sheaf(struct node_barn *barn, struct slab_sheaf *full)
 	struct slab_sheaf *empty;
 	unsigned long flags;
 
+	/* we don't repeat this check under barn->lock as it's not critical */
+	if (data_race(barn->nr_full) >= MAX_FULL_SHEAVES)
+		return ERR_PTR(-E2BIG);
+	if (!data_race(barn->nr_empty))
+		return ERR_PTR(-ENOMEM);
+
 	spin_lock_irqsave(&barn->lock, flags);
 
-	if (barn->nr_full >= MAX_FULL_SHEAVES) {
-		empty = ERR_PTR(-E2BIG);
-	} else if (!barn->nr_empty) {
-		empty = ERR_PTR(-ENOMEM);
-	} else {
+	if (likely(barn->nr_empty)) {
 		empty = list_first_entry(&barn->sheaves_empty, struct slab_sheaf,
 					 barn_list);
 		list_del(&empty->barn_list);
 		list_add(&full->barn_list, &barn->sheaves_full);
 		barn->nr_empty--;
 		barn->nr_full++;
+	} else {
+		empty = ERR_PTR(-ENOMEM);
 	}
 
 	spin_unlock_irqrestore(&barn->lock, flags);

-- 
2.48.1


Powered by blists - more mailing lists

Powered by Openwall GNU/*/Linux Powered by OpenVZ