| lists.openwall.net | lists / announce owl-users owl-dev john-users john-dev passwdqc-users yescrypt popa3d-users / oss-security kernel-hardening musl sabotage tlsify passwords / crypt-dev xvendor / Bugtraq Full-Disclosure linux-kernel linux-netdev linux-ext4 linux-hardening linux-cve-announce PHC | |
|
Open Source and information security mailing list archives
| ||
|
Message-ID: <20250416162921.513656-16-bigeasy@linutronix.de>
Date: Wed, 16 Apr 2025 18:29:15 +0200
From: Sebastian Andrzej Siewior <bigeasy@...utronix.de>
To: linux-kernel@...r.kernel.org
Cc: André Almeida <andrealmeid@...lia.com>,
Darren Hart <dvhart@...radead.org>,
Davidlohr Bueso <dave@...olabs.net>,
Ingo Molnar <mingo@...hat.com>,
Juri Lelli <juri.lelli@...hat.com>,
Peter Zijlstra <peterz@...radead.org>,
Thomas Gleixner <tglx@...utronix.de>,
Valentin Schneider <vschneid@...hat.com>,
Waiman Long <longman@...hat.com>,
Sebastian Andrzej Siewior <bigeasy@...utronix.de>,
Shrikanth Hegde <sshegde@...ux.ibm.com>
Subject: [PATCH v12 15/21] futex: Allow to make the private hash immutable
My initial testing showed that
perf bench futex hash
reported less operations/sec with private hash. After using the same
amount of buckets in the private hash as used by the global hash then
the operations/sec were about the same.
This changed once the private hash became resizable. This feature added
a RCU section and reference counting via atomic inc+dec operation into
the hot path.
The reference counting can be avoided if the private hash is made
immutable.
Extend PR_FUTEX_HASH_SET_SLOTS by a fourth argument which denotes if the
private should be made immutable. Once set (to true) the a further
resize is not allowed (same if set to global hash).
Add PR_FUTEX_HASH_GET_IMMUTABLE which returns true if the hash can not
be changed.
Update "perf bench" suite.
For comparison, results of "perf bench futex hash -s":
- Xeon CPU E5-2650, 2 NUMA nodes, total 32 CPUs:
- Before the introducing task local hash
shared Averaged 1.487.148 operations/sec (+- 0,53%), total secs = 10
private Averaged 2.192.405 operations/sec (+- 0,07%), total secs = 10
- With the series
shared Averaged 1.326.342 operations/sec (+- 0,41%), total secs = 10
-b128 Averaged 141.394 operations/sec (+- 1,15%), total secs = 10
-Ib128 Averaged 851.490 operations/sec (+- 0,67%), total secs = 10
-b8192 Averaged 131.321 operations/sec (+- 2,13%), total secs = 10
-Ib8192 Averaged 1.923.077 operations/sec (+- 0,61%), total secs = 10
128 is the default allocation of hash buckets.
8192 was the previous amount of allocated hash buckets.
- Xeon(R) CPU E7-8890 v3, 4 NUMA nodes, total 144 CPUs:
- Before the introducing task local hash
shared Averaged 1.810.936 operations/sec (+- 0,26%), total secs = 20
private Averaged 2.505.801 operations/sec (+- 0,05%), total secs = 20
- With the series
shared Averaged 1.589.002 operations/sec (+- 0,25%), total secs = 20
-b1024 Averaged 42.410 operations/sec (+- 0,20%), total secs = 20
-Ib1024 Averaged 740.638 operations/sec (+- 1,51%), total secs = 20
-b65536 Averaged 48.811 operations/sec (+- 1,35%), total secs = 20
-Ib65536 Averaged 1.963.165 operations/sec (+- 0,18%), total secs = 20
1024 is the default allocation of hash buckets.
65536 was the previous amount of allocated hash buckets.
Acked-by: Shrikanth Hegde <sshegde@...ux.ibm.com>
Signed-off-by: Sebastian Andrzej Siewior <bigeasy@...utronix.de>
---
include/uapi/linux/prctl.h | 1 +
kernel/futex/core.c | 44 ++++++++++++++++++++++++++++++++------
2 files changed, 38 insertions(+), 7 deletions(-)
diff --git a/include/uapi/linux/prctl.h b/include/uapi/linux/prctl.h
index 3b93fb906e3c5..21f30b3ded74b 100644
--- a/include/uapi/linux/prctl.h
+++ b/include/uapi/linux/prctl.h
@@ -368,5 +368,6 @@ struct prctl_mm_map {
#define PR_FUTEX_HASH 78
# define PR_FUTEX_HASH_SET_SLOTS 1
# define PR_FUTEX_HASH_GET_SLOTS 2
+# define PR_FUTEX_HASH_GET_IMMUTABLE 3
#endif /* _LINUX_PRCTL_H */
diff --git a/kernel/futex/core.c b/kernel/futex/core.c
index 9e7dad52abea8..81c5705b6af5e 100644
--- a/kernel/futex/core.c
+++ b/kernel/futex/core.c
@@ -63,6 +63,7 @@ struct futex_private_hash {
struct rcu_head rcu;
void *mm;
bool custom;
+ bool immutable;
struct futex_hash_bucket queues[];
};
@@ -132,12 +133,16 @@ static inline bool futex_key_is_private(union futex_key *key)
bool futex_private_hash_get(struct futex_private_hash *fph)
{
+ if (fph->immutable)
+ return true;
return rcuref_get(&fph->users);
}
void futex_private_hash_put(struct futex_private_hash *fph)
{
/* Ignore return value, last put is verified via rcuref_is_dead() */
+ if (fph->immutable)
+ return;
if (rcuref_put(&fph->users))
wake_up_var(fph->mm);
}
@@ -277,6 +282,8 @@ struct futex_private_hash *futex_private_hash(void)
if (!fph)
return NULL;
+ if (fph->immutable)
+ return fph;
if (rcuref_get(&fph->users))
return fph;
}
@@ -1433,7 +1440,7 @@ static bool futex_hash_less(struct futex_private_hash *a,
return false; /* equal */
}
-static int futex_hash_allocate(unsigned int hash_slots, bool custom)
+static int futex_hash_allocate(unsigned int hash_slots, unsigned int immutable, bool custom)
{
struct mm_struct *mm = current->mm;
struct futex_private_hash *fph;
@@ -1441,13 +1448,15 @@ static int futex_hash_allocate(unsigned int hash_slots, bool custom)
if (hash_slots && (hash_slots == 1 || !is_power_of_2(hash_slots)))
return -EINVAL;
+ if (immutable > 2)
+ return -EINVAL;
/*
* Once we've disabled the global hash there is no way back.
*/
scoped_guard(rcu) {
fph = rcu_dereference(mm->futex_phash);
- if (fph && !fph->hash_mask) {
+ if (fph && (!fph->hash_mask || fph->immutable)) {
if (custom)
return -EBUSY;
return 0;
@@ -1461,6 +1470,7 @@ static int futex_hash_allocate(unsigned int hash_slots, bool custom)
rcuref_init(&fph->users, 1);
fph->hash_mask = hash_slots ? hash_slots - 1 : 0;
fph->custom = custom;
+ fph->immutable = !!immutable;
fph->mm = mm;
for (i = 0; i < hash_slots; i++)
@@ -1553,7 +1563,7 @@ int futex_hash_allocate_default(void)
if (current_buckets >= buckets)
return 0;
- return futex_hash_allocate(buckets, false);
+ return futex_hash_allocate(buckets, 0, false);
}
static int futex_hash_get_slots(void)
@@ -1567,9 +1577,22 @@ static int futex_hash_get_slots(void)
return 0;
}
+static int futex_hash_get_immutable(void)
+{
+ struct futex_private_hash *fph;
+
+ guard(rcu)();
+ fph = rcu_dereference(current->mm->futex_phash);
+ if (fph && fph->immutable)
+ return 1;
+ if (fph && !fph->hash_mask)
+ return 1;
+ return 0;
+}
+
#else
-static int futex_hash_allocate(unsigned int hash_slots, bool custom)
+static int futex_hash_allocate(unsigned int hash_slots, unsigned int immutable, bool custom)
{
return -EINVAL;
}
@@ -1578,6 +1601,11 @@ static int futex_hash_get_slots(void)
{
return 0;
}
+
+static int futex_hash_get_immutable(void)
+{
+ return 0;
+}
#endif
int futex_hash_prctl(unsigned long arg2, unsigned long arg3, unsigned long arg4)
@@ -1586,15 +1614,17 @@ int futex_hash_prctl(unsigned long arg2, unsigned long arg3, unsigned long arg4)
switch (arg2) {
case PR_FUTEX_HASH_SET_SLOTS:
- if (arg4 != 0)
- return -EINVAL;
- ret = futex_hash_allocate(arg3, true);
+ ret = futex_hash_allocate(arg3, arg4, true);
break;
case PR_FUTEX_HASH_GET_SLOTS:
ret = futex_hash_get_slots();
break;
+ case PR_FUTEX_HASH_GET_IMMUTABLE:
+ ret = futex_hash_get_immutable();
+ break;
+
default:
ret = -EINVAL;
break;
--
2.49.0
Powered by blists - more mailing lists