| lists.openwall.net | lists / announce owl-users owl-dev john-users john-dev passwdqc-users yescrypt popa3d-users / oss-security kernel-hardening musl sabotage tlsify passwords / crypt-dev xvendor / Bugtraq Full-Disclosure linux-kernel linux-netdev linux-ext4 linux-hardening linux-cve-announce PHC | |
|
Open Source and information security mailing list archives
| ||
|
Message-ID: <20250225170914.289358-11-bigeasy@linutronix.de>
Date: Tue, 25 Feb 2025 18:09:13 +0100
From: Sebastian Andrzej Siewior <bigeasy@...utronix.de>
To: linux-kernel@...r.kernel.org
Cc: André Almeida <andrealmeid@...lia.com>,
Darren Hart <dvhart@...radead.org>,
Davidlohr Bueso <dave@...olabs.net>,
Ingo Molnar <mingo@...hat.com>,
Juri Lelli <juri.lelli@...hat.com>,
Peter Zijlstra <peterz@...radead.org>,
Thomas Gleixner <tglx@...utronix.de>,
Valentin Schneider <vschneid@...hat.com>,
Waiman Long <longman@...hat.com>,
Sebastian Andrzej Siewior <bigeasy@...utronix.de>
Subject: [PATCH v9 10/11] futex: Resize local futex hash table based on number of threads.
Automatically size the local hash based on the number of threads but
don't exceed the number of online CPUs. The logic tries to allocate
between 16 and futex_hashsize (the default for the system wide hash
bucket) and uses 4 * number-of-threads.
On CONFIG_BASE_SMALL configs, the additional members for private hash
resize have been removed in order to save memory on mm_struct and avoid
any additional memory consumption. If we really do this, then I would
re-arrange the code structure in the previous patches to limit the
ifdefs.
The alternatives would be to limit the buckets allocated in
futex_hash_allocate_default() to 2. Avoiding
futex_hash_allocate_default() but allowing PR_FUTEX_HASH_SET_SLOTS to
work would require to hold mm_struct::futex_hash_lock in
exit_pi_state_list() and futex_wait_multiple_setup() that private does
not appear during these operations (which is currently ensured by
holding a reference).
Signed-off-by: Sebastian Andrzej Siewior <bigeasy@...utronix.de>
---
include/linux/futex.h | 21 +++++++--------
include/linux/mm_types.h | 2 +-
kernel/fork.c | 4 +--
kernel/futex/core.c | 57 +++++++++++++++++++++++++++++++++++++---
kernel/futex/futex.h | 8 ++++++
5 files changed, 73 insertions(+), 19 deletions(-)
diff --git a/include/linux/futex.h b/include/linux/futex.h
index bfb38764bac7a..77821a78059f2 100644
--- a/include/linux/futex.h
+++ b/include/linux/futex.h
@@ -78,6 +78,13 @@ void futex_exec_release(struct task_struct *tsk);
long do_futex(u32 __user *uaddr, int op, u32 val, ktime_t *timeout,
u32 __user *uaddr2, u32 val2, u32 val3);
int futex_hash_prctl(unsigned long arg2, unsigned long arg3);
+
+#ifdef CONFIG_BASE_SMALL
+static inline int futex_hash_allocate_default(void) { return 0; }
+static inline void futex_hash_free(struct mm_struct *mm) { }
+static inline void futex_mm_init(struct mm_struct *mm) { }
+#else /* !CONFIG_BASE_SMALL */
+
int futex_hash_allocate_default(void);
void futex_hash_free(struct mm_struct *mm);
@@ -87,14 +94,9 @@ static inline void futex_mm_init(struct mm_struct *mm)
mutex_init(&mm->futex_hash_lock);
}
-static inline bool futex_hash_requires_allocation(void)
-{
- if (current->mm->futex_phash)
- return false;
- return true;
-}
+#endif /* CONFIG_BASE_SMALL */
-#else
+#else /* !CONFIG_FUTEX */
static inline void futex_init_task(struct task_struct *tsk) { }
static inline void futex_exit_recursive(struct task_struct *tsk) { }
static inline void futex_exit_release(struct task_struct *tsk) { }
@@ -116,11 +118,6 @@ static inline int futex_hash_allocate_default(void)
static inline void futex_hash_free(struct mm_struct *mm) { }
static inline void futex_mm_init(struct mm_struct *mm) { }
-static inline bool futex_hash_requires_allocation(void)
-{
- return false;
-}
-
#endif
#endif
diff --git a/include/linux/mm_types.h b/include/linux/mm_types.h
index 19abbc870e0a9..72e68de850745 100644
--- a/include/linux/mm_types.h
+++ b/include/linux/mm_types.h
@@ -937,7 +937,7 @@ struct mm_struct {
*/
seqcount_t mm_lock_seq;
#endif
-#ifdef CONFIG_FUTEX
+#if defined(CONFIG_FUTEX) && !defined(CONFIG_BASE_SMALL)
struct mutex futex_hash_lock;
struct futex_private_hash __rcu *futex_phash;
struct futex_private_hash *futex_phash_new;
diff --git a/kernel/fork.c b/kernel/fork.c
index 824cc55d32ece..5e15e5b24f289 100644
--- a/kernel/fork.c
+++ b/kernel/fork.c
@@ -2142,9 +2142,7 @@ static bool need_futex_hash_allocate_default(u64 clone_flags)
{
if ((clone_flags & (CLONE_THREAD | CLONE_VM)) != (CLONE_THREAD | CLONE_VM))
return false;
- if (!thread_group_empty(current))
- return false;
- return futex_hash_requires_allocation();
+ return true;
}
/*
diff --git a/kernel/futex/core.c b/kernel/futex/core.c
index 4d9ee3bcaa6d0..6d375b9407c85 100644
--- a/kernel/futex/core.c
+++ b/kernel/futex/core.c
@@ -138,6 +138,7 @@ static struct futex_hash_bucket *futex_hash_private(union futex_key *key,
return &fhb[hash & hash_mask];
}
+#ifndef CONFIG_BASE_SMALL
static void futex_rehash_current_users(struct futex_private_hash *old,
struct futex_private_hash *new)
{
@@ -256,6 +257,14 @@ static struct futex_private_hash *futex_get_private_hb(union futex_key *key)
return futex_get_private_hash();
}
+#else
+
+static struct futex_private_hash *futex_get_private_hb(union futex_key *key)
+{
+ return NULL;
+}
+#endif
+
/**
* futex_hash - Return the hash bucket in the global or local hash
* @key: Pointer to the futex key for which the hash is calculated
@@ -279,6 +288,7 @@ struct futex_hash_bucket *__futex_hash(union futex_key *key)
return &futex_queues[hash & (futex_hashsize - 1)];
}
+#ifndef CONFIG_BASE_SMALL
bool futex_put_private_hash(struct futex_private_hash *hb_p)
{
bool released;
@@ -315,6 +325,7 @@ void futex_hash_put(struct futex_hash_bucket *hb)
return;
futex_put_private_hash(hb_p);
}
+#endif
/**
* futex_setup_timer - set up the sleeping hrtimer.
@@ -1366,12 +1377,15 @@ void futex_exit_release(struct task_struct *tsk)
static void futex_hash_bucket_init(struct futex_hash_bucket *fhb,
struct futex_private_hash *hb_p)
{
+#ifndef CONFIG_BASE_SMALL
fhb->hb_p = hb_p;
+#endif
atomic_set(&fhb->waiters, 0);
plist_head_init(&fhb->chain);
spin_lock_init(&fhb->lock);
}
+#ifndef CONFIG_BASE_SMALL
void futex_hash_free(struct mm_struct *mm)
{
struct futex_private_hash *hb_p;
@@ -1406,8 +1420,8 @@ static int futex_hash_allocate(unsigned int hash_slots)
hash_slots = 16;
if (hash_slots < 2)
hash_slots = 2;
- if (hash_slots > 131072)
- hash_slots = 131072;
+ if (hash_slots > futex_hashsize)
+ hash_slots = futex_hashsize;
if (!is_power_of_2(hash_slots))
hash_slots = rounddown_pow_of_two(hash_slots);
@@ -1449,7 +1463,31 @@ static int futex_hash_allocate(unsigned int hash_slots)
int futex_hash_allocate_default(void)
{
- return futex_hash_allocate(0);
+ unsigned int threads, buckets, current_buckets = 0;
+ struct futex_private_hash *hb_p;
+
+ if (!current->mm)
+ return 0;
+
+ scoped_guard(rcu) {
+ threads = min_t(unsigned int, get_nr_threads(current), num_online_cpus());
+ hb_p = rcu_dereference(current->mm->futex_phash);
+ if (hb_p)
+ current_buckets = hb_p->hash_mask + 1;
+ }
+
+ /*
+ * The default allocation will remain within
+ * 16 <= threads * 4 <= global hash size
+ */
+ buckets = roundup_pow_of_two(4 * threads);
+ buckets = max(buckets, 16);
+ buckets = min(buckets, futex_hashsize);
+
+ if (current_buckets >= buckets)
+ return 0;
+
+ return futex_hash_allocate(buckets);
}
static int futex_hash_get_slots(void)
@@ -1463,6 +1501,19 @@ static int futex_hash_get_slots(void)
return 0;
}
+#else
+
+static int futex_hash_allocate(unsigned int hash_slots)
+{
+ return -EINVAL;
+}
+
+static int futex_hash_get_slots(void)
+{
+ return 0;
+}
+#endif
+
int futex_hash_prctl(unsigned long arg2, unsigned long arg3)
{
int ret;
diff --git a/kernel/futex/futex.h b/kernel/futex/futex.h
index 973efcca2e01b..d1149739f3110 100644
--- a/kernel/futex/futex.h
+++ b/kernel/futex/futex.h
@@ -205,10 +205,18 @@ futex_setup_timer(ktime_t *time, struct hrtimer_sleeper *timeout,
int flags, u64 range_ns);
extern struct futex_hash_bucket *__futex_hash(union futex_key *key);
+#ifdef CONFIG_BASE_SMALL
+static inline void futex_hash_get(struct futex_hash_bucket *hb) { }
+static inline void futex_hash_put(struct futex_hash_bucket *hb) { }
+static inline struct futex_private_hash *futex_get_private_hash(void) { return NULL; }
+static inline bool futex_put_private_hash(struct futex_private_hash *hb_p) { return false; }
+
+#else /* !CONFIG_BASE_SMALL */
extern void futex_hash_get(struct futex_hash_bucket *hb);
extern void futex_hash_put(struct futex_hash_bucket *hb);
extern struct futex_private_hash *futex_get_private_hash(void);
extern bool futex_put_private_hash(struct futex_private_hash *hb_p);
+#endif
DEFINE_CLASS(hb, struct futex_hash_bucket *,
if (_T) futex_hash_put(_T),
--
2.47.2
Powered by blists - more mailing lists