| lists.openwall.net | lists / announce owl-users owl-dev john-users john-dev passwdqc-users yescrypt popa3d-users / oss-security kernel-hardening musl sabotage tlsify passwords / crypt-dev xvendor / Bugtraq Full-Disclosure linux-kernel linux-netdev linux-ext4 linux-hardening linux-cve-announce PHC | |
|
Open Source and information security mailing list archives
| ||
|
Message-ID: <20241203164335.1125381-8-bigeasy@linutronix.de>
Date: Tue, 3 Dec 2024 17:42:15 +0100
From: Sebastian Andrzej Siewior <bigeasy@...utronix.de>
To: linux-kernel@...r.kernel.org
Cc: André Almeida <andrealmeid@...lia.com>,
Darren Hart <dvhart@...radead.org>,
Davidlohr Bueso <dave@...olabs.net>,
Ingo Molnar <mingo@...hat.com>,
Juri Lelli <juri.lelli@...hat.com>,
Peter Zijlstra <peterz@...radead.org>,
Thomas Gleixner <tglx@...utronix.de>,
Valentin Schneider <vschneid@...hat.com>,
Waiman Long <longman@...hat.com>,
Sebastian Andrzej Siewior <bigeasy@...utronix.de>
Subject: [PATCH v4 07/11] futex: Allow to make the number of slots invariant.
Add an option to freeze the number of hash buckets. The idea is to have
fixed once a certain size is acceptable so that it can be avoided to
acquire mm_struct::futex_hash_lock on certain operations.
Signed-off-by: Sebastian Andrzej Siewior <bigeasy@...utronix.de>
---
include/uapi/linux/prctl.h | 2 ++
kernel/futex/core.c | 54 ++++++++++++++++++++++++++++++++++++--
kernel/futex/futex.h | 1 +
kernel/futex/pi.c | 2 +-
kernel/futex/requeue.c | 3 ++-
5 files changed, 58 insertions(+), 4 deletions(-)
diff --git a/include/uapi/linux/prctl.h b/include/uapi/linux/prctl.h
index 55b843644c51a..d1f4b3dea565c 100644
--- a/include/uapi/linux/prctl.h
+++ b/include/uapi/linux/prctl.h
@@ -357,5 +357,7 @@ struct prctl_mm_map {
#define PR_FUTEX_HASH 77
# define PR_FUTEX_HASH_SET_SLOTS 1
# define PR_FUTEX_HASH_GET_SLOTS 2
+# define PR_FUTEX_HASH_SET_INVARIANT 3
+# define PR_FUTEX_HASH_GET_INVARIANT 4
#endif /* _LINUX_PRCTL_H */
diff --git a/kernel/futex/core.c b/kernel/futex/core.c
index 0dd7100e36419..1abea8f9abd22 100644
--- a/kernel/futex/core.c
+++ b/kernel/futex/core.c
@@ -61,6 +61,7 @@ struct futex_hash_bucket_private {
rcuref_t users;
unsigned int hash_mask;
struct rcu_head rcu;
+ bool slots_invariant;
struct futex_hash_bucket queues[];
};
@@ -1266,6 +1267,7 @@ static int futex_hash_allocate(unsigned int hash_slots)
struct futex_hash_bucket_private *hb_p, *hb_p_old = NULL;
struct mm_struct *mm;
size_t alloc_size;
+ int ret = 0;
int i;
if (hash_slots == 0)
@@ -1291,20 +1293,30 @@ static int futex_hash_allocate(unsigned int hash_slots)
rcuref_init(&hb_p->users, 1);
hb_p->hash_mask = hash_slots - 1;
+ hb_p->slots_invariant = false;
for (i = 0; i < hash_slots; i++)
futex_hash_bucket_init(&hb_p->queues[i], i + 1);
mm = current->mm;
scoped_guard(rwsem_write, &mm->futex_hash_lock) {
+
hb_p_old = rcu_dereference_check(mm->futex_hash_bucket,
lockdep_is_held(&mm->futex_hash_lock));
- rcu_assign_pointer(mm->futex_hash_bucket, hb_p);
+ if (hb_p_old && hb_p_old->slots_invariant)
+ ret = -EINVAL;
+ else
+ rcu_assign_pointer(mm->futex_hash_bucket, hb_p);
}
+ if (ret) {
+ kvfree(hb_p);
+ return ret;
+ }
+
if (hb_p_old)
futex_put_old_hb_p(hb_p_old);
- return 0;
+ return ret;
}
int futex_hash_allocate_default(void)
@@ -1323,6 +1335,36 @@ static int futex_hash_get_slots(void)
return 0;
}
+static int futex_hash_set_invariant(void)
+{
+ struct futex_hash_bucket_private *hb_p;
+ struct mm_struct *mm;
+
+ mm = current->mm;
+ guard(rwsem_write)(&mm->futex_hash_lock);
+ hb_p = rcu_dereference_check(mm->futex_hash_bucket,
+ lockdep_is_held(&mm->futex_hash_lock));
+ if (!hb_p)
+ return -EINVAL;
+ if (hb_p->slots_invariant)
+ return -EALREADY;
+ hb_p->slots_invariant = true;
+ return 0;
+}
+
+bool futex_hash_is_invariant(void)
+{
+ struct futex_hash_bucket_private *hb_p;
+ struct mm_struct *mm;
+
+ mm = current->mm;
+ guard(rcu)();
+ hb_p = rcu_dereference(mm->futex_hash_bucket);
+ if (!hb_p)
+ return -EINVAL;
+ return hb_p->slots_invariant;
+}
+
int futex_hash_prctl(unsigned long arg2, unsigned long arg3,
unsigned long arg4, unsigned long arg5)
{
@@ -1337,6 +1379,14 @@ int futex_hash_prctl(unsigned long arg2, unsigned long arg3,
ret = futex_hash_get_slots();
break;
+ case PR_FUTEX_HASH_SET_INVARIANT:
+ ret = futex_hash_set_invariant();
+ break;
+
+ case PR_FUTEX_HASH_GET_INVARIANT:
+ ret = futex_hash_is_invariant();
+ break;
+
default:
ret = -EINVAL;
break;
diff --git a/kernel/futex/futex.h b/kernel/futex/futex.h
index 503f56643a966..e81820a393027 100644
--- a/kernel/futex/futex.h
+++ b/kernel/futex/futex.h
@@ -208,6 +208,7 @@ extern void futex_hash_get(struct futex_hash_bucket *hb);
extern bool futex_check_hb_valid(struct futex_hash_bucket *hb);
extern bool check_pi_lock_owner(u32 __user *uaddr);
extern void reset_pi_state_owner(struct futex_pi_state *pi_state);
+extern bool futex_hash_is_invariant(void);
static inline struct futex_hash_bucket *futex_hb_from_futex_q(struct futex_q *q)
{
diff --git a/kernel/futex/pi.c b/kernel/futex/pi.c
index b4156d1cc6608..9df320be750c3 100644
--- a/kernel/futex/pi.c
+++ b/kernel/futex/pi.c
@@ -1202,7 +1202,7 @@ int futex_unlock_pi(u32 __user *uaddr, unsigned int flags)
if (!IS_ENABLED(CONFIG_FUTEX_PI))
return -ENOSYS;
- if (!(flags & FLAGS_SHARED))
+ if (!(flags & FLAGS_SHARED) && !futex_hash_is_invariant())
futex_hash_lock = ¤t->mm->futex_hash_lock;
retry:
if (get_user(uval, uaddr))
diff --git a/kernel/futex/requeue.c b/kernel/futex/requeue.c
index 6b3c4413fbf47..904c68abfb8f3 100644
--- a/kernel/futex/requeue.c
+++ b/kernel/futex/requeue.c
@@ -431,7 +431,8 @@ int futex_requeue(u32 __user *uaddr1, unsigned int flags1,
if (refill_pi_state_cache())
return -ENOMEM;
- if (!(flags1 & FLAGS_SHARED) || !(flags2 & FLAGS_SHARED))
+ if ((!(flags1 & FLAGS_SHARED) || !(flags2 & FLAGS_SHARED)) &&
+ !futex_hash_is_invariant())
futex_hash_lock = ¤t->mm->futex_hash_lock;
}
--
2.45.2
Powered by blists - more mailing lists