| lists.openwall.net | lists / announce owl-users owl-dev john-users john-dev passwdqc-users yescrypt popa3d-users / oss-security kernel-hardening musl sabotage tlsify passwords / crypt-dev xvendor / Bugtraq Full-Disclosure linux-kernel linux-netdev linux-ext4 linux-hardening linux-cve-announce PHC | |
|
Open Source and information security mailing list archives
| ||
|
Message-Id: <20251218014531.3793471-5-mathieu.desnoyers@efficios.com>
Date: Wed, 17 Dec 2025 20:45:31 -0500
From: Mathieu Desnoyers <mathieu.desnoyers@...icios.com>
To: Boqun Feng <boqun.feng@...il.com>,
Joel Fernandes <joel@...lfernandes.org>,
"Paul E. McKenney" <paulmck@...nel.org>
Cc: linux-kernel@...r.kernel.org,
Mathieu Desnoyers <mathieu.desnoyers@...icios.com>,
Nicholas Piggin <npiggin@...il.com>,
Michael Ellerman <mpe@...erman.id.au>,
Greg Kroah-Hartman <gregkh@...uxfoundation.org>,
Sebastian Andrzej Siewior <bigeasy@...utronix.de>,
Will Deacon <will@...nel.org>,
Peter Zijlstra <peterz@...radead.org>,
Alan Stern <stern@...land.harvard.edu>,
John Stultz <jstultz@...gle.com>,
Neeraj Upadhyay <Neeraj.Upadhyay@....com>,
Linus Torvalds <torvalds@...ux-foundation.org>,
Andrew Morton <akpm@...ux-foundation.org>,
Frederic Weisbecker <frederic@...nel.org>,
Josh Triplett <josh@...htriplett.org>,
Uladzislau Rezki <urezki@...il.com>,
Steven Rostedt <rostedt@...dmis.org>,
Lai Jiangshan <jiangshanlai@...il.com>,
Zqiang <qiang.zhang1211@...il.com>,
Ingo Molnar <mingo@...hat.com>,
Waiman Long <longman@...hat.com>,
Mark Rutland <mark.rutland@....com>,
Thomas Gleixner <tglx@...utronix.de>,
Vlastimil Babka <vbabka@...e.cz>,
maged.michael@...il.com,
Mateusz Guzik <mjguzik@...il.com>,
Jonas Oberhauser <jonas.oberhauser@...weicloud.com>,
rcu@...r.kernel.org,
linux-mm@...ck.org,
lkmm@...ts.linux.dev
Subject: [RFC PATCH v4 4/4] hazptr: Migrate per-CPU slots to backup slot on context switch
Integrate with the scheduler to migrate per-CPU slots to the backup slot
on context switch. This ensures that the per-CPU slots won't be used by
blocked or preempted tasks holding on hazard pointers for a long time.
Signed-off-by: Mathieu Desnoyers <mathieu.desnoyers@...icios.com>
Cc: Nicholas Piggin <npiggin@...il.com>
Cc: Michael Ellerman <mpe@...erman.id.au>
Cc: Greg Kroah-Hartman <gregkh@...uxfoundation.org>
Cc: Sebastian Andrzej Siewior <bigeasy@...utronix.de>
Cc: "Paul E. McKenney" <paulmck@...nel.org>
Cc: Will Deacon <will@...nel.org>
Cc: Peter Zijlstra <peterz@...radead.org>
Cc: Boqun Feng <boqun.feng@...il.com>
Cc: Alan Stern <stern@...land.harvard.edu>
Cc: John Stultz <jstultz@...gle.com>
Cc: Neeraj Upadhyay <Neeraj.Upadhyay@....com>
Cc: Linus Torvalds <torvalds@...ux-foundation.org>
Cc: Andrew Morton <akpm@...ux-foundation.org>
Cc: Boqun Feng <boqun.feng@...il.com>
Cc: Frederic Weisbecker <frederic@...nel.org>
Cc: Joel Fernandes <joel@...lfernandes.org>
Cc: Josh Triplett <josh@...htriplett.org>
Cc: Uladzislau Rezki <urezki@...il.com>
Cc: Steven Rostedt <rostedt@...dmis.org>
Cc: Lai Jiangshan <jiangshanlai@...il.com>
Cc: Zqiang <qiang.zhang1211@...il.com>
Cc: Ingo Molnar <mingo@...hat.com>
Cc: Waiman Long <longman@...hat.com>
Cc: Mark Rutland <mark.rutland@....com>
Cc: Thomas Gleixner <tglx@...utronix.de>
Cc: Vlastimil Babka <vbabka@...e.cz>
Cc: maged.michael@...il.com
Cc: Mateusz Guzik <mjguzik@...il.com>
Cc: Jonas Oberhauser <jonas.oberhauser@...weicloud.com>
Cc: rcu@...r.kernel.org
Cc: linux-mm@...ck.org
Cc: lkmm@...ts.linux.dev
---
include/linux/hazptr.h | 63 ++++++++++++++++++++++++++++++++++++++++--
include/linux/sched.h | 4 +++
init/init_task.c | 3 ++
kernel/Kconfig.preempt | 10 +++++++
kernel/fork.c | 3 ++
kernel/sched/core.c | 2 ++
6 files changed, 83 insertions(+), 2 deletions(-)
diff --git a/include/linux/hazptr.h b/include/linux/hazptr.h
index 70c066ddb0f5..10ac53a42a7a 100644
--- a/include/linux/hazptr.h
+++ b/include/linux/hazptr.h
@@ -24,6 +24,7 @@
#include <linux/percpu.h>
#include <linux/types.h>
#include <linux/cleanup.h>
+#include <linux/sched.h>
/* 8 slots (each sizeof(void *)) fit in a single cache line. */
#define NR_HAZPTR_PERCPU_SLOTS 8
@@ -46,6 +47,9 @@ struct hazptr_ctx {
struct hazptr_slot *slot;
/* Backup slot in case all per-CPU slots are used. */
struct hazptr_backup_slot backup_slot;
+#ifdef CONFIG_PREEMPT_HAZPTR
+ struct list_head preempt_node;
+#endif
};
struct hazptr_percpu_slots {
@@ -98,6 +102,50 @@ bool hazptr_slot_is_backup(struct hazptr_ctx *ctx, struct hazptr_slot *slot)
return slot == &ctx->backup_slot.slot;
}
+#ifdef CONFIG_PREEMPT_HAZPTR
+static inline
+void hazptr_chain_task_ctx(struct hazptr_ctx *ctx)
+{
+ list_add(&ctx->preempt_node, ¤t->hazptr_ctx_list);
+}
+
+static inline
+void hazptr_unchain_task_ctx(struct hazptr_ctx *ctx)
+{
+ list_del(&ctx->preempt_node);
+}
+
+static inline
+void hazptr_note_context_switch(void)
+{
+ struct hazptr_ctx *ctx;
+
+ list_for_each_entry(ctx, ¤t->hazptr_ctx_list, preempt_node) {
+ struct hazptr_slot *slot;
+
+ if (hazptr_slot_is_backup(ctx, ctx->slot))
+ continue;
+ slot = hazptr_chain_backup_slot(ctx);
+ /*
+ * Move hazard pointer from per-CPU slot to backup slot.
+ * This requires hazard pointer synchronize to iterate
+ * on per-CPU slots with load-acquire before iterating
+ * on the overflow list.
+ */
+ WRITE_ONCE(slot->addr, ctx->slot->addr);
+ /*
+ * store-release orders store to backup slot addr before
+ * store to per-CPU slot addr.
+ */
+ smp_store_release(&ctx->slot->addr, NULL);
+ }
+}
+#else
+static inline void hazptr_chain_task_ctx(struct hazptr_ctx *ctx) { }
+static inline void hazptr_unchain_task_ctx(struct hazptr_ctx *ctx) { }
+static inline void hazptr_note_context_switch(void) { }
+#endif
+
/*
* hazptr_acquire: Load pointer at address and protect with hazard pointer.
*
@@ -114,6 +162,7 @@ void *hazptr_acquire(struct hazptr_ctx *ctx, void * const * addr_p)
struct hazptr_slot *slot = NULL;
void *addr, *addr2;
+ ctx->slot = NULL;
/*
* Load @addr_p to know which address should be protected.
*/
@@ -121,7 +170,9 @@ void *hazptr_acquire(struct hazptr_ctx *ctx, void * const * addr_p)
for (;;) {
if (!addr)
return NULL;
+
guard(preempt)();
+ hazptr_chain_task_ctx(ctx);
if (likely(!hazptr_slot_is_backup(ctx, slot))) {
slot = hazptr_get_free_percpu_slot();
/*
@@ -140,8 +191,11 @@ void *hazptr_acquire(struct hazptr_ctx *ctx, void * const * addr_p)
* Re-load @addr_p after storing it to the hazard pointer slot.
*/
addr2 = READ_ONCE(*addr_p); /* Load A */
- if (likely(ptr_eq(addr2, addr)))
+ if (likely(ptr_eq(addr2, addr))) {
+ ctx->slot = slot;
+ /* Success. Break loop, enable preemption and return. */
break;
+ }
/*
* If @addr_p content has changed since the first load,
* release the hazard pointer and try again.
@@ -150,11 +204,14 @@ void *hazptr_acquire(struct hazptr_ctx *ctx, void * const * addr_p)
if (!addr2) {
if (hazptr_slot_is_backup(ctx, slot))
hazptr_unchain_backup_slot(ctx);
+ hazptr_unchain_task_ctx(ctx);
+ /* Loaded NULL. Enable preemption and return NULL. */
return NULL;
}
addr = addr2;
+ hazptr_unchain_task_ctx(ctx);
+ /* Enable preemption and retry. */
}
- ctx->slot = slot;
/*
* Use addr2 loaded from the second READ_ONCE() to preserve
* address dependency ordering.
@@ -170,11 +227,13 @@ void hazptr_release(struct hazptr_ctx *ctx, void *addr)
if (!addr)
return;
+ guard(preempt)();
slot = ctx->slot;
WARN_ON_ONCE(slot->addr != addr);
smp_store_release(&slot->addr, NULL);
if (unlikely(hazptr_slot_is_backup(ctx, slot)))
hazptr_unchain_backup_slot(ctx);
+ hazptr_unchain_task_ctx(ctx);
}
void hazptr_init(void);
diff --git a/include/linux/sched.h b/include/linux/sched.h
index b469878de25c..bbec9fd6b163 100644
--- a/include/linux/sched.h
+++ b/include/linux/sched.h
@@ -933,6 +933,10 @@ struct task_struct {
struct rcu_node *rcu_blocked_node;
#endif /* #ifdef CONFIG_PREEMPT_RCU */
+#ifdef CONFIG_PREEMPT_HAZPTR
+ struct list_head hazptr_ctx_list;
+#endif
+
#ifdef CONFIG_TASKS_RCU
unsigned long rcu_tasks_nvcsw;
u8 rcu_tasks_holdout;
diff --git a/init/init_task.c b/init/init_task.c
index a55e2189206f..117aebf5573a 100644
--- a/init/init_task.c
+++ b/init/init_task.c
@@ -160,6 +160,9 @@ struct task_struct init_task __aligned(L1_CACHE_BYTES) = {
.rcu_node_entry = LIST_HEAD_INIT(init_task.rcu_node_entry),
.rcu_blocked_node = NULL,
#endif
+#ifdef CONFIG_PREEMPT_HAZPTR
+ .hazptr_ctx_list = LIST_HEAD_INIT(init_task.hazptr_ctx_list),
+#endif
#ifdef CONFIG_TASKS_RCU
.rcu_tasks_holdout = false,
.rcu_tasks_holdout_list = LIST_HEAD_INIT(init_task.rcu_tasks_holdout_list),
diff --git a/kernel/Kconfig.preempt b/kernel/Kconfig.preempt
index da326800c1c9..beb351b42b7c 100644
--- a/kernel/Kconfig.preempt
+++ b/kernel/Kconfig.preempt
@@ -189,3 +189,13 @@ config SCHED_CLASS_EXT
For more information:
Documentation/scheduler/sched-ext.rst
https://github.com/sched-ext/scx
+
+config PREEMPT_HAZPTR
+ bool "Move Hazard Pointers to Task Slots on Context Switch"
+ help
+ Integrate hazard pointers with the scheduler so the active
+ hazard pointers using preallocated per-CPU slots are moved to
+ their context local slot on context switch. This prevents
+ blocked or preempted tasks to hold on to per-CPU slots for
+ a long time, which would cause higher overhead for short
+ hazard pointer critical sections.
diff --git a/kernel/fork.c b/kernel/fork.c
index 3da0f08615a9..35c810fe744e 100644
--- a/kernel/fork.c
+++ b/kernel/fork.c
@@ -1780,6 +1780,9 @@ static inline void rcu_copy_process(struct task_struct *p)
p->rcu_blocked_node = NULL;
INIT_LIST_HEAD(&p->rcu_node_entry);
#endif /* #ifdef CONFIG_PREEMPT_RCU */
+#ifdef CONFIG_PREEMPT_HAZPTR
+ INIT_LIST_HEAD(&p->hazptr_ctx_list);
+#endif /* #ifdef CONFIG_PREEMPT_HAZPTR */
#ifdef CONFIG_TASKS_RCU
p->rcu_tasks_holdout = false;
INIT_LIST_HEAD(&p->rcu_tasks_holdout_list);
diff --git a/kernel/sched/core.c b/kernel/sched/core.c
index f754a60de848..ac8bf2708140 100644
--- a/kernel/sched/core.c
+++ b/kernel/sched/core.c
@@ -60,6 +60,7 @@
#include <linux/profile.h>
#include <linux/psi.h>
#include <linux/rcuwait_api.h>
+#include <linux/hazptr.h>
#include <linux/rseq.h>
#include <linux/sched/wake_q.h>
#include <linux/scs.h>
@@ -6812,6 +6813,7 @@ static void __sched notrace __schedule(int sched_mode)
local_irq_disable();
rcu_note_context_switch(preempt);
+ hazptr_note_context_switch();
/*
* Make sure that signal_pending_state()->signal_pending() below
--
2.39.5
Powered by blists - more mailing lists