| lists.openwall.net | lists / announce owl-users owl-dev john-users john-dev passwdqc-users yescrypt popa3d-users / oss-security kernel-hardening musl sabotage tlsify passwords / crypt-dev xvendor / Bugtraq Full-Disclosure linux-kernel linux-netdev linux-ext4 linux-hardening linux-cve-announce PHC | |
|
Open Source and information security mailing list archives
| ||
|
Message-ID: <aUR9RfVChdcDncwX@tardis-2.local>
Date: Fri, 19 Dec 2025 07:16:37 +0900
From: Boqun Feng <boqun.feng@...il.com>
To: Mathieu Desnoyers <mathieu.desnoyers@...icios.com>
Cc: Joel Fernandes <joel@...lfernandes.org>,
"Paul E. McKenney" <paulmck@...nel.org>, linux-kernel@...r.kernel.org,
Nicholas Piggin <npiggin@...il.com>,
Michael Ellerman <mpe@...erman.id.au>,
Greg Kroah-Hartman <gregkh@...uxfoundation.org>,
Sebastian Andrzej Siewior <bigeasy@...utronix.de>,
Will Deacon <will@...nel.org>, Peter Zijlstra <peterz@...radead.org>,
Alan Stern <stern@...land.harvard.edu>, John Stultz <jstultz@...gle.com>,
Neeraj Upadhyay <Neeraj.Upadhyay@....com>,
Linus Torvalds <torvalds@...ux-foundation.org>,
Andrew Morton <akpm@...ux-foundation.org>,
Frederic Weisbecker <frederic@...nel.org>,
Josh Triplett <josh@...htriplett.org>,
Uladzislau Rezki <urezki@...il.com>,
Steven Rostedt <rostedt@...dmis.org>,
Lai Jiangshan <jiangshanlai@...il.com>,
Zqiang <qiang.zhang1211@...il.com>, Ingo Molnar <mingo@...hat.com>,
Waiman Long <longman@...hat.com>, Mark Rutland <mark.rutland@....com>,
Thomas Gleixner <tglx@...utronix.de>, Vlastimil Babka <vbabka@...e.cz>,
maged.michael@...il.com, Mateusz Guzik <mjguzik@...il.com>,
Jonas Oberhauser <jonas.oberhauser@...weicloud.com>, rcu@...r.kernel.org,
linux-mm@...ck.org, lkmm@...ts.linux.dev
Subject: Re: [RFC PATCH v4 4/4] hazptr: Migrate per-CPU slots to backup slot
on context switch
On Wed, Dec 17, 2025 at 08:45:31PM -0500, Mathieu Desnoyers wrote:
> Integrate with the scheduler to migrate per-CPU slots to the backup slot
> on context switch. This ensures that the per-CPU slots won't be used by
> blocked or preempted tasks holding on hazard pointers for a long time.
>
> Signed-off-by: Mathieu Desnoyers <mathieu.desnoyers@...icios.com>
> Cc: Nicholas Piggin <npiggin@...il.com>
> Cc: Michael Ellerman <mpe@...erman.id.au>
> Cc: Greg Kroah-Hartman <gregkh@...uxfoundation.org>
> Cc: Sebastian Andrzej Siewior <bigeasy@...utronix.de>
> Cc: "Paul E. McKenney" <paulmck@...nel.org>
> Cc: Will Deacon <will@...nel.org>
> Cc: Peter Zijlstra <peterz@...radead.org>
> Cc: Boqun Feng <boqun.feng@...il.com>
> Cc: Alan Stern <stern@...land.harvard.edu>
> Cc: John Stultz <jstultz@...gle.com>
> Cc: Neeraj Upadhyay <Neeraj.Upadhyay@....com>
> Cc: Linus Torvalds <torvalds@...ux-foundation.org>
> Cc: Andrew Morton <akpm@...ux-foundation.org>
> Cc: Boqun Feng <boqun.feng@...il.com>
> Cc: Frederic Weisbecker <frederic@...nel.org>
> Cc: Joel Fernandes <joel@...lfernandes.org>
> Cc: Josh Triplett <josh@...htriplett.org>
> Cc: Uladzislau Rezki <urezki@...il.com>
> Cc: Steven Rostedt <rostedt@...dmis.org>
> Cc: Lai Jiangshan <jiangshanlai@...il.com>
> Cc: Zqiang <qiang.zhang1211@...il.com>
> Cc: Ingo Molnar <mingo@...hat.com>
> Cc: Waiman Long <longman@...hat.com>
> Cc: Mark Rutland <mark.rutland@....com>
> Cc: Thomas Gleixner <tglx@...utronix.de>
> Cc: Vlastimil Babka <vbabka@...e.cz>
> Cc: maged.michael@...il.com
> Cc: Mateusz Guzik <mjguzik@...il.com>
> Cc: Jonas Oberhauser <jonas.oberhauser@...weicloud.com>
> Cc: rcu@...r.kernel.org
> Cc: linux-mm@...ck.org
> Cc: lkmm@...ts.linux.dev
> ---
> include/linux/hazptr.h | 63 ++++++++++++++++++++++++++++++++++++++++--
> include/linux/sched.h | 4 +++
> init/init_task.c | 3 ++
> kernel/Kconfig.preempt | 10 +++++++
> kernel/fork.c | 3 ++
> kernel/sched/core.c | 2 ++
> 6 files changed, 83 insertions(+), 2 deletions(-)
>
> diff --git a/include/linux/hazptr.h b/include/linux/hazptr.h
> index 70c066ddb0f5..10ac53a42a7a 100644
> --- a/include/linux/hazptr.h
> +++ b/include/linux/hazptr.h
> @@ -24,6 +24,7 @@
> #include <linux/percpu.h>
> #include <linux/types.h>
> #include <linux/cleanup.h>
> +#include <linux/sched.h>
>
> /* 8 slots (each sizeof(void *)) fit in a single cache line. */
> #define NR_HAZPTR_PERCPU_SLOTS 8
> @@ -46,6 +47,9 @@ struct hazptr_ctx {
> struct hazptr_slot *slot;
> /* Backup slot in case all per-CPU slots are used. */
> struct hazptr_backup_slot backup_slot;
> +#ifdef CONFIG_PREEMPT_HAZPTR
I would suggest we make CONFIG_PREEMPT_HAZPTR always enabled hence no
need for a config, do we have the measurement of the additional cost?
> + struct list_head preempt_node;
> +#endif
> };
>
> struct hazptr_percpu_slots {
> @@ -98,6 +102,50 @@ bool hazptr_slot_is_backup(struct hazptr_ctx *ctx, struct hazptr_slot *slot)
> return slot == &ctx->backup_slot.slot;
> }
>
> +#ifdef CONFIG_PREEMPT_HAZPTR
> +static inline
> +void hazptr_chain_task_ctx(struct hazptr_ctx *ctx)
> +{
> + list_add(&ctx->preempt_node, ¤t->hazptr_ctx_list);
> +}
> +
> +static inline
> +void hazptr_unchain_task_ctx(struct hazptr_ctx *ctx)
> +{
> + list_del(&ctx->preempt_node);
> +}
> +
I think you need to add interrupt disabling for chain/unchain because of
the potential readers in interrupt and then you can avoid the preempt
disabling in hazptr_release() I think. Let's aim for supporting readers
in interrupt handler, because at least lockdep needs that.
Regards,
Boqun
> +static inline
> +void hazptr_note_context_switch(void)
> +{
> + struct hazptr_ctx *ctx;
> +
> + list_for_each_entry(ctx, ¤t->hazptr_ctx_list, preempt_node) {
> + struct hazptr_slot *slot;
> +
> + if (hazptr_slot_is_backup(ctx, ctx->slot))
> + continue;
> + slot = hazptr_chain_backup_slot(ctx);
> + /*
> + * Move hazard pointer from per-CPU slot to backup slot.
> + * This requires hazard pointer synchronize to iterate
> + * on per-CPU slots with load-acquire before iterating
> + * on the overflow list.
> + */
> + WRITE_ONCE(slot->addr, ctx->slot->addr);
> + /*
> + * store-release orders store to backup slot addr before
> + * store to per-CPU slot addr.
> + */
> + smp_store_release(&ctx->slot->addr, NULL);
> + }
> +}
> +#else
> +static inline void hazptr_chain_task_ctx(struct hazptr_ctx *ctx) { }
> +static inline void hazptr_unchain_task_ctx(struct hazptr_ctx *ctx) { }
> +static inline void hazptr_note_context_switch(void) { }
> +#endif
> +
> /*
> * hazptr_acquire: Load pointer at address and protect with hazard pointer.
> *
> @@ -114,6 +162,7 @@ void *hazptr_acquire(struct hazptr_ctx *ctx, void * const * addr_p)
> struct hazptr_slot *slot = NULL;
> void *addr, *addr2;
>
> + ctx->slot = NULL;
> /*
> * Load @addr_p to know which address should be protected.
> */
> @@ -121,7 +170,9 @@ void *hazptr_acquire(struct hazptr_ctx *ctx, void * const * addr_p)
> for (;;) {
> if (!addr)
> return NULL;
> +
> guard(preempt)();
> + hazptr_chain_task_ctx(ctx);
> if (likely(!hazptr_slot_is_backup(ctx, slot))) {
> slot = hazptr_get_free_percpu_slot();
> /*
> @@ -140,8 +191,11 @@ void *hazptr_acquire(struct hazptr_ctx *ctx, void * const * addr_p)
> * Re-load @addr_p after storing it to the hazard pointer slot.
> */
> addr2 = READ_ONCE(*addr_p); /* Load A */
> - if (likely(ptr_eq(addr2, addr)))
> + if (likely(ptr_eq(addr2, addr))) {
> + ctx->slot = slot;
> + /* Success. Break loop, enable preemption and return. */
> break;
> + }
> /*
> * If @addr_p content has changed since the first load,
> * release the hazard pointer and try again.
> @@ -150,11 +204,14 @@ void *hazptr_acquire(struct hazptr_ctx *ctx, void * const * addr_p)
> if (!addr2) {
> if (hazptr_slot_is_backup(ctx, slot))
> hazptr_unchain_backup_slot(ctx);
> + hazptr_unchain_task_ctx(ctx);
> + /* Loaded NULL. Enable preemption and return NULL. */
> return NULL;
> }
> addr = addr2;
> + hazptr_unchain_task_ctx(ctx);
> + /* Enable preemption and retry. */
> }
> - ctx->slot = slot;
> /*
> * Use addr2 loaded from the second READ_ONCE() to preserve
> * address dependency ordering.
> @@ -170,11 +227,13 @@ void hazptr_release(struct hazptr_ctx *ctx, void *addr)
>
> if (!addr)
> return;
> + guard(preempt)();
> slot = ctx->slot;
> WARN_ON_ONCE(slot->addr != addr);
> smp_store_release(&slot->addr, NULL);
> if (unlikely(hazptr_slot_is_backup(ctx, slot)))
> hazptr_unchain_backup_slot(ctx);
> + hazptr_unchain_task_ctx(ctx);
> }
[...]
Powered by blists - more mailing lists