| lists.openwall.net | lists / announce owl-users owl-dev john-users john-dev passwdqc-users yescrypt popa3d-users / oss-security kernel-hardening musl sabotage tlsify passwords / crypt-dev xvendor / Bugtraq Full-Disclosure linux-kernel linux-netdev linux-ext4 linux-hardening linux-cve-announce PHC | |
|
Open Source and information security mailing list archives
| ||
|
Message-Id: <20251023-sheaves-for-all-v1-12-6ffa2c9941c0@suse.cz>
Date: Thu, 23 Oct 2025 15:52:34 +0200
From: Vlastimil Babka <vbabka@...e.cz>
To: Andrew Morton <akpm@...ux-foundation.org>,
Christoph Lameter <cl@...two.org>, David Rientjes <rientjes@...gle.com>,
Roman Gushchin <roman.gushchin@...ux.dev>, Harry Yoo <harry.yoo@...cle.com>
Cc: Uladzislau Rezki <urezki@...il.com>,
"Liam R. Howlett" <Liam.Howlett@...cle.com>,
Suren Baghdasaryan <surenb@...gle.com>,
Sebastian Andrzej Siewior <bigeasy@...utronix.de>,
Alexei Starovoitov <ast@...nel.org>, linux-mm@...ck.org,
linux-kernel@...r.kernel.org, linux-rt-devel@...ts.linux.dev,
bpf@...r.kernel.org, kasan-dev@...glegroups.com,
Vlastimil Babka <vbabka@...e.cz>
Subject: [PATCH RFC 12/19] slab: remove the do_slab_free() fastpath
We have removed cpu slab usage from allocation paths. Now remove
do_slab_free() which was freeing objects to the cpu slab when
the object belonged to it. Instead call __slab_free() directly,
which was previously the fallback.
This simplifies kfree_nolock() - when freeing to percpu sheaf
fails, we can call defer_free() directly.
Also remove functions that became unused.
Signed-off-by: Vlastimil Babka <vbabka@...e.cz>
---
mm/slub.c | 149 ++++++--------------------------------------------------------
1 file changed, 13 insertions(+), 136 deletions(-)
diff --git a/mm/slub.c b/mm/slub.c
index d8891d852a8f..a35eb397caa9 100644
--- a/mm/slub.c
+++ b/mm/slub.c
@@ -3671,29 +3671,6 @@ static inline unsigned int init_tid(int cpu)
return cpu;
}
-static inline void note_cmpxchg_failure(const char *n,
- const struct kmem_cache *s, unsigned long tid)
-{
-#ifdef SLUB_DEBUG_CMPXCHG
- unsigned long actual_tid = __this_cpu_read(s->cpu_slab->tid);
-
- pr_info("%s %s: cmpxchg redo ", n, s->name);
-
- if (IS_ENABLED(CONFIG_PREEMPTION) &&
- tid_to_cpu(tid) != tid_to_cpu(actual_tid)) {
- pr_warn("due to cpu change %d -> %d\n",
- tid_to_cpu(tid), tid_to_cpu(actual_tid));
- } else if (tid_to_event(tid) != tid_to_event(actual_tid)) {
- pr_warn("due to cpu running other code. Event %ld->%ld\n",
- tid_to_event(tid), tid_to_event(actual_tid));
- } else {
- pr_warn("for unknown reason: actual=%lx was=%lx target=%lx\n",
- actual_tid, tid, next_tid(tid));
- }
-#endif
- stat(s, CMPXCHG_DOUBLE_CPU_FAIL);
-}
-
static void init_kmem_cache_cpus(struct kmem_cache *s)
{
#ifdef CONFIG_PREEMPT_RT
@@ -4231,18 +4208,6 @@ static inline bool pfmemalloc_match(struct slab *slab, gfp_t gfpflags)
return true;
}
-static inline bool
-__update_cpu_freelist_fast(struct kmem_cache *s,
- void *freelist_old, void *freelist_new,
- unsigned long tid)
-{
- freelist_aba_t old = { .freelist = freelist_old, .counter = tid };
- freelist_aba_t new = { .freelist = freelist_new, .counter = next_tid(tid) };
-
- return this_cpu_try_cmpxchg_freelist(s->cpu_slab->freelist_tid.full,
- &old.full, new.full);
-}
-
/*
* Get the slab's freelist and do not freeze it.
*
@@ -6076,99 +6041,6 @@ void defer_free_barrier(void)
irq_work_sync(&per_cpu_ptr(&defer_free_objects, cpu)->work);
}
-/*
- * Fastpath with forced inlining to produce a kfree and kmem_cache_free that
- * can perform fastpath freeing without additional function calls.
- *
- * The fastpath is only possible if we are freeing to the current cpu slab
- * of this processor. This typically the case if we have just allocated
- * the item before.
- *
- * If fastpath is not possible then fall back to __slab_free where we deal
- * with all sorts of special processing.
- *
- * Bulk free of a freelist with several objects (all pointing to the
- * same slab) possible by specifying head and tail ptr, plus objects
- * count (cnt). Bulk free indicated by tail pointer being set.
- */
-static __always_inline void do_slab_free(struct kmem_cache *s,
- struct slab *slab, void *head, void *tail,
- int cnt, unsigned long addr)
-{
- /* cnt == 0 signals that it's called from kfree_nolock() */
- bool allow_spin = cnt;
- struct kmem_cache_cpu *c;
- unsigned long tid;
- void **freelist;
-
-redo:
- /*
- * Determine the currently cpus per cpu slab.
- * The cpu may change afterward. However that does not matter since
- * data is retrieved via this pointer. If we are on the same cpu
- * during the cmpxchg then the free will succeed.
- */
- c = raw_cpu_ptr(s->cpu_slab);
- tid = READ_ONCE(c->tid);
-
- /* Same with comment on barrier() in __slab_alloc_node() */
- barrier();
-
- if (unlikely(slab != c->slab)) {
- if (unlikely(!allow_spin)) {
- /*
- * __slab_free() can locklessly cmpxchg16 into a slab,
- * but then it might need to take spin_lock
- * for further processing.
- * Avoid the complexity and simply add to a deferred list.
- */
- defer_free(s, head);
- } else {
- __slab_free(s, slab, head, tail, cnt, addr);
- }
- return;
- }
-
- if (unlikely(!allow_spin)) {
- if ((in_nmi() || !USE_LOCKLESS_FAST_PATH()) &&
- local_lock_is_locked(&s->cpu_slab->lock)) {
- defer_free(s, head);
- return;
- }
- cnt = 1; /* restore cnt. kfree_nolock() frees one object at a time */
- }
-
- if (USE_LOCKLESS_FAST_PATH()) {
- freelist = READ_ONCE(c->freelist);
-
- set_freepointer(s, tail, freelist);
-
- if (unlikely(!__update_cpu_freelist_fast(s, freelist, head, tid))) {
- note_cmpxchg_failure("slab_free", s, tid);
- goto redo;
- }
- } else {
- __maybe_unused unsigned long flags = 0;
-
- /* Update the free list under the local lock */
- local_lock_cpu_slab(s, flags);
- c = this_cpu_ptr(s->cpu_slab);
- if (unlikely(slab != c->slab)) {
- local_unlock_cpu_slab(s, flags);
- goto redo;
- }
- tid = c->tid;
- freelist = c->freelist;
-
- set_freepointer(s, tail, freelist);
- c->freelist = head;
- c->tid = next_tid(tid);
-
- local_unlock_cpu_slab(s, flags);
- }
- stat_add(s, FREE_FASTPATH, cnt);
-}
-
static __fastpath_inline
void slab_free(struct kmem_cache *s, struct slab *slab, void *object,
unsigned long addr)
@@ -6185,7 +6057,7 @@ void slab_free(struct kmem_cache *s, struct slab *slab, void *object,
return;
}
- do_slab_free(s, slab, object, object, 1, addr);
+ __slab_free(s, slab, object, object, 1, addr);
}
#ifdef CONFIG_MEMCG
@@ -6194,7 +6066,7 @@ static noinline
void memcg_alloc_abort_single(struct kmem_cache *s, void *object)
{
if (likely(slab_free_hook(s, object, slab_want_init_on_free(s), false)))
- do_slab_free(s, virt_to_slab(object), object, object, 1, _RET_IP_);
+ __slab_free(s, virt_to_slab(object), object, object, 1, _RET_IP_);
}
#endif
@@ -6209,7 +6081,7 @@ void slab_free_bulk(struct kmem_cache *s, struct slab *slab, void *head,
* to remove objects, whose reuse must be delayed.
*/
if (likely(slab_free_freelist_hook(s, &head, &tail, &cnt)))
- do_slab_free(s, slab, head, tail, cnt, addr);
+ __slab_free(s, slab, head, tail, cnt, addr);
}
#ifdef CONFIG_SLUB_RCU_DEBUG
@@ -6235,14 +6107,14 @@ static void slab_free_after_rcu_debug(struct rcu_head *rcu_head)
/* resume freeing */
if (slab_free_hook(s, object, slab_want_init_on_free(s), true))
- do_slab_free(s, slab, object, object, 1, _THIS_IP_);
+ __slab_free(s, slab, object, object, 1, _THIS_IP_);
}
#endif /* CONFIG_SLUB_RCU_DEBUG */
#ifdef CONFIG_KASAN_GENERIC
void ___cache_free(struct kmem_cache *cache, void *x, unsigned long addr)
{
- do_slab_free(cache, virt_to_slab(x), x, x, 1, addr);
+ __slab_free(cache, virt_to_slab(x), x, x, 1, addr);
}
#endif
@@ -6444,8 +6316,13 @@ void kfree_nolock(const void *object)
* since kasan quarantine takes locks and not supported from NMI.
*/
kasan_slab_free(s, x, false, false, /* skip quarantine */true);
+ /*
+ * __slab_free() can locklessly cmpxchg16 into a slab, but then it might
+ * need to take spin_lock for further processing.
+ * Avoid the complexity and simply add to a deferred list.
+ */
if (!free_to_pcs(s, x, false))
- do_slab_free(s, slab, x, x, 0, _RET_IP_);
+ defer_free(s, x);
}
EXPORT_SYMBOL_GPL(kfree_nolock);
@@ -6862,7 +6739,7 @@ static void __kmem_cache_free_bulk(struct kmem_cache *s, size_t size, void **p)
if (kfence_free(df.freelist))
continue;
- do_slab_free(df.s, df.slab, df.freelist, df.tail, df.cnt,
+ __slab_free(df.s, df.slab, df.freelist, df.tail, df.cnt,
_RET_IP_);
} while (likely(size));
}
@@ -6945,7 +6822,7 @@ __refill_objects(struct kmem_cache *s, void **p, gfp_t gfp, unsigned int min,
cnt++;
object = get_freepointer(s, object);
} while (object);
- do_slab_free(s, slab, head, tail, cnt, _RET_IP_);
+ __slab_free(s, slab, head, tail, cnt, _RET_IP_);
}
if (refilled >= max)
--
2.51.1
Powered by blists - more mailing lists