| lists.openwall.net | lists / announce owl-users owl-dev john-users john-dev passwdqc-users yescrypt popa3d-users / oss-security kernel-hardening musl sabotage tlsify passwords / crypt-dev xvendor / Bugtraq Full-Disclosure linux-kernel linux-netdev linux-ext4 linux-hardening linux-cve-announce PHC | |
|
Open Source and information security mailing list archives
| ||
|
Message-Id: <20251023-sheaves-for-all-v1-13-6ffa2c9941c0@suse.cz>
Date: Thu, 23 Oct 2025 15:52:35 +0200
From: Vlastimil Babka <vbabka@...e.cz>
To: Andrew Morton <akpm@...ux-foundation.org>,
Christoph Lameter <cl@...two.org>, David Rientjes <rientjes@...gle.com>,
Roman Gushchin <roman.gushchin@...ux.dev>, Harry Yoo <harry.yoo@...cle.com>
Cc: Uladzislau Rezki <urezki@...il.com>,
"Liam R. Howlett" <Liam.Howlett@...cle.com>,
Suren Baghdasaryan <surenb@...gle.com>,
Sebastian Andrzej Siewior <bigeasy@...utronix.de>,
Alexei Starovoitov <ast@...nel.org>, linux-mm@...ck.org,
linux-kernel@...r.kernel.org, linux-rt-devel@...ts.linux.dev,
bpf@...r.kernel.org, kasan-dev@...glegroups.com,
Vlastimil Babka <vbabka@...e.cz>
Subject: [PATCH RFC 13/19] slab: remove defer_deactivate_slab()
There are no more cpu slabs so we don't need their deferred
deactivation. The function is now only used from a place where we
allocate a new slab but then can't spin on node list_lock to put it on
the partial list. Instead of the deferred action we can free it directly
via __free_slab(), we just need to tell it to use _nolock() freeing of
the underlying pages and take care of the accounting.
Since free_frozen_pages_nolock() variant does not yet exist for code
outside of the page allocator, create it as a trivial wrapper for
__free_frozen_pages(..., FPI_TRYLOCK).
Signed-off-by: Vlastimil Babka <vbabka@...e.cz>
---
mm/internal.h | 1 +
mm/page_alloc.c | 5 +++++
mm/slab.h | 8 +-------
mm/slub.c | 50 +++++++++++++++-----------------------------------
4 files changed, 22 insertions(+), 42 deletions(-)
diff --git a/mm/internal.h b/mm/internal.h
index 1561fc2ff5b8..64c5eda7c1ae 100644
--- a/mm/internal.h
+++ b/mm/internal.h
@@ -846,6 +846,7 @@ static inline struct page *alloc_frozen_pages_noprof(gfp_t gfp, unsigned int ord
struct page *alloc_frozen_pages_nolock_noprof(gfp_t gfp_flags, int nid, unsigned int order);
#define alloc_frozen_pages_nolock(...) \
alloc_hooks(alloc_frozen_pages_nolock_noprof(__VA_ARGS__))
+void free_frozen_pages_nolock(struct page *page, unsigned int order);
extern void zone_pcp_reset(struct zone *zone);
extern void zone_pcp_disable(struct zone *zone);
diff --git a/mm/page_alloc.c b/mm/page_alloc.c
index 600d9e981c23..f8ac3232db41 100644
--- a/mm/page_alloc.c
+++ b/mm/page_alloc.c
@@ -2944,6 +2944,11 @@ void free_frozen_pages(struct page *page, unsigned int order)
__free_frozen_pages(page, order, FPI_NONE);
}
+void free_frozen_pages_nolock(struct page *page, unsigned int order)
+{
+ __free_frozen_pages(page, order, FPI_TRYLOCK);
+}
+
/*
* Free a batch of folios
*/
diff --git a/mm/slab.h b/mm/slab.h
index a103da44ab9d..b2663cc594f3 100644
--- a/mm/slab.h
+++ b/mm/slab.h
@@ -55,13 +55,7 @@ struct slab {
struct kmem_cache *slab_cache;
union {
struct {
- union {
- struct list_head slab_list;
- struct { /* For deferred deactivate_slab() */
- struct llist_node llnode;
- void *flush_freelist;
- };
- };
+ struct list_head slab_list;
/* Double-word boundary */
union {
struct {
diff --git a/mm/slub.c b/mm/slub.c
index a35eb397caa9..6f5ca26bbb00 100644
--- a/mm/slub.c
+++ b/mm/slub.c
@@ -3246,7 +3246,7 @@ static struct slab *new_slab(struct kmem_cache *s, gfp_t flags, int node)
flags & (GFP_RECLAIM_MASK | GFP_CONSTRAINT_MASK), node);
}
-static void __free_slab(struct kmem_cache *s, struct slab *slab)
+static void __free_slab(struct kmem_cache *s, struct slab *slab, bool allow_spin)
{
struct folio *folio = slab_folio(slab);
int order = folio_order(folio);
@@ -3257,14 +3257,18 @@ static void __free_slab(struct kmem_cache *s, struct slab *slab)
__folio_clear_slab(folio);
mm_account_reclaimed_pages(pages);
unaccount_slab(slab, order, s);
- free_frozen_pages(&folio->page, order);
+
+ if (allow_spin)
+ free_frozen_pages(&folio->page, order);
+ else
+ free_frozen_pages_nolock(&folio->page, order);
}
static void rcu_free_slab(struct rcu_head *h)
{
struct slab *slab = container_of(h, struct slab, rcu_head);
- __free_slab(slab->slab_cache, slab);
+ __free_slab(slab->slab_cache, slab, true);
}
static void free_slab(struct kmem_cache *s, struct slab *slab)
@@ -3280,7 +3284,7 @@ static void free_slab(struct kmem_cache *s, struct slab *slab)
if (unlikely(s->flags & SLAB_TYPESAFE_BY_RCU))
call_rcu(&slab->rcu_head, rcu_free_slab);
else
- __free_slab(s, slab);
+ __free_slab(s, slab, true);
}
static void discard_slab(struct kmem_cache *s, struct slab *slab)
@@ -3373,8 +3377,6 @@ static void *alloc_single_from_partial(struct kmem_cache *s,
return object;
}
-static void defer_deactivate_slab(struct slab *slab, void *flush_freelist);
-
/*
* Called only for kmem_cache_debug() caches to allocate from a freshly
* allocated slab. Allocate a single object instead of whole freelist
@@ -3390,8 +3392,12 @@ static void *alloc_single_from_new_slab(struct kmem_cache *s, struct slab *slab,
void *object;
if (!allow_spin && !spin_trylock_irqsave(&n->list_lock, flags)) {
- /* Unlucky, discard newly allocated slab */
- defer_deactivate_slab(slab, NULL);
+ /*
+ * Unlucky, discard newly allocated slab.
+ * Since it was just allocated, we can skip the actions
+ * in discard_slab() and free_slab().
+ */
+ __free_slab(s, slab, false);
return NULL;
}
@@ -5949,7 +5955,6 @@ static void free_to_pcs_bulk(struct kmem_cache *s, size_t size, void **p)
struct defer_free {
struct llist_head objects;
- struct llist_head slabs;
struct irq_work work;
};
@@ -5957,7 +5962,6 @@ static void free_deferred_objects(struct irq_work *work);
static DEFINE_PER_CPU(struct defer_free, defer_free_objects) = {
.objects = LLIST_HEAD_INIT(objects),
- .slabs = LLIST_HEAD_INIT(slabs),
.work = IRQ_WORK_INIT(free_deferred_objects),
};
@@ -5970,10 +5974,9 @@ static void free_deferred_objects(struct irq_work *work)
{
struct defer_free *df = container_of(work, struct defer_free, work);
struct llist_head *objs = &df->objects;
- struct llist_head *slabs = &df->slabs;
struct llist_node *llnode, *pos, *t;
- if (llist_empty(objs) && llist_empty(slabs))
+ if (llist_empty(objs))
return;
llnode = llist_del_all(objs);
@@ -5997,16 +6000,6 @@ static void free_deferred_objects(struct irq_work *work)
__slab_free(s, slab, x, x, 1, _THIS_IP_);
}
-
- llnode = llist_del_all(slabs);
- llist_for_each_safe(pos, t, llnode) {
- struct slab *slab = container_of(pos, struct slab, llnode);
-
- if (slab->frozen)
- deactivate_slab(slab->slab_cache, slab, slab->flush_freelist);
- else
- free_slab(slab->slab_cache, slab);
- }
}
static void defer_free(struct kmem_cache *s, void *head)
@@ -6020,19 +6013,6 @@ static void defer_free(struct kmem_cache *s, void *head)
irq_work_queue(&df->work);
}
-static void defer_deactivate_slab(struct slab *slab, void *flush_freelist)
-{
- struct defer_free *df;
-
- slab->flush_freelist = flush_freelist;
-
- guard(preempt)();
-
- df = this_cpu_ptr(&defer_free_objects);
- if (llist_add(&slab->llnode, &df->slabs))
- irq_work_queue(&df->work);
-}
-
void defer_free_barrier(void)
{
int cpu;
--
2.51.1
Powered by blists - more mailing lists