| lists.openwall.net | lists / announce owl-users owl-dev john-users john-dev passwdqc-users yescrypt popa3d-users / oss-security kernel-hardening musl sabotage tlsify passwords / crypt-dev xvendor / Bugtraq Full-Disclosure linux-kernel linux-netdev linux-ext4 linux-hardening linux-cve-announce PHC | |
|
Open Source and information security mailing list archives
| ||
|
Message-ID: <20251113084022.1255121-7-hch@lst.de>
Date: Thu, 13 Nov 2025 09:39:47 +0100
From: Christoph Hellwig <hch@....de>
To: Vlastimil Babka <vbabka@...e.cz>,
Andrew Morton <akpm@...ux-foundation.org>
Cc: Christoph Lameter <cl@...two.org>,
David Rientjes <rientjes@...gle.com>,
Roman Gushchin <roman.gushchin@...ux.dev>,
Harry Yoo <harry.yoo@...cle.com>,
Suren Baghdasaryan <surenb@...gle.com>,
Michal Hocko <mhocko@...e.com>,
Brendan Jackman <jackmanb@...gle.com>,
Zi Yan <ziy@...dia.com>,
Eric Biggers <ebiggers@...nel.org>,
linux-mm@...ck.org,
linux-kernel@...r.kernel.org
Subject: [PATCH 06/11] mempool: factor out a mempool_alloc_from_pool helper
Add a helper for the mempool_alloc slowpath to better separate it from the
fast path, and also use it to implement mempool_alloc_preallocated which
shares the same logic.
Signed-off-by: Christoph Hellwig <hch@....de>
---
mm/mempool.c | 121 ++++++++++++++++++++++++---------------------------
1 file changed, 57 insertions(+), 64 deletions(-)
diff --git a/mm/mempool.c b/mm/mempool.c
index a0718a35c34f..c28087a3b8a9 100644
--- a/mm/mempool.c
+++ b/mm/mempool.c
@@ -380,6 +380,50 @@ int mempool_resize(mempool_t *pool, int new_min_nr)
}
EXPORT_SYMBOL(mempool_resize);
+static void *mempool_alloc_from_pool(struct mempool *pool, gfp_t gfp_mask)
+{
+ unsigned long flags;
+ void *element;
+
+ spin_lock_irqsave(&pool->lock, flags);
+ if (unlikely(!pool->curr_nr))
+ goto fail;
+ element = remove_element(pool);
+ spin_unlock_irqrestore(&pool->lock, flags);
+
+ /* Paired with rmb in mempool_free(), read comment there. */
+ smp_wmb();
+
+ /*
+ * Update the allocation stack trace as this is more useful for
+ * debugging.
+ */
+ kmemleak_update_trace(element);
+ return element;
+
+fail:
+ if (gfp_mask & __GFP_DIRECT_RECLAIM) {
+ DEFINE_WAIT(wait);
+
+ prepare_to_wait(&pool->wait, &wait, TASK_UNINTERRUPTIBLE);
+ spin_unlock_irqrestore(&pool->lock, flags);
+
+ /*
+ * Wait for someone else to return an element to @pool.
+ *
+ * FIXME: this should be io_schedule(). The timeout is there as
+ * a workaround for some DM problems in 2.6.18.
+ */
+ io_schedule_timeout(5 * HZ);
+ finish_wait(&pool->wait, &wait);
+ } else {
+ /* We must not sleep if __GFP_DIRECT_RECLAIM is not set. */
+ spin_unlock_irqrestore(&pool->lock, flags);
+ }
+
+ return NULL;
+}
+
/*
* Adjust the gfp flags for mempool allocations, as we never want to dip into
* the global emergency reserves or retry in the page allocator.
@@ -413,8 +457,6 @@ void *mempool_alloc_noprof(mempool_t *pool, gfp_t gfp_mask)
{
gfp_t gfp_temp = mempool_adjust_gfp(&gfp_mask);
void *element;
- unsigned long flags;
- wait_queue_entry_t wait;
VM_WARN_ON_ONCE(gfp_mask & __GFP_ZERO);
might_alloc(gfp_mask);
@@ -428,53 +470,22 @@ void *mempool_alloc_noprof(mempool_t *pool, gfp_t gfp_mask)
element = pool->alloc(gfp_temp, pool->pool_data);
}
- if (likely(element))
- return element;
-
- spin_lock_irqsave(&pool->lock, flags);
- if (likely(pool->curr_nr)) {
- element = remove_element(pool);
- spin_unlock_irqrestore(&pool->lock, flags);
- /* paired with rmb in mempool_free(), read comment there */
- smp_wmb();
+ if (unlikely(!element)) {
/*
- * Update the allocation stack trace as this is more useful
- * for debugging.
+ * Try to allocate an element from the pool.
+ *
+ * The first pass won't have __GFP_DIRECT_RECLAIM and won't
+ * sleep in mempool_alloc_from_pool. Retry the allocation
+ * with all flags set in that case.
*/
- kmemleak_update_trace(element);
- return element;
- }
-
- /*
- * We use gfp mask w/o direct reclaim or IO for the first round. If
- * alloc failed with that and @pool was empty, retry immediately.
- */
- if (gfp_temp != gfp_mask) {
- spin_unlock_irqrestore(&pool->lock, flags);
- gfp_temp = gfp_mask;
- goto repeat_alloc;
- }
-
- /* We must not sleep if !__GFP_DIRECT_RECLAIM */
- if (!(gfp_mask & __GFP_DIRECT_RECLAIM)) {
- spin_unlock_irqrestore(&pool->lock, flags);
- return NULL;
+ element = mempool_alloc_from_pool(pool, gfp_mask);
+ if (!element && gfp_temp != gfp_mask) {
+ gfp_temp = gfp_mask;
+ goto repeat_alloc;
+ }
}
- /* Let's wait for someone else to return an element to @pool */
- init_wait(&wait);
- prepare_to_wait(&pool->wait, &wait, TASK_UNINTERRUPTIBLE);
-
- spin_unlock_irqrestore(&pool->lock, flags);
-
- /*
- * FIXME: this should be io_schedule(). The timeout is there as a
- * workaround for some DM problems in 2.6.18.
- */
- io_schedule_timeout(5*HZ);
-
- finish_wait(&pool->wait, &wait);
- goto repeat_alloc;
+ return element;
}
EXPORT_SYMBOL(mempool_alloc_noprof);
@@ -492,25 +503,7 @@ EXPORT_SYMBOL(mempool_alloc_noprof);
*/
void *mempool_alloc_preallocated(mempool_t *pool)
{
- void *element;
- unsigned long flags;
-
- spin_lock_irqsave(&pool->lock, flags);
- if (likely(pool->curr_nr)) {
- element = remove_element(pool);
- spin_unlock_irqrestore(&pool->lock, flags);
- /* paired with rmb in mempool_free(), read comment there */
- smp_wmb();
- /*
- * Update the allocation stack trace as this is more useful
- * for debugging.
- */
- kmemleak_update_trace(element);
- return element;
- }
- spin_unlock_irqrestore(&pool->lock, flags);
-
- return NULL;
+ return mempool_alloc_from_pool(pool, GFP_NOWAIT);
}
EXPORT_SYMBOL(mempool_alloc_preallocated);
--
2.47.3
Powered by blists - more mailing lists