| lists.openwall.net | lists / announce owl-users owl-dev john-users john-dev passwdqc-users yescrypt popa3d-users / oss-security kernel-hardening musl sabotage tlsify passwords / crypt-dev xvendor / Bugtraq Full-Disclosure linux-kernel linux-netdev linux-ext4 linux-hardening linux-cve-announce PHC | |
|
Open Source and information security mailing list archives
| ||
|
Message-ID: <20250924-b4-asi-page-alloc-v1-16-2d861768041f@google.com>
Date: Wed, 24 Sep 2025 14:59:51 +0000
From: Brendan Jackman <jackmanb@...gle.com>
To: jackmanb@...gle.com, Andy Lutomirski <luto@...nel.org>,
Lorenzo Stoakes <lorenzo.stoakes@...cle.com>, "Liam R. Howlett" <Liam.Howlett@...cle.com>,
Suren Baghdasaryan <surenb@...gle.com>, Michal Hocko <mhocko@...e.com>,
Johannes Weiner <hannes@...xchg.org>, Zi Yan <ziy@...dia.com>,
Axel Rasmussen <axelrasmussen@...gle.com>, Yuanchu Xie <yuanchu@...gle.com>,
Roman Gushchin <roman.gushchin@...ux.dev>
Cc: peterz@...radead.org, bp@...en8.de, dave.hansen@...ux.intel.com,
mingo@...hat.com, tglx@...utronix.de, akpm@...ux-foundation.org,
david@...hat.com, derkling@...gle.com, junaids@...gle.com,
linux-kernel@...r.kernel.org, linux-mm@...ck.org, reijiw@...gle.com,
rientjes@...gle.com, rppt@...nel.org, vbabka@...e.cz, x86@...nel.org,
yosry.ahmed@...ux.dev
Subject: [PATCH 16/21] mm/page_alloc: introduce ALLOC_NOBLOCK
This flag is set unless we can be sure the caller isn't in an atomic
context.
The allocator will soon start needing to call set_direct_map_* APIs
which cannot be called with IRQs off. It will need to do this even
before direct reclaim is possible.
Despite the fact that, in principle, ALLOC_NOBLOCK is distinct from
__GFP_DIRECT_RECLAIM, in order to avoid introducing a GFP flag, just
infer the former based on whether the caller set the latter. This means
that, in practice, ALLOC_NOBLOCK is just !__GFP_DIRECT_RECLAIM, except
that it is not influenced by gfp_allowed_mask.
Call it ALLOC_NOBLOCK in order to try and mitigate confusion vs the
recently-removed ALLOC_NON_BLOCK, which meant something different.
Signed-off-by: Brendan Jackman <jackmanb@...gle.com>
---
mm/internal.h | 1 +
mm/page_alloc.c | 29 ++++++++++++++++++++++-------
2 files changed, 23 insertions(+), 7 deletions(-)
diff --git a/mm/internal.h b/mm/internal.h
index 513aba6c00bed813c9e38464aec5a15e65edaa58..c697ed35a8ca3376445d1e4249e9ce03097f15b8 100644
--- a/mm/internal.h
+++ b/mm/internal.h
@@ -1315,6 +1315,7 @@ unsigned int reclaim_clean_pages_from_list(struct zone *zone,
#define ALLOC_HIGHATOMIC 0x200 /* Allows access to MIGRATE_HIGHATOMIC */
#define ALLOC_TRYLOCK 0x400 /* Only use spin_trylock in allocation path */
#define ALLOC_KSWAPD 0x800 /* allow waking of kswapd, __GFP_KSWAPD_RECLAIM set */
+#define ALLOC_NOBLOCK 0x1000 /* Caller may be atomic */
/* Flags that allow allocations below the min watermark. */
#define ALLOC_RESERVES (ALLOC_HARDER|ALLOC_MIN_RESERVE|ALLOC_HIGHATOMIC|ALLOC_OOM)
diff --git a/mm/page_alloc.c b/mm/page_alloc.c
index cd47cfaae820ce696d2e6e0c47436e00d3feef60..b0aeb97baa13af038fff0edae33affbbf49e825c 100644
--- a/mm/page_alloc.c
+++ b/mm/page_alloc.c
@@ -4552,6 +4552,8 @@ gfp_to_alloc_flags(gfp_t gfp_mask, unsigned int order)
(gfp_mask & (__GFP_HIGH | __GFP_KSWAPD_RECLAIM));
if (!(gfp_mask & __GFP_DIRECT_RECLAIM)) {
+ alloc_flags |= ALLOC_NOBLOCK;
+
/*
* Not worth trying to allocate harder for __GFP_NOMEMALLOC even
* if it can't schedule.
@@ -4745,14 +4747,13 @@ check_retry_cpuset(int cpuset_mems_cookie, struct alloc_context *ac)
static inline struct page *
__alloc_pages_slowpath(gfp_t gfp_mask, unsigned int order,
- struct alloc_context *ac)
+ struct alloc_context *ac, unsigned int alloc_flags)
{
bool can_direct_reclaim = gfp_mask & __GFP_DIRECT_RECLAIM;
bool can_compact = gfp_compaction_allowed(gfp_mask);
bool nofail = gfp_mask & __GFP_NOFAIL;
const bool costly_order = order > PAGE_ALLOC_COSTLY_ORDER;
struct page *page = NULL;
- unsigned int alloc_flags;
unsigned long did_some_progress;
enum compact_priority compact_priority;
enum compact_result compact_result;
@@ -4795,7 +4796,7 @@ __alloc_pages_slowpath(gfp_t gfp_mask, unsigned int order,
* kswapd needs to be woken up, and to avoid the cost of setting up
* alloc_flags precisely. So we do that now.
*/
- alloc_flags = gfp_to_alloc_flags(gfp_mask, order);
+ alloc_flags |= gfp_to_alloc_flags(gfp_mask, order);
/*
* We need to recalculate the starting point for the zonelist iterator
@@ -5045,6 +5046,19 @@ __alloc_pages_slowpath(gfp_t gfp_mask, unsigned int order,
return page;
}
+static inline unsigned int init_alloc_flags(gfp_t gfp_mask, unsigned int flags)
+{
+ /*
+ * If the caller allowed __GFP_DIRECT_RECLAIM, they can't be atomic.
+ * Note this is a separate determination from whether direct
+ * reclaim is actually allowed, it must happen before applying
+ * gfp_allowed_mask.
+ */
+ if (!(gfp_mask & __GFP_DIRECT_RECLAIM))
+ flags |= ALLOC_NOBLOCK;
+ return flags;
+}
+
static inline bool prepare_alloc_pages(gfp_t gfp_mask, unsigned int order,
int preferred_nid, nodemask_t *nodemask,
struct alloc_context *ac, gfp_t *alloc_gfp,
@@ -5121,7 +5135,7 @@ unsigned long alloc_pages_bulk_noprof(gfp_t gfp, int preferred_nid,
struct list_head *pcp_list;
struct alloc_context ac;
gfp_t alloc_gfp;
- unsigned int alloc_flags = ALLOC_WMARK_LOW;
+ unsigned int alloc_flags = init_alloc_flags(gfp, ALLOC_WMARK_LOW);
int nr_populated = 0, nr_account = 0;
/*
@@ -5267,7 +5281,7 @@ struct page *__alloc_frozen_pages_noprof(gfp_t gfp, unsigned int order,
int preferred_nid, nodemask_t *nodemask)
{
struct page *page;
- unsigned int alloc_flags = ALLOC_WMARK_LOW;
+ unsigned int alloc_flags = init_alloc_flags(gfp, ALLOC_WMARK_LOW);
gfp_t alloc_gfp; /* The gfp_t that was actually used for allocation */
struct alloc_context ac = { };
@@ -5319,7 +5333,7 @@ struct page *__alloc_frozen_pages_noprof(gfp_t gfp, unsigned int order,
*/
ac.nodemask = nodemask;
- page = __alloc_pages_slowpath(alloc_gfp, order, &ac);
+ page = __alloc_pages_slowpath(alloc_gfp, order, &ac, alloc_flags);
out:
if (memcg_kmem_online() && (gfp & __GFP_ACCOUNT) && page &&
@@ -7684,10 +7698,11 @@ struct page *alloc_frozen_pages_nolock_noprof(gfp_t gfp_flags, int nid, unsigned
*/
gfp_t alloc_gfp = __GFP_NOWARN | __GFP_ZERO | __GFP_NOMEMALLOC | __GFP_COMP
| gfp_flags;
- unsigned int alloc_flags = ALLOC_TRYLOCK;
+ unsigned int alloc_flags = init_alloc_flags(gfp_flags, ALLOC_TRYLOCK);
struct alloc_context ac = { };
struct page *page;
+ VM_WARN_ON_ONCE(!(alloc_flags & ALLOC_NOBLOCK));
VM_WARN_ON_ONCE(gfp_flags & ~__GFP_ACCOUNT);
/*
* In PREEMPT_RT spin_trylock() will call raw_spin_lock() which is
--
2.50.1
Powered by blists - more mailing lists