| lists.openwall.net | lists / announce owl-users owl-dev john-users john-dev passwdqc-users yescrypt popa3d-users / oss-security kernel-hardening musl sabotage tlsify passwords / crypt-dev xvendor / Bugtraq Full-Disclosure linux-kernel linux-netdev linux-ext4 linux-hardening linux-cve-announce PHC | |
|
Open Source and information security mailing list archives
| ||
|
Message-ID: <20240709132741.47751-9-linyunsheng@huawei.com>
Date: Tue, 9 Jul 2024 21:27:33 +0800
From: Yunsheng Lin <linyunsheng@...wei.com>
To: <davem@...emloft.net>, <kuba@...nel.org>, <pabeni@...hat.com>
CC: <netdev@...r.kernel.org>, <linux-kernel@...r.kernel.org>, Yunsheng Lin
<linyunsheng@...wei.com>, Alexander Duyck <alexander.duyck@...il.com>, Andrew
Morton <akpm@...ux-foundation.org>, <linux-mm@...ck.org>
Subject: [PATCH net-next v10 08/15] mm: page_frag: some minor refactoring before adding new API
Refactor common codes from __page_frag_alloc_va_align()
to __page_frag_cache_refill(), so that the new API can
make use of them.
CC: Alexander Duyck <alexander.duyck@...il.com>
Signed-off-by: Yunsheng Lin <linyunsheng@...wei.com>
---
include/linux/page_frag_cache.h | 2 +-
mm/page_frag_cache.c | 96 +++++++++++++++++----------------
2 files changed, 50 insertions(+), 48 deletions(-)
diff --git a/include/linux/page_frag_cache.h b/include/linux/page_frag_cache.h
index 87c3eb728f95..71e08db1eb2f 100644
--- a/include/linux/page_frag_cache.h
+++ b/include/linux/page_frag_cache.h
@@ -68,7 +68,7 @@ struct page_frag_cache {
static inline void page_frag_cache_init(struct page_frag_cache *nc)
{
- nc->encoded_va = 0;
+ memset(nc, 0, sizeof(*nc));
}
static inline bool page_frag_cache_is_pfmemalloc(struct page_frag_cache *nc)
diff --git a/mm/page_frag_cache.c b/mm/page_frag_cache.c
index 02e4ec92f948..73164d2482f8 100644
--- a/mm/page_frag_cache.c
+++ b/mm/page_frag_cache.c
@@ -19,6 +19,28 @@
#include <linux/page_frag_cache.h>
#include "internal.h"
+static struct page *__page_frag_cache_recharge(struct page_frag_cache *nc)
+{
+ unsigned long encoded_va = nc->encoded_va;
+ struct page *page;
+
+ page = virt_to_page((void *)encoded_va);
+ if (!page_ref_sub_and_test(page, nc->pagecnt_bias))
+ return NULL;
+
+ if (unlikely(encoded_page_pfmemalloc(encoded_va))) {
+ VM_BUG_ON(compound_order(page) !=
+ encoded_page_order(encoded_va));
+ free_unref_page(page, encoded_page_order(encoded_va));
+ return NULL;
+ }
+
+ /* OK, page count is 0, we can safely set it */
+ set_page_count(page, PAGE_FRAG_CACHE_MAX_SIZE + 1);
+
+ return page;
+}
+
static struct page *__page_frag_cache_refill(struct page_frag_cache *nc,
gfp_t gfp_mask)
{
@@ -26,6 +48,14 @@ static struct page *__page_frag_cache_refill(struct page_frag_cache *nc,
struct page *page = NULL;
gfp_t gfp = gfp_mask;
+ if (likely(nc->encoded_va)) {
+ page = __page_frag_cache_recharge(nc);
+ if (page) {
+ order = encoded_page_order(nc->encoded_va);
+ goto out;
+ }
+ }
+
#if (PAGE_SIZE < PAGE_FRAG_CACHE_MAX_SIZE)
gfp_mask = (gfp_mask & ~__GFP_DIRECT_RECLAIM) | __GFP_COMP |
__GFP_NOWARN | __GFP_NORETRY | __GFP_NOMEMALLOC;
@@ -35,7 +65,7 @@ static struct page *__page_frag_cache_refill(struct page_frag_cache *nc,
if (unlikely(!page)) {
page = alloc_pages_node(NUMA_NO_NODE, gfp, 0);
if (unlikely(!page)) {
- nc->encoded_va = 0;
+ memset(nc, 0, sizeof(*nc));
return NULL;
}
@@ -45,6 +75,16 @@ static struct page *__page_frag_cache_refill(struct page_frag_cache *nc,
nc->encoded_va = encode_aligned_va(page_address(page), order,
page_is_pfmemalloc(page));
+ /* Even if we own the page, we do not use atomic_set().
+ * This would break get_page_unless_zero() users.
+ */
+ page_ref_add(page, PAGE_FRAG_CACHE_MAX_SIZE);
+
+out:
+ /* reset page count bias and remaining to start of new frag */
+ nc->pagecnt_bias = PAGE_FRAG_CACHE_MAX_SIZE + 1;
+ nc->remaining = PAGE_SIZE << order;
+
return page;
}
@@ -55,7 +95,7 @@ void page_frag_cache_drain(struct page_frag_cache *nc)
__page_frag_cache_drain(virt_to_head_page((void *)nc->encoded_va),
nc->pagecnt_bias);
- nc->encoded_va = 0;
+ memset(nc, 0, sizeof(*nc));
}
EXPORT_SYMBOL(page_frag_cache_drain);
@@ -72,53 +112,15 @@ void *__page_frag_alloc_va_align(struct page_frag_cache *nc,
unsigned int fragsz, gfp_t gfp_mask,
unsigned int align_mask)
{
- unsigned long encoded_va = nc->encoded_va;
- int aligned_remaining, remaining;
- unsigned int size;
- struct page *page;
-
- if (unlikely(!encoded_va)) {
-refill:
- page = __page_frag_cache_refill(nc, gfp_mask);
- if (!page)
- return NULL;
-
- encoded_va = nc->encoded_va;
- size = page_frag_cache_page_size(encoded_va);
+ unsigned int size = page_frag_cache_page_size(nc->encoded_va);
+ int aligned_remaining = nc->remaining & align_mask;
+ int remaining = aligned_remaining - fragsz;
- /* Even if we own the page, we do not use atomic_set().
- * This would break get_page_unless_zero() users.
- */
- page_ref_add(page, PAGE_FRAG_CACHE_MAX_SIZE);
-
- /* reset page count bias and remaining to start of new frag */
- nc->pagecnt_bias = PAGE_FRAG_CACHE_MAX_SIZE + 1;
- nc->remaining = size;
- }
-
- size = page_frag_cache_page_size(encoded_va);
-
- aligned_remaining = nc->remaining & align_mask;
- remaining = aligned_remaining - fragsz;
if (unlikely(remaining < 0)) {
- page = virt_to_page((void *)encoded_va);
-
- if (!page_ref_sub_and_test(page, nc->pagecnt_bias))
- goto refill;
-
- if (unlikely(encoded_page_pfmemalloc(encoded_va))) {
- VM_BUG_ON(compound_order(page) !=
- encoded_page_order(encoded_va));
- free_unref_page(page, encoded_page_order(encoded_va));
- goto refill;
- }
-
- /* OK, page count is 0, we can safely set it */
- set_page_count(page, PAGE_FRAG_CACHE_MAX_SIZE + 1);
+ if (!__page_frag_cache_refill(nc, gfp_mask))
+ return NULL;
- /* reset page count bias and remaining to start of new frag */
- nc->pagecnt_bias = PAGE_FRAG_CACHE_MAX_SIZE + 1;
- nc->remaining = size;
+ size = page_frag_cache_page_size(nc->encoded_va);
aligned_remaining = size;
remaining = aligned_remaining - fragsz;
@@ -139,7 +141,7 @@ void *__page_frag_alloc_va_align(struct page_frag_cache *nc,
nc->pagecnt_bias--;
nc->remaining = remaining;
- return encoded_page_address(encoded_va) + (size - aligned_remaining);
+ return encoded_page_address(nc->encoded_va) + (size - aligned_remaining);
}
EXPORT_SYMBOL(__page_frag_alloc_va_align);
--
2.33.0
Powered by blists - more mailing lists