| lists.openwall.net | lists / announce owl-users owl-dev john-users john-dev passwdqc-users yescrypt popa3d-users / oss-security kernel-hardening musl sabotage tlsify passwords / crypt-dev xvendor / Bugtraq Full-Disclosure linux-kernel linux-netdev linux-ext4 linux-hardening linux-cve-announce PHC | |
|
Open Source and information security mailing list archives
| ||
|
Date: Tue, 3 Jul 2012 11:57:15 +0800
From: Jiang Liu <jiang.liu@...wei.com>
To: Christoph Lameter <cl@...ux-foundation.org>,
Pekka Enberg <penberg@...nel.org>,
Matt Mackall <mpm@...enic.com>, Mel Gorman <mgorman@...e.de>,
Yinghai Lu <yinghai@...nel.org>
CC: Jiang Liu <jiang.liu@...wei.com>, Tony Luck <tony.luck@...el.com>,
KAMEZAWA Hiroyuki <kamezawa.hiroyu@...fujitsu.com>,
KOSAKI Motohiro <kosaki.motohiro@...fujitsu.com>,
David Rientjes <rientjes@...gle.com>,
Minchan Kim <minchan@...nel.org>,
Keping Chen <chenkeping@...wei.com>, <linux-mm@...ck.org>,
<linux-kernel@...r.kernel.org>, Jiang Liu <liuj97@...il.com>
Subject: [RFC PATCH 2/4] mm: make consistent use of PG_slab flag
Currently there is some inconsistency with the usages of PG_slab flag.
The SLAB/SLUB/SLOB allocator uses PG_slab flag to mark whether a (compound)
page contains managed objects, but other subsystems use PG_slab flag
to detect whether a (compound) page is allocated/managed by SLAB/SLUB/SLOB.
It's OK with SLAB allocator because all pages allocated by SLAB will be
used to host SLAB object, thus with PG_slab flag set. But it may run into
trouble with SLUB/SLOB allocators. If the requested object is bigger enough,
SLUB/SLOB allocator directly depends on the page allocator to fulfill object
allocate/release requests. To distinguish whether a (compound) page hosts
SLUB/SLOB objects, SLUB/SLOB allocator only sets PG_slab flag on small pages
hosting SLUB/SLOB objects. So the PG_slab flag won't be set for large pages
allocated/managed by SLUB/SLOB allocator.
This patch splits the traditional PG_slab flag into two flags:
PG_slab: mark whether a (compound) page is allocated/managed by
SLAB/SLUB/SLOB.
PG_slabobject: mark whether a (compound) page hosts SLUB/SLOB objects.
The PG_slabobject flag won't be set on pages allocated by SLAB allocator
because it's redundant.
Signed-off-by: Jiang Liu <liuj97@...il.com>
---
include/linux/page-flags.h | 4 +++-
include/linux/slub_def.h | 3 +++
mm/slob.c | 21 +++++++++++++++++----
mm/slub.c | 22 ++++++++++++++--------
4 files changed, 37 insertions(+), 13 deletions(-)
diff --git a/include/linux/page-flags.h b/include/linux/page-flags.h
index c88d2a9..5fcf0b8 100644
--- a/include/linux/page-flags.h
+++ b/include/linux/page-flags.h
@@ -123,8 +123,9 @@ enum pageflags {
PG_pinned = PG_owner_priv_1,
PG_savepinned = PG_dirty,
- /* SLOB */
+ /* SLOB & SLUB */
PG_slob_free = PG_private,
+ PG_slab_object = PG_private_2,
};
#ifndef __GENERATING_BOUNDS_H
@@ -208,6 +209,7 @@ PAGEFLAG(SavePinned, savepinned); /* Xen */
PAGEFLAG(Reserved, reserved) __CLEARPAGEFLAG(Reserved, reserved)
PAGEFLAG(SwapBacked, swapbacked) __CLEARPAGEFLAG(SwapBacked, swapbacked)
+__PAGEFLAG(SlabObject, slab_object)
__PAGEFLAG(SlobFree, slob_free)
/*
diff --git a/include/linux/slub_def.h b/include/linux/slub_def.h
index c2f8c8b..e357c8d 100644
--- a/include/linux/slub_def.h
+++ b/include/linux/slub_def.h
@@ -11,6 +11,7 @@
#include <linux/bug.h>
#include <linux/workqueue.h>
#include <linux/kobject.h>
+#include <linux/page-flags.h>
#include <linux/kmemleak.h>
@@ -224,6 +225,8 @@ static __always_inline void *
kmalloc_order(size_t size, gfp_t flags, unsigned int order)
{
void *ret = (void *) __get_free_pages(flags | __GFP_COMP, order);
+ if (ret)
+ __SetPageSlab(virt_to_page(ret));
kmemleak_alloc(ret, size, 1, flags);
return ret;
}
diff --git a/mm/slob.c b/mm/slob.c
index 8105be4..2c1fa9c 100644
--- a/mm/slob.c
+++ b/mm/slob.c
@@ -135,17 +135,17 @@ static LIST_HEAD(free_slob_large);
*/
static inline int is_slob_page(struct slob_page *sp)
{
- return PageSlab((struct page *)sp);
+ return PageSlabObject((struct page *)sp);
}
static inline void set_slob_page(struct slob_page *sp)
{
- __SetPageSlab((struct page *)sp);
+ __SetPageSlabObject((struct page *)sp);
}
static inline void clear_slob_page(struct slob_page *sp)
{
- __ClearPageSlab((struct page *)sp);
+ __ClearPageSlabObject((struct page *)sp);
}
static inline struct slob_page *slob_page(const void *addr)
@@ -242,7 +242,7 @@ static int slob_last(slob_t *s)
static void *slob_new_pages(gfp_t gfp, int order, int node)
{
- void *page;
+ struct page *page, *pos;
#ifdef CONFIG_NUMA
if (node != -1)
@@ -254,11 +254,24 @@ static void *slob_new_pages(gfp_t gfp, int order, int node)
if (!page)
return NULL;
+ /* Only set PG_slab flag on head page in case of compound page */
+ if (gfp & __GFP_COMP)
+ order = 0;
+ for (pos = page + (1 << order) - 1; pos >= page; pos--)
+ __SetPageSlab(pos);
+
return page_address(page);
}
static void slob_free_pages(void *b, int order)
{
+ struct page *pos, *end;
+
+ pos = virt_to_page(b);
+ end = pos + (PageCompound(pos) ? 1 : 1 << order);
+ for (; pos < end; pos++)
+ __ClearPageSlab(pos);
+
if (current->reclaim_state)
current->reclaim_state->reclaimed_slab += 1 << order;
free_pages((unsigned long)b, order);
diff --git a/mm/slub.c b/mm/slub.c
index 8c691fa..9dc6524 100644
--- a/mm/slub.c
+++ b/mm/slub.c
@@ -840,7 +840,7 @@ static int check_slab(struct kmem_cache *s, struct page *page)
VM_BUG_ON(!irqs_disabled());
- if (!PageSlab(page)) {
+ if (!PageSlabObject(page)) {
slab_err(s, page, "Not a valid slab page");
return 0;
}
@@ -1069,7 +1069,7 @@ static noinline int alloc_debug_processing(struct kmem_cache *s, struct page *pa
return 1;
bad:
- if (PageSlab(page)) {
+ if (PageSlabObject(page)) {
/*
* If this is a slab page then lets do the best we can
* to avoid issues in the future. Marking all objects
@@ -1108,7 +1108,7 @@ static noinline int free_debug_processing(struct kmem_cache *s,
goto out;
if (unlikely(s != page->slab)) {
- if (!PageSlab(page)) {
+ if (!PageSlabObject(page)) {
slab_err(s, page, "Attempt to free object(0x%p) "
"outside of slab", object);
} else if (!page->slab) {
@@ -1370,6 +1370,7 @@ static struct page *new_slab(struct kmem_cache *s, gfp_t flags, int node)
inc_slabs_node(s, page_to_nid(page), page->objects);
page->slab = s;
__SetPageSlab(page);
+ __SetPageSlabObject(page);
start = page_address(page);
@@ -1413,6 +1414,7 @@ static void __free_slab(struct kmem_cache *s, struct page *page)
NR_SLAB_RECLAIMABLE : NR_SLAB_UNRECLAIMABLE,
-pages);
+ __ClearPageSlabObject(page);
__ClearPageSlab(page);
reset_page_mapcount(page);
if (current->reclaim_state)
@@ -3369,8 +3371,10 @@ static void *kmalloc_large_node(size_t size, gfp_t flags, int node)
flags |= __GFP_COMP | __GFP_NOTRACK;
page = alloc_pages_node(node, flags, get_order(size));
- if (page)
+ if (page) {
+ __SetPageSlab(page);
ptr = page_address(page);
+ }
kmemleak_alloc(ptr, size, 1, flags);
return ptr;
@@ -3414,7 +3418,7 @@ size_t ksize(const void *object)
page = virt_to_head_page(object);
- if (unlikely(!PageSlab(page))) {
+ if (unlikely(!PageSlabObject(page))) {
WARN_ON(!PageCompound(page));
return PAGE_SIZE << compound_order(page);
}
@@ -3437,7 +3441,7 @@ bool verify_mem_not_deleted(const void *x)
local_irq_save(flags);
page = virt_to_head_page(x);
- if (unlikely(!PageSlab(page))) {
+ if (unlikely(!PageSlabObject(page))) {
/* maybe it was from stack? */
rv = true;
goto out_unlock;
@@ -3470,9 +3474,10 @@ void kfree(const void *x)
return;
page = virt_to_head_page(x);
- if (unlikely(!PageSlab(page))) {
+ if (unlikely(!PageSlabObject(page))) {
BUG_ON(!PageCompound(page));
kmemleak_free(x);
+ __ClearPageSlab(page);
put_page(page);
return;
}
@@ -3715,7 +3720,8 @@ void __init kmem_cache_init(void)
/* Allocate two kmem_caches from the page allocator */
kmalloc_size = ALIGN(kmem_size, cache_line_size());
order = get_order(2 * kmalloc_size);
- kmem_cache = (void *)__get_free_pages(GFP_NOWAIT, order);
+ kmem_cache = (void *)__get_free_pages(GFP_NOWAIT | __GFP_COMP, order);
+ __SetPageSlab(virt_to_page(kmem_cache));
/*
* Must first have the slab cache available for the allocations of the
--
1.7.1
--
To unsubscribe from this list: send the line "unsubscribe linux-kernel" in
the body of a message to majordomo@...r.kernel.org
More majordomo info at http://vger.kernel.org/majordomo-info.html
Please read the FAQ at http://www.tux.org/lkml/
Powered by blists - more mailing lists