linux-kernel - Re: [RFC][PATCH] mm/slub: Skip CPU slab activation when debugging

lists.openwall.net		lists / announce owl-users owl-dev john-users john-dev passwdqc-users yescrypt popa3d-users / oss-security kernel-hardening musl sabotage tlsify passwords / crypt-dev xvendor / Bugtraq Full-Disclosure linux-kernel linux-netdev linux-ext4 linux-hardening linux-cve-announce PHC
Open Source and information security mailing list archives

Hash Suite: Windows password security audit tool. GUI, reports in PDF.

[<prev] [next>] [<thread-prev] [thread-next>] [day] [month] [year] [list]

Message-ID: <56F9DFC3.501@redhat.com>
Date:	Mon, 28 Mar 2016 18:52:03 -0700
From:	Laura Abbott <labbott@...hat.com>
To:	Christoph Lameter <cl@...ux.com>,
	Pekka Enberg <penberg@...nel.org>,
	David Rientjes <rientjes@...gle.com>,
	Joonsoo Kim <js1304@...il.com>,
	Andrew Morton <akpm@...ux-foundation.org>
Cc:	linux-mm@...ck.org, linux-kernel@...r.kernel.org,
	Kees Cook <keescook@...omium.org>
Subject: Re: [RFC][PATCH] mm/slub: Skip CPU slab activation when debugging

On 03/28/2016 03:53 PM, Laura Abbott wrote:
> The per-cpu slab is designed to be the primary path for allocation in SLUB
> since it assumed allocations will go through the fast path if possible.
> When debugging is enabled, the fast path is disabled and per-cpu
> allocations are not used. The current debugging code path still activates
> the cpu slab for allocations and then immediately deactivates it. This
> is useless work. When a slab is enabled for debugging, skip cpu
> activation.
>
> Signed-off-by: Laura Abbott <labbott@...oraproject.org>
> ---
> This is a follow on to the optimization of the debug paths for poisoning
> With this I get ~2 second drop on hackbench -g 20 -l 1000 with slub_debug=P
> and no noticable change with slub_debug=- .

zero day robot pointed out this is triggering one of the BUG_ON on bootup.
I'll take a deeper look tomorrow unless the approach is actually worthless.
> ---
>   mm/slub.c | 82 +++++++++++++++++++++++++++++++++++++++++++++++++++++++++++----
>   1 file changed, 77 insertions(+), 5 deletions(-)
>
> diff --git a/mm/slub.c b/mm/slub.c
> index 7277413..4507bd8 100644
> --- a/mm/slub.c
> +++ b/mm/slub.c
> @@ -1482,8 +1482,8 @@ static struct page *allocate_slab(struct kmem_cache *s, gfp_t flags, int node)
>   	}
>
>   	page->freelist = fixup_red_left(s, start);
> -	page->inuse = page->objects;
> -	page->frozen = 1;
> +	page->inuse = kmem_cache_debug(s) ? 1 : page->objects;
> +	page->frozen = kmem_cache_debug(s) ? 0 : 1;
>
>   out:
>   	if (gfpflags_allow_blocking(flags))
> @@ -1658,6 +1658,64 @@ static inline void *acquire_slab(struct kmem_cache *s,
>   	return freelist;
>   }
>
> +
> +static inline void *acquire_slab_debug(struct kmem_cache *s,
> +		struct kmem_cache_node *n, struct page *page,
> +		int mode, int *objects)
> +{
> +	void *freelist;
> +	unsigned long counters;
> +	struct page new;
> +	void *next;
> +
> +	lockdep_assert_held(&n->list_lock);
> +
> +
> +	/*
> +	 * Zap the freelist and set the frozen bit.
> +	 * The old freelist is the list of objects for the
> +	 * per cpu allocation list.
> +	 */
> +	freelist = page->freelist;
> +	counters = page->counters;
> +
> +	BUG_ON(!freelist);
> +
> +	next = get_freepointer_safe(s, freelist);
> +
> +	new.counters = counters;
> +	*objects = new.objects - new.inuse;
> +	if (mode) {
> +		new.inuse++;
> +		new.freelist = next;
> +	} else {
> +		BUG();
> +	}
> +
> +	VM_BUG_ON(new.frozen);
> +
> +	if (!new.freelist) {
> +		remove_partial(n, page);
> +		add_full(s, n, page);
> +	}
> +
> +	if (!__cmpxchg_double_slab(s, page,
> +			freelist, counters,
> +			new.freelist, new.counters,
> +			"acquire_slab")) {
> +		if (!new.freelist) {
> +			remove_full(s, n, page);
> +			add_partial(n, page, DEACTIVATE_TO_HEAD);
> +		}
> +		return NULL;
> +	}
> +
> +	WARN_ON(!freelist);
> +	return freelist;
> +}
> +
> +
> +
>   static void put_cpu_partial(struct kmem_cache *s, struct page *page, int drain);
>   static inline bool pfmemalloc_match(struct page *page, gfp_t gfpflags);
>
> @@ -1688,7 +1746,11 @@ static void *get_partial_node(struct kmem_cache *s, struct kmem_cache_node *n,
>   		if (!pfmemalloc_match(page, flags))
>   			continue;
>
> -		t = acquire_slab(s, n, page, object == NULL, &objects);
> +		if (kmem_cache_debug(s))
> +			t = acquire_slab_debug(s, n, page, object == NULL, &objects);
> +		else
> +			t = acquire_slab(s, n, page, object == NULL, &objects);
> +
>   		if (!t)
>   			break;
>
> @@ -2284,7 +2346,17 @@ static inline void *new_slab_objects(struct kmem_cache *s, gfp_t flags,
>   		 * muck around with it freely without cmpxchg
>   		 */
>   		freelist = page->freelist;
> -		page->freelist = NULL;
> +		page->freelist = kmem_cache_debug(s) ?
> +				get_freepointer(s, freelist) : NULL;
> +
> +		if (kmem_cache_debug(s)) {
> +			struct kmem_cache_node *n;
> +
> +			n = get_node(s, page_to_nid(page));
> +			spin_lock(&n->list_lock);
> +			add_partial(n, page, DEACTIVATE_TO_HEAD);
> +			spin_unlock(&n->list_lock);
> +		}
>
>   		stat(s, ALLOC_SLAB);
>   		c->page = page;
> @@ -2446,7 +2518,7 @@ new_slab:
>   			!alloc_debug_processing(s, page, freelist, addr))
>   		goto new_slab;	/* Slab failed checks. Next slab needed */
>
> -	deactivate_slab(s, page, get_freepointer(s, freelist));
> +	/* No need to deactivate, no cpu slab */
>   	c->page = NULL;
>   	c->freelist = NULL;
>   	return freelist;
>