>From 825f5179174be825f1219e23bdde769ef9febd45 Mon Sep 17 00:00:00 2001 From: Glauber Costa Date: Mon, 22 Oct 2012 15:20:26 +0400 Subject: [PATCH] slab: move kmem_cache_free to common code In the effort of commonizing the slab allocators, it would be better if we had a single entry-point for kmem_cache_free. At the same time, the different layout of the allocators lead to routines that are necessarily different. Because we would also want the allocators to be able to free objects in their fast paths without issuing a call instruction, we will rely on the pre-processor to aid us. Signed-off-by: Glauber Costa CC: Christoph Lameter CC: Pekka Enberg CC: David Rientjes --- mm/slab.c | 15 +++------------ mm/slab.h | 26 ++++++++++++++++++++++++++ mm/slob.c | 6 ++---- mm/slub.c | 6 ++---- 4 files changed, 33 insertions(+), 20 deletions(-) diff --git a/mm/slab.c b/mm/slab.c index 98b3460..bceffcc 100644 --- a/mm/slab.c +++ b/mm/slab.c @@ -3903,15 +3903,8 @@ void *__kmalloc(size_t size, gfp_t flags) EXPORT_SYMBOL(__kmalloc); #endif -/** - * kmem_cache_free - Deallocate an object - * @cachep: The cache the allocation was from. - * @objp: The previously allocated object. - * - * Free an object which was previously allocated from this - * cache. - */ -void kmem_cache_free(struct kmem_cache *cachep, void *objp) +static __always_inline void +__kmem_cache_free(struct kmem_cache *cachep, void *objp) { unsigned long flags; @@ -3921,10 +3914,8 @@ void kmem_cache_free(struct kmem_cache *cachep, void *objp) debug_check_no_obj_freed(objp, cachep->object_size); __cache_free(cachep, objp, __builtin_return_address(0)); local_irq_restore(flags); - - trace_kmem_cache_free(_RET_IP_, objp); } -EXPORT_SYMBOL(kmem_cache_free); +KMEM_CACHE_FREE(__kmem_cache_free); /** * kfree - free previously allocated memory diff --git a/mm/slab.h b/mm/slab.h index 66a62d3..32063f8 100644 --- a/mm/slab.h +++ b/mm/slab.h @@ -92,4 +92,30 @@ void get_slabinfo(struct kmem_cache *s, struct slabinfo *sinfo); void slabinfo_show_stats(struct seq_file *m, struct kmem_cache *s); ssize_t slabinfo_write(struct file *file, const char __user *buffer, size_t count, loff_t *ppos); + +/* + * What goes below for kmem_cache_free is not pretty. But because this + * is an extremely hot path, we would like to avoid function calls as + * much as we can. + * + * As ugly as it is, this is a way to guarantee that different allocators, + * with different layouts, and therefore, different free functions, can + * still live in different files and inline the whole of kmem_cache_free. + */ +/** + * kmem_cache_free - Deallocate an object + * @cachep: The cache the allocation was from. + * @objp: The previously allocated object. + * + * Free an object which was previously allocated from this + * cache. + * + */ +#define KMEM_CACHE_FREE(allocator_fn) \ +void kmem_cache_free(struct kmem_cache *s, void *x) \ +{ \ + allocator_fn(s, x); \ + trace_kmem_cache_free(_RET_IP_, x); \ +} \ +EXPORT_SYMBOL(kmem_cache_free) #endif diff --git a/mm/slob.c b/mm/slob.c index 3edfeaa..1580371 100644 --- a/mm/slob.c +++ b/mm/slob.c @@ -571,7 +571,7 @@ static void kmem_rcu_free(struct rcu_head *head) __kmem_cache_free(b, slob_rcu->size); } -void kmem_cache_free(struct kmem_cache *c, void *b) +static __always_inline void do_kmem_cache_free(struct kmem_cache *c, void *b) { kmemleak_free_recursive(b, c->flags); if (unlikely(c->flags & SLAB_DESTROY_BY_RCU)) { @@ -582,10 +582,8 @@ void kmem_cache_free(struct kmem_cache *c, void *b) } else { __kmem_cache_free(b, c->size); } - - trace_kmem_cache_free(_RET_IP_, b); } -EXPORT_SYMBOL(kmem_cache_free); +KMEM_CACHE_FREE(do_kmem_cache_free); unsigned int kmem_cache_size(struct kmem_cache *c) { diff --git a/mm/slub.c b/mm/slub.c index 259bc2c..7dd41d7 100644 --- a/mm/slub.c +++ b/mm/slub.c @@ -2606,7 +2606,7 @@ redo: } -void kmem_cache_free(struct kmem_cache *s, void *x) +static __always_inline void __kmem_cache_free(struct kmem_cache *s, void *x) { struct page *page; @@ -2620,10 +2620,8 @@ void kmem_cache_free(struct kmem_cache *s, void *x) } slab_free(s, page, x, _RET_IP_); - - trace_kmem_cache_free(_RET_IP_, x); } -EXPORT_SYMBOL(kmem_cache_free); +KMEM_CACHE_FREE(__kmem_cache_free); /* * Object placement in a slab is made very easy because we always start at -- 1.7.11.7