There is no need anymore to return the pointer to a slab page from get_partial() since it can be assigned to the kmem_cache_cpu structures "page" field. Return an object pointer instead. That in turn allows a simplification of the spaghetti code in __slab_alloc(). Signed-off-by: Christoph Lameter --- mm/slub.c | 130 ++++++++++++++++++++++++++++++++++---------------------------- 1 file changed, 73 insertions(+), 57 deletions(-) Index: linux-2.6/mm/slub.c =================================================================== --- linux-2.6.orig/mm/slub.c 2011-05-31 10:20:23.322974925 -0500 +++ linux-2.6/mm/slub.c 2011-05-31 10:20:26.822974896 -0500 @@ -1448,9 +1448,11 @@ static inline void remove_partial(struct * Lock slab, remove from the partial list and put the object into the * per cpu freelist. * + * Returns a list of objects or NULL if it fails. + * * Must hold list_lock. */ -static inline int acquire_slab(struct kmem_cache *s, +static inline void *acquire_slab(struct kmem_cache *s, struct kmem_cache_node *n, struct page *page, struct kmem_cache_cpu *c) { @@ -1481,10 +1483,11 @@ static inline int acquire_slab(struct km if (freelist) { /* Populate the per cpu freelist */ - c->freelist = freelist; c->page = page; c->node = page_to_nid(page); - return 1; + stat(s, ALLOC_FROM_PARTIAL); + + return freelist; } else { /* * Slab page came from the wrong list. No object to allocate @@ -1493,17 +1496,18 @@ static inline int acquire_slab(struct km */ printk(KERN_ERR "SLUB: %s : Page without available objects on" " partial list\n", s->name); - return 0; + return NULL; } } /* * Try to allocate a partial slab from a specific node. */ -static struct page *get_partial_node(struct kmem_cache *s, +static void *get_partial_node(struct kmem_cache *s, struct kmem_cache_node *n, struct kmem_cache_cpu *c) { struct page *page; + void *object; /* * Racy check. If we mistakenly see no partial slabs then we @@ -1515,13 +1519,15 @@ static struct page *get_partial_node(str return NULL; spin_lock(&n->list_lock); - list_for_each_entry(page, &n->partial, lru) - if (acquire_slab(s, n, page, c)) + list_for_each_entry(page, &n->partial, lru) { + object = acquire_slab(s, n, page, c); + if (object) goto out; - page = NULL; + } + object = NULL; out: spin_unlock(&n->list_lock); - return page; + return object; } /* @@ -1535,7 +1541,7 @@ static struct page *get_any_partial(stru struct zoneref *z; struct zone *zone; enum zone_type high_zoneidx = gfp_zone(flags); - struct page *page; + void *object; /* * The defrag ratio allows a configuration of the tradeoffs between @@ -1568,10 +1574,10 @@ static struct page *get_any_partial(stru if (n && cpuset_zone_allowed_hardwall(zone, flags) && n->nr_partial > s->min_partial) { - page = get_partial_node(s, n, c); - if (page) { + object = get_partial_node(s, n, c); + if (object) { put_mems_allowed(); - return page; + return object; } } } @@ -1583,15 +1589,15 @@ static struct page *get_any_partial(stru /* * Get a partial page, lock it and return it. */ -static struct page *get_partial(struct kmem_cache *s, gfp_t flags, int node, +static void *get_partial(struct kmem_cache *s, gfp_t flags, int node, struct kmem_cache_cpu *c) { - struct page *page; + void *object; int searchnode = (node == NUMA_NO_NODE) ? numa_node_id() : node; - page = get_partial_node(s, get_node(s, searchnode), c); - if (page || node != NUMA_NO_NODE) - return page; + object = get_partial_node(s, get_node(s, searchnode), c); + if (object || node != NUMA_NO_NODE) + return object; return get_any_partial(s, flags, c); } @@ -1921,6 +1927,35 @@ slab_out_of_memory(struct kmem_cache *s, } } +static inline void *new_slab_objects(struct kmem_cache *s, gfp_t flags, + int node, struct kmem_cache_cpu **pc) +{ + void *object; + struct kmem_cache_cpu *c; + struct page *page = new_slab(s, flags, node); + + if (page) { + c = __this_cpu_ptr(s->cpu_slab); + if (c->page) + flush_slab(s, c); + + /* + * No other reference to the page yet so we can + * muck around with it freely without cmpxchg + */ + object = page->freelist; + page->freelist = NULL; + + stat(s, ALLOC_SLAB); + c->node = page_to_nid(page); + c->page = page; + *pc = c; + } else + object = NULL; + + return object; +} + /* * Slow path. The lockless freelist is empty or we need to perform * debugging duties. @@ -1943,7 +1978,6 @@ static void *__slab_alloc(struct kmem_ca unsigned long addr, struct kmem_cache_cpu *c) { void **object; - struct page *page; unsigned long flags; struct page new; unsigned long counters; @@ -1961,8 +1995,7 @@ static void *__slab_alloc(struct kmem_ca /* We handle __GFP_ZERO in the caller */ gfpflags &= ~__GFP_ZERO; - page = c->page; - if (!page) + if (!c->page) goto new_slab; if (unlikely(!node_match(c, node))) { @@ -1974,8 +2007,8 @@ static void *__slab_alloc(struct kmem_ca stat(s, ALLOC_SLOWPATH); do { - object = page->freelist; - counters = page->counters; + object = c->page->freelist; + counters = c->page->counters; new.counters = counters; VM_BUG_ON(!new.frozen); @@ -1987,12 +2020,12 @@ static void *__slab_alloc(struct kmem_ca * * If there are objects left then we retrieve them * and use them to refill the per cpu queue. - */ + */ - new.inuse = page->objects; + new.inuse = c->page->objects; new.frozen = object != NULL; - } while (!cmpxchg_double_slab(s, page, + } while (!cmpxchg_double_slab(s, c->page, object, counters, NULL, new.counters, "__slab_alloc")); @@ -2006,50 +2039,33 @@ static void *__slab_alloc(struct kmem_ca stat(s, ALLOC_REFILL); load_freelist: - VM_BUG_ON(!page->frozen); c->freelist = get_freepointer(s, object); c->tid = next_tid(c->tid); local_irq_restore(flags); return object; new_slab: - page = get_partial(s, gfpflags, node, c); - if (page) { - stat(s, ALLOC_FROM_PARTIAL); - object = c->freelist; + object = get_partial(s, gfpflags, node, c); - if (kmem_cache_debug(s)) - goto debug; - goto load_freelist; - } + if (unlikely(!object)) { - page = new_slab(s, gfpflags, node); + object = new_slab_objects(s, gfpflags, node, &c); - if (page) { - c = __this_cpu_ptr(s->cpu_slab); - if (c->page) - flush_slab(s, c); + if (unlikely(!object)) { + if (!(gfpflags & __GFP_NOWARN) && printk_ratelimit()) + slab_out_of_memory(s, gfpflags, node); - /* - * No other reference to the page yet so we can - * muck around with it freely without cmpxchg - */ - object = page->freelist; - page->freelist = NULL; + local_irq_restore(flags); + return NULL; + } + } - stat(s, ALLOC_SLAB); - c->node = page_to_nid(page); - c->page = page; + if (likely(!kmem_cache_debug(s))) goto load_freelist; - } - if (!(gfpflags & __GFP_NOWARN) && printk_ratelimit()) - slab_out_of_memory(s, gfpflags, node); - local_irq_restore(flags); - return NULL; -debug: - if (!object || !alloc_debug_processing(s, page, object, addr)) - goto new_slab; + /* Only entered in the debug case */ + if (!alloc_debug_processing(s, c->page, object, addr)) + goto new_slab; /* Slab failed checks. Next slab needed */ c->freelist = get_freepointer(s, object); deactivate_slab(s, c); -- To unsubscribe from this list: send the line "unsubscribe linux-kernel" in the body of a message to majordomo@vger.kernel.org More majordomo info at http://vger.kernel.org/majordomo-info.html Please read the FAQ at http://www.tux.org/lkml/