This patch gets rid constructors and destructors and replaces them with a slab operations structure that is passed into SLUB. For backward compatibility we provide a kmem_cache_create() emulation that can construct a slab operations structure on the fly. The new API's to create slabs are: Without any callbacks: slabhandle = KMEM_CACHE(, ) Creates a slab based on the structure definition with the structure alignment, size and name. This is cleaner because the name showing up in /sys/slab/xxx will be the structure name. One can search the source for the name. The common alignment attributs to the struct can control slab alignmnet. Note: SLAB_HWCACHE_ALIGN is *not* supported as a flag. The flags do *not* specify alignments. The alignment is done to the structure and please nowhere else. Create a slabcache with slab_ops (please use only for special slabs): KMEM_CACHE_OPS(, , ) Old kmem_cache_create() support: kmem_cache_create alone still accepts the specification of SLAB_HWCACHE_ALIGN *if* there is no other alignment specified. In that case kmem_cache_create will generate a proper alignment depending on the size of the structure. Signed-off-by: Christoph Lameter --- include/linux/slab.h | 60 ++++++++++++++++++++++++++++++------ include/linux/slub_def.h | 3 - mm/slub.c | 77 +++++++++++++++++------------------------------ 3 files changed, 80 insertions(+), 60 deletions(-) Index: slub/include/linux/slab.h =================================================================== --- slub.orig/include/linux/slab.h 2007-05-03 20:53:00.000000000 -0700 +++ slub/include/linux/slab.h 2007-05-04 02:38:38.000000000 -0700 @@ -23,7 +23,6 @@ typedef struct kmem_cache kmem_cache_t _ #define SLAB_DEBUG_FREE 0x00000100UL /* DEBUG: Perform (expensive) checks on free */ #define SLAB_RED_ZONE 0x00000400UL /* DEBUG: Red zone objs in a cache */ #define SLAB_POISON 0x00000800UL /* DEBUG: Poison objects */ -#define SLAB_HWCACHE_ALIGN 0x00002000UL /* Align objs on cache lines */ #define SLAB_CACHE_DMA 0x00004000UL /* Use GFP_DMA memory */ #define SLAB_STORE_USER 0x00010000UL /* DEBUG: Store the last owner for bug hunting */ #define SLAB_RECLAIM_ACCOUNT 0x00020000UL /* Objects are reclaimable */ @@ -32,19 +31,21 @@ typedef struct kmem_cache kmem_cache_t _ #define SLAB_MEM_SPREAD 0x00100000UL /* Spread some memory over cpuset */ #define SLAB_TRACE 0x00200000UL /* Trace allocations and frees */ -/* Flags passed to a constructor functions */ -#define SLAB_CTOR_CONSTRUCTOR 0x001UL /* If not set, then deconstructor */ - /* * struct kmem_cache related prototypes */ void __init kmem_cache_init(void); int slab_is_available(void); -struct kmem_cache *kmem_cache_create(const char *, size_t, size_t, - unsigned long, - void (*)(void *, struct kmem_cache *, unsigned long), - void (*)(void *, struct kmem_cache *, unsigned long)); +struct slab_ops { + /* FIXME: ctor should only take the object as an argument. */ + void (*ctor)(void *, struct kmem_cache *, unsigned long); + /* FIXME: Remove all destructors ? */ + void (*dtor)(void *, struct kmem_cache *, unsigned long); +}; + +struct kmem_cache *__kmem_cache_create(const char *, size_t, size_t, + unsigned long, struct slab_ops *s); void kmem_cache_destroy(struct kmem_cache *); int kmem_cache_shrink(struct kmem_cache *); void *kmem_cache_alloc(struct kmem_cache *, gfp_t); @@ -62,9 +63,14 @@ int kmem_ptr_validate(struct kmem_cache * f.e. add ____cacheline_aligned_in_smp to the struct declaration * then the objects will be properly aligned in SMP configurations. */ -#define KMEM_CACHE(__struct, __flags) kmem_cache_create(#__struct,\ +#define KMEM_CACHE(__struct, __flags) __kmem_cache_create(#__struct,\ sizeof(struct __struct), __alignof__(struct __struct),\ - (__flags), NULL, NULL) + (__flags), NULL) + +#define KMEM_CACHE_OPS(__struct, __flags, __ops) \ + __kmem_cache_create(#__struct, sizeof(struct __struct), \ + __alignof__(struct __struct), (__flags), (__ops)) + #ifdef CONFIG_NUMA extern void *kmem_cache_alloc_node(struct kmem_cache *, gfp_t flags, int node); @@ -236,6 +242,40 @@ extern void *__kmalloc_node_track_caller extern const struct seq_operations slabinfo_op; ssize_t slabinfo_write(struct file *, const char __user *, size_t, loff_t *); +/* + * Legacy functions + * + * The sole reason that these definitions are here is because of their + * frequent use. Remove when all call sites have been updated. + */ +#define SLAB_HWCACHE_ALIGN 0x8000000000UL +#define SLAB_CTOR_CONSTRUCTOR 0x001UL + +static inline struct kmem_cache *kmem_cache_create(const char *s, + size_t size, size_t align, unsigned long flags, + void (*ctor)(void *, struct kmem_cache *, unsigned long), + void (*dtor)(void *, struct kmem_cache *, unsigned long)) +{ + struct slab_ops *so = NULL; + + if ((flags & SLAB_HWCACHE_ALIGN) && size > L1_CACHE_BYTES / 2) { + /* Clear the align flag. It is no longer supported */ + flags &= ~SLAB_HWCACHE_ALIGN; + + /* Do not allow conflicting alignment specificiations */ + BUG_ON(align); + + /* And set the cacheline alignment */ + align = L1_CACHE_BYTES; + } + if (ctor || dtor) { + so = kzalloc(sizeof(struct slab_ops), GFP_KERNEL); + so->ctor = ctor; + so->dtor = dtor; + } + return __kmem_cache_create(s, size, align, flags, so); +} + #endif /* __KERNEL__ */ #endif /* _LINUX_SLAB_H */ Index: slub/mm/slub.c =================================================================== --- slub.orig/mm/slub.c 2007-05-04 02:23:34.000000000 -0700 +++ slub/mm/slub.c 2007-05-04 02:40:13.000000000 -0700 @@ -209,6 +209,11 @@ static inline struct kmem_cache_node *ge #endif } +struct slab_ops default_slab_ops = { + NULL, + NULL +}; + /* * Object debugging */ @@ -809,8 +814,8 @@ static void setup_object(struct kmem_cac init_tracking(s, object); } - if (unlikely(s->ctor)) - s->ctor(object, s, SLAB_CTOR_CONSTRUCTOR); + if (unlikely(s->slab_ops->ctor)) + s->slab_ops->ctor(object, s, SLAB_CTOR_CONSTRUCTOR); } static struct page *new_slab(struct kmem_cache *s, gfp_t flags, int node) @@ -867,16 +872,18 @@ out: static void __free_slab(struct kmem_cache *s, struct page *page) { int pages = 1 << s->order; + void (*dtor)(void *, struct kmem_cache *, unsigned long) = + s->slab_ops->dtor; - if (unlikely(PageError(page) || s->dtor)) { + if (unlikely(PageError(page) || dtor)) { void *start = page_address(page); void *end = start + (pages << PAGE_SHIFT); void *p; slab_pad_check(s, page); for (p = start; p <= end - s->size; p += s->size) { - if (s->dtor) - s->dtor(p, s, 0); + if (dtor) + dtor(p, s, 0); check_object(s, page, p, 0); } } @@ -1618,7 +1625,7 @@ static int calculate_sizes(struct kmem_c * then we should never poison the object itself. */ if ((flags & SLAB_POISON) && !(flags & SLAB_DESTROY_BY_RCU) && - !s->ctor && !s->dtor) + s->slab_ops->ctor && !s->slab_ops->dtor) s->flags |= __OBJECT_POISON; else s->flags &= ~__OBJECT_POISON; @@ -1647,7 +1654,7 @@ static int calculate_sizes(struct kmem_c s->inuse = size; if (((flags & (SLAB_DESTROY_BY_RCU | SLAB_POISON)) || - s->ctor || s->dtor)) { + s->slab_ops->ctor || s->slab_ops->dtor)) { /* * Relocate free pointer after the object if it is not * permitted to overwrite the first word of the object on @@ -1731,13 +1738,11 @@ static int __init finish_bootstrap(void) static int kmem_cache_open(struct kmem_cache *s, gfp_t gfpflags, const char *name, size_t size, size_t align, unsigned long flags, - void (*ctor)(void *, struct kmem_cache *, unsigned long), - void (*dtor)(void *, struct kmem_cache *, unsigned long)) + struct slab_ops *slab_ops) { memset(s, 0, kmem_size); s->name = name; - s->ctor = ctor; - s->dtor = dtor; + s->slab_ops = slab_ops; s->objsize = size; s->flags = flags; s->align = align; @@ -1757,7 +1762,7 @@ static int kmem_cache_open(struct kmem_c if (s->size >= 65535 * sizeof(void *)) { BUG_ON(flags & (SLAB_RED_ZONE | SLAB_POISON | SLAB_STORE_USER | SLAB_DESTROY_BY_RCU)); - BUG_ON(ctor || dtor); + BUG_ON(slab_ops->ctor || slab_ops->dtor); } else /* @@ -1992,7 +1997,7 @@ static struct kmem_cache *create_kmalloc down_write(&slub_lock); if (!kmem_cache_open(s, gfp_flags, name, size, ARCH_KMALLOC_MINALIGN, - flags, NULL, NULL)) + flags, &default_slab_ops)) goto panic; list_add(&s->list, &slab_caches); @@ -2313,23 +2318,21 @@ static int slab_unmergeable(struct kmem_ if (slub_nomerge || (s->flags & SLUB_NEVER_MERGE)) return 1; - if (s->ctor || s->dtor) + if (s->slab_ops != &default_slab_ops) return 1; return 0; } static struct kmem_cache *find_mergeable(size_t size, - size_t align, unsigned long flags, - void (*ctor)(void *, struct kmem_cache *, unsigned long), - void (*dtor)(void *, struct kmem_cache *, unsigned long)) + size_t align, unsigned long flags, struct slab_ops *slab_ops) { struct list_head *h; if (slub_nomerge || (flags & SLUB_NEVER_MERGE)) return NULL; - if (ctor || dtor) + if (slab_ops != &default_slab_ops) return NULL; size = ALIGN(size, sizeof(void *)); @@ -2364,15 +2367,17 @@ static struct kmem_cache *find_mergeable return NULL; } -struct kmem_cache *kmem_cache_create(const char *name, size_t size, +struct kmem_cache *__kmem_cache_create(const char *name, size_t size, size_t align, unsigned long flags, - void (*ctor)(void *, struct kmem_cache *, unsigned long), - void (*dtor)(void *, struct kmem_cache *, unsigned long)) + struct slab_ops *slab_ops) { struct kmem_cache *s; + if (!slab_ops) + slab_ops = &default_slab_ops; + down_write(&slub_lock); - s = find_mergeable(size, align, flags, dtor, ctor); + s = find_mergeable(size, align, flags, slab_ops); if (s) { s->refcount++; /* @@ -2386,7 +2391,7 @@ struct kmem_cache *kmem_cache_create(con } else { s = kmalloc(kmem_size, GFP_KERNEL); if (s && kmem_cache_open(s, GFP_KERNEL, name, - size, align, flags, ctor, dtor)) { + size, align, flags, slab_ops)) { if (sysfs_slab_add(s)) { kfree(s); goto err; @@ -2406,7 +2411,7 @@ err: s = NULL; return s; } -EXPORT_SYMBOL(kmem_cache_create); +EXPORT_SYMBOL(__kmem_cache_create); void *kmem_cache_zalloc(struct kmem_cache *s, gfp_t flags) { @@ -2961,28 +2966,6 @@ static ssize_t order_show(struct kmem_ca } SLAB_ATTR_RO(order); -static ssize_t ctor_show(struct kmem_cache *s, char *buf) -{ - if (s->ctor) { - int n = sprint_symbol(buf, (unsigned long)s->ctor); - - return n + sprintf(buf + n, "\n"); - } - return 0; -} -SLAB_ATTR_RO(ctor); - -static ssize_t dtor_show(struct kmem_cache *s, char *buf) -{ - if (s->dtor) { - int n = sprint_symbol(buf, (unsigned long)s->dtor); - - return n + sprintf(buf + n, "\n"); - } - return 0; -} -SLAB_ATTR_RO(dtor); - static ssize_t aliases_show(struct kmem_cache *s, char *buf) { return sprintf(buf, "%d\n", s->refcount - 1); @@ -3213,8 +3196,6 @@ static struct attribute * slab_attrs[] = &slabs_attr.attr, &partial_attr.attr, &cpu_slabs_attr.attr, - &ctor_attr.attr, - &dtor_attr.attr, &aliases_attr.attr, &align_attr.attr, &sanity_checks_attr.attr, Index: slub/include/linux/slub_def.h =================================================================== --- slub.orig/include/linux/slub_def.h 2007-05-04 02:23:51.000000000 -0700 +++ slub/include/linux/slub_def.h 2007-05-04 02:24:27.000000000 -0700 @@ -39,8 +39,7 @@ struct kmem_cache { /* Allocation and freeing of slabs */ int objects; /* Number of objects in slab */ int refcount; /* Refcount for slab cache destroy */ - void (*ctor)(void *, struct kmem_cache *, unsigned long); - void (*dtor)(void *, struct kmem_cache *, unsigned long); + struct slab_ops *slab_ops; int inuse; /* Offset to metadata */ int align; /* Alignment */ const char *name; /* Name (only for display!) */ -- - To unsubscribe from this list: send the line "unsubscribe linux-kernel" in the body of a message to majordomo@vger.kernel.org More majordomo info at http://vger.kernel.org/majordomo-info.html Please read the FAQ at http://www.tux.org/lkml/