Provide scalable per backing_dev_info statistics counters modeled on the ZVC code. Signed-off-by: Peter Zijlstra --- block/ll_rw_blk.c | 1 drivers/block/rd.c | 2 drivers/char/mem.c | 2 fs/char_dev.c | 1 include/linux/backing-dev.h | 86 ++++++++++++++++++++++++++++++++++++ mm/backing-dev.c | 103 ++++++++++++++++++++++++++++++++++++++++++++ 6 files changed, 195 insertions(+) Index: linux-2.6/block/ll_rw_blk.c =================================================================== --- linux-2.6.orig/block/ll_rw_blk.c +++ linux-2.6/block/ll_rw_blk.c @@ -211,6 +211,7 @@ void blk_queue_make_request(request_queu q->backing_dev_info.ra_pages = (VM_MAX_READAHEAD * 1024) / PAGE_CACHE_SIZE; q->backing_dev_info.state = 0; q->backing_dev_info.capabilities = BDI_CAP_MAP_COPY; + bdi_stat_init(&q->backing_dev_info); blk_queue_max_sectors(q, SAFE_MAX_SECTORS); blk_queue_hardsect_size(q, 512); blk_queue_dma_alignment(q, 511); Index: linux-2.6/include/linux/backing-dev.h =================================================================== --- linux-2.6.orig/include/linux/backing-dev.h +++ linux-2.6/include/linux/backing-dev.h @@ -22,6 +22,17 @@ enum bdi_state { BDI_unused, /* Available bits start here */ }; +enum bdi_stat_item { + NR_BDI_STAT_ITEMS +}; + +#ifdef CONFIG_SMP +struct bdi_per_cpu_data { + s8 stat_threshold; + s8 bdi_stat_diff[NR_BDI_STAT_ITEMS]; +} ____cacheline_aligned_in_smp; +#endif + typedef int (congested_fn)(void *, int); struct backing_dev_info { @@ -32,8 +43,83 @@ struct backing_dev_info { void *congested_data; /* Pointer to aux data for congested func */ void (*unplug_io_fn)(struct backing_dev_info *, struct page *); void *unplug_io_data; + + atomic_long_t bdi_stats[NR_BDI_STAT_ITEMS]; +#ifdef CONFIG_SMP + struct bdi_per_cpu_data pcd[NR_CPUS]; +#endif }; +extern atomic_long_t bdi_stats[NR_BDI_STAT_ITEMS]; + +static inline void bdi_stat_add(long x, struct backing_dev_info *bdi, + enum bdi_stat_item item) +{ + atomic_long_add(x, &bdi->bdi_stats[item]); + atomic_long_add(x, &bdi_stats[item]); +} + +/* + * cannot be unsigned long and clip on 0. + */ +static inline unsigned long global_bdi_stat(enum bdi_stat_item item) +{ + long x = atomic_long_read(&bdi_stats[item]); +#ifdef CONFIG_SMP + if (x < 0) + x = 0; +#endif + return x; +} + +static inline unsigned long bdi_stat(struct backing_dev_info *bdi, + enum bdi_stat_item item) +{ + long x = atomic_long_read(&bdi->bdi_stats[item]); +#ifdef CONFIG_SMP + if (x < 0) + x = 0; +#endif + return x; +} + +#ifdef CONFIG_SMP +void __mod_bdi_stat(struct backing_dev_info *bdi, enum bdi_stat_item item, int delta); +void __inc_bdi_stat(struct backing_dev_info *bdi, enum bdi_stat_item item); +void __dec_bdi_stat(struct backing_dev_info *bdi, enum bdi_stat_item item); + +void mod_bdi_stat(struct backing_dev_info *bdi, enum bdi_stat_item item, int delta); +void inc_bdi_stat(struct backing_dev_info *bdi, enum bdi_stat_item item); +void dec_bdi_stat(struct backing_dev_info *bdi, enum bdi_stat_item item); + +#else /* CONFIG_SMP */ + +static inline void __mod_bdi_stat(struct backing_dev_info *bdi, + enum bdi_stat_item item, int delta) +{ + bdi_stat_add(delta, bdi, item); +} + +static inline void __inc_bdi_stat(struct backing_dev_info *bdi, + enum bdi_stat_item item) +{ + atomic_long_inc(&bdi->bdi_stats[item]); + atomic_long_inc(&bdi_stats[item]); +} + +static inline void __dec_bdi_stat(struct backing_dev_info *bdi, + enum bdi_stat_item item) +{ + atomic_long_dec(&bdi->bdi_stats[item]); + atomic_long_dec(&bdi_stats[item]); +} + +#define mod_bdi_stat __mod_bdi_stat +#define inc_bdi_stat __inc_bdi_stat +#define dec_bdi_stat __dec_bdi_stat +#endif + +void bdi_stat_init(struct backing_dev_info *bdi); /* * Flags in backing_dev_info::capability Index: linux-2.6/mm/backing-dev.c =================================================================== --- linux-2.6.orig/mm/backing-dev.c +++ linux-2.6/mm/backing-dev.c @@ -67,3 +67,106 @@ void congestion_end(int rw) wake_up(wqh); } EXPORT_SYMBOL(congestion_end); + +atomic_long_t bdi_stats[NR_BDI_STAT_ITEMS]; +EXPORT_SYMBOL(bdi_stats); + +void bdi_stat_init(struct backing_dev_info *bdi) +{ + int i; + + for (i = 0; i < NR_BDI_STAT_ITEMS; i++) + atomic_long_set(&bdi->bdi_stats[i], 0); + +#ifdef CONFIG_SMP + for (i = 0; i < NR_CPUS; i++) { + int j; + for (j = 0; j < NR_BDI_STAT_ITEMS; j++) + bdi->pcd[i].bdi_stat_diff[j] = 0; + bdi->pcd[i].stat_threshold = 8 * ilog2(num_online_cpus()); + } +#endif +} +EXPORT_SYMBOL(bdi_stat_init); + +#ifdef CONFIG_SMP +void __mod_bdi_stat(struct backing_dev_info *bdi, + enum bdi_stat_item item, int delta) +{ + struct bdi_per_cpu_data *pcd = &bdi->pcd[smp_processor_id()]; + s8 *p = pcd->bdi_stat_diff + item; + long x; + + x = delta + *p; + + if (unlikely(x > pcd->stat_threshold || x < -pcd->stat_threshold)) { + bdi_stat_add(x, bdi, item); + x = 0; + } + *p = x; +} +EXPORT_SYMBOL(__mod_bdi_stat); + +void mod_bdi_stat(struct backing_dev_info *bdi, + enum bdi_stat_item item, int delta) +{ + unsigned long flags; + + local_irq_save(flags); + __mod_bdi_stat(bdi, item, delta); + local_irq_restore(flags); +} +EXPORT_SYMBOL(mod_bdi_stat); + +void __inc_bdi_stat(struct backing_dev_info *bdi, enum bdi_stat_item item) +{ + struct bdi_per_cpu_data *pcd = &bdi->pcd[smp_processor_id()]; + s8 *p = pcd->bdi_stat_diff + item; + + (*p)++; + + if (unlikely(*p > pcd->stat_threshold)) { + int overstep = pcd->stat_threshold / 2; + + bdi_stat_add(*p + overstep, bdi, item); + *p = -overstep; + } +} +EXPORT_SYMBOL(__inc_bdi_stat); + +void inc_bdi_stat(struct backing_dev_info *bdi, enum bdi_stat_item item) +{ + unsigned long flags; + + local_irq_save(flags); + __inc_bdi_stat(bdi, item); + local_irq_restore(flags); +} +EXPORT_SYMBOL(inc_bdi_stat); + +void __dec_bdi_stat(struct backing_dev_info *bdi, enum bdi_stat_item item) +{ + struct bdi_per_cpu_data *pcd = &bdi->pcd[smp_processor_id()]; + s8 *p = pcd->bdi_stat_diff + item; + + (*p)--; + + if (unlikely(*p < -pcd->stat_threshold)) { + int overstep = pcd->stat_threshold / 2; + + bdi_stat_add(*p - overstep, bdi, item); + *p = overstep; + } +} +EXPORT_SYMBOL(__dec_bdi_stat); + +void dec_bdi_stat(struct backing_dev_info *bdi, enum bdi_stat_item item) +{ + unsigned long flags; + + local_irq_save(flags); + __dec_bdi_stat(bdi, item); + local_irq_restore(flags); +} +EXPORT_SYMBOL(dec_bdi_stat); +#endif Index: linux-2.6/drivers/block/rd.c =================================================================== --- linux-2.6.orig/drivers/block/rd.c +++ linux-2.6/drivers/block/rd.c @@ -421,6 +421,8 @@ static int __init rd_init(void) int i; int err = -ENOMEM; + bdi_stat_init(&rd_file_backing_dev_info); + if (rd_blocksize > PAGE_SIZE || rd_blocksize < 512 || (rd_blocksize & (rd_blocksize-1))) { printk("RAMDISK: wrong blocksize %d, reverting to defaults\n", Index: linux-2.6/drivers/char/mem.c =================================================================== --- linux-2.6.orig/drivers/char/mem.c +++ linux-2.6/drivers/char/mem.c @@ -988,6 +988,8 @@ static int __init chr_dev_init(void) MKDEV(MEM_MAJOR, devlist[i].minor), devlist[i].name); + bdi_stat_init(&zero_bdi); + return 0; } Index: linux-2.6/fs/char_dev.c =================================================================== --- linux-2.6.orig/fs/char_dev.c +++ linux-2.6/fs/char_dev.c @@ -545,6 +545,7 @@ static struct kobject *base_probe(dev_t void __init chrdev_init(void) { cdev_map = kobj_map_init(base_probe, &chrdevs_lock); + bdi_stat_init(&directly_mappable_cdev_bdi); } -- - To unsubscribe from this list: send the line "unsubscribe linux-kernel" in the body of a message to majordomo@vger.kernel.org More majordomo info at http://vger.kernel.org/majordomo-info.html Please read the FAQ at http://www.tux.org/lkml/