/* * Read-Copy Update read-side-primitive performance test module. * * This program is free software; you can redistribute it and/or modify * it under the terms of the GNU General Public License as published by * the Free Software Foundation; either version 2 of the License, or * (at your option) any later version. * * This program is distributed in the hope that it will be useful, * but WITHOUT ANY WARRANTY; without even the implied warranty of * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the * GNU General Public License for more details. * * You should have received a copy of the GNU General Public License * along with this program; if not, write to the Free Software * Foundation, Inc., 59 Temple Place - Suite 330, Boston, MA 02111-1307, USA. * * Copyright (C) IBM Corporation, 2006 * * Authors: Paul E. McKenney */ #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include MODULE_LICENSE("GPL"); static int nreaders = -1; /* # reader threads, defaults to ncpus-1 */ static int test_duration = 0; /* Test duration, in seconds. */ static int verbose = 0; /* Print more debug info. */ module_param(nreaders, int, 0); MODULE_PARM_DESC(nreaders, "Number of RCU reader threads"); module_param(test_duration, int, 0); MODULE_PARM_DESC(test_duration, "Test duration, in seconds."); module_param(verbose, int, 0); MODULE_PARM_DESC(verbose, "Enable verbose debugging printk()s"); #define READPERF_FLAG "rcureadperf: " #define PRINTK_STRING(s) \ do { printk(KERN_ALERT READPERF_FLAG s "\n"); } while (0) #define VERBOSE_PRINTK_STRING(s) \ do { if (verbose) printk(KERN_ALERT READPERF_FLAG s "\n"); } while (0) #define VERBOSE_PRINTK_ERRSTRING(s) \ do { if (verbose) printk(KERN_ALERT READPERF_FLAG "!!! " s "\n"); } while (0) static char printk_buf[4096]; static int nrealreaders; static struct task_struct **reader_tasks; static struct task_struct *stats_task; enum reader_ctl { reader_ctl_init, /* Initial state. */ reader_ctl_ready, /* Reader ready to go. */ reader_ctl_go, /* Command readers to go. */ reader_ctl_pause, /* Command readers to stop. */ }; static __cacheline_aligned enum reader_ctl reader_ctl = reader_ctl_init; struct reader_area { unsigned long cscount; /* Count of RCU read-side critical sections */ enum reader_ctl *ctl; /* Global reader control variable. */ enum reader_ctl ack; /* Per-reader acknowledgement of global ctl. */ char pad[256]; /* There should be a better way... */ }; #define CSCOUNT_SCALE (100 * 1024) /* scaling factor for cscount. */ static __cacheline_aligned struct reader_area *reader_areas; static __cacheline_aligned int fullstop = 0; /* * RCU reader kthread. Repeatedly enters and exits an RCU read-side * critical section, counting the number of such entries. */ static int rcu_perf_reader(void *arg) { int i; struct reader_area *rap = (struct reader_area *)arg; VERBOSE_PRINTK_STRING("rcu_perf_reader task started"); set_user_nice(current, 19); do { barrier(); switch (*rap->ctl) { case reader_ctl_init: if (rap->ack != reader_ctl_ready) { rap->ack = reader_ctl_ready; } break; case reader_ctl_go: for (i = 0; i < CSCOUNT_SCALE; i++) { rcu_read_lock(); rcu_read_unlock(); } rap->cscount++; break; case reader_ctl_pause: if (rap->ack != reader_ctl_ready) { rap->ack = reader_ctl_ready; } break; case reader_ctl_ready: printk(KERN_ALERT "Invalid rcureadperf state"); } cond_resched(); } while (!kthread_should_stop() && !fullstop); VERBOSE_PRINTK_STRING("rcu_perf_reader task stopping"); while (!kthread_should_stop()) schedule_timeout_uninterruptible(1); return 0; } /* * Create an RCU-perf statistics message in the specified buffer. * Also clear the counts in preparation for another collection * interval. */ static int rcu_perf_printk_clear(char *page) { int cnt = 0; int i; cnt += sprintf(&page[cnt], "rcureadperf: duration: %d scale: %d /", test_duration, CSCOUNT_SCALE); for (i = 0; i < nrealreaders; i++) { cnt += sprintf(&page[cnt], " %ld", reader_areas[i].cscount); reader_areas[i].cscount = 0; } cnt += sprintf(&page[cnt], "\n"); return cnt; } /* * Print torture statistics. Caller must ensure that there is only * one call to this function at a given time!!! This is normally * accomplished by relying on the module system to only have one copy * of the module loaded, and then by giving the rcu_perf_stats * kthread full control (or the init/cleanup functions when rcu_perf_stats * thread is not running). */ static void rcu_perf_stats_print_clear(void) { int cnt; cnt = rcu_perf_printk_clear(printk_buf); printk(KERN_ALERT "%s", printk_buf); } /* * Wait until all readers are ready to go. */ static int rcu_wait_readers_ready(void) { int i; for (i = 0; i < nrealreaders; i++) { while (reader_areas[i].ack != reader_ctl_ready) { schedule_timeout_interruptible(1 + HZ / 10); if (kthread_should_stop()) return 1; } } return 0; } /* * Run repeated tests and print stats for RCU read-side-primitive * performance. */ static int rcu_perf_stats(void *arg) { VERBOSE_PRINTK_STRING("rcu_perf_stats task started"); (void)rcu_wait_readers_ready(); do { reader_ctl = reader_ctl_go; schedule_timeout_interruptible(test_duration * HZ); reader_ctl = reader_ctl_pause; if (rcu_wait_readers_ready()) break; rcu_perf_stats_print_clear(); } while (!kthread_should_stop()); VERBOSE_PRINTK_STRING("rcu_perf_stats task stopping"); return 0; } static void rcu_perf_cleanup(void) { int i; fullstop = 1; if (reader_tasks != NULL) { for (i = 0; i < nrealreaders; i++) { if (reader_tasks[i] != NULL) { VERBOSE_PRINTK_STRING( "Stopping rcu_perf_reader task"); kthread_stop(reader_tasks[i]); } reader_tasks[i] = NULL; } kfree(reader_tasks); reader_tasks = NULL; } if (stats_task != NULL) { VERBOSE_PRINTK_STRING("Stopping rcu_perf_stats task"); kthread_stop(stats_task); } stats_task = NULL; kfree(reader_areas); /* Wait for all RCU callbacks to fire. */ rcu_barrier(); printk(KERN_ALERT READPERF_FLAG "--- End of test\n"); } static int rcu_perf_init(void) { int i; int firsterr = 0; /* Process args and tell the world that the torturer is on the job. */ if (nreaders >= 0) nrealreaders = nreaders; else nrealreaders = num_online_cpus() - 1; printk(KERN_ALERT READPERF_FLAG "--- Start of test: nreaders=%d test_duration=%d verbose=%d\n", nrealreaders, test_duration, verbose); fullstop = 0; /* Initialize the statistics so that each run gets its own numbers. */ reader_areas = kmalloc(nrealreaders * sizeof(reader_areas[0]), GFP_KERNEL); if (reader_areas == NULL) { VERBOSE_PRINTK_ERRSTRING("out of memory"); firsterr = -ENOMEM; goto unwind; } for (i = 0; i < nrealreaders; i++) { reader_areas[i].cscount = 0; reader_areas[i].ctl = &reader_ctl; reader_areas[i].ack = reader_ctl_init; } /* Start up the kthreads. */ reader_tasks = kmalloc(nrealreaders * sizeof(reader_tasks[0]), GFP_KERNEL); if (reader_tasks == NULL) { VERBOSE_PRINTK_ERRSTRING("out of memory"); firsterr = -ENOMEM; goto unwind; } for (i = 0; i < nrealreaders; i++) { VERBOSE_PRINTK_STRING("Creating rcu_perf_reader task"); reader_tasks[i] = kthread_run(rcu_perf_reader, &reader_areas[i], "rcureadperf"); if (IS_ERR(reader_tasks[i])) { firsterr = PTR_ERR(reader_tasks[i]); VERBOSE_PRINTK_ERRSTRING("Failed to create reader"); reader_tasks[i] = NULL; goto unwind; } } if (test_duration > 0) { VERBOSE_PRINTK_STRING("Creating rcu_perf_stats task"); stats_task = kthread_run(rcu_perf_stats, NULL, "rcu_perf_stats"); if (IS_ERR(stats_task)) { firsterr = PTR_ERR(stats_task); VERBOSE_PRINTK_ERRSTRING("Failed to create stats"); stats_task = NULL; goto unwind; } } return 0; unwind: rcu_perf_cleanup(); return firsterr; } module_init(rcu_perf_init); module_exit(rcu_perf_cleanup);