[<prev] [next>] [<thread-prev] [thread-next>] [day] [month] [year] [list]
Message-Id: <1258391726-30264-22-git-send-email-tj@kernel.org>
Date: Tue, 17 Nov 2009 02:15:26 +0900
From: Tejun Heo <tj@...nel.org>
To: linux-kernel@...r.kernel.org, jeff@...zik.org, mingo@...e.hu,
akpm@...ux-foundation.org, jens.axboe@...cle.com,
rusty@...tcorp.com.au, cl@...ux-foundation.org,
dhowells@...hat.com, arjan@...ux.intel.com,
torvalds@...ux-foundation.org, avi@...hat.com,
peterz@...radead.org, andi@...stfloor.org, fweisbec@...il.com
Cc: Tejun Heo <tj@...nel.org>
Subject: [PATCH 21/21] workqueue: reimplement workqueue freeze using cwq->frozen_works queue
Currently, workqueue freezing is implemented by marking the worker
freezeable and calling try_to_freeze() from dispatch loop.
Reimplement it so that the workqueue is frozen instead of the worker.
* cwq->cur_worklist and cwq->frozen_works are added. During normal
operation cwq->cur_worklist points to cwq->worklist.
* When freezing starts, cwq->cur_worklist is switched to
cwq->frozen_works so that new works are stored in cwq->frozen_works
instead of being processed.
* Freezing is complete when cwq->nr_in_flight equals the number of
works on cwq->frozen_works for all cwqs of all freezeable
workqueues.
* Thawing is done by restoring cwq->cur_worklist to cwq->worklist and
splicing cwq->frozen_works to cwq->worklist.
This new implementation allows having multiple shared workers per cpu.
Signed-off-by: Tejun Heo <tj@...nel.org>
---
include/linux/workqueue.h | 7 ++
kernel/power/process.c | 22 +++++-
kernel/workqueue.c | 182 ++++++++++++++++++++++++++++++++++++++++++---
3 files changed, 199 insertions(+), 12 deletions(-)
diff --git a/include/linux/workqueue.h b/include/linux/workqueue.h
index 53d1410..d7efa66 100644
--- a/include/linux/workqueue.h
+++ b/include/linux/workqueue.h
@@ -326,4 +326,11 @@ static inline long work_on_cpu(unsigned int cpu, long (*fn)(void *), void *arg)
#else
long work_on_cpu(unsigned int cpu, long (*fn)(void *), void *arg);
#endif /* CONFIG_SMP */
+
+#ifdef CONFIG_FREEZER
+extern void freeze_workqueues_begin(void);
+extern bool freeze_workqueues_busy(void);
+extern void thaw_workqueues(void);
+#endif /* CONFIG_FREEZER */
+
#endif
diff --git a/kernel/power/process.c b/kernel/power/process.c
index cc2e553..701e17f 100644
--- a/kernel/power/process.c
+++ b/kernel/power/process.c
@@ -14,6 +14,7 @@
#include <linux/module.h>
#include <linux/syscalls.h>
#include <linux/freezer.h>
+#include <linux/workqueue.h>
/*
* Timeout for stopping processes
@@ -34,6 +35,7 @@ static int try_to_freeze_tasks(bool sig_only)
struct task_struct *g, *p;
unsigned long end_time;
unsigned int todo;
+ bool wq_busy = false;
struct timeval start, end;
u64 elapsed_csecs64;
unsigned int elapsed_csecs;
@@ -41,6 +43,10 @@ static int try_to_freeze_tasks(bool sig_only)
do_gettimeofday(&start);
end_time = jiffies + TIMEOUT;
+
+ if (!sig_only)
+ freeze_workqueues_begin();
+
do {
todo = 0;
read_lock(&tasklist_lock);
@@ -62,7 +68,14 @@ static int try_to_freeze_tasks(bool sig_only)
todo++;
} while_each_thread(g, p);
read_unlock(&tasklist_lock);
+
yield(); /* Yield is okay here */
+
+ if (!sig_only) {
+ wq_busy = freeze_workqueues_busy();
+ todo += wq_busy;
+ }
+
if (time_after(jiffies, end_time))
break;
} while (todo);
@@ -80,9 +93,13 @@ static int try_to_freeze_tasks(bool sig_only)
*/
printk("\n");
printk(KERN_ERR "Freezing of tasks failed after %d.%02d seconds "
- "(%d tasks refusing to freeze):\n",
- elapsed_csecs / 100, elapsed_csecs % 100, todo);
+ "(%d tasks refusing to freeze, wq_busy=%d):\n",
+ elapsed_csecs / 100, elapsed_csecs % 100,
+ todo - wq_busy, wq_busy);
show_state();
+
+ thaw_workqueues();
+
read_lock(&tasklist_lock);
do_each_thread(g, p) {
task_lock(p);
@@ -152,6 +169,7 @@ void thaw_processes(void)
oom_killer_enable();
printk("Restarting tasks ... ");
+ thaw_workqueues();
thaw_tasks(true);
thaw_tasks(false);
schedule();
diff --git a/kernel/workqueue.c b/kernel/workqueue.c
index 3029bb2..933eb84 100644
--- a/kernel/workqueue.c
+++ b/kernel/workqueue.c
@@ -46,6 +46,10 @@
* F: wq->flush_mutex protected.
*
* W: workqueue_lock protected.
+ *
+ * V: Similar to L except that operation is limited to only one
+ * direction if workqueues are frozen (ie. can be added but can't
+ * be removed).
*/
struct cpu_workqueue_struct;
@@ -72,12 +76,14 @@ struct cpu_workqueue_struct {
wait_queue_head_t more_work;
unsigned int cpu;
struct worker *worker;
+ struct list_head *cur_worklist; /* L: current worklist */
struct workqueue_struct *wq; /* I: the owning workqueue */
int work_color; /* L: current color */
int flush_color; /* L: flushing color */
int nr_in_flight[WORK_COLORS];
/* L: nr of in_flight works */
+ struct list_head frozen_works; /* V: used while frozen */
} __attribute__((aligned(1 << WORK_STRUCT_FLAG_BITS)));
/*
@@ -227,6 +233,7 @@ static inline void debug_work_deactivate(struct work_struct *work) { }
static DEFINE_SPINLOCK(workqueue_lock);
static LIST_HEAD(workqueues);
static DEFINE_PER_CPU(struct ida, worker_ida);
+static bool workqueue_frozen;
static int worker_thread(void *__worker);
@@ -314,7 +321,7 @@ static void __queue_work(unsigned int cpu, struct workqueue_struct *wq,
debug_work_activate(work);
spin_lock_irqsave(&cwq->lock, flags);
BUG_ON(!list_empty(&work->entry));
- insert_work(cwq, work, &cwq->worklist, 0);
+ insert_work(cwq, work, cwq->cur_worklist, 0);
spin_unlock_irqrestore(&cwq->lock, flags);
}
@@ -711,19 +718,13 @@ static int worker_thread(void *__worker)
struct cpu_workqueue_struct *cwq = worker->cwq;
DEFINE_WAIT(wait);
- if (cwq->wq->flags & WQ_FREEZEABLE)
- set_freezable();
-
for (;;) {
prepare_to_wait(&cwq->more_work, &wait, TASK_INTERRUPTIBLE);
- if (!freezing(current) &&
- !kthread_should_stop() &&
+ if (!kthread_should_stop() &&
list_empty(&cwq->worklist))
schedule();
finish_wait(&cwq->more_work, &wait);
- try_to_freeze();
-
if (kthread_should_stop())
break;
@@ -1450,6 +1451,14 @@ struct workqueue_struct *__create_workqueue_key(const char *name,
*/
spin_lock(&workqueue_lock);
list_add(&wq->list, &workqueues);
+ for_each_possible_cpu(cpu) {
+ struct cpu_workqueue_struct *cwq = get_cwq(cpu, wq);
+
+ if (workqueue_frozen && wq->flags & WQ_FREEZEABLE)
+ cwq->cur_worklist = &cwq->frozen_works;
+ else
+ cwq->cur_worklist = &cwq->worklist;
+ }
spin_unlock(&workqueue_lock);
/*
* We must initialize cwqs for each possible cpu even if we
@@ -1466,6 +1475,7 @@ struct workqueue_struct *__create_workqueue_key(const char *name,
spin_lock_init(&cwq->lock);
INIT_LIST_HEAD(&cwq->worklist);
init_waitqueue_head(&cwq->more_work);
+ INIT_LIST_HEAD(&cwq->frozen_works);
if (failed || !cpu_online(cpu))
continue;
@@ -1502,12 +1512,17 @@ void destroy_workqueue(struct workqueue_struct *wq)
int cpu;
cpu_maps_update_begin();
+
+ flush_workqueue(wq);
+
+ /*
+ * wq list is used to freeze wq, remove from list after
+ * flushing is complete in case freeze races us.
+ */
spin_lock(&workqueue_lock);
list_del(&wq->list);
spin_unlock(&workqueue_lock);
- flush_workqueue(wq);
-
for_each_possible_cpu(cpu) {
struct cpu_workqueue_struct *cwq = get_cwq(cpu, wq);
int i;
@@ -1520,6 +1535,7 @@ void destroy_workqueue(struct workqueue_struct *wq)
for (i = 0; i < WORK_COLORS; i++)
BUG_ON(cwq->nr_in_flight[i]);
+ BUG_ON(!list_empty(&cwq->frozen_works));
}
cpu_maps_update_done();
@@ -1623,6 +1639,152 @@ long work_on_cpu(unsigned int cpu, long (*fn)(void *), void *arg)
EXPORT_SYMBOL_GPL(work_on_cpu);
#endif /* CONFIG_SMP */
+#ifdef CONFIG_FREEZER
+/**
+ * freeze_workqueues_begin - begin freezing workqueues
+ *
+ * Start freezing workqueues. After this function returns, all
+ * freezeable workqueues will queue new works to their frozen_works
+ * list instead of the cwq ones.
+ *
+ * CONTEXT:
+ * Grabs and releases workqueue_lock and cwq->lock's.
+ */
+void freeze_workqueues_begin(void)
+{
+ struct workqueue_struct *wq;
+ unsigned int cpu;
+
+ spin_lock(&workqueue_lock);
+
+ BUG_ON(workqueue_frozen);
+ workqueue_frozen = true;
+
+ for_each_possible_cpu(cpu) {
+ list_for_each_entry(wq, &workqueues, list) {
+ struct cpu_workqueue_struct *cwq = get_cwq(cpu, wq);
+
+ if (!(wq->flags & WQ_FREEZEABLE))
+ continue;
+
+ spin_lock_irq(&cwq->lock);
+
+ BUG_ON(cwq->cur_worklist != &cwq->worklist);
+ BUG_ON(!list_empty(&cwq->frozen_works));
+
+ cwq->cur_worklist = &cwq->frozen_works;
+
+ spin_unlock_irq(&cwq->lock);
+ }
+ }
+ spin_unlock(&workqueue_lock);
+}
+
+/**
+ * freeze_workqueues_busy - are freezeable workqueues still busy?
+ *
+ * Check whether freezing is complete. This function must be called
+ * between freeeze_workqueues_begin() and thaw_workqueues().
+ *
+ * CONTEXT:
+ * Grabs and releases workqueue_lock.
+ *
+ * RETURNS:
+ * %true if some freezeable workqueues are still busy. %false if
+ * freezing is complete.
+ */
+bool freeze_workqueues_busy(void)
+{
+ struct workqueue_struct *wq;
+ unsigned int cpu;
+ bool busy = false;
+
+ spin_lock(&workqueue_lock);
+
+ BUG_ON(!workqueue_frozen);
+
+ for_each_possible_cpu(cpu) {
+ list_for_each_entry(wq, &workqueues, list) {
+ struct cpu_workqueue_struct *cwq = get_cwq(cpu, wq);
+ struct work_struct *work;
+ int i, nr_in_flight;
+
+ if (!(wq->flags & WQ_FREEZEABLE))
+ continue;
+
+ spin_lock_irq(&cwq->lock);
+
+ BUG_ON(cwq->cur_worklist != &cwq->frozen_works);
+
+ nr_in_flight = 0;
+ for (i = 0; i < WORK_COLORS; i++)
+ nr_in_flight += cwq->nr_in_flight[i];
+
+ list_for_each_entry(work, &cwq->frozen_works, entry)
+ nr_in_flight--;
+
+ spin_unlock_irq(&cwq->lock);
+
+ BUG_ON(nr_in_flight < 0);
+ if (nr_in_flight) {
+ busy = true;
+ break;
+ }
+ }
+ if (busy)
+ break;
+ }
+ spin_unlock(&workqueue_lock);
+ return busy;
+}
+
+/**
+ * thaw_workqueues - thaw workqueues
+ *
+ * Thaw workqueues. Normal queueing is restored and all collected
+ * frozen works are transferred to their respective cwq worklists.
+ *
+ * CONTEXT:
+ * Grabs and releases workqueue_lock and cwq->lock's.
+ */
+void thaw_workqueues(void)
+{
+ struct workqueue_struct *wq;
+ unsigned int cpu;
+
+ spin_lock(&workqueue_lock);
+
+ if (!workqueue_frozen)
+ goto out_unlock;
+
+ workqueue_frozen = false;
+
+ for_each_possible_cpu(cpu) {
+ list_for_each_entry(wq, &workqueues, list) {
+ struct cpu_workqueue_struct *cwq = get_cwq(cpu, wq);
+
+ if (!(wq->flags & WQ_FREEZEABLE))
+ continue;
+
+ spin_lock_irq(&cwq->lock);
+
+ /* switch to normal work queueing */
+ BUG_ON(cwq->cur_worklist != &cwq->frozen_works);
+ cwq->cur_worklist = &cwq->worklist;
+
+ /* transfer frozen tasks to cwq worklist */
+ list_splice_tail_init(&cwq->frozen_works,
+ &cwq->worklist);
+ wake_up(&cwq->more_work);
+
+ spin_unlock_irq(&cwq->lock);
+ }
+ }
+out_unlock:
+ spin_unlock(&workqueue_lock);
+}
+#endif /* CONFIG_FREEZER */
+
void __init init_workqueues(void)
{
unsigned int cpu;
--
1.6.4.2
--
To unsubscribe from this list: send the line "unsubscribe linux-kernel" in
the body of a message to majordomo@...r.kernel.org
More majordomo info at http://vger.kernel.org/majordomo-info.html
Please read the FAQ at http://www.tux.org/lkml/
Powered by blists - more mailing lists