| lists.openwall.net | lists / announce owl-users owl-dev john-users john-dev passwdqc-users yescrypt popa3d-users / oss-security kernel-hardening musl sabotage tlsify passwords / crypt-dev xvendor / Bugtraq Full-Disclosure linux-kernel linux-netdev linux-ext4 linux-hardening linux-cve-announce PHC | |
|
Open Source and information security mailing list archives
| ||
|
Message-Id: <20251113163426.2950-1-jiangshanlai@gmail.com>
Date: Fri, 14 Nov 2025 00:34:26 +0800
From: Lai Jiangshan <jiangshanlai@...il.com>
To: linux-kernel@...r.kernel.org
Cc: Lai Jiangshan <jiangshan.ljs@...group.com>,
ying chen <yc1082463@...il.com>,
Tejun Heo <tj@...nel.org>,
Lai Jiangshan <jiangshanlai@...il.com>
Subject: [PATCH] workqueue: Process rescuer work items one-by-one using a positional marker
From: Lai Jiangshan <jiangshan.ljs@...group.com>
Previously, the rescuer scanned for all matching work items at once and
processed them within a single rescuer thread, which could cause one
blocking work item to stall all others.
Make the rescuer process work items one-by-one instead of slurping all
matches in a single pass.
Break the rescuer loop after finding and processing the first matching
work item, then restart the search to pick up the next. This gives normal
worker threads a chance to process other items and prevents a blocking
work item from stalling the rest once memory pressure is relieved.
Introduce a positional dummy work item to avoid potentially O(N^2)
rescans of the work list. The marker records the resume position for
the next scan, eliminating redundant traversals.
The positional marker also has an additional benefit:
if it is processed by a normal worker rather than the rescuer, this
indicates that the pool has made forward progress. In that case, the pwq
does not need to be rescued until the next mayday event, allowing the
rescuer to focus on other stalled pools.
Processing items one-by-one also naturally handles newly scheduled work
items during rescue. For example, work items dispatched via
pwq_activate_first_inactive() or chained queueing will be picked up in
subsequent rescuer iterations without special handling.
Reported-by: ying chen <yc1082463@...il.com>
Cc: ying chen <yc1082463@...il.com>
Fixes: e22bee782b3b ("workqueue: implement concurrency managed dynamic worker pool")
Signed-off-by: Lai Jiangshan <jiangshan.ljs@...group.com>
---
kernel/workqueue.c | 99 +++++++++++++++++++++++++++++++---------------
1 file changed, 67 insertions(+), 32 deletions(-)
diff --git a/kernel/workqueue.c b/kernel/workqueue.c
index 45320e27a16c..af182a19a8b1 100644
--- a/kernel/workqueue.c
+++ b/kernel/workqueue.c
@@ -286,6 +286,7 @@ struct pool_workqueue {
struct list_head pending_node; /* LN: node on wq_node_nr_active->pending_pwqs */
struct list_head pwqs_node; /* WR: node on wq->pwqs */
struct list_head mayday_node; /* MD: node on wq->maydays */
+ struct work_struct mayday_pos_work;/* L: position on pool->worklist */
u64 stats[PWQ_NR_STATS];
@@ -2982,6 +2983,50 @@ static void idle_cull_fn(struct work_struct *work)
reap_dying_workers(&cull_list);
}
+static void mayday_pos_func(struct work_struct *work)
+{
+}
+
+/**
+ * insert_mayday_pos - Insert the positional work for mayday
+ * @pwq: The pwq that needs to be rescued
+ * @next: The next work where positional work will be inserted before
+ *
+ * CONTEXT:
+ * raw_spin_lock_irq(pool->lock).
+ */
+static void insert_mayday_pos(struct pool_workqueue *pwq, struct work_struct *next)
+{
+ unsigned int work_flags;
+ unsigned int work_color;
+
+ __set_bit(WORK_STRUCT_PENDING_BIT, work_data_bits(&pwq->mayday_pos_work));
+
+ /* The mayday positional work item does not participate in nr_active. */
+ work_flags = WORK_STRUCT_INACTIVE;
+ work_color = pwq->work_color;
+ work_flags |= work_color_to_flags(work_color);
+ pwq->nr_in_flight[work_color]++;
+ insert_work(pwq, &pwq->mayday_pos_work, &next->entry, work_flags);
+}
+
+/**
+ * remove_mayday_pos - Remove the inserted positional work
+ * @pwq: The pwq that needs to be rescued
+ *
+ * WORK_STRUCT_INACTIVE ensures that the pool lock is not dropped in
+ * pwq_dec_nr_in_flight()
+ *
+ * CONTEXT:
+ * raw_spin_lock_irq(pool->lock).
+ */
+static void remove_mayday_pos(struct pool_workqueue *pwq)
+{
+ list_del_init(&pwq->mayday_pos_work.entry);
+ pwq_dec_nr_in_flight(pwq, *work_data_bits(&pwq->mayday_pos_work));
+ INIT_WORK(&pwq->mayday_pos_work, mayday_pos_func);
+}
+
static void send_mayday(struct work_struct *work)
{
struct pool_workqueue *pwq = get_work_pwq(work);
@@ -2992,6 +3037,9 @@ static void send_mayday(struct work_struct *work)
if (!wq->rescuer)
return;
+ if (!work_pending(&pwq->mayday_pos_work))
+ insert_mayday_pos(pwq, work);
+
/* mayday mayday mayday */
if (list_empty(&pwq->mayday_node)) {
/*
@@ -3508,41 +3556,27 @@ static int rescuer_thread(void *__rescuer)
raw_spin_lock_irq(&pool->lock);
- /*
- * Slurp in all works issued via this workqueue and
- * process'em.
- */
- WARN_ON_ONCE(!list_empty(&rescuer->scheduled));
- list_for_each_entry_safe(work, n, &pool->worklist, entry) {
- if (get_work_pwq(work) == pwq &&
- assign_work(work, rescuer, &n))
+rescan:
+ /* If the positional work is processed by a normal worker,
+ * the pwq doesn't need to be rescued. */
+ if (work_pending(&pwq->mayday_pos_work)) {
+ /* scan from the positional work to avoid potentially O(N^2) scanning */
+ work = &pwq->mayday_pos_work;
+ list_for_each_entry_safe_continue(work, n, &pool->worklist, entry) {
+ if (get_work_pwq(work) != pwq)
+ continue;
+ if (!assign_work(work, rescuer, &n))
+ continue;
pwq->stats[PWQ_STAT_RESCUED]++;
- }
-
- if (!list_empty(&rescuer->scheduled)) {
- process_scheduled_works(rescuer);
-
- /*
- * The above execution of rescued work items could
- * have created more to rescue through
- * pwq_activate_first_inactive() or chained
- * queueing. Let's put @pwq back on mayday list so
- * that such back-to-back work items, which may be
- * being used to relieve memory pressure, don't
- * incur MAYDAY_INTERVAL delay inbetween.
- */
- if (pwq->nr_active && need_to_create_worker(pool)) {
- raw_spin_lock(&wq_mayday_lock);
- /*
- * Queue iff we aren't racing destruction
- * and somebody else hasn't queued it already.
- */
- if (wq->rescuer && list_empty(&pwq->mayday_node)) {
- get_pwq(pwq);
- list_add_tail(&pwq->mayday_node, &wq->maydays);
+ /* reset the position and handle the assigned work */
+ if (list_next_entry(&pwq->mayday_pos_work, entry) != n) {
+ remove_mayday_pos(pwq);
+ insert_mayday_pos(pwq, n);
}
- raw_spin_unlock(&wq_mayday_lock);
+ process_scheduled_works(rescuer);
+ goto rescan;
}
+ remove_mayday_pos(pwq);
}
/*
@@ -5162,6 +5196,7 @@ static void init_pwq(struct pool_workqueue *pwq, struct workqueue_struct *wq,
INIT_LIST_HEAD(&pwq->pending_node);
INIT_LIST_HEAD(&pwq->pwqs_node);
INIT_LIST_HEAD(&pwq->mayday_node);
+ INIT_WORK(&pwq->mayday_pos_work, mayday_pos_func);
kthread_init_work(&pwq->release_work, pwq_release_workfn);
}
--
2.19.1.6.gb485710b
Powered by blists - more mailing lists