linux-kernel - [PATCH 2/4] softirq: Introduce raise_ksoftirqd

lists.openwall.net		lists / announce owl-users owl-dev john-users john-dev passwdqc-users yescrypt popa3d-users / oss-security kernel-hardening musl sabotage tlsify passwords / crypt-dev xvendor / Bugtraq Full-Disclosure linux-kernel linux-netdev linux-ext4 linux-hardening linux-cve-announce PHC
Open Source and information security mailing list archives

Hash Suite: Windows password security audit tool. GUI, reports in PDF.

[<prev] [next>] [<thread-prev] [thread-next>] [day] [month] [year] [list]

Message-Id: <20231019233543.1243121-3-frederic@kernel.org>
Date:   Fri, 20 Oct 2023 01:35:41 +0200
From:   Frederic Weisbecker <frederic@...nel.org>
To:     LKML <linux-kernel@...r.kernel.org>
Cc:     Frederic Weisbecker <frederic@...nel.org>,
        Boqun Feng <boqun.feng@...il.com>,
        Joel Fernandes <joel@...lfernandes.org>,
        Josh Triplett <josh@...htriplett.org>,
        Mathieu Desnoyers <mathieu.desnoyers@...icios.com>,
        Neeraj Upadhyay <neeraj.upadhyay@....com>,
        "Paul E . McKenney" <paulmck@...nel.org>,
        Steven Rostedt <rostedt@...dmis.org>,
        Uladzislau Rezki <urezki@...il.com>, rcu <rcu@...r.kernel.org>,
        Zqiang <qiang.zhang1211@...il.com>,
        Lai Jiangshan <jiangshanlai@...il.com>,
        "Liam R . Howlett" <Liam.Howlett@...cle.com>,
        Peter Zijlstra <peterz@...radead.org>,
        Sebastian Siewior <bigeasy@...utronix.de>,
        Thomas Gleixner <tglx@...utronix.de>
Subject: [PATCH 2/4] softirq: Introduce raise_ksoftirqd_irqoff()

Provide a function to raise a softirq vector and force the wakeup of
ksoftirqd along the way, irrespective of the current interrupt context.

This is going to be used by rcutiny to fix and optimize the triggering
of quiescent states from idle.

Fixes: cff9b2332ab7 ("kernel/sched: Modify initial boot task idle setup")
Cc: Liam R. Howlett <Liam.Howlett@...cle.com>
Cc: Peter Zijlstra (Intel) <peterz@...radead.org>
Cc: Sebastian Siewior <bigeasy@...utronix.de>
Cc: Thomas Gleixner <tglx@...utronix.de>
Signed-off-by: Frederic Weisbecker <frederic@...nel.org>
---
 include/linux/interrupt.h |  1 +
 kernel/softirq.c          | 71 +++++++++++++++++++++++----------------
 2 files changed, 43 insertions(+), 29 deletions(-)

diff --git a/include/linux/interrupt.h b/include/linux/interrupt.h
index 558a1a329da9..301d2956e746 100644
--- a/include/linux/interrupt.h
+++ b/include/linux/interrupt.h
@@ -608,6 +608,7 @@ extern void raise_softirq_no_wake(unsigned int nr);
 
 extern void raise_softirq_irqoff(unsigned int nr);
 extern void raise_softirq(unsigned int nr);
+extern void raise_ksoftirqd_irqoff(unsigned int nr);
 
 DECLARE_PER_CPU(struct task_struct *, ksoftirqd);
 
diff --git a/kernel/softirq.c b/kernel/softirq.c
index acfed6f3701d..9c29a8ced1c3 100644
--- a/kernel/softirq.c
+++ b/kernel/softirq.c
@@ -659,35 +659,6 @@ void irq_exit(void)
 	lockdep_hardirq_exit();
 }
 
-/*
- * This function must run with irqs disabled!
- */
-inline void raise_softirq_irqoff(unsigned int nr)
-{
-	raise_softirq_no_wake(nr);
-
-	/*
-	 * If we're in an interrupt or softirq, we're done
-	 * (this also catches softirq-disabled code). We will
-	 * actually run the softirq once we return from
-	 * the irq or softirq.
-	 *
-	 * Otherwise we wake up ksoftirqd to make sure we
-	 * schedule the softirq soon.
-	 */
-	if (!in_interrupt() && should_wake_ksoftirqd())
-		wakeup_softirqd();
-}
-
-void raise_softirq(unsigned int nr)
-{
-	unsigned long flags;
-
-	local_irq_save(flags);
-	raise_softirq_irqoff(nr);
-	local_irq_restore(flags);
-}
-
 void raise_softirq_no_wake(unsigned int nr)
 {
 	lockdep_assert_irqs_disabled();
@@ -695,6 +666,48 @@ void raise_softirq_no_wake(unsigned int nr)
 	or_softirq_pending(1UL << nr);
 }
 
+/*
+ * This function must run with irqs disabled!
+ */
+static inline void __raise_softirq_irqoff(unsigned int nr, bool threaded)
+{
+	raise_softirq_no_wake(nr);
+
+	if (threaded && should_wake_ksoftirqd())
+		wakeup_softirqd();
+}
+
+/*
+ * This function must run with irqs disabled!
+ */
+inline void raise_softirq_irqoff(unsigned int nr)
+{
+	bool threaded;
+	/*
+	 * If in an interrupt or softirq (servicing or disabled
+	 * section), the vector will be handled at the end of
+	 * the interrupt or softirq servicing/disabled section.
+	 * Otherwise the vector must rely on ksoftirqd.
+	 */
+	threaded = !in_interrupt();
+
+	__raise_softirq_irqoff(nr, threaded);
+}
+
+void raise_softirq(unsigned int nr)
+{
+	unsigned long flags;
+
+	local_irq_save(flags);
+	raise_softirq_irqoff(nr);
+	local_irq_restore(flags);
+}
+
+void raise_ksoftirqd_irqoff(unsigned int nr)
+{
+	__raise_softirq_irqoff(nr, true);
+}
+
 void open_softirq(int nr, void (*action)(struct softirq_action *))
 {
 	softirq_vec[nr].action = action;
-- 
2.34.1