linux-kernel - [PATCH 1/3] kernel/watchdog: show irq percentage if irq floods

lists.openwall.net		lists / announce owl-users owl-dev john-users john-dev passwdqc-users yescrypt popa3d-users / oss-security kernel-hardening musl sabotage tlsify passwords / crypt-dev xvendor / Bugtraq Full-Disclosure linux-kernel linux-netdev linux-ext4 linux-hardening linux-cve-announce PHC
Open Source and information security mailing list archives
Hash Suite: Windows password security audit tool. GUI, reports in PDF.
[<prev] [next>] [<thread-prev] [thread-next>] [day] [month] [year] [list]
Date:   Thu, 22 Oct 2020 13:56:01 +0800
From:   Pingfan Liu <kernelfans@...il.com>
To:     linux-kernel@...r.kernel.org
Cc:     Pingfan Liu <kernelfans@...il.com>,
        Thomas Gleixner <tglx@...utronix.de>,
        Peter Zijlstra <peterz@...radead.org>,
        Jisheng Zhang <Jisheng.Zhang@...aptics.com>,
        Andrew Morton <akpm@...ux-foundation.org>,
        "Guilherme G. Piccoli" <gpiccoli@...onical.com>,
        Petr Mladek <pmladek@...e.com>, Marc Zyngier <maz@...nel.org>,
        Linus Walleij <linus.walleij@...aro.org>,
        afzal mohammed <afzal.mohd.ma@...il.com>,
        Lina Iyer <ilina@...eaurora.org>,
        "Gustavo A. R. Silva" <gustavo@...eddedor.com>,
        Maulik Shah <mkshah@...eaurora.org>,
        Al Viro <viro@...iv.linux.org.uk>,
        Jonathan Corbet <corbet@....net>,
        Pawan Gupta <pawan.kumar.gupta@...ux.intel.com>,
        Mike Kravetz <mike.kravetz@...cle.com>,
        Oliver Neukum <oneukum@...e.com>, linux-doc@...r.kernel.org,
        kexec@...ts.infradead.org
Subject: [PATCH 1/3] kernel/watchdog: show irq percentage if irq floods

In kdump case, the capture kernel has no chance to shutdown devices, and
leave the devices in unstable state. Irq flood is one of the consequence.

When irq floods, the kernel is totally occupies by irq.  Currently, there
may be nothing or just soft lockup warning showed in console. It is better
to warn users with irq flood info.

Soft lockup watchdog is a good foundation to implement the detector. A irq
flood is reported if the following two conditions are met:
  -1. the irq occupies too much (98%) of the past interval.
  -2. no tasks has been scheduled in the past interval. This is implemented
      by check_hint.
      A note: is_softlockup() implies the 2nd condition, but unfortunately, irq
      flood can come from anywhere. If irq_enter_rcu()->tick_irq_enter(), then
      touch_softlockup_watchdog_sched() will reset watchdog, and softlockup is
      never detected.

Signed-off-by: Pingfan Liu <kernelfans@...il.com>
Cc: Thomas Gleixner <tglx@...utronix.de>
Cc: Peter Zijlstra <peterz@...radead.org>
Cc: Jisheng Zhang <Jisheng.Zhang@...aptics.com>
Cc: Andrew Morton <akpm@...ux-foundation.org>
Cc: "Guilherme G. Piccoli" <gpiccoli@...onical.com>
Cc: Petr Mladek <pmladek@...e.com>
Cc: Marc Zyngier <maz@...nel.org>
Cc: Linus Walleij <linus.walleij@...aro.org>
Cc: afzal mohammed <afzal.mohd.ma@...il.com>
Cc: Lina Iyer <ilina@...eaurora.org>
Cc: "Gustavo A. R. Silva" <gustavo@...eddedor.com>
Cc: Maulik Shah <mkshah@...eaurora.org>
Cc: Al Viro <viro@...iv.linux.org.uk>
Cc: Jonathan Corbet <corbet@....net>
Cc: Pawan Gupta <pawan.kumar.gupta@...ux.intel.com>
Cc: Mike Kravetz <mike.kravetz@...cle.com>
Cc: Oliver Neukum <oneukum@...e.com>
To: linux-kernel@...r.kernel.org
Cc: linux-doc@...r.kernel.org
Cc: kexec@...ts.infradead.org
---
 kernel/watchdog.c | 41 +++++++++++++++++++++++++++++++++++++++++
 1 file changed, 41 insertions(+)

diff --git a/kernel/watchdog.c b/kernel/watchdog.c
index 5abb5b2..230ac38 100644
--- a/kernel/watchdog.c
+++ b/kernel/watchdog.c
@@ -23,6 +23,7 @@
 #include <linux/sched/debug.h>
 #include <linux/sched/isolation.h>
 #include <linux/stop_machine.h>
+#include <linux/kernel_stat.h>
 
 #include <asm/irq_regs.h>
 #include <linux/kvm_para.h>
@@ -175,6 +176,13 @@ static DEFINE_PER_CPU(bool, softlockup_touch_sync);
 static DEFINE_PER_CPU(bool, soft_watchdog_warn);
 static DEFINE_PER_CPU(unsigned long, hrtimer_interrupts);
 static DEFINE_PER_CPU(unsigned long, hrtimer_interrupts_saved);
+
+#ifdef CONFIG_IRQ_TIME_ACCOUNTING
+static DEFINE_PER_CPU(long, check_hint) = {-1};
+static DEFINE_PER_CPU(unsigned long, last_irq_ts);
+static DEFINE_PER_CPU(unsigned long, last_total_ts);
+#endif
+
 static unsigned long soft_lockup_nmi_warn;
 
 static int __init nowatchdog_setup(char *str)
@@ -331,12 +339,43 @@ static DEFINE_PER_CPU(struct cpu_stop_work, softlockup_stop_work);
  */
 static int softlockup_fn(void *data)
 {
+#ifdef CONFIG_IRQ_TIME_ACCOUNTING
+	__this_cpu_write(check_hint, -1);
+#endif
 	__touch_watchdog();
 	complete(this_cpu_ptr(&softlockup_completion));
 
 	return 0;
 }
 
+#ifdef CONFIG_IRQ_TIME_ACCOUNTING
+static void check_irq_flood(void)
+{
+	unsigned long irqts, totalts, percent, cnt;
+	u64 *cpustat = kcpustat_this_cpu->cpustat;
+
+	totalts = running_clock();
+	irqts = cpustat[CPUTIME_IRQ] + cpustat[CPUTIME_SOFTIRQ];
+	cnt = __this_cpu_inc_return(check_hint);
+
+	if (cnt >= 5) {
+		totalts = totalts - __this_cpu_read(last_total_ts);
+		irqts = irqts - __this_cpu_read(last_irq_ts);
+		percent = irqts * 100 / totalts;
+		percent =  percent < 100 ? percent : 100;
+		__this_cpu_write(check_hint, -1);
+		if (percent >= 98)
+			pr_info("Irq flood occupies more than %lu%% of the past %lu seconds\n",
+				percent, totalts >> 30);
+	} else if (cnt == 0) {
+		__this_cpu_write(last_total_ts, totalts);
+		__this_cpu_write(last_irq_ts, irqts);
+	}
+}
+#else
+static void check_irq_flood(void){}
+#endif
+
 /* watchdog kicker functions */
 static enum hrtimer_restart watchdog_timer_fn(struct hrtimer *hrtimer)
 {
@@ -348,6 +387,8 @@ static enum hrtimer_restart watchdog_timer_fn(struct hrtimer *hrtimer)
 	if (!watchdog_enabled)
 		return HRTIMER_NORESTART;
 
+	/* When irq floods, watchdog may be still touched. Hence it can not be done inside lockup */
+	check_irq_flood();
 	/* kick the hardlockup detector */
 	watchdog_interrupt_count();
 
-- 
2.7.5