| lists.openwall.net | lists / announce owl-users owl-dev john-users john-dev passwdqc-users yescrypt popa3d-users / oss-security kernel-hardening musl sabotage tlsify passwords / crypt-dev xvendor / Bugtraq Full-Disclosure linux-kernel linux-netdev linux-ext4 linux-hardening linux-cve-announce PHC | |
|
Open Source and information security mailing list archives
| ||
|
Message-Id: <20250721-vmcore_hw_error-v2-1-ab65a6b43c5a@debian.org>
Date: Mon, 21 Jul 2025 03:13:40 -0700
From: Breno Leitao <leitao@...ian.org>
To: "Rafael J. Wysocki" <rafael@...nel.org>, Len Brown <lenb@...nel.org>,
James Morse <james.morse@....com>, Tony Luck <tony.luck@...el.com>,
Borislav Petkov <bp@...en8.de>, Robert Moore <robert.moore@...el.com>,
Thomas Gleixner <tglx@...utronix.de>, Ingo Molnar <mingo@...hat.com>,
Dave Hansen <dave.hansen@...ux.intel.com>, x86@...nel.org,
"H. Peter Anvin" <hpa@...or.com>, Hanjun Guo <guohanjun@...wei.com>,
Mauro Carvalho Chehab <mchehab@...nel.org>,
Mahesh J Salgaonkar <mahesh@...ux.ibm.com>,
Oliver O'Halloran <oohall@...il.com>, Bjorn Helgaas <bhelgaas@...gle.com>
Cc: linux-acpi@...r.kernel.org, linux-kernel@...r.kernel.org,
acpica-devel@...ts.linux.dev, osandov@...ndov.com,
xueshuai@...ux.alibaba.com, konrad.wilk@...cle.com,
linux-edac@...r.kernel.org, linuxppc-dev@...ts.ozlabs.org,
linux-pci@...r.kernel.org, kernel-team@...a.com,
Breno Leitao <leitao@...ian.org>
Subject: [PATCH v2] vmcoreinfo: Track and log recoverable hardware errors
Introduce a generic infrastructure for tracking recoverable hardware
errors (HW errors that did not cause a panic) and record them for vmcore
consumption. This aids post-mortem crash analysis tools by preserving
a count and timestamp for the last occurrence of such errors.
This patch adds centralized logging for three common sources of
recoverable hardware errors:
- PCIe AER Correctable errors
- x86 Machine Check Exceptions (MCE)
- APEI/CPER GHES corrected or recoverable errors
hwerror_tracking is write-only at kernel runtime, and it is meant to be
read from vmcore using tools like crash/drgn. For example, this is how
it looks like when opening the crashdump from drgn.
>>> prog['hwerror_tracking']
(struct hwerror_tracking_info [3]){
{
.count = (int)844,
.timestamp = (time64_t)1752852018,
},
...
Suggested-by: Tony Luck <tony.luck@...el.com>
Signed-off-by: Breno Leitao <leitao@...ian.org>
---
Changes in v2:
- Split the counter by recoverable error (Tony Luck)
- Link to v1: https://lore.kernel.org/r/20250714-vmcore_hw_error-v1-1-8cf45edb6334@debian.org
---
arch/x86/kernel/cpu/mce/core.c | 3 +++
drivers/acpi/apei/ghes.c | 8 ++++++--
drivers/pci/pcie/aer.c | 2 ++
include/linux/vmcore_info.h | 14 ++++++++++++++
kernel/vmcore_info.c | 18 ++++++++++++++++++
5 files changed, 43 insertions(+), 2 deletions(-)
diff --git a/arch/x86/kernel/cpu/mce/core.c b/arch/x86/kernel/cpu/mce/core.c
index 4da4eab56c81d..781cf574642eb 100644
--- a/arch/x86/kernel/cpu/mce/core.c
+++ b/arch/x86/kernel/cpu/mce/core.c
@@ -45,6 +45,7 @@
#include <linux/task_work.h>
#include <linux/hardirq.h>
#include <linux/kexec.h>
+#include <linux/vmcore_info.h>
#include <asm/fred.h>
#include <asm/cpu_device_id.h>
@@ -1692,6 +1693,8 @@ noinstr void do_machine_check(struct pt_regs *regs)
out:
instrumentation_end();
+ /* Given it didn't panic, mark it as recoverable */
+ hwerror_tracking_log(HWE_RECOV_MCE);
clear:
mce_wrmsrq(MSR_IA32_MCG_STATUS, 0);
}
diff --git a/drivers/acpi/apei/ghes.c b/drivers/acpi/apei/ghes.c
index a0d54993edb3b..396cdffbe6a37 100644
--- a/drivers/acpi/apei/ghes.c
+++ b/drivers/acpi/apei/ghes.c
@@ -43,6 +43,7 @@
#include <linux/uuid.h>
#include <linux/ras.h>
#include <linux/task_work.h>
+#include <linux/vmcore_info.h>
#include <acpi/actbl1.h>
#include <acpi/ghes.h>
@@ -1136,13 +1137,16 @@ static int ghes_proc(struct ghes *ghes)
{
struct acpi_hest_generic_status *estatus = ghes->estatus;
u64 buf_paddr;
- int rc;
+ int rc, sev;
rc = ghes_read_estatus(ghes, estatus, &buf_paddr, FIX_APEI_GHES_IRQ);
if (rc)
goto out;
- if (ghes_severity(estatus->error_severity) >= GHES_SEV_PANIC)
+ sev = ghes_severity(estatus->error_severity);
+ if (sev == GHES_SEV_RECOVERABLE || sev == GHES_SEV_CORRECTED)
+ hwerror_tracking_log(HWE_RECOV_GHES);
+ else if (sev >= GHES_SEV_PANIC)
__ghes_panic(ghes, estatus, buf_paddr, FIX_APEI_GHES_IRQ);
if (!ghes_estatus_cached(estatus)) {
diff --git a/drivers/pci/pcie/aer.c b/drivers/pci/pcie/aer.c
index e286c197d7167..064d220564b59 100644
--- a/drivers/pci/pcie/aer.c
+++ b/drivers/pci/pcie/aer.c
@@ -30,6 +30,7 @@
#include <linux/kfifo.h>
#include <linux/ratelimit.h>
#include <linux/slab.h>
+#include <linux/vmcore_info.h>
#include <acpi/apei.h>
#include <acpi/ghes.h>
#include <ras/ras_event.h>
@@ -746,6 +747,7 @@ static void pci_dev_aer_stats_incr(struct pci_dev *pdev,
switch (info->severity) {
case AER_CORRECTABLE:
aer_info->dev_total_cor_errs++;
+ hwerror_tracking_log(HWE_RECOV_AER);
counter = &aer_info->dev_cor_errs[0];
max = AER_MAX_TYPEOF_COR_ERRS;
break;
diff --git a/include/linux/vmcore_info.h b/include/linux/vmcore_info.h
index 37e003ae52626..5894da92a6ba4 100644
--- a/include/linux/vmcore_info.h
+++ b/include/linux/vmcore_info.h
@@ -77,4 +77,18 @@ extern u32 *vmcoreinfo_note;
Elf_Word *append_elf_note(Elf_Word *buf, char *name, unsigned int type,
void *data, size_t data_len);
void final_note(Elf_Word *buf);
+
+enum hwerror_tracking_source {
+ HWE_RECOV_AER,
+ HWE_RECOV_MCE,
+ HWE_RECOV_GHES,
+ HWE_RECOV_MAX,
+};
+
+#ifdef CONFIG_VMCORE_INFO
+void hwerror_tracking_log(enum hwerror_tracking_source src);
+#else
+void hwerror_tracking_log(enum hwerror_tracking_source src) {};
+#endif
+
#endif /* LINUX_VMCORE_INFO_H */
diff --git a/kernel/vmcore_info.c b/kernel/vmcore_info.c
index e066d31d08f89..23d7ddcd55cdd 100644
--- a/kernel/vmcore_info.c
+++ b/kernel/vmcore_info.c
@@ -31,6 +31,13 @@ u32 *vmcoreinfo_note;
/* trusted vmcoreinfo, e.g. we can make a copy in the crash memory */
static unsigned char *vmcoreinfo_data_safecopy;
+struct hwerror_tracking_info {
+ int __data_racy count;
+ time64_t __data_racy timestamp;
+};
+
+static struct hwerror_tracking_info hwerror_tracking[HWE_RECOV_MAX];
+
Elf_Word *append_elf_note(Elf_Word *buf, char *name, unsigned int type,
void *data, size_t data_len)
{
@@ -118,6 +125,17 @@ phys_addr_t __weak paddr_vmcoreinfo_note(void)
}
EXPORT_SYMBOL(paddr_vmcoreinfo_note);
+void hwerror_tracking_log(enum hwerror_tracking_source src)
+{
+ if (src < 0 || src >= HWE_RECOV_MAX)
+ return;
+
+ /* No need to atomics/locks given the precision is not important */
+ hwerror_tracking[src].count++;
+ hwerror_tracking[src].timestamp = ktime_get_real_seconds();
+}
+EXPORT_SYMBOL_GPL(hwerror_tracking_log);
+
static int __init crash_save_vmcoreinfo_init(void)
{
vmcoreinfo_data = (unsigned char *)get_zeroed_page(GFP_KERNEL);
---
base-commit: 97987520025658f30bb787a99ffbd9bbff9ffc9d
change-id: 20250707-vmcore_hw_error-322429e6c316
Best regards,
--
Breno Leitao <leitao@...ian.org>
Powered by blists - more mailing lists