Message-ID: <20240614034433.602622-5-liaochang1@huawei.com>
Date: Fri, 14 Jun 2024 03:44:27 +0000
From: Liao Chang <liaochang1@...wei.com>
To: <catalin.marinas@....com>, <will@...nel.org>, <ryabinin.a.a@...il.com>,
	<glider@...gle.com>, <andreyknvl@...il.com>, <dvyukov@...gle.com>,
	<vincenzo.frascino@....com>, <maz@...nel.org>, <oliver.upton@...ux.dev>,
	<james.morse@....com>, <suzuki.poulose@....com>, <yuzenghui@...wei.com>,
	<mark.rutland@....com>, <lpieralisi@...nel.org>, <tglx@...utronix.de>,
	<ardb@...nel.org>, <broonie@...nel.org>, <liaochang1@...wei.com>,
	<steven.price@....com>, <ryan.roberts@....com>, <pcc@...gle.com>,
	<anshuman.khandual@....com>, <eric.auger@...hat.com>,
	<miguel.luis@...cle.com>, <shiqiliu@...t.edu.cn>, <quic_jiles@...cinc.com>,
	<rafael@...nel.org>, <sudeep.holla@....com>, <dwmw@...zon.co.uk>,
	<joey.gouly@....com>, <jeremy.linton@....com>, <robh@...nel.org>,
	<scott@...amperecomputing.com>, <songshuaishuai@...ylab.org>,
	<swboyd@...omium.org>, <dianders@...omium.org>,
	<shijie@...amperecomputing.com>, <bhe@...hat.com>,
	<akpm@...ux-foundation.org>, <rppt@...nel.org>, <mhiramat@...nel.org>,
	<mcgrof@...nel.org>, <rmk+kernel@...linux.org.uk>,
	<Jonathan.Cameron@...wei.com>, <takakura@...inux.co.jp>,
	<sumit.garg@...aro.org>, <frederic@...nel.org>, <tabba@...gle.com>,
	<kristina.martsenko@....com>, <ruanjinjie@...wei.com>
CC: <linux-arm-kernel@...ts.infradead.org>, <linux-kernel@...r.kernel.org>,
	<kasan-dev@...glegroups.com>, <kvmarm@...ts.linux.dev>
Subject: [PATCH v4 04/10] arm64: daifflags: Introduce logical interrupt masking

Motivation
----------

This patch introduces a series of functions for managing interrupt
masking on ARM64, adopting a "logical" approach. This approach builds
upon the suggestion made by Mark Rutland in th FEAT_NMI patchset [1] to
refactor DAIF management.

Implementation
--------------

- A new union data type is defined to represent the combined interrupt
  masking context, includes ICC_PMR, PSTATE.DAIF and PSTATE.ALLINT.

- New helper functions offer a similar interface (starting with
  "local_allint_") to their existing couterparts (starting with
  "local_daif_"), ensuing compatibility with existing code.

- For platform or kernel that does not support FEAT_NMI, this patch uses
  local_allint_save_flags() to determine NMI masking behavior instead of
  relying on the PSTATE.A field which is not a straightforward way to
  understand and maintain for kernel with PSEUDO_NMI enabled.

Benefits
--------

This patch introduces a robust approach for managing interrupt context,
it removes the need to explicitly check the PSTATE.A field to determine
NMIs masking status. Additionally, the new series of interrupt context
save/restore/mask/inherit functions uses names that reflect their
purpose directly, ensuring consistent behavior regardless of platform
support for FEAT_NMI or PSEUDO_NMI.

[1] https://lore.kernel.org/linux-arm-kernel/Y4sH5qX5bK9xfEBp@lpieralisi/

Signed-off-by: Liao Chang <liaochang1@...wei.com>
---
 arch/arm64/include/asm/daifflags.h   | 243 +++++++++++++++++++++++++++
 arch/arm64/include/uapi/asm/ptrace.h |   1 +
 2 files changed, 244 insertions(+)

diff --git a/arch/arm64/include/asm/daifflags.h b/arch/arm64/include/asm/daifflags.h
index 55f57dfa8e2f..5d502cc0dac2 100644
--- a/arch/arm64/include/asm/daifflags.h
+++ b/arch/arm64/include/asm/daifflags.h
@@ -141,4 +141,247 @@ static inline void local_daif_inherit(struct pt_regs *regs)
 	 */
 	write_sysreg(flags, daif);
 }
+
+/*
+ * For Arm64 processor support Armv8.8 or later, kernel supports three types
+ * of irqflags, they used for corresponding configuration depicted as below:
+ *
+ * 1. When CONFIG_ARM64_PSEUDO_NMI and CONFIG_ARM64_NMI are not 'y', kernel
+ *    does not support handling NMI.
+ *
+ * 2. When CONFIG_ARM64_PSEUDO_NMI=y and irqchip.gicv3_pseudo_nmi=1, kernel
+ *    makes use of the CPU Interface PMR and GIC priority feature to support
+ *    handling NMI.
+ *
+ * 3. When CONFIG_ARM64_NMI=y and irqchip.gicv3_pseudo_nmi is not enabled,
+ *    kernel makes use of the FEAT_NMI extension added since Armv8.8 to
+ *    support handling NMI.
+ *
+ * The table below depicts the relationship between fields in struct
+ * arch_irqflags and corresponding interrupt masking behavior reflected in
+ * hardware registers.
+ *
+ * Legend:
+ *  IRQ = IRQ and FIQ.
+ *  NMI = PSEUDO_NMI or IRQ with superpriority for ARMv8.8.
+ *    M = Interrupt is masked.
+ *    U = Interrupt is unmasked.
+ *    * = Non relevant.
+ *
+ * IRQ | NMI | SError | ICC_PMR_EL1                       | PSR.DAIF | PSR.ALLINT
+ * ------------------------------------------------------------------------------
+ *  U  |  U  |   *    | GIC_PRIO_IRQON                    | 0b **00  | 0b 0
+ * ------------------------------------------------------------------------------
+ *  M  |  U  |   *    | GIC_PRIO_IRQOFF                   | 0b **00  | 0b 0
+ * ------------------------------------------------------------------------------
+ *  M  |  M  |   *    | (GIC_PRIO_IRQON | GIC_PRIO_I_SET) | 0b **11  | 0b 1
+ * ------------------------------------------------------------------------------
+ *  M  |  M  |   M    | (GIC_PRIO_IRQON | GIC_PRIO_I_SET) | 0b *111  | 0b 1
+ */
+union arch_irqflags {
+	unsigned long flags;
+	struct {
+		unsigned long pmr : 8;     // SYS_ICC_PMR_EL1
+		unsigned long daif : 10;   // PSTATE.DAIF at bits[6-9]
+		unsigned long allint : 14; // PSTATE.ALLINT at bits[13]
+	} fields;
+};
+
+typedef union arch_irqflags arch_irqflags_t;
+
+static inline void __local_pmr_mask(void)
+{
+	WARN_ON(system_has_prio_mask_debugging() &&
+		(read_sysreg_s(SYS_ICC_PMR_EL1) ==
+		 (GIC_PRIO_IRQOFF | GIC_PRIO_PSR_I_SET)));
+	/*
+	 * Don't really care for a dsb here, we don't intend to enable
+	 * IRQs.
+	 */
+	gic_write_pmr(GIC_PRIO_IRQON | GIC_PRIO_PSR_I_SET);
+}
+
+static inline void __local_nmi_mask(void)
+{
+	msr_pstate_allint(1);
+}
+
+static inline void local_allint_mask_notrace(void)
+{
+	asm volatile ("msr daifset, #0xf" : : : "memory");
+
+	if (system_uses_irq_prio_masking())
+		__local_pmr_mask();
+	else if (system_uses_nmi())
+		__local_nmi_mask();
+}
+
+static inline void local_allint_mask(void)
+{
+	local_allint_mask_notrace();
+	trace_hardirqs_off();
+}
+
+static inline arch_irqflags_t __local_save_pmr_daif_flags(void)
+{
+	arch_irqflags_t irqflags;
+
+	irqflags.fields.pmr = read_sysreg_s(SYS_ICC_PMR_EL1);
+	irqflags.fields.daif = read_sysreg(daif);
+
+	/*
+	 * If IRQs are masked with PMR, reflect it in the daif of irqflags.
+	 * If NMIs and IRQs are masked with PMR, reflect it in the allint
+	 * of irqflags, this avoid the need of checking PSTATE.A in
+	 * local_allint_restore() to determine if NMIs are masked.
+	 */
+	switch (irqflags.fields.pmr) {
+	case GIC_PRIO_IRQON:
+		irqflags.fields.allint = 0;
+		break;
+
+	case __GIC_PRIO_IRQOFF:
+	case __GIC_PRIO_IRQOFF_NS:
+		irqflags.fields.daif |= PSR_I_BIT | PSR_F_BIT;
+		irqflags.fields.allint = 0;
+		break;
+
+	case GIC_PRIO_IRQON | GIC_PRIO_PSR_I_SET:
+		irqflags.fields.daif |= PSR_I_BIT | PSR_F_BIT;
+		irqflags.fields.allint = PSR_ALLINT_BIT;
+		break;
+
+	default:
+		WARN_ON(1);
+	}
+
+	return irqflags;
+}
+
+static inline arch_irqflags_t __local_save_nmi_daif_flags(void)
+{
+	arch_irqflags_t irqflags;
+
+	irqflags.fields.daif = read_sysreg(daif);
+	irqflags.fields.allint = read_sysreg_s(SYS_ALLINT);
+
+	return irqflags;
+}
+
+static inline arch_irqflags_t local_allint_save_flags(void)
+{
+	arch_irqflags_t irqflags = { .flags = 0UL };
+
+	if (system_uses_irq_prio_masking())
+		return __local_save_pmr_daif_flags();
+
+	if (system_uses_nmi())
+		return __local_save_nmi_daif_flags();
+
+	irqflags.fields.daif = read_sysreg(daif);
+	return irqflags;
+}
+
+static inline arch_irqflags_t local_allint_save(void)
+{
+	arch_irqflags_t irqflags;
+
+	irqflags = local_allint_save_flags();
+
+	local_allint_mask();
+
+	return irqflags;
+}
+
+static inline void __local_pmr_restore(arch_irqflags_t irqflags)
+{
+	/*
+	 * There has been concern that the write to daif
+	 * might be reordered before this write to PMR.
+	 * From the ARM ARM DDI 0487D.a, section D1.7.1
+	 * "Accessing PSTATE fields":
+	 *   Writes to the PSTATE fields have side-effects on
+	 *   various aspects of the PE operation. All of these
+	 *   side-effects are guaranteed:
+	 *     - Not to be visible to earlier instructions in
+	 *       the execution stream.
+	 *     - To be visible to later instructions in the
+	 *       execution stream
+	 *
+	 * Also, writes to PMR are self-synchronizing, so no
+	 * interrupts with a lower priority than PMR is signaled
+	 * to the PE after the write.
+	 *
+	 * So we don't need additional synchronization here.
+	 */
+	gic_write_pmr(irqflags.fields.pmr);
+}
+
+static inline void __local_nmi_restore(arch_irqflags_t irqflags)
+{
+	msr_pstate_allint(!!irqflags.fields.allint ? 1 : 0);
+}
+
+static inline int local_hardirqs_disabled(arch_irqflags_t irqflags)
+{
+	return irqflags.fields.allint || (irqflags.fields.daif & PSR_I_BIT);
+}
+
+static inline void __local_allint_restore(arch_irqflags_t irqflags)
+{
+	if (system_uses_irq_prio_masking())
+		__local_pmr_restore(irqflags);
+	else if (system_uses_nmi())
+		__local_nmi_restore(irqflags);
+
+	write_sysreg(irqflags.fields.daif, daif);
+}
+
+static inline void local_allint_restore_notrace(arch_irqflags_t irqflags)
+{
+	/*
+	 * Use arch_allint.fields.allint to indicates we can take
+	 * NMIs, instead of the old hacking style that use PSTATE.A.
+	 */
+	if (system_uses_irq_prio_masking() && !irqflags.fields.allint)
+		irqflags.fields.daif &= ~(PSR_I_BIT | PSR_F_BIT);
+
+	__local_allint_restore(irqflags);
+}
+
+/*
+ * It has to conside the different kernel configure and parameters, that need
+ * to use corresponding operations to mask interrupts properly. For example,
+ * the kernel disable PSEUDO_NMI, the kernel uses prio masking to support
+ * PSEUDO_NMI, or the kernel uses FEAT_NMI extension to support ARM64_NMI.
+ */
+static inline void local_allint_restore(arch_irqflags_t irqflags)
+{
+	int irq_disabled = local_hardirqs_disabled(irqflags);
+
+	if (!irq_disabled)
+		trace_hardirqs_on();
+
+	local_allint_restore_notrace(irqflags);
+
+	if (irq_disabled)
+		trace_hardirqs_off();
+}
+
+/*
+ * Called by synchronous exception handlers to restore the DAIF bits that were
+ * modified by taking an exception.
+ */
+static inline void local_allint_inherit(struct pt_regs *regs)
+{
+	arch_irqflags_t irqflags;
+
+	if (interrupts_enabled(regs))
+		trace_hardirqs_on();
+
+	irqflags.fields.pmr = regs->pmr_save;
+	irqflags.fields.daif = regs->pstate & DAIF_MASK;
+	irqflags.fields.allint = regs->pstate & PSR_ALLINT_BIT;
+	__local_allint_restore(irqflags);
+}
 #endif
diff --git a/arch/arm64/include/uapi/asm/ptrace.h b/arch/arm64/include/uapi/asm/ptrace.h
index 7fa2f7036aa7..8a125a1986be 100644
--- a/arch/arm64/include/uapi/asm/ptrace.h
+++ b/arch/arm64/include/uapi/asm/ptrace.h
@@ -48,6 +48,7 @@
 #define PSR_D_BIT	0x00000200
 #define PSR_BTYPE_MASK	0x00000c00
 #define PSR_SSBS_BIT	0x00001000
+#define PSR_ALLINT_BIT	0x00002000
 #define PSR_PAN_BIT	0x00400000
 #define PSR_UAO_BIT	0x00800000
 #define PSR_DIT_BIT	0x01000000
-- 
2.34.1

Powered by blists - more mailing lists