| lists.openwall.net | lists / announce owl-users owl-dev john-users john-dev passwdqc-users yescrypt popa3d-users / oss-security kernel-hardening musl sabotage tlsify passwords / crypt-dev xvendor / Bugtraq Full-Disclosure linux-kernel linux-netdev linux-ext4 linux-hardening linux-cve-announce PHC | |
|
Open Source and information security mailing list archives
| ||
|
Message-Id: <20220311101940.3403607-4-tarumizu.kohei@fujitsu.com>
Date: Fri, 11 Mar 2022 19:19:35 +0900
From: Kohei Tarumizu <tarumizu.kohei@...itsu.com>
To: catalin.marinas@....com, will@...nel.org, tglx@...utronix.de,
mingo@...hat.com, bp@...en8.de, dave.hansen@...ux.intel.com,
x86@...nel.org, hpa@...or.com,
linux-arm-kernel@...ts.infradead.org, linux-kernel@...r.kernel.org
Cc: tarumizu.kohei@...itsu.com
Subject: [PATCH v2 3/8] arm64: Add hardware prefetch control support for ARM64
This adds module init/exit code, and creates sysfs attribute files for
"stream_detect_prefetcher_enable", "stream_detect_prefetcher_strong"
and "stream_detect_prefetcher_dist". This driver works only if part
number is FUJITSU_CPU_PART_A64FX at this point. The details of the
registers to be read and written in this patch are described below.
"https://github.com/fujitsu/A64FX/tree/master/doc/"
A64FX_Specification_HPC_Extension_v1_EN.pdf
Signed-off-by: Kohei Tarumizu <tarumizu.kohei@...itsu.com>
---
arch/arm64/kernel/pfctl.c | 368 ++++++++++++++++++++++++++++++++++++++
1 file changed, 368 insertions(+)
create mode 100644 arch/arm64/kernel/pfctl.c
diff --git a/arch/arm64/kernel/pfctl.c b/arch/arm64/kernel/pfctl.c
new file mode 100644
index 000000000000..0487c763b206
--- /dev/null
+++ b/arch/arm64/kernel/pfctl.c
@@ -0,0 +1,368 @@
+// SPDX-License-Identifier: GPL-2.0
+/*
+ * Copyright 2022 FUJITSU LIMITED
+ *
+ * ARM64 Hardware Prefetch Control support
+ */
+
+#include <asm/cputype.h>
+#include <linux/bitfield.h>
+#include <linux/cacheinfo.h>
+#include <linux/module.h>
+#include <linux/pfctl.h>
+#include <linux/parser.h>
+
+struct pfctl_driver arm64_pfctl_driver;
+
+/**************************************
+ * FUJITSU A64FX support
+ **************************************/
+
+/*
+ * Constants for these add the "A64FX_SDPF" prefix to the name described in
+ * section "1.3.4.2. IMP_PF_STREAM_DETECT_CTRL_EL0" of "A64FX specification".
+ * (https://github.com/fujitsu/A64FX/tree/master/doc/A64FX_Specification_HPC_Extension_v1_EN.pdf")
+ * See this document for register specification details.
+ */
+#define A64FX_SDPF_IMP_PF_STREAM_DETECT_CTRL_EL0 sys_reg(3, 3, 11, 4, 0)
+#define A64FX_SDPF_V BIT_ULL(63)
+#define A64FX_SDPF_L1PF_DIS BIT_ULL(59)
+#define A64FX_SDPF_L2PF_DIS BIT_ULL(58)
+#define A64FX_SDPF_L1W BIT_ULL(55)
+#define A64FX_SDPF_L2W BIT_ULL(54)
+#define A64FX_SDPF_L1_DIST GENMASK_ULL(27, 24)
+#define A64FX_SDPF_L2_DIST GENMASK_ULL(19, 16)
+
+#define A64FX_SDPF_MIN_DIST_L1 256
+#define A64FX_SDPF_MIN_DIST_L2 1024
+
+struct a64fx_read_info {
+ enum pfctl_attr pattr;
+ u64 val;
+ unsigned int level;
+ int ret;
+};
+
+struct a64fx_write_info {
+ enum pfctl_attr pattr;
+ u64 val;
+ unsigned int level;
+ int ret;
+};
+
+static int a64fx_get_sdpf_enable(u64 reg, unsigned int level)
+{
+ u64 val;
+
+ switch (level) {
+ case 1:
+ val = FIELD_GET(A64FX_SDPF_L1PF_DIS, reg);
+ break;
+ case 2:
+ val = FIELD_GET(A64FX_SDPF_L2PF_DIS, reg);
+ break;
+ default:
+ return -EINVAL;
+ }
+
+ if (val == 0)
+ return PFCTL_ENABLE_VAL;
+ else if (val == 1)
+ return PFCTL_DISABLE_VAL;
+ else
+ return -EINVAL;
+}
+
+static int a64fx_modify_sdpf_enable(u64 *reg, unsigned int level, u64 val)
+{
+ if (val == PFCTL_ENABLE_VAL)
+ val = 0;
+ else
+ val = 1;
+
+ switch (level) {
+ case 1:
+ *reg &= ~A64FX_SDPF_L1PF_DIS;
+ *reg |= FIELD_PREP(A64FX_SDPF_L1PF_DIS, val);
+ break;
+ case 2:
+ *reg &= ~A64FX_SDPF_L2PF_DIS;
+ *reg |= FIELD_PREP(A64FX_SDPF_L2PF_DIS, val);
+ break;
+ default:
+ return -EINVAL;
+ }
+
+ return 0;
+}
+
+static int a64fx_get_sdpf_strong(u64 reg, unsigned int level)
+{
+ u64 val;
+
+ switch (level) {
+ case 1:
+ val = FIELD_GET(A64FX_SDPF_L1W, reg);
+ break;
+ case 2:
+ val = FIELD_GET(A64FX_SDPF_L2W, reg);
+ break;
+ default:
+ return -EINVAL;
+ }
+
+ if (val == 0)
+ return PFCTL_STRONG_VAL;
+ else if (val == 1)
+ return PFCTL_WEAK_VAL;
+ else
+ return -EINVAL;
+}
+
+static int a64fx_modify_sdpf_strong(u64 *reg, unsigned int level, u64 val)
+{
+ if (val == PFCTL_STRONG_VAL)
+ val = 0;
+ else
+ val = 1;
+
+ switch (level) {
+ case 1:
+ *reg &= ~A64FX_SDPF_L1W;
+ *reg |= FIELD_PREP(A64FX_SDPF_L1W, val);
+ break;
+ case 2:
+ *reg &= ~A64FX_SDPF_L2W;
+ *reg |= FIELD_PREP(A64FX_SDPF_L2W, val);
+ break;
+ default:
+ return -EINVAL;
+ }
+
+ return 0;
+}
+
+static int a64fx_get_sdpf_dist(u64 reg, unsigned int level)
+{
+ switch (level) {
+ case 1:
+ return FIELD_GET(A64FX_SDPF_L1_DIST, reg) *
+ A64FX_SDPF_MIN_DIST_L1;
+ case 2:
+ return FIELD_GET(A64FX_SDPF_L2_DIST, reg) *
+ A64FX_SDPF_MIN_DIST_L2;
+ default:
+ return -EINVAL;
+ }
+}
+
+static int a64fx_modify_sdpf_dist(u64 *reg, unsigned int level, u64 val)
+{
+ switch (level) {
+ case 1:
+ val = roundup(val, A64FX_SDPF_MIN_DIST_L1) /
+ A64FX_SDPF_MIN_DIST_L1;
+ if (!FIELD_FIT(A64FX_SDPF_L1_DIST, val))
+ return -EINVAL;
+ *reg &= ~A64FX_SDPF_L1_DIST;
+ *reg |= FIELD_PREP(A64FX_SDPF_L1_DIST, val);
+ break;
+ case 2:
+ val = roundup(val, A64FX_SDPF_MIN_DIST_L2) /
+ A64FX_SDPF_MIN_DIST_L2;
+ if (!FIELD_FIT(A64FX_SDPF_L2_DIST, val))
+ return -EINVAL;
+ *reg &= ~A64FX_SDPF_L2_DIST;
+ *reg |= FIELD_PREP(A64FX_SDPF_L2_DIST, val);
+ break;
+ default:
+ return -EINVAL;
+ }
+
+ return 0;
+}
+
+static void a64fx_enable_sdpf_verify(u64 *reg)
+{
+ *reg &= ~A64FX_SDPF_V;
+ *reg |= FIELD_PREP(A64FX_SDPF_V, 1);
+}
+
+static int a64fx_get_sdpf_params(enum pfctl_attr pattr, u64 reg,
+ unsigned int level, u64 *val)
+{
+ int ret;
+
+ switch (pattr) {
+ case SDPF_ENABLE:
+ ret = a64fx_get_sdpf_enable(reg, level);
+ break;
+ case SDPF_STRONG:
+ ret = a64fx_get_sdpf_strong(reg, level);
+ break;
+ case SDPF_DIST:
+ ret = a64fx_get_sdpf_dist(reg, level);
+ break;
+ default:
+ return -ENOENT;
+ }
+
+ if (ret < 0)
+ return ret;
+ *val = ret;
+
+ return 0;
+}
+
+static int a64fx_modify_pfreg_val(enum pfctl_attr pattr, u64 *reg,
+ unsigned int level, u64 val)
+{
+ int ret;
+
+ switch (pattr) {
+ case SDPF_ENABLE:
+ ret = a64fx_modify_sdpf_enable(reg, level, val);
+ break;
+ case SDPF_STRONG:
+ ret = a64fx_modify_sdpf_strong(reg, level, val);
+ break;
+ case SDPF_DIST:
+ ret = a64fx_modify_sdpf_dist(reg, level, val);
+ break;
+ default:
+ return -ENOENT;
+ }
+
+ if (ret < 0)
+ return ret;
+
+ a64fx_enable_sdpf_verify(reg);
+
+ return 0;
+}
+
+static void _a64fx_read_pfreg(void *info)
+{
+ u64 reg;
+ struct a64fx_read_info *rinfo = info;
+
+ reg = read_sysreg_s(A64FX_SDPF_IMP_PF_STREAM_DETECT_CTRL_EL0);
+
+ rinfo->ret = a64fx_get_sdpf_params(rinfo->pattr, reg, rinfo->level,
+ &rinfo->val);
+}
+
+static int a64fx_read_pfreg(enum pfctl_attr pattr, unsigned int cpu,
+ unsigned int level, u64 *val)
+{
+ struct a64fx_read_info info = {
+ .level = level,
+ .pattr = pattr,
+ };
+
+ smp_call_function_single(cpu, _a64fx_read_pfreg, &info, true);
+
+ if (info.ret < 0)
+ return info.ret;
+
+ *val = info.val;
+ return 0;
+}
+
+static void _a64fx_write_pfreg(void *info)
+{
+ int ret;
+ u64 reg;
+ struct a64fx_write_info *winfo = info;
+
+ reg = read_sysreg_s(A64FX_SDPF_IMP_PF_STREAM_DETECT_CTRL_EL0);
+
+ ret = a64fx_modify_pfreg_val(winfo->pattr, ®, winfo->level,
+ winfo->val);
+ if (ret < 0) {
+ winfo->ret = ret;
+ return;
+ }
+
+ write_sysreg_s(reg, A64FX_SDPF_IMP_PF_STREAM_DETECT_CTRL_EL0);
+
+ winfo->ret = 0;
+}
+
+static int a64fx_write_pfreg(enum pfctl_attr pattr, unsigned int cpu,
+ unsigned int level, u64 val)
+{
+ struct a64fx_write_info info = {
+ .level = level,
+ .pattr = pattr,
+ .val = val,
+ };
+
+ smp_call_function_single(cpu, _a64fx_write_pfreg, &info, true);
+ return info.ret;
+}
+
+/***** end of FUJITSU A64FX support *****/
+
+/*
+ * This driver returns a negative value if it does not support the Hardware
+ * Prefetch Control or if it is running on a VM guest.
+ */
+static int __init setup_pfctl_driver_params(void)
+{
+ unsigned long implementor = read_cpuid_implementor();
+ unsigned long part_number = read_cpuid_part_number();
+
+ if (!is_kernel_in_hyp_mode())
+ return -EINVAL;
+
+ switch (implementor) {
+ case ARM_CPU_IMP_FUJITSU:
+ switch (part_number) {
+ case FUJITSU_CPU_PART_A64FX:
+ /* A64FX register requires EL2 access */
+ if (!has_vhe())
+ return -EINVAL;
+
+ arm64_pfctl_driver.supported_l1d_prefetcher = SDPF;
+ arm64_pfctl_driver.supported_l2_prefetcher = SDPF;
+ arm64_pfctl_driver.read_pfreg = a64fx_read_pfreg;
+ arm64_pfctl_driver.write_pfreg = a64fx_write_pfreg;
+ break;
+ default:
+ return -ENODEV;
+ }
+ break;
+ default:
+ return -ENODEV;
+ }
+
+ return 0;
+}
+
+static int __init arm64_pfctl_init(void)
+{
+ int ret;
+
+ ret = setup_pfctl_driver_params();
+ if (ret < 0)
+ return ret;
+
+ ret = pfctl_register_driver(&arm64_pfctl_driver);
+ if (ret < 0)
+ return ret;
+
+ return 0;
+}
+
+static void __exit arm64_pfctl_exit(void)
+{
+ pfctl_unregister_driver(&arm64_pfctl_driver);
+}
+
+late_initcall(arm64_pfctl_init);
+module_exit(arm64_pfctl_exit);
+
+MODULE_LICENSE("GPL v2");
+MODULE_AUTHOR("FUJITSU LIMITED");
+MODULE_DESCRIPTION("ARM64 Hardware Prefetch Control Driver");
--
2.27.0
Powered by blists - more mailing lists