| lists.openwall.net | lists / announce owl-users owl-dev john-users john-dev passwdqc-users yescrypt popa3d-users / oss-security kernel-hardening musl sabotage tlsify passwords / crypt-dev xvendor / Bugtraq Full-Disclosure linux-kernel linux-netdev linux-ext4 linux-hardening linux-cve-announce PHC | |
|
Open Source and information security mailing list archives
| ||
|
Message-Id: <20211110010906.1923210-3-eric.dumazet@gmail.com>
Date: Tue, 9 Nov 2021 17:09:06 -0800
From: Eric Dumazet <eric.dumazet@...il.com>
To: Peter Zijlstra <peterz@...radead.org>
Cc: linux-kernel <linux-kernel@...r.kernel.org>,
Eric Dumazet <edumazet@...gle.com>,
Eric Dumazet <eric.dumazet@...il.com>
Subject: [PATCH 2/2] jump_label: refine placement of static_keys
From: Eric Dumazet <edumazet@...gle.com>
With CONFIG_JUMP_LABEL=y, "struct static_key" content is only
used for the control path.
Marking them __read_mostly is only needed when CONFIG_JUMP_LABEL=n.
Otherwise we place them out of the way to increase data locality.
This patch adds __static_key to centralize this new policy.
Signed-off-by: Eric Dumazet <edumazet@...gle.com>
---
arch/x86/kvm/lapic.c | 4 ++--
arch/x86/kvm/x86.c | 2 +-
include/linux/jump_label.h | 25 +++++++++++++++++--------
kernel/events/core.c | 2 +-
kernel/sched/fair.c | 2 +-
net/core/dev.c | 8 ++++----
net/netfilter/core.c | 2 +-
net/netfilter/x_tables.c | 2 +-
8 files changed, 28 insertions(+), 19 deletions(-)
diff --git a/arch/x86/kvm/lapic.c b/arch/x86/kvm/lapic.c
index d6ac32f3f650c066733ff9afe96ece36bca01523..a4e8ec38cd09aca2969e71242bced1d95e9e9c1f 100644
--- a/arch/x86/kvm/lapic.c
+++ b/arch/x86/kvm/lapic.c
@@ -91,8 +91,8 @@ static inline int __apic_test_and_clear_vector(int vec, void *bitmap)
return __test_and_clear_bit(VEC_POS(vec), (bitmap) + REG_POS(vec));
}
-__read_mostly DEFINE_STATIC_KEY_DEFERRED_FALSE(apic_hw_disabled, HZ);
-__read_mostly DEFINE_STATIC_KEY_DEFERRED_FALSE(apic_sw_disabled, HZ);
+DEFINE_STATIC_KEY_DEFERRED_FALSE(apic_hw_disabled, HZ);
+DEFINE_STATIC_KEY_DEFERRED_FALSE(apic_sw_disabled, HZ);
static inline int apic_enabled(struct kvm_lapic *apic)
{
diff --git a/arch/x86/kvm/x86.c b/arch/x86/kvm/x86.c
index c1c4e2b05a63abd53e01d0671f740a3b900d98e6..051236fedee39e197d6795fc62bdb29849c7ada2 100644
--- a/arch/x86/kvm/x86.c
+++ b/arch/x86/kvm/x86.c
@@ -11204,7 +11204,7 @@ bool kvm_vcpu_is_bsp(struct kvm_vcpu *vcpu)
return (vcpu->arch.apic_base & MSR_IA32_APICBASE_BSP) != 0;
}
-__read_mostly DEFINE_STATIC_KEY_FALSE(kvm_has_noapic_vcpu);
+DEFINE_STATIC_KEY_FALSE(kvm_has_noapic_vcpu);
EXPORT_SYMBOL_GPL(kvm_has_noapic_vcpu);
void kvm_arch_sched_in(struct kvm_vcpu *vcpu, int cpu)
diff --git a/include/linux/jump_label.h b/include/linux/jump_label.h
index 48b9b2a82767d74ec95f102ab1470f5fa79cc97d..13ad61a845a30a91c9f18eabff7ff93cac4028e2 100644
--- a/include/linux/jump_label.h
+++ b/include/linux/jump_label.h
@@ -106,10 +106,19 @@ struct static_key {
};
};
+/*
+ * With CONFIG_JUMP_LABEL, "struct static_key" are not used in fast path.
+ * We can put them in a separate section to increase data locality.
+ */
+#define __static_key __section(".data.unlikely")
+
#else
struct static_key {
atomic_t enabled;
};
+
+#define __static_key __read_mostly
+
#endif /* CONFIG_JUMP_LABEL */
#endif /* __ASSEMBLY__ */
@@ -367,7 +376,7 @@ struct static_key_false {
#define STATIC_KEY_FALSE_INIT (struct static_key_false){ .key = STATIC_KEY_INIT_FALSE, }
#define DEFINE_STATIC_KEY_TRUE(name) \
- struct static_key_true name = STATIC_KEY_TRUE_INIT
+ struct static_key_true __static_key name = STATIC_KEY_TRUE_INIT
#define DEFINE_STATIC_KEY_TRUE_RO(name) \
struct static_key_true name __ro_after_init = STATIC_KEY_TRUE_INIT
@@ -376,7 +385,7 @@ struct static_key_false {
extern struct static_key_true name
#define DEFINE_STATIC_KEY_FALSE(name) \
- struct static_key_false name = STATIC_KEY_FALSE_INIT
+ struct static_key_false __static_key name = STATIC_KEY_FALSE_INIT
#define DEFINE_STATIC_KEY_FALSE_RO(name) \
struct static_key_false name __ro_after_init = STATIC_KEY_FALSE_INIT
@@ -384,14 +393,14 @@ struct static_key_false {
#define DECLARE_STATIC_KEY_FALSE(name) \
extern struct static_key_false name
-#define DEFINE_STATIC_KEY_ARRAY_TRUE(name, count) \
- struct static_key_true name[count] = { \
- [0 ... (count) - 1] = STATIC_KEY_TRUE_INIT, \
+#define DEFINE_STATIC_KEY_ARRAY_TRUE(name, count) \
+ struct static_key_true __static_key name[count] = { \
+ [0 ... (count) - 1] = STATIC_KEY_TRUE_INIT, \
}
-#define DEFINE_STATIC_KEY_ARRAY_FALSE(name, count) \
- struct static_key_false name[count] = { \
- [0 ... (count) - 1] = STATIC_KEY_FALSE_INIT, \
+#define DEFINE_STATIC_KEY_ARRAY_FALSE(name, count) \
+ struct static_key_false __static_key name[count] = { \
+ [0 ... (count) - 1] = STATIC_KEY_FALSE_INIT, \
}
#define _DEFINE_STATIC_KEY_1(name) DEFINE_STATIC_KEY_TRUE(name)
diff --git a/kernel/events/core.c b/kernel/events/core.c
index f2253ea729a2732b0261f32280f77f1d1a2ebefb..5cd140b3a489bb51c2536bd5f869290bf8395e7b 100644
--- a/kernel/events/core.c
+++ b/kernel/events/core.c
@@ -9600,7 +9600,7 @@ static int swevent_hlist_get(void)
return err;
}
-struct static_key perf_swevent_enabled[PERF_COUNT_SW_MAX];
+struct static_key __static_key perf_swevent_enabled[PERF_COUNT_SW_MAX];
static void sw_perf_event_destroy(struct perf_event *event)
{
diff --git a/kernel/sched/fair.c b/kernel/sched/fair.c
index 13950beb01a251bfb65bf5b75952e3dea9cbc62b..de9b18a244cadd3b47a2ec4ed4c000a034a7fb96 100644
--- a/kernel/sched/fair.c
+++ b/kernel/sched/fair.c
@@ -4586,7 +4586,7 @@ entity_tick(struct cfs_rq *cfs_rq, struct sched_entity *curr, int queued)
#ifdef CONFIG_CFS_BANDWIDTH
#ifdef CONFIG_JUMP_LABEL
-static struct static_key __cfs_bandwidth_used;
+static struct static_key __static_key __cfs_bandwidth_used;
static inline bool cfs_bandwidth_used(void)
{
diff --git a/net/core/dev.c b/net/core/dev.c
index edeb811c454e6caf4298aae723237aea2c363c3c..21d53ca2730d1abb5dc3103ccca053e1cdf676b7 100644
--- a/net/core/dev.c
+++ b/net/core/dev.c
@@ -2390,8 +2390,8 @@ int netdev_txq_to_tc(struct net_device *dev, unsigned int txq)
EXPORT_SYMBOL(netdev_txq_to_tc);
#ifdef CONFIG_XPS
-static struct static_key xps_needed __read_mostly;
-static struct static_key xps_rxqs_needed __read_mostly;
+static struct static_key __static_key xps_needed;
+static struct static_key __static_key xps_rxqs_needed;
static DEFINE_MUTEX(xps_map_mutex);
#define xmap_dereference(P) \
rcu_dereference_protected((P), lockdep_is_held(&xps_map_mutex))
@@ -4362,9 +4362,9 @@ EXPORT_SYMBOL(rps_sock_flow_table);
u32 rps_cpu_mask __read_mostly;
EXPORT_SYMBOL(rps_cpu_mask);
-struct static_key_false rps_needed __read_mostly;
+struct static_key_false __static_key rps_needed;
EXPORT_SYMBOL(rps_needed);
-struct static_key_false rfs_needed __read_mostly;
+struct static_key_false __static_key rfs_needed;
EXPORT_SYMBOL(rfs_needed);
static struct rps_dev_flow *
diff --git a/net/netfilter/core.c b/net/netfilter/core.c
index 6dec9cd395f1570c845346b6956ba2586a4235d6..3fd26326a55d95c28ab634eb3caf5a4a851ad20c 100644
--- a/net/netfilter/core.c
+++ b/net/netfilter/core.c
@@ -35,7 +35,7 @@ DEFINE_PER_CPU(bool, nf_skb_duplicated);
EXPORT_SYMBOL_GPL(nf_skb_duplicated);
#ifdef CONFIG_JUMP_LABEL
-struct static_key nf_hooks_needed[NFPROTO_NUMPROTO][NF_MAX_HOOKS];
+struct static_key __static_key nf_hooks_needed[NFPROTO_NUMPROTO][NF_MAX_HOOKS];
EXPORT_SYMBOL(nf_hooks_needed);
#endif
diff --git a/net/netfilter/x_tables.c b/net/netfilter/x_tables.c
index 25524e3933496098f68748286e56839454ee842c..aaf78481b04a0decaa09b03b9f69fa3737740cf4 100644
--- a/net/netfilter/x_tables.c
+++ b/net/netfilter/x_tables.c
@@ -1319,7 +1319,7 @@ EXPORT_SYMBOL_GPL(xt_compat_unlock);
DEFINE_PER_CPU(seqcount_t, xt_recseq);
EXPORT_PER_CPU_SYMBOL_GPL(xt_recseq);
-struct static_key xt_tee_enabled __read_mostly;
+struct static_key __static_key xt_tee_enabled;
EXPORT_SYMBOL_GPL(xt_tee_enabled);
static int xt_jumpstack_alloc(struct xt_table_info *i)
--
2.34.0.rc0.344.g81b53c2807-goog
Powered by blists - more mailing lists