| lists.openwall.net | lists / announce owl-users owl-dev john-users john-dev passwdqc-users yescrypt popa3d-users / oss-security kernel-hardening musl sabotage tlsify passwords / crypt-dev xvendor / Bugtraq Full-Disclosure linux-kernel linux-netdev linux-ext4 linux-hardening linux-cve-announce PHC | |
|
Open Source and information security mailing list archives
| ||
|
Date: Mon, 07 Oct 2019 10:27:20 +0200
From: Peter Zijlstra <peterz@...radead.org>
To: x86@...nel.org
Cc: peterz@...radead.org, linux-kernel@...r.kernel.org,
rostedt@...dmis.org, mhiramat@...nel.org, bristot@...hat.com,
jbaron@...mai.com, torvalds@...ux-foundation.org,
tglx@...utronix.de, mingo@...nel.org, namit@...are.com,
hpa@...or.com, luto@...nel.org, ard.biesheuvel@...aro.org,
jpoimboe@...hat.com
Subject: [PATCH v2 12/13] static_call: Allow early init
In order to use static_call() to wire up x86_pmu, we need to
initialize earlier; copy some of the tricks from jump_label to enable
this.
Primarily we overload key->next to store a sites pointer when there
are no modules, this avoids having to use kmalloc() to initialize the
sites and allows us to run much earlier.
(arguably, this is much much earlier than needed for perf, but it
might allow other uses.)
Signed-off-by: Peter Zijlstra (Intel) <peterz@...radead.org>
---
arch/x86/kernel/setup.c | 2 +
arch/x86/kernel/static_call.c | 3 ++
include/linux/static_call.h | 15 ++++++++++--
kernel/static_call.c | 52 +++++++++++++++++++++++++++++++++++++++---
4 files changed, 67 insertions(+), 5 deletions(-)
--- a/arch/x86/kernel/setup.c
+++ b/arch/x86/kernel/setup.c
@@ -73,6 +73,7 @@
#include <linux/jiffies.h>
#include <linux/mem_encrypt.h>
#include <linux/sizes.h>
+#include <linux/static_call.h>
#include <linux/usb/xhci-dbgp.h>
#include <video/edid.h>
@@ -896,6 +897,7 @@ void __init setup_arch(char **cmdline_p)
early_cpu_init();
arch_init_ideal_nops();
jump_label_init();
+ static_call_init();
early_ioremap_init();
setup_olpc_ofw_pgd();
--- a/arch/x86/kernel/static_call.c
+++ b/arch/x86/kernel/static_call.c
@@ -38,6 +38,9 @@ static void __static_call_transform(void
if (memcmp(insn, code, size) == 0)
return;
+ if (unlikely(system_state == SYSTEM_BOOTING))
+ return text_poke_early(insn, code, size);
+
text_poke_bp(insn, code, size, NULL);
}
--- a/include/linux/static_call.h
+++ b/include/linux/static_call.h
@@ -81,6 +81,8 @@ extern void arch_static_call_transform(v
#ifdef CONFIG_HAVE_STATIC_CALL_INLINE
+extern void __init static_call_init(void);
+
struct static_call_mod {
struct static_call_mod *next;
struct module *mod; /* for vmlinux, mod == NULL */
@@ -89,7 +91,12 @@ struct static_call_mod {
struct static_call_key {
void *func;
- struct static_call_mod *next;
+ union {
+ /* bit0 => 0 - next, 1 - sites */
+ unsigned long type;
+ struct static_call_mod *next;
+ struct static_call_site *sites;
+ };
};
extern void __static_call_update(struct static_call_key *key, void *tramp, void *func);
@@ -100,7 +107,7 @@ extern int static_call_text_reserved(voi
DECLARE_STATIC_CALL(name, _func); \
struct static_call_key STATIC_CALL_NAME(name) = { \
.func = _func, \
- .next = NULL, \
+ .type = 1, \
}; \
__ADDRESSABLE(STATIC_CALL_NAME(name)); \
ARCH_DEFINE_STATIC_CALL_TRAMP(name, _func)
@@ -118,6 +125,8 @@ extern int static_call_text_reserved(voi
#elif defined(CONFIG_HAVE_STATIC_CALL)
+static inline void static_call_init(void) { }
+
struct static_call_key {
void *func;
};
@@ -151,6 +160,8 @@ static inline int static_call_text_reser
#else /* Generic implementation */
+static inline void static_call_init(void) { }
+
struct static_call_key {
void *func;
};
--- a/kernel/static_call.c
+++ b/kernel/static_call.c
@@ -94,10 +94,31 @@ static inline void static_call_sort_entr
static_call_site_cmp, static_call_site_swap);
}
+static inline bool static_call_key_has_next(struct static_call_key *key)
+{
+ return !(key->type & 1);
+}
+
+static inline struct static_call_mod *static_call_key_next(struct static_call_key *key)
+{
+ if (static_call_key_has_next(key))
+ return key->next->next;
+
+ return NULL;
+}
+
+static inline struct static_call_site *static_call_key_sites(struct static_call_key *key)
+{
+ if (static_call_key_has_next(key))
+ return key->next->sites;
+
+ return (struct static_call_site *)(key->type & ~1);
+}
+
void __static_call_update(struct static_call_key *key, void *tramp, void *func)
{
struct static_call_site *site, *stop;
- struct static_call_mod *site_mod;
+ struct static_call_mod *site_mod, first;
cpus_read_lock();
static_call_lock();
@@ -116,7 +137,13 @@ void __static_call_update(struct static_
if (WARN_ON_ONCE(!static_call_initialized))
goto done;
- for (site_mod = key->next; site_mod; site_mod = site_mod->next) {
+ first = (struct static_call_mod){
+ .next = static_call_key_next(key),
+ .mod = NULL,
+ .sites = static_call_key_sites(key),
+ };
+
+ for (site_mod = &first; site_mod; site_mod = site_mod->next) {
if (!site_mod->sites) {
/*
* This can happen if the static call key is defined in
@@ -191,16 +218,35 @@ static int __static_call_init(struct mod
if (key != prev_key) {
prev_key = key;
+ if (!mod) {
+ key->sites = site;
+ key->type |= 1;
+ goto do_transform;
+ }
+
site_mod = kzalloc(sizeof(*site_mod), GFP_KERNEL);
if (!site_mod)
return -ENOMEM;
+ if (!static_call_key_has_next(key)) {
+ site_mod->mod = NULL;
+ site_mod->next = NULL;
+ site_mod->sites = static_call_key_sites(key);
+
+ key->next = site_mod;
+
+ site_mod = kzalloc(sizeof(*site_mod), GFP_KERNEL);
+ if (!site_mod)
+ return -ENOMEM;
+ }
+
site_mod->mod = mod;
site_mod->sites = site;
site_mod->next = key->next;
key->next = site_mod;
}
+do_transform:
arch_static_call_transform(site_addr, NULL, key->func,
static_call_is_tail(site));
}
@@ -343,7 +389,7 @@ int static_call_text_reserved(void *star
return ret;
}
-static void __init static_call_init(void)
+void __init static_call_init(void)
{
int ret;
Powered by blists - more mailing lists