| lists.openwall.net | lists / announce owl-users owl-dev john-users john-dev passwdqc-users yescrypt popa3d-users / oss-security kernel-hardening musl sabotage tlsify passwords / crypt-dev xvendor / Bugtraq Full-Disclosure linux-kernel linux-netdev linux-ext4 linux-hardening linux-cve-announce PHC | |
|
Open Source and information security mailing list archives
| ||
|
Message-ID: <166601841630.401.8138685499148178070.tip-bot2@tip-bot2>
Date: Mon, 17 Oct 2022 14:53:36 -0000
From: "tip-bot2 for Peter Zijlstra" <tip-bot2@...utronix.de>
To: linux-tip-commits@...r.kernel.org
Cc: "Peter Zijlstra (Intel)" <peterz@...radead.org>,
Thomas Gleixner <tglx@...utronix.de>, x86@...nel.org,
linux-kernel@...r.kernel.org
Subject: [tip: x86/core] static_call: Add call depth tracking support
The following commit has been merged into the x86/core branch of tip:
Commit-ID: 7825451fa4dc04660f1f53d236e4302161d0ebd1
Gitweb: https://git.kernel.org/tip/7825451fa4dc04660f1f53d236e4302161d0ebd1
Author: Peter Zijlstra <peterz@...radead.org>
AuthorDate: Thu, 15 Sep 2022 13:11:31 +02:00
Committer: Peter Zijlstra <peterz@...radead.org>
CommitterDate: Mon, 17 Oct 2022 16:41:16 +02:00
static_call: Add call depth tracking support
When indirect calls are switched to direct calls then it has to be ensured
that the call target is not the function, but the call thunk when call
depth tracking is enabled. But static calls are available before call
thunks have been set up.
Ensure a second run through the static call patching code after call thunks
have been created. When call thunks are not enabled this has no side
effects.
Signed-off-by: Peter Zijlstra (Intel) <peterz@...radead.org>
Signed-off-by: Thomas Gleixner <tglx@...utronix.de>
Signed-off-by: Peter Zijlstra (Intel) <peterz@...radead.org>
Link: https://lore.kernel.org/r/20220915111148.306100465@infradead.org
---
arch/x86/include/asm/alternative.h | 5 +++++
arch/x86/kernel/callthunks.c | 18 ++++++++++++++++++
arch/x86/kernel/static_call.c | 1 +
include/linux/static_call.h | 2 ++
kernel/static_call_inline.c | 23 ++++++++++++++++++-----
5 files changed, 44 insertions(+), 5 deletions(-)
diff --git a/arch/x86/include/asm/alternative.h b/arch/x86/include/asm/alternative.h
index 4c416b2..07ac257 100644
--- a/arch/x86/include/asm/alternative.h
+++ b/arch/x86/include/asm/alternative.h
@@ -91,11 +91,16 @@ struct callthunk_sites {
extern void callthunks_patch_builtin_calls(void);
extern void callthunks_patch_module_calls(struct callthunk_sites *sites,
struct module *mod);
+extern void *callthunks_translate_call_dest(void *dest);
#else
static __always_inline void callthunks_patch_builtin_calls(void) {}
static __always_inline void
callthunks_patch_module_calls(struct callthunk_sites *sites,
struct module *mod) {}
+static __always_inline void *callthunks_translate_call_dest(void *dest)
+{
+ return dest;
+}
#endif
#ifdef CONFIG_SMP
diff --git a/arch/x86/kernel/callthunks.c b/arch/x86/kernel/callthunks.c
index dfe7fff..0710036 100644
--- a/arch/x86/kernel/callthunks.c
+++ b/arch/x86/kernel/callthunks.c
@@ -6,6 +6,7 @@
#include <linux/kallsyms.h>
#include <linux/memory.h>
#include <linux/moduleloader.h>
+#include <linux/static_call.h>
#include <asm/alternative.h>
#include <asm/asm-offsets.h>
@@ -271,10 +272,27 @@ void __init callthunks_patch_builtin_calls(void)
pr_info("Setting up call depth tracking\n");
mutex_lock(&text_mutex);
callthunks_setup(&cs, &builtin_coretext);
+ static_call_force_reinit();
thunks_initialized = true;
mutex_unlock(&text_mutex);
}
+void *callthunks_translate_call_dest(void *dest)
+{
+ void *target;
+
+ lockdep_assert_held(&text_mutex);
+
+ if (!thunks_initialized || skip_addr(dest))
+ return dest;
+
+ if (!is_coretext(NULL, dest))
+ return dest;
+
+ target = patch_dest(dest, false);
+ return target ? : dest;
+}
+
#ifdef CONFIG_MODULES
void noinline callthunks_patch_module_calls(struct callthunk_sites *cs,
struct module *mod)
diff --git a/arch/x86/kernel/static_call.c b/arch/x86/kernel/static_call.c
index 5d3844a..2ebc338 100644
--- a/arch/x86/kernel/static_call.c
+++ b/arch/x86/kernel/static_call.c
@@ -34,6 +34,7 @@ static void __ref __static_call_transform(void *insn, enum insn_type type,
switch (type) {
case CALL:
+ func = callthunks_translate_call_dest(func);
code = text_gen_insn(CALL_INSN_OPCODE, insn, func);
if (func == &__static_call_return0) {
emulate = code;
diff --git a/include/linux/static_call.h b/include/linux/static_call.h
index df53bed..141e6b1 100644
--- a/include/linux/static_call.h
+++ b/include/linux/static_call.h
@@ -162,6 +162,8 @@ extern void arch_static_call_transform(void *site, void *tramp, void *func, bool
extern int __init static_call_init(void);
+extern void static_call_force_reinit(void);
+
struct static_call_mod {
struct static_call_mod *next;
struct module *mod; /* for vmlinux, mod == NULL */
diff --git a/kernel/static_call_inline.c b/kernel/static_call_inline.c
index dc5665b..639397b 100644
--- a/kernel/static_call_inline.c
+++ b/kernel/static_call_inline.c
@@ -15,7 +15,18 @@ extern struct static_call_site __start_static_call_sites[],
extern struct static_call_tramp_key __start_static_call_tramp_key[],
__stop_static_call_tramp_key[];
-static bool static_call_initialized;
+static int static_call_initialized;
+
+/*
+ * Must be called before early_initcall() to be effective.
+ */
+void static_call_force_reinit(void)
+{
+ if (WARN_ON_ONCE(!static_call_initialized))
+ return;
+
+ static_call_initialized++;
+}
/* mutex to protect key modules/sites */
static DEFINE_MUTEX(static_call_mutex);
@@ -475,7 +486,8 @@ int __init static_call_init(void)
{
int ret;
- if (static_call_initialized)
+ /* See static_call_force_reinit(). */
+ if (static_call_initialized == 1)
return 0;
cpus_read_lock();
@@ -490,11 +502,12 @@ int __init static_call_init(void)
BUG();
}
- static_call_initialized = true;
-
#ifdef CONFIG_MODULES
- register_module_notifier(&static_call_module_nb);
+ if (!static_call_initialized)
+ register_module_notifier(&static_call_module_nb);
#endif
+
+ static_call_initialized = 1;
return 0;
}
early_initcall(static_call_init);
Powered by blists - more mailing lists