| lists.openwall.net | lists / announce owl-users owl-dev john-users john-dev passwdqc-users yescrypt popa3d-users / oss-security kernel-hardening musl sabotage tlsify passwords / crypt-dev xvendor / Bugtraq Full-Disclosure linux-kernel linux-netdev linux-ext4 linux-hardening linux-cve-announce PHC | |
|
Open Source and information security mailing list archives
| ||
|
Date: Sat, 28 Jan 2012 11:09:30 -0500
From: Steven Rostedt <rostedt@...dmis.org>
To: linux-kernel@...r.kernel.org
Cc: Ingo Molnar <mingo@...e.hu>,
Andrew Morton <akpm@...ux-foundation.org>,
Jason Baron <jbaron@...hat.com>,
"H. Peter Anvin" <hpa@...or.com>,
Frederic Weisbecker <fweisbec@...il.com>
Subject: [PATCH 5/5 v2] x86/jump labels: Use etiher 5 byte or 2 byte jumps
From: Steven Rostedt <srostedt@...hat.com>
Have the jump labels add a "jmp" in the assembly instead
of a default nop. This will cause the assembler to put in
either a 2 byte or 5 byte jmp depending on where the target
lable is.
Then at compile time, the update_jump_label code will replace
the jmps with either 2 or 5 byte nops.
On boot up, the code can be examined to see if the jump label
uses either a 2 or 5 byte nop and replace it.
By allowing the jump labels to be 2 bytes, it speeds up the
nops, not only 2 byte nops are faster than 5 byte nops, but also
because it saves on cache foot print.
text data bss dec hex filename
13403667 3666856 2998272 20068795 13239bb ../nobackup/mxtest/vmlinux-old
13398536 3666856 2998272 20063664 13225b0 ../nobackup/mxtest/vmlinux-new
Converting the current v3.2 trace points saved 5,131 bytes.
As more places use jump labels, this will have a bigger savings.
Signed-off-by: Steven Rostedt <rostedt@...dmis.org>
---
arch/x86/Kconfig | 1 +
arch/x86/include/asm/jump_label.h | 7 +++-
arch/x86/kernel/jump_label.c | 85 ++++++++++++++++++++++++++++--------
3 files changed, 73 insertions(+), 20 deletions(-)
diff --git a/arch/x86/Kconfig b/arch/x86/Kconfig
index efb4294..b5004c1 100644
--- a/arch/x86/Kconfig
+++ b/arch/x86/Kconfig
@@ -61,6 +61,7 @@ config X86
select HAVE_ARCH_KMEMCHECK
select HAVE_USER_RETURN_NOTIFIER
select HAVE_ARCH_JUMP_LABEL
+ select HAVE_BUILD_TIME_JUMP_LABEL
select HAVE_TEXT_POKE_SMP
select HAVE_GENERIC_HARDIRQS
select HAVE_SPARSE_IRQ
diff --git a/arch/x86/include/asm/jump_label.h b/arch/x86/include/asm/jump_label.h
index 7082743..ba12f7e 100644
--- a/arch/x86/include/asm/jump_label.h
+++ b/arch/x86/include/asm/jump_label.h
@@ -10,6 +10,11 @@
#define JUMP_LABEL_NOP_SIZE 5
+/*
+ * The JUMP_LABEL_INIT_NOP must match the nops used in
+ * scripts/update_jump_label.c. Otherwise the boot time checks
+ * will fail and trigger a BUG() on boot up.
+ */
#ifdef CONFIG_X86_64
# define JUMP_LABEL_INIT_NOP P6_NOP5_ATOMIC
#else
@@ -19,7 +24,7 @@
static __always_inline bool arch_static_branch(struct jump_label_key *key)
{
asm goto("1:"
- ".byte " __stringify(JUMP_LABEL_INIT_NOP) "\n\t"
+ "jmp %l[l_yes]\n"
".pushsection __jump_table, \"aw\" \n\t"
_ASM_ALIGN "\n\t"
_ASM_PTR "1b, %l[l_yes], %c0 \n\t"
diff --git a/arch/x86/kernel/jump_label.c b/arch/x86/kernel/jump_label.c
index 12a8812..7112a27 100644
--- a/arch/x86/kernel/jump_label.c
+++ b/arch/x86/kernel/jump_label.c
@@ -16,12 +16,20 @@
#ifdef HAVE_JUMP_LABEL
+/* These are the nops added at compile time */
+static const unsigned char nop_short[] = { P6_NOP2 };
+static const unsigned char default_nop[] = { JUMP_LABEL_INIT_NOP };
+
union jump_code_union {
char code[JUMP_LABEL_NOP_SIZE];
struct {
char jump;
int offset;
- } __attribute__((packed));
+ } __packed;
+ struct {
+ char jump_short;
+ char offset_short;
+ } __packed;
};
static void __jump_label_transform(struct jump_entry *entry,
@@ -30,18 +38,33 @@ static void __jump_label_transform(struct jump_entry *entry,
int init)
{
union jump_code_union code;
+ unsigned char nop;
+ unsigned char op;
+ unsigned size;
+ void *ip = (void *)entry->code;
const unsigned char *ideal_nop = ideal_nops[NOP_ATOMIC5];
- if (type == JUMP_LABEL_ENABLE) {
- /*
- * We are enabling this jump label. If it is not a nop
- * then something must have gone wrong.
- */
- BUG_ON(memcmp((void *)entry->code, ideal_nop, 5) != 0);
+ /* Use probe_kernel_read()? */
+ op = *(unsigned char *)ip;
+ nop = ideal_nops[NOP_ATOMIC5][0];
- code.jump = 0xe9;
- code.offset = entry->target -
- (entry->code + JUMP_LABEL_NOP_SIZE);
+ if (type == JUMP_LABEL_ENABLE) {
+ if (memcmp(ip, nop_short, 2) == 0) {
+ size = 2;
+ code.jump_short = 0xeb;
+ code.offset = entry->target - (entry->code + 2);
+ /* Check for overflow ? */
+ } else if (memcmp(ip, ideal_nop, 5) == 0) {
+ size = JUMP_LABEL_NOP_SIZE;
+ code.jump = 0xe9;
+ code.offset = entry->target - (entry->code + size);
+ } else
+ /*
+ * The location is not a nop that we were expecting,
+ * something went wrong. Crash the box, as something could be
+ * corrupting the kernel.
+ */
+ BUG();
} else {
/*
* We are disabling this jump label. If it is not what
@@ -50,18 +73,44 @@ static void __jump_label_transform(struct jump_entry *entry,
* are converting the default nop to the ideal nop.
*/
if (init) {
- const unsigned char default_nop[] = { JUMP_LABEL_INIT_NOP };
- BUG_ON(memcmp((void *)entry->code, default_nop, 5) != 0);
- } else {
+ /* Ignore short nops, we do not change them */
+ if (memcmp(ip, nop_short, 2) == 0)
+ return;
+
+ /* We are initializing from the default nop */
+ BUG_ON(memcmp(ip, default_nop, 5) != 0);
+
+ /* Set to the ideal nop */
+ size = JUMP_LABEL_NOP_SIZE;
+ memcpy(&code, ideal_nops[NOP_ATOMIC5], size);
+
+ } else if (op == 0xe9) {
+ /* Replace a 5 byte jmp */
+
+ /* Make sure this is what we expected it to be */
code.jump = 0xe9;
code.offset = entry->target -
(entry->code + JUMP_LABEL_NOP_SIZE);
- BUG_ON(memcmp((void *)entry->code, &code, 5) != 0);
- }
- memcpy(&code, ideal_nops[NOP_ATOMIC5], JUMP_LABEL_NOP_SIZE);
+ BUG_ON(memcmp(ip, &code, 5) != 0);
+
+ size = JUMP_LABEL_NOP_SIZE;
+ memcpy(&code, ideal_nops[NOP_ATOMIC5], size);
+ } else if (op == 0xeb) {
+ /* Replace a 2 byte jmp */
+
+ /* Had better be a 2 byte jmp */
+ code.jump_short = 0xeb;
+ code.offset = entry->target - (entry->code + 2);
+ BUG_ON(memcmp(ip, &code, 2) != 0);
+
+ size = 2;
+ memcpy(&code, nop_short, size);
+ } else
+ /* The code was not what we expected! */
+ BUG();
}
- (*poker)((void *)entry->code, &code, JUMP_LABEL_NOP_SIZE);
+ (*poker)(ip, &code, size);
}
void arch_jump_label_transform(struct jump_entry *entry,
@@ -88,7 +137,6 @@ void arch_jump_label_transform_static(struct jump_entry *entry,
* If it is not, then we need to update the nop to the ideal nop.
*/
if (!once) {
- const unsigned char default_nop[] = { JUMP_LABEL_INIT_NOP };
const unsigned char *ideal_nop = ideal_nops[NOP_ATOMIC5];
once++;
if (memcmp(ideal_nop, default_nop, 5) != 0)
@@ -97,5 +145,4 @@ void arch_jump_label_transform_static(struct jump_entry *entry,
if (update)
__jump_label_transform(entry, type, text_poke_early, 1);
}
-
#endif
--
1.7.8.3
Download attachment "signature.asc" of type "application/pgp-signature" (837 bytes)
Powered by blists - more mailing lists