| lists.openwall.net | lists / announce owl-users owl-dev john-users john-dev passwdqc-users yescrypt popa3d-users / oss-security kernel-hardening musl sabotage tlsify passwords / crypt-dev xvendor / Bugtraq Full-Disclosure linux-kernel linux-netdev linux-ext4 linux-hardening linux-cve-announce PHC | |
|
Open Source and information security mailing list archives
| ||
|
Date: Thu, 29 Sep 2011 15:20:07 -0700
From: Jeremy Fitzhardinge <jeremy@...p.org>
To: Peter Zijlstra <peterz@...radead.org>
CC: Jason Baron <jbaron@...hat.com>,
Steven Rostedt <rostedt@...dmis.org>,
Linux Kernel Mailing List <linux-kernel@...r.kernel.org>,
Michael Ellerman <michael@...erman.id.au>,
Jan Glauber <jang@...ux.vnet.ibm.com>,
David Daney <ddaney@...iumnetworks.com>,
"David S. Miller" <davem@...emloft.net>
Subject: Re: Jump Label initialization
On 09/29/2011 12:37 AM, Peter Zijlstra wrote:
> On Wed, 2011-09-28 at 19:14 -0700, Jeremy Fitzhardinge wrote:
>> Hi all,
>>
>> I'm trying to use the jump label machinery as part of the pv ticketlock
>> work I'm doing on x86.
>>
>> The problem I'm having at the moment is that I do my spinlock setup in
>> smp_prepare_boot_cpu(), which happens before jump_label_init() gets
>> called, and so the latter goes and nops out all my enabled jump label key.
>>
>> I'm experimenting at the moment with a patch to allow
>> jump_label_enable() to be called fairly early, and have that be
>> respected by jump_label_init(). I'm doing this by replacing
>> arch_jump_label_poke_text_early() with
>> arch_jump_label_transform_early(), which shares most of its code with
>> its non-early variant, except that it expects to run in a pre-SMP
>> environment.
>>
>> Does this seem plausible? (I haven't tested it yet.)
>>
>> The x86, mips and sparc patches are fairly simple; I forgot to look at
>> powerpc, and I didn't fully investigate s390.
>>
>> While my current use-case is x86-specific, it seems generally useful to
>> make the jump_label machinery available as early as possible. I wonder
>> if you have any suggestions about how to handle this?
> I also remember talking to Jason about a way to initialize a jump label
> enabled, instead of the default disabled.
>
> So yeah, I think your stuff would be useful..
>
Here's what I have, which appears to work on x86. The non-x86
architecture changes are completely untested.
But I still question whether the _early variant is necessary at all. If
it were, then jump_label_enable() should use when called early as well.
J
diff --git a/arch/mips/kernel/jump_label.c b/arch/mips/kernel/jump_label.c
index 6001610..ddc9a65 100644
--- a/arch/mips/kernel/jump_label.c
+++ b/arch/mips/kernel/jump_label.c
@@ -20,8 +20,8 @@
#define J_RANGE_MASK ((1ul << 28) - 1)
-void arch_jump_label_transform(struct jump_entry *e,
- enum jump_label_type type)
+static void __jump_label_transform(struct jump_entry *e,
+ enum jump_label_type type)
{
union mips_instruction insn;
union mips_instruction *insn_p =
@@ -40,15 +40,28 @@ void arch_jump_label_transform(struct jump_entry *e,
insn.word = 0; /* nop */
}
- get_online_cpus();
- mutex_lock(&text_mutex);
*insn_p = insn;
flush_icache_range((unsigned long)insn_p,
(unsigned long)insn_p + sizeof(*insn_p));
+}
+
+void arch_jump_label_transform(struct jump_entry *e,
+ enum jump_label_type type)
+{
+ get_online_cpus();
+ mutex_lock(&text_mutex);
+
+ __jump_label_tranform(e, type);
mutex_unlock(&text_mutex);
put_online_cpus();
}
+void __init arch_jump_label_transform_early(struct jump_entry *e,
+ enum jump_label_type type)
+{
+ __jump_label_tranform(e, type);
+}
+
#endif /* HAVE_JUMP_LABEL */
diff --git a/arch/powerpc/kernel/jump_label.c b/arch/powerpc/kernel/jump_label.c
index 368d158..20c82fb 100644
--- a/arch/powerpc/kernel/jump_label.c
+++ b/arch/powerpc/kernel/jump_label.c
@@ -21,3 +21,9 @@ void arch_jump_label_transform(struct jump_entry *entry,
else
patch_instruction(addr, PPC_INST_NOP);
}
+
+void __init arch_jump_label_transform_early(struct jump_entry *entry,
+ enum jump_label_type type)
+{
+ arch_jump_label_transform(entry, type);
+}
diff --git a/arch/s390/kernel/jump_label.c b/arch/s390/kernel/jump_label.c
index 44cc06b..6001228 100644
--- a/arch/s390/kernel/jump_label.c
+++ b/arch/s390/kernel/jump_label.c
@@ -18,23 +18,12 @@ struct insn {
} __packed;
struct insn_args {
- unsigned long *target;
- struct insn *insn;
- ssize_t size;
+ struct jump_entry *entry;
+ enum jump_label_type type;
};
-static int __arch_jump_label_transform(void *data)
-{
- struct insn_args *args = data;
- int rc;
-
- rc = probe_kernel_write(args->target, args->insn, args->size);
- WARN_ON_ONCE(rc < 0);
- return 0;
-}
-
-void arch_jump_label_transform(struct jump_entry *entry,
- enum jump_label_type type)
+static void __jump_label_transform(struct jump_entry *entry,
+ enum jump_label_type type)
{
struct insn_args args;
struct insn insn;
@@ -49,11 +38,33 @@ void arch_jump_label_transform(struct jump_entry *entry,
insn.offset = 0;
}
- args.target = (void *) entry->code;
- args.insn = &insn;
- args.size = JUMP_LABEL_NOP_SIZE;
+ rc = probe_kernel_write(entry->code, &insn, JUMP_LABEL_NOP_SIZE);
+ WARN_ON_ONCE(rc < 0);
+}
+
+static int __sm_arch_jump_label_transform(void *data)
+{
+ struct insn_args *args = data;
+
+ __jump_label_transform(args->entry, args->type);
+ return 0;
+}
- stop_machine(__arch_jump_label_transform, &args, NULL);
+void arch_jump_label_transform(struct jump_entry *entry,
+ enum jump_label_type type)
+{
+ struct insn_args args;
+
+ args.entry = entry;
+ args.type = type;
+
+ stop_machine(__sm_arch_jump_label_transform, &args, NULL);
+}
+
+void __init arch_jump_label_transform_early(struct jump_entry *entry,
+ enum jump_label_type type)
+{
+ __jump_label_transform(entry, type);
}
#endif
diff --git a/arch/sparc/kernel/jump_label.c b/arch/sparc/kernel/jump_label.c
index ea2dafc..d3dad25 100644
--- a/arch/sparc/kernel/jump_label.c
+++ b/arch/sparc/kernel/jump_label.c
@@ -8,8 +8,8 @@
#ifdef HAVE_JUMP_LABEL
-void arch_jump_label_transform(struct jump_entry *entry,
- enum jump_label_type type)
+static void __jump_label_transform(struct jump_entry *entry,
+ enum jump_label_type type)
{
u32 val;
u32 *insn = (u32 *) (unsigned long) entry->code;
@@ -28,20 +28,26 @@ void arch_jump_label_transform(struct jump_entry *entry,
val = 0x01000000;
}
- get_online_cpus();
- mutex_lock(&text_mutex);
*insn = val;
flushi(insn);
+}
+
+void arch_jump_label_transform(struct jump_entry *entry,
+ enum jump_label_type type)
+{
+ get_online_cpus();
+ mutex_lock(&text_mutex);
+
+ __jump_label_transform(entry, type);
+
mutex_unlock(&text_mutex);
put_online_cpus();
}
-void arch_jump_label_text_poke_early(jump_label_t addr)
+void __init arch_jump_label_transform_early(struct jump_entry *entry,
+ enum jump_label_type type)
{
- u32 *insn_p = (u32 *) (unsigned long) addr;
-
- *insn_p = 0x01000000;
- flushi(insn_p);
+ __jump_label_transform(entry, type);
}
#endif
diff --git a/arch/x86/kernel/jump_label.c b/arch/x86/kernel/jump_label.c
index 3fee346..0887b59 100644
--- a/arch/x86/kernel/jump_label.c
+++ b/arch/x86/kernel/jump_label.c
@@ -24,8 +24,9 @@ union jump_code_union {
} __attribute__((packed));
};
-void arch_jump_label_transform(struct jump_entry *entry,
- enum jump_label_type type)
+static void __jump_label_transform(struct jump_entry *entry,
+ enum jump_label_type type,
+ void *(*poker)(void *, const void *, size_t))
{
union jump_code_union code;
@@ -35,17 +36,24 @@ void arch_jump_label_transform(struct jump_entry *entry,
(entry->code + JUMP_LABEL_NOP_SIZE);
} else
memcpy(&code, ideal_nops[NOP_ATOMIC5], JUMP_LABEL_NOP_SIZE);
+
+ (*poker)((void *)entry->code, &code, JUMP_LABEL_NOP_SIZE);
+}
+
+void arch_jump_label_transform(struct jump_entry *entry,
+ enum jump_label_type type)
+{
get_online_cpus();
mutex_lock(&text_mutex);
- text_poke_smp((void *)entry->code, &code, JUMP_LABEL_NOP_SIZE);
+ __jump_label_transform(entry, type, text_poke_smp);
mutex_unlock(&text_mutex);
put_online_cpus();
}
-void arch_jump_label_text_poke_early(jump_label_t addr)
+void __init arch_jump_label_transform_early(struct jump_entry *entry,
+ enum jump_label_type type)
{
- text_poke_early((void *)addr, ideal_nops[NOP_ATOMIC5],
- JUMP_LABEL_NOP_SIZE);
+ __jump_label_transform(entry, type, text_poke_early);
}
#endif
diff --git a/include/linux/jump_label.h b/include/linux/jump_label.h
index 66f23dc..c8fb1b3 100644
--- a/include/linux/jump_label.h
+++ b/include/linux/jump_label.h
@@ -28,9 +28,9 @@ struct module;
#ifdef HAVE_JUMP_LABEL
#ifdef CONFIG_MODULES
-#define JUMP_LABEL_INIT {{ 0 }, NULL, NULL}
+#define JUMP_LABEL_INIT {ATOMIC_INIT(0), NULL, NULL}
#else
-#define JUMP_LABEL_INIT {{ 0 }, NULL}
+#define JUMP_LABEL_INIT {ATOMIC_INIT(0), NULL}
#endif
static __always_inline bool static_branch(struct jump_label_key *key)
@@ -45,7 +45,8 @@ extern void jump_label_lock(void);
extern void jump_label_unlock(void);
extern void arch_jump_label_transform(struct jump_entry *entry,
enum jump_label_type type);
-extern void arch_jump_label_text_poke_early(jump_label_t addr);
+extern void arch_jump_label_transform_early(struct jump_entry *entry,
+ enum jump_label_type type);
extern int jump_label_text_reserved(void *start, void *end);
extern void jump_label_inc(struct jump_label_key *key);
extern void jump_label_dec(struct jump_label_key *key);
diff --git a/kernel/jump_label.c b/kernel/jump_label.c
index a8ce450..3010bcf 100644
--- a/kernel/jump_label.c
+++ b/kernel/jump_label.c
@@ -121,13 +121,6 @@ static void __jump_label_update(struct jump_label_key *key,
}
}
-/*
- * Not all archs need this.
- */
-void __weak arch_jump_label_text_poke_early(jump_label_t addr)
-{
-}
-
static __init int jump_label_init(void)
{
struct jump_entry *iter_start = __start___jump_table;
@@ -139,12 +132,15 @@ static __init int jump_label_init(void)
jump_label_sort_entries(iter_start, iter_stop);
for (iter = iter_start; iter < iter_stop; iter++) {
- arch_jump_label_text_poke_early(iter->code);
- if (iter->key == (jump_label_t)(unsigned long)key)
+ struct jump_label_key *iterk;
+
+ iterk = (struct jump_label_key *)(unsigned long)iter->key;
+ arch_jump_label_transform_early(iter, jump_label_enabled(iterk) ?
+ JUMP_LABEL_ENABLE : JUMP_LABEL_DISABLE);
+ if (iterk == key)
continue;
- key = (struct jump_label_key *)(unsigned long)iter->key;
- atomic_set(&key->enabled, 0);
+ key = iterk;
key->entries = iter;
#ifdef CONFIG_MODULES
key->next = NULL;
@@ -212,7 +208,7 @@ void jump_label_apply_nops(struct module *mod)
return;
for (iter = iter_start; iter < iter_stop; iter++)
- arch_jump_label_text_poke_early(iter->code);
+ arch_jump_label_transform(iter, JUMP_LABEL_DISABLE);
}
static int jump_label_add_module(struct module *mod)
J
--
To unsubscribe from this list: send the line "unsubscribe linux-kernel" in
the body of a message to majordomo@...r.kernel.org
More majordomo info at http://vger.kernel.org/majordomo-info.html
Please read the FAQ at http://www.tux.org/lkml/
Powered by blists - more mailing lists