| lists.openwall.net | lists / announce owl-users owl-dev john-users john-dev passwdqc-users yescrypt popa3d-users / oss-security kernel-hardening musl sabotage tlsify passwords / crypt-dev xvendor / Bugtraq Full-Disclosure linux-kernel linux-netdev linux-ext4 linux-hardening linux-cve-announce PHC | |
|
Open Source and information security mailing list archives
| ||
|
Message-Id: <1304502103-3228-2-git-send-email-jolsa@redhat.com>
Date: Wed, 4 May 2011 11:41:42 +0200
From: Jiri Olsa <jolsa@...hat.com>
To: jbaron@...hat.com, rostedt@...dmis.org, mingo@...e.hu
Cc: linux-kernel@...r.kernel.org, Jiri Olsa <jolsa@...hat.com>
Subject: [PATCHv2 1/2] jump_label,x86: use text_poke_smp_batch for entries update
Changing the jump label update code to use batch processing
for x86 architectures.
Currently each jump label update calls text_poke_smp for each
jump label key entry. Thus one key update ends up calling stop
machine multiple times.
This patch is using text_poke_smp_batch, which is called for
all the key's entries. Thus ensuring the stop machine is called
only once per jump_label key.
Added jump_label_update_end function which is paired with
the key's entries update.
The jump_label_update_end calls arch_jump_label_update_end
(with generic weak definition) which is overloaded by x86
arch and makes the batch update of all the entries queued
by arch_jump_label_transform function.
Added dynamic array/handling for queueing jump_label entries
with the same key.
Tested this on x86 and s390 archs.
Signed-off-by: Jiri Olsa <jolsa@...hat.com>
---
arch/x86/kernel/jump_label.c | 128 +++++++++++++++++++++++++++++++++++++++---
include/linux/jump_label.h | 1 +
kernel/jump_label.c | 16 +++++-
3 files changed, 134 insertions(+), 11 deletions(-)
diff --git a/arch/x86/kernel/jump_label.c b/arch/x86/kernel/jump_label.c
index 3fee346..7a4dd32 100644
--- a/arch/x86/kernel/jump_label.c
+++ b/arch/x86/kernel/jump_label.c
@@ -11,6 +11,7 @@
#include <linux/list.h>
#include <linux/jhash.h>
#include <linux/cpu.h>
+#include <linux/slab.h>
#include <asm/kprobes.h>
#include <asm/alternative.h>
@@ -24,24 +25,133 @@ union jump_code_union {
} __attribute__((packed));
};
-void arch_jump_label_transform(struct jump_entry *entry,
- enum jump_label_type type)
+struct text_poke_buffer {
+ u8 code[JUMP_LABEL_NOP_SIZE];
+};
+
+#define POKE_ALLOC_CNT 30
+
+static struct text_poke_param *poke_pars;
+static struct text_poke_buffer *poke_bufs;
+static int poke_cnt, poke_size;
+
+static void poke_setup(struct text_poke_param *param, u8 *buf,
+ int enable,
+ struct jump_entry *entry)
{
- union jump_code_union code;
+ union jump_code_union *code = (union jump_code_union *) buf;
- if (type == JUMP_LABEL_ENABLE) {
- code.jump = 0xe9;
- code.offset = entry->target -
- (entry->code + JUMP_LABEL_NOP_SIZE);
+ if (enable == JUMP_LABEL_ENABLE) {
+ code->jump = 0xe9;
+ code->offset = entry->target -
+ (entry->code + JUMP_LABEL_NOP_SIZE);
} else
- memcpy(&code, ideal_nops[NOP_ATOMIC5], JUMP_LABEL_NOP_SIZE);
+ memcpy(code, ideal_nops[NOP_ATOMIC5], JUMP_LABEL_NOP_SIZE);
+
+ param->addr = (void *) entry->code;
+ param->opcode = code;
+ param->len = JUMP_LABEL_NOP_SIZE;
+}
+
+static int poke_alloc(void)
+{
+ struct text_poke_param *pars;
+ struct text_poke_buffer *bufs;
+
+ if (poke_cnt % POKE_ALLOC_CNT)
+ return 0;
+
+ poke_size += POKE_ALLOC_CNT;
+
+ pars = krealloc(poke_pars, poke_size * sizeof(*pars),
+ GFP_KERNEL);
+ if (!pars)
+ return -ENOMEM;
+
+ bufs = krealloc(poke_bufs, poke_size * sizeof(*bufs),
+ GFP_KERNEL);
+ if (!bufs) {
+ kfree(pars);
+ return -ENOMEM;
+ }
+
+ poke_pars = pars;
+ poke_bufs = bufs;
+ return 0;
+}
+
+static void poke_free(void)
+{
+ kfree(poke_pars);
+ kfree(poke_bufs);
+
+ poke_cnt = poke_size = 0;
+ poke_pars = NULL;
+ poke_bufs = NULL;
+}
+
+static void poke_process(struct text_poke_param *par, int cnt)
+{
get_online_cpus();
mutex_lock(&text_mutex);
- text_poke_smp((void *)entry->code, &code, JUMP_LABEL_NOP_SIZE);
+ text_poke_smp_batch(par, cnt);
mutex_unlock(&text_mutex);
put_online_cpus();
}
+static void poke_end(void)
+{
+ if (!poke_cnt)
+ return;
+
+ poke_process(poke_pars, poke_cnt);
+ poke_free();
+}
+
+static void process_entry(struct jump_entry *entry,
+ enum jump_label_type enable)
+{
+ struct text_poke_param par;
+ struct text_poke_buffer buf;
+
+ poke_setup(&par, buf.code, enable, entry);
+ poke_process(&par, 1);
+}
+
+void arch_jump_label_transform(struct jump_entry *entry,
+ enum jump_label_type enable)
+{
+ struct text_poke_param *par;
+ struct text_poke_buffer *buf;
+
+ if (poke_alloc()) {
+ /*
+ * Allocation failed, let's finish what we
+ * gather so far and process this entry
+ * separatelly. Next call we try the allocation
+ * again.
+ */
+ poke_end();
+ process_entry(entry, enable);
+ return;
+ }
+
+ par = &poke_pars[poke_cnt];
+ buf = &poke_bufs[poke_cnt];
+
+ poke_setup(par, buf->code, enable, entry);
+ poke_cnt++;
+}
+
+/*
+ * Called after arch_jump_label_transform is called for
+ * all entries of a single key.
+ */
+void arch_jump_label_update_end(void)
+{
+ poke_end();
+}
+
void arch_jump_label_text_poke_early(jump_label_t addr)
{
text_poke_early((void *)addr, ideal_nops[NOP_ATOMIC5],
diff --git a/include/linux/jump_label.h b/include/linux/jump_label.h
index 83e745f..e7a8fa3 100644
--- a/include/linux/jump_label.h
+++ b/include/linux/jump_label.h
@@ -46,6 +46,7 @@ extern void jump_label_unlock(void);
extern void arch_jump_label_transform(struct jump_entry *entry,
enum jump_label_type type);
extern void arch_jump_label_text_poke_early(jump_label_t addr);
+extern void arch_jump_label_update_end(void);
extern int jump_label_text_reserved(void *start, void *end);
extern void jump_label_inc(struct jump_label_key *key);
extern void jump_label_dec(struct jump_label_key *key);
diff --git a/kernel/jump_label.c b/kernel/jump_label.c
index 74d1c09..6657a37 100644
--- a/kernel/jump_label.c
+++ b/kernel/jump_label.c
@@ -125,6 +125,15 @@ void __weak arch_jump_label_text_poke_early(jump_label_t addr)
{
}
+void __weak arch_jump_label_update_end(void)
+{
+}
+
+void jump_label_update_end(void)
+{
+ arch_jump_label_update_end();
+}
+
static __init int jump_label_init(void)
{
struct jump_entry *iter_start = __start___jump_table;
@@ -244,10 +253,11 @@ static int jump_label_add_module(struct module *mod)
jlm->next = key->next;
key->next = jlm;
- if (jump_label_enabled(key))
+ if (jump_label_enabled(key)) {
__jump_label_update(key, iter, JUMP_LABEL_ENABLE);
+ jump_label_update_end();
+ }
}
-
return 0;
}
@@ -376,6 +386,8 @@ static void jump_label_update(struct jump_label_key *key, int enable)
#ifdef CONFIG_MODULES
__jump_label_mod_update(key, enable);
#endif
+
+ jump_label_update_end();
}
#endif
--
1.7.1
--
To unsubscribe from this list: send the line "unsubscribe linux-kernel" in
the body of a message to majordomo@...r.kernel.org
More majordomo info at http://vger.kernel.org/majordomo-info.html
Please read the FAQ at http://www.tux.org/lkml/
Powered by blists - more mailing lists