Date:   Wed, 10 Oct 2018 12:52:37 -0500
From:   Josh Poimboeuf <jpoimboe@...hat.com>
To:     Steven Rostedt <rostedt@...dmis.org>
Cc:     Andy Lutomirski <luto@...capital.net>,
        Peter Zijlstra <peterz@...radead.org>,
        linux-kernel@...r.kernel.org,
        Linus Torvalds <torvalds@...ux-foundation.org>,
        Ingo Molnar <mingo@...nel.org>,
        Andrew Morton <akpm@...ux-foundation.org>,
        Thomas Gleixner <tglx@...utronix.de>,
        Masami Hiramatsu <mhiramat@...nel.org>,
        Mathieu Desnoyers <mathieu.desnoyers@...icios.com>,
        Matthew Helsley <mhelsley@...are.com>,
        "Rafael J . Wysocki" <rafael.j.wysocki@...el.com>,
        David Woodhouse <dwmw2@...radead.org>,
        Paolo Bonzini <pbonzini@...hat.com>,
        Jason Baron <jbaron@...mai.com>, Jiri Kosina <jkosina@...e.cz>,
        ard.biesheuvel@...aro.org, Andy Lutomirski <luto@...nel.org>
Subject: Re: [POC][RFC][PATCH 1/2] jump_function: Addition of new feature
 "jump_function"

On Mon, Oct 08, 2018 at 11:57:50PM -0400, Steven Rostedt wrote:
> On Mon, 8 Oct 2018 21:17:10 -0500
> Josh Poimboeuf <jpoimboe@...hat.com> wrote:
> 
> > I'm not really convinced we need objtool for this, maybe I'll try
> > whipping up a POC.
> 
> Awesome!
> 
> I wasn't thinking of actually having objtool itself perform this task,
> but instead breaking the internals of objtool up into more of a generic
> infrastructure, that recordmcount.c, objtool, and whatever this does
> can use.

So I had been thinking that we could find the call sites at runtime, by
looking at the relocations.  But I managed to forget that vmlinux
relocations are resolved during linking.  So yeah, some kind of tooling
magic would be needed.

I worked up a POC using objtool.  It doesn't *have* to be done with
objtool, but since it's already reading/writing all the ELF stuff
anyway, it was pretty easy to add this on.

This patch has at least a few issues:

- No module support.

- For some reason, the sync_cores in text_poke_bp() don't always seem to
  be working as expected.  Running this patch on my VM, the test code in
  cmdline_proc_show() works *most* of the time, but it occasionally
  branches off into the weeds.  I have no idea what the problem is yet.

diff --git a/arch/Kconfig b/arch/Kconfig
index 9d329608913e..20ff5624dad7 100644
--- a/arch/Kconfig
+++ b/arch/Kconfig
@@ -865,6 +865,9 @@ config HAVE_ARCH_PREL32_RELOCATIONS
 	  architectures, and don't require runtime relocation on relocatable
 	  kernels.
 
+config HAVE_ARCH_STATIC_CALL
+	bool
+
 source "kernel/gcov/Kconfig"
 
 source "scripts/gcc-plugins/Kconfig"
diff --git a/arch/x86/Kconfig b/arch/x86/Kconfig
index 5136a1281870..1a14c8f87876 100644
--- a/arch/x86/Kconfig
+++ b/arch/x86/Kconfig
@@ -128,6 +128,7 @@ config X86
 	select HAVE_ARCH_COMPAT_MMAP_BASES	if MMU && COMPAT
 	select HAVE_ARCH_PREL32_RELOCATIONS
 	select HAVE_ARCH_SECCOMP_FILTER
+	select HAVE_ARCH_STATIC_CALL		if X86_64
 	select HAVE_ARCH_THREAD_STRUCT_WHITELIST
 	select HAVE_ARCH_TRACEHOOK
 	select HAVE_ARCH_TRANSPARENT_HUGEPAGE
diff --git a/arch/x86/include/asm/static_call.h b/arch/x86/include/asm/static_call.h
new file mode 100644
index 000000000000..40fec631b760
--- /dev/null
+++ b/arch/x86/include/asm/static_call.h
@@ -0,0 +1,35 @@
+/* SPDX-License-Identifier: GPL-2.0 */
+#ifndef _ASM_STATIC_CALL_H
+#define _ASM_STATIC_CALL_H
+
+#ifdef CONFIG_X86_64
+
+#include <linux/frame.h>
+
+void static_call_init(void);
+extern void __static_call_update(void *tramp, void *func);
+
+#define DECLARE_STATIC_CALL(tramp, func)				\
+	extern typeof(func) tramp;					\
+	static void __used __section(.discard.static_call_tramps)	\
+		*__static_call_tramp_##tramp = tramp
+
+#define DEFINE_STATIC_CALL(tramp, func)					\
+	DECLARE_STATIC_CALL(tramp, func);				\
+	asm(".pushsection .text, \"ax\"				\n"	\
+	    ".align 4						\n"	\
+	    ".globl " #tramp "					\n"	\
+	    ".type " #tramp ", @function			\n"	\
+	    #tramp ":						\n"	\
+	    "jmp " #func "					\n"	\
+	    ASM_NOP3 "						\n"	\
+	    ".popsection					\n")
+
+#define static_call_update(tramp, func)					\
+	__static_call_update(tramp, func)
+
+#else /* !CONFIG_X86_64 */
+static inline void static_call_init(void) {}
+#endif
+
+#endif /* _ASM_STATIC_CALL_H */
diff --git a/arch/x86/kernel/Makefile b/arch/x86/kernel/Makefile
index 8824d01c0c35..e5d9f3a1e73f 100644
--- a/arch/x86/kernel/Makefile
+++ b/arch/x86/kernel/Makefile
@@ -62,6 +62,7 @@ obj-y			+= tsc.o tsc_msr.o io_delay.o rtc.o
 obj-y			+= pci-iommu_table.o
 obj-y			+= resource.o
 obj-y			+= irqflags.o
+obj-$(CONFIG_X86_64)	+= static_call.o
 
 obj-y				+= process.o
 obj-y				+= fpu/
diff --git a/arch/x86/kernel/setup.c b/arch/x86/kernel/setup.c
index b4866badb235..447401fc8d65 100644
--- a/arch/x86/kernel/setup.c
+++ b/arch/x86/kernel/setup.c
@@ -117,6 +117,7 @@
 #include <asm/microcode.h>
 #include <asm/kaslr.h>
 #include <asm/unwind.h>
+#include <asm/static_call.h>
 
 /*
  * max_low_pfn_mapped: highest direct mapped pfn under 4GB
@@ -874,6 +875,7 @@ void __init setup_arch(char **cmdline_p)
 	early_cpu_init();
 	arch_init_ideal_nops();
 	jump_label_init();
+	static_call_init();
 	early_ioremap_init();
 
 	setup_olpc_ofw_pgd();
diff --git a/arch/x86/kernel/static_call.c b/arch/x86/kernel/static_call.c
new file mode 100644
index 000000000000..e7a17ee6942d
--- /dev/null
+++ b/arch/x86/kernel/static_call.c
@@ -0,0 +1,117 @@
+/* SPDX-License-Identifier: GPL-2.0 */
+
+#include <linux/init.h>
+#include <linux/static_call.h>
+#include <linux/printk.h>
+#include <linux/bug.h>
+#include <linux/smp.h>
+#include <linux/memory.h>
+#include <asm/text-patching.h>
+#include <asm/processor.h>
+
+extern int cmdline_proc_show(void);
+
+/* The static call table is created by objtool */
+struct static_call_entry {
+	s32 insn, tramp;
+};
+extern struct static_call_entry __start_static_call_table[],
+			        __stop_static_call_table[];
+
+void __init static_call_init(void)
+{
+	struct static_call_entry *entry;
+	unsigned long insn, tramp, func;
+	unsigned char insn_opcode, tramp_opcode;
+	s32 call_dest;
+
+	for (entry = __start_static_call_table;
+	     entry < __stop_static_call_table; entry++) {
+
+		insn = (long)entry->insn + (unsigned long)&entry->insn;
+		tramp = (long)entry->tramp + (unsigned long)&entry->tramp;
+
+		insn_opcode = *(unsigned char *)insn;
+		if (insn_opcode != 0xe8 && insn_opcode != 0xe9) {
+			WARN_ONCE(1, "unexpected static call insn opcode %x at %pS",
+				  insn_opcode, (void *)insn);
+			continue;
+		}
+
+		tramp_opcode = *(unsigned char *)tramp;
+		if (tramp_opcode != 0xeb && tramp_opcode != 0xe9) {
+			WARN_ONCE(1, "unexpected trampoline jump opcode %x at %ps",
+				 tramp_opcode, (void *)tramp);
+			continue;
+		}
+
+		if (tramp_opcode == 0xeb)
+			func = *(s8 *)(tramp + 1) + (tramp + 2);
+		else
+			func = *(s32 *)(tramp + 1) + (tramp + 5);
+
+		call_dest = (long)(func) - (long)(insn + 5);
+
+		printk("static_call_init: poking %lx at %lx\n", (unsigned long)call_dest, (insn+1));
+
+		text_poke_early((void *)(insn + 1), &call_dest, 4);
+	}
+}
+
+/* cribbed from arch/x86/kernel/alternative.c */
+static void do_sync_core(void *info)
+{
+	sync_core();
+}
+
+void __static_call_update(void *tramp, void *func)
+{
+	struct static_call_entry *entry;
+	unsigned long insn, t;
+	s32 call_dest;
+	unsigned char opcodes[5];
+
+	mutex_lock(&text_mutex);
+
+	/*
+	 * Reuse the (now unused) trampoline to be the fallback handler
+	 * for text_poke_bp():
+	 */
+	call_dest = (long)(func) - (long)(tramp + 5);
+	opcodes[0] = 0xe8;
+	memcpy(&opcodes[1], &call_dest, 4);
+	text_poke(tramp, opcodes, 5);
+	on_each_cpu(do_sync_core, NULL, 1);
+
+	/* Patch the call sites: */
+	for (entry = __start_static_call_table;
+	     entry < __stop_static_call_table; entry++) {
+
+		t = (long)entry->tramp + (unsigned long)&entry->tramp;
+		if ((void *)t != tramp)
+			continue;
+
+		insn = (long)entry->insn + (unsigned long)&entry->insn;
+		call_dest = (long)(func) - (long)(insn + 5);
+		opcodes[0] = 0xe8;
+		memcpy(&opcodes[1], &call_dest, 4);
+
+		text_poke_bp((void *)insn, opcodes, 5, tramp);
+	}
+
+	mutex_unlock(&text_mutex);
+}
+
+/*** TEST CODE BELOW - called from cmdline_proc_show() ***/
+
+int my_func_add(int arg1, int arg2)
+{
+	return arg1 + arg2;
+}
+
+int my_func_sub(int arg1, int arg2)
+{
+	return arg1 - arg2;
+}
+
+DEFINE_STATIC_CALL(my_static_call, my_func_add);
diff --git a/arch/x86/kernel/vmlinux.lds.S b/arch/x86/kernel/vmlinux.lds.S
index 0d618ee634ac..cf0566f8a13c 100644
--- a/arch/x86/kernel/vmlinux.lds.S
+++ b/arch/x86/kernel/vmlinux.lds.S
@@ -185,6 +185,9 @@ SECTIONS
 
 	BUG_TABLE
 
+	/* FIXME: move to read-only section */
+	STATIC_CALL_TABLE
+
 	ORC_UNWIND_TABLE
 
 	. = ALIGN(PAGE_SIZE);
diff --git a/fs/proc/cmdline.c b/fs/proc/cmdline.c
index fa762c5fbcb2..c704b9e1fe5f 100644
--- a/fs/proc/cmdline.c
+++ b/fs/proc/cmdline.c
@@ -3,9 +3,27 @@
 #include <linux/init.h>
 #include <linux/proc_fs.h>
 #include <linux/seq_file.h>
+#include <linux/static_call.h>
+
+extern int my_func_add(int arg1, int arg2);
+extern int my_func_sub(int arg1, int arg2);
+DECLARE_STATIC_CALL(my_static_call, my_func_add);
 
 static int cmdline_proc_show(struct seq_file *m, void *v)
 {
+	int ret;
+
+	ret = my_static_call(1, 2);
+	printk("static call (orig): ret=%d\n", ret);
+
+	static_call_update(my_static_call, my_func_sub);
+	ret = my_static_call(1, 2);
+	printk("static call (sub): ret=%d\n", ret);
+
+	static_call_update(my_static_call, my_func_add);
+	ret = my_static_call(1, 2);
+	printk("static call (add): ret=%d\n", ret);
+
 	seq_puts(m, saved_command_line);
 	seq_putc(m, '\n');
 	return 0;
diff --git a/include/asm-generic/vmlinux.lds.h b/include/asm-generic/vmlinux.lds.h
index f09ee3c544bc..a1c7bda1b22a 100644
--- a/include/asm-generic/vmlinux.lds.h
+++ b/include/asm-generic/vmlinux.lds.h
@@ -722,6 +722,14 @@
 #define BUG_TABLE
 #endif
 
+#define STATIC_CALL_TABLE						\
+	. = ALIGN(8);							\
+	__static_call_table : AT(ADDR(__static_call_table) - LOAD_OFFSET) { \
+		__start_static_call_table = .;			\
+		KEEP(*(__static_call_table))				\
+		__stop_static_call_table = .;				\
+	}
+
 #ifdef CONFIG_UNWINDER_ORC
 #define ORC_UNWIND_TABLE						\
 	. = ALIGN(4);							\
diff --git a/include/linux/static_call.h b/include/linux/static_call.h
new file mode 100644
index 000000000000..729e7ee4c66b
--- /dev/null
+++ b/include/linux/static_call.h
@@ -0,0 +1,20 @@
+/* SPDX-License-Identifier: GPL-2.0 */
+#ifndef _LINUX_STATIC_CALL_H
+#define _LINUX_STATIC_CALL_H
+
+#ifdef CONFIG_HAVE_ARCH_STATIC_CALL
+#include <asm/static_call.h>
+#else
+
+#define DECLARE_STATIC_CALL(ptr, func)					\
+	extern typeof(func) *ptr
+
+#define DEFINE_STATIC_CALL(ptr, func)					\
+	typeof(func) *ptr = func
+
+#define static_call_update(ptr, func)					\
+	WRITE_ONCE(ptr, func)
+
+#endif /* !CONFIG_HAVE_ARCH_STATIC_CALL */
+
+#endif /* _LINUX_STATIC_CALL_H */
diff --git a/tools/objtool/check.c b/tools/objtool/check.c
index 0414a0d52262..a8e7d3b92513 100644
--- a/tools/objtool/check.c
+++ b/tools/objtool/check.c
@@ -525,6 +525,10 @@ static int add_jump_destinations(struct objtool_file *file)
 		} else {
 			/* sibling call */
 			insn->jump_dest = 0;
+			if (rela->sym->static_call_tramp) {
+				list_add_tail(&insn->static_call_node,
+					      &file->static_call_list);
+			}
 			continue;
 		}
 
@@ -1202,6 +1206,21 @@ static int read_retpoline_hints(struct objtool_file *file)
 	return 0;
 }
 
+static int read_static_call_tramps(struct objtool_file *file)
+{
+	struct section *sec;
+	struct rela *rela;
+
+	sec = find_section_by_name(file->elf, ".rela.discard.static_call_tramps");
+	if (!sec)
+		return 0;
+
+	list_for_each_entry(rela, &sec->rela_list, list)
+		rela->sym->static_call_tramp = true;
+
+	return 0;
+}
+
 static void mark_rodata(struct objtool_file *file)
 {
 	struct section *sec;
@@ -1267,6 +1286,10 @@ static int decode_sections(struct objtool_file *file)
 	if (ret)
 		return ret;
 
+	ret = read_static_call_tramps(file);
+	if (ret)
+		return ret;
+
 	return 0;
 }
 
@@ -1920,6 +1943,11 @@ static int validate_branch(struct objtool_file *file, struct instruction *first,
 			if (is_fentry_call(insn))
 				break;
 
+			if (insn->call_dest->static_call_tramp) {
+				list_add_tail(&insn->static_call_node,
+					      &file->static_call_list);
+			}
+
 			ret = dead_end_function(file, insn->call_dest);
 			if (ret == 1)
 				return 0;
@@ -2167,6 +2195,83 @@ static int validate_reachable_instructions(struct objtool_file *file)
 	return 0;
 }
 
+struct static_call_entry {
+	s32 insn, tramp;
+};
+
+static int create_static_call_sections(struct objtool_file *file)
+{
+	struct section *sec, *rela_sec;
+	struct rela *rela;
+	struct static_call_entry *entry;
+	struct instruction *insn;
+	int idx;
+
+	sec = find_section_by_name(file->elf, "__static_call_table");
+	if (sec) {
+		WARN("file already has __static_call_table section, skipping");
+		return -1;
+	}
+
+	if (list_empty(&file->static_call_list))
+		return 0;
+
+	idx = 0;
+	list_for_each_entry(insn, &file->static_call_list, static_call_node)
+		idx++;
+
+	sec = elf_create_section(file->elf, "__static_call_table",
+				 sizeof(struct static_call_entry), idx);
+	if (!sec)
+		return -1;
+
+	rela_sec = elf_create_rela_section(file->elf, sec);
+	if (!rela_sec)
+		return -1;
+
+	idx = 0;
+	list_for_each_entry(insn, &file->static_call_list, static_call_node) {
+
+		entry = (struct static_call_entry *)sec->data->d_buf + idx;
+		memset(entry, 0, sizeof(struct static_call_entry));
+
+		/* populate rela for 'insn' */
+		rela = malloc(sizeof(*rela));
+		if (!rela) {
+			perror("malloc");
+			return -1;
+		}
+		memset(rela, 0, sizeof(*rela));
+		rela->sym = insn->sec->sym;
+		rela->addend = insn->offset;
+		rela->type = R_X86_64_PC32;
+		rela->offset = idx * sizeof(struct static_call_entry);
+		list_add_tail(&rela->list, &rela_sec->rela_list);
+		hash_add(rela_sec->rela_hash, &rela->hash, rela->offset);
+
+		/* populate rela for 'tramp' */
+		rela = malloc(sizeof(*rela));
+		if (!rela) {
+			perror("malloc");
+			return -1;
+		}
+		memset(rela, 0, sizeof(*rela));
+		rela->sym = insn->call_dest;
+		rela->addend = 0;
+		rela->type = R_X86_64_PC32;
+		rela->offset = idx * sizeof(struct static_call_entry) + 4;
+		list_add_tail(&rela->list, &rela_sec->rela_list);
+		hash_add(rela_sec->rela_hash, &rela->hash, rela->offset);
+
+		idx++;
+	}
+
+	if (elf_rebuild_rela_section(rela_sec))
+		return -1;
+
+	return 0;
+}
+
 static void cleanup(struct objtool_file *file)
 {
 	struct instruction *insn, *tmpinsn;
@@ -2197,6 +2302,7 @@ int check(const char *_objname, bool orc)
 
 	INIT_LIST_HEAD(&file.insn_list);
 	hash_init(file.insn_hash);
+	INIT_LIST_HEAD(&file.static_call_list);
 	file.whitelist = find_section_by_name(file.elf, ".discard.func_stack_frame_non_standard");
 	file.c_file = find_section_by_name(file.elf, ".comment");
 	file.ignore_unreachables = no_unreachable;
@@ -2236,6 +2342,11 @@ int check(const char *_objname, bool orc)
 		warnings += ret;
 	}
 
+	ret = create_static_call_sections(&file);
+	if (ret < 0)
+		goto out;
+	warnings += ret;
+
 	if (orc) {
 		ret = create_orc(&file);
 		if (ret < 0)
@@ -2244,7 +2355,9 @@ int check(const char *_objname, bool orc)
 		ret = create_orc_sections(&file);
 		if (ret < 0)
 			goto out;
+	}
 
+	if (orc || !list_empty(&file.static_call_list)) {
 		ret = elf_write(file.elf);
 		if (ret < 0)
 			goto out;
diff --git a/tools/objtool/check.h b/tools/objtool/check.h
index e6e8a655b556..56b8b7fb1bd1 100644
--- a/tools/objtool/check.h
+++ b/tools/objtool/check.h
@@ -39,6 +39,7 @@ struct insn_state {
 struct instruction {
 	struct list_head list;
 	struct hlist_node hash;
+	struct list_head static_call_node;
 	struct section *sec;
 	unsigned long offset;
 	unsigned int len;
@@ -60,6 +61,7 @@ struct objtool_file {
 	struct elf *elf;
 	struct list_head insn_list;
 	DECLARE_HASHTABLE(insn_hash, 16);
+	struct list_head static_call_list;
 	struct section *whitelist;
 	bool ignore_unreachables, c_file, hints, rodata;
 };
diff --git a/tools/objtool/elf.h b/tools/objtool/elf.h
index bc97ed86b9cd..3cf44d7cc3ac 100644
--- a/tools/objtool/elf.h
+++ b/tools/objtool/elf.h
@@ -62,6 +62,7 @@ struct symbol {
 	unsigned long offset;
 	unsigned int len;
 	struct symbol *pfunc, *cfunc;
+	bool static_call_tramp;
 };
 
 struct rela {

Powered by blists - more mailing lists