From 0441f6cf7f593502ecfec0da954ef3e15cae7b70 Mon Sep 17 00:00:00 2001 From: David Woodhouse Date: Sat, 23 Dec 2017 02:11:18 +0000 Subject: [PATCH 3/8] x86: Use retpoline for calls in inline asm Fix up PV operations and hypercalls to avoid indirect calls. There is perhaps scope for optimising the pvops case. For ops which aren't going to be changed at runtime (all of them?), we could just patch the code to make it use *direct* jumps instead of indirect. Signed-off-by: David Woodhouse --- arch/x86/include/asm/mshyperv.h | 18 ++++++++++-------- arch/x86/include/asm/paravirt_types.h | 15 ++++++++++++--- arch/x86/include/asm/spec_ctrl.h | 13 +++++++++++++ arch/x86/include/asm/xen/hypercall.h | 5 +++-- 4 files changed, 38 insertions(+), 13 deletions(-) diff --git a/arch/x86/include/asm/mshyperv.h b/arch/x86/include/asm/mshyperv.h index 581bb54..ca7463b 100644 --- a/arch/x86/include/asm/mshyperv.h +++ b/arch/x86/include/asm/mshyperv.h @@ -7,6 +7,7 @@ #include #include #include +#include /* * The below CPUID leaves are present if VersionAndFeatures.HypervisorPresent @@ -186,10 +187,11 @@ static inline u64 hv_do_hypercall(u64 control, void *input, void *output) return U64_MAX; __asm__ __volatile__("mov %4, %%r8\n" - "call *%5" + CALL_THUNK : "=a" (hv_status), ASM_CALL_CONSTRAINT, "+c" (control), "+d" (input_address) - : "r" (output_address), "m" (hv_hypercall_pg) + : "r" (output_address), + THUNK_TARGET(hv_hypercall_pg) : "cc", "memory", "r8", "r9", "r10", "r11"); #else u32 input_address_hi = upper_32_bits(input_address); @@ -200,13 +202,13 @@ static inline u64 hv_do_hypercall(u64 control, void *input, void *output) if (!hv_hypercall_pg) return U64_MAX; - __asm__ __volatile__("call *%7" + __asm__ __volatile__(CALL_THUNK : "=A" (hv_status), "+c" (input_address_lo), ASM_CALL_CONSTRAINT : "A" (control), "b" (input_address_hi), "D"(output_address_hi), "S"(output_address_lo), - "m" (hv_hypercall_pg) + THUNK_TARGET(hv_hypercall_pg) : "cc", "memory"); #endif /* !x86_64 */ return hv_status; @@ -227,10 +229,10 @@ static inline u64 hv_do_fast_hypercall8(u16 code, u64 input1) #ifdef CONFIG_X86_64 { - __asm__ __volatile__("call *%4" + __asm__ __volatile__(CALL_THUNK : "=a" (hv_status), ASM_CALL_CONSTRAINT, "+c" (control), "+d" (input1) - : "m" (hv_hypercall_pg) + : THUNK_TARGET(hv_hypercall_pg) : "cc", "r8", "r9", "r10", "r11"); } #else @@ -238,13 +240,13 @@ static inline u64 hv_do_fast_hypercall8(u16 code, u64 input1) u32 input1_hi = upper_32_bits(input1); u32 input1_lo = lower_32_bits(input1); - __asm__ __volatile__ ("call *%5" + __asm__ __volatile__ (CALL_THUNK : "=A"(hv_status), "+c"(input1_lo), ASM_CALL_CONSTRAINT : "A" (control), "b" (input1_hi), - "m" (hv_hypercall_pg) + THUNK_TARGET(hv_hypercall_pg) : "cc", "edi", "esi"); } #endif diff --git a/arch/x86/include/asm/paravirt_types.h b/arch/x86/include/asm/paravirt_types.h index 6ec54d0..dfff11e 100644 --- a/arch/x86/include/asm/paravirt_types.h +++ b/arch/x86/include/asm/paravirt_types.h @@ -336,11 +336,17 @@ extern struct pv_lock_ops pv_lock_ops; #define PARAVIRT_PATCH(x) \ (offsetof(struct paravirt_patch_template, x) / sizeof(void *)) +#define paravirt_clobber(clobber) \ + [paravirt_clobber] "i" (clobber) +#ifdef RETPOLINE +#define paravirt_type(op) \ + [paravirt_typenum] "i" (PARAVIRT_PATCH(op)), \ + [paravirt_opptr] "r" ((op)) +#else #define paravirt_type(op) \ [paravirt_typenum] "i" (PARAVIRT_PATCH(op)), \ [paravirt_opptr] "i" (&(op)) -#define paravirt_clobber(clobber) \ - [paravirt_clobber] "i" (clobber) +#endif /* * Generate some code, and mark it as patchable by the @@ -392,8 +398,11 @@ int paravirt_disable_iospace(void); * offset into the paravirt_patch_template structure, and can therefore be * freely converted back into a structure offset. */ +#ifdef RETPOLINE +#define PARAVIRT_CALL "call __x86.indirect_thunk.%V[paravirt_opptr];" +#else #define PARAVIRT_CALL "call *%c[paravirt_opptr];" - +#endif /* * These macros are intended to wrap calls through one of the paravirt * ops structs, so that they can be later identified and patched at diff --git a/arch/x86/include/asm/spec_ctrl.h b/arch/x86/include/asm/spec_ctrl.h index 59b664e..58a49a2 100644 --- a/arch/x86/include/asm/spec_ctrl.h +++ b/arch/x86/include/asm/spec_ctrl.h @@ -196,5 +196,18 @@ ALTERNATIVE __stringify(__ASM_STUFF_RSB), "", X86_FEATURE_SMEP #endif .endm +#else /* __ASSEMBLY__ */ + +#ifdef RETPOLINE +#define CALL_THUNK ALTERNATIVE( \ + "call __x86.indirect_thunk.%V[thunk_target]", \ + "call *%[thunk_target]", X86_FEATURE_IBRS_ATT) +#define THUNK_TARGET(addr) [thunk_target] "r" (addr) +#else +#define CALL_THUNK "call *%[thunk_target]" +#define THUNK_TARGET(addr) [thunk_target] "rm" (addr) +#endif + #endif /* __ASSEMBLY__ */ + #endif /* _ASM_X86_SPEC_CTRL_H */ diff --git a/arch/x86/include/asm/xen/hypercall.h b/arch/x86/include/asm/xen/hypercall.h index 7cb282e..90df832 100644 --- a/arch/x86/include/asm/xen/hypercall.h +++ b/arch/x86/include/asm/xen/hypercall.h @@ -44,6 +44,7 @@ #include #include #include +#include #include #include @@ -217,9 +218,9 @@ privcmd_call(unsigned call, __HYPERCALL_5ARG(a1, a2, a3, a4, a5); stac(); - asm volatile("call *%[call]" + asm volatile(CALL_THUNK : __HYPERCALL_5PARAM - : [call] "a" (&hypercall_page[call]) + : [thunk_target] "a" (&hypercall_page[call]) : __HYPERCALL_CLOBBER5); clac(); -- 2.7.4