From 35f99255f2bc0174aba90c44b3e70415f0658257 Mon Sep 17 00:00:00 2001 From: "H. Peter Anvin" Date: Sat, 17 Aug 2024 16:45:24 -0700 Subject: [PATCH] x86/extable: implement EX_TYPE_FUNC_REWIND Add a new exception type, which allows emulating an exception as if it had happened at or near the call site of a function. This allows a function call inside an alternative for instruction emulation to "kick back" the exception into the alternatives pattern, possibly invoking a different exception handling pattern there, or at least indicating the "real" location of the fault. Untested-by: H. Peter Anvin (Intel) Signed-off-by: H. Peter Anvin (Intel) --- arch/x86/include/asm/asm.h | 6 + arch/x86/include/asm/extable_fixup_types.h | 1 + arch/x86/mm/extable.c | 136 +++++++++++++-------- 3 files changed, 92 insertions(+), 51 deletions(-) diff --git a/arch/x86/include/asm/asm.h b/arch/x86/include/asm/asm.h index 2bec0c89a95c..7398261b0f4a 100644 --- a/arch/x86/include/asm/asm.h +++ b/arch/x86/include/asm/asm.h @@ -232,5 +232,11 @@ register unsigned long current_stack_pointer asm(_ASM_SP); #define _ASM_EXTABLE_FAULT(from, to) \ _ASM_EXTABLE_TYPE(from, to, EX_TYPE_FAULT) +#define _ASM_EXTABLE_FUNC_REWIND(from, ipdelta, spdelta) \ + _ASM_EXTABLE_TYPE(from, from /* unused */, \ + EX_TYPE_FUNC_REWIND | \ + EX_DATA_REG(spdelta) | \ + EX_DATA_IMM(ipdelta)) + #endif /* __KERNEL__ */ #endif /* _ASM_X86_ASM_H */ diff --git a/arch/x86/include/asm/extable_fixup_types.h b/arch/x86/include/asm/extable_fixup_types.h index 906b0d5541e8..2eed7f39893f 100644 --- a/arch/x86/include/asm/extable_fixup_types.h +++ b/arch/x86/include/asm/extable_fixup_types.h @@ -67,5 +67,6 @@ #define EX_TYPE_ZEROPAD 20 /* longword load with zeropad on fault */ #define EX_TYPE_ERETU 21 +#define EX_TYPE_FUNC_REWIND 22 #endif diff --git a/arch/x86/mm/extable.c b/arch/x86/mm/extable.c index 51986e8a9d35..076f1492b935 100644 --- a/arch/x86/mm/extable.c +++ b/arch/x86/mm/extable.c @@ -290,6 +290,28 @@ static bool ex_handler_eretu(const struct exception_table_entry *fixup, } #endif +/* + * Emulate a fault taken at the call site of a function. + * The combined reg and flags field are used as an unsigned + * number of machine words to pop off the stack before the + * return address, then the signed imm field is used as a delta + * from the return IP address. + */ +static bool ex_handler_func_rewind(struct pt_regs *regs, int data) +{ + const long ipdelta = FIELD_GET(EX_DATA_IMM_MASK, data); + const unsigned long pops = + FIELD_GET(EX_DATA_REG_MASK|EX_DATA_FLAG_MASK, data); + unsigned long *sp; + + sp = (unsigned long *)regs->sp; + sp += pops; + regs->ip = *sp++ + ipdelta; + regs->sp = (unsigned long)sp; + + return true; +} + int ex_get_fixup_type(unsigned long ip) { const struct exception_table_entry *e = search_exception_tables(ip); @@ -302,6 +324,7 @@ int fixup_exception(struct pt_regs *regs, int trapnr, unsigned long error_code, { const struct exception_table_entry *e; int type, reg, imm; + bool again; #ifdef CONFIG_PNPBIOS if (unlikely(SEGMENT_IS_PNP_CODE(regs->cs))) { @@ -317,60 +340,71 @@ int fixup_exception(struct pt_regs *regs, int trapnr, unsigned long error_code, } #endif - e = search_exception_tables(regs->ip); - if (!e) - return 0; - - type = FIELD_GET(EX_DATA_TYPE_MASK, e->data); - reg = FIELD_GET(EX_DATA_REG_MASK, e->data); - imm = FIELD_GET(EX_DATA_IMM_MASK, e->data); - - switch (type) { - case EX_TYPE_DEFAULT: - case EX_TYPE_DEFAULT_MCE_SAFE: - return ex_handler_default(e, regs); - case EX_TYPE_FAULT: - case EX_TYPE_FAULT_MCE_SAFE: - return ex_handler_fault(e, regs, trapnr); - case EX_TYPE_UACCESS: - return ex_handler_uaccess(e, regs, trapnr, fault_addr); - case EX_TYPE_CLEAR_FS: - return ex_handler_clear_fs(e, regs); - case EX_TYPE_FPU_RESTORE: - return ex_handler_fprestore(e, regs); - case EX_TYPE_BPF: - return ex_handler_bpf(e, regs); - case EX_TYPE_WRMSR: - return ex_handler_msr(e, regs, true, false, reg); - case EX_TYPE_RDMSR: - return ex_handler_msr(e, regs, false, false, reg); - case EX_TYPE_WRMSR_SAFE: - return ex_handler_msr(e, regs, true, true, reg); - case EX_TYPE_RDMSR_SAFE: - return ex_handler_msr(e, regs, false, true, reg); - case EX_TYPE_WRMSR_IN_MCE: - ex_handler_msr_mce(regs, true); - break; - case EX_TYPE_RDMSR_IN_MCE: - ex_handler_msr_mce(regs, false); - break; - case EX_TYPE_POP_REG: - regs->sp += sizeof(long); - fallthrough; - case EX_TYPE_IMM_REG: - return ex_handler_imm_reg(e, regs, reg, imm); - case EX_TYPE_FAULT_SGX: - return ex_handler_sgx(e, regs, trapnr); - case EX_TYPE_UCOPY_LEN: - return ex_handler_ucopy_len(e, regs, trapnr, fault_addr, reg, imm); - case EX_TYPE_ZEROPAD: - return ex_handler_zeropad(e, regs, fault_addr); + do { + e = search_exception_tables(regs->ip); + if (!e) + return 0; + + again = false; + + type = FIELD_GET(EX_DATA_TYPE_MASK, e->data); + reg = FIELD_GET(EX_DATA_REG_MASK, e->data); + imm = FIELD_GET(EX_DATA_IMM_MASK, e->data); + + switch (type) { + case EX_TYPE_DEFAULT: + case EX_TYPE_DEFAULT_MCE_SAFE: + return ex_handler_default(e, regs); + case EX_TYPE_FAULT: + case EX_TYPE_FAULT_MCE_SAFE: + return ex_handler_fault(e, regs, trapnr); + case EX_TYPE_UACCESS: + return ex_handler_uaccess(e, regs, trapnr, fault_addr); + case EX_TYPE_CLEAR_FS: + return ex_handler_clear_fs(e, regs); + case EX_TYPE_FPU_RESTORE: + return ex_handler_fprestore(e, regs); + case EX_TYPE_BPF: + return ex_handler_bpf(e, regs); + case EX_TYPE_WRMSR: + return ex_handler_msr(e, regs, true, false, reg); + case EX_TYPE_RDMSR: + return ex_handler_msr(e, regs, false, false, reg); + case EX_TYPE_WRMSR_SAFE: + return ex_handler_msr(e, regs, true, true, reg); + case EX_TYPE_RDMSR_SAFE: + return ex_handler_msr(e, regs, false, true, reg); + case EX_TYPE_WRMSR_IN_MCE: + ex_handler_msr_mce(regs, true); + break; + case EX_TYPE_RDMSR_IN_MCE: + ex_handler_msr_mce(regs, false); + break; + case EX_TYPE_POP_REG: + regs->sp += sizeof(long); + fallthrough; + case EX_TYPE_IMM_REG: + return ex_handler_imm_reg(e, regs, reg, imm); + case EX_TYPE_FAULT_SGX: + return ex_handler_sgx(e, regs, trapnr); + case EX_TYPE_UCOPY_LEN: + return ex_handler_ucopy_len(e, regs, trapnr, fault_addr, reg, imm); + case EX_TYPE_ZEROPAD: + return ex_handler_zeropad(e, regs, fault_addr); #ifdef CONFIG_X86_FRED - case EX_TYPE_ERETU: - return ex_handler_eretu(e, regs, error_code); + case EX_TYPE_ERETU: + return ex_handler_eretu(e, regs, error_code); #endif - } + case EX_TYPE_FUNC_REWIND: + again = ex_handler_func_rewind(regs, e->data); + break; + default: + break; /* Will BUG() */ + } + } while (again); + BUG(); + return 0; } extern unsigned int early_recursion_flag; -- 2.46.0