| lists.openwall.net | lists / announce owl-users owl-dev john-users john-dev passwdqc-users yescrypt popa3d-users / oss-security kernel-hardening musl sabotage tlsify passwords / crypt-dev xvendor / Bugtraq Full-Disclosure linux-kernel linux-netdev linux-ext4 linux-hardening linux-cve-announce PHC | |
|
Open Source and information security mailing list archives
| ||
|
Message-Id: <20210831175025.27570-18-jiangshanlai@gmail.com>
Date: Wed, 1 Sep 2021 01:50:18 +0800
From: Lai Jiangshan <jiangshanlai@...il.com>
To: linux-kernel@...r.kernel.org
Cc: Lai Jiangshan <laijs@...ux.alibaba.com>,
Andy Lutomirski <luto@...nel.org>,
Thomas Gleixner <tglx@...utronix.de>,
Ingo Molnar <mingo@...hat.com>, Borislav Petkov <bp@...en8.de>,
x86@...nel.org, "H. Peter Anvin" <hpa@...or.com>,
Joerg Roedel <jroedel@...e.de>,
Youquan Song <youquan.song@...el.com>,
Tony Luck <tony.luck@...el.com>,
Juergen Gross <jgross@...e.com>,
"Peter Zijlstra (Intel)" <peterz@...radead.org>
Subject: [PATCH 17/24] x86/entry: Introduce struct ist_regs
From: Lai Jiangshan <laijs@...ux.alibaba.com>
struct ist_regs is the upmost stack frame for IST interrupt, it
consists of struct pt_regs and other fields which will be added in
later patch.
Make vc_switch_off_ist() take ist_regs as its argument and it will switch
the whole ist_regs if needed.
Make the frame before calling paranoid_entry() and paranoid_exit() be
struct ist_regs.
This patch is prepared for converting paranoid_entry() and paranoid_exit()
into C code which will need the additinal fields to store the results in
paranoid_entry() and to use them in paranoid_exit().
The C interrupt handlers don't use struct ist_regs due to they don't need
the additional fields in the struct ist_regs, and they still use pt_regs.
No functional change intended and IST_pt_reg is still 0.
Signed-off-by: Lai Jiangshan <laijs@...ux.alibaba.com>
---
arch/x86/entry/entry_64.S | 37 ++++++++++++++++++--------------
arch/x86/entry/traps.c | 16 +++++++-------
arch/x86/include/asm/traps.h | 10 ++++++++-
arch/x86/kernel/asm-offsets_64.c | 2 ++
4 files changed, 40 insertions(+), 25 deletions(-)
diff --git a/arch/x86/entry/entry_64.S b/arch/x86/entry/entry_64.S
index e968074046c3..1ae10ca351f4 100644
--- a/arch/x86/entry/entry_64.S
+++ b/arch/x86/entry/entry_64.S
@@ -431,13 +431,14 @@ SYM_CODE_START(\asmsym)
/* paranoid_entry returns GS information for paranoid_exit in EBX. */
call paranoid_entry
- UNWIND_HINT_REGS
+ UNWIND_HINT_REGS offset=IST_pt_regs
- movq %rsp, %rdi /* pt_regs pointer */
+ leaq IST_pt_regs(%rsp), %rdi /* pt_regs pointer */
call \cfunc
call paranoid_exit
+ addq $IST_pt_regs, %rsp /* put %rsp back to pt_regs */
jmp restore_regs_and_return_to_kernel
/* Switch to the regular task stack and use the noist entry point */
@@ -488,7 +489,7 @@ SYM_CODE_START(\asmsym)
*/
call paranoid_entry
- UNWIND_HINT_REGS
+ UNWIND_HINT_REGS offset=IST_pt_regs
/*
* Switch off the IST stack to make it free for nested exceptions. The
@@ -496,17 +497,17 @@ SYM_CODE_START(\asmsym)
* stack if it is safe to do so. If not it switches to the VC fall-back
* stack.
*/
- movq %rsp, %rdi /* pt_regs pointer */
+ movq %rsp, %rdi /* ist_regs pointer */
call vc_switch_off_ist
movq %rax, %rsp /* Switch to new stack */
- UNWIND_HINT_REGS
+ UNWIND_HINT_REGS offset=IST_pt_regs
- /* Update pt_regs */
- movq ORIG_RAX(%rsp), %rsi /* get error code into 2nd argument*/
- movq $-1, ORIG_RAX(%rsp) /* no syscall to restart */
+ leaq IST_pt_regs(%rsp), %rdi /* pt_regs pointer */
- movq %rsp, %rdi /* pt_regs pointer */
+ /* Update pt_regs */
+ movq ORIG_RAX(%rdi), %rsi /* get error code into 2nd argument*/
+ movq $-1, ORIG_RAX(%rdi) /* no syscall to restart */
call kernel_\cfunc
@@ -516,6 +517,7 @@ SYM_CODE_START(\asmsym)
* so it is definitely mapped even with PTI enabled.
*/
call paranoid_exit
+ addq $IST_pt_regs, %rsp /* put %rsp back to pt_regs */
jmp restore_regs_and_return_to_kernel
/* Switch to the regular task stack */
@@ -539,14 +541,15 @@ SYM_CODE_START(\asmsym)
/* paranoid_entry returns GS information for paranoid_exit in EBX. */
call paranoid_entry
- UNWIND_HINT_REGS
+ UNWIND_HINT_REGS offset=IST_pt_regs
- movq %rsp, %rdi /* pt_regs pointer into first argument */
- movq ORIG_RAX(%rsp), %rsi /* get error code into 2nd argument*/
- movq $-1, ORIG_RAX(%rsp) /* no syscall to restart */
+ leaq IST_pt_regs(%rsp), %rdi /* pt_regs pointer into first argument */
+ movq ORIG_RAX(%rdi), %rsi /* get error code into 2nd argument*/
+ movq $-1, ORIG_RAX(%rdi) /* no syscall to restart */
call \cfunc
call paranoid_exit
+ addq $IST_pt_regs, %rsp /* put %rsp back to pt_regs */
jmp restore_regs_and_return_to_kernel
_ASM_NOKPROBE(\asmsym)
@@ -852,8 +855,9 @@ SYM_CODE_START_LOCAL(paranoid_entry)
PUSH_AND_CLEAR_REGS skip_rdi=1
movq RDI(%rsp), %rsi /* temporarily store the return address in %rsi */
movq %rdi, RDI(%rsp) /* put %rdi onto pt_regs */
+ subq $IST_pt_regs, %rsp /* reserve room for ist_regs */
pushq %rsi /* put the return address onto the stack */
- ENCODE_FRAME_POINTER 8
+ ENCODE_FRAME_POINTER 8+IST_pt_regs
/*
* Always stash CR3 in %r14. This value will be restored,
@@ -1294,9 +1298,9 @@ end_repeat_nmi:
* Use paranoid_entry to handle SWAPGS and CR3.
*/
call paranoid_entry
- UNWIND_HINT_REGS
+ UNWIND_HINT_REGS offset=IST_pt_regs
- movq %rsp, %rdi
+ leaq IST_pt_regs(%rsp), %rdi /* pt_regs pointer */
movq $-1, %rsi
call exc_nmi
@@ -1305,6 +1309,7 @@ end_repeat_nmi:
* restore_regs_and_return_to_kernel as we must handle nested NMI.
*/
call paranoid_exit
+ addq $IST_pt_regs, %rsp /* put %rsp back to pt_regs */
POP_REGS
diff --git a/arch/x86/entry/traps.c b/arch/x86/entry/traps.c
index 40722a2f61ae..da55565d43ef 100644
--- a/arch/x86/entry/traps.c
+++ b/arch/x86/entry/traps.c
@@ -693,17 +693,17 @@ struct pt_regs *sync_regs(struct pt_regs *eregs)
}
#ifdef CONFIG_AMD_MEM_ENCRYPT
-asmlinkage __visible noinstr struct pt_regs *vc_switch_off_ist(struct pt_regs *regs)
+asmlinkage __visible noinstr struct ist_regs *vc_switch_off_ist(struct ist_regs *ist)
{
unsigned long sp, *stack;
struct stack_info info;
- struct pt_regs *regs_ret;
+ struct ist_regs *ist_ret;
/*
* In the SYSCALL entry path the RSP value comes from user-space - don't
* trust it and switch to the current kernel stack
*/
- if (ip_within_syscall_gap(regs)) {
+ if (ip_within_syscall_gap(&ist->regs)) {
sp = this_cpu_read(cpu_current_top_of_stack);
goto sync;
}
@@ -713,7 +713,7 @@ asmlinkage __visible noinstr struct pt_regs *vc_switch_off_ist(struct pt_regs *r
* happened from a safe stack. Not safe are the entry or unknown stacks,
* use the fall-back stack instead in this case.
*/
- sp = regs->sp;
+ sp = ist->regs.sp;
stack = (unsigned long *)sp;
if (!get_stack_info_noinstr(stack, current, &info) || info.type == STACK_TYPE_ENTRY ||
@@ -726,12 +726,12 @@ asmlinkage __visible noinstr struct pt_regs *vc_switch_off_ist(struct pt_regs *r
* IST stack. The code below only copies pt_regs, the real switch happens
* in assembly code.
*/
- sp = ALIGN_DOWN(sp, 8) - sizeof(*regs_ret);
+ sp = ALIGN_DOWN(sp, 8) - sizeof(*ist_ret);
- regs_ret = (struct pt_regs *)sp;
- *regs_ret = *regs;
+ ist_ret = (struct ist_regs *)sp;
+ *ist_ret = *ist;
- return regs_ret;
+ return ist_ret;
}
#endif
diff --git a/arch/x86/include/asm/traps.h b/arch/x86/include/asm/traps.h
index 5c41a279c1e0..e24c63bbc30a 100644
--- a/arch/x86/include/asm/traps.h
+++ b/arch/x86/include/asm/traps.h
@@ -11,9 +11,17 @@
#include <asm/trap_pf.h>
#ifdef CONFIG_X86_64
+struct ist_regs {
+ /*
+ * ist specific fields must be defined before pt_regs
+ * and they are located below pt_regs on the stacks.
+ */
+ struct pt_regs regs;
+};
+
asmlinkage __visible notrace struct pt_regs *do_error_entry(struct pt_regs *eregs);
void __init trap_init(void);
-asmlinkage __visible noinstr struct pt_regs *vc_switch_off_ist(struct pt_regs *eregs);
+asmlinkage __visible noinstr struct ist_regs *vc_switch_off_ist(struct ist_regs *ist);
#endif
#ifdef CONFIG_X86_F00F_BUG
diff --git a/arch/x86/kernel/asm-offsets_64.c b/arch/x86/kernel/asm-offsets_64.c
index b14533af7676..3cffadf64ac2 100644
--- a/arch/x86/kernel/asm-offsets_64.c
+++ b/arch/x86/kernel/asm-offsets_64.c
@@ -4,6 +4,7 @@
#endif
#include <asm/ia32.h>
+#include <asm/traps.h>
#if defined(CONFIG_KVM_GUEST) && defined(CONFIG_PARAVIRT_SPINLOCKS)
#include <asm/kvm_para.h>
@@ -55,6 +56,7 @@ int main(void)
#undef ENTRY
BLANK();
+ DEFINE(IST_pt_regs, offsetof(struct ist_regs, regs));
#ifdef CONFIG_STACKPROTECTOR
DEFINE(stack_canary_offset, offsetof(struct fixed_percpu_data, stack_canary));
--
2.19.1.6.gb485710b
Powered by blists - more mailing lists