linux-kernel - [RFT PATCH v1 6/7] amd64: avoid saving and restoring FPSIMD registers until threads access them

lists.openwall.net		lists / announce owl-users owl-dev john-users john-dev passwdqc-users yescrypt popa3d-users / oss-security kernel-hardening musl sabotage tlsify passwords / crypt-dev xvendor / Bugtraq Full-Disclosure linux-kernel linux-netdev linux-ext4 linux-hardening linux-cve-announce PHC
Open Source and information security mailing list archives

Hash Suite: Windows password security audit tool. GUI, reports in PDF.

[<prev] [next>] [<thread-prev] [thread-next>] [day] [month] [year] [list]

Date:	Fri, 27 Sep 2013 16:04:46 +0800
From:	Jiang Liu <liuj97@...il.com>
To:	Catalin Marinas <catalin.marinas@....com>,
	Will Deacon <will.deacon@....com>,
	Jiang Liu <jiang.liu@...wei.com>,
	Ard Biesheuvel <ard.biesheuvel@...aro.org>,
	linux-arm-kernel@...ts.infradead.org, linux-kernel@...r.kernel.org
Cc:	Jiang Liu <liuj97@...il.com>
Subject: [RFT PATCH v1 6/7] amd64: avoid saving and restoring FPSIMD registers until threads access them

From: Jiang Liu <jiang.liu@...wei.com>

Use PF_USED_MATH flag to mark whether the thread has accessed any FPSIMD
registers, so we could avoid saving and restroing FPSIMD registers until
threads access them. This may improve performance when lazy FPSIMD restore
is disabled.

Signed-off-by: Jiang Liu <jiang.liu@...wei.com>
Cc: Jiang Liu <liuj97@...il.com>
---
 arch/arm64/kernel/fpsimd.c | 38 +++++++++++++++++++++++---------------
 1 file changed, 23 insertions(+), 15 deletions(-)

diff --git a/arch/arm64/kernel/fpsimd.c b/arch/arm64/kernel/fpsimd.c
index 267e54a..a81af5f 100644
--- a/arch/arm64/kernel/fpsimd.c
+++ b/arch/arm64/kernel/fpsimd.c
@@ -99,7 +99,8 @@ void fpsimd_disable_lazy_restore(void)
  * If lazy mode is enabled, caller needs to disable preemption
  * when calling fpsimd_load_state_lazy() and fpsimd_save_state_lazy().
  */
-static void fpsimd_load_state_lazy(struct fpsimd_state *state)
+static void fpsimd_load_state_lazy(struct fpsimd_state *state,
+				   struct task_struct *tsk)
 {
 	/* Could we reuse the hardware context? */
 	if (state->last_cpu == smp_processor_id() &&
@@ -109,13 +110,19 @@ static void fpsimd_load_state_lazy(struct fpsimd_state *state)
 	if (static_key_false(&fpsimd_lazy_mode)) {
 		fpsimd_clear_on_hw(state);
 		fpsimd_enable_trap();
-	} else {
+	} else if (tsk_used_math(tsk)) {
+		fpsimd_disable_trap();
 		fpsimd_load_state(state);
+	} else {
+		fpsimd_enable_trap();
 	}
 }
 
 static void fpsimd_save_state_lazy(struct fpsimd_state *state)
 {
+	if (!used_math())
+		return;
+
 	if (static_key_false(&fpsimd_lazy_mode)) {
 		if (!fpsimd_is_on_hw(state))
 			return;
@@ -133,12 +140,14 @@ void do_fpsimd_acc(unsigned int esr, struct pt_regs *regs)
 {
 	struct fpsimd_state *state = &current->thread.fpsimd_state;
 
+	fpsimd_disable_trap();
+	fpsimd_load_state(state);
 	if (static_key_false(&fpsimd_lazy_mode)) {
-		fpsimd_disable_trap();
-		fpsimd_load_state(state);
 		fpsimd_set_on_hw(state);
+		if (!used_math())
+			set_used_math();
 	} else {
-		WARN_ON(1);
+		set_used_math();
 	}
 }
 
@@ -175,7 +184,7 @@ void fpsimd_thread_switch(struct task_struct *next)
 	if (current->mm)
 		fpsimd_save_state_lazy(&current->thread.fpsimd_state);
 	if (next->mm)
-		fpsimd_load_state_lazy(&next->thread.fpsimd_state);
+		fpsimd_load_state_lazy(&next->thread.fpsimd_state, next);
 }
 
 void fpsimd_flush_thread(void)
@@ -187,8 +196,9 @@ void fpsimd_flush_thread(void)
 #if (AARCH64_FPCR_DEFAULT_VAL != 0)
 	state->fpcr = AARCH64_FPCR_DEFAULT_VAL;
 #endif
+	clear_used_math();
 	fpsimd_set_last_cpu(state, -1);
-	fpsimd_load_state_lazy(state);
+	fpsimd_load_state_lazy(state, current);
 	preempt_enable();
 }
 
@@ -205,7 +215,7 @@ void fpsimd_dup_state(struct fpsimd_state *src, struct fpsimd_state *dst)
 			fpsimd_save_state(dst);
 		fpsimd_clear_on_hw(dst);
 		preempt_enable();
-	} else {
+	} else if (used_math()) {
 		fpsimd_save_state(dst);
 	}
 	fpsimd_set_last_cpu(dst, -1);
@@ -226,12 +236,12 @@ void fpsimd_prepare_sigctx(struct fpsimd_state *ctx)
 		preempt_disable();
 		if (fpsimd_is_on_hw(ctx)) {
 			fpsimd_init_hw_state();
-		} else {
+		} else if (used_math()) {
 			ctx->fpsr = 0;
 			ctx->fpcr = AARCH64_FPCR_DEFAULT_VAL;
 		}
 		preempt_enable();
-	} else {
+	} else if (used_math()) {
 		fpsimd_init_hw_state();
 	}
 }
@@ -243,7 +253,7 @@ void fpsimd_restore_sigctx(struct fpsimd_state *ctx)
 	preempt_disable();
 	*state = *ctx;
 	__this_cpu_write(fpsimd_owner, NULL);
-	fpsimd_load_state_lazy(state);
+	fpsimd_load_state_lazy(state, current);
 	preempt_enable();
 }
 
@@ -261,9 +271,7 @@ void kernel_neon_begin(void)
 	if (current->mm)
 		fpsimd_save_state_lazy(&current->thread.fpsimd_state);
 
-	if (static_key_false(&fpsimd_lazy_mode))
-		fpsimd_disable_trap();
-
+	fpsimd_disable_trap();
 	fpsimd_init_hw_state();
 	__this_cpu_write(fpsimd_owner, NULL);
 }
@@ -272,7 +280,7 @@ EXPORT_SYMBOL(kernel_neon_begin);
 void kernel_neon_end(void)
 {
 	if (current->mm)
-		fpsimd_load_state_lazy(&current->thread.fpsimd_state);
+		fpsimd_load_state_lazy(&current->thread.fpsimd_state, current);
 
 	preempt_enable();
 }
-- 
1.8.1.2

--
To unsubscribe from this list: send the line "unsubscribe linux-kernel" in
the body of a message to majordomo@...r.kernel.org
More majordomo info at  http://vger.kernel.org/majordomo-info.html
Please read the FAQ at  http://www.tux.org/lkml/