Message-Id: <20200605073052.23044-1-wooy88.kim@samsung.com>
Date:   Fri,  5 Jun 2020 16:30:52 +0900
From:   Wooyeon Kim <wooy88.kim@...sung.com>
To:     Catalin Marinas <catalin.marinas@....com>,
        Will Deacon <will@...nel.org>
Cc:     jihun.kim@...sung.com, yhwan.joo@...sung.com, yb.song@...sung.com,
        junik.lee@...sung.com, yj.yim@...sung.com, sgun.bae@...sung.com,
        hk92.kim@...sung.com, dongww.kim@...sung.com,
        jinsoo37.kim@...sung.com, hyeyeon5.shim@...sung.com,
        hyewon.ryu@...sung.com, dh.han@...sung.com, kgene.kim@...sung.com,
        Allison Randal <allison@...utok.net>,
        Greg Kroah-Hartman <gregkh@...uxfoundation.org>,
        Thomas Gleixner <tglx@...utronix.de>,
        Mark Rutland <mark.rutland@....com>,
        Vincenzo Frascino <vincenzo.frascino@....com>,
        Kees Cook <keescook@...omium.org>,
        Kristina Martsenko <kristina.martsenko@....com>,
        Steve Capper <steve.capper@....com>,
        Bhupesh Sharma <bhsharma@...hat.com>,
        James Morse <james.morse@....com>,
        Marc Zyngier <maz@...nel.org>,
        Dave Martin <Dave.Martin@....com>,
        Sudeep Holla <sudeep.holla@....com>,
        Anisse Astier <aastier@...ebox.fr>,
        Julien Grall <julien.grall@....com>,
        Suzuki K Poulose <suzuki.poulose@....com>,
        linux-arm-kernel@...ts.infradead.org, linux-kernel@...r.kernel.org,
        Wooki Min <wooki.min@...sung.com>,
        Jeongtae Park <jtp.park@...sung.com>,
        Sanghoon Lee <shoon114.lee@...sung.com>,
        Wooyeon Kim <wooy88.kim@...sung.com>
Subject: [PATCH] arm64: fpsimd: Added API to manage fpsimd state inside
 kernel

From: Wooki Min <wooki.min@...sung.com>

     This is an patch to use FPSIMD register in Kernel space.
     It need to manage to use FPSIMD register without damaging it
     of the user task.
     Following items have been implemented and added.

     1. Using FPSIMD in ISR (in_interrupt)
	It can used __efi_fpsimd_begin/__efi_fpsimd_end
	which is already implemented.
	Save fpsimd state before entering ISR,
	and restore fpsimd state after ISR ends.
	For use in external kernel module,
	it is declared as EXPORT_SYMBOL.

     2. User task -> Function in kernel
        Add fpsimd_get/fpsimd_put API to save/restore
	FPSIMD in use by the user.
	In this case, depth variable is used to set fpsimd usage
	depth between User and Kernel space.
	 * fpsimd_get: Save the FPSIMD of the user task.
	 * fpsimd_put: Restore the FPSIMD of the user task.

	 EX> fpsimd_get();
	     API in kernel space();
	     fpsimd_put();

     3. Add kernel task FPSIMD save/restore in "fpsimd_thread_switch"
        It checks the depth value in current task structure and
	does save/restore action for FP/SIMD register used by kernel
	context.

Signed-off-by: Wooki Min <wooki.min@...sung.com>
Signed-off-by: Jeongtae Park <jtp.park@...sung.com>
Signed-off-by: Sanghoon Lee <shoon114.lee@...sung.com>
Signed-off-by: Wooyeon Kim <wooy88.kim@...sung.com>
---
 arch/arm64/include/asm/fpsimd.h      |  6 +++
 arch/arm64/include/asm/processor.h   | 13 +++++
 arch/arm64/include/uapi/asm/ptrace.h |  8 ++++
 arch/arm64/kernel/fpsimd.c           | 71 +++++++++++++++++++++++++---
 4 files changed, 92 insertions(+), 6 deletions(-)

diff --git a/arch/arm64/include/asm/fpsimd.h b/arch/arm64/include/asm/fpsimd.h
index 59f10dd13f12..462c434fc57c 100644
--- a/arch/arm64/include/asm/fpsimd.h
+++ b/arch/arm64/include/asm/fpsimd.h
@@ -167,6 +167,12 @@ static inline void sve_setup(void) { }
 extern void __efi_fpsimd_begin(void);
 extern void __efi_fpsimd_end(void);
 
+void fpsimd_set_task_using(struct task_struct *t);
+void fpsimd_clr_task_using(struct task_struct *t);
+
+void fpsimd_get(void);
+void fpsimd_put(void);
+
 #endif
 
 #endif
diff --git a/arch/arm64/include/asm/processor.h b/arch/arm64/include/asm/processor.h
index 240fe5e5b720..265669456bcb 100644
--- a/arch/arm64/include/asm/processor.h
+++ b/arch/arm64/include/asm/processor.h
@@ -139,6 +139,19 @@ struct thread_struct {
 		unsigned long	tp2_value;
 		struct user_fpsimd_state fpsimd_state;
 	} uw;
+	struct fpsimd_kernel_state fpsimd_kernel_state;
+
+	/*
+	 * indicate the depth of using FP/SIMD registers in kernel mode.
+	 * above kernel state should be preserved at first time
+	 * before FP/SIMD registers be used by other tasks
+	 * and the state should be restored before they be used by own.
+	 *
+	 * a kernel thread which uses FP/SIMD registers have to
+	 * set this depth and it could utilize for a tasks executes
+	 * some NEON instructions without preemption disable.
+	 */
+	atomic_t fpsimd_kernel_depth;
 
 	unsigned int		fpsimd_cpu;
 	void			*sve_state;	/* SVE registers, if any */
diff --git a/arch/arm64/include/uapi/asm/ptrace.h b/arch/arm64/include/uapi/asm/ptrace.h
index 42cbe34d95ce..0327e719721e 100644
--- a/arch/arm64/include/uapi/asm/ptrace.h
+++ b/arch/arm64/include/uapi/asm/ptrace.h
@@ -105,6 +105,14 @@ struct user_hwdebug_state {
 	}		dbg_regs[16];
 };
 
+/* Kernel structure for floating point */
+struct fpsimd_kernel_state {
+	__uint128_t	vregs[32];
+	__u32		fpsr;
+	__u32		fpcr;
+	unsigned int	cpu;
+};
+
 /* SVE/FP/SIMD state (NT_ARM_SVE) */
 
 struct user_sve_header {
diff --git a/arch/arm64/kernel/fpsimd.c b/arch/arm64/kernel/fpsimd.c
index 35cb5e66c504..07597423fcfc 100644
--- a/arch/arm64/kernel/fpsimd.c
+++ b/arch/arm64/kernel/fpsimd.c
@@ -269,9 +269,6 @@ static void sve_free(struct task_struct *task)
  */
 static void task_fpsimd_load(void)
 {
-	WARN_ON(!system_supports_fpsimd());
-	WARN_ON(!have_cpu_fpsimd_context());
-
 	if (system_supports_sve() && test_thread_flag(TIF_SVE))
 		sve_load_state(sve_pffr(&current->thread),
 			       &current->thread.uw.fpsimd_state.fpsr,
@@ -290,9 +287,6 @@ static void fpsimd_save(void)
 		this_cpu_ptr(&fpsimd_last_state);
 	/* set by fpsimd_bind_task_to_cpu() or fpsimd_bind_state_to_cpu() */
 
-	WARN_ON(!system_supports_fpsimd());
-	WARN_ON(!have_cpu_fpsimd_context());
-
 	if (!test_thread_flag(TIF_FOREIGN_FPSTATE)) {
 		if (system_supports_sve() && test_thread_flag(TIF_SVE)) {
 			if (WARN_ON(sve_get_vl() != last->sve_vl)) {
@@ -982,6 +976,10 @@ void do_fpsimd_exc(unsigned int esr, struct pt_regs *regs)
 void fpsimd_thread_switch(struct task_struct *next)
 {
 	bool wrong_task, wrong_cpu;
+	struct fpsimd_kernel_state *cur_kst
+			= &current->thread.fpsimd_kernel_state;
+	struct fpsimd_kernel_state *nxt_kst
+			= &next->thread.fpsimd_kernel_state;
 
 	if (!system_supports_fpsimd())
 		return;
@@ -991,6 +989,16 @@ void fpsimd_thread_switch(struct task_struct *next)
 	/* Save unsaved fpsimd state, if any: */
 	fpsimd_save();
 
+	if (atomic_read(&current->thread.fpsimd_kernel_depth))
+		fpsimd_save_state((struct user_fpsimd_state *)cur_kst);
+
+	if (atomic_read(&next->thread.fpsimd_kernel_depth)) {
+		fpsimd_load_state((struct user_fpsimd_state *)nxt_kst);
+		this_cpu_write(fpsimd_last_state.st,
+				(struct user_fpsimd_state *)nxt_kst);
+		nxt_kst->cpu = smp_processor_id();
+	}
+
 	/*
 	 * Fix up TIF_FOREIGN_FPSTATE to correctly describe next's
 	 * state.  For kernel threads, FPSIMD registers are never loaded
@@ -1233,6 +1241,55 @@ void fpsimd_save_and_flush_cpu_state(void)
 	__put_cpu_fpsimd_context();
 }
 
+void fpsimd_set_task_using(struct task_struct *t)
+{
+	atomic_set(&t->thread.fpsimd_kernel_depth, 1);
+}
+EXPORT_SYMBOL(fpsimd_set_task_using);
+
+void fpsimd_clr_task_using(struct task_struct *t)
+{
+	atomic_set(&t->thread.fpsimd_kernel_depth, 0);
+}
+EXPORT_SYMBOL(fpsimd_clr_task_using);
+
+void fpsimd_get(void)
+{
+	if (in_interrupt())
+		return;
+
+	if (atomic_inc_return(&current->thread.fpsimd_kernel_depth) == 1) {
+		preempt_disable();
+		if (current->mm) {
+			fpsimd_save();
+			fpsimd_flush_task_state(current);
+		}
+		fpsimd_flush_cpu_state();
+		preempt_enable();
+	}
+}
+EXPORT_SYMBOL(fpsimd_get);
+
+void fpsimd_put(void)
+{
+	if (in_interrupt())
+		return;
+
+	WARN_ON(atomic_dec_return(
+		&current->thread.fpsimd_kernel_depth) < 0);
+
+	if (atomic_read(&current->thread.fpsimd_kernel_depth) == 0) {
+		preempt_disable();
+		if (current->mm && test_thread_flag(TIF_FOREIGN_FPSTATE)) {
+			task_fpsimd_load();
+			fpsimd_bind_task_to_cpu();
+			clear_thread_flag(TIF_FOREIGN_FPSTATE);
+		}
+		preempt_enable();
+	}
+}
+EXPORT_SYMBOL(fpsimd_put);
+
 #ifdef CONFIG_KERNEL_MODE_NEON
 
 /*
@@ -1338,6 +1395,7 @@ void __efi_fpsimd_begin(void)
 		__this_cpu_write(efi_fpsimd_state_used, true);
 	}
 }
+EXPORT_SYMBOL(__efi_fpsimd_begin);
 
 /*
  * __efi_fpsimd_end(): clean up FPSIMD after an EFI runtime services call
@@ -1364,6 +1422,7 @@ void __efi_fpsimd_end(void)
 		}
 	}
 }
+EXPORT_SYMBOL(__efi_fpsimd_end);
 
 #endif /* CONFIG_EFI */
 
-- 
2.27.0.rc0

Powered by blists - more mailing lists