diff --git a/arch/x86/include/asm/uaccess_64.h b/arch/x86/include/asm/uaccess_64.h index 5cd1caa8bc65..db58c4436ce3 100644 --- a/arch/x86/include/asm/uaccess_64.h +++ b/arch/x86/include/asm/uaccess_64.h @@ -62,6 +62,8 @@ copy_to_user_mcsafe(void *to, const void *from, unsigned len) return ret; } +#define marker(x) asm volatile("rorx $" #x ",%rax,%rdx") + static __always_inline __must_check unsigned long raw_copy_from_user(void *dst, const void __user *src, unsigned long size) { @@ -72,30 +74,35 @@ raw_copy_from_user(void *dst, const void __user *src, unsigned long size) switch (size) { case 1: __uaccess_begin_nospec(); + marker(1); __get_user_asm_nozero(*(u8 *)dst, (u8 __user *)src, ret, "b", "b", "=q", 1); __uaccess_end(); return ret; case 2: __uaccess_begin_nospec(); + marker(2); __get_user_asm_nozero(*(u16 *)dst, (u16 __user *)src, ret, "w", "w", "=r", 2); __uaccess_end(); return ret; case 4: __uaccess_begin_nospec(); + marker(4); __get_user_asm_nozero(*(u32 *)dst, (u32 __user *)src, ret, "l", "k", "=r", 4); __uaccess_end(); return ret; case 8: __uaccess_begin_nospec(); + marker(8); __get_user_asm_nozero(*(u64 *)dst, (u64 __user *)src, ret, "q", "", "=r", 8); __uaccess_end(); return ret; case 10: __uaccess_begin_nospec(); + marker(10); __get_user_asm_nozero(*(u64 *)dst, (u64 __user *)src, ret, "q", "", "=r", 10); if (likely(!ret)) @@ -106,6 +113,7 @@ raw_copy_from_user(void *dst, const void __user *src, unsigned long size) return ret; case 16: __uaccess_begin_nospec(); + marker(16); __get_user_asm_nozero(*(u64 *)dst, (u64 __user *)src, ret, "q", "", "=r", 16); if (likely(!ret)) @@ -129,30 +137,35 @@ raw_copy_to_user(void __user *dst, const void *src, unsigned long size) switch (size) { case 1: __uaccess_begin(); + marker(51); __put_user_asm(*(u8 *)src, (u8 __user *)dst, ret, "b", "b", "iq", 1); __uaccess_end(); return ret; case 2: __uaccess_begin(); + marker(52); __put_user_asm(*(u16 *)src, (u16 __user *)dst, ret, "w", "w", "ir", 2); __uaccess_end(); return ret; case 4: __uaccess_begin(); + marker(54); __put_user_asm(*(u32 *)src, (u32 __user *)dst, ret, "l", "k", "ir", 4); __uaccess_end(); return ret; case 8: __uaccess_begin(); + marker(58); __put_user_asm(*(u64 *)src, (u64 __user *)dst, ret, "q", "", "er", 8); __uaccess_end(); return ret; case 10: __uaccess_begin(); + marker(60); __put_user_asm(*(u64 *)src, (u64 __user *)dst, ret, "q", "", "er", 10); if (likely(!ret)) { @@ -164,6 +177,7 @@ raw_copy_to_user(void __user *dst, const void *src, unsigned long size) return ret; case 16: __uaccess_begin(); + marker(66); __put_user_asm(*(u64 *)src, (u64 __user *)dst, ret, "q", "", "er", 16); if (likely(!ret)) {