diff --git a/arch/x86_64/lib/clear_page.S b/arch/x86_64/lib/clear_page.S index 9a10a78..4e34c12 100644 --- a/arch/x86_64/lib/clear_page.S +++ b/arch/x86_64/lib/clear_page.S @@ -53,6 +53,8 @@ ENDPROC(clear_page) .align 8 .quad clear_page .quad 1b + .byte 0x00 + .byte 0x00 .byte X86_FEATURE_REP_GOOD .byte .Lclear_page_end - clear_page .byte 2b - 1b diff --git a/arch/x86_64/lib/copy_page.S b/arch/x86_64/lib/copy_page.S index 727a5d4..2aa51c2 100644 --- a/arch/x86_64/lib/copy_page.S +++ b/arch/x86_64/lib/copy_page.S @@ -113,6 +113,8 @@ ENDPROC(copy_page) .align 8 .quad copy_page .quad 1b + .byte 0x00 + .byte 0x00 .byte X86_FEATURE_REP_GOOD .byte .Lcopy_page_end - copy_page .byte 2b - 1b diff --git a/arch/x86_64/lib/copy_user.S b/arch/x86_64/lib/copy_user.S index 70bebd3..2dd39d1 100644 --- a/arch/x86_64/lib/copy_user.S +++ b/arch/x86_64/lib/copy_user.S @@ -27,6 +27,8 @@ .align 8 .quad 0b .quad 2b + .byte 0x00 + .byte 0x00 .byte \feature /* when feature is set */ .byte 5 .byte 5 diff --git a/arch/x86_64/lib/memcpy.S b/arch/x86_64/lib/memcpy.S index 0ea0ddc..ee42a08 100644 --- a/arch/x86_64/lib/memcpy.S +++ b/arch/x86_64/lib/memcpy.S @@ -123,6 +123,8 @@ ENDPROC(__memcpy) .align 8 .quad memcpy .quad 1b + .byte 0x00 + .byte 0x00 .byte X86_FEATURE_REP_GOOD .byte .Lfinal - memcpy .byte 2b - 1b diff --git a/arch/x86_64/lib/memset.S b/arch/x86_64/lib/memset.S index 2c59481..814e091 100644 --- a/arch/x86_64/lib/memset.S +++ b/arch/x86_64/lib/memset.S @@ -127,6 +127,8 @@ ENDPROC(__memset) .align 8 .quad memset .quad 1b + .byte 0x00 + .byte 0x00 .byte X86_FEATURE_REP_GOOD .byte .Lfinal - memset .byte 2b - 1b --- a/include/asm-x86_64/alternative.h +++ b/include/asm-x86_64/alternative.h @@ -7,14 +7,31 @@ #include #include +/* struct alt_instr - define replacement sequences + * + * this struct is used in 2 ways: + * - as the first entry for a replace sequence (used == 0) + * In this case *instr points to the original instruction and + * instr_cpuid is ignored + * - as a following entry in a replace sequence (used == [1|2]) + * In this case *instr is used as a replacement pointer too + * (supporting up to two replacements per struct) and + * instr_cpuid is its cpuid value + * + * The first matching replacement in a sequence is used + */ struct alt_instr { u8 *instr; /* original instruction */ u8 *replacement; + u8 used; /* count the number of replacements in + this struct (only for succeeding entries) */ + u8 instr_cpuid; /* cpuid bit set if instr is used + as replacement */ u8 cpuid; /* cpuid bit set for replacement */ u8 instrlen; /* length of original instruction */ u8 replacementlen; /* length of new instruction, <= instrlen */ - u8 pad[5]; -}; + u8 pad[3]; +} __attribute__ ((packed)); extern void apply_alternatives(struct alt_instr *start, struct alt_instr *end); @@ -36,6 +53,13 @@ static inline void alternatives_smp_switch(int smp) {} #endif + +/* + * Use or extend the following macros if you need more than one + * output argument in the alternative_io() macro + */ +#define ALTERNATIVE_OUTPUT2(a,b) a,b + /* * Alternative instructions for different CPU types or capabilities. * @@ -54,6 +78,8 @@ static inline void alternatives_smp_switch(int smp) {} " .align 8\n" \ " .quad 661b\n" /* label */ \ " .quad 663f\n" /* new instruction */ \ + " .byte 0x00\n" /* first entry */ \ + " .byte 0x00\n" /* zero for first entry */ \ " .byte %c0\n" /* feature bit */ \ " .byte 662b-661b\n" /* sourcelen */ \ " .byte 664f-663f\n" /* replacementlen */ \ @@ -78,6 +104,8 @@ static inline void alternatives_smp_switch(int smp) {} " .align 8\n" \ " .quad 661b\n" /* label */ \ " .quad 663f\n" /* new instruction */ \ + " .byte 0x00\n" /* first entry */ \ + " .byte 0x00\n" /* zero for first entry */ \ " .byte %c0\n" /* feature bit */ \ " .byte 662b-661b\n" /* sourcelen */ \ " .byte 664f-663f\n" /* replacementlen */ \ @@ -93,6 +121,8 @@ static inline void alternatives_smp_switch(int smp) {} " .align 8\n" \ " .quad 661b\n" /* label */ \ " .quad 663f\n" /* new instruction */ \ + " .byte 0x00\n" /* first entry */ \ + " .byte 0x00\n" /* zero for first entry */ \ " .byte %c[feat]\n" /* feature bit */ \ " .byte 662b-661b\n" /* sourcelen */ \ " .byte 664f-663f\n" /* replacementlen */ \ @@ -102,6 +132,45 @@ static inline void alternatives_smp_switch(int smp) {} ".previous" : output : [feat] "i" (feature), ##input) /* + * additional alternatives + * + * In the case where more than one alternative for an instruction exist, + * the two following macros could be used. They must appear immediately + * after the use alternative_io, alternative_input or alternative macros. + */ + +#define alternative_add_one(newinstr2, feature2) \ + asm volatile(".section .altinstructions,\"a\"\n" \ + " .align 8\n" \ + " .quad 661f\n" \ + " .quad 0x00\n" \ + " .byte 0x01\n" \ + " .byte %c[feat2]\n" \ + " .byte 0x00\n" \ + " .byte 662f-661f\n" \ + " .byte 0x00\n" \ + ".previous\n" \ + ".section .altinstr_replacement,\"ax\"\n" \ + "661:\n\t" newinstr2 "\n662:\n" \ + ".previous" : : [feat2] "i" (feature2) ) + +#define alternative_add_two(newinstr2, feature2, newinstr3, feature3) \ + asm volatile(".section .altinstructions,\"a\"\n" \ + " .align 8\n" \ + " .quad 661f\n" \ + " .quad 663f\n" \ + " .byte 0x02\n" \ + " .byte %c[feat2]\n" \ + " .byte %c[feat3]\n" \ + " .byte 662f-661f\n" \ + " .byte 664f-663f\n" \ + ".previous\n" \ + ".section .altinstr_replacement,\"ax\"\n" \ + "661:\n\t" newinstr2 "\n662:\n" \ + "663:\n\t" newinstr3 "\n664:\n" \ + ".previous" : : [feat2] "i" (feature2), [feat3] "i" (feature3) ) + +/* * Alternative inline assembly for SMP. * * The LOCK_PREFIX macro defined here replaces the LOCK and