#include #include typedef unsigned long long int hp_timing_t; #define MAXSAMPLESTPT 1000 #define MAXCOPYSIZE (1024 * 1024 * 100) #define ORIG 0 #define NEW 1 static char* buf1 = NULL; static char* buf2 = NULL; static int repeat_one_test = 32; hp_timing_t _dl_hp_timing_overhead; # define HP_TIMING_NOW(Var) \ ({ unsigned long long _hi, _lo; \ asm volatile ("rdtsc" : "=a" (_lo), "=d" (_hi)); \ (Var) = _hi << 32 | _lo; }) #define HP_TIMING_DIFF(Diff, Start, End) (Diff) = ((End) - (Start)) #define HP_TIMING_TOTAL(total_time, start, end) \ do \ { \ hp_timing_t tmptime; \ HP_TIMING_DIFF (tmptime, start + _dl_hp_timing_overhead, end); \ total_time += tmptime; \ } \ while (0) #define HP_TIMING_BEST(best_time, start, end) \ do \ { \ hp_timing_t tmptime; \ HP_TIMING_DIFF (tmptime, start + _dl_hp_timing_overhead, end); \ if (best_time > tmptime) \ best_time = tmptime; \ } \ while (0) void memcpy_orig(char *dst, char *src, int len); void memcpy_new(char *dst, char *src, int len); void (*do_memcpy)(char *dst, char *src, int len); static void do_one_test ( char *dst, char *src, size_t len) { hp_timing_t start __attribute ((unused)); hp_timing_t stop __attribute ((unused)); hp_timing_t best_time = ~ (hp_timing_t) 0; size_t i,j; for (i = 0; i < repeat_one_test; ++i) { HP_TIMING_NOW (start); do_memcpy ( dst, src, len); HP_TIMING_NOW (stop); HP_TIMING_BEST (best_time, start, stop); } printf ("\t%zd", (size_t) best_time); } static void do_test (size_t align1, size_t align2, size_t len) { size_t i, j; char *s1, *s2; s1 = (char *) (buf1 + align1); s2 = (char *) (buf2 + align2); printf ("TPT: Len %4zd, alignment %2zd/%2zd:", len, align1, align2); do_memcpy = memcpy_orig; do_one_test (s2, s1, len); do_memcpy = memcpy_new; do_one_test (s2, s1, len); putchar ('\n'); } static test_init(void) { int i; buf1 = valloc(MAXCOPYSIZE); buf2 = valloc(MAXCOPYSIZE); for (i = 0; i < MAXCOPYSIZE ; i = i + 64) { buf1[i] = buf2[i] = i & 0xff; } } void memset_c(char *dst, char *src, int len) { __asm__("mov %rdx, %rcx"); __asm__("shr $3, %rcx"); __asm__("rep stosq"); } void memset_2(char *dst, char *src, int len) { __asm__("sub $128, %rdx"); __asm__("1:"); __asm__("sub $128, %rdx"); __asm__("movdqa %xmm0, (%rdi)"); __asm__("movdqa %xmm0, 16(%rdi)"); __asm__("movdqa %xmm0, 32(%rdi)"); __asm__("movdqa %xmm0, 48(%rdi)"); __asm__("movdqa %xmm0, 64(%rdi)"); __asm__("movdqa %xmm0, 80(%rdi)"); __asm__("movdqa %xmm0, 96(%rdi)"); __asm__("movdqa %xmm0, 112(%rdi)"); __asm__("jae 1b"); } void memcpy_new(char *dst, char *src, int len) { __asm__("movq %rdi, %rax"); __asm__("cmp $0x20, %edx"); __asm__("jb 1f"); /* * We check whether memory false dependece could occur, * then jump to corresponding copy mode. */ __asm__("cmp %dil, %sil"); __asm__("jl 2f"); __asm__("subl $0x20, %edx"); __asm__("3:"); __asm__("subq $0x20, %rdx"); /* * Move in blocks of 4x8 bytes: */ __asm__("movq 0*8(%rsi), %r8"); __asm__("movq 1*8(%rsi), %r9"); __asm__("movq 2*8(%rsi), %r10"); __asm__("movq 3*8(%rsi), %r11"); __asm__("leaq 4*8(%rsi), %rsi"); __asm__("movq %r8, 0*8(%rdi)"); __asm__("movq %r9, 1*8(%rdi)"); __asm__("movq %r10, 2*8(%rdi)"); __asm__("movq %r11, 3*8(%rdi)"); __asm__("leaq 4*8(%rdi), %rdi"); __asm__("jae 3b"); __asm__("addq $0x20, %rdx"); __asm__("jmp 1f"); __asm__("2:"); /* * Calculate copy position to tail. */ __asm__("addq %rdx, %rsi"); __asm__("addq %rdx, %rdi"); __asm__("subq $0x20, %rdx"); __asm__(".p2align 4"); __asm__("4:"); __asm__("subq $0x20, %rdx"); __asm__("movq -1*8(%rsi), %r8"); __asm__("movq -2*8(%rsi), %r9"); __asm__("movq -3*8(%rsi), %r10"); __asm__("movq -4*8(%rsi), %r11"); __asm__("leaq -4*8(%rsi), %rsi"); __asm__("movq %r8, -1*8(%rdi)"); __asm__("movq %r9, -2*8(%rdi)"); __asm__("movq %r10, -3*8(%rdi)"); __asm__("movq %r11, -4*8(%rdi)"); __asm__("leaq -4*8(%rdi), %rdi"); __asm__("jae 4b"); /* * Calculate copy position to head. */ __asm__("addq $0x20, %rdx"); __asm__("subq %rdx, %rsi"); __asm__("subq %rdx, %rdi"); __asm__("1:"); __asm__("cmpq $16, %rdx"); __asm__("jb 5f"); /* * Move data from 16 bytes to 31 bytes. */ __asm__("movq 0*8(%rsi), %r8"); __asm__("movq 1*8(%rsi), %r9"); __asm__("movq -2*8(%rsi, %rdx), %r10"); __asm__("movq -1*8(%rsi, %rdx), %r11"); __asm__("movq %r8, 0*8(%rdi)"); __asm__("movq %r9, 1*8(%rdi)"); __asm__("movq %r10, -2*8(%rdi, %rdx)"); __asm__("movq %r11, -1*8(%rdi, %rdx)"); __asm__("retq"); __asm__(".p2align 4"); __asm__("5:"); __asm__("cmpq $8, %rdx"); __asm__("jb 6f"); /* * Move data from 8 bytes to 15 bytes. */ __asm__("movq 0*8(%rsi), %r8"); __asm__("movq -1*8(%rsi, %rdx), %r9"); __asm__("movq %r8, 0*8(%rdi)"); __asm__("movq %r9, -1*8(%rdi, %rdx)"); __asm__("retq"); __asm__(".p2align 4"); __asm__("6:"); __asm__("cmpq $4, %rdx"); __asm__("jb 7f"); /* * Move data from 4 bytes to 7 bytes. */ __asm__("movl (%rsi), %ecx"); __asm__("movl -4(%rsi, %rdx), %r8d"); __asm__("movl %ecx, (%rdi)"); __asm__("movl %r8d, -4(%rdi, %rdx)"); __asm__("retq"); __asm__(".p2align 4"); __asm__("7:"); __asm__("cmpl $0, %edx"); __asm__("je 8f"); /* * Move data from 1 bytes to 3 bytes. */ __asm__("9:"); __asm__("movb (%rsi), %r8b"); __asm__("movb %r8b, (%rdi)"); __asm__("incq %rdi"); __asm__("incq %rsi"); __asm__("decl %edx"); __asm__("jnz 9b"); __asm__("8:"); } void memcpy_orig(char *dst, char *src, int len) { __asm__("mov %rdi, %rax"); __asm__("movl %edx, %ecx"); __asm__("shrl $6, %ecx"); __asm__("jz 2f"); __asm__("mov $0x80, %r8d "); /*aligned case for loop 1 */ __asm__("1:"); __asm__("decl %ecx"); __asm__("mov 0*8(%rsi), %r11"); __asm__("mov 1*8(%rdi), %r8"); __asm__("mov %r11, 0*8(%rdi)"); __asm__("mov %r8, 1*8(%rdi)"); __asm__("mov 2*8(%rsi), %r9"); __asm__("mov 3*8(%rdi), %r10"); __asm__("mov %r9, 2*8(%rdi)"); __asm__("mov %r10, 3*8(%rdi)"); __asm__("mov 4*8(%rsi), %r11"); __asm__("mov 5*8(%rdi), %r8"); __asm__("mov %r11, 4*8(%rdi)"); __asm__("mov %r8, 5*8(%rdi)"); __asm__("mov 6*8(%rsi), %r9"); __asm__("mov 7*8(%rdi), %r10"); __asm__("mov %r9, 6*8(%rdi)"); __asm__("mov %r10, 7*8(%rdi)"); __asm__("leaq 64(%rsi), %rsi"); __asm__("leaq 64(%rdi), %rdi"); __asm__("jnz 1b"); __asm__("2:"); __asm__("movl %edx, %ecx"); __asm__("andl $63, %ecx"); __asm__("shrl $3, %ecx"); __asm__("jz 4f"); __asm__("3:"); __asm__("decl %ecx"); __asm__("mov (%rsi), %r8"); __asm__("mov %r8, (%rdi)"); __asm__("leaq 8(%rdi), %rdi"); __asm__("leaq 8(%rsi), %rsi"); __asm__("jnz 3b"); __asm__("4:"); __asm__("movl %edx, %ecx"); __asm__("andl $7, %ecx"); __asm__("jz 6f"); __asm__("5:"); __asm__("movb (%rsi), %r8b"); __asm__("movb %r8b, (%rdi)"); __asm__("incq %rdi"); __asm__("incq %rsi"); __asm__("decl %ecx"); __asm__("jnz 5b"); __asm__("6:"); } void main(void) { int i; test_init(); printf ("%23s", ""); printf ("\t%s\t%s\n", "memcpy_orig", "memcpy_new"); for(i = 0; i< 128;i= 1 + i) { do_test(0, 0, i); do_test(0, 8, i); do_test(4, 16, i); do_test(0, 16, i); } do_test(4, 16, 1024); do_test(0, 0, 1024); do_test(4, 16, 4096); do_test(0, 8, 4096); do_test(0, 16, 4096); do_test(0, 64, 4096); do_test(0, 0, 4096); do_test(16, 0, 4096*2); do_test(0, 16, 4096*2); return ; }