function test_src:long(type:long, off:long, len:long, zero:long)
%{
        unsigned char *ptr;
        unsigned long addr;
        int ret = 0, cu_ret = 0, i;
        printk("testing src with '%s', off = %lld, len = %lld, zero tail = %lld\n", THIS->type ? "string" : "unrolled", THIS->off, THIS->len, THIS->zero);

        // vmalloc makes a hole after new vma
        ptr = vmalloc(PAGE_SIZE);
        printk("ptr = %p\n", ptr);

        // poison memory
        memset(ptr, 0xaa, PAGE_SIZE >> 1);

        // fill memory with pattern
        for (i = 0; i < PAGE_SIZE >> 1; ++i)
                ptr[i + (PAGE_SIZE >> 1)] = i & 0xff;

        // raise GPF

        addr = THIS->type ? _STRING_ : _UNROLLED_;
        asm ("call %%rax;"
             : "=a"(cu_ret)
             : "S"(ptr + PAGE_SIZE - THIS->off), "D" (ptr), "d"(THIS->len), "c"(THIS->zero),
               "a"(addr)
                );

        if (cu_ret)
                printk("copy_user GPF (%d bytes uncopied)\n", cu_ret);

        // dump
        for (i = 0; i < 256; ++i) {
                if (!(i % 8))
                        printk("\n%04x: ", i);
                printk("%02x ", ptr[i]);
        }

        printk("\n");

        // check for bug in copy_user_generic_unrolled
        for (i = 0; i < (THIS->off > THIS->len ? THIS->len : THIS->off) && !(ret & 1); ++i)
                if  (ptr[i] != ptr[i + PAGE_SIZE - THIS->off])
                        ret |= 1;

        if (cu_ret) {
                if (THIS->len - THIS->off != cu_ret)
                        ret |= 4;

                if (THIS->zero)
                // check for bug in copy_user_generic_c/string
                for (i = 0; i < cu_ret && !(ret & 2); ++i)
                        if (ptr[THIS->off + i])
                                ret |= 2;
        }
        vfree(ptr);
        THIS->__retvalue = ret;
%}

function test_dst:long(type:long, off:long, len:long, zero:long)
%{
        unsigned char *ptr;
        unsigned long addr;
        int ret = 0, cu_ret = 0, i;
        printk("testing dst with '%s', off = %lld, len = %lld, zero tail = %lld\n", THIS->type ? "string" : "unrolled", THIS->off, THIS->len, THIS->zero);

        // vmalloc makes a hole after new vma
        ptr = vmalloc(PAGE_SIZE);
        printk("ptr = %p\n", ptr);

        // poison memory
        memset(ptr + (PAGE_SIZE >> 1), 0xaa, PAGE_SIZE >> 1);

        // fill memory with pattern
        for (i = 0; i < PAGE_SIZE >> 1; ++i)
                ptr[i] = i & 0xff;

        // raise GPF

        addr = THIS->type ? _STRING_ : _UNROLLED_;
        asm ("call %%rax;"
             : "=a"(cu_ret)
             : "S"(ptr), "D" (ptr + PAGE_SIZE - THIS->off), "d"(THIS->len), "c"(THIS->zero),
               "a"(addr)
                );

        if (cu_ret)
                printk("copy_user GPF (%d bytes uncopied)\n", cu_ret);

        // dump
        for (i = PAGE_SIZE - 256; i < PAGE_SIZE; ++i) {
                if (!(i % 8))
                        printk("\n%04x: ", i);
                printk("%02x ", ptr[i]);
        }

        printk("\n");

        for (i = 0; i < (THIS->off > THIS->len ? THIS->len : THIS->off) && !(ret & 1); ++i)
                if (ptr[i] != ptr[PAGE_SIZE - THIS->off + i])
                        ret |= 1;

        vfree(ptr);
        THIS->__retvalue = ret;
%}

function run_test:long(type:long, off:long, len:long, zero:long)
{
        m = test_src(type, off, len, zero);
        printf("testing src '%s', off = %d, len = %d, zero tail = %d, buggy: %s\n", type ? "string" : "unrolled", off, len, zero, m ? "yes" : "no");
        if (m) {
                if (m & 1)
                        printf("copy_user BUG #0: valid data was not stored\n");
                if (m & 2)
                        printf("copy_user BUG #1: tail not zeroed\n");
                if (m & 4)
                        printf("copy_user BUG #2: uncopied bytes miscalculation\n");
        }
        n = test_dst(type, off, len, zero);
        printf("testing dst '%s', off = %d, len = %d, zero tail = %d, buggy: %s\n", type ? "string" : "unrolled", off, len, zero, n ? "yes" : "no");
        if (n) {
                if (n & 1)
                        printf("copy_user BUG #3: copied data unequal\n");
        }
        return m + n;
}

%( kernel_v <= "2.6.18" %?
        function get_feature:long()
        %{
                 THIS->__retvalue = boot_cpu_has(3*32+ 4); // X86_FEATURE_K8_C in RHEL-4 or X86_FEATURE_REP_GOOD in RHEL-5
        %}

        %:

        function get_feature:long()
        %{
                 THIS->__retvalue = boot_cpu_has(3*32+ 16); // X86_FEATURE_REP_GOOD in upstream
        %}
%)

probe begin
{
        printf("begin\n");
        printf("kernel uses '%s' version of copy_user\n", get_feature() ? "string" : "unrolled");
        i = 0;

        i += run_test(0, 56, 3, 1); // zero tail
        i += run_test(1, 56, 3, 1); // zero tail
        i += run_test(0, 56, 3, 0);
        i += run_test(1, 56, 3, 0);

        i += run_test(0, 56, 128, 1); // unrolled loop, zero tail
        i += run_test(1, 56, 128, 1); // movsq, zero tail
        i += run_test(0, 56, 128, 0); // unrolled loop
        i += run_test(1, 56, 128, 0); // movsq

        i += run_test(0, 9, 128, 1);  // unrolled loop, zero tail
        i += run_test(1, 9, 128, 1);  // movsq, zero tail
        i += run_test(0, 9, 128, 0);  // unrolled loop
        i += run_test(1, 9, 128, 0);  // movsq

        i += run_test(0, 9, 9, 1);    // quad word loop + byte loop, zero tail
        i += run_test(1, 9, 9, 1);    // movsq + movsb, zero tail
        i += run_test(0, 9, 9, 0);    // quad word loop + byte loop
        i += run_test(1, 9, 9, 0);    // movsq + movsb

        i += run_test(0, 8, 9, 1);    // quad word loop + byte loop, zero tail
        i += run_test(1, 8, 9, 1);    // movsq + movsb, zero tail
        i += run_test(0, 8, 9, 0);    // quad word loop + byte loop
        i += run_test(1, 8, 9, 0);    // movsq + movsb

        i += run_test(0, 1, 8, 1);    // quad word loop, zero tail
        i += run_test(1, 1, 8, 1);    // movsq, zero tail
        i += run_test(0, 1, 8, 0);    // quad word loop
        i += run_test(1, 1, 8, 0);    // movsq

        i += run_test(0, 0, 1, 1);    // byte loop, zero tail
        i += run_test(1, 0, 1, 1);    // movsb, zero tail
        i += run_test(0, 0, 1, 0);    // byte loop
        i += run_test(1, 0, 1, 0);    // movsb

        i += run_test(0, 1, 1, 1);    // byte loop, zero tail
        i += run_test(1, 1, 1, 1);    // movsb, zero tail
        i += run_test(0, 1, 1, 0);    // byte loop
        i += run_test(1, 1, 1, 0);    // movsb

        i += run_test(0, 1, 0, 1);    // sanity check
        i += run_test(1, 1, 0, 1);    // sanity check
        i += run_test(0, 1, 0, 0);    // sanity check
        i += run_test(1, 1, 0, 0);    // sanity check

        i += run_test(0, 0, 0, 1);    // sanity check
        i += run_test(1, 0, 0, 1);    // sanity check
        i += run_test(0, 0, 0, 0);    // sanity check
        i += run_test(1, 0, 0, 0);    // sanity check

        if (i) {
                printf("copy_user is buggy\n");
        } else {
                printf("copy user is sane\n");
        }
        printf("end\n");
        exit();
}