Date:   Tue, 13 Dec 2022 12:19:34 +0000
From:   Arseniy Krasnov <AVKrasnov@...rdevices.ru>
To:     Stefano Garzarella <sgarzare@...hat.com>
CC:     Jakub Kicinski <kuba@...nel.org>,
        "David S. Miller" <davem@...emloft.net>,
        Paolo Abeni <pabeni@...hat.com>,
        "edumazet@...gle.com" <edumazet@...gle.com>,
        Bobby Eshleman <bobby.eshleman@...edance.com>,
        Krasnov Arseniy <oxffffaa@...il.com>,
        "linux-kernel@...r.kernel.org" <linux-kernel@...r.kernel.org>,
        "netdev@...r.kernel.org" <netdev@...r.kernel.org>,
        "virtualization@...ts.linux-foundation.org" 
        <virtualization@...ts.linux-foundation.org>,
        kernel <kernel@...rdevices.ru>
Subject: Re: [RFC PATCH v4 4/4] test/vsock: vsock_perf utility

On 13.12.2022 14:17, Stefano Garzarella wrote:
> On Tue, Dec 06, 2022 at 08:54:28PM +0000, Arseniy Krasnov wrote:
>> This adds utility to check vsock rx/tx performance.
>>
>> Usage as sender:
>> ./vsock_perf --sender <cid> --port <port> --bytes <bytes to send)
> 
> Little typo "<bytes to send>". I mean replace `)` with `>`
> 
>> Usage as receiver:
>> ./vsock_perf --port <port> --rcvlowat <SO_RCVLOWAT>
>>
>> Signed-off-by: Arseniy Krasnov <AVKrasnov@...rdevices.ru>
>> ---
>> tools/testing/vsock/Makefile     |   3 +-
>> tools/testing/vsock/README       |  34 +++
>> tools/testing/vsock/vsock_perf.c | 434 +++++++++++++++++++++++++++++++
>> 3 files changed, 470 insertions(+), 1 deletion(-)
>> create mode 100644 tools/testing/vsock/vsock_perf.c
>>
>> diff --git a/tools/testing/vsock/Makefile b/tools/testing/vsock/Makefile
>> index f8293c6910c9..43a254f0e14d 100644
>> --- a/tools/testing/vsock/Makefile
>> +++ b/tools/testing/vsock/Makefile
>> @@ -1,8 +1,9 @@
>> # SPDX-License-Identifier: GPL-2.0-only
>> -all: test
>> +all: test vsock_perf
>> test: vsock_test vsock_diag_test
>> vsock_test: vsock_test.o timeout.o control.o util.o
>> vsock_diag_test: vsock_diag_test.o timeout.o control.o util.o
>> +vsock_perf: vsock_perf.o
>>
>> CFLAGS += -g -O2 -Werror -Wall -I. -I../../include -I../../../usr/include -Wno-pointer-sign -fno-strict-overflow -fno-strict-aliasing -fno-common -MMD -U_FORTIFY_SOURCE -D_GNU_SOURCE
>> .PHONY: all test clean
>> diff --git a/tools/testing/vsock/README b/tools/testing/vsock/README
>> index 4d5045e7d2c3..e6f6735bba05 100644
>> --- a/tools/testing/vsock/README
>> +++ b/tools/testing/vsock/README
>> @@ -35,3 +35,37 @@ Invoke test binaries in both directions as follows:
>>                        --control-port=$GUEST_IP \
>>                        --control-port=1234 \
>>                        --peer-cid=3
>> +
>> +vsock_perf utility
>> +-------------------
>> +'vsock_perf' is a simple tool to measure vsock performance. It works in
>> +sender/receiver modes: sender connect to peer at the specified port and
>> +starts data transmission to the receiver. After data processing is done,
>> +it prints several metrics(see below).
>> +
>> +Usage:
>> +# run as sender
>> +# connect to CID 2, port 1234, send 1G of data, tx buf size is 1M
>> +./vsock_perf --sender 2 --port 1234 --bytes 1G --buf-size 1M
>> +
>> +Output:
>> +tx performance: A Gb/s
>> +
>> +Output explanation:
>> +A is calculated as "number of bytes to send" / "time in tx loop"
>> +
>> +# run as receiver
>> +# listen port 1234, rx buf size is 1M, socket buf size is 1G, SO_RCVLOWAT is 64K
>> +./vsock_perf --port 1234 --buf-size 1M --vsk-size 1G --rcvlowat 64K
>> +
>> +Output:
>> +rx performance: A Gb/s
>> +total in 'read()': B sec
>> +POLLIN wakeups: C
>> +average in 'read()': D ns
>> +
>> +Output explanation:
>> +A is calculated as "number of received bytes" / "time in rx loop".
>> +B is time, spent in 'read()' system call(excluding 'poll()')
>> +C is number of 'poll()' wake ups with POLLIN bit set.
>> +D is B / C, e.g. average amount of time, spent in single 'read()'.
>> diff --git a/tools/testing/vsock/vsock_perf.c b/tools/testing/vsock/vsock_perf.c
>> new file mode 100644
>> index 000000000000..248fc3285d5c
>> --- /dev/null
>> +++ b/tools/testing/vsock/vsock_perf.c
>> @@ -0,0 +1,434 @@
>> +// SPDX-License-Identifier: GPL-2.0-only
>> +/*
>> + * vsock_perf - benchmark utility for vsock.
>> + *
>> + * Copyright (C) 2022 SberDevices.
>> + *
>> + * Author: Arseniy Krasnov <AVKrasnov@...rdevices.ru>
>> + */
>> +#include <getopt.h>
>> +#include <stdio.h>
>> +#include <stdlib.h>
>> +#include <stdbool.h>
>> +#include <string.h>
>> +#include <errno.h>
>> +#include <unistd.h>
>> +#include <time.h>
>> +#include <stdint.h>
>> +#include <poll.h>
>> +#include <sys/socket.h>
>> +#include <linux/vm_sockets.h>
>> +
>> +#define DEFAULT_BUF_SIZE_BYTES    (128 * 1024)
>> +#define DEFAULT_TO_SEND_BYTES    (64 * 1024)
>> +#define DEFAULT_VSOCK_BUF_BYTES (256 * 1024)
>> +#define DEFAULT_RCVLOWAT_BYTES    1
>> +#define DEFAULT_PORT        1234
>> +
>> +#define BYTES_PER_GB        (1024 * 1024 * 1024ULL)
>> +#define NSEC_PER_SEC        (1000000000ULL)
>> +
>> +static unsigned int port = DEFAULT_PORT;
>> +static unsigned long buf_size_bytes = DEFAULT_BUF_SIZE_BYTES;
>> +static unsigned long vsock_buf_bytes = DEFAULT_VSOCK_BUF_BYTES;
>> +
>> +static inline time_t current_nsec(void)
>> +{
>> +    struct timespec ts;
>> +
>> +    if (clock_gettime(CLOCK_REALTIME, &ts)) {
>> +        perror("clock_gettime");
>> +        exit(EXIT_FAILURE);
>> +    }
>> +
>> +    return (ts.tv_sec * NSEC_PER_SEC) + ts.tv_nsec;
>> +}
>> +
>> +/* From lib/cmdline.c. */
>> +static unsigned long memparse(const char *ptr)
>> +{
>> +    char *endptr;
>> +
>> +    unsigned long long ret = strtoull(ptr, &endptr, 0);
>> +
>> +    switch (*endptr) {
>> +    case 'E':
>> +    case 'e':
>> +        ret <<= 10;
>> +    case 'P':
>> +    case 'p':
>> +        ret <<= 10;
>> +    case 'T':
>> +    case 't':
>> +        ret <<= 10;
>> +    case 'G':
>> +    case 'g':
>> +        ret <<= 10;
>> +    case 'M':
>> +    case 'm':
>> +        ret <<= 10;
>> +    case 'K':
>> +    case 'k':
>> +        ret <<= 10;
>> +        endptr++;
>> +    default:
>> +        break;
>> +    }
>> +
>> +    return ret;
>> +}
>> +
>> +static void vsock_increase_buf_size(int fd)
>> +{
>> +    if (setsockopt(fd, AF_VSOCK, SO_VM_SOCKETS_BUFFER_MAX_SIZE,
>> +               &vsock_buf_bytes, sizeof(vsock_buf_bytes))) {
>> +        perror("setsockopt(SO_VM_SOCKETS_BUFFER_MAX_SIZE)");
>> +        exit(EXIT_FAILURE);
>> +    }
>> +
>> +    if (setsockopt(fd, AF_VSOCK, SO_VM_SOCKETS_BUFFER_SIZE,
>> +               &vsock_buf_bytes, sizeof(vsock_buf_bytes))) {
>> +        perror("setsockopt(SO_VM_SOCKETS_BUFFER_SIZE)");
>> +        exit(EXIT_FAILURE);
>> +    }
>> +}
>> +
>> +static int vsock_connect(unsigned int cid, unsigned int port)
>> +{
>> +    union {
>> +        struct sockaddr sa;
>> +        struct sockaddr_vm svm;
>> +    } addr = {
>> +        .svm = {
>> +            .svm_family = AF_VSOCK,
>> +            .svm_port = port,
>> +            .svm_cid = cid,
>> +        },
>> +    };
>> +    int fd;
>> +
>> +    fd = socket(AF_VSOCK, SOCK_STREAM, 0);
>> +
>> +    if (fd < 0) {
>> +        perror("socket");
>> +        return -1;
>> +    }
>> +
>> +    if (connect(fd, &addr.sa, sizeof(addr.svm)) < 0) {
>> +        perror("connect");
>> +        close(fd);
>> +        return -1;
>> +    }
>> +
>> +    return fd;
>> +}
>> +
>> +static float get_gbps(unsigned long bytes, time_t ns_delta)
>> +{
>> +    return ((float)bytes / BYTES_PER_GB) /
>> +           ((float)ns_delta / NSEC_PER_SEC);
>> +}
>> +
>> +static void run_receiver(unsigned long rcvlowat_bytes)
>> +{
>> +    unsigned int read_cnt;
>> +    time_t rx_begin_ns;
>> +    time_t in_read_ns;
>> +    size_t total_recv;
>> +    int client_fd;
>> +    char *data;
>> +    int fd;
>> +    union {
>> +        struct sockaddr sa;
>> +        struct sockaddr_vm svm;
>> +    } addr = {
>> +        .svm = {
>> +            .svm_family = AF_VSOCK,
>> +            .svm_port = port,
>> +            .svm_cid = VMADDR_CID_ANY,
>> +        },
>> +    };
>> +    union {
>> +        struct sockaddr sa;
>> +        struct sockaddr_vm svm;
>> +    } clientaddr;
>> +
>> +    socklen_t clientaddr_len = sizeof(clientaddr.svm);
>> +
>> +    printf("Run as receiver\n");
>> +    printf("Listen port %u\n", port);
>> +    printf("RX buffer %lu bytes\n", buf_size_bytes);
>> +    printf("vsock buffer %lu bytes\n", vsock_buf_bytes);
>> +    printf("SO_RCVLOWAT %lu bytes\n", rcvlowat_bytes);
>> +
>> +    fd = socket(AF_VSOCK, SOCK_STREAM, 0);
>> +
>> +    if (fd < 0) {
>> +        perror("socket");
>> +        exit(EXIT_FAILURE);
>> +    }
>> +
>> +    if (bind(fd, &addr.sa, sizeof(addr.svm)) < 0) {
>> +        perror("bind");
>> +        exit(EXIT_FAILURE);
>> +    }
>> +
>> +    if (listen(fd, 1) < 0) {
>> +        perror("listen");
>> +        exit(EXIT_FAILURE);
>> +    }
>> +
>> +    client_fd = accept(fd, &clientaddr.sa, &clientaddr_len);
>> +
>> +    if (client_fd < 0) {
>> +        perror("accept");
>> +        exit(EXIT_FAILURE);
>> +    }
>> +
>> +    vsock_increase_buf_size(client_fd);
>> +
>> +    if (setsockopt(client_fd, SOL_SOCKET, SO_RCVLOWAT,
>> +               &rcvlowat_bytes,
>> +               sizeof(rcvlowat_bytes))) {
>> +        perror("setsockopt(SO_RCVLOWAT)");
>> +        exit(EXIT_FAILURE);
>> +    }
>> +
>> +    data = malloc(buf_size_bytes);
>> +
>> +    if (!data) {
>> +        printf("malloc failed\n");
> 
> This is an error message, so we can use fprintf(stderr, ...)
> 
>> +        exit(EXIT_FAILURE);
>> +    }
>> +
>> +    read_cnt = 0;
>> +    in_read_ns = 0;
>> +    total_recv = 0;
>> +    rx_begin_ns = current_nsec();
>> +
>> +    while (1) {
>> +        struct pollfd fds = { 0 };
>> +
>> +        fds.fd = client_fd;
>> +        fds.events = POLLIN | POLLERR |
>> +                 POLLHUP | POLLRDHUP;
>> +
>> +        if (poll(&fds, 1, -1) < 0) {
>> +            perror("poll");
>> +            exit(EXIT_FAILURE);
>> +        }
>> +
>> +        if (fds.revents & POLLERR) {
>> +            printf("'poll()' error\n");
> 
> Ditto.
> 
>> +            exit(EXIT_FAILURE);
>> +        }
>> +
>> +        if (fds.revents & POLLIN) {
>> +            ssize_t bytes_read;
>> +            time_t t;
>> +
>> +            t = current_nsec();
>> +            bytes_read = read(fds.fd, data, buf_size_bytes);
>> +            in_read_ns += (current_nsec() - t);
>> +            read_cnt++;
>> +
>> +            if (!bytes_read)
>> +                break;
>> +
>> +            if (bytes_read < 0) {
>> +                perror("read");
>> +                exit(EXIT_FAILURE);
>> +            }
>> +
>> +            total_recv += bytes_read;
>> +        }
>> +
>> +        if (fds.revents & (POLLHUP | POLLRDHUP))
>> +            break;
>> +    }
>> +
>> +    printf("rx performance: %f Gb/s\n",
>> +           get_gbps(total_recv, current_nsec() - rx_begin_ns));
>> +    printf("total time in 'read()': %f sec\n", (float)in_read_ns / NSEC_PER_SEC);
>> +    printf("POLLIN wakeups: %i\n", read_cnt);
>> +    printf("average time in 'read()': %f ns\n", (float)in_read_ns / read_cnt);
>> +
>> +    free(data);
>> +    close(client_fd);
>> +    close(fd);
>> +}
>> +
>> +static void run_sender(int peer_cid, unsigned long to_send_bytes)
>> +{
>> +    time_t tx_begin_ns;
>> +    size_t total_send;
>> +    void *data;
>> +    int fd;
>> +
>> +    printf("Run as sender\n");
>> +    printf("Connect to %i:%u\n", peer_cid, port);
>> +    printf("Send %lu bytes\n", to_send_bytes);
>> +    printf("TX buffer %lu bytes\n", buf_size_bytes);
>> +
>> +    fd = vsock_connect(peer_cid, port);
>> +
>> +    if (fd < 0)
>> +        exit(EXIT_FAILURE);
>> +
>> +    data = malloc(buf_size_bytes);
>> +
>> +    if (!data) {
>> +        printf("malloc failed\n");
> 
> Ditto.
> 
>> +        exit(EXIT_FAILURE);
>> +    }
>> +
>> +    memset(data, 0, buf_size_bytes);
>> +    total_send = 0;
>> +    tx_begin_ns = current_nsec();
>> +
>> +    while (total_send < to_send_bytes) {
>> +        ssize_t sent;
>> +
>> +        sent = write(fd, data, buf_size_bytes);
>> +
>> +        if (sent <= 0) {
>> +            perror("write");
>> +            exit(EXIT_FAILURE);
>> +        }
>> +
>> +        total_send += sent;
>> +    }
> 
> I don't know if it would be useful, but maybe we can also print the total number of bytes and the time spent (the same in the receiver as well).
> 
> Here I would add
> 
>         tx_end_ns = current_nsec();
> 
> Then use it later.
> 
>> +
>> +    printf("tx performance: %f Gb/s\n",
>> +           get_gbps(total_send, current_nsec() - tx_begin_ns));
>> +
> 
> Did you compared the performance with iperf-vsock and uperf?
> 
> I see an order of magnitude difference but I did not have time to check all the parameters to make a fair comparison. It would be helpful if you can compare and report it.
Thanks for review,

Sounds interestring, i'll check it and post results in cv of the next version(along with minor changes in patchset).

Thank You
> 
> Thanks,
> Stefano
> 
>> +    close(fd);
>> +    free(data);
>> +}
>> +
>> +static const char optstring[] = "";
>> +static const struct option longopts[] = {
>> +    {
>> +        .name = "help",
>> +        .has_arg = no_argument,
>> +        .val = 'H',
>> +    },
>> +    {
>> +        .name = "sender",
>> +        .has_arg = required_argument,
>> +        .val = 'S',
>> +    },
>> +    {
>> +        .name = "port",
>> +        .has_arg = required_argument,
>> +        .val = 'P',
>> +    },
>> +    {
>> +        .name = "bytes",
>> +        .has_arg = required_argument,
>> +        .val = 'M',
>> +    },
>> +    {
>> +        .name = "buf-size",
>> +        .has_arg = required_argument,
>> +        .val = 'B',
>> +    },
>> +    {
>> +        .name = "vsk-size",
>> +        .has_arg = required_argument,
>> +        .val = 'V',
>> +    },
>> +    {
>> +        .name = "rcvlowat",
>> +        .has_arg = required_argument,
>> +        .val = 'R',
>> +    },
>> +    {},
>> +};
>> +
>> +static void usage(void)
>> +{
>> +    printf("Usage: ./vsock_perf [--help] [options]\n"
>> +           "\n"
>> +           "This is benchmarking utility, to test vsock performance.\n"
>> +           "It runs in two modes: sender or receiver. In sender mode, it\n"
>> +           "connects to the specified CID and starts data transmission.\n"
>> +           "\n"
>> +           "Options:\n"
>> +           "  --help            This message\n"
>> +           "  --sender   <cid>        Sender mode (receiver default)\n"
>> +           "                                <cid> of the receiver to connect to\n"
>> +           "  --port     <port>        Port (default %d)\n"
>> +           "  --bytes    <bytes>KMG        Bytes to send (default %d)\n"
>> +           "  --buf-size <bytes>KMG        Data buffer size (default %d). In sender mode\n"
>> +           "                                it is the buffer size, passed to 'write()'. In\n"
>> +           "                                receiver mode it is the buffer size passed to 'read()'.\n"
>> +           "  --vsk-size <bytes>KMG        Socket buffer size (default %d)\n"
>> +           "  --rcvlowat <bytes>KMG        SO_RCVLOWAT value (default %d)\n"
>> +           "\n", DEFAULT_PORT, DEFAULT_TO_SEND_BYTES,
>> +           DEFAULT_BUF_SIZE_BYTES, DEFAULT_VSOCK_BUF_BYTES,
>> +           DEFAULT_RCVLOWAT_BYTES);
>> +    exit(EXIT_FAILURE);
>> +}
>> +
>> +static long strtolx(const char *arg)
>> +{
>> +    long value;
>> +    char *end;
>> +
>> +    value = strtol(arg, &end, 10);
>> +
>> +    if (end != arg + strlen(arg))
>> +        usage();
>> +
>> +    return value;
>> +}
>> +
>> +int main(int argc, char **argv)
>> +{
>> +    unsigned long to_send_bytes = DEFAULT_TO_SEND_BYTES;
>> +    unsigned long rcvlowat_bytes = DEFAULT_RCVLOWAT_BYTES;
>> +    int peer_cid = -1;
>> +    bool sender = false;
>> +
>> +    while (1) {
>> +        int opt = getopt_long(argc, argv, optstring, longopts, NULL);
>> +
>> +        if (opt == -1)
>> +            break;
>> +
>> +        switch (opt) {
>> +        case 'V': /* Peer buffer size. */
>> +            vsock_buf_bytes = memparse(optarg);
>> +            break;
>> +        case 'R': /* SO_RCVLOWAT value. */
>> +            rcvlowat_bytes = memparse(optarg);
>> +            break;
>> +        case 'P': /* Port to connect to. */
>> +            port = strtolx(optarg);
>> +            break;
>> +        case 'M': /* Bytes to send. */
>> +            to_send_bytes = memparse(optarg);
>> +            break;
>> +        case 'B': /* Size of rx/tx buffer. */
>> +            buf_size_bytes = memparse(optarg);
>> +            break;
>> +        case 'S': /* Sender mode. CID to connect to. */
>> +            peer_cid = strtolx(optarg);
>> +            sender = true;
>> +            break;
>> +        case 'H': /* Help. */
>> +            usage();
>> +            break;
>> +        default:
>> +            usage();
>> +        }
>> +    }
>> +
>> +    if (!sender)
>> +        run_receiver(rcvlowat_bytes);
>> +    else
>> +        run_sender(peer_cid, to_send_bytes);
>> +
>> +    return 0;
>> +}
>> -- 
>> 2.25.1
> 

Powered by blists - more mailing lists