Message-ID: <a89d3362-ced4-2851-05ce-98028f558b2f@iogearbox.net>
Date:   Wed, 7 Nov 2018 22:44:05 +0100
From:   Daniel Borkmann <daniel@...earbox.net>
To:     John Fastabend <john.fastabend@...il.com>,
        Shannon Nelson <shannon.nelson@...cle.com>, ast@...nel.org
Cc:     netdev@...r.kernel.org, eric.dumazet@...il.com,
        silviu.smarandache@...cle.com, shannon.lee.nelson@...il.com
Subject: Re: [PATCH bpf-next] xdp: sample code for redirecting vlan packets to
 specific cpus

On 10/29/2018 11:11 PM, John Fastabend wrote:
> On 10/29/2018 02:19 PM, Shannon Nelson wrote:
>> This is an example of using XDP to redirect the processing of
>> particular vlan packets to specific CPUs.  This is in response
>> to comments received on a kernel patch put forth previously
>> to do something similar using RPS.
>>      https://www.spinics.net/lists/netdev/msg528210.html
>>      [PATCH net-next] net: enable RPS on vlan devices
>>
>> This XDP application watches for inbound vlan-tagged packets
>> and redirects those packets to be processed on a specific CPU
>> as configured in a BPF map.  The BPF map can be modified by
>> this user program, which can also load and unload the kernel
>> XDP code.
>>
>> One example use is for supporting VMs where we can't control the
>> OS being used: we'd like to separate the VM CPU processing from
>> the host's CPUs as a way to help mitigate L1TF related issues.
>> When running the VM's traffic on a vlan we can stick the host's
>> Rx processing on one set of CPUs separate from the VM's CPUs.
>>
>> This example currently uses a vlan key and cpu value in the
>> BPF map, so only can do one CPU per vlan.  This could easily
>> be modified to use a bitpattern of CPUs rather than a CPU id
>> to allow multiple CPUs per vlan.
> 
> Great, so does this solve your use case then? At least on drivers
> with XDP support?
> 
>>
>> Signed-off-by: Shannon Nelson <shannon.nelson@...cle.com>
>> ---
> 
> Some really small and trivial nits below.
> 
> Acked-by: John Fastabend <john.fastabend@...il.com>
> 
> [...]
> 
>> +	if (install) {
>> +
> 
> new line probably not needed. 
> 
>> +		/* check to see if already installed */
>> +		errno = 0;
>> +		access(pin_prog_name, R_OK);
>> +		if (errno != ENOENT) {
>> +			fprintf(stderr, "ERR: %s is already installed\n", argv[0]);
>> +			return -1;
>> +		}
>> +
>> +		/* load the XDP program and maps with the convenient library */
>> +		if (load_bpf_file(filename)) {
>> +			fprintf(stderr, "ERR: load_bpf_file(%s): \n%s",
>> +				filename, bpf_log_buf);
>> +			return -1;
>> +		}
>> +		if (!prog_fd[0]) {
>> +			fprintf(stderr, "ERR: load_bpf_file(%s): %d %s\n",
>> +				filename, errno, strerror(errno));
>> +			return -1;
>> +		}
>> +
>> +		/* pin the XDP program and maps */
>> +		if (bpf_obj_pin(prog_fd[0], pin_prog_name) < 0) {
>> +			fprintf(stderr, "ERR: bpf_obj_pin(%s): %d %s\n",
>> +				pin_prog_name, errno, strerror(errno));
>> +			if (errno == 2)
>> +				fprintf(stderr, "     (is the BPF fs mounted on /sys/fs/bpf?)\n");
>> +			return -1;
>> +		}
>> +		if (bpf_obj_pin(map_fd[0], pin_vlanmap_name) < 0) {
>> +			fprintf(stderr, "ERR: bpf_obj_pin(%s): %d %s\n",
>> +				pin_vlanmap_name, errno, strerror(errno));
>> +			return -1;
>> +		}
>> +		if (bpf_obj_pin(map_fd[2], pin_countermap_name) < 0) {
>> +			fprintf(stderr, "ERR: bpf_obj_pin(%s): %d %s\n",
>> +				pin_countermap_name, errno, strerror(errno));
>> +			return -1;
>> +		}
>> +
>> +		/* prep the vlan map with "not used" values */
>> +		c64 = UNDEF_CPU;
>> +		for (v64 = 0; v64 < 4096; v64++) {
> 
> maybe #define MAX_VLANS 4096 just to avoid constants.
> 
>> +			if (bpf_map_update_elem(map_fd[0], &v64, &c64, 0)) {
>> +				fprintf(stderr, "ERR: preping vlan map failed on v=%llu: %d %s\n",
>> +					v64, errno, strerror(errno));
>> +				return -1;
>> +			}
>> +		}
>> +
>> +		/* prep the cpumap with queue sizes */
>> +		c64 = 128+64;  /* see note in xdp_redirect_cpu_user.c */
>> +		for (v64 = 0; v64 < MAX_CPUS; v64++) {
>> +			if (bpf_map_update_elem(map_fd[1], &v64, &c64, 0)) {
>> +				if (errno == ENODEV) {
>> +					/* Save the last CPU number attempted
>> +					 * into the counters map
>> +					 */
>> +					c64 = CPU_COUNT;
>> +					ret = bpf_map_update_elem(map_fd[2], &c64, &v64, 0);
>> +					break;
>> +				}
>> +
>> +				fprintf(stderr, "ERR: preping cpu map failed on v=%llu: %d %s\n",
>> +					v64, errno, strerror(errno));
>> +				return -1;
>> +			}
>> +		}
>> +
>> +		/* wire the XDP program to the device */
>> +		if (bpf_set_link_xdp_fd(ifindex, prog_fd[0], 0) < 0) {
>> +			fprintf(stderr, "ERR: bpf_set_link_xdp_fd(): %d %s\n",
>> +				errno, strerror(errno));
>> +			return -1;
>> +		}
>> +
>> +		return 0;
>> +	}
>> +
>> +	if (remove) {
>> +

Ditto

>> +		/* unlink the program from the device */
>> +		if (bpf_set_link_xdp_fd(ifindex, -1, 0) < 0)
>> +			fprintf(stderr, "ERR: bpf_set_link_xdp_fd(): %d %s\n",
>> +				errno, strerror(errno));
>> +
>> +		/* unlink pinned files */
>> +		if (unlink(pin_prog_name))
>> +			fprintf(stderr, "ERR: unlink(%s): %d %s\n",
>> +				pin_prog_name, errno, strerror(errno));
>> +		if (unlink(pin_vlanmap_name))
>> +			fprintf(stderr, "ERR: unlink(%s): %d %s\n",
>> +				pin_vlanmap_name, errno, strerror(errno));
>> +		if (unlink(pin_countermap_name))
>> +			fprintf(stderr, "ERR: unlink(%s): %d %s\n",
>> +				pin_countermap_name, errno, strerror(errno));
>> +
>> +		return 0;
>> +	}
>> +
>> +	if (vlan == 0) {
>> +		fprintf(stderr, "ERR: required option --vlan missing\n");
>> +		goto error;
>> +	}
>> +
>> +	if (cpu == MAX_CPUS && vlan > 0) {
>> +		fprintf(stderr, "ERR: required option --cpu missing\n");
>> +		goto error;
>> +	}
>> +
>> +	vfd = bpf_obj_get(pin_vlanmap_name);
>> +	if (vfd < 0) {
>> +		fprintf(stderr, "ERR: can't find pinned map %s: %d %s\n",
>> +			pin_vlanmap_name, errno, strerror(errno));
>> +		if (errno == ENOENT)
>> +			fprintf(stderr, "   (has %s been installed yet?)\n", argv[0]);
>> +		return -1;
>> +	}
>> +
>> +	/* decode the requested action */
>> +	if (vlan > 0) {
>> +		/* check cpu against the max value found */
>> +		cfd = bpf_obj_get(pin_countermap_name);
>> +		if (cfd < 0) {
>> +			fprintf(stderr, "ERR: can't find pinned map %s: %d %s\n",
>> +				pin_countermap_name, errno, strerror(errno));
>> +			return -1;
>> +		}
>> +		c64 = CPU_COUNT;
>> +		ret = bpf_map_lookup_elem(cfd, &c64, &v64);
>> +		if (cpu >= v64) {
> 
> Need to check ret code?

Yeah, I expect this would have another respin, thanks!

>> +			fprintf(stderr, "ERR: cpu %d greater than max %llu\n", cpu, v64);
>> +			return -1;
>> +		}
>> +
>> +		/* Note that the value and key pointers really do need to be
>> +		 * pointers to 64-bit values, else things get a bit muddled.
>> +		 */
>> +		v64 = vlan;
>> +		c64 = cpu;
>> +		ret = bpf_map_update_elem(vfd, &v64, &c64, 0);
>> +		if (ret) {
>> +			fprintf(stderr, "Adding vlan %d CPU %d failed: %d %s\n",
>> +				vlan, cpu, errno, strerror(errno));
>> +			return -1;
>> +		}
>> +
>> +	} else {
>> +		v64 = -vlan;
>> +		c64 = UNDEF_CPU;
>> +
>> +		/* We can't actually delete from a TYPE_ARRAY map, so we
>> +		 * simply set it to an undefined value.
>> +		 */
>> +		ret = bpf_map_update_elem(vfd, &v64, &c64, 0);
>> +		if (ret) {
>> +			fprintf(stderr, "Delete of vlan %llu failed: %d %s\n",
>> +				v64, errno, strerror(errno));
>> +			return -1;
>> +		}
>> +	}
>> +
>> +	return 0;
>> +
>> +error:
>> +	usage(argv);
>> +	return -1;
>> +}
>>
> 

Powered by blists - more mailing lists