lists.openwall.net   lists  /  announce  owl-users  owl-dev  john-users  john-dev  passwdqc-users  yescrypt  popa3d-users  /  oss-security  kernel-hardening  musl  sabotage  tlsify  passwords  /  crypt-dev  xvendor  /  Bugtraq  Full-Disclosure  linux-kernel  linux-netdev  linux-ext4  linux-hardening  linux-cve-announce  PHC 
Open Source and information security mailing list archives
 
Hash Suite: Windows password security audit tool. GUI, reports in PDF.
[<prev] [next>] [thread-next>] [day] [month] [year] [list]
Message-Id: <01cd8afa22786b2c8a4cd7250d165741e990a771.1618927173.git.lorenzo@kernel.org>
Date:   Tue, 20 Apr 2021 16:05:14 +0200
From:   Lorenzo Bianconi <lorenzo@...nel.org>
To:     bpf@...r.kernel.org
Cc:     netdev@...r.kernel.org, lorenzo.bianconi@...hat.com,
        davem@...emloft.net, kuba@...nel.org, ast@...nel.org,
        daniel@...earbox.net, brouer@...hat.com, song@...nel.org,
        toke@...hat.com
Subject: [PATCH v3 bpf-next] cpumap: bulk skb using netif_receive_skb_list

Rely on netif_receive_skb_list routine to send skbs converted from
xdp_frames in cpu_map_kthread_run in order to improve i-cache usage.
The proposed patch has been tested running xdp_redirect_cpu bpf sample
available in the kernel tree that is used to redirect UDP frames from
ixgbe driver to a cpumap entry and then to the networking stack.
UDP frames are generated using pkt_gen. Packets are discarded by the
UDP layer.

$xdp_redirect_cpu  --cpu <cpu> --progname xdp_cpu_map0 --dev <eth>

bpf-next: ~2.35Mpps
bpf-next + cpumap skb-list: ~2.72Mpps

Since netif_receive_skb_list does not return number of discarded packets,
remove drop counter from xdp_cpumap_kthread tracepoint and update related
xdp samples.

Signed-off-by: Lorenzo Bianconi <lorenzo@...nel.org>
---
Changes since v2:
- remove drop counter and update related xdp samples
- rebased on top of bpf-next

Changes since v1:
- fixed comment
- rebased on top of bpf-next tree
---
 include/trace/events/xdp.h          | 14 +++++---------
 kernel/bpf/cpumap.c                 | 16 +++++++---------
 samples/bpf/xdp_monitor_kern.c      |  6 ++----
 samples/bpf/xdp_monitor_user.c      | 14 ++++++--------
 samples/bpf/xdp_redirect_cpu_kern.c | 12 +++++-------
 samples/bpf/xdp_redirect_cpu_user.c | 10 ++++------
 6 files changed, 29 insertions(+), 43 deletions(-)

diff --git a/include/trace/events/xdp.h b/include/trace/events/xdp.h
index fcad3645a70b..52ecfe9c7e25 100644
--- a/include/trace/events/xdp.h
+++ b/include/trace/events/xdp.h
@@ -184,16 +184,15 @@ DEFINE_EVENT(xdp_redirect_template, xdp_redirect_map_err,
 
 TRACE_EVENT(xdp_cpumap_kthread,
 
-	TP_PROTO(int map_id, unsigned int processed,  unsigned int drops,
-		 int sched, struct xdp_cpumap_stats *xdp_stats),
+	TP_PROTO(int map_id, unsigned int processed, int sched,
+		 struct xdp_cpumap_stats *xdp_stats),
 
-	TP_ARGS(map_id, processed, drops, sched, xdp_stats),
+	TP_ARGS(map_id, processed, sched, xdp_stats),
 
 	TP_STRUCT__entry(
 		__field(int, map_id)
 		__field(u32, act)
 		__field(int, cpu)
-		__field(unsigned int, drops)
 		__field(unsigned int, processed)
 		__field(int, sched)
 		__field(unsigned int, xdp_pass)
@@ -205,7 +204,6 @@ TRACE_EVENT(xdp_cpumap_kthread,
 		__entry->map_id		= map_id;
 		__entry->act		= XDP_REDIRECT;
 		__entry->cpu		= smp_processor_id();
-		__entry->drops		= drops;
 		__entry->processed	= processed;
 		__entry->sched	= sched;
 		__entry->xdp_pass	= xdp_stats->pass;
@@ -215,13 +213,11 @@ TRACE_EVENT(xdp_cpumap_kthread,
 
 	TP_printk("kthread"
 		  " cpu=%d map_id=%d action=%s"
-		  " processed=%u drops=%u"
-		  " sched=%d"
+		  " processed=%u sched=%u"
 		  " xdp_pass=%u xdp_drop=%u xdp_redirect=%u",
 		  __entry->cpu, __entry->map_id,
 		  __print_symbolic(__entry->act, __XDP_ACT_SYM_TAB),
-		  __entry->processed, __entry->drops,
-		  __entry->sched,
+		  __entry->processed, __entry->sched,
 		  __entry->xdp_pass, __entry->xdp_drop, __entry->xdp_redirect)
 );
 
diff --git a/kernel/bpf/cpumap.c b/kernel/bpf/cpumap.c
index 0cf2791d5099..1c22caadf78f 100644
--- a/kernel/bpf/cpumap.c
+++ b/kernel/bpf/cpumap.c
@@ -27,7 +27,7 @@
 #include <linux/capability.h>
 #include <trace/events/xdp.h>
 
-#include <linux/netdevice.h>   /* netif_receive_skb_core */
+#include <linux/netdevice.h>   /* netif_receive_skb_list */
 #include <linux/etherdevice.h> /* eth_type_trans */
 
 /* General idea: XDP packets getting XDP redirected to another CPU,
@@ -253,10 +253,11 @@ static int cpu_map_kthread_run(void *data)
 	while (!kthread_should_stop() || !__ptr_ring_empty(rcpu->queue)) {
 		struct xdp_cpumap_stats stats = {}; /* zero stats */
 		gfp_t gfp = __GFP_ZERO | GFP_ATOMIC;
-		unsigned int drops = 0, sched = 0;
 		void *frames[CPUMAP_BATCH];
 		void *skbs[CPUMAP_BATCH];
+		unsigned int sched = 0;
 		int i, n, m, nframes;
+		LIST_HEAD(list);
 
 		/* Release CPU reschedule checks */
 		if (__ptr_ring_empty(rcpu->queue)) {
@@ -297,7 +298,6 @@ static int cpu_map_kthread_run(void *data)
 			if (unlikely(m == 0)) {
 				for (i = 0; i < nframes; i++)
 					skbs[i] = NULL; /* effect: xdp_return_frame */
-				drops += nframes;
 			}
 		}
 
@@ -305,7 +305,6 @@ static int cpu_map_kthread_run(void *data)
 		for (i = 0; i < nframes; i++) {
 			struct xdp_frame *xdpf = frames[i];
 			struct sk_buff *skb = skbs[i];
-			int ret;
 
 			skb = __xdp_build_skb_from_frame(xdpf, skb,
 							 xdpf->dev_rx);
@@ -314,13 +313,12 @@ static int cpu_map_kthread_run(void *data)
 				continue;
 			}
 
-			/* Inject into network stack */
-			ret = netif_receive_skb_core(skb);
-			if (ret == NET_RX_DROP)
-				drops++;
+			list_add_tail(&skb->list, &list);
 		}
+		netif_receive_skb_list(&list);
+
 		/* Feedback loop via tracepoint */
-		trace_xdp_cpumap_kthread(rcpu->map_id, n, drops, sched, &stats);
+		trace_xdp_cpumap_kthread(rcpu->map_id, n, sched, &stats);
 
 		local_bh_enable(); /* resched point, may call do_softirq() */
 	}
diff --git a/samples/bpf/xdp_monitor_kern.c b/samples/bpf/xdp_monitor_kern.c
index 5c955b812c47..f7aeecfa046b 100644
--- a/samples/bpf/xdp_monitor_kern.c
+++ b/samples/bpf/xdp_monitor_kern.c
@@ -186,9 +186,8 @@ struct cpumap_kthread_ctx {
 	int map_id;		//	offset:8;  size:4; signed:1;
 	u32 act;		//	offset:12; size:4; signed:0;
 	int cpu;		//	offset:16; size:4; signed:1;
-	unsigned int drops;	//	offset:20; size:4; signed:0;
-	unsigned int processed;	//	offset:24; size:4; signed:0;
-	int sched;		//	offset:28; size:4; signed:1;
+	unsigned int processed;	//	offset:20; size:4; signed:0;
+	int sched;		//	offset:24; size:4; signed:1;
 };
 
 SEC("tracepoint/xdp/xdp_cpumap_kthread")
@@ -201,7 +200,6 @@ int trace_xdp_cpumap_kthread(struct cpumap_kthread_ctx *ctx)
 	if (!rec)
 		return 0;
 	rec->processed += ctx->processed;
-	rec->dropped   += ctx->drops;
 
 	/* Count times kthread yielded CPU via schedule call */
 	if (ctx->sched)
diff --git a/samples/bpf/xdp_monitor_user.c b/samples/bpf/xdp_monitor_user.c
index 49ebc49aefc3..f31796809cce 100644
--- a/samples/bpf/xdp_monitor_user.c
+++ b/samples/bpf/xdp_monitor_user.c
@@ -432,11 +432,11 @@ static void stats_print(struct stats_record *stats_rec,
 
 	/* cpumap kthread stats */
 	{
-		char *fmt1 = "%-15s %-7d %'-12.0f %'-12.0f %'-10.0f %s\n";
-		char *fmt2 = "%-15s %-7s %'-12.0f %'-12.0f %'-10.0f %s\n";
+		char *fmt1 = "%-15s %-7d %'-12.0f %-12s %'-10.0f %s\n";
+		char *fmt2 = "%-15s %-7s %'-12.0f %-12s %'-10.0f %s\n";
 		struct record *rec, *prev;
-		double drop, info;
 		char *i_str = "";
+		double info;
 
 		rec  =  &stats_rec->xdp_cpumap_kthread;
 		prev = &stats_prev->xdp_cpumap_kthread;
@@ -446,20 +446,18 @@ static void stats_print(struct stats_record *stats_rec,
 			struct datarec *p = &prev->cpu[i];
 
 			pps  = calc_pps(r, p, t);
-			drop = calc_drop(r, p, t);
 			info = calc_info(r, p, t);
 			if (info > 0)
 				i_str = "sched";
-			if (pps > 0 || drop > 0)
+			if (pps > 0)
 				printf(fmt1, "cpumap-kthread",
-				       i, pps, drop, info, i_str);
+				       i, pps, "-", info, i_str);
 		}
 		pps = calc_pps(&rec->total, &prev->total, t);
-		drop = calc_drop(&rec->total, &prev->total, t);
 		info = calc_info(&rec->total, &prev->total, t);
 		if (info > 0)
 			i_str = "sched-sum";
-		printf(fmt2, "cpumap-kthread", "total", pps, drop, info, i_str);
+		printf(fmt2, "cpumap-kthread", "total", pps, "-", info, i_str);
 	}
 
 	/* devmap ndo_xdp_xmit stats */
diff --git a/samples/bpf/xdp_redirect_cpu_kern.c b/samples/bpf/xdp_redirect_cpu_kern.c
index 8255025dea97..34332b0b7020 100644
--- a/samples/bpf/xdp_redirect_cpu_kern.c
+++ b/samples/bpf/xdp_redirect_cpu_kern.c
@@ -699,12 +699,11 @@ struct cpumap_kthread_ctx {
 	int map_id;			//	offset:8;  size:4; signed:1;
 	u32 act;			//	offset:12; size:4; signed:0;
 	int cpu;			//	offset:16; size:4; signed:1;
-	unsigned int drops;		//	offset:20; size:4; signed:0;
-	unsigned int processed;		//	offset:24; size:4; signed:0;
-	int sched;			//	offset:28; size:4; signed:1;
-	unsigned int xdp_pass;		//	offset:32; size:4; signed:0;
-	unsigned int xdp_drop;		//	offset:36; size:4; signed:0;
-	unsigned int xdp_redirect;	//	offset:40; size:4; signed:0;
+	unsigned int processed;		//	offset:20; size:4; signed:0;
+	int sched;			//	offset:24; size:4; signed:1;
+	unsigned int xdp_pass;		//	offset:28; size:4; signed:0;
+	unsigned int xdp_drop;		//	offset:32; size:4; signed:0;
+	unsigned int xdp_redirect;	//	offset:36; size:4; signed:0;
 };
 
 SEC("tracepoint/xdp/xdp_cpumap_kthread")
@@ -717,7 +716,6 @@ int trace_xdp_cpumap_kthread(struct cpumap_kthread_ctx *ctx)
 	if (!rec)
 		return 0;
 	rec->processed += ctx->processed;
-	rec->dropped   += ctx->drops;
 	rec->xdp_pass  += ctx->xdp_pass;
 	rec->xdp_drop  += ctx->xdp_drop;
 	rec->xdp_redirect  += ctx->xdp_redirect;
diff --git a/samples/bpf/xdp_redirect_cpu_user.c b/samples/bpf/xdp_redirect_cpu_user.c
index 576411612523..76346647fd5b 100644
--- a/samples/bpf/xdp_redirect_cpu_user.c
+++ b/samples/bpf/xdp_redirect_cpu_user.c
@@ -436,8 +436,8 @@ static void stats_print(struct stats_record *stats_rec,
 
 	/* cpumap kthread stats */
 	{
-		char *fmt_k = "%-15s %-7d %'-14.0f %'-11.0f %'-10.0f %s\n";
-		char *fm2_k = "%-15s %-7s %'-14.0f %'-11.0f %'-10.0f %s\n";
+		char *fmt_k = "%-15s %-7d %'-14.0f %-11s %'-10.0f %s\n";
+		char *fm2_k = "%-15s %-7s %'-14.0f %-11s %'-10.0f %s\n";
 		char *e_str = "";
 
 		rec  = &stats_rec->kthread;
@@ -448,20 +448,18 @@ static void stats_print(struct stats_record *stats_rec,
 			struct datarec *p = &prev->cpu[i];
 
 			pps  = calc_pps(r, p, t);
-			drop = calc_drop_pps(r, p, t);
 			err  = calc_errs_pps(r, p, t);
 			if (err > 0)
 				e_str = "sched";
 			if (pps > 0)
 				printf(fmt_k, "cpumap_kthread",
-				       i, pps, drop, err, e_str);
+				       i, pps, "-", err, e_str);
 		}
 		pps = calc_pps(&rec->total, &prev->total, t);
-		drop = calc_drop_pps(&rec->total, &prev->total, t);
 		err  = calc_errs_pps(&rec->total, &prev->total, t);
 		if (err > 0)
 			e_str = "sched-sum";
-		printf(fm2_k, "cpumap_kthread", "total", pps, drop, err, e_str);
+		printf(fm2_k, "cpumap_kthread", "total", pps, "-", err, e_str);
 	}
 
 	/* XDP redirect err tracepoints (very unlikely) */
-- 
2.30.2

Powered by blists - more mailing lists

Powered by Openwall GNU/*/Linux Powered by OpenVZ