Message-ID: <20220819165940.ett7n4vwbw6hvqvi@skbuf>
Date:   Fri, 19 Aug 2022 16:59:41 +0000
From:   Vladimir Oltean <vladimir.oltean@....com>
To:     Kurt Kanzenbach <kurt@...utronix.de>
CC:     "netdev@...r.kernel.org" <netdev@...r.kernel.org>,
        "David S. Miller" <davem@...emloft.net>,
        Eric Dumazet <edumazet@...gle.com>,
        Jakub Kicinski <kuba@...nel.org>,
        Paolo Abeni <pabeni@...hat.com>,
        Michal Kubecek <mkubecek@...e.cz>,
        Claudiu Manoil <claudiu.manoil@....com>,
        Vinicius Costa Gomes <vinicius.gomes@...el.com>,
        Xiaoliang Yang <xiaoliang.yang_1@....com>,
        Rui Sousa <rui.sousa@....com>,
        Ferenc Fejes <ferenc.fejes@...csson.com>
Subject: Re: [RFC PATCH net-next 0/7] 802.1Q Frame Preemption and 802.3 MAC
 Merge support via ethtool

Hi Kurt,

On Fri, Aug 19, 2022 at 10:16:20AM +0200, Kurt Kanzenbach wrote:
> On Wed Aug 17 2022, Vladimir Oltean wrote:
> > Vinicius' progress on upstreaming frame preemption support for Intel I226
> > seemed to stall, so I decided to give it a go using my own view as well.
> > https://patchwork.kernel.org/project/netdevbpf/cover/20220520011538.1098888-1-vinicius.gomes@intel.com/
> 
> Great to see progress on FPE :-).

Let's hope it lasts ;)

> > - Finally, the hardware I'm working with (here, the test vehicle is the
> >   NXP ENETC from LS1028A, although I have patches for the Felix switch
> >   as well, but those need a bit of a revolution in the driver to go in
> >   first). This hardware is not without its flaws, but at least allows me
> >   to concentrate on the UAPI portions for this series.
> >
> > I also have a kselftest written, but it's for the Felix switch (covers
> > forwarding latency) and so it's not included here.
> 
> What kind of selftest did you implement? So far I've been doing this:
> Using a cyclic real time application to create high priority frames and
> running iperf3 in parallel to simulate low priority traffic
> constantly. Afterwards, checking the NIC statistics for fragments and so
> on. Also checking the latency of the RT frames with FPE on/off.
> 
> BTW, if you guys need help with testing of patches, i do have access to
> i225 and stmmacs which both support FPE. Also the Hirschmann switches
> should support it.

Blah, I didn't want to spoil the surprise just yet. I am orchestrating 2
isochron senders at specific times, one of PT traffic and one of ET.

There are actually 2 variants of this: one is for endpoint FP and the
other is for bridge FP. I only had time to convert the bridge FP to
kselftest format; not the endpoint one (for enetc).

In the endpoint case, interference is created on the sender interface.
I compare HW TX timestamps to the expected TX times to calculate how
long it took until PT was preempted. I repeat the test millions of times
until I can plot the latencies having the PT <-> ET base time offset on
the X axis. It looks very cool.

The bridge case is similar, except for the fact that interference is
created on a bridge port going to a common receiver of 2 isochron
senders. What I plot is the path delay, and again, this shows actual
preemption times with a nanosecond resolution.

Here's a small snapshot of the main script. Not shown are the supporting
scripts for this:

#!/bin/bash
# SPDX-License-Identifier: GPL-2.0
# Copyright 2022 NXP
#
# Selftest for Frame Preemption on a switch. Creates controlled packet
# collisions between a priority configured as preemptable traffic (PT) and a
# priority configured as express traffic (ET).
#
#                           +-----------------------------+
#                           |            fp.sh            |
#                           |             br0             |
#                           |     +--------+--------+     |
#                           |     |        |        |     |
#                           |   $swp1    $swp2    $swp3   |
#                           +-----------------------------+
#   +--------------------+        ^        ^        |       +----------------+
#   |                    |        |        |        |       |                |
#   | fp_node_et_send.sh |--------+        |        +------>| fp_node_rcv.sh |
#   |        $h1         |                 |                |       $h3      |
#   +--------------------+                 |                +----------------+
#                                          |
#   +--------------------+                 |
#   |                    |                 |
#   | fp_node_pt_send.sh |-----------------+
#   |        $h2         |
#   +--------------------+
#
# Normally, a packet collision plot with no frame preemption looks as follows.
# The first packet starts being transmitted by the MAC, and the second packet
# sees a latency proportional to how much transmission time of the previous
# packet there is left.
#
#        ^
#   ET   |
# frame  |        ---.
#  path  |       /    --.
# delay  |       |       --.
#        |       /          --.
#        |       |             --.
#        |      /                 --.
#        |      |                    ---.
#        |-----/                         -----------------------
#        |
#     ---+----------------------------------------------------------->
#        |      ET base time offset relative to PT
#
# Depending on the preemption point (where the express packet hits the
# preemptable packet that is undergoing transmission), the express packet will
# have a higher or lower path delay (latency).
#
# The packet collision can be broken into 3 distinct intervals:
# (a) when the packets start interfering, but not enough of the PT packet has
#     been transmitted yet so as to preempt it (MAC merge fragments must still
#     obey the minimum Ethernet frame size rule). There are no frame
#     preemptions when packets collide here.
# (b) the preemptable area, where the latency should be equal to the
#     transmission time of a MAC merge fragment rather than a full packet
#     (preemption should happen right away).
# (c) where the packets still interfere, but there are not enough bytes of the
#     PT frame left to transmit so as to construct a valid fragment out of the
#     remainder. No preemptions are expected in this region.
#
#        ^
#   ET   |      a         b            c
# frame  |   <----><---------------><---->
#  path  |
# delay  |
#        |
#        |       /|
#        |      /  -----------------.
#        |      |                    ---.
#        |-----/                         -----------------------
#        |
#     ---+----------------------------------------------------------->
#        |      ET base time offset relative to PT
#
# The selftest evaluates the preemption performance of a given switch by
# plotting the ET latency as a function of the ET base time offset.

WAIT_TIME=1
NUM_NETIFS=3
NETIF_CREATE=no
REQUIRE_MZ=no
lib_dir=$(dirname $0)/../../../net/forwarding
source $lib_dir/lib.sh
source $lib_dir/tsn_lib.sh
source $(dirname $0)/fp_topology.sh

require_command gnuplot

swp1=${NETIFS[p1]}
swp2=${NETIFS[p2]}
swp3=${NETIFS[p3]}

signal_received=false

ethtool_save_fp_admin_status()
{
	admin_status=$(ethtool --json --show-fp $1 | \
		jq "(.[].\"parameter-table\"[] | select(.prio == $2)).\"admin-status\"")
}

ethtool_restore_fp_admin_status()
{
	case $admin_status in
	express)
		ethtool --set-fp $1 admin-status $2:E
		;;
	preemptable)
		ethtool --set-fp $1 admin-status $2:P
		;;
	esac
}

avg_pdelay()
{
	local file=$1
	local tmp=$(mktemp)

	printf "print((" > $tmp
	isochron report \
		--input-file "${file}" \
		--printf-format "print(\"{} + \".format(%d - %d), end='')\n" \
		--printf-args "RT" | python3 - >> $tmp
	printf "0) / $NUM_FRAMES)" >> $tmp
	cat $tmp | python3 -
	rm -f $tmp
}

validate_coordination()
{
	local et=$1; shift
	local pt=$1; shift
	local et_start
	local pt_start

	et_start=$(($(isochron report --input-file $et \
		--printf-format "%u - %u" --printf-args "SB" --stop 1)))
	pt_start=$(($(isochron report --input-file $pt \
		--printf-format "%u - %u" --printf-args "SB" --stop 1)))
	if ! [ $et_start = $pt_start ]; then
		printf "ET test %s / PT test %s: ET starts at %s, PT starts at %s\n" \
			"$et" "$pt" \
			"$(isochron report --input-file $et \
				--printf-format "%T" --printf-args "S" --stop 1)" \
			"$(isochron report --input-file $pt \
				--printf-format "%T" --printf-args "S" --stop 1)"
		exit 1
	fi
}

plot()
{
	local pt_frame_size=$1; shift
	local num_tests=$1; shift
	local fp_label=$1; shift
	local pt_svg="plots/pt-${pt_frame_size}-${fp_label}-pt.svg"
	local pt_plot="plots/pt-${pt_frame_size}-${fp_label}-pt.plot"
	local et_svg="plots/pt-${pt_frame_size}-${fp_label}-et.svg"
	local et_plot="plots/pt-${pt_frame_size}-${fp_label}-et.plot"
	local pt_avg_pdelay
	local et_avg_pdelay
	local base_time
	local et
	local pt
	local i

	printf "Plotting charts for collision between PT@%d and ET@%d\n" \
		"${pt_frame_size}" "${ET_FRAME_SIZE}"

	mkdir -p plots
	rm -f $et_plot $pt_plot

	for ((i = 0; i < num_tests; i++)); do
		et="reports/pt-${pt_frame_size}-test-${i}-${fp_label}-et.dat"
		pt="reports/pt-${pt_frame_size}-test-${i}-${fp_label}-pt.dat"

		base_time=$(isochron report --input-file "${et}" \
				--printf-format "%u" --printf-args "B" \
				--stop 1)

		pt_avg_pdelay=$(avg_pdelay $pt)
		et_avg_pdelay=$(avg_pdelay $et)

		echo "$base_time $pt_avg_pdelay" >> $pt_plot
		echo "$base_time $et_avg_pdelay" >> $et_plot

		if [ "$signal_received" = "true" ]; then
			exit 1
		fi
	done

	gnuplot --persist <<- EOF
		set xlabel "Base time offset (ns)"
		set ylabel "Average PT path delay (ns)"
		set term svg
		set terminal svg enhanced background rgb "white"
		set output "${pt_svg}"
		plot "${pt_plot}" using 1:2 with lines title "PT latency"
	EOF

	gnuplot --persist <<- EOF
		set xlabel "Base time offset (ns)"
		set ylabel "Average ET path delay (ns)"
		set term svg
		set terminal svg enhanced background rgb "white"
		set output "${et_svg}"
		plot "${et_plot}" using 1:2 with lines title "ET latency"
	EOF

	rm -f ${et_plot} ${pt_plot}
}

test_collision()
{
	local test_num=$1; shift
	local pt_frame_size=$1; shift
	local base_time_offset=$1; shift
	local fp_label=$1; shift
	local vrf_name=$(master_name_get br0)
	local et="reports/pt-${pt_frame_size}-test-${test_num}-${fp_label}-et.dat"
	local pt="reports/pt-${pt_frame_size}-test-${test_num}-${fp_label}-pt.dat"
	local orchestration=$(mktemp)
	local et_extra_args
	local pt_extra_args

	if [ "${H1_REQUIRE_PTP4L}" = "no" ]; then
		et_extra_args="${et_extra_args} --omit-sync"
	fi

	if [ "${H2_REQUIRE_PTP4L}" = "no" ]; then
		pt_extra_args="${pt_extra_args} --omit-sync"
	fi

	if [ "${H3_REQUIRE_PTP4L}" = "no" ]; then
		et_extra_args="${et_extra_args} --omit-remote-sync"
		pt_extra_args="${pt_extra_args} --omit-remote-sync"
	fi

	mkdir -p reports

	printf "Collision between PT packets of size %d and ET packets of size %d at %s, test %d (base time offset %d)\n" \
		${pt_frame_size} ${ET_FRAME_SIZE} ${fp_label} ${test_num} ${base_time_offset}

	cat <<- EOF > ${orchestration}
	[ET]
	host = $H1_IPV4%$vrf_name
	port = $H1_PORT
	exec = isochron send \\
		--client $H3_IPV4%$vrf_name \\
		--stats-port $H3_ET_PORT \\
		--interface $ET_IF_NAME \\
		--unix-domain-socket $UDS_ADDRESS_H1 \\
		--num-frames $NUM_FRAMES \\
		--priority $ET_PRIO \\
		--vid 0 \\
		--base-time ${base_time_offset} \\
		--cycle-time ${CYCLE_TIME_NS} \\
		--frame-size $ET_FRAME_SIZE \\
		--sync-threshold $SYNC_THRESHOLD_NS \\
		--cpu-mask $((1 << ${ISOCHRON_CPU})) \\
		--sched-fifo \\
		--sched-priority 98 \\
		--etype 0xdead \\
		--txtime \\
		${et_extra_args} \\
		--output-file ${et}

	[PT]
	host = $H2_IPV4%$vrf_name
	port = $H2_PORT
	exec = isochron send \\
		--client $H3_IPV4%$vrf_name \\
		--stats-port $H3_PT_PORT \\
		--interface $PT_IF_NAME \\
		--unix-domain-socket $UDS_ADDRESS_H2 \\
		--num-frames $NUM_FRAMES \\
		--priority $PT_PRIO \\
		--vid 0 \\
		--base-time 0.000000000 \\
		--cycle-time ${CYCLE_TIME_NS} \\
		--frame-size $pt_frame_size \\
		--sync-threshold $SYNC_THRESHOLD_NS \\
		--cpu-mask $((1 << ${ISOCHRON_CPU})) \\
		--sched-fifo \\
		--sched-priority 98 \\
		--etype 0xdeaf \\
		--txtime \\
		${pt_extra_args} \\
		--output-file ${pt}
	EOF

	isochron orchestrate --input-file ${orchestration}

	rm -rf ${orchestration}

	validate_coordination $et $pt
}

test_collision_sweep_base_time()
{
	local pt_frame_size=$1; shift
	local base_time_start=$1; shift
	local base_time_stop=$1; shift
	local base_time_increment=$1; shift
	local fp_label=$1; shift
	local num_tests
	local i

	num_tests=$(((base_time_stop - base_time_start + 1) / base_time_increment))
	for ((i = 0; i < num_tests; i++)); do
		base_time=$((base_time_start + i * base_time_increment))

		test_collision "$i" "$pt_frame_size" "$base_time" "$fp_label"

		if [ "$signal_received" = "true" ]; then
			exit 1
		fi
	done

	plot $pt_frame_size $num_tests $fp_label
}

test_pt_124()
{
	local start=$1; shift
	local stop=$1; shift
	local fp_label=$1; shift

	test_collision_sweep_base_time 124 $start $stop 40 $fp_label
}

test_pt_300()
{
	local start=$1; shift
	local stop=$1; shift
	local fp_label=$1; shift

	test_collision_sweep_base_time 300 $start $stop 40 $fp_label
}

test_pt_600()
{
	local start=$1; shift
	local stop=$1; shift
	local fp_label=$1; shift

	test_collision_sweep_base_time 600 $start $stop 40 $fp_label
}

test_pt_1000()
{
	local start=$1; shift
	local stop=$1; shift
	local fp_label=$1; shift

	test_collision_sweep_base_time 1000 $start $stop 40 $fp_label
}

test_pt_1500()
{
	local start=$1; shift
	local stop=$1; shift
	local fp_label=$1

	test_collision_sweep_base_time 1500 $start $stop 40 $fp_label
}

test_no_fp_cut_through()
{
	local fp_label="no-fp"

	test_pt_124 0 1800 $fp_label
	test_pt_300 0 3400 $fp_label
	test_pt_600 0 7000 $fp_label
	test_pt_1000 1500 12000 $fp_label
	test_pt_1500 3000 17000 $fp_label
}

test_fp_quick()
{
	ethtool_save_fp_admin_status $swp3 $PT_PRIO
	ethtool --set-mm $swp3 verify-disable off enabled on add-frag-size 0
	ethtool --set-fp $swp3 admin-status $PT_PRIO:P

	test_pt_1500 12000 26000 "quick"

	ethtool_restore_fp_admin_status $swp3 $PT_PRIO
}

test_fp()
{
	local fp_label=$1; shift
	local add_frag_size=$1; shift

	ethtool_save_fp_admin_status $swp3 $PT_PRIO
	ethtool --set-mm $swp3 verify-disable off enabled on add-frag-size $add_frag_size
	ethtool --set-fp $swp3 admin-status $PT_PRIO:P

	test_pt_124 200 2000 $fp_label
	test_pt_300 1500 5000 $fp_label
	test_pt_600 4500 10000 $fp_label
	test_pt_1000 8800 17000 $fp_label
	test_pt_1500 12000 26000 $fp_label

	ethtool_restore_fp_admin_status $swp3 $PT_PRIO
}

test_fp_add_frag_size_0()
{
	test_fp "add-frag-size-0" 0
}

test_fp_add_frag_size_1()
{
	test_fp "add-frag-size-1" 1
}

test_fp_add_frag_size_2()
{
	test_fp "add-frag-size-2" 2
}

test_fp_add_frag_size_3()
{
	test_fp "add-frag-size-2" 3
}

setup_prepare()
{
	vrf_prepare

	switch_create
	ptp4l_start "$swp1 $swp2 $swp3" false ${UDS_ADDRESS_SWITCH}
	phc2sys_start ${UDS_ADDRESS_SWITCH}
}

cleanup()
{
	pre_cleanup

	ptp4l_stop "$swp1 $swp2 $swp3"
	phc2sys_stop
	switch_destroy

	vrf_cleanup
}

signal()
{
	signal_received=true
}

trap cleanup EXIT
trap signal SIGTERM
trap signal SIGINT

#	test_no_fp_cut_through
#	test_fp_add_frag_size_0
#	test_fp_add_frag_size_1
#	test_fp_add_frag_size_2
#	test_fp_add_frag_size_3
ALL_TESTS="
	test_fp_quick
"

setup_prepare
setup_wait

tests_run

exit $EXIT_STATUS

Powered by blists - more mailing lists