testing: incremental improvements to the router benchmark tooling (scionproto#4518)

Summary of changes:
* The packet size is now configurable. CI uses a constant while the
benchmark command accepts an optional argument.
* The benchmark command sets the mtu to 9000 on the host side and
instructs the user to do the same on the router side. In the CI test
variant: an mtu of 9000 (not 8000) is used on both sides.
* The performance index formula gained a tentative value for the
NIC_CONSTANT constant.
* The benchmark's total number of packets (per testcase) is no longer
tunable. Instead, the duration is what is configurable. Both the CI test
and the benchmark command use 13 seconds per test case.
* The benchmark's output includes a bit more information: bit rate and
raw (not corrected for preemptions) packet rate.
* A useless and noisy error message was silenced in brload: that which
was output for every foreign packet received (turns out that some
Realtek hardware sends proprietary management traffic as Ethernet
frames).
* Improved the relevance of mmbm by deliberately misaligning buffers.
Also make sure to not use more buffers than can fit in L3 cache when
probing for L2 cache + TLB misses.
* Added zip as a dependency after finding it isn't part of Debian by
default.
* Added an option (via constant for now) to profile the router during
the CI test.

---------

Co-authored-by: Jean-Christophe Hugly <[email protected]>

Author: jiceathome

acceptance/router_benchmark/benchmark.py

@@ -95,6 +95,8 @@ class RouterBMTool(cli.Application, RouterBM):
                               help="The coremark score of the subject machine.")
     mmbm = cli.SwitchAttr(["m", "mmbm"], int, default=0,
                           help="The mmbm score of the subject machine.")
+    packet_size = cli.SwitchAttr(["s", "size"], int, default=172,
+                                 help="Test packet size (includes all headers - floored at 154).")
     intf_map: dict[str, Intf] = {}
     brload: LocalCommand = local["./bin/brload"]
     brload_cpus: list[int] = []
@@ -158,8 +160,7 @@ def config_interface(self, req: IntfReq):
             if i.name == host_intf:
                 break
         else:
-            # TODO: instructions/warning regarding inability to enable jumbo frames.
-            # sudo("ip", "link", "set", host_intf, "mtu", "8000")
+            sudo("ip", "link", "set", host_intf, "mtu", "9000")
 
             # Do not assign the host addresses but create one link-local addr.
             # Brload needs some src IP to send arp requests. (This requires rp_filter
@@ -291,16 +292,16 @@ def instructions(self):
     bmtools includes two microbenchmarks: scion-coremark and scion-mmbm. Those will run
     automatically and the results will be used to improve the benchmark report.
 
-    Optinal: If you did not install bmtools.ipk, install and run those microbenchmark and make a
+    Optional: If you did not install bmtools.ipk, install and run those microbenchmarks and make a
     note of the results: (scion-coremark; scion-mmbm).
 
 2 - Configure the following interfaces on your router (The procedure depends on your router
-    UI):
+    UI) - All interfaces should have the mtu set to 9000:
     - One physical interface with addresses: {", ".join(multiplexed)}
 {nl.join(['    - One physical interface with address: ' + s for s in exclusives])}
 
     IMPORTANT: if you're using a partitioned network (eg. multiple switches or no switches),
-    the "must reach" annotation matters. The 'h' number is the order in which the corresponding host
+    the "must reach" annotation matters. The '#' number is the order in which the corresponding host
     interface must be given on the command line in step 7.
 
 3 - Connect the corresponding ports into your test switch (best if dedicated for the test).

acceptance/router_benchmark/benchmarklib.py

@@ -33,13 +33,11 @@
 # to retrieve a frame. That one is hardware dependent and must be found by a third benchmark, so
 # it is not theoretically a constant, but keeping it here to not forget. Until then, our performance
 # index isn't really valid cross-hardware. M_COEF=400 gives roughly consistent results with the
-# hardware we have. So, using that until we know more.
+# hardware we have. So, using that until we know more. NIC_CONSTANT seems to be around
+# 1 microsecond. Using that, provisionally.
 
 M_COEF = 400
-NIC_CONSTANT = 0
-
-# TODO(jiceatscion): get it from or give it to brload?
-BM_PACKET_LEN = 172
+NIC_CONSTANT = 1.0/1000000
 
 # Intf: description of an interface configured for brload's use. Depending on context
 # mac and peermac may be unused. "mac" is the MAC address configured on the side of the subject
@@ -55,31 +53,36 @@ class Results:
     cores: int = 0
     coremark: int = 0
     mmbm: int = 0
+    packet_size: int = 0
     cases: list[dict] = []
     failed: list[dict] = []
     checked: bool = False
 
-    def __init__(self, cores: int, coremark: int, mmbm: int):
+    def __init__(self, cores: int, coremark: int, mmbm: int, packet_size: int):
         self.cores = cores
         self.coremark = coremark
         self.mmbm = mmbm
+        self.packet_size = packet_size
 
     def perf_index(self, rate: int) -> float:
         # TODO(jiceatscion): The perf index assumes that line speed isn't the bottleneck.
         # It almost never is, but ideally we'd need to run iperf3 to verify.
         # mmbm is in mebiBytes/s, rate is in pkt/s
         return rate * (1.0 / self.coremark +
-                       M_COEF * BM_PACKET_LEN / (self.mmbm * 1024 * 1024) +
+                       M_COEF * self.packet_size / (self.mmbm * 1024 * 1024) +
                        NIC_CONSTANT)
 
-    def add_case(self, name: str, rate: int, droppage: int):
+    def add_case(self, name: str, rate: int, droppage: int, raw_rate: int):
         dropRatio = round(float(droppage) / (rate + droppage), 2)
         saturated = dropRatio > 0.03
         perf = 0.0
         if self.cores == 3 and self.coremark and self.mmbm:
             perf = round(self.perf_index(rate), 1)
         self.cases.append({"case": name,
-                           "perf": perf, "rate": rate, "drop": dropRatio, "full": saturated})
+                           "perf": perf, "rate": rate, "drop": dropRatio,
+                           "bit_rate": rate * self.packet_size * 8,
+                           "raw_pkt_rate": raw_rate,
+                           "full": saturated})
 
     def CI_check(self, expectations: dict[str, int]):
         self.checked = True
@@ -147,14 +150,15 @@ class RouterBM():
     This class is a Mixin that borrows the following attributes from the host class:
     * coremark: the coremark benchmark results.
     * mmbm: the mmbm benchmark results.
+    * packet_size: the packet_size to use in the test cases.
     * intf_map: the map "label->actual_interface" map to be passed to brload.
     * brload: "localCmd" wraper for the brload executable (plumbum.machines.LocalCommand)
     * brload_cpus: [int] cpus where it is acceptable to run brload ([] means any)
     * artifacts: the data directory (passed to docker).
     * prom_address: the address of the prometheus API a string in the form "host:port"
     """
 
-    def exec_br_load(self, case: str, map_args: list[str], count: int) -> str:
+    def exec_br_load(self, case: str, map_args: list[str], duration: int) -> str:
         # For num-streams, attempt to distribute uniformly on many possible number of cores.
         # 840 is a multiple of 1, 2, 3, 4, 5, 6, 7, 8, 10, 12, 14, 15, 20, 21, 24, 28, ...
         brload_args = [
@@ -163,8 +167,9 @@ def exec_br_load(self, case: str, map_args: list[str], count: int) -> str:
             "--artifacts", self.artifacts,
             *map_args,
             "--case", case,
-            "--num-packets", str(count),
+            "--duration", f"{duration}s",
             "--num-streams", "840",
+            "--packet-size", f"{self.packet_size}",
         ]
         if self.brload_cpus:
             brload_args = [
@@ -176,20 +181,20 @@ def exec_br_load(self, case: str, map_args: list[str], count: int) -> str:
     def run_test_case(self, case: str, map_args: list[str]) -> (int, int):
         logger.debug(f"==> Starting load {case}")
 
-        output = self.exec_br_load(case, map_args, 10000000)
-        beg = "0"
+        # We transmit for 13 seconds and then ignore the first 3.
+        output = self.exec_br_load(case, map_args, 13)
         end = "0"
         for line in output.splitlines():
             if line.startswith("metricsBegin"):
-                _, beg, _, end = line.split()
+                end = line.split()[3]  # "... metricsEnd: <end>"
 
         logger.debug(f"==> Collecting {case} performance metrics...")
 
         # The raw metrics are expressed in terms of core*seconds. We convert to machine*seconds
         # which allows us to provide a projected packet/s; ...more intuitive than packets/core*s.
-        # We measure the rate over 10s. For best results we sample the end of the middle 10s of the
-        # run. "beg" is the start time of the real action and "end" is the end time.
-        sampleTime = (int(beg) + int(end) + 10) / 2
+        # We measure the rate over 10s. For best results we only look at the last 10 seconds.
+        # "end" reports a time when the transmission was still going on at maximum rate.
+        sampleTime = int(end)
         prom_query = urlencode({
             'time': f'{sampleTime}',
             'query': (
@@ -218,6 +223,31 @@ def run_test_case(self, case: str, map_args: list[str]) -> (int, int):
             processed = int(float(val))
             break
 
+        # Collect the raw packet rate too. Just so we can discover if the cpu-availability
+        # correction is bad.
+        prom_query = urlencode({
+            'time': f'{sampleTime}',
+            'query': (
+                'sum by (instance, job) ('
+                f'  rate(router_output_pkts_total{{job="BR", type="{case}"}}[10s])'
+                ')'
+            )
+        })
+        conn = HTTPConnection(self.prom_address)
+        conn.request("GET", f"/api/v1/query?{prom_query}")
+        resp = conn.getresponse()
+        if resp.status != 200:
+            raise RuntimeError(f"Unexpected response: {resp.status} {resp.reason}")
+
+        # There's only one router, so whichever metric we get is the right one.
+        pld = json.loads(resp.read().decode("utf-8"))
+        raw = 0
+        results = pld["data"]["result"]
+        for result in results:
+            ts, val = result["value"]
+            raw = int(float(val))
+            break
+
         # Collect dropped packets metrics, so we can verify that the router was well saturated.
         # If not, the metrics aren't very useful.
         prom_query = urlencode({
@@ -248,7 +278,7 @@ def run_test_case(self, case: str, map_args: list[str]) -> (int, int):
             dropped = int(float(val))
             break
 
-        return processed, dropped
+        return processed, dropped, raw
 
     # Fetch and log the number of cores used by Go. This may inform performance
     # modeling later.
@@ -289,18 +319,18 @@ def run_bm(self, test_cases: [str]) -> Results:
         # Run one test (30% size) as warm-up to trigger any frequency scaling, else the first test
         # can get much lower performance.
         logger.debug("Warmup")
-        self.exec_br_load(test_cases[0], map_args, 3000000)
+        self.exec_br_load(test_cases[0], map_args, 5)
 
         # Fetch the core count once. It doesn't change while the router is running.
         # We can't get it until the router has done some work, but the warmup is enough.
         cores = self.core_count()
 
         # At long last, run the tests.
-        results = Results(cores, self.coremark, self.mmbm)
+        results = Results(cores, self.coremark, self.mmbm, self.packet_size)
         for test_case in test_cases:
             logger.info(f"Case: {test_case}")
-            rate, droppage = self.run_test_case(test_case, map_args)
-            results.add_case(test_case, rate, droppage)
+            rate, droppage, raw = self.run_test_case(test_case, map_args)
+            results.add_case(test_case, rate or 1, droppage, raw)
 
         return results
         logger.info("Benchmarked")

acceptance/router_benchmark/brload/main.go

@@ -15,6 +15,7 @@
 package main
 
 import (
+	"bytes"
 	"encoding/binary"
 	"errors"
 	"fmt"
@@ -37,7 +38,7 @@ import (
 	"github.com/scionproto/scion/private/keyconf"
 )
 
-type Case func(payload string, mac hash.Hash) (string, string, []byte)
+type Case func(packetSize int, mac hash.Hash) (string, string, []byte, []byte)
 
 type caseChoice string
 
@@ -70,12 +71,13 @@ var (
 		"out_transit": cases.OutTransit,
 		"br_transit":  cases.BrTransit,
 	}
-	logConsole string
-	dir        string
-	numPackets int
-	numStreams uint16
-	caseToRun  caseChoice
-	interfaces []string
+	logConsole   string
+	dir          string
+	testDuration time.Duration
+	packetSize   int
+	numStreams   uint16
+	caseToRun    caseChoice
+	interfaces   []string
 )
 
 func main() {
@@ -97,7 +99,8 @@ func main() {
 			os.Exit(run(cmd))
 		},
 	}
-	runCmd.Flags().IntVar(&numPackets, "num-packets", 10, "Number of packets to send")
+	runCmd.Flags().DurationVar(&testDuration, "duration", time.Second*15, "Test duration")
+	runCmd.Flags().IntVar(&packetSize, "packet-size", 172, "Total size of each packet sent")
 	runCmd.Flags().Uint16Var(&numStreams, "num-streams", 4,
 		"Number of independent streams (flowID) to use")
 	runCmd.Flags().StringVar(&logConsole, "log.console", "error",
@@ -160,10 +163,8 @@ func run(cmd *cobra.Command) int {
 	registerScionPorts()
 
 	log.Info("BRLoad acceptance tests:")
-
-	payloadString := "actualpayloadbytes"
 	caseFunc := allCases[string(caseToRun)] // key already checked.
-	caseDevIn, caseDevOut, rawPkt := caseFunc(payloadString, hfMAC)
+	caseDevIn, caseDevOut, payload, rawPkt := caseFunc(packetSize, hfMAC)
 
 	writePktTo, ok := handles[caseDevIn]
 	if !ok {
@@ -186,30 +187,40 @@ func run(cmd *cobra.Command) int {
 	go func() {
 		defer log.HandlePanic()
 		defer close(listenerChan)
-		listenerChan <- receivePackets(packetChan, payloadString)
+		listenerChan <- receivePackets(packetChan, payload)
 	}()
 
-	// We started everything that could be started. So the best window for perf mertics
-	// opens somewhere around now.
-	metricsBegin := time.Now().Unix()
 	// Because we're using IPV4 only, the UDP checksum is optional, so we are allowed to
 	// just set it to zero instead of recomputing it. The IP checksum does not cover the payload, so
 	// we don't need to update it.
 	binary.BigEndian.PutUint16(rawPkt[40:42], 0)
 
-	for i := 0; i < numPackets; i++ {
-		// Rotate through flowIDs. We patch it directly into the SCION header of the packet.  The
-		// SCION header starts at offset 42. The flowID is the 20 least significant bits of the
-		// first 32 bit field. To make our life simpler, we only use the last 16 bits (so no more
-		// than 64K flows).
-		binary.BigEndian.PutUint16(rawPkt[44:46], uint16(i%int(numStreams)))
-		if err := writePktTo.WritePacketData(rawPkt); err != nil {
-			log.Error("writing input packet", "case", string(caseToRun), "error", err)
-			return 1
+	// We started everything that could be started. So the best window for perf mertics
+	// opens somewhere around now.
+	begin := time.Now()
+	metricsBegin := begin.Unix()
+
+	numPkt := 0
+	for time.Since(begin) < testDuration {
+		// Check the time only once every 10000 packets
+		for i := 0; i < 10000; i++ {
+			// Rotate through flowIDs. We patch it directly into the SCION header of the packet. The
+			// SCION header starts at offset 42. The flowID is the 20 least significant bits of the
+			// first 32 bit field. To make our life simpler, we only use the last 16 bits (so no
+			// more than 64K flows).
+			binary.BigEndian.PutUint16(rawPkt[44:46], uint16(numPkt%int(numStreams)))
+			numPkt++
+			if err := writePktTo.WritePacketData(rawPkt); err != nil {
+				log.Error("writing input packet", "case", string(caseToRun), "error", err)
+				return 1
+			}
 		}
 	}
+
 	metricsEnd := time.Now().Unix()
-	// The test harness looks for this output.
+
+	// The test harness looks for this output. [metricsBegin, metricsEnd] needs to be fully
+	// contained in the period when we were actually transmitting, but can be a bit smaller.
 	fmt.Printf("metricsBegin: %d metricsEnd: %d\n", metricsBegin, metricsEnd)
 
 	// Get the results from the packet listener.
@@ -240,7 +251,7 @@ func run(cmd *cobra.Command) int {
 // The number of consumed packets is returned.
 // Currently we are content with receiving a single correct packet and we terminate after
 // that.
-func receivePackets(packetChan chan gopacket.Packet, payload string) int {
+func receivePackets(packetChan chan gopacket.Packet, payload []byte) int {
 	numRcv := 0
 
 	for {
@@ -251,15 +262,18 @@ func receivePackets(packetChan chan gopacket.Packet, payload string) int {
 			return numRcv
 		}
 		if err := got.ErrorLayer(); err != nil {
-			log.Error("error decoding packet", "err", err)
+			// This isn't an error. There is all sort of traffic that we might not know about
+			// and not be able to read.
+			// log.Error("error decoding packet", "err", err)
 			continue
 		}
 		layer := got.Layer(gopacket.LayerTypePayload)
 		if layer == nil {
-			log.Error("error fetching packet payload: no PayLoad")
+			// Don't treat this as an error. This could be random traffic we don't know about.
 			continue
 		}
-		if string(layer.LayerContents()) == payload {
+		if bytes.Equal(layer.LayerContents(), payload) {
+			// That's ours.
 			// To return the count of all packets received, just remove the "return" below.
 			// Return will occur once packetChan closes (which happens after a short timeout at
 			// the end of the test).

acceptance/router_benchmark/cases/BUILD.bazel

@@ -4,6 +4,7 @@ go_library(
     name = "go_default_library",
     srcs = [
         "br_transit.go",
+        "helpers.go",
         "in.go",
         "in_transit.go",
         "out.go",