testing: improved stream synthesis and cores use in router benchmark (scionproto#4457)

* testing: improved stream synthesis in router benchmark

Changes:
* Update the flowID of packets (and a couple of checksums) on the fly (just before
  sending) as opposed to pre-constructing a different packet for each stream.
* As a result, the test cases are simplified, they no-longer need to provide one
  packet per flowID.
* In passing, deleted the dumping of /proc/cpuinfo in the log. That has served its purpose.
* In the test harness, left in the form of comments, a suggestion of core partitioning. Not
  sure how to apply that in general. This is what provides the best results on my machine.

* testing: simplify the checksum updates out of the blaster.

Also enable core-partitioning to see how it behaves on the CI system.

* testing: Add check for proper saturation in the router benchmark.

In passing: fixed a couple of function names (camel-> snake)

* testing: fix expectation for droppage in the router benchmark

Was expressed in pkts instead of %.

* testing: cosmetic change to the output of expected drop ratio.

* testing: linter friendship

* testing: include a cpu selection in the router benchmark

Also: fixed the broken prom query string.

* testing: please the linter.

* testing: log lscpus on the CI system.

* testing: fix bad string formating.

* testing: Improve core selection in the router benchmark.

Added L2 cache criteria. The test harness will prefer configuration where either cores share no L2 cache or all cores share the same one.

* testing: make the router benchmark test harness compatible with more variants
of lscpu (hopefully).

* testing: log the CPU serial number in the router benchmark.

This is done as an attempt at detecting live migration.

* build: added dmidecode as a required package.

* build: Complete the list of files that affect the list of packages to install.

Three files were missing, most notably the pkgs.txt files which are typically the ones
modified when adding a dependency. As a result new packages weren't getting installed.

* build: Delete use of dmidecode.

This proved useless as a means of spotting migrations. The information doesn't
even change from instance to instance.

* build: fix trailing blank line.

* router: reduce the benchmarks expectations for saturation.

After fixing an arithmetic error, it seems we are not able to reliably cause the
router to drop more than 1% of the packets (and then sometimes no even that).
Until we figure out how to reach higher levels, making sure that other PRs
aren't blocked.

* testing: fixed bugs in router_benchmark core selection code.

* testing: update the expectations of the router benchmark.

After switching to non-overbooked aws instances, performance has become much better
and repeatable.

Author: jiceathome

.buildkite/provision-agent.sh

@@ -13,4 +13,4 @@ echo "~~~ Install build tools"
 tools/install_bazel
 tools/install_deps
 
-sha1sum tools/install_bazel tools/install_deps tools/env/pip3/deps tools/env/pip3/requirements.txt tools/env/rhel/deps > /tmp/buildkite-scionproto-runner-provision.sum
+sha1sum tools/install_bazel tools/install_deps tools/env/pip3/deps tools/env/pip3/requirements.txt tools/env/rhel/deps tools/env/rhel/pkgs.txt tools/env/debian/deps tools/env/debian/pkgs.txt > /tmp/buildkite-scionproto-runner-provision.sum

acceptance/router_benchmark/brload/main.go

@@ -15,6 +15,7 @@
 package main
 
 import (
+	"encoding/binary"
 	"errors"
 	"fmt"
 	"hash"
@@ -38,7 +39,7 @@ import (
 	"github.com/scionproto/scion/private/keyconf"
 )
 
-type Case func(payload string, mac hash.Hash, numDistinct int) (string, string, [][]byte)
+type Case func(payload string, mac hash.Hash) (string, string, []byte)
 
 type caseChoice string
 
@@ -74,7 +75,7 @@ var (
 	logConsole string
 	dir        string
 	numPackets int
-	numStreams int
+	numStreams uint16
 	caseToRun  caseChoice
 	interfaces []string
 )
@@ -99,7 +100,7 @@ func main() {
 		},
 	}
 	runCmd.Flags().IntVar(&numPackets, "num-packets", 10, "Number of packets to send")
-	runCmd.Flags().IntVar(&numStreams, "num-streams", 4,
+	runCmd.Flags().Uint16Var(&numStreams, "num-streams", 4,
 		"Number of independent streams (flowID) to use")
 	runCmd.Flags().StringVar(&logConsole, "log.console", "error",
 		"Console logging level: debug|info|error|etc.")
@@ -136,8 +137,6 @@ func run(cmd *cobra.Command) int {
 	}
 	defer log.HandlePanic()
 
-	caseFunc := allCases[string(caseToRun)] // key already checked.
-
 	artifactsDir := dir
 	if v := os.Getenv("TEST_ARTIFACTS_DIR"); v != "" {
 		artifactsDir = v
@@ -166,7 +165,8 @@ func run(cmd *cobra.Command) int {
 	log.Info("BRLoad acceptance tests:")
 
 	payloadString := "actualpayloadbytes"
-	caseDevIn, caseDevOut, rawPkts := caseFunc(payloadString, hfMAC, numStreams)
+	caseFunc := allCases[string(caseToRun)] // key already checked.
+	caseDevIn, caseDevOut, rawPkt := caseFunc(payloadString, hfMAC)
 
 	writePktTo, ok := handles[caseDevIn]
 	if !ok {
@@ -195,8 +195,18 @@ func run(cmd *cobra.Command) int {
 	// We started everything that could be started. So the best window for perf mertics
 	// opens somewhere around now.
 	metricsBegin := time.Now().Unix()
+	// Because we're using IPV4 only, the UDP checksum is optional, so we are allowed to
+	// just set it to zero instead of recomputing it. The IP checksum does not cover the payload, so
+	// we don't need to update it.
+	binary.BigEndian.PutUint16(rawPkt[40:42], 0)
+
 	for i := 0; i < numPackets; i++ {
-		if err := writePktTo.WritePacketData(rawPkts[i%numStreams]); err != nil {
+		// Rotate through flowIDs. We patch it directly into the SCION header of the packet.  The
+		// SCION header starts at offset 42. The flowID is the 20 least significant bits of the
+		// first 32 bit field. To make our life simpler, we only use the last 16 bits (so no more
+		// than 64K flows).
+		binary.BigEndian.PutUint16(rawPkt[44:46], uint16(i%int(numStreams)))
+		if err := writePktTo.WritePacketData(rawPkt); err != nil {
 			log.Error("writing input packet", "case", string(caseToRun), "error", err)
 			return 1
 		}
@@ -255,7 +265,7 @@ func receivePackets(packetChan chan gopacket.Packet, payload string) int {
 		if string(layer.LayerContents()) == payload {
 			// To return the count of all packets received, just remove the "return" below.
 			// Return will occur once packetChan closes (which happens after a short timeout at
-			// the end of the test.
+			// the end of the test).
 			numRcv++
 			return numRcv
 		}

acceptance/router_benchmark/cases/br_transit.go

@@ -32,9 +32,9 @@ import (
 //    AS3 (br3) ---+
 // See topo.go
 
-// oneBrTransit generates one packet of "br_transit" traffic over the router under test.
-// The outcome is a raw packet which the test must feed to the router.
-func oneBrTransit(payload string, mac hash.Hash, flowId uint32) []byte {
+// BrTransit generates one packet of "br_transit" traffic over the router under test.
+// The outcome is a raw packet which the test must feed to the router. The flowID field is 0.
+func BrTransit(payload string, mac hash.Hash) (string, string, []byte) {
 
 	var (
 		originIA       = ISDAS(2)
@@ -100,7 +100,7 @@ func oneBrTransit(payload string, mac hash.Hash, flowId uint32) []byte {
 	scionL := &slayers.SCION{
 		Version:      0,
 		TrafficClass: 0xb8,
-		FlowID:       flowId,
+		FlowID:       0,
 		NextHdr:      slayers.L4UDP,
 		PathType:     scion.PathType,
 		SrcIA:        originIA,
@@ -128,17 +128,6 @@ func oneBrTransit(payload string, mac hash.Hash, flowId uint32) []byte {
 	); err != nil {
 		panic(err)
 	}
-	return input.Bytes()
-}
 
-// BrTransit generates numDistinct packets (each with a unique flowID) with the given payload
-// constructed to cause br_transit traffic at the br1a router.
-// numDistrinct is a small number, only to enable multiple parallel streams. Each distinct packet
-// is meant to be replayed a large number of times for performance measurement.
-func BrTransit(payload string, mac hash.Hash, numDistinct int) (string, string, [][]byte) {
-	packets := make([][]byte, numDistinct)
-	for i := 0; i < numDistinct; i++ {
-		packets[i] = oneBrTransit(payload, mac, uint32(i+1))
-	}
-	return DeviceName(1, 2), DeviceName(1, 3), packets
+	return DeviceName(1, 2), DeviceName(1, 3), input.Bytes()
 }

acceptance/router_benchmark/cases/in.go

@@ -33,8 +33,8 @@ import (
 // See topo.go
 
 // oneIn generates one packet of incoming traffic into AS1 at br1a. The outcome is a raw packet
-// that the test must feed into the router.
-func oneIn(payload string, mac hash.Hash, flowId uint32) []byte {
+// that the test must feed into the router. The flow ID is 0.
+func In(payload string, mac hash.Hash) (string, string, []byte) {
 
 	var (
 		originIA       = ISDAS(2)
@@ -87,7 +87,7 @@ func oneIn(payload string, mac hash.Hash, flowId uint32) []byte {
 	scionL := &slayers.SCION{
 		Version:      0,
 		TrafficClass: 0xb8,
-		FlowID:       flowId,
+		FlowID:       0,
 		NextHdr:      slayers.L4UDP,
 		PathType:     scion.PathType,
 		SrcIA:        originIA,
@@ -115,17 +115,5 @@ func oneIn(payload string, mac hash.Hash, flowId uint32) []byte {
 	); err != nil {
 		panic(err)
 	}
-	return input.Bytes()
-}
-
-// In generates numDistinct packets (each with a unique flowID) with the given payload
-// constructed to cause "in" traffic at the br1a router.
-// numDistrinct is a small number, only to enable multiple parallel streams. Each distinct packet
-// is meant to be replayed a large number of times for performance measurement.
-func In(payload string, mac hash.Hash, numDistinct int) (string, string, [][]byte) {
-	packets := make([][]byte, numDistinct)
-	for i := 0; i < numDistinct; i++ {
-		packets[i] = oneIn(payload, mac, uint32(i+1))
-	}
-	return DeviceName(1, 2), DeviceName(1, 0), packets
+	return DeviceName(1, 2), DeviceName(1, 0), input.Bytes()
 }

acceptance/router_benchmark/cases/in_transit.go

@@ -33,8 +33,8 @@ import (
 // See topo.go
 
 // oneInTransit generates one packet of "in_transit" traffic over the router under test.
-// The outcome is a raw packet that the test must feed into the router.
-func oneInTransit(payload string, mac hash.Hash, flowId uint32) []byte {
+// The outcome is a raw packet that the test must feed into the router. The flow ID is 0.
+func InTransit(payload string, mac hash.Hash) (string, string, []byte) {
 
 	var (
 		originIA       = ISDAS(2)
@@ -100,7 +100,7 @@ func oneInTransit(payload string, mac hash.Hash, flowId uint32) []byte {
 	scionL := &slayers.SCION{
 		Version:      0,
 		TrafficClass: 0xb8,
-		FlowID:       flowId,
+		FlowID:       0,
 		NextHdr:      slayers.L4UDP,
 		PathType:     scion.PathType,
 		SrcIA:        originIA,
@@ -128,17 +128,5 @@ func oneInTransit(payload string, mac hash.Hash, flowId uint32) []byte {
 	); err != nil {
 		panic(err)
 	}
-	return input.Bytes()
-}
-
-// InTransit generates numDistinct packets (each with a unique flowID) with the given payload
-// constructed to cause in_transit traffic at the br1a router.
-// numDistrinct is a small number, only to enable multiple parallel streams. Each distinct packet
-// is meant to be replayed a large number of times for performance measurement.
-func InTransit(payload string, mac hash.Hash, numDistinct int) (string, string, [][]byte) {
-	packets := make([][]byte, numDistinct)
-	for i := 0; i < numDistinct; i++ {
-		packets[i] = oneInTransit(payload, mac, uint32(i+1))
-	}
-	return DeviceName(1, 2), DeviceName(1, 0), packets
+	return DeviceName(1, 2), DeviceName(1, 0), input.Bytes()
 }

acceptance/router_benchmark/cases/out.go

@@ -33,8 +33,8 @@ import (
 // See topo.go
 
 // oneOut generates one packet of outgoing traffic from AS1 at br1a. The outcome is a raw packet
-// that the test must feed into the router.
-func oneOut(payload string, mac hash.Hash, flowId uint32) []byte {
+// that the test must feed into the router. The flow ID is 0.
+func Out(payload string, mac hash.Hash) (string, string, []byte) {
 
 	var (
 		originIA       = ISDAS(1)
@@ -88,7 +88,7 @@ func oneOut(payload string, mac hash.Hash, flowId uint32) []byte {
 	scionL := &slayers.SCION{
 		Version:      0,
 		TrafficClass: 0xb8,
-		FlowID:       flowId,
+		FlowID:       0,
 		NextHdr:      slayers.L4UDP,
 		PathType:     scion.PathType,
 		SrcIA:        originIA,
@@ -117,17 +117,5 @@ func oneOut(payload string, mac hash.Hash, flowId uint32) []byte {
 		panic(err)
 	}
 
-	return input.Bytes()
-}
-
-// Out generates numDistinct packets (each with a unique flowID) with the given payload
-// constructed to cause "out" traffic at the br1a router.
-// numDistrinct is a small number, only to enable multiple parallel streams. Each distinct packet
-// is meant to be replayed a large number of times for performance measurement.
-func Out(payload string, mac hash.Hash, numDistinct int) (string, string, [][]byte) {
-	packets := make([][]byte, numDistinct)
-	for i := 0; i < numDistinct; i++ {
-		packets[i] = oneOut(payload, mac, uint32(i+1))
-	}
-	return DeviceName(1, 0), DeviceName(1, 2), packets
+	return DeviceName(1, 0), DeviceName(1, 2), input.Bytes()
 }

acceptance/router_benchmark/cases/out_transit.go

@@ -33,8 +33,8 @@ import (
 // See topo.go
 
 // oneOutTransit generates one packet of "out_transit" traffic over the router under test.
-// The outcome is a raw packet that the test must feed to the router.
-func oneOutTransit(payload string, mac hash.Hash, flowId uint32) []byte {
+// The outcome is a raw packet that the test must feed to the router. The flow ID is 0.
+func OutTransit(payload string, mac hash.Hash) (string, string, []byte) {
 
 	var (
 		originIA       = ISDAS(4)
@@ -101,7 +101,7 @@ func oneOutTransit(payload string, mac hash.Hash, flowId uint32) []byte {
 	scionL := &slayers.SCION{
 		Version:      0,
 		TrafficClass: 0xb8,
-		FlowID:       flowId,
+		FlowID:       0,
 		NextHdr:      slayers.L4UDP,
 		PathType:     scion.PathType,
 		SrcIA:        originIA,
@@ -129,17 +129,5 @@ func oneOutTransit(payload string, mac hash.Hash, flowId uint32) []byte {
 	); err != nil {
 		panic(err)
 	}
-	return input.Bytes()
-}
-
-// OutTransit generates numDistinct packets (each with a unique flowID) with the given payload
-// constructed to cause out_transit traffic at the br1a router.
-// numDistrinct is a small number, only to enable multiple parallel streams. Each distinct packet
-// is meant to be replayed a large number of times for performance measurement.
-func OutTransit(payload string, mac hash.Hash, numDistinct int) (string, string, [][]byte) {
-	packets := make([][]byte, numDistinct)
-	for i := 0; i < numDistinct; i++ {
-		packets[i] = oneOutTransit(payload, mac, uint32(i+1))
-	}
-	return DeviceName(1, 0), DeviceName(1, 2), packets
+	return DeviceName(1, 0), DeviceName(1, 2), input.Bytes()
 }