integration test cleanup (#206)

Cleanup and simplify integration tests and test framework.

- collect full stdout/sterr output into a buffer to allow matching on
  complete output. This simplifies the output matchers (in particular,
  `NoPanic`) and allows to write out much more helpful error messages.

  Unfortunately, the os/exec package has some quirks when capturing
  stdout/stderr of grand-child processes, requiring some workarounds.
  This is described in the new helper function `InputPipeScript`, used
  (only) in the sensorapp integration test.
- add `DefaultIAPairs` function which returns shortened list of test
  pairs with reasonable subset of relevant cases.
- simplify table-driven tests with single case entry
- netcat: completely overhaul integration test setup. Needs new -q flag.
- bwtester: provide duration in test parameters to shorten run
- camerapp: change regex to allow 'T' (timeout with re-try) in output
- change `appnet.Listen` to ensure "ReadySignal" is only printed when
  Listen is successful
- remove various quirks with Chdir around that are not/no longer needed
- call `integration.Init` from `TestMain` in integration tests instead of
  during the test functions. Previously, flags were not be parsed
  correctly and repeated invocation with `go test --count` would panic.
- Makefile: run `go test` with `--count=1` to ensure integration tests
  are re-run when test files are unchanged.

Author: matzf

Makefile

@@ -50,7 +50,7 @@ install: all
 	cp -t $(DESTDIR) $(BIN)/scion-*
 
 integration: build
-	go test -v -tags=integration,$(TAGS) ./... ./_examples/helloworld/
+	go test -tags=integration,$(TAGS) --count=1 ./... ./_examples/helloworld/
 
 .PHONY: scion-bat
 scion-bat:

_examples/helloworld/helloworld_integration_test.go

@@ -17,67 +17,38 @@
 package main
 
 import (
-	"strings"
 	"testing"
 
 	"github.com/netsec-ethz/scion-apps/pkg/integration"
 )
 
 const (
-	name = "helloworld"
-	bin  = "example-helloworld"
+	bin = "example-helloworld"
 )
 
+func TestMain(m *testing.M) {
+	integration.TestMain(m)
+}
+
 func TestHelloworldSample(t *testing.T) {
-	if err := integration.Init(name); err != nil {
-		t.Fatalf("Failed to init: %s\n", err)
-	}
 	cmd := integration.AppBinPath(bin)
-	// Common arguments
-	cmnArgs := []string{}
 	// Server
 	serverPort := "12345"
 	serverArgs := []string{"-port", serverPort}
-	serverArgs = append(serverArgs, cmnArgs...)
-
-	testCases := []struct {
-		Name              string
-		Args              []string
-		ServerOutMatchFun func(bool, string) bool
-		ServerErrMatchFun func(bool, string) bool
-		ClientOutMatchFun func(bool, string) bool
-		ClientErrMatchFun func(bool, string) bool
-	}{
-		{
-			"client_hello",
-			append([]string{"-remote", integration.DstAddrPattern + ":" + serverPort}, cmnArgs...),
-			func(prev bool, line string) bool {
-				res := strings.Contains(line, "hello world")
-				return prev || res // return true if any output line contains the string
-			},
-			nil,
-			integration.Contains("Done. Wrote 11 bytes."),
-			nil,
-		},
-	}
 
-	for _, tc := range testCases {
-		in := integration.NewAppsIntegration(name, tc.Name, cmd, cmd, tc.Args, serverArgs, true)
-		in.ServerStdout(tc.ServerOutMatchFun)
-		in.ServerStderr(tc.ServerErrMatchFun)
-		in.ClientStdout(tc.ClientOutMatchFun)
-		in.ClientStderr(tc.ClientErrMatchFun)
-		// Host address pattern
-		hostAddr := integration.HostAddr
-		// Cartesian product of src and dst IAs, is a random permutation
-		// can be restricted to a subset to reduce the number of tests to run without significant
-		// loss of coverage
-		IAPairs := integration.IAPairs(hostAddr)
-		IAPairs = IAPairs[:len(IAPairs)/2]
-		// Run the tests to completion or until a test fails,
-		// increase the client timeout if clients need more time to start
-		if err := integration.RunTests(in, IAPairs, integration.DefaultClientTimeout, 0); err != nil {
-			t.Fatalf("Error during tests err: %v", err)
-		}
+	// Client
+	clientArgs := []string{"-remote", integration.DstAddrPattern + ":" + serverPort}
+
+	in := integration.NewAppsIntegration(cmd, cmd, clientArgs, serverArgs)
+	in.ServerOutMatch = integration.Contains("hello world")
+	in.ClientOutMatch = integration.Contains("Done. Wrote 11 bytes.")
+	// Cartesian product of src and dst IAs, a random permutation
+	// restricted to a subset to reduce the number of tests to run without significant
+	// loss of coverage
+	iaPairs := integration.DefaultIAPairs()
+	// Run the tests to completion or until a test fails,
+	// increase the ClientTimeout if clients need more time to start
+	if err := in.Run(t, iaPairs); err != nil {
+		t.Error(err)
 	}
 }

bwtester/bwtestclient/bwtestclient_integration_test.go

@@ -17,66 +17,36 @@
 package main
 
 import (
-	"github.com/netsec-ethz/scion-apps/pkg/integration"
-	"strings"
 	"testing"
+
+	"github.com/netsec-ethz/scion-apps/pkg/integration"
 )
 
 const (
-	name      = "bwtester"
 	clientBin = "scion-bwtestclient"
 	serverBin = "scion-bwtestserver"
 )
 
+func TestMain(m *testing.M) {
+	integration.TestMain(m)
+}
+
 func TestIntegrationBwtestclient(t *testing.T) {
-	if err := integration.Init(name); err != nil {
-		t.Fatalf("Failed to init: %s\n", err)
-	}
-  clientCmd := integration.AppBinPath(clientBin)
-  serverCmd := integration.AppBinPath(serverBin)
+	clientCmd := integration.AppBinPath(clientBin)
+	serverCmd := integration.AppBinPath(serverBin)
 
-	// Common arguments
-	cmnArgs := []string{}
 	// Server
 	serverPort := "40002"
 	serverArgs := []string{"-p", serverPort}
-	serverArgs = append(serverArgs, cmnArgs...)
-
-	testCases := []struct {
-		Name              string
-		Args              []string
-		ServerOutMatchFun func(bool, string) bool
-		ServerErrMatchFun func(bool, string) bool
-		ClientOutMatchFun func(bool, string) bool
-		ClientErrMatchFun func(bool, string) bool
-	}{
-		{
-			"bandwidth_client",
-			append([]string{"-s", integration.DstAddrPattern + ":" + serverPort, "-cs", "1Mbps"}, cmnArgs...),
-			func(prev bool, line string) bool {
-				res := strings.Contains(line, "Received request")
-				return prev || res // return true if any output line contains the string
-			},
-			nil,
-			integration.RegExp("^Achieved bandwidth: \\d+ bps / \\d+.\\d+ [Mk]bps$"),
-			nil,
-		},
-	}
-
-	for _, tc := range testCases {
-		in := integration.NewAppsIntegration(name, tc.Name, clientCmd, serverCmd, tc.Args, serverArgs, true)
-		in.ServerStdout(tc.ServerOutMatchFun)
-		in.ServerStderr(tc.ServerErrMatchFun)
-		in.ClientStdout(tc.ClientOutMatchFun)
-		in.ClientStderr(tc.ClientErrMatchFun)
-
-		hostAddr := integration.HostAddr
+	// Client
+	clientArgs := []string{"-s", integration.DstAddrPattern + ":" + serverPort, "-cs", "1,?,?,1Mbps"}
 
-		IAPairs := integration.IAPairs(hostAddr)
-		IAPairs = IAPairs[:1]
+	in := integration.NewAppsIntegration(clientCmd, serverCmd, clientArgs, serverArgs)
+	in.ServerOutMatch = integration.Contains("Received request")
+	in.ClientOutMatch = integration.RegExp("(?m)^Achieved bandwidth: \\d+ bps / \\d+.\\d+ [Mk]bps$")
 
-		if err := integration.RunTests(in, IAPairs, integration.DefaultClientTimeout, 0); err != nil {
-			t.Fatalf("Error during tests err: %v", err)
-		}
+	iaPairs := integration.DefaultIAPairs()
+	if err := in.Run(t, iaPairs); err != nil {
+		t.Error(err)
 	}
 }

camerapp/camerapp_integration_test.go

@@ -18,8 +18,6 @@ package main
 
 import (
 	"fmt"
-	"io/ioutil"
-	"os"
 	"os/exec"
 	"path"
 	"regexp"
@@ -29,85 +27,51 @@ import (
 )
 
 const (
-	name      = "camerapp"
 	clientBin = "scion-imagefetcher"
 	serverBin = "scion-imageserver"
 )
 
+func TestMain(m *testing.M) {
+	integration.TestMain(m)
+}
+
 func TestIntegrationImagefetcher(t *testing.T) {
-	if err := integration.Init(name); err != nil {
-		t.Fatalf("Failed to init: %s\n", err)
-	}
 	clientCmd := integration.AppBinPath(clientBin)
 	serverCmd := integration.AppBinPath(serverBin)
 
-	// Common arguments
-	cmnArgs := []string{}
 	// Server
 	serverPort := "42002"
 	serverArgs := []string{"-p", serverPort, "-d", "testdata"}
-	serverArgs = append(serverArgs, cmnArgs...)
 
 	// Sample file path
 	sample := path.Join("testdata", "logo.jpg")
-	// Image fetcher output directory
-	sampleOutputDir, err := ioutil.TempDir("", fmt.Sprintf("%s_integration_output", name))
-	sampleOuput := path.Join(sampleOutputDir, "download.jpg")
-	if err != nil {
-		t.Fatalf("Error during setup err: %v", err)
-	}
-	defer os.RemoveAll(sampleOutputDir)
 
-	testCases := []struct {
-		Name              string
-		Args              []string
-		ServerOutMatchFun func(bool, string) bool
-		ServerErrMatchFun func(bool, string) bool
-		ClientOutMatchFun func(bool, string) bool
-		ClientErrMatchFun func(bool, string) bool
-	}{
-		{
-			"fetch_image",
-			append([]string{"-s", integration.DstAddrPattern + ":" + serverPort, "-output", sampleOuput}, cmnArgs...),
-			nil,
-			nil,
-			func(prev bool, line string) bool {
-				if !prev {
-					matched, err := regexp.MatchString("^r+[.r]+$", line)
-					if err == nil {
-						return matched
-					}
-				} else {
-					matched, err := regexp.MatchString("^Done, exiting. Total duration \\d+\\.\\d+m?s$", line)
-					if err != nil || !matched {
-						return false
-					}
-					// The image was downloaded, compare it with the source
-					cmd := exec.Command("cmp", "-l", sampleOuput, sample)
-					// cmp exits with 0 exit status if the files are identical, and err is nil if the exit status is 0
-					err = cmd.Run()
-					return err == nil
-				}
-				return prev
-			},
-			nil,
-		},
-	}
-
-	for _, tc := range testCases {
-		in := integration.NewAppsIntegration(name, tc.Name, clientCmd, serverCmd, tc.Args, serverArgs, true)
-		in.ServerStdout(tc.ServerOutMatchFun)
-		in.ServerStderr(tc.ServerErrMatchFun)
-		in.ClientStdout(tc.ClientOutMatchFun)
-		in.ClientStderr(tc.ClientErrMatchFun)
+	// Client
+	// Image fetcher output directory
+	outputDir := t.TempDir()
+	sampleOutput := path.Join(outputDir, "download.jpg")
 
-		hostAddr := integration.HostAddr
+	clientArgs := []string{"-s", integration.DstAddrPattern + ":" + serverPort, "-output", sampleOutput}
 
-		IAPairs := integration.IAPairs(hostAddr)
-		IAPairs = IAPairs[:1]
+	in := integration.NewAppsIntegration(clientCmd, serverCmd, clientArgs, serverArgs)
+	in.ClientOutMatch = func(out string) error {
+		re := regexp.MustCompile(`^r[.rT]+\nDone, exiting. Total duration \d+\.\d+m?s\n$`)
+		if !re.MatchString(out) {
+			return fmt.Errorf("does not match regexp '%s'", re)
+		}
 
-		if err := integration.RunTests(in, IAPairs, integration.DefaultClientTimeout, 0); err != nil {
-			t.Fatalf("Error during tests err: %v", err)
+		// The image was downloaded, compare it with the source
+		cmd := exec.Command("cmp", "--verbose", sampleOutput, sample)
+		// cmp exits with 0 exit status if the files are identical, and err is nil if the exit status is 0
+		if out, err := cmd.CombinedOutput(); err != nil {
+			return fmt.Errorf("comparing downloaded to ground truth: %w\ncommand:\n%s\noutput:\n%s\n",
+				err, cmd, out)
 		}
+		return nil
+	}
+
+	iaPairs := integration.DefaultIAPairs()
+	if err := in.Run(t, iaPairs); err != nil {
+		t.Error(err)
 	}
 }