Merge pull request #658 from kinvolk/alban/inceptiontap

Zhou Hao · web-flow · commit 2e13089356c7 · 2018-08-21T13:22:35.000+08:00
Fix TAP output with multiple RuntimeInsideValidate
diff --git a/docs/devel_guidelines.md b/docs/devel_guidelines.md
@@ -7,9 +7,52 @@ Each validation test has a `.go` file in the `validation/` directory and can be
 ### TAP output
 
 Each validation test prints TAP output.
-So far, we have two kinds of validation tests and they print the TAP output differently:
-* tests using `util.RuntimeInsideValidate`: they start the process `runtimetest` inside the container and `runtimetest` prints the TAP output. The test process itself must not output anything to avoid mixing its output with the TAP output. Each test can only call `util.RuntimeInsideValidate` one time because several TAP outputs cannot be concatenated.
-* tests using `util.RuntimeOutsideValidate`: they create a container but without executing `runtimetest`. The test program itself must print the TAP output.
+So far, we have three kinds of validation tests and they print the TAP output differently:
+
+#### Using `util.RuntimeOutsideValidate`
+
+They create a container but without executing `runtimetest`. The test program itself must print the TAP output.
+
+Example:
+```go
+err = util.RuntimeOutsideValidate(g, t, func(config *rspec.Spec, t *tap.T, state *rspec.State) error {
+        err := testFoo()
+        t.Ok((err == nil), "check foo")
+        if err != nil {
+                t.Diagnostic(err.Error())
+                return nil
+        }
+        return nil
+})
+
+```
+#### Using `util.RuntimeInsideValidate` and passthrough
+
+They start the process `runtimetest` inside the container and `runtimetest` prints the TAP output.
+The test process itself must not output anything to avoid mixing its output with the TAP output.
+Each test can only call `util.RuntimeInsideValidate` one time because several TAP outputs cannot be concatenated.
+
+Example:
+```go
+err = util.RuntimeInsideValidate(g, nil, nil)
+if err != nil {
+        util.Fatal(err)
+}
+```
+
+#### Using `util.RuntimeInsideValidate` and encapsulation
+
+Similar to the passthrough variant but the test consumes the output from `runtimetest` and re-emit a single TAP result for the container run.
+For that, the TAP object must be passed as parameter to `util.RuntimeInsideValidate`.
+
+Example:
+```go
+g.AddAnnotation("TestName", "check foo")
+err = util.RuntimeInsideValidate(g, t, nil)
+if err != nil {
+        util.Fatal(err)
+}
+```
 
 ### Exit status
 
diff --git a/validation/default/default.go b/validation/default/default.go
@@ -9,7 +9,7 @@ func main() {
 	if err != nil {
 		util.Fatal(err)
 	}
-	err = util.RuntimeInsideValidate(g, nil)
+	err = util.RuntimeInsideValidate(g, nil, nil)
 	if err != nil {
 		util.Fatal(err)
 	}
diff --git a/validation/hostname/hostname.go b/validation/hostname/hostname.go
@@ -15,7 +15,8 @@ func testHostname(t *tap.T, hostname string) error {
 	}
 
 	g.SetHostname(hostname)
-	err = util.RuntimeInsideValidate(g, nil)
+	g.AddAnnotation("TestName", fmt.Sprintf("check hostname %q", hostname))
+	err = util.RuntimeInsideValidate(g, t, nil)
 	t.Ok(err == nil, "hostname is set correctly")
 	if err != nil {
 		t.Diagnosticf("expect: err == nil, actual: err != nil")
diff --git a/validation/linux_devices/linux_devices.go b/validation/linux_devices/linux_devices.go
@@ -51,7 +51,7 @@ func main() {
 	pdev.FileMode = &pmode
 	g.AddDevice(pdev)
 
-	err = util.RuntimeInsideValidate(g, nil)
+	err = util.RuntimeInsideValidate(g, nil, nil)
 	if err != nil {
 		util.Fatal(err)
 	}
diff --git a/validation/linux_masked_paths/linux_masked_paths.go b/validation/linux_masked_paths/linux_masked_paths.go
@@ -6,11 +6,12 @@ import (
 	"os"
 	"path/filepath"
 
+	"github.com/mndrix/tap-go"
 	"github.com/opencontainers/runtime-tools/validation/util"
 	"golang.org/x/sys/unix"
 )
 
-func checkMaskedPaths() error {
+func checkMaskedPaths(t *tap.T) error {
 	g, err := util.GetDefaultGenerator()
 	if err != nil {
 		return err
@@ -32,7 +33,8 @@ func checkMaskedPaths() error {
 	g.AddLinuxMaskedPaths(maskedDirSub)
 	g.AddLinuxMaskedPaths(maskedFileSub)
 	g.AddLinuxMaskedPaths(maskedFileSubSub)
-	err = util.RuntimeInsideValidate(g, func(path string) error {
+	g.AddAnnotation("TestName", "check masked paths")
+	err = util.RuntimeInsideValidate(g, t, func(path string) error {
 		testDir := filepath.Join(path, maskedDirSub)
 		err = os.MkdirAll(testDir, 0777)
 		if err != nil {
@@ -63,7 +65,7 @@ func checkMaskedPaths() error {
 	return err
 }
 
-func checkMaskedRelPaths() error {
+func checkMaskedRelPaths(t *tap.T) error {
 	g, err := util.GetDefaultGenerator()
 	if err != nil {
 		return err
@@ -73,7 +75,8 @@ func checkMaskedRelPaths() error {
 	maskedRelPath := "masked-relpath"
 
 	g.AddLinuxMaskedPaths(maskedRelPath)
-	err = util.RuntimeInsideValidate(g, func(path string) error {
+	g.AddAnnotation("TestName", "check masked relative paths")
+	err = util.RuntimeInsideValidate(g, t, func(path string) error {
 		testFile := filepath.Join(path, maskedRelPath)
 		if _, err := os.Stat(testFile); err != nil && os.IsNotExist(err) {
 			return err
@@ -87,7 +90,7 @@ func checkMaskedRelPaths() error {
 	return fmt.Errorf("expected: err != nil, actual: err == nil")
 }
 
-func checkMaskedSymlinks() error {
+func checkMaskedSymlinks(t *tap.T) error {
 	g, err := util.GetDefaultGenerator()
 	if err != nil {
 		return err
@@ -98,7 +101,8 @@ func checkMaskedSymlinks() error {
 	maskedSymlink := "/masked-symlink"
 
 	g.AddLinuxMaskedPaths(maskedSymlink)
-	err = util.RuntimeInsideValidate(g, func(path string) error {
+	g.AddAnnotation("TestName", "check masked symlinks")
+	err = util.RuntimeInsideValidate(g, t, func(path string) error {
 		testFile := filepath.Join(path, maskedSymlink)
 		// ln -s .. /masked-symlink ; readlink -f /masked-symlink; ls -L /masked-symlink
 		if err := os.Symlink("../masked-symlink", testFile); err != nil {
@@ -121,7 +125,7 @@ func checkMaskedSymlinks() error {
 	return fmt.Errorf("expected: err != nil, actual: err == nil")
 }
 
-func checkMaskedDeviceNodes(mode uint32) error {
+func checkMaskedDeviceNodes(t *tap.T, mode uint32) error {
 	g, err := util.GetDefaultGenerator()
 	if err != nil {
 		return err
@@ -130,7 +134,8 @@ func checkMaskedDeviceNodes(mode uint32) error {
 	maskedDevice := "/masked-device"
 
 	g.AddLinuxMaskedPaths(maskedDevice)
-	return util.RuntimeInsideValidate(g, func(path string) error {
+	g.AddAnnotation("TestName", "check masked device nodes")
+	return util.RuntimeInsideValidate(g, t, func(path string) error {
 		testFile := filepath.Join(path, maskedDevice)
 
 		if err := unix.Mknod(testFile, mode, 0); err != nil {
@@ -146,15 +151,19 @@ func checkMaskedDeviceNodes(mode uint32) error {
 }
 
 func main() {
-	if err := checkMaskedPaths(); err != nil {
+	t := tap.New()
+	t.Header(0)
+	defer t.AutoPlan()
+
+	if err := checkMaskedPaths(t); err != nil {
 		util.Fatal(err)
 	}
 
-	if err := checkMaskedRelPaths(); err != nil {
+	if err := checkMaskedRelPaths(t); err != nil {
 		util.Fatal(err)
 	}
 
-	if err := checkMaskedSymlinks(); err != nil {
+	if err := checkMaskedSymlinks(t); err != nil {
 		util.Fatal(err)
 	}
 
@@ -167,7 +176,7 @@ func main() {
 	}
 
 	for _, m := range modes {
-		if err := checkMaskedDeviceNodes(m); err != nil {
+		if err := checkMaskedDeviceNodes(t, m); err != nil {
 			util.Fatal(err)
 		}
 	}
diff --git a/validation/linux_readonly_paths/linux_readonly_paths.go b/validation/linux_readonly_paths/linux_readonly_paths.go
@@ -6,11 +6,12 @@ import (
 	"os"
 	"path/filepath"
 
+	"github.com/mndrix/tap-go"
 	"github.com/opencontainers/runtime-tools/validation/util"
 	"golang.org/x/sys/unix"
 )
 
-func checkReadonlyPaths() error {
+func checkReadonlyPaths(t *tap.T) error {
 	g, err := util.GetDefaultGenerator()
 	if err != nil {
 		return err
@@ -32,7 +33,8 @@ func checkReadonlyPaths() error {
 	g.AddLinuxReadonlyPaths(readonlyDirSub)
 	g.AddLinuxReadonlyPaths(readonlyFileSub)
 	g.AddLinuxReadonlyPaths(readonlyFileSubSub)
-	err = util.RuntimeInsideValidate(g, func(path string) error {
+	g.AddAnnotation("TestName", "check read-only paths")
+	err = util.RuntimeInsideValidate(g, t, func(path string) error {
 		testDir := filepath.Join(path, readonlyDirSub)
 		err = os.MkdirAll(testDir, 0777)
 		if err != nil {
@@ -63,7 +65,7 @@ func checkReadonlyPaths() error {
 	return err
 }
 
-func checkReadonlyRelPaths() error {
+func checkReadonlyRelPaths(t *tap.T) error {
 	g, err := util.GetDefaultGenerator()
 	if err != nil {
 		return err
@@ -73,7 +75,8 @@ func checkReadonlyRelPaths() error {
 	readonlyRelPath := "readonly-relpath"
 
 	g.AddLinuxReadonlyPaths(readonlyRelPath)
-	err = util.RuntimeInsideValidate(g, func(path string) error {
+	g.AddAnnotation("TestName", "check read-only relative paths")
+	err = util.RuntimeInsideValidate(g, t, func(path string) error {
 		testFile := filepath.Join(path, readonlyRelPath)
 		if _, err := os.Stat(testFile); err != nil && os.IsNotExist(err) {
 			return err
@@ -87,7 +90,7 @@ func checkReadonlyRelPaths() error {
 	return fmt.Errorf("expected: err != nil, actual: err == nil")
 }
 
-func checkReadonlySymlinks() error {
+func checkReadonlySymlinks(t *tap.T) error {
 	g, err := util.GetDefaultGenerator()
 	if err != nil {
 		return err
@@ -98,7 +101,8 @@ func checkReadonlySymlinks() error {
 	readonlySymlink := "/readonly-symlink"
 
 	g.AddLinuxReadonlyPaths(readonlySymlink)
-	err = util.RuntimeInsideValidate(g, func(path string) error {
+	g.AddAnnotation("TestName", "check read-only symlinks")
+	err = util.RuntimeInsideValidate(g, t, func(path string) error {
 		testFile := filepath.Join(path, readonlySymlink)
 		// ln -s .. /readonly-symlink ; readlink -f /readonly-symlink; ls -L /readonly-symlink
 		if err := os.Symlink("../readonly-symlink", testFile); err != nil {
@@ -121,7 +125,7 @@ func checkReadonlySymlinks() error {
 	return fmt.Errorf("expected: err != nil, actual: err == nil")
 }
 
-func checkReadonlyDeviceNodes(mode uint32) error {
+func checkReadonlyDeviceNodes(t *tap.T, mode uint32) error {
 	g, err := util.GetDefaultGenerator()
 	if err != nil {
 		return err
@@ -130,7 +134,8 @@ func checkReadonlyDeviceNodes(mode uint32) error {
 	readonlyDevice := "/readonly-device"
 
 	g.AddLinuxReadonlyPaths(readonlyDevice)
-	return util.RuntimeInsideValidate(g, func(path string) error {
+	g.AddAnnotation("TestName", "check read-only device nodes")
+	return util.RuntimeInsideValidate(g, t, func(path string) error {
 		testFile := filepath.Join(path, readonlyDevice)
 
 		if err := unix.Mknod(testFile, mode, 0); err != nil {
@@ -146,15 +151,19 @@ func checkReadonlyDeviceNodes(mode uint32) error {
 }
 
 func main() {
-	if err := checkReadonlyPaths(); err != nil {
+	t := tap.New()
+	t.Header(0)
+	defer t.AutoPlan()
+
+	if err := checkReadonlyPaths(t); err != nil {
 		util.Fatal(err)
 	}
 
-	if err := checkReadonlyRelPaths(); err != nil {
+	if err := checkReadonlyRelPaths(t); err != nil {
 		util.Fatal(err)
 	}
 
-	if err := checkReadonlySymlinks(); err != nil {
+	if err := checkReadonlySymlinks(t); err != nil {
 		util.Fatal(err)
 	}
 
@@ -167,7 +176,7 @@ func main() {
 	}
 
 	for _, m := range modes {
-		if err := checkReadonlyDeviceNodes(m); err != nil {
+		if err := checkReadonlyDeviceNodes(t, m); err != nil {
 			util.Fatal(err)
 		}
 	}
diff --git a/validation/linux_rootfs_propagation/linux_rootfs_propagation.go b/validation/linux_rootfs_propagation/linux_rootfs_propagation.go
@@ -5,14 +5,15 @@ import (
 	"github.com/opencontainers/runtime-tools/validation/util"
 )
 
-func testLinuxRootPropagation(propMode string) error {
+func testLinuxRootPropagation(t *tap.T, propMode string) error {
 	g, err := util.GetDefaultGenerator()
 	if err != nil {
 		util.Fatal(err)
 	}
 	g.SetupPrivileged(true)
 	g.SetLinuxRootPropagation(propMode)
-	return util.RuntimeInsideValidate(g, nil)
+	g.AddAnnotation("TestName", "check root propagation")
+	return util.RuntimeInsideValidate(g, t, nil)
 }
 
 func main() {
@@ -28,7 +29,7 @@ func main() {
 	}
 
 	for _, c := range cases {
-		if err := testLinuxRootPropagation(c); err != nil {
+		if err := testLinuxRootPropagation(t, c); err != nil {
 			t.Fail(err.Error())
 		}
 	}
diff --git a/validation/linux_seccomp/linux_seccomp.go b/validation/linux_seccomp/linux_seccomp.go
@@ -16,7 +16,7 @@ func main() {
 	}
 	g.SetDefaultSeccompAction("allow")
 	g.SetSyscallAction(syscallArgs)
-	err = util.RuntimeInsideValidate(g, nil)
+	err = util.RuntimeInsideValidate(g, nil, nil)
 	if err != nil {
 		util.Fatal(err)
 	}
diff --git a/validation/linux_sysctl/linux_sysctl.go b/validation/linux_sysctl/linux_sysctl.go
@@ -10,7 +10,7 @@ func main() {
 		util.Fatal(err)
 	}
 	g.AddLinuxSysctl("net.ipv4.ip_forward", "1")
-	err = util.RuntimeInsideValidate(g, nil)
+	err = util.RuntimeInsideValidate(g, nil, nil)
 	if err != nil {
 		util.Fatal(err)
 	}
diff --git a/validation/linux_uid_mappings/linux_uid_mappings.go b/validation/linux_uid_mappings/linux_uid_mappings.go
@@ -12,7 +12,7 @@ func main() {
 	g.AddOrReplaceLinuxNamespace("user", "")
 	g.AddLinuxUIDMapping(uint32(1000), uint32(0), uint32(2000))
 	g.AddLinuxGIDMapping(uint32(1000), uint32(0), uint32(3000))
-	err = util.RuntimeInsideValidate(g, nil)
+	err = util.RuntimeInsideValidate(g, nil, nil)
 	if err != nil {
 		util.Fatal(err)
 	}
diff --git a/validation/mounts/mounts.go b/validation/mounts/mounts.go
@@ -22,7 +22,7 @@ func main() {
 		},
 	}
 	g.AddMount(mount)
-	err = util.RuntimeInsideValidate(g, nil)
+	err = util.RuntimeInsideValidate(g, nil, nil)
 	if err != nil {
 		util.Fatal(err)
 	}
diff --git a/validation/process/process.go b/validation/process/process.go
@@ -16,7 +16,7 @@ func main() {
 	g.AddProcessEnv("testa", "valuea")
 	g.AddProcessEnv("testb", "123")
 
-	err = util.RuntimeInsideValidate(g, func(path string) error {
+	err = util.RuntimeInsideValidate(g, nil, func(path string) error {
 		pathName := filepath.Join(path, "test")
 		return os.MkdirAll(pathName, 0700)
 	})
diff --git a/validation/process_capabilities/process_capabilities.go b/validation/process_capabilities/process_capabilities.go
@@ -18,7 +18,7 @@ func main() {
 		util.Fatal(err)
 	}
 	g.SetupPrivileged(true)
-	err = util.RuntimeInsideValidate(g, nil)
+	err = util.RuntimeInsideValidate(g, nil, nil)
 	if err != nil {
 		util.Fatal(err)
 	}
diff --git a/validation/process_capabilities_fail/process_capabilities_fail.go b/validation/process_capabilities_fail/process_capabilities_fail.go
@@ -21,7 +21,7 @@ func main() {
 		util.Fatal(err)
 	}
 	g.AddProcessCapabilityBounding("CAP_TEST")
-	err = util.RuntimeInsideValidate(g, nil)
+	err = util.RuntimeInsideValidate(g, nil, nil)
 	if err == nil {
 		util.Fatal(specerror.NewError(specerror.LinuxProcCapError, fmt.Errorf("Any value which cannot be mapped to a relevant kernel interface MUST cause an error"), rspecs.Version))
 	}
diff --git a/validation/process_oom_score_adj/process_oom_score_adj.go b/validation/process_oom_score_adj/process_oom_score_adj.go
diff --git a/validation/process_rlimits/process_rlimits.go b/validation/process_rlimits/process_rlimits.go
diff --git a/validation/process_rlimits_fail/process_rlimits_fail.go b/validation/process_rlimits_fail/process_rlimits_fail.go
diff --git a/validation/process_user/process_user.go b/validation/process_user/process_user.go
diff --git a/validation/root_readonly_true/root_readonly_true.go b/validation/root_readonly_true/root_readonly_true.go
diff --git a/validation/util/test.go b/validation/util/test.go

Original file line number	Diff line number	Diff line change
`@@ -9,7 +9,7 @@ func main() {`
`9`	`9`	`if err != nil {`
`10`	`10`	`util.Fatal(err)`
`11`	`11`	`}`
`12`		`- err = util.RuntimeInsideValidate(g, nil)`
	`12`	`+ err = util.RuntimeInsideValidate(g, nil, nil)`
`13`	`13`	`if err != nil {`
`14`	`14`	`util.Fatal(err)`
`15`	`15`	`}`
Original file line number	Diff line number	Diff line change
`@@ -15,7 +15,8 @@ func testHostname(t *tap.T, hostname string) error {`
`15`	`15`	`}`
`16`	`16`
`17`	`17`	`g.SetHostname(hostname)`
`18`		`- err = util.RuntimeInsideValidate(g, nil)`
	`18`	`+ g.AddAnnotation("TestName", fmt.Sprintf("check hostname %q", hostname))`
	`19`	`+ err = util.RuntimeInsideValidate(g, t, nil)`
`19`	`20`	`t.Ok(err == nil, "hostname is set correctly")`
`20`	`21`	`if err != nil {`
`21`	`22`	`t.Diagnosticf("expect: err == nil, actual: err != nil")`
Original file line number	Diff line number	Diff line change
`@@ -51,7 +51,7 @@ func main() {`
`51`	`51`	`pdev.FileMode = &pmode`
`52`	`52`	`g.AddDevice(pdev)`
`53`	`53`
`54`		`- err = util.RuntimeInsideValidate(g, nil)`
	`54`	`+ err = util.RuntimeInsideValidate(g, nil, nil)`
`55`	`55`	`if err != nil {`
`56`	`56`	`util.Fatal(err)`
`57`	`57`	`}`
Original file line number	Diff line number	Diff line change
`@@ -5,14 +5,15 @@ import (`
`5`	`5`	`"github.com/opencontainers/runtime-tools/validation/util"`
`6`	`6`	`)`
`7`	`7`
`8`		`-func testLinuxRootPropagation(propMode string) error {`
	`8`	`+func testLinuxRootPropagation(t *tap.T, propMode string) error {`
`9`	`9`	`g, err := util.GetDefaultGenerator()`
`10`	`10`	`if err != nil {`
`11`	`11`	`util.Fatal(err)`
`12`	`12`	`}`
`13`	`13`	`g.SetupPrivileged(true)`
`14`	`14`	`g.SetLinuxRootPropagation(propMode)`
`15`		`- return util.RuntimeInsideValidate(g, nil)`
	`15`	`+ g.AddAnnotation("TestName", "check root propagation")`
	`16`	`+ return util.RuntimeInsideValidate(g, t, nil)`
`16`	`17`	`}`
`17`	`18`
`18`	`19`	`func main() {`
`@@ -28,7 +29,7 @@ func main() {`
`28`	`29`	`}`
`29`	`30`
`30`	`31`	`for _, c := range cases {`
`31`		`- if err := testLinuxRootPropagation(c); err != nil {`
	`32`	`+ if err := testLinuxRootPropagation(t, c); err != nil {`
`32`	`33`	`t.Fail(err.Error())`
`33`	`34`	`}`
`34`	`35`	`}`
Original file line number	Diff line number	Diff line change
`@@ -16,7 +16,7 @@ func main() {`
`16`	`16`	`}`
`17`	`17`	`g.SetDefaultSeccompAction("allow")`
`18`	`18`	`g.SetSyscallAction(syscallArgs)`
`19`		`- err = util.RuntimeInsideValidate(g, nil)`
	`19`	`+ err = util.RuntimeInsideValidate(g, nil, nil)`
`20`	`20`	`if err != nil {`
`21`	`21`	`util.Fatal(err)`
`22`	`22`	`}`
Original file line number	Diff line number	Diff line change
`@@ -10,7 +10,7 @@ func main() {`
`10`	`10`	`util.Fatal(err)`
`11`	`11`	`}`
`12`	`12`	`g.AddLinuxSysctl("net.ipv4.ip_forward", "1")`
`13`		`- err = util.RuntimeInsideValidate(g, nil)`
	`13`	`+ err = util.RuntimeInsideValidate(g, nil, nil)`
`14`	`14`	`if err != nil {`
`15`	`15`	`util.Fatal(err)`
`16`	`16`	`}`
Original file line number	Diff line number	Diff line change
`@@ -12,7 +12,7 @@ func main() {`
`12`	`12`	`g.AddOrReplaceLinuxNamespace("user", "")`
`13`	`13`	`g.AddLinuxUIDMapping(uint32(1000), uint32(0), uint32(2000))`
`14`	`14`	`g.AddLinuxGIDMapping(uint32(1000), uint32(0), uint32(3000))`
`15`		`- err = util.RuntimeInsideValidate(g, nil)`
	`15`	`+ err = util.RuntimeInsideValidate(g, nil, nil)`
`16`	`16`	`if err != nil {`
`17`	`17`	`util.Fatal(err)`
`18`	`18`	`}`