RuntimeInsideValidate: can now be called several times

alban · alban · commit 270145ae9a84 · 2018-08-14T15:50:51.000+02:00
Signed-off-by: Alban Crequy &lt;alban@kinvolk.io&gt;
diff --git a/validation/default/default.go b/validation/default/default.go
@@ -9,7 +9,7 @@ func main() {
 	if err != nil {
 		util.Fatal(err)
 	}
-	err = util.RuntimeInsideValidate(g, nil)
+	err = util.RuntimeInsideValidate(g, nil, nil)
 	if err != nil {
 		util.Fatal(err)
 	}
diff --git a/validation/hostname/hostname.go b/validation/hostname/hostname.go
@@ -15,7 +15,7 @@ func testHostname(t *tap.T, hostname string) error {
 	}
 
 	g.SetHostname(hostname)
-	err = util.RuntimeInsideValidate(g, nil)
+	err = util.RuntimeInsideValidate(g, nil, nil)
 	t.Ok(err == nil, "hostname is set correctly")
 	if err != nil {
 		t.Diagnosticf("expect: err == nil, actual: err != nil")
diff --git a/validation/linux_devices/linux_devices.go b/validation/linux_devices/linux_devices.go
@@ -51,7 +51,7 @@ func main() {
 	pdev.FileMode = &pmode
 	g.AddDevice(pdev)
 
-	err = util.RuntimeInsideValidate(g, nil)
+	err = util.RuntimeInsideValidate(g, nil, nil)
 	if err != nil {
 		util.Fatal(err)
 	}
diff --git a/validation/linux_masked_paths/linux_masked_paths.go b/validation/linux_masked_paths/linux_masked_paths.go
@@ -32,7 +32,7 @@ func checkMaskedPaths() error {
 	g.AddLinuxMaskedPaths(maskedDirSub)
 	g.AddLinuxMaskedPaths(maskedFileSub)
 	g.AddLinuxMaskedPaths(maskedFileSubSub)
-	err = util.RuntimeInsideValidate(g, func(path string) error {
+	err = util.RuntimeInsideValidate(g, nil, func(path string) error {
 		testDir := filepath.Join(path, maskedDirSub)
 		err = os.MkdirAll(testDir, 0777)
 		if err != nil {
@@ -73,7 +73,7 @@ func checkMaskedRelPaths() error {
 	maskedRelPath := "masked-relpath"
 
 	g.AddLinuxMaskedPaths(maskedRelPath)
-	err = util.RuntimeInsideValidate(g, func(path string) error {
+	err = util.RuntimeInsideValidate(g, nil, func(path string) error {
 		testFile := filepath.Join(path, maskedRelPath)
 		if _, err := os.Stat(testFile); err != nil && os.IsNotExist(err) {
 			return err
@@ -98,7 +98,7 @@ func checkMaskedSymlinks() error {
 	maskedSymlink := "/masked-symlink"
 
 	g.AddLinuxMaskedPaths(maskedSymlink)
-	err = util.RuntimeInsideValidate(g, func(path string) error {
+	err = util.RuntimeInsideValidate(g, nil, func(path string) error {
 		testFile := filepath.Join(path, maskedSymlink)
 		// ln -s .. /masked-symlink ; readlink -f /masked-symlink; ls -L /masked-symlink
 		if err := os.Symlink("../masked-symlink", testFile); err != nil {
@@ -130,7 +130,7 @@ func checkMaskedDeviceNodes(mode uint32) error {
 	maskedDevice := "/masked-device"
 
 	g.AddLinuxMaskedPaths(maskedDevice)
-	return util.RuntimeInsideValidate(g, func(path string) error {
+	return util.RuntimeInsideValidate(g, nil, func(path string) error {
 		testFile := filepath.Join(path, maskedDevice)
 
 		if err := unix.Mknod(testFile, mode, 0); err != nil {
diff --git a/validation/linux_readonly_paths/linux_readonly_paths.go b/validation/linux_readonly_paths/linux_readonly_paths.go
@@ -32,7 +32,7 @@ func checkReadonlyPaths() error {
 	g.AddLinuxReadonlyPaths(readonlyDirSub)
 	g.AddLinuxReadonlyPaths(readonlyFileSub)
 	g.AddLinuxReadonlyPaths(readonlyFileSubSub)
-	err = util.RuntimeInsideValidate(g, func(path string) error {
+	err = util.RuntimeInsideValidate(g, nil, func(path string) error {
 		testDir := filepath.Join(path, readonlyDirSub)
 		err = os.MkdirAll(testDir, 0777)
 		if err != nil {
@@ -73,7 +73,7 @@ func checkReadonlyRelPaths() error {
 	readonlyRelPath := "readonly-relpath"
 
 	g.AddLinuxReadonlyPaths(readonlyRelPath)
-	err = util.RuntimeInsideValidate(g, func(path string) error {
+	err = util.RuntimeInsideValidate(g, nil, func(path string) error {
 		testFile := filepath.Join(path, readonlyRelPath)
 		if _, err := os.Stat(testFile); err != nil && os.IsNotExist(err) {
 			return err
@@ -98,7 +98,7 @@ func checkReadonlySymlinks() error {
 	readonlySymlink := "/readonly-symlink"
 
 	g.AddLinuxReadonlyPaths(readonlySymlink)
-	err = util.RuntimeInsideValidate(g, func(path string) error {
+	err = util.RuntimeInsideValidate(g, nil, func(path string) error {
 		testFile := filepath.Join(path, readonlySymlink)
 		// ln -s .. /readonly-symlink ; readlink -f /readonly-symlink; ls -L /readonly-symlink
 		if err := os.Symlink("../readonly-symlink", testFile); err != nil {
@@ -130,7 +130,7 @@ func checkReadonlyDeviceNodes(mode uint32) error {
 	readonlyDevice := "/readonly-device"
 
 	g.AddLinuxReadonlyPaths(readonlyDevice)
-	return util.RuntimeInsideValidate(g, func(path string) error {
+	return util.RuntimeInsideValidate(g, nil, func(path string) error {
 		testFile := filepath.Join(path, readonlyDevice)
 
 		if err := unix.Mknod(testFile, mode, 0); err != nil {
diff --git a/validation/linux_rootfs_propagation/linux_rootfs_propagation.go b/validation/linux_rootfs_propagation/linux_rootfs_propagation.go
@@ -12,7 +12,7 @@ func testLinuxRootPropagation(propMode string) error {
 	}
 	g.SetupPrivileged(true)
 	g.SetLinuxRootPropagation(propMode)
-	return util.RuntimeInsideValidate(g, nil)
+	return util.RuntimeInsideValidate(g, nil, nil)
 }
 
 func main() {
diff --git a/validation/linux_seccomp/linux_seccomp.go b/validation/linux_seccomp/linux_seccomp.go
@@ -16,7 +16,7 @@ func main() {
 	}
 	g.SetDefaultSeccompAction("allow")
 	g.SetSyscallAction(syscallArgs)
-	err = util.RuntimeInsideValidate(g, nil)
+	err = util.RuntimeInsideValidate(g, nil, nil)
 	if err != nil {
 		util.Fatal(err)
 	}
diff --git a/validation/linux_sysctl/linux_sysctl.go b/validation/linux_sysctl/linux_sysctl.go
@@ -10,7 +10,7 @@ func main() {
 		util.Fatal(err)
 	}
 	g.AddLinuxSysctl("net.ipv4.ip_forward", "1")
-	err = util.RuntimeInsideValidate(g, nil)
+	err = util.RuntimeInsideValidate(g, nil, nil)
 	if err != nil {
 		util.Fatal(err)
 	}
diff --git a/validation/linux_uid_mappings/linux_uid_mappings.go b/validation/linux_uid_mappings/linux_uid_mappings.go
@@ -12,7 +12,7 @@ func main() {
 	g.AddOrReplaceLinuxNamespace("user", "")
 	g.AddLinuxUIDMapping(uint32(1000), uint32(0), uint32(2000))
 	g.AddLinuxGIDMapping(uint32(1000), uint32(0), uint32(3000))
-	err = util.RuntimeInsideValidate(g, nil)
+	err = util.RuntimeInsideValidate(g, nil, nil)
 	if err != nil {
 		util.Fatal(err)
 	}
diff --git a/validation/mounts/mounts.go b/validation/mounts/mounts.go
@@ -22,7 +22,7 @@ func main() {
 		},
 	}
 	g.AddMount(mount)
-	err = util.RuntimeInsideValidate(g, nil)
+	err = util.RuntimeInsideValidate(g, nil, nil)
 	if err != nil {
 		util.Fatal(err)
 	}
diff --git a/validation/process/process.go b/validation/process/process.go
@@ -16,7 +16,7 @@ func main() {
 	g.AddProcessEnv("testa", "valuea")
 	g.AddProcessEnv("testb", "123")
 
-	err = util.RuntimeInsideValidate(g, func(path string) error {
+	err = util.RuntimeInsideValidate(g, nil, func(path string) error {
 		pathName := filepath.Join(path, "test")
 		return os.MkdirAll(pathName, 0700)
 	})
diff --git a/validation/process_capabilities/process_capabilities.go b/validation/process_capabilities/process_capabilities.go
@@ -18,7 +18,7 @@ func main() {
 		util.Fatal(err)
 	}
 	g.SetupPrivileged(true)
-	err = util.RuntimeInsideValidate(g, nil)
+	err = util.RuntimeInsideValidate(g, nil, nil)
 	if err != nil {
 		util.Fatal(err)
 	}
diff --git a/validation/process_capabilities_fail/process_capabilities_fail.go b/validation/process_capabilities_fail/process_capabilities_fail.go
@@ -21,7 +21,7 @@ func main() {
 		util.Fatal(err)
 	}
 	g.AddProcessCapabilityBounding("CAP_TEST")
-	err = util.RuntimeInsideValidate(g, nil)
+	err = util.RuntimeInsideValidate(g, nil, nil)
 	if err == nil {
 		util.Fatal(specerror.NewError(specerror.LinuxProcCapError, fmt.Errorf("Any value which cannot be mapped to a relevant kernel interface MUST cause an error"), rspecs.Version))
 	}
diff --git a/validation/process_oom_score_adj/process_oom_score_adj.go b/validation/process_oom_score_adj/process_oom_score_adj.go
@@ -10,7 +10,7 @@ func main() {
 		util.Fatal(err)
 	}
 	g.SetProcessOOMScoreAdj(500)
-	err = util.RuntimeInsideValidate(g, nil)
+	err = util.RuntimeInsideValidate(g, nil, nil)
 	if err != nil {
 		util.Fatal(err)
 	}
diff --git a/validation/process_rlimits/process_rlimits.go b/validation/process_rlimits/process_rlimits.go
@@ -27,7 +27,7 @@ func main() {
 
 	g.AddProcessRlimits("RLIMIT_CPU", 120, 60)       // seconds
 	g.AddProcessRlimits("RLIMIT_NOFILE", 4000, 3000) // number of files
-	err = util.RuntimeInsideValidate(g, nil)
+	err = util.RuntimeInsideValidate(g, nil, nil)
 	if err != nil {
 		util.Fatal(err)
 	}
diff --git a/validation/process_rlimits_fail/process_rlimits_fail.go b/validation/process_rlimits_fail/process_rlimits_fail.go
@@ -21,7 +21,7 @@ func main() {
 		util.Fatal(err)
 	}
 	g.AddProcessRlimits("RLIMIT_TEST", 1024, 1024)
-	err = util.RuntimeInsideValidate(g, nil)
+	err = util.RuntimeInsideValidate(g, nil, nil)
 	if err == nil {
 		util.Fatal(specerror.NewError(specerror.PosixProcRlimitsTypeGenError, fmt.Errorf("The runtime MUST generate an error for any values which cannot be mapped to a relevant kernel interface"), rspecs.Version))
 	}
diff --git a/validation/process_user/process_user.go b/validation/process_user/process_user.go
@@ -22,7 +22,7 @@ func main() {
 	default:
 	}
 
-	err = util.RuntimeInsideValidate(g, nil)
+	err = util.RuntimeInsideValidate(g, nil, nil)
 	if err != nil {
 		util.Fatal(err)
 	}
diff --git a/validation/root_readonly_true/root_readonly_true.go b/validation/root_readonly_true/root_readonly_true.go
@@ -18,7 +18,7 @@ func main() {
 		util.Fatal(err)
 	}
 	g.SetRootReadonly(true)
-	err = util.RuntimeInsideValidate(g, nil)
+	err = util.RuntimeInsideValidate(g, nil, nil)
 	if err != nil {
 		util.Fatal(err)
 	}
diff --git a/validation/util/test.go b/validation/util/test.go
@@ -8,6 +8,7 @@ import (
 	"os/exec"
 	"path/filepath"
 	"runtime"
+	"strings"
 	"time"
 
 	"github.com/mndrix/tap-go"
@@ -173,8 +174,10 @@ func WaitingForStatus(r Runtime, status LifecycleStatus, retryTimeout time.Durat
 	return errors.New("timeout in waiting for the container status")
 }
 
+var runtimeInsideValidateCalled bool
+
 // RuntimeInsideValidate runs runtimetest inside a container.
-func RuntimeInsideValidate(g *generate.Generator, f PreFunc) (err error) {
+func RuntimeInsideValidate(g *generate.Generator, t *tap.T, f PreFunc) (err error) {
 	bundleDir, err := PrepareBundle()
 	if err != nil {
 		return err
@@ -235,7 +238,24 @@ func RuntimeInsideValidate(g *generate.Generator, f PreFunc) (err error) {
 		return err
 	}
 
-	os.Stdout.Write(stdout)
+	// Write stdout in the outter TAP
+	if t != nil {
+		diagnostic := map[string]string{
+			"stdout": string(stdout),
+			"stderr": string(stderr),
+		}
+		if err != nil {
+			diagnostic["error"] = fmt.Sprintf("%v", err)
+		}
+		t.YAML(diagnostic)
+		t.Ok(err == nil && !strings.Contains(string(stdout), "not ok"), g.Config.Annotations["TestName"])
+	} else {
+		if runtimeInsideValidateCalled {
+			Fatal(errors.New("RuntimeInsideValidate called several times in the same test without passing TAP"))
+		}
+		runtimeInsideValidateCalled = true
+		os.Stdout.Write(stdout)
+	}
 	return nil
 }
 

Original file line number	Diff line number	Diff line change
`@@ -9,7 +9,7 @@ func main() {`
`9`	`9`	`if err != nil {`
`10`	`10`	`util.Fatal(err)`
`11`	`11`	`}`
`12`		`- err = util.RuntimeInsideValidate(g, nil)`
	`12`	`+ err = util.RuntimeInsideValidate(g, nil, nil)`
`13`	`13`	`if err != nil {`
`14`	`14`	`util.Fatal(err)`
`15`	`15`	`}`
Original file line number	Diff line number	Diff line change
`@@ -15,7 +15,7 @@ func testHostname(t *tap.T, hostname string) error {`
`15`	`15`	`}`
`16`	`16`
`17`	`17`	`g.SetHostname(hostname)`
`18`		`- err = util.RuntimeInsideValidate(g, nil)`
	`18`	`+ err = util.RuntimeInsideValidate(g, nil, nil)`
`19`	`19`	`t.Ok(err == nil, "hostname is set correctly")`
`20`	`20`	`if err != nil {`
`21`	`21`	`t.Diagnosticf("expect: err == nil, actual: err != nil")`
Original file line number	Diff line number	Diff line change
`@@ -51,7 +51,7 @@ func main() {`
`51`	`51`	`pdev.FileMode = &pmode`
`52`	`52`	`g.AddDevice(pdev)`
`53`	`53`
`54`		`- err = util.RuntimeInsideValidate(g, nil)`
	`54`	`+ err = util.RuntimeInsideValidate(g, nil, nil)`
`55`	`55`	`if err != nil {`
`56`	`56`	`util.Fatal(err)`
`57`	`57`	`}`
Original file line number	Diff line number	Diff line change
`@@ -12,7 +12,7 @@ func testLinuxRootPropagation(propMode string) error {`
`12`	`12`	`}`
`13`	`13`	`g.SetupPrivileged(true)`
`14`	`14`	`g.SetLinuxRootPropagation(propMode)`
`15`		`- return util.RuntimeInsideValidate(g, nil)`
	`15`	`+ return util.RuntimeInsideValidate(g, nil, nil)`
`16`	`16`	`}`
`17`	`17`
`18`	`18`	`func main() {`
Original file line number	Diff line number	Diff line change
`@@ -16,7 +16,7 @@ func main() {`
`16`	`16`	`}`
`17`	`17`	`g.SetDefaultSeccompAction("allow")`
`18`	`18`	`g.SetSyscallAction(syscallArgs)`
`19`		`- err = util.RuntimeInsideValidate(g, nil)`
	`19`	`+ err = util.RuntimeInsideValidate(g, nil, nil)`
`20`	`20`	`if err != nil {`
`21`	`21`	`util.Fatal(err)`
`22`	`22`	`}`
Original file line number	Diff line number	Diff line change
`@@ -10,7 +10,7 @@ func main() {`
`10`	`10`	`util.Fatal(err)`
`11`	`11`	`}`
`12`	`12`	`g.AddLinuxSysctl("net.ipv4.ip_forward", "1")`
`13`		`- err = util.RuntimeInsideValidate(g, nil)`
	`13`	`+ err = util.RuntimeInsideValidate(g, nil, nil)`
`14`	`14`	`if err != nil {`
`15`	`15`	`util.Fatal(err)`
`16`	`16`	`}`
Original file line number	Diff line number	Diff line change
`@@ -12,7 +12,7 @@ func main() {`
`12`	`12`	`g.AddOrReplaceLinuxNamespace("user", "")`
`13`	`13`	`g.AddLinuxUIDMapping(uint32(1000), uint32(0), uint32(2000))`
`14`	`14`	`g.AddLinuxGIDMapping(uint32(1000), uint32(0), uint32(3000))`
`15`		`- err = util.RuntimeInsideValidate(g, nil)`
	`15`	`+ err = util.RuntimeInsideValidate(g, nil, nil)`
`16`	`16`	`if err != nil {`
`17`	`17`	`util.Fatal(err)`
`18`	`18`	`}`
Original file line number	Diff line number	Diff line change
`@@ -22,7 +22,7 @@ func main() {`
`22`	`22`	`},`
`23`	`23`	`}`
`24`	`24`	`g.AddMount(mount)`
`25`		`- err = util.RuntimeInsideValidate(g, nil)`
	`25`	`+ err = util.RuntimeInsideValidate(g, nil, nil)`
`26`	`26`	`if err != nil {`
`27`	`27`	`util.Fatal(err)`
`28`	`28`	`}`