runtimetest: count correctly TAP tests

alban · alban · commit c9b4d6689c56 · 2018-03-13T10:37:14.000+01:00
Don't create a new tap.T in the middle of a test but reuse the existing one instead. Symptoms: ``` $ sudo validation/linux_readonly_paths.t TAP version 13 ok 1 - root filesystem ok 2 - hostname ok 3 - process ok 4 - mounts ok 5 - user ok 6 - rlimits ok 7 - capabilities ok 8 - default symlinks ok 9 - default file system ok 10 - default devices ok 11 - linux devices ok 12 - linux process ok 13 - masked paths ok 14 - oom score adj ok 1 # SKIP syscall action SCMP_ACT_ALLOW ok 2 # SKIP syscall action SCMP_ACT_ALLOW ok 3 # SKIP syscall action SCMP_ACT_ALLOW ok 4 # SKIP syscall action SCMP_ACT_ALLOW ok 5 # SKIP syscall action SCMP_ACT_ALLOW ok 6 # SKIP syscall action SCMP_ACT_ALLOW ok 15 - seccomp ok 16 - read only paths ok 17 - rootfs propagation ok 18 - sysctls ok 19 - uid mappings ok 20 - gid mappings 1..20 ``` After this patch is applied: ``` $ sudo validation/linux_readonly_paths.t TAP version 13 ok 1 - root filesystem ok 2 - hostname ok 3 - process ok 4 - mounts ok 5 - user ok 6 - rlimits ok 7 - capabilities ok 8 - default symlinks ok 9 - default file system ok 10 - default devices ok 11 - linux devices ok 12 - linux process ok 13 - masked paths ok 14 - oom score adj ok 15 # SKIP syscall action SCMP_ACT_ALLOW ok 16 # SKIP syscall action SCMP_ACT_ALLOW ok 17 # SKIP syscall action SCMP_ACT_ALLOW ok 18 # SKIP syscall action SCMP_ACT_ALLOW ok 19 # SKIP syscall action SCMP_ACT_ALLOW ok 20 # SKIP syscall action SCMP_ACT_ALLOW ok 21 - seccomp ok 22 - read only paths ok 23 - rootfs propagation ok 24 - sysctls ok 25 - uid mappings ok 26 - gid mappings 1..26 ``` Fixes #593 Signed-off-by: Alban Crequy <alban@kinvolk.io>
diff --git a/cmd/runtimetest/main.go b/cmd/runtimetest/main.go
@@ -69,7 +69,7 @@ var (
 )
 
 type validation struct {
-	test        func(*rspec.Spec) error
+	test        func(*rspec.Spec, *tap.T) error
 	description string
 }
 
@@ -91,7 +91,7 @@ func loadSpecConfig(path string) (spec *rspec.Spec, err error) {
 	return spec, nil
 }
 
-func validatePosixUser(spec *rspec.Spec) error {
+func validatePosixUser(spec *rspec.Spec, t *tap.T) error {
 	if spec.Process == nil {
 		return nil
 	}
@@ -124,7 +124,7 @@ func validatePosixUser(spec *rspec.Spec) error {
 	return nil
 }
 
-func validateProcess(spec *rspec.Spec) error {
+func validateProcess(spec *rspec.Spec, t *tap.T) error {
 	if spec.Process == nil {
 		return nil
 	}
@@ -152,7 +152,7 @@ func validateProcess(spec *rspec.Spec) error {
 	return nil
 }
 
-func validateLinuxProcess(spec *rspec.Spec) error {
+func validateLinuxProcess(spec *rspec.Spec, t *tap.T) error {
 	if spec.Process == nil {
 		return nil
 	}
@@ -186,7 +186,7 @@ func validateLinuxProcess(spec *rspec.Spec) error {
 	return nil
 }
 
-func validateCapabilities(spec *rspec.Spec) error {
+func validateCapabilities(spec *rspec.Spec, t *tap.T) error {
 	if spec.Process == nil || spec.Process.Capabilities == nil {
 		return nil
 	}
@@ -251,7 +251,7 @@ func validateCapabilities(spec *rspec.Spec) error {
 	return nil
 }
 
-func validateHostname(spec *rspec.Spec) error {
+func validateHostname(spec *rspec.Spec, t *tap.T) error {
 	hostname, err := os.Hostname()
 	if err != nil {
 		return err
@@ -262,7 +262,7 @@ func validateHostname(spec *rspec.Spec) error {
 	return nil
 }
 
-func validateRlimits(spec *rspec.Spec) error {
+func validateRlimits(spec *rspec.Spec, t *tap.T) error {
 	if spec.Process == nil {
 		return nil
 	}
@@ -288,7 +288,7 @@ func validateRlimits(spec *rspec.Spec) error {
 	return nil
 }
 
-func validateSysctls(spec *rspec.Spec) error {
+func validateSysctls(spec *rspec.Spec, t *tap.T) error {
 	if spec.Linux == nil {
 		return nil
 	}
@@ -318,7 +318,7 @@ func testWriteAccess(path string) error {
 	return nil
 }
 
-func validateRootFS(spec *rspec.Spec) error {
+func validateRootFS(spec *rspec.Spec, t *tap.T) error {
 	if spec.Root == nil {
 		return nil
 	}
@@ -338,7 +338,7 @@ func validateRootFS(spec *rspec.Spec) error {
 	return nil
 }
 
-func validateRootfsPropagation(spec *rspec.Spec) error {
+func validateRootfsPropagation(spec *rspec.Spec, t *tap.T) error {
 	if spec.Linux == nil || spec.Linux.RootfsPropagation == "" {
 		return nil
 	}
@@ -403,7 +403,7 @@ func validateRootfsPropagation(spec *rspec.Spec) error {
 	return nil
 }
 
-func validateDefaultFS(spec *rspec.Spec) error {
+func validateDefaultFS(spec *rspec.Spec, t *tap.T) error {
 	mountInfos, err := mount.GetMounts()
 	if err != nil {
 		specerror.NewError(specerror.DefaultFilesystems, err, spec.Version)
@@ -423,7 +423,7 @@ func validateDefaultFS(spec *rspec.Spec) error {
 	return nil
 }
 
-func validateLinuxDevices(spec *rspec.Spec) error {
+func validateLinuxDevices(spec *rspec.Spec, t *tap.T) error {
 	if spec.Linux == nil {
 		return nil
 	}
@@ -480,7 +480,7 @@ func validateLinuxDevices(spec *rspec.Spec) error {
 	return nil
 }
 
-func validateDefaultSymlinks(spec *rspec.Spec) error {
+func validateDefaultSymlinks(spec *rspec.Spec, t *tap.T) error {
 	for symlink, dest := range defaultSymlinks {
 		fi, err := os.Lstat(symlink)
 		if err != nil {
@@ -506,7 +506,7 @@ func validateDefaultSymlinks(spec *rspec.Spec) error {
 	return nil
 }
 
-func validateDefaultDevices(spec *rspec.Spec) error {
+func validateDefaultDevices(spec *rspec.Spec, t *tap.T) error {
 	if spec.Process != nil && spec.Process.Terminal {
 		defaultDevices = append(defaultDevices, "/dev/console")
 	}
@@ -531,7 +531,7 @@ func validateDefaultDevices(spec *rspec.Spec) error {
 	return nil
 }
 
-func validateMaskedPaths(spec *rspec.Spec) error {
+func validateMaskedPaths(spec *rspec.Spec, t *tap.T) error {
 	if spec.Linux == nil {
 		return nil
 	}
@@ -550,11 +550,10 @@ func validateMaskedPaths(spec *rspec.Spec) error {
 	return nil
 }
 
-func validateSeccomp(spec *rspec.Spec) error {
+func validateSeccomp(spec *rspec.Spec, t *tap.T) error {
 	if spec.Linux == nil || spec.Linux.Seccomp == nil {
 		return nil
 	}
-	t := tap.New()
 	for _, sys := range spec.Linux.Seccomp.Syscalls {
 		if sys.Action == "SCMP_ACT_ERRNO" {
 			for _, name := range sys.Names {
@@ -574,7 +573,7 @@ func validateSeccomp(spec *rspec.Spec) error {
 	return nil
 }
 
-func validateROPaths(spec *rspec.Spec) error {
+func validateROPaths(spec *rspec.Spec, t *tap.T) error {
 	if spec.Linux == nil {
 		return nil
 	}
@@ -588,7 +587,7 @@ func validateROPaths(spec *rspec.Spec) error {
 	return nil
 }
 
-func validateOOMScoreAdj(spec *rspec.Spec) error {
+func validateOOMScoreAdj(spec *rspec.Spec, t *tap.T) error {
 	if spec.Process != nil && spec.Process.OOMScoreAdj != nil {
 		expected := *spec.Process.OOMScoreAdj
 		f, err := os.Open("/proc/self/oom_score_adj")
@@ -679,14 +678,14 @@ func validateIDMappings(mappings []rspec.LinuxIDMapping, path string, property s
 	return nil
 }
 
-func validateUIDMappings(spec *rspec.Spec) error {
+func validateUIDMappings(spec *rspec.Spec, t *tap.T) error {
 	if spec.Linux == nil {
 		return nil
 	}
 	return validateIDMappings(spec.Linux.UIDMappings, "/proc/self/uid_map", "linux.uidMappings")
 }
 
-func validateGIDMappings(spec *rspec.Spec) error {
+func validateGIDMappings(spec *rspec.Spec, t *tap.T) error {
 	if spec.Linux == nil {
 		return nil
 	}
@@ -715,7 +714,7 @@ func mountMatch(configMount rspec.Mount, sysMount *mount.Info) error {
 	return nil
 }
 
-func validatePosixMounts(spec *rspec.Spec) error {
+func validatePosixMounts(spec *rspec.Spec, t *tap.T) error {
 	mountInfos, err := mount.GetMounts()
 	if err != nil {
 		return err
@@ -909,7 +908,7 @@ func run(context *cli.Context) error {
 	}
 
 	for _, v := range validations {
-		err := v.test(spec)
+		err := v.test(spec, t)
 		if err == nil {
 			t.Pass(v.description)
 		} else {

Original file line number	Diff line number	Diff line change
`@@ -69,7 +69,7 @@ var (`
`69`	`69`	`)`
`70`	`70`
`71`	`71`	`type validation struct {`
`72`		`- test func(*rspec.Spec) error`
	`72`	`+ test func(rspec.Spec, tap.T) error`
`73`	`73`	`description string`
`74`	`74`	`}`
`75`	`75`
`@@ -91,7 +91,7 @@ func loadSpecConfig(path string) (spec *rspec.Spec, err error) {`
`91`	`91`	`return spec, nil`
`92`	`92`	`}`
`93`	`93`
`94`		`-func validatePosixUser(spec *rspec.Spec) error {`
	`94`	`+func validatePosixUser(spec rspec.Spec, t tap.T) error {`
`95`	`95`	`if spec.Process == nil {`
`96`	`96`	`return nil`
`97`	`97`	`}`
`@@ -124,7 +124,7 @@ func validatePosixUser(spec *rspec.Spec) error {`
`124`	`124`	`return nil`
`125`	`125`	`}`
`126`	`126`
`127`		`-func validateProcess(spec *rspec.Spec) error {`
	`127`	`+func validateProcess(spec rspec.Spec, t tap.T) error {`
`128`	`128`	`if spec.Process == nil {`
`129`	`129`	`return nil`
`130`	`130`	`}`
`@@ -152,7 +152,7 @@ func validateProcess(spec *rspec.Spec) error {`
`152`	`152`	`return nil`
`153`	`153`	`}`
`154`	`154`
`155`		`-func validateLinuxProcess(spec *rspec.Spec) error {`
	`155`	`+func validateLinuxProcess(spec rspec.Spec, t tap.T) error {`
`156`	`156`	`if spec.Process == nil {`
`157`	`157`	`return nil`
`158`	`158`	`}`
`@@ -186,7 +186,7 @@ func validateLinuxProcess(spec *rspec.Spec) error {`
`186`	`186`	`return nil`
`187`	`187`	`}`
`188`	`188`
`189`		`-func validateCapabilities(spec *rspec.Spec) error {`
	`189`	`+func validateCapabilities(spec rspec.Spec, t tap.T) error {`
`190`	`190`	`if spec.Process == nil \|\| spec.Process.Capabilities == nil {`
`191`	`191`	`return nil`
`192`	`192`	`}`
`@@ -251,7 +251,7 @@ func validateCapabilities(spec *rspec.Spec) error {`
`251`	`251`	`return nil`
`252`	`252`	`}`
`253`	`253`
`254`		`-func validateHostname(spec *rspec.Spec) error {`
	`254`	`+func validateHostname(spec rspec.Spec, t tap.T) error {`
`255`	`255`	`hostname, err := os.Hostname()`
`256`	`256`	`if err != nil {`
`257`	`257`	`return err`
`@@ -262,7 +262,7 @@ func validateHostname(spec *rspec.Spec) error {`
`262`	`262`	`return nil`
`263`	`263`	`}`
`264`	`264`
`265`		`-func validateRlimits(spec *rspec.Spec) error {`
	`265`	`+func validateRlimits(spec rspec.Spec, t tap.T) error {`
`266`	`266`	`if spec.Process == nil {`
`267`	`267`	`return nil`
`268`	`268`	`}`
`@@ -288,7 +288,7 @@ func validateRlimits(spec *rspec.Spec) error {`
`288`	`288`	`return nil`
`289`	`289`	`}`
`290`	`290`
`291`		`-func validateSysctls(spec *rspec.Spec) error {`
	`291`	`+func validateSysctls(spec rspec.Spec, t tap.T) error {`
`292`	`292`	`if spec.Linux == nil {`
`293`	`293`	`return nil`
`294`	`294`	`}`
`@@ -318,7 +318,7 @@ func testWriteAccess(path string) error {`
`318`	`318`	`return nil`
`319`	`319`	`}`
`320`	`320`
`321`		`-func validateRootFS(spec *rspec.Spec) error {`
	`321`	`+func validateRootFS(spec rspec.Spec, t tap.T) error {`
`322`	`322`	`if spec.Root == nil {`
`323`	`323`	`return nil`
`324`	`324`	`}`
`@@ -338,7 +338,7 @@ func validateRootFS(spec *rspec.Spec) error {`
`338`	`338`	`return nil`
`339`	`339`	`}`
`340`	`340`
`341`		`-func validateRootfsPropagation(spec *rspec.Spec) error {`
	`341`	`+func validateRootfsPropagation(spec rspec.Spec, t tap.T) error {`
`342`	`342`	`if spec.Linux == nil \|\| spec.Linux.RootfsPropagation == "" {`
`343`	`343`	`return nil`
`344`	`344`	`}`
`@@ -403,7 +403,7 @@ func validateRootfsPropagation(spec *rspec.Spec) error {`
`403`	`403`	`return nil`
`404`	`404`	`}`
`405`	`405`
`406`		`-func validateDefaultFS(spec *rspec.Spec) error {`
	`406`	`+func validateDefaultFS(spec rspec.Spec, t tap.T) error {`
`407`	`407`	`mountInfos, err := mount.GetMounts()`
`408`	`408`	`if err != nil {`
`409`	`409`	`specerror.NewError(specerror.DefaultFilesystems, err, spec.Version)`
`@@ -423,7 +423,7 @@ func validateDefaultFS(spec *rspec.Spec) error {`
`423`	`423`	`return nil`
`424`	`424`	`}`
`425`	`425`
`426`		`-func validateLinuxDevices(spec *rspec.Spec) error {`
	`426`	`+func validateLinuxDevices(spec rspec.Spec, t tap.T) error {`
`427`	`427`	`if spec.Linux == nil {`
`428`	`428`	`return nil`
`429`	`429`	`}`
`@@ -480,7 +480,7 @@ func validateLinuxDevices(spec *rspec.Spec) error {`
`480`	`480`	`return nil`
`481`	`481`	`}`
`482`	`482`
`483`		`-func validateDefaultSymlinks(spec *rspec.Spec) error {`
	`483`	`+func validateDefaultSymlinks(spec rspec.Spec, t tap.T) error {`
`484`	`484`	`for symlink, dest := range defaultSymlinks {`
`485`	`485`	`fi, err := os.Lstat(symlink)`
`486`	`486`	`if err != nil {`
`@@ -506,7 +506,7 @@ func validateDefaultSymlinks(spec *rspec.Spec) error {`
`506`	`506`	`return nil`
`507`	`507`	`}`
`508`	`508`
`509`		`-func validateDefaultDevices(spec *rspec.Spec) error {`
	`509`	`+func validateDefaultDevices(spec rspec.Spec, t tap.T) error {`
`510`	`510`	`if spec.Process != nil && spec.Process.Terminal {`
`511`	`511`	`defaultDevices = append(defaultDevices, "/dev/console")`
`512`	`512`	`}`
`@@ -531,7 +531,7 @@ func validateDefaultDevices(spec *rspec.Spec) error {`
`531`	`531`	`return nil`
`532`	`532`	`}`
`533`	`533`
`534`		`-func validateMaskedPaths(spec *rspec.Spec) error {`
	`534`	`+func validateMaskedPaths(spec rspec.Spec, t tap.T) error {`
`535`	`535`	`if spec.Linux == nil {`
`536`	`536`	`return nil`
`537`	`537`	`}`
`@@ -550,11 +550,10 @@ func validateMaskedPaths(spec *rspec.Spec) error {`
`550`	`550`	`return nil`
`551`	`551`	`}`
`552`	`552`
`553`		`-func validateSeccomp(spec *rspec.Spec) error {`
	`553`	`+func validateSeccomp(spec rspec.Spec, t tap.T) error {`
`554`	`554`	`if spec.Linux == nil \|\| spec.Linux.Seccomp == nil {`
`555`	`555`	`return nil`
`556`	`556`	`}`
`557`		`- t := tap.New()`
`558`	`557`	`for _, sys := range spec.Linux.Seccomp.Syscalls {`
`559`	`558`	`if sys.Action == "SCMP_ACT_ERRNO" {`
`560`	`559`	`for _, name := range sys.Names {`
`@@ -574,7 +573,7 @@ func validateSeccomp(spec *rspec.Spec) error {`
`574`	`573`	`return nil`
`575`	`574`	`}`
`576`	`575`
`577`		`-func validateROPaths(spec *rspec.Spec) error {`
	`576`	`+func validateROPaths(spec rspec.Spec, t tap.T) error {`
`578`	`577`	`if spec.Linux == nil {`
`579`	`578`	`return nil`
`580`	`579`	`}`
`@@ -588,7 +587,7 @@ func validateROPaths(spec *rspec.Spec) error {`
`588`	`587`	`return nil`
`589`	`588`	`}`
`590`	`589`
`591`		`-func validateOOMScoreAdj(spec *rspec.Spec) error {`
	`590`	`+func validateOOMScoreAdj(spec rspec.Spec, t tap.T) error {`
`592`	`591`	`if spec.Process != nil && spec.Process.OOMScoreAdj != nil {`
`593`	`592`	`expected := *spec.Process.OOMScoreAdj`
`594`	`593`	`f, err := os.Open("/proc/self/oom_score_adj")`
`@@ -679,14 +678,14 @@ func validateIDMappings(mappings []rspec.LinuxIDMapping, path string, property s`
`679`	`678`	`return nil`
`680`	`679`	`}`
`681`	`680`
`682`		`-func validateUIDMappings(spec *rspec.Spec) error {`
	`681`	`+func validateUIDMappings(spec rspec.Spec, t tap.T) error {`
`683`	`682`	`if spec.Linux == nil {`
`684`	`683`	`return nil`
`685`	`684`	`}`
`686`	`685`	`return validateIDMappings(spec.Linux.UIDMappings, "/proc/self/uid_map", "linux.uidMappings")`
`687`	`686`	`}`
`688`	`687`
`689`		`-func validateGIDMappings(spec *rspec.Spec) error {`
	`688`	`+func validateGIDMappings(spec rspec.Spec, t tap.T) error {`
`690`	`689`	`if spec.Linux == nil {`
`691`	`690`	`return nil`
`692`	`691`	`}`
`@@ -715,7 +714,7 @@ func mountMatch(configMount rspec.Mount, sysMount *mount.Info) error {`
`715`	`714`	`return nil`
`716`	`715`	`}`
`717`	`716`
`718`		`-func validatePosixMounts(spec *rspec.Spec) error {`
	`717`	`+func validatePosixMounts(spec rspec.Spec, t tap.T) error {`
`719`	`718`	`mountInfos, err := mount.GetMounts()`
`720`	`719`	`if err != nil {`
`721`	`720`	`return err`
`@@ -909,7 +908,7 @@ func run(context *cli.Context) error {`
`909`	`908`	`}`
`910`	`909`
`911`	`910`	`for _, v := range validations {`
`912`		`- err := v.test(spec)`
	`911`	`+ err := v.test(spec, t)`
`913`	`912`	`if err == nil {`
`914`	`913`	`t.Pass(v.description)`
`915`	`914`	`} else {`