squash! Add support for tap output from runtime test

* Use the new T.Skip(...)
* Get TAP out of test_runtime.sh (move the harness, e.g. prove, to a
  higher level).  Also make sure non-TAP stuff goes to stderr (and not
  stdout).

Author: wking

README.md

@@ -29,32 +29,16 @@ INFO[0000] Bundle validation succeeded.
 
 ## Testing OCI runtimes
 
-With [prove][] installed:
+You can run [`test_runtime.sh`][test_runtime.sh] with any [TAP consumer][tap-consumer].
+For example, with [prove][]:
 
 ```
 $ make
 $ sudo make install
-$ sudo ./test_runtime.sh -r /usr/bin/runc
-/usr/bin/runc .. ok
+$ sudo prove ./test_runtime.sh -r /usr/bin/runc
+./test_runtime.sh .. ok
 All tests successful.
-Files=1, Tests=90,  0 wallclock secs ( 0.02 usr  0.01 sys +  0.02 cusr  0.01 csys =  0.06 CPU)
-Result: PASS
-```
-
-You can also use the usual prove options (separated by `--`):
-
-```
-$ sudo ./test_runtime.sh -r /usr/bin/runc -- -v
-/home/wking/bin/runc ..
-TAP version 13
-ok 1 - # SKIP root.readonly falsy
-ok 2 - hostname matches expected value
-…
-ok 90 - # SKIP linux.gidMappings checks (no mappings specified)
-1..90
-ok
-All tests successful.
-Files=1, Tests=90,  0 wallclock secs ( 0.02 usr  0.00 sys +  0.02 cusr  0.01 csys =  0.05 CPU)
+Files=1, Tests=90,  0 wallclock secs ( 0.02 usr  0.00 sys +  0.04 cusr  0.00 csys =  0.06 CPU)
 Result: PASS
 ```
 
@@ -63,6 +47,7 @@ Result: PASS
 [prove]: http://perldoc.perl.org/prove.html
 [runC]: https://github.com/opencontainers/runc
 [runtime-spec]: https://github.com/opencontainers/runtime-spec
+[tap-consumer]: https://testanything.org/consumers.html
 
 [generate.1]: man/oci-runtime-tool-generate.1.md
 [validate.1]: man/oci-runtime-tool-validate.1.md

cmd/runtimetest/main.go

@@ -73,7 +73,7 @@ func loadSpecConfig() (spec *rspec.Spec, err error) {
 // should be included by other platform specified process validation
 func validateGeneralProcess(harness *tap.T, spec *rspec.Spec) error {
 	if spec.Process.Cwd == "" {
-		harness.Pass("# SKIP process.cwd not set")
+		harness.Skip(1, "process.cwd not set")
 	} else {
 		cwd, err := os.Getwd()
 		if err != nil {
@@ -192,7 +192,7 @@ func validateCapabilities(harness *tap.T, spec *rspec.Spec) error {
 
 func validateHostname(harness *tap.T, spec *rspec.Spec) error {
 	if spec.Hostname == "" {
-		harness.Pass("# SKIP hostname not set")
+		harness.Skip(1, "hostname not set")
 	} else {
 		hostname, err := os.Hostname()
 		if err != nil {
@@ -273,7 +273,7 @@ func validateRootFS(harness *tap.T, spec *rspec.Spec) error {
 			return err
 		}
 	} else {
-		harness.Pass("# SKIP root.readonly falsy")
+		harness.Skip(1, "root.readonly falsy")
 	}
 
 	return nil
@@ -302,7 +302,7 @@ func validateDefaultFS(harness *tap.T, spec *rspec.Spec) error {
 
 func validateLinuxDevices(harness *tap.T, spec *rspec.Spec) error {
 	if len(spec.Linux.Devices) == 0 {
-		harness.Pass("# SKIP linux.devices (no devices configured)")
+		harness.Skip(1, "linux.devices (no devices configured)")
 	}
 	for _, device := range spec.Linux.Devices {
 		fi, err := os.Stat(device.Path)
@@ -345,7 +345,7 @@ func validateLinuxDevices(harness *tap.T, spec *rspec.Spec) error {
 			}
 		}
 		if device.FileMode == nil {
-			harness.Pass(fmt.Sprintf("# SKIP device %v has unconfigured permissions", device.Path))
+			harness.Skip(1, fmt.Sprintf("device %v has unconfigured permissions", device.Path))
 		} else {
 			expectedPerm := *device.FileMode & os.ModePerm
 			actualPerm := fi.Mode() & os.ModePerm
@@ -355,15 +355,15 @@ func validateLinuxDevices(harness *tap.T, spec *rspec.Spec) error {
 			}
 		}
 		if device.UID == nil {
-			harness.Pass(fmt.Sprintf("# SKIP device %v has an unconfigured user ID", device.Path))
+			harness.Skip(1, fmt.Sprintf("device %v has an unconfigured user ID", device.Path))
 		} else {
 			harness.Ok(fStat.Uid == *device.UID, fmt.Sprintf("device %v has expected user ID", device.Path))
 			if fStat.Uid != *device.UID {
 				harness.Diagnosticf("device %v user ID epected: %v, actual: %v", device.Path, *device.UID, fStat.Uid)
 			}
 		}
 		if device.GID == nil {
-			harness.Pass(fmt.Sprintf("# SKIP device %v has an unconfigured group ID", device.Path))
+			harness.Skip(1, fmt.Sprintf("device %v has an unconfigured group ID", device.Path))
 		} else {
 			harness.Ok(fStat.Gid == *device.GID, fmt.Sprintf("device %v has expected group ID", device.Path))
 			if fStat.Gid != *device.GID {
@@ -380,7 +380,7 @@ func validateDefaultSymlinks(harness *tap.T, spec *rspec.Spec) error {
 		fi, err := os.Lstat(symlink)
 		harness.Ok(err == nil, fmt.Sprintf("lstat default symlink %v", symlink))
 		if err != nil {
-			harness.Pass(fmt.Sprintf("# SKIP default symlink %v checks (failed lstat)", symlink))
+			harness.Skip(1, fmt.Sprintf("default symlink %v checks (failed lstat)", symlink))
 		} else {
 			harness.Ok(fi.Mode()&os.ModeSymlink == os.ModeSymlink, fmt.Sprintf("default symlink %v is a symlink", symlink))
 			realDest, err := os.Readlink(symlink)
@@ -405,7 +405,7 @@ func validateDefaultDevices(harness *tap.T, spec *rspec.Spec) error {
 		fi, err := os.Stat(device)
 		harness.Ok(err == nil, fmt.Sprintf("stat default device %v", device))
 		if err != nil {
-			harness.Pass(fmt.Sprintf("# SKIP default device %v checks (failed stat)", device))
+			harness.Skip(1, fmt.Sprintf("default device %v checks (failed stat)", device))
 		} else {
 			harness.Ok(fi.Mode()&os.ModeDevice == os.ModeDevice, fmt.Sprintf("default device %v is a device", device))
 		}
@@ -419,7 +419,7 @@ func validateMaskedPaths(harness *tap.T, spec *rspec.Spec) error {
 		f, err := os.Open(maskedPath)
 		harness.Ok(err == nil, fmt.Sprintf("open masked path %v", maskedPath))
 		if err != nil {
-			harness.Pass(fmt.Sprintf("# SKIP masked path %v checks (failed open)", maskedPath))
+			harness.Skip(1, fmt.Sprintf("masked path %v checks (failed open)", maskedPath))
 		} else {
 			defer f.Close()
 			b := make([]byte, 1)
@@ -443,13 +443,13 @@ func validateROPaths(harness *tap.T, spec *rspec.Spec) error {
 
 func validateOOMScoreAdj(harness *tap.T, spec *rspec.Spec) error {
 	if spec.Linux.Resources == nil || spec.Linux.Resources.OOMScoreAdj == nil {
-		harness.Pass("# SKIP linux.resources.oomScoreAdj falsy")
+		harness.Skip(1, "linux.resources.oomScoreAdj falsy")
 	} else {
 		expected := *spec.Linux.Resources.OOMScoreAdj
 		f, err := os.Open("/proc/1/oom_score_adj")
 		harness.Ok(err == nil, "open /proc/1/oom_score_adj")
 		if err != nil {
-			harness.Pass("# SKIP oomScoreAdj checks (failed open)")
+			harness.Skip(1, "oomScoreAdj checks (failed open)")
 			return nil
 		}
 		defer f.Close()
@@ -459,14 +459,14 @@ func validateOOMScoreAdj(harness *tap.T, spec *rspec.Spec) error {
 			err := s.Err()
 			harness.Ok(err == nil, "scan /proc/1/oom_score_adj")
 			if err != nil {
-				harness.Pass("# SKIP oomScoreAdj checks (failed scan)")
+				harness.Skip(1, "oomScoreAdj checks (failed scan)")
 				return nil
 			}
 			text := strings.TrimSpace(s.Text())
 			actual, err := strconv.Atoi(text)
 			harness.Ok(err == nil, "convert scanned /proc/1/oom_score_adj value to an integer")
 			if err != nil {
-				harness.Pass("# SKIP oomScoreAdj checks (failed integer conversion)")
+				harness.Skip(1, "oomScoreAdj checks (failed integer conversion)")
 				return nil
 			}
 			harness.Ok(actual == expected, "has expected oomScoreAdj")
@@ -518,13 +518,13 @@ func getIDMappings(path string) ([]rspec.IDMapping, error) {
 
 func validateIDMappings(harness *tap.T, mappings []rspec.IDMapping, path string, property string) error {
 	if len(mappings) == 0 {
-		harness.Pass(fmt.Sprintf("# SKIP %s checks (no mappings specified)", property))
+		harness.Skip(1, fmt.Sprintf("%s checks (no mappings specified)", property))
 		return nil
 	}
 	idMaps, err := getIDMappings(path)
 	harness.Ok(err == nil, fmt.Sprintf("get ID mappings from %s", path))
 	if err != nil {
-		harness.Pass(fmt.Sprintf("# SKIP %s checks (failed to get mappings)", property))
+		harness.Skip(1, fmt.Sprintf("%s checks (failed to get mappings)", property))
 		return nil
 	}
 	harness.Ok(len(idMaps) == len(mappings), fmt.Sprintf("%s has expected number of mappings", path))

test_runtime.sh

@@ -19,16 +19,16 @@ TEST_ARGS=('--args' '/runtimetest')
 KEEP=0 # Track whether we keep the test directory around or clean it up
 
 usage() {
-	echo "$0 -l <log-level> -r <runtime> -k -h [-- [prove-options]]"
+	echo "$0 -l <log-level> -r <runtime> -k -h"
 }
 
 error() {
-	echo $*
+	echo "$*" >&2
 	exit 1
 }
 
 info() {
-	echo $*
+	echo "$*" >&2
 }
 
 while getopts "l:r:kh" opt; do
@@ -52,7 +52,6 @@ while getopts "l:r:kh" opt; do
 			;;
 	esac
 done
-shift $((${OPTIND} - 1))
 
 if ! command -v ${RUNTIME} > /dev/null; then
 	error "Runtime ${RUNTIME} not found in the path"
@@ -77,4 +76,4 @@ cp runtimetest ${TESTDIR}
 oci-runtime-tool generate --output "${TESTDIR}/config.json" "${TEST_ARGS[@]}" --rootfs-path '.'
 
 cd ${TESTDIR}
-prove "${@}" ${RUNTIME} :: start $(uuidgen)
+${RUNTIME} start $(uuidgen)