Skip to content

Commit 36cf932

Browse files
liangchenyeliangchenye
authored
Merge pull request #439 from wking/tap-diagnostics
cmd/runtimetest/main: Use TAP diagnostics for errors
2 parents 1b3d6bd + ad47e7d commit 36cf932

26 files changed

+769
-417
lines changed

‎Makefile‎

Lines changed: 14 additions & 4 deletions
Original file line numberDiff line numberDiff line change
@@ -1,12 +1,14 @@
11
PREFIX ?= $(DESTDIR)/usr
22
BINDIR ?= $(DESTDIR)/usr/bin
3+
TAP ?= tap
34

45
BUILDTAGS=
56
RUNTIME ?= runc
67
COMMIT=$(shell git rev-parse HEAD 2> /dev/null || true)
78
VERSION := ${shell cat ./VERSION}
9+
VALIDATION_TESTS ?= $(patsubst %.go,%.t,$(wildcard validation/*.go))
810

9-
all: tool runtimetest
11+
all: tool runtimetest validation-executables
1012

1113
tool:
1214
go build -tags "$(BUILDTAGS)" -ldflags "-X main.gitCommit=${COMMIT} -X main.version=${VERSION}" -o oci-runtime-tool ./cmd/oci-runtime-tool
@@ -35,10 +37,18 @@ uninstall:
3537
rm -f $(PREFIX)/share/bash-completion/completions/oci-runtime-tool
3638

3739
clean:
38-
rm -f oci-runtime-tool runtimetest *.1
40+
rm -f oci-runtime-tool runtimetest *.1 $(VALIDATION_TESTS)
3941

40-
localvalidation: runtimetest
41-
RUNTIME=$(RUNTIME) go test -tags "$(BUILDTAGS)" ${TESTFLAGS} -v github.com/opencontainers/runtime-tools/validation
42+
localvalidation:
43+
RUNTIME=$(RUNTIME) $(TAP) $(VALIDATION_TESTS)
44+
45+
.PHONY: validation-executables
46+
validation-executables: $(VALIDATION_TESTS)
47+
48+
.PRECIOUS: $(VALIDATION_TESTS)
49+
.PHONY: $(VALIDATION_TESTS)
50+
$(VALIDATION_TESTS): %.t: %.go
51+
go build -tags "$(BUILDTAGS)" ${TESTFLAGS} -o $@ $<
4252

4353
.PHONY: test .gofmt .govet .golint
4454

‎README.md‎

Lines changed: 140 additions & 42 deletions
Original file line numberDiff line numberDiff line change
@@ -8,7 +8,7 @@ To build from source code, runtime-tools requires Go 1.7.x or above.
88
[`oci-runtime-tool generate`][generate.1] generates [configuration JSON][config.json] for an [OCI bundle][bundle].
99
[OCI-compatible runtimes][runtime-spec] like [runC][] expect to read the configuration from `config.json`.
1010

11-
```sh
11+
```console
1212
$ oci-runtime-tool generate --output config.json
1313
$ cat config.json
1414
{
@@ -22,63 +22,161 @@ $ cat config.json
2222
[`oci-runtime-tool validate`][validate.1] validates an OCI bundle.
2323
The error message will be printed if the OCI bundle failed the validation procedure.
2424

25-
```sh
25+
```console
2626
$ oci-runtime-tool generate
2727
$ oci-runtime-tool validate
2828
INFO[0000] Bundle validation succeeded.
2929
```
3030

3131
## Testing OCI runtimes
3232

33-
```sh
34-
$ sudo make RUNTIME=runc localvalidation
35-
RUNTIME=runc go test -tags "" -v github.com/opencontainers/runtime-tools/validation
36-
=== RUN TestValidateBasic
37-
TAP version 13
38-
ok 1 - root filesystem
39-
ok 2 - hostname
40-
ok 3 - mounts
41-
ok 4 - capabilities
42-
ok 5 - default symlinks
43-
ok 6 - default devices
44-
ok 7 - linux devices
45-
ok 8 - linux process
46-
ok 9 - masked paths
47-
ok 10 - oom score adj
48-
ok 11 - read only paths
49-
ok 12 - rlimits
50-
ok 13 - sysctls
51-
ok 14 - uid mappings
52-
ok 15 - gid mappings
53-
1..15
54-
--- PASS: TestValidateBasic (0.08s)
55-
=== RUN TestValidateSysctls
33+
The runtime validation suite uses [node-tap][], which is packaged for some distributions (for example, it is in [Debian's `node-tap` package][debian-node-tap]).
34+
If your distribution does not package node-tap, you can install [npm][] (for example, from [Gentoo's `nodejs` package][gentoo-nodejs]) and use it:
35+
36+
```console
37+
$ npm install tap
38+
```
39+
40+
```console
41+
$ make runtimetest validation-executables
42+
RUNTIME=runc tap validation/linux_rootfs_propagation_shared.t validation/create.t validation/default.t validation/linux_readonly_paths.t validation/linux_masked_paths.t validation/mounts.t validation/process.t validation/root_readonly_false.t validation/linux_sysctl.t validation/linux_devices.t validation/linux_gid_mappings.t validation/process_oom_score_adj.t validation/process_capabilities.t validation/process_rlimits.t validation/root_readonly_true.t validation/linux_rootfs_propagation_unbindable.t validation/hostname.t validation/linux_uid_mappings.t
43+
validation/linux_rootfs_propagation_shared.t ........ 18/19
44+
not ok rootfs propagation
45+
46+
validation/create.t ................................... 4/4
47+
validation/default.t ................................ 19/19
48+
validation/linux_readonly_paths.t ................... 19/19
49+
validation/linux_masked_paths.t ..................... 18/19
50+
not ok masked paths
51+
52+
validation/mounts.t ................................... 0/1
53+
Skipped: 1
54+
TODO: mounts generation options have not been implemented
55+
56+
validation/process.t ................................ 19/19
57+
validation/root_readonly_false.t .................... 19/19
58+
validation/linux_sysctl.t ........................... 19/19
59+
validation/linux_devices.t .......................... 19/19
60+
validation/linux_gid_mappings.t ..................... 18/19
61+
not ok gid mappings
62+
63+
validation/process_oom_score_adj.t .................. 19/19
64+
validation/process_capabilities.t ................... 19/19
65+
validation/process_rlimits.t ........................ 19/19
66+
validation/root_readonly_true.t ...................failed to create the container
67+
rootfsPropagation=unbindable is not supported
68+
exit status 1
69+
validation/root_readonly_true.t ..................... 19/19
70+
validation/linux_rootfs_propagation_unbindable.t ...... 0/1
71+
not ok validation/linux_rootfs_propagation_unbindable.t
72+
timeout: 30000
73+
file: validation/linux_rootfs_propagation_unbindable.t
74+
command: validation/linux_rootfs_propagation_unbindable.t
75+
args: []
76+
stdio:
77+
- 0
78+
- pipe
79+
- 2
80+
cwd: /…/go/src/github.com/opencontainers/runtime-tools
81+
exitCode: 1
82+
83+
validation/hostname.t ...................failed to create the container
84+
User namespace mappings specified, but USER namespace isn't enabled in the config
85+
exit status 1
86+
validation/hostname.t ............................... 19/19
87+
validation/linux_uid_mappings.t ....................... 0/1
88+
not ok validation/linux_uid_mappings.t
89+
timeout: 30000
90+
file: validation/linux_uid_mappings.t
91+
command: validation/linux_uid_mappings.t
92+
args: []
93+
stdio:
94+
- 0
95+
- pipe
96+
- 2
97+
cwd: /…/go/src/github.com/opencontainers/runtime-tools
98+
exitCode: 1
99+
100+
total ............................................. 267/273
101+
102+
103+
267 passing (31s)
104+
1 pending
105+
5 failing
106+
107+
make: *** [Makefile:43: localvalidation] Error 1
108+
```
109+
110+
You can also run an individual test executable directly:
111+
112+
```console
113+
$ RUNTIME=runc validation/default.t
56114
TAP version 13
57115
ok 1 - root filesystem
58116
ok 2 - hostname
59-
ok 3 - mounts
60-
ok 4 - capabilities
61-
ok 5 - default symlinks
62-
ok 6 - default devices
63-
ok 7 - linux devices
64-
ok 8 - linux process
65-
ok 9 - masked paths
66-
ok 10 - oom score adj
67-
ok 11 - read only paths
68-
ok 12 - rlimits
69-
ok 13 - sysctls
70-
ok 14 - uid mappings
71-
ok 15 - gid mappings
72-
1..15
73-
--- PASS: TestValidateSysctls (0.20s)
74-
PASS
75-
ok github.com/opencontainers/runtime-tools/validation 0.281s
117+
ok 3 - process
118+
ok 4 - mounts
119+
ok 5 - user
120+
ok 6 - rlimits
121+
ok 7 - capabilities
122+
ok 8 - default symlinks
123+
ok 9 - default file system
124+
ok 10 - default devices
125+
ok 11 - linux devices
126+
ok 12 - linux process
127+
ok 13 - masked paths
128+
ok 14 - oom score adj
129+
ok 15 - read only paths
130+
ok 16 - rootfs propagation
131+
ok 17 - sysctls
132+
ok 18 - uid mappings
133+
ok 19 - gid mappings
134+
1..19
135+
```
136+
137+
If you cannot install node-tap, you can probably run the test suite with another [TAP consumer][tap-consumers].
138+
For example, with [`prove`][prove]:
139+
140+
```console
141+
$ sudo make TAP='prove -Q -j9' RUNTIME=runc localvalidation
142+
RUNTIME=runc prove -Q -j9 validation/linux_rootfs_propagation_shared.t validation/create.t validation/default.t validation/linux_readonly_paths.t validation/linux_masked_paths.t validation/mounts.t validation/process.t validation/root_readonly_false.t validation/linux_sysctl.t validation/linux_devices.t validation/linux_gid_mappings.t validation/process_oom_score_adj.t validation/process_capabilities.t validation/process_rlimits.t validation/root_readonly_true.t validation/linux_rootfs_propagation_unbindable.t validation/hostname.t validation/linux_uid_mappings.t
143+
failed to create the container
144+
rootfsPropagation=unbindable is not supported
145+
exit status 1
146+
failed to create the container
147+
User namespace mappings specified, but USER namespace isn't enabled in the config
148+
exit status 1
149+
150+
Test Summary Report
151+
-------------------
152+
validation/linux_rootfs_propagation_shared.t (Wstat: 0 Tests: 19 Failed: 1)
153+
Failed test: 16
154+
validation/linux_masked_paths.t (Wstat: 0 Tests: 19 Failed: 1)
155+
Failed test: 13
156+
validation/linux_rootfs_propagation_unbindable.t (Wstat: 256 Tests: 0 Failed: 0)
157+
Non-zero exit status: 1
158+
Parse errors: No plan found in TAP output
159+
validation/linux_uid_mappings.t (Wstat: 256 Tests: 0 Failed: 0)
160+
Non-zero exit status: 1
161+
Parse errors: No plan found in TAP output
162+
validation/linux_gid_mappings.t (Wstat: 0 Tests: 19 Failed: 1)
163+
Failed test: 19
164+
Files=18, Tests=271, 6 wallclock secs ( 0.06 usr 0.01 sys + 0.59 cusr 0.24 csys = 0.90 CPU)
165+
Result: FAIL
166+
make: *** [Makefile:43: localvalidation] Error 1
76167
```
77168

78169
[bundle]: https://github.com/opencontainers/runtime-spec/blob/master/bundle.md
79170
[config.json]: https://github.com/opencontainers/runtime-spec/blob/master/config.md
171+
[debian-node-tap]: https://packages.debian.org/stretch/node-tap
172+
[debian-nodejs]: https://packages.debian.org/stretch/nodejs
173+
[gentoo-nodejs]: https://packages.gentoo.org/packages/net-libs/nodejs
174+
[node-tap]: http://www.node-tap.org/
175+
[npm]: https://www.npmjs.com/
176+
[prove]: http://search.cpan.org/~leont/Test-Harness-3.39/bin/prove
80177
[runC]: https://github.com/opencontainers/runc
81178
[runtime-spec]: https://github.com/opencontainers/runtime-spec
179+
[tap-consumers]: https://testanything.org/consumers.html
82180

83181
[generate.1]: man/oci-runtime-tool-generate.1.md
84182
[validate.1]: man/oci-runtime-tool-validate.1.md

‎cmd/runtimetest/main.go‎

Lines changed: 28 additions & 31 deletions
Original file line numberDiff line numberDiff line change
@@ -895,46 +895,43 @@ func run(context *cli.Context) error {
895895
complianceLevel = rfc2119.Must
896896
logrus.Warningf("%s, using 'MUST' by default.", err.Error())
897897
}
898-
var validationErrors error
899-
for _, v := range defaultValidations {
900-
err := v.test(spec)
901-
t.Ok(err == nil, v.description)
902-
if err != nil {
903-
if e, ok := err.(*specerror.Error); ok && e.Err.Level < complianceLevel {
904-
continue
905-
}
906-
validationErrors = multierror.Append(validationErrors, err)
907-
}
908-
}
909898

910-
if platform == "linux" || platform == "solaris" {
911-
for _, v := range posixValidations {
912-
err := v.test(spec)
913-
t.Ok(err == nil, v.description)
914-
if err != nil {
915-
if e, ok := err.(*specerror.Error); ok && e.Err.Level < complianceLevel {
916-
continue
917-
}
918-
validationErrors = multierror.Append(validationErrors, err)
919-
}
920-
}
899+
validations := defaultValidations
900+
if platform == "linux" {
901+
validations = append(validations, posixValidations...)
902+
validations = append(validations, linuxValidations...)
903+
} else if platform == "solaris" {
904+
validations = append(validations, posixValidations...)
921905
}
922906

923-
if platform == "linux" {
924-
for _, v := range linuxValidations {
925-
err := v.test(spec)
926-
t.Ok(err == nil, v.description)
927-
if err != nil {
928-
if e, ok := err.(*specerror.Error); ok && e.Err.Level < complianceLevel {
929-
continue
907+
for _, v := range validations {
908+
err := v.test(spec)
909+
if err == nil {
910+
t.Pass(v.description)
911+
} else {
912+
merr, ok := err.(*multierror.Error)
913+
if ok {
914+
for _, err = range merr.Errors {
915+
if e, ok := err.(*rfc2119.Error); ok {
916+
t.Ok(e.Level < complianceLevel, v.description)
917+
} else {
918+
t.Fail(v.description)
919+
}
920+
t.Diagnostic(err.Error())
921+
}
922+
} else {
923+
if e, ok := err.(*rfc2119.Error); ok {
924+
t.Ok(e.Level < complianceLevel, v.description)
925+
} else {
926+
t.Fail(v.description)
930927
}
931-
validationErrors = multierror.Append(validationErrors, err)
928+
t.Diagnostic(err.Error())
932929
}
933930
}
934931
}
935932
t.AutoPlan()
936933

937-
return validationErrors
934+
return nil
938935
}
939936

940937
func main() {

‎validation/generate_test.go‎ renamed to ‎generate/generate_test.go‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -1,4 +1,4 @@
1-
package validation
1+
package generate_test
22

33
import (
44
"io/ioutil"

‎validation/.gitignore‎

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1 @@
1+
/*.t

0 commit comments

Comments
 (0)