Skip to content

Commit 772feb4

Browse files
AkihiroSudaAkihiroSuda
authored
Merge pull request containerd#3418 from apostasie/dev-3130-tesutil-nextgen
New test tooling
2 parents 9ece613 + a74f0f3 commit 772feb4

23 files changed

+2599
-1239
lines changed

‎.github/workflows/test-canary.yml

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -112,5 +112,5 @@ jobs:
112112
ctrdVersion: ${{ env.CONTAINERD_VERSION }}
113113
run: powershell hack/configure-windows-ci.ps1
114114
- name: "Run integration tests"
115-
# See https://github.com/containerd/nerdctl/blob/main/docs/dev/testing.md#about-parallelization
115+
# See https://github.com/containerd/nerdctl/blob/main/docs/testing/README.md#about-parallelization
116116
run: go test -p 1 -v ./cmd/nerdctl/...

‎.github/workflows/test-kube.yml

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -22,7 +22,7 @@ jobs:
2222
with:
2323
fetch-depth: 1
2424
- name: "Run Kubernetes integration tests"
25-
# See https://github.com/containerd/nerdctl/blob/main/docs/dev/testing.md#about-parallelization
25+
# See https://github.com/containerd/nerdctl/blob/main/docs/testing/README.md#about-parallelization
2626
run: |
2727
./hack/build-integration-kubernetes.sh
2828
sudo ./_output/nerdctl exec nerdctl-test-control-plane bash -c -- 'export TMPDIR="$HOME"/tmp; mkdir -p "$TMPDIR"; cd /nerdctl-source; /usr/local/go/bin/go test -p 1 ./cmd/nerdctl/... -test.only-kubernetes'

‎.github/workflows/test.yml

Lines changed: 3 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -290,15 +290,15 @@ jobs:
290290
timeout_minutes: 30
291291
max_attempts: 2
292292
retry_on: error
293-
# See https://github.com/containerd/nerdctl/blob/main/docs/dev/testing.md#about-parallelization
293+
# See https://github.com/containerd/nerdctl/blob/main/docs/testing/README.md#about-parallelization
294294
command: go test -p 1 -timeout 20m -v -exec sudo ./cmd/nerdctl/... -args -test.target=docker -test.allow-kill-daemon
295295
- name: "Ensure that the IPv6 integration test suite is compatible with Docker"
296296
uses: nick-fields/retry@v3
297297
with:
298298
timeout_minutes: 30
299299
max_attempts: 2
300300
retry_on: error
301-
# See https://github.com/containerd/nerdctl/blob/main/docs/dev/testing.md#about-parallelization
301+
# See https://github.com/containerd/nerdctl/blob/main/docs/testing/README.md#about-parallelization
302302
command: go test -p 1 -timeout 20m -v -exec sudo ./cmd/nerdctl/... -args -test.target=docker -test.allow-kill-daemon -test.only-ipv6
303303

304304
test-integration-windows:
@@ -332,7 +332,7 @@ jobs:
332332
run: powershell hack/configure-windows-ci.ps1
333333
# TODO: Run unit tests
334334
- name: "Run integration tests"
335-
# See https://github.com/containerd/nerdctl/blob/main/docs/dev/testing.md#about-parallelization
335+
# See https://github.com/containerd/nerdctl/blob/main/docs/testing/README.md#about-parallelization
336336
run: go test -p 1 -v ./cmd/nerdctl/...
337337

338338
test-integration-freebsd:

‎README.md

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -253,7 +253,7 @@ Using `go install github.com/containerd/nerdctl/v2/cmd/nerdctl` is possible, but
253253

254254
### Testing
255255

256-
See [testing nerdctl](docs/dev/testing.md).
256+
See [testing nerdctl](docs/testing/README.md).
257257

258258
### Contributing to nerdctl
259259

‎cmd/nerdctl/main_linux_test.go

Lines changed: 30 additions & 8 deletions
Original file line numberDiff line numberDiff line change
@@ -20,17 +20,39 @@ import (
2020
"testing"
2121

2222
"github.com/containerd/nerdctl/v2/pkg/testutil"
23+
"github.com/containerd/nerdctl/v2/pkg/testutil/nerdtest"
24+
"github.com/containerd/nerdctl/v2/pkg/testutil/test"
2325
)
2426

2527
// TestIssue108 tests https://github.com/containerd/nerdctl/issues/108
2628
// ("`nerdctl run --net=host -it` fails while `nerdctl run -it --net=host` works")
2729
func TestIssue108(t *testing.T) {
28-
base := testutil.NewBase(t)
29-
// unbuffer(1) emulates tty, which is required by `nerdctl run -t`.
30-
// unbuffer(1) can be installed with `apt-get install expect`.
31-
unbuffer := []string{"unbuffer"}
32-
base.CmdWithHelper(unbuffer, "run", "-it", "--rm", "--net=host", testutil.AlpineImage,
33-
"echo", "this was always working").AssertOK()
34-
base.CmdWithHelper(unbuffer, "run", "--rm", "--net=host", "-it", testutil.AlpineImage,
35-
"echo", "this was not working due to issue #108").AssertOK()
30+
nerdtest.Setup()
31+
32+
testGroup := &test.Group{
33+
{
34+
Description: "-it --net=host",
35+
Require: test.Binary("unbuffer"),
36+
Command: func(data test.Data, helpers test.Helpers) test.Command {
37+
return helpers.
38+
Command("run", "-it", "--rm", "--net=host", testutil.AlpineImage, "echo", "this was always working").
39+
WithWrapper("unbuffer")
40+
},
41+
// Note: unbuffer will merge stdout and stderr, preventing exact match here
42+
Expected: test.Expects(0, nil, test.Contains("this was always working")),
43+
},
44+
{
45+
Description: "--net=host -it",
46+
Require: test.Binary("unbuffer"),
47+
Command: func(data test.Data, helpers test.Helpers) test.Command {
48+
return helpers.
49+
Command("run", "--rm", "--net=host", "-it", testutil.AlpineImage, "echo", "this was not working due to issue #108").
50+
WithWrapper("unbuffer")
51+
},
52+
// Note: unbuffer will merge stdout and stderr, preventing exact match here
53+
Expected: test.Expects(0, nil, test.Contains("this was not working due to issue #108")),
54+
},
55+
}
56+
57+
testGroup.Run(t)
3658
}

‎cmd/nerdctl/main_test.go

Lines changed: 96 additions & 53 deletions
Original file line numberDiff line numberDiff line change
@@ -17,15 +17,14 @@
1717
package main
1818

1919
import (
20-
"os"
21-
"path/filepath"
20+
"errors"
2221
"testing"
2322

24-
"gotest.tools/v3/assert"
25-
2623
"github.com/containerd/containerd/v2/defaults"
2724

2825
"github.com/containerd/nerdctl/v2/pkg/testutil"
26+
"github.com/containerd/nerdctl/v2/pkg/testutil/nerdtest"
27+
"github.com/containerd/nerdctl/v2/pkg/testutil/test"
2928
)
3029

3130
func TestMain(m *testing.M) {
@@ -34,56 +33,100 @@ func TestMain(m *testing.M) {
3433

3534
// TestUnknownCommand tests https://github.com/containerd/nerdctl/issues/487
3635
func TestUnknownCommand(t *testing.T) {
37-
t.Parallel()
38-
base := testutil.NewBase(t)
39-
base.Cmd("non-existent-command").AssertFail()
40-
base.Cmd("non-existent-command", "info").AssertFail()
41-
base.Cmd("system", "non-existent-command").AssertFail()
42-
base.Cmd("system", "non-existent-command", "info").AssertFail()
43-
base.Cmd("system").AssertOK() // show help without error
44-
base.Cmd("system", "info").AssertOutContains("Kernel Version:")
45-
base.Cmd("info").AssertOutContains("Kernel Version:")
36+
nerdtest.Setup()
37+
38+
var unknownSubCommand = errors.New("unknown subcommand")
39+
40+
testGroup := &test.Group{
41+
{
42+
Description: "non-existent-command",
43+
Command: test.RunCommand("non-existent-command"),
44+
Expected: test.Expects(1, []error{unknownSubCommand}, nil),
45+
},
46+
{
47+
Description: "non-existent-command info",
48+
Command: test.RunCommand("non-existent-command", "info"),
49+
Expected: test.Expects(1, []error{unknownSubCommand}, nil),
50+
},
51+
{
52+
Description: "system non-existent-command",
53+
Command: test.RunCommand("system", "non-existent-command"),
54+
Expected: test.Expects(1, []error{unknownSubCommand}, nil),
55+
},
56+
{
57+
Description: "system non-existent-command info",
58+
Command: test.RunCommand("system", "non-existent-command", "info"),
59+
Expected: test.Expects(1, []error{unknownSubCommand}, nil),
60+
},
61+
{
62+
Description: "system",
63+
Command: test.RunCommand("system"),
64+
Expected: test.Expects(0, nil, nil),
65+
},
66+
{
67+
Description: "system info",
68+
Command: test.RunCommand("system", "info"),
69+
Expected: test.Expects(0, nil, test.Contains("Kernel Version:")),
70+
},
71+
{
72+
Description: "info",
73+
Command: test.RunCommand("info"),
74+
Expected: test.Expects(0, nil, test.Contains("Kernel Version:")),
75+
},
76+
}
77+
78+
testGroup.Run(t)
4679
}
4780

48-
// TestNerdctlConfig validates the configuration precedence [CLI, Env, TOML, Default].
81+
// TestNerdctlConfig validates the configuration precedence [CLI, Env, TOML, Default] and broken config rejection
4982
func TestNerdctlConfig(t *testing.T) {
50-
testutil.DockerIncompatible(t)
51-
t.Parallel()
52-
tomlPath := filepath.Join(t.TempDir(), "nerdctl.toml")
53-
err := os.WriteFile(tomlPath, []byte(`
54-
snapshotter = "dummy-snapshotter-via-toml"
55-
`), 0400)
56-
assert.NilError(t, err)
57-
base := testutil.NewBase(t)
58-
59-
// [Default]
60-
base.Cmd("info", "-f", "{{.Driver}}").AssertOutExactly(defaults.DefaultSnapshotter + "\n")
61-
62-
// [TOML, Default]
63-
base.Env = append(base.Env, "NERDCTL_TOML="+tomlPath)
64-
base.Cmd("info", "-f", "{{.Driver}}").AssertOutExactly("dummy-snapshotter-via-toml\n")
65-
66-
// [CLI, TOML, Default]
67-
base.Cmd("info", "-f", "{{.Driver}}", "--snapshotter=dummy-snapshotter-via-cli").AssertOutExactly("dummy-snapshotter-via-cli\n")
68-
69-
// [Env, TOML, Default]
70-
base.Env = append(base.Env, "CONTAINERD_SNAPSHOTTER=dummy-snapshotter-via-env")
71-
base.Cmd("info", "-f", "{{.Driver}}").AssertOutExactly("dummy-snapshotter-via-env\n")
72-
73-
// [CLI, Env, TOML, Default]
74-
base.Cmd("info", "-f", "{{.Driver}}", "--snapshotter=dummy-snapshotter-via-cli").AssertOutExactly("dummy-snapshotter-via-cli\n")
75-
}
76-
77-
func TestNerdctlConfigBad(t *testing.T) {
78-
testutil.DockerIncompatible(t)
79-
t.Parallel()
80-
tomlPath := filepath.Join(t.TempDir(), "config.toml")
81-
err := os.WriteFile(tomlPath, []byte(`
82-
# containerd config, not nerdctl config
83-
version = 2
84-
`), 0400)
85-
assert.NilError(t, err)
86-
base := testutil.NewBase(t)
87-
base.Env = append(base.Env, "NERDCTL_TOML="+tomlPath)
88-
base.Cmd("info").AssertFail()
83+
nerdtest.Setup()
84+
85+
tc := &test.Case{
86+
Description: "Nerdctl configuration",
87+
// Docker does not support nerdctl.toml obviously
88+
Require: test.Not(nerdtest.Docker),
89+
SubTests: []*test.Case{
90+
{
91+
Description: "Default",
92+
Command: test.RunCommand("info", "-f", "{{.Driver}}"),
93+
Expected: test.Expects(0, nil, test.Equals(defaults.DefaultSnapshotter+"\n")),
94+
},
95+
{
96+
Description: "TOML > Default",
97+
Command: test.RunCommand("info", "-f", "{{.Driver}}"),
98+
Expected: test.Expects(0, nil, test.Equals("dummy-snapshotter-via-toml\n")),
99+
Data: test.WithConfig(nerdtest.NerdctlToml, `snapshotter = "dummy-snapshotter-via-toml"`),
100+
},
101+
{
102+
Description: "Cli > TOML > Default",
103+
Command: test.RunCommand("info", "-f", "{{.Driver}}", "--snapshotter=dummy-snapshotter-via-cli"),
104+
Expected: test.Expects(0, nil, test.Equals("dummy-snapshotter-via-cli\n")),
105+
Data: test.WithConfig(nerdtest.NerdctlToml, `snapshotter = "dummy-snapshotter-via-toml"`),
106+
},
107+
{
108+
Description: "Env > TOML > Default",
109+
Command: test.RunCommand("info", "-f", "{{.Driver}}"),
110+
Env: map[string]string{"CONTAINERD_SNAPSHOTTER": "dummy-snapshotter-via-env"},
111+
Expected: test.Expects(0, nil, test.Equals("dummy-snapshotter-via-env\n")),
112+
Data: test.WithConfig(nerdtest.NerdctlToml, `snapshotter = "dummy-snapshotter-via-toml"`),
113+
},
114+
{
115+
Description: "Cli > Env > TOML > Default",
116+
Command: test.RunCommand("info", "-f", "{{.Driver}}", "--snapshotter=dummy-snapshotter-via-cli"),
117+
Env: map[string]string{"CONTAINERD_SNAPSHOTTER": "dummy-snapshotter-via-env"},
118+
Expected: test.Expects(0, nil, test.Equals("dummy-snapshotter-via-cli\n")),
119+
Data: test.WithConfig(nerdtest.NerdctlToml, `snapshotter = "dummy-snapshotter-via-toml"`),
120+
},
121+
{
122+
Description: "Broken config",
123+
Command: test.RunCommand("info"),
124+
Expected: test.Expects(1, []error{errors.New("failed to load nerdctl config")}, nil),
125+
Data: test.WithConfig(nerdtest.NerdctlToml, `# containerd config, not nerdctl config
126+
version = 2`),
127+
},
128+
},
129+
}
130+
131+
tc.Run(t)
89132
}

0 commit comments

Comments
 (0)