Merge pull request containerd#3384 from apostasie/dev-testing-fix

Testing-rig fixes

Author: AkihiroSuda

.github/workflows/test-canary.yml

@@ -112,4 +112,5 @@ jobs:
           ctrdVersion: ${{ env.CONTAINERD_VERSION }}
         run: powershell hack/configure-windows-ci.ps1
       - name: "Run integration tests"
-        run: go test -v ./cmd/...
+        # See https://github.com/containerd/nerdctl/blob/main/docs/testing.md#about-parallelization
+        run: go test -p 1 -v ./cmd/nerdctl/...

.github/workflows/test-kube.yml

@@ -22,6 +22,7 @@ jobs:
         with:
           fetch-depth: 1
       - name: "Run Kubernetes integration tests"
+        # See https://github.com/containerd/nerdctl/blob/main/docs/testing.md#about-parallelization
         run: |
           ./hack/build-integration-kubernetes.sh
-          sudo ./_output/nerdctl exec nerdctl-test-control-plane bash -c -- 'export TMPDIR="$HOME"/tmp; mkdir -p "$TMPDIR"; cd /nerdctl-source; /usr/local/go/bin/go test ./cmd/nerdctl/ -test.only-kubernetes'
+          sudo ./_output/nerdctl exec nerdctl-test-control-plane bash -c -- 'export TMPDIR="$HOME"/tmp; mkdir -p "$TMPDIR"; cd /nerdctl-source; /usr/local/go/bin/go test -p 1 ./cmd/nerdctl/... -test.only-kubernetes'

.github/workflows/test.yml

@@ -290,14 +290,16 @@ jobs:
           timeout_minutes: 30
           max_attempts: 2
           retry_on: error
-          command: go test -timeout 20m -v -exec sudo ./cmd/nerdctl/... -args -test.target=docker -test.allow-kill-daemon
+          # See https://github.com/containerd/nerdctl/blob/main/docs/testing.md#about-parallelization
+          command: go test -p 1 -timeout 20m -v -exec sudo ./cmd/nerdctl/... -args -test.target=docker -test.allow-kill-daemon
       - name: "Ensure that the IPv6 integration test suite is compatible with Docker"
         uses: nick-fields/retry@v3
         with:
           timeout_minutes: 30
           max_attempts: 2
           retry_on: error
-          command: go test -timeout 20m -v -exec sudo ./cmd/nerdctl/... -args -test.target=docker -test.allow-kill-daemon -test.only-ipv6
+          # See https://github.com/containerd/nerdctl/blob/main/docs/testing.md#about-parallelization
+          command: go test -p 1 -timeout 20m -v -exec sudo ./cmd/nerdctl/... -args -test.target=docker -test.allow-kill-daemon -test.only-ipv6
 
   test-integration-windows:
     runs-on: windows-2022
@@ -330,7 +332,8 @@ jobs:
         run: powershell hack/configure-windows-ci.ps1
       # TODO: Run unit tests
       - name: "Run integration tests"
-        run: go test -v ./cmd/...
+        # See https://github.com/containerd/nerdctl/blob/main/docs/testing.md#about-parallelization
+        run: go test -p 1 -v ./cmd/nerdctl/...
 
   test-integration-freebsd:
     name: FreeBSD

Dockerfile

@@ -318,8 +318,8 @@ RUN curl -o nydus-static.tgz -fsSL --proto '=https' --tlsv1.2 "https://github.co
   tar xzf nydus-static.tgz && \
   mv nydus-static/nydus-image nydus-static/nydusd nydus-static/nydusify /usr/bin/ && \
   rm nydus-static.tgz
-CMD ["gotestsum", "--format=testname", "--rerun-fails=2", "--packages=github.com/containerd/nerdctl/v2/cmd/nerdctl/...", \
-  "--", "-timeout=60m", "-args", "-test.allow-kill-daemon"]
+CMD ["gotestsum", "--format=testname", "--rerun-fails=2", "--packages=./cmd/nerdctl/...", \
+  "--", "-timeout=60m", "-p", "1", "-args", "-test.allow-kill-daemon"]
 
 FROM test-integration AS test-integration-rootless
 # Install SSH for creating systemd user session.
@@ -340,20 +340,19 @@ COPY ./Dockerfile.d/[email protected]_delegate.conf /etc/system
 # ipfs daemon for rootless containerd will be enabled in /test-integration-rootless.sh
 RUN systemctl disable test-integration-ipfs-offline
 VOLUME /home/rootless/.local/share
-RUN go test -o /usr/local/bin/nerdctl.test -c ./cmd/nerdctl
 COPY ./Dockerfile.d/test-integration-rootless.sh /
+RUN chmod a+rx /test-integration-rootless.sh
 CMD ["/test-integration-rootless.sh", \
-  "gotestsum", "--format=testname", "--rerun-fails=2", "--raw-command", \
-  "--", "/usr/local/go/bin/go", "tool", "test2json", "-t", "-p", "github.com/containerd/nerdctl/v2/cmd/nerdctl",  \
-  "/usr/local/bin/nerdctl.test", "-test.v", "-test.timeout=60m", "-test.allow-kill-daemon"]
+  "gotestsum", "--format=testname", "--rerun-fails=2", "--packages=./cmd/nerdctl/...", \
+  "--", "-timeout=60m", "-p", "1", "-args", "-test.allow-kill-daemon"]
 
 # test for CONTAINERD_ROOTLESS_ROOTLESSKIT_PORT_DRIVER=slirp4netns
 FROM test-integration-rootless AS test-integration-rootless-port-slirp4netns
 COPY ./Dockerfile.d/home_rootless_.config_systemd_user_containerd.service.d_port-slirp4netns.conf /home/rootless/.config/systemd/user/containerd.service.d/port-slirp4netns.conf
 RUN chown -R rootless:rootless /home/rootless/.config
 
 FROM test-integration AS test-integration-ipv6
-CMD ["gotestsum", "--format=testname", "--rerun-fails=2", "--packages=github.com/containerd/nerdctl/v2/cmd/nerdctl/...", \
-  "--", "-timeout=60m", "-args", "-test.allow-kill-daemon", "-test.only-ipv6"]
+CMD ["gotestsum", "--format=testname", "--rerun-fails=2", "--packages=./cmd/nerdctl/...", \
+  "--", "-timeout=60m", "-p", "1", "-args", "-test.allow-kill-daemon", "-test.only-ipv6"]
 
 FROM base AS demo

Dockerfile.d/test-integration-rootless.sh

@@ -56,5 +56,8 @@ EOF
 	systemctl --user restart stargz-snapshotter.service
 	export IPFS_PATH="/home/rootless/.local/share/ipfs"
 	containerd-rootless-setuptool.sh install-bypass4netnsd
-	exec "$@"
+	# Once ssh-ed, we lost the Dockerfile working dir, so, get back in the nerdctl checkout
+	cd /go/src/github.com/containerd/nerdctl
+	# We also lose the PATH (and SendEnv=PATH would require sshd config changes)
+	exec env PATH="/usr/local/go/bin:$PATH" "$@"
 fi

cmd/nerdctl/container_run_mount_linux_test.go

@@ -98,6 +98,7 @@ func TestRunAnonymousVolume(t *testing.T) {
 func TestRunVolumeRelativePath(t *testing.T) {
 	t.Parallel()
 	base := testutil.NewBase(t)
+	base.Dir = t.TempDir()
 	base.Cmd("run", "--rm", "-v", "./foo:/mnt/foo", testutil.AlpineImage).AssertOK()
 	base.Cmd("run", "--rm", "-v", "./foo", testutil.AlpineImage).AssertOK()
 

docs/testing.md

@@ -21,18 +21,34 @@ Be sure to first `make && sudo make install`
 
 ```bash
 # Test all with nerdctl (rootless mode, if running go as a non-root user)
-go test ./cmd/nerdctl/...
+go test -p 1 ./cmd/nerdctl/...
 
 # Test all with nerdctl rootful
-go test -exec sudo ./cmd/nerdctl/...
+go test -p 1 -exec sudo ./cmd/nerdctl/...
 
 # Test all with docker
-go test ./cmd/nerdctl/... -args -test.target=docker
+go test -p 1 ./cmd/nerdctl/... -args -test.target=docker
 
 # Test just the tests(s) which names match TestVolume.*
-go test ./cmd/nerdctl/... -run "TestVolume.*"
+go test -p 1 ./cmd/nerdctl/... -run "TestVolume.*"
+# Or alternatively, just test the subpackage
+go test ./cmd/nerdctl/volume
 ```
 
+### About parallelization
+
+By default, when `go test ./foo/...` finds subpackages, it does create _a separate test binary
+per sub-package_, and execute them _in parallel_.
+This effectively will make distinct tests in different subpackages to be executed in
+parallel, regardless of whether they called `t.Parallel` or not.
+
+The `-p 1` flag does inhibit this behavior, and forces go to run each sub-package
+sequentially.
+
+Note that this is different from the `--parallel` flag, which controls the amount of
+parallelization that a single go test binary will use when faced with tests that do
+explicitly allow it (with a call to `t.Parallel()`).
+
 ### Or test in a container
 
 ```bash

pkg/testutil/testutil.go

@@ -58,6 +58,7 @@ type Base struct {
 	Binary               string
 	Args                 []string
 	Env                  []string
+	Dir                  string
 }
 
 // WithStdin sets the standard input of Cmd to the specified reader
@@ -70,6 +71,7 @@ func WithStdin(r io.Reader) func(*Cmd) {
 func (b *Base) Cmd(args ...string) *Cmd {
 	icmdCmd := icmd.Command(b.Binary, append(b.Args, args...)...)
 	icmdCmd.Env = b.Env
+	icmdCmd.Dir = b.Dir
 	cmd := &Cmd{
 		Cmd:  icmdCmd,
 		Base: b,
@@ -83,6 +85,7 @@ func (b *Base) ComposeCmd(args ...string) *Cmd {
 	binaryArgs := append(b.Args, append([]string{"compose"}, args...)...)
 	icmdCmd := icmd.Command(binary, binaryArgs...)
 	icmdCmd.Env = b.Env
+	icmdCmd.Dir = b.Dir
 	cmd := &Cmd{
 		Cmd:  icmdCmd,
 		Base: b,