Merge pull request moby#5027 from profnandaa/integration-tests-4485

tests: enabling integration tests on windows

Author: tonistiigi

.github/workflows/test-os.yml

@@ -77,7 +77,7 @@ jobs:
     needs:
       - build
     env:
-      TESTFLAGS: "-v --parallel=6 --timeout=30m"
+      TESTFLAGS: "-v --timeout=60m"
       GOTESTSUM_FORMAT: "standard-verbose"
     strategy:
       fail-fast: false
@@ -102,7 +102,7 @@ jobs:
           echo "TEST_REPORT_NAME=${{ github.job }}-$(echo "${{ matrix.pkg }}-${{ matrix.skip-integration-tests }}-${{ matrix.worker }}" | tr -dc '[:alnum:]-\n\r' | tr '[:upper:]' '[:lower:]')" >> $GITHUB_ENV
           testFlags="${{ env.TESTFLAGS }}"
           if [ -n "${{ matrix.worker }}" ]; then
-            testFlags="${testFlags} --run=//worker=${{ matrix.worker }}$"
+            testFlags="${testFlags} --run=TestIntegration/.*/worker=${{ matrix.worker }}"
           fi
           echo "TESTFLAGS=${testFlags}" >> $GITHUB_ENV
         shell: bash

client/client_test.go

@@ -6297,6 +6297,7 @@ func testExportLocalNoPlatformSplit(t *testing.T, sb integration.Sandbox) {
 }
 
 func testExportLocalNoPlatformSplitOverwrite(t *testing.T, sb integration.Sandbox) {
+	integration.SkipOnPlatform(t, "windows")
 	workers.CheckFeatureCompat(t, sb, workers.FeatureOCIExporter, workers.FeatureMultiPlatform)
 	c, err := New(sb.Context(), sb.Address())
 	require.NoError(t, err)
@@ -8131,6 +8132,7 @@ func testPullWithLayerLimit(t *testing.T, sb integration.Sandbox) {
 }
 
 func testCallInfo(t *testing.T, sb integration.Sandbox) {
+	integration.SkipOnPlatform(t, "windows")
 	workers.CheckFeatureCompat(t, sb, workers.FeatureInfo)
 	c, err := New(sb.Context(), sb.Address())
 	require.NoError(t, err)
@@ -10568,6 +10570,7 @@ func testLayerLimitOnMounts(t *testing.T, sb integration.Sandbox) {
 }
 
 func testClientCustomGRPCOpts(t *testing.T, sb integration.Sandbox) {
+	integration.SkipOnPlatform(t, "windows")
 	var interceptedMethods []string
 	intercept := func(ctx context.Context, method string, req, reply interface{}, cc *grpc.ClientConn, invoker grpc.UnaryInvoker, opts ...grpc.CallOption) error {
 		interceptedMethods = append(interceptedMethods, method)

frontend/dockerfile/dockerfile_test.go

@@ -232,8 +232,11 @@ type frontend interface {
 func init() {
 	frontends := map[string]interface{}{}
 
+	images := integration.UnixOrWindows(
+		[]string{"busybox:latest", "alpine:latest"},
+		[]string{"nanoserver:latest"})
 	opts = []integration.TestOpt{
-		integration.WithMirroredImages(integration.OfficialImages("busybox:latest", "alpine:latest")),
+		integration.WithMirroredImages(integration.OfficialImages(images...)),
 		integration.WithMatrix("frontend", frontends),
 	}
 
@@ -256,27 +259,32 @@ func init() {
 func TestIntegration(t *testing.T) {
 	integration.Run(t, allTests, opts...)
 
-	integration.Run(t, securityTests, append(append(opts, securityOpts...),
-		integration.WithMatrix("security.insecure", map[string]interface{}{
-			"granted": securityInsecureGranted,
-			"denied":  securityInsecureDenied,
-		}))...)
-	integration.Run(t, networkTests, append(opts,
-		integration.WithMatrix("network.host", map[string]interface{}{
-			"granted": networkHostGranted,
-			"denied":  networkHostDenied,
-		}))...)
 	integration.Run(t, lintTests, opts...)
 	integration.Run(t, heredocTests, opts...)
 	integration.Run(t, outlineTests, opts...)
 	integration.Run(t, targetsTests, opts...)
 
+	// the rest of the tests are meant for non-Windows, skipping on Windows.
+	integration.SkipOnPlatform(t, "windows")
+
 	integration.Run(t, reproTests, append(opts,
 		// Only use the amd64 digest,  regardless to the host platform
 		integration.WithMirroredImages(map[string]string{
 			"amd64/bullseye-20230109-slim:latest": "docker.io/amd64/debian:bullseye-20230109-slim@sha256:1acb06a0c31fb467eb8327ad361f1091ab265e0bf26d452dea45dcb0c0ea5e75",
 		}),
 	)...)
+
+	integration.Run(t, securityTests, append(append(opts, securityOpts...),
+		integration.WithMatrix("security.insecure", map[string]interface{}{
+			"granted": securityInsecureGranted,
+			"denied":  securityInsecureDenied,
+		}))...)
+
+	integration.Run(t, networkTests, append(opts,
+		integration.WithMatrix("network.host", map[string]interface{}{
+			"granted": networkHostGranted,
+			"denied":  networkHostDenied,
+		}))...)
 }
 
 func testEmptyStringArgInEnv(t *testing.T, sb integration.Sandbox) {
@@ -389,14 +397,20 @@ echo -n $my_arg $* > /out
 }
 
 func testEnvEmptyFormatting(t *testing.T, sb integration.Sandbox) {
-	integration.SkipOnPlatform(t, "windows")
 	f := getFrontend(t, sb)
 
-	dockerfile := []byte(`
+	dockerfile := []byte(integration.UnixOrWindows(
+		`
 FROM busybox AS build
 ENV myenv foo%sbar
 RUN [ "$myenv" = 'foo%sbar' ]
-`)
+`,
+		`
+FROM nanoserver AS build
+ENV myenv foo%sbar
+RUN if %myenv% NEQ foo%sbar (exit 1)
+`,
+	))
 
 	dir := integration.Tmpdir(
 		t,
@@ -425,23 +439,36 @@ RUN [ "$myenv" = 'foo%sbar' ]
 }
 
 func testDockerignoreOverride(t *testing.T, sb integration.Sandbox) {
-	integration.SkipOnPlatform(t, "windows")
 	f := getFrontend(t, sb)
-	dockerfile := []byte(`
+	dockerfile := []byte(integration.UnixOrWindows(
+		`
 FROM busybox
 COPY . .
 RUN [ -f foo ] && [ ! -f bar ]
-`)
+`,
+		`
+FROM nanoserver
+COPY . .
+RUN if exist foo (if not exist bar (exit 0) else (exit 1))
+`,
+	))
 
 	ignore := []byte(`
 bar
 `)
 
-	dockerfile2 := []byte(`
+	dockerfile2 := []byte(integration.UnixOrWindows(
+		`
 FROM busybox
 COPY . .
 RUN [ ! -f foo ] && [ -f bar ]
-`)
+`,
+		`
+FROM nanoserver
+COPY . .
+RUN if not exist foo (if exist bar (exit 0) else (exit 1))
+`,
+	))
 
 	ignore2 := []byte(`
 foo
@@ -482,15 +509,22 @@ foo
 }
 
 func testEmptyDestDir(t *testing.T, sb integration.Sandbox) {
-	integration.SkipOnPlatform(t, "windows")
 	f := getFrontend(t, sb)
 
-	dockerfile := []byte(`
+	dockerfile := []byte(integration.UnixOrWindows(
+		`
 FROM busybox
 ENV empty=""
 COPY testfile $empty
 RUN [ "$(cat testfile)" == "contents0" ]
-`)
+`,
+		`
+FROM nanoserver
+COPY testfile ''
+RUN cmd /V:on /C "set /p tfcontent=<testfile \
+	& if !tfcontent! NEQ contents0 (exit 1)"
+`,
+	))
 
 	dir := integration.Tmpdir(
 		t,

util/testutil/integration/pins.go

@@ -1,3 +1,6 @@
+//go:build !windows
+// +build !windows
+
 package integration
 
 var pins = map[string]map[string]string{

util/testutil/integration/run.go

@@ -189,7 +189,9 @@ func Run(t *testing.T, testCases []Test, opt ...TestOpt) {
 							t.Skip("rootless")
 						}
 						ctx := appcontext.Context()
-						if !strings.HasSuffix(fn, "NoParallel") {
+						// TODO(profnandaa): to revisit this to allow tests run
+						// in parallel on Windows in a stable way. Is flaky currently.
+						if !strings.HasSuffix(fn, "NoParallel") && runtime.GOOS != "windows" {
 							t.Parallel()
 						}
 						require.NoError(t, sandboxLimiter.Acquire(context.TODO(), 1))
@@ -273,23 +275,7 @@ func copyImagesLocal(t *testing.T, host string, images map[string]string) error
 }
 
 func OfficialImages(names ...string) map[string]string {
-	ns := runtime.GOARCH
-	if ns == "arm64" {
-		ns = "arm64v8"
-	} else if ns != "amd64" {
-		ns = "library"
-	}
-	m := map[string]string{}
-	for _, name := range names {
-		ref := "docker.io/" + ns + "/" + name
-		if pns, ok := pins[name]; ok {
-			if dgst, ok := pns[ns]; ok {
-				ref += "@" + dgst
-			}
-		}
-		m["library/"+name] = ref
-	}
-	return m
+	return officialImages(names...)
 }
 
 func withMirrorConfig(mirror string) ConfigUpdater {
@@ -439,9 +425,19 @@ func prepareValueMatrix(tc testConf) []matrixValue {
 	return m
 }
 
-// Skips tests on Windows
+// Skips tests on platform
 func SkipOnPlatform(t *testing.T, goos string) {
 	if runtime.GOOS == goos {
 		t.Skipf("Skipped on %s", goos)
 	}
 }
+
+// Selects between two types, returns second
+// argument if on Windows or else first argument.
+// Typically used for selecting test cases.
+func UnixOrWindows[T any](unix, windows T) T {
+	if runtime.GOOS == "windows" {
+		return windows
+	}
+	return unix
+}

util/testutil/integration/run_unix.go

@@ -0,0 +1,26 @@
+//go:build !windows
+// +build !windows
+
+package integration
+
+import "runtime"
+
+func officialImages(names ...string) map[string]string {
+	ns := runtime.GOARCH
+	if ns == "arm64" {
+		ns = "arm64v8"
+	} else if ns != "amd64" {
+		ns = "library"
+	}
+	m := map[string]string{}
+	for _, name := range names {
+		ref := "docker.io/" + ns + "/" + name
+		if pns, ok := pins[name]; ok {
+			if dgst, ok := pns[ns]; ok {
+				ref += "@" + dgst
+			}
+		}
+		m["library/"+name] = ref
+	}
+	return m
+}

util/testutil/integration/run_windows.go

@@ -0,0 +1,13 @@
+package integration
+
+func officialImages(names ...string) map[string]string {
+	m := map[string]string{}
+	for _, name := range names {
+		// select available refs from the mirror map
+		// so that we mirror only those needed for the tests
+		if ref, ok := windowsImagesMirrorMap[name]; ok {
+			m["library/"+name] = ref
+		}
+	}
+	return m
+}

util/testutil/integration/util.go

@@ -94,8 +94,12 @@ func StartCmd(cmd *exec.Cmd, logs map[string]*bytes.Buffer) (func() error, error
 		case <-ctx.Done():
 		case <-stopped:
 		case <-stop:
+			// windows processes not responding to SIGTERM
+			signal := UnixOrWindows(syscall.SIGTERM, syscall.SIGKILL)
+			signalStr := UnixOrWindows("SIGTERM", "SIGKILL")
 			fmt.Fprintf(cmd.Stderr, "> sending sigterm %v\n", time.Now())
-			cmd.Process.Signal(syscall.SIGTERM)
+			fmt.Fprintf(cmd.Stderr, "> sending %s %v\n", signalStr, time.Now())
+			cmd.Process.Signal(signal)
 			go func() {
 				select {
 				case <-stopped:

util/testutil/integration/util_windows.go

@@ -11,6 +11,13 @@ import (
 
 var socketScheme = "npipe://"
 
+var windowsImagesMirrorMap = map[string]string{
+	// TODO(profnandaa): currently, amd64 only, to revisit for other archs.
+	"nanoserver:latest": "mcr.microsoft.com/windows/nanoserver:ltsc2022",
+	"servercore:latest": "mcr.microsoft.com/windows/servercore:ltsc2022",
+	"busybox:latest":    "registry.k8s.io/e2e-test-images/busybox@sha256:6d854ffad9666d2041b879a1c128c9922d77faced7745ad676639b07111ab650",
+}
+
 // abstracted function to handle pipe dialing on windows.
 // some simplification has been made to discard timeout param.
 func dialPipe(address string) (net.Conn, error) {