Merge pull request moby#3612 from crazy-max/itg-dockerd-flags

integration: set custom flags for dockerd worker

Author: tonistiigi

.github/workflows/dockerd.yml

@@ -137,3 +137,6 @@ jobs:
           TESTFLAGS: "${{ env.TESTFLAGS }} --run=//worker=${{ matrix.worker }}$"
           SKIP_INTEGRATION_TESTS: "${{ matrix.skip-integration-tests }}"
           CACHE_FROM: "type=gha,scope=${{ env.CACHE_GHA_SCOPE_IT }} type=gha,scope=${{ env.CACHE_GHA_SCOPE_BINARIES }}"
+          BUILDKIT_INTEGRATION_DOCKERD_FLAGS: |
+            --bip=10.66.66.1/24
+            --default-address-pool=base=10.66.66.0/16,size=24

hack/test

@@ -78,7 +78,7 @@ if ! docker container inspect "$cacheVolume" >/dev/null 2>/dev/null; then
 fi
 
 if [ "$TEST_INTEGRATION" == 1 ]; then
-  cid=$(docker create --rm -v /tmp $coverageVol --volumes-from=$cacheVolume -e GITHUB_REF -e ACTIONS_RUNTIME_TOKEN -e ACTIONS_CACHE_URL -e TEST_DOCKERD -e SKIP_INTEGRATION_TESTS ${BUILDKIT_INTEGRATION_SNAPSHOTTER:+"-eBUILDKIT_INTEGRATION_SNAPSHOTTER"} -e BUILDKIT_REGISTRY_MIRROR_DIR=/root/.cache/registry --privileged $iid go test $coverageFlags ${TESTFLAGS:--v} ${TESTPKGS:-./...})
+  cid=$(docker create --rm -v /tmp $coverageVol --volumes-from=$cacheVolume -e GITHUB_REF -e ACTIONS_RUNTIME_TOKEN -e ACTIONS_CACHE_URL -e TEST_DOCKERD -e SKIP_INTEGRATION_TESTS ${BUILDKIT_INTEGRATION_SNAPSHOTTER:+"-eBUILDKIT_INTEGRATION_SNAPSHOTTER"} -e BUILDKIT_REGISTRY_MIRROR_DIR=/root/.cache/registry -e BUILDKIT_INTEGRATION_DOCKERD_FLAGS --privileged $iid go test $coverageFlags ${TESTFLAGS:--v} ${TESTPKGS:-./...})
   if [ "$TEST_DOCKERD" = "1" ]; then
     docker cp "$TEST_DOCKERD_BINARY" $cid:/usr/bin/dockerd
   fi
@@ -118,7 +118,7 @@ if [ "$TEST_DOCKERFILE" == 1 ]; then
 
     if [ -s $tarout ]; then
       if [ "$release" = "mainline" ] || [ "$release" = "labs" ] || [ -n "$DOCKERFILE_RELEASES_CUSTOM" ] || [ "$GITHUB_ACTIONS" = "true" ]; then
-        cid=$(docker create -v /tmp $coverageVol --rm --privileged --volumes-from=$cacheVolume -e GITHUB_REF -e ACTIONS_RUNTIME_TOKEN -e ACTIONS_CACHE_URL -e TEST_DOCKERD -e BUILDKIT_REGISTRY_MIRROR_DIR=/root/.cache/registry -e BUILDKIT_WORKER_RANDOM -e FRONTEND_GATEWAY_ONLY=local:/$release.tar -e EXTERNAL_DF_FRONTEND=/dockerfile-frontend $iid go test $coverageFlags --count=1 -tags "$buildtags" ${TESTFLAGS:--v} ./frontend/dockerfile)
+        cid=$(docker create -v /tmp $coverageVol --rm --privileged --volumes-from=$cacheVolume -e GITHUB_REF -e ACTIONS_RUNTIME_TOKEN -e ACTIONS_CACHE_URL -e TEST_DOCKERD -e BUILDKIT_REGISTRY_MIRROR_DIR=/root/.cache/registry -e BUILDKIT_WORKER_RANDOM -e BUILDKIT_INTEGRATION_DOCKERD_FLAGS -e FRONTEND_GATEWAY_ONLY=local:/$release.tar -e EXTERNAL_DF_FRONTEND=/dockerfile-frontend $iid go test $coverageFlags --count=1 -tags "$buildtags" ${TESTFLAGS:--v} ./frontend/dockerfile)
         docker cp $tarout $cid:/$release.tar
         if [ "$TEST_DOCKERD" = "1" ]; then
           docker cp "$TEST_DOCKERD_BINARY" $cid:/usr/bin/dockerd

util/testutil/dockerd/daemon.go

@@ -2,6 +2,7 @@ package dockerd
 
 import (
 	"bytes"
+	"fmt"
 	"io"
 	"os"
 	"os/exec"
@@ -137,6 +138,7 @@ func (d *Daemon) StartWithError(daemonLogs map[string]*bytes.Buffer, providedArg
 		d.cmd.Stderr = &lockingWriter{Writer: b}
 	}
 
+	fmt.Fprintf(d.cmd.Stderr, "> startCmd %v %+v\n", time.Now(), d.cmd.String())
 	if err := d.cmd.Start(); err != nil {
 		return errors.Wrapf(err, "[%s] could not start daemon container", d.id)
 	}

util/testutil/integration/dockerd.go

@@ -1,13 +1,13 @@
 package integration
 
 import (
-	"bytes"
 	"context"
 	"encoding/json"
 	"io"
 	"net"
 	"os"
 	"path/filepath"
+	"strings"
 	"time"
 
 	"github.com/docker/docker/client"
@@ -125,21 +125,23 @@ func (c moby) New(ctx context.Context, cfg *BackendConfig) (b Backend, cl func()
 		return nil, nil, err
 	}
 
-	err = d.StartWithError(cfg.Logs,
+	dockerdFlags := []string{
 		"--config-file", dockerdConfigFile,
 		"--userland-proxy=false",
-		"--bip", "10.66.66.1/24",
-		"--default-address-pool", "base=10.66.66.0/16,size=24",
 		"--debug",
-	)
+	}
+	if s := os.Getenv("BUILDKIT_INTEGRATION_DOCKERD_FLAGS"); s != "" {
+		dockerdFlags = append(dockerdFlags, strings.Split(strings.TrimSpace(s), "\n")...)
+	}
+
+	err = d.StartWithError(cfg.Logs, dockerdFlags...)
 	if err != nil {
 		return nil, nil, err
 	}
 	deferF.append(d.StopWithError)
 
-	logs := map[string]*bytes.Buffer{}
 	if err := waitUnix(d.Sock(), 5*time.Second); err != nil {
-		return nil, nil, errors.Errorf("dockerd did not start up: %q, %s", err, formatLogs(logs))
+		return nil, nil, errors.Errorf("dockerd did not start up: %q, %s", err, formatLogs(cfg.Logs))
 	}
 
 	dockerAPI, err := client.NewClientWithOpts(client.WithHost(d.Sock()))