diff --git a/e2e/build_test.go b/e2e/build_test.go index 6aa71092b..a0a45f543 100644 --- a/e2e/build_test.go +++ b/e2e/build_test.go @@ -21,7 +21,7 @@ import ( ) func TestBuild(t *testing.T) { - runWithDindSwarmAndRegistry(t, func(info dindSwarmAndRegistryInfo) { + runWithDindSwarmAndRegistry(t, func(info OrchestratorAndRegistryInfo) { cmd := info.configuredCmd tmp := fs.NewDir(t, "TestBuild") @@ -59,7 +59,7 @@ func TestBuild(t *testing.T) { } func TestBuildMultiTag(t *testing.T) { - runWithDindSwarmAndRegistry(t, func(info dindSwarmAndRegistryInfo) { + runWithDindSwarmAndRegistry(t, func(info OrchestratorAndRegistryInfo) { cmd := info.configuredCmd tmp := fs.NewDir(t, "TestBuild") testDir := path.Join("testdata", "build") @@ -96,7 +96,7 @@ func TestBuildMultiTag(t *testing.T) { } func TestQuietBuild(t *testing.T) { - runWithDindSwarmAndRegistry(t, func(info dindSwarmAndRegistryInfo) { + runWithDindSwarmAndRegistry(t, func(info OrchestratorAndRegistryInfo) { cmd := info.configuredCmd tmp := fs.NewDir(t, "TestBuild") @@ -117,7 +117,7 @@ func TestQuietBuild(t *testing.T) { } func TestBuildWithoutTag(t *testing.T) { - runWithDindSwarmAndRegistry(t, func(info dindSwarmAndRegistryInfo) { + runWithDindSwarmAndRegistry(t, func(info OrchestratorAndRegistryInfo) { cmd := info.configuredCmd testDir := path.Join("testdata", "build") @@ -148,7 +148,7 @@ func TestBuildWithoutTag(t *testing.T) { } func TestBuildWithArgs(t *testing.T) { - runWithDindSwarmAndRegistry(t, func(info dindSwarmAndRegistryInfo) { + runWithDindSwarmAndRegistry(t, func(info OrchestratorAndRegistryInfo) { cmd := info.configuredCmd testDir := path.Join("testdata", "build") @@ -179,7 +179,7 @@ func TestBuildWithArgs(t *testing.T) { } func TestBuildWithArgsDefinedTwice(t *testing.T) { - runWithDindSwarmAndRegistry(t, func(info dindSwarmAndRegistryInfo) { + runWithDindSwarmAndRegistry(t, func(info OrchestratorAndRegistryInfo) { cmd := info.configuredCmd testDir := path.Join("testdata", "build") diff --git a/e2e/commands_test.go b/e2e/commands_test.go index 1f3d6e110..50472967c 100644 --- a/e2e/commands_test.go +++ b/e2e/commands_test.go @@ -91,7 +91,7 @@ func TestRenderAppNotFound(t *testing.T) { } func TestRenderFormatters(t *testing.T) { - runWithDindSwarmAndRegistry(t, func(info dindSwarmAndRegistryInfo) { + runWithDindSwarmAndRegistry(t, func(info OrchestratorAndRegistryInfo) { cmd := info.configuredCmd contextPath := filepath.Join("testdata", "simple") @@ -239,7 +239,7 @@ func TestInitWithInvalidCompose(t *testing.T) { } func TestInspectApp(t *testing.T) { - runWithDindSwarmAndRegistry(t, func(info dindSwarmAndRegistryInfo) { + runWithDindSwarmAndRegistry(t, func(info OrchestratorAndRegistryInfo) { cmd := info.configuredCmd // cwd = e2e @@ -284,7 +284,7 @@ func TestRunOnlyOne(t *testing.T) { } func TestRunWithLabels(t *testing.T) { - runWithDindSwarmAndRegistry(t, func(info dindSwarmAndRegistryInfo) { + runWithDindSwarmAndRegistry(t, func(info OrchestratorAndRegistryInfo) { cmd := info.configuredCmd contextPath := filepath.Join("testdata", "simple") @@ -308,7 +308,7 @@ func TestRunWithLabels(t *testing.T) { } func TestDockerAppLifecycle(t *testing.T) { - runWithDindSwarmAndRegistry(t, func(info dindSwarmAndRegistryInfo) { + runWithDindSwarmAndRegistry(t, func(info OrchestratorAndRegistryInfo) { cmd := info.configuredCmd appName := strings.ToLower(strings.Replace(t.Name(), "/", "_", 1)) tmpDir := fs.NewDir(t, appName) @@ -376,7 +376,7 @@ func TestDockerAppLifecycle(t *testing.T) { } func TestDockerAppLifecycleMultiRm(t *testing.T) { - runWithDindSwarmAndRegistry(t, func(info dindSwarmAndRegistryInfo) { + runWithDindSwarmAndRegistry(t, func(info OrchestratorAndRegistryInfo) { cmd := info.configuredCmd appName := strings.ToLower(strings.Replace(t.Name(), "/", "_", 1)) tmpDir := fs.NewDir(t, appName) diff --git a/e2e/compatibility_test.go b/e2e/compatibility_test.go index 5d73a7c6f..a85a27c03 100644 --- a/e2e/compatibility_test.go +++ b/e2e/compatibility_test.go @@ -1,187 +1,153 @@ package e2e import ( - "errors" "fmt" - "io" "io/ioutil" - "net/http" "os" "path/filepath" "regexp" - "strings" "testing" "time" "gotest.tools/assert" "k8s.io/apimachinery/pkg/util/wait" - - "gotest.tools/fs" ) const ( pollTimeout = 30 * time.Second ) -func loadAndTagImage(info dindSwarmAndRegistryInfo, tmpDir *fs.Dir, tag string, url string) error { - err := downloadImageTarball(tmpDir.Join("image.tar"), url) - if err != nil { - return err - } - - digest := "" - combined := info.dockerCmd("load", "-q", "-i", tmpDir.Join("image.tar")) - for _, line := range strings.Split(combined, "\n") { - if strings.Contains(line, "sha256:") { - digest = strings.Split(line, "sha256:")[1] - } - } - if digest == "" { - return errors.New("Image digest not found in docker load's stdout") - } - - digest = strings.Trim(digest, " \r\n") - info.dockerCmd("tag", digest, tag) - - return nil -} - -func downloadImageTarball(filepath string, url string) error { - client := http.Client{Timeout: time.Minute * 1} - res, err := client.Get(url) - if err != nil { - return err +func runAppE2E(t *testing.T, info OrchestratorAndRegistryInfo, expectedOutput map[string][]string) { + appName := "app-e2e" + appImage := filepath.Join(info.registryAddress, "app-e2e:v0.9.0") + imgPrefix := filepath.Join(info.registryAddress, "app-e2e") + data, err := ioutil.ReadFile(filepath.Join("testdata", "compatibility", "bundle-v0.9.0.json")) + assert.NilError(t, err) + + // update bundle + bundleDir := filepath.Join(info.configDir, "app", "bundles", "docker.io", "library", "app-e2e", "_tags", "v0.9.0") + + assert.NilError(t, os.MkdirAll(bundleDir, os.FileMode(0777))) + assert.NilError(t, ioutil.WriteFile(filepath.Join(bundleDir, "bundle.json"), data, os.FileMode(0644))) + + // load images build with an old Docker App version + assert.NilError(t, info.LoadImageFromWeb("app-e2e:0.1.0-invoc", "https://github.com/docker/app-e2e/raw/master/images/v0.9.0/app-e2e-invoc.tar")) + assert.NilError(t, info.LoadImageFromWeb("app-e2e/backend", "https://github.com/docker/app-e2e/raw/master/images/v0.9.0/backend.tar")) + assert.NilError(t, info.LoadImageFromWeb("app-e2e/frontend", "https://github.com/docker/app-e2e/raw/master/images/v0.9.0/frontend.tar")) + + // tag app image + info.dockerCmd("app", "image", "tag", "app-e2e:v0.9.0", appImage) + // list images + output := info.dockerCmd("app", "image", "ls") + checkContains(t, output, []string{appName}) + // push app to registry + info.dockerCmd("app", "push", appImage) + // pull app from registry + info.dockerCmd("app", "pull", appImage) + // inspect bundle + output = info.dockerCmd("app", "image", "inspect", appImage, "--pretty") + checkContains(t, output, + []string{ + `name:\s+app-e2e`, + fmt.Sprintf(`backend\s+1\s+.+%s`, imgPrefix), + fmt.Sprintf(`frontend\s+1\s+8080\s+.+%s`, imgPrefix), + `ports.frontend\s+8080`, + }) + + // render bundle + output = info.dockerCmd("app", "image", "render", appImage) + checkContains(t, output, + []string{ + fmt.Sprintf(`image:.+%s`, imgPrefix), + fmt.Sprintf(`image:.+%s`, imgPrefix), + "published: 8080", + "target: 80", + }) + + // Install app + output = info.dockerCmd("app", "run", appImage, "--name", appName) + checkContains(t, output, expectedOutput["run"]) + + // Status check -- poll app list + checkStatus := func(lastAction string) { + err = wait.Poll(2*time.Second, pollTimeout, func() (bool, error) { + output = info.dockerCmd("app", "ls") + fmt.Println(output) + expectedLines := []string{ + `RUNNING APP\s+APP NAME\s+SERVICES\s+LAST ACTION\s+RESULT\s+CREATED\s+MODIFIED\s+REFERENCE`, + fmt.Sprintf(`%s\s+%s \(0.1.0\)\s+.+%s\s+success\s+.+second[s]?\sago\s+.+second[s]?\sago\s+`, appName, appName, lastAction), + } + matches := true + for _, expected := range expectedLines { + exp := regexp.MustCompile(expected) + matches = matches && exp.MatchString(output) + } + return matches, nil + }) + assert.NilError(t, err) } - defer res.Body.Close() - // Create the file - out, err := os.Create(filepath) - if err != nil { - return err - } - defer out.Close() - // Write the body to file - _, err = io.Copy(out, res.Body) - return err + // Inspect app + output = info.dockerCmd("app", "inspect", appName, "--pretty") + checkContains(t, output, + []string{ + "Running App:", + fmt.Sprintf("Name: %s", appName), + "Result: success", + `ports.frontend: "8080"`, + }) + + // Update the application, changing the port + output = info.dockerCmd("app", "update", appName, "--set", "ports.frontend=8081") + checkContains(t, output, expectedOutput["upgrade"]) + + // check status on upgrade + checkStatus("upgrade") + + // Uninstall the application + output = info.dockerCmd("app", "rm", appName) + checkContains(t, output, expectedOutput["rm"]) } func TestBackwardsCompatibilityV1(t *testing.T) { - runWithDindSwarmAndRegistry(t, func(info dindSwarmAndRegistryInfo) { - appName := "app-e2e" - - data, err := ioutil.ReadFile(filepath.Join("testdata", "compatibility", "bundle-v0.9.0.json")) - assert.NilError(t, err) - // update bundle - bundleDir := filepath.Join(info.configDir, "app", "bundles", "docker.io", "library", "app-e2e", "_tags", "v0.9.0") - assert.NilError(t, os.MkdirAll(bundleDir, os.FileMode(0777))) - assert.NilError(t, ioutil.WriteFile(filepath.Join(bundleDir, "bundle.json"), data, os.FileMode(0644))) - - // load images build with an old Docker App version - assert.NilError(t, loadAndTagImage(info, info.tmpDir, "app-e2e:0.1.0-invoc", "https://github.com/docker/app-e2e/raw/master/images/v0.9.0/app-e2e-invoc.tar")) - assert.NilError(t, loadAndTagImage(info, info.tmpDir, "app-e2e/backend", "https://github.com/docker/app-e2e/raw/master/images/v0.9.0/backend.tar")) - assert.NilError(t, loadAndTagImage(info, info.tmpDir, "app-e2e/frontend", "https://github.com/docker/app-e2e/raw/master/images/v0.9.0/frontend.tar")) - - // list images - output := info.dockerCmd("app", "image", "ls") - checkContains(t, output, []string{appName}) - // inspect bundle - output = info.dockerCmd("app", "image", "inspect", "app-e2e:v0.9.0", "--pretty") - checkContains(t, output, - []string{ - `name:\s+app-e2e`, - `backend\s+1\s+app-e2e/backend`, - `frontend\s+1\s+8080\s+app-e2e/frontend`, - `ports.frontend\s+8080`, - }) - - // render bundle - output = info.dockerCmd("app", "image", "render", "app-e2e:v0.9.0") - checkContains(t, output, - []string{ - "image: app-e2e/frontend", - "image: app-e2e/backend", - "published: 8080", - "target: 80", - }) - - // Install app - output = info.dockerCmd("app", "run", "app-e2e:v0.9.0", "--name", appName) - checkContains(t, output, - []string{ - fmt.Sprintf("Creating service %s_backend", appName), - fmt.Sprintf("Creating service %s_frontend", appName), - fmt.Sprintf("Creating network %s_default", appName), - }) - - // Status check -- poll app list - checkStatus := func(lastAction string) { - err = wait.Poll(2*time.Second, pollTimeout, func() (bool, error) { - output = info.dockerCmd("app", "ls") - expectedLines := []string{ - `RUNNING APP\s+APP NAME\s+SERVICES\s+LAST ACTION\s+RESULT\s+CREATED\s+MODIFIED\s+REFERENCE`, - fmt.Sprintf(`%s\s+%s \(0.1.0\)\s+2/2\s+%s\s+success\s+.+second[s]?\sago\s+.+second[s]?\sago\s+`, appName, appName, lastAction), - } - matches := true - for _, expected := range expectedLines { - exp := regexp.MustCompile(expected) - matches = matches && exp.MatchString(output) - } - return matches, nil - }) - assert.NilError(t, err) - } - - queryService := func(port string) { - err = wait.Poll(2*time.Second, pollTimeout, func() (bool, error) { - // Check the frontend service responds - url := `http://localhost:` + port - output = info.execCmd("/usr/bin/wget", "-O", "-", url) - return strings.Contains(output, `Hi there, I love Docker!`), nil - }) - output = "" - if err != nil { - output = info.dockerCmd("stack", "ps", appName) + t.Run("Swarm", func(t *testing.T) { + runWithDindSwarmAndRegistry(t, func(info OrchestratorAndRegistryInfo) { + //expected_outputs + expectedOutput := map[string][]string{ + "run": { + `Creating service app-e2e_backend`, + `Creating service app-e2e_frontend`, + `Creating network app-e2e_default`, + }, + "upgrade": { + `Updating service app-e2e_backend`, + `Updating service app-e2e_frontend`, + }, + "rm": { + `Removing service app-e2e_backend`, + `Removing service app-e2e_frontend`, + `Removing network app-e2e_default`, + }} + runAppE2E(t, info, expectedOutput) + }) + }) + t.Run("Kubernetes", func(t *testing.T) { + runWithKindAndRegistry(t, func(info OrchestratorAndRegistryInfo) { + //expected_outputs + expectedOutput := map[string][]string{ + "run": { + `backend: Ready`, + `frontend: Ready`, + `Stack app-e2e is stable and running`, + }, + "upgrade": { + `Waiting for the stack to be stable and running...`, + `backend: Ready`, + `frontend: Ready`, + }, + "rm": {}, } - assert.NilError(t, err, output) - } - - // Check status on install - checkStatus("install") - - // query deployed service - queryService("8080") - - // Inspect app - output = info.dockerCmd("app", "inspect", appName, "--pretty") - checkContains(t, output, - []string{ - "Running App:", - fmt.Sprintf("Name: %s", appName), - "Result: success", - `ports.frontend: "8080"`, - }) - - // Update the application, changing the port - output = info.dockerCmd("app", "update", appName, "--set", "ports.frontend=8081") - checkContains(t, output, - []string{ - fmt.Sprintf("Updating service %s_backend", appName), - fmt.Sprintf("Updating service %s_frontend", appName), - }) - - // check status on upgrade - checkStatus("upgrade") - - // Check the frontend service responds on the new port - queryService("8081") - - // Uninstall the application - output = info.dockerCmd("app", "rm", appName) - checkContains(t, output, - []string{ - fmt.Sprintf("Removing service %s_backend", appName), - fmt.Sprintf("Removing service %s_frontend", appName), - fmt.Sprintf("Removing network %s_default", appName), - }) + runAppE2E(t, info, expectedOutput) + }) }) } diff --git a/e2e/helper_test.go b/e2e/helper_test.go index ee1cb93d8..7563995b2 100644 --- a/e2e/helper_test.go +++ b/e2e/helper_test.go @@ -1,9 +1,18 @@ package e2e import ( + "archive/tar" + "archive/zip" + "compress/gzip" + "errors" "fmt" + "io" "io/ioutil" "net" + "net/http" + "os" + "path/filepath" + "runtime" "strconv" "strings" "testing" @@ -14,6 +23,7 @@ import ( "gotest.tools/assert" "gotest.tools/fs" "gotest.tools/icmd" + "k8s.io/apimachinery/pkg/util/wait" ) // readFile returns the content of the file at the designated path normalizing @@ -25,25 +35,162 @@ func readFile(t *testing.T, path string) string { return strings.Replace(string(content), "\r", "", -1) } -type dindSwarmAndRegistryInfo struct { - swarmAddress string - registryAddress string - configuredCmd icmd.Cmd - configDir string - tmpDir *fs.Dir - stopRegistry func() - registryLogs func() string - dockerCmd func(...string) string - execCmd func(...string) string - localCmd func(...string) string +func getHostIPAddress() (string, error) { + addrs, err := net.InterfaceAddrs() + if err != nil { + return "", err + } + for _, address := range addrs { + if ipnet, ok := address.(*net.IPNet); ok && !ipnet.IP.IsLoopback() { + if ipnet.IP.To4() != nil { + return ipnet.IP.String(), nil + } + } + } + return "", errors.New("No IP address detected") } -func runWithDindSwarmAndRegistry(t *testing.T, todo func(dindSwarmAndRegistryInfo)) { - cmd, cleanup := dockerCli.createTestCmd() - defer cleanup() +type OrchestratorAndRegistryInfo struct { + orchestratorName string + orchestratorAddress string + registryAddress string + configuredCmd icmd.Cmd + configDir string + tmpDir *fs.Dir + cleanup func() + stopRegistry func() + registryLogs func() string + dockerCmd func(...string) string + execCmd func(...string) string + localCmd func(...string) string +} + +func (info *OrchestratorAndRegistryInfo) LoadImageFromWeb(tag string, url string) error { + filename := filepath.Base(url) + err := info.Download(info.tmpDir.Join(filename), url) + if err != nil { + return err + } + combined := info.dockerCmd("load", "-q", "-i", info.tmpDir.Join(filename)) + digest := "" + for _, line := range strings.Split(combined, "\n") { + if strings.Contains(line, "sha256:") { + digest = strings.Split(line, "sha256:")[1] + } + } + if digest == "" { + return errors.New("Image digest not found in docker load's stdout") + } + // tag image + digest = strings.Trim(digest, " \r\n") + info.dockerCmd("tag", digest, tag) + return nil +} + +func (info *OrchestratorAndRegistryInfo) Download(filename string, url string) error { + client := http.Client{Timeout: time.Minute * 5} + res, err := client.Get(url) + if err != nil { + return err + } + defer res.Body.Close() + if res.StatusCode != http.StatusOK { + return errors.New("Page not found: " + url) + } + // Create the file + out, err := os.Create(filename) + if err != nil { + return err + } + defer out.Close() + // Write the body to file + _, err = io.Copy(out, res.Body) + return err +} +func extractZIP(archive string, binary string, filename string) error { + zipReader, err := zip.OpenReader(archive) + if err != nil { + return err + } + for _, file := range zipReader.Reader.File { + if filename == file.Name { + zippedFile, err := file.Open() + if err != nil { + return err + } + defer zippedFile.Close() + binaryFile, err := os.OpenFile( + binary, + os.O_WRONLY|os.O_CREATE|os.O_TRUNC, + file.Mode(), + ) + if err != nil { + return err + } + defer binaryFile.Close() + _, err = io.Copy(binaryFile, zippedFile) + if err != nil { + return err + } + } + } + return nil +} + +func (info *OrchestratorAndRegistryInfo) ExtractBinaryFromArchive(binary string, archive string, archiveFilename string) error { + f, err := os.Open(archive) + if err != nil { + return err + } + defer f.Close() + + if strings.HasSuffix(archive, ".zip") { + return extractZIP(archive, binary, archiveFilename) + } + + var fileReader io.ReadCloser = f + // just in case we are reading a tar.gz file, add a filter to handle gzipped file + if strings.HasSuffix(archive, ".gz") { + if fileReader, err = gzip.NewReader(f); err != nil { + return err + } + defer fileReader.Close() + } + tarReader := tar.NewReader(fileReader) + for { + file, err := tarReader.Next() + if err != nil { + if err != io.EOF { + return err + } + return nil + } + if file.Typeflag != tar.TypeReg { + continue + } + if archiveFilename == file.Name { + // handle normal file + writer, err := os.Create(binary) + if err != nil { + return err + } + _, err = io.Copy(writer, tarReader) + if err != nil { + return err + } + err = os.Chmod(binary, os.FileMode(file.Mode)) + if err != nil { + return err + } + writer.Close() + } + } +} + +func initInfoAndRegistry(t *testing.T) *OrchestratorAndRegistryInfo { + cmd, cleanup := dockerCli.createTestCmd() tmpDir := fs.NewDir(t, t.Name()) - defer tmpDir.Remove() var configDir string for _, val := range cmd.Env { @@ -51,9 +198,8 @@ func runWithDindSwarmAndRegistry(t *testing.T, todo func(dindSwarmAndRegistryInf configDir = strings.Replace(val, "DOCKER_CONFIG=", "", 1) } } - // Initialize the info struct - runner := dindSwarmAndRegistryInfo{configuredCmd: cmd, configDir: configDir, tmpDir: tmpDir} + runner := &OrchestratorAndRegistryInfo{configuredCmd: cmd, configDir: configDir, tmpDir: tmpDir} // Func to execute command locally runLocalCmd := func(params ...string) string { @@ -61,6 +207,7 @@ func runWithDindSwarmAndRegistry(t *testing.T, todo func(dindSwarmAndRegistryInf return "" } cmd := icmd.Command(params[0], params[1:]...) + cmd.Env = runner.configuredCmd.Env result := icmd.RunCmd(cmd) result.Assert(t, icmd.Success) return result.Combined() @@ -73,6 +220,13 @@ func runWithDindSwarmAndRegistry(t *testing.T, todo func(dindSwarmAndRegistryInf return result.Combined() } + runner.localCmd = runLocalCmd + runner.dockerCmd = runDockerCmd + runner.execCmd = func(params ...string) string { + args := append([]string{"docker", "exec", "-t", runner.orchestratorName}, params...) + return runLocalCmd(args...) + } + // The dind doesn't have the cnab-app-base image so we save it in order to load it later runDockerCmd("save", fmt.Sprintf("docker/cnab-app-base:%s", internal.Version), "-o", tmpDir.Join("cnab-app-base.tar.gz")) @@ -85,42 +239,184 @@ func runWithDindSwarmAndRegistry(t *testing.T, todo func(dindSwarmAndRegistryInf // - the registry must be reachable from the dind daemon on the same address/port // - the installer image need to target the same docker context (dind) as the client, while running on default (or another) context, which means we can't use 'localhost' // Solution found is: use host external IP (not loopback) so accessing from within installer container will reach the right container - + hostIP, err := getHostIPAddress() + assert.NilError(t, err) registry := NewContainer("registry:2", 5000) - registry.Start(t, "-e", "REGISTRY_VALIDATION_MANIFESTS_URLS_ALLOW=[^http]", + registry.Start(t, "-p", fmt.Sprintf(`%s:5000:5000`, hostIP), "-e", "REGISTRY_VALIDATION_MANIFESTS_URLS_ALLOW=[^http]", "-e", "REGISTRY_HTTP_ADDR=0.0.0.0:5000") - defer registry.StopNoFail() + //defer registry.StopNoFail() registryAddress := registry.GetAddress(t) - - swarm := NewContainer("docker:19.03.3-dind", 2375, "--insecure-registry", registryAddress) - swarm.Start(t, "-e", "DOCKER_TLS_CERTDIR=") // Disable certificate generate on DinD startup - defer swarm.Stop(t) - swarmAddress := swarm.GetAddress(t) - // Initialize the info struct runner.registryAddress = registryAddress - runner.swarmAddress = swarmAddress runner.stopRegistry = registry.StopNoFail runner.registryLogs = registry.Logs(t) - runDockerCmd("context", "create", "swarm-context", "--docker", fmt.Sprintf(`"host=tcp://%s"`, swarmAddress), "--default-stack-orchestrator", "swarm") + runner.cleanup = func() { + runner.stopRegistry() + runner.tmpDir.Remove() + cleanup() + } + return runner +} + +func runWithDindSwarmAndRegistry(t *testing.T, todo func(OrchestratorAndRegistryInfo)) { + runner := initInfoAndRegistry(t) + defer runner.cleanup() + tmpDir := runner.tmpDir + swarm := NewContainer("docker:19.03.3-dind", 2375, "--insecure-registry", runner.registryAddress) + swarm.Start(t, "--name", "dind", "-e", "DOCKER_TLS_CERTDIR=", "-P") // Disable certificate generate on DinD startup + defer swarm.Stop(t) + swarmAddress := swarm.GetAddress(t) + + runner.orchestratorName = swarm.container + runner.orchestratorAddress = swarmAddress + + runner.dockerCmd("context", "create", "swarm-context", "--docker", fmt.Sprintf(`"host=tcp://%s"`, swarmAddress), "--default-stack-orchestrator", "swarm") runner.configuredCmd.Env = append(runner.configuredCmd.Env, "DOCKER_CONTEXT=swarm-context", "DOCKER_INSTALLER_CONTEXT=swarm-context") // Initialize the swarm - runDockerCmd("swarm", "init") + runner.dockerCmd("swarm", "init") // Load the needed base cnab image into the swarm docker engine - runDockerCmd("load", "-i", tmpDir.Join("cnab-app-base.tar.gz")) + runner.dockerCmd("load", "-i", tmpDir.Join("cnab-app-base.tar.gz")) // Pre-load busybox image used by a few e2e tests - runDockerCmd("load", "-i", tmpDir.Join("busybox.tar.gz")) + runner.dockerCmd("load", "-i", tmpDir.Join("busybox.tar.gz")) - runner.localCmd = runLocalCmd - runner.dockerCmd = runDockerCmd - runner.execCmd = func(params ...string) string { - args := append([]string{"docker", "exec", "-t", swarm.container}, params...) - return runLocalCmd(args...) + todo(*runner) +} + +func runWithKindAndRegistry(t *testing.T, todo func(OrchestratorAndRegistryInfo)) { + runner := initInfoAndRegistry(t) + + extension := "" + archive := ".tar.gz" + if runtime.GOOS == "windows" { + extension = ".exe" + archive = ".zip" } - todo(runner) + // paths to binaries used during install + kind := runner.tmpDir.Join("kind" + extension) + kubectl := runner.tmpDir.Join("kubectl" + extension) + helm := runner.tmpDir.Join("helm" + extension) + conk := runner.tmpDir.Join(fmt.Sprintf("installer-%s%s", runtime.GOOS, extension)) + + // detect host's platform + err := runner.Download(kind, fmt.Sprintf(`https://github.com/kubernetes-sigs/kind/releases/download/v0.6.0/kind-%s-amd64`, runtime.GOOS)) + assert.NilError(t, err) + // make kind binary executable + err = os.Chmod(kind, os.FileMode(0111)) + assert.NilError(t, err) + //get hosts's ip address + ipaddress, err := getHostIPAddress() + assert.NilError(t, err) + kindconf := []byte(fmt.Sprintf(`kind: Cluster +apiVersion: kind.x-k8s.io/v1alpha4 +nodes: +- role: control-plane +networking: + apiServerAddress: %s +containerdConfigPatches: +- |- + [plugins."io.containerd.grpc.v1.cri".registry.mirrors."%s"] + endpoint = ["http://%s"] +`, ipaddress, runner.registryAddress, runner.registryAddress)) + + // write kind config file + assert.NilError(t, ioutil.WriteFile(runner.tmpDir.Join("kind.conf"), kindconf, os.FileMode(0644))) + + runner.localCmd(kind, "create", "cluster", "--name", "kindcluster", "--config", runner.tmpDir.Join("kind.conf"), "--kubeconfig", runner.tmpDir.Join("kube.conf")) + // add kube config to env for kubectl + runner.configuredCmd.Env = append(runner.configuredCmd.Env, "KUBECONFIG="+runner.tmpDir.Join("kube.conf")) + runner.orchestratorName = "kindcluster-control-plane" + // download and install kubectl, helm and compose-on-kubernetes + err = runner.Download(kubectl, fmt.Sprintf(`https://storage.googleapis.com/kubernetes-release/release/v1.16.0/bin/%s/amd64/kubectl%s`, runtime.GOOS, extension)) + assert.NilError(t, err) + err = os.Chmod(kubectl, os.FileMode(0111)) + assert.NilError(t, err) + time.Sleep(time.Second * 60) + + output := runner.localCmd(kubectl, `create`, `namespace`, `compose`) + checkContains(t, output, []string{`namespace/compose created`}) + + // wait until all containers are in running state + waitReady := func() { + areRunning := func(s []string) bool { + if len(s) == 0 { + return true + } + if s[0] != "Running" { + return false + } + for i := 1; i < len(s); i++ { + if s[i] != s[0] { + return false + } + } + return true + } + err := wait.Poll(time.Second*2, time.Second*240, func() (bool, error) { + output := runner.localCmd(kubectl, `get`, `pods`, `-o=jsonpath={.items[*].status.phase}`, `--all-namespaces`) + containerStatus := strings.Split(strings.Trim(output, `\n`), ` `) + return areRunning(containerStatus), nil + }) + assert.NilError(t, err) + } + waitReady() + + output = runner.localCmd(kubectl, `-n`, `kube-system`, `create`, `serviceaccount`, `tiller`) + checkContains(t, output, []string{`serviceaccount/tiller created`}) + output = runner.localCmd(kubectl, `-n`, `kube-system`, `create`, `clusterrolebinding`, `tiller`, `--clusterrole`, `cluster-admin`, `--serviceaccount`, `kube-system:tiller`) + checkContains(t, output, []string{`clusterrolebinding.rbac.authorization.k8s.io/tiller created`}) + waitReady() + // download and install helm + err = runner.Download(runner.tmpDir.Join("helm"+archive), fmt.Sprintf(`https://get.helm.sh/helm-v2.16.1-%s-amd64%s`, runtime.GOOS, archive)) + assert.NilError(t, err) + err = runner.ExtractBinaryFromArchive(helm, runner.tmpDir.Join("helm"+archive), fmt.Sprintf("%s-amd64/helm%s", runtime.GOOS, extension)) + assert.NilError(t, err) + err = os.Chmod(helm, os.FileMode(0111)) + assert.NilError(t, err) + runner.localCmd(helm, `init`, `--service-account`, `tiller`) + time.Sleep(time.Second * 30) + waitReady() + + output = runner.localCmd(helm, `install`, `--name`, `etcd-operator`, `stable/etcd-operator`, `--namespace`, `compose`) + checkContains(t, output, []string{`1. etcd-operator deployed.`}) + time.Sleep(time.Second * 5) + waitReady() + + // write compose file for etcd deployment + data, err := ioutil.ReadFile(filepath.Join("testdata", "compatibility", "compose-etcd.yaml")) + assert.NilError(t, err) + assert.NilError(t, ioutil.WriteFile(runner.tmpDir.Join("compose-etcd.yaml"), data, os.FileMode(0644))) + // deploy etcd for compose on kube + runner.localCmd(kubectl, `apply`, `-f`, runner.tmpDir.Join("compose-etcd.yaml")) + + time.Sleep(time.Second * 10) + waitReady() + + // download compose-on-kube binary + err = runner.Download(conk, fmt.Sprintf(`https://github.com/docker/compose-on-kubernetes/releases/download/v0.5.0-alpha1/installer-%s%s`, runtime.GOOS, extension)) + assert.NilError(t, err) + err = os.Chmod(conk, os.FileMode(0111)) + assert.NilError(t, err) + // Install Compose on Kube + output = runner.localCmd(conk, `-namespace=compose`, `-etcd-servers=http://compose-etcd-client:2379`) + checkContains(t, output, []string{`Controller: image: `}) + time.Sleep(time.Second * 5) + waitReady() + + // setup docker context + runner.dockerCmd(`context`, `create`, `kind-context`, `--docker`, `"host=unix:///var/run/docker.sock"`, `--default-stack-orchestrator`, + `kubernetes`, `--kubernetes`, fmt.Sprintf(`"config-file=%s"`, runner.tmpDir.Join("kube.conf"))) + runner.configuredCmd.Env = append(runner.configuredCmd.Env, "DOCKER_CONTEXT=kind-context", "DOCKER_INSTALLER_CONTEXT=kind-context") + + cleanAll := func() { + runner.dockerCmd(`rm`, `--force`, `--volumes`, runner.orchestratorName) + runner.cleanup() + } + defer cleanAll() + todo(*runner) + } func build(t *testing.T, cmd icmd.Cmd, dockerCli dockerCliCommand, ref, path string) { @@ -149,7 +445,7 @@ func NewContainer(image string, privatePort int, args ...string) *Container { // Start starts a new docker container on a random port func (c *Container) Start(t *testing.T, dockerArgs ...string) { - args := []string{"run", "--rm", "--privileged", "-d", "-P"} + args := []string{"run", "--rm", "--privileged", "-d"} args = append(args, dockerArgs...) args = append(args, c.image) args = append(args, c.args...) diff --git a/e2e/images_test.go b/e2e/images_test.go index af29a1108..c5bb7bdb6 100644 --- a/e2e/images_test.go +++ b/e2e/images_test.go @@ -60,7 +60,7 @@ func verifyImageIDListOutput(t *testing.T, cmd icmd.Cmd, expectedCount int) { } func TestImageList(t *testing.T) { - runWithDindSwarmAndRegistry(t, func(info dindSwarmAndRegistryInfo) { + runWithDindSwarmAndRegistry(t, func(info OrchestratorAndRegistryInfo) { cmd := info.configuredCmd insertBundles(t, cmd) @@ -76,7 +76,7 @@ my.registry:5000/c-myapp latest [a-f0-9]{12} push-pull } func TestImageListQuiet(t *testing.T) { - runWithDindSwarmAndRegistry(t, func(info dindSwarmAndRegistryInfo) { + runWithDindSwarmAndRegistry(t, func(info OrchestratorAndRegistryInfo) { cmd := info.configuredCmd insertBundles(t, cmd) verifyImageIDListOutput(t, cmd, 3) @@ -84,7 +84,7 @@ func TestImageListQuiet(t *testing.T) { } func TestImageListDigests(t *testing.T) { - runWithDindSwarmAndRegistry(t, func(info dindSwarmAndRegistryInfo) { + runWithDindSwarmAndRegistry(t, func(info OrchestratorAndRegistryInfo) { cmd := info.configuredCmd insertBundles(t, cmd) expected := `REPOSITORY TAG DIGEST APP IMAGE ID APP NAME @@ -97,7 +97,7 @@ my.registry:5000/c-myapp latest [a-f0-9]{12} } func TestImageRmForce(t *testing.T) { - runWithDindSwarmAndRegistry(t, func(info dindSwarmAndRegistryInfo) { + runWithDindSwarmAndRegistry(t, func(info OrchestratorAndRegistryInfo) { cmd := info.configuredCmd iidfile := fs.NewFile(t, "iid").Path() @@ -127,7 +127,7 @@ func TestImageRmForce(t *testing.T) { } func TestImageRm(t *testing.T) { - runWithDindSwarmAndRegistry(t, func(info dindSwarmAndRegistryInfo) { + runWithDindSwarmAndRegistry(t, func(info OrchestratorAndRegistryInfo) { cmd := info.configuredCmd insertBundles(t, cmd) @@ -157,7 +157,7 @@ Deleted: b-simple-app:latest`, } func TestImageTag(t *testing.T) { - runWithDindSwarmAndRegistry(t, func(info dindSwarmAndRegistryInfo) { + runWithDindSwarmAndRegistry(t, func(info OrchestratorAndRegistryInfo) { cmd := info.configuredCmd dockerAppImageTag := func(args ...string) { diff --git a/e2e/pushpull_test.go b/e2e/pushpull_test.go index c90f1678a..3ff6211c9 100644 --- a/e2e/pushpull_test.go +++ b/e2e/pushpull_test.go @@ -33,7 +33,7 @@ func TestPushUnknown(t *testing.T) { } func TestPushInsecureRegistry(t *testing.T) { - runWithDindSwarmAndRegistry(t, func(info dindSwarmAndRegistryInfo) { + runWithDindSwarmAndRegistry(t, func(info OrchestratorAndRegistryInfo) { path := filepath.Join("testdata", "local") ref := info.registryAddress + "/test/push-insecure" @@ -53,7 +53,7 @@ func TestPushInsecureRegistry(t *testing.T) { } func TestPushInstall(t *testing.T) { - runWithDindSwarmAndRegistry(t, func(info dindSwarmAndRegistryInfo) { + runWithDindSwarmAndRegistry(t, func(info OrchestratorAndRegistryInfo) { cmd := info.configuredCmd ref := info.registryAddress + "/test/push-pull" build(t, cmd, dockerCli, ref, filepath.Join("testdata", "push-pull")) @@ -69,7 +69,7 @@ func TestPushInstall(t *testing.T) { } func TestPushPullInstall(t *testing.T) { - runWithDindSwarmAndRegistry(t, func(info dindSwarmAndRegistryInfo) { + runWithDindSwarmAndRegistry(t, func(info OrchestratorAndRegistryInfo) { cmd := info.configuredCmd ref := info.registryAddress + "/test/push-pull" tag := ":v.0.0.1" @@ -108,7 +108,7 @@ Unable to find App "unknown": failed to resolve bundle manifest "docker.io/libra } func TestPushInstallBundle(t *testing.T) { - runWithDindSwarmAndRegistry(t, func(info dindSwarmAndRegistryInfo) { + runWithDindSwarmAndRegistry(t, func(info OrchestratorAndRegistryInfo) { cmd := info.configuredCmd ref := info.registryAddress + "/test/push-bundle" @@ -159,7 +159,7 @@ func TestPushInstallBundle(t *testing.T) { cmdIsolatedStore, cleanupIsolatedStore := dockerCli.createTestCmd() // Enter the same context as `cmd` to run commands within the same environment - cmdIsolatedStore.Command = dockerCli.Command("context", "create", "swarm-context", "--docker", fmt.Sprintf(`"host=tcp://%s"`, info.swarmAddress)) + cmdIsolatedStore.Command = dockerCli.Command("context", "create", "swarm-context", "--docker", fmt.Sprintf(`"host=tcp://%s"`, info.orchestratorAddress)) icmd.RunCmd(cmdIsolatedStore).Assert(t, icmd.Success) cmdIsolatedStore.Env = append(cmdIsolatedStore.Env, "DOCKER_CONTEXT=swarm-context") diff --git a/e2e/relocation_test.go b/e2e/relocation_test.go index da9cf742c..48f7033f0 100644 --- a/e2e/relocation_test.go +++ b/e2e/relocation_test.go @@ -14,7 +14,7 @@ import ( ) func TestRelocationMapCreatedOnPull(t *testing.T) { - runWithDindSwarmAndRegistry(t, func(info dindSwarmAndRegistryInfo) { + runWithDindSwarmAndRegistry(t, func(info OrchestratorAndRegistryInfo) { cmd := info.configuredCmd cfg := getDockerConfigDir(t, cmd) @@ -42,7 +42,7 @@ func TestRelocationMapCreatedOnPull(t *testing.T) { } func TestRelocationMapRun(t *testing.T) { - runWithDindSwarmAndRegistry(t, func(info dindSwarmAndRegistryInfo) { + runWithDindSwarmAndRegistry(t, func(info OrchestratorAndRegistryInfo) { cmd := info.configuredCmd cfg := getDockerConfigDir(t, cmd) @@ -92,7 +92,7 @@ func TestRelocationMapRun(t *testing.T) { } func TestPushPulledApplication(t *testing.T) { - runWithDindSwarmAndRegistry(t, func(info dindSwarmAndRegistryInfo) { + runWithDindSwarmAndRegistry(t, func(info OrchestratorAndRegistryInfo) { cmd := info.configuredCmd cfg := getDockerConfigDir(t, cmd) @@ -130,7 +130,7 @@ func TestPushPulledApplication(t *testing.T) { } func TestRelocationMapOnInspect(t *testing.T) { - runWithDindSwarmAndRegistry(t, func(info dindSwarmAndRegistryInfo) { + runWithDindSwarmAndRegistry(t, func(info OrchestratorAndRegistryInfo) { cmd := info.configuredCmd cfg := getDockerConfigDir(t, cmd) diff --git a/e2e/run_test.go b/e2e/run_test.go index 6220eac9f..4dbc63c8a 100644 --- a/e2e/run_test.go +++ b/e2e/run_test.go @@ -11,7 +11,7 @@ func TestRunTwice(t *testing.T) { // Test that we are indeed generating random app names // We had a problem where the second run would fail with an error // "Installation "gallant_poitras" already exists, use 'docker app update' instead" - runWithDindSwarmAndRegistry(t, func(info dindSwarmAndRegistryInfo) { + runWithDindSwarmAndRegistry(t, func(info OrchestratorAndRegistryInfo) { cmd := info.configuredCmd contextPath := filepath.Join("testdata", "simple") diff --git a/e2e/testdata/compatibility/bundle-v0.9.0.json b/e2e/testdata/compatibility/bundle-v0.9.0.json index d1f733aff..d29fbabbe 100644 --- a/e2e/testdata/compatibility/bundle-v0.9.0.json +++ b/e2e/testdata/compatibility/bundle-v0.9.0.json @@ -20,7 +20,7 @@ "custom": { "com.docker.app": { "payload": { - "created": "2019-11-18T08:22:31.570871111Z" + "app-version": "v0.9.0-zeta1-223-g012748d754" }, "version": "1.0.0" } @@ -83,26 +83,26 @@ "description": "sample description", "images": { "backend": { - "contentDigest": "", + "contentDigest": "sha256:469e3e749d7c78386f5c9886ad5cad158c70934bb782c00ef0e195bc74903db6", "description": "app-e2e/backend", "image": "app-e2e/backend", "imageType": "docker", "size": 7394886 }, "frontend": { - "contentDigest": "", + "contentDigest": "sha256:8299ba94a840aefc2763a426917aec6b9fbc7cbcba1f6195c6f226fe0efcae4f", "description": "app-e2e/frontend", "image": "app-e2e/frontend", "imageType": "docker", - "size": 21365920 + "size": 21461559 } }, "invocationImages": [ { - "contentDigest": "", + "contentDigest": "sha256:22bf63159a078013a073da8e9d21d58d17b749ab34407ebf1cc709a237cd5cba", "image": "app-e2e:0.1.0-invoc", "imageType": "docker", - "size": 47261342 + "size": 47347352 } ], "maintainers": [ diff --git a/e2e/testdata/compatibility/compose-etcd.yaml b/e2e/testdata/compatibility/compose-etcd.yaml new file mode 100644 index 000000000..6449030a2 --- /dev/null +++ b/e2e/testdata/compatibility/compose-etcd.yaml @@ -0,0 +1,21 @@ +apiVersion: "etcd.database.coreos.com/v1beta2" +kind: "EtcdCluster" +metadata: + name: "compose-etcd" + namespace: "compose" +spec: + size: 3 + version: "3.3.15" + pod: + affinity: + podAntiAffinity: + preferredDuringSchedulingIgnoredDuringExecution: + - weight: 100 + podAffinityTerm: + labelSelector: + matchExpressions: + - key: etcd_cluster + operator: In + values: + - compose-etcd + topologyKey: kubernetes.io/hostname \ No newline at end of file