Skip to content
This repository was archived by the owner on Jul 18, 2025. It is now read-only.

Commit 020761c

Browse files
ndeloofndeloof
committed
We can't access DinD by localhost
from within the installer image, this would only resolve to container's loopback so we use computer external IP Signed-off-by: Nicolas De Loof <[email protected]>
1 parent f9acea2 commit 020761c

File tree

10 files changed

+145
-215
lines changed

10 files changed

+145
-215
lines changed

‎e2e/commands_test.go

Lines changed: 78 additions & 122 deletions
Original file line numberDiff line numberDiff line change
@@ -65,12 +65,16 @@ func testRenderApp(appPath string, env ...string) func(*testing.T) {
6565
cmd.Env = append(cmd.Env, env...)
6666
t.Run("stdout", func(t *testing.T) {
6767
result := icmd.RunCmd(cmd).Assert(t, icmd.Success)
68-
assert.Assert(t, is.Equal(readFile(t, filepath.Join(appPath, "expected.txt")), result.Stdout()), "rendering mismatch")
68+
expected := readFile(t, filepath.Join(appPath, "expected.txt"))
69+
actual := result.Stdout()
70+
assert.Assert(t, is.Equal(expected, actual), "rendering mismatch")
6971
})
7072
t.Run("file", func(t *testing.T) {
7173
cmd.Command = append(cmd.Command, "--output="+dir.Join("actual.yaml"))
7274
icmd.RunCmd(cmd).Assert(t, icmd.Success)
73-
assert.Assert(t, is.Equal(readFile(t, filepath.Join(appPath, "expected.txt")), readFile(t, dir.Join("actual.yaml"))), "rendering mismatch")
75+
expected := readFile(t, filepath.Join(appPath, "expected.txt"))
76+
actual := readFile(t, dir.Join("actual.yaml"))
77+
assert.Assert(t, is.Equal(expected, actual), "rendering mismatch")
7478
})
7579
}
7680
}
@@ -233,100 +237,86 @@ func TestRunWithLabels(t *testing.T) {
233237
}
234238

235239
func TestDockerAppLifecycle(t *testing.T) {
236-
t.Run("withBindMounts", func(t *testing.T) {
237-
testDockerAppLifecycle(t, true)
238-
})
239-
t.Run("withoutBindMounts", func(t *testing.T) {
240-
testDockerAppLifecycle(t, false)
241-
})
242-
}
240+
runWithDindSwarmAndRegistry(t, func(info dindSwarmAndRegistryInfo) {
241+
cmd := info.configuredCmd
242+
appName := strings.ToLower(strings.Replace(t.Name(), "/", "_", 1))
243+
tmpDir := fs.NewDir(t, appName)
244+
defer tmpDir.Remove()
243245

244-
func testDockerAppLifecycle(t *testing.T, useBindMount bool) {
245-
cmd, cleanup := dockerCli.createTestCmd()
246-
defer cleanup()
247-
appName := strings.ToLower(strings.Replace(t.Name(), "/", "_", 1))
248-
tmpDir := fs.NewDir(t, appName)
249-
defer tmpDir.Remove()
250-
// Running a swarm using docker in docker to install the application
251-
// and run the invocation image
252-
swarm := NewContainer("docker:19.03.3-dind", 2375)
253-
swarm.Start(t, "-e", "DOCKER_TLS_CERTDIR=")
254-
defer swarm.Stop(t)
255-
initializeDockerAppEnvironment(t, &cmd, tmpDir, swarm, useBindMount)
256-
257-
cmd.Command = dockerCli.Command("app", "build", "--tag", appName, "testdata/simple")
258-
icmd.RunCmd(cmd).Assert(t, icmd.Success)
259-
260-
// Install an illformed Docker Application Package
261-
cmd.Command = dockerCli.Command("app", "run", appName, "--set", "web_port=-1", "--name", appName)
262-
icmd.RunCmd(cmd).Assert(t, icmd.Expected{
263-
ExitCode: 1,
264-
Err: "error decoding 'Ports': Invalid hostPort: -1",
265-
})
246+
cmd.Command = dockerCli.Command("app", "build", "--tag", appName, "testdata/simple")
247+
icmd.RunCmd(cmd).Assert(t, icmd.Success)
266248

267-
// List the installation and check the failed status
268-
cmd.Command = dockerCli.Command("app", "ls")
269-
checkContains(t, icmd.RunCmd(cmd).Assert(t, icmd.Success).Combined(),
270-
[]string{
271-
`INSTALLATION\s+APPLICATION\s+LAST ACTION\s+RESULT\s+CREATED\s+MODIFIED\s+REFERENCE`,
272-
fmt.Sprintf(`%s\s+simple \(1.1.0-beta1\)\s+install\s+failure\s+.+second[s]?\sago\s+.+second[s]?\sago\s+`, appName),
249+
// Install an illformed Docker Application Package
250+
cmd.Command = dockerCli.Command("app", "run", appName, "--set", "web_port=-1", "--name", appName)
251+
icmd.RunCmd(cmd).Assert(t, icmd.Expected{
252+
ExitCode: 1,
253+
Err: "error decoding 'Ports': Invalid hostPort: -1",
273254
})
274255

275-
// Upgrading a failed installation is not allowed
276-
cmd.Command = dockerCli.Command("app", "update", appName)
277-
icmd.RunCmd(cmd).Assert(t, icmd.Expected{
278-
ExitCode: 1,
279-
Err: fmt.Sprintf("Running App %q cannot be updated, please use 'docker app run' instead", appName),
280-
})
281-
282-
// Install a Docker Application Package with an existing failed installation is fine
283-
cmd.Command = dockerCli.Command("app", "run", appName, "--name", appName)
284-
checkContains(t, icmd.RunCmd(cmd).Assert(t, icmd.Success).Combined(),
285-
[]string{
286-
fmt.Sprintf("WARNING: installing over previously failed installation %q", appName),
287-
fmt.Sprintf("Creating network %s_back", appName),
288-
fmt.Sprintf("Creating network %s_front", appName),
289-
fmt.Sprintf("Creating service %s_db", appName),
290-
fmt.Sprintf("Creating service %s_api", appName),
291-
fmt.Sprintf("Creating service %s_web", appName),
292-
})
293-
assertAppLabels(t, &cmd, appName, "db")
256+
// List the installation and check the failed status
257+
cmd.Command = dockerCli.Command("app", "ls")
258+
checkContains(t, icmd.RunCmd(cmd).Assert(t, icmd.Success).Combined(),
259+
[]string{
260+
`INSTALLATION\s+APPLICATION\s+LAST ACTION\s+RESULT\s+CREATED\s+MODIFIED\s+REFERENCE`,
261+
fmt.Sprintf(`%s\s+simple \(1.1.0-beta1\)\s+install\s+failure\s+.+second[s]?\sago\s+.+second[s]?\sago\s+`, appName),
262+
})
294263

295-
// List the installed application
296-
cmd.Command = dockerCli.Command("app", "ls")
297-
checkContains(t, icmd.RunCmd(cmd).Assert(t, icmd.Success).Combined(),
298-
[]string{
299-
`INSTALLATION\s+APPLICATION\s+LAST ACTION\s+RESULT\s+CREATED\s+MODIFIED\s+REFERENCE`,
300-
fmt.Sprintf(`%s\s+simple \(1.1.0-beta1\)\s+install\s+success\s+.+second[s]?\sago\s+.+second[s]?\sago\s+`, appName),
264+
// Upgrading a failed installation is not allowed
265+
cmd.Command = dockerCli.Command("app", "update", appName)
266+
icmd.RunCmd(cmd).Assert(t, icmd.Expected{
267+
ExitCode: 1,
268+
Err: fmt.Sprintf("Running App %q cannot be updated, please use 'docker app run' instead", appName),
301269
})
302270

303-
// Installing again the same application is forbidden
304-
cmd.Command = dockerCli.Command("app", "run", appName, "--name", appName)
305-
icmd.RunCmd(cmd).Assert(t, icmd.Expected{
306-
ExitCode: 1,
307-
Err: fmt.Sprintf("Installation %q already exists, use 'docker app update' instead", appName),
308-
})
271+
// Install a Docker Application Package with an existing failed installation is fine
272+
cmd.Command = dockerCli.Command("app", "run", appName, "--name", appName)
273+
checkContains(t, icmd.RunCmd(cmd).Assert(t, icmd.Success).Combined(),
274+
[]string{
275+
fmt.Sprintf("WARNING: installing over previously failed installation %q", appName),
276+
fmt.Sprintf("Creating network %s_back", appName),
277+
fmt.Sprintf("Creating network %s_front", appName),
278+
fmt.Sprintf("Creating service %s_db", appName),
279+
fmt.Sprintf("Creating service %s_api", appName),
280+
fmt.Sprintf("Creating service %s_web", appName),
281+
})
282+
assertAppLabels(t, &cmd, appName, "db")
283+
284+
// List the installed application
285+
cmd.Command = dockerCli.Command("app", "ls")
286+
checkContains(t, icmd.RunCmd(cmd).Assert(t, icmd.Success).Combined(),
287+
[]string{
288+
`INSTALLATION\s+APPLICATION\s+LAST ACTION\s+RESULT\s+CREATED\s+MODIFIED\s+REFERENCE`,
289+
fmt.Sprintf(`%s\s+simple \(1.1.0-beta1\)\s+install\s+success\s+.+second[s]?\sago\s+.+second[s]?\sago\s+`, appName),
290+
})
309291

310-
// Update the application, changing the port
311-
cmd.Command = dockerCli.Command("app", "update", appName, "--set", "web_port=8081")
312-
checkContains(t, icmd.RunCmd(cmd).Assert(t, icmd.Success).Combined(),
313-
[]string{
314-
fmt.Sprintf("Updating service %s_db", appName),
315-
fmt.Sprintf("Updating service %s_api", appName),
316-
fmt.Sprintf("Updating service %s_web", appName),
292+
// Installing again the same application is forbidden
293+
cmd.Command = dockerCli.Command("app", "run", appName, "--name", appName)
294+
icmd.RunCmd(cmd).Assert(t, icmd.Expected{
295+
ExitCode: 1,
296+
Err: fmt.Sprintf("Installation %q already exists, use 'docker app update' instead", appName),
317297
})
318-
assertAppLabels(t, &cmd, appName, "db")
319298

320-
// Uninstall the application
321-
cmd.Command = dockerCli.Command("app", "rm", appName)
322-
checkContains(t, icmd.RunCmd(cmd).Assert(t, icmd.Success).Combined(),
323-
[]string{
324-
fmt.Sprintf("Removing service %s_api", appName),
325-
fmt.Sprintf("Removing service %s_db", appName),
326-
fmt.Sprintf("Removing service %s_web", appName),
327-
fmt.Sprintf("Removing network %s_front", appName),
328-
fmt.Sprintf("Removing network %s_back", appName),
329-
})
299+
// Update the application, changing the port
300+
cmd.Command = dockerCli.Command("app", "update", appName, "--set", "web_port=8081")
301+
checkContains(t, icmd.RunCmd(cmd).Assert(t, icmd.Success).Combined(),
302+
[]string{
303+
fmt.Sprintf("Updating service %s_db", appName),
304+
fmt.Sprintf("Updating service %s_api", appName),
305+
fmt.Sprintf("Updating service %s_web", appName),
306+
})
307+
assertAppLabels(t, &cmd, appName, "db")
308+
309+
// Uninstall the application
310+
cmd.Command = dockerCli.Command("app", "rm", appName)
311+
checkContains(t, icmd.RunCmd(cmd).Assert(t, icmd.Success).Combined(),
312+
[]string{
313+
fmt.Sprintf("Removing service %s_api", appName),
314+
fmt.Sprintf("Removing service %s_db", appName),
315+
fmt.Sprintf("Removing service %s_web", appName),
316+
fmt.Sprintf("Removing network %s_front", appName),
317+
fmt.Sprintf("Removing network %s_back", appName),
318+
})
319+
})
330320
}
331321

332322
func TestCredentials(t *testing.T) {
@@ -417,7 +407,8 @@ func TestCredentials(t *testing.T) {
417407
"--cnab-bundle-json", bundle,
418408
)
419409
result := icmd.RunCmd(cmd).Assert(t, icmd.Success)
420-
golden.Assert(t, result.Stdout(), "credential-install-mixed-local-cred.golden")
410+
stdout := result.Stdout()
411+
golden.Assert(t, stdout, "credential-install-mixed-local-cred.golden")
421412
})
422413

423414
t.Run("overload", func(t *testing.T) {
@@ -437,41 +428,6 @@ func TestCredentials(t *testing.T) {
437428
})
438429
}
439430

440-
func initializeDockerAppEnvironment(t *testing.T, cmd *icmd.Cmd, tmpDir *fs.Dir, swarm *Container, useBindMount bool) {
441-
cmd.Env = append(cmd.Env, "DOCKER_TARGET_CONTEXT=swarm-target-context")
442-
443-
// The dind doesn't have the cnab-app-base image so we save it in order to load it later
444-
icmd.RunCommand(dockerCli.path, "save", fmt.Sprintf("docker/cnab-app-base:%s", internal.Version), "--output", tmpDir.Join("cnab-app-base.tar.gz")).Assert(t, icmd.Success)
445-
446-
// We need two contexts:
447-
// - one for `docker` so that it connects to the dind swarm created before
448-
// - the target context for the invocation image to install within the swarm
449-
cmd.Command = dockerCli.Command("context", "create", "swarm-context", "--docker", fmt.Sprintf(`"host=tcp://%s"`, swarm.GetAddress(t)), "--default-stack-orchestrator", "swarm")
450-
icmd.RunCmd(*cmd).Assert(t, icmd.Success)
451-
452-
// When creating a context on a Windows host we cannot use
453-
// the unix socket but it's needed inside the invocation image.
454-
// The workaround is to create a context with an empty host.
455-
// This host will default to the unix socket inside the
456-
// invocation image
457-
host := "host="
458-
if !useBindMount {
459-
host += fmt.Sprintf("tcp://%s", swarm.GetPrivateAddress(t))
460-
}
461-
462-
cmd.Command = dockerCli.Command("context", "create", "swarm-target-context", "--docker", host, "--default-stack-orchestrator", "swarm")
463-
icmd.RunCmd(*cmd).Assert(t, icmd.Success)
464-
465-
// Initialize the swarm
466-
cmd.Env = append(cmd.Env, "DOCKER_CONTEXT=swarm-context")
467-
cmd.Command = dockerCli.Command("swarm", "init")
468-
icmd.RunCmd(*cmd).Assert(t, icmd.Success)
469-
470-
// Load the needed base cnab image into the swarm docker engine
471-
cmd.Command = dockerCli.Command("load", "--input", tmpDir.Join("cnab-app-base.tar.gz"))
472-
icmd.RunCmd(*cmd).Assert(t, icmd.Success)
473-
}
474-
475431
func assertAppLabels(t *testing.T, cmd *icmd.Cmd, appName, containerName string) {
476432
cmd.Command = dockerCli.Command("inspect", fmt.Sprintf("%s_%s", appName, containerName))
477433
checkContains(t, icmd.RunCmd(*cmd).Assert(t, icmd.Success).Combined(),

‎e2e/helper_test.go

Lines changed: 45 additions & 44 deletions
Original file line numberDiff line numberDiff line change
@@ -3,6 +3,7 @@ package e2e
33
import (
44
"fmt"
55
"io/ioutil"
6+
"net"
67
"strconv"
78
"strings"
89
"testing"
@@ -27,63 +28,54 @@ func runWithDindSwarmAndRegistry(t *testing.T, todo func(dindSwarmAndRegistryInf
2728
cmd, cleanup := dockerCli.createTestCmd()
2829
defer cleanup()
2930

30-
registryPort := findAvailablePort()
3131
tmpDir := fs.NewDir(t, t.Name())
3232
defer tmpDir.Remove()
3333

34-
cmd.Env = append(cmd.Env, "DOCKER_TARGET_CONTEXT=swarm-target-context")
35-
3634
// The dind doesn't have the cnab-app-base image so we save it in order to load it later
3735
saveCmd := icmd.Cmd{Command: dockerCli.Command("save", fmt.Sprintf("docker/cnab-app-base:%s", internal.Version), "-o", tmpDir.Join("cnab-app-base.tar.gz"))}
3836
icmd.RunCmd(saveCmd).Assert(t, icmd.Success)
3937

38+
// Busybox is used in a few e2e test, let's pre-load it
39+
cmd.Command = dockerCli.Command("pull", "busybox:1.30.1")
40+
icmd.RunCmd(cmd).Assert(t, icmd.Success)
41+
saveCmd = icmd.Cmd{Command: dockerCli.Command("save", "busybox:1.30.1", "-o", tmpDir.Join("busybox.tar.gz"))}
42+
icmd.RunCmd(saveCmd).Assert(t, icmd.Success)
43+
4044
// we have a difficult constraint here:
4145
// - the registry must be reachable from the client side (for cnab-to-oci, which does not use the docker daemon to access the registry)
4246
// - the registry must be reachable from the dind daemon on the same address/port
43-
// Solution found is: fix the port of the registry to be the same internally and externally
44-
// and run the dind container in the same network namespace: this way 127.0.0.1:<registry-port> both resolves to the registry from the client and from dind
45-
46-
swarm := NewContainer("docker:19.03.3-dind", 2375, "--insecure-registry", fmt.Sprintf("127.0.0.1:%d", registryPort))
47-
swarm.Start(t, "--expose", strconv.FormatInt(int64(registryPort), 10),
48-
"-p", fmt.Sprintf("%d:%d", registryPort, registryPort),
49-
"-p", "2375",
50-
"-e", "DOCKER_TLS_CERTDIR=") // Disable certificate generate on DinD startup
51-
defer swarm.Stop(t)
47+
// - the installer image need to target the same docker context (dind) as the client, while running on default (or another) context, which means we can't use 'localhost'
48+
// Solution found is: use host external IP (not loopback) so accessing from within installer container will reach the right container
5249

53-
registry := NewContainer("registry:2", registryPort)
54-
registry.StartWithContainerNetwork(t, swarm, "-e", "REGISTRY_VALIDATION_MANIFESTS_URLS_ALLOW=[^http]",
55-
"-e", fmt.Sprintf("REGISTRY_HTTP_ADDR=0.0.0.0:%d", registryPort))
50+
registry := NewContainer("registry:2", 5000)
51+
registry.Start(t, "-e", "REGISTRY_VALIDATION_MANIFESTS_URLS_ALLOW=[^http]",
52+
"-e", "REGISTRY_HTTP_ADDR=0.0.0.0:5000")
5653
defer registry.StopNoFail()
54+
registryAddress := registry.GetAddress(t)
5755

58-
// We need two contexts:
59-
// - one for `docker` so that it connects to the dind swarm created before
60-
// - the target context for the invocation image to install within the swarm
61-
cmd.Command = dockerCli.Command("context", "create", "swarm-context", "--docker", fmt.Sprintf(`"host=tcp://%s"`, swarm.GetAddress(t)), "--default-stack-orchestrator", "swarm")
62-
icmd.RunCmd(cmd).Assert(t, icmd.Success)
56+
swarm := NewContainer("docker:19.03.3-dind", 2375, "--insecure-registry", registryAddress)
57+
swarm.Start(t, "-e", "DOCKER_TLS_CERTDIR=") // Disable certificate generate on DinD startup
58+
defer swarm.Stop(t)
59+
swarmAddress := swarm.GetAddress(t)
6360

64-
// When creating a context on a Windows host we cannot use
65-
// the unix socket but it's needed inside the invocation image.
66-
// The workaround is to create a context with an empty host.
67-
// This host will default to the unix socket inside the
68-
// invocation image
69-
cmd.Command = dockerCli.Command("context", "create", "swarm-target-context", "--docker", "host=", "--default-stack-orchestrator", "swarm")
61+
cmd.Command = dockerCli.Command("context", "create", "swarm-context", "--docker", fmt.Sprintf(`"host=tcp://%s"`, swarmAddress), "--default-stack-orchestrator", "swarm")
7062
icmd.RunCmd(cmd).Assert(t, icmd.Success)
7163

64+
cmd.Env = append(cmd.Env, "DOCKER_CONTEXT=swarm-context", "DOCKER_INSTALLER_CONTEXT=swarm-context")
7265
// Initialize the swarm
73-
cmd.Env = append(cmd.Env, "DOCKER_CONTEXT=swarm-context")
7466
cmd.Command = dockerCli.Command("swarm", "init")
7567
icmd.RunCmd(cmd).Assert(t, icmd.Success)
7668
// Load the needed base cnab image into the swarm docker engine
7769
cmd.Command = dockerCli.Command("load", "-i", tmpDir.Join("cnab-app-base.tar.gz"))
7870
icmd.RunCmd(cmd).Assert(t, icmd.Success)
79-
80-
cmd.Command = dockerCli.Command("pull", "busybox:1.30.1")
71+
// Pre-load busybox image used by a few e2e tests
72+
cmd.Command = dockerCli.Command("load", "-i", tmpDir.Join("busybox.tar.gz"))
8173
icmd.RunCmd(cmd).Assert(t, icmd.Success)
8274

8375
info := dindSwarmAndRegistryInfo{
8476
configuredCmd: cmd,
85-
registryAddress: registry.GetAddress(t),
86-
swarmAddress: swarm.GetAddress(t),
77+
registryAddress: registryAddress,
78+
swarmAddress: swarmAddress,
8779
stopRegistry: registry.StopNoFail,
8880
registryLogs: registry.Logs(t),
8981
}
@@ -147,23 +139,32 @@ func (c *Container) GetAddress(t *testing.T) string {
147139
if c.address != "" {
148140
return c.address
149141
}
150-
container := c.parentContainer
151-
if container == "" {
152-
container = c.container
153-
}
154-
result := icmd.RunCommand(dockerCli.path, "port", container, strconv.Itoa(c.privatePort)).Assert(t, icmd.Success)
155-
c.address = fmt.Sprintf("127.0.0.1:%v", strings.Trim(strings.Split(result.Stdout(), ":")[1], " \r\n"))
142+
ip := c.getIP(t)
143+
port := c.getPort(t)
144+
c.address = fmt.Sprintf("%s:%v", ip, port)
156145
return c.address
157146
}
158147

159-
// GetPrivateAddress returns the host:port this container listens on
160-
func (c *Container) GetPrivateAddress(t *testing.T) string {
161-
container := c.parentContainer
162-
if container == "" {
163-
container = c.container
148+
func (c *Container) getPort(t *testing.T) string {
149+
result := icmd.RunCommand(dockerCli.path, "port", c.container, strconv.Itoa(c.privatePort)).Assert(t, icmd.Success)
150+
port := strings.Trim(strings.Split(result.Stdout(), ":")[1], " \r\n")
151+
return port
152+
}
153+
154+
func (c *Container) getIP(t *testing.T) string {
155+
ip := "127.0.0.1"
156+
// This won't work, but we can't guarantee computer has another IP
157+
addrs, err := net.InterfaceAddrs()
158+
assert.NilError(t, err)
159+
for _, a := range addrs {
160+
if ipnet, ok := a.(*net.IPNet); ok && !ipnet.IP.IsLoopback() {
161+
if ipnet.IP.To4() != nil {
162+
ip = ipnet.IP.String()
163+
break
164+
}
165+
}
164166
}
165-
result := icmd.RunCommand(dockerCli.path, "inspect", container, "-f", "{{.NetworkSettings.IPAddress}}").Assert(t, icmd.Success)
166-
return fmt.Sprintf("%s:%d", strings.TrimSpace(result.Stdout()), c.privatePort)
167+
return ip
167168
}
168169

169170
func (c *Container) Logs(t *testing.T) func() string {

0 commit comments

Comments
 (0)