Make DockerComposeContainer only pull necessary images when multiple compose files were given (#3787)

Co-authored-by: tianjie <[email protected]>

Author: askfor

core/src/main/java/org/testcontainers/containers/DockerComposeContainer.java

@@ -77,7 +77,7 @@ public class DockerComposeContainer<SELF extends DockerComposeContainer<SELF>> e
      */
     private final String identifier;
     private final List<File> composeFiles;
-    private Set<ParsedDockerComposeFile> parsedComposeFiles;
+    private DockerComposeFiles dockerComposeFiles;
     private final Map<String, Integer> scalingPreferences = new HashMap<>();
     private DockerClient dockerClient;
     private boolean localCompose;
@@ -126,7 +126,7 @@ public DockerComposeContainer(String identifier, File... composeFiles) {
     public DockerComposeContainer(String identifier, List<File> composeFiles) {
 
         this.composeFiles = composeFiles;
-        this.parsedComposeFiles = composeFiles.stream().map(ParsedDockerComposeFile::new).collect(Collectors.toSet());
+        this.dockerComposeFiles = new DockerComposeFiles(composeFiles);
 
         // Use a unique identifier so that containers created for this compose environment can be identified
         this.identifier = identifier;
@@ -184,8 +184,7 @@ private void pullImages() {
         // Pull images using our docker client rather than compose itself,
         // (a) as a workaround for https://github.com/docker/compose/issues/5854, which prevents authenticated image pulls being possible when credential helpers are in use
         // (b) so that credential helper-based auth still works when compose is running from within a container
-        parsedComposeFiles.stream()
-            .flatMap(it -> it.getDependencyImageNames().stream())
+        dockerComposeFiles.getDependencyImages()
             .forEach(imageName -> {
                 try {
                     log.info("Preemptively checking local images for '{}', referenced via a compose file or transitive Dockerfile. If not available, it will be pulled.", imageName);

core/src/main/java/org/testcontainers/containers/DockerComposeFiles.java

@@ -0,0 +1,43 @@
+package org.testcontainers.containers;
+
+import java.io.File;
+import java.util.HashMap;
+import java.util.HashSet;
+import java.util.List;
+import java.util.Map;
+import java.util.Map.Entry;
+import java.util.Set;
+import java.util.stream.Collectors;
+
+public class DockerComposeFiles {
+
+    private List<ParsedDockerComposeFile> parsedComposeFiles;
+
+    public DockerComposeFiles(List<File> composeFiles) {
+        this.parsedComposeFiles = composeFiles.stream().map(ParsedDockerComposeFile::new).collect(Collectors.toList());
+    }
+
+    public Set<String> getDependencyImages() {
+
+        Map<String, Set<String>> mergedServiceNameToImageNames = mergeServiceDependencyImageNames();
+
+        return getImageNames(mergedServiceNameToImageNames);
+    }
+
+    private Map<String, Set<String>> mergeServiceDependencyImageNames() {
+        Map<String, Set<String>> mergedServiceNameToImageNames = new HashMap();
+        for (ParsedDockerComposeFile parsedComposeFile : parsedComposeFiles) {
+            for (Entry<String, Set<String>> entry : parsedComposeFile.getServiceNameToImageNames().entrySet()) {
+                mergedServiceNameToImageNames.put(entry.getKey(), entry.getValue());
+            }
+        }
+        return mergedServiceNameToImageNames;
+    }
+
+    private Set<String> getImageNames(Map<String, Set<String>> serviceToImageNames) {
+        Set<String> imageNames = new HashSet<>();
+        serviceToImageNames.values().stream().forEach(imageNames::addAll);
+        return imageNames;
+    }
+
+}

core/src/main/java/org/testcontainers/containers/ParsedDockerComposeFile.java

@@ -1,6 +1,7 @@
 package org.testcontainers.containers;
 
 import com.google.common.annotations.VisibleForTesting;
+import com.google.common.collect.Sets;
 import lombok.EqualsAndHashCode;
 import lombok.Getter;
 import lombok.extern.slf4j.Slf4j;
@@ -12,7 +13,7 @@
 import java.io.FileInputStream;
 import java.nio.file.Files;
 import java.nio.file.Path;
-import java.util.HashSet;
+import java.util.HashMap;
 import java.util.Map;
 import java.util.Set;
 
@@ -29,7 +30,7 @@ class ParsedDockerComposeFile {
     private final File composeFile;
 
     @Getter
-    private Set<String> dependencyImageNames = new HashSet<>();
+    private Map<String, Set<String>> serviceNameToImageNames = new HashMap<>();
 
     ParsedDockerComposeFile(File composeFile) {
         Yaml yaml = new Yaml();
@@ -87,8 +88,8 @@ private void parseAndValidate() {
             final Map serviceDefinitionMap = (Map) serviceDefinition;
 
             validateNoContainerNameSpecified(serviceName, serviceDefinitionMap);
-            findServiceImageName(serviceDefinitionMap);
-            findImageNamesInDockerfile(serviceDefinitionMap);
+            findServiceImageName(serviceName, serviceDefinitionMap);
+            findImageNamesInDockerfile(serviceName, serviceDefinitionMap);
         }
     }
 
@@ -102,15 +103,15 @@ private void validateNoContainerNameSpecified(String serviceName, Map serviceDef
         }
     }
 
-    private void findServiceImageName(Map serviceDefinitionMap) {
+    private void findServiceImageName(String serviceName, Map serviceDefinitionMap) {
         if (serviceDefinitionMap.containsKey("image") && serviceDefinitionMap.get("image") instanceof String) {
             final String imageName = (String) serviceDefinitionMap.get("image");
             log.debug("Resolved dependency image for Docker Compose in {}: {}", composeFileName, imageName);
-            dependencyImageNames.add(imageName);
+            serviceNameToImageNames.put(serviceName, Sets.newHashSet(imageName));
         }
     }
 
-    private void findImageNamesInDockerfile(Map serviceDefinitionMap) {
+    private void findImageNamesInDockerfile(String serviceName, Map serviceDefinitionMap) {
         final Object buildNode = serviceDefinitionMap.get("build");
         Path dockerfilePath = null;
 
@@ -141,7 +142,7 @@ private void findImageNamesInDockerfile(Map serviceDefinitionMap) {
             Set<String> resolvedImageNames = new ParsedDockerfile(dockerfilePath).getDependencyImageNames();
             if (!resolvedImageNames.isEmpty()) {
                 log.debug("Resolved Dockerfile dependency images for Docker Compose in {} -> {}: {}", composeFileName, dockerfilePath, resolvedImageNames);
-                this.dependencyImageNames.addAll(resolvedImageNames);
+                this.serviceNameToImageNames.put(serviceName, resolvedImageNames);
             }
         }
     }

core/src/test/java/org/testcontainers/containers/DockerComposeFilesTest.java

@@ -0,0 +1,25 @@
+package org.testcontainers.containers;
+
+
+import com.google.common.collect.Lists;
+import java.io.File;
+import org.assertj.core.api.Assertions;
+import org.junit.Test;
+
+public class DockerComposeFilesTest {
+    @Test
+    public void shouldGetDependencyImages() {
+        DockerComposeFiles dockerComposeFiles = new DockerComposeFiles(Lists.newArrayList(new File("src/test/resources/docker-compose-imagename-parsing-v2.yml")));
+        Assertions.assertThat(dockerComposeFiles.getDependencyImages())
+            .containsExactlyInAnyOrder("postgres", "redis", "mysql");
+    }
+
+    @Test
+    public void shouldGetDependencyImagesWhenOverriding() {
+        DockerComposeFiles dockerComposeFiles = new DockerComposeFiles(
+            Lists.newArrayList(new File("src/test/resources/docker-compose-imagename-overriding-a.yml"),
+                               new File("src/test/resources/docker-compose-imagename-overriding-b.yml")));
+        Assertions.assertThat(dockerComposeFiles.getDependencyImages())
+            .containsExactlyInAnyOrder("alpine:3.2", "redis:b", "mysql:b", "aservice");
+    }
+}

core/src/test/java/org/testcontainers/containers/ParsedDockerComposeFileValidationTest.java

@@ -9,7 +9,7 @@
 
 import static java.util.Collections.emptyList;
 import static java.util.Collections.emptyMap;
-import static org.rnorth.visibleassertions.VisibleAssertions.assertEquals;
+import static org.assertj.core.api.Assertions.entry;
 
 public class ParsedDockerComposeFileValidationTest {
 
@@ -76,27 +76,47 @@ public void shouldIgnoreUnknownStructure() {
     public void shouldObtainImageNamesV1() {
         File file = new File("src/test/resources/docker-compose-imagename-parsing-v1.yml");
         ParsedDockerComposeFile parsedFile = new ParsedDockerComposeFile(file);
-        assertEquals("all defined images are found", Sets.newHashSet("redis", "mysql", "postgres"), parsedFile.getDependencyImageNames()); // redis, mysql from compose file, postgres from Dockerfile build
+        Assertions.assertThat(parsedFile.getServiceNameToImageNames())
+            .contains(
+                entry("mysql", Sets.newHashSet("mysql")),
+                entry("redis", Sets.newHashSet("redis")),
+                entry("custom", Sets.newHashSet("postgres")))
+            .as("all defined images are found"); // redis, mysql from compose file, postgres from Dockerfile build
     }
 
     @Test
     public void shouldObtainImageNamesV2() {
         File file = new File("src/test/resources/docker-compose-imagename-parsing-v2.yml");
         ParsedDockerComposeFile parsedFile = new ParsedDockerComposeFile(file);
-        assertEquals("all defined images are found", Sets.newHashSet("redis", "mysql", "postgres"), parsedFile.getDependencyImageNames()); // redis, mysql from compose file, postgres from Dockerfile build
+        Assertions.assertThat(parsedFile.getServiceNameToImageNames())
+            .contains(
+                entry("mysql", Sets.newHashSet("mysql")),
+                entry("redis", Sets.newHashSet("redis")),
+                entry("custom", Sets.newHashSet("postgres")))
+            .as("all defined images are found");
     }
 
     @Test
     public void shouldObtainImageFromDockerfileBuild() {
         File file = new File("src/test/resources/docker-compose-imagename-parsing-dockerfile.yml");
         ParsedDockerComposeFile parsedFile = new ParsedDockerComposeFile(file);
-        assertEquals("all defined images are found", Sets.newHashSet("redis", "mysql", "alpine:3.2"), parsedFile.getDependencyImageNames()); // redis, mysql from compose file, alpine:3.2 from Dockerfile build
+        Assertions.assertThat(parsedFile.getServiceNameToImageNames())
+            .contains(
+                entry("mysql", Sets.newHashSet("mysql")),
+                entry("redis", Sets.newHashSet("redis")),
+                entry("custom", Sets.newHashSet("alpine:3.2")))
+            .as("all defined images are found"); // r/ redis, mysql from compose file, alpine:3.2 from Dockerfile build
     }
 
     @Test
     public void shouldObtainImageFromDockerfileBuildWithContext() {
         File file = new File("src/test/resources/docker-compose-imagename-parsing-dockerfile-with-context.yml");
         ParsedDockerComposeFile parsedFile = new ParsedDockerComposeFile(file);
-        assertEquals("all defined images are found", Sets.newHashSet("redis", "mysql", "alpine:3.2"), parsedFile.getDependencyImageNames()); // redis, mysql from compose file, alpine:3.2 from Dockerfile build
+        Assertions.assertThat(parsedFile.getServiceNameToImageNames())
+            .contains(
+                entry("mysql", Sets.newHashSet("mysql")),
+                entry("redis", Sets.newHashSet("redis")),
+                entry("custom", Sets.newHashSet("alpine:3.2")))
+            .as("all defined images are found"); // redis, mysql from compose file, alpine:3.2 from Dockerfile build
     }
 }

core/src/test/resources/docker-compose-imagename-overriding-a.yml

@@ -0,0 +1,12 @@
+version: "2.1"
+services:
+    aservice:
+      image: aservice
+    redis:
+        image: redis:a
+    mysql:
+        image: mysql:a
+    custom:
+        build: .
+networks:
+    custom_network: {}

core/src/test/resources/docker-compose-imagename-overriding-b.yml

@@ -0,0 +1,10 @@
+version: "2.1"
+services:
+    redis:
+        image: redis:b
+    mysql:
+        image: mysql:b
+    custom:
+        build: compose-dockerfile
+networks:
+    custom_network: {}