From 7260695e5e1590235ddb85e016a2e4c8e7e3e5d1 Mon Sep 17 00:00:00 2001 From: Johannes Zottele Date: Tue, 29 Oct 2024 19:08:07 +0100 Subject: [PATCH 1/3] Add ContainerState#copyArchiveFromContainer as an alternative to copyFileFromContainer While the copyFileFromContainer allows consuming a generic InputStream of the Archives' first entry, it does not support directories. The new copyArchiveFromContainer method allows users to process the raw TAR archive InputStream. --- .../containers/ContainerState.java | 32 ++++++++++++++++--- 1 file changed, 27 insertions(+), 5 deletions(-) diff --git a/core/src/main/java/org/testcontainers/containers/ContainerState.java b/core/src/main/java/org/testcontainers/containers/ContainerState.java index e19f7a85310..3dddeed362e 100644 --- a/core/src/main/java/org/testcontainers/containers/ContainerState.java +++ b/core/src/main/java/org/testcontainers/containers/ContainerState.java @@ -8,6 +8,7 @@ import com.github.dockerjava.api.model.PortBinding; import com.github.dockerjava.api.model.Ports; import com.google.common.base.Preconditions; +import java.nio.file.Path; import lombok.SneakyThrows; import org.apache.commons.compress.archivers.tar.TarArchiveInputStream; import org.apache.commons.compress.archivers.tar.TarArchiveOutputStream; @@ -402,17 +403,38 @@ default void copyFileFromContainer(String containerPath, String destinationPath) */ @SneakyThrows default T copyFileFromContainer(String containerPath, ThrowingFunction function) { + return copyArchiveFromContainer(containerPath, (stream) -> { + try( + TarArchiveInputStream tarStream = new TarArchiveInputStream(stream) + ) { + tarStream.getNextTarEntry(); + return function.apply(tarStream); + } + }); + } + + /** + * Copies an archive from the container at the specified path + * and processes it using the provided function. + * The path can be a file or directory and is automatically archived. + * + * @param containerPath the path inside the container to the file or directory to copy + * @param function a function that takes an {@link InputStream} of the copied + * archive and returns a result + * @return the result of applying the function to the InputStream + */ + @SneakyThrows + default T copyArchiveFromContainer(String containerPath, ThrowingFunction function) { if (getContainerId() == null) { throw new IllegalStateException("copyFileFromContainer can only be used when the Container is created."); } - DockerClient dockerClient = getDockerClient(); try ( - InputStream inputStream = dockerClient.copyArchiveFromContainerCmd(getContainerId(), containerPath).exec(); - TarArchiveInputStream tarInputStream = new TarArchiveInputStream(inputStream) + InputStream inputStream = dockerClient.copyArchiveFromContainerCmd(getContainerId(), containerPath).exec(); ) { - tarInputStream.getNextTarEntry(); - return function.apply(tarInputStream); + return function.apply(inputStream); } } + + } From ea17c1c2eaf8c23994a39ec397235799d93cbaed Mon Sep 17 00:00:00 2001 From: Johannes Zottele Date: Tue, 29 Oct 2024 19:22:04 +0100 Subject: [PATCH 2/3] Add method to copy files and directories from container to host Introduces a new `copyPathFromContainer` method in `ContainerState.java` that enables copying files or directories from a Docker container to the host. This method handles both file and directory copying, ensuring proper directory structure on the host. It also includes error handling for various states such as existing files conflicting with directories. --- .../containers/ContainerState.java | 80 ++++++++++++++++++- 1 file changed, 79 insertions(+), 1 deletion(-) diff --git a/core/src/main/java/org/testcontainers/containers/ContainerState.java b/core/src/main/java/org/testcontainers/containers/ContainerState.java index 3dddeed362e..6a927a971ef 100644 --- a/core/src/main/java/org/testcontainers/containers/ContainerState.java +++ b/core/src/main/java/org/testcontainers/containers/ContainerState.java @@ -9,10 +9,13 @@ import com.github.dockerjava.api.model.Ports; import com.google.common.base.Preconditions; import java.nio.file.Path; +import java.nio.file.Paths; import lombok.SneakyThrows; +import org.apache.commons.compress.archivers.tar.TarArchiveEntry; import org.apache.commons.compress.archivers.tar.TarArchiveInputStream; import org.apache.commons.compress.archivers.tar.TarArchiveOutputStream; import org.apache.commons.compress.utils.IOUtils; +import org.apache.commons.io.FileUtils; import org.apache.commons.lang3.math.NumberUtils; import org.slf4j.Logger; import org.slf4j.LoggerFactory; @@ -395,6 +398,77 @@ default void copyFileFromContainer(String containerPath, String destinationPath) ); } + + /** + * Copies a file or directory from a container to the host. + * + * @param containerPath the path within the container from which to copy + * @param hostPath the path on the host to which the contents will be copied + */ + default void copyPathFromContainer(String containerPath, String hostPath) { + if (getContainerId() == null) { + throw new IllegalStateException("copyPathFromContainer can only be used when the Container is created."); + } + + copyArchiveFromContainer(containerPath, (inputStream) -> { + try(TarArchiveInputStream tarStream = new TarArchiveInputStream(inputStream)) { + // advance to first tar entry + TarArchiveEntry currentEntry = tarStream.getNextTarEntry(); + + // copy is file only (no directory copy) + boolean fileOnly = currentEntry.isFile(); + // in case of coping a directory, + // we don't want to emit the root directory (would result in double nested directories) + String dirPrefixToRemove = currentEntry.isDirectory() ? currentEntry.getName() : ""; + + Path hostPathObj = Paths.get(hostPath); + + while (currentEntry != null) { + File destFile; + if (fileOnly) { + // if we copy only a single file, we use the specified hostPath as destination + destFile = hostPathObj.toFile(); + } else { + // if we copy a directory we have to resolve the path + destFile = hostPathObj.resolve(currentEntry.getName() + // remove the root directory of the copied TAR + .replaceFirst("^" + dirPrefixToRemove, "") + ).toFile(); + } + + if (currentEntry.isFile()) { + // create parent directory if they do not exist yet + FileUtils.forceMkdirParent(destFile); + // copy file to destination + try (FileOutputStream output = new FileOutputStream(destFile)) { + IOUtils.copy(tarStream, output); + } + } else if (currentEntry.isDirectory()) { + if (destFile.exists() && !destFile.isDirectory()) { + // throw exception if directory would override already existing file + throw new IOException( + "copyPathFromContainer cannot create directory '" + destFile + "' as a file at this path already exists."); + } + // create a destination directory + FileUtils.forceMkdir(destFile); + } else { + // if we cannot handle the entry, we throw an exception + throw new UnsupportedOperationException( + "copyPathFromContainer can only copy files and directories. '" + + currentEntry.getName() + + "' is neither a file nor a directory."); + } + + // jump to next tar entry + currentEntry = tarStream.getNextTarEntry(); + } + + return true; + } + }); + } + + /** * Streams a file which resides inside the container * @@ -403,6 +477,10 @@ default void copyFileFromContainer(String containerPath, String destinationPath) */ @SneakyThrows default T copyFileFromContainer(String containerPath, ThrowingFunction function) { + if (getContainerId() == null) { + throw new IllegalStateException("copyFileFromContainer can only be used when the Container is created."); + } + return copyArchiveFromContainer(containerPath, (stream) -> { try( TarArchiveInputStream tarStream = new TarArchiveInputStream(stream) @@ -426,7 +504,7 @@ default T copyFileFromContainer(String containerPath, ThrowingFunction T copyArchiveFromContainer(String containerPath, ThrowingFunction function) { if (getContainerId() == null) { - throw new IllegalStateException("copyFileFromContainer can only be used when the Container is created."); + throw new IllegalStateException("copyArchiveFromContainer can only be used when the Container is created."); } DockerClient dockerClient = getDockerClient(); try ( From 9367f07a205c8f82f3c0e36f951f962b5fe87c28 Mon Sep 17 00:00:00 2001 From: Johannes Zottele Date: Tue, 29 Oct 2024 20:42:26 +0100 Subject: [PATCH 3/3] Add tests for copyPathFromContainer method These tests cover various scenarios such as copying single files, nested directories, and handling conflicts and errors. --- .../junit/FileOperationsTest.java | 113 ++++++++++++++++++ 1 file changed, 113 insertions(+) diff --git a/core/src/test/java/org/testcontainers/junit/FileOperationsTest.java b/core/src/test/java/org/testcontainers/junit/FileOperationsTest.java index 767deacfb00..505b81cd02c 100644 --- a/core/src/test/java/org/testcontainers/junit/FileOperationsTest.java +++ b/core/src/test/java/org/testcontainers/junit/FileOperationsTest.java @@ -1,6 +1,9 @@ package org.testcontainers.junit; import com.github.dockerjava.api.exception.NotFoundException; +import java.nio.file.Files; +import java.nio.file.Path; +import java.nio.file.Paths; import org.apache.commons.io.FileUtils; import org.apache.commons.io.IOUtils; import org.apache.commons.io.output.CountingOutputStream; @@ -185,4 +188,114 @@ public void shouldCopyFileFromExitedContainerTest() throws IOException { ); } } + + @Test + public void copyPathFromContainer_singleFileTest() throws IOException { + try ( + GenericContainer container = new GenericContainer(TestImages.ALPINE_IMAGE) // + .withCommand("top") + ) { + container.start(); + final MountableFile mountableFile = MountableFile.forClasspathResource("test_copy_to_container.txt"); + container.copyFileToContainer(mountableFile, "/home/"); + + File actualFile = new File(temporaryFolder.getRoot().getAbsolutePath() + "/test_copy_from_container.txt"); + container.copyPathFromContainer("/home/test_copy_to_container.txt", actualFile.getPath()); + + File expectedFile = new File(mountableFile.getResolvedPath()); + assertThat(FileUtils.contentEquals(expectedFile, actualFile)).as("Files aren't same ").isTrue(); + } + } + + @Test + public void copyPathFromContainer_nestedDirectoryStructureWithoutFilesTest() + throws IOException, InterruptedException { + try (GenericContainer container = new GenericContainer<>(TestImages.ALPINE_IMAGE) + .withCommand("sleep", "infinity")) { + container.start(); + + // create directory structure + container.execInContainer("mkdir", "-p", "/home/nested/directoryOne/inner"); + container.execInContainer("mkdir", "-p", "/home/nested/directoryTwo/inner"); + + File hostPath = temporaryFolder.getRoot(); + container.copyPathFromContainer("/home", hostPath.getPath()); + + // assert layer 1 structure + assertDirectoryContents(hostPath.toPath(), "nested"); + + // assert layer 2 structure (nested/) + assertDirectoryContents(Paths.get(hostPath.getPath(), "nested"), "directoryOne", "directoryTwo"); + + // assert layer 3 structure (nested/directoryOne/ and nested/directoryTwo/) + assertDirectoryContents(Paths.get(hostPath.getPath(), "nested", "directoryOne"), "inner"); + assertDirectoryContents(Paths.get(hostPath.getPath(), "nested", "directoryTwo"), "inner"); + } + } + + @Test + public void copyPathFromContainer_filesInNestedDirectory() + throws IOException, InterruptedException { + try (GenericContainer container = new GenericContainer<>(TestImages.ALPINE_IMAGE) + .withCommand("sleep", "infinity")) { + container.start(); + + // create directory and files + container.execInContainer("mkdir", "-p", "/home/test/nested"); + container.execInContainer("sh","-c", "echo 'rootfile' > /home/test/root.txt "); + container.execInContainer("sh","-c", "echo 'nestedfile' > /home/test/nested/nested.txt"); + + Path actualRootPath = temporaryFolder.getRoot().toPath(); + container.copyPathFromContainer("/home/test", actualRootPath.toString()); + + // assert correct file structure + assertDirectoryContents(actualRootPath, "nested", "root.txt"); + assertDirectoryContents(actualRootPath.resolve("nested"), "nested.txt"); + + // assert correct file content + assertThat(actualRootPath.resolve("root.txt")) + .hasContent("rootfile"); + assertThat(actualRootPath.resolve("nested/nested.txt")) + .hasContent("nestedfile"); + } + } + + @Test(expected = NotFoundException.class) + public void copyPathFromContainer_nonExistingDirectory_shouldFail() + throws IOException, InterruptedException { + try (GenericContainer container = new GenericContainer<>(TestImages.ALPINE_IMAGE) + .withCommand("sleep", "infinity")) { + container.start(); + + Path actualRootPath = temporaryFolder.getRoot().toPath(); + container.copyPathFromContainer("/home/test", actualRootPath.toString()); + } + } + + @Test + public void copyPathFromContainer_conflictingHostFile_shouldFail() + throws IOException, InterruptedException { + try (GenericContainer container = new GenericContainer<>(TestImages.ALPINE_IMAGE) + .withCommand("sleep", "infinity")) { + container.start(); + + container.execInContainer("mkdir", "-p", "/home/test/conflict/dir"); + + Path actualRootPath = temporaryFolder.getRoot().toPath(); + // create file `conflict` ... will conflict the copied directory `conflict/dir` + FileUtils.touch(actualRootPath.resolve("conflict").toFile()); + + assertThatThrownBy(() -> { + container.copyPathFromContainer("/home/test", actualRootPath.toString()); + }) + .isInstanceOf(IOException.class) + .hasMessageContaining("copyPathFromContainer cannot create directory") + .hasMessageContaining("as a file at this path already exists."); + } + } + + private void assertDirectoryContents(Path directory, String... expectedContents) { + assertThat(directory.toFile().list()).containsExactlyInAnyOrder(expectedContents); + } + }