Merge pull request #1107 from sarveshkaushal/cp-test-fix

k8s-ci-robot · web-flow · commit 6e64a04a0e38 · 2020-08-19T01:01:10.000-07:00
Supporting copy operation when tar is not present
diff --git a/util/src/main/java/io/kubernetes/client/Copy.java b/util/src/main/java/io/kubernetes/client/Copy.java
@@ -19,14 +19,17 @@
 import io.kubernetes.client.openapi.models.V1Pod;
 import io.kubernetes.client.util.exception.CopyNotSupportedException;
 import java.io.BufferedInputStream;
+import java.io.BufferedReader;
 import java.io.ByteArrayInputStream;
 import java.io.File;
 import java.io.FileInputStream;
 import java.io.FileOutputStream;
 import java.io.IOException;
 import java.io.InputStream;
+import java.io.InputStreamReader;
 import java.io.OutputStream;
 import java.nio.file.Path;
+import java.nio.file.Paths;
 import org.apache.commons.codec.binary.Base64InputStream;
 import org.apache.commons.codec.binary.Base64OutputStream;
 import org.apache.commons.compress.archivers.ArchiveEntry;
@@ -121,6 +124,35 @@ public void copyDirectoryFromPod(
     if (!isTarPresentInContainer(namespace, pod, container)) {
       throw new CopyNotSupportedException("Tar is not present in the target container");
     }
+    copyDirectoryFromPod(namespace, pod, container, srcPath, destination, true);
+  }
+
+  /**
+   * Copy directory from a pod to local.
+   *
+   * @param namespace
+   * @param pod
+   * @param container
+   * @param srcPath
+   * @param destination
+   * @param enableTarCompressing: false if tar is not present in target container
+   * @throws IOException
+   * @throws ApiException
+   */
+  public void copyDirectoryFromPod(
+      String namespace,
+      String pod,
+      String container,
+      String srcPath,
+      Path destination,
+      boolean enableTarCompressing)
+      throws IOException, ApiException {
+    if (!enableTarCompressing) {
+      TreeNode tree = new TreeNode(true, srcPath, true);
+      createDirectoryTree(tree, namespace, pod, container, srcPath);
+      createDirectoryStructureFromTree(tree, namespace, pod, container, srcPath, destination);
+      return;
+    }
     final Process proc =
         this.exec(
             namespace,
@@ -165,6 +197,125 @@ public void copyDirectoryFromPod(
     }
   }
 
+  // This creates directories and files using tree of files and directories under container
+  private void createDirectoryStructureFromTree(
+      TreeNode tree,
+      String namespace,
+      String pod,
+      String container,
+      String srcPath,
+      Path destination)
+      throws IOException, ApiException {
+    // Code for creating invidual directory and files
+    createDirectory(tree, destination);
+    createFiles(tree, namespace, pod, container, srcPath, destination);
+  }
+
+  // Method to create files from directories/files tree in destination
+  private void createFiles(
+      TreeNode node,
+      String namespace,
+      String pod,
+      String container,
+      String srcPath,
+      Path destination)
+      throws IOException, ApiException {
+    if (node == null) {
+      return;
+    }
+    for (TreeNode childNode : node.getChildren()) {
+      if (!childNode.isDir) {
+        // Create file which is under 'node'
+        String filePath = genericPathBuilder(destination.toString(), childNode.name);
+        File f = new File(filePath);
+        if (!f.createNewFile()) {
+          throw new IOException("Failed to create file: " + f);
+        }
+        String modifiedSrcPath = genericPathBuilder(srcPath, childNode.name);
+        try (InputStream is = copyFileFromPod(namespace, pod, modifiedSrcPath);
+            OutputStream fs = new FileOutputStream(f)) {
+          ByteStreams.copy(is, fs);
+          fs.flush();
+        }
+      } else {
+        String newSrcPath = genericPathBuilder(srcPath, childNode.name);
+        // TODO: Change this once the method genericPathBuilder is changed to varargs
+        Path newDestination = Paths.get(destination.toString(), childNode.name);
+        createFiles(childNode, namespace, pod, container, newSrcPath, newDestination);
+      }
+    }
+  }
+
+  // Method to create directories in destination path
+  private void createDirectory(TreeNode node, Path destination) throws IOException {
+    if (node == null) {
+      return;
+    }
+    if (!node.isRoot) {
+      // String directoryPath = genericPathBuilder(destination.toString(), node.name);
+      File f = new File(destination.toString());
+      // File f = new File(directoryPath);
+      if (!f.mkdirs()) {
+        throw new IOException("Failed to create directory: " + f);
+      }
+    }
+    for (TreeNode childNode : node.getChildren()) {
+      if (childNode.isDir) {
+        String path = genericPathBuilder(destination.toString(), childNode.name);
+        Path newPath = Paths.get(path);
+        createDirectory(childNode, newPath);
+      }
+    }
+  }
+
+  // Method to create a tree of files and directory of location inside container
+  private void createDirectoryTree(
+      TreeNode node, String namespace, String pod, String container, String srcPath)
+      throws IOException, ApiException {
+    if (node.isDir) {
+      final Process proc =
+          this.exec(
+              namespace,
+              pod,
+              new String[] {"sh", "-c", "ls -F " + srcPath},
+              container,
+              false,
+              false);
+
+      try (InputStream is = proc.getInputStream();
+          BufferedReader reader = new BufferedReader(new InputStreamReader(is))) {
+        String line = "";
+        while ((line = reader.readLine()) != null) {
+          int len = line.length();
+          // line = stripFileSeparators(line);
+          if (line.charAt(line.length() - 1) == '/') {
+            TreeNode subDirTree =
+                new TreeNode(
+                    true,
+                    line.substring(0, len - 1),
+                    false); // Stripping off '/' in the end of directory
+            String path = genericPathBuilder(srcPath, subDirTree.name);
+            createDirectoryTree(subDirTree, namespace, pod, container, path);
+            node.getChildren().add(subDirTree);
+          } else {
+            node.getChildren().add(new TreeNode(false, line, false));
+          }
+        }
+      }
+    }
+  }
+
+  /*
+  Generic method to create path.
+  TODO: Change this to varargs
+  */
+  private String genericPathBuilder(String first, String second) {
+    StringBuilder pathBuilder = new StringBuilder(first);
+    pathBuilder.append(File.separator);
+    pathBuilder.append(second);
+    return pathBuilder.toString();
+  }
+
   public static void copyFileFromPod(String namespace, String pod, String srcPath, Path dest)
       throws ApiException, IOException {
     Copy c = new Copy();
diff --git a/util/src/main/java/io/kubernetes/client/TreeNode.java b/util/src/main/java/io/kubernetes/client/TreeNode.java
@@ -0,0 +1,69 @@
+/*
+Copyright 2020 The Kubernetes Authors.
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+http://www.apache.org/licenses/LICENSE-2.0
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+*/
+package io.kubernetes.client;
+
+import com.google.gson.Gson;
+import java.util.ArrayList;
+import java.util.List;
+
+/** Helper class to represent directory tree */
+public class TreeNode {
+  boolean isDir;
+  String name;
+  boolean isRoot;
+  List<TreeNode> children;
+
+  public TreeNode(boolean isDir, String name, boolean isRoot) {
+    this.isDir = isDir;
+    this.name = name;
+    this.isRoot = isRoot;
+    this.children = new ArrayList<>();
+  }
+
+  public boolean isDir() {
+    return isDir;
+  }
+
+  public void setDir(boolean dir) {
+    isDir = dir;
+  }
+
+  public String getName() {
+    return name;
+  }
+
+  public void setName(String name) {
+    this.name = name;
+  }
+
+  public boolean isRoot() {
+    return isRoot;
+  }
+
+  public void setRoot(boolean root) {
+    isRoot = root;
+  }
+
+  public List<TreeNode> getChildren() {
+    return children;
+  }
+
+  public void setChildren(List<TreeNode> children) {
+    this.children = children;
+  }
+
+  @Override
+  public String toString() {
+    return new Gson().toJson(this).toString();
+  }
+}
diff --git a/util/src/test/java/io/kubernetes/client/CopyTest.java b/util/src/test/java/io/kubernetes/client/CopyTest.java
@@ -26,13 +26,15 @@
 import io.kubernetes.client.openapi.models.V1ObjectMeta;
 import io.kubernetes.client.openapi.models.V1Pod;
 import io.kubernetes.client.util.ClientBuilder;
+import io.kubernetes.client.util.exception.CopyNotSupportedException;
 import java.io.File;
 import java.io.IOException;
 import java.io.InputStream;
 import java.nio.file.Paths;
 import org.junit.Before;
 import org.junit.Rule;
 import org.junit.Test;
+import org.junit.rules.TemporaryFolder;
 
 /** Tests for the Copy helper class */
 public class CopyTest {
@@ -44,6 +46,7 @@ public class CopyTest {
 
   private static final int PORT = 8089;
   @Rule public WireMockRule wireMockRule = new WireMockRule(PORT);
+  @Rule public TemporaryFolder folder = new TemporaryFolder();
 
   @Before
   public void setup() throws IOException {
@@ -185,4 +188,51 @@ public void run() {
             .withQueryParam("command", equalTo("-c"))
             .withQueryParam("command", equalTo("base64 -d | tar -xmf - -C /")));
   }
+
+  public void testCopyDirectoryFromPod() throws IOException, ApiException, InterruptedException {
+
+    // Create a temp directory
+    File tempFolder = folder.newFolder("destinationFolder");
+
+    Copy copy = new Copy(client);
+
+    wireMockRule.stubFor(
+        get(urlPathEqualTo("/api/v1/namespaces/" + namespace + "/pods/" + podName + "/exec"))
+            .willReturn(
+                aResponse()
+                    .withStatus(404)
+                    .withHeader("Content-Type", "application/json")
+                    .withBody("{}")));
+
+    Thread t =
+        new Thread(
+            new Runnable() {
+              public void run() {
+                try {
+                  copy.copyDirectoryFromPod(
+                      namespace,
+                      podName,
+                      "",
+                      tempFolder.toPath().toString(),
+                      Paths.get("/copied-testDir"));
+                } catch (IOException | ApiException | CopyNotSupportedException ex) {
+                  ex.printStackTrace();
+                }
+              }
+            });
+    t.start();
+    Thread.sleep(2000);
+    t.interrupt();
+
+    verify(
+        getRequestedFor(
+                urlPathEqualTo("/api/v1/namespaces/" + namespace + "/pods/" + podName + "/exec"))
+            .withQueryParam("stdin", equalTo("false"))
+            .withQueryParam("stdout", equalTo("true"))
+            .withQueryParam("stderr", equalTo("true"))
+            .withQueryParam("tty", equalTo("false"))
+            .withQueryParam("command", equalTo("sh"))
+            .withQueryParam("command", equalTo("-c"))
+            .withQueryParam("command", equalTo("tar --version")));
+  }
 }
