Merge pull request #967 from chids/download-repository-archives

Add support for downloading zip and tar archives of repositories.

Author: bitwiseman

src/main/java/org/kohsuke/github/GHRepository.java

@@ -30,6 +30,7 @@
 import edu.umd.cs.findbugs.annotations.NonNull;
 import edu.umd.cs.findbugs.annotations.SuppressFBWarnings;
 import org.apache.commons.lang3.StringUtils;
+import org.kohsuke.github.function.InputStreamFunction;
 
 import java.io.FileNotFoundException;
 import java.io.IOException;
@@ -49,21 +50,15 @@
 import java.util.LinkedHashSet;
 import java.util.List;
 import java.util.Map;
-import java.util.Objects;
 import java.util.Set;
 import java.util.TreeMap;
 import java.util.WeakHashMap;
 
 import javax.annotation.Nonnull;
 
 import static java.util.Arrays.*;
-import static org.kohsuke.github.internal.Previews.ANTIOPE;
-import static org.kohsuke.github.internal.Previews.ANT_MAN;
-import static org.kohsuke.github.internal.Previews.BAPTISTE;
-import static org.kohsuke.github.internal.Previews.FLASH;
-import static org.kohsuke.github.internal.Previews.INERTIA;
-import static org.kohsuke.github.internal.Previews.MERCY;
-import static org.kohsuke.github.internal.Previews.SHADOW_CAT;
+import static java.util.Objects.requireNonNull;
+import static org.kohsuke.github.internal.Previews.*;
 
 /**
  * A repository on GitHub.
@@ -1788,7 +1783,7 @@ public InputStream readBlob(String blobSha) throws IOException {
         return root.createRequest()
                 .withHeader("Accept", "application/vnd.github.v3.raw")
                 .withUrlPath(target)
-                .fetchStream();
+                .fetchStream(Requester::copyInputStream);
     }
 
     /**
@@ -2815,7 +2810,7 @@ public Reader renderMarkdown(String text, MarkdownMode mode) throws IOException
                         .with("mode", mode == null ? null : mode.toString())
                         .with("context", getFullName())
                         .withUrlPath("/markdown")
-                        .fetchStream(),
+                        .fetchStream(Requester::copyInputStream),
                 "UTF-8");
     }
 
@@ -2969,6 +2964,52 @@ public GHTagObject createTag(String tag, String message, String object, String t
                 .wrap(this);
     }
 
+    /**
+     * Streams a zip archive of the repository, optionally at a given <code>ref</code>.
+     *
+     * @param <T>
+     *            the type of result
+     * @param streamFunction
+     *            The {@link InputStreamFunction} that will process the stream
+     * @param ref
+     *            if <code>null</code> the repository's default branch, usually <code>master</code>,
+     * @throws IOException
+     *             The IO exception.
+     * @return the result of reading the stream.
+     */
+    public <T> T readZip(InputStreamFunction<T> streamFunction, String ref) throws IOException {
+        return downloadArchive("zip", ref, streamFunction);
+    }
+
+    /**
+     * Streams a tar archive of the repository, optionally at a given <code>ref</code>.
+     *
+     * @param <T>
+     *            the type of result
+     * @param streamFunction
+     *            The {@link InputStreamFunction} that will process the stream
+     * @param ref
+     *            if <code>null</code> the repository's default branch, usually <code>master</code>,
+     * @throws IOException
+     *             The IO exception.
+     * @return the result of reading the stream.
+     */
+    public <T> T readTar(InputStreamFunction<T> streamFunction, String ref) throws IOException {
+        return downloadArchive("tar", ref, streamFunction);
+    }
+
+    private <T> T downloadArchive(@Nonnull String type,
+            @CheckForNull String ref,
+            @Nonnull InputStreamFunction<T> streamFunction) throws IOException {
+        requireNonNull(streamFunction, "Sink must not be null");
+        String tailUrl = getApiTailUrl(type + "ball");
+        if (ref != null) {
+            tailUrl += "/" + ref;
+        }
+        final Requester builder = root.createRequest().method("GET").withUrlPath(tailUrl);
+        return builder.fetchStream(streamFunction);
+    }
+
     /**
      * Populate this object.
      *
@@ -2980,7 +3021,7 @@ void populate() throws IOException {
             return; // can't populate if the root is offline
         }
 
-        final URL url = Objects.requireNonNull(getUrl(), "Missing instance URL!");
+        final URL url = requireNonNull(getUrl(), "Missing instance URL!");
 
         try {
             // IMPORTANT: the url for repository records does not reliably point to the API url.

src/main/java/org/kohsuke/github/GitHub.java

@@ -1278,7 +1278,7 @@ public Reader renderMarkdown(String text) throws IOException {
                         .with(new ByteArrayInputStream(text.getBytes("UTF-8")))
                         .contentType("text/plain;charset=UTF-8")
                         .withUrlPath("/markdown/raw")
-                        .fetchStream(),
+                        .fetchStream(Requester::copyInputStream),
                 "UTF-8");
     }
 

src/main/java/org/kohsuke/github/GitHubResponse.java

@@ -4,6 +4,7 @@
 import com.fasterxml.jackson.databind.InjectableValues;
 import com.fasterxml.jackson.databind.JsonMappingException;
 import org.apache.commons.io.IOUtils;
+import org.kohsuke.github.function.FunctionThrows;
 
 import java.io.Closeable;
 import java.io.IOException;
@@ -194,24 +195,11 @@ public T body() {
     /**
      * Represents a supplier of results that can throw.
      *
-     * <p>
-     * This is a <a href="package-summary.html">functional interface</a> whose functional method is
-     * {@link #apply(ResponseInfo)}.
-     *
      * @param <T>
      *            the type of results supplied by this supplier
      */
     @FunctionalInterface
-    interface BodyHandler<T> {
-
-        /**
-         * Gets a result.
-         *
-         * @return a result
-         * @throws IOException
-         *             if an I/O Exception occurs.
-         */
-        T apply(ResponseInfo input) throws IOException;
+    interface BodyHandler<T> extends FunctionThrows<ResponseInfo, T, IOException> {
     }
 
     /**

src/main/java/org/kohsuke/github/Requester.java

@@ -23,7 +23,9 @@
  */
 package org.kohsuke.github;
 
+import edu.umd.cs.findbugs.annotations.NonNull;
 import org.apache.commons.io.IOUtils;
+import org.kohsuke.github.function.InputStreamFunction;
 
 import java.io.ByteArrayInputStream;
 import java.io.IOException;
@@ -106,15 +108,31 @@ public int fetchHttpStatusCode() throws IOException {
      * Response input stream. There are scenarios where direct stream reading is needed, however it is better to use
      * {@link #fetch(Class)} where possible.
      *
-     * @return the input stream
      * @throws IOException
      *             the io exception
      */
-    public InputStream fetchStream() throws IOException {
-        return client
-                .sendRequest(this,
-                        (responseInfo) -> new ByteArrayInputStream(IOUtils.toByteArray(responseInfo.bodyStream())))
-                .body();
+    public <T> T fetchStream(@Nonnull InputStreamFunction<T> handler) throws IOException {
+        return client.sendRequest(this, (responseInfo) -> handler.apply(responseInfo.bodyStream())).body();
+    }
+
+    /**
+     * Helper function to make it easy to pull streams.
+     *
+     * Copies an input stream to an in-memory input stream. The performance on this is not great but
+     * {@link GitHubResponse.ResponseInfo#bodyStream()} is closed at the end of every call to
+     * {@link GitHubClient#sendRequest(GitHubRequest, GitHubResponse.BodyHandler)}, so any reads to the original input
+     * stream must be completed before then. There are a number of deprecated methods that return {@link InputStream}.
+     * This method keeps all of them using the same code path.
+     *
+     * @param inputStream
+     *            the input stream to be copied
+     * @return an in-memory copy of the passed input stream
+     * @throws IOException
+     *             if an error occurs while copying the stream
+     */
+    @NonNull
+    public static InputStream copyInputStream(InputStream inputStream) throws IOException {
+        return new ByteArrayInputStream(IOUtils.toByteArray(inputStream));
     }
 
     /**

src/main/java/org/kohsuke/github/function/FunctionThrows.java

@@ -0,0 +1,25 @@
+package org.kohsuke.github.function;
+
+/**
+ * A functional interface, equivalent to {@link java.util.function.Function} but that allows throwing {@link Throwable}
+ *
+ * @param <T>
+ *            the type of input
+ * @param <R>
+ *            the type of output
+ * @param <E>
+ *            the type of error
+ */
+@FunctionalInterface
+public interface FunctionThrows<T, R, E extends Throwable> {
+    /**
+     * Apply r.
+     *
+     * @param input
+     *            the input
+     * @return the r
+     * @throws E
+     *             the e
+     */
+    R apply(T input) throws E;
+}

src/main/java/org/kohsuke/github/function/InputStreamFunction.java

@@ -0,0 +1,14 @@
+package org.kohsuke.github.function;
+
+import java.io.IOException;
+import java.io.InputStream;
+
+/**
+ * A functional interface, equivalent to {@link java.util.function.Function} but that allows throwing {@link Throwable}
+ *
+ * @param <R>
+ *            the type to of object to be returned
+ */
+@FunctionalInterface
+public interface InputStreamFunction<R> extends FunctionThrows<InputStream, R, IOException> {
+}

src/test/java/org/kohsuke/github/GHRepositoryTest.java

@@ -4,8 +4,10 @@
 import org.apache.commons.io.IOUtils;
 import org.junit.Test;
 
+import java.io.ByteArrayInputStream;
 import java.io.FileNotFoundException;
 import java.io.IOException;
+import java.io.InputStream;
 import java.net.URL;
 import java.util.ArrayList;
 import java.util.Date;
@@ -29,6 +31,20 @@ private GHRepository getRepository(GitHub gitHub) throws IOException {
         return gitHub.getOrganization("hub4j-test-org").getRepository("github-api");
     }
 
+    @Test
+    public void testZipball() throws IOException {
+        getTempRepository().readZip((InputStream inputstream) -> {
+            return new ByteArrayInputStream(IOUtils.toByteArray(inputstream));
+        }, null);
+    }
+
+    @Test
+    public void testTarball() throws IOException {
+        getTempRepository().readTar((InputStream inputstream) -> {
+            return new ByteArrayInputStream(IOUtils.toByteArray(inputstream));
+        }, null);
+    }
+
     @Test
     public void testGetters() throws IOException {
         GHRepository r = getTempRepository();