jenkins-client-217 - Added ability to stream logs and retrieve chunks

ReleaseNotes.md

@@ -2,6 +2,20 @@
 
 ## Release 0.3.8 (NOT RELEASED YET)
 
+ * [Fixed Issue 217][issue- 217]
+
+   Added new api for streaming build logs
+
+   ```java
+    BuildWithDetails build = ...
+    // Get log with initial offset
+    int offset = 40;
+    String output = build.getConsoleOutputText(offset);
+    // Stream logs (when build is in progress)
+    BuildConsoleStreamListener buildListener = ...
+    build.streamConsoleOutput(listener, 3, 420);
+   ```
+
  * [Fixed Issue 222][issue-222]
 
    Fixed WARNING during build.

jenkins-client/src/main/java/com/offbytwo/jenkins/client/JenkinsHttpClient.java

@@ -6,22 +6,25 @@
 
 package com.offbytwo.jenkins.client;
 
-import static com.fasterxml.jackson.databind.DeserializationFeature.FAIL_ON_UNKNOWN_PROPERTIES;
-import static org.apache.commons.lang.StringUtils.isNotBlank;
-
-import java.io.IOException;
-import java.io.InputStream;
-import java.net.URI;
-import java.util.List;
-import java.util.Map;
-
+import com.fasterxml.jackson.databind.ObjectMapper;
+import com.google.common.collect.Lists;
+import com.google.common.io.ByteStreams;
+import com.offbytwo.jenkins.client.util.EncodingUtils;
+import com.offbytwo.jenkins.client.util.RequestReleasingInputStream;
+import com.offbytwo.jenkins.client.validator.HttpResponseValidator;
+import com.offbytwo.jenkins.model.BaseModel;
+import com.offbytwo.jenkins.model.Crumb;
+import com.offbytwo.jenkins.model.ExtractHeader;
+import net.sf.json.JSONObject;
 import org.apache.commons.io.IOUtils;
 import org.apache.commons.lang.StringUtils;
 import org.apache.http.Header;
 import org.apache.http.HttpResponse;
+import org.apache.http.NameValuePair;
 import org.apache.http.auth.AuthScope;
 import org.apache.http.auth.UsernamePasswordCredentials;
 import org.apache.http.client.CredentialsProvider;
+import org.apache.http.client.entity.UrlEncodedFormEntity;
 import org.apache.http.client.methods.HttpGet;
 import org.apache.http.client.methods.HttpPost;
 import org.apache.http.client.methods.HttpRequestBase;
@@ -38,18 +41,16 @@
 import org.slf4j.Logger;
 import org.slf4j.LoggerFactory;
 
-import com.fasterxml.jackson.databind.ObjectMapper;
-import com.google.common.collect.Lists;
-import com.google.common.io.ByteStreams;
-import com.offbytwo.jenkins.client.util.EncodingUtils;
-import com.offbytwo.jenkins.client.util.RequestReleasingInputStream;
-//import com.offbytwo.jenkins.client.util.HttpResponseContentExtractor;
-import com.offbytwo.jenkins.client.validator.HttpResponseValidator;
-import com.offbytwo.jenkins.model.BaseModel;
-import com.offbytwo.jenkins.model.Crumb;
-import com.offbytwo.jenkins.model.ExtractHeader;
+import java.io.IOException;
+import java.io.InputStream;
+import java.net.URI;
+import java.util.List;
+import java.util.Map;
 
-import net.sf.json.JSONObject;
+import static com.fasterxml.jackson.databind.DeserializationFeature.FAIL_ON_UNKNOWN_PROPERTIES;
+import static org.apache.commons.lang.StringUtils.isNotBlank;
+
+//import com.offbytwo.jenkins.client.util.HttpResponseContentExtractor;
 
 public class JenkinsHttpClient {
     private final Logger LOGGER = LoggerFactory.getLogger(getClass());
@@ -312,6 +313,39 @@ public void post_form(String path, Map<String, String> data, boolean crumbFlag)
         }
     }
 
+
+    /**
+     * Perform a POST request using form url encoding and return HttpResponse object
+     * This method is not performing validation and can be used for more generic queries to jenkins.
+     *
+     * @param path
+     *            path to request, can be relative or absolute
+     * @param data
+     *            data to post
+     * @throws IOException,
+     *             HttpResponseException
+     */
+    public HttpResponse post_form_with_result(String path, List<NameValuePair> data, boolean crumbFlag) throws IOException {
+        HttpPost request;
+        if (data != null) {
+            UrlEncodedFormEntity urlEncodedFormEntity = new UrlEncodedFormEntity(data);
+            request = new HttpPost(noapi(path));
+            request.setEntity(urlEncodedFormEntity);
+        } else {
+            request = new HttpPost(noapi(path));
+        }
+
+        if (crumbFlag == true) {
+            Crumb crumb = get("/crumbIssuer", Crumb.class);
+            if (crumb != null) {
+                request.addHeader(new BasicHeader(crumb.getCrumbRequestField(), crumb.getCrumb()));
+            }
+        }
+        HttpResponse response = client.execute(request, localContext);
+        getJenkinsVersionFromHeader(response);
+        return response;
+    }
+
     /**
      * Perform a POST request of XML (instead of using json mapper) and return a
      * string rendering of the response entity.

jenkins-client/src/main/java/com/offbytwo/jenkins/helper/BuildConsoleStreamListener.java

@@ -0,0 +1,19 @@
+package com.offbytwo.jenkins.helper;
+
+/**
+ * Listener interface used to obtain build console logs
+ */
+public interface BuildConsoleStreamListener {
+
+    /**
+     * Called by api when new log data available.
+     *
+     * @param newLogChunk - string containing latest chunk of logs
+     */
+    void onData(String newLogChunk);
+
+    /**
+     * Called when streaming console output is finished
+     */
+     void finished();
+}

jenkins-client/src/main/java/com/offbytwo/jenkins/model/BuildWithDetails.java

@@ -9,12 +9,26 @@
 import com.google.common.base.Predicate;
 import com.google.common.base.Strings;
 import com.google.common.collect.ImmutableMap;
+import com.offbytwo.jenkins.helper.BuildConsoleStreamListener;
+import org.apache.http.Header;
+import org.apache.http.HttpResponse;
+import org.apache.http.NameValuePair;
+import org.apache.http.message.BasicNameValuePair;
+import org.apache.http.util.EntityUtils;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
 
 import java.io.IOException;
 import java.io.InputStream;
 import java.net.URI;
 import java.net.URISyntaxException;
-import java.util.*;
+import java.util.ArrayList;
+import java.util.Collection;
+import java.util.Collections;
+import java.util.HashMap;
+import java.util.List;
+import java.util.Map;
+import java.util.Objects;
 
 import static com.google.common.collect.Collections2.filter;
 
@@ -25,6 +39,11 @@
  */
 public class BuildWithDetails extends Build {
 
+    private final Logger LOGGER = LoggerFactory.getLogger(getClass());
+
+    public final static String TEXT_SIZE_HEADER = "x-text-size";
+    public final static String MORE_DATA_HEADER = "x-more-data";
+
     /**
      * This will be returned by the API in cases where the build has never run.
      * For example {@link Build#BUILD_HAS_NEVER_RUN}
@@ -350,24 +369,104 @@ public boolean apply(Map<String, Object> action) {
     }
 
     /**
-     * @return The console output of the build. The line separation is done by
+     * @return The full console output of the build. The line separation is done by
      *         {@code CR+LF}.
+     *
+     * @see streamConsoleOutput method for obtaining logs for running build
+     *
      * @throws IOException in case of a failure.
      */
     public String getConsoleOutputText() throws IOException {
         return client.get(getUrl() + "/logText/progressiveText");
     }
 
     /**
-     * The console output with HTML.
-     * 
+     * The full console output with HTML.
+     *
+     * @see streamConsoleOutput method for obtaining logs for running build
+     *
      * @return The console output as HTML.
      * @throws IOException in case of an error.
      */
     public String getConsoleOutputHtml() throws IOException {
         return client.get(getUrl() + "/logText/progressiveHtml");
     }
 
+
+    /**
+     * Stream build console output log as text using BuildConsoleStreamListener
+     * Method can be used to asynchronously obtain logs for running build.
+     *
+     * @param listener interface used to asynchronously obtain logs
+     * @param poolingInterval interval (seconds) used to pool jenkins for logs
+     * @param poolingTimeout pooling timeout (seconds) used to break pooling in case build stuck
+     *
+     */
+    public void streamConsoleOutput(final BuildConsoleStreamListener listener, final int poolingInterval, final int poolingTimeout) throws InterruptedException, IOException {
+        // Calculate start and timeout
+        final long startTime = System.currentTimeMillis();
+        final long timeoutTime = startTime + (poolingTimeout * 1000);
+
+        int bufferOffset = 0;
+        while (true) {
+            Thread.sleep(poolingInterval * 1000);
+
+            ConsoleLog consoleLog = null;
+            consoleLog = getConsoleOutputText(bufferOffset);
+            String logString = consoleLog.getConsoleLog();
+            if (logString != null && !logString.isEmpty()) {
+                listener.onData(logString);
+            }
+            if (consoleLog.getHasMoreData()) {
+                bufferOffset = consoleLog.getCurrentBufferSize();
+            } else {
+                listener.finished();
+                break;
+            }
+            long currentTime = System.currentTimeMillis();
+
+            if (currentTime > timeoutTime) {
+                LOGGER.warn("Pooling for build {0} for {2} timeout! Check if job stuck in jenkins",
+                        BuildWithDetails.this.getDisplayName(), BuildWithDetails.this.getNumber());
+                break;
+            }
+        }
+    }
+
+    /**
+     * Get build console output log as text.
+     * Use this method to periodically obtain logs from jenkins and skip chunks that were already received
+     *
+     * @param bufferOffset offset in console lo
+     * @return ConsoleLog object containing console output of the build. The line separation is done by
+     * {@code CR+LF}.
+     * @throws IOException in case of a failure.
+     */
+    public ConsoleLog getConsoleOutputText(int bufferOffset) throws IOException {
+        List<NameValuePair> formData = new ArrayList<>();
+        formData.add(new BasicNameValuePair("start", Integer.toString(bufferOffset)));
+        String path = getUrl() + "logText/progressiveText";
+        HttpResponse httpResponse = client.post_form_with_result(path, formData, false);
+
+        Header moreDataHeader = httpResponse.getFirstHeader(MORE_DATA_HEADER);
+        Header textSizeHeader = httpResponse.getFirstHeader(TEXT_SIZE_HEADER);
+        String response = EntityUtils.toString(httpResponse.getEntity());
+        boolean hasMoreData = false;
+        if (moreDataHeader != null) {
+            hasMoreData = Boolean.TRUE.toString().equals(moreDataHeader.getValue());
+        }
+        Integer currentBufferSize = bufferOffset;
+        if (textSizeHeader != null) {
+            try {
+                currentBufferSize = Integer.parseInt(textSizeHeader.getValue());
+            } catch (NumberFormatException e) {
+                LOGGER.warn("Cannot parse buffer size for job {0} build {1}. Using current offset!", this.getDisplayName(), this.getNumber());
+            }
+        }
+        return new ConsoleLog(response, hasMoreData, currentBufferSize);
+    }
+
+
     public BuildChangeSet getChangeSet() {
         return changeSet;
     }

jenkins-client/src/main/java/com/offbytwo/jenkins/model/ConsoleLog.java

@@ -0,0 +1,41 @@
+package com.offbytwo.jenkins.model;
+
+/**
+ * Represents build console log
+ */
+public class ConsoleLog {
+
+    private String consoleLog;
+    private Boolean hasMoreData;
+    private Integer currentBufferSize;
+
+    public ConsoleLog(String consoleLog, Boolean hasMoreData, Integer currentBufferSize) {
+        this.consoleLog = consoleLog;
+        this.hasMoreData = hasMoreData;
+        this.currentBufferSize = currentBufferSize;
+    }
+
+    public String getConsoleLog() {
+        return consoleLog;
+    }
+
+    public void setConsoleLog(String consoleLog) {
+        this.consoleLog = consoleLog;
+    }
+
+    public Boolean getHasMoreData() {
+        return hasMoreData;
+    }
+
+    public void setHasMoreData(Boolean hasMoreData) {
+        this.hasMoreData = hasMoreData;
+    }
+
+    public Integer getCurrentBufferSize() {
+        return currentBufferSize;
+    }
+
+    public void setCurrentBufferSize(Integer currentBufferSize) {
+        this.currentBufferSize = currentBufferSize;
+    }
+}