From 4c59d43103c57e4e802eaf701a94c9c5439ce1c3 Mon Sep 17 00:00:00 2001 From: Wojciech Trocki Date: Mon, 19 Dec 2016 13:14:12 +0000 Subject: [PATCH] jenkins-client-217 - Added ability to stream logs and retrieve chunks for logs --- ReleaseNotes.md | 14 ++ .../jenkins/client/JenkinsHttpClient.java | 74 ++++++++--- .../helper/BuildConsoleStreamListener.java | 19 +++ .../jenkins/model/BuildWithDetails.java | 107 ++++++++++++++- .../offbytwo/jenkins/model/ConsoleLog.java | 41 ++++++ .../jenkins/model/BuildWithDetailsTest.java | 122 ++++++++++++++++++ 6 files changed, 353 insertions(+), 24 deletions(-) create mode 100644 jenkins-client/src/main/java/com/offbytwo/jenkins/helper/BuildConsoleStreamListener.java create mode 100644 jenkins-client/src/main/java/com/offbytwo/jenkins/model/ConsoleLog.java create mode 100644 jenkins-client/src/test/java/com/offbytwo/jenkins/model/BuildWithDetailsTest.java diff --git a/ReleaseNotes.md b/ReleaseNotes.md index 2f4d1d80..a5bae98c 100644 --- a/ReleaseNotes.md +++ b/ReleaseNotes.md @@ -2,6 +2,20 @@ ## Release 0.3.8 (NOT RELEASED YET) + * [Fixed Issue 217][issue- 217] + + Added new api for streaming build logs + + ```java + BuildWithDetails build = ... + // Get log with initial offset + int offset = 40; + String output = build.getConsoleOutputText(offset); + // Stream logs (when build is in progress) + BuildConsoleStreamListener buildListener = ... + build.streamConsoleOutput(listener, 3, 420); + ``` + * [Fixed Issue 222][issue-222] Fixed WARNING during build. diff --git a/jenkins-client/src/main/java/com/offbytwo/jenkins/client/JenkinsHttpClient.java b/jenkins-client/src/main/java/com/offbytwo/jenkins/client/JenkinsHttpClient.java index df58360f..40abca9a 100644 --- a/jenkins-client/src/main/java/com/offbytwo/jenkins/client/JenkinsHttpClient.java +++ b/jenkins-client/src/main/java/com/offbytwo/jenkins/client/JenkinsHttpClient.java @@ -6,22 +6,25 @@ package com.offbytwo.jenkins.client; -import static com.fasterxml.jackson.databind.DeserializationFeature.FAIL_ON_UNKNOWN_PROPERTIES; -import static org.apache.commons.lang.StringUtils.isNotBlank; - -import java.io.IOException; -import java.io.InputStream; -import java.net.URI; -import java.util.List; -import java.util.Map; - +import com.fasterxml.jackson.databind.ObjectMapper; +import com.google.common.collect.Lists; +import com.google.common.io.ByteStreams; +import com.offbytwo.jenkins.client.util.EncodingUtils; +import com.offbytwo.jenkins.client.util.RequestReleasingInputStream; +import com.offbytwo.jenkins.client.validator.HttpResponseValidator; +import com.offbytwo.jenkins.model.BaseModel; +import com.offbytwo.jenkins.model.Crumb; +import com.offbytwo.jenkins.model.ExtractHeader; +import net.sf.json.JSONObject; import org.apache.commons.io.IOUtils; import org.apache.commons.lang.StringUtils; import org.apache.http.Header; import org.apache.http.HttpResponse; +import org.apache.http.NameValuePair; import org.apache.http.auth.AuthScope; import org.apache.http.auth.UsernamePasswordCredentials; import org.apache.http.client.CredentialsProvider; +import org.apache.http.client.entity.UrlEncodedFormEntity; import org.apache.http.client.methods.HttpGet; import org.apache.http.client.methods.HttpPost; import org.apache.http.client.methods.HttpRequestBase; @@ -38,18 +41,16 @@ import org.slf4j.Logger; import org.slf4j.LoggerFactory; -import com.fasterxml.jackson.databind.ObjectMapper; -import com.google.common.collect.Lists; -import com.google.common.io.ByteStreams; -import com.offbytwo.jenkins.client.util.EncodingUtils; -import com.offbytwo.jenkins.client.util.RequestReleasingInputStream; -//import com.offbytwo.jenkins.client.util.HttpResponseContentExtractor; -import com.offbytwo.jenkins.client.validator.HttpResponseValidator; -import com.offbytwo.jenkins.model.BaseModel; -import com.offbytwo.jenkins.model.Crumb; -import com.offbytwo.jenkins.model.ExtractHeader; +import java.io.IOException; +import java.io.InputStream; +import java.net.URI; +import java.util.List; +import java.util.Map; -import net.sf.json.JSONObject; +import static com.fasterxml.jackson.databind.DeserializationFeature.FAIL_ON_UNKNOWN_PROPERTIES; +import static org.apache.commons.lang.StringUtils.isNotBlank; + +//import com.offbytwo.jenkins.client.util.HttpResponseContentExtractor; public class JenkinsHttpClient { private final Logger LOGGER = LoggerFactory.getLogger(getClass()); @@ -312,6 +313,39 @@ public void post_form(String path, Map data, boolean crumbFlag) } } + + /** + * Perform a POST request using form url encoding and return HttpResponse object + * This method is not performing validation and can be used for more generic queries to jenkins. + * + * @param path + * path to request, can be relative or absolute + * @param data + * data to post + * @throws IOException, + * HttpResponseException + */ + public HttpResponse post_form_with_result(String path, List data, boolean crumbFlag) throws IOException { + HttpPost request; + if (data != null) { + UrlEncodedFormEntity urlEncodedFormEntity = new UrlEncodedFormEntity(data); + request = new HttpPost(noapi(path)); + request.setEntity(urlEncodedFormEntity); + } else { + request = new HttpPost(noapi(path)); + } + + if (crumbFlag == true) { + Crumb crumb = get("/crumbIssuer", Crumb.class); + if (crumb != null) { + request.addHeader(new BasicHeader(crumb.getCrumbRequestField(), crumb.getCrumb())); + } + } + HttpResponse response = client.execute(request, localContext); + getJenkinsVersionFromHeader(response); + return response; + } + /** * Perform a POST request of XML (instead of using json mapper) and return a * string rendering of the response entity. diff --git a/jenkins-client/src/main/java/com/offbytwo/jenkins/helper/BuildConsoleStreamListener.java b/jenkins-client/src/main/java/com/offbytwo/jenkins/helper/BuildConsoleStreamListener.java new file mode 100644 index 00000000..41671fbb --- /dev/null +++ b/jenkins-client/src/main/java/com/offbytwo/jenkins/helper/BuildConsoleStreamListener.java @@ -0,0 +1,19 @@ +package com.offbytwo.jenkins.helper; + +/** + * Listener interface used to obtain build console logs + */ +public interface BuildConsoleStreamListener { + + /** + * Called by api when new log data available. + * + * @param newLogChunk - string containing latest chunk of logs + */ + void onData(String newLogChunk); + + /** + * Called when streaming console output is finished + */ + void finished(); +} diff --git a/jenkins-client/src/main/java/com/offbytwo/jenkins/model/BuildWithDetails.java b/jenkins-client/src/main/java/com/offbytwo/jenkins/model/BuildWithDetails.java index 4fbf463c..9812a00d 100644 --- a/jenkins-client/src/main/java/com/offbytwo/jenkins/model/BuildWithDetails.java +++ b/jenkins-client/src/main/java/com/offbytwo/jenkins/model/BuildWithDetails.java @@ -9,12 +9,26 @@ import com.google.common.base.Predicate; import com.google.common.base.Strings; import com.google.common.collect.ImmutableMap; +import com.offbytwo.jenkins.helper.BuildConsoleStreamListener; +import org.apache.http.Header; +import org.apache.http.HttpResponse; +import org.apache.http.NameValuePair; +import org.apache.http.message.BasicNameValuePair; +import org.apache.http.util.EntityUtils; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; import java.io.IOException; import java.io.InputStream; import java.net.URI; import java.net.URISyntaxException; -import java.util.*; +import java.util.ArrayList; +import java.util.Collection; +import java.util.Collections; +import java.util.HashMap; +import java.util.List; +import java.util.Map; +import java.util.Objects; import static com.google.common.collect.Collections2.filter; @@ -25,6 +39,11 @@ */ public class BuildWithDetails extends Build { + private final Logger LOGGER = LoggerFactory.getLogger(getClass()); + + public final static String TEXT_SIZE_HEADER = "x-text-size"; + public final static String MORE_DATA_HEADER = "x-more-data"; + /** * This will be returned by the API in cases where the build has never run. * For example {@link Build#BUILD_HAS_NEVER_RUN} @@ -350,8 +369,11 @@ public boolean apply(Map action) { } /** - * @return The console output of the build. The line separation is done by + * @return The full console output of the build. The line separation is done by * {@code CR+LF}. + * + * @see streamConsoleOutput method for obtaining logs for running build + * * @throws IOException in case of a failure. */ public String getConsoleOutputText() throws IOException { @@ -359,8 +381,10 @@ public String getConsoleOutputText() throws IOException { } /** - * The console output with HTML. - * + * The full console output with HTML. + * + * @see streamConsoleOutput method for obtaining logs for running build + * * @return The console output as HTML. * @throws IOException in case of an error. */ @@ -368,6 +392,81 @@ public String getConsoleOutputHtml() throws IOException { return client.get(getUrl() + "/logText/progressiveHtml"); } + + /** + * Stream build console output log as text using BuildConsoleStreamListener + * Method can be used to asynchronously obtain logs for running build. + * + * @param listener interface used to asynchronously obtain logs + * @param poolingInterval interval (seconds) used to pool jenkins for logs + * @param poolingTimeout pooling timeout (seconds) used to break pooling in case build stuck + * + */ + public void streamConsoleOutput(final BuildConsoleStreamListener listener, final int poolingInterval, final int poolingTimeout) throws InterruptedException, IOException { + // Calculate start and timeout + final long startTime = System.currentTimeMillis(); + final long timeoutTime = startTime + (poolingTimeout * 1000); + + int bufferOffset = 0; + while (true) { + Thread.sleep(poolingInterval * 1000); + + ConsoleLog consoleLog = null; + consoleLog = getConsoleOutputText(bufferOffset); + String logString = consoleLog.getConsoleLog(); + if (logString != null && !logString.isEmpty()) { + listener.onData(logString); + } + if (consoleLog.getHasMoreData()) { + bufferOffset = consoleLog.getCurrentBufferSize(); + } else { + listener.finished(); + break; + } + long currentTime = System.currentTimeMillis(); + + if (currentTime > timeoutTime) { + LOGGER.warn("Pooling for build {0} for {2} timeout! Check if job stuck in jenkins", + BuildWithDetails.this.getDisplayName(), BuildWithDetails.this.getNumber()); + break; + } + } + } + + /** + * Get build console output log as text. + * Use this method to periodically obtain logs from jenkins and skip chunks that were already received + * + * @param bufferOffset offset in console lo + * @return ConsoleLog object containing console output of the build. The line separation is done by + * {@code CR+LF}. + * @throws IOException in case of a failure. + */ + public ConsoleLog getConsoleOutputText(int bufferOffset) throws IOException { + List formData = new ArrayList<>(); + formData.add(new BasicNameValuePair("start", Integer.toString(bufferOffset))); + String path = getUrl() + "logText/progressiveText"; + HttpResponse httpResponse = client.post_form_with_result(path, formData, false); + + Header moreDataHeader = httpResponse.getFirstHeader(MORE_DATA_HEADER); + Header textSizeHeader = httpResponse.getFirstHeader(TEXT_SIZE_HEADER); + String response = EntityUtils.toString(httpResponse.getEntity()); + boolean hasMoreData = false; + if (moreDataHeader != null) { + hasMoreData = Boolean.TRUE.toString().equals(moreDataHeader.getValue()); + } + Integer currentBufferSize = bufferOffset; + if (textSizeHeader != null) { + try { + currentBufferSize = Integer.parseInt(textSizeHeader.getValue()); + } catch (NumberFormatException e) { + LOGGER.warn("Cannot parse buffer size for job {0} build {1}. Using current offset!", this.getDisplayName(), this.getNumber()); + } + } + return new ConsoleLog(response, hasMoreData, currentBufferSize); + } + + public BuildChangeSet getChangeSet() { return changeSet; } diff --git a/jenkins-client/src/main/java/com/offbytwo/jenkins/model/ConsoleLog.java b/jenkins-client/src/main/java/com/offbytwo/jenkins/model/ConsoleLog.java new file mode 100644 index 00000000..59ef5c6d --- /dev/null +++ b/jenkins-client/src/main/java/com/offbytwo/jenkins/model/ConsoleLog.java @@ -0,0 +1,41 @@ +package com.offbytwo.jenkins.model; + +/** + * Represents build console log + */ +public class ConsoleLog { + + private String consoleLog; + private Boolean hasMoreData; + private Integer currentBufferSize; + + public ConsoleLog(String consoleLog, Boolean hasMoreData, Integer currentBufferSize) { + this.consoleLog = consoleLog; + this.hasMoreData = hasMoreData; + this.currentBufferSize = currentBufferSize; + } + + public String getConsoleLog() { + return consoleLog; + } + + public void setConsoleLog(String consoleLog) { + this.consoleLog = consoleLog; + } + + public Boolean getHasMoreData() { + return hasMoreData; + } + + public void setHasMoreData(Boolean hasMoreData) { + this.hasMoreData = hasMoreData; + } + + public Integer getCurrentBufferSize() { + return currentBufferSize; + } + + public void setCurrentBufferSize(Integer currentBufferSize) { + this.currentBufferSize = currentBufferSize; + } +} diff --git a/jenkins-client/src/test/java/com/offbytwo/jenkins/model/BuildWithDetailsTest.java b/jenkins-client/src/test/java/com/offbytwo/jenkins/model/BuildWithDetailsTest.java new file mode 100644 index 00000000..af25f266 --- /dev/null +++ b/jenkins-client/src/test/java/com/offbytwo/jenkins/model/BuildWithDetailsTest.java @@ -0,0 +1,122 @@ +package com.offbytwo.jenkins.model; + +import com.offbytwo.jenkins.BaseUnitTest; +import com.offbytwo.jenkins.client.JenkinsHttpClient; +import com.offbytwo.jenkins.helper.BuildConsoleStreamListener; +import org.apache.http.HttpResponse; +import org.apache.http.NameValuePair; +import org.apache.http.entity.StringEntity; +import org.apache.http.message.BasicHeader; +import org.junit.Before; +import org.junit.Test; + +import java.io.IOException; + +import static org.assertj.core.api.Assertions.assertThat; +import static org.assertj.core.api.Assertions.fail; +import static org.mockito.BDDMockito.given; +import static org.mockito.Matchers.anyBoolean; +import static org.mockito.Matchers.anyListOf; +import static org.mockito.Matchers.anyString; +import static org.mockito.Mockito.mock; + +/** + */ +public class BuildWithDetailsTest extends BaseUnitTest { + + private JenkinsHttpClient client; + private BuildWithDetails buildWithDetails; + + @Before + public void setUp() { + client = mock(JenkinsHttpClient.class); + buildWithDetails = givenBuild(); + } + + private BuildWithDetails givenBuild() { + BuildWithDetails buildWithDetails = new BuildWithDetails(); + buildWithDetails.setClient(client); + return buildWithDetails; + } + + @Test + public void getBuildLogWithBuffer() { + try { + HttpResponse response = mock(HttpResponse.class); + String text = "test test test"; + int textLength = text.length(); + given(response.getEntity()).willReturn(new StringEntity(text)); + BasicHeader moreDataHeader = new BasicHeader(BuildWithDetails.MORE_DATA_HEADER, "false"); + BasicHeader textSizeHeader = new BasicHeader(BuildWithDetails.TEXT_SIZE_HEADER, Integer.toString(textLength)); + given(response.getFirstHeader(BuildWithDetails.MORE_DATA_HEADER)).willReturn(moreDataHeader); + given(response.getFirstHeader(BuildWithDetails.TEXT_SIZE_HEADER)).willReturn(textSizeHeader); + given(client.post_form_with_result(anyString(),anyListOf(NameValuePair.class),anyBoolean())).willReturn(response); + ConsoleLog consoleOutputText = buildWithDetails.getConsoleOutputText(500); + assertThat(consoleOutputText.getConsoleLog()).isEqualTo(text); + assertThat(consoleOutputText.getCurrentBufferSize()).isEqualTo(textLength); + assertThat(consoleOutputText.getHasMoreData()).isFalse(); + } catch (IOException e) { + fail("Should not return exception",e); + } + } + + + @Test + public void poolBuildLog() throws InterruptedException { + try { + HttpResponse response = mock(HttpResponse.class); + final String text = "test test test"; + int textLength = text.length(); + given(response.getEntity()).willReturn(new StringEntity(text)); + BasicHeader moreDataHeader = new BasicHeader(BuildWithDetails.MORE_DATA_HEADER, "false"); + BasicHeader textSizeHeader = new BasicHeader(BuildWithDetails.TEXT_SIZE_HEADER, Integer.toString(textLength)); + given(response.getFirstHeader(BuildWithDetails.MORE_DATA_HEADER)).willReturn(moreDataHeader); + given(response.getFirstHeader(BuildWithDetails.TEXT_SIZE_HEADER)).willReturn(textSizeHeader); + given(client.post_form_with_result(anyString(),anyListOf(NameValuePair.class),anyBoolean())).willReturn(response); + final StringBuffer buffer=new StringBuffer(); + buildWithDetails.streamConsoleOutput(new BuildConsoleStreamListener() { + @Override + public void onData(String newLogChunk) { + assertThat(newLogChunk).isEqualTo(text); + buffer.append(text); + } + + @Override + public void finished() { + assertThat(buffer.toString()).isEqualTo(text); + } + },1,2); + } catch (IOException e) { + fail("Should not return exception",e); + } + } + + @Test + public void poolBuildLogShouldTimeout() throws InterruptedException { + try { + HttpResponse response = mock(HttpResponse.class); + final String text = "test test test"; + int textLength = text.length(); + given(response.getEntity()).willReturn(new StringEntity(text)); + BasicHeader moreDataHeader = new BasicHeader(BuildWithDetails.MORE_DATA_HEADER, "true"); + BasicHeader textSizeHeader = new BasicHeader(BuildWithDetails.TEXT_SIZE_HEADER, Integer.toString(textLength)); + given(response.getFirstHeader(BuildWithDetails.MORE_DATA_HEADER)).willReturn(moreDataHeader); + given(response.getFirstHeader(BuildWithDetails.TEXT_SIZE_HEADER)).willReturn(textSizeHeader); + given(client.post_form_with_result(anyString(),anyListOf(NameValuePair.class),anyBoolean())).willReturn(response); + buildWithDetails.streamConsoleOutput(new BuildConsoleStreamListener() { + @Override + public void onData(String newLogChunk) { + assertThat(newLogChunk).isEqualTo(text); + } + + @Override + public void finished() { + fail("Should timeout"); + } + },1,2); + } catch (IOException e) { + fail("Should not return exception",e); + } + } + +}