Skip to content

Commit 887532c

Browse files
wtrockiwtrocki
committed
jenkins-client-217 - Added ability to stream logs and retrieve chunks for logs
1 parent 330dcb6 commit 887532c

File tree

6 files changed

+348
-24
lines changed

6 files changed

+348
-24
lines changed

‎ReleaseNotes.md‎

Lines changed: 9 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -2,6 +2,15 @@
22

33
## Release 0.3.8 (NOT RELEASED YET)
44

5+
* [Fixed Issue 217][issue- 217]
6+
7+
Added new method for streaming logs
8+
9+
```java
10+
11+
12+
```
13+
514
* [Fixed Issue 222][issue-222]
615

716
Fixed WARNING during build.

‎jenkins-client/src/main/java/com/offbytwo/jenkins/client/JenkinsHttpClient.java‎

Lines changed: 54 additions & 20 deletions
Original file line numberDiff line numberDiff line change
@@ -6,22 +6,25 @@
66

77
package com.offbytwo.jenkins.client;
88

9-
import static com.fasterxml.jackson.databind.DeserializationFeature.FAIL_ON_UNKNOWN_PROPERTIES;
10-
import static org.apache.commons.lang.StringUtils.isNotBlank;
11-
12-
import java.io.IOException;
13-
import java.io.InputStream;
14-
import java.net.URI;
15-
import java.util.List;
16-
import java.util.Map;
17-
9+
import com.fasterxml.jackson.databind.ObjectMapper;
10+
import com.google.common.collect.Lists;
11+
import com.google.common.io.ByteStreams;
12+
import com.offbytwo.jenkins.client.util.EncodingUtils;
13+
import com.offbytwo.jenkins.client.util.RequestReleasingInputStream;
14+
import com.offbytwo.jenkins.client.validator.HttpResponseValidator;
15+
import com.offbytwo.jenkins.model.BaseModel;
16+
import com.offbytwo.jenkins.model.Crumb;
17+
import com.offbytwo.jenkins.model.ExtractHeader;
18+
import net.sf.json.JSONObject;
1819
import org.apache.commons.io.IOUtils;
1920
import org.apache.commons.lang.StringUtils;
2021
import org.apache.http.Header;
2122
import org.apache.http.HttpResponse;
23+
import org.apache.http.NameValuePair;
2224
import org.apache.http.auth.AuthScope;
2325
import org.apache.http.auth.UsernamePasswordCredentials;
2426
import org.apache.http.client.CredentialsProvider;
27+
import org.apache.http.client.entity.UrlEncodedFormEntity;
2528
import org.apache.http.client.methods.HttpGet;
2629
import org.apache.http.client.methods.HttpPost;
2730
import org.apache.http.client.methods.HttpRequestBase;
@@ -38,18 +41,16 @@
3841
import org.slf4j.Logger;
3942
import org.slf4j.LoggerFactory;
4043

41-
import com.fasterxml.jackson.databind.ObjectMapper;
42-
import com.google.common.collect.Lists;
43-
import com.google.common.io.ByteStreams;
44-
import com.offbytwo.jenkins.client.util.EncodingUtils;
45-
import com.offbytwo.jenkins.client.util.RequestReleasingInputStream;
46-
//import com.offbytwo.jenkins.client.util.HttpResponseContentExtractor;
47-
import com.offbytwo.jenkins.client.validator.HttpResponseValidator;
48-
import com.offbytwo.jenkins.model.BaseModel;
49-
import com.offbytwo.jenkins.model.Crumb;
50-
import com.offbytwo.jenkins.model.ExtractHeader;
44+
import java.io.IOException;
45+
import java.io.InputStream;
46+
import java.net.URI;
47+
import java.util.List;
48+
import java.util.Map;
5149

52-
import net.sf.json.JSONObject;
50+
import static com.fasterxml.jackson.databind.DeserializationFeature.FAIL_ON_UNKNOWN_PROPERTIES;
51+
import static org.apache.commons.lang.StringUtils.isNotBlank;
52+
53+
//import com.offbytwo.jenkins.client.util.HttpResponseContentExtractor;
5354

5455
public class JenkinsHttpClient {
5556
private final Logger LOGGER = LoggerFactory.getLogger(getClass());
@@ -312,6 +313,39 @@ public void post_form(String path, Map<String, String> data, boolean crumbFlag)
312313
}
313314
}
314315

316+
317+
/**
318+
* Perform a POST request using form url encoding and return HttpResponse object
319+
* This method is not performing validation and can be used for more generic queries to jenkins.
320+
*
321+
* @param path
322+
* path to request, can be relative or absolute
323+
* @param data
324+
* data to post
325+
* @throws IOException,
326+
* HttpResponseException
327+
*/
328+
public HttpResponse post_form_with_result(String path, List<NameValuePair> data, boolean crumbFlag) throws IOException {
329+
HttpPost request;
330+
if (data != null) {
331+
UrlEncodedFormEntity urlEncodedFormEntity = new UrlEncodedFormEntity(data);
332+
request = new HttpPost(noapi(path));
333+
request.setEntity(urlEncodedFormEntity);
334+
} else {
335+
request = new HttpPost(noapi(path));
336+
}
337+
338+
if (crumbFlag == true) {
339+
Crumb crumb = get("/crumbIssuer", Crumb.class);
340+
if (crumb != null) {
341+
request.addHeader(new BasicHeader(crumb.getCrumbRequestField(), crumb.getCrumb()));
342+
}
343+
}
344+
HttpResponse response = client.execute(request, localContext);
345+
getJenkinsVersionFromHeader(response);
346+
return response;
347+
}
348+
315349
/**
316350
* Perform a POST request of XML (instead of using json mapper) and return a
317351
* string rendering of the response entity.

‎jenkins-client/src/main/java/com/offbytwo/jenkins/helper/BuildConsoleStreamListener.java‎

Lines changed: 19 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,19 @@
1+
package com.offbytwo.jenkins.helper;
2+
3+
/**
4+
* Listener interface used to obtain build console logs
5+
*/
6+
public interface BuildConsoleStreamListener {
7+
8+
/**
9+
* Called by api when new log data available.
10+
*
11+
* @param newLogChunk - string containing latest chunk of logs
12+
*/
13+
void onData(String newLogChunk);
14+
15+
/**
16+
* Called when streaming console output is finished
17+
*/
18+
void finished();
19+
}

‎jenkins-client/src/main/java/com/offbytwo/jenkins/model/BuildWithDetails.java‎

Lines changed: 103 additions & 4 deletions
Original file line numberDiff line numberDiff line change
@@ -9,12 +9,26 @@
99
import com.google.common.base.Predicate;
1010
import com.google.common.base.Strings;
1111
import com.google.common.collect.ImmutableMap;
12+
import com.offbytwo.jenkins.helper.BuildConsoleStreamListener;
13+
import org.apache.http.Header;
14+
import org.apache.http.HttpResponse;
15+
import org.apache.http.NameValuePair;
16+
import org.apache.http.message.BasicNameValuePair;
17+
import org.apache.http.util.EntityUtils;
18+
import org.slf4j.Logger;
19+
import org.slf4j.LoggerFactory;
1220

1321
import java.io.IOException;
1422
import java.io.InputStream;
1523
import java.net.URI;
1624
import java.net.URISyntaxException;
17-
import java.util.*;
25+
import java.util.ArrayList;
26+
import java.util.Collection;
27+
import java.util.Collections;
28+
import java.util.HashMap;
29+
import java.util.List;
30+
import java.util.Map;
31+
import java.util.Objects;
1832

1933
import static com.google.common.collect.Collections2.filter;
2034

@@ -25,6 +39,11 @@
2539
*/
2640
public class BuildWithDetails extends Build {
2741

42+
private final Logger LOGGER = LoggerFactory.getLogger(getClass());
43+
44+
public final static String TEXT_SIZE_HEADER = "x-text-size";
45+
public final static String MORE_DATA_HEADER = "x-more-data";
46+
2847
/**
2948
* This will be returned by the API in cases where the build has never run.
3049
* For example {@link Build#BUILD_HAS_NEVER_RUN}
@@ -350,24 +369,104 @@ public boolean apply(Map<String, Object> action) {
350369
}
351370

352371
/**
353-
* @return The console output of the build. The line separation is done by
372+
* @return The full console output of the build. The line separation is done by
354373
* {@code CR+LF}.
374+
*
375+
* @see streamConsoleOutput method for obtaining logs for running build
376+
*
355377
* @throws IOException in case of a failure.
356378
*/
357379
public String getConsoleOutputText() throws IOException {
358380
return client.get(getUrl() + "/logText/progressiveText");
359381
}
360382

361383
/**
362-
* The console output with HTML.
363-
*
384+
* The full console output with HTML.
385+
*
386+
* @see streamConsoleOutput method for obtaining logs for running build
387+
*
364388
* @return The console output as HTML.
365389
* @throws IOException in case of an error.
366390
*/
367391
public String getConsoleOutputHtml() throws IOException {
368392
return client.get(getUrl() + "/logText/progressiveHtml");
369393
}
370394

395+
396+
/**
397+
* Stream build console output log as text using BuildConsoleStreamListener
398+
* Method can be used to asynchronously obtain logs for running build.
399+
*
400+
* @param listener interface used to asynchronously obtain logs
401+
* @param poolingInterval interval (seconds) used to pool jenkins for logs
402+
* @param poolingTimeout pooling timeout (seconds) used to break pooling in case build stuck
403+
*
404+
*/
405+
public void streamConsoleOutput(final BuildConsoleStreamListener listener, final int poolingInterval, final int poolingTimeout) throws InterruptedException, IOException {
406+
// Calculate start and timeout
407+
final long startTime = System.currentTimeMillis();
408+
final long timeoutTime = startTime + (poolingTimeout * 1000);
409+
410+
int bufferOffset = 0;
411+
while (true) {
412+
Thread.sleep(poolingInterval * 1000);
413+
414+
ConsoleLog consoleLog = null;
415+
consoleLog = getConsoleOutputText(bufferOffset);
416+
String logString = consoleLog.getConsoleLog();
417+
if (logString != null && !logString.isEmpty()) {
418+
listener.onData(logString);
419+
}
420+
if (consoleLog.getHasMoreData()) {
421+
bufferOffset = consoleLog.getCurrentBufferSize();
422+
} else {
423+
listener.finished();
424+
break;
425+
}
426+
long currentTime = System.currentTimeMillis();
427+
428+
if (currentTime > timeoutTime) {
429+
LOGGER.warn("Pooling for build {0} for {2} timeout! Check if job stuck in jenkins",
430+
BuildWithDetails.this.getDisplayName(), BuildWithDetails.this.getNumber());
431+
break;
432+
}
433+
}
434+
}
435+
436+
/**
437+
* Get build console output log as text.
438+
* Use this method to periodically obtain logs from jenkins and skip chunks that were already received
439+
*
440+
* @param bufferOffset offset in console lo
441+
* @return ConsoleLog object containing console output of the build. The line separation is done by
442+
* {@code CR+LF}.
443+
* @throws IOException in case of a failure.
444+
*/
445+
public ConsoleLog getConsoleOutputText(int bufferOffset) throws IOException {
446+
List<NameValuePair> formData = new ArrayList<>();
447+
formData.add(new BasicNameValuePair("start", Integer.toString(bufferOffset)));
448+
String path = getUrl() + "logText/progressiveText";
449+
HttpResponse httpResponse = client.post_form_with_result(path, formData, false);
450+
451+
Header moreDataHeader = httpResponse.getFirstHeader(MORE_DATA_HEADER);
452+
Header textSizeHeader = httpResponse.getFirstHeader(TEXT_SIZE_HEADER);
453+
String response = EntityUtils.toString(httpResponse.getEntity());
454+
boolean hasMoreData = false;
455+
if (moreDataHeader != null) {
456+
hasMoreData = Boolean.TRUE.toString().equals(moreDataHeader.getValue());
457+
}
458+
Integer currentBufferSize = bufferOffset;
459+
if (textSizeHeader != null) {
460+
try {
461+
currentBufferSize = Integer.parseInt(textSizeHeader.getValue());
462+
} catch (NumberFormatException e) {
463+
LOGGER.warn("Cannot parse buffer size for job {0} build {1}. Using current offset!", this.getDisplayName(), this.getNumber());
464+
}
465+
}
466+
return new ConsoleLog(response, hasMoreData, currentBufferSize);
467+
}
468+
469+
371470
public BuildChangeSet getChangeSet() {
372471
return changeSet;
373472
}

‎jenkins-client/src/main/java/com/offbytwo/jenkins/model/ConsoleLog.java‎

Lines changed: 41 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,41 @@
1+
package com.offbytwo.jenkins.model;
2+
3+
/**
4+
* Represents build console log
5+
*/
6+
public class ConsoleLog {
7+
8+
private String consoleLog;
9+
private Boolean hasMoreData;
10+
private Integer currentBufferSize;
11+
12+
public ConsoleLog(String consoleLog, Boolean hasMoreData, Integer currentBufferSize) {
13+
this.consoleLog = consoleLog;
14+
this.hasMoreData = hasMoreData;
15+
this.currentBufferSize = currentBufferSize;
16+
}
17+
18+
public String getConsoleLog() {
19+
return consoleLog;
20+
}
21+
22+
public void setConsoleLog(String consoleLog) {
23+
this.consoleLog = consoleLog;
24+
}
25+
26+
public Boolean getHasMoreData() {
27+
return hasMoreData;
28+
}
29+
30+
public void setHasMoreData(Boolean hasMoreData) {
31+
this.hasMoreData = hasMoreData;
32+
}
33+
34+
public Integer getCurrentBufferSize() {
35+
return currentBufferSize;
36+
}
37+
38+
public void setCurrentBufferSize(Integer currentBufferSize) {
39+
this.currentBufferSize = currentBufferSize;
40+
}
41+
}

0 commit comments

Comments
 (0)