Skip to content

Commit bfa7557

Browse files
wtrockiwtrocki
committed
jenkins-client-217 - Added ability to stream logs and retrieve chunks for logs
1 parent 74eac36 commit bfa7557

File tree

5 files changed

+256
-24
lines changed

5 files changed

+256
-24
lines changed

‎jenkins-client/src/main/java/com/offbytwo/jenkins/Main.java‎

Lines changed: 46 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,46 @@
1+
package com.offbytwo.jenkins;
2+
3+
import com.offbytwo.jenkins.helper.BuildConsoleStreamListener;
4+
import com.offbytwo.jenkins.model.Build;
5+
import com.offbytwo.jenkins.model.BuildWithDetails;
6+
import com.offbytwo.jenkins.model.JobWithDetails;
7+
8+
import java.io.IOException;
9+
import java.net.URI;
10+
import java.net.URISyntaxException;
11+
12+
/**
13+
*/
14+
public class Main {
15+
// This code is used only to demo solution and it would be removed later and replaced with unit tests.
16+
// Posting now for early feedback
17+
// TODO This would be replaced by unit tests
18+
public static void main(String[] args) throws URISyntaxException, IOException, InterruptedException {
19+
JenkinsServer jenkinsServer = new JenkinsServer(new URI("https://jenkins.net"),"admin","admin");
20+
21+
// Get chunk of the logs
22+
// JobWithDetails job = jenkinsServer.getJob("wtr-demo");
23+
// Build buildByNumber = job.getBuildByNumber(6);
24+
// BuildWithDetails details = buildByNumber.details();
25+
// ConsoleLog consoleOutputText = details.getConsoleOutputText(50);
26+
// System.out.print(consoleOutputText.getConsoleLog());
27+
28+
29+
System.out.println("------------------------------------------------------------------------------");
30+
// Pool logs
31+
JobWithDetails job2 = jenkinsServer.getJob("wtr-demo");
32+
Build buildByNumber2 = job2.getBuildByNumber(7);
33+
BuildWithDetails details2 = buildByNumber2.details();
34+
details2.streamConsoleOutput(new BuildConsoleStreamListener() {
35+
@Override
36+
public void onData(String newLogChunk) {
37+
System.out.print(newLogChunk);
38+
}
39+
40+
@Override
41+
public void finished() {
42+
// Do something when finished
43+
}
44+
}, 3, 400);
45+
}
46+
}

‎jenkins-client/src/main/java/com/offbytwo/jenkins/client/JenkinsHttpClient.java‎

Lines changed: 54 additions & 20 deletions
Original file line numberDiff line numberDiff line change
@@ -6,22 +6,25 @@
66

77
package com.offbytwo.jenkins.client;
88

9-
import static com.fasterxml.jackson.databind.DeserializationFeature.FAIL_ON_UNKNOWN_PROPERTIES;
10-
import static org.apache.commons.lang.StringUtils.isNotBlank;
11-
12-
import java.io.IOException;
13-
import java.io.InputStream;
14-
import java.net.URI;
15-
import java.util.List;
16-
import java.util.Map;
17-
9+
import com.fasterxml.jackson.databind.ObjectMapper;
10+
import com.google.common.collect.Lists;
11+
import com.google.common.io.ByteStreams;
12+
import com.offbytwo.jenkins.client.util.EncodingUtils;
13+
import com.offbytwo.jenkins.client.util.RequestReleasingInputStream;
14+
import com.offbytwo.jenkins.client.validator.HttpResponseValidator;
15+
import com.offbytwo.jenkins.model.BaseModel;
16+
import com.offbytwo.jenkins.model.Crumb;
17+
import com.offbytwo.jenkins.model.ExtractHeader;
18+
import net.sf.json.JSONObject;
1819
import org.apache.commons.io.IOUtils;
1920
import org.apache.commons.lang.StringUtils;
2021
import org.apache.http.Header;
2122
import org.apache.http.HttpResponse;
23+
import org.apache.http.NameValuePair;
2224
import org.apache.http.auth.AuthScope;
2325
import org.apache.http.auth.UsernamePasswordCredentials;
2426
import org.apache.http.client.CredentialsProvider;
27+
import org.apache.http.client.entity.UrlEncodedFormEntity;
2528
import org.apache.http.client.methods.HttpGet;
2629
import org.apache.http.client.methods.HttpPost;
2730
import org.apache.http.client.methods.HttpRequestBase;
@@ -38,18 +41,16 @@
3841
import org.slf4j.Logger;
3942
import org.slf4j.LoggerFactory;
4043

41-
import com.fasterxml.jackson.databind.ObjectMapper;
42-
import com.google.common.collect.Lists;
43-
import com.google.common.io.ByteStreams;
44-
import com.offbytwo.jenkins.client.util.EncodingUtils;
45-
import com.offbytwo.jenkins.client.util.RequestReleasingInputStream;
46-
//import com.offbytwo.jenkins.client.util.HttpResponseContentExtractor;
47-
import com.offbytwo.jenkins.client.validator.HttpResponseValidator;
48-
import com.offbytwo.jenkins.model.BaseModel;
49-
import com.offbytwo.jenkins.model.Crumb;
50-
import com.offbytwo.jenkins.model.ExtractHeader;
44+
import java.io.IOException;
45+
import java.io.InputStream;
46+
import java.net.URI;
47+
import java.util.List;
48+
import java.util.Map;
5149

52-
import net.sf.json.JSONObject;
50+
import static com.fasterxml.jackson.databind.DeserializationFeature.FAIL_ON_UNKNOWN_PROPERTIES;
51+
import static org.apache.commons.lang.StringUtils.isNotBlank;
52+
53+
//import com.offbytwo.jenkins.client.util.HttpResponseContentExtractor;
5354

5455
public class JenkinsHttpClient {
5556
private final Logger LOGGER = LoggerFactory.getLogger( getClass() );
@@ -337,6 +338,39 @@ public void post_form(String path, Map<String, String> data, boolean crumbFlag)
337338
}
338339
}
339340

341+
342+
/**
343+
* Perform a POST request using form url encoding and return HttpResponse object
344+
* This method is not performing validation and can be used for more generic queries to jenkins.
345+
*
346+
* @param path
347+
* path to request, can be relative or absolute
348+
* @param data
349+
* data to post
350+
* @throws IOException,
351+
* HttpResponseException
352+
*/
353+
public HttpResponse post_form_with_result(String path, List<NameValuePair> data, boolean crumbFlag) throws IOException {
354+
HttpPost request;
355+
if (data != null) {
356+
UrlEncodedFormEntity urlEncodedFormEntity = new UrlEncodedFormEntity(data);
357+
request = new HttpPost(noapi(path));
358+
request.setEntity(urlEncodedFormEntity);
359+
} else {
360+
request = new HttpPost(noapi(path));
361+
}
362+
363+
if (crumbFlag == true) {
364+
Crumb crumb = get("/crumbIssuer", Crumb.class);
365+
if (crumb != null) {
366+
request.addHeader(new BasicHeader(crumb.getCrumbRequestField(), crumb.getCrumb()));
367+
}
368+
}
369+
HttpResponse response = client.execute(request, localContext);
370+
getJenkinsVersionFromHeader(response);
371+
return response;
372+
}
373+
340374
/**
341375
* Perform a POST request of XML (instead of using json mapper) and return a
342376
* string rendering of the response entity.

‎jenkins-client/src/main/java/com/offbytwo/jenkins/helper/BuildConsoleStreamListener.java‎

Lines changed: 19 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,19 @@
1+
package com.offbytwo.jenkins.helper;
2+
3+
/**
4+
* Listener interface used to obtain build console logs
5+
*/
6+
public interface BuildConsoleStreamListener {
7+
8+
/**
9+
* Called by api when new log data available.
10+
*
11+
* @param newLogChunk - string containing latest chunk of logs
12+
*/
13+
void onData(String newLogChunk);
14+
15+
/**
16+
* Called when streaming console output is finished
17+
*/
18+
void finished();
19+
}

‎jenkins-client/src/main/java/com/offbytwo/jenkins/model/BuildWithDetails.java‎

Lines changed: 96 additions & 4 deletions
Original file line numberDiff line numberDiff line change
@@ -9,12 +9,26 @@
99
import com.google.common.base.Predicate;
1010
import com.google.common.base.Strings;
1111
import com.google.common.collect.ImmutableMap;
12+
import com.offbytwo.jenkins.helper.BuildConsoleStreamListener;
13+
import org.apache.http.Header;
14+
import org.apache.http.HttpResponse;
15+
import org.apache.http.NameValuePair;
16+
import org.apache.http.message.BasicNameValuePair;
17+
import org.apache.http.util.EntityUtils;
18+
import org.slf4j.Logger;
19+
import org.slf4j.LoggerFactory;
1220

1321
import java.io.IOException;
1422
import java.io.InputStream;
1523
import java.net.URI;
1624
import java.net.URISyntaxException;
17-
import java.util.*;
25+
import java.util.ArrayList;
26+
import java.util.Collection;
27+
import java.util.Collections;
28+
import java.util.HashMap;
29+
import java.util.List;
30+
import java.util.Map;
31+
import java.util.Objects;
1832

1933
import static com.google.common.collect.Collections2.filter;
2034

@@ -25,6 +39,8 @@
2539
*/
2640
public class BuildWithDetails extends Build {
2741

42+
private final Logger LOGGER = LoggerFactory.getLogger(getClass());
43+
2844
/**
2945
* This will be returned by the API in cases where the build has never run.
3046
* For example {@link Build#BUILD_HAS_NEVER_RUN}
@@ -350,24 +366,100 @@ public boolean apply(Map<String, Object> action) {
350366
}
351367

352368
/**
353-
* @return The console output of the build. The line separation is done by
369+
* @return The full console output of the build. The line separation is done by
354370
* {@code CR+LF}.
371+
*
372+
* @see streamConsoleOutput method for obtaining logs for running build
373+
*
355374
* @throws IOException in case of a failure.
356375
*/
357376
public String getConsoleOutputText() throws IOException {
358377
return client.get(getUrl() + "/logText/progressiveText");
359378
}
360379

361380
/**
362-
* The console output with HTML.
363-
*
381+
* The full console output with HTML.
382+
*
383+
* @see streamConsoleOutput method for obtaining logs for running build
384+
*
364385
* @return The console output as HTML.
365386
* @throws IOException
366387
*/
367388
public String getConsoleOutputHtml() throws IOException {
368389
return client.get(getUrl() + "/logText/progressiveHtml");
369390
}
370391

392+
393+
/**
394+
* Stream build console output log as text using BuildConsoleStreamListener
395+
* Method can be used to asynchronously obtain logs for running build.
396+
*
397+
* @param listener interface used to asynchronously obtain logs
398+
* @param poolingInterval interval (seconds) used to pool jenkins for logs
399+
* @param poolingTimeout pooling timeout (seconds) used to break pooling in case build stuck
400+
*
401+
*/
402+
public void streamConsoleOutput(BuildConsoleStreamListener listener, int poolingInterval, int poolingTimeout) throws IOException, InterruptedException {
403+
int bufferOffset = 0;
404+
// Calculate start and timeout
405+
long startTime = System.currentTimeMillis();
406+
long timeoutTime = startTime + (poolingTimeout * 1000);
407+
// Pool for logs
408+
while (true) {
409+
Thread.sleep(poolingInterval);
410+
ConsoleLog consoleLog = getConsoleOutputText(bufferOffset);
411+
String logString = consoleLog.getConsoleLog();
412+
if (logString != null && !logString.isEmpty()) {
413+
listener.onData(logString);
414+
}
415+
if (consoleLog.getHasMoreData()) {
416+
bufferOffset = consoleLog.getCurrentBufferSize();
417+
} else {
418+
listener.finished();
419+
break;
420+
}
421+
long currentTime = System.currentTimeMillis();
422+
423+
if(currentTime> timeoutTime){
424+
LOGGER.warn("Pooling for build {0} for {2} timeout! Check if job stuck in jenkins", this.getDisplayName(), this.getNumber());
425+
break;
426+
}
427+
}
428+
}
429+
430+
/**
431+
* Get build console output log as text.
432+
* Use this method to periodically obtain logs from jenkins and skip chunks that were already received
433+
*
434+
* @param bufferOffset offset in console lo
435+
* @return ConsoleLog object containing console output of the build. The line separation is done by
436+
* {@code CR+LF}.
437+
* @throws IOException in case of a failure.
438+
*/
439+
public ConsoleLog getConsoleOutputText(int bufferOffset) throws IOException {
440+
List<NameValuePair> formData = new ArrayList<>();
441+
formData.add(new BasicNameValuePair("start", Integer.toString(bufferOffset)));
442+
String path = getUrl() + "logText/progressiveText";
443+
HttpResponse httpResponse = client.post_form_with_result(path, formData, false);
444+
Header moreDataHeader = httpResponse.getFirstHeader("x-more-data");
445+
Header textSizeHeader = httpResponse.getFirstHeader("x-text-size");
446+
String response = EntityUtils.toString(httpResponse.getEntity());
447+
boolean hasMoreData = false;
448+
if (moreDataHeader != null) {
449+
hasMoreData = Boolean.TRUE.toString().equals(moreDataHeader.getValue());
450+
}
451+
Integer currentBufferSize = bufferOffset;
452+
if (textSizeHeader != null) {
453+
try {
454+
currentBufferSize = Integer.parseInt(textSizeHeader.getValue());
455+
} catch (NumberFormatException e) {
456+
LOGGER.warn("Cannot parse buffer size for job {0} build {1}. Using current offset!", this.getDisplayName(), this.getNumber());
457+
}
458+
}
459+
return new ConsoleLog(response, hasMoreData, currentBufferSize);
460+
}
461+
462+
371463
public BuildChangeSet getChangeSet() {
372464
return changeSet;
373465
}

‎jenkins-client/src/main/java/com/offbytwo/jenkins/model/ConsoleLog.java‎

Lines changed: 41 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,41 @@
1+
package com.offbytwo.jenkins.model;
2+
3+
/**
4+
* Represents build console log
5+
*/
6+
public class ConsoleLog {
7+
8+
private String consoleLog;
9+
private Boolean hasMoreData;
10+
private Integer currentBufferSize;
11+
12+
public ConsoleLog(String consoleLog, Boolean hasMoreData, Integer currentBufferSize) {
13+
this.consoleLog = consoleLog;
14+
this.hasMoreData = hasMoreData;
15+
this.currentBufferSize = currentBufferSize;
16+
}
17+
18+
public String getConsoleLog() {
19+
return consoleLog;
20+
}
21+
22+
public void setConsoleLog(String consoleLog) {
23+
this.consoleLog = consoleLog;
24+
}
25+
26+
public Boolean getHasMoreData() {
27+
return hasMoreData;
28+
}
29+
30+
public void setHasMoreData(Boolean hasMoreData) {
31+
this.hasMoreData = hasMoreData;
32+
}
33+
34+
public Integer getCurrentBufferSize() {
35+
return currentBufferSize;
36+
}
37+
38+
public void setCurrentBufferSize(Integer currentBufferSize) {
39+
this.currentBufferSize = currentBufferSize;
40+
}
41+
}

0 commit comments

Comments
 (0)