Skip to content

Commit 7f484f3

Browse files
tkohegyitkohegyi
committed
adding proxy support of Brotli content encoding - #13 
1 parent 7fa1837 commit 7f484f3

23 files changed

+623
-120
lines changed

‎build.gradle‎

Lines changed: 5 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -19,7 +19,7 @@ ext.myBuildNumber='SNAPSHOT'
1919
if ( hasProperty('buildNumber') ) {
2020
myBuildNumber = "${project.ext.buildNumber}"
2121
}
22-
version = "$wilmaVersion" + ".21." + "${project.ext.myBuildNumber}"
22+
version = "$wilmaVersion" + ".22." + "${project.ext.myBuildNumber}"
2323

2424
def isSnapshot = project.version.contains('SNAPSHOT')
2525

@@ -94,6 +94,7 @@ dependencies {
9494
implementation group: 'org.slf4j', name: 'jul-to-slf4j', version:'1.7.30'
9595
api group: 'org.apache.httpcomponents', name: 'httpmime', version:'4.5.13' //this includes httpclient too
9696
implementation group: 'org.apache.commons', name: 'commons-io', version:'1.3.2'
97+
implementation group: 'org.brotli', name: 'dec', version: '0.1.2'
9798
implementation group: 'org.apache.ant', name: 'ant', version:'1.10.10'
9899
implementation group: 'org.bouncycastle', name: 'bcpkix-jdk15on', version:'1.69' //this includes bcprov-jdk15on too
99100
implementation group: 'org.eclipse.jetty', name: 'jetty-io', version: '9.4.42.v20210604'
@@ -105,6 +106,9 @@ dependencies {
105106
testImplementation group: 'io.netty', name: 'netty-all', version: '4.1.65.Final'
106107
testImplementation group: 'org.eclipse.jetty', name: 'jetty-server', version: '9.4.42.v20210604'
107108
testImplementation group: 'org.springframework', name: 'spring-core', version: '5.3.8'
109+
testImplementation group: 'com.nixxcode.jvmbrotli', name: 'jvmbrotli', version: '0.2.0'
110+
testImplementation group: 'com.nixxcode.jvmbrotli', name: 'jvmbrotli-win32-x86-amd64', version: '0.2.0'
111+
testImplementation group: 'com.nixxcode.jvmbrotli', name: 'jvmbrotli-linux-x86-amd64', version: '0.2.0'
108112
}
109113

110114
def myCopySpec = project.copySpec {

‎src/main/java/net/lightbody/bmp/proxy/http/BrowserMobHttpClient.java‎

Lines changed: 39 additions & 20 deletions
Original file line numberDiff line numberDiff line change
@@ -1,10 +1,5 @@
11
package net.lightbody.bmp.proxy.http;
22

3-
import website.magyar.mitm.proxy.ProxyServer;
4-
import website.magyar.mitm.proxy.RequestInterceptor;
5-
import website.magyar.mitm.proxy.ResponseInterceptor;
6-
import website.magyar.mitm.proxy.http.MitmJavaProxyHttpRequest;
7-
import website.magyar.mitm.proxy.http.MitmJavaProxyHttpResponse;
83
import net.lightbody.bmp.core.har.Har;
94
import net.lightbody.bmp.core.har.HarCookie;
105
import net.lightbody.bmp.core.har.HarEntry;
@@ -65,10 +60,16 @@
6560
import org.apache.http.protocol.BasicHttpContext;
6661
import org.apache.http.protocol.ExecutionContext;
6762
import org.apache.http.protocol.HttpContext;
63+
import org.brotli.dec.BrotliInputStream;
6864
import org.slf4j.Logger;
6965
import org.slf4j.LoggerFactory;
7066
import org.xbill.DNS.Cache;
7167
import org.xbill.DNS.DClass;
68+
import website.magyar.mitm.proxy.ProxyServer;
69+
import website.magyar.mitm.proxy.RequestInterceptor;
70+
import website.magyar.mitm.proxy.ResponseInterceptor;
71+
import website.magyar.mitm.proxy.http.MitmJavaProxyHttpRequest;
72+
import website.magyar.mitm.proxy.http.MitmJavaProxyHttpResponse;
7273

7374
import java.io.ByteArrayInputStream;
7475
import java.io.ByteArrayOutputStream;
@@ -115,15 +116,15 @@ public class BrowserMobHttpClient {
115116
// not using CopyOnWriteArray because we're WRITE heavy and it is for READ heavy operations
116117
// instead doing it the old fashioned way with a synchronized block
117118
private final Set<ActiveRequest> activeRequests = new HashSet<ActiveRequest>();
119+
private final SimulatedSocketFactory socketFactory;
120+
private final TrustingSSLSocketFactory sslSocketFactory;
121+
private final PoolingHttpClientConnectionManager httpClientConnMgr;
122+
private final HttpClient httpClient;
118123
private Har har;
119124
private String harPageRef;
120125
private boolean captureHeaders;
121126
private boolean captureContent; // if captureContent is set, default policy is to capture binary contents too
122127
private boolean captureBinaryContent = true;
123-
private final SimulatedSocketFactory socketFactory;
124-
private final TrustingSSLSocketFactory sslSocketFactory;
125-
private final PoolingHttpClientConnectionManager httpClientConnMgr;
126-
private final HttpClient httpClient;
127128
private int requestTimeout;
128129
private BrowserMobHostNameResolver hostNameResolver;
129130
private boolean decompress = true;
@@ -425,6 +426,7 @@ private MitmJavaProxyHttpResponse execute(final MitmJavaProxyHttpRequest req, in
425426
long bytes = 0;
426427
boolean gzipping = false;
427428
boolean deflating = false;
429+
boolean brotling = false;
428430
OutputStream os = req.getOutputStream();
429431
if (os == null) {
430432
os = new CappedByteArrayOutputStream(MAX_BUFFER_SIZE);
@@ -459,7 +461,7 @@ private MitmJavaProxyHttpResponse execute(final MitmJavaProxyHttpRequest req, in
459461
try {
460462
// set the User-Agent if it's not already set
461463
if (method.getHeaders("User-Agent").length == 0) {
462-
method.addHeader("User-Agent", "MITM-JavaProxy V/1.0");
464+
method.addHeader("User-Agent", "MITM-JavaProxy V-22");
463465
}
464466

465467
response = httpClient.execute(method, ctx);
@@ -480,14 +482,21 @@ private MitmJavaProxyHttpResponse execute(final MitmJavaProxyHttpRequest req, in
480482
if (is != null) {
481483
Header contentEncodingHeader = response.getFirstHeader("Content-Encoding");
482484
if (contentEncodingHeader != null) {
483-
if ("gzip".equalsIgnoreCase(contentEncodingHeader.getValue())) {
485+
String value = contentEncodingHeader.getValue();
486+
if ("gzip".equalsIgnoreCase(value)) {
484487
gzipping = true;
485-
} else if ("deflate".equalsIgnoreCase(contentEncodingHeader.getValue())) {
486-
deflating = true;
488+
} else {
489+
if ("deflate".equalsIgnoreCase(value)) {
490+
deflating = true;
491+
} else {
492+
if ("br".equalsIgnoreCase(value)) {
493+
brotling = true;
494+
}
495+
}
487496
}
488497
}
489498

490-
// deal with GZIP content!
499+
// deal with compressed content!
491500
if (decompress && response.getEntity().getContentLength() != 0) { //getContentLength<0 if unknown
492501
if (gzipping) {
493502
is = new GZIPInputStream(is);
@@ -496,6 +505,10 @@ private MitmJavaProxyHttpResponse execute(final MitmJavaProxyHttpRequest req, in
496505
// WARN : if system is using zlib<=1.1.4 the stream must be append with a dummy byte
497506
// that is not required for zlib>1.1.4 (not mentioned on current Inflater javadoc)
498507
is = new InflaterInputStream(is, new Inflater(true));
508+
} else {
509+
if (brotling) {
510+
is = new BrotliInputStream(is);
511+
}
499512
}
500513
}
501514
}
@@ -638,17 +651,23 @@ private MitmJavaProxyHttpResponse execute(final MitmJavaProxyHttpRequest req, in
638651
copy = bos;
639652
}
640653
if (captureContent && enableWorkWithCopy) {
641-
if (entry.getResponse().getBodySize() != 0 && (gzipping || deflating)) {
654+
if (entry.getResponse().getBodySize() != 0 && (gzipping || deflating || brotling)) {
642655
// ok, we need to decompress it before we can put it in the har file
643656
try {
644657
InputStream temp = null;
645658
if (gzipping) {
646659
temp = new GZIPInputStream(new ByteArrayInputStream(copy.toByteArray()));
647-
} else if (deflating) {
648-
//RAW deflate only
649-
// WARN : if system is using zlib<=1.1.4 the stream must be append with a dummy byte
650-
// that is not required for zlib>1.1.4 (not mentioned on current Inflater javadoc)
651-
temp = new InflaterInputStream(new ByteArrayInputStream(copy.toByteArray()), new Inflater(true));
660+
} else {
661+
if (deflating) {
662+
//RAW deflate only
663+
// WARN : if system is using zlib<=1.1.4 the stream must be append with a dummy byte
664+
// that is not required for zlib>1.1.4 (not mentioned on current Inflater javadoc)
665+
temp = new InflaterInputStream(new ByteArrayInputStream(copy.toByteArray()), new Inflater(true));
666+
} else {
667+
if (brotling) {
668+
temp = new BrotliInputStream(new ByteArrayInputStream(copy.toByteArray()));
669+
}
670+
}
652671
}
653672
copy = new ByteArrayOutputStream();
654673
IOUtils.copy(temp, copy);

‎src/test/java/net/lightbody/bmp/proxy/http/ContentEncodingWithNoProxyTest.java‎

Lines changed: 93 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,93 @@
1+
package net.lightbody.bmp.proxy.http;
2+
3+
import org.apache.http.Header;
4+
import org.junit.Test;
5+
import website.magyar.mitm.proxy.help.AbstractComplexProxyTool;
6+
import website.magyar.mitm.proxy.help.ContentEncoding;
7+
import website.magyar.mitm.proxy.help.DefaultRequestInterceptor;
8+
import website.magyar.mitm.proxy.help.DefaultResponseInterceptor;
9+
import website.magyar.mitm.proxy.help.ResponseInfo;
10+
11+
import static org.junit.Assert.assertEquals;
12+
import static org.junit.Assert.assertNull;
13+
import static org.junit.Assert.assertTrue;
14+
15+
public class ContentEncodingWithNoProxyTest extends AbstractComplexProxyTool {
16+
17+
@Override
18+
protected void setUp() {
19+
String stubUrl = "http://127.0.0.1:" + getStubServerPort() + "/stub";
20+
LOGGER.info("STUB URL used: {}", stubUrl);
21+
DefaultRequestInterceptor defaultRequestInterceptor = new DefaultRequestInterceptor(getRequestCount(), NEED_STUB_RESPONSE, stubUrl);
22+
DefaultResponseInterceptor defaultResponseInterceptor = new DefaultResponseInterceptor(getResponseCount());
23+
getProxyServer().addRequestInterceptor(defaultRequestInterceptor);
24+
getProxyServer().addResponseInterceptor(defaultResponseInterceptor);
25+
getProxyServer().setCaptureBinaryContent(true);
26+
getProxyServer().setCaptureContent(true);
27+
}
28+
29+
private void checkEncoding(Header contentEncodingHeader, ContentEncoding contentEncoding) {
30+
if (contentEncoding == ContentEncoding.NONE) {
31+
assertNull(contentEncodingHeader);
32+
} else {
33+
assertTrue("Response is not encoded with:" + contentEncoding.getValue(),
34+
contentEncodingHeader != null && contentEncodingHeader.getValue().contains(contentEncoding.getValue()));
35+
}
36+
}
37+
38+
@Test
39+
public void testSimpleGetRequest_NoEncoding() throws Exception {
40+
ContentEncoding contentEncoding = ContentEncoding.NONE;
41+
ResponseInfo proxiedResponse = httpGetWithApacheClient(getWebHost(), NO_NEED_STUB_RESPONSE, false, false, contentEncoding);
42+
assertEquals(200, proxiedResponse.getStatusCode());
43+
assertEquals(SERVER_BACKEND, proxiedResponse.getBody());
44+
assertEquals(0, getResponseCount().get());
45+
assertEquals(0, getRequestCount().get());
46+
checkEncoding(proxiedResponse.getContentEncoding(), contentEncoding);
47+
}
48+
49+
@Test
50+
public void testSimpleGetRequest_GzipEncoding() throws Exception {
51+
ContentEncoding contentEncoding = ContentEncoding.GZIP;
52+
ResponseInfo proxiedResponse = httpGetWithApacheClient(getWebHost(), NO_NEED_STUB_RESPONSE, false, false, contentEncoding);
53+
assertEquals(200, proxiedResponse.getStatusCode());
54+
assertEquals(SERVER_BACKEND, proxiedResponse.getBody());
55+
assertEquals(0, getResponseCount().get());
56+
assertEquals(0, getRequestCount().get());
57+
checkEncoding(proxiedResponse.getContentEncoding(), contentEncoding);
58+
}
59+
60+
@Test
61+
public void testSimpleGetRequest_Deflate() throws Exception {
62+
ContentEncoding contentEncoding = ContentEncoding.DEFLATE;
63+
ResponseInfo proxiedResponse = httpGetWithApacheClient(getWebHost(), NO_NEED_STUB_RESPONSE, false, false, contentEncoding);
64+
assertEquals(200, proxiedResponse.getStatusCode());
65+
assertEquals(SERVER_BACKEND, proxiedResponse.getBody());
66+
assertEquals(0, getResponseCount().get());
67+
assertEquals(0, getRequestCount().get());
68+
checkEncoding(proxiedResponse.getContentEncoding(), contentEncoding);
69+
}
70+
71+
@Test
72+
public void testSimpleGetRequest_Brotli() throws Exception {
73+
ContentEncoding contentEncoding = ContentEncoding.BROTLI;
74+
ResponseInfo proxiedResponse = httpGetWithApacheClient(getWebHost(), NO_NEED_STUB_RESPONSE, false, false, contentEncoding);
75+
assertEquals(200, proxiedResponse.getStatusCode());
76+
assertEquals(SERVER_BACKEND, proxiedResponse.getBody());
77+
assertEquals(0, getResponseCount().get());
78+
assertEquals(0, getRequestCount().get());
79+
checkEncoding(proxiedResponse.getContentEncoding(), contentEncoding);
80+
}
81+
82+
@Test
83+
public void testSimpleGetRequest_Any() throws Exception {
84+
ContentEncoding contentEncoding = ContentEncoding.ANY;
85+
ResponseInfo proxiedResponse = httpGetWithApacheClient(getWebHost(), NO_NEED_STUB_RESPONSE, false, false, contentEncoding);
86+
assertEquals(200, proxiedResponse.getStatusCode());
87+
assertEquals(SERVER_BACKEND, proxiedResponse.getBody());
88+
assertEquals(0, getResponseCount().get());
89+
assertEquals(0, getRequestCount().get());
90+
//received content encoding is not checked
91+
}
92+
93+
}

‎src/test/java/net/lightbody/bmp/proxy/http/ContentEncodingWithProxyTest.java‎

Lines changed: 94 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,94 @@
1+
package net.lightbody.bmp.proxy.http;
2+
3+
import org.apache.http.Header;
4+
import org.junit.Test;
5+
import website.magyar.mitm.proxy.ProxyServer;
6+
import website.magyar.mitm.proxy.help.AbstractComplexProxyTool;
7+
import website.magyar.mitm.proxy.help.ContentEncoding;
8+
import website.magyar.mitm.proxy.help.DefaultRequestInterceptor;
9+
import website.magyar.mitm.proxy.help.DefaultResponseInterceptor;
10+
import website.magyar.mitm.proxy.help.ResponseInfo;
11+
12+
import static org.junit.Assert.assertEquals;
13+
import static org.junit.Assert.assertNull;
14+
import static org.junit.Assert.assertTrue;
15+
16+
public class ContentEncodingWithProxyTest extends AbstractComplexProxyTool {
17+
18+
@Override
19+
protected void setUp() {
20+
String stubUrl = "http://127.0.0.1:" + getStubServerPort() + "/stub";
21+
LOGGER.info("STUB URL used: {}", stubUrl);
22+
DefaultRequestInterceptor defaultRequestInterceptor = new DefaultRequestInterceptor(getRequestCount(), NEED_STUB_RESPONSE, stubUrl);
23+
DefaultResponseInterceptor defaultResponseInterceptor = new DefaultResponseInterceptor(getResponseCount());
24+
getProxyServer().addRequestInterceptor(defaultRequestInterceptor);
25+
getProxyServer().addResponseInterceptor(defaultResponseInterceptor);
26+
getProxyServer().setCaptureBinaryContent(true);
27+
getProxyServer().setCaptureContent(true);
28+
}
29+
30+
private void checkEncoding(Header contentEncodingHeader, ContentEncoding contentEncoding) {
31+
if (contentEncoding == ContentEncoding.NONE) {
32+
assertNull(contentEncodingHeader);
33+
} else {
34+
assertTrue("Response is not encoded with:" + contentEncoding.getValue(),
35+
contentEncodingHeader != null && contentEncodingHeader.getValue().contains(contentEncoding.getValue()));
36+
}
37+
}
38+
39+
@Test
40+
public void testSimpleGetRequest_NoEncoding() throws Exception {
41+
ContentEncoding contentEncoding = ContentEncoding.NONE;
42+
ResponseInfo proxiedResponse = httpGetWithApacheClient(getWebHost(), NO_NEED_STUB_RESPONSE, true, false, contentEncoding);
43+
assertEquals(200, proxiedResponse.getStatusCode());
44+
assertEquals(SERVER_BACKEND, proxiedResponse.getBody());
45+
assertEquals(1, getResponseCount().get());
46+
assertEquals(1, getRequestCount().get());
47+
checkEncoding(proxiedResponse.getContentEncoding(), contentEncoding);
48+
}
49+
50+
@Test
51+
public void testSimpleGetRequest_GzipEncoding() throws Exception {
52+
ContentEncoding contentEncoding = ContentEncoding.GZIP;
53+
ResponseInfo proxiedResponse = httpGetWithApacheClient(getWebHost(), NO_NEED_STUB_RESPONSE, true, false, contentEncoding);
54+
assertEquals(200, proxiedResponse.getStatusCode());
55+
assertEquals(SERVER_BACKEND, proxiedResponse.getBody());
56+
assertEquals(1, getResponseCount().get());
57+
assertEquals(1, getRequestCount().get());
58+
//gzipped response is decoded by proxy - no checkEncoding(proxiedResponse.getContentEncoding(), contentEncoding);
59+
}
60+
61+
@Test
62+
public void testSimpleGetRequest_Deflate() throws Exception {
63+
ContentEncoding contentEncoding = ContentEncoding.DEFLATE;
64+
ResponseInfo proxiedResponse = httpGetWithApacheClient(getWebHost(), NO_NEED_STUB_RESPONSE, true, false, contentEncoding);
65+
assertEquals(200, proxiedResponse.getStatusCode());
66+
assertEquals(SERVER_BACKEND, proxiedResponse.getBody());
67+
assertEquals(1, getResponseCount().get());
68+
assertEquals(1, getRequestCount().get());
69+
//deflated response is decoded by proxy - no checkEncoding(proxiedResponse.getContentEncoding(), contentEncoding);
70+
}
71+
72+
@Test
73+
public void testSimpleGetRequest_Brotli() throws Exception {
74+
ContentEncoding contentEncoding = ContentEncoding.BROTLI;
75+
ResponseInfo proxiedResponse = httpGetWithApacheClient(getWebHost(), NO_NEED_STUB_RESPONSE, true, false, contentEncoding);
76+
assertEquals(200, proxiedResponse.getStatusCode());
77+
assertEquals(SERVER_BACKEND, proxiedResponse.getBody());
78+
assertEquals(1, getResponseCount().get());
79+
assertEquals(1, getRequestCount().get());
80+
checkEncoding(proxiedResponse.getContentEncoding(), contentEncoding);
81+
}
82+
83+
@Test
84+
public void testSimpleGetRequest_Any() throws Exception {
85+
ContentEncoding contentEncoding = ContentEncoding.ANY;
86+
ResponseInfo proxiedResponse = httpGetWithApacheClient(getWebHost(), NO_NEED_STUB_RESPONSE, true, false, contentEncoding);
87+
assertEquals(200, proxiedResponse.getStatusCode());
88+
assertEquals(SERVER_BACKEND, proxiedResponse.getBody());
89+
assertEquals(1, getResponseCount().get());
90+
assertEquals(1, getRequestCount().get());
91+
//received content encoding is not checked
92+
}
93+
94+
}

0 commit comments

Comments
 (0)