Skip to content

Commit 9a2e01e

Browse files
author
cduby
committed
[CYB-207] use https for mock server for all tests.
1 parent f5bfb2d commit 9a2e01e

File tree

12 files changed

+110
-217
lines changed

12 files changed

+110
-217
lines changed

‎flink-cyber/flink-enrichment/flink-enrichment-lookup-rest/src/main/java/com/cloudera/cyber/enrichment/rest/AsyncHttpRequest.java‎

Lines changed: 0 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -26,7 +26,6 @@
2626
import java.util.ArrayList;
2727
import java.util.Collection;
2828
import java.util.Collections;
29-
import java.util.function.Predicate;
3029
import java.util.List;
3130

3231
import static com.cloudera.cyber.DataQualityMessageLevel.ERROR;

‎flink-cyber/flink-enrichment/flink-enrichment-lookup-rest/src/main/java/com/cloudera/cyber/enrichment/rest/RestLookupJob.java‎

Lines changed: 2 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -20,7 +20,6 @@
2020
import com.fasterxml.jackson.databind.jsontype.BasicPolymorphicTypeValidator;
2121
import lombok.extern.slf4j.Slf4j;
2222
import org.apache.flink.api.java.utils.ParameterTool;
23-
import org.apache.flink.streaming.api.TimeCharacteristic;
2423
import org.apache.flink.streaming.api.datastream.AsyncDataStream;
2524
import org.apache.flink.streaming.api.datastream.DataStream;
2625
import org.apache.flink.streaming.api.environment.StreamExecutionEnvironment;
@@ -30,6 +29,7 @@
3029
import java.nio.file.Files;
3130
import java.nio.file.Paths;
3231
import java.util.ArrayList;
32+
import java.util.Base64;
3333
import java.util.List;
3434
import java.util.Map;
3535
import java.util.concurrent.TimeUnit;
@@ -60,7 +60,7 @@ private static DataStream<Message> enrich(DataStream<Message> source, List<RestE
6060
Preconditions.checkNotNull(config.getSources(),"specify a list of source names or ANY to match any source");
6161
Preconditions.checkArgument(!config.getSources().isEmpty(), "specify a list of source names or ANY to match any source");
6262
AsyncHttpRequest asyncHttpRequest = new AsyncHttpRequest(config);
63-
String processId = "rest-" + md5(source + config.getEndpointTemplate());
63+
String processId = "rest-" + Base64.getEncoder().encodeToString(md5(source + config.getEndpointTemplate()));
6464

6565
return AsyncDataStream.unorderedWait(
6666
in,
@@ -72,7 +72,6 @@ private static DataStream<Message> enrich(DataStream<Message> source, List<RestE
7272

7373
protected StreamExecutionEnvironment createPipeline(ParameterTool params) throws IOException {
7474
StreamExecutionEnvironment env = StreamExecutionEnvironment.getExecutionEnvironment();
75-
env.setStreamTimeCharacteristic(TimeCharacteristic.EventTime);
7675
env.enableCheckpointing(5000);
7776
FlinkUtils.setupEnv(env, params);
7877

‎flink-cyber/flink-enrichment/flink-enrichment-lookup-rest/src/test/java/com/cloudera/cyber/enrichment/rest/AsyncHttpRequestTest.java‎

Lines changed: 6 additions & 8 deletions
Original file line numberDiff line numberDiff line change
@@ -32,16 +32,14 @@
3232

3333
public class AsyncHttpRequestTest extends RestRequestTest {
3434

35-
private static MockRestServer mockRestServer;
36-
3735
@BeforeClass
38-
public static void startMockServer() {
39-
mockRestServer = new MockRestServer(false);
36+
public static void createMockServices() {
37+
startMockServer();
4038
}
4139

4240
@AfterClass
43-
public static void stopMockServer() {
44-
mockRestServer.close();
41+
public static void stopMockServices() {
42+
stopMockServer();
4543
}
4644

4745
private Message buildMessage(String source, String fieldName, String fieldValue) {
@@ -133,7 +131,7 @@ public void testGetError() throws Exception {
133131
}};
134132

135133
String mockHostAndPort = mockRestServer.getMockHostAndPort();
136-
String url = String.format("http://%s/asset?id=56", mockHostAndPort);
134+
String url = String.format("%s://%s/asset?id=56", MockRestServer.ENCRYPTED_PROTOCOL, mockHostAndPort);
137135
String exceptionMessage = String.format(RestRequest.REST_REQUEST_HTTP_FAILURE, "HTTP/1.1 503 Service Unavailable");
138136
List<DataQualityMessage> expectedQualityMessages = Collections.singletonList(DataQualityMessage.builder().
139137
feature(AsyncHttpRequest.REST_ENRICHMENT_FEATURE).
@@ -179,7 +177,7 @@ private void testGetAsset(String fieldName, String assetId, Map<String, Object>
179177
}
180178

181179
private void testGetAsset(ArrayList<String> enrichmentSources, String messageSource, String fieldName, String assetId, Map<String, Object> expectedExtensions, Collection<DataQualityMessage> expectedDataQualityMessages) throws Exception {
182-
AsyncHttpRequest request = new AsyncHttpRequest(mockRestServer.configureGetAssetRequest().sources(enrichmentSources).build());
180+
AsyncHttpRequest request = new AsyncHttpRequest(mockRestServer.configureGetAssetRequest(MockRestServer.ENCRYPTED_PROTOCOL).sources(enrichmentSources).build());
183181
request.open(new Configuration());
184182

185183
AsyncResult result = new AsyncResult();

‎flink-cyber/flink-enrichment/flink-enrichment-lookup-rest/src/test/java/com/cloudera/cyber/enrichment/rest/GetRestRequestTest.java‎

Lines changed: 17 additions & 12 deletions
Original file line numberDiff line numberDiff line change
@@ -15,25 +15,30 @@
1515
import com.cloudera.cyber.enrichment.rest.impl.MockRestServer;
1616
import com.google.common.collect.Lists;
1717
import lombok.extern.slf4j.Slf4j;
18+
import org.junit.AfterClass;
1819
import org.junit.Assert;
19-
import org.junit.Ignore;
20+
import org.junit.BeforeClass;
2021
import org.junit.Test;
2122

2223
import java.util.HashMap;
2324
import java.util.Map;
2425

2526
@Slf4j
26-
// tests will run either with or without tls when junit runs derived classes
27-
@Ignore
2827
public class GetRestRequestTest extends RestRequestTest {
2928

30-
public static void createMockService(boolean enableTlsMutualAuth) {
31-
startMockServer(enableTlsMutualAuth);
29+
@BeforeClass
30+
public static void createMockServices() {
31+
startMockServer();
32+
}
33+
34+
@AfterClass
35+
public static void stopMockServices() {
36+
stopMockServer();
3237
}
3338

3439
@Test
3540
public void testSimpleGet() throws Exception {
36-
RestEnrichmentConfig config = mockRestServer.configureGetUserRequest().build();
41+
RestEnrichmentConfig config = mockRestServer.configureGetUserRequest(MockRestServer.ENCRYPTED_PROTOCOL).build();
3742

3843
Map<String, String> variables = new HashMap<String, String>() {{
3944
put("name", "Chris");
@@ -44,7 +49,7 @@ public void testSimpleGet() throws Exception {
4449

4550
@Test
4651
public void testPropertiesNull() throws Exception {
47-
RestEnrichmentConfig config = mockRestServer.getBuilder(null).sources(Lists.newArrayList(MockRestServer.USER_SOURCE)).
52+
RestEnrichmentConfig config = mockRestServer.getBuilder(null, MockRestServer.ENCRYPTED_PROTOCOL).sources(Lists.newArrayList(MockRestServer.USER_SOURCE)).
4853
endpointTemplate(String.format("%s://%s/user?name=${name}", mockRestServer.getMockProtocol(), mockRestServer.getMockHostAndPort())).build();
4954

5055
Map<String, String> variables = new HashMap<String, String>() {{
@@ -57,18 +62,18 @@ public void testPropertiesNull() throws Exception {
5762

5863
@Test
5964
public void testBasicAuthGet() throws Exception {
60-
RestRequestResult result = makeAssetRequest(MockRestServer.ASSET_ID);
65+
RestRequestResult result = makeAssetRequest(MockRestServer.ASSET_ID, mockRestServer);
6166
Assert.assertEquals(MockRestServer.ASSET_LOCATION, result.getExtensions().get(MockRestServer.ASSET_LOCATION_PROPERTY));
6267
}
6368

6469
@Test
6570
public void testServerError() throws Exception {
66-
RestRequestResult result = makeAssetRequest(MockRestServer.SERVER_ERROR_ASSET_ID);
67-
verifyErrorResult(result, "Rest request url='%s://%s/asset?id=56' entity='null' failed 'Rest request failed due to 'HTTP/1.1 503 Service Unavailable'.'");
71+
RestRequestResult result = makeAssetRequest(MockRestServer.SERVER_ERROR_ASSET_ID, mockRestServer);
72+
verifyErrorResult(result, "Rest request url='%s://%s/asset?id=56' entity='null' failed 'Rest request failed due to 'HTTP/1.1 503 Service Unavailable'.'", mockRestServer);
6873
}
6974

70-
private RestRequestResult makeAssetRequest(String assetId) throws Exception {
71-
RestEnrichmentConfig config = mockRestServer.configureGetAssetRequest().build();
75+
private RestRequestResult makeAssetRequest(String assetId, MockRestServer mockRestServer) throws Exception {
76+
RestEnrichmentConfig config = mockRestServer.configureGetAssetRequest(MockRestServer.ENCRYPTED_PROTOCOL).build();
7277
Map<String, String> variables = new HashMap<String, String>() {{
7378
put("id", assetId);
7479
}};

‎flink-cyber/flink-enrichment/flink-enrichment-lookup-rest/src/test/java/com/cloudera/cyber/enrichment/rest/NonSecureGetRequestTest.java‎

Lines changed: 0 additions & 29 deletions
This file was deleted.

‎flink-cyber/flink-enrichment/flink-enrichment-lookup-rest/src/test/java/com/cloudera/cyber/enrichment/rest/NonSecurePostRestRequestTest.java‎

Lines changed: 0 additions & 30 deletions
This file was deleted.

‎flink-cyber/flink-enrichment/flink-enrichment-lookup-rest/src/test/java/com/cloudera/cyber/enrichment/rest/PostRestRequestTest.java‎

Lines changed: 49 additions & 18 deletions
Original file line numberDiff line numberDiff line change
@@ -15,8 +15,10 @@
1515
import com.cloudera.cyber.enrichment.rest.impl.MockRestServer;
1616
import com.google.common.collect.Lists;
1717
import org.apache.http.client.methods.HttpPost;
18+
import org.hamcrest.Matcher;
19+
import org.junit.AfterClass;
1820
import org.junit.Assert;
19-
import org.junit.Ignore;
21+
import org.junit.BeforeClass;
2022
import org.junit.Test;
2123

2224
import javax.annotation.Nonnull;
@@ -28,22 +30,29 @@
2830
import java.util.concurrent.CompletableFuture;
2931
import java.util.concurrent.Executors;
3032

31-
// tests will run either with or without tls when junit runs derived classes
32-
@Ignore
33+
import static org.hamcrest.Matchers.containsString;
34+
import static org.junit.Assert.assertThat;
35+
3336
public class PostRestRequestTest extends RestRequestTest {
3437
private static RestEnrichmentConfig modelResultPostRequest;
3538

36-
public static void createMockService(boolean enableTlsMutualAuth) {
37-
startMockServer(enableTlsMutualAuth);
38-
modelResultPostRequest = mockRestServer.configureModelPostRequest().build();
39+
@BeforeClass
40+
public static void createMockServices() {
41+
startMockServer();
42+
modelResultPostRequest = mockRestServer.configureModelPostRequest(MockRestServer.ENCRYPTED_PROTOCOL).build();
43+
}
44+
45+
@AfterClass
46+
public static void stopMockServices() {
47+
stopMockServer();
3948
}
4049

4150
@Test
4251
public void testSimplePostCModel() throws Exception {
4352
testSuccessfulModelPosts(modelResultPostRequest);
4453
}
4554

46-
protected void testSuccessfulModelPosts(RestEnrichmentConfig config) throws Exception {
55+
private void testSuccessfulModelPosts(RestEnrichmentConfig config) throws Exception {
4756
for(MockRestServer.ExpectedModelResult expectedModelResult : MockRestServer.expectedModelResults) {
4857
if (expectedModelResult.isSuccess()) {
4958
testDomain(config, expectedModelResult.getDomainName(), expectedModelResult.isLegit());
@@ -54,24 +63,24 @@ protected void testSuccessfulModelPosts(RestEnrichmentConfig config) throws Exce
5463
@Test
5564
public void testHttp200StatusButResultIndicatesError() throws Exception {
5665
String expectedErrorFormat = "Rest request url='%s://%s/model' entity='{\"accessKey\":\"mup8kz1hsl3erczwepbt8jupamita6y6\",\"request\":{\"domain\":\"unsuccessfuldomain\"}}' failed 'Rest request returned a success code but the content indicated failure.'";
57-
testHttpFailedRequest(MockRestServer.UNSUCCESSFUL_DOMAIN, expectedErrorFormat);
58-
66+
testHttpFailedRequest(MockRestServer.UNSUCCESSFUL_DOMAIN, expectedErrorFormat, modelResultPostRequest, mockRestServer);
5967
}
68+
6069
@Test
6170
public void testHttp400Exception() throws Exception {
6271
String expectedErrorFormat = "Rest request url='%s://%s/model' entity='{\"accessKey\":\"mup8kz1hsl3erczwepbt8jupamita6y6\",\"request\":{\"domain\":\"clienterror\"}}' failed 'Rest request failed due to 'HTTP/1.1 400 Bad Request'.'";
63-
testHttpFailedRequest(MockRestServer.CLIENT_ERROR_DOMAIN, expectedErrorFormat);
72+
testHttpFailedRequest(MockRestServer.CLIENT_ERROR_DOMAIN, expectedErrorFormat, modelResultPostRequest, mockRestServer);
6473
}
6574

66-
public void testHttpFailedRequest(String domainName, String errorMessageFormat) throws Exception {
75+
private void testHttpFailedRequest(String domainName, String errorMessageFormat, RestEnrichmentConfig modelResultPostRequest, MockRestServer mockRestServer) throws Exception {
6776
RestRequest request = modelResultPostRequest.createRestEnrichmentRequest();
6877
Map<String, String> variables = new HashMap<String, String>() {
6978
{
7079
put(MockRestServer.DOMAIN_EXTENSION_NAME, domainName);
7180
}
7281
};
7382
RestRequestResult result = request.getResult(true, variables).get();
74-
verifyErrorResult(result, errorMessageFormat);
83+
verifyErrorResult(result, errorMessageFormat, mockRestServer);
7584
}
7685

7786
private static class MockThrowingPostRequest extends PostRestRequest {
@@ -87,30 +96,52 @@ protected void addEntityToRequest(@Nonnull HttpPost postRequest, @Nonnull String
8796

8897
@Test
8998
public void testUnsupportedCharacterSetException() throws Exception {
90-
9199
MockThrowingPostRequest throwingPostRequest = new MockThrowingPostRequest(modelResultPostRequest);
92100
RestRequestKey key = throwingPostRequest.getKey(new HashMap<String, String>() {{ put(MockRestServer.DOMAIN_EXTENSION_NAME, "mydomain");}});
93101
CompletableFuture<RestRequestResult> future = new MockThrowingPostRequest(modelResultPostRequest).asyncLoad(key, Executors.newFixedThreadPool(1));
94102
RestRequestResult result = future.get();
95103
String expectedErrorFormat = "Rest request url='%s://%s/model' entity='{\"accessKey\":\"mup8kz1hsl3erczwepbt8jupamita6y6\",\"request\":{\"domain\":\"mydomain\"}}' failed 'Default UnsupportedEndcodingException message'";
96-
verifyErrorResult(result, expectedErrorFormat);
104+
verifyErrorResult(result, expectedErrorFormat, mockRestServer);
97105
}
98106

99107
@Test
100108
public void testUndefinedVariableWithAnyMatchIgnoresKeyErrors() throws Exception {
101-
RestEnrichmentConfig anyMatchRequest = mockRestServer.configureModelPostRequest().sources(Lists.newArrayList(AsyncHttpRequest.ANY_SOURCE_NAME)).build();
109+
RestEnrichmentConfig anyMatchRequest = mockRestServer.configureModelPostRequest(MockRestServer.ENCRYPTED_PROTOCOL).sources(Lists.newArrayList(AsyncHttpRequest.ANY_SOURCE_NAME)).build();
102110
// accessing an undefined field in a key with any source match does not report an error
103111
testDomain(anyMatchRequest, new ArrayList<>());
104112
}
105113

114+
@Test
115+
public void testTlsWithAlias() throws Exception {
116+
RestEnrichmentConfig.RestEnrichmentConfigBuilder modelResultPostRequestWithKeyAlias = mockRestServer.configureTLS(mockRestServer.configureModelPostRequest(MockRestServer.ENCRYPTED_PROTOCOL), "client");
117+
testSuccessfulModelPosts(modelResultPostRequestWithKeyAlias.build());
118+
}
119+
120+
@Test
121+
public void testTlsHandshakeException() throws Exception {
122+
RestEnrichmentConfig.RestEnrichmentConfigBuilder modelResultPostRequestWithKeyAlias = mockRestServer.configureTLS(mockRestServer.configureModelPostRequest(MockRestServer.ENCRYPTED_PROTOCOL), "nosuchkeyalias");
123+
RestRequest badHandshakePost = new PostRestRequest(modelResultPostRequestWithKeyAlias.build());
124+
MockRestServer.ExpectedModelResult expectedResult = MockRestServer.expectedModelResults.get(0);
125+
Map<String, String> extensions = new HashMap<String, String>() {{
126+
put(MockRestServer.DOMAIN_EXTENSION_NAME, expectedResult.getDomainName());
127+
}};
128+
RestRequestResult result = badHandshakePost.getResult(true, extensions).get();
129+
130+
Assert.assertTrue(result.getExtensions().isEmpty());
131+
List<String> errors = result.getErrors();
132+
Assert.assertEquals(1, errors.size());
133+
Matcher<String> expectedString = containsString(String.format("Rest request url='%s://%s/model' entity='{\"accessKey\":\"mup8kz1hsl3erczwepbt8jupamita6y6\",\"request\":{\"domain\":\"google\"}}'", mockRestServer.getMockProtocol(), mockRestServer.getMockHostAndPort()));
134+
assertThat(errors.get(0), expectedString);
135+
}
136+
106137
@Test
107138
public void testUndefinedVariableWithSpecificMatchReturnsKeyErrors() throws Exception {
108-
RestEnrichmentConfig anyMatchRequest = mockRestServer.configureModelPostRequest().build();
139+
RestEnrichmentConfig anyMatchRequest = mockRestServer.configureModelPostRequest(MockRestServer.ENCRYPTED_PROTOCOL).build();
109140
// accessing an undefined field in a key with a specific source match fails with a key error
110141
testDomain(anyMatchRequest, Lists.newArrayList("Variable(s) 'domain' required by rest entity are undefined"));
111142
}
112143

113-
protected void testDomain(RestEnrichmentConfig config, String domainName, Boolean legit) throws Exception {
144+
private void testDomain(RestEnrichmentConfig config, String domainName, Boolean legit) throws Exception {
114145
Map<String, String> variables = new HashMap<String, String>() {{
115146
put(MockRestServer.DOMAIN_EXTENSION_NAME, domainName);
116147
}};
@@ -119,7 +150,7 @@ protected void testDomain(RestEnrichmentConfig config, String domainName, Boolea
119150
Assert.assertTrue(result.getErrors().isEmpty());
120151
}
121152

122-
protected void testDomain(RestEnrichmentConfig config, List<String> expectedErrors) throws Exception {
153+
private void testDomain(RestEnrichmentConfig config, List<String> expectedErrors) throws Exception {
123154
Map<String, String> variables = new HashMap<String, String>() {{
124155
put("wrong field name", "testdomain");
125156
}};

‎flink-cyber/flink-enrichment/flink-enrichment-lookup-rest/src/test/java/com/cloudera/cyber/enrichment/rest/RestLookupJobTest.java‎

Lines changed: 3 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -59,9 +59,9 @@ public static void startMockRestServer() throws IOException {
5959
mockRestServer = new MockRestServer(true);
6060
File configFile = configTempFolder.newFile("rest-job-test.json");
6161
List<RestEnrichmentConfig> modelRestConfig = new ArrayList<>();
62-
modelRestConfig.add(mockRestServer.configureModelPostRequest().build());
63-
modelRestConfig.add(mockRestServer.configureGetAssetRequest().build());
64-
modelRestConfig.add(mockRestServer.configureGetUserRequest().build());
62+
modelRestConfig.add(mockRestServer.configureModelPostRequest(MockRestServer.ENCRYPTED_PROTOCOL).build());
63+
modelRestConfig.add(mockRestServer.configureGetAssetRequest(MockRestServer.ENCRYPTED_PROTOCOL).build());
64+
modelRestConfig.add(mockRestServer.configureGetUserRequest(MockRestServer.ENCRYPTED_PROTOCOL).build());
6565
ObjectWriter ow = getConfigObjectMapper().writerFor(new TypeReference<ArrayList<RestEnrichmentConfig>>() {
6666
});
6767
ow.writeValue(configFile, modelRestConfig);

‎flink-cyber/flink-enrichment/flink-enrichment-lookup-rest/src/test/java/com/cloudera/cyber/enrichment/rest/RestRequestTest.java‎

Lines changed: 9 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -22,8 +22,14 @@ public abstract class RestRequestTest {
2222
protected static MockRestServer mockRestServer;
2323

2424

25-
protected static void startMockServer(boolean enableTlsMutualAuth) {
26-
mockRestServer = new MockRestServer(enableTlsMutualAuth);
25+
protected static void startMockServer() {
26+
mockRestServer = new MockRestServer(true);
27+
}
28+
29+
protected static void stopMockServer() {
30+
if (mockRestServer != null) {
31+
mockRestServer.close();
32+
}
2733
}
2834

2935
protected RestRequestResult makeRequest(RestEnrichmentConfig config, Map<String, String> variables) throws Exception {
@@ -32,7 +38,7 @@ protected RestRequestResult makeRequest(RestEnrichmentConfig config, Map<String,
3238
}
3339
}
3440

35-
protected void verifyErrorResult(RestRequestResult result, String expectedErrorFormat) {
41+
protected void verifyErrorResult(RestRequestResult result, String expectedErrorFormat, MockRestServer mockRestServer) {
3642
Assert.assertTrue(result.getExtensions().isEmpty());
3743
List<String> errors = result.getErrors();
3844
Assert.assertEquals(1, errors.size());

0 commit comments

Comments
 (0)