Skip to content

Commit d807259

Browse files
paxtonharepaxtonhare
committed
prepping 2.0.3.1 patch
1 parent 94ab9b1 commit d807259

File tree

6 files changed

+309
-29
lines changed

6 files changed

+309
-29
lines changed

‎marklogic-data-hub/src/main/java/com/marklogic/hub/collector/impl/CollectorImpl.java‎

Lines changed: 10 additions & 9 deletions
Original file line numberDiff line numberDiff line change
@@ -103,7 +103,7 @@ public DiskQueue<String> run(String jobId, String entity, String flow, int threa
103103
RestTemplate template = newRestTemplate(appConfig.getAppServicesUsername(), appConfig.getAppServicesPassword());
104104
String uriString = String.format(
105105
"%s://%s:%d%s?job-id=%s&entity-name=%s&flow-name=%s&database=%s",
106-
"http",
106+
client.getSecurityContext().getSSLContext() != null ? "https" : "http",
107107
client.getHost(),
108108
client.getPort(),
109109
"/com.marklogic.hub/endpoints/collector.xqy",
@@ -120,15 +120,16 @@ public DiskQueue<String> run(String jobId, String entity, String flow, int threa
120120
URI uri = new URI(uriString);
121121
HttpHeaders headers = new HttpHeaders();
122122
headers.set("Accept", MediaType.TEXT_PLAIN_VALUE);
123-
InputStream inputStream = template.exchange(uri, HttpMethod.GET, new HttpEntity<>(headers), Resource.class).getBody().getInputStream();
124-
125-
BufferedReader bufferedReader = new BufferedReader(new InputStreamReader(inputStream));
126-
String line;
127-
while((line = bufferedReader.readLine()) != null) {
128-
results.add(line);
123+
Resource responseBody = template.exchange(uri, HttpMethod.GET, new HttpEntity<>(headers), Resource.class).getBody();
124+
if(responseBody != null) {
125+
InputStream inputStream = responseBody.getInputStream();
126+
BufferedReader bufferedReader = new BufferedReader(new InputStreamReader(inputStream));
127+
String line;
128+
while((line = bufferedReader.readLine()) != null) {
129+
results.add(line);
130+
}
131+
inputStream.close();
129132
}
130-
inputStream.close();
131-
132133
return results;
133134
}
134135
catch(Exception e) {

‎marklogic-data-hub/src/main/resources/ml-modules/root/com.marklogic.hub/lib/hub-entities.xqy‎

Lines changed: 11 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -5,6 +5,8 @@ module namespace hent = "http://marklogic.com/data-hub/hub-entities";
55
import module namespace es = "http://marklogic.com/entity-services"
66
at "/MarkLogic/entity-services/entity-services.xqy";
77

8+
declare namespace search = "http://marklogic.com/appservices/search";
9+
810
declare variable $ENTITY-MODEL-COLLECTION := "http://marklogic.com/entity-services/models";
911

1012
declare option xdmp:mapping "false";
@@ -81,6 +83,15 @@ declare %private function hent:fix-options($nodes as node()*)
8183
for $n in $nodes
8284
return
8385
typeswitch($n)
86+
case element(search:options) return
87+
element { fn:node-name($n) } {
88+
<search:constraint name="Collection">
89+
<search:collection/>
90+
</search:constraint>,
91+
hent:fix-options(($n/@*, $n/node()))
92+
}
93+
case element(search:additional-query) return ()
94+
case element(search:return-facets) return <search:return-facets>true</search:return-facets>
8495
case element() return
8596
element { fn:node-name($n) } { hent:fix-options(($n/@*, $n/node())) }
8697
case text() return

‎marklogic-data-hub/src/test/java/com/marklogic/hub/collector/EmptyCollectorTest.java‎

Lines changed: 99 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,99 @@
1+
package com.marklogic.hub.collector;
2+
3+
import com.fasterxml.jackson.databind.JsonNode;
4+
import com.marklogic.client.datamovement.DataMovementManager;
5+
import com.marklogic.client.datamovement.JobTicket;
6+
import com.marklogic.client.datamovement.WriteBatcher;
7+
import com.marklogic.client.io.DocumentMetadataHandle;
8+
import com.marklogic.client.io.Format;
9+
import com.marklogic.client.io.JacksonHandle;
10+
import com.marklogic.client.io.StringHandle;
11+
import com.marklogic.hub.DataHub;
12+
import com.marklogic.hub.FlowManager;
13+
import com.marklogic.hub.HubConfig;
14+
import com.marklogic.hub.HubTestBase;
15+
import com.marklogic.hub.flow.*;
16+
import com.marklogic.hub.scaffold.Scaffolding;
17+
import com.marklogic.hub.util.FileUtil;
18+
import org.apache.commons.io.FileUtils;
19+
import org.custommonkey.xmlunit.XMLUnit;
20+
import org.junit.AfterClass;
21+
import org.junit.Before;
22+
import org.junit.BeforeClass;
23+
import org.junit.Test;
24+
25+
import java.io.File;
26+
import java.io.IOException;
27+
import java.nio.file.Path;
28+
import java.nio.file.Paths;
29+
import java.util.ArrayList;
30+
import java.util.HashMap;
31+
32+
import static org.junit.Assert.assertFalse;
33+
import static org.junit.Assert.assertTrue;
34+
import static org.junit.Assert.assertEquals;
35+
36+
public class EmptyCollectorTest extends HubTestBase {
37+
38+
private static final String ENTITY = "streamentity";
39+
private static Path projectDir = Paths.get(".", "ye-olde-project");
40+
41+
@BeforeClass
42+
public static void setup() throws IOException {
43+
XMLUnit.setIgnoreWhitespace(true);
44+
File projectDirFile = projectDir.toFile();
45+
if (projectDirFile.isDirectory() && projectDirFile.exists()) {
46+
FileUtils.deleteDirectory(projectDirFile);
47+
}
48+
49+
createProjectDir();
50+
51+
installHub();
52+
53+
Scaffolding scaffolding = new Scaffolding(projectDir.toString(), stagingClient);
54+
scaffolding.createEntity(ENTITY);
55+
scaffolding.createFlow(ENTITY, "testharmonize", FlowType.HARMONIZE,
56+
CodeFormat.XQUERY, DataFormat.XML);
57+
58+
DataHub dh = new DataHub(getHubConfig());
59+
dh.clearUserModules();
60+
installUserModules(getHubConfig(), false);
61+
clearDatabases(HubConfig.DEFAULT_STAGING_NAME, HubConfig.DEFAULT_FINAL_NAME, HubConfig.DEFAULT_TRACE_NAME, HubConfig.DEFAULT_JOB_NAME);
62+
}
63+
64+
65+
@AfterClass
66+
public static void teardown() {
67+
uninstallHub();
68+
}
69+
70+
@Test
71+
public void runCollector() {
72+
assertEquals(0, getStagingDocCount());
73+
assertEquals(0, getFinalDocCount());
74+
FlowManager fm = new FlowManager(getHubConfig());
75+
Flow harmonizeFlow = fm.getFlow(ENTITY, "testharmonize",
76+
FlowType.HARMONIZE);
77+
HashMap<String, Object> options = new HashMap<>();
78+
79+
// a sneaky attempt to test passing options. this value makes the collector work.
80+
options.put("returnStuff", true);
81+
FlowRunner flowRunner = fm.newFlowRunner()
82+
.withFlow(harmonizeFlow)
83+
.withBatchSize(10)
84+
.withThreadCount(1)
85+
.withOptions(options)
86+
.withStopOnFailure(true);
87+
JobTicket ticket = flowRunner.run();
88+
flowRunner.awaitCompletion();
89+
assertEquals(0, getFinalDocCount());
90+
91+
JsonNode node = jobDocMgr.read("/jobs/" + ticket.getJobId() + ".json").next().getContent(new JacksonHandle()).get();
92+
assertEquals(ticket.getJobId(), node.get("jobId").asText());
93+
assertEquals(0, node.get("successfulEvents").asInt());
94+
assertEquals(0, node.get("failedEvents").asInt());
95+
assertEquals(0, node.get("failedBatches").asInt());
96+
assertEquals("FAILED", node.get("status").asText());
97+
98+
}
99+
}

‎marklogic-data-hub/src/test/resources/entity-manager-test/options.xml‎

Lines changed: 4 additions & 7 deletions
Original file line numberDiff line numberDiff line change
@@ -1,5 +1,8 @@
11
<?xml version="1.0" encoding="UTF-8"?>
22
<search:options xmlns:search="http://marklogic.com/appservices/search">
3+
<search:constraint name="Collection">
4+
<search:collection/>
5+
</search:constraint>
36
<search:constraint name="entity-type">
47
<search:value>
58
<search:element ns="http://marklogic.com/entity-services" name="title"/>
@@ -43,14 +46,8 @@
4346
<search:extract-path>//*:instance/(Employee)</search:extract-path>
4447
</search:extract-document-data>
4548
<!--Change or remove this additional-query to broaden search beyond entity instance documents-->
46-
<search:additional-query>
47-
<cts:element-query xmlns:cts="http://marklogic.com/cts">
48-
<cts:element>*:instance</cts:element>
49-
<cts:true-query/>
50-
</cts:element-query>
51-
</search:additional-query>
5249
<!--To return facets, change this option to 'true' and edit constraints-->
53-
<search:return-facets>false</search:return-facets>
50+
<search:return-facets>true</search:return-facets>
5451
<!--To return snippets, comment out or remove this option-->
5552
<search:transform-results apply="empty-snippet"/>
5653
</search:options>

‎ml-data-hub-plugin/src/test/groovy/com/marklogic/gradle/task/SslTest.groovy‎

Lines changed: 89 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -2,11 +2,16 @@ package com.marklogic.gradle.task
22

33
import com.marklogic.client.DatabaseClientFactory
44
import com.marklogic.client.ext.modulesloader.ssl.SimpleX509TrustManager
5+
import com.marklogic.client.io.DOMHandle
6+
import com.marklogic.client.io.DocumentMetadataHandle
7+
import com.marklogic.hub.HubConfig
58
import org.gradle.testkit.runner.UnexpectedBuildFailure
69

710
import javax.net.ssl.SSLContext
811
import javax.net.ssl.TrustManager
912

13+
import static org.custommonkey.xmlunit.XMLAssert.assertXMLEqual
14+
import static org.custommonkey.xmlunit.XMLAssert.assertXMLEqual
1015
import static org.gradle.testkit.runner.TaskOutcome.SUCCESS
1116

1217

@@ -68,18 +73,18 @@ class SslTest extends BaseTest {
6873
manageClient.putJson("/manage/v2/servers/Admin/properties?group-id=Default", '{"ssl-certificate-template": ""}')
6974
manageClient.putJson("/manage/v2/servers/App-Services/properties?group-id=Default", '{"ssl-certificate-template": ""}')
7075
manageClient.putJson("/manage/v2/servers/Manage/properties?group-id=Default", '{"ssl-certificate-template": ""}')
71-
76+
7277
def adminConfig = getProject().property("mlAdminConfig")
7378
adminConfig.setScheme("http")
7479
adminConfig.setConfigureSimpleSsl(false)
7580
def adminManager = new com.marklogic.mgmt.admin.AdminManager(adminConfig)
7681
adminManager.waitForRestart()
77-
82+
7883
def manageConfig = getProject().property("mlManageConfig")
7984
manageConfig.setScheme("http")
8085
manageConfig.setConfigureSimpleSsl(false)
8186
def mgClient = new com.marklogic.mgmt.ManageClient(manageConfig)
82-
87+
8388
def certManager = new com.marklogic.mgmt.resource.security.CertificateTemplateManager(mgClient)
8489
certManager.delete(adminCert())
8590
}
@@ -192,4 +197,85 @@ class SslTest extends BaseTest {
192197
modCount == 83 || modCount == 63
193198
result.task(":mlDeploy").outcome == SUCCESS
194199
}
200+
201+
def "runHarmonizeFlow with default src and dest"() {
202+
given:
203+
println(runTask('hubCreateHarmonizeFlow', '-PentityName=my-new-entity', '-PflowName=my-new-harmonize-flow', '-PdataFormat=xml', '-PpluginFormat=xqy').getOutput())
204+
println(runTask('mlReLoadModules'))
205+
206+
def newSslContext = SSLContext.getInstance("TLSv1.2")
207+
newSslContext.init(null, [new SimpleX509TrustManager()] as TrustManager[], null)
208+
hubConfig().stagingSslContext = newSslContext
209+
hubConfig().stagingSslHostnameVerifier = DatabaseClientFactory.SSLHostnameVerifier.ANY
210+
hubConfig().finalSslContext = newSslContext
211+
hubConfig().finalSslHostnameVerifier = DatabaseClientFactory.SSLHostnameVerifier.ANY
212+
213+
clearDatabases(HubConfig.DEFAULT_STAGING_NAME, HubConfig.DEFAULT_FINAL_NAME)
214+
215+
assert (getStagingDocCount() == 0)
216+
assert (getFinalDocCount() == 0)
217+
218+
DocumentMetadataHandle meta = new DocumentMetadataHandle();
219+
meta.getCollections().add("my-new-entity");
220+
installStagingDoc("/employee1.xml", meta, new File("src/test/resources/run-flow-test/employee1.xml").text)
221+
installStagingDoc("/employee2.xml", meta, new File("src/test/resources/run-flow-test/employee2.xml").text)
222+
assert (getStagingDocCount() == 2)
223+
assert (getFinalDocCount() == 0)
224+
225+
installModule("/entities/my-new-entity/harmonize/my-new-harmonize-flow/content/content.xqy", "run-flow-test/content.xqy")
226+
227+
when:
228+
println(runTask('hubRunFlow', '-PentityName=my-new-entity', '-PflowName=my-new-harmonize-flow', '-i').getOutput())
229+
230+
then:
231+
notThrown(UnexpectedBuildFailure)
232+
getStagingDocCount() == 2
233+
getFinalDocCount() == 2
234+
assertXMLEqual(getXmlFromResource("run-flow-test/harmonized1.xml"), hubConfig().newFinalClient().newDocumentManager().read("/employee1.xml").next().getContent(new DOMHandle()).get())
235+
assertXMLEqual(getXmlFromResource("run-flow-test/harmonized2.xml"), hubConfig().newFinalClient().newDocumentManager().read("/employee2.xml").next().getContent(new DOMHandle()).get())
236+
}
237+
238+
def "runHarmonizeFlow with swapped src and dest"() {
239+
given:
240+
println(runTask('hubCreateHarmonizeFlow', '-PentityName=my-new-entity', '-PflowName=my-new-harmonize-flow', '-PdataFormat=xml', '-PpluginFormat=xqy').getOutput())
241+
println(runTask('mlReLoadModules'))
242+
243+
def newSslContext = SSLContext.getInstance("TLSv1.2")
244+
newSslContext.init(null, [new SimpleX509TrustManager()] as TrustManager[], null)
245+
hubConfig().stagingSslContext = newSslContext
246+
hubConfig().stagingSslHostnameVerifier = DatabaseClientFactory.SSLHostnameVerifier.ANY
247+
hubConfig().finalSslContext = newSslContext
248+
hubConfig().finalSslHostnameVerifier = DatabaseClientFactory.SSLHostnameVerifier.ANY
249+
250+
clearDatabases(HubConfig.DEFAULT_STAGING_NAME, HubConfig.DEFAULT_FINAL_NAME)
251+
assert (getStagingDocCount() == 0)
252+
assert (getFinalDocCount() == 0)
253+
254+
DocumentMetadataHandle meta = new DocumentMetadataHandle();
255+
meta.getCollections().add("my-new-entity");
256+
installFinalDoc("/employee1.xml", meta, new File("src/test/resources/run-flow-test/employee1.xml").text)
257+
installFinalDoc("/employee2.xml", meta, new File("src/test/resources/run-flow-test/employee2.xml").text)
258+
259+
assert (getStagingDocCount() == 0)
260+
assert (getFinalDocCount() == 2)
261+
installModule("/entities/my-new-entity/harmonize/my-new-harmonize-flow/content/content.xqy", "run-flow-test/content.xqy")
262+
263+
when:
264+
println(runTask(
265+
'hubRunFlow',
266+
'-PentityName=my-new-entity',
267+
'-PflowName=my-new-harmonize-flow',
268+
'-PsourceDB=data-hub-FINAL',
269+
'-PdestDB=data-hub-STAGING',
270+
'-i'
271+
).getOutput())
272+
273+
then:
274+
notThrown(UnexpectedBuildFailure)
275+
getStagingDocCount() == 2
276+
getFinalDocCount() == 2
277+
278+
assertXMLEqual(getXmlFromResource("run-flow-test/harmonized1.xml"), hubConfig().newStagingClient().newDocumentManager().read("/employee1.xml").next().getContent(new DOMHandle()).get())
279+
assertXMLEqual(getXmlFromResource("run-flow-test/harmonized2.xml"), hubConfig().newStagingClient().newDocumentManager().read("/employee2.xml").next().getContent(new DOMHandle()).get())
280+
}
195281
}

0 commit comments

Comments
 (0)