Skip to content

Commit de58697

Browse files
srinathgitSameeraPriyathamTadikonda
srinathgit
authored and
SameeraPriyathamTadikonda
committed
DHFPROD-6162:Enable writing temporal documents
1 parent d6624e8 commit de58697

File tree

14 files changed

+404
-67
lines changed

14 files changed

+404
-67
lines changed

‎marklogic-data-hub/src/main/resources/hub-internal-config/security/amps/amps-dhf-get-temporal-collections.json‎

Lines changed: 6 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,6 @@
1+
{
2+
"local-name" : "getTemporalCollections",
3+
"document-uri" : "/data-hub/5/temporal/hub-temporal.sjs",
4+
"modules-database" : "%%mlModulesDbName%%",
5+
"role" : [ "temporal-admin" ]
6+
}

‎marklogic-data-hub/src/main/resources/ml-modules/root/data-hub/5/impl/hub-utils.sjs‎

Lines changed: 4 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -53,8 +53,8 @@ class HubUtils {
5353
writeDocuments(writeQueue, permissions = 'xdmp.defaultPermissions()', collections = [], database = xdmp.databaseName(xdmp.database())){
5454
return fn.head(xdmp.eval(`
5555
const temporal = require("/MarkLogic/temporal.xqy");
56-
57-
const temporalCollections = temporal.collections().toArray().reduce((acc, col) => {
56+
const temporalLib = require("/data-hub/5/temporal/hub-temporal.sjs");
57+
const temporalCollections = temporalLib.getTemporalCollections().toArray().reduce((acc, col) => {
5858
acc[col] = true;
5959
return acc;
6060
}, {});
@@ -79,10 +79,11 @@ class HubUtils {
7979
if (metadata) {
8080
delete metadata.temporalDocURI;
8181
}
82+
const collectionsReservedForTemporal = ['latest', content.uri];
8283
temporal.documentInsert(temporalCollection, content.uri, content.value,
8384
{
8485
permissions,
85-
collections: collections.filter((col) => !temporalCollections[col]),
86+
collections: collections.filter((col) => !(temporalCollections[col] || collectionsReservedForTemporal.includes(col))),
8687
metadata
8788
}
8889
);

‎marklogic-data-hub/src/main/resources/ml-modules/root/data-hub/5/temporal/hub-temporal.sjs‎

Lines changed: 24 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,24 @@
1+
/*
2+
* Copyright 2020 MarkLogic Corporation
3+
*
4+
* Licensed under the Apache License, Version 2.0 (the "License");
5+
* you may not use this file except in compliance with the License.
6+
* You may obtain a copy of the License at
7+
*
8+
* http://www.apache.org/licenses/LICENSE-2.0
9+
*
10+
* Unless required by applicable law or agreed to in writing, software
11+
* distributed under the License is distributed on an "AS IS" BASIS,
12+
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
13+
* See the License for the specific language governing permissions and
14+
* limitations under the License.
15+
*/
16+
'use strict';
17+
const temporal = require("/MarkLogic/temporal.xqy");
18+
19+
module.exports.getTemporalCollections = module.amp(
20+
function getTemporalCollections() {
21+
return temporal.collections();
22+
}
23+
);
24+

‎marklogic-data-hub/src/test/java/com/marklogic/hub/HubTestBase.java‎

Lines changed: 71 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -55,6 +55,7 @@
5555
import com.marklogic.mgmt.ManageClient;
5656
import com.marklogic.mgmt.ManageConfig;
5757
import com.marklogic.mgmt.admin.AdminConfig;
58+
import com.marklogic.mgmt.resource.databases.DatabaseManager;
5859
import com.marklogic.mgmt.util.ObjectMapperFactory;
5960
import org.apache.commons.io.FileUtils;
6061
import org.apache.commons.io.FilenameUtils;
@@ -66,6 +67,7 @@
6667
import org.slf4j.LoggerFactory;
6768
import org.springframework.beans.factory.annotation.Autowired;
6869
import org.springframework.context.ApplicationContext;
70+
import org.springframework.core.io.ClassPathResource;
6971
import org.springframework.stereotype.Component;
7072
import org.springframework.util.ReflectionUtils;
7173
import org.w3c.dom.Document;
@@ -462,7 +464,7 @@ protected HubConfigImpl runAsDataHubOperator() {
462464
protected HubConfigImpl runAsAdmin() {
463465
return runAsUser("test-admin-for-data-hub-tests", "password");
464466
}
465-
467+
466468
protected HubConfigImpl runAsUser(String mlUsername, String mlPassword) {
467469
adminHubConfig.setMlUsername(mlUsername);
468470
adminHubConfig.setMlPassword(mlPassword);
@@ -1295,4 +1297,72 @@ protected void verifyCollectionCountsFromRunningTestFlow() {
12951297
assertEquals(1, getDocCount(HubConfig.DEFAULT_FINAL_NAME, "json-map"));
12961298
assertEquals(1, getDocCount(HubConfig.DEFAULT_FINAL_NAME, "xml-map"));
12971299
}
1300+
protected ObjectNode getDatabaseProperties(String database) {
1301+
DatabaseManager mgr = new DatabaseManager(adminHubConfig.getManageClient());
1302+
try {
1303+
return (ObjectNode) ObjectMapperFactory.getObjectMapper().readTree(mgr.getPropertiesAsJson(database));
1304+
} catch (IOException e) {
1305+
throw new RuntimeException(e);
1306+
}
1307+
}
1308+
1309+
/**
1310+
* Intended to make it easy to specify a set of project files to load for a particular test.
1311+
*
1312+
* @param folderInClasspath
1313+
*/
1314+
protected void installProjectInFolder(String folderInClasspath) {
1315+
long start = System.currentTimeMillis();
1316+
HubProject hubProject = runAsDataHubDeveloper().getHubProject();
1317+
try {
1318+
File testProjectDir = new ClassPathResource(folderInClasspath).getFile();
1319+
1320+
File dataDir = new File(testProjectDir, "data");
1321+
if (dataDir.exists()) {
1322+
FileUtils.copyDirectory(dataDir, new File(hubProject.getProjectDir().toFile(), "data"));
1323+
}
1324+
1325+
File entitiesDir = new File(testProjectDir, "entities");
1326+
if (entitiesDir.exists()) {
1327+
FileUtils.copyDirectory(entitiesDir, hubProject.getHubEntitiesDir().toFile());
1328+
}
1329+
1330+
File flowsDir = new File(testProjectDir, "flows");
1331+
if (flowsDir.exists()) {
1332+
FileUtils.copyDirectory(flowsDir, hubProject.getFlowsDir().toFile());
1333+
}
1334+
1335+
File inputDir = new File(testProjectDir, "input");
1336+
if (inputDir.exists()) {
1337+
FileUtils.copyDirectory(inputDir, new File(hubProject.getProjectDir().toFile(), "input"));
1338+
}
1339+
1340+
File mappingsDir = new File(testProjectDir, "mappings");
1341+
if (mappingsDir.exists()) {
1342+
FileUtils.copyDirectory(mappingsDir, hubProject.getHubMappingsDir().toFile());
1343+
}
1344+
1345+
File stepDefinitionsDir = new File(testProjectDir, "step-definitions");
1346+
if (stepDefinitionsDir.exists()) {
1347+
FileUtils.copyDirectory(stepDefinitionsDir, hubProject.getStepDefinitionsDir().toFile());
1348+
}
1349+
1350+
File modulesDir = new File(testProjectDir, "modules");
1351+
if (modulesDir.exists()) {
1352+
FileUtils.copyDirectory(modulesDir, hubProject.getModulesDir().toFile());
1353+
}
1354+
1355+
File configDir = new File(testProjectDir, "ml-config");
1356+
if (configDir.exists()) {
1357+
FileUtils.copyDirectory(configDir, hubProject.getUserConfigDir().toFile());
1358+
}
1359+
} catch (IOException e) {
1360+
throw new RuntimeException("Unable to load project files: " + e.getMessage(), e);
1361+
}
1362+
1363+
installUserModules(runAsDataHubDeveloper(), true);
1364+
1365+
logger.info("Installed project from folder in classpath: " + folderInClasspath + "; time: " +
1366+
(System.currentTimeMillis() - start));
1367+
}
12981368
}

‎marklogic-data-hub/src/test/java/com/marklogic/hub/deploy/commands/DeployDatabaseFieldCommandTest.java‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -30,7 +30,7 @@ public void test() {
3030
}
3131

3232
private void givenTheFinalDatabaseHasACustomFieldAndIndexes() {
33-
ObjectNode db = getFinalDatabaseProperties();
33+
ObjectNode db = getDatabaseProperties("data-hub-FINAL");
3434

3535
ObjectNode newNode = ObjectMapperFactory.getObjectMapper().createObjectNode();
3636
newNode.put("database-name", "data-hub-FINAL");

‎marklogic-data-hub/src/test/java/com/marklogic/hub/flow/TemporalWriteTest.java‎

Lines changed: 141 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,141 @@
1+
package com.marklogic.hub.flow;
2+
3+
import com.fasterxml.jackson.databind.ObjectMapper;
4+
import com.fasterxml.jackson.databind.node.ArrayNode;
5+
import com.fasterxml.jackson.databind.node.ObjectNode;
6+
import com.marklogic.appdeployer.command.temporal.DeployTemporalAxesCommand;
7+
import com.marklogic.appdeployer.command.temporal.DeployTemporalCollectionsCommand;
8+
import com.marklogic.appdeployer.impl.SimpleAppDeployer;
9+
import com.marklogic.client.io.DocumentMetadataHandle;
10+
import com.marklogic.client.io.JacksonHandle;
11+
import com.marklogic.hub.ApplicationConfig;
12+
import com.marklogic.hub.HubConfig;
13+
import com.marklogic.hub.HubTestBase;
14+
import com.marklogic.hub.flow.impl.FlowRunnerImpl;
15+
import com.marklogic.mgmt.resource.databases.DatabaseManager;
16+
import com.marklogic.mgmt.util.ObjectMapperFactory;
17+
import org.junit.jupiter.api.AfterEach;
18+
import org.junit.jupiter.api.Assertions;
19+
import org.junit.jupiter.api.BeforeEach;
20+
import org.junit.jupiter.api.Test;
21+
import org.junit.jupiter.api.extension.ExtendWith;
22+
import org.springframework.test.context.ContextConfiguration;
23+
import org.springframework.test.context.junit.jupiter.SpringExtension;
24+
25+
import static com.marklogic.client.io.DocumentMetadataHandle.Capability.*;
26+
import static org.junit.jupiter.api.Assertions.assertEquals;
27+
28+
@ExtendWith(SpringExtension.class)
29+
@ContextConfiguration(classes = ApplicationConfig.class)
30+
public class TemporalWriteTest extends HubTestBase {
31+
private FlowRunner flowRunner;
32+
33+
@BeforeEach
34+
void setUp() {
35+
runAsDataHubDeveloper();
36+
clearStagingFinalAndJobDatabases();
37+
installHubArtifacts(runAsDataHubDeveloper(), true);
38+
installProjectInFolder("test-projects/temporal-test");
39+
addFieldAndIndexes();
40+
new SimpleAppDeployer(new DeployTemporalAxesCommand(), new DeployTemporalCollectionsCommand())
41+
.deploy(adminHubConfig.getAppConfig());
42+
DocumentMetadataHandle meta = new DocumentMetadataHandle();
43+
meta.getCollections().add("testTemporal");
44+
meta.getPermissions().add("data-hub-operator", READ, UPDATE, EXECUTE);
45+
ObjectMapper mapper = new ObjectMapper();
46+
ObjectNode sourceNode = mapper.createObjectNode();
47+
sourceNode.put("content", "v1-content");
48+
stagingClient.newJSONDocumentManager().write("/test.json", meta, new JacksonHandle(sourceNode));
49+
}
50+
51+
@AfterEach
52+
void tearDown(){
53+
removeIndexesAndFields();
54+
new DatabaseManager(runAsAdmin().getManageClient()).clearDatabase(HubConfig.DEFAULT_STAGING_SCHEMAS_DB_NAME);
55+
}
56+
57+
@Test
58+
void writeTemporalFile() {
59+
flowRunner = new FlowRunnerImpl(runAsDataHubOperator());
60+
final String flowName = "temporal-test";
61+
flowRunner.runFlow(new FlowInputs(flowName));
62+
flowRunner.awaitCompletion();
63+
assertEquals(1, getDocCount(HubConfig.DEFAULT_STAGING_NAME, "temporal/test"));
64+
String updateTemporalDoc = "var temporal = require('/MarkLogic/temporal.xqy');\n" +
65+
"var root ={'envelope':{'headers':{}, 'triples':[], 'instance':{'content':'v2-content'}, 'attachments':null}};\n" +
66+
"declareUpdate();\n" +
67+
"temporal.documentInsert('temporal/test', '/temporal/ingestion/test.json', root, {permissions : [xdmp.permission('data-hub-operator', 'read'),xdmp.permission('data-hub-operator', 'update')]});";
68+
69+
try{
70+
runAsDataHubOperator().newStagingClient().newServerEval().javascript(updateTemporalDoc).eval();
71+
}
72+
catch (Exception e){
73+
logger.error("Document update failed: ", e);
74+
Assertions.fail("After the step is run, a temporal document should have been created and it's update should not fail. " +
75+
"If it's not a temporal document, temporal update would fail");
76+
}
77+
78+
String deleteTemporalDoc = "var temporal = require('/MarkLogic/temporal.xqy');\n" +
79+
"declareUpdate();\n" +
80+
"temporal.documentDelete('temporal/test', '/temporal/ingestion/test.json');";
81+
82+
try{
83+
runAsDataHubOperator().newStagingClient().newServerEval().javascript(deleteTemporalDoc).eval();
84+
}
85+
catch (Exception e){
86+
logger.error("Document deletion failed: ", e);
87+
Assertions.fail("dh-operator should be able to delete the document");
88+
}
89+
}
90+
91+
private void removeIndexesAndFields() {
92+
String indexesDeletion = " const admin = require('/MarkLogic/admin.xqy');\n" +
93+
" var config = admin.getConfiguration();\n" +
94+
" var dbid = xdmp.database('data-hub-STAGING');\n" +
95+
" var rangespec1 = admin.databaseRangeFieldIndex('dateTime', 'systemStart', '', fn.false());\n" +
96+
" var rangespec2 = admin.databaseRangeFieldIndex('dateTime', 'systemEnd', '', fn.false());\n" +
97+
" config = admin.databaseDeleteRangeFieldIndex(config, dbid, rangespec1);\n" +
98+
" config = admin.databaseDeleteRangeFieldIndex(config, dbid, rangespec2);\n" +
99+
" config = admin.databaseDeleteField(config, dbid, 'systemStart');\n" +
100+
" config = admin.databaseDeleteField(config, dbid, 'systemEnd');\n" +
101+
" admin.saveConfigurationWithoutRestart(config);";
102+
runAsAdmin().newStagingClient().newServerEval().javascript(indexesDeletion).eval();
103+
}
104+
105+
private void addFieldAndIndexes() {
106+
ObjectNode db = getDatabaseProperties("data-hub-STAGING");
107+
108+
ObjectNode newNode = ObjectMapperFactory.getObjectMapper().createObjectNode();
109+
newNode.put("database-name", "data-hub-STAGING");
110+
newNode.set("field", db.get("field"));
111+
112+
Object field = newNode.get("field");
113+
ObjectNode systemStart = ((ArrayNode) field).addObject();
114+
systemStart.put("field-name", "systemStart");
115+
systemStart.put("metadata", true);
116+
ObjectNode systemEnd = ((ArrayNode) field).addObject();
117+
systemEnd.put("field-name", "systemEnd");
118+
systemEnd.put("metadata", true);
119+
120+
121+
ArrayNode indexes;
122+
if (db.has("range-field-index")) {
123+
indexes = (ArrayNode) db.get("range-field-index");
124+
newNode.set("range-field-index", indexes);
125+
} else {
126+
indexes = newNode.putArray("range-field-index");
127+
}
128+
ObjectNode systemStartIndex = indexes.addObject();
129+
systemStartIndex.put("scalar-type", "dateTime");
130+
systemStartIndex.put("field-name", "systemStart");
131+
systemStartIndex.put("invalid-values", "reject");
132+
systemStartIndex.put("range-value-positions", false);
133+
134+
ObjectNode systemEndIndex = indexes.addObject();
135+
systemEndIndex.put("scalar-type", "dateTime");
136+
systemEndIndex.put("field-name", "systemEnd");
137+
systemEndIndex.put("invalid-values", "reject");
138+
systemEndIndex.put("range-value-positions", false);
139+
new DatabaseManager(adminHubConfig.getManageClient()).save(newNode.toString());
140+
}
141+
}

0 commit comments

Comments
 (0)