Skip to content

Commit 974782e

Browse files
rahulvudutalaMarkLogic Builder
rahulvudutala
authored and
MarkLogic Builder
committed
DHFPROD-10566: Ignore flow when properties file is missing in legacy flow directory
1 parent 970aefd commit 974782e

File tree

9 files changed

+164
-25
lines changed

9 files changed

+164
-25
lines changed

‎marklogic-data-hub/src/main/java/com/marklogic/hub/impl/HubProjectImpl.java‎

Lines changed: 16 additions & 15 deletions
Original file line numberDiff line numberDiff line change
@@ -497,10 +497,10 @@ public void upgradeProject(FlowManager flowManager) throws IOException {
497497
}
498498

499499
public int upgradeLegacyFlows(FlowManager flowManager) {
500-
return upgradeLegacyFlows(flowManager, new ArrayList<>(), new ArrayList<>(), new ArrayList<>());
500+
return upgradeLegacyFlows(flowManager, new ArrayList<>(), new ArrayList<>(), new ArrayList<>(), "data-hub-STAGING", "data-hub-FINAL");
501501
}
502502

503-
public int upgradeLegacyFlows(FlowManager flowManager, List<String> legacyEntities, List<String> legacyFlowTypes, List<String> legacyFlowNames) {
503+
public int upgradeLegacyFlows(FlowManager flowManager, List<String> legacyEntities, List<String> legacyFlowTypes, List<String> legacyFlowNames, String sourceDb, String targetDb) {
504504
Set<String> legacyEntitiesSet = new HashSet<>(legacyEntities);
505505
Set<String> legacyFlowTypesSet = new HashSet<>(legacyFlowTypes);
506506
Set<String> legacyFlowNamesSet = new HashSet<>(legacyFlowNames);
@@ -579,12 +579,12 @@ public int upgradeLegacyFlows(FlowManager flowManager, List<String> legacyEntiti
579579
JsonNode stepPayLoad = scaffolding.getStepConfig(newStepName, stepType, newStepName, null, acceptSourceModule);
580580
// Save StepDefinition to local file
581581
scaffolding.saveStepDefinition(newStepName, newStepName, stepType, true);
582-
updateStepOptionsFor4xFlow(stepName, stepFile, stepPayLoad, mainModulePath, legacyEntityDir.getName());
582+
updateStepOptionsFor4xFlow(stepName, stepFile, stepPayLoad, mainModulePath, legacyEntityDir.getName(), sourceDb, targetDb);
583583
// Save Step to local file
584584
scaffolding.saveLocalStep(stepType, stepPayLoad);
585585
// Add step to local Flow
586586
ObjectNode stepIdObj = objectMapper.createObjectNode();
587-
steps.put(Integer.toString(++stepNumber), stepIdObj);
587+
steps.putIfAbsent(Integer.toString(++stepNumber), stepIdObj);
588588
stepIdObj.put("stepId", newStepName.concat("-").concat(stepType));
589589
flowsUpdated++;
590590
}
@@ -598,14 +598,14 @@ public int upgradeLegacyFlows(FlowManager flowManager, List<String> legacyEntiti
598598
return flowsUpdated;
599599
}
600600

601-
private void updateStepOptionsFor4xFlow(String stepName, File stepFile, JsonNode stepPayLoad, String mainModulePath, String entityType) {
601+
private void updateStepOptionsFor4xFlow(String stepName, File stepFile, JsonNode stepPayLoad, String mainModulePath, String entityType, String sourceDb, String finalDb) {
602602
ObjectNode step = (ObjectNode) stepPayLoad;
603603
ObjectMapper mapper = new ObjectMapper();
604604
Properties properties = new Properties();
605605
try {
606606
File propsFile = stepFile.listFiles((File file, String name) -> name.equals(stepName.concat(".properties")))[0];
607607
properties.load(Files.newInputStream(propsFile.toPath()));
608-
} catch (IOException e) {
608+
} catch (IOException | ArrayIndexOutOfBoundsException e) {
609609
logger.warn("%s.properties file is missing in the %s directory. The dataFormat and mainModule is defaulted to json and main.sjs" +
610610
"If the default values are inappropriate, change the values in steps/%s file", stepName, stepName, stepPayLoad.get("name").asText());
611611
properties.put("mainModule", "main.sjs");
@@ -614,30 +614,31 @@ private void updateStepOptionsFor4xFlow(String stepName, File stepFile, JsonNode
614614
ObjectNode optionsNode = mapper.createObjectNode();
615615
optionsNode.put("flow", stepName);
616616
optionsNode.put("entity", "");
617-
optionsNode.put("dataFormat", properties.get("dataFormat").toString());
618-
optionsNode.put("mainModuleUri", mainModulePath.concat("/").concat(properties.get("mainModule").toString()));
617+
optionsNode.put("dataFormat", properties.getOrDefault("dataFormat", "json").toString());
618+
optionsNode.put("mainModuleUri", mainModulePath.concat("/").concat(properties.getOrDefault("mainModule", "main.sjs").toString()));
619619

620620
if(step.get("stepDefinitionType").asText().equals("custom")) {
621-
step.put("sourceDatabase", "data-hub-STAGING");
622-
step.put("targetDatabase", "data-hub-FINAL");
621+
step.put("sourceDatabase", sourceDb);
622+
step.put("targetDatabase", finalDb);
623623
step.put("sourceQueryIsModule", true);
624-
mainModulePath = mainModulePath.concat("/").concat(properties.get("collectorModule").toString());
624+
mainModulePath = mainModulePath.concat("/").concat(properties.getOrDefault("collectorModule", "collector.sjs").toString());
625625
ObjectNode sourceModuleNode = (ObjectNode) step.get("sourceModule");
626626
sourceModuleNode.put("modulePath", mainModulePath);
627627
sourceModuleNode.put("functionName", "collect");
628628

629629
optionsNode.put("entity", entityType);
630630
} else {
631-
step.put("targetDatabase", "data-hub-STAGING");
631+
step.put("targetDatabase", sourceDb);
632+
step.put("inputFilePath", "");
632633
}
633-
step.put("options", optionsNode);
634+
step.putIfAbsent("options", optionsNode);
634635

635636
step.putArray("collections").add(stepPayLoad.get("name").asText()).add(entityType);
636637
step.put("permissions", "data-hub-common,read,data-hub-common,update");
637638
step.put("stepId", step.get("name").asText().concat("-").concat(step.get("stepDefinitionType").asText()));
638639
step.put("isUpgradedLegacyFlow", true);
639-
step.put("sourceFormat", properties.get("dataFormat").toString());
640-
step.put("targetFormat", properties.get("dataFormat").toString());
640+
step.put("sourceFormat", properties.getOrDefault("dataFormat", "json").toString());
641+
step.put("targetFormat", properties.getOrDefault("dataFormat", "json").toString());
641642
}
642643

643644
private JsonNode retrieveEntityFromCommunityNode(String modelName, JsonNode modelNodes, Map<String, JsonNode> entityModels) throws IOException {

‎marklogic-data-hub/src/test/java/com/marklogic/hub/impl/UpgradeProjectTest.java‎

Lines changed: 21 additions & 9 deletions
Original file line numberDiff line numberDiff line change
@@ -81,7 +81,7 @@ public void upgrade43xToCurrentVersion() throws IOException {
8181
File mappingDir = new File(projectDir, "mappings");
8282
File entitiesDir = new File(projectDir, "entities");
8383
verifyDirContents(mappingDir, 1);
84-
verifyDirContents(entitiesDir, 3);
84+
verifyDirContents(entitiesDir, 4);
8585

8686
File finalDbFile = hubProject.getUserConfigDir().resolve("databases").resolve("final-database.json").toFile();
8787
ObjectNode db = (ObjectNode) ObjectMapperFactory.getObjectMapper().readTree(finalDbFile);
@@ -228,7 +228,7 @@ public void onlyLegacyEntitiesUpgrade() throws IOException {
228228
List<String> legacyFlowNames = new ArrayList<>();
229229
legacyEntities.add("Customer");
230230
legacyEntities.add("Product");
231-
hubProject.upgradeLegacyFlows(flowManager, legacyEntities, legacyFlowTypes, legacyFlowNames);
231+
hubProject.upgradeLegacyFlows(flowManager, legacyEntities, legacyFlowTypes, legacyFlowNames, getHubConfig().getStagingDbName(), getHubConfig().getFinalDbName());
232232
assertTrue(hubProject.getFlowsDir().resolve("dh_Upgrade_CustomerFlow.flow.json").toFile().exists());
233233
assertTrue(hubProject.getFlowsDir().resolve("dh_Upgrade_ProductFlow.flow.json").toFile().exists());
234234
assertFalse(hubProject.getFlowsDir().resolve("dh_Upgrade_OrderFlow.flow.json").toFile().exists());
@@ -241,7 +241,7 @@ public void onlyFlowTypeUpgrade() throws IOException {
241241
List<String> legacyFlowTypes = new ArrayList<>();
242242
List<String> legacyFlowNames = new ArrayList<>();
243243
legacyFlowTypes.add("input");
244-
hubProject.upgradeLegacyFlows(flowManager, legacyEntities, legacyFlowTypes, legacyFlowNames);
244+
hubProject.upgradeLegacyFlows(flowManager, legacyEntities, legacyFlowTypes, legacyFlowNames, getHubConfig().getStagingDbName(), getHubConfig().getFinalDbName());
245245
assertTrue(hubProject.getStepsPath(StepDefinition.StepDefinitionType.INGESTION).toFile().exists());
246246
assertFalse(hubProject.getStepsPath(StepDefinition.StepDefinitionType.CUSTOM).toFile().exists());
247247
}
@@ -254,7 +254,7 @@ public void onlyFlowNamesUpgrade() throws IOException {
254254
List<String> legacyFlowNames = new ArrayList<>();
255255
legacyFlowNames.add("Load Customers");
256256
legacyFlowNames.add("Harmonize Products");
257-
hubProject.upgradeLegacyFlows(flowManager, legacyEntities, legacyFlowTypes, legacyFlowNames);
257+
hubProject.upgradeLegacyFlows(flowManager, legacyEntities, legacyFlowTypes, legacyFlowNames, getHubConfig().getStagingDbName(), getHubConfig().getFinalDbName());
258258
assertTrue(hubProject.getStepsPath(StepDefinition.StepDefinitionType.INGESTION).resolve("LoadCustomers.step.json").toFile().exists());
259259
assertFalse(hubProject.getStepsPath(StepDefinition.StepDefinitionType.INGESTION).resolve("LoadOrders.step.json").toFile().exists());
260260
assertFalse(hubProject.getStepsPath(StepDefinition.StepDefinitionType.INGESTION).resolve("LoadProducts.step.json").toFile().exists());
@@ -270,7 +270,7 @@ public void entityAndFlowNameUpgrade() throws IOException {
270270
legacyEntities.add("Customer");
271271
legacyFlowNames.add("Load Customers");
272272
legacyFlowNames.add("Harmonize Products");
273-
hubProject.upgradeLegacyFlows(flowManager, legacyEntities, legacyFlowTypes, legacyFlowNames);
273+
hubProject.upgradeLegacyFlows(flowManager, legacyEntities, legacyFlowTypes, legacyFlowNames, getHubConfig().getStagingDbName(), getHubConfig().getFinalDbName());
274274
assertTrue(hubProject.getStepsPath(StepDefinition.StepDefinitionType.INGESTION).resolve("LoadCustomers.step.json").toFile().exists());
275275
assertFalse(hubProject.getStepsPath(StepDefinition.StepDefinitionType.CUSTOM).resolve("HarmonizeProducts.step.json").toFile().exists());
276276
}
@@ -283,7 +283,7 @@ public void entityAndFlowTypeUpgrade() throws IOException {
283283
List<String> legacyFlowNames = new ArrayList<>();
284284
legacyEntities.add("Product");
285285
legacyFlowTypes.add("input");
286-
hubProject.upgradeLegacyFlows(flowManager, legacyEntities, legacyFlowTypes, legacyFlowNames);
286+
hubProject.upgradeLegacyFlows(flowManager, legacyEntities, legacyFlowTypes, legacyFlowNames, getHubConfig().getStagingDbName(), getHubConfig().getFinalDbName());
287287
assertTrue(hubProject.getStepsPath(StepDefinition.StepDefinitionType.INGESTION).resolve("LoadProducts.step.json").toFile().exists());
288288
assertFalse(hubProject.getStepsPath(StepDefinition.StepDefinitionType.CUSTOM).resolve("HarmonizeProducts.step.json").toFile().exists());
289289
}
@@ -295,7 +295,7 @@ public void nonExistentFlowName() throws IOException {
295295
List<String> legacyFlowTypes = new ArrayList<>();
296296
List<String> legacyFlowNames = new ArrayList<>();
297297
legacyFlowNames.add("NonExistentFlow");
298-
hubProject.upgradeLegacyFlows(flowManager, legacyEntities, legacyFlowTypes, legacyFlowNames);
298+
hubProject.upgradeLegacyFlows(flowManager, legacyEntities, legacyFlowTypes, legacyFlowNames, getHubConfig().getStagingDbName(), getHubConfig().getFinalDbName());
299299
assertEquals(0, hubProject.getFlowsDir().toFile().listFiles().length);
300300
}
301301

@@ -307,13 +307,13 @@ public void multipleLegacyFlowUpgrades() throws IOException {
307307
List<String> legacyFlowTypes = new ArrayList<>();
308308
List<String> legacyFlowNames = new ArrayList<>();
309309
legacyFlowTypes.add("input");
310-
hubProject.upgradeLegacyFlows(flowManager, legacyEntities, legacyFlowTypes, legacyFlowNames);
310+
hubProject.upgradeLegacyFlows(flowManager, legacyEntities, legacyFlowTypes, legacyFlowNames, getHubConfig().getStagingDbName(), getHubConfig().getFinalDbName());
311311

312312
assertTrue(hubProject.getStepsPath(StepDefinition.StepDefinitionType.INGESTION).toFile().exists());
313313
assertFalse(hubProject.getStepsPath(StepDefinition.StepDefinitionType.CUSTOM).toFile().exists());
314314

315315
legacyFlowTypes = new ArrayList<>();
316-
hubProject.upgradeLegacyFlows(flowManager, legacyEntities, legacyFlowTypes, legacyFlowNames);
316+
hubProject.upgradeLegacyFlows(flowManager, legacyEntities, legacyFlowTypes, legacyFlowNames, getHubConfig().getStagingDbName(), getHubConfig().getFinalDbName());
317317
verify4xUpgradedFlows();
318318
}
319319

@@ -520,6 +520,18 @@ private void verify4xUpgradedFlows() throws IOException {
520520
assertEquals("Product", node.get("options").get("entity").asText());
521521
assertEquals("json", node.get("options").get("dataFormat").asText());
522522
assertEquals("/entities/Product/harmonize/Harmonize Products/main.sjs", node.get("options").get("mainModuleUri").asText());
523+
524+
// validate the step default configuration when properties file is missing
525+
assertTrue(hubProject.getFlowsDir().resolve("dh_Upgrade_EmployeeFlow.flow.json").toFile().exists());
526+
assertTrue(hubProject.getStepsPath(StepDefinition.StepDefinitionType.CUSTOM).resolve("NoProperties.step.json").toFile().exists());
527+
assertTrue(hubProject.getStepDefinitionPath(StepDefinition.StepDefinitionType.CUSTOM).resolve("NoProperties").toFile().exists());
528+
assertNotNull(hubProject.getCustomModuleDir("NoProperties", StepDefinition.StepDefinitionType.CUSTOM.toString()));
529+
JsonNode noPropertiesStep = mapper.readTree(hubProject.getStepFile(StepDefinition.StepDefinitionType.CUSTOM, "NoProperties"));
530+
assertEquals("json", noPropertiesStep.get("sourceFormat").asText());
531+
assertEquals("json", noPropertiesStep.get("targetFormat").asText());
532+
assertEquals("json", noPropertiesStep.get("options").get("dataFormat").asText());
533+
assertEquals("/entities/Employee/harmonize/NoProperties/main.sjs", noPropertiesStep.get("options").get("mainModuleUri").asText());
534+
assertEquals("/entities/Employee/harmonize/NoProperties/collector.sjs", noPropertiesStep.get("sourceModule").get("modulePath").asText());
523535
}
524536

525537
private HubProjectImpl setUpProject(String sourceProjectName, String destProjectName) throws IOException {

‎marklogic-data-hub/src/test/resources/upgrade-projects/dhf43x/plugins/entities/Employee/Employee.entity.json‎

Lines changed: 14 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,14 @@
1+
{
2+
"info" : {
3+
"title" : "Employee",
4+
"version" : "0.0.1"
5+
},
6+
"definitions" : {
7+
"Employee" : {
8+
"required" : [ ],
9+
"rangeIndex" : [ ],
10+
"wordLexicon" : [ ],
11+
"properties" : { }
12+
}
13+
}
14+
}

‎marklogic-data-hub/src/test/resources/upgrade-projects/dhf43x/plugins/entities/Employee/harmonize/NoProperties/content.sjs‎

Lines changed: 16 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,16 @@
1+
/*
2+
* Create Content Plugin
3+
*
4+
* @param id - the identifier returned by the collector
5+
* @param rawContent - the raw content being loaded.
6+
* @param options - an object containing options. Options are sent from Java
7+
*
8+
* @return - your content
9+
*/
10+
function createContent(id, rawContent, options) {
11+
return rawContent;
12+
}
13+
14+
module.exports = {
15+
createContent: createContent
16+
};

‎marklogic-data-hub/src/test/resources/upgrade-projects/dhf43x/plugins/entities/Employee/harmonize/NoProperties/headers.sjs‎

Lines changed: 17 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,17 @@
1+
/*
2+
* Create Headers Plugin
3+
*
4+
* @param id - the identifier returned by the collector
5+
* @param content - the output of your content plugin
6+
* @param options - an object containing options. Options are sent from Java
7+
*
8+
* @return - an object of headers
9+
*/
10+
function createHeaders(id, content, options) {
11+
return {};
12+
}
13+
14+
module.exports = {
15+
createHeaders: createHeaders
16+
};
17+

‎marklogic-data-hub/src/test/resources/upgrade-projects/dhf43x/plugins/entities/Employee/harmonize/NoProperties/main.sjs‎

Lines changed: 45 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,45 @@
1+
// dhf.sjs exposes helper functions to make your life easier
2+
// See documentation at:
3+
// https://marklogic.github.io/marklogic-data-hub/docs/server-side/
4+
const dhf = require('/data-hub/4/dhf.sjs');
5+
6+
const contentPlugin = require('./content.sjs');
7+
const headersPlugin = require('./headers.sjs');
8+
const triplesPlugin = require('./triples.sjs');
9+
10+
/*
11+
* Plugin Entry point
12+
*
13+
* @param id - the identifier returned by the collector
14+
* @param rawContent - the raw content being loaded
15+
* @param options - a map containing options. Options are sent from Java
16+
*
17+
*/
18+
function main(id, rawContent, options) {
19+
var contentContext = dhf.contentContext(rawContent);
20+
var content = dhf.run(contentContext, function() {
21+
return contentPlugin.createContent(id, rawContent, options);
22+
});
23+
24+
var headerContext = dhf.headersContext(content);
25+
var headers = dhf.run(headerContext, function() {
26+
return headersPlugin.createHeaders(id, content, options);
27+
});
28+
29+
var tripleContext = dhf.triplesContext(content, headers);
30+
var triples = dhf.run(tripleContext, function() {
31+
return triplesPlugin.createTriples(id, content, headers, options);
32+
});
33+
34+
var envelope = dhf.makeEnvelope(content, headers, triples, options.dataFormat);
35+
36+
// log the final envelope as a trace
37+
// only fires if tracing is enabled
38+
dhf.logTrace(dhf.writerContext(envelope));
39+
40+
return envelope;
41+
}
42+
43+
module.exports = {
44+
main: main
45+
};

‎marklogic-data-hub/src/test/resources/upgrade-projects/dhf43x/plugins/entities/Employee/harmonize/NoProperties/triples.sjs‎

Lines changed: 18 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,18 @@
1+
/*
2+
* Create Triples Plugin
3+
*
4+
* @param id - the identifier returned by the collector
5+
* @param content - the output of your content plugin
6+
* @param headers - the output of your heaaders plugin
7+
* @param options - an object containing options. Options are sent from Java
8+
*
9+
* @return - an array of triples
10+
*/
11+
function createTriples(id, content, headers, options) {
12+
return [];
13+
}
14+
15+
module.exports = {
16+
createTriples: createTriples
17+
};
18+

‎marklogic-data-hub/src/test/resources/upgrade-projects/dhf43x/plugins/entities/Employee/harmonize/NoProperties/writer.sjs‎

Lines changed: 14 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,14 @@
1+
/*~
2+
* Writer Plugin
3+
*
4+
* @param id - the identifier returned by the collector
5+
* @param envelope - the final envelope
6+
* @param options - an object options. Options are sent from Java
7+
*
8+
* @return - nothing
9+
*/
10+
function write(id, envelope, options) {
11+
xdmp.documentInsert(id, envelope, xdmp.defaultPermissions(), options.flow);
12+
}
13+
14+
module.exports = write;

‎ml-data-hub-plugin/src/main/groovy/com/marklogic/gradle/task/UpdateLegacyFlowsTask.groovy‎

Lines changed: 3 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -17,6 +17,7 @@
1717

1818
package com.marklogic.gradle.task
1919

20+
import com.marklogic.hub.impl.HubConfigImpl
2021
import org.gradle.api.tasks.Input
2122
import org.gradle.api.tasks.Optional
2223
import org.gradle.api.tasks.TaskAction
@@ -51,7 +52,8 @@ class UpdateLegacyFlowsTask extends HubTask {
5152
}
5253

5354
println "start upgradeLegacyFlows task ."
54-
int flowsUpdated = getHubProject().upgradeLegacyFlows(getFlowManager(), legacyEntities, legacyFlowTypes, legacyFlowNames)
55+
HubConfigImpl config = (HubConfigImpl) getHubConfig()
56+
int flowsUpdated = getHubProject().upgradeLegacyFlows(getFlowManager(), legacyEntities, legacyFlowTypes, legacyFlowNames, config.getStagingDbName(), config.getFinalDbName())
5557
if(flowsUpdated == 0) {
5658
println("No legacy Flows found in plugins/entities directory to upgrade")
5759
}

0 commit comments

Comments
 (0)