DHFPROD-10580: Source Header Fields Populated Incorrectly

Author: rahulvudutala

examples/reference-entity-model/src/main/ml-modules/root/custom-modules/ingestion/zipTransform/main.sjs

@@ -5,14 +5,14 @@
  */
 
 function main(content, options) {
-  const value = content.value.toObject();
+  const value = content.value;
 
   // Concatenating latitude and longitude allows for a geospatial element index to be easily added
-  value.geoPoint = value.latitude + " " + value.longitude;
+  value.geoPoint = value.root.latitude + " " + value.root.longitude;
 
   // "zipCode" is a bit more descriptive than "zip"
-  value.zipCode = value.zip;
-  delete value.zip;
+  value.zipCode = value.root.zip;
+  delete value.root.zip;
 
   content.value = value;
   return content;

marklogic-data-hub/src/main/resources/ml-modules/root/data-hub/5/impl/flow-utils.mjs

@@ -495,16 +495,17 @@ function evalSubstituteVal(value) {
 }
 
 function createHeaders(options) {
+  const newOptions = JSON.parse(JSON.stringify(options));
   let headers = {};
-  for (let key in options.headers) {
-    headers[key] = evalSubstituteVal(options.headers[key]);
+  for (let key in newOptions.headers) {
+    headers[key] = evalSubstituteVal(newOptions.headers[key]);
   }
-  if (options.file) {
-    headers["createdUsingFile"] = options.file;
+  if (newOptions.file) {
+    headers["createdUsingFile"] = newOptions.file;
   }
 
-  const sourceName = options.sourceName ? options.sourceName : null;
-  const sourceType = options.sourceType ? options.sourceType : null;
+  const sourceName = newOptions.sourceName ? newOptions.sourceName : null;
+  const sourceType = newOptions.sourceType ? newOptions.sourceType : null;
 
   if (headers["sources"] && !Array.isArray(headers["sources"])) {
     headers.sources = [headers.sources];

marklogic-data-hub/src/test/java/com/marklogic/hub/step/impl/IngestWithSourceNameAndSourceTypeTest.java

@@ -26,7 +26,7 @@ void beforeEach() {
 
     @Test
     void runDefaultIngestionWithSourceNameAndSourceType(){
-        String path = "entity-reference-model/data/customers";
+        String path = "entity-reference-model/data/employees";
         ObjectNode info = new ObjectMapper().createObjectNode();
         info.put("name", "sources-test");
         info.put("description", "step to test source name and source type");
@@ -49,14 +49,18 @@ void runDefaultIngestionWithSourceNameAndSourceType(){
         RunFlowResponse response = flowRunner.runFlow(inputs);
         flowRunner.awaitCompletion();
         assertEquals(JobStatus.FINISHED.toString(), response.getJobStatus());
-        JsonNode sourceDoc = getStagingDoc("customer1.json");
-        assertNotNull(sourceDoc);
-        JsonNode sourcesNode = sourceDoc.get("envelope").get("headers").get("sources").get(0);
-        assertEquals("sources-test", sourcesNode.get("name").asText());
 
-        sourcesNode = sourceDoc.get("envelope").get("headers").get("sources").get(1);
-        assertEquals("sample-source", sourcesNode.get("datahubSourceName").asText());
-        assertEquals("employee", sourcesNode.get("datahubSourceType").asText());
+        for(int i=1; i<=3; i++) {
+            JsonNode sourceDoc = getStagingDoc("employee" + i + ".json");
+            assertNotNull(sourceDoc);
+            assertEquals(2, sourceDoc.get("envelope").get("headers").get("sources").size(), "sources should contain name and only 1 object of sourceName and sourceType");
+            JsonNode sourcesNode = sourceDoc.get("envelope").get("headers").get("sources").get(0);
+            assertEquals("sources-test", sourcesNode.get("name").asText());
+
+            sourcesNode = sourceDoc.get("envelope").get("headers").get("sources").get(1);
+            assertEquals("sample-source", sourcesNode.get("datahubSourceName").asText());
+            assertEquals("employee", sourcesNode.get("datahubSourceType").asText());
+        }
     }
 
     @Test

marklogic-data-hub/src/test/resources/entity-reference-model/data/employees/employee1.json

@@ -0,0 +1,3 @@
+{
+  "EmployeeID": "1"
+}

marklogic-data-hub/src/test/resources/entity-reference-model/data/employees/employee2.json

@@ -0,0 +1,3 @@
+{
+  "EmployeeID": "2"
+}

marklogic-data-hub/src/test/resources/entity-reference-model/data/employees/employee3.json

@@ -0,0 +1,3 @@
+{
+  "EmployeeID": "3"
+}