DHFPROD-2592: test cleanup, fixing how file collector recognizes binaries after changes to allowing multiple values per type

Author: aebadirad

marklogic-data-hub/src/main/java/com/marklogic/hub/collector/impl/FileCollector.java

@@ -59,10 +59,11 @@ public DiskQueue<String> run() {
                 (filePath, fileAttr) -> fileAttr.isRegularFile())
                 .forEach(path -> {
                     File file = path.toFile();
-                    if (fileFormat.get(inputFormat.toLowerCase()).contains(FilenameUtils.getExtension(file.getName()).toLowerCase())) {
+                    String fileName = FilenameUtils.getExtension(file.getName()).toLowerCase();
+                    if (fileFormat.containsKey(inputFormat.toLowerCase()) && fileFormat.get(inputFormat.toLowerCase()).contains(fileName)) {
                         results.add(path.toFile().getAbsolutePath());
                     }
-                    else if("binary".equalsIgnoreCase(inputFormat) && !fileFormat.values().contains(FilenameUtils.getExtension(file.getName().toLowerCase()))){
+                    else if("binary".equalsIgnoreCase(inputFormat) && !csvExts.contains(fileName) && !jsonExts.contains(fileName) && !xmlExts.contains(fileName)){
                         results.add(path.toFile().getAbsolutePath());
                     }
                 });

marklogic-data-hub/src/test/java/com/marklogic/hub/flow/FlowRunnerTest.java

@@ -282,7 +282,7 @@ public void testIngestBinaryAndTxt(){
 
         stepConfig.put("fileLocations", stepDetails);
         stepConfig.put("batchSize", "1");
-        opts.put("outputFormat", "Binary");
+        opts.put("outputFormat", "binary");
         opts.put("collections", coll);
         opts.put("permissions", "rest-reader,read");
         RunFlowResponse resp = fr.runFlow("testFlow",steps, UUID.randomUUID().toString(), opts, stepConfig);
@@ -295,7 +295,7 @@ public void testIngestBinaryAndTxt(){
 
         stepConfig.put("fileLocations", stepDetails);
         stepConfig.put("batchSize", "1");
-        opts.put("outputFormat", "Text");
+        opts.put("outputFormat", "text");
         RunFlowResponse resp1 = fr.runFlow("testFlow",steps, UUID.randomUUID().toString(), opts, stepConfig);
         fr.awaitCompletion();
 

marklogic-data-hub/src/test/resources/flow-runner-test/flows/testFlow.flow.json

@@ -58,7 +58,7 @@
         "inputFilePath": "input/tabs",
         "inputFileType": "csv",
         "outputURIReplacement": ".*/input,''",
-        "separator": "\t"
+        "separator": "\\t"
       },
       "options": {
         "outputFormat": "json",