Merge pull request #95 from paxtonhare/94_add_mlcp_attrs

fixed #92

Author: paxtonhare

data-hub/build.gradle

@@ -26,7 +26,7 @@ dependencies {
     compile 'com.marklogic:ml-javaclient-util:2.6.1'
     compile 'com.marklogic:ml-app-deployer:2.0'
     compile 'commons-io:commons-io:2.4'
-    compile 'com.google.code.gson:gson:2.6.1'
+    compile 'org.apache.commons:commons-csv:1.2'
     compile("com.marklogic:mlcp:8.0-4") {
         exclude module : 'servlet-api'
         exclude group: 'com.google.guava', module: 'guava'

data-hub/src/main/java/com/marklogic/hub/FlowManager.java

@@ -117,7 +117,7 @@ protected void initializeDefaultSpringBatchComponents() {
 
     /**
      * Retrieves a list of flows installed on the MarkLogic server
-     * 
+     *
      * @param entityName
      *            - the entity from which to fetch the flows
      * @return - a list of flows for the given entity
@@ -151,7 +151,7 @@ public List<Flow> getFlows(String entityName) {
 
     /**
      * Retrieves a named flow from a given entity
-     * 
+     *
      * @param entityName
      *            - the entity that the flow belongs to
      * @param flowName
@@ -199,7 +199,7 @@ public JobExecution runFlow(Flow flow, int batchSize) {
      * @param flow - the flow to run
      * @param batchSize - the size to use for batching transactions
      * @param listener - the JobExecutionListener to receive status updates about the job
-     * @return 
+     * @return
      */
     public JobExecution runFlow(Flow flow, int batchSize, JobExecutionListener listener) {
         Collector c = flow.getCollector();

data-hub/src/main/java/com/marklogic/hub/Mlcp.java

@@ -134,6 +134,11 @@ public List<String> getMlcpArguments() throws IOException {
                 arguments.add(sourceOptions.getInputFilePattern());
             }
 
+            if (sourceOptions.getCollection() != null) {
+                arguments.add("-output_collections");
+                arguments.add(sourceOptions.getCollection());
+            }
+
             // by default, cut the source directory path to make URIs shorter
             String uriReplace = canonicalPath + ",''";
             uriReplace = uriReplace.replaceAll("\\\\", "/");
@@ -157,6 +162,7 @@ public static class SourceOptions {
         private String flowType;
         private String inputFileType;
         private String inputFilePattern;
+        private String collection;
 
         public SourceOptions(String entityName, String flowName, String flowType) {
             this.entityName = entityName;
@@ -192,6 +198,14 @@ public void setInputFilePattern(String inputFilePattern) {
             this.inputFilePattern = inputFilePattern;
         }
 
+        public String getCollection() {
+            return collection;
+        }
+
+        public void setCollection(String collection) {
+            this.collection = collection;
+        }
+
         protected String getTransformParams() {
             return String
                     .format("<params><entity-name>%s</entity-name><flow-name>%s</flow-name><flow-type>%s</flow-type></params>",

quick-start/build.gradle

@@ -4,13 +4,17 @@ buildscript {
         maven {url 'http://developer.marklogic.com/maven2/'}
     }
     dependencies {
-        classpath("org.springframework.boot:spring-boot-gradle-plugin:1.3.3.RELEASE")
+        classpath('org.springframework.boot:spring-boot-gradle-plugin:1.3.3.RELEASE')
     }
 }
 
-apply plugin: 'java'
-apply plugin: 'eclipse'
-apply plugin: 'idea'
+plugins {
+    id 'java'
+    id 'eclipse'
+    id 'idea'
+    id 'com.eriwen.gradle.js' version '1.12.1'
+}
+
 apply plugin: 'spring-boot'
 
 repositories {
@@ -26,10 +30,28 @@ configurations {
 }
 
 dependencies {
-    compile("org.springframework.boot:spring-boot-starter")
-    compile("org.springframework.boot:spring-boot-starter-web")
+    compile('org.springframework.boot:spring-boot-starter')
+    compile('org.springframework.boot:spring-boot-starter-web')
     compile project(':data-hub')
-    providedRuntime("org.springframework.boot:spring-boot-starter-tomcat")
-    testCompile("junit:junit")
+    providedRuntime('org.springframework.boot:spring-boot-starter-tomcat')
+    testCompile('junit:junit')
 }
 
+
+javascript.source {
+    dev {
+        js {
+            srcDir "src/main/resources/static/app"
+            include "**/*.js"
+        }
+    }
+}
+
+jshint {
+    tasks.jshint.source = javascript.source.dev.js.files
+    tasks.jshint.dest = file("${buildDir}/jshint.out")
+    tasks.jshint.ignoreExitCode = false
+    tasks.jshint.outputToStdOut = true
+    tasks.jshint.reporter = 'checkstyle'
+    options = ["--config":"src/main/resources/static/.jshintrc"]
+}

quick-start/src/main/java/com/marklogic/hub/model/RunFlowModel.java

@@ -4,6 +4,8 @@ public class RunFlowModel {
     private String entityName;
     private String flowName;
     private String inputPath;
+    private String collection;
+    private String dataFormat;
 
     public String getEntityName() {
         return entityName;
@@ -16,4 +18,12 @@ public String getFlowName() {
     public String getInputPath() {
         return inputPath;
     }
+
+    public String getCollection() {
+        return collection;
+    }
+
+    public String getDataFormat() {
+        return dataFormat;
+    }
 }

quick-start/src/main/java/com/marklogic/hub/service/SyncStatusService.java

@@ -77,9 +77,12 @@ public FileVisitResult preVisitDirectory(Path dir, BasicFileAttributes attrs) th
 
         @Override
         public FileVisitResult visitFile(Path file, BasicFileAttributes attrs) throws IOException {
-            synched = synched && isSynched(file.normalize().toAbsolutePath().toFile());
-            if (!synched) {
-                return FileVisitResult.TERMINATE;
+            // don't look at hidden files like .DS_Store
+            if (!file.getFileName().toString().startsWith(".")) {
+                synched = synched && isSynched(file.normalize().toAbsolutePath().toFile());
+                if (!synched) {
+                    return FileVisitResult.TERMINATE;
+                }
             }
 
             return FileVisitResult.CONTINUE;

quick-start/src/main/java/com/marklogic/hub/web/controller/api/FlowApiController.java

@@ -192,6 +192,8 @@ public void run(BasicFuture<?> resultFuture) {
             SourceOptions sourceOptions = new SourceOptions(
                     runFlow.getEntityName(), runFlow.getFlowName(),
                     FlowType.INPUT.toString());
+            sourceOptions.setInputFileType(runFlow.getDataFormat());
+            sourceOptions.setCollection(runFlow.getCollection());
             mlcp.addSourceDirectory(runFlow.getInputPath(), sourceOptions);
             mlcp.loadContent();
 

quick-start/src/main/resources/static/.jshintrc

@@ -0,0 +1,25 @@
+{
+  "browser": true,
+  "node": true,
+  "camelcase": true,
+  "curly": true,
+  "eqeqeq": true,
+  "immed": true,
+  "indent": 2,
+  "latedef": "nofunc",
+  "newcap": true,
+  "quotmark": "single",
+  "undef": true,
+  "unused": "vars",
+  "strict": true,
+  "trailing": true,
+  "globals": {
+    "angular": false,
+    "google": false,
+    "$": false,
+    "Highcharts": false,
+    "_": false,
+    "moment": false,
+    "MLSearchController": false
+  }
+}

quick-start/src/main/resources/static/app/directives/footer.html

@@ -1,3 +1,3 @@
-<div id="footer" class="container-fluid">
-    <span>&copy;2016.&nbsp;Data Hub in a Box.</span>
-</div>
+<div id="footer" class="container-fluid">
+  <span>&copy;2016.&nbsp;<a href="http://www.marklogic.com">MarkLogic</a></span>
+</div>

quick-start/src/main/resources/static/app/directives/footerDirective.js

@@ -1,26 +1,20 @@
-var dependencies = [
-];
-var module = angular.module('dhib.quickstart.directives.footer', dependencies);
-module.directive('footer', [
-    '$http'
-    ,'$location'
-    ,function(
-        $http
-        ,$location
-    ) {
-        return {
-            restrict: 'E'
-            ,scope : {
-                activeTab : '='
-            }
-            ,transclude: true
-            ,templateUrl : function(element, attrs) {
-                return attrs.templateUrl || 'app/directives/footer.html';
-            }
-            ,link : function(scope, element, attrs) {
-            }
-            ,controller : function($scope, $element, $attrs, $transclude) {
-            }
-        };
-    }
-]);
+(function () {
+
+  'use strict';
+
+  angular.module('dhib.quickstart.directives.footer', [])
+    .directive('footer', FooterDirective);
+
+  function FooterDirective($http, $location) {
+    return {
+      restrict: 'E',
+      scope : {
+        activeTab : '='
+      },
+      transclude: true,
+      templateUrl : function(element, attrs) {
+        return attrs.templateUrl || 'app/directives/footer.html';
+      }
+    };
+  }
+})();