Merge branch 'release/1.0.2' into 1.0-master

paxtonhare · paxtonhare · commit b7e2a02a32e3 · 2016-11-22T15:11:17.000-05:00
diff --git a/CHANGELOG.md b/CHANGELOG.md
@@ -1,7 +1,21 @@
 # Change Log
 
-## [v1.0.1](https://github.com/marklogic/marklogic-data-hub/tree/v1.0.1)
+## [v1.0.2](https://github.com/marklogic/marklogic-data-hub/tree/v1.0.2)
 
+[Full Changelog](https://github.com/marklogic/marklogic-data-hub/compare/v1.0.1...v1.0.2)
+
+**Fixed bugs:**
+
+- mlcp load from QuickStart GUI not loading data [\#331](https://github.com/marklogic/marklogic-data-hub/issues/331)
+
+**Closed issues:**
+
+- Possible typo on wiki page [\#333](https://github.com/marklogic/marklogic-data-hub/issues/333)
+- Tutorial instruction - create entity "Employees" instead of "Employee" [\#330](https://github.com/marklogic/marklogic-data-hub/issues/330)
+- mlcp job is not getting run + console log button not showing [\#329](https://github.com/marklogic/marklogic-data-hub/issues/329)
+- Error Extension hubstats does not exist. [\#328](https://github.com/marklogic/marklogic-data-hub/issues/328)
+
+## [v1.0.1](https://github.com/marklogic/marklogic-data-hub/tree/v1.0.1) (2016-11-08)
 [Full Changelog](https://github.com/marklogic/marklogic-data-hub/compare/v1.0.0...v1.0.1)
 
 **Fixed bugs:**
diff --git a/CONTRIBUTING.md b/CONTRIBUTING.md
@@ -24,7 +24,7 @@ Looking to build the code from source? Look no further.
 You need these to get started
 
 - Java 8 JDK
-- Gradle (the newer the better)
+- Gradle (3.1 or greater)
 - Node JS 6.5 or newer
 - Typings `npm -g install typings`
 - A decent IDE. IntelliJ is nice.
@@ -90,10 +90,10 @@ your vision does not align with that of a project maintainer.
 #### Create a branch for your changes
 
 Okay, so you have decided to fix something. Create a feature branch
-and start hacking. **Note** that we use git flow and thus our most recent changes live on the develop branch.
+and start hacking. **Note** that we use git flow and thus our most recent changes live on the 1.0-develop branch.
 
 ```sh
-$ git checkout -b my-feature-branch -t origin/develop
+$ git checkout -b my-feature-branch -t origin/1.0-develop
 ```
 
 #### Formatting code
@@ -143,7 +143,7 @@ Use `git rebase` (not `git merge`) to sync your work from time to time.
 
 ```sh
 $ git fetch upstream
-$ git rebase upstream/develop
+$ git rebase upstream/1.0-develop
 ```
 
 
@@ -187,10 +187,10 @@ from the main (upstream) repository:
     git push origin --delete my-feature-branch
     ```
 
-* Check out the develop branch:
+* Check out the 1.0-develop branch:
 
     ```shell
-    git checkout develop -f
+    git checkout 1.0-develop -f
     ```
 
 * Delete the local branch:
@@ -199,10 +199,10 @@ from the main (upstream) repository:
     git branch -D my-feature-branch
     ```
 
-* Update your develop with the latest upstream version:
+* Update your 1.0-develop with the latest upstream version:
 
     ```shell
-    git pull --ff upstream develop
+    git pull --ff upstream 1.0-develop
     ```
 
 [issue tracker]: https://github.com/marklogic/marklogic-data-hub/issues
diff --git a/README.md b/README.md
@@ -2,7 +2,7 @@
 
 OS | Status
 --- | --- | ---
-Linux/Mac | [![Build Status](https://travis-ci.org/marklogic/marklogic-data-hub.svg?branch=master)](https://travis-ci.org/marklogic/marklogic-data-hub)
+Linux/Mac | [![Build Status](https://travis-ci.org/marklogic/marklogic-data-hub.svg?branch=1.0-master)](https://travis-ci.org/marklogic/marklogic-data-hub)
 Windows | [![Windows Build status](https://ci.appveyor.com/api/projects/status/kgj0k5na59uhkvbv?svg=true)](https://ci.appveyor.com/project/paxtonhare/marklogic-data-hub)
 
 # MarkLogic Data Hub
@@ -13,6 +13,14 @@ This project allows you to deploy a skeleton Data Hub into MarkLogic. With some
 
 # Getting Started
 
+###Prerequisites
+
+You need these to get started
+
+- Java 8 JDK
+- MarkLogic 8.0-2 or greater
+- Gradle 3.1 or greater **(Optional)**
+
 ###TL;DR
 
 Head over to our [Getting Started Tutorial](https://marklogic.github.io/marklogic-data-hub/) to get up and running with the Data Hub.
@@ -36,7 +44,7 @@ Alternatively you can include the jar file as a build dependency in your Java pr
 **Gradle**
 
 ```groovy
-compile('com.marklogic:marklogic-data-hub:1.0.1')
+compile('com.marklogic:marklogic-data-hub:1.0.2')
 ```
 
 **Maven**
@@ -45,15 +53,15 @@ compile('com.marklogic:marklogic-data-hub:1.0.1')
 <dependency>
   <groupId>com.marklogic</groupId>
   <artifactId>marklogic-data-hub</artifactId>
-  <version>1.0.1</version>
+  <version>1.0.2</version>
   <type>pom</type>
 </dependency>
 ```
 
 **Ivy**
 
 ```xml
-<dependency org='com.marklogic' name='marklogic-data-hub' rev='1.0.1'>
+<dependency org='com.marklogic' name='marklogic-data-hub' rev='1.0.2'>
   <artifact name='$AID' ext='pom'></artifact>
 </dependency>
 ```
@@ -64,7 +72,7 @@ If you prefer to use gradle for all of your hub interactions then you can includ
 
 ```groovy
 plugins {
-    id 'com.marklogic.ml-data-hub' version '1.0.1'
+    id 'com.marklogic.ml-data-hub' version '1.0.2'
 }
 ```
 
diff --git a/examples/spring-batch/build.gradle b/examples/spring-batch/build.gradle
@@ -4,7 +4,7 @@ plugins {
     id 'idea'
     id 'application'
     id 'net.saliman.properties' version '1.4.6'
-    id 'com.marklogic.ml-data-hub' version '1.0.1'
+    id 'com.marklogic.ml-data-hub' version '1.0.2'
 }
 
 repositories {
@@ -13,7 +13,7 @@ repositories {
 }
 
 dependencies {
-    compile 'com.marklogic:marklogic-data-hub:1.0.1'
+    compile 'com.marklogic:marklogic-data-hub:1.0.2'
     compile 'com.marklogic:marklogic-spring-batch-core:0.6.0'
     compile 'com.marklogic:ml-javaclient-util:2.9.1'
     testCompile 'com.marklogic:marklogic-spring-batch-test:0.6.0'
diff --git a/marklogic-data-hub/gradle.properties b/marklogic-data-hub/gradle.properties
@@ -36,6 +36,6 @@ hubProjectDir=examples/hr-hub
 mlReplaceTokensInModules=true
 mlUseRoxyTokenPrefix=false
 
-version=1.0.1
+version=1.0.2
 
 
diff --git a/marklogic-data-hub/src/main/resources/scaffolding/build_gradle b/marklogic-data-hub/src/main/resources/scaffolding/build_gradle
@@ -3,7 +3,7 @@ plugins {
     id 'eclipse'
     id 'idea'
     id 'net.saliman.properties' version '1.4.6'
-    id 'com.marklogic.ml-data-hub' version '1.0.1'
+    id 'com.marklogic.ml-data-hub' version '1.0.2'
 }
 
 repositories {
@@ -12,6 +12,6 @@ repositories {
 }
 
 dependencies {
-    compile 'com.marklogic:marklogic-data-hub:1.0.1'
+    compile 'com.marklogic:marklogic-data-hub:1.0.2'
     compile 'com.marklogic:marklogic-xcc:8.0.5'
 }
diff --git a/marklogic-data-hub/src/trace-ui/package.json b/marklogic-data-hub/src/trace-ui/package.json
@@ -1,6 +1,6 @@
 {
   "name": "data-hub-quickstart",
-  "version": "1.0.1",
+  "version": "1.0.2",
   "license": "APACHE",
   "homepage": "https://github.com/marklogic/marklogic-data-hub",
   "scripts": {
diff --git a/ml-data-hub-plugin/gradle.properties b/ml-data-hub-plugin/gradle.properties
@@ -1,2 +1,2 @@
-version=1.0.1
+version=1.0.2
 group=com.marklogic
diff --git a/quick-start/gradle.properties b/quick-start/gradle.properties
@@ -1 +1 @@
-version=1.0.1
+version=1.0.2
diff --git a/quick-start/package.json b/quick-start/package.json
@@ -1,6 +1,6 @@
 {
   "name": "data-hub-quickstart",
-  "version": "1.0.1",
+  "version": "1.0.2",
   "license": "APACHE",
   "homepage": "https://github.com/marklogic/marklogic-data-hub",
   "angular-cli": {},
diff --git a/quick-start/src/main/java/com/marklogic/quickstart/Application.java b/quick-start/src/main/java/com/marklogic/quickstart/Application.java
@@ -15,6 +15,8 @@
  */
 package com.marklogic.quickstart;
 
+import com.marklogic.contentpump.ContentPump;
+import org.apache.commons.lang.ArrayUtils;
 import org.springframework.boot.SpringApplication;
 import org.springframework.boot.autoconfigure.SpringBootApplication;
 import org.springframework.boot.builder.SpringApplicationBuilder;
@@ -31,7 +33,13 @@ protected SpringApplicationBuilder configure(SpringApplicationBuilder applicatio
         return application.sources(Application.class);
     }
 
-    public static void main(String[] args) {
-        SpringApplication.run(Application.class, args);
+    public static void main(String[] args) throws Exception {
+        if (args.length > 0 && args[0].equals("mlcp")) {
+            String[] newArgs = (String[]) ArrayUtils.subarray(args, 1, args.length);
+            ContentPump.main(newArgs);
+        }
+        else {
+            SpringApplication.run(Application.class, args);
+        }
     }
 }
diff --git a/quick-start/src/main/java/com/marklogic/quickstart/service/MlcpTasklet.java b/quick-start/src/main/java/com/marklogic/quickstart/service/MlcpTasklet.java
@@ -2,25 +2,30 @@
 
 import com.fasterxml.jackson.databind.JsonNode;
 import com.fasterxml.jackson.databind.ObjectMapper;
-import com.marklogic.contentpump.ContentPump;
 import com.marklogic.contentpump.bean.MlcpBean;
 import com.marklogic.hub.HubConfig;
 import com.marklogic.hub.JobStatusListener;
 import com.marklogic.quickstart.util.StreamGobbler;
 import org.apache.commons.io.FileUtils;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
 import org.springframework.batch.core.StepContribution;
 import org.springframework.batch.core.scope.context.ChunkContext;
 import org.springframework.batch.core.step.tasklet.Tasklet;
 import org.springframework.batch.repeat.RepeatStatus;
 
 import java.io.File;
 import java.io.IOException;
+import java.net.URL;
+import java.net.URLClassLoader;
 import java.util.ArrayList;
 import java.util.Arrays;
 import java.util.function.Consumer;
 
 class MlcpTasklet implements Tasklet {
 
+    protected final Logger logger = LoggerFactory.getLogger(getClass());
+
     private HubConfig hubConfig;
     private JsonNode mlcpOptions;
     private JobStatusListener statusListener;
@@ -66,34 +71,53 @@ public RepeatStatus execute(StepContribution contribution, ChunkContext chunkCon
     }
 
     private void runMlcp(long jobId, MlcpBean bean) throws IOException, InterruptedException {
+        ClassLoader cl = ClassLoader.getSystemClassLoader();
+
+        URL[] urls = ((URLClassLoader)cl).getURLs();
+
         String javaHome = System.getProperty("java.home");
         String javaBin = javaHome +
             File.separator + "bin" +
             File.separator + "java";
         String classpath = System.getProperty("java.class.path");
-        String className = ContentPump.class.getCanonicalName();
-
-        File loggerFile = File.createTempFile("mlcp-", "-logger");
-        String loggerData = "log4j.rootLogger=INFO,console\n" +
-            "log4j.appender.console=org.apache.log4j.ConsoleAppender\n" +
-            "log4j.appender.console.target=System.err\n" +
-            "log4j.appender.console.layout=org.apache.log4j.PatternLayout\n" +
-            "log4j.appender.console.layout.ConversionPattern=%d{yy/MM/dd HH:mm:ss} %p %c{2}: %m%n\n" +
+
+        File loggerFile = File.createTempFile("mlcp-", "-logger.xml");
+        String loggerData = "<configuration>\n" +
             "\n" +
-            "# To suppress not native warn on Mac and Solaris\n" +
-            "log4j.logger.org.apache.hadoop.util.NativeCodeLoader=ERROR\n" +
+            "  <appender name=\"STDOUT\" class=\"ch.qos.logback.core.ConsoleAppender\">\n" +
+            "    <!-- encoders are assigned the type\n" +
+            "         ch.qos.logback.classic.encoder.PatternLayoutEncoder by default -->\n" +
+            "    <encoder>\n" +
+            "      <pattern>%d{HH:mm:ss.SSS} [%thread] %-5level %logger{36} - %msg%n</pattern>\n" +
+            "    </encoder>\n" +
+            "  </appender>\n" +
             "\n" +
-            "# To enable debug\n" +
-            "#log4j.logger.com.marklogic.mapreduce=DEBUG\n" +
-            "#log4j.logger.com.marklogic.contentpump=DEBUG\n";
+            "  <logger name=\"org.apache.http\" level=\"WARN\"/>\n" +
+            "\n" +
+            "  <logger name=\"com.marklogic.spring.batch.core.repository.dao.MarkLogicStepExecutionDao\" level=\"WARN\"/>\n" +
+            "  <logger name=\"com.marklogic.spring.batch.core.repository.dao.MarkLogicJobExecutionDao\" level=\"WARN\"/>\n" +
+            "  <logger name=\"com.marklogic.client.impl.DocumentManagerImpl\" level=\"WARN\"/>\n" +
+            "  <logger name=\"com.marklogic.client.impl.DatabaseClientImpl\" level=\"WARN\"/>\n" +
+            "  <logger name=\"com.marklogic\" level=\"INFO\"/>\n" +
+            "  <logger name=\"com.marklogic.appdeployer\" level=\"INFO\"/>\n" +
+            "  <logger name=\"com.marklogic.hub\" level=\"INFO\"/>\n" +
+            "  <logger name=\"com.marklogic.contentpump\" level=\"INFO\"/>\n" +
+            "  <logger name=\"org.apache.catalina.webresources.Cache\" level=\"ERROR\"/>\n" +
+            "  <logger name=\"org.apache.hadoop.util.Shell\" level=\"OFF\"/>\n" +
+            "  <logger name=\"org.apache.hadoop.util.NativeCodeLoader\" level=\"ERROR\"/>\n" +
+            "\n" +
+            "  <root level=\"WARN\">\n" +
+            "    <appender-ref ref=\"STDOUT\" />\n" +
+            "  </root>\n" +
+            "</configuration>\n";
         FileUtils.writeStringToFile(loggerFile, loggerData);
 
         ArrayList<String> args = new ArrayList<>();
         args.add(javaBin);
-        args.add("-Dlog4j.configurationFile=" + loggerFile.getAbsolutePath());
-        args.add("-cp");
+        args.add("-Dlogback.configurationFile=" + loggerFile.toURI());//logback);
+        args.add("-jar");
         args.add(classpath);
-        args.add(className);
+        args.add("mlcp");
         args.addAll(Arrays.asList(bean.buildArgs()));
 
         ProcessBuilder pb = new ProcessBuilder(args);
@@ -104,7 +128,7 @@ private void runMlcp(long jobId, MlcpBean bean) throws IOException, InterruptedE
 
             @Override
             public void accept(String status) {
-
+                System.out.println(status);
                 // don't log an error if the winutils binary is missing
                 if (status.contains("ERROR") && !status.contains("winutils binary")) {
                     hasError = true;
diff --git a/quick-start/src/main/java/com/marklogic/quickstart/util/StreamGobbler.java b/quick-start/src/main/java/com/marklogic/quickstart/util/StreamGobbler.java
@@ -16,7 +16,6 @@ public StreamGobbler(InputStream inputStream, Consumer<String> consumeInputLine)
 
     @Override
     public void run() {
-        System.out.println("gobbler running");
         new BufferedReader(new InputStreamReader(inputStream)).lines().forEach(consumeInputLine);
     }
 }
diff --git a/quick-start/src/main/resources/log4j.properties b/quick-start/src/main/resources/log4j.properties
@@ -1 +1,4 @@
 log4j.logger.httpclient.wire=DEBUG
+log4j.logger.org.apache.hadoop.util.NativeCodeLoader=ERROR
+log4j.logger.com.marklogic.mapreduce=ERROR
+log4j.logger.com.marklogic.contentpump=ERROR
diff --git a/quick-start/src/main/resources/logback-spring.xml b/quick-start/src/main/resources/logback-spring.xml
@@ -19,7 +19,9 @@
   <logger name="com.marklogic.appdeployer" level="INFO"/>
   <logger name="com.marklogic.hub" level="INFO"/>
   <logger name="com.marklogic.contentpump" level="INFO"/>
+  <logger name="com.marklogic.mapreduce" level="INFO"/>
   <logger name="org.apache.catalina.webresources.Cache" level="SEVERE"/>
+  <logger name="org.apache.hadoop.util.NativeCodeLoader" level="SEVERE"/>
 
   <root level="WARN">
     <appender-ref ref="STDOUT" />

Original file line number	Diff line number	Diff line change
`@@ -3,7 +3,7 @@ plugins {`
`3`	`3`	`id 'eclipse'`
`4`	`4`	`id 'idea'`
`5`	`5`	`id 'net.saliman.properties' version '1.4.6'`
`6`		`- id 'com.marklogic.ml-data-hub' version '1.0.1'`
	`6`	`+ id 'com.marklogic.ml-data-hub' version '1.0.2'`
`7`	`7`	`}`
`8`	`8`
`9`	`9`	`repositories {`
`@@ -12,6 +12,6 @@ repositories {`
`12`	`12`	`}`
`13`	`13`
`14`	`14`	`dependencies {`
`15`		`- compile 'com.marklogic:marklogic-data-hub:1.0.1'`
	`15`	`+ compile 'com.marklogic:marklogic-data-hub:1.0.2'`
`16`	`16`	`compile 'com.marklogic:marklogic-xcc:8.0.5'`
`17`	`17`	`}`
Original file line number	Diff line number	Diff line change
`@@ -1,6 +1,6 @@`
`1`	`1`	`{`
`2`	`2`	`"name": "data-hub-quickstart",`
`3`		`- "version": "1.0.1",`
	`3`	`+ "version": "1.0.2",`
`4`	`4`	`"license": "APACHE",`
`5`	`5`	`"homepage": "https://github.com/marklogic/marklogic-data-hub",`
`6`	`6`	`"scripts": {`
Original file line number	Diff line number	Diff line change
`@@ -1,2 +1,2 @@`
`1`		`-version=1.0.1`
	`1`	`+version=1.0.2`
`2`	`2`	`group=com.marklogic`