Merge branch 'release/1.1.3' into 1.0-master

Author: paxtonhare

.travis/Dockerfile-ml

@@ -1,13 +1,25 @@
 FROM centos:centos7
 
 ENV LANG C.UTF-8
-
+ENV JAVA_VERSION 8u131
+ENV BUILD_VERSION b11
+ENV SUM d54c1d3a095b4ff2b6607d096fa80163
 RUN yum -y install glibc.i686 \
     gdb.x86_64 redhat-lsb.x86_64 vim \
     bzip2 \
     unzip \
-    xz-utils \
-    java-1.8.0-openjdk-devel 2>&1 > /dev/null
+    wget \
+    xz-utils 2>&1 > /dev/null
+
+# Downloading Java
+RUN wget --no-cookies --no-check-certificate --header "Cookie: oraclelicense=accept-securebackup-cookie" "http://download.oracle.com/otn-pub/java/jdk/$JAVA_VERSION-$BUILD_VERSION/$SUM/jdk-$JAVA_VERSION-linux-x64.rpm" -O /tmp/jdk-8-linux-x64.rpm
+RUN yum -y install /tmp/jdk-8-linux-x64.rpm
+
+RUN alternatives --install /usr/bin/java jar /usr/java/latest/bin/java 200000
+RUN alternatives --install /usr/bin/javaws javaws /usr/java/latest/bin/javaws 200000
+RUN alternatives --install /usr/bin/javac javac /usr/java/latest/bin/javac 200000
+
+ENV JAVA_HOME /usr/java/latest
 
 # Install MarkLogic
 COPY ./MarkLogic.rpm /tmp/MarkLogic.rpm
@@ -16,7 +28,7 @@ RUN yum -y install /tmp/MarkLogic.rpm 2>&1 > /dev/null
 
 # Expose MarkLogic Server ports - add additional ones for your REST, etc
 # endpoints
-EXPOSE 7997-8002
+EXPOSE 7997-8020
 
 # init
 COPY .travis/startml.sh /tmp/startml.sh

.travis/download.sh

@@ -4,4 +4,4 @@ echo "Logging in to Docker registry"
 docker login -u ${DOCKER_USER} -p ${DOCKER_PASSWORD}
 
 echo "Getting Docker Image"
-docker pull paxtonhare/marklogic-datahub-1x
+docker pull marklogiccommunity/marklogic-datahub-1x

CHANGELOG.md

@@ -1,7 +1,59 @@
 # Change Log
 
-## [v1.1.2](https://github.com/marklogic/marklogic-data-hub/tree/v1.1.2)
+## [v1.1.3](https://github.com/marklogic/marklogic-data-hub/tree/v1.1.3)
 
+[Full Changelog](https://github.com/marklogic/marklogic-data-hub/compare/v2.0.0-alpha.4...v1.1.3)
+
+**Implemented enhancements:**
+
+- Expose the ability to pass custom properties via gradle [\#416](https://github.com/marklogic/marklogic-data-hub/issues/416)
+- Add gradle wrapper to scaffolded project [\#415](https://github.com/marklogic/marklogic-data-hub/issues/415)
+- Entity view: show "Loading entities..." rather than "You don't have any entities yet" [\#411](https://github.com/marklogic/marklogic-data-hub/issues/411)
+- Better default document format for input flow [\#410](https://github.com/marklogic/marklogic-data-hub/issues/410)
+- Add button to sync indexes [\#393](https://github.com/marklogic/marklogic-data-hub/issues/393)
+- ugly scrollbars appear on project list in quickstart [\#385](https://github.com/marklogic/marklogic-data-hub/issues/385)
+- Handle Batch Flow Errors [\#379](https://github.com/marklogic/marklogic-data-hub/issues/379)
+- Default the harmonize collector to only get items in a standard input collection [\#344](https://github.com/marklogic/marklogic-data-hub/issues/344)
+- Add ability to specify source/target database for a Harmonize flow [\#319](https://github.com/marklogic/marklogic-data-hub/issues/319)
+
+**Fixed bugs:**
+
+- MLCP options: Output URI Replace is not working as expected [\#414](https://github.com/marklogic/marklogic-data-hub/issues/414)
+- Can't do load on Windows [\#407](https://github.com/marklogic/marklogic-data-hub/issues/407)
+- Illegal/unsupported escape sequence in Windows 10 when creating entities [\#406](https://github.com/marklogic/marklogic-data-hub/issues/406)
+- Better feedback for client-side validation failures [\#398](https://github.com/marklogic/marklogic-data-hub/issues/398)
+- XQuery bug detected but not shown on QuickStart GUI [\#395](https://github.com/marklogic/marklogic-data-hub/issues/395)
+- Save Options in Input Flow doesn't save changes to 'Output URI Replace' [\#390](https://github.com/marklogic/marklogic-data-hub/issues/390)
+- Default forests are created/attached even with custom forest JSON definitions [\#389](https://github.com/marklogic/marklogic-data-hub/issues/389)
+- mlReloadModules may not work [\#386](https://github.com/marklogic/marklogic-data-hub/issues/386)
+- Can't run harmonize on 2.0.0-alpha.2 [\#382](https://github.com/marklogic/marklogic-data-hub/issues/382)
+- Compile issue [\#381](https://github.com/marklogic/marklogic-data-hub/issues/381)
+- When one item fails in a harmonize batch run, other items in the chunk do not get processed [\#279](https://github.com/marklogic/marklogic-data-hub/issues/279)
+
+**Closed issues:**
+
+- Need to update my forest location while setting up the datahub framework; [\#356](https://github.com/marklogic/marklogic-data-hub/issues/356)
+- Harmonization flow not hitting staging port defined in gradle.properties [\#342](https://github.com/marklogic/marklogic-data-hub/issues/342)
+- gradle commands for flow creation [\#339](https://github.com/marklogic/marklogic-data-hub/issues/339)
+- gradle hubInit failing [\#307](https://github.com/marklogic/marklogic-data-hub/issues/307)
+- QuickStart App data loaded to incorrect directory? [\#251](https://github.com/marklogic/marklogic-data-hub/issues/251)
+- Allow Ingest to feed directly to conform w/o storing data [\#186](https://github.com/marklogic/marklogic-data-hub/issues/186)
+- DataHub::isInstalled\(\) is not sufficient [\#165](https://github.com/marklogic/marklogic-data-hub/issues/165)
+- Auto Generate Indexes based on entity defs [\#105](https://github.com/marklogic/marklogic-data-hub/issues/105)
+- Allow Exploration on the staged, raw data [\#2](https://github.com/marklogic/marklogic-data-hub/issues/2)
+
+**Merged pull requests:**
+
+- display a "Loading" message while retrieving entities [\#412](https://github.com/marklogic/marklogic-data-hub/pull/412) ([dmcassel](https://github.com/dmcassel))
+- disabled frame options in quickstart so it can be run inside an iframe [\#364](https://github.com/marklogic/marklogic-data-hub/pull/364) ([ryan321](https://github.com/ryan321))
+
+## [v2.0.0-alpha.4](https://github.com/marklogic/marklogic-data-hub/tree/v2.0.0-alpha.4) (2017-05-12)
+[Full Changelog](https://github.com/marklogic/marklogic-data-hub/compare/v2.0.0-alpha.3...v2.0.0-alpha.4)
+
+## [v2.0.0-alpha.3](https://github.com/marklogic/marklogic-data-hub/tree/v2.0.0-alpha.3) (2017-05-08)
+[Full Changelog](https://github.com/marklogic/marklogic-data-hub/compare/v1.1.2...v2.0.0-alpha.3)
+
+## [v1.1.2](https://github.com/marklogic/marklogic-data-hub/tree/v1.1.2) (2017-04-12)
 [Full Changelog](https://github.com/marklogic/marklogic-data-hub/compare/v1.1.1...v1.1.2)
 
 **Fixed bugs:**
@@ -14,7 +66,7 @@
 - QuickStart Template Compile Error 2.0-alpha.2 [\#380](https://github.com/marklogic/marklogic-data-hub/issues/380)
 
 ## [v1.1.1](https://github.com/marklogic/marklogic-data-hub/tree/v1.1.1) (2017-03-20)
-[Full Changelog](https://github.com/marklogic/marklogic-data-hub/compare/v1.1.0...v1.1.1)
+[Full Changelog](https://github.com/marklogic/marklogic-data-hub/compare/v2.0.0-alpha.2...v1.1.1)
 
 **Implemented enhancements:**
 
@@ -39,8 +91,27 @@
 - Adds CLA requirement \(\#361\) [\#362](https://github.com/marklogic/marklogic-data-hub/pull/362) ([jmakeig](https://github.com/jmakeig))
 - Updates Java Client API dependency to stable build [\#359](https://github.com/marklogic/marklogic-data-hub/pull/359) ([jmakeig](https://github.com/jmakeig))
 
+## [v2.0.0-alpha.2](https://github.com/marklogic/marklogic-data-hub/tree/v2.0.0-alpha.2) (2017-01-16)
+[Full Changelog](https://github.com/marklogic/marklogic-data-hub/compare/v1.1.0...v2.0.0-alpha.2)
+
+**Implemented enhancements:**
+
+- Allow exernal data to be passed in to a flow's options-map [\#334](https://github.com/marklogic/marklogic-data-hub/issues/334)
+- Integrate with Entity Services [\#283](https://github.com/marklogic/marklogic-data-hub/issues/283)
+- show traces for a given flow [\#277](https://github.com/marklogic/marklogic-data-hub/issues/277)
+
+**Fixed bugs:**
+
+- Returning json:object\(\) isn't invoking ES serialization in flow [\#345](https://github.com/marklogic/marklogic-data-hub/issues/345)
+
+**Closed issues:**
+
+- "No message available" when following Quick Start [\#352](https://github.com/marklogic/marklogic-data-hub/issues/352)
+- Add job-id to harmonized documents [\#351](https://github.com/marklogic/marklogic-data-hub/issues/351)
+- Entity modeling rendering incorrectly [\#347](https://github.com/marklogic/marklogic-data-hub/issues/347)
+
 ## [v1.1.0](https://github.com/marklogic/marklogic-data-hub/tree/v1.1.0) (2016-12-16)
-[Full Changelog](https://github.com/marklogic/marklogic-data-hub/compare/v1.0.2...v1.1.0)
+[Full Changelog](https://github.com/marklogic/marklogic-data-hub/compare/v2.0.0-alpha.1...v1.1.0)
 
 **Implemented enhancements:**
 
@@ -61,6 +132,13 @@
 - Keep user database settings separate from hub database settings [\#325](https://github.com/marklogic/marklogic-data-hub/issues/325)
 - Create a non-admin user for doing hub stuff [\#200](https://github.com/marklogic/marklogic-data-hub/issues/200)
 
+## [v2.0.0-alpha.1](https://github.com/marklogic/marklogic-data-hub/tree/v2.0.0-alpha.1) (2016-11-30)
+[Full Changelog](https://github.com/marklogic/marklogic-data-hub/compare/v1.0.2...v2.0.0-alpha.1)
+
+**Implemented enhancements:**
+
+- Visual entity type model editor [\#286](https://github.com/marklogic/marklogic-data-hub/issues/286)
+
 ## [v1.0.2](https://github.com/marklogic/marklogic-data-hub/tree/v1.0.2) (2016-11-22)
 [Full Changelog](https://github.com/marklogic/marklogic-data-hub/compare/v1.0.1...v1.0.2)
 

CONTRIBUTING.md

@@ -71,7 +71,7 @@ plugins {
    ...
 
    // comment out this line. It pulls the version from the cloud
-   // id 'com.marklogic.ml-data-hub' version '1.1.2'
+   // id 'com.marklogic.ml-data-hub' version '1.1.3'
 }
 
 // this tells gradle to apply the plugin you included above in the buildscript section

README.md

@@ -46,7 +46,7 @@ Alternatively you can include the jar file as a build dependency in your Java pr
 **Gradle**
 
 ```groovy
-compile('com.marklogic:marklogic-data-hub:1.1.2')
+compile('com.marklogic:marklogic-data-hub:1.1.3')
 ```
 
 **Maven**
@@ -55,15 +55,15 @@ compile('com.marklogic:marklogic-data-hub:1.1.2')
 <dependency>
   <groupId>com.marklogic</groupId>
   <artifactId>marklogic-data-hub</artifactId>
-  <version>1.1.2</version>
+  <version>1.1.3</version>
   <type>pom</type>
 </dependency>
 ```
 
 **Ivy**
 
 ```xml
-<dependency org='com.marklogic' name='marklogic-data-hub' rev='1.1.2'>
+<dependency org='com.marklogic' name='marklogic-data-hub' rev='1.1.3'>
   <artifact name='$AID' ext='pom'></artifact>
 </dependency>
 ```
@@ -74,7 +74,7 @@ If you prefer to use gradle for all of your hub interactions then you can includ
 
 ```groovy
 plugins {
-    id 'com.marklogic.ml-data-hub' version '1.1.2'
+    id 'com.marklogic.ml-data-hub' version '1.1.3'
 }
 ```
 

docker-compose.yml

@@ -1,7 +1,7 @@
 version: '2'
 services:
   marklogic:
-    image: paxtonhare/marklogic-datahub-1x
+    image: marklogiccommunity/marklogic-datahub-1x
     ports:
       - "8000-8020"
     volumes:

examples/spring-batch/build.gradle

@@ -1,34 +1,38 @@
-buildscript {
-  repositories {
-    maven {url 'http://developer.marklogic.com/maven2/'}
-  }
-}
-
 plugins {
     id 'java'
-    id 'eclipse'
-    id 'idea'
     id 'application'
     id 'net.saliman.properties' version '1.4.6'
-    id 'com.marklogic.ml-data-hub' version '1.1.2'
+    id 'com.marklogic.ml-data-hub' version '1.1.3'
 }
 
 repositories {
+    mavenLocal()
     jcenter()
     maven {url 'http://developer.marklogic.com/maven2/'}
 }
 
 dependencies {
-    compile 'com.marklogic:marklogic-data-hub:1.1.2'
-    compile 'com.marklogic:marklogic-spring-batch-core:0.6.0'
-    compile 'com.marklogic:ml-javaclient-util:2.13.0'
-    testCompile 'com.marklogic:marklogic-spring-batch-test:0.6.0'
-}
+    compile 'com.marklogic:marklogic-data-hub:1.1.3'
+    compile "com.marklogic:marklogic-spring-batch-core:0.7.4"
+    compile 'com.marklogic:ml-javaclient-util:4.0.alpha4'
 
-distributions {
-    main {
-        baseName = 'baseJob'
-    }
+    testCompile "com.marklogic:marklogic-spring-batch-test:0.7.4"
+    runtime "com.marklogic:marklogic-spring-batch-core:0.7.2"
 }
 
-mainClassName = "com.marklogic.spring.batch.hub.HubJobRunner"
+mainClassName = "com.marklogic.spring.batch.Main"
+
+task importMonsters(type: JavaExec) {
+    classpath = sourceSets.main.runtimeClasspath
+    main = "com.marklogic.spring.batch.Main"
+    args = [
+            "--config", "example.LoadAndRunFlow",
+            "--project_dir", ".",
+            "--env", "local",
+            "--input_file_path", "./input",
+            "--input_file_pattern", ".*\\.xml",
+            "--entity_name", "Monster",
+            "--flow_name", "ingest-monster",
+            "--chunk", "100"
+    ]
+}

examples/spring-batch/gradle.properties

@@ -8,31 +8,65 @@
 #
 # ....
 mlHost=localhost
-mlAppName=data-hub
 
+# Your MarkLogic Username and Password
 mlUsername=admin
 mlPassword=admin
 
+# If specified, the manage username/password combo is used with the ML Management REST API for managing application
+# resources; this user must have the manage-admin and rest-admin roles.
+#
+# If these are not set, then mlUsername/mlPassword is used for managing application resources.
+# mlManageUsername=
+# mlManagePassword=
+
+# If specified, these values can override where the DHF thinks
+# MarkLogic default ports are at. You would only use this if you
+# have changed the ports on which MarkLogic listens
+#
+# mlAppServicesPort=8000
+# mlAdminPort=8001
+# mlManagePort=8002
+
+# If specified, the admin username/password combo is used with the ML Management REST API for creating users and roles. This
+# user must have the manage-admin or admin role. A good practice is to use your admin account here to create app-specific
+# users and roles, which can then be used as mlManageUsername/mlManagePassword and mlUsername/mlPassword.
+#
+# These properties are also used for connecting to the admin application on port 8001 - e.g. for initializing ML and for
+# waiting for ML to restart.
+#
+# If these properties are not set, then mlUsername/mlPassword will be used.
+# mlAdminUsername=
+# mlAdminPassword=
+
 mlStagingAppserverName=data-hub-STAGING
 mlStagingPort=8010
 mlStagingDbName=data-hub-STAGING
 mlStagingForestsPerHost=4
+mlStagingAuth=digest
 
 mlFinalAppserverName=data-hub-FINAL
 mlFinalPort=8011
 mlFinalDbName=data-hub-FINAL
 mlFinalForestsPerHost=4
+mlFinalAuth=digest
 
 mlTraceAppserverName=data-hub-TRACING
 mlTracePort=8012
 mlTraceDbName=data-hub-TRACING
 mlTraceForestsPerHost=1
+mlTraceAuth=digest
 
 mlJobAppserverName=data-hub-JOBS
 mlJobPort=8013
 mlJobDbName=data-hub-JOBS
 mlJobForestsPerHost=1
+mlJobAuth=digest
 
 mlModulesDbName=data-hub-MODULES
 mlTriggersDbName=data-hub-TRIGGERS
 mlSchemasDbName=data-hub-SCHEMAS
+
+# The name of the Role to create for Hub Access
+mlHubUserRole=data-hub-ROLE
+mlHubUserName=data-hub-user