4.0.3 example changes

rahulvudutala · rahulvudutala · commit c1d69805c853 · 2018-11-09T15:25:44.000-08:00
diff --git a/examples/README.md b/examples/README.md
@@ -2,8 +2,11 @@
 
 This folder contains working examples of various DHF usage scenarios.
 
+1. [DHS-e2e](https://github.com/marklogic/marklogic-data-hub/tree/master/examples/DHS-e2e) - an end to end example demonstrating ingestion, harmonization and retrieval of data in DHS environment
 1. [barebones](https://github.com/marklogic/marklogic-data-hub/tree/master/examples/barebones) - an example of the minimum configuration necessary to run a Gradle based Data Hub
 1. [custom-tokens](https://github.com/marklogic/marklogic-data-hub/tree/master/examples/custom-tokens) - an example where gradle environment properties are used in substitutions of configuration files that is based off of the barebones example
+1. [gradle-tasks-demo](https://github.com/marklogic/marklogic-data-hub/tree/master/examples/gradle-tasks-demo) - a demonstration of gradle commands in DHS 
+environment
 1. [healthcare](https://github.com/marklogic/marklogic-data-hub/tree/master/examples/healthcare) - an example of a Healthcare 360 Data Hub
 1. [hr-hub](https://github.com/marklogic/marklogic-data-hub/tree/master/examples/hr-hub) - an example used for our 1.x tutorial. This example harmonizes data from various HR systems
 1. [load-binaries](https://github.com/marklogic/marklogic-data-hub/tree/master/examples/load-binaries) - an example of how to ingest binaries via an MLCP Input Flow
diff --git a/examples/barebones/README.md b/examples/barebones/README.md
@@ -13,7 +13,7 @@ plugins {
     // it includes ml-gradle. This plugin is what lets you
     // run DHF (Data Hub Framework) tasks from the
     // command line
-    id 'com.marklogic.ml-data-hub' version '4.0.1'
+    id 'com.marklogic.ml-data-hub' version '4.0.3'
 }
 ```
 
diff --git a/examples/barebones/build.gradle b/examples/barebones/build.gradle
@@ -7,5 +7,5 @@ plugins {
     // it includes ml-gradle. This plugin is what lets you
     // run DHF (Data Hub Framework) tasks from the
     // command line
-    id 'com.marklogic.ml-data-hub' version '4.0.1'
+    id 'com.marklogic.ml-data-hub' version '4.0.3'
 }
diff --git a/examples/custom-tokens/build.gradle b/examples/custom-tokens/build.gradle
@@ -7,5 +7,5 @@ plugins {
     // it includes ml-gradle. This plugin is what lets you
     // run DHF (Data Hub Framework) tasks from the
     // command line
-    id 'com.marklogic.ml-data-hub' version '4.0.1'
+    id 'com.marklogic.ml-data-hub' version '4.0.3'
 }
diff --git a/examples/custom-tokens/gradle.properties b/examples/custom-tokens/gradle.properties
@@ -7,7 +7,7 @@
 # named gradle-prod.properties.
 #
 # ....
-mlDHFVersion=4.0.1
+mlDHFVersion=4.0.3
 mlHost=localhost
 
 # If you are working with a load balancer please specify it here
@@ -30,7 +30,13 @@ mlPassword=
 # To use SSL on the Manage appserver (port 8002 by default)
 # mlManageScheme=https
 # mlManageSimpleSsl=true
-
+#
+# If specified, mlSecurityUsername/mlSecurityPassword is used for talking to Security end points on port 8002; this 
+# user must have the "manage-admin" and "security" roles.
+#
+# mlSecurityUsername=
+# mlSecurityPassword=
+#
 # If specified, the admin username/password combo is used with the ML Management REST API for creating users and roles. This
 # user must have the manage-admin or admin role. A good practice is to use your admin account here to create app-specific
 # users and roles, which can then be used as mlManageUsername/mlManagePassword and mlUsername/mlPassword.
@@ -125,6 +131,9 @@ mlHubAdminUserPassword=o@r*'.2HAt(3iAoXR/*v
 # Default module permissions which allow data-hub-role to execute flows
 mlModulePermissions=rest-reader,read,rest-writer,insert,rest-writer,update,rest-extension-user,execute,data-hub-role,read,data-hub-role,execute
 
+# If DHF is running in a provisioned environment please specify it here
+# mlIsProvisionedEnvironment=false
+
 # Custom properties defined here
 TEST_DATABASE_NAME=custom-tokens-test-database
 TEST_SERVER_NAME=custom-tokens-test-database-server
diff --git a/examples/data-integration-tests/build.gradle b/examples/data-integration-tests/build.gradle
@@ -19,7 +19,7 @@ plugins {
 
     // This gradle plugin extends the ml-gradle plugin with
     // commands that make the Data Hub Framework do its magic
-    id 'com.marklogic.ml-data-hub' version '4.0.1'
+    id 'com.marklogic.ml-data-hub' version '4.0.3'
 
     id "org.jetbrains.kotlin.jvm" version "1.2.21"
 }
@@ -34,7 +34,7 @@ repositories {
 dependencies {
     // this allows you to write custom java code that depends
     // on the Data Hub Framework library
-    compile 'com.marklogic:marklogic-data-hub:4.0.1'
+    compile 'com.marklogic:marklogic-data-hub:4.0.3'
     compile 'com.marklogic:marklogic-xcc:9.0.7'
     compile 'com.marklogic:marklogic-client-api:4.1.1'
     // prototyping with kotlin tests
diff --git a/examples/disconnected-project/build.gradle b/examples/disconnected-project/build.gradle
@@ -19,7 +19,7 @@ plugins {
 
     // This gradle plugin extends the ml-gradle plugin with
     // commands that make the Data Hub Framework do its magic
-    id 'com.marklogic.ml-data-hub' version '4.0.1'
+    id 'com.marklogic.ml-data-hub' version '4.0.3'
 }
 
 repositories {
@@ -42,7 +42,7 @@ dependencies {
     // this allows you to write custom java code that depends
     // on the Data Hub Framework library
 	// Remove if not needed
-    compile 'com.marklogic:marklogic-data-hub:4.0.1'
+    compile 'com.marklogic:marklogic-data-hub:4.0.3'
     compile 'com.marklogic:marklogic-xcc:9.0.7'
 
     //sample mlcp dependency. Remove if not required
diff --git a/examples/disconnected-project/gradle.properties b/examples/disconnected-project/gradle.properties
@@ -13,7 +13,7 @@ projectGradleHome=build/gradle-home
 # named gradle-prod.properties.
 #
 # ....
-mlDHFVersion=4.0.1
+mlDHFVersion=4.0.3
 mlHost=localhost
 
 # If you are working with a load balancer please specify it here
@@ -36,7 +36,13 @@ mlPassword=
 # To use SSL on the Manage appserver (port 8002 by default)
 # mlManageScheme=https
 # mlManageSimpleSsl=true
-
+#
+# If specified, mlSecurityUsername/mlSecurityPassword is used for talking to Security end points on port 8002; this 
+# user must have the "manage-admin" and "security" roles.
+#
+# mlSecurityUsername=
+# mlSecurityPassword=
+#
 # If specified, the admin username/password combo is used with the ML Management REST API for creating users and roles. This
 # user must have the manage-admin or admin role. A good practice is to use your admin account here to create app-specific
 # users and roles, which can then be used as mlManageUsername/mlManagePassword and mlUsername/mlPassword.
@@ -130,3 +136,6 @@ mlHubAdminUserPassword=o@r*'.2HAt(3iAoXR/*v
 
 # Default module permissions which allow data-hub-role to execute flows
 mlModulePermissions=rest-reader,read,rest-writer,insert,rest-writer,update,rest-extension-user,execute,data-hub-role,read,data-hub-role,execute
+
+# If DHF is running in a provisioned environment please specify it here
+# mlIsProvisionedEnvironment=false
diff --git a/examples/external-security/build.gradle b/examples/external-security/build.gradle
@@ -7,7 +7,7 @@ plugins {
     // it includes ml-gradle. This plugin is what lets you
     // run DHF (Data Hub Framework) tasks from the
     // command line
-    id 'com.marklogic.ml-data-hub' version '4.0.1'
+    id 'com.marklogic.ml-data-hub' version '4.0.3'
 }
 
 ext {
diff --git a/examples/external-security/gradle-example.properties b/examples/external-security/gradle-example.properties
@@ -7,7 +7,7 @@
 # named gradle-prod.properties.
 #
 # ....
-mlDHFVersion=4.0.1
+mlDHFVersion=4.0.3
 mlHost=localhost
 
 # If you are working with a load balancer please specify it here
@@ -30,7 +30,13 @@ mlPassword=
 # To use SSL on the Manage appserver (port 8002 by default)
 # mlManageScheme=https
 # mlManageSimpleSsl=true
-
+#
+# If specified, mlSecurityUsername/mlSecurityPassword is used for talking to Security end points on port 8002; this 
+# user must have the "manage-admin" and "security" roles.
+#
+# mlSecurityUsername=
+# mlSecurityPassword=
+#
 # If specified, the admin username/password combo is used with the ML Management REST API for creating users and roles. This
 # user must have the manage-admin or admin role. A good practice is to use your admin account here to create app-specific
 # users and roles, which can then be used as mlManageUsername/mlManagePassword and mlUsername/mlPassword.
@@ -125,6 +131,9 @@ mlHubAdminUserPassword=o@r*'.2HAt(3iAoXR/*v
 # Default module permissions which allow data-hub-role to execute flows
 mlModulePermissions=rest-reader,read,rest-writer,insert,rest-writer,update,rest-extension-user,execute,data-hub-role,read,data-hub-role,execute
 
+# If DHF is running in a provisioned environment please specify it here
+# mlIsProvisionedEnvironment=false
+
 #cp are Custom properties see https://github.com/marklogic/marklogic-data-hub/tree/master/examples/custom-tokens for details
 #this should match the name of the exteranl security configuration 
 cpExternalSecurity=LDAPConfig
diff --git a/examples/gradle-tasks-demo/README.md b/examples/gradle-tasks-demo/README.md
@@ -1,10 +1,20 @@
-Sample Project to work on DHS:
+Assumptions:
+1) This project assumes that DHS environment is already provisioned (DHS environment is provisioned in AWS. All the app servers, databases and required roles 
+are provisioned and the users are created)
+2) You have information about load balancer hosts. The loadbalancerhosts can be configured to be public. If the loadbalancerhosts are public, you can run this project from local environment. If the loadbalancerhosts are not public, then these hosts are only accessible from the VPC that is peered to the MarkLogic VPC (This can be accessed from your local by ssh tunneling)
+
+Example Demonstration:
+This example demonstrates the usage of gradle tasks to ingest and harmonize sample Product data into MarkLogic.
+The example will walk you through an end to end flow of loading datahub modules into DHS environment, creating entites, mappings, input/harmonize flows and 
+running the flows in DHS environment
+
+Steps to work on DHS:
 
 1) Add the build.gradle file from the docs folder into the current directory
 2) install gradle and run gradle wrapper --gradle-version 3.4
 3) ./gradlew build
 4) ./gradlew hubInit
-5) Copy the gradle.properties file from docs and update the user credentials
+5) Copy the gradle.properties file from docs and update the user credentials (Refer to this documentation: https://marklogic.github.io/marklogic-data-hub/project/dhsdeploy/)
 6) ./gradlew hubInstallModules (Installs dhf modules into DHS env)
 7) ./gradlew hubCreateEntity -PentityName=Product
 8) Copy the file Product.entity.json file from docs/entities folder into plugins/entities/Product in your current directory
@@ -18,4 +28,4 @@ Sample Project to work on DHS:
 16) ./gradlew mlLoadModules
 17) run create_hm_flows.sh file
 18) ./gradlew mlLoadModules
-19) run run_hm_flows.sh file
+19) run run_hm_flows.sh file
diff --git a/examples/gradle-tasks-demo/docs/build.gradle b/examples/gradle-tasks-demo/docs/build.gradle
@@ -11,5 +11,5 @@ plugins {
     // it includes ml-gradle. This plugin is what lets you
     // run DHF (Data Hub Framework) tasks from the
     // command line
-    id 'com.marklogic.ml-data-hub' version '4.0.1'
+    id 'com.marklogic.ml-data-hub' version '4.0.3'
 }
diff --git a/examples/gradle-tasks-demo/docs/gradle.properties b/examples/gradle-tasks-demo/docs/gradle.properties
@@ -0,0 +1,53 @@
+#
+# Copyright 2012-2018 MarkLogic Corporation
+#
+#  Licensed under the Apache License, Version 2.0 (the "License");
+#  you may not use this file except in compliance with the License.
+#  You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+#  Unless required by applicable law or agreed to in writing, software
+#  distributed under the License is distributed on an "AS IS" BASIS,
+#  WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+#  See the License for the specific language governing permissions and
+#  limitations under the License.
+mlDHFVersion=4.0.3
+mlHost=
+mlAppName=data-hub
+
+mlIsHostLoadBalancer=true
+
+mlUsername=
+mlPassword=
+mlManageUsername=
+mlManagePassword=
+#mlSecurityUsername=
+#mlSecurityPassword=
+
+mlStagingAppserverName=data-hub-STAGING
+mlStagingPort=8006
+mlStagingDbName=data-hub-STAGING
+mlStagingForestsPerHost=1
+
+mlFinalAppserverName=data-hub-FINAL
+mlFinalPort=8004
+mlFinalDbName=data-hub-FINAL
+mlFinalForestsPerHost=1
+
+mlJobAppserverName=data-hub-JOBS
+mlJobPort=8007
+mlJobDbName=data-hub-JOBS
+mlJobForestsPerHost=1
+
+mlModulesDbName=data-hub-MODULES
+mlStagingTriggersDbName=data-hub-staging-TRIGGERS
+mlStagingSchemasDbName=data-hub-staging-SCHEMAS
+
+mlFinalTriggersDbName=data-hub-final-TRIGGERS
+mlFinalSchemasDbName=data-hub-final-SCHEMAS
+
+mlModulePermissions=flowDeveloper,read,flowDeveloper,execute,flowDeveloper,insert,flowOperator,read,flowOperator,execute,flowOperator,insert
+
+# If DHF is running in a provisioned environment please specify it here
+mlIsProvisionedEnvironment=true
diff --git a/examples/healthcare/README.md b/examples/healthcare/README.md
@@ -15,9 +15,9 @@ The sample data is located in the input/ folder.
 ```
 
 # TLDR; How do I run it?
-1. Download the [latest quick-start war](https://github.com/marklogic/marklogic-data-hub/releases/download/v4.0.1/quick-start-4.0.1.war) into this folder.
+1. Download the [latest quick-start war](https://github.com/marklogic/marklogic-data-hub/releases/download/4.0.3/quick-start-4.0.3.war) into this folder.
 
-1. Run the quick-start jar `java -jar quick-start-4.0.1.war`
+1. Run the quick-start jar `java -jar quick-start-4.0.3.war`
 
 1. Open your web browser to [http://localhost:8080](http://localhost:8080).
 
diff --git a/examples/healthcare/build.gradle b/examples/healthcare/build.gradle
@@ -7,5 +7,5 @@ plugins {
     // it includes ml-gradle. This plugin is what lets you
     // run DHF (Data Hub Framework) tasks from the
     // command line
-    id 'com.marklogic.ml-data-hub' version '4.0.0'
+    id 'com.marklogic.ml-data-hub' version '4.0.3'
 }
diff --git a/examples/load-binaries/README.md b/examples/load-binaries/README.md
@@ -2,9 +2,9 @@
 This example shows how to load binary documents with the Hub Framework.
 
 # TLDR; How do I run it?
-1. Download the [latest quick-start war](https://github.com/marklogic/marklogic-data-hub/releases/download/v4.0.1/quick-start-4.0.1.war) into this folder.
+1. Download the [latest quick-start war](https://github.com/marklogic/marklogic-data-hub/releases/download/4.0.3/quick-start-4.0.3.war) into this folder.
 
-1. Run the quick-start war `java -jar quick-start-4.0.1.war`
+1. Run the quick-start war `java -jar quick-start-4.0.3.war`
 
 1. Open your web browser to [http://localhost:8080](http://localhost:8080).
 
diff --git a/examples/online-store/README.md b/examples/online-store/README.md
@@ -16,7 +16,7 @@ The data are separated into 2 folders under the input/ folder.
 # TLDR; How do I run it?
 Check out the [Getting Started Tutorial](https://marklogic.github.io/marklogic-data-hub/) on the Datahub Site.
 
-You will want to download the latest 4.x release of the [QuickStart Application](https://github.com/marklogic/marklogic-data-hub/releases/download/v4.0.1/quick-start-4.0.1.war) and save it in this directory.
+You will want to download the latest 4.x release of the [QuickStart Application](https://github.com/marklogic/marklogic-data-hub/releases/download/4.0.3/quick-start-4.0.3.war) and save it in this directory.
 
 # Wait. What is a data Hub?
 Get started over at the [Data Hub Site](https://marklogic.github.io/marklogic-data-hub/)
diff --git a/examples/online-store/build.gradle b/examples/online-store/build.gradle
@@ -7,5 +7,5 @@ plugins {
     // it includes ml-gradle. This plugin is what lets you
     // run DHF (Data Hub Framework) tasks from the
     // command line
-    id 'com.marklogic.ml-data-hub' version '4.0.1'
+    id 'com.marklogic.ml-data-hub' version '4.0.3'
 }
diff --git a/examples/single-step-ingest/README.md b/examples/single-step-ingest/README.md
@@ -11,9 +11,9 @@ The sample data is located in the input/ folder
 ```
 
 # TLDR; How do I run it?
-1. Download the [latest quick-start war](https://github.com/marklogic/marklogic-data-hub/releases/download/v4.0.1/quick-start-4.0.1.war) into this folder.
+1. Download the [latest quick-start war](https://github.com/marklogic/marklogic-data-hub/releases/download/4.0.3/quick-start-4.0.3.war) into this folder.
 
-1. Run the quick-start jar `java -jar quick-start-4.0.1.war`
+1. Run the quick-start jar `java -jar quick-start-4.0.3.war`
 
 1. Open your web browser to [http://localhost:8080](http://localhost:8080).
 
diff --git a/examples/spring-batch/README.md b/examples/spring-batch/README.md
@@ -8,7 +8,7 @@ This example loads relational data from an [Invoice Database](./invoices-sql-dia
 
 ## How do I run this example?
 
-1. [Deploy the MarkLogic Job Repository](https://github.com/marklogic/marklogic-spring-batch/wiki/MarkLogicJobRepository). When the application starts it needs to persist the job metadata into MarkLogic.  Note that this metadata is different than datahub job metadata.  
+1. [Deploy the MarkLogic Job Repository](https://github.com/marklogic-community/marklogic-spring-batch/wiki/SetupMarkLogicJobRepository). When the application starts it needs to persist the job metadata into MarkLogic.  Note that this metadata is different than datahub job metadata.  
 1. Start up the Data Hub Quick Start, deploy the project from the examples/spring-batch directory.  Verify that the Customer entity and customer-flow exist.  
 1. Verify the property values in src/test/resources/job.properties for your environment.  
    1. Verify the hosts for the STAGING database and mlJobRepo
diff --git a/examples/spring-batch/build.gradle b/examples/spring-batch/build.gradle
@@ -3,7 +3,7 @@ plugins {
     id 'application'
     id 'idea'
     id 'net.saliman.properties' version '1.4.6'
-    id 'com.marklogic.ml-data-hub' version '4.0.1'
+    id 'com.marklogic.ml-data-hub' version '4.0.3'
 }
 
 repositories {
@@ -12,7 +12,7 @@ repositories {
 }
 
 dependencies {
-    compile 'com.marklogic:marklogic-data-hub:4.0.1'
+    compile 'com.marklogic:marklogic-data-hub:4.0.3'
     compile "com.marklogic:marklogic-spring-batch-core:1.7.0"
     compile 'com.marklogic:spring-batch-rdbms:1.7.0'
 
diff --git a/examples/ssl/build.gradle b/examples/ssl/build.gradle
@@ -7,7 +7,7 @@ plugins {
     // it includes ml-gradle. This plugin is what lets you
     // run DHF (Data Hub Framework) tasks from the
     // command line
-    id 'com.marklogic.ml-data-hub' version '4.0.1'
+    id 'com.marklogic.ml-data-hub' version '4.0.3'
 }
 
 ext {
diff --git a/examples/ssl/gradle.properties b/examples/ssl/gradle.properties
@@ -7,7 +7,7 @@
 # named gradle-prod.properties.
 #
 # ....
-mlDHFVersion=4.0.1
+mlDHFVersion=4.0.3
 mlHost=localhost
 
 # If you are working with a load balancer please specify it here
@@ -30,7 +30,13 @@ mlPassword=
 # To use SSL on the Manage appserver (port 8002 by default)
 # mlManageScheme=https
 # mlManageSimpleSsl=true
-
+#
+# If specified, mlSecurityUsername/mlSecurityPassword is used for talking to Security end points on port 8002; this 
+# user must have the "manage-admin" and "security" roles.
+#
+# mlSecurityUsername=
+# mlSecurityPassword=
+#
 # If specified, the admin username/password combo is used with the ML Management REST API for creating users and roles. This
 # user must have the manage-admin or admin role. A good practice is to use your admin account here to create app-specific
 # users and roles, which can then be used as mlManageUsername/mlManagePassword and mlUsername/mlPassword.
@@ -125,6 +131,9 @@ mlHubAdminUserPassword=o@r*'.2HAt(3iAoXR/*v
 # Default module permissions which allow data-hub-role to execute flows
 mlModulePermissions=rest-reader,read,rest-writer,insert,rest-writer,update,rest-extension-user,execute,data-hub-role,read,data-hub-role,execute
 
+# If DHF is running in a provisioned environment please specify it here
+# mlIsProvisionedEnvironment=false
+
 # To use SSL on port 8001
 mlAdminScheme=https
 mlAdminSimpleSsl=true

Original file line number	Diff line number	Diff line change
`@@ -13,7 +13,7 @@ plugins {`
`13`	`13`	`// it includes ml-gradle. This plugin is what lets you`
`14`	`14`	`// run DHF (Data Hub Framework) tasks from the`
`15`	`15`	`// command line`
`16`		`- id 'com.marklogic.ml-data-hub' version '4.0.1'`
	`16`	`+ id 'com.marklogic.ml-data-hub' version '4.0.3'`
`17`	`17`	`}`
`18`	`18`	```
`19`	`19`
Original file line number	Diff line number	Diff line change
`@@ -7,5 +7,5 @@ plugins {`
`7`	`7`	`// it includes ml-gradle. This plugin is what lets you`
`8`	`8`	`// run DHF (Data Hub Framework) tasks from the`
`9`	`9`	`// command line`
`10`		`- id 'com.marklogic.ml-data-hub' version '4.0.1'`
	`10`	`+ id 'com.marklogic.ml-data-hub' version '4.0.3'`
`11`	`11`	`}`
Original file line number	Diff line number	Diff line change
`@@ -7,7 +7,7 @@ plugins {`
`7`	`7`	`// it includes ml-gradle. This plugin is what lets you`
`8`	`8`	`// run DHF (Data Hub Framework) tasks from the`
`9`	`9`	`// command line`
`10`		`- id 'com.marklogic.ml-data-hub' version '4.0.1'`
	`10`	`+ id 'com.marklogic.ml-data-hub' version '4.0.3'`
`11`	`11`	`}`
`12`	`12`
`13`	`13`	`ext {`