Merge branch 'release/2.0.0-alpha.1' into 2.0-master

Author: paxtonhare

.gitignore

@@ -21,3 +21,4 @@ flow.properties
 node_modules/
 .tmp/
 *.sublime-*
+.vscode/

CHANGELOG.md

@@ -1,7 +1,28 @@
 # Change Log
 
-## [v1.0.1](https://github.com/marklogic/marklogic-data-hub/tree/v1.0.1)
+## [v2.0.1-alpha.1](https://github.com/marklogic/marklogic-data-hub/tree/v2.0.1-alpha.1)
 
+[Full Changelog](https://github.com/marklogic/marklogic-data-hub/compare/v1.0.2...v2.0.1-alpha.1)
+
+**Implemented enhancements:**
+
+- Visual entity type model editor [\#286](https://github.com/marklogic/marklogic-data-hub/issues/286)
+
+## [v1.0.2](https://github.com/marklogic/marklogic-data-hub/tree/v1.0.2) (2016-11-22)
+[Full Changelog](https://github.com/marklogic/marklogic-data-hub/compare/v1.0.1...v1.0.2)
+
+**Fixed bugs:**
+
+- mlcp load from QuickStart GUI not loading data [\#331](https://github.com/marklogic/marklogic-data-hub/issues/331)
+
+**Closed issues:**
+
+- Possible typo on wiki page [\#333](https://github.com/marklogic/marklogic-data-hub/issues/333)
+- Tutorial instruction - create entity "Employees" instead of "Employee" [\#330](https://github.com/marklogic/marklogic-data-hub/issues/330)
+- mlcp job is not getting run + console log button not showing [\#329](https://github.com/marklogic/marklogic-data-hub/issues/329)
+- Error Extension hubstats does not exist. [\#328](https://github.com/marklogic/marklogic-data-hub/issues/328)
+
+## [v1.0.1](https://github.com/marklogic/marklogic-data-hub/tree/v1.0.1) (2016-11-08)
 [Full Changelog](https://github.com/marklogic/marklogic-data-hub/compare/v1.0.0...v1.0.1)
 
 **Fixed bugs:**

README.md

@@ -2,15 +2,19 @@
 
 OS | Status
 --- | --- | ---
-Linux/Mac | [![Build Status](https://travis-ci.org/marklogic/marklogic-data-hub.svg?branch=master)](https://travis-ci.org/marklogic/marklogic-data-hub)
+Linux/Mac | [![Build Status](https://travis-ci.org/marklogic/marklogic-data-hub.svg?branch=2.0-develop)](https://travis-ci.org/marklogic/marklogic-data-hub)
 Windows | [![Windows Build status](https://ci.appveyor.com/api/projects/status/kgj0k5na59uhkvbv?svg=true)](https://ci.appveyor.com/project/paxtonhare/marklogic-data-hub)
 
 # MarkLogic Data Hub
 
+## WARNING!!!! This is the the 2.0 branch of The MarkLogic Data Hub Framework. The 2.0 version is only compatible with MarkLogic 9.0 which is currently in Early Access and not generally available.
+
 Go from nothing to Enterprise Data Hub in a matter of minutes.  
 
 This project allows you to deploy a skeleton Data Hub into MarkLogic. With some basic configuration you will be running an Enterprise Data Hub in no time.
 
+
+
 # Getting Started
 
 ###TL;DR
@@ -36,7 +40,7 @@ Alternatively you can include the jar file as a build dependency in your Java pr
 **Gradle**
 
 ```groovy
-compile('com.marklogic:marklogic-data-hub:1.0.1')
+compile('com.marklogic:marklogic-data-hub:2.0.0-alpha.1')
 ```
 
 **Maven**
@@ -45,15 +49,15 @@ compile('com.marklogic:marklogic-data-hub:1.0.1')
 <dependency>
   <groupId>com.marklogic</groupId>
   <artifactId>marklogic-data-hub</artifactId>
-  <version>1.0.1</version>
+  <version>2.0.0-alpha.1</version>
   <type>pom</type>
 </dependency>
 ```
 
 **Ivy**
 
 ```xml
-<dependency org='com.marklogic' name='marklogic-data-hub' rev='1.0.1'>
+<dependency org='com.marklogic' name='marklogic-data-hub' rev='2.0.0-alpha.1'>
   <artifact name='$AID' ext='pom'></artifact>
 </dependency>
 ```
@@ -64,7 +68,7 @@ If you prefer to use gradle for all of your hub interactions then you can includ
 
 ```groovy
 plugins {
-    id 'com.marklogic.ml-data-hub' version '1.0.1'
+    id 'com.marklogic.ml-data-hub' version '2.0.0-alpha.1'
 }
 ```
 

examples/spring-batch/build.gradle

@@ -4,7 +4,7 @@ plugins {
     id 'idea'
     id 'application'
     id 'net.saliman.properties' version '1.4.6'
-    id 'com.marklogic.ml-data-hub' version '1.0.1'
+    id 'com.marklogic.ml-data-hub' version '2.0.0-alpha.1'
 }
 
 repositories {
@@ -13,7 +13,7 @@ repositories {
 }
 
 dependencies {
-    compile 'com.marklogic:marklogic-data-hub:1.0.1'
+    compile 'com.marklogic:marklogic-data-hub:2.0.0-alpha.1'
     compile 'com.marklogic:marklogic-spring-batch-core:0.6.0'
     compile 'com.marklogic:ml-javaclient-util:2.9.1'
     testCompile 'com.marklogic:marklogic-spring-batch-test:0.6.0'

marklogic-data-hub/gradle.properties

@@ -36,6 +36,6 @@ hubProjectDir=examples/hr-hub
 mlReplaceTokensInModules=true
 mlUseRoxyTokenPrefix=false
 
-version=1.0.1
+version=2.0.0-alpha.1
 
 

marklogic-data-hub/src/main/java/com/marklogic/hub/DataHub.java

@@ -174,9 +174,9 @@ public void validateServer() throws ServerValidationException {
                 alteredString += "0";
             }
             int major = Integer.parseInt(alteredString.substring(0, 1));
-            int ver = Integer.parseInt(alteredString.substring(0, 3));
-            boolean isNightly = versionString.matches("[^-]+-\\d{8}");
-            if (major < 8 || (!isNightly && ver < 804)) {
+//            int ver = Integer.parseInt(alteredString.substring(0, 3));
+//            boolean isNightly = versionString.matches("[^-]+-\\d{8}");
+            if (major < 9) {
                 throw new ServerValidationException("Invalid MarkLogic Server Version: " + versionString);
             }
 

marklogic-data-hub/src/main/java/com/marklogic/hub/deploy/commands/LoadUserModulesCommand.java

@@ -85,7 +85,7 @@ public void execute(CommandContext context) {
         DefaultModulesLoader modulesLoader = getStagingModulesLoader(config);
         modulesLoader.loadModules(baseDir, new AssetModulesFinder(), stagingClient);
 
-        JSONDocumentManager entityDocMgr = stagingClient.newJSONDocumentManager();
+        JSONDocumentManager entityDocMgr = finalClient.newJSONDocumentManager();
 
         AllButAssetsModulesFinder allButAssetsModulesFinder = new AllButAssetsModulesFinder();
 

marklogic-data-hub/src/main/java/com/marklogic/hub/scaffold/Scaffolding.java

@@ -15,29 +15,39 @@
  */
 package com.marklogic.hub.scaffold;
 
-import com.marklogic.client.helper.LoggingObject;
-import com.marklogic.client.io.Format;
-import com.marklogic.hub.error.ScaffoldingValidationException;
-import com.marklogic.hub.flow.FlowType;
-import com.marklogic.hub.flow.SimpleFlow;
-import com.marklogic.hub.plugin.PluginFormat;
-
 import java.io.*;
+import java.nio.charset.StandardCharsets;
 import java.nio.file.Files;
 import java.nio.file.Path;
 import java.nio.file.Paths;
 
+import com.marklogic.client.DatabaseClient;
+import com.marklogic.client.extensions.ResourceManager;
+import com.marklogic.client.extensions.ResourceServices;
+import com.marklogic.client.helper.LoggingObject;
+import com.marklogic.client.io.StringHandle;
+import com.marklogic.client.util.RequestParameters;
+import com.marklogic.hub.error.ScaffoldingValidationException;
+import com.marklogic.hub.plugin.PluginFormat;
+import com.sun.jersey.api.client.ClientHandlerException;
+
+import com.marklogic.client.io.Format;
+import com.marklogic.hub.flow.FlowType;
+import com.marklogic.hub.flow.SimpleFlow;
+
 public class Scaffolding extends LoggingObject {
 
     private String projectDir;
     private Path pluginsDir;
     private Path entitiesDir;
     private ScaffoldingValidator validator;
+    private DatabaseClient databaseClient;
 
-    public Scaffolding(String projectDir) {
+    public Scaffolding(String projectDir, DatabaseClient databaseClient) {
         this.projectDir = projectDir;
         this.pluginsDir = Paths.get(this.projectDir, "plugins");
         this.entitiesDir = this.pluginsDir.resolve("entities");
+        this.databaseClient = databaseClient;
         validator = new ScaffoldingValidator(projectDir);
     }
 
@@ -55,7 +65,14 @@ public void createEntity(String entityName) throws FileNotFoundException {
     }
 
     public void createFlow(String entityName, String flowName,
-                           FlowType flowType, PluginFormat pluginFormat, Format dataFormat)
+                           FlowType flowType, PluginFormat pluginFormat,
+                           Format dataFormat) throws IOException {
+        createFlow(entityName, flowName, flowType, pluginFormat, dataFormat, false);
+    }
+
+    public void createFlow(String entityName, String flowName,
+                           FlowType flowType, PluginFormat pluginFormat,
+                           Format dataFormat, boolean useEsModel)
             throws IOException {
         Path flowDir = getFlowDir(entityName, flowName, flowType);
 
@@ -73,8 +90,17 @@ public void createFlow(String entityName, String flowName,
 
         Path contentDir = flowDir.resolve("content");
         contentDir.toFile().mkdirs();
-        writeFile("scaffolding/" + flowType + "/" + pluginFormat + "/content." + pluginFormat,
+
+
+        if (useEsModel) {
+            ContentPlugin cp = new ContentPlugin(databaseClient);
+            String content = cp.getContents(entityName, pluginFormat);
+            writeBuffer(content, contentDir.resolve("content." + pluginFormat));
+        }
+        else {
+            writeFile("scaffolding/" + flowType + "/" + pluginFormat + "/content." + pluginFormat,
                 contentDir.resolve("content." + pluginFormat));
+        }
 
         Path headerDir = flowDir.resolve("headers");
         headerDir.toFile().mkdirs();
@@ -104,6 +130,14 @@ private void writeFile(String srcFile, Path dstFile) throws IOException {
         }
     }
 
+    private void writeBuffer(String buffer, Path dstFile) throws IOException {
+        logger.info("writing: " + dstFile.toString());
+        if (!dstFile.toFile().exists()) {
+            InputStream inputStream = new ByteArrayInputStream(buffer.getBytes(StandardCharsets.UTF_8));
+            Files.copy(inputStream, dstFile);
+        }
+    }
+
     public void createRestExtension(String entityName, String extensionName,
             FlowType flowType, PluginFormat pluginFormat) throws IOException, ScaffoldingValidationException {
         logger.info(extensionName);
@@ -224,4 +258,33 @@ public static String getAbsolutePath(String first, String... more) {
         }
         return absolutePath.toString();
     }
+
+    public class ContentPlugin extends ResourceManager {
+        private static final String NAME = "scaffold-content";
+
+        private RequestParameters params = new RequestParameters();
+
+        public ContentPlugin(DatabaseClient client) {
+            super();
+            client.init(NAME, this);
+        }
+
+        public String getContents(String entityName, PluginFormat pluginFormat) throws IOException {
+            try {
+                params.add("entity", entityName);
+                params.add("pluginFormat", pluginFormat.toString());
+                ResourceServices.ServiceResultIterator resultItr = this.getServices().get(params);
+                if (resultItr == null || ! resultItr.hasNext()) {
+                    throw new IOException("Unable to get Content Plugin scaffold");
+                }
+                ResourceServices.ServiceResult res = resultItr.next();
+                return res.getContent(new StringHandle()).get();
+            }
+            catch(ClientHandlerException e) {
+            }
+            return "{}";
+        }
+
+    }
+
 }

marklogic-data-hub/src/main/resources/ml-modules/root/com.marklogic.hub/lib/flow-lib.xqy

@@ -562,15 +562,15 @@ declare function flow:make-envelope(
         map:put($o, "headers", $headers),
         map:put($o, "triples", $triples),
         map:put($o, "instance",
-          if ($content instance of map:map) then
+          if ($content castable as map:map) then
             flow:instance-to-canonical-json($content)
           else
             $content
         ),
         map:put($o, "attachments",
           let $content := map:get($map, "content")
           return
-            if ($content instance of map:map) then
+            if ($content castable as map:map) then
               map:get($content, "$attachments")
             else
               ()
@@ -590,9 +590,8 @@ declare function flow:make-envelope(
         <instance>
         {
           let $content := map:get($map, "content")
-          let $_ := xdmp:log(("content: [", $content, "]"))
           return
-            if ($content instance of map:map) then
+            if ($content castable as map:map) then
               flow:instance-to-canonical-xml($content)
             else
               $content
@@ -602,7 +601,7 @@ declare function flow:make-envelope(
         {
           let $content := map:get($map, "content")
           return
-            if ($content instance of map:map) then
+            if ($content castable as map:map) then
               map:get($content, "$attachments")
             else
               ()
@@ -844,10 +843,12 @@ declare function flow:run-plugin(
 
     let $resp :=
       typeswitch($resp)
-        case map:map return
-          $resp
         case object-node() | json:object return
-          if ($data-format = $XML) then
+          (: map:map lands here too :)
+          (: map:map is ES response type :)
+          if ($resp castable as map:map) then
+            $resp
+          else if ($data-format = $XML) then
             json:transform-from-json($resp, json:config("custom"))
           else
             $resp
@@ -938,14 +939,22 @@ declare function flow:run-writer(
       xdmp:rethrow()
     }
   let $duration := xdmp:elapsed-time() - $before
+  let $is-xml :=
+    let $e :=
+      if ($envelope instance of document-node()) then
+        $envelope/node()
+      else
+        $envelope
+    return
+      $e instance of element()
   let $_ :=
     trace:plugin-trace(
       $identifier,
       $module-uri,
       "writer",
       $flow-type,
       $envelope,
-      if ($envelope instance of element()) then ()
+      if ($is-xml) then ()
       else null-node {},
       $duration
     )

marklogic-data-hub/src/main/resources/ml-modules/root/com.marklogic.hub/lib/hub-entities.xqy

@@ -9,6 +9,25 @@ declare variable $ENTITY-MODEL-COLLECTION := "http://marklogic.com/entity-servic
 
 declare option xdmp:mapping "false";
 
+declare function hent:get-model($entity-name as xs:string)
+{
+  let $model := fn:collection($ENTITY-MODEL-COLLECTION)[info/title = $entity-name]
+  where fn:exists($model)
+  return
+    let $model-map as map:map? := $model
+    let $refs := $model//*[fn:local-name(.) = '$ref'][fn:starts-with(., "#/definitions")] ! fn:replace(., "#/definitions/", "")
+    let $_ :=
+      let $definitions := $model-map=>map:get("definitions")
+      for $ref in $refs
+      let $other-model as map:map? := hent:get-model($ref)
+      let $other-defs := $other-model=>map:get("definitions")
+      for $key in map:keys($other-defs)
+      return
+        $definitions=>map:put($key, $other-defs=>map:get($key))
+    return
+      $model-map
+};
+
 declare function hent:dump-indexes()
 {
   let $uber-model := map:map()