Merge branch 'feature/entity-services' into develop

Author: paxtonhare

.github/ISSUE_TEMPLATE.md

@@ -0,0 +1,29 @@
+### Welcome Hub enthusiast!
+
+Let's get the most out of this ticket.
+
+#### The issue
+
+Short description of the problem:
+
+What behavior are you expecting?
+
+#### Tech details
+
+Which Operating System are you using?
+
+Which version of MarkLogic are you using?
+
+Which version of the Data Hub Framework are you using?
+
+Is this a QuickStart UI bug? If so, which browser are you using?
+
+#### The devil is in the details...
+
+If possible, rerun the command with -d for debugging output and attach the output:
+
+  **for quickstart:**  
+  `java -jar quickstart.war -d > myoutput.txt`
+
+  **for gradle:**  
+  `gradle mlYourCommand -d > myoutput.txt`

marklogic-data-hub/build.gradle

@@ -22,7 +22,7 @@ dependencies {
     compile 'org.springframework.batch:spring-batch-core:3.0.6.RELEASE'
     compile 'org.springframework:spring-jdbc:4.2.6.RELEASE'
     compile 'com.marklogic:java-client-api:3.0.5'
-    compile 'com.marklogic:ml-javaclient-util:2.9.1'
+    compile 'com.marklogic:ml-javaclient-util:2.10.0'
     compile 'com.marklogic:ml-app-deployer:2.3.0'
     compile 'com.marklogic:marklogic-spring-batch-core:0.7.0'
     compile 'commons-io:commons-io:2.4'

marklogic-data-hub/src/main/java/com/marklogic/hub/DataHub.java

@@ -51,9 +51,12 @@
 import com.marklogic.client.helper.LoggingObject;
 import com.marklogic.client.io.JacksonHandle;
 import com.marklogic.client.util.RequestParameters;
-import com.marklogic.hub.commands.DeployHubDatabasesCommand;
-import com.marklogic.hub.commands.LoadHubModulesCommand;
-import com.marklogic.hub.commands.LoadUserModulesCommand;
+import com.marklogic.hub.deploy.commands.DeployHubDatabasesCommand;
+import com.marklogic.hub.deploy.commands.LoadHubModulesCommand;
+import com.marklogic.hub.deploy.commands.LoadUserModulesCommand;
+import com.marklogic.hub.deploy.HubAppDeployer;
+import com.marklogic.hub.deploy.util.HubDeployStatusListener;
+import com.marklogic.hub.error.ServerValidationException;
 import com.marklogic.mgmt.ManageClient;
 import com.marklogic.mgmt.ManageConfig;
 import com.marklogic.mgmt.admin.AdminConfig;
@@ -94,19 +97,24 @@ public DataHub(String host, String username, String password) {
 
     private void init(HubConfig hubConfig) {
         this.hubConfig = hubConfig;
-        config = new ManageConfig(hubConfig.host, 8002, hubConfig.username, hubConfig.password);
-        client = new ManageClient(config);
-        databaseManager = new DatabaseManager(client);
-        serverManager = new ServerManager(client);
-
-        AdminConfig adminConfig = new AdminConfig();
-        adminConfig.setHost(hubConfig.host);
-        adminConfig.setUsername(hubConfig.username);
-        adminConfig.setPassword(hubConfig.password);
-        adminManager = new AdminManager(adminConfig);
+        if (hubConfig.username != null && hubConfig.password != null) {
+            config = new ManageConfig(hubConfig.host, 8002, hubConfig.username, hubConfig.password);
+            client = new ManageClient(config);
+            databaseManager = new DatabaseManager(client);
+            serverManager = new ServerManager(client);
+
+            AdminConfig adminConfig = new AdminConfig();
+            adminConfig.setHost(hubConfig.host);
+            adminConfig.setUsername(hubConfig.adminUsername);
+            adminConfig.setPassword(hubConfig.adminPassword);
+            adminManager = new AdminManager(adminConfig);
+        }
+        else {
+            logger.info("Missing username and/or password.");
+        }
     }
 
-    public void setAdminManager(AdminManager manager) {
+    void setAdminManager(AdminManager manager) {
         this.adminManager = manager;
     }
 
@@ -203,6 +211,10 @@ private AppConfig getAppConfig() {
         return config;
     }
 
+    /**
+     * Updates the given AppConfig with values from this DataHub
+     * @param config - the AppConfig instance to update
+     */
     public void updateAppConfig(AppConfig config) {
         config.setHost(hubConfig.host);
         config.setRestPort(hubConfig.stagingPort);
@@ -253,6 +265,10 @@ public void updateAppConfig(AppConfig config) {
         customTokens.put("%%mlModulesDbName%%", hubConfig.modulesDbName);
         customTokens.put("%%mlTriggersDbName%%", hubConfig.triggersDbName);
         customTokens.put("%%mlSchemasDbName%%", hubConfig.schemasDbName);
+
+        customTokens.put("%%mlHubUserName%%", hubConfig.hubUserName);
+        customTokens.put("%%mlHubUserPassword%%", hubConfig.hubUserPassword);
+        customTokens.put("%%mlHubUserRole%%", hubConfig.hubUserRole);
     }
 
     public void initProject() {
@@ -426,17 +442,26 @@ private List<Command> getCommands(AppConfig config) {
 
         // SQL Views
         List<Command> viewCommands = new ArrayList<>();
-        viewCommands.add(new DeployViewSchemasCommand());
+        DeployViewSchemasCommand deployViewSchemasCommand = new DeployViewSchemasCommand();
+        deployViewSchemasCommand.setDatabaseIdOrName(hubConfig.finalDbName);
+        viewCommands.add(deployViewSchemasCommand);
         commands.addAll(viewCommands);
 
         return commands;
     }
 
+    /**
+     * Installs the data hub configuration and server-side modules into MarkLogic
+     */
+    public void install() {
+        install(null);
+    }
+
     /**
      * Installs the data hub configuration and server-side modules into MarkLogic
      * @param listener - the callback method to receive status updates
      */
-    public void install(StatusListener listener) {
+    public void install(HubDeployStatusListener listener) {
         initProject();
 
         logger.info("Installing the Data Hub into MarkLogic");
@@ -447,11 +472,18 @@ public void install(StatusListener listener) {
         deployer.deploy(config);
     }
 
+    /**
+     * Uninstalls the data hub configuration and server-side modules from MarkLogic
+     */
+    public void uninstall() {
+        uninstall(null);
+    }
+
     /**
      * Uninstalls the data hub configuration and server-side modules from MarkLogic
      * @param listener - the callback method to receive status updates
      */
-    public void uninstall(StatusListener listener) {
+    public void uninstall(HubDeployStatusListener listener) {
         logger.debug("Uninstalling the Data Hub from MarkLogic");
 
         AppConfig config = getAppConfig();

marklogic-data-hub/src/main/java/com/marklogic/hub/Debugging.java

@@ -16,7 +16,7 @@ public Debugging(DatabaseClient client) {
     }
 
     /**
-     * Enables tracing
+     * Enables debugging
      */
     public void enable() {
         RequestParameters params = new RequestParameters();
@@ -25,7 +25,7 @@ public void enable() {
     }
 
     /**
-     * Disables tracing
+     * Disables debugging
      */
     public void disable() {
         RequestParameters params = new RequestParameters();
@@ -34,7 +34,7 @@ public void disable() {
     }
 
     /**
-     * Determines if the hub has tracing enabled or not
+     * Determines if the hub has debugging enabled or not
      *
      * @return - true if enabled, false otherwise
      */

marklogic-data-hub/src/main/java/com/marklogic/hub/FinishedListener.java

@@ -23,6 +23,7 @@
 import com.marklogic.client.io.DOMHandle;
 import com.marklogic.client.util.RequestParameters;
 import com.marklogic.hub.flow.Flow;
+import com.marklogic.hub.flow.FlowComplexity;
 import com.marklogic.hub.flow.FlowType;
 import com.marklogic.hub.flow.SimpleFlow;
 import com.marklogic.spring.batch.hub.FlowConfig;
@@ -51,21 +52,10 @@ public class FlowManager extends ResourceManager {
     private DatabaseClient client;
     private HubConfig hubConfig;
 
-    private DatabaseClient getClient() {
-        DatabaseClientFactory.Authentication authMethod = DatabaseClientFactory.Authentication
-            .valueOf(hubConfig.authMethod.toUpperCase());
-
-        return DatabaseClientFactory.newClient(
-            hubConfig.host,
-            hubConfig.stagingPort,
-            hubConfig.username,
-            hubConfig.password, authMethod);
-    }
-
     public FlowManager(HubConfig hubConfig) {
         super();
         this.hubConfig = hubConfig;
-        this.client = getClient();
+        this.client = hubConfig.newStagingClient();
         this.client.init(NAME, this);
     }
 

marklogic-data-hub/src/main/java/com/marklogic/hub/FlowManager.java

@@ -16,13 +16,18 @@
 package com.marklogic.hub;
 
 import com.fasterxml.jackson.annotation.JsonIgnore;
+import com.marklogic.client.DatabaseClient;
+import com.marklogic.client.DatabaseClientFactory;
 import com.marklogic.client.helper.LoggingObject;
 
 import java.io.File;
 import java.io.FileInputStream;
 import java.io.InputStream;
 import java.util.Properties;
 
+/**
+ * A class for passing around the Data Hub's Configuration
+ */
 public class HubConfig extends LoggingObject {
 
     public static final String DEFAULT_HOST = "localhost";
@@ -35,13 +40,16 @@ public class HubConfig extends LoggingObject {
     public static final String DEFAULT_TRIGGERS_DB_NAME = "data-hub-TRIGGERS";
     public static final String DEFAULT_SCHEMAS_DB_NAME = "data-hub-SCHEMAS";
 
+    public static final String DEFAULT_HUB_USER_ROLE = "data-hub-user";
+    public static final String DEFAULT_HUB_USER_NAME = "data-hub-user";
+
     public static final Integer DEFAULT_STAGING_PORT = 8010;
     public static final Integer DEFAULT_FINAL_PORT = 8011;
     public static final Integer DEFAULT_TRACE_PORT = 8012;
     public static final Integer DEFAULT_JOB_PORT = 8013;
 
     public static final String DEFAULT_APP_NAME = "data-hub";
-    public static final String DEFAULT_MODULES_PATH = "src/data-hub";
+    public static final String DEFAULT_PROJECT_DIR = ".";
 
     public static final String DEFAULT_AUTH_METHOD = "digest";
 
@@ -55,6 +63,10 @@ public class HubConfig extends LoggingObject {
     @JsonIgnore
     public String password;
 
+    public String adminUsername;
+    @JsonIgnore
+    public String adminPassword;
+
     public String host = DEFAULT_HOST;
 
     public String stagingDbName = DEFAULT_STAGING_NAME;
@@ -81,12 +93,16 @@ public class HubConfig extends LoggingObject {
     public String triggersDbName = DEFAULT_TRIGGERS_DB_NAME;
     public String schemasDbName = DEFAULT_SCHEMAS_DB_NAME;
 
+    public String hubUserName = DEFAULT_HUB_USER_NAME;
+    public String hubUserPassword = DEFAULT_HUB_USER_NAME;
+    public String hubUserRole = DEFAULT_HUB_USER_ROLE;
+
     public String authMethod = DEFAULT_AUTH_METHOD;
 
     public String projectDir;
 
     public HubConfig() {
-        this(DEFAULT_MODULES_PATH);
+        this(DEFAULT_PROJECT_DIR);
     }
 
     public HubConfig(String projectDir) {
@@ -116,7 +132,7 @@ private Properties getProperties(String environment) {
         return environmentProperties;
     }
 
-    public void loadConfigurationFromProperties(Properties environmentProperties) {
+    private void loadConfigurationFromProperties(Properties environmentProperties) {
         name = getEnvPropString(environmentProperties, "mlAppName", name);
 
         host = getEnvPropString(environmentProperties, "mlHost", host);
@@ -151,6 +167,58 @@ public void loadConfigurationFromProperties(Properties environmentProperties) {
         password = getEnvPropString(environmentProperties, "mlAdminPassword", password);
     }
 
+    /**
+     * Creates a new DatabaseClient for accessing the Staging database
+     * @return - a DatabaseClient
+     */
+    public DatabaseClient newStagingClient() {
+        return DatabaseClientFactory.newClient(
+            host,
+            stagingPort,
+            username,
+            password,
+            DatabaseClientFactory.Authentication.valueOf(authMethod.toUpperCase()));
+    }
+
+    /**
+     * Creates a new DatabaseClient for accessing the Final database
+     * @return - a DatabaseClient
+     */
+    public DatabaseClient newFinalClient() {
+        return DatabaseClientFactory.newClient(
+            host,
+            finalPort,
+            username,
+            password,
+            DatabaseClientFactory.Authentication.valueOf(authMethod.toUpperCase()));
+    }
+
+    /**
+     * Creates a new DatabaseClient for accessing the Job database
+     * @return - a DatabaseClient
+     */
+    public DatabaseClient newJobDbClient() {
+        return DatabaseClientFactory.newClient(
+            host,
+            jobPort,
+            username,
+            password,
+            DatabaseClientFactory.Authentication.valueOf(authMethod.toUpperCase()));
+    }
+
+    /**
+     * Creates a new DatabaseClient for accessing the Trace database
+     * @return - a DatabaseClient
+     */
+    public DatabaseClient newTraceDbClient() {
+        return DatabaseClientFactory.newClient(
+            host,
+            tracePort,
+            username,
+            password,
+            DatabaseClientFactory.Authentication.valueOf(authMethod.toUpperCase()));
+    }
+
     private String getEnvPropString(Properties environmentProperties, String key, String fallback) {
         String value = environmentProperties.getProperty(key);
         if (value == null) {
@@ -172,7 +240,7 @@ private int getEnvPropInteger(Properties environmentProperties, String key, int
     }
 
     private void loadConfigurationFromFile(Properties configProperties, String fileName) {
-        InputStream is = null;
+        InputStream is;
         try {
             File file = new File(this.projectDir, fileName);
             if(file.exists()) {
@@ -182,7 +250,7 @@ private void loadConfigurationFromFile(Properties configProperties, String fileN
             }
         }
         catch ( Exception e ) {
-            is = null;
+            e.printStackTrace();
         }
     }