adding trace server, db, forests

paxtonhare · paxtonhare · commit c55829d001ba · 2016-04-08T14:38:32.000-04:00
diff --git a/examples/gradle-advanced/build.gradle b/examples/gradle-advanced/build.gradle
@@ -42,6 +42,10 @@ ext {
         customTokens.put("%%FINAL_SERVER_PORT%%", mlFinalAppserverPort)
         customTokens.put("%%FINAL_DB_NAME%%", mlFinalDbName)
 
+        customTokens.put("%%TRACE_SERVER_NAME%%", mlTraceAppserverName)
+        customTokens.put("%%TRACE_SERVER_PORT%%", mlTraceAppserverPort)
+        customTokens.put("%%TRACE_DB_NAME%%", mlTraceDbName)
+
         customTokens.put("%%MODULES_DB_NAME%%", mlModulesDbName)
     }
 }
@@ -66,6 +70,11 @@ ext {
     mlAppDeployer.commands.add(finalDbCommand)
     mlDatabaseCommands.add(finalDbCommand)
 
+    // install the trace database
+    def traceDbCommand = new com.marklogic.appdeployer.command.databases.DeployDatabaseCommand("trace-database.json")
+    mlAppDeployer.commands.add(traceDbCommand)
+    mlDatabaseCommands.add(traceDbCommand)
+
     // install the modules database
     def modulesDbCommand = new com.marklogic.appdeployer.command.databases.DeployDatabaseCommand("modules-database.json")
     mlAppDeployer.commands.add(modulesDbCommand)
diff --git a/examples/gradle-advanced/gradle.properties b/examples/gradle-advanced/gradle.properties
@@ -9,6 +9,10 @@ mlFinalAppserverName=data-hub-FINAL
 mlFinalAppserverPort=8101
 mlFinalDbName=data-hub-FINAL
 
+mlTraceAppserverName=data-hub-TRACING
+mlTraceAppserverPort=8102
+mlTraceDbName=data-hub-TRACING
+
 mlModulesDbName=data-hub-MODULES
 mlTriggersDbName=data-hub-TRIGGERS
 mlSchemasDbName=data-hub-SCHEMAS
diff --git a/examples/gradle-advanced/src/main/ml-config/databases/trace-database.json b/examples/gradle-advanced/src/main/ml-config/databases/trace-database.json
@@ -0,0 +1,9 @@
+{
+  "database-name": "%%TRACE_DB_NAME%%",
+  "range-element-index": [],
+  "schema-database": "%%SCHEMAS_DATABASE%%",
+  "triggers-database": "%%TRIGGERS_DATABASE%%",
+  "triple-index": true,
+  "collection-lexicon": true,
+  "uri-lexicon": true
+}
diff --git a/examples/gradle-advanced/src/main/ml-config/servers/trace-server.json b/examples/gradle-advanced/src/main/ml-config/servers/trace-server.json
@@ -0,0 +1,14 @@
+{
+  "server-name": "%%TRACE_SERVER_NAME%%",
+  "server-type": "http",
+  "root": "/",
+  "group-name": "%%GROUP%%",
+  "port": %%TRACE_SERVER_PORT%%,
+  "modules-database": "%%MODULES_DB_NAME%%",
+  "content-database": "%%TRACE_DB_NAME%%",
+  "authentication": "digest",
+  "default-error-format": "json",
+  "error-handler": "/MarkLogic/rest-api/error-handler.xqy",
+  "url-rewriter": "/MarkLogic/rest-api/rewriter.xml",
+  "rewrite-resolves-globally": true
+}
diff --git a/examples/gradle-basic/gradle.properties b/examples/gradle-basic/gradle.properties
@@ -1,4 +1,5 @@
 mlHost=localhost
 mlStagingPort=8100
 mlFinalPort=8101
+mlTracePort=8102
 hubModulesPath=./plugins
diff --git a/marklogic-data-hub/gradle.properties b/marklogic-data-hub/gradle.properties
@@ -3,6 +3,7 @@ mlConfigDir=marklogic-data-hub/src/main/resources/ml-config
 publishUrl=file:../marklogic-data-hub/releases
 mlStagingRestPort=8010
 mlFinalRestPort=8011
+mlTraceRestPort=8012
 mlUsername=admin
 mlPassword=admin
 mlHost=localhost
diff --git a/marklogic-data-hub/src/main/java/com/marklogic/hub/DataHub.java b/marklogic-data-hub/src/main/java/com/marklogic/hub/DataHub.java
@@ -72,6 +72,7 @@ public class DataHub {
     static final private Logger LOGGER = LoggerFactory.getLogger(DataHub.class);
     static final public String STAGING_NAME = "data-hub-STAGING";
     static final public String FINAL_NAME = "data-hub-FINAL";
+    static final public String TRACING_NAME = "data-hub-TRACING";
     static final public String MODULES_DB_NAME = "data-hub-modules";
     private ManageConfig config;
     private ManageClient client;
@@ -80,28 +81,31 @@ public class DataHub {
     private String host;
     private int stagingRestPort;
     private int finalRestPort;
+    private int tracingRestPort;
     private String username;
     private String password;
 
     private File assetInstallTimeFile = new File("./assetInstallTime.properties");
 
     private final static int DEFAULT_STAGING_REST_PORT = 8010;
     private final static int DEFAULT_FINAL_REST_PORT = 8011;
+    private final static int DEFAULT_TRACING_REST_PORT = 8012;
 
     public DataHub(HubConfig config) {
-        this(config.getHost(), config.getStagingPort(), config.getFinalPort(), config.getAdminUsername(), config.getAdminPassword());
+        this(config.getHost(), config.getStagingPort(), config.getFinalPort(), config.getTracePort(), config.getAdminUsername(), config.getAdminPassword());
     }
 
     public DataHub(String host, String username, String password) {
-        this(host, DEFAULT_STAGING_REST_PORT, DEFAULT_FINAL_REST_PORT, username, password);
+        this(host, DEFAULT_STAGING_REST_PORT, DEFAULT_FINAL_REST_PORT, DEFAULT_TRACING_REST_PORT, username, password);
     }
 
-    public DataHub(String host, int stagingRestPort, int finalRestPort, String username, String password) {
+    public DataHub(String host, int stagingRestPort, int finalRestPort, int tracingRestPort, String username, String password) {
         config = new ManageConfig(host, 8002, username, password);
         client = new ManageClient(config);
         this.host = host;
         this.stagingRestPort = stagingRestPort;
         this.finalRestPort = finalRestPort;
+        this.tracingRestPort = tracingRestPort;
         this.username = username;
         this.password = password;
     }
@@ -214,6 +218,7 @@ private DatabaseClient getDatabaseClient(int port) {
                 config.getRestAuthentication(), config.getRestSslContext(), config.getRestSslHostnameVerifier());
         return client;
     }
+
     /**
      * Installs User Provided modules into the Data Hub
      *
@@ -298,6 +303,9 @@ private List<Command> getCommands(AppConfig config) {
         finalDb.setForestsPerHost(FORESTS_PER_HOST);
         dbCommands.add(finalDb);
 
+        DeployHubDatabaseCommand tracingDb = new DeployHubDatabaseCommand(TRACING_NAME);
+        dbCommands.add(tracingDb);
+
         dbCommands.add(new DeployModulesDatabaseCommand(MODULES_DB_NAME));
         dbCommands.add(new DeployTriggersDatabaseCommand());
         dbCommands.add(new DeploySchemasDatabaseCommand());
@@ -306,6 +314,7 @@ private List<Command> getCommands(AppConfig config) {
         // App Servers
         commands.add(new DeployRestApiCommand(STAGING_NAME, stagingRestPort));
         commands.add(new DeployRestApiCommand(FINAL_NAME, finalRestPort));
+        commands.add(new DeployRestApiCommand(TRACING_NAME, tracingRestPort));
 
         // Modules
         commands.add(new LoadModulesCommand());
diff --git a/marklogic-data-hub/src/main/java/com/marklogic/hub/DefaultHubConfigFactory.java b/marklogic-data-hub/src/main/java/com/marklogic/hub/DefaultHubConfigFactory.java
@@ -59,6 +59,12 @@ public HubConfig newHubConfig() {
             c.setFinalPort(Integer.parseInt(prop));
         }
 
+        prop = getProperty("mlTracePort");
+        if (prop != null) {
+            logger.info("Trace App REST port: " + prop);
+            c.setTracePort(Integer.parseInt(prop));
+        }
+
         prop = getProperty("mlAdminUsername");
         if (prop != null) {
             logger.info("REST admin username: " + prop);
diff --git a/marklogic-data-hub/src/main/java/com/marklogic/hub/HubConfig.java b/marklogic-data-hub/src/main/java/com/marklogic/hub/HubConfig.java
@@ -22,6 +22,7 @@ public class HubConfig {
     public static final String DEFAULT_HOST = "localhost";
     public static final Integer DEFAULT_STAGING_PORT = 8010;
     public static final Integer DEFAULT_FINAL_PORT = 8011;
+    public static final Integer DEFAULT_TRACE_PORT = 8012;
     public static final String DEFAULT_APP_NAME = "my-data-hub";
     public final static String DEFAULT_MODULES_PATH = "src/data-hub";
     public static final String DEFAULT_AUTH_METHOD = "digest";
@@ -32,6 +33,7 @@ public class HubConfig {
     private String host = DEFAULT_HOST;
     private Integer stagingPort = DEFAULT_STAGING_PORT;
     private Integer finalPort = DEFAULT_FINAL_PORT;
+    private Integer tracePort = DEFAULT_TRACE_PORT;
     private String authMethod = DEFAULT_AUTH_METHOD;
 
     private String modulesPath;
@@ -111,6 +113,17 @@ public void setFinalPort(Integer port) {
         this.finalPort = port;
     }
 
+    /**
+     * @return the port of the Final REST API server used for loading modules
+     */
+    public Integer getTracePort() {
+        return tracePort;
+    }
+
+    public void setTracePort(Integer port) {
+        this.tracePort = port;
+    }
+
     public String getModulesPath() {
         return this.modulesPath;
     }
diff --git a/marklogic-data-hub/src/main/resources/ml-modules/root/com.marklogic.hub/lib/flow-lib.xqy b/marklogic-data-hub/src/main/resources/ml-modules/root/com.marklogic.hub/lib/flow-lib.xqy
@@ -279,7 +279,6 @@ declare %private function flow:get-flow(
     return
       fn:doc($uri)/hub:flow
   })
-  where $flow
   return
     <flow xmlns="http://marklogic.com/data-hub">
       <name>{$flow-name}</name>
diff --git a/marklogic-data-hub/src/test/java/com/marklogic/hub/DataHubTest.java b/marklogic-data-hub/src/test/java/com/marklogic/hub/DataHubTest.java
@@ -44,7 +44,7 @@ public static void setup() {
 
     @Test
     public void testValidateServer() throws ServerValidationException {
-        DataHub dh = new DataHub(host, stagingPort, finalPort, user, password);
+        DataHub dh = new DataHub(host, stagingPort, finalPort, tracePort, user, password);
         dh.validateServer();
     }
 
@@ -58,7 +58,7 @@ public void testValidateInvalidServer() throws ServerValidationException {
 
     @Test
     public void testInstallUserModules() throws IOException, ParserConfigurationException, SAXException {
-        DataHub dh = new DataHub(host, stagingPort, finalPort, user, password);
+        DataHub dh = new DataHub(host, stagingPort, finalPort, tracePort, user, password);
         if (dh.isInstalled()) {
             dh.uninstall();
         }
diff --git a/marklogic-data-hub/src/test/java/com/marklogic/hub/EndToEndTestSjsXml.java b/marklogic-data-hub/src/test/java/com/marklogic/hub/EndToEndTestSjsXml.java
@@ -42,7 +42,7 @@ public static void setup() throws IOException {
         Scaffolding.createFlow(ENTITY, "testconformance", FlowType.CONFORMANCE,
                 PluginFormat.JAVASCRIPT, Format.XML, pluginsDir);
 
-        new DataHub(host, stagingPort, finalPort, user, password).installUserModules(pluginsDir.toString());
+        new DataHub(host, stagingPort, finalPort, tracePort, user, password).installUserModules(pluginsDir.toString());
     }
 
     @AfterClass
diff --git a/marklogic-data-hub/src/test/java/com/marklogic/hub/EndToEndTestXqyXml.java b/marklogic-data-hub/src/test/java/com/marklogic/hub/EndToEndTestXqyXml.java
@@ -42,7 +42,7 @@ public static void setup() throws IOException {
         Scaffolding.createFlow(ENTITY, "testconformance", FlowType.CONFORMANCE,
                 PluginFormat.XQUERY, Format.XML, pluginsDir);
 
-        new DataHub(host, stagingPort, finalPort, user, password).installUserModules(pluginsDir.toString());
+        new DataHub(host, stagingPort, finalPort, tracePort, user, password).installUserModules(pluginsDir.toString());
     }
 
     @AfterClass
diff --git a/marklogic-data-hub/src/test/java/com/marklogic/hub/FlowManagerTest.java b/marklogic-data-hub/src/test/java/com/marklogic/hub/FlowManagerTest.java
@@ -107,6 +107,7 @@ public void testSimpleFlowFromXml() throws IOException, ParserConfigurationExcep
     @Test
     public void testEmptyFlow() throws IOException {
         runInModules("xdmp:directory-create(\"/entities/test/conformance/empty-flow/\")");
+        installModule("/entities/test/conformance/empty-flow/empty-flow.xml", "flow-manager-test/my-test-flow1/my-test-flow1.xml");
         installModule("/entities/test/conformance/empty-flow/collector/collector.xqy", "flow-manager-test/my-test-flow1/collector/collector.xqy");
 
         FlowManager fm = new FlowManager(stagingClient);
diff --git a/marklogic-data-hub/src/test/java/com/marklogic/hub/HubTestBase.java b/marklogic-data-hub/src/test/java/com/marklogic/hub/HubTestBase.java
@@ -52,13 +52,16 @@ public class HubTestBase {
     public static String host;
     public static int stagingPort;
     public static int finalPort;
+    public static int tracePort;
     public static String user;
     public static String password;
     public static Authentication authMethod;
     public static DatabaseClient stagingClient = null;
     public static DatabaseClient stagingModulesClient = null;
     public static DatabaseClient finalClient = null;
     public static DatabaseClient finalModulesClient = null;
+    public static DatabaseClient traceClient = null;
+    public static DatabaseClient traceModulesClient = null;
     private static Properties properties = new Properties();
     private static boolean initialized = false;
     public static XMLDocumentManager stagingDocMgr = getStagingMgr();
@@ -113,6 +116,7 @@ private static void init() {
         host = properties.getProperty("mlHost");
         stagingPort = Integer.parseInt(properties.getProperty("mlStagingRestPort"));
         finalPort = Integer.parseInt(properties.getProperty("mlFinalRestPort"));
+        tracePort = Integer.parseInt(properties.getProperty("mlTraceRestPort"));
         user = properties.getProperty("mlUsername");
         password = properties.getProperty("mlPassword");
         authMethod = Authentication.valueOf(properties.getProperty("auth").toUpperCase());
@@ -121,6 +125,8 @@ private static void init() {
         stagingModulesClient  = DatabaseClientFactory.newClient(host, stagingPort, DataHub.MODULES_DB_NAME, user, password, authMethod);
         finalClient = DatabaseClientFactory.newClient(host, finalPort, user, password, authMethod);
         finalModulesClient  = DatabaseClientFactory.newClient(host, stagingPort, DataHub.MODULES_DB_NAME, user, password, authMethod);
+        traceClient = DatabaseClientFactory.newClient(host, tracePort, user, password, authMethod);
+        traceModulesClient  = DatabaseClientFactory.newClient(host, stagingPort, DataHub.MODULES_DB_NAME, user, password, authMethod);
     }
 
     public HubTestBase() {
@@ -142,11 +148,11 @@ protected static HubConfig getHubConfig(String pluginDir) {
     }
 
     protected static void installHub() throws IOException {
-        new DataHub(host, stagingPort, finalPort, user, password).install();
+        new DataHub(host, stagingPort, finalPort, tracePort, user, password).install();
     }
 
     protected static void uninstallHub() throws IOException {
-        new DataHub(host, stagingPort, finalPort, user, password).uninstall();
+        new DataHub(host, stagingPort, finalPort, tracePort, user, password).uninstall();
     }
     protected static String getResource(String resourceName) throws IOException {
         try {
diff --git a/quick-start/src/main/java/com/marklogic/hub/config/EnvironmentConfiguration.java b/quick-start/src/main/java/com/marklogic/hub/config/EnvironmentConfiguration.java
@@ -36,6 +36,7 @@ public class EnvironmentConfiguration {
 	private static final String ML_PASSWORD = "mlPassword";
 	private static final String ML_STAGING_REST_PORT = "mlStagingRestPort";
 	private static final String ML_FINAL_REST_PORT = "mlFinalRestPort";
+	private static final String ML_TRACE_REST_PORT = "mlTraceRestPort";
 	private static final String ML_AUTH = "mlAuth";
 	private static final String USER_PLUGIN_DIR = "userPluginDir";
 	private static final String ASSET_INSTALL_TIME_FILE = "assetInstallTimeFile";
@@ -115,6 +116,19 @@ public String getMLFinalRestPort() {
         return this.environment.getProperty(ML_FINAL_REST_PORT + DEFAULT_SUFFIX);
     }
 
+    public String getMLTraceRestPort() {
+        String value = this.environmentProperties.getProperty(ML_TRACE_REST_PORT);
+        if (value != null) {
+            return value;
+        }
+        value = this.environment.getProperty(ML_TRACE_REST_PORT);
+        if (value != null) {
+            this.environmentProperties.setProperty(ML_TRACE_REST_PORT, value);
+            return value;
+        }
+        return this.environment.getProperty(ML_TRACE_REST_PORT + DEFAULT_SUFFIX);
+    }
+
 	public String getMLAuth() {
 		String value = this.environmentProperties.getProperty(ML_AUTH);
 		if (value != null) {
@@ -161,11 +175,15 @@ public void setMLHost(String mlHost) {
 	public void setMLStagingRestPort(String mlStagingRestPort) {
 		this.environmentProperties.setProperty(ML_STAGING_REST_PORT, mlStagingRestPort);
 	}
-	
+
 	public void setMLFinalRestPort(String mlFinalRestPort) {
         this.environmentProperties.setProperty(ML_FINAL_REST_PORT, mlFinalRestPort);
     }
 
+	public void setMlTraceRestPort(String mlTraceRestPort) {
+	    this.environmentProperties.setProperty(ML_TRACE_REST_PORT, mlTraceRestPort);
+	}
+
 	public void setMLUsername(String mlUsername) {
 		this.environmentProperties.setProperty(ML_USERNAME, mlUsername);
 	}
@@ -186,7 +204,7 @@ public void loadConfigurationFromFiles() {
 	    loadConfigurationFromFile(environmentProperties, ENVIRONMENT_PROPERTIES_FILENAME);
 	    loadConfigurationFromFile(environmentProperties, FLOW_PROPERTIES_FILENAME);
 	}
-	
+
 	public void loadConfigurationFromFile(Properties configProperties, String fileName) {
         InputStream is = null;
         try {
@@ -203,7 +221,7 @@ public void loadConfigurationFromFile(Properties configProperties, String fileNa
 	public void saveConfigurationToFile() {
 		saveConfigurationToFile(environmentProperties, ENVIRONMENT_PROPERTIES_FILENAME);
 	}
-	
+
 	private void saveConfigurationToFile(Properties configProperties, String fileName) {
         OutputStream out = null;
         try {
@@ -223,12 +241,12 @@ private void saveConfigurationToFile(Properties configProperties, String fileNam
             }
         }
     }
-	
+
 	public void saveOrUpdateFlowInputPath(String entityName, String flowName, String inputPath) {
 	    this.flowProperties.setProperty(entityName + "-" + flowName, inputPath);
 	    saveConfigurationToFile(flowProperties, FLOW_PROPERTIES_FILENAME);
     }
-	
+
 	public String getFlowInputPath(String entityName, String flowName) {
 	    return this.flowProperties.getProperty(entityName + "-" + flowName);
     }
diff --git a/quick-start/src/main/java/com/marklogic/hub/factory/FlowModelFactory.java b/quick-start/src/main/java/com/marklogic/hub/factory/FlowModelFactory.java
@@ -2,7 +2,6 @@
 
 import java.io.File;
 import java.io.IOException;
-import java.nio.file.Path;
 import java.util.LinkedHashMap;
 import java.util.List;
 import java.util.Map;
diff --git a/quick-start/src/main/java/com/marklogic/hub/service/DataHubService.java b/quick-start/src/main/java/com/marklogic/hub/service/DataHubService.java
@@ -55,6 +55,7 @@ private DataHub getDataHub() throws DataHubException {
             DataHub dataHub = new DataHub(environmentConfiguration.getMLHost(),
                     Integer.parseInt(environmentConfiguration.getMLStagingRestPort()),
                     Integer.parseInt(environmentConfiguration.getMLFinalRestPort()),
+                    Integer.parseInt(environmentConfiguration.getMLTraceRestPort()),
                     environmentConfiguration.getMLUsername(),
                     environmentConfiguration.getMLPassword());
             dataHub.setAssetInstallTimeFile(new File(environmentConfiguration.getAssetInstallTimeFilePath()));
diff --git a/quick-start/src/main/java/com/marklogic/hub/service/EntityManagerService.java b/quick-start/src/main/java/com/marklogic/hub/service/EntityManagerService.java
diff --git a/quick-start/src/main/java/com/marklogic/hub/web/controller/api/DataHubServerApiController.java b/quick-start/src/main/java/com/marklogic/hub/web/controller/api/DataHubServerApiController.java
diff --git a/quick-start/src/main/java/com/marklogic/hub/web/form/LoginForm.java b/quick-start/src/main/java/com/marklogic/hub/web/form/LoginForm.java
diff --git a/quick-start/src/main/resources/application.properties b/quick-start/src/main/resources/application.properties
diff --git a/quick-start/src/main/resources/static/index.html b/quick-start/src/main/resources/static/index.html
diff --git a/quick-start/src/main/resources/static/login/login.html b/quick-start/src/main/resources/static/login/login.html
