merging fixes from 2.x into 1.x

Author: paxtonhare

.travis/Dockerfile-ml

@@ -16,7 +16,7 @@ RUN yum -y install /tmp/MarkLogic.rpm 2>&1 > /dev/null
 
 # Expose MarkLogic Server ports - add additional ones for your REST, etc
 # endpoints
-EXPOSE 7997-8002
+EXPOSE 7997-8020
 
 # init
 COPY .travis/startml.sh /tmp/startml.sh

.travis/download.sh

@@ -4,4 +4,4 @@ echo "Logging in to Docker registry"
 docker login -u ${DOCKER_USER} -p ${DOCKER_PASSWORD}
 
 echo "Getting Docker Image"
-docker pull paxtonhare/marklogic-datahub-1x
+docker pull marklogiccommunity/marklogic-datahub-1x

docker-compose.yml

@@ -1,7 +1,7 @@
 version: '2'
 services:
   marklogic:
-    image: paxtonhare/marklogic-datahub-1x
+    image: marklogiccommunity/marklogic-datahub-1x
     ports:
       - "8000-8020"
     volumes:

examples/spring-batch/build.gradle

@@ -21,7 +21,7 @@ repositories {
 dependencies {
     compile 'com.marklogic:marklogic-data-hub:1.1.2'
     compile 'com.marklogic:marklogic-spring-batch-core:0.6.0'
-    compile 'com.marklogic:ml-javaclient-util:2.13.0'
+    compile 'com.marklogic:ml-javaclient-util:2.14.0'
     testCompile 'com.marklogic:marklogic-spring-batch-test:0.6.0'
 }
 

marklogic-data-hub/build.gradle

@@ -19,14 +19,10 @@ sourceCompatibility = 1.8
 targetCompatibility = 1.8
 
 dependencies {
-    compile 'org.springframework.batch:spring-batch-core:3.0.6.RELEASE'
-    compile 'org.springframework:spring-jdbc:4.2.6.RELEASE'
-    compile 'com.marklogic:java-client-api:3.0.7'
-    compile 'com.marklogic:ml-javaclient-util:2.13.0'
+    compile 'com.marklogic:marklogic-client-api:4.0.1'
+    compile 'com.marklogic:ml-javaclient-util:4.0.alpha4'
     compile 'com.marklogic:ml-app-deployer:2.6.0'
-    compile 'com.marklogic:marklogic-spring-batch-core:0.7.0'
     compile 'commons-io:commons-io:2.4'
-    testCompile 'org.springframework.batch:spring-batch-test:3.0.6.RELEASE'
     testCompile 'junit:junit:4.12'
     testCompile 'xmlunit:xmlunit:1.3'
     testCompile 'org.skyscreamer:jsonassert:1.3.0'

marklogic-data-hub/src/main/java/com/marklogic/client/datamovement/impl/DataMovementServices.java

@@ -0,0 +1,116 @@
+/*
+ * Copyright 2015-2017 MarkLogic Corporation
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *    http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package com.marklogic.client.datamovement.impl;
+
+import java.util.ArrayList;
+import java.util.UUID;
+
+import com.fasterxml.jackson.databind.JsonNode;
+import com.marklogic.client.DatabaseClient;
+import com.marklogic.client.impl.DatabaseClientImpl;
+import com.marklogic.client.io.JacksonHandle;
+import com.marklogic.client.datamovement.Batcher;
+import com.marklogic.client.datamovement.JobTicket;
+import com.marklogic.client.datamovement.JobTicket.JobType;
+import com.marklogic.client.datamovement.QueryBatcher;
+import com.marklogic.client.datamovement.WriteBatcher;
+import com.marklogic.client.datamovement.JobReport;
+import com.marklogic.client.datamovement.impl.ForestConfigurationImpl;
+import java.util.List;
+
+public class DataMovementServices {
+    private DatabaseClient client;
+
+    public DatabaseClient getClient() {
+        return client;
+    }
+
+    public DataMovementServices setClient(DatabaseClient client) {
+        this.client = client;
+        return this;
+    }
+
+    public ForestConfigurationImpl readForestConfig() {
+        List<ForestImpl> forests = new ArrayList<>();
+        JsonNode results = ((DatabaseClientImpl) client).getServices()
+            .getResource(null, "resources/forest-info", null, null, new JacksonHandle())
+            .get();
+        for ( JsonNode forestNode : results ) {
+            String id = forestNode.get("id").asText();
+            String name = forestNode.get("name").asText();
+            String database = forestNode.get("database").asText();
+            String host = forestNode.get("host").asText();
+            String openReplicaHost = null;
+            if ( forestNode.get("openReplicaHost") != null ) openReplicaHost = forestNode.get("openReplicaHost").asText();
+            String alternateHost = null;
+            if ( forestNode.get("alternateHost") != null ) alternateHost = forestNode.get("alternateHost").asText();
+            boolean isUpdateable = "all".equals(forestNode.get("updatesAllowed").asText());
+            boolean isDeleteOnly = false; // TODO: get this for real after we start using a REST endpoint
+            forests.add(new ForestImpl(host, alternateHost, openReplicaHost, database, name, id, isUpdateable, isDeleteOnly));
+        }
+
+        return new ForestConfigurationImpl(forests.toArray(new ForestImpl[forests.size()]));
+    }
+
+    public JobTicket startJob(WriteBatcher batcher) {
+        JobTicket jobTicket = new JobTicketImpl(generateJobId(), JobTicket.JobType.WRITE_BATCHER)
+            .withWriteBatcher((WriteBatcherImpl) batcher);
+        ((WriteBatcherImpl) batcher).start(jobTicket);
+        return jobTicket;
+    }
+
+    public JobTicket startJob(QueryBatcher batcher) {
+        JobTicket jobTicket = new JobTicketImpl(generateJobId(), JobTicket.JobType.QUERY_BATCHER)
+            .withQueryBatcher((QueryBatcherImpl) batcher);
+        ((QueryBatcherImpl) batcher).start(jobTicket);
+        return jobTicket;
+    }
+
+    public JobReport getJobReport(JobTicket ticket) {
+        if ( ticket instanceof JobTicketImpl ) {
+            JobTicketImpl ticketImpl = (JobTicketImpl) ticket;
+            if ( ticketImpl.getJobType() == JobType.WRITE_BATCHER ) {
+                return new JobReportImpl(ticketImpl.getWriteBatcher());
+            } else if ( ticketImpl.getJobType() == JobType.QUERY_BATCHER ) {
+                return new JobReportImpl(ticketImpl.getQueryBatcher());
+            }
+        }
+        return null;
+    }
+
+    public void stopJob(JobTicket ticket) {
+        if ( ticket instanceof JobTicketImpl ) {
+            JobTicketImpl ticketImpl = (JobTicketImpl) ticket;
+            if ( ticketImpl.getJobType() == JobType.WRITE_BATCHER ) {
+                ticketImpl.getWriteBatcher().stop();
+            } else if ( ticketImpl.getJobType() == JobType.QUERY_BATCHER ) {
+                ticketImpl.getQueryBatcher().stop();
+            }
+        }
+    }
+
+    public void stopJob(Batcher batcher) {
+        if ( batcher instanceof QueryBatcherImpl ) {
+            ((QueryBatcherImpl) batcher).stop();
+        } else if ( batcher instanceof WriteBatcherImpl ) {
+            ((WriteBatcherImpl) batcher).stop();
+        }
+    }
+
+    private String generateJobId() {
+        return UUID.randomUUID().toString();
+    }
+}

marklogic-data-hub/src/main/java/com/marklogic/hub/DataHub.java

@@ -21,6 +21,8 @@
 import com.marklogic.appdeployer.command.Command;
 import com.marklogic.appdeployer.command.CommandMapBuilder;
 import com.marklogic.appdeployer.command.appservers.DeployOtherServersCommand;
+import com.marklogic.appdeployer.command.security.DeployRolesCommand;
+import com.marklogic.appdeployer.command.security.DeployUsersCommand;
 import com.marklogic.appdeployer.impl.SimpleAppDeployer;
 import com.marklogic.client.DatabaseClient;
 import com.marklogic.client.FailedRequestException;
@@ -305,6 +307,10 @@ public List<Command> getCommandList() {
     public Map<String, List<Command>> getCommands() {
         Map<String, List<Command>> commandMap = new CommandMapBuilder().buildCommandMap();
 
+        List<Command> securityCommands = commandMap.get("mlSecurityCommands");
+        securityCommands.set(0, new DeployHubRolesCommand(hubConfig));
+        securityCommands.set(1, new DeployHubUsersCommand(hubConfig));
+
         List<Command> dbCommands = new ArrayList<>();
         dbCommands.add(new DeployHubDatabasesCommand(hubConfig));
         dbCommands.add(new DeployHubOtherDatabasesCommand(hubConfig));
@@ -355,6 +361,15 @@ public void install(HubDeployStatusListener listener) {
         deployer.deploy(config);
     }
 
+    public void updateIndexes() {
+        AppConfig config = hubConfig.getAppConfig();
+        HubAppDeployer deployer = new HubAppDeployer(getManageClient(), getAdminManager(), null);
+        List<Command> commands = new ArrayList<>();
+        commands.add(new DeployHubDatabasesCommand(hubConfig));
+        deployer.setCommands(commands);
+        deployer.deploy(config);
+    }
+
     /**
      * Uninstalls the data hub configuration and server-side modules from MarkLogic
      */

marklogic-data-hub/src/main/java/com/marklogic/hub/FlowManager.java

@@ -16,21 +16,22 @@
 package com.marklogic.hub;
 
 import com.marklogic.client.DatabaseClient;
+import com.marklogic.client.datamovement.DataMovementManager;
 import com.marklogic.client.extensions.ResourceManager;
 import com.marklogic.client.extensions.ResourceServices.ServiceResult;
 import com.marklogic.client.extensions.ResourceServices.ServiceResultIterator;
 import com.marklogic.client.io.DOMHandle;
 import com.marklogic.client.util.RequestParameters;
 import com.marklogic.hub.flow.*;
 import com.marklogic.hub.flow.impl.FlowRunnerImpl;
+import com.marklogic.hub.job.JobManager;
 import org.w3c.dom.Document;
 import org.w3c.dom.Element;
 import org.w3c.dom.Node;
 import org.w3c.dom.NodeList;
 
 import java.util.ArrayList;
 import java.util.List;
-import java.util.Map;
 
 public class FlowManager extends ResourceManager {
     private static final String HUB_NS = "http://marklogic.com/data-hub";
@@ -40,14 +41,18 @@ public class FlowManager extends ResourceManager {
     private DatabaseClient finalClient;
     private DatabaseClient jobClient;
     private HubConfig hubConfig;
-    private Map<String, Object> userOptions;
+    private JobManager jobManager;
+
+    private DataMovementManager dataMovementManager;
 
     public FlowManager(HubConfig hubConfig) {
         super();
         this.hubConfig = hubConfig;
         this.stagingClient = hubConfig.newStagingClient();
         this.finalClient = hubConfig.newFinalClient();
         this.jobClient = hubConfig.newJobDbClient();
+        this.jobManager = new JobManager(this.jobClient);
+        this.dataMovementManager = this.stagingClient.newDataMovementManager();
         this.stagingClient.init(NAME, this);
     }
 

marklogic-data-hub/src/main/java/com/marklogic/hub/HubConfig.java

@@ -43,6 +43,9 @@
  */
 public class HubConfig {
 
+    public static final String USER_MODULES_DEPLOY_TIMESTAMPS_PROPERTIES = "user-modules-deploy-timestamps.properties";
+    public static final String USER_CONTENT_DEPLOY_TIMESTAMPS_PROPERTIES = "user-content-deploy-timestamps.properties";
+
     public static final String OLD_HUB_CONFIG_DIR = "marklogic-config";
     public static final String HUB_CONFIG_DIR = "hub-internal-config";
     public static final String USER_CONFIG_DIR = "user-config";
@@ -58,11 +61,18 @@ public class HubConfig {
     public static final String DEFAULT_TRIGGERS_DB_NAME = "data-hub-TRIGGERS";
     public static final String DEFAULT_SCHEMAS_DB_NAME = "data-hub-SCHEMAS";
 
+    public static final String DEFAULT_ROLE_NAME = "data-hub-ROLE";
+    public static final String DEFAULT_USER_NAME = "data-hub-user";
+
     public static final Integer DEFAULT_STAGING_PORT = 8010;
     public static final Integer DEFAULT_FINAL_PORT = 8011;
     public static final Integer DEFAULT_TRACE_PORT = 8012;
     public static final Integer DEFAULT_JOB_PORT = 8013;
 
+    public static final Integer DEFAULT_APP_SERVICES_PORT = 8000;
+    public static final Integer DEFAULT_ADMIN_PORT = 8001;
+    public static final Integer DEFAULT_MANAGE_PORT = 8002;
+
     public static final String DEFAULT_PROJECT_DIR = ".";
 
     public static final String DEFAULT_AUTH_METHOD = "digest";
@@ -113,6 +123,13 @@ public class HubConfig {
     public String triggersDbName = DEFAULT_TRIGGERS_DB_NAME;
     public String schemasDbName = DEFAULT_SCHEMAS_DB_NAME;
 
+    public Integer appServicesPort = DEFAULT_APP_SERVICES_PORT;
+    public Integer adminPort = DEFAULT_ADMIN_PORT;
+    public Integer managePort = DEFAULT_MANAGE_PORT;
+
+    public String hubRoleName = DEFAULT_ROLE_NAME;
+    public String hubUserName = DEFAULT_USER_NAME;
+
     public String projectDir;
 
     private Properties environmentProperties;
@@ -149,12 +166,24 @@ private Properties getProperties(String environment) {
         return environmentProperties;
     }
 
+    public File getModulesDeployTimestampFile() {
+        return Paths.get(projectDir, ".tmp", USER_MODULES_DEPLOY_TIMESTAMPS_PROPERTIES).toFile();
+    }
+
+    public File getContentDeployTimestampFile() {
+        return Paths.get(projectDir, ".tmp", USER_CONTENT_DEPLOY_TIMESTAMPS_PROPERTIES).toFile();
+    }
+
     public void loadConfigurationFromProperties(Properties environmentProperties) {
         this.environmentProperties = environmentProperties;
 
         if (this.environmentProperties != null) {
             host = getEnvPropString(environmentProperties, "mlHost", host);
 
+            appServicesPort = getEnvPropInteger(environmentProperties, "mlAppServicesPort", appServicesPort);
+            adminPort = getEnvPropInteger(environmentProperties, "mlAdminPort", adminPort);
+            managePort = getEnvPropInteger(environmentProperties, "mlManagePort", managePort);
+
             stagingDbName = getEnvPropString(environmentProperties, "mlStagingDbName", stagingDbName);
             stagingHttpName = getEnvPropString(environmentProperties, "mlStagingAppserverName", stagingHttpName);
             stagingForestsPerHost = getEnvPropInteger(environmentProperties, "mlStagingForestsPerHost", stagingForestsPerHost);
@@ -183,6 +212,9 @@ public void loadConfigurationFromProperties(Properties environmentProperties) {
             triggersDbName = getEnvPropString(environmentProperties, "mlTriggersDbName", triggersDbName);
             schemasDbName = getEnvPropString(environmentProperties, "mlSchemasDbName", schemasDbName);
 
+            hubRoleName = getEnvPropString(environmentProperties, "mlHubUserRole", hubRoleName);
+            hubUserName = getEnvPropString(environmentProperties, "mlHubUserName", hubUserName);
+
             adminUsername = getEnvPropString(environmentProperties, "mlAdminUsername", adminUsername);
             adminPassword = getEnvPropString(environmentProperties, "mlAdminPassword", adminPassword);
 
@@ -205,13 +237,14 @@ public void loadConfigurationFromProperties(Properties environmentProperties) {
     }
 
     public ManageClient newManageClient() {
-        ManageConfig config = new ManageConfig(host, 8002, username, password);
+        ManageConfig config = new ManageConfig(host, managePort, getManageUsername(), getManagePassword());
         return new ManageClient(config);
     }
 
     public AdminManager newAdminManager() {
         AdminConfig adminConfig = new AdminConfig();
         adminConfig.setHost(host);
+        adminConfig.setPort(adminPort);
         adminConfig.setUsername(getAdminUsername());
         adminConfig.setPassword(getAdminPassword());
         return new AdminManager(adminConfig);
@@ -348,6 +381,10 @@ public Path getHubSecurityDir() {
         return Paths.get(this.projectDir, HUB_CONFIG_DIR, "security");
     }
 
+    public Path getUserSecurityDir() {
+        return Paths.get(this.projectDir, USER_CONFIG_DIR, "security");
+    }
+
     public Path getUserConfigDir() {
         return Paths.get(this.projectDir, USER_CONFIG_DIR);
     }
@@ -487,6 +524,7 @@ public AppConfig getAppConfig() {
     public void updateAppConfig(AppConfig config) {
         config.setHost(host);
         config.setRestPort(stagingPort);
+        config.setAppServicesPort(appServicesPort);
         config.setRestAdminUsername(getRestAdminUsername());
         config.setRestAdminPassword(getRestAdminPassword());
         config.setModulesDatabaseName(modulesDbName);
@@ -508,6 +546,8 @@ public void updateAppConfig(AppConfig config) {
         ConfigDir configDir = new ConfigDir(getUserConfigDir().toFile());
         config.setConfigDir(configDir);
 
+        config.setSchemasPath(getUserConfigDir().resolve("schemas").toString());
+
         Map<String, String> customTokens = config.getCustomTokens();
 
         customTokens.put("%%mlStagingAppserverName%%", stagingHttpName);
@@ -534,6 +574,9 @@ public void updateAppConfig(AppConfig config) {
         customTokens.put("%%mlTriggersDbName%%", triggersDbName);
         customTokens.put("%%mlSchemasDbName%%", schemasDbName);
 
+        customTokens.put("%%mlHubUserRole%%", hubRoleName);
+        customTokens.put("%%mlHubUserName%%", hubUserName);
+
         if (environmentProperties != null) {
             Enumeration keyEnum = environmentProperties.propertyNames();
             while (keyEnum.hasMoreElements()) {

marklogic-data-hub/src/main/java/com/marklogic/hub/HubProject.java

@@ -62,6 +62,9 @@ public HubProject(HubConfig config) {
         customTokens.put("%%mlModulesDbName%%", hubConfig.modulesDbName);
         customTokens.put("%%mlTriggersDbName%%", hubConfig.triggersDbName);
         customTokens.put("%%mlSchemasDbName%%", hubConfig.schemasDbName);
+
+        customTokens.put("%%mlHubUserRole%%", hubConfig.hubRoleName);
+        customTokens.put("%%mlHubUserName%%", hubConfig.hubUserName);
     }
 
     /**