Merge branch 'dev' of https://github.com/rjrudin/ml-app-deployer into feature/issue-28-add-temporal-config

# Conflicts:
#	src/main/java/com/marklogic/appdeployer/command/SortOrderConstants.java

Author: Dermot Smyth

build.gradle

@@ -15,10 +15,10 @@ repositories {
 }
 
 dependencies {
-  compile 'com.marklogic:ml-javaclient-util:2.11.0'
+  compile 'com.marklogic:ml-javaclient-util:2.13.0'
   compile 'jaxen:jaxen:1.1.6'
   compile 'org.apache.httpcomponents:httpclient:4.3.5'
-  compile 'org.springframework:spring-web:4.1.5.RELEASE'
+  compile 'org.springframework:spring-web:4.3.5.RELEASE'
   testCompile ('com.marklogic:ml-junit:2.6.0') {
     exclude module: "ml-javaclient-util"
   }

gradle.properties

@@ -1,3 +1,3 @@
 group=com.marklogic
 javadocsDir=../gh-pages-marklogic-java/javadocs
-version=2.5.0
+version=2.6.0

src/main/java/com/marklogic/appdeployer/AppConfig.java

@@ -133,6 +133,8 @@ public class AppConfig {
 	private boolean generateExtractionTemplate = true;
 	private boolean generateSearchOptions = true;
 
+	private String[] resourceFilenamesToIgnore;
+
 	public AppConfig() {
         this(DEFAULT_MODULES_PATH, DEFAULT_SCHEMAS_PATH);
     }
@@ -696,4 +698,12 @@ public String getModuleTimestampsPath() {
 	public void setModuleTimestampsPath(String moduleTimestampsPath) {
 		this.moduleTimestampsPath = moduleTimestampsPath;
 	}
+
+	public String[] getResourceFilenamesToIgnore() {
+		return resourceFilenamesToIgnore;
+	}
+
+	public void setResourceFilenamesToIgnore(String... resourceFilenamesToIgnore) {
+		this.resourceFilenamesToIgnore = resourceFilenamesToIgnore;
+	}
 }

src/main/java/com/marklogic/appdeployer/ConfigDir.java

@@ -60,6 +60,10 @@ public File getCpfDir() {
         return new File(baseDir, "cpf");
     }
 
+    public File getClustersDir() {
+        return new File(baseDir, "clusters");
+    }
+
     public File getAlertDir() {
         return new File(baseDir, "alert");
     }

src/main/java/com/marklogic/appdeployer/DefaultAppConfigFactory.java

@@ -281,47 +281,60 @@ public AppConfig newAppConfig() {
 			c.setStaticCheckLibraryAssets(Boolean.parseBoolean(prop));
 		}
 
+		/**
+		 * The following properties are all for generating Entity Services artifacts.
+		 */
 		prop = getProperty("mlModelsPath");
 		if (prop != null) {
 			logger.info("Entity Services models path: " + prop);
 			c.setModelsPath(prop);
 		}
-
 		prop = getProperty("mlInstanceConverterPath");
 		if (prop != null) {
 			logger.info("Entity Services instance converter path: " + prop);
 			c.setInstanceConverterPath(prop);
 		}
-
 		prop = getProperty("mlGenerateInstanceConverter");
 		if (prop != null) {
 			logger.info("Entity Services generate instance converter: " + prop);
 			c.setGenerateInstanceConverter(Boolean.parseBoolean(prop));
 		}
-
 		prop = getProperty("mlGenerateSchema");
 		if (prop != null) {
 			logger.info("Entity Services generate schema: " + prop);
 			c.setGenerateSchema(Boolean.parseBoolean(prop));
 		}
-
 		prop = getProperty("mlGenerateSearchOptions");
 		if (prop != null) {
 			logger.info("Entity Services generate search options: " + prop);
 			c.setGenerateSearchOptions(Boolean.parseBoolean(prop));
 		}
-
 		prop = getProperty("mlGenerateDatabaseProperties");
 		if (prop != null) {
 			logger.info("Entity Services generate database properties: " + prop);
 			c.setGenerateDatabaseProperties(Boolean.parseBoolean(prop));
 		}
-
 		prop = getProperty("mlGenerateExtractionTemplate");
 		if (prop != null) {
 			logger.info("Entity Services generate extraction template: " + prop);
 			c.setGenerateExtractionTemplate(Boolean.parseBoolean(prop));
 		}
+		// End Entity Services properties
+
+		/**
+		 * Sets resource filenames to ignore on ALL commands. Be careful here, in case you have files for different kinds
+		 * of resources, but with the same filename (this should be very rare and easily avoided).
+		 *
+		 * Also that as of version 2.6.0 of ml-app-deployer, this property is processed by AbstractAppDeployer, NOT by
+		 * the Command itself. So in order for this property to be applied, you must execute a Command via a subclass of
+		 * AbstractAppDeployer (most commonly SimpleAppDeployer).
+		 */
+		prop = getProperty("mlResourceFilenamesToIgnore");
+		if (prop != null) {
+			String[] values = prop.split(",");
+			logger.info("Ignoring resource filenames: " + Arrays.asList(values));
+			c.setResourceFilenamesToIgnore(values);
+		}
 
 		return c;
 	}

src/main/java/com/marklogic/appdeployer/command/AbstractCommand.java

@@ -41,6 +41,9 @@ public Integer getExecuteSortOrder() {
      * @param filenames
      */
     public void setFilenamesToIgnore(String... filenames) {
+        if (filenames == null || filenames.length == 0) {
+            return;
+        }
         if (resourceFilenameFilter != null && resourceFilenameFilter instanceof ResourceFilenameFilter) {
             ResourceFilenameFilter rff = (ResourceFilenameFilter) resourceFilenameFilter;
             Set<String> set = null;
@@ -52,6 +55,7 @@ public void setFilenamesToIgnore(String... filenames) {
             for (String f : filenames) {
                 set.add(f);
             }
+            rff.setFilenamesToIgnore(set);
         } else {
             this.resourceFilenameFilter = new ResourceFilenameFilter(filenames);
         }

src/main/java/com/marklogic/appdeployer/command/AbstractResourceCommand.java

@@ -23,18 +23,22 @@ public abstract class AbstractResourceCommand extends AbstractUndoableCommand {
     @Override
     public void execute(CommandContext context) {
         for (File resourceDir : getResourceDirs(context)) {
-            if (resourceDir.exists()) {
-                ResourceManager mgr = getResourceManager(context);
+            processExecuteOnResourceDir(context, resourceDir);
+        }
+    }
+
+    protected void processExecuteOnResourceDir(CommandContext context, File resourceDir) {
+        if (resourceDir.exists()) {
+            ResourceManager mgr = getResourceManager(context);
+            if (logger.isInfoEnabled()) {
+                logger.info("Processing files in directory: " + resourceDir.getAbsolutePath());
+            }
+            for (File f : listFilesInDirectory(resourceDir)) {
                 if (logger.isInfoEnabled()) {
-                    logger.info("Processing files in directory: " + resourceDir.getAbsolutePath());
-                }
-                for (File f : listFilesInDirectory(resourceDir)) {
-                    if (logger.isInfoEnabled()) {
-                        logger.info("Processing file: " + f.getAbsolutePath());
-                    }
-                    SaveReceipt receipt = saveResource(mgr, context, f);
-                    afterResourceSaved(mgr, context, f, receipt);
+                    logger.info("Processing file: " + f.getAbsolutePath());
                 }
+                SaveReceipt receipt = saveResource(mgr, context, f);
+                afterResourceSaved(mgr, context, f, receipt);
             }
         }
     }
@@ -56,18 +60,22 @@ protected void afterResourceSaved(ResourceManager mgr, CommandContext context, F
     public void undo(CommandContext context) {
         if (deleteResourcesOnUndo) {
             for (File resourceDir : getResourceDirs(context)) {
-                if (resourceDir.exists()) {
-                    if (logger.isInfoEnabled()) {
-                        logger.info("Processing files in directory: " + resourceDir.getAbsolutePath());
-                    }
-                    final ResourceManager mgr = getResourceManager(context);
-                    for (File f : listFilesInDirectory(resourceDir)) {
-                        if (logger.isInfoEnabled()) {
-                            logger.info("Processing file: " + f.getAbsolutePath());
-                        }
-                        deleteResource(mgr, context, f);
-                    }
+                processUndoOnResourceDir(context, resourceDir);
+            }
+        }
+    }
+
+    protected void processUndoOnResourceDir(CommandContext context, File resourceDir) {
+        if (resourceDir.exists()) {
+            if (logger.isInfoEnabled()) {
+                logger.info("Processing files in directory: " + resourceDir.getAbsolutePath());
+            }
+            final ResourceManager mgr = getResourceManager(context);
+            for (File f : listFilesInDirectory(resourceDir)) {
+                if (logger.isInfoEnabled()) {
+                    logger.info("Processing file: " + f.getAbsolutePath());
                 }
+                deleteResource(mgr, context, f);
             }
         }
     }

src/main/java/com/marklogic/appdeployer/command/CommandMapBuilder.java

@@ -0,0 +1,152 @@
+package com.marklogic.appdeployer.command;
+
+import com.marklogic.appdeployer.command.alert.DeployAlertActionsCommand;
+import com.marklogic.appdeployer.command.alert.DeployAlertConfigsCommand;
+import com.marklogic.appdeployer.command.alert.DeployAlertRulesCommand;
+import com.marklogic.appdeployer.command.appservers.DeployOtherServersCommand;
+import com.marklogic.appdeployer.command.appservers.UpdateRestApiServersCommand;
+import com.marklogic.appdeployer.command.clusters.ModifyLocalClusterCommand;
+import com.marklogic.appdeployer.command.cpf.DeployCpfConfigsCommand;
+import com.marklogic.appdeployer.command.cpf.DeployDomainsCommand;
+import com.marklogic.appdeployer.command.cpf.DeployPipelinesCommand;
+import com.marklogic.appdeployer.command.databases.DeployContentDatabasesCommand;
+import com.marklogic.appdeployer.command.databases.DeployOtherDatabasesCommand;
+import com.marklogic.appdeployer.command.databases.DeploySchemasDatabaseCommand;
+import com.marklogic.appdeployer.command.databases.DeployTriggersDatabaseCommand;
+import com.marklogic.appdeployer.command.flexrep.DeployConfigsCommand;
+import com.marklogic.appdeployer.command.flexrep.DeployFlexrepCommand;
+import com.marklogic.appdeployer.command.flexrep.DeployTargetsCommand;
+import com.marklogic.appdeployer.command.forests.ConfigureForestReplicasCommand;
+import com.marklogic.appdeployer.command.forests.DeployCustomForestsCommand;
+import com.marklogic.appdeployer.command.groups.DeployGroupsCommand;
+import com.marklogic.appdeployer.command.mimetypes.DeployMimetypesCommand;
+import com.marklogic.appdeployer.command.modules.LoadModulesCommand;
+import com.marklogic.appdeployer.command.restapis.DeployRestApiServersCommand;
+import com.marklogic.appdeployer.command.schemas.LoadSchemasCommand;
+import com.marklogic.appdeployer.command.security.*;
+import com.marklogic.appdeployer.command.tasks.DeployScheduledTasksCommand;
+import com.marklogic.appdeployer.command.triggers.DeployTriggersCommand;
+import com.marklogic.appdeployer.command.viewschemas.DeployViewSchemasCommand;
+
+import java.util.ArrayList;
+import java.util.HashMap;
+import java.util.List;
+import java.util.Map;
+
+/**
+ * The intent of this class is to construct a map of commonly used commands that can used in a variety of contexts - i.e.
+ * ml-gradle or the Data Hub Framework - thus preventing those clients from having to duplicate this code.
+ *
+ * A map is returned so that the commands can be grouped into lists, which is convenient for e.g. ml-gradle tasks that
+ * want to execute all of the commands for a particular resource or set of resources - e.g. mlSecurityCommands for
+ * invoking all commands pertaining to security resources.
+ */
+public class CommandMapBuilder {
+
+	public Map<String, List<Command>> buildCommandMap() {
+		Map<String, List<Command>> map = new HashMap<>();
+
+		// Security
+		List<Command> securityCommands = new ArrayList<Command>();
+		securityCommands.add(new DeployRolesCommand());
+		securityCommands.add(new DeployUsersCommand());
+		securityCommands.add(new DeployAmpsCommand());
+		securityCommands.add(new DeployCertificateTemplatesCommand());
+		securityCommands.add(new DeployCertificateAuthoritiesCommand());
+		securityCommands.add(new DeployExternalSecurityCommand());
+		securityCommands.add(new DeployPrivilegesCommand());
+		securityCommands.add(new DeployProtectedCollectionsCommand());
+		map.put("mlSecurityCommands", securityCommands);
+
+		// Cluster
+		List<Command> clusterCommands = new ArrayList<Command>();
+		clusterCommands.add(new ModifyLocalClusterCommand());
+		map.put("mlClusterCommands", clusterCommands);
+
+		// Databases
+		List<Command> dbCommands = new ArrayList<Command>();
+		dbCommands.add(new DeployContentDatabasesCommand());
+		dbCommands.add(new DeployTriggersDatabaseCommand());
+		dbCommands.add(new DeploySchemasDatabaseCommand());
+		dbCommands.add(new DeployOtherDatabasesCommand());
+		map.put("mlDatabaseCommands", dbCommands);
+
+		// Schemas
+		List<Command> schemaCommands = new ArrayList<>();
+		schemaCommands.add(new LoadSchemasCommand());
+		map.put("mlSchemaCommands", schemaCommands);
+
+		// REST API instance creation
+		List<Command> restApiCommands = new ArrayList<>();
+		restApiCommands.add(new DeployRestApiServersCommand());
+		map.put("mlRestApiCommands", restApiCommands);
+
+		// App servers
+		List<Command> serverCommands = new ArrayList<>();
+		serverCommands.add(new DeployOtherServersCommand());
+		serverCommands.add(new UpdateRestApiServersCommand());
+		map.put("mlServerCommands", serverCommands);
+
+		// Modules
+		List<Command> moduleCommands = new ArrayList<>();
+		moduleCommands.add(new LoadModulesCommand());
+		map.put("mlModuleCommands", moduleCommands);
+
+		// Alerting
+		List<Command> alertCommands = new ArrayList<Command>();
+		alertCommands.add(new DeployAlertConfigsCommand());
+		alertCommands.add(new DeployAlertActionsCommand());
+		alertCommands.add(new DeployAlertRulesCommand());
+		map.put("mlAlertCommands", alertCommands);
+
+		// CPF
+		List<Command> cpfCommands = new ArrayList<Command>();
+		cpfCommands.add(new DeployCpfConfigsCommand());
+		cpfCommands.add(new DeployDomainsCommand());
+		cpfCommands.add(new DeployPipelinesCommand());
+		map.put("mlCpfCommands", cpfCommands);
+
+		// Flexrep
+		List<Command> flexrepCommands = new ArrayList<Command>();
+		flexrepCommands.add(new DeployConfigsCommand());
+		flexrepCommands.add(new DeployTargetsCommand());
+		flexrepCommands.add(new DeployFlexrepCommand());
+		map.put("mlFlexrepCommands", flexrepCommands);
+
+		// Groups
+		List<Command> groupCommands = new ArrayList<Command>();
+		groupCommands.add(new DeployGroupsCommand());
+		map.put("mlGroupCommands", groupCommands);
+
+		List<Command> mimetypeCommands = new ArrayList<Command>();
+		mimetypeCommands.add(new DeployMimetypesCommand());
+		map.put("mlMimetypeCommands", mimetypeCommands);
+
+		// Forests
+		List<Command> forestCommands = new ArrayList<Command>();
+		forestCommands.add(new DeployCustomForestsCommand());
+		map.put("mlForestCommands", forestCommands);
+
+		// Forest replicas
+		List<Command> replicaCommands = new ArrayList<Command>();
+		replicaCommands.add(new ConfigureForestReplicasCommand());
+		map.put("mlForestReplicaCommands", replicaCommands);
+
+		// Tasks
+		List<Command> taskCommands = new ArrayList<Command>();
+		taskCommands.add(new DeployScheduledTasksCommand());
+		map.put("mlTaskCommands", taskCommands);
+
+		// Triggers
+		List<Command> triggerCommands = new ArrayList<Command>();
+		triggerCommands.add(new DeployTriggersCommand());
+		map.put("mlTriggerCommands", triggerCommands);
+
+		// SQL Views
+		List<Command> viewCommands = new ArrayList<Command>();
+		viewCommands.add(new DeployViewSchemasCommand());
+		map.put("mlViewCommands", viewCommands);
+
+		return map;
+	}
+}

src/main/java/com/marklogic/appdeployer/command/clusters/ModifyLocalClusterCommand.java

@@ -0,0 +1,35 @@
+package com.marklogic.appdeployer.command.clusters;
+
+import com.marklogic.appdeployer.command.AbstractCommand;
+import com.marklogic.appdeployer.command.CommandContext;
+import com.marklogic.appdeployer.command.SortOrderConstants;
+import com.marklogic.mgmt.clusters.ClusterManager;
+
+import java.io.File;
+
+/**
+ * Looks for a file with a name starting with "local-cluster" (e.g. "local-cluster.json" or "local-cluster.xml") in the
+ * "clusters" directory. The cluster endpoints - https://docs.marklogic.com/REST/management/clusters - offer a lot more
+ * functionality; this command is just for the https://docs.marklogic.com/REST/PUT/manage/v2/properties endpoint.
+ */
+public class ModifyLocalClusterCommand extends AbstractCommand {
+
+	public ModifyLocalClusterCommand() {
+		setExecuteSortOrder(SortOrderConstants.MODIFY_LOCAL_CLUSTER);
+	}
+
+	@Override
+	public void execute(CommandContext context) {
+		File configDir = context.getAppConfig().getConfigDir().getClustersDir();
+		if (configDir != null && configDir.exists()) {
+			for (File f : configDir.listFiles()) {
+				if (f.isFile() && f.getName().startsWith("local-cluster")) {
+					String payload = copyFileToString(f);
+					payload = tokenReplacer.replaceTokens(payload, context.getAppConfig(), false);
+					new ClusterManager(context.getManageClient()).modifyLocalCluster(payload, context.getAdminManager());
+				}
+			}
+		}
+	}
+
+}