Merge branch 'dev'

Author: rjrudin

gradle.properties

@@ -1,6 +1,6 @@
 group=com.marklogic
 javadocsDir=../gh-pages-marklogic-java/javadocs
-version=3.2.0
-mlJavaclientUtilVersion=3.2.0
+version=3.3.0
+mlJavaclientUtilVersion=3.3.0
 mlJunitVersion=3.1.0
 

src/main/java/com/marklogic/appdeployer/AppConfig.java

@@ -116,7 +116,8 @@ public class AppConfig {
 	private int modulesLoaderThreadCount = 8;
 
     private String schemasPath;
-    private ConfigDir configDir;
+
+    private List<ConfigDir> configDirs;
 
     // Passed into the PayloadTokenReplacer that subclasses of AbstractCommand use
     private Map<String, String> customTokens = new HashMap<>();
@@ -154,6 +155,7 @@ public class AppConfig {
     private Set<String> databasesWithForestsOnOneHost;
 
     private Map<String, Set<String>> databaseHosts;
+    private Map<String, Set<String>> databaseGroups;
 
     // Data/fast/large directories for default forests
     private String forestDataDirectory;
@@ -183,7 +185,9 @@ public class AppConfig {
     // Additional PropertiesSources instance to use for replacing module tokens
     private List<PropertiesSource> moduleTokensPropertiesSources = new ArrayList<>();
 
-    private Map<String, Integer> forestCounts = new HashMap<>();
+	private Pattern moduleFilenamesIncludePattern;
+
+	private Map<String, Integer> forestCounts = new HashMap<>();
 
     // Entity Services properties
     private String modelsPath = "data/entity-services";
@@ -212,7 +216,8 @@ public AppConfig(String defaultModulePath) {
     public AppConfig(String defaultModulePath, String defaultSchemasPath) {
         modulePaths = new ArrayList<String>();
         modulePaths.add(defaultModulePath);
-        configDir = new ConfigDir();
+        configDirs = new ArrayList<>();
+        configDirs.add(new ConfigDir());
         schemasPath = defaultSchemasPath;
     }
 
@@ -523,16 +528,40 @@ public void setRestAuthentication(Authentication authentication) {
     }
 
     /**
+     * As of 3.3.0, this now returns the first ConfigDir in the List of ConfigsDir that this class now maintains.
+     *
      * @return a {@code ConfigDir} instance that defines the location of the configuration directory (where files are
      * stored that are then loaded via MarkLogic Management API endpoints) as well as paths to specific
      * resources within that directory
      */
+    @Deprecated
     public ConfigDir getConfigDir() {
-        return configDir;
+    	return getFirstConfigDir();
+    }
+
+	/**
+	 * Starting in 3.3.0, use this when you only care about the first ConfigDir in the List of ConfigDirs maintained by
+	 * this class.
+	 *
+	 * @return
+	 */
+	public ConfigDir getFirstConfigDir() {
+	    if (configDirs == null || configDirs.isEmpty()) {
+		    this.configDirs = new ArrayList<>();
+		    this.configDirs.add(new ConfigDir());
+	    }
+	    return configDirs.get(0);
     }
 
-    public void setConfigDir(ConfigDir configDir) {
-        this.configDir = configDir;
+	/**
+	 * As of 3.3.0, this is instead clearing and adding the ConfigDir to the List of ConfigDirs that this class now
+	 * maintains.
+	 *
+	 * @param configDir
+	 */
+	public void setConfigDir(ConfigDir configDir) {
+		this.configDirs = new ArrayList<>();
+		this.configDirs.add(configDir);
     }
 
     /**
@@ -1106,4 +1135,28 @@ public Map<String, Set<String>> getDatabaseHosts() {
 	public void setDatabaseHosts(Map<String, Set<String>> databaseHosts) {
 		this.databaseHosts = databaseHosts;
 	}
+
+	public Map<String, Set<String>> getDatabaseGroups() {
+		return databaseGroups;
+	}
+
+	public void setDatabaseGroups(Map<String, Set<String>> databaseGroups) {
+		this.databaseGroups = databaseGroups;
+	}
+
+	public List<ConfigDir> getConfigDirs() {
+		return configDirs;
+	}
+
+	public void setConfigDirs(List<ConfigDir> configDirs) {
+		this.configDirs = configDirs;
+	}
+
+	public Pattern getModuleFilenamesIncludePattern() {
+		return moduleFilenamesIncludePattern;
+	}
+
+	public void setModuleFilenamesIncludePattern(Pattern moduleFilenamesIncludePattern) {
+		this.moduleFilenamesIncludePattern = moduleFilenamesIncludePattern;
+	}
 }

src/main/java/com/marklogic/appdeployer/ConfigDir.java

@@ -80,21 +80,41 @@ public File getAmpsDir() {
 		return new File(getSecurityDir(), "amps");
 	}
 
+	public File getCertificateAuthoritiesDir() {
+		return new File(getSecurityDir(), "certificate-authorities");
+	}
+
+	public File getCertificateTemplatesDir() {
+		return new File(getSecurityDir(), "certificate-templates");
+	}
+
+	public File getExternalSecuritiesDir() {
+		return new File(getSecurityDir(), "external-security");
+	}
+
 	public File getPrivilegesDir() {
 		return new File(getSecurityDir(), "privileges");
 	}
 
+	public File getProtectedCollectionsDir() {
+		return new File(getSecurityDir(), "protected-collections");
+	}
+
 	public File getRolesDir() {
 		return new File(getSecurityDir(), "roles");
 	}
 
+	public File getTriggersDir() {
+		return new File(getBaseDir(), "triggers");
+	}
+
 	public File getUsersDir() {
 		return new File(getSecurityDir(), "users");
 	}
 
 	public File getProtectedPathsDir() { return new File(getSecurityDir(), "protected-paths"); }
 
-	public File getQueryRoleSetsDir() { return new File(getSecurityDir(), "query-rolesets"); }
+	public File getQueryRolesetsDir() { return new File(getSecurityDir(), "query-rolesets"); }
 
 	public File getServersDir() {
 		return new File(baseDir, "servers");
@@ -132,6 +152,10 @@ public File getGroupsDir() {
 		return new File(baseDir, "groups");
 	}
 
+	public File getMimetypesDir() {
+		return new File(baseDir, "mimetypes");
+	}
+
 	public File getViewSchemasDir() {
 		return new File(baseDir, "view-schemas");
 	}

src/main/java/com/marklogic/appdeployer/DefaultAppConfigFactory.java

@@ -58,16 +58,30 @@ public AppConfig newAppConfig() {
 		 * The path to the directory containing all the resource configuration files. Defaults to src/main/ml-config.
 		 * mlConfigPath is the preferred one, as its name is consistent with other properties that refer to a path.
 		 * mlConfigDir is deprecated but still supported.
+		 *
+		 * As of 3.3.0, mlConfigPaths is the preferred property, and mlConfigDir and mlConfigPath will be ignored if
+		 * it's set.
 		 */
-		prop = getProperty("mlConfigDir");
+		prop = getProperty("mlConfigPaths");
 		if (prop != null) {
-			logger.info("mlConfigDir is deprecated; please use mlConfigPath; Config dir: " + prop);
-			c.setConfigDir(new ConfigDir(new File(prop)));
+			logger.info("Config paths: " + prop);
+			List<ConfigDir> list = new ArrayList<>();
+			for (String path : prop.split(",")) {
+				list.add(new ConfigDir(new File(path)));
+			}
+			c.setConfigDirs(list);
 		}
-		prop = getProperty("mlConfigPath");
-		if (prop != null) {
-			logger.info("Config path: " + prop);
-			c.setConfigDir(new ConfigDir(new File(prop)));
+		else {
+			prop = getProperty("mlConfigDir");
+			if (prop != null) {
+				logger.info("mlConfigDir is deprecated; please use mlConfigPath; Config dir: " + prop);
+				c.setConfigDir(new ConfigDir(new File(prop)));
+			}
+			prop = getProperty("mlConfigPath");
+			if (prop != null) {
+				logger.info("Config path: " + prop);
+				c.setConfigDir(new ConfigDir(new File(prop)));
+			}
 		}
 
 		/**
@@ -287,21 +301,16 @@ public AppConfig newAppConfig() {
 			c.setDatabasesWithForestsOnOneHost(set);
 		}
 
+		prop = getProperty("mlDatabaseGroups");
+		if (prop != null) {
+			logger.info("Databases and the groups containing the hosts that their forests will be created on: " + prop);
+			c.setDatabaseGroups(buildSetMapFromDelimitedString(prop));
+		}
+
 		prop = getProperty("mlDatabaseHosts");
 		if (prop != null) {
 			logger.info("Databases and the hosts that their forests will be created on: " + prop);
-			String[] tokens = prop.split(",");
-			Map<String, Set<String>> map = new HashMap<>();
-			for (int i = 0; i < tokens.length; i += 2) {
-				String dbName = tokens[i];
-				String[] hostNames = tokens[i + 1].split("\\|");
-				Set<String> names = new HashSet<>();
-				for (String name : hostNames) {
-					names.add(name);
-				}
-				map.put(dbName, names);
-			}
-			c.setDatabaseHosts(map);
+			c.setDatabaseHosts(buildSetMapFromDelimitedString(prop));
 		}
 
 		prop = getProperty("mlForestDataDirectory");
@@ -509,6 +518,12 @@ public AppConfig newAppConfig() {
 			c.setModuleTimestampsPath(prop);
 		}
 
+		prop = getProperty("mlModulesRegex");
+		if (prop != null) {
+			logger.info("Including module filenames matching regex: " + prop);
+			c.setModuleFilenamesIncludePattern(Pattern.compile(prop));
+		}
+
 		/**
 		 * Whether or not to load asset modules in bulk - i.e. in one transaction. Defaults to true.
 		 */
@@ -644,4 +659,19 @@ protected Map<String, String> buildMapFromCommaDelimitedString(String str) {
 		}
 		return map;
 	}
+
+	protected Map<String, Set<String>> buildSetMapFromDelimitedString(String str) {
+		String[] tokens = str.split(",");
+		Map<String, Set<String>> map = new LinkedHashMap<>();
+		for (int i = 0; i < tokens.length; i += 2) {
+			String dbName = tokens[i];
+			String[] hostNames = tokens[i + 1].split("\\|");
+			Set<String> names = new LinkedHashSet<>();
+			for (String name : hostNames) {
+				names.add(name);
+			}
+			map.put(dbName, names);
+		}
+		return map;
+	}
 }

src/main/java/com/marklogic/appdeployer/command/AbstractCommand.java

@@ -206,6 +206,12 @@ protected File[] listFilesInDirectory(File dir) {
         return files;
     }
 
+    protected void logResourceDirectoryNotFound(File dir) {
+    	 if (dir != null && logger.isInfoEnabled()) {
+		    logger.info("No resource directory found at: " + dir.getAbsolutePath());
+	    }
+    }
+
     public void setPayloadTokenReplacer(PayloadTokenReplacer payloadTokenReplacer) {
         this.payloadTokenReplacer = payloadTokenReplacer;
     }

src/main/java/com/marklogic/appdeployer/command/AbstractResourceCommand.java

@@ -1,15 +1,18 @@
 package com.marklogic.appdeployer.command;
 
+import com.marklogic.appdeployer.AppConfig;
+import com.marklogic.appdeployer.ConfigDir;
 import com.marklogic.mgmt.admin.AdminManager;
 import com.marklogic.mgmt.resource.ResourceManager;
 import com.marklogic.mgmt.SaveReceipt;
 import com.marklogic.mgmt.admin.ActionRequiringRestart;
-import com.marklogic.mgmt.admin.AdminManager;
 import org.springframework.http.HttpHeaders;
 import org.springframework.http.ResponseEntity;
 
 import java.io.File;
 import java.net.URI;
+import java.util.ArrayList;
+import java.util.List;
 
 /**
  * Provides a basic implementation for creating/updating a resource while an app is being deployed and then deleting it
@@ -32,21 +35,54 @@ public void execute(CommandContext context) {
         }
     }
 
-    protected void processExecuteOnResourceDir(CommandContext context, File resourceDir) {
-        if (resourceDir.exists()) {
-            ResourceManager mgr = getResourceManager(context);
-            if (logger.isInfoEnabled()) {
-                logger.info("Processing files in directory: " + resourceDir.getAbsolutePath());
-            }
-            for (File f : listFilesInDirectory(resourceDir, context)) {
-                if (logger.isInfoEnabled()) {
-                    logger.info("Processing file: " + f.getAbsolutePath());
-                }
-                SaveReceipt receipt = saveResource(mgr, context, f);
-                afterResourceSaved(mgr, context, f, receipt);
-            }
-        }
-    }
+	protected File[] findResourceDirs(CommandContext context, ResourceDirFinder resourceDirFinder) {
+		return findResourceDirs(context.getAppConfig(), resourceDirFinder);
+	}
+
+	/**
+	 * A subclass is likely to use this as a simple way of selecting all of the resource directories, specific to the
+	 * subclass's resource, within each ConfigDir on the AppConfig.
+	 *
+	 * @param appConfig
+	 * @param resourceDirFinder
+	 * @return
+	 */
+	protected File[] findResourceDirs(AppConfig appConfig, ResourceDirFinder resourceDirFinder) {
+		List<File> list = new ArrayList<>();
+		List<ConfigDir> configDirs = appConfig.getConfigDirs();
+		if (configDirs != null && !configDirs.isEmpty()) {
+			for (ConfigDir configDir : appConfig.getConfigDirs()) {
+				File dir = resourceDirFinder.getResourceDir(configDir);
+				if (dir != null && dir.exists()) {
+					list.add(dir);
+				} else {
+					logResourceDirectoryNotFound(dir);
+				}
+			}
+		}
+		else {
+			logger.warn("No ConfigDir objects found in AppConfig, unable to find resource directories");
+		}
+		return list.toArray(new File[]{});
+	}
+
+	protected void processExecuteOnResourceDir(CommandContext context, File resourceDir) {
+		if (resourceDir.exists()) {
+			ResourceManager mgr = getResourceManager(context);
+			if (logger.isInfoEnabled()) {
+				logger.info("Processing files in directory: " + resourceDir.getAbsolutePath());
+			}
+			for (File f : listFilesInDirectory(resourceDir, context)) {
+				if (logger.isInfoEnabled()) {
+					logger.info("Processing file: " + f.getAbsolutePath());
+				}
+				SaveReceipt receipt = saveResource(mgr, context, f);
+				afterResourceSaved(mgr, context, f, receipt);
+			}
+		} else {
+			logResourceDirectoryNotFound(resourceDir);
+		}
+	}
 
 	/**
 	 * Defaults to the parent method. This was extracted so that a subclass can override it and have access to the

src/main/java/com/marklogic/appdeployer/command/CommandMapBuilder.java

@@ -61,6 +61,8 @@ public Map<String, List<Command>> buildCommandMap() {
 		securityCommands.add(new DeployExternalSecurityCommand());
 		securityCommands.add(new DeployPrivilegesCommand());
 		securityCommands.add(new DeployProtectedCollectionsCommand());
+		securityCommands.add(new DeployProtectedPathsCommand());
+		securityCommands.add(new DeployQueryRolesetsCommand());
 		map.put("mlSecurityCommands", securityCommands);
 
 		// Cluster

src/main/java/com/marklogic/appdeployer/command/ResourceDirFinder.java

@@ -0,0 +1,12 @@
+package com.marklogic.appdeployer.command;
+
+import com.marklogic.appdeployer.ConfigDir;
+
+import java.io.File;
+
+/**
+ * Exists solely for the "findResourceDirs" method in AbstractResourceCommand.
+ */
+public interface ResourceDirFinder {
+	File getResourceDir(ConfigDir configDir);
+}