#184 Can include/exclude resource filenames based on a regex

Author: rjrudin

src/main/java/com/marklogic/appdeployer/AppConfig.java

@@ -18,6 +18,7 @@
 import java.util.HashMap;
 import java.util.List;
 import java.util.Map;
+import java.util.regex.Pattern;
 
 /**
  * Encapsulates common configuration properties for an application deployed to MarkLogic. These properties include not
@@ -172,6 +173,8 @@ public class AppConfig {
 	private boolean generateSearchOptions = true;
 
 	private String[] resourceFilenamesToIgnore;
+	private Pattern resourceFilenamesExcludePattern;
+	private Pattern resourceFilenamesIncludePattern;
 
 	private Map<String, Object> additionalProperties = new HashMap<>();
 
@@ -903,4 +906,20 @@ public ConfiguredDatabaseClientFactory getConfiguredDatabaseClientFactory() {
 	public void setConfiguredDatabaseClientFactory(ConfiguredDatabaseClientFactory configuredDatabaseClientFactory) {
 		this.configuredDatabaseClientFactory = configuredDatabaseClientFactory;
 	}
+
+	public Pattern getResourceFilenamesExcludePattern() {
+		return resourceFilenamesExcludePattern;
+	}
+
+	public void setResourceFilenamesExcludePattern(Pattern resourceFilenamesExcludePattern) {
+		this.resourceFilenamesExcludePattern = resourceFilenamesExcludePattern;
+	}
+
+	public Pattern getResourceFilenamesIncludePattern() {
+		return resourceFilenamesIncludePattern;
+	}
+
+	public void setResourceFilenamesIncludePattern(Pattern resourceFilenamesIncludePattern) {
+		this.resourceFilenamesIncludePattern = resourceFilenamesIncludePattern;
+	}
 }

src/main/java/com/marklogic/appdeployer/DefaultAppConfigFactory.java

@@ -9,6 +9,7 @@
 import java.util.ArrayList;
 import java.util.Arrays;
 import java.util.List;
+import java.util.regex.Pattern;
 
 public class DefaultAppConfigFactory extends PropertySourceFactory implements AppConfigFactory {
 
@@ -500,6 +501,18 @@ public AppConfig newAppConfig() {
 			c.setResourceFilenamesToIgnore(values);
 		}
 
+		prop = getProperty("mlResourceFilenamesToExcludeRegex");
+		if (prop != null) {
+			logger.info("Excluding resource filenames matching regex: " + prop);
+			c.setResourceFilenamesExcludePattern(Pattern.compile(prop));
+		}
+
+		prop = getProperty("mlResourceFilenamesToIncludeRegex");
+		if (prop != null) {
+			logger.info("Including resource filenames matching regex: " + prop);
+			c.setResourceFilenamesIncludePattern(Pattern.compile(prop));
+		}
+
 		/**
 		 * Version 2.9.0 of ml-app-deployer, by default, sorts role files by reading each file and looking at the role
 		 * dependencies. You can disable this behavior by setting this property to false.

src/main/java/com/marklogic/appdeployer/command/AbstractCommand.java

@@ -12,6 +12,7 @@
 import java.util.Arrays;
 import java.util.HashSet;
 import java.util.Set;
+import java.util.regex.Pattern;
 
 /**
  * Abstract base class that provides some convenience methods for implementing a command. Subclasses will typically
@@ -43,23 +44,55 @@ public void setFilenamesToIgnore(String... filenames) {
         if (filenames == null || filenames.length == 0) {
             return;
         }
-        if (resourceFilenameFilter != null && resourceFilenameFilter instanceof ResourceFilenameFilter) {
-            ResourceFilenameFilter rff = (ResourceFilenameFilter) resourceFilenameFilter;
-            Set<String> set = null;
-            if (rff.getFilenamesToIgnore() != null) {
-                set = rff.getFilenamesToIgnore();
-            } else {
-                set = new HashSet<>();
-            }
-            for (String f : filenames) {
-                set.add(f);
-            }
-            rff.setFilenamesToIgnore(set);
+        if (resourceFilenameFilter != null) {
+        	if (resourceFilenameFilter instanceof ResourceFilenameFilter) {
+		        ResourceFilenameFilter rff = (ResourceFilenameFilter) resourceFilenameFilter;
+		        Set<String> set = null;
+		        if (rff.getFilenamesToIgnore() != null) {
+			        set = rff.getFilenamesToIgnore();
+		        } else {
+			        set = new HashSet<>();
+		        }
+		        for (String f : filenames) {
+			        set.add(f);
+		        }
+		        rff.setFilenamesToIgnore(set);
+	        } else {
+		        logger.warn("resourceFilenameFilter is not an instanceof ResourceFilenameFilter, so unable to set resource filenames to ignore");
+	        }
         } else {
             this.resourceFilenameFilter = new ResourceFilenameFilter(filenames);
         }
     }
 
+    public void setResourceFilenamesExcludePattern(Pattern pattern) {
+    	if (resourceFilenameFilter != null) {
+    		if (resourceFilenameFilter instanceof ResourceFilenameFilter) {
+			    ((ResourceFilenameFilter)resourceFilenameFilter).setExcludePattern(pattern);
+		    } else {
+    			logger.warn("resourceFilenameFilter is not an instanceof ResourceFilenameFilter, so unable to set exclude pattern");
+		    }
+	    } else {
+    		ResourceFilenameFilter rff = new ResourceFilenameFilter();
+    		rff.setExcludePattern(pattern);
+    		this.resourceFilenameFilter = rff;
+	    }
+    }
+
+	public void setResourceFilenamesIncludePattern(Pattern pattern) {
+		if (resourceFilenameFilter != null) {
+			if (resourceFilenameFilter instanceof ResourceFilenameFilter) {
+				((ResourceFilenameFilter)resourceFilenameFilter).setIncludePattern(pattern);
+			} else {
+				logger.warn("resourceFilenameFilter is not an instanceof ResourceFilenameFilter, so unable to set include pattern");
+			}
+		} else {
+			ResourceFilenameFilter rff = new ResourceFilenameFilter();
+			rff.setIncludePattern(pattern);
+			this.resourceFilenameFilter = rff;
+		}
+	}
+
     /**
      * Simplifies reading the contents of a File into a String.
      *

src/main/java/com/marklogic/appdeployer/command/ResourceFilenameFilter.java

@@ -6,13 +6,16 @@
 import java.io.FilenameFilter;
 import java.util.HashSet;
 import java.util.Set;
+import java.util.regex.Pattern;
 
 /**
  * Simple filter implementation that returns true for .json and .xml files.
  */
 public class ResourceFilenameFilter extends LoggingObject implements FilenameFilter {
 
     private Set<String> filenamesToIgnore;
+    private Pattern excludePattern;
+    private Pattern includePattern;
 
     public ResourceFilenameFilter() {
     }
@@ -29,14 +32,37 @@ public ResourceFilenameFilter(Set<String> filenamesToIgnore) {
     }
 
     @Override
-    public boolean accept(File dir, String name) {
-        if (filenamesToIgnore != null && filenamesToIgnore.contains(name)) {
-            if (logger.isDebugEnabled()) {
-                logger.debug("Ignoring filename: " + name);
+    public boolean accept(File dir, String filename) {
+    	if (excludePattern != null && includePattern != null) {
+    		throw new IllegalStateException("Both excludePattern and includePattern cannot be specified");
+	    }
+
+	    if (excludePattern != null) {
+    		if (excludePattern.matcher(filename).matches()) {
+    			if (logger.isInfoEnabled()) {
+    				logger.info(format("Filename %s matches excludePattern, so ignoring", filename));
+			    }
+			    return false;
+		    }
+	    }
+
+	    if (includePattern != null) {
+    		if (!includePattern.matcher(filename).matches()) {
+    			if (logger.isInfoEnabled()) {
+    				logger.info(format("Filename %s doesn't match includePattern, so ignoring", filename));
+			    }
+			    return false;
+		    }
+	    }
+
+        if (filenamesToIgnore != null && filenamesToIgnore.contains(filename)) {
+            if (logger.isInfoEnabled()) {
+                logger.info("Ignoring filename: " + filename);
             }
             return false;
         }
-        return name.endsWith(".json") || name.endsWith(".xml");
+
+        return filename.endsWith(".json") || filename.endsWith(".xml");
     }
 
     public void setFilenamesToIgnore(Set<String> ignoreFilenames) {
@@ -47,4 +73,19 @@ public Set<String> getFilenamesToIgnore() {
         return filenamesToIgnore;
     }
 
+	public Pattern getExcludePattern() {
+		return excludePattern;
+	}
+
+	public void setExcludePattern(Pattern excludePattern) {
+		this.excludePattern = excludePattern;
+	}
+
+	public Pattern getIncludePattern() {
+		return includePattern;
+	}
+
+	public void setIncludePattern(Pattern includePattern) {
+		this.includePattern = includePattern;
+	}
 }

src/main/java/com/marklogic/appdeployer/impl/AbstractAppDeployer.java

@@ -14,6 +14,7 @@
 import java.util.Collections;
 import java.util.Comparator;
 import java.util.List;
+import java.util.regex.Pattern;
 
 /**
  * Abstract base class that just needs the subclass to define the list of Command instances to use. Handles executing
@@ -54,12 +55,26 @@ public void deploy(AppConfig appConfig) {
         CommandContext context = new CommandContext(appConfig, manageClient, adminManager);
 
         String[] filenamesToIgnore = appConfig.getResourceFilenamesToIgnore();
+	    Pattern excludePattern = appConfig.getResourceFilenamesExcludePattern();
+	    Pattern includePattern = appConfig.getResourceFilenamesIncludePattern();
+
         for (Command command : commands) {
             String name = command.getClass().getName();
             logger.info(format("Executing command [%s] with sort order [%d]", name, command.getExecuteSortOrder()));
+
             if (command instanceof AbstractCommand) {
-	            ((AbstractCommand)command).setFilenamesToIgnore(filenamesToIgnore);
+            	AbstractCommand abstractCommand = (AbstractCommand)command;
+            	if (filenamesToIgnore != null) {
+		            abstractCommand.setFilenamesToIgnore(filenamesToIgnore);
+	            }
+	            if (excludePattern != null) {
+            		abstractCommand.setResourceFilenamesExcludePattern(excludePattern);
+	            }
+	            if (includePattern != null) {
+            		abstractCommand.setResourceFilenamesIncludePattern(includePattern);
+	            }
             }
+
             command.execute(context);
             logger.info(format("Finished executing command [%s]\n", name));
         }

src/test/java/com/marklogic/appdeployer/DefaultAppConfigFactoryTest.java

@@ -129,7 +129,10 @@ public void allProperties() {
 	    p.setProperty("mlGenerateSchema", "false");
 	    p.setProperty("mlGenerateSearchOptions", "false");
 	    p.setProperty("mlGenerateExtractionTemplate", "false");
+
 	    p.setProperty("mlResourceFilenamesToIgnore", "role1.json,role2.xml");
+	    p.setProperty("mlResourceFilenamesToExcludeRegex", "dev-.*");
+	    p.setProperty("mlResourceFilenamesToIncludeRegex", "qa-.*");
 
 	    p.setProperty("mlDatabaseNamesAndReplicaCounts", "Documents,1,Security,2");
 	    p.setProperty("mlReplicaForestDataDirectory", "/var/data");
@@ -208,6 +211,8 @@ public void allProperties() {
 
         assertEquals("role1.json", config.getResourceFilenamesToIgnore()[0]);
 	    assertEquals("role2.xml", config.getResourceFilenamesToIgnore()[1]);
+	    assertEquals("dev-.*", config.getResourceFilenamesExcludePattern().pattern());
+	    assertEquals("qa-.*", config.getResourceFilenamesIncludePattern().pattern());
 
 	    assertEquals("Documents,1,Security,2", config.getDatabaseNamesAndReplicaCounts());
 	    assertEquals("/var/data", config.getReplicaForestDataDirectory());

src/test/java/com/marklogic/appdeployer/command/security/IgnoreRoleTest.java

@@ -4,6 +4,8 @@
 import com.marklogic.mgmt.resource.security.RoleManager;
 import org.junit.Test;
 
+import java.util.regex.Pattern;
+
 public class IgnoreRoleTest extends AbstractAppDeployerTest {
 
 	@Test
@@ -20,4 +22,49 @@ public void test() {
 			undeploySampleApp();
 		}
 	}
+
+	@Test
+	public void excludeRoles() {
+		appConfig.setResourceFilenamesExcludePattern(Pattern.compile(".*role1.*"));
+		initializeAppDeployer(new DeployRolesCommand());
+		appDeployer.deploy(appConfig);
+
+		try {
+			RoleManager mgr = new RoleManager(manageClient);
+			assertTrue(mgr.exists("sample-app-role2"));
+			assertFalse("Role should not have been created because its filename was excluded via regex", mgr.exists("sample-app-role1"));
+		} finally {
+			undeploySampleApp();
+		}
+	}
+
+	@Test
+	public void includeRoles() {
+		appConfig.setResourceFilenamesIncludePattern(Pattern.compile(".*role1.*"));
+		initializeAppDeployer(new DeployRolesCommand());
+		appDeployer.deploy(appConfig);
+
+		try {
+			RoleManager mgr = new RoleManager(manageClient);
+			assertTrue(mgr.exists("sample-app-role1"));
+			assertFalse("Role should not have been created because its filename was not included via regex", mgr.exists("sample-app-role2"));
+		} finally {
+			undeploySampleApp();
+		}
+	}
+
+	@Test
+	public void includeAndExcludeRoles() {
+		appConfig.setResourceFilenamesExcludePattern(Pattern.compile(".*role2.*"));
+		appConfig.setResourceFilenamesIncludePattern(Pattern.compile(".*role1.*"));
+		initializeAppDeployer(new DeployRolesCommand());
+		try {
+			appDeployer.deploy(appConfig);
+			fail("Deployment should have failed because exclude and include patterns can't both be set");
+		} catch (Exception ex) {
+			assertEquals("Both excludePattern and includePattern cannot be specified", ex.getMessage());
+		} finally {
+			undeploySampleApp();
+		}
+	}
 }

src/test/java/com/marklogic/mgmt/api/ConnectTest.java

@@ -9,9 +9,6 @@ public class ConnectTest extends AbstractApiTest {
 	 */
 	@Test
 	public void test() {
-		manageConfig.setScheme("https");
-		manageConfig.setConfigureSimpleSsl(true);
-
 		api.connect(manageConfig.getHost(), manageConfig);
 		api.getDb().list();