#214 Can now specify the hosts for database forests

Author: rjrudin

src/main/java/com/marklogic/appdeployer/AppConfig.java

@@ -151,7 +151,9 @@ public class AppConfig {
     private String databaseNamesAndReplicaCounts;
 
     // Comma-delimited string of database names that should only have forests (most likely just one) created on one host
-    private Set<String> databaseNamesWithForestsOnOneHost;
+    private Set<String> databasesWithForestsOnOneHost;
+
+    private Map<String, Set<String>> databaseHosts;
 
     // Data/fast/large directories for default forests
     private String forestDataDirectory;
@@ -1056,11 +1058,19 @@ public void setCatchUndeployExceptions(boolean catchUndeployExceptions) {
 		this.catchUndeployExceptions = catchUndeployExceptions;
 	}
 
-	public Set<String> getDatabaseNamesWithForestsOnOneHost() {
-		return databaseNamesWithForestsOnOneHost;
+	public Set<String> getDatabasesWithForestsOnOneHost() {
+		return databasesWithForestsOnOneHost;
+	}
+
+	public void setDatabasesWithForestsOnOneHost(Set<String> databasesWithForestsOnOneHost) {
+		this.databasesWithForestsOnOneHost = databasesWithForestsOnOneHost;
+	}
+
+	public Map<String, Set<String>> getDatabaseHosts() {
+		return databaseHosts;
 	}
 
-	public void setDatabaseNamesWithForestsOnOneHost(Set<String> databaseNamesWithForestsOnOneHost) {
-		this.databaseNamesWithForestsOnOneHost = databaseNamesWithForestsOnOneHost;
+	public void setDatabaseHosts(Map<String, Set<String>> databaseHosts) {
+		this.databaseHosts = databaseHosts;
 	}
 }

src/main/java/com/marklogic/appdeployer/DefaultAppConfigFactory.java

@@ -276,15 +276,32 @@ public AppConfig newAppConfig() {
 			c.setDatabaseNamesAndReplicaCounts(prop);
 		}
 
-		prop = getProperty("mlDatabaseNamesWithForestsOnOneHost");
+		prop = getProperty("mlDatabasesWithForestsOnOneHost");
 		if (prop != null) {
 			logger.info("Databases that will have their forest(s) created on a single host: " + prop);
 			String[] names = prop.split(",");
 			Set<String> set = new HashSet<>();
 			for (String name : names) {
 				set.add(name);
 			}
-			c.setDatabaseNamesWithForestsOnOneHost(set);
+			c.setDatabasesWithForestsOnOneHost(set);
+		}
+
+		prop = getProperty("mlDatabaseHosts");
+		if (prop != null) {
+			logger.info("Databases and the hosts that their forests will be created on: " + prop);
+			String[] tokens = prop.split(",");
+			Map<String, Set<String>> map = new HashMap<>();
+			for (int i = 0; i < tokens.length; i += 2) {
+				String dbName = tokens[i];
+				String[] hostNames = tokens[i + 1].split("\\|");
+				Set<String> names = new HashSet<>();
+				for (String name : hostNames) {
+					names.add(name);
+				}
+				map.put(dbName, names);
+			}
+			c.setDatabaseHosts(map);
 		}
 
 		prop = getProperty("mlForestDataDirectory");

src/main/java/com/marklogic/appdeployer/command/databases/DeployDatabaseCommand.java

@@ -16,7 +16,6 @@
 import java.util.ArrayList;
 import java.util.List;
 import java.util.Map;
-import java.util.Set;
 
 /**
  * Can be used for creating any kind of database with any sorts of forests. Specifying a config file for the database or
@@ -284,9 +283,6 @@ protected boolean customForestsExist(CommandContext context, String dbName) {
     /**
      * Allows for how an instance of DeployForestsCommand is built to be overridden by a subclass.
      *
-     * Starting in 3.2.0, the "createForestsOnEachHost" property is overridden if the database name is in the
-     * databaseNamesWithForestsOnOneHost property in AppConfig.
-     *
      * @param dbPayload
      *            Needed so we can look up forest counts based on the database name
      * @param receipt
@@ -298,17 +294,7 @@ protected DeployForestsCommand buildDeployForestsCommand(String dbPayload, SaveR
     	final String databaseName = receipt.getResourceId();
 
         DeployForestsCommand c = new DeployForestsCommand();
-
-	    Set<String> names = context.getAppConfig().getDatabaseNamesWithForestsOnOneHost();
-	    if (names != null && names.contains(databaseName)) {
-	    	if (logger.isInfoEnabled()) {
-	    		logger.info("Will only create forest(s) on one host for database: " + databaseName);
-		    }
-	    	c.setCreateForestsOnEachHost(false);
-	    } else {
-		    c.setCreateForestsOnEachHost(createForestsOnEachHost);
-	    }
-
+        c.setCreateForestsOnEachHost(createForestsOnEachHost);
         c.setForestsPerHost(determineForestCountPerHost(dbPayload, context));
         c.setForestFilename(forestFilename);
         c.setDatabaseName(databaseName);

src/main/java/com/marklogic/appdeployer/command/forests/DeployForestsCommand.java

@@ -12,6 +12,8 @@
 import java.io.File;
 import java.util.ArrayList;
 import java.util.List;
+import java.util.Map;
+import java.util.Set;
 
 /**
  * This command is for a simple use case where all the forests created for a database have the same structure,
@@ -76,12 +78,7 @@ protected void createForests(String originalPayload, CommandContext context) {
 
         // Find out which hosts to create forests on
         List<String> hostNames = new HostManager(context.getManageClient()).getHostNames();
-        if (!createForestsOnEachHost) {
-            String first = hostNames.get(0);
-            logger.info(format("Only creating forests on the first host: " + first));
-            hostNames = new ArrayList<>();
-            hostNames.add(first);
-        }
+        hostNames = determineHostNamesForForest(context, hostNames);
 
         // Find out how many forests exist already
         int countOfExistingForests = new DatabaseManager(context.getManageClient()).getPrimaryForestIds(getForestDatabaseName(appConfig)).size();
@@ -104,6 +101,42 @@ protected void createForests(String originalPayload, CommandContext context) {
         }
     }
 
+	/**
+	 * @param context
+	 * @param hostNames
+	 * @return
+	 */
+	protected List<String> determineHostNamesForForest(CommandContext context, List<String> hostNames) {
+	    Set<String> databaseNames = context.getAppConfig().getDatabasesWithForestsOnOneHost();
+	    boolean onlyOnOneHost = databaseNames != null && databaseNames.contains(this.databaseName);
+
+	    if (!createForestsOnEachHost || onlyOnOneHost) {
+		    String first = hostNames.get(0);
+		    logger.info(format("Only creating forests on the first host: " + first));
+		    hostNames = new ArrayList<>();
+		    hostNames.add(first);
+		    return hostNames;
+	    }
+
+	    Map<String, Set<String>> databaseHosts = context.getAppConfig().getDatabaseHosts();
+	    if (databaseHosts != null) {
+	    	Set<String> selectedHostNames = databaseHosts.get(this.databaseName);
+	    	if (selectedHostNames != null) {
+	    		List<String> newHostNames = new ArrayList<>();
+	    		for (String name : selectedHostNames) {
+	    			if (hostNames.contains(name)) {
+	    				newHostNames.add(name);
+				    } else {
+	    				logger.warn(format("Host '%s' for database '%s' is not recognized, ignoring", name, this.databaseName));
+				    }
+			    }
+			    return newHostNames;
+		    }
+	    }
+
+	    return hostNames;
+    }
+
     protected String getForestName(AppConfig appConfig, int forestNumber) {
         return databaseName + "-" + forestNumber;
     }

src/test/java/com/marklogic/appdeployer/DefaultAppConfigFactoryTest.java

@@ -140,7 +140,8 @@ public void allProperties() {
 	    p.setProperty("mlResourceFilenamesToIncludeRegex", "qa-.*");
 
 	    p.setProperty("mlDatabaseNamesAndReplicaCounts", "Documents,1,Security,2");
-	    p.setProperty("mlDatabaseNamesWithForestsOnOneHost", "Documents,Security");
+	    p.setProperty("mlDatabasesWithForestsOnOneHost", "Documents,Security");
+	    p.setProperty("mlDatabaseHosts", "Documents,host1|host2|host3,Security,host1|host2");
 
 	    p.setProperty("mlForestDataDirectory", "/data/path");
 	    p.setProperty("mlForestFastDataDirectory", "/fast/path");
@@ -241,11 +242,21 @@ public void allProperties() {
 
 	    assertEquals("Documents,1,Security,2", config.getDatabaseNamesAndReplicaCounts());
 
-	    Set<String> set = config.getDatabaseNamesWithForestsOnOneHost();
+	    Set<String> set = config.getDatabasesWithForestsOnOneHost();
 	    assertEquals(2, set.size());
 	    assertTrue(set.contains("Documents"));
 	    assertTrue(set.contains("Security"));
 
+	    Map<String, Set<String>> databaseHosts = config.getDatabaseHosts();
+	    assertEquals(2, databaseHosts.size());
+	    assertEquals(3, databaseHosts.get("Documents").size());
+	    assertTrue(databaseHosts.get("Documents").contains("host1"));
+	    assertTrue(databaseHosts.get("Documents").contains("host2"));
+	    assertTrue(databaseHosts.get("Documents").contains("host3"));
+	    assertEquals(2, databaseHosts.get("Security").size());
+	    assertTrue(databaseHosts.get("Security").contains("host1"));
+	    assertTrue(databaseHosts.get("Security").contains("host2"));
+
 	    assertEquals("/data/path", config.getForestDataDirectory());
 	    assertEquals("/fast/path", config.getForestFastDataDirectory());
 	    assertEquals("/large/path", config.getForestLargeDataDirectory());

src/test/java/com/marklogic/appdeployer/command/databases/CreateForestsOnOneHostTest.java

@@ -25,7 +25,7 @@ public static void main(String[] args) {
 		AppConfig appConfig = new AppConfig();
 		Set<String> names = new HashSet<>();
 		names.add("testdb");
-		appConfig.setDatabaseNamesWithForestsOnOneHost(names);
+		appConfig.setDatabasesWithForestsOnOneHost(names);
 		CommandContext context = new CommandContext(appConfig, manageClient, null);
 
 		DeployDatabaseCommand ddc = new DeployDatabaseCommand();

src/test/java/com/marklogic/appdeployer/command/forests/CreateForestsOnOneHostDebug.java

@@ -0,0 +1,46 @@
+package com.marklogic.appdeployer.command.forests;
+
+import com.marklogic.appdeployer.AppConfig;
+import com.marklogic.appdeployer.command.CommandContext;
+import org.junit.Assert;
+import org.junit.Test;
+
+import java.util.ArrayList;
+import java.util.HashSet;
+import java.util.List;
+import java.util.Set;
+
+public class CreateForestsOnOneHostTest extends Assert {
+
+	@Test
+	public void test() {
+		AppConfig appConfig = new AppConfig();
+		CommandContext context = new CommandContext(appConfig, null, null);
+
+		DeployForestsCommand command = new DeployForestsCommand();
+		command.setDatabaseName("test-db");
+
+		List<String> fakeHostNames = new ArrayList<>();
+		fakeHostNames.add("host1");
+		fakeHostNames.add("host2");
+		fakeHostNames.add("host3");
+
+		List<String> hostNames = command.determineHostNamesForForest(context, fakeHostNames);
+		assertEquals(3, hostNames.size());
+
+		command.setCreateForestsOnEachHost(false);
+		hostNames = command.determineHostNamesForForest(context, fakeHostNames);
+		assertEquals(1, hostNames.size());
+		assertEquals("host1", hostNames.get(0));
+
+		command.setCreateForestsOnEachHost(true);
+		hostNames = command.determineHostNamesForForest(context, fakeHostNames);
+		assertEquals(3, hostNames.size());
+
+		Set<String> names = new HashSet<>();
+		names.add("test-db");
+		appConfig.setDatabasesWithForestsOnOneHost(names);
+		hostNames = command.determineHostNamesForForest(context, fakeHostNames);
+		assertEquals(1, hostNames.size());
+	}
+}

src/test/java/com/marklogic/appdeployer/command/forests/CreateForestsOnOneHostTest.java

@@ -0,0 +1,52 @@
+package com.marklogic.appdeployer.command.forests;
+
+import com.marklogic.appdeployer.AppConfig;
+import com.marklogic.appdeployer.DefaultAppConfigFactory;
+import com.marklogic.appdeployer.command.CommandContext;
+import com.marklogic.mgmt.util.SimplePropertySource;
+import org.junit.Assert;
+import org.junit.Test;
+import org.springframework.core.env.PropertiesPropertySource;
+
+import java.util.*;
+
+public class CreateForestsOnSpecificHostsTest extends Assert {
+
+	@Test
+	public void test() {
+		AppConfig appConfig = new AppConfig();
+		CommandContext context = new CommandContext(appConfig, null, null);
+
+		DeployForestsCommand command = new DeployForestsCommand();
+		command.setDatabaseName("test-db");
+
+		List<String> fakeHostNames = new ArrayList<>();
+		fakeHostNames.add("host1");
+		fakeHostNames.add("host2");
+		fakeHostNames.add("host3");
+
+		// Verify we get all 3 hosts back when nothing special is configured
+		List<String> hostNames = command.determineHostNamesForForest(context, fakeHostNames);
+		assertEquals(3, hostNames.size());
+
+		// Select 2 of the 3 hosts for test-db
+		Properties props = new Properties();
+		props.setProperty("mlDatabaseHosts", "test-db,host1|host2");
+		DefaultAppConfigFactory factory = new DefaultAppConfigFactory(new SimplePropertySource(props));
+		appConfig = factory.newAppConfig();
+		context = new CommandContext(appConfig, null, null);
+
+		hostNames = command.determineHostNamesForForest(context, fakeHostNames);
+		assertEquals(2, hostNames.size());
+		assertTrue(hostNames.contains("host1"));
+		assertTrue(hostNames.contains("host2"));
+
+		// Select 1 of the 3 hosts, and include a bad host; the bad one should be ignored
+		props.setProperty("mlDatabaseHosts", "some-other-db,host2,test-db,bad-host|host2");
+		appConfig = factory.newAppConfig();
+		context = new CommandContext(appConfig, null, null);
+		hostNames = command.determineHostNamesForForest(context, fakeHostNames);
+		assertEquals(1, hostNames.size());
+		assertTrue(hostNames.contains("host2"));
+	}
+}