#431 Replicas now work with mlDatabasesWithForestsOnOneHost

HostCalculator now calculates two lists - one for primary forests, one for replica forests.

Author: rjrudin

src/main/java/com/marklogic/appdeployer/AppConfig.java

@@ -1151,6 +1151,13 @@ public boolean isDatabaseWithForestsOnOneHost(String databaseName) {
 		return databasesWithForestsOnOneHost.contains(databaseName);
 	}
 
+	public void addDatabaseWithForestsOnOneHost(String databaseName) {
+		if (databasesWithForestsOnOneHost == null) {
+			databasesWithForestsOnOneHost = new HashSet<>();
+		}
+		databasesWithForestsOnOneHost.add(databaseName);
+	}
+
 	public Set<String> getDatabasesWithForestsOnOneHost() {
 		return databasesWithForestsOnOneHost;
 	}

src/main/java/com/marklogic/appdeployer/command/forests/DefaultHostCalculator.java

@@ -2,6 +2,7 @@
 
 import com.marklogic.appdeployer.command.CommandContext;
 import com.marklogic.client.ext.helper.LoggingObject;
+import com.marklogic.mgmt.api.forest.Forest;
 import com.marklogic.mgmt.resource.hosts.HostNameProvider;
 
 import java.util.ArrayList;
@@ -17,21 +18,44 @@ public DefaultHostCalculator(HostNameProvider hostNameProvider) {
 	}
 
 	@Override
-	public List<String> calculateHostNames(String databaseName, CommandContext context) {
-		List<String> hostNamesFromDatabaseGroups = determineHostsNamesBasedOnDatabaseGroups(databaseName, context);
-		if (hostNamesFromDatabaseGroups != null) {
-			if (hostNamesFromDatabaseGroups.size() > 1 && context.getAppConfig().isDatabaseWithForestsOnOneHost(databaseName)) {
-				return hostNamesFromDatabaseGroups.subList(0, 1);
+	public ForestHostNames calculateHostNames(String databaseName, CommandContext context, List<Forest> existingPrimaryForests) {
+		final List<String> candidateHostNames = getCandidateHostNames(databaseName, context);
+		if (candidateHostNames.isEmpty()) {
+			throw new RuntimeException("Unable to determine host names for forests for database: " + databaseName + "; please check the " +
+				"properties you've set for creating forests for this database to ensure that forests can be created on at " +
+				"least one host in your cluster");
+		}
+
+		final List<String> primaryForestHostNames = new ArrayList<>();
+		final List<String> replicaForestHostNames = new ArrayList<>();
+
+		if (context.getAppConfig().isDatabaseWithForestsOnOneHost(databaseName)) {
+			if (existingPrimaryForests.size() > 0) {
+				primaryForestHostNames.add(existingPrimaryForests.get(0).getHost());
+				replicaForestHostNames.addAll(candidateHostNames);
+			} else {
+				primaryForestHostNames.add(candidateHostNames.get(0));
+				replicaForestHostNames.addAll(candidateHostNames);
 			}
-			return hostNamesFromDatabaseGroups;
+		} else {
+			primaryForestHostNames.addAll(candidateHostNames);
+			replicaForestHostNames.addAll(candidateHostNames);
 		}
 
+		return new ForestHostNames(primaryForestHostNames, replicaForestHostNames);
+	}
+
+	protected List<String> getCandidateHostNames(String databaseName, CommandContext context) {
 		if (logger.isInfoEnabled()) {
 			logger.info("Finding eligible hosts for forests for database: " + databaseName);
 		}
 
-		List<String> hostNames = hostNameProvider.getHostNames();
+		List<String> hostNamesFromDatabaseGroups = determineHostNamesBasedOnDatabaseGroups(databaseName, context);
+		if (hostNamesFromDatabaseGroups != null) {
+			return hostNamesFromDatabaseGroups;
+		}
 
+		List<String> hostNames = hostNameProvider.getHostNames();
 		List<String> hostNamesFromDatabaseHosts = determineHostNamesBasedOnDatabaseHosts(databaseName, context, hostNames);
 		if (hostNamesFromDatabaseHosts != null) {
 			return hostNamesFromDatabaseHosts;
@@ -44,7 +68,7 @@ public List<String> calculateHostNames(String databaseName, CommandContext conte
 	 * @param context
 	 * @return
 	 */
-	protected List<String> determineHostsNamesBasedOnDatabaseGroups(String databaseName, CommandContext context) {
+	protected List<String> determineHostNamesBasedOnDatabaseGroups(String databaseName, CommandContext context) {
 		Map<String, List<String>> databaseGroups = context.getAppConfig().getDatabaseGroups();
 		if (databaseGroups != null) {
 			List<String> selectedGroupNames = databaseGroups.get(databaseName);

src/main/java/com/marklogic/appdeployer/command/forests/DeployForestsCommand.java

@@ -19,7 +19,6 @@
 import java.util.ArrayList;
 import java.util.List;
 import java.util.Map;
-import java.util.Set;
 
 /**
  * This command constructs forests based on properties found in the AppConfig object associated with the incoming
@@ -197,33 +196,17 @@ protected String buildForestTemplate(CommandContext context, ForestManager fores
 	 * configured to only have forests on one host, or when the deprecated setCreateForestsOnOneHost method is used.
 	 */
 	protected ForestHostNames determineHostNamesForForest(CommandContext context, List<Forest> existingPrimaryForests) {
-		Set<String> databaseNames = context.getAppConfig().getDatabasesWithForestsOnOneHost();
-		boolean onlyOnOneHost = databaseNames != null && databaseNames.contains(this.databaseName);
-
 		if (hostCalculator == null) {
 			hostCalculator = new DefaultHostCalculator(new DefaultHostNameProvider(context.getManageClient()));
 		}
 
-		// "candidate" = hosts that can be used for primary and replica forests
-		List<String> candidateHostNames = hostCalculator.calculateHostNames(this.databaseName, context);
-		List<String> primaryForestHostNames = new ArrayList<>();
-
-		if (!createForestsOnEachHost || onlyOnOneHost) {
-			// If the database should only have forests on one host and such forests already exist, then use the host
-			// that the forests exist on
-			if (!existingPrimaryForests.isEmpty()) {
-				primaryForestHostNames.add(existingPrimaryForests.get(0).getHost());
-			}
-			else {
-				String first = candidateHostNames.get(0);
-				logger.info(format("Only creating forests on the first host: " + first));
-				primaryForestHostNames.add(first);
-			}
-		} else {
-			primaryForestHostNames.addAll(candidateHostNames);
+		// If this deprecated feature is used, then configure the AppConfig object so that the hostCalculator can be
+		// aware of it.
+		if (!createForestsOnEachHost) {
+			context.getAppConfig().addDatabaseWithForestsOnOneHost(this.databaseName);
 		}
 
-		return new ForestHostNames(primaryForestHostNames, candidateHostNames);
+		return hostCalculator.calculateHostNames(this.databaseName, context, existingPrimaryForests);
 	}
 
 	public int getForestsPerHost() {
@@ -285,21 +268,3 @@ public void setForestBuilder(ForestBuilder forestBuilder) {
 		this.forestBuilder = forestBuilder;
 	}
 }
-
-class ForestHostNames {
-	private List<String> primaryForestHostNames;
-	private List<String> replicaForestHostNames;
-
-	public ForestHostNames(List<String> primaryForestHostNames, List<String> replicaForestHostNames) {
-		this.primaryForestHostNames = primaryForestHostNames;
-		this.replicaForestHostNames = replicaForestHostNames;
-	}
-
-	public List<String> getPrimaryForestHostNames() {
-		return primaryForestHostNames;
-	}
-
-	public List<String> getReplicaForestHostNames() {
-		return replicaForestHostNames;
-	}
-}

src/main/java/com/marklogic/appdeployer/command/forests/ForestHostNames.java

@@ -0,0 +1,22 @@
+package com.marklogic.appdeployer.command.forests;
+
+import java.util.List;
+
+public class ForestHostNames {
+
+	private List<String> primaryForestHostNames;
+	private List<String> replicaForestHostNames;
+
+	public ForestHostNames(List<String> primaryForestHostNames, List<String> replicaForestHostNames) {
+		this.primaryForestHostNames = primaryForestHostNames;
+		this.replicaForestHostNames = replicaForestHostNames;
+	}
+
+	public List<String> getPrimaryForestHostNames() {
+		return primaryForestHostNames;
+	}
+
+	public List<String> getReplicaForestHostNames() {
+		return replicaForestHostNames;
+	}
+}

src/main/java/com/marklogic/appdeployer/command/forests/HostCalculator.java

@@ -1,11 +1,25 @@
 package com.marklogic.appdeployer.command.forests;
 
 import com.marklogic.appdeployer.command.CommandContext;
+import com.marklogic.mgmt.api.forest.Forest;
 
 import java.util.List;
 
 public interface HostCalculator {
 
-	List<String> calculateHostNames(String databaseName, CommandContext context);
+	/**
+	 * Calculates which hosts can be used for primary and replica forests. The results are affected by the AppConfig
+	 * properties that configure which groups and hosts are allowed to have forests for a particular database.
+	 *
+	 * In addition, if the database is configured to only have forests on one host, then the list of primary forests
+	 * will have a single host. However, the list of replica forests will still have all candidate hosts so that
+	 * replicas can still be created.
+	 *
+	 * @param databaseName
+	 * @param context
+	 * @param existingPrimaryForests
+	 * @return
+	 */
+	ForestHostNames calculateHostNames(String databaseName, CommandContext context, List<Forest> existingPrimaryForests);
 
 }

src/test/java/com/marklogic/appdeployer/command/forests/BuildForestReplicaTest.java

@@ -5,7 +5,6 @@
 import com.marklogic.mgmt.api.forest.Forest;
 import com.marklogic.mgmt.api.forest.ForestReplica;
 import com.marklogic.mgmt.util.SimplePropertySource;
-import static org.junit.jupiter.api.Assertions.*;
 import org.junit.jupiter.api.Test;
 
 import java.util.Arrays;
@@ -15,7 +14,10 @@
 import java.util.concurrent.atomic.AtomicInteger;
 import java.util.stream.Stream;
 
-public class BuildForestReplicaTest  {
+import static org.junit.jupiter.api.Assertions.assertEquals;
+import static org.junit.jupiter.api.Assertions.assertTrue;
+
+public class BuildForestReplicaTest {
 
 	private ForestBuilder builder = new ForestBuilder();
 
@@ -120,6 +122,23 @@ public void primaryForestsOnOneHost() {
 		});
 	}
 
+	@Test
+	void databaseHasPrimaryForestsOnOneHost() {
+		AppConfig config = newAppConfig("mlDatabasesWithForestsOnOneHost", "my-database");
+
+		Forest forest = builder.buildForests(new ForestPlan("my-database", "host1", "host2", "host3").withReplicaCount(2), config).get(0);
+		assertEquals("host1", forest.getHost(), "host1 should be selected since it's the first host");
+		assertEquals(2, forest.getForestReplica().size(), "Expecting 2 replicas since replica count is 2");
+
+		ForestReplica replica = forest.getForestReplica().get(0);
+		assertEquals("my-database-1-replica-1", replica.getReplicaName());
+		assertEquals("host2", replica.getHost());
+
+		replica = forest.getForestReplica().get(1);
+		assertEquals("my-database-1-replica-2", replica.getReplicaName());
+		assertEquals("host3", replica.getHost());
+	}
+
 	@Test
 	public void customNamingStrategyWithDistributedStrategy() {
 		AppConfig appConfig = newAppConfig("mlForestsPerHost", "my-database,2");

src/test/java/com/marklogic/appdeployer/command/forests/CalculateForestHostsTest.java

@@ -7,6 +7,7 @@
 import static org.junit.jupiter.api.Assertions.*;
 import org.junit.jupiter.api.Test;
 
+import java.util.ArrayList;
 import java.util.List;
 import java.util.Properties;
 
@@ -25,7 +26,7 @@ public void test() {
 		DefaultHostCalculator hostCalculator = new DefaultHostCalculator(hostNameProvider);
 
 		// Verify we get all 5 hosts back when nothing special is configured
-		List<String> hostNames = hostCalculator.calculateHostNames("test-db", context);
+		List<String> hostNames = hostCalculator.calculateHostNames("test-db", context, new ArrayList<>()).getPrimaryForestHostNames();
 		assertEquals(5, hostNames.size());
 
 		// Select 2 of the 3 hosts for test-db
@@ -35,7 +36,7 @@ public void test() {
 		appConfig = factory.newAppConfig();
 		context = new CommandContext(appConfig, null, null);
 
-		hostNames = hostCalculator.calculateHostNames("test-db", context);
+		hostNames = hostCalculator.calculateHostNames("test-db", context, new ArrayList<>()).getPrimaryForestHostNames();
 		assertEquals(3, hostNames.size());
 		assertTrue(hostNames.contains("name1"));
 		assertTrue(hostNames.contains("name2"));
@@ -44,7 +45,7 @@ public void test() {
 		props.setProperty("mlDatabaseGroups", "test-db,group3");
 		appConfig = factory.newAppConfig();
 		context = new CommandContext(appConfig, null, null);
-		hostNames = hostCalculator.calculateHostNames("test-db", context);
+		hostNames = hostCalculator.calculateHostNames("test-db", context, new ArrayList<>()).getPrimaryForestHostNames();
 		assertEquals(2, hostNames.size());
 		assertTrue(hostNames.contains("name4"));
 		assertTrue(hostNames.contains("name5"));
@@ -63,7 +64,7 @@ public void databaseWithForestOnOneHost() {
 		props.setProperty("mlDatabasesWithForestsOnOneHost", "test-db");
 
 		CommandContext context = new CommandContext(new DefaultAppConfigFactory(new SimplePropertySource(props)).newAppConfig(), null, null);
-		List<String> hostNames = hostCalculator.calculateHostNames("test-db", context);
+		List<String> hostNames = hostCalculator.calculateHostNames("test-db", context, new ArrayList<>()).getPrimaryForestHostNames();
 		assertEquals(1, hostNames.size(), "The database should only have a forest on the first host");
 		assertEquals("name2", hostNames.get(0));
 	}

src/test/java/com/marklogic/appdeployer/command/forests/CreateForestsOnOneHostTest.java

@@ -7,7 +7,9 @@
 import com.marklogic.mgmt.util.SimplePropertySource;
 import org.junit.jupiter.api.Test;
 
-import java.util.*;
+import java.util.ArrayList;
+import java.util.Arrays;
+import java.util.List;
 
 import static org.junit.jupiter.api.Assertions.assertEquals;
 
@@ -25,25 +27,36 @@ public void test() {
 
 		ForestHostNames hostNames = command.determineHostNamesForForest(context, new ArrayList<>());
 		assertEquals(3, hostNames.getPrimaryForestHostNames().size());
+		assertEquals(3, hostNames.getReplicaForestHostNames().size());
 
-		command.setCreateForestsOnEachHost(false);
+		appConfig.addDatabaseWithForestsOnOneHost("test-db");
 		hostNames = command.determineHostNamesForForest(context, new ArrayList<>());
 		assertEquals(1, hostNames.getPrimaryForestHostNames().size());
-		assertEquals("host1", hostNames.getPrimaryForestHostNames().get(0));
-		assertEquals(3, hostNames.getReplicaForestHostNames().size(),
-			"When forests aren't created on each host, all hosts should still be available for replica forests, " +
-				"with the expectation that the primary forest host will still not be used");
+		assertEquals(3, hostNames.getReplicaForestHostNames().size());
+	}
+
+	@Test
+	void deprecatedWayOfCreatingForestsOnOneHost() {
+		AppConfig appConfig = new AppConfig();
+		CommandContext context = new CommandContext(appConfig, null, null);
+
+		DeployForestsCommand command = new DeployForestsCommand("test-db");
+
+		HostCalculator hostCalculator = new DefaultHostCalculator(new TestHostNameProvider("host1", "host2", "host3"));
+		command.setHostCalculator(hostCalculator);
 
 		command.setCreateForestsOnEachHost(true);
-		hostNames = command.determineHostNamesForForest(context, new ArrayList<>());
+		ForestHostNames hostNames = command.determineHostNamesForForest(context, new ArrayList<>());
 		assertEquals(3, hostNames.getPrimaryForestHostNames().size());
+		assertEquals(3, hostNames.getReplicaForestHostNames().size());
 
-		Set<String> names = new HashSet<>();
-		names.add("test-db");
-		appConfig.setDatabasesWithForestsOnOneHost(names);
+		command.setCreateForestsOnEachHost(false);
 		hostNames = command.determineHostNamesForForest(context, new ArrayList<>());
 		assertEquals(1, hostNames.getPrimaryForestHostNames().size());
-		assertEquals(3, hostNames.getReplicaForestHostNames().size());
+		assertEquals("host1", hostNames.getPrimaryForestHostNames().get(0));
+		assertEquals(3, hostNames.getReplicaForestHostNames().size(),
+			"When forests aren't created on each host, all hosts should still be available for replica forests, " +
+				"with the expectation that the primary forest host will still not be used");
 	}
 
 	/**