DEVEXP-564 Using CMA to retrieve forest details

Allows for deploying databases much more quickly by getting all forest details in a single request as opposed to one request per forest via /manage/v2. Any issue results in /manage/v2 being used instead. 

A simple test was added for this, but the method is being tested by dozens of existing tests - i.e. any test that involves deploying a database is hitting this. There's no actual change in functionality too - it's purely a performance optimization.

Author: rjrudin

src/main/java/com/marklogic/appdeployer/command/databases/DeployOtherDatabasesCommand.java

@@ -34,13 +34,22 @@
 import com.marklogic.mgmt.mapper.DefaultResourceMapper;
 import com.marklogic.mgmt.mapper.ResourceMapper;
 import com.marklogic.mgmt.resource.databases.DatabaseManager;
+import com.marklogic.mgmt.resource.forests.ForestManager;
 import com.marklogic.mgmt.util.ObjectMapperFactory;
 import com.marklogic.rest.util.JsonNodeUtil;
 
 import java.io.File;
 import java.io.FilenameFilter;
 import java.io.IOException;
-import java.util.*;
+import java.util.ArrayList;
+import java.util.Collections;
+import java.util.HashMap;
+import java.util.HashSet;
+import java.util.LinkedHashMap;
+import java.util.List;
+import java.util.Map;
+import java.util.Set;
+import java.util.stream.Collectors;
 
 /**
  * As of release 3.14.0, this now handles all databases, not just "databases other than the default content database".
@@ -161,7 +170,7 @@ public void undo(CommandContext context) {
 	/**
 	 * If no database files are found, may still need to delete the content database in case no file exists for it.
 	 * That's because the command for creating a REST API server will not delete the content database by default.
-	 *
+	 * <p>
 	 * Per ticket #404, this will now do a check to see if the default content database filename is ignored. If so,
 	 * and there are no database files found, then the content database will not be deleted.
 	 *
@@ -392,6 +401,8 @@ protected void buildDeployDatabaseCommands(CommandContext context, List<Database
 	 * @param databasePlans
 	 */
 	protected void deployDatabasesAndForestsViaCma(CommandContext context, List<DatabasePlan> databasePlans) {
+		addForestMapToCommandContext(context, databasePlans);
+
 		Configuration dbConfig = new Configuration();
 		// Forests must be included in a separate configuration object
 		Configuration forestConfig = new Configuration();
@@ -415,6 +426,28 @@ protected void deployDatabasesAndForestsViaCma(CommandContext context, List<Data
 		});
 	}
 
+	/**
+	 * Added to greatly speed up performance when getting details about all the existing primary forests for each
+	 * database referenced by a plan. In the event that anything fails, the map won't be added to the command context
+	 * and any code expecting to use the map will just have to fall back to use /manage/v2.
+	 *
+	 * @param context
+	 * @param databasePlans
+	 */
+	private void addForestMapToCommandContext(CommandContext context, List<DatabasePlan> databasePlans) {
+		try {
+			Set<String> dbNames = databasePlans.stream().map(plan -> plan.getDatabaseName()).collect(Collectors.toSet());
+			logger.info("Retrieving all forest details via CMA");
+			long start = System.currentTimeMillis();
+			Map<String, List<Forest>> forestMap = new ForestManager(context.getManageClient()).getPrimaryForestsForDatabases(dbNames.toArray(new String[]{}));
+			logger.info("Finished retrieving all forests details via CMA; duration: " + (System.currentTimeMillis() - start));
+			context.getContextMap().put("ml-app-deployer-forestMap", forestMap);
+		} catch (Exception ex) {
+			logger.warn("Unable to retrieve all forest details, cause: " + ex.getMessage() + "; will fall back to " +
+				"using /manage/v2 when needed for getting details for a forest.");
+		}
+	}
+
 	/**
 	 * Each DatabasePlan is expected to have constructed a DeployForestCommand, but not executed it. Each
 	 * DeployForestCommand can then be used to build a list of forests. All of those forests can be combined into a

src/main/java/com/marklogic/appdeployer/command/forests/DeployForestsCommand.java

@@ -128,12 +128,25 @@ protected void createForestsViaForestEndpoint(CommandContext context, List<Fores
 	 */
 	public List<Forest> buildForests(CommandContext context, boolean includeReplicas) {
 		// Need to know what primary forests exist already in case more need to be added, or a new host has been added
-		List<Forest> existingPrimaryForests = getExistingPrimaryForests(context, this.databaseName);
+		List<Forest> existingPrimaryForests = null;
+
+		// As of 4.5.3, if CMA is enabled, then the context should contain a map of all the forests for each database
+		// being deployed. If it's not there, then /manage/v2 will be used instead.
+		if (context.getAppConfig().getCmaConfig().isDeployForests()) {
+			Map<String, List<Forest>> map = (Map<String, List<Forest>>) context.getContextMap().get("ml-app-deployer-forestMap");
+			if (map != null && map.containsKey(this.databaseName)) {
+				existingPrimaryForests = map.get(this.databaseName);
+			}
+		}
+
+		if (existingPrimaryForests == null) {
+			existingPrimaryForests = getExistingPrimaryForests(context, this.databaseName);
+		}
+
 		return buildForests(context, includeReplicas, existingPrimaryForests);
 	}
 
 	/**
-	 *
 	 * @param context
 	 * @param includeReplicas
 	 * @param existingPrimaryForests
@@ -163,6 +176,14 @@ protected List<Forest> buildForests(CommandContext context, boolean includeRepli
 		return forestBuilder.buildForests(forestPlan, context.getAppConfig());
 	}
 
+	/**
+	 * @param context
+	 * @param databaseName
+	 * @return
+	 * @deprecated in 4.5.3, as getting forest details one at a time can be very slow for applications with a large
+	 * number of forests.
+	 */
+	@Deprecated
 	protected List<Forest> getExistingPrimaryForests(CommandContext context, String databaseName) {
 		List<String> forestIds = new DatabaseManager(context.getManageClient()).getPrimaryForestIds(databaseName);
 		ForestManager forestMgr = new ForestManager(context.getManageClient());
@@ -259,6 +280,7 @@ public boolean isCreateForestsOnEachHost() {
 
 	/**
 	 * Use appConfig.setDatabasesWithForestsOnOneHost
+	 *
 	 * @param createForestsOnEachHost
 	 */
 	@Deprecated