#269 Can create forests via CMA

Author: rjrudin

src/main/java/com/marklogic/appdeployer/AppConfig.java

@@ -3,7 +3,6 @@
 import com.marklogic.appdeployer.command.forests.ForestNamingStrategy;
 import com.marklogic.client.DatabaseClient;
 import com.marklogic.client.DatabaseClientFactory;
-import com.marklogic.client.DatabaseClientFactory.Authentication;
 import com.marklogic.client.DatabaseClientFactory.SSLHostnameVerifier;
 import com.marklogic.client.ext.ConfiguredDatabaseClientFactory;
 import com.marklogic.client.ext.DatabaseClientConfig;
@@ -67,6 +66,11 @@ public class AppConfig {
     private boolean catchDeployExceptions = false;
     private boolean catchUndeployExceptions = false;
 
+    // Whether to optimize certain operations with the Configuration Management API (CMA) if it's available
+	// For version 3.8.0, defaulting this to false as it's a new feature.
+	// Will default it to true once we have more experience with CMA under our belts.
+    private boolean optimizeWithCma = false;
+
     // Used to construct DatabaseClient instances based on inputs defined in this class
     private ConfiguredDatabaseClientFactory configuredDatabaseClientFactory = new DefaultConfiguredDatabaseClientFactory();
 
@@ -1204,4 +1208,12 @@ public Map<String, ForestNamingStrategy> getForestNamingStrategies() {
 	public void setForestNamingStrategies(Map<String, ForestNamingStrategy> forestNamingStrategies) {
 		this.forestNamingStrategies = forestNamingStrategies;
 	}
+
+	public boolean isOptimizeWithCma() {
+		return optimizeWithCma;
+	}
+
+	public void setOptimizeWithCma(boolean optimizeWithCma) {
+		this.optimizeWithCma = optimizeWithCma;
+	}
 }

src/main/java/com/marklogic/appdeployer/DefaultAppConfigFactory.java

@@ -40,6 +40,11 @@ public void initialize() {
 			config.setCatchUndeployExceptions(Boolean.parseBoolean(prop));
 		});
 
+		propertyConsumerMap.put("mlOptimizeWithCma", (config, prop) -> {
+			logger.info("Optimize with the Configuration Management API (CMA): " + prop);
+			config.setOptimizeWithCma(Boolean.parseBoolean(prop));
+		});
+
 		/**
 		 * The application name is used as a prefix for default names for a variety of resources, such as REST API servers
 		 * and databases.

src/main/java/com/marklogic/appdeployer/command/AbstractCommand.java

@@ -3,6 +3,7 @@
 import com.marklogic.client.ext.helper.LoggingObject;
 import com.marklogic.mgmt.PayloadParser;
 import com.marklogic.mgmt.admin.AdminManager;
+import com.marklogic.mgmt.cma.ConfigurationManager;
 import com.marklogic.mgmt.resource.ResourceManager;
 import com.marklogic.mgmt.SaveReceipt;
 import org.springframework.http.HttpHeaders;
@@ -274,6 +275,13 @@ protected void logResourceDirectoryNotFound(File dir) {
 	    }
     }
 
+    protected boolean shouldOptimizeWithCma(CommandContext context) {
+    	if (context.getAppConfig().isOptimizeWithCma()) {
+		    return new ConfigurationManager(context.getManageClient()).endpointExists();
+	    }
+	    return false;
+    }
+
     public void setPayloadTokenReplacer(PayloadTokenReplacer payloadTokenReplacer) {
         this.payloadTokenReplacer = payloadTokenReplacer;
     }

src/main/java/com/marklogic/appdeployer/command/forests/DeployForestsCommand.java

@@ -5,10 +5,12 @@
 import com.marklogic.appdeployer.command.CommandContext;
 import com.marklogic.appdeployer.command.SortOrderConstants;
 import com.marklogic.mgmt.api.forest.Forest;
+import com.marklogic.mgmt.cma.ConfigurationManager;
 import com.marklogic.mgmt.resource.databases.DatabaseManager;
 import com.marklogic.mgmt.resource.forests.ForestManager;
 import com.marklogic.mgmt.resource.hosts.DefaultHostNameProvider;
 import com.marklogic.mgmt.resource.hosts.HostManager;
+import org.springframework.http.ResponseEntity;
 import org.springframework.util.StringUtils;
 
 import java.io.File;
@@ -51,6 +53,39 @@ public DeployForestsCommand(String databaseName) {
 	public void execute(CommandContext context) {
 		// Replicas are currently handled by ConfigureForestReplicasCommand
 		List<Forest> forests = buildForests(context, false);
+
+		if (shouldOptimizeWithCma(context) && !forests.isEmpty()) {
+			createForestsViaCma(context, forests);
+		} else {
+			if (logger.isInfoEnabled()) {
+				logger.info("Configuration Management API is not available at " + ConfigurationManager.PATH + ", so forests will be created one at a time via the forests endpoint");
+			}
+			createForestsViaForestEndpoint(context, forests);
+		}
+	}
+
+	protected void createForestsViaCma(CommandContext context, List<Forest> forests) {
+		StringBuilder sb = new StringBuilder("{\"config\":[{\"forest\":[");
+		for (int i = 0; i < forests.size(); i++) {
+			if (i > 0) {
+				sb.append(",");
+			}
+			sb.append(forests.get(i).getJson());
+		}
+		sb.append("]}]}");
+
+		// For version 3.8.0, logging the configuration package at the info level for better visibility and
+		// easier debugging of this new feature
+		if (logger.isInfoEnabled()) {
+			logger.info("Submitting configuration with forests: " + sb);
+		}
+		context.getManageClient().postJson("/manage/v3", sb.toString());
+		if (logger.isInfoEnabled()) {
+			logger.info("Successfully submitted configuration with forests");
+		}
+	}
+
+	protected void createForestsViaForestEndpoint(CommandContext context, List<Forest> forests) {
 		ForestManager forestManager = new ForestManager(context.getManageClient());
 		for (Forest f : forests) {
 			forestManager.save(f.getJson());

src/main/java/com/marklogic/mgmt/cma/ConfigurationManager.java

@@ -3,7 +3,9 @@
 import com.marklogic.mgmt.AbstractManager;
 import com.marklogic.mgmt.ManageClient;
 import com.marklogic.mgmt.SaveReceipt;
+import com.marklogic.rest.util.MgmtResponseErrorHandler;
 import org.springframework.http.ResponseEntity;
+import org.springframework.web.client.HttpClientErrorException;
 
 /**
  * This doesn't extend AbstractResourceManager because a configuration isn't really a resource, it's a collection of
@@ -15,12 +17,34 @@
  */
 public class ConfigurationManager extends AbstractManager {
 
+	public final static String PATH = "/manage/v3";
+
 	private ManageClient manageClient;
 
 	public ConfigurationManager(ManageClient manageClient) {
 		this.manageClient = manageClient;
 	}
 
+	/**
+	 * Returns true if the CMA endpoint exists. This temporarily disables logging in MgmtResponseErrorHandler so that
+	 * a client doesn't see the 404 error being logged, which could be mistakenly perceived as a real error.
+	 * @return
+	 */
+	public boolean endpointExists() {
+		try {
+			MgmtResponseErrorHandler.errorLoggingEnabled = false;
+			if (logger.isInfoEnabled()) {
+				logger.info("Checking to see if Configuration Management API is available at: " + PATH);
+			}
+			manageClient.postJson(PATH, "{}");
+			return true;
+		} catch (HttpClientErrorException ex) {
+			return false;
+		} finally {
+			MgmtResponseErrorHandler.errorLoggingEnabled = true;
+		}
+	}
+
 	public SaveReceipt save(String payload) {
 		String configurationName = payloadParser.getPayloadFieldValue(payload, "name", false);
 		if (configurationName == null) {
@@ -31,13 +55,13 @@ public SaveReceipt save(String payload) {
 			logger.info("Applying configuration " + configurationName);
 		}
 
-		final String path = "/manage/v3";
-		ResponseEntity<String> response = postPayload(manageClient, path, payload);
+		ResponseEntity<String> response = postPayload(manageClient, PATH, payload);
 
 		if (logger.isInfoEnabled()) {
 			logger.info("Applied configuration " + configurationName);
 		}
 
-		return new SaveReceipt(null, payload, path, response);
+		return new SaveReceipt(null, payload, PATH, response);
 	}
+
 }

src/main/java/com/marklogic/rest/util/MgmtResponseErrorHandler.java

@@ -12,24 +12,29 @@
 
 public class MgmtResponseErrorHandler extends DefaultResponseErrorHandler {
 
-    protected final Logger logger = LoggerFactory.getLogger(getClass());
+	protected final Logger logger = LoggerFactory.getLogger(getClass());
 
-    @Override
-    public void handleError(ClientHttpResponse response) throws IOException {
-        try {
-            super.handleError(response);
-        } catch (HttpClientErrorException | HttpServerErrorException ex) {
+	/**
+	 * Allows clients to disable error logging. Typical use case is for when error logging isn't useful and may
+	 * confuse someone looking at the logs into thinking there's an actual problem.
+	 */
+	public static boolean errorLoggingEnabled = true;
+
+	@Override
+	public void handleError(ClientHttpResponse response) throws IOException {
+		try {
+			super.handleError(response);
+		} catch (HttpClientErrorException | HttpServerErrorException ex) {
 			String message = "Logging HTTP response body to assist with debugging: " + ex.getResponseBodyAsString();
 			if (HttpStatus.SERVICE_UNAVAILABLE.equals(ex.getStatusCode())) {
 				if (logger.isDebugEnabled()) {
 					logger.debug(message);
 				}
+			} else if (logger.isErrorEnabled() && errorLoggingEnabled) {
+				logger.error(message);
 			}
-            else if (logger.isErrorEnabled()) {
-                logger.error(message);
-            }
-            throw ex;
-        }
-    }
+			throw ex;
+		}
+	}
 
 }

src/test/java/com/marklogic/appdeployer/DefaultAppConfigFactoryTest.java

@@ -107,6 +107,7 @@ public void allProperties() {
 
         p.setProperty("mlCatchDeployExceptions", "true");
 	    p.setProperty("mlCatchUndeployExceptions", "true");
+	    p.setProperty("mlOptimizeWithCma", "true");
 
 	    p.setProperty("mlHost", "prophost");
         p.setProperty("mlAppName", "propname");
@@ -201,6 +202,7 @@ public void allProperties() {
 
         assertTrue(config.isCatchDeployExceptions());
         assertTrue(config.isCatchUndeployExceptions());
+        assertTrue(config.isOptimizeWithCma());
 
         assertEquals("prophost", config.getHost());
         assertEquals("propname", config.getName());

src/test/java/com/marklogic/appdeployer/command/forests/DeployForestsViaCmaTest.java

@@ -0,0 +1,41 @@
+package com.marklogic.appdeployer.command.forests;
+
+import com.marklogic.appdeployer.AbstractAppDeployerTest;
+import com.marklogic.appdeployer.command.databases.DeployContentDatabasesCommand;
+import com.marklogic.mgmt.ManageClient;
+import com.marklogic.mgmt.cma.ConfigurationManager;
+import com.marklogic.mgmt.resource.clusters.ClusterManager;
+import com.marklogic.mgmt.resource.forests.ForestManager;
+import org.junit.Test;
+
+import java.io.File;
+
+public class DeployForestsViaCmaTest extends AbstractAppDeployerTest {
+
+	/**
+	 * We don't really know that the forests were created via CMA, as if the ML cluster isn't 9.0-5, the call to
+	 * /manage/v3 should fail silently and switch over to the forests endpoint for creating forests. The intent of this
+	 * test then is just to have a focused test on creating a handful of forests with inspection of logging done manually.
+	 */
+	@Test
+	public void test() {
+		//appConfig.setOptimizeWithCma(true);
+		appConfig.getFirstConfigDir().setBaseDir(new File("src/test/resources/sample-app/db-only-config"));
+
+		initializeAppDeployer(new DeployContentDatabasesCommand(6));
+
+		ForestManager mgr = new ForestManager(manageClient);
+
+		try {
+			deploySampleApp();
+			for (int i = 1; i <= 6; i++) {
+				assertTrue(mgr.exists(appConfig.getContentDatabaseName() + "-" + i));
+			}
+		} finally {
+			undeploySampleApp();
+			for (int i = 1; i <= 6; i++) {
+				assertFalse(mgr.exists(appConfig.getContentDatabaseName() + "-" + i));
+			}
+		}
+	}
+}