#346 Can now deploy databases via CMA

Doesn't yet support sub-databases though

Author: rjrudin

gradle.properties

@@ -1,6 +1,6 @@
 group=com.marklogic
 javadocsDir=../gh-pages-marklogic-java/javadocs
-version=3.14.0
+version=3.14.346
 mlJavaclientUtilVersion=3.12.0
 mlJunitVersion=3.1.0
 

src/main/java/com/marklogic/appdeployer/AppConfig.java

@@ -74,6 +74,7 @@ public class AppConfig {
     private boolean catchUndeployExceptions = false;
 
     private boolean deployAmpsWithCma = false;
+    private boolean deployDatabasesWithCma = false;
     private boolean deployForestsWithCma = false;
     private boolean deployPrivilegesWithCma = false;
     private boolean mergeResources = true;
@@ -1396,4 +1397,12 @@ public DatabaseClient.ConnectionType getAppServicesConnectionType() {
 	public void setAppServicesConnectionType(DatabaseClient.ConnectionType appServicesConnectionType) {
 		this.appServicesConnectionType = appServicesConnectionType;
 	}
+
+	public boolean isDeployDatabasesWithCma() {
+		return deployDatabasesWithCma;
+	}
+
+	public void setDeployDatabasesWithCma(boolean deployDatabasesWithCma) {
+		this.deployDatabasesWithCma = deployDatabasesWithCma;
+	}
 }

src/main/java/com/marklogic/appdeployer/DefaultAppConfigFactory.java

@@ -76,6 +76,11 @@ public void initialize() {
 			config.setDeployAmpsWithCma(Boolean.parseBoolean(prop));
 		});
 
+		propertyConsumerMap.put("mlDeployDatabasesWithCma", (config, prop) -> {
+			logger.info("Deploy databases and forests" + cmaMessage + prop);
+			config.setDeployDatabasesWithCma(Boolean.parseBoolean(prop));
+		});
+
 		propertyConsumerMap.put("mlDeployForestsWithCma", (config, prop) -> {
 			logger.info("Deploy forests" + cmaMessage + prop);
 			config.setDeployForestsWithCma(Boolean.parseBoolean(prop));

src/main/java/com/marklogic/appdeployer/command/AbstractCommand.java

@@ -157,7 +157,7 @@ protected String copyFileToString(File f, CommandContext context) {
 	 * @return
 	 */
 	protected SaveReceipt saveResource(ResourceManager mgr, CommandContext context, File resourceFile) {
-		String payload = readResourceFromFile(mgr, context, resourceFile);
+		String payload = readResourceFromFile(context, resourceFile);
 
 		if (payload != null && resourceMergingIsSupported(context)) {
 			try {
@@ -344,9 +344,9 @@ protected BiPredicate<ResourceReference, ResourceReference> getBiPredicateForMer
 	 * @param f
 	 * @return
 	 */
-	protected String readResourceFromFile(ResourceManager mgr, CommandContext context, File f) {
+	protected String readResourceFromFile(CommandContext context, File f) {
 		String payload = copyFileToString(f, context);
-		return adjustPayloadBeforeSavingResource(mgr, context, f, payload);
+		return adjustPayloadBeforeSavingResource(context, f, payload);
 	}
 
 	/**
@@ -386,13 +386,12 @@ protected void afterResourceSaved(ResourceManager mgr, CommandContext context, R
 	 * <p>
 	 * A subclass can return null from this method to indicate that the resource should not be saved.
 	 *
-	 * @param mgr
 	 * @param context
 	 * @param f
 	 * @param payload
 	 * @return
 	 */
-	protected String adjustPayloadBeforeSavingResource(ResourceManager mgr, CommandContext context, File f, String payload) {
+	protected String adjustPayloadBeforeSavingResource(CommandContext context, File f, String payload) {
 		String[] props = context.getAppConfig().getExcludeProperties();
 		if (props != null && props.length > 0) {
 			logger.info(format("Excluding properties %s from payload", Arrays.asList(props).toString()));

src/main/java/com/marklogic/appdeployer/command/AbstractResourceCommand.java

@@ -10,8 +10,10 @@
 import com.marklogic.mgmt.mapper.DefaultResourceMapper;
 import com.marklogic.mgmt.mapper.ResourceMapper;
 import com.marklogic.mgmt.resource.ResourceManager;
+import com.marklogic.mgmt.util.ObjectMapperFactory;
 
 import java.io.File;
+import java.io.IOException;
 import java.util.ArrayList;
 import java.util.List;
 
@@ -152,15 +154,21 @@ protected boolean useCmaForDeployingResources(CommandContext context) {
 	 * @param resourceDir
 	 */
 	protected void deployResourcesViaCma(CommandContext context, File resourceDir) {
-		ResourceMapper resourceMapper = new DefaultResourceMapper(new API(context.getManageClient()));
 		Configuration config = new Configuration();
 		for (File f : listFilesInDirectory(resourceDir, context)) {
 			if (logger.isInfoEnabled()) {
 				logger.info("Processing file: " + f.getAbsolutePath());
 			}
-			String payload = readResourceFromFile(null, context, f);
+			String payload = readResourceFromFile(context, f);
 			if (payload != null && payload.trim().length() > 0) {
-				((SupportsCmaCommand) this).addResourceToConfiguration(payload, resourceMapper, config);
+				payload = convertXmlPayloadToJsonIfNecessary(context, payload);
+				ObjectNode objectNode;
+				try {
+					objectNode = (ObjectNode)ObjectMapperFactory.getObjectMapper().readTree(payload);
+				} catch (IOException e) {
+					throw new RuntimeException("Unable to read JSON, cause: " + e.getMessage(), e);
+				}
+				((SupportsCmaCommand) this).addResourceToConfiguration(objectNode, config);
 			}
 		}
 		new Configurations(config).submit(context.getManageClient());
@@ -173,10 +181,8 @@ protected void deployResourcesViaCma(CommandContext context, File resourceDir) {
 	 */
 	protected void saveMergedResourcesViaCma(CommandContext context, List<ResourceReference> mergedReferences) {
 		Configuration config = new Configuration();
-		ResourceMapper resourceMapper = new DefaultResourceMapper(new API(context.getManageClient()));
 		for (ResourceReference reference : mergedReferences) {
-			String payload = reference.getObjectNode().toString();
-			((SupportsCmaCommand)this).addResourceToConfiguration(payload, resourceMapper, config);
+			((SupportsCmaCommand)this).addResourceToConfiguration(reference.getObjectNode(), config);
 		}
 		new Configurations(config).submit(context.getManageClient());
 	}

src/main/java/com/marklogic/appdeployer/command/SupportsCmaCommand.java

@@ -1,7 +1,7 @@
 package com.marklogic.appdeployer.command;
 
+import com.fasterxml.jackson.databind.node.ObjectNode;
 import com.marklogic.mgmt.api.configuration.Configuration;
-import com.marklogic.mgmt.mapper.ResourceMapper;
 
 /**
  * Commands that wish to optimize the deployment of resources in a single resource directory should implement this
@@ -13,11 +13,10 @@ public interface SupportsCmaCommand {
 	boolean cmaShouldBeUsed(CommandContext context);
 
 	/**
-	 * @param payload        the contents of the resource file, with all tokens replaced
-	 * @param resourceMapper the implementer is likely to use this to convert the payload into a subclass of Resource
-	 *                       so that it doesn't matter if the payload is JSON or XML
-	 * @param configuration  the CMA Configuration object that the payload should be added to
+	 * @param resource      the contents of the resource file, with all tokens replaced
+	 * @param configuration the CMA Configuration object that the payload should be added to
 	 */
-	void addResourceToConfiguration(String payload, ResourceMapper resourceMapper, Configuration configuration);
+	void addResourceToConfiguration(ObjectNode resource, Configuration configuration);
+
 
 }

src/main/java/com/marklogic/appdeployer/command/appservers/UpdateRestApiServersCommand.java

@@ -42,7 +42,7 @@ public void execute(CommandContext context) {
 			if (appConfig.isTestPortSet()) {
 				String payload = copyFileToString(f);
 				payload = payloadTokenReplacer.replaceTokens(payload, appConfig, true);
-				payload = adjustPayloadBeforeSavingResource(mgr, context, f, payload);
+				payload = adjustPayloadBeforeSavingResource(context, f, payload);
 				mgr.save(payload);
 			}
 		}

src/main/java/com/marklogic/appdeployer/command/cma/DeployConfigurationsCommand.java

@@ -29,7 +29,7 @@ public void execute(CommandContext context) {
 					if (logger.isInfoEnabled()) {
 						logger.info("Processing file: " + f.getAbsolutePath());
 					}
-					String payload = readResourceFromFile(null, context, f);
+					String payload = readResourceFromFile(context, f);
 					SaveReceipt receipt = mgr.save(payload);
 					afterResourceSaved(null, context, new ResourceReference(f, null), receipt);
 				}

src/main/java/com/marklogic/appdeployer/command/databases/DeployDatabaseCommand.java

@@ -123,29 +123,53 @@ public Integer getUndoSortOrder() {
 
 	@Override
     public void execute(CommandContext context) {
-        String payload = buildPayload(context);
+        String payload = buildPayloadForSaving(context);
         if (payload != null) {
-            DatabaseManager dbMgr = new DatabaseManager(context.getManageClient());
-            databaseName = dbMgr.getResourceId(payload);
+	        DatabaseManager dbMgr = new DatabaseManager(context.getManageClient());
+	        databaseName = dbMgr.getResourceId(payload);
 
-            payload = adjustPayloadBeforeSavingResource(dbMgr, context, null, payload);
             dbMgr.save(payload);
 
-            if (shouldCreateForests(context, payload)) {
-	            deployForestsCommand = buildDeployForestsCommand(databaseName, context);
+            DeployForestsCommand tempCommand = buildDeployForestsCommand(databaseName, context);
+            if (tempCommand != null) {
+            	this.deployForestsCommand = tempCommand;
 	            if (postponeForestCreation) {
 		            logger.info("Postponing creation of forests for database: " + databaseName);
 	            } else {
 		            deployForestsCommand.execute(context);
 	            }
             }
 
-            if (!isSubDatabase()) {
-            	new DeploySubDatabasesCommand(databaseName, deployDatabaseCommandFactory).execute(context);
-            }
+            deploySubDatabases(this.databaseName, context);
         }
     }
 
+	/**
+	 * If this is not a sub-database, then deploy sub-databases if any have been configured for this database.
+	 *
+	 * @param context
+	 */
+	public void deploySubDatabases(String dbName, CommandContext context) {
+	    if (!isSubDatabase()) {
+		    new DeploySubDatabasesCommand(dbName, deployDatabaseCommandFactory).execute(context);
+	    }
+    }
+
+	/**
+	 * Performs all work necessary to construct a payload that can either be saved immediately or included in a CMA
+	 * configuration.
+	 *
+	 * @param context
+	 * @return
+	 */
+	public String buildPayloadForSaving(CommandContext context) {
+	    String payload = buildPayload(context);
+	    if (payload != null) {
+		    payload = adjustPayloadBeforeSavingResource(context, null, payload);
+	    }
+	    return payload;
+    }
+
 	/**
 	 * When deleting databases, resource files are not yet merged together first. This should only mean that some
 	 * unnecessary delete calls are made.
@@ -267,11 +291,11 @@ else if (databaseFilename != null) {
 	 * name is extracted from the payload via a PayloadParser. This check can be disabled by setting
 	 * checkForCustomForests to false.
 	 *
+	 * @param databaseName
 	 * @param context
-	 * @param payload
 	 * @return
 	 */
-	protected boolean shouldCreateForests(CommandContext context, String payload) {
+	protected boolean shouldCreateForests(String databaseName, CommandContext context) {
 		if (!context.getAppConfig().isCreateForests()) {
 			if (logger.isInfoEnabled()) {
 				logger.info("Forest creation is disabled, so not creating any forests");
@@ -280,11 +304,9 @@ protected boolean shouldCreateForests(CommandContext context, String payload) {
 		}
 
 		if (isCheckForCustomForests()) {
-			PayloadParser parser = new PayloadParser();
-			String dbName = parser.getPayloadFieldValue(payload, "database-name");
-			boolean customForestsDontExist = !customForestsExist(context, dbName);
+			boolean customForestsDontExist = !customForestsExist(context, databaseName);
 			if (!customForestsDontExist && logger.isInfoEnabled()) {
-				logger.info("Found custom forests for database " + dbName + ", so not creating default forests");
+				logger.info("Found custom forests for database " + databaseName + ", so not creating default forests");
 			}
 			return customForestsDontExist;
 		}
@@ -315,14 +337,17 @@ protected boolean customForestsExist(CommandContext context, String dbName) {
      *
      * @param databaseName
      * @param context
-     * @return
+     * @return will return null if it's determined that no forests should be created for the database
      */
     public DeployForestsCommand buildDeployForestsCommand(String databaseName, CommandContext context) {
-        DeployForestsCommand c = new DeployForestsCommand(databaseName);
-        c.setForestsPerHost(getForestsPerHost());
-        c.setCreateForestsOnEachHost(createForestsOnEachHost);
-        c.setForestFilename(forestFilename);
-        return c;
+    	if (shouldCreateForests(databaseName, context)) {
+		    DeployForestsCommand c = new DeployForestsCommand(databaseName);
+		    c.setForestsPerHost(getForestsPerHost());
+		    c.setCreateForestsOnEachHost(createForestsOnEachHost);
+		    c.setForestFilename(forestFilename);
+		    return c;
+	    }
+    	return null;
     }
 
     protected String buildDefaultDatabasePayload(CommandContext context) {

src/main/java/com/marklogic/appdeployer/command/databases/DeployOtherDatabasesCommand.java

@@ -6,6 +6,7 @@
 import com.marklogic.appdeployer.command.AbstractUndoableCommand;
 import com.marklogic.appdeployer.command.CommandContext;
 import com.marklogic.appdeployer.command.SortOrderConstants;
+import com.marklogic.appdeployer.command.SupportsCmaCommand;
 import com.marklogic.appdeployer.command.forests.DeployForestsCommand;
 import com.marklogic.mgmt.PayloadParser;
 import com.marklogic.mgmt.api.API;
@@ -87,20 +88,27 @@ public void execute(CommandContext context) {
 			logger.info("Not sorting databases by dependencies, will sort them by their filenames instead");
 		}
 
-		databasePlans.forEach(databasePlan -> {
-			databasePlan.getDeployDatabaseCommand().execute(context);
-		});
+		if (context.getAppConfig().isDeployDatabasesWithCma()) {
+			deployDatabasesAndForestsViaCma(context, databasePlans);
+		}
 
-		// Either create forests in one bulk CMA request, or via a command per database
-		if (context.getAppConfig().isDeployForestsWithCma()) {
-			deployAllForestsInSingleCmaRequest(context, databasePlans);
-		} else {
+		else {
+			// Otherwise, create each database one at a time, which also handles sub-databases
 			databasePlans.forEach(databasePlan -> {
-				DeployForestsCommand dfc = databasePlan.getDeployDatabaseCommand().getDeployForestsCommand();
-				if (dfc != null) {
-					dfc.execute(context);
-				}
+				databasePlan.getDeployDatabaseCommand().execute(context);
 			});
+
+			// Either create forests in one bulk CMA request, or via a command per database
+			if (context.getAppConfig().isDeployForestsWithCma()) {
+				deployAllForestsInSingleCmaRequest(context, databasePlans);
+			} else {
+				databasePlans.forEach(databasePlan -> {
+					DeployForestsCommand dfc = databasePlan.getDeployDatabaseCommand().getDeployForestsCommand();
+					if (dfc != null) {
+						dfc.execute(context);
+					}
+				});
+			}
 		}
 	}
 
@@ -340,6 +348,42 @@ protected void buildDeployDatabaseCommands(CommandContext context, List<Database
 		});
 	}
 
+	/**
+	 * As of 3.15.0, if databases are to be deployed via CMA, then their forests will also be deployed via CMA,
+	 * regardless of the setting on the AppConfig instance.
+	 *
+	 * Also as of 3.15.0, sub-databases and their forests are never deployed by CMA. Will support this in a future
+	 * release.
+	 *
+	 * @param context
+	 * @param databasePlans
+	 */
+	protected void deployDatabasesAndForestsViaCma(CommandContext context, List<DatabasePlan> databasePlans) {
+		Configuration dbConfig = new Configuration();
+		// Forests must be included in a separate configuration object
+		Configuration forestConfig = new Configuration();
+
+		databasePlans.forEach(plan -> {
+			final DeployDatabaseCommand deployDatabaseCommand = plan.getDeployDatabaseCommand();
+			String payload = deployDatabaseCommand.buildPayloadForSaving(context);
+			payload = convertXmlPayloadToJsonIfNecessary(context, payload);
+			dbConfig.addDatabase(payload);
+
+			DeployForestsCommand deployForestsCommand = deployDatabaseCommand.buildDeployForestsCommand(plan.getDatabaseName(), context);
+			if (deployForestsCommand != null) {
+				deployForestsCommand.buildForests(context, false).forEach(forest -> forestConfig.addForest(forest.toObjectNode()));
+			}
+		});
+
+		new Configurations(dbConfig, forestConfig).submit(context.getManageClient());
+
+		// Now account for sub-databases, but not with CMA
+		databasePlans.forEach(plan -> {
+			final DeployDatabaseCommand deployDatabaseCommand = plan.getDeployDatabaseCommand();
+			deployDatabaseCommand.deploySubDatabases(plan.getDatabaseName(), context);
+		});
+	}
+
 	/**
 	 * Each DatabasePlan is expected to have constructed a DeployForestCommand, but not executed it. Each
 	 * DeployForestCommand can then be used to build a list of forests. All of those forests can be combined into a
@@ -350,15 +394,15 @@ protected void buildDeployDatabaseCommands(CommandContext context, List<Database
 	 */
 	protected void deployAllForestsInSingleCmaRequest(CommandContext context, List<DatabasePlan> databasePlans) {
 		List<Forest> allForests = new ArrayList<>();
-		for (DatabasePlan reference : databasePlans) {
-			DeployForestsCommand deployForestsCommand = reference.getDeployDatabaseCommand().getDeployForestsCommand();
-			if (deployForestsCommand != null) {
-				allForests.addAll(deployForestsCommand.buildForests(context, false));
+		databasePlans.forEach(plan -> {
+			DeployForestsCommand dfc = plan.getDeployDatabaseCommand().getDeployForestsCommand();
+			if (dfc != null) {
+				allForests.addAll(dfc.buildForests(context, false));
 			}
-		}
+		});
 		if (!allForests.isEmpty()) {
 			Configuration config = new Configuration();
-			config.setForests(allForests);
+			allForests.forEach(forest -> config.addForest(forest.toObjectNode()));
 			new Configurations(config).submit(context.getManageClient());
 		}
 	}