#168 Added support for replica forest properties

Author: rjrudin

src/main/java/com/marklogic/appdeployer/AppConfig.java

@@ -23,10 +23,13 @@
  * Encapsulates common configuration properties for an application deployed to MarkLogic. These properties include not
  * just names of application resources - such as app servers and databases - but also connection information for loading
  * modules into an application as well as paths for modules and configuration files.
- * <p>
+ *
  * An instance of this class is passed in as the main argument to the methods in the {@code AppDeployer} interface,
  * meaning that you're free to not just configure this as needed but also subclass it and add anything that you would
  * like.
+ *
+ * Additionally, the additionalProperties Map can used for storing any additional properties that a client of this class
+ * may use, without having to define them as attributes of this class.
  */
 public class AppConfig {
 
@@ -118,6 +121,11 @@ public class AppConfig {
     // Comma-delimited string used for configuring forest replicas
     private String databaseNamesAndReplicaCounts;
 
+    // Configures the data-directory for replica forests built dynamically
+    private String replicaForestDataDirectory;
+    private String replicaForestLargeDataDirectory;
+    private String replicaForestFastDataDirectory;
+
     // Path to use for DeployFlexrepCommand
     private String flexrepPath;
 
@@ -142,6 +150,8 @@ public class AppConfig {
 
 	private String[] resourceFilenamesToIgnore;
 
+	private Map<String, Object> additionalProperties = new HashMap<>();
+
 	public AppConfig() {
         this(DEFAULT_MODULES_PATH, DEFAULT_SCHEMAS_PATH);
     }
@@ -737,4 +747,36 @@ public boolean isNoRestServer() {
     public void setNoRestServer(boolean noRestServer) {
         this.noRestServer = noRestServer;
     }
+
+	public String getReplicaForestDataDirectory() {
+		return replicaForestDataDirectory;
+	}
+
+	public void setReplicaForestDataDirectory(String replicaForestDataDirectory) {
+		this.replicaForestDataDirectory = replicaForestDataDirectory;
+	}
+
+	public String getReplicaForestLargeDataDirectory() {
+		return replicaForestLargeDataDirectory;
+	}
+
+	public void setReplicaForestLargeDataDirectory(String replicaForestLargeDataDirectory) {
+		this.replicaForestLargeDataDirectory = replicaForestLargeDataDirectory;
+	}
+
+	public String getReplicaForestFastDataDirectory() {
+		return replicaForestFastDataDirectory;
+	}
+
+	public void setReplicaForestFastDataDirectory(String replicaForestFastDataDirectory) {
+		this.replicaForestFastDataDirectory = replicaForestFastDataDirectory;
+	}
+
+	public Map<String, Object> getAdditionalProperties() {
+		return additionalProperties;
+	}
+
+	public void setAdditionalProperties(Map<String, Object> additionalProperties) {
+		this.additionalProperties = additionalProperties;
+	}
 }

src/main/java/com/marklogic/appdeployer/DefaultAppConfigFactory.java

@@ -158,6 +158,24 @@ public AppConfig newAppConfig() {
 			c.setDatabaseNamesAndReplicaCounts(prop);
 		}
 
+		prop = getProperty("mlReplicaForestDataDirectory");
+		if (prop != null) {
+			logger.info("Replica forest data directory " + prop);
+			c.setReplicaForestDataDirectory(prop);
+		}
+
+		prop = getProperty("mlReplicaForestLargeDataDirectory");
+		if (prop != null) {
+			logger.info("Replica forest large data directory " + prop);
+			c.setReplicaForestLargeDataDirectory(prop);
+		}
+
+		prop = getProperty("mlReplicaForestFastDataDirectory");
+		if (prop != null) {
+			logger.info("Replica forest fast data directory " + prop);
+			c.setReplicaForestFastDataDirectory(prop);
+		}
+
 		/**
 		 * When undo is invoked on DeployDatabaseCommand (such as via mlUndeploy in ml-gradle), this controls whether
 		 * or not forests are deleted, or just their configuration is deleted. If mlDeleteReplicas is set to true, this

src/main/java/com/marklogic/appdeployer/command/forests/ConfigureForestReplicasCommand.java

@@ -1,13 +1,18 @@
 package com.marklogic.appdeployer.command.forests;
 
+import com.marklogic.appdeployer.AppConfig;
 import com.marklogic.appdeployer.command.AbstractUndoableCommand;
 import com.marklogic.appdeployer.command.CommandContext;
 import com.marklogic.appdeployer.command.SortOrderConstants;
+import com.marklogic.mgmt.api.API;
+import com.marklogic.mgmt.api.forest.Forest;
+import com.marklogic.mgmt.api.forest.ForestReplica;
 import com.marklogic.mgmt.databases.DatabaseManager;
 import com.marklogic.mgmt.forests.ForestManager;
 import com.marklogic.mgmt.forests.ForestStatus;
 import com.marklogic.mgmt.hosts.HostManager;
 
+import java.util.ArrayList;
 import java.util.HashMap;
 import java.util.List;
 import java.util.Map;
@@ -84,7 +89,7 @@ public void execute(CommandContext context) {
 		for (String forestName : forestNamesAndReplicaCounts.keySet()) {
 			int replicaCount = forestNamesAndReplicaCounts.get(forestName);
 			if (replicaCount > 0) {
-				configureReplicaForests(forestName, replicaCount, hostIds, forestMgr);
+				configureReplicaForests(forestName, replicaCount, hostIds, context, forestMgr);
 			}
 		}
 	}
@@ -130,21 +135,20 @@ protected void deleteReplicas(String forestName, ForestManager forestMgr) {
 
 	/**
 	 * For the given database, find all of its primary forests. Then for each primary forest, just call
-	 * configureReplicaForests? And that should be smart enough to say - if the primary forest already has replicas,
+	 * configureReplicaForests. And that should be smart enough to say - if the primary forest already has replicas,
 	 * then don't do anything.
 	 *
 	 * @param databaseName
 	 * @param replicaCount
 	 * @param hostIds
 	 */
 	protected void configureDatabaseReplicaForests(String databaseName, int replicaCount, List<String> hostIds,
-												   CommandContext context) {
+	                                               CommandContext context) {
 		ForestManager forestMgr = new ForestManager(context.getManageClient());
 		DatabaseManager dbMgr = new DatabaseManager(context.getManageClient());
 		List<String> forestNames = dbMgr.getForestNames(databaseName);
-		logger.info("Forests: " + forestNames);
 		for (String name : forestNames) {
-			configureReplicaForests(name, replicaCount, hostIds, forestMgr);
+			configureReplicaForests(name, replicaCount, hostIds, context, forestMgr);
 		}
 	}
 
@@ -155,10 +159,11 @@ protected void configureDatabaseReplicaForests(String databaseName, int replicaC
 	 * @param forestIdOrName
 	 * @param replicaCount
 	 * @param hostIds
+	 * @param context
 	 * @param forestMgr
 	 */
 	protected void configureReplicaForests(String forestIdOrName, int replicaCount, List<String> hostIds,
-										   ForestManager forestMgr) {
+	                                       CommandContext context, ForestManager forestMgr) {
 		ForestStatus status = forestMgr.getForestStatus(forestIdOrName);
 		if (!status.isPrimary()) {
 			logger.info(format("Forest %s is not a primary forest, so not configuring replica forests", forestIdOrName));
@@ -169,12 +174,9 @@ protected void configureReplicaForests(String forestIdOrName, int replicaCount,
 			return;
 		}
 
-		Map<String, String> replicaNamesAndHostIds = createReplicaForests(forestIdOrName, replicaCount, hostIds, forestMgr);
-		logger.info(format("Configuring forest replicas for primary forest %s", forestIdOrName));
-		if (!replicaNamesAndHostIds.isEmpty()) {
-			forestMgr.setReplicas(forestIdOrName, replicaNamesAndHostIds);
-		}
-		logger.info(format("Finished configuring forest replicas for primary forest %s", forestIdOrName));
+		logger.info(format("Creating forest replicas for primary forest %s", forestIdOrName));
+		createReplicaForests(forestIdOrName, replicaCount, hostIds, context, forestMgr);
+		logger.info(format("Finished creating forest replicas for primary forest %s", forestIdOrName));
 	}
 
 	/**
@@ -184,12 +186,20 @@ protected void configureReplicaForests(String forestIdOrName, int replicaCount,
 	 * @param forestIdOrName
 	 * @param replicaCount
 	 * @param hostIds
+	 * @param context
 	 * @param forestMgr
 	 * @return a map where the keys are replica forest names, and the value of each key is the ID of the host that
 	 * the replica was created on
 	 */
 	protected Map<String, String> createReplicaForests(String forestIdOrName, int replicaCount, List<String> hostIds,
-													   ForestManager forestMgr) {
+	                                                   CommandContext context, ForestManager forestMgr) {
+
+		// Using the Forest class to generate JSON
+		API api = new API(context.getManageClient());
+		Forest forest = new Forest(api, null);
+		List<ForestReplica> replicas = new ArrayList<>();
+		forest.setForestReplica(replicas);
+
 		String primaryForestHostId = forestMgr.getHostId(forestIdOrName);
 		Map<String, String> replicaNamesAndHostIds = new HashMap<>();
 		int size = hostIds.size();
@@ -203,15 +213,28 @@ protected Map<String, String> createReplicaForests(String forestIdOrName, int re
 					}
 					String replicaHostId = hostIds.get(nextReplicaHostIndex);
 					String name = forestIdOrName + "-replica-" + j;
-					forestMgr.createJsonForestWithName(name, replicaHostId);
+					replicas.add(buildForestReplica(name, replicaHostId, context.getAppConfig()));
 					replicaNamesAndHostIds.put(name, replicaHostId);
 					nextReplicaHostIndex++;
 				}
 			}
 		}
+
+		String json = forest.getJson();
+		context.getManageClient().putJson(forestMgr.getPropertiesPath(forestIdOrName), json);
 		return replicaNamesAndHostIds;
 	}
 
+	protected ForestReplica buildForestReplica(String name, String replicaHostId, AppConfig appConfig) {
+		ForestReplica replica = new ForestReplica();
+		replica.setHost(replicaHostId);
+		replica.setReplicaName(name);
+		replica.setDataDirectory(appConfig.getReplicaForestDataDirectory());
+		replica.setLargeDataDirectory(appConfig.getReplicaForestLargeDataDirectory());
+		replica.setFastDataDirectory(appConfig.getReplicaForestFastDataDirectory());
+		return replica;
+	}
+
 	public Map<String, Integer> getForestNamesAndReplicaCounts() {
 		return forestNamesAndReplicaCounts;
 	}

src/main/java/com/marklogic/mgmt/api/forest/Forest.java

@@ -25,6 +25,7 @@ public class Forest extends Resource {
     private List<String> failoverHost;
     private List<ForestBackup> forestBackup;
     private List<DatabaseReplication> databaseReplication;
+    private List<ForestReplica> forestReplica;
 
     public Forest() {
     }
@@ -191,4 +192,12 @@ public Boolean getRebalancerEnable() {
     public Boolean getFailoverEnable() {
         return failoverEnable;
     }
+
+	public List<ForestReplica> getForestReplica() {
+		return forestReplica;
+	}
+
+	public void setForestReplica(List<ForestReplica> forestReplica) {
+		this.forestReplica = forestReplica;
+	}
 }

src/test/java/com/marklogic/appdeployer/DefaultAppConfigFactoryTest.java

@@ -79,6 +79,11 @@ public void allProperties() {
 	    p.setProperty("mlGenerateExtractionTemplate", "false");
 	    p.setProperty("mlResourceFilenamesToIgnore", "role1.json,role2.xml");
 
+	    p.setProperty("mlDatabaseNamesAndReplicaCounts", "Documents,1,Security,2");
+	    p.setProperty("mlReplicaForestDataDirectory", "/var/data");
+	    p.setProperty("mlReplicaForestFastDataDirectory", "/var/fast");
+	    p.setProperty("mlReplicaForestLargeDataDirectory", "/var/large");
+
 	    sut = new DefaultAppConfigFactory(new SimplePropertySource(p));
         AppConfig config = sut.newAppConfig();
 
@@ -124,6 +129,11 @@ public void allProperties() {
 
         assertEquals("role1.json", config.getResourceFilenamesToIgnore()[0]);
 	    assertEquals("role2.xml", config.getResourceFilenamesToIgnore()[1]);
+
+	    assertEquals("Documents,1,Security,2", config.getDatabaseNamesAndReplicaCounts());
+	    assertEquals("/var/data", config.getReplicaForestDataDirectory());
+	    assertEquals("/var/fast", config.getReplicaForestFastDataDirectory());
+	    assertEquals("/var/large", config.getReplicaForestLargeDataDirectory());
     }
 
 	/**

src/test/java/com/marklogic/appdeployer/command/forests/BuildForestReplicaTest.java

@@ -0,0 +1,30 @@
+package com.marklogic.appdeployer.command.forests;
+
+import com.marklogic.appdeployer.AppConfig;
+import com.marklogic.appdeployer.command.CommandContext;
+import com.marklogic.mgmt.api.forest.ForestReplica;
+import org.junit.Assert;
+import org.junit.Test;
+
+public class BuildForestReplicaTest extends Assert {
+
+	/**
+	 * Small unit test to verify that a ForestReplica is built correctly based on the AppConfig.ø
+	 */
+	@Test
+	public void test() {
+		AppConfig config = new AppConfig();
+		config.setReplicaForestFastDataDirectory("/var/fast");
+		config.setReplicaForestLargeDataDirectory("/var/large");
+		config.setReplicaForestDataDirectory("/var/data");
+		CommandContext context = new CommandContext(config, null, null);
+
+		ConfigureForestReplicasCommand command = new ConfigureForestReplicasCommand();
+		ForestReplica replica = command.buildForestReplica("test-name", "host-1", config);
+		assertEquals("test-name", replica.getReplicaName());
+		assertEquals("host-1", replica.getHost());
+		assertEquals("/var/data", replica.getDataDirectory());
+		assertEquals("/var/fast", replica.getFastDataDirectory());
+		assertEquals("/var/large", replica.getLargeDataDirectory());
+	}
+}

src/test/java/com/marklogic/appdeployer/command/forests/ConfigureForestReplicasDebug.java

@@ -13,13 +13,16 @@
 public class ConfigureForestReplicasDebug {
 
     public static void main(String[] args) {
-        final String host = args[0];
-        final String password = args[1];
+        final String host = "localhost"; //args[0];
+        final String password = "admin"; //args[1];
 
         ManageConfig config = new ManageConfig(host, 8002, "admin", password);
         ManageClient manageClient = new ManageClient(config);
         AppConfig appConfig = new AppConfig();
         appConfig.setDatabaseNamesAndReplicaCounts("testdb,1");
+        appConfig.setReplicaForestDataDirectory("/var/opt/MarkLogic/Replica");
+        appConfig.setReplicaForestLargeDataDirectory("/var/opt/MarkLogic/Large");
+        appConfig.setReplicaForestFastDataDirectory("/var/opt/MarkLogic/Fast");
         CommandContext context = new CommandContext(appConfig, manageClient, null);
 
         DeployDatabaseCommand ddc = new DeployDatabaseCommand();
@@ -33,7 +36,10 @@ public static void main(String[] args) {
         ddc.execute(context);
         cfrc.execute(context);
 
-        // Then delete the replicas, and then undeploy the database
+        // Deploy again to make sure there are no errors
+	    cfrc.execute(context);
+
+	    // Then delete the replicas, and then undeploy the database
         cfrc.undo(context);
         ddc.undo(context);
     }