#140 Deleting a database defaults to deleting forests and replicas first

Author: Rob Rudin

src/main/java/com/marklogic/appdeployer/AppConfig.java

@@ -97,6 +97,9 @@ public class AppConfig {
     // Allows for creating a triggers database without a config file for one
     private boolean createTriggersDatabase = true;
 
+    // Controls whether forests are deleted when a database is deleted
+    private boolean deleteForests = true;
+
     // Controls whether replicas are deleted or not when undeploying a database
     private boolean deleteReplicas = true;
 
@@ -717,4 +720,12 @@ public boolean isDeleteReplicas() {
     public void setDeleteReplicas(boolean deleteReplicas) {
         this.deleteReplicas = deleteReplicas;
     }
+
+    public boolean isDeleteForests() {
+        return deleteForests;
+    }
+
+    public void setDeleteForests(boolean deleteForests) {
+        this.deleteForests = deleteForests;
+    }
 }

src/main/java/com/marklogic/appdeployer/DefaultAppConfigFactory.java

@@ -144,8 +144,20 @@ public AppConfig newAppConfig() {
 
 		/**
 		 * When undo is invoked on DeployDatabaseCommand (such as via mlUndeploy in ml-gradle), this controls whether
-		 * or not replicas are deleted first. Most of the time, you want this set to true (the default) as otherwise,
-		 * the database can't be deleted and the Management REST API will throw an error.
+		 * or not forests are deleted, or just their configuration is deleted. If mlDeleteReplicas is set to true, this
+		 * has no impact - currently, the forests and their replicas will be deleted for efficiency reasons (results in
+		 * fewer calls to the Management REST API.
+		 */
+		prop = getProperty("mlDeleteForests");
+		if (prop != null) {
+			logger.info("Delete forests when a database is deleted: " + prop);
+			c.setDeleteForests(Boolean.parseBoolean(prop));
+		}
+
+		/**
+		 * When undo is invoked on DeployDatabaseCommand (such as via mlUndeploy in ml-gradle), this controls whether
+		 * primary forests and their replicas are deleted first. Most of the time, you want this set to true
+		 * (the default) as otherwise, the database can't be deleted and the Management REST API will throw an error.
 		 */
 		prop = getProperty("mlDeleteReplicas");
 		if (prop != null) {

src/main/java/com/marklogic/appdeployer/command/databases/DeployContentDatabasesCommand.java

@@ -1,8 +1,5 @@
 package com.marklogic.appdeployer.command.databases;
 
-import java.io.File;
-import java.util.List;
-
 import com.fasterxml.jackson.databind.JsonNode;
 import com.marklogic.appdeployer.AppConfig;
 import com.marklogic.appdeployer.command.CommandContext;
@@ -11,6 +8,9 @@
 import com.marklogic.mgmt.databases.DatabaseManager;
 import com.marklogic.rest.util.JsonNodeUtil;
 
+import java.io.File;
+import java.util.List;
+
 /**
  * For ease of use, this command handles creating forests the the content database, either based on a file in the
  * forests directory, or based on the default payload in the DeployForestsCommand class. This allows a developer to only
@@ -70,18 +70,15 @@ public void undo(CommandContext context) {
             String payload = node.toString();
             String json = tokenReplacer.replaceTokens(payload, appConfig, false);
 
-            DatabaseManager dbMgr = new DatabaseManager(context.getManageClient());
-            dbMgr.setForestDelete(getForestDelete());
+            DatabaseManager dbMgr = newDatabaseManageForDeleting(context);
             dbMgr.delete(json);
-
             if (appConfig.isTestPortSet()) {
                 json = tokenReplacer.replaceTokens(payload, appConfig, true);
                 dbMgr.delete(json);
             }
         } else {
             // Try to delete the content database if it exists
-            DatabaseManager dbMgr = new DatabaseManager(context.getManageClient());
-            dbMgr.setForestDelete(getForestDelete());
+            DatabaseManager dbMgr = newDatabaseManageForDeleting(context);
             dbMgr.deleteByName(appConfig.getContentDatabaseName());
 
             if (appConfig.isTestPortSet()) {

src/main/java/com/marklogic/appdeployer/command/databases/DeployDatabaseCommand.java

@@ -1,8 +1,6 @@
 package com.marklogic.appdeployer.command.databases;
 
-import java.io.File;
-import java.util.Map;
-
+import com.marklogic.appdeployer.AppConfig;
 import com.marklogic.appdeployer.command.AbstractCommand;
 import com.marklogic.appdeployer.command.CommandContext;
 import com.marklogic.appdeployer.command.SortOrderConstants;
@@ -11,6 +9,10 @@
 import com.marklogic.mgmt.PayloadParser;
 import com.marklogic.mgmt.SaveReceipt;
 import com.marklogic.mgmt.databases.DatabaseManager;
+import com.marklogic.mgmt.forests.ForestManager;
+
+import java.io.File;
+import java.util.Map;
 
 /**
  * Can be used for creating any kind of database with any sorts of forests. Specifying a config file for the database or
@@ -51,14 +53,6 @@ public class DeployDatabaseCommand extends AbstractCommand implements UndoableCo
      */
     private boolean createForestsOnEachHost = true;
 
-    /**
-     * Applied when the database is deleted - see
-     * http://docs.marklogic.com/REST/DELETE/manage/v2/databases/[id-or-name].
-     */
-    private String forestDelete = "data";
-
-    private boolean deleteReplicas = true;
-
     private int undoSortOrder;
 
     public DeployDatabaseCommand() {
@@ -95,13 +89,28 @@ public void execute(CommandContext context) {
     public void undo(CommandContext context) {
         String payload = buildPayload(context);
         if (payload != null) {
-            DatabaseManager dbMgr = new DatabaseManager(context.getManageClient());
-            dbMgr.setForestDelete(forestDelete);
-            dbMgr.setDeleteReplicas(deleteReplicas);
-            dbMgr.delete(payload);
+            newDatabaseManageForDeleting(context).delete(payload);
         }
     }
 
+    /**
+     * Configures the DatabaseManager in terms of how it deletes forests based on properties in the AppConfig instance
+     * in the CommandContext.
+     *
+     * @param context
+     * @return
+     */
+    protected DatabaseManager newDatabaseManageForDeleting(CommandContext context) {
+        DatabaseManager dbMgr = new DatabaseManager(context.getManageClient());
+        dbMgr.setForestDelete(getForestDeleteLevel(context.getAppConfig()));
+        dbMgr.setDeleteReplicas(context.getAppConfig().isDeleteReplicas());
+        return dbMgr;
+    }
+
+    protected String getForestDeleteLevel(AppConfig appConfig) {
+        return appConfig.isDeleteForests() ? DatabaseManager.DELETE_FOREST_DATA : DatabaseManager.DELETE_FOREST_CONFIGURATION;
+    }
+
     /**
      * Builds the XML or JSON payload for this command, based on the given CommandContext.
      * 
@@ -188,14 +197,6 @@ protected String buildDefaultDatabasePayload(CommandContext context) {
         return format("{\"database-name\": \"%s\"}", databaseName);
     }
 
-    public String getForestDelete() {
-        return forestDelete;
-    }
-
-    public void setForestDelete(String forestDelete) {
-        this.forestDelete = forestDelete;
-    }
-
     public int getForestsPerHost() {
         return forestsPerHost;
     }
@@ -247,12 +248,4 @@ public boolean isCreateForestsOnEachHost() {
     public void setCreateForestsOnEachHost(boolean createForestsOnEachHost) {
         this.createForestsOnEachHost = createForestsOnEachHost;
     }
-
-    public boolean isDeleteReplicas() {
-        return deleteReplicas;
-    }
-
-    public void setDeleteReplicas(boolean deleteReplicas) {
-        this.deleteReplicas = deleteReplicas;
-    }
 }

src/main/java/com/marklogic/mgmt/AbstractResourceManager.java

@@ -1,9 +1,8 @@
 package com.marklogic.mgmt;
 
-import org.springframework.http.ResponseEntity;
-
 import com.marklogic.rest.util.Fragment;
 import com.marklogic.rest.util.ResourcesFragment;
+import org.springframework.http.ResponseEntity;
 
 /**
  * This class makes a number of assumptions in order to simplify the implementation of common operations for a MarkLogic

src/main/java/com/marklogic/mgmt/databases/DatabaseManager.java

@@ -1,17 +1,18 @@
 package com.marklogic.mgmt.databases;
 
-import java.util.List;
-
 import com.marklogic.mgmt.AbstractResourceManager;
 import com.marklogic.mgmt.ManageClient;
-import com.marklogic.mgmt.ManageConfig;
 import com.marklogic.mgmt.forests.ForestManager;
-import com.marklogic.mgmt.forests.ForestStatus;
 import com.marklogic.rest.util.Fragment;
 
+import java.util.List;
+
 public class DatabaseManager extends AbstractResourceManager {
 
-    private String forestDelete = "data";
+	public final static String DELETE_FOREST_DATA = "data";
+	public final static String DELETE_FOREST_CONFIGURATION = "configuration";
+
+    private String forestDelete = DELETE_FOREST_DATA;
     private boolean deleteReplicas = true;
 
     public DatabaseManager(ManageClient manageClient) {
@@ -60,17 +61,33 @@ public void deleteByName(String databaseName) {
         delete(json);
     }
 
+    /**
+     * Use this to delete all of a database's forests and their replicas, but leave the database in place. This seems
+     * to be the safest way to ensure a database can be deleted.
+     *
+     * @param databaseIdOrName
+     */
+    public void deleteForestsAndReplicas(String databaseIdOrName) {
+        List<String> primaryForestIds = getPrimaryForestIds(databaseIdOrName);
+        detachForests(databaseIdOrName);
+        ForestManager forestManager = new ForestManager(getManageClient());
+        for (String forestId : primaryForestIds) {
+            forestManager.delete(forestId, ForestManager.DELETE_LEVEL_FULL, ForestManager.REPLICAS_DELETE);
+        }
+    }
+
+    public void detachForests(String databaseIdOrName) {
+    	logger.info("Detaching forests from database: " + databaseIdOrName);
+        save(format("{\"database-name\":\"%s\", \"forest\":[]}", databaseIdOrName));
+        logger.info("Finished detaching forests from database: " + databaseIdOrName);
+    }
+
     @Override
     protected void beforeDelete(String resourceId, String path, String... resourceUrlParams) {
         if (deleteReplicas) {
-            logger.info("Deleting any replicas that exist for database: " + resourceId);
-            ForestManager forestManager = new ForestManager(getManageClient());
-            for (String forestId : getPrimaryForestIds(resourceId)) {
-                ForestStatus status = forestManager.getForestStatus(forestId);
-                if (status.isPrimary() && status.hasReplicas()) {
-                    forestManager.deleteReplicas(forestId);
-                }
-            }
+        	logger.info("Deleting forests and replicas for database: " + resourceId);
+        	deleteForestsAndReplicas(resourceId);
+        	logger.info("Finished deleting forests and replicas for database: " + resourceId);
         }
     }
 
@@ -102,7 +119,13 @@ public List<String> getPrimaryForestIds(String databaseNameOrId) {
         return getPropertiesAsXml(databaseNameOrId).getElementValues("/node()/m:forests/m:forest");
     }
 
-    public void deleteReplicaForests(String databaseNameOrId) {
+	/**
+	 * Delete all replicas for the primary forests for the given database, but don't delete the database or the
+	 * primary forests.
+	 *
+	 * @param databaseNameOrId
+	 */
+	public void deleteReplicaForests(String databaseNameOrId) {
         logger.info(format("Deleting replica forests (if any exist) for database %s", databaseNameOrId));
         ForestManager mgr = new ForestManager(getManageClient());
         for (String forestId : getPrimaryForestIds(databaseNameOrId)) {
@@ -133,7 +156,11 @@ public void setForestDelete(String forestDelete) {
         this.forestDelete = forestDelete;
     }
 
-    public boolean isDeleteReplicas() {
+	public String getForestDelete() {
+		return forestDelete;
+	}
+
+	public boolean isDeleteReplicas() {
         return deleteReplicas;
     }
 

src/main/java/com/marklogic/mgmt/forests/ForestManager.java

@@ -1,15 +1,15 @@
 package com.marklogic.mgmt.forests;
 
-import java.util.HashMap;
-import java.util.Iterator;
-import java.util.List;
-import java.util.Map;
-
 import com.fasterxml.jackson.databind.JsonNode;
 import com.marklogic.mgmt.AbstractResourceManager;
 import com.marklogic.mgmt.ManageClient;
 import com.marklogic.rest.util.Fragment;
 
+import java.util.HashMap;
+import java.util.Iterator;
+import java.util.List;
+import java.util.Map;
+
 /**
  * Provides methods wrapping /manage/v2/forests endpoints.
  *
@@ -19,10 +19,15 @@
  */
 public class ForestManager extends AbstractResourceManager {
 
-    public final static String DELETE_FULL = "full";
-    public final static String DELETE_CONFIG = "config-only";
+    // Parameter values for the "level" parameter when deleting a forest
+    public final static String DELETE_LEVEL_FULL = "full";
+    public final static String DELETE_LEVEL_CONFIG_ONLY = "config-only";
 
-    private String deleteLevel = DELETE_FULL;
+    // Parameter values for the "replicas" parameter when deleting a forest
+    public final static String REPLICAS_DETACH = "detach";
+    public final static String REPLICAS_DELETE = "delete";
+
+    private String deleteLevel = DELETE_LEVEL_FULL;
 
     public ForestManager(ManageClient client) {
         super(client);
@@ -40,11 +45,20 @@ public void createJsonForestWithName(String name, String host) {
     }
 
     public void delete(String nameOrId, String level) {
+        delete(nameOrId, level, REPLICAS_DELETE);
+    }
+
+    /**
+     * @param nameOrId Name or ID of forest to delete
+     * @param level either "full" or "config-only"
+     * @param replicas either "detach" or "delete"
+     */
+    public void delete(String nameOrId, String level, String replicas) {
         if (!forestExists(nameOrId)) {
             logger.info(format("Could not find forest with name or ID: %s, so not deleting", nameOrId));
         } else {
             logger.info(format("Deleting forest %s", nameOrId));
-            getManageClient().delete(format("/manage/v2/forests/%s?level=%s", nameOrId, level));
+            getManageClient().delete(format("/manage/v2/forests/%s?level=%s&replicas=%s", nameOrId, level, replicas));
             logger.info(format("Deleted forest %s", nameOrId));
         }
     }
@@ -158,7 +172,7 @@ public void deleteReplicas(String forestIdOrName) {
         }
         setReplicasToNone(forestIdOrName);
         for (String id : replicaIds) {
-            delete(id, DELETE_FULL);
+            delete(id, DELETE_LEVEL_FULL);
         }
     }
 

src/test/java/com/marklogic/appdeployer/DefaultAppConfigFactoryTest.java

@@ -58,6 +58,7 @@ public void allProperties() {
         p.setProperty("mlSimpleSsl", "anyvalue");
         p.setProperty("mlContentDatabaseName", "my-content-db");
         p.setProperty("mlModulesDatabaseName", "my-modules");
+	    p.setProperty("mlDeleteForests", "false");
         p.setProperty("mlDeleteReplicas", "false");
         p.setProperty("mlGroupName", "other-group");
         p.setProperty("mlAppServicesPort", "8123");
@@ -94,6 +95,7 @@ public void allProperties() {
         assertNotNull(config.getRestSslHostnameVerifier());
         assertEquals("my-content-db", config.getContentDatabaseName());
         assertEquals("my-modules", config.getModulesDatabaseName());
+	    assertFalse(config.isDeleteForests());
         assertFalse(config.isDeleteReplicas());
         assertEquals("other-group", config.getGroupName());
         assertEquals((Integer) 8123, config.getAppServicesPort());

src/test/java/com/marklogic/appdeployer/command/es/GenerateModelArtifactsTest.java

@@ -8,10 +8,12 @@
 import com.marklogic.appdeployer.command.restapis.DeployRestApiServersCommand;
 import com.marklogic.appdeployer.command.schemas.LoadSchemasCommand;
 import org.junit.After;
+import org.junit.Ignore;
 import org.junit.Test;
 
 import java.io.File;
 
+@Ignore("Ignoring until ML9 is released")
 public class GenerateModelArtifactsTest extends AbstractAppDeployerTest {
 
 	@After

src/test/java/com/marklogic/appdeployer/command/modules/LoadModulesTest.java

@@ -48,8 +48,8 @@ public void loadModulesWithStaticCheck() {
 			fail("An error should have been thrown because /ext/bad.xqy failed static check");
 		} catch (Exception ex) {
 			String message = ex.getMessage();
-			assertTrue("Loading modules with 2.11.0 of ml-javaclient-util defaults to bulk loading, so static checking should as well",
-				message.contains("Bulk static check failure"));
+			assertTrue("Loading modules with 2.11.0 of ml-javaclient-util defaults to bulk loading, so static checking should as well; message: " + message,
+				message.contains("Unexpected token syntax error"));
 			assertTrue(message.contains("in /ext/bad.xqy, on line 2"));
 		}
 	}