#225 Now sorting databases to deploy based on database property values

Removed OBE mlSortRolesByDependencies too

Author: rjrudin

src/main/java/com/marklogic/appdeployer/AppConfig.java

@@ -134,7 +134,7 @@ public class AppConfig {
     // Controls whether replicas are deleted or not when undeploying a database
     private boolean deleteReplicas = true;
 
-    private boolean sortRolesByDependencies = true;
+    private boolean sortOtherDatabaseByDependencies = true;
 
     // As defined by the REST API
     private String modulePermissions = "rest-admin,read,rest-admin,update,rest-extension-user,execute";
@@ -917,14 +917,6 @@ public void setAppServicesPassword(String appServicesPassword) {
 		this.appServicesPassword = appServicesPassword;
 	}
 
-	public boolean isSortRolesByDependencies() {
-		return sortRolesByDependencies;
-	}
-
-	public void setSortRolesByDependencies(boolean sortRolesByDependencies) {
-		this.sortRolesByDependencies = sortRolesByDependencies;
-	}
-
 	public boolean isDeleteTestModules() {
 		return deleteTestModules;
 	}
@@ -1186,4 +1178,12 @@ public Pattern getModuleFilenamesIncludePattern() {
 	public void setModuleFilenamesIncludePattern(Pattern moduleFilenamesIncludePattern) {
 		this.moduleFilenamesIncludePattern = moduleFilenamesIncludePattern;
 	}
+
+	public boolean isSortOtherDatabaseByDependencies() {
+		return sortOtherDatabaseByDependencies;
+	}
+
+	public void setSortOtherDatabaseByDependencies(boolean sortOtherDatabaseByDependencies) {
+		this.sortOtherDatabaseByDependencies = sortOtherDatabaseByDependencies;
+	}
 }

src/main/java/com/marklogic/appdeployer/DefaultAppConfigFactory.java

@@ -643,16 +643,6 @@ public AppConfig newAppConfig() {
 			c.setResourceFilenamesIncludePattern(Pattern.compile(prop));
 		}
 
-		/**
-		 * Version 2.9.0 of ml-app-deployer, by default, sorts role files by reading each file and looking at the role
-		 * dependencies. You can disable this behavior by setting this property to false.
-		 */
-		prop = getProperty("mlSortRolesByDependencies");
-		if (prop != null) {
-			logger.info("Sort roles by dependencies: " + prop);
-			c.setSortRolesByDependencies(Boolean.parseBoolean(prop));
-		}
-
 		prop = getProperty("mlExcludeProperties");
 		if (prop != null) {
 			String[] values = prop.split(",");

src/main/java/com/marklogic/appdeployer/command/databases/DeployDatabaseCommandComparator.java

@@ -2,6 +2,10 @@
 
 import com.marklogic.appdeployer.command.CommandContext;
 import com.marklogic.client.ext.helper.LoggingObject;
+import com.marklogic.mgmt.api.API;
+import com.marklogic.mgmt.api.database.Database;
+import com.marklogic.mgmt.mapper.DefaultResourceMapper;
+import com.marklogic.mgmt.mapper.ResourceMapper;
 
 import java.util.Comparator;
 
@@ -11,41 +15,44 @@
  */
 public class DeployDatabaseCommandComparator extends LoggingObject implements Comparator<DeployDatabaseCommand> {
 
-    private CommandContext context;
-    private boolean reverseOrder = false;
-
-    public DeployDatabaseCommandComparator(CommandContext context, boolean reverseOrder) {
-        this.context = context;
-        this.reverseOrder = reverseOrder;
-    }
-
-    @Override
-    public int compare(DeployDatabaseCommand o1, DeployDatabaseCommand o2) {
-        String p1 = o1.buildPayload(context);
-        String p2 = o2.buildPayload(context);
-        if (p1 == null || p2 == null) {
-            return 0;
-        }
-
-        boolean b1 = payloadDependsOnOtherDatabase(p1);
-        boolean b2 = payloadDependsOnOtherDatabase(p2);
-        if (b1 && !b2) {
-            return reverseOrder ? -1 : 1;
-        }
-        if (b2 && !b1) {
-            return reverseOrder ? 1 : -1;
-        }
-        return 0;
-    }
-
-    /**
-     * If the payload has a triggers-database or schemas-database, we consider it to depend on some other database. We
-     * don't check for a security database yet, as it's very rare to use a custom security database.
-     *
-     * @param payload
-     * @return
-     */
-    protected boolean payloadDependsOnOtherDatabase(String payload) {
-        return payload.contains("triggers-database") || payload.contains("schema-database");
-    }
+	private CommandContext context;
+	private boolean reverseOrder = false;
+	private ResourceMapper resourceMapper;
+
+	public DeployDatabaseCommandComparator(CommandContext context, boolean reverseOrder) {
+		this.context = context;
+		this.reverseOrder = reverseOrder;
+		this.resourceMapper = new DefaultResourceMapper(new API(null));
+	}
+
+	@Override
+	public int compare(DeployDatabaseCommand o1, DeployDatabaseCommand o2) {
+		String p1 = o1.buildPayload(context);
+		String p2 = o2.buildPayload(context);
+		if (p1 == null || p2 == null) {
+			return 0;
+		}
+
+		Database db1 = null, db2 = null;
+		try {
+			db1 = resourceMapper.readResource(p1, Database.class);
+			db2 = resourceMapper.readResource(p2, Database.class);
+		} catch (Exception ex) {
+			logger.warn("Unable to map database payload to Database object, " +
+				"will not be able to determine the right order in which to deploy this database; cause: " + ex.getMessage());
+			return 0;
+		}
+
+		// This should never happen, but just in case
+		if (db1 == null || db2 == null) {
+			return 0;
+		}
+
+		if (db1.dependsOnDatabase(db2)) {
+			return reverseOrder ? -1 : 1;
+		} else if (db2.dependsOnDatabase(db1)) {
+			return reverseOrder ? 1 : -1;
+		}
+		return 0;
+	}
 }

src/main/java/com/marklogic/appdeployer/command/databases/DeployOtherDatabasesCommand.java

@@ -14,11 +14,13 @@
  * uncommon to need to create additional databases (and perhaps REST API servers to go with them).
  * <p>
  * A key aspect of this class is its attempt to deploy/undeploy databases in the correct order. For each database file
- * that it finds that's not one of the default ones, a DeployDatabaseCommand will be created. All of those commands will
- * then be sorted based on the presence of "triggers-database" or "schema-database" within the payload for the command.
+ * that it finds that's not one of the default ones, a DeployDatabaseCommand will be created. These commands are then
+ * sorted based on their dependencies between each other (a database can depend on another database for schemas, triggers,
+ * or security).
  * <p>
  * If the above strategy doesn't work for you, you can always resort to naming your database files to control the order
- * that they're processed in.
+ * that they're processed in. Be sure to set "setSortOtherDatabasesByDependencies" to false in the AppConfig object that
+ * is passed in via a CommandContext to disable the sorting that this command will otherwise perform.
  * </p>
  */
 public class DeployOtherDatabasesCommand extends AbstractUndoableCommand {
@@ -48,7 +50,12 @@ public void execute(CommandContext context) {
     }
 
     protected void sortCommandsBeforeExecute(List<DeployDatabaseCommand> list, CommandContext context) {
-        Collections.sort(list, new DeployDatabaseCommandComparator(context, false));
+    	if (context.getAppConfig().isSortOtherDatabaseByDependencies()) {
+		    Collections.sort(list, new DeployDatabaseCommandComparator(context, false));
+	    }
+	    else {
+    		logger.info("Not sorting databases by dependencies; they will be sorted based on their filename instead");
+	    }
     }
 
     @Override

src/main/java/com/marklogic/mgmt/api/database/Database.java

@@ -166,6 +166,22 @@ public void detach(String forestName) {
         }
     }
 
+	/**
+	 *
+	 * @param other
+	 * @return true if the name of the other database equals the name of this database object's schema database,
+	 * triggers database, or security database
+	 */
+	public boolean dependsOnDatabase(Database other) {
+    	String otherName = other.getDatabaseName();
+
+    	if (otherName == null) {
+    		return false;
+	    }
+
+	    return otherName.equals(schemaDatabase) || otherName.equals(triggersDatabase) || otherName.equals(securityDatabase);
+    }
+
     public String getDatabaseName() {
         return databaseName;
     }

src/test/java/com/marklogic/appdeployer/DefaultAppConfigFactoryTest.java

@@ -40,7 +40,7 @@ public void moduleTimestampsPath() {
 	    AppConfig config = new DefaultAppConfigFactory(new SimplePropertySource(p)).newAppConfig();
 	    assertNull(config.getModuleTimestampsPath());
     }
-    
+
     @Test
     public void unrecognizedProperties() {
         sut = new DefaultAppConfigFactory(new SimplePropertySource("foo.mlHost", "host", "foo.mlUsername", "user"));
@@ -169,8 +169,6 @@ public void allProperties() {
 	    p.setProperty("mlDatabaseReplicaFastDataDirectories", "Documents,/fast/replicas,Security,/fast/security/replicas");
 	    p.setProperty("mlDatabaseReplicaLargeDataDirectories", "Documents,/large/replicas,Security,/large/security/replicas");
 
-	    p.setProperty("mlSortRolesByDependencies", "false");
-
 	    sut = new DefaultAppConfigFactory(new SimplePropertySource(p));
         AppConfig config = sut.newAppConfig();
 
@@ -308,8 +306,6 @@ public void allProperties() {
 	    map = config.getDatabaseReplicaLargeDataDirectories();
 	    assertEquals("/large/replicas", map.get("Documents"));
 	    assertEquals("/large/security/replicas", map.get("Security"));
-
-	    assertFalse(config.isSortRolesByDependencies());
     }
 
 	/**

src/test/java/com/marklogic/appdeployer/command/databases/DeployDatabasesInOrderTest.java

@@ -0,0 +1,52 @@
+package com.marklogic.appdeployer.command.databases;
+
+import com.marklogic.appdeployer.AbstractAppDeployerTest;
+import com.marklogic.appdeployer.ConfigDir;
+import com.marklogic.mgmt.resource.databases.DatabaseManager;
+import org.junit.After;
+import org.junit.Test;
+
+import java.io.File;
+
+public class DeployDatabasesInOrderTest extends AbstractAppDeployerTest {
+
+	@After
+	public void teardown() {
+		undeploySampleApp();
+
+		DatabaseManager mgr = new DatabaseManager(manageClient);
+		assertFalse(mgr.exists("sample-app-A"));
+		assertFalse(mgr.exists("sample-app-B"));
+		assertFalse(mgr.exists("sample-app-C"));
+	}
+
+	/**
+	 * Tests 3 files that have filenames that would lead them to be processed in the wrong order if the command
+	 * doesn't sort them correctly.
+	 */
+	@Test
+	public void test() {
+		appConfig.setConfigDir(new ConfigDir(new File("src/test/resources/sample-app/databases-in-order")));
+		initializeAppDeployer(new DeployOtherDatabasesCommand(1));
+		deploySampleApp();
+
+		DatabaseManager mgr = new DatabaseManager(manageClient);
+		assertTrue(mgr.exists("sample-app-A"));
+		assertTrue(mgr.exists("sample-app-B"));
+		assertTrue(mgr.exists("sample-app-C"));
+	}
+
+	@Test
+	public void notSortingProducesFailure() {
+		appConfig.setConfigDir(new ConfigDir(new File("src/test/resources/sample-app/databases-in-order")));
+		appConfig.setSortOtherDatabaseByDependencies(false);
+		initializeAppDeployer(new DeployOtherDatabasesCommand(1));
+
+		try {
+			deploySampleApp();
+			fail("Deploy should have failed because the databases were not sorted based on their dependencies");
+		} catch (Exception ex) {
+			logger.info("Caught expected exception: " + ex.getMessage());
+		}
+	}
+}

src/test/resources/sample-app/databases-in-order/databases/database-A.json

@@ -0,0 +1,5 @@
+{
+  "database-name": "%%NAME%%-A",
+	"schema-database": "%%NAME%%-C",
+	"triggers-database": "%%NAME%%-B"
+}

src/test/resources/sample-app/databases-in-order/databases/database-B.json

@@ -0,0 +1,5 @@
+{
+	"database-name": "%%NAME%%-B",
+	"schema-database": "%%NAME%%-C",
+	"triggers-database": "Triggers"
+}

src/test/resources/sample-app/databases-in-order/databases/database-C.json

@@ -0,0 +1,5 @@
+{
+	"database-name": "%%NAME%%-C",
+	"schema-database": "Schemas",
+	"triggers-database": "Triggers"
+}