#156 #157 #158 #159 #160 Can now export databases, servers, forests, users, roles, and privileges

Author: Rob Rudin

build.gradle

@@ -36,6 +36,8 @@ dependencies {
   testCompile ('com.marklogic:ml-junit:2.6.0') {
     exclude module: "ml-javaclient-util"
   }
+
+  testCompile('commons-io:commons-io:2.5')
 }
 
 // This ensures that Gradle includes in the published jar any non-java files under src/main/java

gradle.properties

@@ -1,3 +1,3 @@
 group=com.marklogic
 javadocsDir=../gh-pages-marklogic-java/javadocs
-version=2.6.0
+version=159

src/main/java/com/marklogic/appdeployer/ConfigDir.java

@@ -49,13 +49,19 @@ public File getRestApiFile() {
     }
 
     public File getRestApiServerFile() {
-        return new File(new File(baseDir, "servers"), "rest-api-server.json");
+        return new File(getServersDir(), "rest-api-server.json");
     }
 
     public File getSecurityDir() {
         return new File(baseDir, "security");
     }
 
+    public File getServersDir() { return new File(baseDir, "servers"); }
+
+    public File getForestsDir() {
+        return new File(baseDir, "forests");
+    }
+
     public File getCpfDir() {
         return new File(baseDir, "cpf");
     }
@@ -74,6 +80,10 @@ public File getFlexrepDir() {
 
 	public File getTemporalDir() { return new File(baseDir, "temporal"); }
 
+	public File getTasksDir() {
+    	return new File(baseDir, "tasks");
+	}
+
 	public void setDatabasesPath(String databasesPath) {
         this.databasesPath = databasesPath;
     }

src/main/java/com/marklogic/appdeployer/command/appservers/DeployOtherServersCommand.java

@@ -25,7 +25,7 @@ public DeployOtherServersCommand() {
 
     @Override
     protected File[] getResourceDirs(CommandContext context) {
-        return new File[] { new File(context.getAppConfig().getConfigDir().getBaseDir(), "servers") };
+        return new File[] { context.getAppConfig().getConfigDir().getServersDir() };
     }
 
     @Override

src/main/java/com/marklogic/appdeployer/command/databases/DeployDatabaseCommand.java

@@ -9,7 +9,6 @@
 import com.marklogic.mgmt.PayloadParser;
 import com.marklogic.mgmt.SaveReceipt;
 import com.marklogic.mgmt.databases.DatabaseManager;
-import com.marklogic.mgmt.forests.ForestManager;
 
 import java.io.File;
 import java.util.Map;
@@ -37,14 +36,23 @@ public class DeployDatabaseCommand extends AbstractCommand implements UndoableCo
      */
     private String databaseName;
 
+    /**
+     * If true, this will look for a ./forests/(name of database) directory that defines custom forests for this
+     * database. If such a directory exists, this command will not create any forests for the database. If the
+     * directory does not exist, then this command will create forests for the database as defined by forestsPerHost.
+     */
+    private boolean checkForCustomForests = true;
+
     /**
      * Optional name of the file in the forests directory that will be used to create each forest. If not provided, a
-     * "vanilla" forest is created on each host with a name based on the databaseName attribute.
+     * "vanilla" forest is created on each host with a name based on the databaseName attribute. This is ignored if
+     * custom forests are found in ./forests/(name of forest).
      */
     private String forestFilename;
 
     /**
-     * Number of forests to create per host for this database.
+     * Number of forests to create per host for this database. This is ignored if custom forests are found in
+     * ./forests/(name of forest).
      */
     private int forestsPerHost = 1;
 
@@ -81,7 +89,13 @@ public void execute(CommandContext context) {
         if (payload != null) {
             DatabaseManager dbMgr = new DatabaseManager(context.getManageClient());
             SaveReceipt receipt = dbMgr.save(payload);
-            buildDeployForestsCommand(payload, receipt, context).execute(context);
+            if (shouldCreateForests(context, payload)) {
+	            buildDeployForestsCommand(payload, receipt, context).execute(context);
+            } else {
+            	if (logger.isInfoEnabled()) {
+            		logger.info("Found custom forests for database, so not creating default forests");
+	            }
+            }
         }
     }
 
@@ -146,6 +160,40 @@ protected String getPayload(CommandContext context) {
         }
     }
 
+	/**
+	 * This is where we check to see if a custom forests directory exists at ./forests/(database name). The database
+	 * name is extracted from the payload via a PayloadParser. This check can be disabled by setting
+	 * checkForCustomForests to false.
+	 *
+	 * @param context
+	 * @param payload
+	 * @return
+	 */
+	protected boolean shouldCreateForests(CommandContext context, String payload) {
+		if (isCheckForCustomForests()) {
+			PayloadParser parser = new PayloadParser();
+			String dbName = parser.getPayloadFieldValue(payload, "database-name");
+			return !customForestsExist(context, dbName);
+		}
+		return true;
+	}
+
+	/**
+	 * @param context
+	 * @param dbName
+	 * @return true if any file exists in ./forests/(name of database)
+	 */
+	protected boolean customForestsExist(CommandContext context, String dbName) {
+    	File dir = context.getAppConfig().getConfigDir().getForestsDir();
+    	if (dir.exists()) {
+    		File dbDir = new File(dir, dbName);
+    		if (dbDir.exists()) {
+    			return dbDir.listFiles().length > 0;
+		    }
+	    }
+	    return false;
+    }
+
     /**
      * Allows for how an instance of DeployForestsCommand is built to be overridden by a subclass.
      * 
@@ -248,4 +296,12 @@ public boolean isCreateForestsOnEachHost() {
     public void setCreateForestsOnEachHost(boolean createForestsOnEachHost) {
         this.createForestsOnEachHost = createForestsOnEachHost;
     }
+
+    public boolean isCheckForCustomForests() {
+        return checkForCustomForests;
+    }
+
+    public void setCheckForCustomForests(boolean checkForCustomForests) {
+        this.checkForCustomForests = checkForCustomForests;
+    }
 }

src/main/java/com/marklogic/appdeployer/command/databases/DeployOtherDatabasesCommand.java

@@ -28,9 +28,19 @@
  */
 public class DeployOtherDatabasesCommand extends AbstractUndoableCommand {
 
+	// Each of these is copied to the instances of DeployDatabaseCommand that are created
+    private int forestsPerHost = 1;
+	private boolean checkForCustomForests = true;
+	private String forestFilename;
+	private boolean createForestsOnEachHost = true;
+
     public DeployOtherDatabasesCommand() {
-        setExecuteSortOrder(SortOrderConstants.DEPLOY_OTHER_DATABASES);
-        setUndoSortOrder(SortOrderConstants.DELETE_OTHER_DATABASES);
+        this(1);
+    }
+
+    public DeployOtherDatabasesCommand(int forestsPerHost) {
+	    setExecuteSortOrder(SortOrderConstants.DEPLOY_OTHER_DATABASES);
+	    setUndoSortOrder(SortOrderConstants.DELETE_OTHER_DATABASES);
     }
 
     @Override
@@ -69,14 +79,22 @@ protected List<DeployDatabaseCommand> buildDatabaseCommands(CommandContext conte
                 if (logger.isDebugEnabled()) {
                     logger.debug("Will process other database in file: " + f.getName());
                 }
-                DeployDatabaseCommand c = new DeployDatabaseCommand();
-                c.setDatabaseFilename(f.getName());
-                dbCommands.add(c);
+                dbCommands.add(buildDeployDatabaseCommand(f));
             }
         }
         return dbCommands;
     }
 
+    protected DeployDatabaseCommand buildDeployDatabaseCommand(File file) {
+	    DeployDatabaseCommand c = new DeployDatabaseCommand();
+	    c.setForestsPerHost(getForestsPerHost());
+	    c.setDatabaseFilename(file.getName());
+	    c.setCheckForCustomForests(isCheckForCustomForests());
+	    c.setForestFilename(getForestFilename());
+	    c.setCreateForestsOnEachHost(isCreateForestsOnEachHost());
+	    return c;
+    }
+
     /**
      * Adds to the list of resource filenames to ignore. Some may already have been set via the superclass method.
      *
@@ -91,4 +109,36 @@ protected void ignoreKnownDatabaseFilenames(CommandContext context) {
         ignore.add(DeployTriggersDatabaseCommand.DATABASE_FILENAME);
         setFilenamesToIgnore(ignore.toArray(new String[]{}));
     }
+
+	public int getForestsPerHost() {
+		return forestsPerHost;
+	}
+
+	public void setForestsPerHost(int forestsPerHost) {
+		this.forestsPerHost = forestsPerHost;
+	}
+
+	public boolean isCheckForCustomForests() {
+		return checkForCustomForests;
+	}
+
+	public void setCheckForCustomForests(boolean checkForCustomForests) {
+		this.checkForCustomForests = checkForCustomForests;
+	}
+
+	public String getForestFilename() {
+		return forestFilename;
+	}
+
+	public void setForestFilename(String forestFilename) {
+		this.forestFilename = forestFilename;
+	}
+
+	public boolean isCreateForestsOnEachHost() {
+		return createForestsOnEachHost;
+	}
+
+	public void setCreateForestsOnEachHost(boolean createForestsOnEachHost) {
+		this.createForestsOnEachHost = createForestsOnEachHost;
+	}
 }

src/main/java/com/marklogic/appdeployer/command/forests/DeployCustomForestsCommand.java

@@ -3,6 +3,7 @@
 import com.marklogic.appdeployer.command.AbstractCommand;
 import com.marklogic.appdeployer.command.CommandContext;
 import com.marklogic.appdeployer.command.SortOrderConstants;
+import com.marklogic.mgmt.PayloadParser;
 import com.marklogic.mgmt.forests.ForestManager;
 
 import java.io.File;
@@ -20,6 +21,7 @@
 public class DeployCustomForestsCommand extends AbstractCommand {
 
 	private String customForestsPath = "forests";
+	private PayloadParser payloadParser;
 
 	public DeployCustomForestsCommand() {
 		setExecuteSortOrder(SortOrderConstants.DEPLOY_FORESTS);
@@ -29,6 +31,7 @@ public DeployCustomForestsCommand() {
 	public void execute(CommandContext context) {
 		File dir = new File(context.getAppConfig().getConfigDir().getBaseDir(), customForestsPath);
 		if (dir != null && dir.exists()) {
+			payloadParser = new PayloadParser();
 			for (File f : dir.listFiles()) {
 				if (f.isDirectory()) {
 					processDirectory(f, context);
@@ -37,6 +40,12 @@ public void execute(CommandContext context) {
 		}
 	}
 
+	/**
+	 * Supports JSON files with a single payload or an array of payloads, or an XML file with a single payload.
+	 *
+	 * @param dir
+	 * @param context
+	 */
 	protected void processDirectory(File dir, CommandContext context) {
 		if (logger.isInfoEnabled()) {
 			logger.info("Processing custom forest files in directory: " + dir.getAbsolutePath());
@@ -47,7 +56,11 @@ protected void processDirectory(File dir, CommandContext context) {
 				logger.info("Processing forests in file: " + f.getAbsolutePath());
 			}
 			String payload = copyFileToString(f, context);
-			mgr.saveJsonForests(payload);
+			if (payloadParser.isJsonPayload(payload)) {
+				mgr.saveJsonForests(payload);
+			} else {
+				mgr.save(payload);
+			}
 		}
 	}
 

src/main/java/com/marklogic/appdeployer/command/forests/DeployForestsCommand.java

@@ -46,7 +46,7 @@ public DeployForestsCommand() {
     public void execute(CommandContext context) {
         String payload = null;
         if (forestFilename != null) {
-            File dir = new File(context.getAppConfig().getConfigDir().getBaseDir(), "forests");
+            File dir = context.getAppConfig().getConfigDir().getForestsDir();
             if (dir.exists()) {
                 File f = new File(dir, forestFilename);
                 if (f.exists()) {

src/main/java/com/marklogic/appdeployer/command/tasks/DeployScheduledTasksCommand.java

@@ -20,7 +20,7 @@ public DeployScheduledTasksCommand() {
 
     @Override
     protected File[] getResourceDirs(CommandContext context) {
-        return new File[] { new File(context.getAppConfig().getConfigDir().getBaseDir(), "tasks") };
+        return new File[] { context.getAppConfig().getConfigDir().getTasksDir() };
     }
 
     @Override
@@ -88,4 +88,4 @@ class IsDirectoryFilter implements FileFilter {
 	public boolean accept(File pathname) {
 		return pathname != null && pathname.isDirectory();
 	}
-}
+}