Merge pull request #306 from marklogic-community/feature/286-billfarber

Adding support for Gradle-style incremental deployments

Author: rjrudin

gradle.properties

@@ -1,6 +1,6 @@
 group=com.marklogic
 javadocsDir=../gh-pages-marklogic-java/javadocs
-version=3.9.0
+version=3.10.dev
 mlJavaclientUtilVersion=3.8.1
 mlJunitVersion=3.1.0
 

src/main/java/com/marklogic/appdeployer/AppConfig.java

@@ -121,6 +121,7 @@ public class AppConfig {
 	private String deleteTestModulesPattern = "/test/**";
 	private int modulesLoaderThreadCount = 4;
 	private Integer modulesLoaderBatchSize;
+	private boolean incrementalDeploy = false;
 
     private String schemasPath;
 
@@ -1184,6 +1185,14 @@ public void setModulesLoaderBatchSize(Integer modulesLoaderBatchSize) {
 		this.modulesLoaderBatchSize = modulesLoaderBatchSize;
 	}
 
+	public boolean isIncrementalDeploy() {
+		return incrementalDeploy;
+	}
+
+	public void setIncrementalDeploy(boolean incrementalDeploy) {
+		this.incrementalDeploy = incrementalDeploy;
+	}
+
 	public String getModelsDatabase() {
 		return modelsDatabase;
 	}

src/main/java/com/marklogic/appdeployer/DefaultAppConfigFactory.java

@@ -626,6 +626,11 @@ public void initialize() {
 			config.setIncludeProperties(values);
 		});
 
+		propertyConsumerMap.put("mlIncremental", (config, prop) -> {
+			logger.info("Supported resources will only be deployed if their resource files are new or have been modified since the last deployment: " + prop);
+			config.setIncrementalDeploy(Boolean.parseBoolean(prop));
+		});
+    
 		propertyConsumerMap.put("mlUpdateMimetypeWhenPropertiesAreEqual", (config, prop) -> {
 			logger.info("Update mimetype when properties are equal (defaults to false to avoid unnecessary ML restarts): " + prop);
 			config.setUpdateMimetypeWhenPropertiesAreEqual(Boolean.parseBoolean(prop));

src/main/java/com/marklogic/appdeployer/command/AbstractCommand.java

@@ -16,7 +16,6 @@
 import java.net.URI;
 import java.util.Arrays;
 import java.util.HashSet;
-import java.util.Map;
 import java.util.Set;
 import java.util.regex.Pattern;
 
@@ -282,6 +281,24 @@ protected boolean shouldOptimizeWithCma(CommandContext context) {
 	    return false;
     }
 
+	protected void ignoreIncrementalCheckForFile(File file) {
+    	if (resourceFilenameFilter instanceof IncrementalFilenameFilter) {
+		    ((IncrementalFilenameFilter) resourceFilenameFilter).ignoreIncrementalCheckForFile(file);
+	    } else {
+    		logger.warn("resourceFilenameFilter does not implement " + IncrementalFilenameFilter.class.getName() + ", and thus " +
+			    "ignoreIncrementalCheckForFile for file " + file.getAbsolutePath() + " cannot be invoked");
+	    }
+	}
+
+	protected void setIncrementalMode(boolean incrementalMode) {
+		if (resourceFilenameFilter instanceof IncrementalFilenameFilter) {
+			((IncrementalFilenameFilter) resourceFilenameFilter).setIncrementalMode(incrementalMode);
+		} else {
+			logger.warn("resourceFilenameFilter does not implement " + IncrementalFilenameFilter.class.getName() + ", and thus " +
+				"setIncrementalMode cannot be invoked");
+		}
+	}
+
     public void setPayloadTokenReplacer(PayloadTokenReplacer payloadTokenReplacer) {
         this.payloadTokenReplacer = payloadTokenReplacer;
     }
@@ -297,4 +314,8 @@ public void setStoreResourceIdsAsCustomTokens(boolean storeResourceIdsAsCustomTo
     public void setResourceFilenameFilter(FilenameFilter resourceFilenameFilter) {
         this.resourceFilenameFilter = resourceFilenameFilter;
     }
+
+	public FilenameFilter getResourceFilenameFilter() {
+		return resourceFilenameFilter;
+	}
 }

src/main/java/com/marklogic/appdeployer/command/AbstractResourceCommand.java

@@ -2,15 +2,11 @@
 
 import com.marklogic.appdeployer.AppConfig;
 import com.marklogic.appdeployer.ConfigDir;
-import com.marklogic.mgmt.admin.AdminManager;
-import com.marklogic.mgmt.resource.ResourceManager;
 import com.marklogic.mgmt.SaveReceipt;
 import com.marklogic.mgmt.admin.ActionRequiringRestart;
-import org.springframework.http.HttpHeaders;
-import org.springframework.http.ResponseEntity;
+import com.marklogic.mgmt.resource.ResourceManager;
 
 import java.io.File;
-import java.net.URI;
 import java.util.ArrayList;
 import java.util.List;
 
@@ -30,6 +26,7 @@ public abstract class AbstractResourceCommand extends AbstractUndoableCommand {
 
     @Override
     public void execute(CommandContext context) {
+    	setIncrementalMode(context.getAppConfig().isIncrementalDeploy());
         for (File resourceDir : getResourceDirs(context)) {
             processExecuteOnResourceDir(context, resourceDir);
         }
@@ -100,6 +97,7 @@ protected File[] listFilesInDirectory(File resourceDir, CommandContext context)
     @Override
     public void undo(CommandContext context) {
         if (deleteResourcesOnUndo) {
+        	setIncrementalMode(false);
             for (File resourceDir : getResourceDirs(context)) {
                 processUndoOnResourceDir(context, resourceDir);
             }

src/main/java/com/marklogic/appdeployer/command/IncrementalFilenameFilter.java

@@ -0,0 +1,26 @@
+package com.marklogic.appdeployer.command;
+
+import java.io.File;
+import java.io.FilenameFilter;
+
+/**
+ * Extends FilenameFilter to add support for incremental deployments, where only resources that are new or modified since
+ * a previous deployment will be deployed.
+ */
+public interface IncrementalFilenameFilter extends FilenameFilter {
+
+	/**
+	 * If the given File is processed during a deployment, do not perform an incremental check on it - i.e. essentially
+	 * act as though incremental mode is disabled.
+	 *
+	 * @param resourceFile
+	 */
+	void ignoreIncrementalCheckForFile(File resourceFile);
+
+	/**
+	 * Toggle whether this file performs any incremental check.
+	 *
+	 * @param incrementalMode
+	 */
+	void setIncrementalMode(boolean incrementalMode);
+}

src/main/java/com/marklogic/appdeployer/command/ResourceFileManager.java

@@ -0,0 +1,20 @@
+package com.marklogic.appdeployer.command;
+
+import java.io.File;
+
+/**
+ * Defines operations for whether a resource file should be processed or not during a deployment.
+ */
+public interface ResourceFileManager {
+
+	/**
+	 * Give the implementor a chance to initialize itself - e.g. loading data from a properties file or other resource.
+	 */
+	void initialize();
+
+	/**
+	 * @param file
+	 * @return
+	 */
+	boolean shouldResourceFileBeProcessed(File file);
+}

src/main/java/com/marklogic/appdeployer/command/ResourceFileManagerImpl.java

@@ -0,0 +1,51 @@
+package com.marklogic.appdeployer.command;
+
+import com.marklogic.client.ext.modulesloader.impl.PropertiesModuleManager;
+
+import java.io.File;
+import java.util.Date;
+
+/**
+ * Extends the module-specific PropertiesModuleManager, as that class's behavior is exactly what's needed here as well.
+ * Just uses a different default file path for the properties file that keeps track of timestamps.
+ */
+public class ResourceFileManagerImpl extends PropertiesModuleManager implements ResourceFileManager {
+
+	public static final String DEFAULT_FILE_PATH = "build/com.marklogic.ml-app-deployer/resource-timestamps.properties";
+
+	public ResourceFileManagerImpl() {
+		this(DEFAULT_FILE_PATH);
+	}
+
+	public ResourceFileManagerImpl(String propertiesFilePath) {
+		super(propertiesFilePath);
+	}
+
+	/**
+	 * One difference between this and the parent class is that if the resource file should be processed, its timestamp
+	 * in the properties file is immediately updated, as opposed to that being done via a separate method.
+	 * <p>
+	 * Note that this doesn't indicate that the resource file was successfully deployed. If an error occurs while
+	 * deploying the resource, it is expected that a developer would then make a change to the resource file. If that
+	 * doesn't occur - e.g. if the failure occurred due to an authentication issue, and thus the resource file doesn't
+	 * have to be changed - the developer would need to turn off incremental support or delete the properties file that
+	 * contains the timestamps.
+	 *
+	 * @param file
+	 * @return
+	 */
+	@Override
+	public boolean shouldResourceFileBeProcessed(File file) {
+		boolean shouldBeProcessed = hasFileBeenModifiedSinceLastLoaded(file);
+		if (shouldBeProcessed) {
+			if (logger.isDebugEnabled()) {
+				logger.debug("File is new or has been modified: " + file.getAbsolutePath());
+			}
+			super.saveLastLoadedTimestamp(file, new Date());
+		} else if (logger.isInfoEnabled()) {
+			logger.info("File is neither new nor modified: " + file.getAbsolutePath());
+		}
+		return shouldBeProcessed;
+	}
+
+}