#286 Adding tests for incremental support

Author: rjrudin

gradle.properties

@@ -1,6 +1,6 @@
 group=com.marklogic
 javadocsDir=../gh-pages-marklogic-java/javadocs
-version=3.9.0
+version=3.10.dev
 mlJavaclientUtilVersion=3.8.1
 mlJunitVersion=3.1.0
 

src/main/java/com/marklogic/appdeployer/DefaultAppConfigFactory.java

@@ -627,9 +627,9 @@ public void initialize() {
 		});
 
 		propertyConsumerMap.put("mlIncremental", (config, prop) -> {
-			logger.info("Supported resources will only be deployed if their resource files are now or have been modified since the last deployment: " + prop);
+			logger.info("Supported resources will only be deployed if their resource files are new or have been modified since the last deployment: " + prop);
 			config.setIncrementalDeploy(Boolean.parseBoolean(prop));
-    });
+		});
 
 		propertyConsumerMap.put("mlUpdateMimetypeWhenPropertiesAreEqual", (config, prop) -> {
 			logger.info("Update mimetype when properties are equal (defaults to false to avoid unnecessary ML restarts): " + prop);

src/main/java/com/marklogic/appdeployer/command/AbstractCommand.java

@@ -11,6 +11,7 @@
 import org.springframework.util.FileCopyUtils;
 
 import java.io.File;
+import java.io.FilenameFilter;
 import java.io.IOException;
 import java.net.URI;
 import java.util.Arrays;
@@ -28,7 +29,7 @@ public abstract class AbstractCommand extends LoggingObject implements Command {
     private boolean storeResourceIdsAsCustomTokens = false;
 
     protected PayloadTokenReplacer payloadTokenReplacer = new DefaultPayloadTokenReplacer();
-    private IncrementalFilenameFilter resourceFilenameFilter = new ResourceFilenameFilter();
+    private FilenameFilter resourceFilenameFilter = new ResourceFilenameFilter();
     private PayloadParser payloadParser = new PayloadParser();
 
     /**
@@ -280,6 +281,24 @@ protected boolean shouldOptimizeWithCma(CommandContext context) {
 	    return false;
     }
 
+	protected void ignoreIncrementalCheckForFile(File file) {
+    	if (resourceFilenameFilter instanceof IncrementalFilenameFilter) {
+		    ((IncrementalFilenameFilter) resourceFilenameFilter).ignoreIncrementalCheckForFile(file);
+	    } else {
+    		logger.warn("resourceFilenameFilter does not implement " + IncrementalFilenameFilter.class.getName() + ", and thus " +
+			    "ignoreIncrementalCheckForFile for file " + file.getAbsolutePath() + " cannot be invoked");
+	    }
+	}
+
+	protected void setIncrementalMode(boolean incrementalMode) {
+		if (resourceFilenameFilter instanceof IncrementalFilenameFilter) {
+			((IncrementalFilenameFilter) resourceFilenameFilter).setIncrementalMode(incrementalMode);
+		} else {
+			logger.warn("resourceFilenameFilter does not implement " + IncrementalFilenameFilter.class.getName() + ", and thus " +
+				"setIncrementalMode cannot be invoked");
+		}
+	}
+
     public void setPayloadTokenReplacer(PayloadTokenReplacer payloadTokenReplacer) {
         this.payloadTokenReplacer = payloadTokenReplacer;
     }
@@ -292,15 +311,11 @@ public void setStoreResourceIdsAsCustomTokens(boolean storeResourceIdsAsCustomTo
         this.storeResourceIdsAsCustomTokens = storeResourceIdsAsCustomTokens;
     }
 
-    public void setResourceFilenameFilter(IncrementalFilenameFilter resourceFilenameFilter) {
+    public void setResourceFilenameFilter(FilenameFilter resourceFilenameFilter) {
         this.resourceFilenameFilter = resourceFilenameFilter;
     }
 
-    protected void ignoreHashForFilename(String filename) {
-		((ResourceFilenameFilter) resourceFilenameFilter).addFilenameToIgnoreHash(filename);
-	}
-
-	protected void setIncrementalMode(Boolean incrementalMode) {
-		resourceFilenameFilter.setIncrementalMode(incrementalMode);
+	public FilenameFilter getResourceFilenameFilter() {
+		return resourceFilenameFilter;
 	}
 }

src/main/java/com/marklogic/appdeployer/command/AbstractResourceCommand.java

@@ -2,9 +2,9 @@
 
 import com.marklogic.appdeployer.AppConfig;
 import com.marklogic.appdeployer.ConfigDir;
-import com.marklogic.mgmt.resource.ResourceManager;
 import com.marklogic.mgmt.SaveReceipt;
 import com.marklogic.mgmt.admin.ActionRequiringRestart;
+import com.marklogic.mgmt.resource.ResourceManager;
 
 import java.io.File;
 import java.util.ArrayList;
@@ -26,7 +26,7 @@ public abstract class AbstractResourceCommand extends AbstractUndoableCommand {
 
     @Override
     public void execute(CommandContext context) {
-		setIncrementalMode(context.getAppConfig().isIncrementalDeploy());
+    	setIncrementalMode(context.getAppConfig().isIncrementalDeploy());
         for (File resourceDir : getResourceDirs(context)) {
             processExecuteOnResourceDir(context, resourceDir);
         }
@@ -96,8 +96,8 @@ protected File[] listFilesInDirectory(File resourceDir, CommandContext context)
 
     @Override
     public void undo(CommandContext context) {
-		setIncrementalMode(false);
-		if (deleteResourcesOnUndo) {
+        if (deleteResourcesOnUndo) {
+        	setIncrementalMode(false);
             for (File resourceDir : getResourceDirs(context)) {
                 processUndoOnResourceDir(context, resourceDir);
             }

src/main/java/com/marklogic/appdeployer/command/IncrementalFilenameFilter.java

@@ -1,12 +1,26 @@
 package com.marklogic.appdeployer.command;
 
+import java.io.File;
 import java.io.FilenameFilter;
 
+/**
+ * Extends FilenameFilter to add support for incremental deployments, where only resources that are new or modified since
+ * a previous deployment will be deployed.
+ */
 public interface IncrementalFilenameFilter extends FilenameFilter {
 
-	public void addFilenameToIgnoreHash(String filename);
+	/**
+	 * If the given File is processed during a deployment, do not perform an incremental check on it - i.e. essentially
+	 * act as though incremental mode is disabled.
+	 *
+	 * @param resourceFile
+	 */
+	void ignoreIncrementalCheckForFile(File resourceFile);
 
-	public void clearFilenamesToIgnoreHash();
-
-	public void setIncrementalMode(boolean incrementalMode);
+	/**
+	 * Toggle whether this file performs any incremental check.
+	 *
+	 * @param incrementalMode
+	 */
+	void setIncrementalMode(boolean incrementalMode);
 }

src/main/java/com/marklogic/appdeployer/command/ResourceFileManager.java

@@ -3,16 +3,18 @@
 import java.io.File;
 
 /**
- * Defines operations for whether a resource file should be deployed or not.
+ * Defines operations for whether a resource file should be processed or not during a deployment.
  */
 public interface ResourceFileManager {
 
-    /**
-     * Give the implementor a chance to initialize itself - e.g. loading data from a properties file or other resource.
-     */
-    void initialize();
+	/**
+	 * Give the implementor a chance to initialize itself - e.g. loading data from a properties file or other resource.
+	 */
+	void initialize();
 
-    boolean hasFileBeenModifiedSinceLastDeployed(File file);
-
-    void saveLastDeployedHash(File file);
+	/**
+	 * @param file
+	 * @return
+	 */
+	boolean shouldResourceFileBeProcessed(File file);
 }

src/main/java/com/marklogic/appdeployer/command/ResourceFileManagerImpl.java

@@ -1,114 +1,51 @@
 package com.marklogic.appdeployer.command;
 
-import com.marklogic.client.ext.helper.LoggingObject;
+import com.marklogic.client.ext.modulesloader.impl.PropertiesModuleManager;
 
 import java.io.File;
-import java.io.FileInputStream;
-import java.io.FileWriter;
-import java.util.Properties;
+import java.util.Date;
 
 /**
- * Defines operations for managing whether a module needs to be installed or not.
+ * Extends the module-specific PropertiesModuleManager, as that class's behavior is exactly what's needed here as well.
+ * Just uses a different default file path for the properties file that keeps track of timestamps.
  */
-public class ResourceFileManagerImpl extends LoggingObject implements ResourceFileManager {
+public class ResourceFileManagerImpl extends PropertiesModuleManager implements ResourceFileManager {
 
-	public static final String DEFAULT_FILE_PATH = "build/ml-gradle/resource-timestamps.properties";
-
-	private Properties props;
-	private String propertiesFilePath;
+	public static final String DEFAULT_FILE_PATH = "build/com.marklogic.ml-app-deployer/resource-timestamps.properties";
 
 	public ResourceFileManagerImpl() {
 		this(DEFAULT_FILE_PATH);
 	}
 
 	public ResourceFileManagerImpl(String propertiesFilePath) {
-		props = new Properties();
-		this.propertiesFilePath = propertiesFilePath;
-		initialize();
-	}
-
-	@Override
-	public void initialize() {
-		File propertiesFile = new File(propertiesFilePath);
-		logger.info("Loading properties from: " + propertiesFile.getAbsolutePath());
-		propertiesFile.getParentFile().mkdirs();
-		if (propertiesFile.exists()) {
-			FileInputStream fis = null;
-			try {
-				fis = new FileInputStream(propertiesFile);
-				if (logger.isDebugEnabled()) {
-					logger.debug("Loading properties from: " + propertiesFile.getAbsolutePath());
-				}
-				props.load(fis);
-			} catch (Exception e) {
-				logger.warn("Unable to load properties, cause: " + e.getMessage());
-			} finally {
-				try {
-					fis.close();
-				} catch (Exception e) {
-					logger.warn(e.getMessage());
-				}
-			}
-		}
-	}
-
-	@Override
-	public boolean hasFileBeenModifiedSinceLastDeployed(File file) {
-			String key = buildKey(file);
-			Long lastFileTimestamp = file.lastModified();
-			String lastDeployedTimestampPropertyValue = props.getProperty(key);
-			if (lastDeployedTimestampPropertyValue != null) {
-				Long lastDeployedTimestamp = null;
-				try {
-					lastDeployedTimestamp = Long.parseLong(lastDeployedTimestampPropertyValue);
-				} catch (Exception e) {}
-				if (lastDeployedTimestamp != null) {
-					if (logger.isDebugEnabled()) {
-						logger.debug("lastDeployedTimestamp: " + lastDeployedTimestamp);
-						logger.debug("key: " + key);
-						logger.debug("lastFileTimestamp: " + lastFileTimestamp);
-					}
-					if (lastFileTimestamp != null) {
-						return (lastFileTimestamp > lastDeployedTimestamp);
-					} else {
-						return true;
-					}
-				} else {
-					return true;
-				}
-			} else {
-				return true;
-			}
-	}
-
-	@Override
-	public void saveLastDeployedHash(File file) {
-		String key = buildKey(file);
-		props.setProperty(key, file.lastModified() + "");
-		FileWriter fw = null;
-		try {
-			fw = new FileWriter(new File(propertiesFilePath));
-			props.store(fw, "");
-		} catch (Exception e) {
-			logger.warn("Unable to store properties, cause: " + e.getMessage());
-		} finally {
-			try {
-				fw.close();
-			} catch (Exception e) {
-				logger.warn(e.getMessage());
-			}
-		}
+		super(propertiesFilePath);
 	}
 
 	/**
-	 * Lower-casing avoids some annoying issues on Windows where sometimes you get "C:" at the start, and other times
-	 * you get "c:". This of course will be a problem if you for some reason have modules with the same names but
-	 * differing in some cases, but I'm not sure why anyone would do that.
+	 * One difference between this and the parent class is that if the resource file should be processed, its timestamp
+	 * in the properties file is immediately updated, as opposed to that being done via a separate method.
+	 * <p>
+	 * Note that this doesn't indicate that the resource file was successfully deployed. If an error occurs while
+	 * deploying the resource, it is expected that a developer would then make a change to the resource file. If that
+	 * doesn't occur - e.g. if the failure occurred due to an authentication issue, and thus the resource file doesn't
+	 * have to be changed - the developer would need to turn off incremental support or delete the properties file that
+	 * contains the timestamps.
 	 *
 	 * @param file
 	 * @return
 	 */
-	protected String buildKey(File file) {
-		return file.getAbsolutePath().toLowerCase();
+	@Override
+	public boolean shouldResourceFileBeProcessed(File file) {
+		boolean shouldBeProcessed = hasFileBeenModifiedSinceLastLoaded(file);
+		if (shouldBeProcessed) {
+			if (logger.isDebugEnabled()) {
+				logger.debug("File is new or has been modified: " + file.getAbsolutePath());
+			}
+			super.saveLastLoadedTimestamp(file, new Date());
+		} else if (logger.isInfoEnabled()) {
+			logger.info("File is neither new nor modified: " + file.getAbsolutePath());
+		}
+		return shouldBeProcessed;
 	}
+
 }