#233 New property for setting batch size when loading modules

Author: rjrudin

gradle.properties

@@ -1,6 +1,6 @@
 group=com.marklogic
 javadocsDir=../gh-pages-marklogic-java/javadocs
-version=3.3.1
-mlJavaclientUtilVersion=3.3.1
+version=3.4-beta1
+mlJavaclientUtilVersion=3.4-beta1
 mlJunitVersion=3.1.0
 

src/main/java/com/marklogic/appdeployer/AppConfig.java

@@ -114,6 +114,7 @@ public class AppConfig {
 	private boolean deleteTestModules = false;
 	private String deleteTestModulesPattern = "/test/**";
 	private int modulesLoaderThreadCount = 8;
+	private Integer modulesLoaderBatchSize;
 
     private String schemasPath;
 
@@ -1186,4 +1187,12 @@ public boolean isSortOtherDatabaseByDependencies() {
 	public void setSortOtherDatabaseByDependencies(boolean sortOtherDatabaseByDependencies) {
 		this.sortOtherDatabaseByDependencies = sortOtherDatabaseByDependencies;
 	}
+
+	public Integer getModulesLoaderBatchSize() {
+		return modulesLoaderBatchSize;
+	}
+
+	public void setModulesLoaderBatchSize(Integer modulesLoaderBatchSize) {
+		this.modulesLoaderBatchSize = modulesLoaderBatchSize;
+	}
 }

src/main/java/com/marklogic/appdeployer/DefaultAppConfigFactory.java

@@ -588,6 +588,12 @@ public AppConfig newAppConfig() {
 			c.setModulesLoaderThreadCount(Integer.parseInt(prop));
 		}
 
+		prop = getProperty("mlModulesLoaderBatchSize");
+		if (prop != null) {
+			logger.info("Modules loader batch size: " + prop);
+			c.setModulesLoaderBatchSize(Integer.parseInt(prop));
+		}
+
 		/**
 		 * The following properties are all for generating Entity Services artifacts.
 		 */

src/main/java/com/marklogic/appdeployer/command/modules/DefaultModulesLoaderFactory.java

@@ -32,6 +32,9 @@ public ModulesLoader newModulesLoader(AppConfig appConfig) {
 		RestBatchWriter assetBatchWriter = new RestBatchWriter(appConfig.newModulesDatabaseClient(), false);
 		assetBatchWriter.setThreadCount(threadCount);
 		AssetFileLoader assetFileLoader = new AssetFileLoader(assetBatchWriter, modulesManager);
+		if (appConfig.getModulesLoaderBatchSize() != null) {
+			assetFileLoader.setBatchSize(appConfig.getModulesLoaderBatchSize());
+		}
 
 		String permissions = appConfig.getModulePermissions();
 		if (permissions != null) {

src/test/java/com/marklogic/appdeployer/DefaultAppConfigFactoryTest.java

@@ -138,6 +138,7 @@ public void allProperties() {
         p.setProperty("mlDeleteTestModules", "true");
         p.setProperty("mlDeleteTestModulesPattern", "/some/pattern");
         p.setProperty("mlModulesLoaderThreadCount", "3");
+        p.setProperty("mlModulesLoaderBatchSize", "79");
 
         p.setProperty("mlModelsPath", "ml/models");
         p.setProperty("mlInstanceConverterPath", "ext/my/path");
@@ -239,6 +240,7 @@ public void allProperties() {
         assertTrue(config.isDeleteTestModules());
         assertEquals("/some/pattern", config.getDeleteTestModulesPattern());
         assertEquals(3, config.getModulesLoaderThreadCount());
+        assertEquals(new Integer(79), config.getModulesLoaderBatchSize());
 
         assertEquals("ml/models", config.getModelsPath());
         assertEquals("ext/my/path", config.getInstanceConverterPath());

src/test/java/com/marklogic/appdeployer/command/modules/LoadModulesTest.java

@@ -71,6 +71,8 @@ public void customModuleTimestampsPath() {
 
 	@Test
 	public void loadModulesFromMultiplePaths() {
+		// Setting batch size to make sure nothing blows up
+		appConfig.setModulesLoaderBatchSize(1);
 		appConfig.getModulePaths().add("src/test/resources/sample-app/build/mlRestApi/some-library/ml-modules");
 
 		initializeAppDeployer(new DeployRestApiServersCommand(true), buildLoadModulesCommand());