#333 TDE templates are now validated by default when loading schemas

Author: rjrudin

gradle.properties

@@ -1,6 +1,6 @@
 group=com.marklogic
 javadocsDir=../gh-pages-marklogic-java/javadocs
-version=3.12.0
-mlJavaclientUtilVersion=3.10.0
+version=3.13.dev
+mlJavaclientUtilVersion=3.11.dev
 mlJunitVersion=3.1.0
 

src/main/java/com/marklogic/appdeployer/AppConfig.java

@@ -125,6 +125,7 @@ public class AppConfig {
 	private boolean incrementalDeploy = false;
 
     private String schemasPath;
+    private boolean tdeValidationEnabled = true;
 
     private List<ConfigDir> configDirs;
 
@@ -1286,4 +1287,12 @@ public ReplicaBuilderStrategy getReplicaBuilderStrategy() {
 	public void setReplicaBuilderStrategy(ReplicaBuilderStrategy replicaBuilderStrategy) {
 		this.replicaBuilderStrategy = replicaBuilderStrategy;
 	}
+
+	public boolean isTdeValidationEnabled() {
+		return tdeValidationEnabled;
+	}
+
+	public void setTdeValidationEnabled(boolean tdeValidationEnabled) {
+		this.tdeValidationEnabled = tdeValidationEnabled;
+	}
 }

src/main/java/com/marklogic/appdeployer/DefaultAppConfigFactory.java

@@ -270,6 +270,11 @@ public void initialize() {
 			config.setSchemasPath(path);
 		});
 
+		propertyConsumerMap.put("mlTdeValidationEnabled", (config, prop) -> {
+			logger.info("TDE validation enabled: " + prop);
+			config.setTdeValidationEnabled(Boolean.parseBoolean(prop));
+		});
+
 		propertyConsumerMap.put("mlSchemasDatabaseName", (config, prop) -> {
 			logger.info("Schemas database name: " + prop);
 			config.setSchemasDatabaseName(prop);

src/main/java/com/marklogic/appdeployer/command/schemas/LoadSchemasCommand.java

@@ -1,13 +1,18 @@
 package com.marklogic.appdeployer.command.schemas;
 
 import com.marklogic.appdeployer.AppConfig;
+import com.marklogic.appdeployer.ConfigDir;
 import com.marklogic.appdeployer.command.AbstractCommand;
 import com.marklogic.appdeployer.command.CommandContext;
 import com.marklogic.appdeployer.command.SortOrderConstants;
 import com.marklogic.client.DatabaseClient;
 import com.marklogic.client.FailedRequestException;
 import com.marklogic.client.ext.schemasloader.SchemasLoader;
 import com.marklogic.client.ext.schemasloader.impl.DefaultSchemasLoader;
+import com.marklogic.mgmt.api.API;
+import com.marklogic.mgmt.api.database.Database;
+import com.marklogic.mgmt.mapper.DefaultResourceMapper;
+import com.marklogic.mgmt.mapper.ResourceMapper;
 
 import java.io.File;
 import java.io.FileFilter;
@@ -61,7 +66,7 @@ protected void loadSchemas(String schemasPath, String schemasDatabaseName, Comma
 		logger.info(format("Loading schemas into database %s from: %s", schemasDatabaseName, schemasPath));
 		DatabaseClient client = buildDatabaseClient(schemasDatabaseName, context);
 		try {
-			SchemasLoader schemasLoader = buildSchemasLoader(context, client);
+			SchemasLoader schemasLoader = buildSchemasLoader(context, client, schemasDatabaseName);
 			schemasLoader.loadSchemas(schemasPath);
 			logger.info("Finished loading schemas from: " + schemasPath);
 		} catch (FailedRequestException fre) {
@@ -81,18 +86,64 @@ protected DatabaseClient buildDatabaseClient(String schemasDatabaseName, Command
 
 	/**
 	 * Will utilize schemasFileFilter in AppConfig if it's been set.
+	 * <p>
+	 * So given a schemasDatabaseName,
 	 *
 	 * @param context
 	 * @param client
 	 * @return
 	 */
-	protected SchemasLoader buildSchemasLoader(CommandContext context, DatabaseClient client) {
-		AppConfig config = context.getAppConfig();
-		DefaultSchemasLoader schemasLoader = new DefaultSchemasLoader(client);
-		FileFilter filter = config.getSchemasFileFilter();
+	protected SchemasLoader buildSchemasLoader(CommandContext context, DatabaseClient client, String schemasDatabaseName) {
+		AppConfig appConfig = context.getAppConfig();
+
+		String tdeValidationDatabase = null;
+		if (appConfig.isTdeValidationEnabled()) {
+			tdeValidationDatabase = findContentDatabaseAssociatedWithSchemasDatabase(context, schemasDatabaseName);
+			if (tdeValidationDatabase != null) {
+				logger.info(format("TDE templates loaded into %s will be validated against content database %s",
+					schemasDatabaseName, tdeValidationDatabase));
+			}
+		} else {
+			logger.info("TDE validation is disabled");
+		}
+
+		DefaultSchemasLoader schemasLoader = new DefaultSchemasLoader(client, tdeValidationDatabase);
+		FileFilter filter = appConfig.getSchemasFileFilter();
 		if (filter != null) {
 			schemasLoader.addFileFilter(filter);
 		}
 		return schemasLoader;
 	}
+
+	/**
+	 * In order to validate a TDE template, a content database must be found that is associated with the given
+	 * schemas database. This method looks at the databases directory in each config directory and tries to find a
+	 * database resource file that defines a database associated with the given schemas database.
+	 *
+	 * @param context
+	 * @param schemasDatabaseName
+	 * @return
+	 */
+	protected String findContentDatabaseAssociatedWithSchemasDatabase(CommandContext context, String schemasDatabaseName) {
+		String tdeValidationDatabase = null;
+		ResourceMapper resourceMapper = new DefaultResourceMapper(new API(context.getManageClient()));
+		for (ConfigDir configDir : context.getAppConfig().getConfigDirs()) {
+			File dbDir = configDir.getDatabasesDir();
+			if (dbDir != null && dbDir.exists()) {
+				for (File f : listFilesInDirectory(dbDir)) {
+					String payload = copyFileToString(f, context);
+					try {
+						Database db = resourceMapper.readResource(payload, Database.class);
+						if (schemasDatabaseName.equals(db.getSchemaDatabase())) {
+							tdeValidationDatabase = db.getDatabaseName();
+							break;
+						}
+					} catch (Exception ex) {
+						logger.warn("Unexpected error when reading database file to determine database for TDE validation: " + ex.getMessage());
+					}
+				}
+			}
+		}
+		return tdeValidationDatabase;
+	}
 }

src/test/java/com/marklogic/appdeployer/DefaultAppConfigFactoryTest.java

@@ -211,6 +211,7 @@ public void allProperties() {
 		p.setProperty("mlSchemasDatabaseName", "my-schemas-db");
 		p.setProperty("mlTriggersDatabaseName", "my-triggers-db");
 		p.setProperty("mlSchemasPath", "/my/schemas");
+		p.setProperty("mlTdeValidationEnabled", "false");
 		p.setProperty("mlDeleteForests", "false");
 		p.setProperty("mlDeleteReplicas", "false");
 		p.setProperty("mlGroupName", "other-group");
@@ -323,6 +324,7 @@ public void allProperties() {
 		assertEquals("my-schemas-db", config.getSchemasDatabaseName());
 		assertEquals("my-triggers-db", config.getTriggersDatabaseName());
 		assertEquals("/my/schemas", config.getSchemasPath());
+		assertFalse(config.isTdeValidationEnabled());
 		assertFalse(config.isDeleteForests());
 		assertFalse(config.isDeleteReplicas());
 		assertEquals("other-group", config.getGroupName());

src/test/java/com/marklogic/appdeployer/command/schemas/LoadSchemasTest.java

@@ -63,6 +63,7 @@ public void testCustomSchemasPathWithCustomFileFilter() {
 
 		appConfig.setSchemasPath("src/test/resources/schemas-marklogic9");
 		appConfig.setSchemasFileFilter(new CustomFileFilter());
+		appConfig.setTdeValidationEnabled(false);
 		appDeployer.deploy(appConfig);
 
 		DatabaseClient client = appConfig.newSchemasDatabaseClient();
@@ -89,6 +90,20 @@ public void nullSchemaPath() {
 		logger.info("Verifies that no error occurs when the schemas path is null");
 	}
 
+	@Test
+	public void tdeValidationEnabled() {
+		initializeAppDeployer(new DeployOtherDatabasesCommand(), new DeployContentDatabasesCommand(1), newCommand());
+		appConfig.getFirstConfigDir().setBaseDir(new File("src/test/resources/sample-app/tde-validation"));
+		try {
+			deploySampleApp();
+			fail("The deploy should have failed because of a bad TDE template");
+		} catch (Exception ex) {
+			String message = ex.getCause().getMessage();
+			assertTrue(message.startsWith("TDE template failed validation"));
+			assertTrue(message.contains("TDE-REPEATEDCOLUMN"));
+		}
+	}
+
 	@After
 	public void cleanup() {
 		undeploySampleApp();

src/test/resources/sample-app/tde-validation/databases/content-database.json

@@ -0,0 +1,4 @@
+{
+	"database-name" : "%%DATABASE%%",
+	"schema-database": "%%SCHEMAS_DATABASE%%"
+}

src/test/resources/sample-app/tde-validation/databases/sample-app-schemas/schemas/tde/bad-template.json

@@ -0,0 +1,32 @@
+{
+	"template": {
+		"context": "/example",
+		"description": "This is a description with quotes in it '\"",
+		"collections": [
+			"example"
+		],
+		"rows": [
+			{
+				"schemaName": "Example",
+				"viewName": "default",
+				"columns": [
+					{
+						"name": "Id",
+						"scalarType": "string",
+						"val": "Id"
+					},
+					{
+						"name": "DuplicateColumn",
+						"scalarType": "string",
+						"val": "DuplicateColumn"
+					},
+					{
+						"name": "DuplicateColumn",
+						"scalarType": "string",
+						"val": "DuplicateColumn"
+					}
+				]
+			}
+		]
+	}
+}

src/test/resources/sample-app/tde-validation/databases/schemas-database.json

@@ -0,0 +1,3 @@
+{
+	"database-name": "%%SCHEMAS_DATABASE%%"
+}