Refactoring of TDE/QBV processors

Changes:

- Renamed client to `schemasDatabaseClient` to make it clear the kind of client that's needed.
- Using `FilenameUtil` - not the most glamorous class, but nice to have all the checks in one place. 
- Deprecated some getters/setters that I shouldn't have added. 
- Added a schemas database to the test app and changed schemas tests to use that.

Author: rjrudin

gradle.properties

@@ -16,3 +16,4 @@ mlAppName=ml-javaclient-util-test
 mlRestPort=8028
 mlUsername=admin
 mlPassword=change in gradle-local.properties
+mlConfigPaths=src/test/ml-config

src/main/java/com/marklogic/client/ext/helper/FilenameUtil.java

@@ -20,17 +20,30 @@
 public abstract class FilenameUtil {
 
     public static boolean isXslFile(String filename) {
-        return filename.endsWith(".xsl") || filename.endsWith(".xslt");
+		return endsWithExtension(filename, ".xsl", ".xslt");
     }
 
     public static boolean isXqueryFile(String filename) {
-        return filename.endsWith(".xqy") || filename.endsWith(".xq");
+		return endsWithExtension(filename, ".xqy", ".xqy");
     }
 
     public static boolean isJavascriptFile(String filename) {
-        return filename.endsWith(".sjs") || filename.endsWith(".js");
+		return endsWithExtension(filename, ".sjs", ".js");
     }
 
+	public static boolean endsWithExtension(String filename, String... extensions) {
+		if (filename == null || extensions == null) {
+			return false;
+		}
+		filename = filename.toLowerCase();
+		for (String extension : extensions) {
+			if (extension != null && filename.endsWith(extension.toLowerCase())) {
+				return true;
+			}
+		}
+		return false;
+	}
+
     public static String getFileExtension(File f) {
         String[] split = f.getName().split("\\.");
         if (split.length > 1) {

src/main/java/com/marklogic/client/ext/schemasloader/impl/QbvDocumentFileProcessor.java

@@ -20,6 +20,7 @@
 import com.marklogic.client.eval.ServerEvaluationCall;
 import com.marklogic.client.ext.file.DocumentFile;
 import com.marklogic.client.ext.file.DocumentFileProcessor;
+import com.marklogic.client.ext.helper.FilenameUtil;
 import com.marklogic.client.ext.helper.LoggingObject;
 import com.marklogic.client.extra.jdom.JDOMHandle;
 import com.marklogic.client.io.Format;
@@ -29,32 +30,41 @@
 import org.jdom2.input.SAXBuilder;
 import org.springframework.util.FileCopyUtils;
 
+import java.io.File;
 import java.io.IOException;
 import java.io.StringReader;
 import java.util.ArrayList;
 import java.util.List;
 
+/**
+ * @since 4.6.0
+ */
 public class QbvDocumentFileProcessor extends LoggingObject implements DocumentFileProcessor {
 
 	public static final String QBV_COLLECTION = "http://marklogic.com/xdmp/qbv";
 	private static final String QBV_XML_PLAN_NAMESPACE = "http://marklogic.com/plan";
 	private static final String QBV_XML_ROOT_ELEMENT = "query-based-view";
-	private static final String JAVASCRIPT_EVAL_TEMPLATE = "declareUpdate(); xdmp.invokeFunction(function() {'use strict'; const op = require('/MarkLogic/optic'); return %s }, {database: xdmp.database('%s')})";
-	private static final String XQUERY_EVAL_TEMPLATE = "xquery version \"1.0-ml\"; import module namespace op=\"http://marklogic.com/optic\" at \"/MarkLogic/optic.xqy\"; xdmp:invoke-function(function() {%s},<options xmlns=\"xdmp:eval\"><database>{xdmp:database('%s')}</database></options>)";
+	private static final String JAVASCRIPT_EVAL_TEMPLATE = "declareUpdate(); " +
+		"xdmp.invokeFunction(function() {'use strict'; const op = require('/MarkLogic/optic'); return %s }, " +
+		"{database: xdmp.database('%s')})";
+	private static final String XQUERY_EVAL_TEMPLATE = "xquery version \"1.0-ml\"; " +
+		"import module namespace op=\"http://marklogic.com/optic\" at \"/MarkLogic/optic.xqy\"; " +
+		"xdmp:invoke-function(function() {%s},<options xmlns=\"xdmp:eval\"><database>{xdmp:database('%s')}</database></options>)";
 
-	final private DatabaseClient databaseClient;
+	final private DatabaseClient schemasDatabaseClient;
 	final private String qbvGeneratorDatabaseName;
-	final protected List<DocumentFile> qbvFiles = new ArrayList<>();
+	final private List<DocumentFile> qbvFiles = new ArrayList<>();
 	final private XMLDocumentManager docMgr;
 
 
 	/**
-	 * @param databaseClient - a MarkLogic DatabaseClient object for the Schemas database
+	 * @param schemasDatabaseClient database client for the application's schemas database
+	 * @param qbvGeneratorDatabaseName the database to run a script against for generating a QBV
 	 */
-	public QbvDocumentFileProcessor(DatabaseClient databaseClient, String qbvGeneratorDatabaseName) {
-		this.databaseClient = databaseClient;
+	public QbvDocumentFileProcessor(DatabaseClient schemasDatabaseClient, String qbvGeneratorDatabaseName) {
+		this.schemasDatabaseClient = schemasDatabaseClient;
 		this.qbvGeneratorDatabaseName = qbvGeneratorDatabaseName;
-		this.docMgr = databaseClient.newXMLDocumentManager();
+		this.docMgr = schemasDatabaseClient.newXMLDocumentManager();
 	}
 
 	@Override
@@ -70,17 +80,11 @@ public DocumentFile processDocumentFile(DocumentFile documentFile) {
 
 	private boolean isQbvQuery(DocumentFile documentFile) {
 		String uri = documentFile.getUri();
-		if (uri != null && uri.startsWith("/qbv")) {
-			String extension = documentFile.getFileExtension();
-			if (extension != null) {
-				extension = extension.toLowerCase();
-			}
-			return
-				"sjs".equals(extension)
-					|| "js".equals(extension)
-					|| "xqy".equals(extension)
-					|| "xq".equals(extension);
-		} else return false;
+		File file = documentFile.getFile();
+		return uri != null
+			&& uri.startsWith("/qbv")
+			&& file != null
+			&& (FilenameUtil.isXqueryFile(file.getName()) || FilenameUtil.isJavascriptFile(file.getName()));
 	}
 
 	public void processQbvFiles() {
@@ -125,24 +129,18 @@ private ServerEvaluationCall getServerEvaluationCall(DocumentFile qbvFile) {
 		} catch (IOException e) {
 			throw new RuntimeException(format("Unable to generate Query-Based View; could not read from file %s; cause: %s", qbvFile.getFile().getAbsolutePath(), e.getMessage()));
 		}
-		String extension = qbvFile.getFileExtension();
-		if (extension != null) {
-			extension = extension.toLowerCase();
-		}
-		if (("xqy".equals(extension)) || ("xq".equals(extension))) {
-			return buildXqueryCall(fileContent);
-		} else {
-			return buildJavascriptCall(fileContent);
-		}
+		return FilenameUtil.isXqueryFile(qbvFile.getFile().getName()) ?
+			buildXqueryCall(fileContent) :
+			buildJavascriptCall(fileContent);
 	}
 
 	private ServerEvaluationCall buildJavascriptCall(String fileContent) {
 		String script = format(JAVASCRIPT_EVAL_TEMPLATE, fileContent, qbvGeneratorDatabaseName);
-		return databaseClient.newServerEval().javascript(script);
+		return schemasDatabaseClient.newServerEval().javascript(script);
 	}
 
 	private ServerEvaluationCall buildXqueryCall(String fileContent) {
 		String script = format(XQUERY_EVAL_TEMPLATE, fileContent, qbvGeneratorDatabaseName);
-		return databaseClient.newServerEval().xquery(script);
+		return schemasDatabaseClient.newServerEval().xquery(script);
 	}
 }

src/main/java/com/marklogic/client/ext/schemasloader/impl/TdeDocumentFileProcessor.java

@@ -20,7 +20,7 @@
 import com.marklogic.client.eval.ServerEvaluationCall;
 import com.marklogic.client.ext.file.DocumentFile;
 import com.marklogic.client.ext.file.DocumentFileProcessor;
-import com.marklogic.client.ext.helper.ClientHelper;
+import com.marklogic.client.ext.helper.FilenameUtil;
 import com.marklogic.client.ext.helper.LoggingObject;
 import com.marklogic.client.io.Format;
 import com.marklogic.client.io.JacksonHandle;
@@ -32,7 +32,7 @@
 
 public class TdeDocumentFileProcessor extends LoggingObject implements DocumentFileProcessor {
 
-	private DatabaseClient databaseClient;
+	private DatabaseClient schemasDatabaseClient;
 	private String tdeValidationDatabase;
 	private Boolean templateBatchInsertSupported;
 
@@ -45,31 +45,34 @@ public TdeDocumentFileProcessor() {
 	/**
 	 * Use this constructor when you want to validate a TDE template before writing it to MarkLogic.
 	 *
-	 * @param databaseClient
+	 * @param schemasDatabaseClient database client for the application's schemas database
+	 * @param tdeValidationDatabase the database to run a script against for validating a TDE
 	 */
-	public TdeDocumentFileProcessor(DatabaseClient databaseClient, String tdeValidationDatabase) {
-		this.databaseClient = databaseClient;
+	public TdeDocumentFileProcessor(DatabaseClient schemasDatabaseClient, String tdeValidationDatabase) {
+		this.schemasDatabaseClient = schemasDatabaseClient;
 		this.tdeValidationDatabase = tdeValidationDatabase;
 	}
 
 	@Override
 	public DocumentFile processDocumentFile(DocumentFile documentFile) {
 		String uri = documentFile.getUri();
-		String extension = documentFile.getFileExtension();
-		if (extension != null) {
-			extension = extension.toLowerCase();
-		}
-
-		boolean isTdeTemplate = ("tdej".equals(extension) || "tdex".equals(extension)) || (uri != null && uri.startsWith("/tde"));
-		if (isTdeTemplate) {
+		String filename = documentFile.getFile() != null ? documentFile.getFile().getName() : null;
+		boolean isTdeUri = (uri != null && uri.startsWith("/tde"));
+		boolean isJsonTde = (isTdeUri && FilenameUtil.endsWithExtension(filename, ".json"))
+			|| FilenameUtil.endsWithExtension(filename, ".tdej");
+		boolean isXmlTde = (isTdeUri && FilenameUtil.endsWithExtension(filename, ".xml"))
+			|| FilenameUtil.endsWithExtension(filename, ".tdex");
+
+		// We have a test suggesting that a TDE may not be JSON or XML; that doesn't seem likely, but it also does not
+		// appear to cause any issues.
+		if (isTdeUri || isJsonTde || isXmlTde) {
 			documentFile.getDocumentMetadata().withCollections(TdeUtil.TDE_COLLECTION);
 			validateTdeTemplate(documentFile);
-		}
-
-		if ("tdej".equals(extension) || "json".equals(extension)) {
-			documentFile.setFormat(Format.JSON);
-		} else if ("tdex".equals(extension) || "xml".equals(extension)) {
-			documentFile.setFormat(Format.XML);
+			if (isJsonTde) {
+				documentFile.setFormat(Format.JSON);
+			} else if (isXmlTde) {
+				documentFile.setFormat(Format.XML);
+			}
 		}
 
 		return documentFile;
@@ -79,14 +82,14 @@ private boolean isTemplateBatchInsertSupported() {
 		if (this.templateBatchInsertSupported == null) {
 			// Memoize this to avoid repeated calls; the result will always be the same unless the databaseClient is
 			// modified, in which case templateBatchInsertSupported is set to null
-			this.templateBatchInsertSupported = TdeUtil.templateBatchInsertSupported(databaseClient);
+			this.templateBatchInsertSupported = TdeUtil.templateBatchInsertSupported(schemasDatabaseClient);
 		}
 		return this.templateBatchInsertSupported;
 	}
 
 	protected void validateTdeTemplate(DocumentFile documentFile) {
 		final File file = documentFile.getFile();
-		if (databaseClient == null) {
+		if (schemasDatabaseClient == null) {
 			logger.info("No DatabaseClient provided for TDE validation, so will not validate TDE templates");
 		} else if (tdeValidationDatabase == null) {
 			logger.info("No TDE validation database specified, so will not validate TDE templates");
@@ -127,7 +130,7 @@ protected ServerEvaluationCall buildJavascriptCall(DocumentFile documentFile, St
 			"\nreturn tde.validate([xdmp.toJSON(template)], ['%s'])}, {database: xdmp.database('%s')})",
 			documentFile.getUri(), tdeValidationDatabase
 		));
-		return databaseClient.newServerEval().javascript(script.toString())
+		return schemasDatabaseClient.newServerEval().javascript(script.toString())
 			.addVariable("template", new StringHandle(fileContent).withFormat(Format.JSON));
 	}
 
@@ -139,22 +142,42 @@ protected ServerEvaluationCall buildXqueryCall(DocumentFile documentFile, String
 			"\ntde:validate($template, '%s')}, <options xmlns='xdmp:eval'><database>{xdmp:database('%s')}</database></options>)",
 			documentFile.getUri(), tdeValidationDatabase
 		));
-		return databaseClient.newServerEval().xquery(script.toString()).addVariable("template", new StringHandle(fileContent).withFormat(Format.XML));
+		return schemasDatabaseClient.newServerEval().xquery(script.toString()).addVariable("template", new StringHandle(fileContent).withFormat(Format.XML));
 	}
 
+	/**
+	 * @return
+	 * @deprecated since 4.6.0, will be removed in 5.0.0
+	 */
+	@Deprecated
 	public DatabaseClient getDatabaseClient() {
-		return databaseClient;
+		return schemasDatabaseClient;
 	}
 
+	/**
+	 * @param databaseClient
+	 * @deprecated since 4.6.0, will be removed in 5.0.0
+	 */
+	@Deprecated
 	public void setDatabaseClient(DatabaseClient databaseClient) {
-		this.databaseClient = databaseClient;
+		this.schemasDatabaseClient = databaseClient;
 		this.templateBatchInsertSupported = null;
 	}
 
+	/**
+	 * @return
+	 * @deprecated since 4.6.0, will be removed in 5.0.0
+	 */
+	@Deprecated
 	public String getTdeValidationDatabase() {
 		return tdeValidationDatabase;
 	}
 
+	/**
+	 * @param tdeValidationDatabase
+	 * @deprecated since 4.6.0, will be removed in 5.0.0
+	 */
+	@Deprecated
 	public void setTdeValidationDatabase(String tdeValidationDatabase) {
 		this.tdeValidationDatabase = tdeValidationDatabase;
 	}

src/test/java/com/marklogic/client/ext/schemasloader/impl/AbstractSchemasTest.java

@@ -26,7 +26,7 @@ public abstract class AbstractSchemasTest extends AbstractIntegrationTest {
 	 */
 	@BeforeEach
 	public void setup() {
-		client = newClient("Schemas");
+		client = newClient("ml-javaclient-util-test-schemas");
 		client.newServerEval().xquery("cts:uris((), (), cts:true-query()) ! xdmp:document-delete(.)").eval();
 	}
 

src/test/java/com/marklogic/client/ext/schemasloader/impl/GenerateQbvTest.java

@@ -3,6 +3,7 @@
 import com.marklogic.client.ext.file.DocumentFile;
 import com.marklogic.client.ext.helper.ClientHelper;
 import com.marklogic.client.io.DocumentMetadataHandle;
+import org.junit.jupiter.api.BeforeEach;
 import org.junit.jupiter.api.Test;
 
 import java.nio.file.Path;
@@ -14,9 +15,15 @@
 
 public class GenerateQbvTest extends AbstractSchemasTest {
 
+	private DefaultSchemasLoader loader;
+
+	@BeforeEach
+	void beforeEach() {
+		loader = new DefaultSchemasLoader(client, CONTENT_DATABASE);
+	}
+
 	@Test
 	public void test() {
-		DefaultSchemasLoader loader = new DefaultSchemasLoader(client, "Documents");
 		Path path = Paths.get("src", "test", "resources", "qbv-schemas");
 		List<DocumentFile> files = loader.loadSchemas(path.toString());
 		assertEquals(3, files.size(),
@@ -52,7 +59,6 @@ public void test() {
 
 	@Test
 	public void loadBadOptic() {
-		DefaultSchemasLoader loader = new DefaultSchemasLoader(client, "Documents");
 		Path path = Paths.get("src", "test", "resources", "qbv-bad-schemas");
 		RuntimeException ex = assertThrows(RuntimeException.class, () -> loader.loadSchemas(path.toString()));
 		assertTrue(ex.getMessage().contains("Query-Based View generation failed for file:"), "Unexpected message: " + ex.getMessage());
@@ -62,7 +68,6 @@ public void loadBadOptic() {
 
 	@Test
 	public void schemaViewDoesNotExist() {
-		DefaultSchemasLoader loader = new DefaultSchemasLoader(client, "Documents");
 		Path path = Paths.get("src", "test", "resources", "qbv-no-tde-schemas");
 		RuntimeException ex = assertThrows(RuntimeException.class, () -> loader.loadSchemas(path.toString()));
 		assertTrue(ex.getMessage().contains("Query-Based View generation failed for file:"), "Unexpected message: " + ex.getMessage());
@@ -72,7 +77,6 @@ public void schemaViewDoesNotExist() {
 
 	@Test
 	public void emptyDirectories() {
-		DefaultSchemasLoader loader = new DefaultSchemasLoader(client, "Documents");
 		Path path = Paths.get("src", "test", "resources", "qbv-empty-schemas");
 		List<DocumentFile> files = loader.loadSchemas(path.toString());
 		assertEquals(0, files.size());

src/test/java/com/marklogic/client/ext/schemasloader/impl/LoadSchemasTest.java

@@ -25,7 +25,9 @@
 import java.nio.file.Paths;
 import java.util.List;
 
-import static org.junit.jupiter.api.Assertions.*;
+import static org.junit.jupiter.api.Assertions.assertEquals;
+import static org.junit.jupiter.api.Assertions.assertThrows;
+import static org.junit.jupiter.api.Assertions.assertTrue;
 
 public class LoadSchemasTest extends AbstractSchemasTest {
 
@@ -45,6 +47,9 @@ public void test() {
 		assertTrue(uris.contains("/child/child.tdej"));
 		assertTrue(uris.contains("/child/grandchild/grandchild.tdex"));
 		assertTrue(uris.contains("/parent.tdex"));
+
+		// This assertion seems a little off - a TDE should be either a JSON or XML file. This doesn't seem to cause
+		// any problems, but it also doesn't seem to make sense.
 		assertTrue(uris.contains("/tde/ruleset.txt"));
 	}
 

src/test/ml-config/databases/content-database.json

@@ -0,0 +1,4 @@
+{
+  "database-name": "%%DATABASE%%",
+  "schema-database": "%%SCHEMAS_DATABASE%%"
+}

src/test/ml-config/databases/schemas-database.json

@@ -0,0 +1,3 @@
+{
+  "database-name": "%%SCHEMAS_DATABASE%%"
+}