Merge pull request #194 from marklogic/feature/qbv-tweak

Reworking TDE/QBV processors

Author: rjrudin

src/main/java/com/marklogic/client/ext/schemasloader/impl/DefaultSchemasLoader.java

@@ -23,7 +23,6 @@
 import com.marklogic.client.ext.modulesloader.impl.DefaultFileFilter;
 import com.marklogic.client.ext.schemasloader.SchemasLoader;
 import com.marklogic.client.io.DocumentMetadataHandle;
-import org.springframework.util.StringUtils;
 
 import java.util.ArrayList;
 import java.util.List;
@@ -33,30 +32,25 @@
 
 public class DefaultSchemasLoader extends GenericFileLoader implements SchemasLoader {
 
-	private DatabaseClient schemasDatabaseClient;
-	private String tdeValidationDatabase;
-	protected QbvDocumentFileProcessor qbvDocumentFileProcessor;
+	private final DatabaseClient schemasDatabaseClient;
+	private final DatabaseClient contentDatabaseClient;
+	private final boolean validateTdeTemplates;
+	private QbvDocumentFileProcessor qbvDocumentFileProcessor;
 
 	/**
-	 * Simplest constructor for using this class. Just provide a DatabaseClient, and this will use sensible defaults for
-	 * how documents are read and written. Note that the DatabaseClient will not be released after this class is done
-	 * with it, as this class wasn't the one that created it.
-	 *
-	 * @param schemasDatabaseClient
+	 * @param schemasDatabaseClient for loading files into an application's schemas database
+	 * @param contentDatabaseClient for validating TDEs and generating QBVs
 	 */
-	public DefaultSchemasLoader(DatabaseClient schemasDatabaseClient) {
-		this(schemasDatabaseClient, null);
+	public DefaultSchemasLoader(DatabaseClient schemasDatabaseClient, DatabaseClient contentDatabaseClient) {
+		this(schemasDatabaseClient, contentDatabaseClient, true);
 	}
 
 	/**
-	 * If you want to validate TDE templates before they're loaded, you need to provide a second DatabaseClient that
-	 * connects to the content database associated with the schemas database that schemas will be loaded into. This is
-	 * because the "tde.validate" function must run against the content database.
-	 *
-	 * @param schemasDatabaseClient
-	 * @param tdeValidationDatabase
+	 * @param schemasDatabaseClient for loading files into an application's schemas database
+	 * @param contentDatabaseClient for validating TDEs and generating QBVs
+	 * @param validateTdeTemplates if false, TDEs will not be validated nor loaded via tde.templateBatchInsert
 	 */
-	public DefaultSchemasLoader(DatabaseClient schemasDatabaseClient, String tdeValidationDatabase) {
+	public DefaultSchemasLoader(DatabaseClient schemasDatabaseClient, DatabaseClient contentDatabaseClient, boolean validateTdeTemplates) {
 		super(((Supplier<BatchWriter>) () -> {
 			RestBatchWriter writer = new RestBatchWriter(schemasDatabaseClient);
 			// Default this to 1, as it's not typical to have such a large number of schemas to load that multiple threads
@@ -67,31 +61,20 @@ public DefaultSchemasLoader(DatabaseClient schemasDatabaseClient, String tdeVali
 		}).get());
 
 		this.schemasDatabaseClient = schemasDatabaseClient;
-		this.tdeValidationDatabase = tdeValidationDatabase;
-		initializeDefaultSchemasLoader();
-	}
+		this.contentDatabaseClient = contentDatabaseClient;
+		this.validateTdeTemplates = validateTdeTemplates;
 
-	/**
-	 * Assumes that the BatchWriter has already been initialized.
-	 *
-	 * @param batchWriter
-	 * @deprecated Since 4.6.0; this class needs a DatabaseClient for the schemas database passed to it so that it can
-	 * pass that client on to specific file processors.
-	 */
-	@Deprecated
-	public DefaultSchemasLoader(BatchWriter batchWriter) {
-		super(batchWriter);
-		initializeDefaultSchemasLoader();
-	}
+		if (this.contentDatabaseClient != null) {
+			this.qbvDocumentFileProcessor = new QbvDocumentFileProcessor(this.schemasDatabaseClient, this.contentDatabaseClient);
+			addDocumentFileProcessor(this.qbvDocumentFileProcessor);
+		}
+
+		if (this.validateTdeTemplates && this.contentDatabaseClient != null) {
+			addDocumentFileProcessor(new TdeDocumentFileProcessor(this.contentDatabaseClient));
+		} else {
+			addDocumentFileProcessor(new TdeDocumentFileProcessor(null));
+		}
 
-	/**
-	 * Adds the DocumentFileProcessors and FileFilters specific to loading schemas, which will then be used to construct
-	 * a DocumentFileReader by the parent class.
-	 */
-	protected void initializeDefaultSchemasLoader() {
-		this.qbvDocumentFileProcessor = new QbvDocumentFileProcessor(this.schemasDatabaseClient, this.tdeValidationDatabase);
-		addDocumentFileProcessor(new TdeDocumentFileProcessor(this.schemasDatabaseClient, this.tdeValidationDatabase));
-		addDocumentFileProcessor(this.qbvDocumentFileProcessor);
 		addFileFilter(new DefaultFileFilter());
 	}
 
@@ -107,7 +90,7 @@ public List<DocumentFile> loadSchemas(String... paths) {
 		final List<DocumentFile> documentFiles = super.getDocumentFiles(paths);
 
 		if (!documentFiles.isEmpty()) {
-			if (TdeUtil.templateBatchInsertSupported(schemasDatabaseClient) && StringUtils.hasText(tdeValidationDatabase)) {
+			if (this.validateTdeTemplates && TdeUtil.templateBatchInsertSupported(schemasDatabaseClient) && contentDatabaseClient != null) {
 				SchemaFiles schemaFiles = readSchemaFiles(documentFiles);
 				if (!schemaFiles.tdeFiles.isEmpty()) {
 					loadTdeTemplatesViaBatchInsert(schemaFiles.tdeFiles);
@@ -122,7 +105,10 @@ public List<DocumentFile> loadSchemas(String... paths) {
 				writeDocumentFiles(documentFiles);
 			}
 		}
-		this.qbvDocumentFileProcessor.processQbvFiles();
+
+		if (this.qbvDocumentFileProcessor != null) {
+			this.qbvDocumentFileProcessor.processQbvFiles();
+		}
 
 		return documentFiles;
 	}
@@ -152,11 +138,10 @@ private void loadTdeTemplatesViaBatchInsert(List<DocumentFile> tdeFiles) {
 			tdeFiles.stream().map(documentFile -> documentFile.getFile().getName()).collect(Collectors.toList()));
 
 		String query = buildTdeBatchInsertQuery(tdeFiles);
-		StringBuilder script = new StringBuilder("declareUpdate(); xdmp.invokeFunction(function() {var tde = require('/MarkLogic/tde.xqy');");
+		StringBuilder script = new StringBuilder("declareUpdate(); const tde = require('/MarkLogic/tde.xqy'); ");
 		script.append(query);
-		script.append(format("}, {database: xdmp.database('%s')})", tdeValidationDatabase));
 		try {
-			schemasDatabaseClient.newServerEval().javascript(script.toString()).eval().close();
+			contentDatabaseClient.newServerEval().javascript(script.toString()).eval().close();
 		} catch (Exception ex) {
 			throw new RuntimeException("Unable to load and validate TDE templates via tde.templateBatchInsert; " +
 				"cause: " + ex.getMessage() + "; the following script can be run in Query Console against your content " +
@@ -216,17 +201,4 @@ public SchemaFiles(List<DocumentFile> tdeFiles, List<DocumentFile> nonTdeFiles)
 			this.nonTdeFiles = nonTdeFiles;
 		}
 	}
-
-	public String getTdeValidationDatabase() {
-		return tdeValidationDatabase;
-	}
-
-	/**
-	 * @param tdeValidationDatabase
-	 * @deprecated Should be set via the constructor and not modified.
-	 */
-	@Deprecated
-	public void setTdeValidationDatabase(String tdeValidationDatabase) {
-		this.tdeValidationDatabase = tdeValidationDatabase;
-	}
 }

src/main/java/com/marklogic/client/ext/schemasloader/impl/QbvDocumentFileProcessor.java

@@ -39,32 +39,24 @@
 /**
  * @since 4.6.0
  */
-public class QbvDocumentFileProcessor extends LoggingObject implements DocumentFileProcessor {
+class QbvDocumentFileProcessor extends LoggingObject implements DocumentFileProcessor {
 
 	public static final String QBV_COLLECTION = "http://marklogic.com/xdmp/qbv";
 	private static final String QBV_XML_PLAN_NAMESPACE = "http://marklogic.com/plan";
 	private static final String QBV_XML_ROOT_ELEMENT = "query-based-view";
-	private static final String JAVASCRIPT_EVAL_TEMPLATE = "declareUpdate(); " +
-		"xdmp.invokeFunction(function() {'use strict'; const op = require('/MarkLogic/optic'); return %s }, " +
-		"{database: xdmp.database('%s')})";
-	private static final String XQUERY_EVAL_TEMPLATE = "xquery version \"1.0-ml\"; " +
-		"import module namespace op=\"http://marklogic.com/optic\" at \"/MarkLogic/optic.xqy\"; " +
-		"xdmp:invoke-function(function() {%s},<options xmlns=\"xdmp:eval\"><database>{xdmp:database('%s')}</database></options>)";
 
-	final private DatabaseClient schemasDatabaseClient;
-	final private String qbvGeneratorDatabaseName;
+	final private DatabaseClient contentDatabaseClient;
 	final private List<DocumentFile> qbvFiles = new ArrayList<>();
-	final private XMLDocumentManager docMgr;
+	final private XMLDocumentManager schemasDocumentManager;
 
 
 	/**
-	 * @param schemasDatabaseClient database client for the application's schemas database
-	 * @param qbvGeneratorDatabaseName the database to run a script against for generating a QBV
+	 * @param schemasDatabaseClient used to write the QBV XML document to the application's schemas database
+	 * @param contentDatabaseClient used to generate the QBV based on a user-provided script
 	 */
-	public QbvDocumentFileProcessor(DatabaseClient schemasDatabaseClient, String qbvGeneratorDatabaseName) {
-		this.schemasDatabaseClient = schemasDatabaseClient;
-		this.qbvGeneratorDatabaseName = qbvGeneratorDatabaseName;
-		this.docMgr = schemasDatabaseClient.newXMLDocumentManager();
+	QbvDocumentFileProcessor(DatabaseClient schemasDatabaseClient, DatabaseClient contentDatabaseClient) {
+		this.schemasDocumentManager = schemasDatabaseClient.newXMLDocumentManager();
+		this.contentDatabaseClient = contentDatabaseClient;
 	}
 
 	@Override
@@ -99,29 +91,29 @@ private void processQbvFile(DocumentFile qbvFile) {
 		ServerEvaluationCall call = getServerEvaluationCall(qbvFile);
 		if (call != null) {
 			StringHandle handleString = new StringHandle();
+			try {
+				call.eval(handleString);
+			} catch (Exception e) {
+				throw new RuntimeException(format("Query-Based View generation failed for file: %s; cause: %s", qbvFile.getFile().getAbsolutePath(), e.getMessage()));
+			}
+			if (Format.XML.equals(handleString.getFormat())) {
+				Document xmlDocument;
 				try {
-					call.eval(handleString);
+					xmlDocument = new SAXBuilder().build(new StringReader(handleString.get()));
 				} catch (Exception e) {
 					throw new RuntimeException(format("Query-Based View generation failed for file: %s; cause: %s", qbvFile.getFile().getAbsolutePath(), e.getMessage()));
 				}
-				if (Format.XML.equals(handleString.getFormat())) {
-					Document xmlDocument;
-					try {
-						xmlDocument = new SAXBuilder().build(new StringReader(handleString.get()));
-					} catch (Exception e) {
-						throw new RuntimeException(format("Query-Based View generation failed for file: %s; cause: %s", qbvFile.getFile().getAbsolutePath(), e.getMessage()));
-					}
-					Element root = xmlDocument.getRootElement();
-					if (QBV_XML_ROOT_ELEMENT.equals(root.getName()) & (root.getNamespace() != null && root.getNamespace().getURI().equals(QBV_XML_PLAN_NAMESPACE))) {
-						qbvFile.getDocumentMetadata().getCollections().add(QBV_COLLECTION);
-						String uri = qbvFile.getUri() + ".xml";
-						docMgr.write(uri, qbvFile.getDocumentMetadata(), new JDOMHandle(xmlDocument));
-					} else {
-						throw new RuntimeException(format("Query-Based view generation failed for file: %s; received unexpected response from server: %s", qbvFile.getFile().getAbsolutePath(), handleString.get()));
-					}
+				Element root = xmlDocument.getRootElement();
+				if (QBV_XML_ROOT_ELEMENT.equals(root.getName()) & (root.getNamespace() != null && root.getNamespace().getURI().equals(QBV_XML_PLAN_NAMESPACE))) {
+					qbvFile.getDocumentMetadata().getCollections().add(QBV_COLLECTION);
+					String uri = qbvFile.getUri() + ".xml";
+					schemasDocumentManager.write(uri, qbvFile.getDocumentMetadata(), new JDOMHandle(xmlDocument));
 				} else {
-					throw new RuntimeException(format("Query-Based View generation failed for file: %s; ensure your Optic script includes a call to generate a view; received unexpected response from server: %s", qbvFile.getFile().getAbsolutePath(), handleString.get()));
+					throw new RuntimeException(format("Query-Based view generation failed for file: %s; received unexpected response from server: %s", qbvFile.getFile().getAbsolutePath(), handleString.get()));
 				}
+			} else {
+				throw new RuntimeException(format("Query-Based View generation failed for file: %s; ensure your Optic script includes a call to generate a view; received unexpected response from server: %s", qbvFile.getFile().getAbsolutePath(), handleString.get()));
+			}
 		}
 	}
 
@@ -133,17 +125,7 @@ private ServerEvaluationCall getServerEvaluationCall(DocumentFile qbvFile) {
 			throw new RuntimeException(format("Unable to generate Query-Based View; could not read from file %s; cause: %s", qbvFile.getFile().getAbsolutePath(), e.getMessage()));
 		}
 		return FilenameUtil.isXqueryFile(qbvFile.getFile().getName()) ?
-			buildXqueryCall(fileContent) :
-			buildJavascriptCall(fileContent);
-	}
-
-	private ServerEvaluationCall buildJavascriptCall(String fileContent) {
-		String script = format(JAVASCRIPT_EVAL_TEMPLATE, fileContent, qbvGeneratorDatabaseName);
-		return schemasDatabaseClient.newServerEval().javascript(script);
-	}
-
-	private ServerEvaluationCall buildXqueryCall(String fileContent) {
-		String script = format(XQUERY_EVAL_TEMPLATE, fileContent, qbvGeneratorDatabaseName);
-		return schemasDatabaseClient.newServerEval().xquery(script);
+			contentDatabaseClient.newServerEval().xquery(fileContent) :
+			contentDatabaseClient.newServerEval().javascript(fileContent);
 	}
 }

src/main/java/com/marklogic/client/ext/schemasloader/impl/TdeDocumentFileProcessor.java

@@ -30,27 +30,17 @@
 import java.io.File;
 import java.io.IOException;
 
-public class TdeDocumentFileProcessor extends LoggingObject implements DocumentFileProcessor {
+class TdeDocumentFileProcessor extends LoggingObject implements DocumentFileProcessor {
 
-	private DatabaseClient schemasDatabaseClient;
-	private String tdeValidationDatabase;
+	private final DatabaseClient contentDatabaseClient;
 	private Boolean templateBatchInsertSupported;
 
 	/**
-	 * Use this constructor when you don't want any TDE validation to occur.
+	 * @param contentDatabaseClient the database to run a script against for validating a TDE. If null, TDE validation
+	 *                              will not be performed.
 	 */
-	public TdeDocumentFileProcessor() {
-	}
-
-	/**
-	 * Use this constructor when you want to validate a TDE template before writing it to MarkLogic.
-	 *
-	 * @param schemasDatabaseClient database client for the application's schemas database
-	 * @param tdeValidationDatabase the database to run a script against for validating a TDE
-	 */
-	public TdeDocumentFileProcessor(DatabaseClient schemasDatabaseClient, String tdeValidationDatabase) {
-		this.schemasDatabaseClient = schemasDatabaseClient;
-		this.tdeValidationDatabase = tdeValidationDatabase;
+	TdeDocumentFileProcessor(DatabaseClient contentDatabaseClient) {
+		this.contentDatabaseClient = contentDatabaseClient;
 	}
 
 	@Override
@@ -79,22 +69,25 @@ public DocumentFile processDocumentFile(DocumentFile documentFile) {
 	}
 
 	private boolean isTemplateBatchInsertSupported() {
-		if (this.templateBatchInsertSupported == null) {
+		if (this.templateBatchInsertSupported == null && contentDatabaseClient != null) {
 			// Memoize this to avoid repeated calls; the result will always be the same unless the databaseClient is
 			// modified, in which case templateBatchInsertSupported is set to null
-			this.templateBatchInsertSupported = TdeUtil.templateBatchInsertSupported(schemasDatabaseClient);
+			this.templateBatchInsertSupported = TdeUtil.templateBatchInsertSupported(contentDatabaseClient);
 		}
 		return this.templateBatchInsertSupported;
 	}
 
-	protected void validateTdeTemplate(DocumentFile documentFile) {
+	/**
+	 * This mechanism is only needed on older versions of MarkLogic that do not support tde.templateBatchInsert.
+	 *
+	 * @param documentFile
+	 */
+	private void validateTdeTemplate(DocumentFile documentFile) {
 		final File file = documentFile.getFile();
-		if (schemasDatabaseClient == null) {
-			logger.info("No DatabaseClient provided for TDE validation, so will not validate TDE templates");
-		} else if (tdeValidationDatabase == null) {
-			logger.info("No TDE validation database specified, so will not validate TDE templates");
+		if (contentDatabaseClient == null) {
+			logger.info("No content database client provided, so will not validate TDE templates.");
 		} else if (isTemplateBatchInsertSupported()) {
-			logger.debug("Not performing TDE validation; it will be performed automatically via tde.templateBatchInsert");
+			logger.debug("Not performing TDE validation; it will be performed automatically via tde.templateBatchInsert.");
 		} else {
 			String fileContent = null;
 			try {
@@ -124,61 +117,17 @@ protected void validateTdeTemplate(DocumentFile documentFile) {
 		}
 	}
 
-	protected ServerEvaluationCall buildJavascriptCall(DocumentFile documentFile, String fileContent) {
-		StringBuilder script = new StringBuilder("var template; xdmp.invokeFunction(function() {var tde = require('/MarkLogic/tde.xqy');");
-		script.append(format(
-			"\nreturn tde.validate([xdmp.toJSON(template)], ['%s'])}, {database: xdmp.database('%s')})",
-			documentFile.getUri(), tdeValidationDatabase
-		));
-		return schemasDatabaseClient.newServerEval().javascript(script.toString())
+	private ServerEvaluationCall buildJavascriptCall(DocumentFile documentFile, String fileContent) {
+		StringBuilder script = new StringBuilder("const tde = require('/MarkLogic/tde.xqy'); var template; ");
+		script.append(format("\ntde.validate([xdmp.toJSON(template)], ['%s'])", documentFile.getUri()));
+		return contentDatabaseClient.newServerEval().javascript(script.toString())
 			.addVariable("template", new StringHandle(fileContent).withFormat(Format.JSON));
 	}
 
-	protected ServerEvaluationCall buildXqueryCall(DocumentFile documentFile, String fileContent) {
+	private ServerEvaluationCall buildXqueryCall(DocumentFile documentFile, String fileContent) {
 		StringBuilder script = new StringBuilder("import module namespace tde = 'http://marklogic.com/xdmp/tde' at '/MarkLogic/tde.xqy'; ");
 		script.append("\ndeclare variable $template external; ");
-		script.append("\nxdmp:invoke-function(function() { ");
-		script.append(format(
-			"\ntde:validate($template, '%s')}, <options xmlns='xdmp:eval'><database>{xdmp:database('%s')}</database></options>)",
-			documentFile.getUri(), tdeValidationDatabase
-		));
-		return schemasDatabaseClient.newServerEval().xquery(script.toString()).addVariable("template", new StringHandle(fileContent).withFormat(Format.XML));
-	}
-
-	/**
-	 * @return
-	 * @deprecated since 4.6.0, will be removed in 5.0.0
-	 */
-	@Deprecated
-	public DatabaseClient getDatabaseClient() {
-		return schemasDatabaseClient;
-	}
-
-	/**
-	 * @param databaseClient
-	 * @deprecated since 4.6.0, will be removed in 5.0.0
-	 */
-	@Deprecated
-	public void setDatabaseClient(DatabaseClient databaseClient) {
-		this.schemasDatabaseClient = databaseClient;
-		this.templateBatchInsertSupported = null;
-	}
-
-	/**
-	 * @return
-	 * @deprecated since 4.6.0, will be removed in 5.0.0
-	 */
-	@Deprecated
-	public String getTdeValidationDatabase() {
-		return tdeValidationDatabase;
-	}
-
-	/**
-	 * @param tdeValidationDatabase
-	 * @deprecated since 4.6.0, will be removed in 5.0.0
-	 */
-	@Deprecated
-	public void setTdeValidationDatabase(String tdeValidationDatabase) {
-		this.tdeValidationDatabase = tdeValidationDatabase;
+		script.append(format("\ntde:validate($template, '%s')", documentFile.getUri()));
+		return contentDatabaseClient.newServerEval().xquery(script.toString()).addVariable("template", new StringHandle(fileContent).withFormat(Format.XML));
 	}
 }