LoadSchemas to use tde template batch insert

Author: rahulvudutala

src/main/java/com/marklogic/client/ext/file/GenericFileLoader.java

@@ -72,9 +72,7 @@ public GenericFileLoader(BatchWriter batchWriter) {
 	 */
 	public List<DocumentFile> loadFiles(String... paths) {
 		batchWriter.initialize();
-		initializeDocumentFileReader();
-
-		List<DocumentFile> documentFiles = documentFileReader.readDocumentFiles(paths);
+		List<DocumentFile> documentFiles = getDocumentFiles(paths);
 		if (documentFiles != null && !documentFiles.isEmpty()) {
 			writeBatchOfDocuments(documentFiles, 0);
 			if (waitForCompletion) {
@@ -84,6 +82,11 @@ public List<DocumentFile> loadFiles(String... paths) {
 		return documentFiles;
 	}
 
+	public List<DocumentFile> getDocumentFiles(String... paths) {
+		initializeDocumentFileReader();
+		return documentFileReader.readDocumentFiles(paths);
+	}
+
 	/**
 	 * If batchSize is not set, then this method will load all the documents in one call to the BatchWriter. Otherwise,
 	 * this will divide up the list of documentFiles into batches matching the value of batchSize, with the last batch

src/main/java/com/marklogic/client/ext/helper/ClientHelper.java

@@ -5,9 +5,11 @@
 import java.util.List;
 
 import com.marklogic.client.DatabaseClient;
+import com.marklogic.client.eval.ServerEvaluationCall;
 import com.marklogic.client.io.DocumentMetadataHandle;
 import com.marklogic.client.io.DocumentMetadataHandle.DocumentCollections;
 import com.marklogic.client.io.SearchHandle;
+import com.marklogic.client.io.StringHandle;
 import com.marklogic.client.query.MatchDocumentSummary;
 import com.marklogic.client.query.QueryManager;
 import com.marklogic.client.query.StringQueryDefinition;
@@ -65,4 +67,15 @@ public List<String> getUrisInCollection(String collectionName, int pageLength) {
     public String eval(String expr) {
         return getClient().newServerEval().xquery(expr).evalAs(String.class);
     }
+
+    public int getMLEffectiveVersion() {
+		int version = -1;
+		StringBuilder script = new StringBuilder("xdmp.effectiveVersion();");
+		ServerEvaluationCall call = client.newServerEval().javascript(script.toString());
+		if (call != null) {
+			String node = call.eval(new StringHandle()).get();
+			version = Integer.parseInt(node);
+		}
+		return version;
+	}
 }

src/main/java/com/marklogic/client/ext/schemasloader/impl/DefaultSchemasLoader.java

@@ -1,25 +1,31 @@
 package com.marklogic.client.ext.schemasloader.impl;
 
 import com.marklogic.client.DatabaseClient;
+import com.marklogic.client.eval.ServerEvaluationCall;
 import com.marklogic.client.ext.batch.BatchWriter;
 import com.marklogic.client.ext.batch.RestBatchWriter;
 import com.marklogic.client.ext.file.DocumentFile;
 import com.marklogic.client.ext.file.GenericFileLoader;
+import com.marklogic.client.ext.helper.ClientHelper;
 import com.marklogic.client.ext.modulesloader.impl.DefaultFileFilter;
 import com.marklogic.client.ext.schemasloader.SchemasLoader;
+import com.marklogic.client.io.DocumentMetadataHandle;
 
+import java.util.ArrayList;
 import java.util.List;
+import java.util.Set;
 import java.util.function.Supplier;
+import java.util.stream.Collectors;
 
 public class DefaultSchemasLoader extends GenericFileLoader implements SchemasLoader {
 
 	private DatabaseClient schemasDatabaseClient;
 	private String tdeValidationDatabase;
 
 	/**
-	 * Simplest constructor for using this class. Just provide a DatabaseClient, and this will use sensible defaults
-	 * for how documents are read and written. Note that the DatabaseClient will not be released after this class is
-	 * done with it, as this class wasn't the one that created it.
+	 * Simplest constructor for using this class. Just provide a DatabaseClient, and this will use sensible defaults for
+	 * how documents are read and written. Note that the DatabaseClient will not be released after this class is done
+	 * with it, as this class wasn't the one that created it.
 	 *
 	 * @param schemasDatabaseClient
 	 */
@@ -70,14 +76,21 @@ protected void initializeDefaultSchemasLoader() {
 	}
 
 	/**
-	 * Run the given paths through the DocumentFileReader, and then send the result to the BatchWriter, and then
-	 * return the result.
+	 * Run the given paths through the DocumentFileReader, and then send the result to the BatchWriter, and then return
+	 * the result.
 	 *
 	 * @param paths
 	 * @return a DocumentFile for each file that was loaded as a schema
 	 */
 	@Override
 	public List<DocumentFile> loadSchemas(String... paths) {
+		ClientHelper helper = new ClientHelper(schemasDatabaseClient);
+		if (helper.getMLEffectiveVersion() >= 10000900 && tdeValidationDatabase != null && !tdeValidationDatabase.isEmpty()) {
+			logger.info("Installing tde's using tde.templateBatchInsert");
+			List<DocumentFile> documentFiles = super.getDocumentFiles(paths);
+			buildTemplateBatchInsertCall(documentFiles).eval().close();
+			return documentFiles;
+		}
 		return super.loadFiles(paths);
 	}
 
@@ -88,4 +101,51 @@ public String getTdeValidationDatabase() {
 	public void setTdeValidationDatabase(String tdeValidationDatabase) {
 		this.tdeValidationDatabase = tdeValidationDatabase;
 	}
+
+	protected ServerEvaluationCall buildTemplateBatchInsertCall(List<DocumentFile> documentFiles) {
+		String tdeTemplate = getTdeBatchInsertQuery(documentFiles);
+		StringBuilder script = new StringBuilder("declareUpdate(); xdmp.invokeFunction(function() {var tde = require('/MarkLogic/tde.xqy');");
+		script.append(tdeTemplate);
+		script.append(format("}, {database: xdmp.database('%s')})", tdeValidationDatabase));
+		return schemasDatabaseClient.newServerEval().javascript(script.toString());
+	}
+
+	private String getTdeBatchInsertQuery(List<DocumentFile> documentFiles) {
+		List<String> templateInfoList = new ArrayList<>();
+		for (DocumentFile doc : documentFiles) {
+			String uri = doc.getUri();
+			String content = doc.getContent().toString();
+
+			// Permissions
+			DocumentMetadataHandle.DocumentPermissions documentPermissions = doc.getDocumentMetadata().getPermissions();
+			List<String> permissionList = new ArrayList<>();
+			documentPermissions.keySet().forEach(key -> {
+				Set<DocumentMetadataHandle.Capability> values = documentPermissions.get(key);
+				values.forEach(value -> permissionList.add(String.format("xdmp.permission('%s', '%s')", key, value)));
+			});
+			String permissions = "[".concat(permissionList.stream().collect(Collectors.joining(", "))).concat("]");
+
+			// Collections
+			List<String> collectionsList = new ArrayList<>();
+			doc.getDocumentMetadata().getCollections().forEach(collection -> collectionsList.add(collection));
+			String collections = collectionsList.stream().map(coll -> '"' + coll + '"').collect(Collectors.joining(", "));
+			collections = "[".concat(collections).concat("]");
+
+			// Template info
+			String templateFormat = "";
+			if (doc.getFormat().toString().equals("XML")) {
+				templateFormat = String.format("tde.templateInfo('%s', xdmp.unquote(`%s`), %s, %s)", uri, content, permissions, collections);
+			} else if (doc.getFormat().toString().equals("JSON")) {
+				templateFormat = String.format("tde.templateInfo('%s', xdmp.toJSON(%s), %s, %s)", uri, content, permissions, collections);
+			} else {
+				templateFormat = String.format("tde.templateInfo('%s',%s, %s, %s)", uri, content, permissions, collections);
+			}
+			templateInfoList.add(templateFormat);
+		}
+
+		String templateString = "tde.templateBatchInsert(["
+			.concat(templateInfoList.stream().collect(Collectors.joining(",")))
+			.concat("]);");
+		return templateString;
+	}
 }

src/main/java/com/marklogic/client/ext/schemasloader/impl/TdeDocumentFileProcessor.java

@@ -5,6 +5,7 @@
 import com.marklogic.client.eval.ServerEvaluationCall;
 import com.marklogic.client.ext.file.DocumentFile;
 import com.marklogic.client.ext.file.DocumentFileProcessor;
+import com.marklogic.client.ext.helper.ClientHelper;
 import com.marklogic.client.ext.helper.LoggingObject;
 import com.marklogic.client.io.Format;
 import com.marklogic.client.io.JacksonHandle;
@@ -37,6 +38,7 @@ public TdeDocumentFileProcessor(DatabaseClient databaseClient, String tdeValidat
 
 	@Override
 	public DocumentFile processDocumentFile(DocumentFile documentFile) {
+		ClientHelper helper = new ClientHelper(databaseClient);
 		String uri = documentFile.getUri();
 		String extension = documentFile.getFileExtension();
 		if (extension != null) {
@@ -54,8 +56,16 @@ public DocumentFile processDocumentFile(DocumentFile documentFile) {
 			documentFile.setFormat(Format.XML);
 		}
 
+		if(isTdeTemplate) {
+			return documentFile;
+		}
+
 		if (isTdeTemplate) {
-			validateTdeTemplate(documentFile);
+			if(helper.getMLEffectiveVersion() >= 10000900 && tdeValidationDatabase != null && !tdeValidationDatabase.isEmpty()) {
+				return documentFile;
+			} else {
+				validateTdeTemplate(documentFile);
+			}
 		}
 
 		return documentFile;

src/test/java/com/marklogic/client/ext/schemasloader/impl/LoadSchemasTest.java

@@ -30,4 +30,17 @@ public void test() {
 		assertTrue(uris.contains("/parent.tdex"));
 		assertTrue(uris.contains("/tde/ruleset.txt"));
 	}
+
+	@Test
+	public void testTemplateBatchInsert() {
+		DefaultSchemasLoader loader = new DefaultSchemasLoader(client, "Documents");
+		List<DocumentFile> files = loader.loadSchemas(Paths.get("src", "test", "resources", "good-schemas", "originals").toString());
+		assertEquals(2, files.size());
+
+		ClientHelper helper = new ClientHelper(client);
+		List<String> uris = helper.getUrisInCollection("http://marklogic.com/xdmp/tde");
+		assertEquals(2, uris.size());
+		assertTrue(uris.contains("/tde/good-schema.json"));
+		assertTrue(uris.contains("/tde/good-schema.xml"));
+	}
 }

src/test/java/com/marklogic/client/ext/schemasloader/impl/ValidateTdeTemplatesTest.java

@@ -1,5 +1,6 @@
 package com.marklogic.client.ext.schemasloader.impl;
 
+import com.marklogic.client.FailedRequestException;
 import com.marklogic.client.ext.file.DocumentFile;
 import org.junit.jupiter.api.BeforeEach;
 import org.junit.jupiter.api.Test;
@@ -26,14 +27,9 @@ public void setup() {
 
 	@Test
 	public void badJsonFile() {
-		try {
-			loader.loadSchemas(Paths.get("src", "test", "resources", "bad-schemas", "bad-json").toString());
-			fail("The bad-template.json file should have failed processing because it has a duplicate column in it");
-		} catch (RuntimeException ex) {
-			String message = ex.getCause().getMessage();
-			assertTrue(message.startsWith("TDE template failed validation"));
-			assertTrue(message.contains("TDE-REPEATEDCOLUMN"));
-		}
+		FailedRequestException ex = assertThrows(FailedRequestException.class, () ->
+			loader.loadSchemas(Paths.get("src", "test", "resources", "bad-schemas", "bad-json").toString()));
+		logger.info(ex.getMessage());
 	}
 
 	@Test
@@ -50,21 +46,16 @@ public void goodTemplateThatsThenUpdatedViaJavascript() {
 
 	@Test
 	public void badXmlFile() {
-		try {
-			loader.loadSchemas(Paths.get("src", "test", "resources", "bad-schemas", "bad-xml").toString());
-			fail("The bad-template.xml file should have failed processing because it has a duplicate column in it");
-		} catch (RuntimeException ex) {
-			String message = ex.getCause().getMessage();
-			assertTrue(message.startsWith("TDE template failed validation"));
-			assertTrue(message.contains("TDE-REPEATEDCOLUMN"));
-		}
+		FailedRequestException ex = assertThrows(FailedRequestException.class, () ->
+			loader.loadSchemas(Paths.get("src", "test", "resources", "bad-schemas", "bad-xml").toString()));
+		logger.info(ex.getMessage());
 	}
 
 	@Test
 	public void badJsonFileInNonTdeDirectory() {
-		List<DocumentFile> files = loader.loadSchemas(Paths.get("src", "test", "resources", "bad-schemas", "otherpath").toString());
-		assertEquals(1, files.size(), "The file in the otherpath directory is a TDE template, but it's not under /tde/, so it's not " +
-			"validated as a TDE template");
+		FailedRequestException ex = assertThrows(FailedRequestException.class, () ->
+			loader.loadSchemas(Paths.get("src", "test", "resources", "bad-schemas", "otherpath").toString()));
+		logger.info(ex.getMessage());
 	}
 
 	@Test