Skip to content
This repository was archived by the owner on Sep 16, 2024. It is now read-only.

Commit d18e525

Browse files
rjrudinrjrudin
committed
#106 TDE templates are now validated by default
In addition, if an error is thrown when executing any DocumentFileProcessor, the error is propagated instead of just being logged by default
1 parent 7ab3412 commit d18e525

File tree

12 files changed

+332
-37
lines changed

12 files changed

+332
-37
lines changed

‎gradle.properties‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -1,3 +1,3 @@
11
group=com.marklogic
22
javadocsDir=../gh-pages-marklogic-java/javadocs
3-
version=3.10.0
3+
version=3.11.dev

‎src/main/java/com/marklogic/client/ext/file/AbstractDocumentFileReader.java‎

Lines changed: 22 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -13,6 +13,7 @@ public abstract class AbstractDocumentFileReader extends LoggingObject {
1313

1414
private List<DocumentFileProcessor> documentFileProcessors = new ArrayList<>();
1515
private FormatDocumentFileProcessor formatDocumentFileProcessor = new FormatDocumentFileProcessor();
16+
private boolean catchProcessingError = false;
1617

1718
protected AbstractDocumentFileReader() {
1819
documentFileProcessors.add(formatDocumentFileProcessor);
@@ -41,6 +42,13 @@ public void addDocumentFileProcessor(DocumentFileProcessor processor) {
4142
documentFileProcessors.add(processor);
4243
}
4344

45+
/**
46+
* The catchProcessingError field controls whether an exception thrown by a processor will be caught or not. Starting
47+
* in 3.11.0, it defaults to false, as an exception typically indicates that the processing should stop.
48+
*
49+
* @param documentFile
50+
* @return
51+
*/
4452
protected DocumentFile processDocumentFile(DocumentFile documentFile) {
4553
for (DocumentFileProcessor processor : documentFileProcessors) {
4654
try {
@@ -49,8 +57,12 @@ protected DocumentFile processDocumentFile(DocumentFile documentFile) {
4957
}
5058
documentFile = processor.processDocumentFile(documentFile);
5159
} catch (Exception e) {
52-
logger.error("Error while processing document file; file: " + documentFile.getFile().getAbsolutePath()
53-
+ "; cause: " + e.getMessage(), e);
60+
final String message = "Error while processing document file: " + documentFile.getFile();
61+
if (catchProcessingError) {
62+
logger.error(message, e);
63+
} else {
64+
throw new RuntimeException(message, e);
65+
}
5466
}
5567
if (documentFile == null) {
5668
break;
@@ -74,4 +86,12 @@ public FormatDocumentFileProcessor getFormatDocumentFileProcessor() {
7486
public void setFormatDocumentFileProcessor(FormatDocumentFileProcessor formatDocumentFileProcessor) {
7587
this.formatDocumentFileProcessor = formatDocumentFileProcessor;
7688
}
89+
90+
public boolean isCatchProcessingError() {
91+
return catchProcessingError;
92+
}
93+
94+
public void setCatchProcessingError(boolean catchProcessingError) {
95+
this.catchProcessingError = catchProcessingError;
96+
}
7797
}

‎src/main/java/com/marklogic/client/ext/file/GenericFileLoader.java‎

Lines changed: 4 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -271,4 +271,8 @@ public List<DocumentFileProcessor> getDocumentFileProcessors() {
271271
public void setBatchSize(Integer batchSize) {
272272
this.batchSize = batchSize;
273273
}
274+
275+
public void setBatchWriter(BatchWriter batchWriter) {
276+
this.batchWriter = batchWriter;
277+
}
274278
}

‎src/main/java/com/marklogic/client/ext/schemasloader/impl/DefaultSchemasLoader.java‎

Lines changed: 30 additions & 5 deletions
Original file line numberDiff line numberDiff line change
@@ -2,24 +2,41 @@
22

33
import com.marklogic.client.DatabaseClient;
44
import com.marklogic.client.ext.batch.BatchWriter;
5-
import com.marklogic.client.ext.file.GenericFileLoader;
65
import com.marklogic.client.ext.file.DocumentFile;
6+
import com.marklogic.client.ext.file.GenericFileLoader;
77
import com.marklogic.client.ext.modulesloader.impl.DefaultFileFilter;
88
import com.marklogic.client.ext.schemasloader.SchemasLoader;
99

1010
import java.util.List;
1111

1212
public class DefaultSchemasLoader extends GenericFileLoader implements SchemasLoader {
1313

14+
private DatabaseClient schemasDatabaseClient;
15+
private String tdeValidationDatabase;
16+
1417
/**
1518
* Simplest constructor for using this class. Just provide a DatabaseClient, and this will use sensible defaults
1619
* for how documents are read and written. Note that the DatabaseClient will not be released after this class is
1720
* done with it, as this class wasn't the one that created it.
1821
*
19-
* @param databaseClient
22+
* @param schemasDatabaseClient
23+
*/
24+
public DefaultSchemasLoader(DatabaseClient schemasDatabaseClient) {
25+
this(schemasDatabaseClient, null);
26+
}
27+
28+
/**
29+
* If you want to validate TDE templates before they're loaded, you need to provide a second DatabaseClient that
30+
* connects to the content database associated with the schemas database that schemas will be loaded into. This is
31+
* because the "tde.validate" function must run against the content database.
32+
*
33+
* @param schemasDatabaseClient
34+
* @param tdeValidationDatabase
2035
*/
21-
public DefaultSchemasLoader(DatabaseClient databaseClient) {
22-
super(databaseClient);
36+
public DefaultSchemasLoader(DatabaseClient schemasDatabaseClient, String tdeValidationDatabase) {
37+
super(schemasDatabaseClient);
38+
this.schemasDatabaseClient = schemasDatabaseClient;
39+
this.tdeValidationDatabase = tdeValidationDatabase;
2340
initializeDefaultSchemasLoader();
2441
}
2542

@@ -38,7 +55,7 @@ public DefaultSchemasLoader(BatchWriter batchWriter) {
3855
* a DocumentFileReader by the parent class.
3956
*/
4057
protected void initializeDefaultSchemasLoader() {
41-
addDocumentFileProcessor(new TdeDocumentFileProcessor());
58+
addDocumentFileProcessor(new TdeDocumentFileProcessor(this.schemasDatabaseClient, this.tdeValidationDatabase));
4259
addFileFilter(new DefaultFileFilter());
4360
}
4461

@@ -53,4 +70,12 @@ protected void initializeDefaultSchemasLoader() {
5370
public List<DocumentFile> loadSchemas(String... paths) {
5471
return super.loadFiles(paths);
5572
}
73+
74+
public String getTdeValidationDatabase() {
75+
return tdeValidationDatabase;
76+
}
77+
78+
public void setTdeValidationDatabase(String tdeValidationDatabase) {
79+
this.tdeValidationDatabase = tdeValidationDatabase;
80+
}
5681
}

‎src/main/java/com/marklogic/client/ext/schemasloader/impl/TdeDocumentFileProcessor.java‎

Lines changed: 107 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -1,27 +1,131 @@
11
package com.marklogic.client.ext.schemasloader.impl;
22

3+
import com.fasterxml.jackson.databind.node.ObjectNode;
4+
import com.marklogic.client.DatabaseClient;
5+
import com.marklogic.client.FailedRequestException;
6+
import com.marklogic.client.eval.ServerEvaluationCall;
37
import com.marklogic.client.ext.file.DocumentFile;
48
import com.marklogic.client.ext.file.DocumentFileProcessor;
59
import com.marklogic.client.ext.helper.LoggingObject;
610
import com.marklogic.client.io.Format;
11+
import com.marklogic.client.io.JacksonHandle;
12+
import org.springframework.util.FileCopyUtils;
13+
14+
import java.io.File;
15+
import java.io.IOException;
716

817
public class TdeDocumentFileProcessor extends LoggingObject implements DocumentFileProcessor {
918

19+
private DatabaseClient databaseClient;
20+
private String tdeValidationDatabase;
21+
22+
/**
23+
* Use this constructor when you don't want any TDE validation to occur.
24+
*/
25+
public TdeDocumentFileProcessor() {
26+
}
27+
28+
/**
29+
* Use this constructor when you want to validate a TDE template before writing it to MarkLogic.
30+
*
31+
* @param databaseClient
32+
*/
33+
public TdeDocumentFileProcessor(DatabaseClient databaseClient, String tdeValidationDatabase) {
34+
this.databaseClient = databaseClient;
35+
this.tdeValidationDatabase = tdeValidationDatabase;
36+
}
37+
1038
@Override
1139
public DocumentFile processDocumentFile(DocumentFile documentFile) {
1240
String uri = documentFile.getUri();
1341
String extension = documentFile.getFileExtension();
42+
if (extension != null) {
43+
extension = extension.toLowerCase();
44+
}
1445

15-
if (("tdej".equals(extension) || "tdex".equals(extension)) || (uri != null && uri.startsWith("/tde"))) {
46+
boolean isTdeTemplate = ("tdej".equals(extension) || "tdex".equals(extension)) || (uri != null && uri.startsWith("/tde"));
47+
if (isTdeTemplate) {
1648
documentFile.getDocumentMetadata().withCollections("http://marklogic.com/xdmp/tde");
1749
}
1850

19-
if ("tdej".equals(extension)) {
51+
if ("tdej".equals(extension) || "json".equals(extension)) {
2052
documentFile.setFormat(Format.JSON);
21-
} else if ("tdex".equals(extension)) {
53+
} else if ("tdex".equals(extension) || "xml".equals(extension)) {
2254
documentFile.setFormat(Format.XML);
2355
}
2456

57+
if (isTdeTemplate) {
58+
validateTdeTemplate(documentFile);
59+
}
60+
2561
return documentFile;
2662
}
63+
64+
protected void validateTdeTemplate(DocumentFile documentFile) {
65+
final File file = documentFile.getFile();
66+
if (databaseClient == null) {
67+
logger.info("No DatabaseClient provided for TDE validation, so will not validate TDE templates");
68+
} else if (tdeValidationDatabase == null) {
69+
logger.info("No TDE validation database specified, so will not validate TDE templates");
70+
} else {
71+
String fileContent = null;
72+
try {
73+
fileContent = new String(FileCopyUtils.copyToByteArray(file));
74+
} catch (IOException e) {
75+
logger.warn("Could not read TDE template from file, will not validate; cause: " + e.getMessage());
76+
}
77+
if (fileContent != null) {
78+
try {
79+
ServerEvaluationCall call = null;
80+
if (Format.XML.equals(documentFile.getFormat())) {
81+
call = buildXqueryCall(fileContent);
82+
} else if (Format.JSON.equals(documentFile.getFormat())) {
83+
call = buildJavascriptCall(fileContent);
84+
} else {
85+
logger.info("Unrecognized file format, will not try to validate TDE template in file: " + file + "; format: " + documentFile.getFormat());
86+
}
87+
88+
if (call != null) {
89+
ObjectNode node = (ObjectNode) call.eval(new JacksonHandle()).get();
90+
if (node.get("valid").asBoolean()) {
91+
logger.info("TDE template passed validation: " + file);
92+
} else {
93+
throw new RuntimeException(format("TDE template failed validation; file: %s; cause: %s", file, node.get("message").asText()));
94+
}
95+
}
96+
} catch (FailedRequestException e) {
97+
logger.warn("Unexpected error when trying to validate TDE template in file: " + file + "; cause: " + e.getMessage());
98+
}
99+
}
100+
}
101+
}
102+
103+
protected ServerEvaluationCall buildJavascriptCall(String fileContent) {
104+
StringBuilder script = new StringBuilder("xdmp.invokeFunction(function() {var tde = require('/MarkLogic/tde.xqy');");
105+
script.append(format("\nreturn tde.validate([xdmp.toJSON(%s)])}, {database: xdmp.database('%s')})", fileContent, tdeValidationDatabase));
106+
return databaseClient.newServerEval().javascript(script.toString());
107+
}
108+
109+
protected ServerEvaluationCall buildXqueryCall(String fileContent) {
110+
StringBuilder script = new StringBuilder("import module namespace tde = 'http://marklogic.com/xdmp/tde' at '/MarkLogic/tde.xqy'; ");
111+
script.append(format("\nxdmp:invoke-function(function() { \nlet $t := %s ", fileContent));
112+
script.append(format("\nreturn tde:validate($t)}, <options xmlns='xdmp:eval'><database>{xdmp:database('%s')}</database></options>)", tdeValidationDatabase));
113+
return databaseClient.newServerEval().xquery(script.toString());
114+
}
115+
116+
public DatabaseClient getDatabaseClient() {
117+
return databaseClient;
118+
}
119+
120+
public void setDatabaseClient(DatabaseClient databaseClient) {
121+
this.databaseClient = databaseClient;
122+
}
123+
124+
public String getTdeValidationDatabase() {
125+
return tdeValidationDatabase;
126+
}
127+
128+
public void setTdeValidationDatabase(String tdeValidationDatabase) {
129+
this.tdeValidationDatabase = tdeValidationDatabase;
130+
}
27131
}

‎src/test/java/com/marklogic/client/ext/schemasloader/impl/AbstractSchemasTest.java‎

Lines changed: 18 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,18 @@
1+
package com.marklogic.client.ext.schemasloader.impl;
2+
3+
import com.marklogic.client.ext.AbstractIntegrationTest;
4+
import org.junit.Before;
5+
6+
public abstract class AbstractSchemasTest extends AbstractIntegrationTest {
7+
8+
/**
9+
* Wipes out the Schemas database - it's assumed you're not using the Schemas database for
10+
* anything besides ad hoc testing like this.
11+
*/
12+
@Before
13+
public void setup() {
14+
client = newClient("Schemas");
15+
client.newServerEval().xquery("cts:uris((), (), cts:true-query()) ! xdmp:document-delete(.)").eval();
16+
}
17+
18+
}

‎src/test/java/com/marklogic/client/ext/schemasloader/impl/LoadRulesetsTest.java‎

Lines changed: 1 addition & 13 deletions
Original file line numberDiff line numberDiff line change
@@ -1,26 +1,14 @@
11
package com.marklogic.client.ext.schemasloader.impl;
22

3-
import com.marklogic.client.ext.AbstractIntegrationTest;
43
import com.marklogic.client.ext.file.DocumentFile;
54
import com.marklogic.client.ext.helper.ClientHelper;
65
import com.marklogic.client.io.DocumentMetadataHandle;
7-
import org.junit.Before;
86
import org.junit.Test;
97

108
import java.nio.file.Paths;
119
import java.util.List;
1210

13-
public class LoadRulesetsTest extends AbstractIntegrationTest {
14-
15-
/**
16-
* Wipes out the Schemas database - it's assumed you're not using the Schemas database for
17-
* anything besides ad hoc testing like this.
18-
*/
19-
@Before
20-
public void setup() {
21-
client = newClient("Schemas");
22-
client.newServerEval().xquery("cts:uris((), (), cts:true-query()) ! xdmp:document-delete(.)").eval();
23-
}
11+
public class LoadRulesetsTest extends AbstractSchemasTest {
2412

2513
@Test
2614
public void test() {

‎src/test/java/com/marklogic/client/ext/schemasloader/impl/LoadSchemasTest.java‎

Lines changed: 2 additions & 13 deletions
Original file line numberDiff line numberDiff line change
@@ -1,29 +1,18 @@
11
package com.marklogic.client.ext.schemasloader.impl;
22

3-
import com.marklogic.client.ext.AbstractIntegrationTest;
43
import com.marklogic.client.ext.file.DocumentFile;
54
import com.marklogic.client.ext.helper.ClientHelper;
6-
import org.junit.Before;
75
import org.junit.Test;
86

97
import java.nio.file.Paths;
108
import java.util.List;
119

12-
public class LoadSchemasTest extends AbstractIntegrationTest {
13-
14-
/**
15-
* Wipes out the Schemas database - it's assumed you're not using the Schemas database for
16-
* anything besides ad hoc testing like this.
17-
*/
18-
@Before
19-
public void setup() {
20-
client = newClient("Schemas");
21-
client.newServerEval().xquery("cts:uris((), (), cts:true-query()) ! xdmp:document-delete(.)").eval();
22-
}
10+
public class LoadSchemasTest extends AbstractSchemasTest {
2311

2412
@Test
2513
public void test() {
2614
DefaultSchemasLoader loader = new DefaultSchemasLoader(client);
15+
2716
List<DocumentFile> files = loader.loadSchemas(Paths.get("src", "test", "resources", "schemas").toString());
2817
assertEquals(5, files.size());
2918

0 commit comments

Comments
 (0)