Skip to content
This repository was archived by the owner on Sep 16, 2024. It is now read-only.

Commit 1c2792a

Browse files
rjrudinrjrudin
authored
Merge pull request #150 from marklogic-community/feature/149-tde-fix
TDE validation now works on ML versions < 10.0-9
2 parents 9c422c3 + c1621d8 commit 1c2792a

File tree

10 files changed

+137
-58
lines changed

10 files changed

+137
-58
lines changed

‎src/main/java/com/marklogic/client/ext/file/AbstractDocumentFileReader.java‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -55,7 +55,7 @@ protected DocumentFile processDocumentFile(DocumentFile documentFile) {
5555
}
5656
documentFile = processor.processDocumentFile(documentFile);
5757
} catch (Exception e) {
58-
final String message = "Error while processing document file: " + documentFile.getFile();
58+
final String message = "Error while processing file: " + documentFile.getFile() + "; cause: " + e.getMessage();
5959
if (catchProcessingError) {
6060
logger.error(message, e);
6161
} else {

‎src/main/java/com/marklogic/client/ext/file/DocumentFile.java‎

Lines changed: 1 addition & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -7,7 +7,6 @@
77
import com.marklogic.client.io.InputStreamHandle;
88
import com.marklogic.client.io.StringHandle;
99
import com.marklogic.client.io.marker.AbstractWriteHandle;
10-
import com.marklogic.client.io.marker.DocumentMetadataWriteHandle;
1110
import org.springframework.core.io.FileSystemResource;
1211
import org.springframework.core.io.Resource;
1312

@@ -65,7 +64,7 @@ public DocumentWriteOperation.OperationType getOperationType() {
6564
return DocumentWriteOperation.OperationType.DOCUMENT_WRITE;
6665
}
6766

68-
public DocumentMetadataWriteHandle getMetadata() {
67+
public DocumentMetadataHandle getMetadata() {
6968
return documentMetadata;
7069
}
7170

‎src/main/java/com/marklogic/client/ext/file/GenericFileLoader.java‎

Lines changed: 10 additions & 6 deletions
Original file line numberDiff line numberDiff line change
@@ -73,18 +73,22 @@ public GenericFileLoader(BatchWriter batchWriter) {
7373
public List<DocumentFile> loadFiles(String... paths) {
7474
batchWriter.initialize();
7575
List<DocumentFile> documentFiles = getDocumentFiles(paths);
76+
writeDocumentFiles(documentFiles);
77+
return documentFiles;
78+
}
79+
80+
protected final List<DocumentFile> getDocumentFiles(String... paths) {
81+
initializeDocumentFileReader();
82+
return documentFileReader.readDocumentFiles(paths);
83+
}
84+
85+
protected final void writeDocumentFiles(List<DocumentFile> documentFiles) {
7686
if (documentFiles != null && !documentFiles.isEmpty()) {
7787
writeBatchOfDocuments(documentFiles, 0);
7888
if (waitForCompletion) {
7989
batchWriter.waitForCompletion();
8090
}
8191
}
82-
return documentFiles;
83-
}
84-
85-
public List<DocumentFile> getDocumentFiles(String... paths) {
86-
initializeDocumentFileReader();
87-
return documentFileReader.readDocumentFiles(paths);
8892
}
8993

9094
/**

‎src/main/java/com/marklogic/client/ext/helper/ClientHelper.java‎

Lines changed: 0 additions & 12 deletions
Original file line numberDiff line numberDiff line change
@@ -5,11 +5,9 @@
55
import java.util.List;
66

77
import com.marklogic.client.DatabaseClient;
8-
import com.marklogic.client.eval.ServerEvaluationCall;
98
import com.marklogic.client.io.DocumentMetadataHandle;
109
import com.marklogic.client.io.DocumentMetadataHandle.DocumentCollections;
1110
import com.marklogic.client.io.SearchHandle;
12-
import com.marklogic.client.io.StringHandle;
1311
import com.marklogic.client.query.MatchDocumentSummary;
1412
import com.marklogic.client.query.QueryManager;
1513
import com.marklogic.client.query.StringQueryDefinition;
@@ -68,14 +66,4 @@ public String eval(String expr) {
6866
return getClient().newServerEval().xquery(expr).evalAs(String.class);
6967
}
7068

71-
public int getMLEffectiveVersion() {
72-
int version = -1;
73-
StringBuilder script = new StringBuilder("xdmp.effectiveVersion();");
74-
ServerEvaluationCall call = client.newServerEval().javascript(script.toString());
75-
if (call != null) {
76-
String node = call.eval(new StringHandle()).get();
77-
version = Integer.parseInt(node);
78-
}
79-
return version;
80-
}
8169
}

‎src/main/java/com/marklogic/client/ext/schemasloader/impl/DefaultSchemasLoader.java‎

Lines changed: 43 additions & 11 deletions
Original file line numberDiff line numberDiff line change
@@ -1,7 +1,6 @@
11
package com.marklogic.client.ext.schemasloader.impl;
22

33
import com.marklogic.client.DatabaseClient;
4-
import com.marklogic.client.eval.ServerEvaluationCall;
54
import com.marklogic.client.ext.batch.BatchWriter;
65
import com.marklogic.client.ext.batch.RestBatchWriter;
76
import com.marklogic.client.ext.file.DocumentFile;
@@ -10,6 +9,7 @@
109
import com.marklogic.client.ext.modulesloader.impl.DefaultFileFilter;
1110
import com.marklogic.client.ext.schemasloader.SchemasLoader;
1211
import com.marklogic.client.io.DocumentMetadataHandle;
12+
import org.springframework.util.StringUtils;
1313

1414
import java.util.ArrayList;
1515
import java.util.List;
@@ -84,13 +84,33 @@ protected void initializeDefaultSchemasLoader() {
8484
*/
8585
@Override
8686
public List<DocumentFile> loadSchemas(String... paths) {
87-
ClientHelper helper = new ClientHelper(schemasDatabaseClient);
88-
if (helper.getMLEffectiveVersion() >= 10000900 && tdeValidationDatabase != null && !tdeValidationDatabase.isEmpty()) {
89-
logger.info("Installing tde's using tde.templateBatchInsert");
90-
List<DocumentFile> documentFiles = super.getDocumentFiles(paths);
91-
buildTemplateBatchInsertCall(documentFiles).eval().close();
87+
if (TdeUtil.templateBatchInsertSupported(schemasDatabaseClient) && StringUtils.hasText(tdeValidationDatabase)) {
88+
final List<DocumentFile> documentFiles = super.getDocumentFiles(paths);
89+
final List<DocumentFile> tdeFiles = new ArrayList<>();
90+
final List<DocumentFile> nonTdeFiles = new ArrayList<>();
91+
92+
for (DocumentFile file : documentFiles) {
93+
DocumentMetadataHandle metadata = file.getMetadata();
94+
if (metadata != null && metadata.getCollections().contains(TdeUtil.TDE_COLLECTION)) {
95+
tdeFiles.add(file);
96+
} else {
97+
nonTdeFiles.add(file);
98+
}
99+
}
100+
101+
if (!tdeFiles.isEmpty()) {
102+
loadTdeTemplatesViaBatchInsert(tdeFiles);
103+
}
104+
if (!nonTdeFiles.isEmpty()) {
105+
if (logger.isDebugEnabled()) {
106+
logger.debug("Non-TDE files: " + nonTdeFiles);
107+
}
108+
super.writeDocumentFiles(nonTdeFiles);
109+
}
110+
92111
return documentFiles;
93112
}
113+
94114
return super.loadFiles(paths);
95115
}
96116

@@ -102,15 +122,27 @@ public void setTdeValidationDatabase(String tdeValidationDatabase) {
102122
this.tdeValidationDatabase = tdeValidationDatabase;
103123
}
104124

105-
protected ServerEvaluationCall buildTemplateBatchInsertCall(List<DocumentFile> documentFiles) {
106-
String tdeTemplate = getTdeBatchInsertQuery(documentFiles);
125+
private void loadTdeTemplatesViaBatchInsert(List<DocumentFile> tdeFiles) {
126+
logger.info("Loading and validating TDE templates via tde.templateBatchInsert; templates: " +
127+
tdeFiles.stream().map(documentFile -> documentFile.getFile().getName()).collect(Collectors.toList()));
128+
129+
String query = buildTdeBatchInsertQuery(tdeFiles);
107130
StringBuilder script = new StringBuilder("declareUpdate(); xdmp.invokeFunction(function() {var tde = require('/MarkLogic/tde.xqy');");
108-
script.append(tdeTemplate);
131+
script.append(query);
109132
script.append(format("}, {database: xdmp.database('%s')})", tdeValidationDatabase));
110-
return schemasDatabaseClient.newServerEval().javascript(script.toString());
133+
try {
134+
schemasDatabaseClient.newServerEval().javascript(script.toString()).eval().close();
135+
} catch (Exception ex) {
136+
throw new RuntimeException("Unable to load and validate TDE templates via tde.templateBatchInsert; cause: " + ex.getMessage(), ex);
137+
}
111138
}
112139

113-
private String getTdeBatchInsertQuery(List<DocumentFile> documentFiles) {
140+
/**
141+
*
142+
* @param documentFiles
143+
* @return a JavaScript query that uses the tde.templateBatchInsert function introduced in ML 10.0-9
144+
*/
145+
private String buildTdeBatchInsertQuery(List<DocumentFile> documentFiles) {
114146
List<String> templateInfoList = new ArrayList<>();
115147
for (DocumentFile doc : documentFiles) {
116148
String uri = doc.getUri();

‎src/main/java/com/marklogic/client/ext/schemasloader/impl/TdeDocumentFileProcessor.java‎

Lines changed: 14 additions & 13 deletions
Original file line numberDiff line numberDiff line change
@@ -19,6 +19,7 @@ public class TdeDocumentFileProcessor extends LoggingObject implements DocumentF
1919

2020
private DatabaseClient databaseClient;
2121
private String tdeValidationDatabase;
22+
private Boolean templateBatchInsertSupported;
2223

2324
/**
2425
* Use this constructor when you don't want any TDE validation to occur.
@@ -38,7 +39,6 @@ public TdeDocumentFileProcessor(DatabaseClient databaseClient, String tdeValidat
3839

3940
@Override
4041
public DocumentFile processDocumentFile(DocumentFile documentFile) {
41-
ClientHelper helper = new ClientHelper(databaseClient);
4242
String uri = documentFile.getUri();
4343
String extension = documentFile.getFileExtension();
4444
if (extension != null) {
@@ -47,7 +47,8 @@ public DocumentFile processDocumentFile(DocumentFile documentFile) {
4747

4848
boolean isTdeTemplate = ("tdej".equals(extension) || "tdex".equals(extension)) || (uri != null && uri.startsWith("/tde"));
4949
if (isTdeTemplate) {
50-
documentFile.getDocumentMetadata().withCollections("http://marklogic.com/xdmp/tde");
50+
documentFile.getDocumentMetadata().withCollections(TdeUtil.TDE_COLLECTION);
51+
validateTdeTemplate(documentFile);
5152
}
5253

5354
if ("tdej".equals(extension) || "json".equals(extension)) {
@@ -56,19 +57,16 @@ public DocumentFile processDocumentFile(DocumentFile documentFile) {
5657
documentFile.setFormat(Format.XML);
5758
}
5859

59-
if(isTdeTemplate) {
60-
return documentFile;
61-
}
60+
return documentFile;
61+
}
6262

63-
if (isTdeTemplate) {
64-
if(helper.getMLEffectiveVersion() >= 10000900 && tdeValidationDatabase != null && !tdeValidationDatabase.isEmpty()) {
65-
return documentFile;
66-
} else {
67-
validateTdeTemplate(documentFile);
68-
}
63+
private boolean isTemplateBatchInsertSupported() {
64+
if (this.templateBatchInsertSupported == null) {
65+
// Memoize this to avoid repeated calls; the result will always be the same unless the databaseClient is
66+
// modified, in which case templateBatchInsertSupported is set to null
67+
this.templateBatchInsertSupported = TdeUtil.templateBatchInsertSupported(databaseClient);
6968
}
70-
71-
return documentFile;
69+
return this.templateBatchInsertSupported;
7270
}
7371

7472
protected void validateTdeTemplate(DocumentFile documentFile) {
@@ -77,6 +75,8 @@ protected void validateTdeTemplate(DocumentFile documentFile) {
7775
logger.info("No DatabaseClient provided for TDE validation, so will not validate TDE templates");
7876
} else if (tdeValidationDatabase == null) {
7977
logger.info("No TDE validation database specified, so will not validate TDE templates");
78+
} else if (isTemplateBatchInsertSupported()) {
79+
logger.debug("Not performing TDE validation; it will be performed automatically via tde.templateBatchInsert");
8080
} else {
8181
String fileContent = null;
8282
try {
@@ -133,6 +133,7 @@ public DatabaseClient getDatabaseClient() {
133133

134134
public void setDatabaseClient(DatabaseClient databaseClient) {
135135
this.databaseClient = databaseClient;
136+
this.templateBatchInsertSupported = null;
136137
}
137138

138139
public String getTdeValidationDatabase() {

‎src/main/java/com/marklogic/client/ext/schemasloader/impl/TdeUtil.java‎

Lines changed: 22 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,22 @@
1+
package com.marklogic.client.ext.schemasloader.impl;
2+
3+
import com.marklogic.client.DatabaseClient;
4+
import com.marklogic.client.io.StringHandle;
5+
import org.slf4j.Logger;
6+
import org.slf4j.LoggerFactory;
7+
8+
abstract class TdeUtil {
9+
10+
public static final String TDE_COLLECTION = "http://marklogic.com/xdmp/tde";
11+
12+
private final static Logger logger = LoggerFactory.getLogger(TdeUtil.class);
13+
14+
public static boolean templateBatchInsertSupported(DatabaseClient client) {
15+
final int markLogic10Dot9 = 10000900;
16+
String result = client.newServerEval().javascript("xdmp.effectiveVersion()").eval(new StringHandle()).get();
17+
if (logger.isDebugEnabled()) {
18+
logger.debug("Checking if templateBatchInsert is supported; MarkLogic version: " + result);
19+
}
20+
return Integer.parseInt(result) >= markLogic10Dot9;
21+
}
22+
}

‎src/test/java/com/marklogic/client/ext/schemasloader/impl/LoadRulesetsTest.java‎

Lines changed: 2 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -14,7 +14,8 @@ public class LoadRulesetsTest extends AbstractSchemasTest {
1414

1515
@Test
1616
public void test() {
17-
DefaultSchemasLoader loader = new DefaultSchemasLoader(client);
17+
// Pass in a TDE validation database to ensure that TDE validation doesn't happen for these files
18+
DefaultSchemasLoader loader = new DefaultSchemasLoader(client, "Documents");
1819
List<DocumentFile> files = loader.loadSchemas(Paths.get("src", "test", "resources", "rulesets", "collection-test").toString());
1920
assertEquals(2, files.size());
2021

‎src/test/java/com/marklogic/client/ext/schemasloader/impl/LoadSchemasTest.java‎

Lines changed: 4 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -8,7 +8,8 @@
88
import java.nio.file.Paths;
99
import java.util.List;
1010

11-
import static org.junit.jupiter.api.Assertions.*;
11+
import static org.junit.jupiter.api.Assertions.assertEquals;
12+
import static org.junit.jupiter.api.Assertions.assertTrue;
1213

1314
public class LoadSchemasTest extends AbstractSchemasTest {
1415

@@ -23,7 +24,7 @@ public void test() {
2324
assertEquals(5, files.size());
2425

2526
ClientHelper helper = new ClientHelper(client);
26-
List<String> uris = helper.getUrisInCollection("http://marklogic.com/xdmp/tde");
27+
List<String> uris = helper.getUrisInCollection(TdeUtil.TDE_COLLECTION);
2728
assertEquals(4, uris.size(), "The non-tde/ruleset.txt file should not be in the TDE collection");
2829
assertTrue(uris.contains("/child/child.tdej"));
2930
assertTrue(uris.contains("/child/grandchild/grandchild.tdex"));
@@ -38,7 +39,7 @@ public void testTemplateBatchInsert() {
3839
assertEquals(2, files.size());
3940

4041
ClientHelper helper = new ClientHelper(client);
41-
List<String> uris = helper.getUrisInCollection("http://marklogic.com/xdmp/tde");
42+
List<String> uris = helper.getUrisInCollection(TdeUtil.TDE_COLLECTION);
4243
assertEquals(2, uris.size());
4344
assertTrue(uris.contains("/tde/good-schema.json"));
4445
assertTrue(uris.contains("/tde/good-schema.xml"));

‎src/test/java/com/marklogic/client/ext/schemasloader/impl/ValidateTdeTemplatesTest.java‎

Lines changed: 40 additions & 9 deletions
Original file line numberDiff line numberDiff line change
@@ -27,9 +27,20 @@ public void setup() {
2727

2828
@Test
2929
public void badJsonFile() {
30-
FailedRequestException ex = assertThrows(FailedRequestException.class, () ->
31-
loader.loadSchemas(Paths.get("src", "test", "resources", "bad-schemas", "bad-json").toString()));
32-
logger.info(ex.getMessage());
30+
final String path = Paths.get("src", "test", "resources", "bad-schemas", "bad-json").toString();
31+
if (TdeUtil.templateBatchInsertSupported(client)) {
32+
FailedRequestException ex = assertThrows(FailedRequestException.class, () -> loader.loadSchemas(path));
33+
logger.info(ex.getMessage());
34+
} else {
35+
try {
36+
loader.loadSchemas(path);
37+
fail("The bad-template.json file should have failed processing because it has a duplicate column in it");
38+
} catch (RuntimeException ex) {
39+
String message = ex.getCause().getMessage();
40+
assertTrue(message.startsWith("TDE template failed validation"));
41+
assertTrue(message.contains("TDE-REPEATEDCOLUMN"));
42+
}
43+
}
3344
}
3445

3546
@Test
@@ -46,16 +57,28 @@ public void goodTemplateThatsThenUpdatedViaJavascript() {
4657

4758
@Test
4859
public void badXmlFile() {
49-
FailedRequestException ex = assertThrows(FailedRequestException.class, () ->
50-
loader.loadSchemas(Paths.get("src", "test", "resources", "bad-schemas", "bad-xml").toString()));
51-
logger.info(ex.getMessage());
60+
final String path = Paths.get("src", "test", "resources", "bad-schemas", "bad-xml").toString();
61+
if (TdeUtil.templateBatchInsertSupported(client)) {
62+
FailedRequestException ex = assertThrows(FailedRequestException.class, () -> loader.loadSchemas(path));
63+
logger.info(ex.getMessage());
64+
} else {
65+
try {
66+
loader.loadSchemas(path);
67+
fail("The bad-template.xml file should have failed processing because it has a duplicate column in it");
68+
} catch (RuntimeException ex) {
69+
String message = ex.getCause().getMessage();
70+
assertTrue(message.startsWith("TDE template failed validation"));
71+
assertTrue(message.contains("TDE-REPEATEDCOLUMN"));
72+
}
73+
}
5274
}
5375

5476
@Test
5577
public void badJsonFileInNonTdeDirectory() {
56-
FailedRequestException ex = assertThrows(FailedRequestException.class, () ->
57-
loader.loadSchemas(Paths.get("src", "test", "resources", "bad-schemas", "otherpath").toString()));
58-
logger.info(ex.getMessage());
78+
final String path = Paths.get("src", "test", "resources", "bad-schemas", "otherpath").toString();
79+
List<DocumentFile> files = loader.loadSchemas(path);
80+
assertEquals(1, files.size(), "The file in the otherpath directory is a TDE template, but it's not under /tde/, so it's not " +
81+
"validated as a TDE template");
5982
}
6083

6184
@Test
@@ -70,4 +93,12 @@ public void mixedContent() {
7093
List<DocumentFile> files = loader.loadSchemas(Paths.get("src", "test", "resources", "good-schemas", "xml-schemas").toString());
7194
assertEquals(1, files.size(), "Verifying that the file still loads correctly even with processing instructions and comments in it");
7295
}
96+
97+
@Test
98+
public void schemasThatArentTdeTemplates() {
99+
List<DocumentFile> files = loader.loadSchemas(Paths.get("src", "test", "resources", "schemas", "not-tde").toString());
100+
assertEquals(1, files.size());
101+
assertEquals("ruleset.txt", files.get(0).getFile().getName(), "The loader should recognize" +
102+
"that the file is not a TDE, and thus it should not be loaded via templateBatchInsert");
103+
}
73104
}

0 commit comments

Comments
 (0)