Skip to content

Commit aa7e2eb

Browse files
maeisabelle08maeisabelle08
committed
Merge branch 'master' into dino/197-Refactor-to-autodeploy
* master: fixing indentation. adding 2 files I missed before fixing regression with uninstalling hub updating trace library to write 1 doc per transaction added range indexes on trace db added more robust tests for tracing added more verbose logging for unit tests updated travis installer to get its marklogic version from a variable fixing bug in trace for errors
2 parents 3cf67e9 + be718b2 commit aa7e2eb

File tree

42 files changed

+722
-247
lines changed

Some content is hidden

Large Commits have some content hidden by default. Use the searchbox below for content that may be hidden.

42 files changed

+722
-247
lines changed

‎.gitignore‎

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -14,3 +14,4 @@ assetInstallTime.properties
1414
/quick-start/input
1515
/quick-start/plugins
1616
flow.properties
17+
node_modules/

‎examples/gradle-basic/build.gradle‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -10,7 +10,7 @@ plugins {
1010
id 'eclipse'
1111
id 'idea'
1212
id 'net.saliman.properties' version '1.4.5'
13-
id 'com.marklogic.ml-data-hub-plugin' version '1.0.0-beta.1'
13+
id 'com.marklogic.ml-data-hub-plugin' version '1.0.0-beta.4'
1414
id 'com.marklogic.ml-gradle' version '2.1.0'
1515
}
1616

‎marklogic-data-hub/src/main/java/com/marklogic/hub/DataHub.java‎

Lines changed: 4 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -55,6 +55,7 @@
5555
import com.marklogic.hub.commands.DeployRestApiCommand;
5656
import com.marklogic.hub.commands.LoadModulesCommand;
5757
import com.marklogic.hub.commands.UpdateRestApiServersCommand;
58+
import com.marklogic.hub.commands.DeployHubDatabaseCommand.DBType;
5859
import com.marklogic.hub.util.HubFileFilter;
5960
import com.marklogic.hub.util.HubModulesLoader;
6061
import com.marklogic.mgmt.ManageClient;
@@ -301,14 +302,17 @@ private List<Command> getCommands(AppConfig config) {
301302
List<Command> dbCommands = new ArrayList<Command>();
302303
DeployHubDatabaseCommand staging = new DeployHubDatabaseCommand(hubConfig.stagingDbName);
303304
staging.setForestsPerHost(hubConfig.stagingForestsPerHost);
305+
staging.setDbType(DBType.STAGING);
304306
dbCommands.add(staging);
305307

306308
DeployHubDatabaseCommand finalDb = new DeployHubDatabaseCommand(hubConfig.finalDbName);
307309
finalDb.setForestsPerHost(hubConfig.finalForestsPerHost);
310+
finalDb.setDbType(DBType.FINAL);
308311
dbCommands.add(finalDb);
309312

310313
DeployHubDatabaseCommand tracingDb = new DeployHubDatabaseCommand(hubConfig.tracingDbName);
311314
tracingDb.setForestsPerHost(hubConfig.tracingForestsPerHost);
315+
tracingDb.setDbType(DBType.TRACING);
312316
dbCommands.add(tracingDb);
313317

314318
dbCommands.add(new DeployModulesDatabaseCommand(hubConfig.modulesDbName));
@@ -347,7 +351,6 @@ public void uninstall() throws IOException {
347351
LOGGER.debug("Uninstalling the Data Hub from MarkLogic");
348352
AppConfig config = getAppConfig();
349353
AdminManager adminManager = getAdminManager();
350-
adminManager.setWaitForRestartCheckInterval(250);
351354
SimpleAppDeployer deployer = new SimpleAppDeployer(client, adminManager);
352355
deployer.setCommands(getCommands(config));
353356
deployer.undeploy(config);

‎marklogic-data-hub/src/main/java/com/marklogic/hub/commands/DeployHubDatabaseCommand.java‎

Lines changed: 47 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -10,6 +10,16 @@
1010
*/
1111
public class DeployHubDatabaseCommand extends DeployDatabaseCommand {
1212

13+
public enum DBType {
14+
STAGING, FINAL, TRACING;
15+
}
16+
17+
private DBType dbType;
18+
19+
public void setDbType(DBType dbType) {
20+
this.dbType = dbType;
21+
}
22+
1323
public DeployHubDatabaseCommand(String databaseName) {
1424
super();
1525
this.setDatabaseName(databaseName);
@@ -23,8 +33,42 @@ public void undo(CommandContext context) {
2333

2434
@Override
2535
protected String getPayload(CommandContext context) {
26-
return format("{\"database-name\": \"%s\","
27-
+ "\"triple-index\": true,"
28-
+ "\"collection-lexicon\":true}", getDatabaseName());
36+
String payload =
37+
"{" +
38+
" \"database-name\": \"%s\"," +
39+
" \"triple-index\": true," +
40+
" \"collection-lexicon\":true";
41+
if (this.dbType.equals(DBType.STAGING)) {
42+
payload += "," +
43+
"\"range-element-index\": [" +
44+
" {" +
45+
" \"scalar-type\": \"unsignedInt\"," +
46+
" \"namespace-uri\": \"http://marklogic.com/data-hub/trace\"," +
47+
" \"localname\": \"is-tracing-enabled\"," +
48+
" \"collation\": \"\"," +
49+
" \"range-value-positions\": false," +
50+
" \"invalid-values\": \"reject\"" +
51+
" }" +
52+
"]";
53+
}
54+
else if (this.dbType.equals(DBType.FINAL)) {
55+
56+
}
57+
else if (this.dbType.equals(DBType.TRACING)) {
58+
payload += "," +
59+
"\"range-element-index\": [" +
60+
" {" +
61+
" \"scalar-type\": \"string\"," +
62+
" \"namespace-uri\": \"\"," +
63+
" \"localname\": \"trace-id\"," +
64+
" \"collation\": \"http://marklogic.com/collation/codepoint\"," +
65+
" \"range-value-positions\": false," +
66+
" \"invalid-values\": \"reject\"" +
67+
" }" +
68+
"]";
69+
}
70+
payload += "}";
71+
72+
return format(payload, getDatabaseName());
2973
}
3074
}

‎marklogic-data-hub/src/main/java/com/marklogic/hub/commands/LoadModulesCommand.java‎

Lines changed: 15 additions & 4 deletions
Original file line numberDiff line numberDiff line change
@@ -19,14 +19,15 @@
1919
import com.marklogic.client.admin.ResourceExtensionsManager.MethodParameters;
2020
import com.marklogic.client.modulesloader.ExtensionMetadataAndParams;
2121
import com.marklogic.client.modulesloader.impl.DefaultModulesLoader;
22+
import com.marklogic.client.modulesloader.impl.XccAssetLoader;
2223
import com.marklogic.client.modulesloader.xcc.CommaDelimitedPermissionsParser;
24+
import com.marklogic.client.modulesloader.xcc.DefaultDocumentFormatGetter;
2325
import com.marklogic.client.modulesloader.xcc.PermissionsParser;
2426
import com.marklogic.xcc.Content;
2527
import com.marklogic.xcc.ContentCreateOptions;
2628
import com.marklogic.xcc.ContentFactory;
2729
import com.marklogic.xcc.ContentSource;
2830
import com.marklogic.xcc.ContentSourceFactory;
29-
import com.marklogic.xcc.DocumentFormat;
3031
import com.marklogic.xcc.SecurityOptions;
3132
import com.marklogic.xcc.Session;
3233
import com.marklogic.xcc.Session.TransactionMode;
@@ -72,7 +73,12 @@ private List<Resource> findResources(String basePath, String... paths) throws IO
7273

7374
protected void loadFile(String uri, InputStream inputStream, AppConfig config) throws IOException {
7475
ContentCreateOptions options = new ContentCreateOptions();
75-
options.setFormat(DocumentFormat.TEXT);
76+
if (uri.endsWith("xml")) {
77+
options.setFormatXml();
78+
}
79+
else {
80+
options.setFormatText();
81+
}
7682
options.setPermissions(permissionsParser.parsePermissions(this.permissions));
7783
if (this.collections != null) {
7884
options.setCollections(collections);
@@ -100,7 +106,12 @@ protected void loadFile(String uri, InputStream inputStream, AppConfig config) t
100106

101107
protected void initializeActiveSession(CommandContext context) {
102108
AppConfig config = context.getAppConfig();
103-
this.modulesLoader = new DefaultModulesLoader(context.getAppConfig().newXccAssetLoader());
109+
XccAssetLoader xccAssetLoader = context.getAppConfig().newXccAssetLoader();
110+
DefaultDocumentFormatGetter documentFormatGetter = new DefaultDocumentFormatGetter();
111+
documentFormatGetter.getBinaryExtensions().add("woff2");
112+
documentFormatGetter.getBinaryExtensions().add("otf");
113+
xccAssetLoader.setDocumentFormatGetter(documentFormatGetter);
114+
this.modulesLoader = new DefaultModulesLoader(xccAssetLoader);
104115
this.modulesLoader.setDatabaseClient(config.newDatabaseClient());
105116
ContentSource cs = ContentSourceFactory.newContentSource(config.getHost(), port, config.getRestAdminUsername(), config.getRestAdminPassword(), config.getModulesDatabaseName(),
106117
securityOptions);
@@ -116,7 +127,7 @@ public void execute(CommandContext context) {
116127
String rootPath = "/ml-modules/root";
117128

118129
AppConfig appConfig = context.getAppConfig();
119-
List<Resource> resources = findResources("classpath:" + rootPath, "/**/*.xqy");
130+
List<Resource> resources = findResources("classpath:" + rootPath, "/**/*.x??");
120131
for (Resource r : resources) {
121132
String path = r.getURL().getPath();
122133
if (path.contains("!")) {

‎marklogic-data-hub/src/main/resources/ml-config/databases/staging-database.json‎

Lines changed: 10 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -1,6 +1,15 @@
11
{
22
"database-name": "%%STAGING_DB_NAME%%",
3-
"range-element-index": [],
3+
"range-element-index": [
4+
{
5+
"scalar-type": "unsignedInt",
6+
"namespace-uri": "http://marklogic.com/data-hub/trace",
7+
"localname": "is-tracing-enabled",
8+
"collation": "",
9+
"range-value-positions": false,
10+
"invalid-values": "reject"
11+
}
12+
],
413
"schema-database": "%%SCHEMAS_DATABASE%%",
514
"triggers-database": "%%TRIGGERS_DATABASE%%",
615
"triple-index": true,

‎marklogic-data-hub/src/main/resources/ml-config/databases/trace-database.json‎

Lines changed: 10 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -1,6 +1,15 @@
11
{
22
"database-name": "%%TRACE_DB_NAME%%",
3-
"range-element-index": [],
3+
"range-element-index": [
4+
{
5+
"scalar-type": "string",
6+
"namespace-uri": "",
7+
"localname": "trace-id",
8+
"collation": "http://marklogic.com/collation/codepoint",
9+
"range-value-positions": false,
10+
"invalid-values": "reject"
11+
}
12+
],
413
"schema-database": "%%SCHEMAS_DATABASE%%",
514
"triggers-database": "%%TRIGGERS_DATABASE%%",
615
"triple-index": true,

0 commit comments

Comments
 (0)