Skip to content

Commit c2cd767

Browse files
maeisabelle08maeisabelle08
committed
Merge branch 'master' into 193-PerformanceTrace
* master: fixing indentation. adding 2 files I missed before fixing regression with uninstalling hub updating trace library to write 1 doc per transaction added range indexes on trace db added more robust tests for tracing added more verbose logging for unit tests updated travis installer to get its marklogic version from a variable fixing bug in trace for errors Conflicts: marklogic-data-hub/src/main/resources/ml-modules/transforms/run-flow.xqy
2 parents 1ab1b86 + be718b2 commit c2cd767

File tree

42 files changed

+725
-247
lines changed

Some content is hidden

Large Commits have some content hidden by default. Use the searchbox below for content that may be hidden.

42 files changed

+725
-247
lines changed

‎.gitignore‎

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -14,3 +14,4 @@ assetInstallTime.properties
1414
/quick-start/input
1515
/quick-start/plugins
1616
flow.properties
17+
node_modules/

‎examples/gradle-basic/build.gradle‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -10,7 +10,7 @@ plugins {
1010
id 'eclipse'
1111
id 'idea'
1212
id 'net.saliman.properties' version '1.4.5'
13-
id 'com.marklogic.ml-data-hub-plugin' version '1.0.0-beta.1'
13+
id 'com.marklogic.ml-data-hub-plugin' version '1.0.0-beta.4'
1414
id 'com.marklogic.ml-gradle' version '2.1.0'
1515
}
1616

‎marklogic-data-hub/src/main/java/com/marklogic/hub/DataHub.java‎

Lines changed: 4 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -56,6 +56,7 @@
5656
import com.marklogic.hub.commands.DeployRestApiCommand;
5757
import com.marklogic.hub.commands.LoadModulesCommand;
5858
import com.marklogic.hub.commands.UpdateRestApiServersCommand;
59+
import com.marklogic.hub.commands.DeployHubDatabaseCommand.DBType;
5960
import com.marklogic.hub.util.HubFileFilter;
6061
import com.marklogic.hub.util.HubModulesLoader;
6162
import com.marklogic.hub.util.PerformanceLogger;
@@ -324,14 +325,17 @@ private List<Command> getCommands(AppConfig config) {
324325
List<Command> dbCommands = new ArrayList<Command>();
325326
DeployHubDatabaseCommand staging = new DeployHubDatabaseCommand(hubConfig.stagingDbName);
326327
staging.setForestsPerHost(hubConfig.stagingForestsPerHost);
328+
staging.setDbType(DBType.STAGING);
327329
dbCommands.add(staging);
328330

329331
DeployHubDatabaseCommand finalDb = new DeployHubDatabaseCommand(hubConfig.finalDbName);
330332
finalDb.setForestsPerHost(hubConfig.finalForestsPerHost);
333+
finalDb.setDbType(DBType.FINAL);
331334
dbCommands.add(finalDb);
332335

333336
DeployHubDatabaseCommand tracingDb = new DeployHubDatabaseCommand(hubConfig.tracingDbName);
334337
tracingDb.setForestsPerHost(hubConfig.tracingForestsPerHost);
338+
tracingDb.setDbType(DBType.TRACING);
335339
dbCommands.add(tracingDb);
336340

337341
dbCommands.add(new DeployModulesDatabaseCommand(hubConfig.modulesDbName));
@@ -371,7 +375,6 @@ public void uninstall() throws IOException {
371375

372376
AppConfig config = getAppConfig();
373377
AdminManager adminManager = getAdminManager();
374-
adminManager.setWaitForRestartCheckInterval(250);
375378
SimpleAppDeployer deployer = new SimpleAppDeployer(client, adminManager);
376379
deployer.setCommands(getCommands(config));
377380
deployer.undeploy(config);

‎marklogic-data-hub/src/main/java/com/marklogic/hub/commands/DeployHubDatabaseCommand.java‎

Lines changed: 47 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -10,6 +10,16 @@
1010
*/
1111
public class DeployHubDatabaseCommand extends DeployDatabaseCommand {
1212

13+
public enum DBType {
14+
STAGING, FINAL, TRACING;
15+
}
16+
17+
private DBType dbType;
18+
19+
public void setDbType(DBType dbType) {
20+
this.dbType = dbType;
21+
}
22+
1323
public DeployHubDatabaseCommand(String databaseName) {
1424
super();
1525
this.setDatabaseName(databaseName);
@@ -23,8 +33,42 @@ public void undo(CommandContext context) {
2333

2434
@Override
2535
protected String getPayload(CommandContext context) {
26-
return format("{\"database-name\": \"%s\","
27-
+ "\"triple-index\": true,"
28-
+ "\"collection-lexicon\":true}", getDatabaseName());
36+
String payload =
37+
"{" +
38+
" \"database-name\": \"%s\"," +
39+
" \"triple-index\": true," +
40+
" \"collection-lexicon\":true";
41+
if (this.dbType.equals(DBType.STAGING)) {
42+
payload += "," +
43+
"\"range-element-index\": [" +
44+
" {" +
45+
" \"scalar-type\": \"unsignedInt\"," +
46+
" \"namespace-uri\": \"http://marklogic.com/data-hub/trace\"," +
47+
" \"localname\": \"is-tracing-enabled\"," +
48+
" \"collation\": \"\"," +
49+
" \"range-value-positions\": false," +
50+
" \"invalid-values\": \"reject\"" +
51+
" }" +
52+
"]";
53+
}
54+
else if (this.dbType.equals(DBType.FINAL)) {
55+
56+
}
57+
else if (this.dbType.equals(DBType.TRACING)) {
58+
payload += "," +
59+
"\"range-element-index\": [" +
60+
" {" +
61+
" \"scalar-type\": \"string\"," +
62+
" \"namespace-uri\": \"\"," +
63+
" \"localname\": \"trace-id\"," +
64+
" \"collation\": \"http://marklogic.com/collation/codepoint\"," +
65+
" \"range-value-positions\": false," +
66+
" \"invalid-values\": \"reject\"" +
67+
" }" +
68+
"]";
69+
}
70+
payload += "}";
71+
72+
return format(payload, getDatabaseName());
2973
}
3074
}

‎marklogic-data-hub/src/main/java/com/marklogic/hub/commands/LoadModulesCommand.java‎

Lines changed: 15 additions & 4 deletions
Original file line numberDiff line numberDiff line change
@@ -19,14 +19,15 @@
1919
import com.marklogic.client.admin.ResourceExtensionsManager.MethodParameters;
2020
import com.marklogic.client.modulesloader.ExtensionMetadataAndParams;
2121
import com.marklogic.client.modulesloader.impl.DefaultModulesLoader;
22+
import com.marklogic.client.modulesloader.impl.XccAssetLoader;
2223
import com.marklogic.client.modulesloader.xcc.CommaDelimitedPermissionsParser;
24+
import com.marklogic.client.modulesloader.xcc.DefaultDocumentFormatGetter;
2325
import com.marklogic.client.modulesloader.xcc.PermissionsParser;
2426
import com.marklogic.xcc.Content;
2527
import com.marklogic.xcc.ContentCreateOptions;
2628
import com.marklogic.xcc.ContentFactory;
2729
import com.marklogic.xcc.ContentSource;
2830
import com.marklogic.xcc.ContentSourceFactory;
29-
import com.marklogic.xcc.DocumentFormat;
3031
import com.marklogic.xcc.SecurityOptions;
3132
import com.marklogic.xcc.Session;
3233
import com.marklogic.xcc.Session.TransactionMode;
@@ -72,7 +73,12 @@ private List<Resource> findResources(String basePath, String... paths) throws IO
7273

7374
protected void loadFile(String uri, InputStream inputStream, AppConfig config) throws IOException {
7475
ContentCreateOptions options = new ContentCreateOptions();
75-
options.setFormat(DocumentFormat.TEXT);
76+
if (uri.endsWith("xml")) {
77+
options.setFormatXml();
78+
}
79+
else {
80+
options.setFormatText();
81+
}
7682
options.setPermissions(permissionsParser.parsePermissions(this.permissions));
7783
if (this.collections != null) {
7884
options.setCollections(collections);
@@ -100,7 +106,12 @@ protected void loadFile(String uri, InputStream inputStream, AppConfig config) t
100106

101107
protected void initializeActiveSession(CommandContext context) {
102108
AppConfig config = context.getAppConfig();
103-
this.modulesLoader = new DefaultModulesLoader(context.getAppConfig().newXccAssetLoader());
109+
XccAssetLoader xccAssetLoader = context.getAppConfig().newXccAssetLoader();
110+
DefaultDocumentFormatGetter documentFormatGetter = new DefaultDocumentFormatGetter();
111+
documentFormatGetter.getBinaryExtensions().add("woff2");
112+
documentFormatGetter.getBinaryExtensions().add("otf");
113+
xccAssetLoader.setDocumentFormatGetter(documentFormatGetter);
114+
this.modulesLoader = new DefaultModulesLoader(xccAssetLoader);
104115
this.modulesLoader.setDatabaseClient(config.newDatabaseClient());
105116
ContentSource cs = ContentSourceFactory.newContentSource(config.getHost(), port, config.getRestAdminUsername(), config.getRestAdminPassword(), config.getModulesDatabaseName(),
106117
securityOptions);
@@ -116,7 +127,7 @@ public void execute(CommandContext context) {
116127
String rootPath = "/ml-modules/root";
117128

118129
AppConfig appConfig = context.getAppConfig();
119-
List<Resource> resources = findResources("classpath:" + rootPath, "/**/*.xqy");
130+
List<Resource> resources = findResources("classpath:" + rootPath, "/**/*.x??");
120131
for (Resource r : resources) {
121132
String path = r.getURL().getPath();
122133
if (path.contains("!")) {

‎marklogic-data-hub/src/main/resources/ml-config/databases/staging-database.json‎

Lines changed: 10 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -1,6 +1,15 @@
11
{
22
"database-name": "%%STAGING_DB_NAME%%",
3-
"range-element-index": [],
3+
"range-element-index": [
4+
{
5+
"scalar-type": "unsignedInt",
6+
"namespace-uri": "http://marklogic.com/data-hub/trace",
7+
"localname": "is-tracing-enabled",
8+
"collation": "",
9+
"range-value-positions": false,
10+
"invalid-values": "reject"
11+
}
12+
],
413
"schema-database": "%%SCHEMAS_DATABASE%%",
514
"triggers-database": "%%TRIGGERS_DATABASE%%",
615
"triple-index": true,

‎marklogic-data-hub/src/main/resources/ml-config/databases/trace-database.json‎

Lines changed: 10 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -1,6 +1,15 @@
11
{
22
"database-name": "%%TRACE_DB_NAME%%",
3-
"range-element-index": [],
3+
"range-element-index": [
4+
{
5+
"scalar-type": "string",
6+
"namespace-uri": "",
7+
"localname": "trace-id",
8+
"collation": "http://marklogic.com/collation/codepoint",
9+
"range-value-positions": false,
10+
"invalid-values": "reject"
11+
}
12+
],
413
"schema-database": "%%SCHEMAS_DATABASE%%",
514
"triggers-database": "%%TRIGGERS_DATABASE%%",
615
"triple-index": true,

0 commit comments

Comments
 (0)