Skip to content

Commit 8c4d1cc

Browse files
paxtonharepaxtonhare
committed
fixing regression with uninstalling hub
updating trace library to write 1 doc per transaction added range indexes on trace db added more robust tests for tracing added more verbose logging for unit tests updated travis installer to get its marklogic version from a variable
1 parent 10ad71f commit 8c4d1cc

File tree

41 files changed

+719
-258
lines changed

Some content is hidden

Large Commits have some content hidden by default. Use the searchbox below for content that may be hidden.

41 files changed

+719
-258
lines changed

‎examples/gradle-basic/build.gradle‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -10,7 +10,7 @@ plugins {
1010
id 'eclipse'
1111
id 'idea'
1212
id 'net.saliman.properties' version '1.4.5'
13-
id 'com.marklogic.ml-data-hub-plugin' version '1.0.0-beta.1'
13+
id 'com.marklogic.ml-data-hub-plugin' version '1.0.0-beta.4'
1414
id 'com.marklogic.ml-gradle' version '2.1.0'
1515
}
1616

‎marklogic-data-hub/src/main/java/com/marklogic/hub/DataHub.java‎

Lines changed: 7 additions & 4 deletions
Original file line numberDiff line numberDiff line change
@@ -55,6 +55,7 @@
5555
import com.marklogic.hub.commands.DeployRestApiCommand;
5656
import com.marklogic.hub.commands.LoadModulesCommand;
5757
import com.marklogic.hub.commands.UpdateRestApiServersCommand;
58+
import com.marklogic.hub.commands.DeployHubDatabaseCommand.DBType;
5859
import com.marklogic.hub.util.HubFileFilter;
5960
import com.marklogic.hub.util.HubModulesLoader;
6061
import com.marklogic.mgmt.ManageClient;
@@ -300,14 +301,17 @@ private List<Command> getCommands(AppConfig config) {
300301
List<Command> dbCommands = new ArrayList<Command>();
301302
DeployHubDatabaseCommand staging = new DeployHubDatabaseCommand(hubConfig.stagingDbName);
302303
staging.setForestsPerHost(hubConfig.stagingForestsPerHost);
304+
staging.setDbType(DBType.STAGING);
303305
dbCommands.add(staging);
304306

305307
DeployHubDatabaseCommand finalDb = new DeployHubDatabaseCommand(hubConfig.finalDbName);
306308
finalDb.setForestsPerHost(hubConfig.finalForestsPerHost);
309+
finalDb.setDbType(DBType.FINAL);
307310
dbCommands.add(finalDb);
308311

309312
DeployHubDatabaseCommand tracingDb = new DeployHubDatabaseCommand(hubConfig.tracingDbName);
310313
tracingDb.setForestsPerHost(hubConfig.tracingForestsPerHost);
314+
tracingDb.setDbType(DBType.TRACING);
311315
dbCommands.add(tracingDb);
312316

313317
dbCommands.add(new DeployModulesDatabaseCommand(hubConfig.modulesDbName));
@@ -318,9 +322,9 @@ private List<Command> getCommands(AppConfig config) {
318322
commands.add(new DeployRestApiCommand(hubConfig.finalHttpName, hubConfig.finalPort, hubConfig.finalDbName, hubConfig.finalForestsPerHost));
319323
commands.add(new DeployRestApiCommand(hubConfig.tracingHttpName, hubConfig.tracePort, hubConfig.tracingDbName, hubConfig.tracingForestsPerHost));
320324

321-
commands.add(new UpdateRestApiServersCommand(hubConfig.stagingHttpName));
322-
commands.add(new UpdateRestApiServersCommand(hubConfig.finalHttpName));
323-
commands.add(new UpdateRestApiServersCommand(hubConfig.tracingHttpName));
325+
commands.add(new UpdateRestApiServersCommand(hubConfig.stagingHttpName, "/com.marklogic.hub/staging-rewriter.xml"));
326+
commands.add(new UpdateRestApiServersCommand(hubConfig.finalHttpName, "/com.marklogic.hub/final-rewriter.xml"));
327+
commands.add(new UpdateRestApiServersCommand(hubConfig.tracingHttpName, "/com.marklogic.hub/tracing-rewriter.xml"));
324328

325329
// Modules
326330
commands.add(new LoadModulesCommand());
@@ -346,7 +350,6 @@ public void uninstall() throws IOException {
346350
LOGGER.debug("Uninstalling the Data Hub from MarkLogic");
347351
AppConfig config = getAppConfig();
348352
AdminManager adminManager = getAdminManager();
349-
adminManager.setWaitForRestartCheckInterval(250);
350353
SimpleAppDeployer deployer = new SimpleAppDeployer(client, adminManager);
351354
deployer.setCommands(getCommands(config));
352355
deployer.undeploy(config);

‎marklogic-data-hub/src/main/java/com/marklogic/hub/commands/DeployHubDatabaseCommand.java‎

Lines changed: 47 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -10,6 +10,16 @@
1010
*/
1111
public class DeployHubDatabaseCommand extends DeployDatabaseCommand {
1212

13+
public enum DBType {
14+
STAGING, FINAL, TRACING;
15+
}
16+
17+
private DBType dbType;
18+
19+
public void setDbType(DBType dbType) {
20+
this.dbType = dbType;
21+
}
22+
1323
public DeployHubDatabaseCommand(String databaseName) {
1424
super();
1525
this.setDatabaseName(databaseName);
@@ -23,8 +33,42 @@ public void undo(CommandContext context) {
2333

2434
@Override
2535
protected String getPayload(CommandContext context) {
26-
return format("{\"database-name\": \"%s\","
27-
+ "\"triple-index\": true,"
28-
+ "\"collection-lexicon\":true}", getDatabaseName());
36+
String payload =
37+
"{" +
38+
" \"database-name\": \"%s\"," +
39+
" \"triple-index\": true," +
40+
" \"collection-lexicon\":true";
41+
if (this.dbType.equals(DBType.STAGING)) {
42+
payload += "," +
43+
"\"range-element-index\": [" +
44+
" {" +
45+
" \"scalar-type\": \"unsignedInt\"," +
46+
" \"namespace-uri\": \"http://marklogic.com/data-hub/trace\"," +
47+
" \"localname\": \"is-tracing-enabled\"," +
48+
" \"collation\": \"\"," +
49+
" \"range-value-positions\": false," +
50+
" \"invalid-values\": \"reject\"" +
51+
" }" +
52+
"]";
53+
}
54+
else if (this.dbType.equals(DBType.FINAL)) {
55+
56+
}
57+
else if (this.dbType.equals(DBType.TRACING)) {
58+
payload += "," +
59+
"\"range-element-index\": [" +
60+
" {" +
61+
" \"scalar-type\": \"string\"," +
62+
" \"namespace-uri\": \"\"," +
63+
" \"localname\": \"trace-id\"," +
64+
" \"collation\": \"http://marklogic.com/collation/codepoint\"," +
65+
" \"range-value-positions\": false," +
66+
" \"invalid-values\": \"reject\"" +
67+
" }" +
68+
"]";
69+
}
70+
payload += "}";
71+
72+
return format(payload, getDatabaseName());
2973
}
3074
}

‎marklogic-data-hub/src/main/java/com/marklogic/hub/commands/LoadModulesCommand.java‎

Lines changed: 16 additions & 4 deletions
Original file line numberDiff line numberDiff line change
@@ -19,14 +19,16 @@
1919
import com.marklogic.client.admin.ResourceExtensionsManager.MethodParameters;
2020
import com.marklogic.client.modulesloader.ExtensionMetadataAndParams;
2121
import com.marklogic.client.modulesloader.impl.DefaultModulesLoader;
22+
import com.marklogic.client.modulesloader.impl.XccAssetLoader;
2223
import com.marklogic.client.modulesloader.xcc.CommaDelimitedPermissionsParser;
24+
import com.marklogic.client.modulesloader.xcc.DefaultDocumentFormatGetter;
25+
import com.marklogic.client.modulesloader.xcc.DocumentFormatGetter;
2326
import com.marklogic.client.modulesloader.xcc.PermissionsParser;
2427
import com.marklogic.xcc.Content;
2528
import com.marklogic.xcc.ContentCreateOptions;
2629
import com.marklogic.xcc.ContentFactory;
2730
import com.marklogic.xcc.ContentSource;
2831
import com.marklogic.xcc.ContentSourceFactory;
29-
import com.marklogic.xcc.DocumentFormat;
3032
import com.marklogic.xcc.SecurityOptions;
3133
import com.marklogic.xcc.Session;
3234
import com.marklogic.xcc.Session.TransactionMode;
@@ -72,7 +74,12 @@ private List<Resource> findResources(String basePath, String... paths) throws IO
7274

7375
protected void loadFile(String uri, InputStream inputStream, AppConfig config) throws IOException {
7476
ContentCreateOptions options = new ContentCreateOptions();
75-
options.setFormat(DocumentFormat.TEXT);
77+
if (uri.endsWith("xml")) {
78+
options.setFormatXml();
79+
}
80+
else {
81+
options.setFormatText();
82+
}
7683
options.setPermissions(permissionsParser.parsePermissions(this.permissions));
7784
if (this.collections != null) {
7885
options.setCollections(collections);
@@ -100,7 +107,12 @@ protected void loadFile(String uri, InputStream inputStream, AppConfig config) t
100107

101108
protected void initializeActiveSession(CommandContext context) {
102109
AppConfig config = context.getAppConfig();
103-
this.modulesLoader = new DefaultModulesLoader(context.getAppConfig().newXccAssetLoader());
110+
XccAssetLoader xccAssetLoader = context.getAppConfig().newXccAssetLoader();
111+
DefaultDocumentFormatGetter documentFormatGetter = new DefaultDocumentFormatGetter();
112+
documentFormatGetter.getBinaryExtensions().add("woff2");
113+
documentFormatGetter.getBinaryExtensions().add("otf");
114+
xccAssetLoader.setDocumentFormatGetter(documentFormatGetter);
115+
this.modulesLoader = new DefaultModulesLoader(xccAssetLoader);
104116
this.modulesLoader.setDatabaseClient(config.newDatabaseClient());
105117
ContentSource cs = ContentSourceFactory.newContentSource(config.getHost(), port, config.getRestAdminUsername(), config.getRestAdminPassword(), config.getModulesDatabaseName(),
106118
securityOptions);
@@ -116,7 +128,7 @@ public void execute(CommandContext context) {
116128
String rootPath = "/ml-modules/root";
117129

118130
AppConfig appConfig = context.getAppConfig();
119-
List<Resource> resources = findResources("classpath:" + rootPath, "/**/*.xqy");
131+
List<Resource> resources = findResources("classpath:" + rootPath, "/**/*.x??");
120132
for (Resource r : resources) {
121133
String path = r.getURL().getPath();
122134
if (path.contains("!")) {

‎marklogic-data-hub/src/main/resources/ml-config/databases/staging-database.json‎

Lines changed: 10 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -1,6 +1,15 @@
11
{
22
"database-name": "%%STAGING_DB_NAME%%",
3-
"range-element-index": [],
3+
"range-element-index": [
4+
{
5+
"scalar-type": "unsignedInt",
6+
"namespace-uri": "http://marklogic.com/data-hub/trace",
7+
"localname": "is-tracing-enabled",
8+
"collation": "",
9+
"range-value-positions": false,
10+
"invalid-values": "reject"
11+
}
12+
],
413
"schema-database": "%%SCHEMAS_DATABASE%%",
514
"triggers-database": "%%TRIGGERS_DATABASE%%",
615
"triple-index": true,

‎marklogic-data-hub/src/main/resources/ml-config/databases/trace-database.json‎

Lines changed: 10 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -1,6 +1,15 @@
11
{
22
"database-name": "%%TRACE_DB_NAME%%",
3-
"range-element-index": [],
3+
"range-element-index": [
4+
{
5+
"scalar-type": "string",
6+
"namespace-uri": "",
7+
"localname": "trace-id",
8+
"collation": "http://marklogic.com/collation/codepoint",
9+
"range-value-positions": false,
10+
"invalid-values": "reject"
11+
}
12+
],
413
"schema-database": "%%SCHEMAS_DATABASE%%",
514
"triggers-database": "%%TRIGGERS_DATABASE%%",
615
"triple-index": true,

0 commit comments

Comments
 (0)