Skip to content

Commit 52d6ebb

Browse files
paxtonharepaxtonhare
committed
fixing job persistence performance issues
1 parent b132145 commit 52d6ebb

File tree

71 files changed

+273
-982
lines changed

Some content is hidden

Large Commits have some content hidden by default. Use the searchbox below for content that may be hidden.

71 files changed

+273
-982
lines changed

‎marklogic-data-hub/src/main/java/com/marklogic/hub/DataHub.java‎

Lines changed: 4 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -21,6 +21,7 @@
2121
import com.marklogic.appdeployer.command.Command;
2222
import com.marklogic.appdeployer.command.CommandMapBuilder;
2323
import com.marklogic.appdeployer.command.appservers.DeployOtherServersCommand;
24+
import com.marklogic.appdeployer.command.mimetypes.DeployMimetypesCommand;
2425
import com.marklogic.appdeployer.impl.SimpleAppDeployer;
2526
import com.marklogic.client.DatabaseClient;
2627
import com.marklogic.client.FailedRequestException;
@@ -327,6 +328,9 @@ public Map<String, List<Command>> getCommands() {
327328
moduleCommands.add(new LoadUserModulesCommand(hubConfig));
328329
commandMap.put("mlModuleCommands", moduleCommands);
329330

331+
List<Command> mimetypeCommands = commandMap.get("mlMimetypeCommands");
332+
mimetypeCommands.add(0, new DeployHubMimetypesCommand(hubConfig));
333+
330334
return commandMap;
331335
}
332336

‎marklogic-data-hub/src/main/java/com/marklogic/hub/HubConfig.java‎

Lines changed: 4 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -360,6 +360,10 @@ public Path getUserServersDir() {
360360
return Paths.get(this.projectDir, USER_CONFIG_DIR, "servers");
361361
}
362362

363+
public Path getHubMimetypesDir() {
364+
return Paths.get(this.projectDir, HUB_CONFIG_DIR, "mimetypes");
365+
}
366+
363367
public void setUsername(String username) {
364368
this.username = username;
365369
}

‎marklogic-data-hub/src/main/java/com/marklogic/hub/HubProject.java‎

Lines changed: 5 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -100,6 +100,11 @@ public void init() {
100100
writeResourceFileWithReplace("ml-config/security/roles/data-hub-user.json", rolesDir.resolve("data-hub-user.json"), true);
101101
writeResourceFileWithReplace("ml-config/security/users/data-hub-user.json", usersDir.resolve("data-hub-user.json"), true);
102102

103+
Path mimetypesDir = hubConfig.getHubMimetypesDir();
104+
mimetypesDir.toFile().mkdirs();
105+
writeResourceFile("ml-config/mimetypes/woff.json", mimetypesDir.resolve("woff.json"), true);
106+
writeResourceFile("ml-config/mimetypes/woff2.json", mimetypesDir.resolve("woff2.json"), true);
107+
103108
hubConfig.getUserServersDir().toFile().mkdirs();
104109
hubConfig.getUserDatabaseDir().toFile().mkdirs();
105110

‎marklogic-data-hub/src/main/java/com/marklogic/hub/deploy/commands/DeployHubMimetypesCommand.java‎

Lines changed: 22 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,22 @@
1+
package com.marklogic.hub.deploy.commands;
2+
3+
import com.marklogic.appdeployer.command.CommandContext;
4+
import com.marklogic.appdeployer.command.mimetypes.DeployMimetypesCommand;
5+
import com.marklogic.hub.HubConfig;
6+
7+
import java.io.File;
8+
9+
public class DeployHubMimetypesCommand extends DeployMimetypesCommand {
10+
11+
private HubConfig config;
12+
13+
public DeployHubMimetypesCommand(HubConfig config) {
14+
super();
15+
this.config = config;
16+
}
17+
18+
@Override
19+
protected File[] getResourceDirs(CommandContext context) {
20+
return new File[]{config.getHubMimetypesDir().toFile()};
21+
}
22+
}

‎marklogic-data-hub/src/main/java/com/marklogic/hub/deploy/commands/LoadHubModulesCommand.java‎

Lines changed: 11 additions & 7 deletions
Original file line numberDiff line numberDiff line change
@@ -76,15 +76,19 @@ private Content prepContent(String uri, InputStream inputStream, AppConfig confi
7676
logger.info(format("Inserting module with URI: %s", uri));
7777
}
7878

79-
String fileContents = IOUtils.toString(inputStream);
80-
Map<String, String> customTokens = config.getCustomTokens();
81-
if (customTokens != null) {
82-
for (String key : customTokens.keySet()) {
83-
fileContents = fileContents.replace(key, customTokens.get(key));
79+
if (uri.endsWith(".xqy")) {
80+
String fileContents = IOUtils.toString(inputStream);
81+
Map<String, String> customTokens = config.getCustomTokens();
82+
if (customTokens != null) {
83+
for (String key : customTokens.keySet()) {
84+
fileContents = fileContents.replace(key, customTokens.get(key));
85+
}
8486
}
87+
88+
return ContentFactory.newContent(uri, fileContents, options);
8589
}
8690

87-
return ContentFactory.newContent(uri, fileContents, options);
91+
return ContentFactory.newContent(uri, inputStream, options);
8892
}
8993

9094
private void initializeActiveSession(CommandContext context) {
@@ -199,7 +203,7 @@ public void execute(CommandContext context) {
199203
// switch to job db to do this:
200204
this.modulesLoader.setDatabaseClient(hubConfig.newJobDbClient());
201205
startTime = System.nanoTime();
202-
resources = findResources("classpath*:/ml-modules/options", "/**/spring-batch.xml");
206+
resources = findResources("classpath*:/ml-modules/options", "/**/jobs.xml");
203207
for (Resource r : resources) {
204208
this.modulesLoader.installQueryOptions(r);
205209
}

‎marklogic-data-hub/src/main/resources/ml-config/databases/job-database.json‎

Lines changed: 25 additions & 55 deletions
Original file line numberDiff line numberDiff line change
@@ -2,95 +2,66 @@
22
"database-name": "%%mlJobDbName%%",
33
"range-element-index": [
44
{
5-
"scalar-type": "dateTime",
6-
"namespace-uri": "http://marklogic.com/xdmp/dls",
7-
"localname": "created",
8-
"collation": "",
9-
"range-value-positions": false,
10-
"invalid-values": "reject"
11-
},
12-
{
13-
"scalar-type": "dateTime",
14-
"namespace-uri": "http://marklogic.com/xdmp/dls",
15-
"localname": "replaced",
16-
"collation": "",
5+
"scalar-type": "string",
6+
"namespace-uri": "",
7+
"localname": "jobId",
8+
"collation": "http://marklogic.com/collation/codepoint",
179
"range-value-positions": false,
1810
"invalid-values": "reject"
1911
},
2012
{
21-
"scalar-type": "unsignedLong",
22-
"namespace-uri": "http://marklogic.com/xdmp/dls",
23-
"localname": "version-id",
24-
"collation": "",
13+
"scalar-type": "string",
14+
"namespace-uri": "",
15+
"localname": "jobName",
16+
"collation": "http://marklogic.com/collation/codepoint",
2517
"range-value-positions": false,
2618
"invalid-values": "reject"
2719
},
2820
{
29-
"scalar-type": "unsignedLong",
30-
"namespace-uri": "http://marklogic.com/spring-batch",
31-
"localname": "id",
21+
"scalar-type": "dateTime",
22+
"namespace-uri": "",
23+
"localname": "startTime",
3224
"collation": "",
3325
"range-value-positions": false,
3426
"invalid-values": "reject"
35-
36-
},
37-
{
38-
"scalar-type": "string",
39-
"namespace-uri": "http://marklogic.com/spring-batch",
40-
"localname": "jobName",
41-
"collation": "http://marklogic.com/collation/",
42-
"range-value-positions": false,
43-
"invalid-values": "reject"
4427
},
4528
{
4629
"scalar-type": "dateTime",
47-
"namespace-uri": "http://marklogic.com/spring-batch",
48-
"localname": "createDateTime",
30+
"namespace-uri": "",
31+
"localname": "endTime",
4932
"collation": "",
5033
"range-value-positions": false,
5134
"invalid-values": "reject"
5235
},
5336
{
5437
"scalar-type": "string",
55-
"namespace-uri": "http://marklogic.com/spring-batch",
38+
"namespace-uri": "",
5639
"localname": "status",
57-
"collation": "http://marklogic.com/collation/",
58-
"range-value-positions": false,
59-
"invalid-values": "reject"
60-
}
61-
],
62-
"path-namespace": [
63-
{
64-
"prefix": "msb",
65-
"namespace-uri": "http://marklogic.com/spring-batch"
66-
}
67-
],
68-
"range-path-index": [
69-
{
70-
"scalar-type": "unsignedLong",
71-
"collation": "",
72-
"path-expression": "/msb:mlJobInstance/msb:jobExecutions/msb:jobExecution/msb:id",
40+
"collation": "http://marklogic.com/collation/codepoint",
7341
"range-value-positions": false,
7442
"invalid-values": "reject"
7543
},
7644
{
7745
"scalar-type": "string",
78-
"collation": "http://marklogic.com/collation/",
79-
"path-expression": "/msb:mlJobInstance/msb:jobExecutions/msb:jobExecution/msb:jobParameters/msb:jobParameter[@key='entity']",
46+
"namespace-uri": "",
47+
"localname": "entityName",
48+
"collation": "http://marklogic.com/collation/codepoint",
8049
"range-value-positions": false,
8150
"invalid-values": "reject"
8251
},
8352
{
8453
"scalar-type": "string",
85-
"collation": "http://marklogic.com/collation/",
86-
"path-expression": "/msb:mlJobInstance/msb:jobExecutions/msb:jobExecution/msb:jobParameters/msb:jobParameter[@key='flow']",
54+
"namespace-uri": "",
55+
"localname": "flowName",
56+
"collation": "http://marklogic.com/collation/codepoint",
8757
"range-value-positions": false,
8858
"invalid-values": "reject"
8959
},
9060
{
9161
"scalar-type": "string",
92-
"collation": "http://marklogic.com/collation/",
93-
"path-expression": "/msb:mlJobInstance/msb:jobExecutions/msb:jobExecution/msb:jobParameters/msb:jobParameter[@key='flowType']",
62+
"namespace-uri": "",
63+
"localname": "flowType",
64+
"collation": "http://marklogic.com/collation/codepoint",
9465
"range-value-positions": false,
9566
"invalid-values": "reject"
9667
}
@@ -99,6 +70,5 @@
9970
"triggers-database": "%%TRIGGERS_DATABASE%%",
10071
"triple-index": true,
10172
"collection-lexicon": true,
102-
"uri-lexicon": true,
103-
"maintain-last-modified": true
73+
"uri-lexicon": true
10474
}

‎marklogic-data-hub/src/main/resources/ml-config/mimetypes/woff.json‎

Lines changed: 7 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,7 @@
1+
{
2+
"name": "application/font-woff",
3+
"extension": [
4+
"woff"
5+
],
6+
"format": "binary"
7+
}

‎marklogic-data-hub/src/main/resources/ml-config/mimetypes/woff2.json‎

Lines changed: 7 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,7 @@
1+
{
2+
"name": "application/font-woff2",
3+
"extension": [
4+
"woff2"
5+
],
6+
"format": "binary"
7+
}

‎marklogic-data-hub/src/main/resources/ml-modules/options/jobs.xml‎

Lines changed: 55 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,55 @@
1+
<options xmlns="http://marklogic.com/appservices/search">
2+
<constraint name="jobName">
3+
<range type="xs:string" collation="http://marklogic.com/collation/codepoint">
4+
<json-property>jobName</json-property>
5+
</range>
6+
</constraint>
7+
<constraint name="status">
8+
<range type="xs:string" collation="http://marklogic.com/collation/codepoint">
9+
<json-property>status</json-property>
10+
</range>
11+
</constraint>
12+
<constraint name="entityName">
13+
<range type="xs:string" collation="http://marklogic.com/collation/codepoint">
14+
<json-property>entityName</json-property>
15+
</range>
16+
</constraint>
17+
<constraint name="flowName">
18+
<range type="xs:string" collation="http://marklogic.com/collation/codepoint">
19+
<json-property>flowName</json-property>
20+
</range>
21+
</constraint>
22+
<constraint name="flowType">
23+
<range type="xs:string" collation="http://marklogic.com/collation/codepoint">
24+
<json-property>flowType</json-property>
25+
</range>
26+
</constraint>
27+
<constraint name="endTime">
28+
<range type="xs:dateTime" facet="false">
29+
<json-property>endTime</json-property>
30+
</range>
31+
</constraint>
32+
<constraint name="startTime">
33+
<range type="xs:dateTime" facet="false">
34+
<json-property>startTime</json-property>
35+
</range>
36+
</constraint>
37+
<operator name="sort">
38+
<state name="date">
39+
<sort-order direction="descending" type="xs:dateTime">
40+
<json-property>startTime</json-property>
41+
</sort-order>
42+
</state>
43+
<state name="date-desc">
44+
<sort-order direction="descending" type="xs:dateTime">
45+
<json-property>startTime</json-property>
46+
</sort-order>
47+
</state>
48+
<state name="date-asc">
49+
<sort-order direction="ascending" type="xs:dateTime">
50+
<json-property>startTime</json-property>
51+
</sort-order>
52+
</state>
53+
</operator>
54+
<transform-results apply="raw" />
55+
</options>

0 commit comments

Comments
 (0)