Skip to content

Commit 0407b1a

Browse files
dmcasseldmcassel
authored
Merge pull request #566 from marklogic-community/feature/delete-old-jobs
Feature/delete old jobs
2 parents 1fe1d3b + d97f1f7 commit 0407b1a

File tree

20 files changed

+695
-23
lines changed

20 files changed

+695
-23
lines changed

‎CONTRIBUTING.md‎

Lines changed: 12 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -80,6 +80,18 @@ plugins {
8080
apply plugin: "com.marklogic.ml-data-hub"
8181
```
8282

83+
To run the plugin's unit tests, cd to the ml-data-hub-plugin directory, then:
84+
85+
Run all unit tests
86+
87+
../gradlew test
88+
89+
90+
Run one unit test
91+
92+
../gradlew -Dtest.single=CreateEntityTask test
93+
94+
8395
**Note**: This change goes in a DHF project's build.gradle. Not the DHF source code's build.gradle.
8496

8597
#### Running the QuickStart UI from source

‎marklogic-data-hub/src/main/java/com/marklogic/hub/job/JobDeleteResponse.java‎

Lines changed: 27 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,27 @@
1+
package com.marklogic.hub.job;
2+
3+
import com.fasterxml.jackson.databind.JsonNode;
4+
5+
import java.util.List;
6+
7+
public class JobDeleteResponse {
8+
public long totalCount = 0;
9+
public long errorCount = 0;
10+
public List<String> deletedJobs;
11+
public List<String> deletedTraces;
12+
public List<String> failedJobs;
13+
public List<String> failedTraces;
14+
public List<JsonNode> errors;
15+
16+
public String toString()
17+
{
18+
return
19+
"JobDeleteResponse:" +
20+
"\n\ttotal jobs deleted: " + totalCount +
21+
"\n\ttotal errors: " + errorCount +
22+
"\n\tjobs deleted: " + deletedJobs +
23+
"\n\ttotal traces deleted: " + deletedTraces.size() +
24+
"\n\terrors: " + errors;
25+
}
26+
27+
}

‎marklogic-data-hub/src/main/java/com/marklogic/hub/job/JobManager.java‎

Lines changed: 55 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -23,15 +23,22 @@
2323
import com.marklogic.client.Transaction;
2424
import com.marklogic.client.document.DocumentWriteSet;
2525
import com.marklogic.client.document.JSONDocumentManager;
26+
import com.marklogic.client.extensions.ResourceManager;
27+
import com.marklogic.client.extensions.ResourceServices;
2628
import com.marklogic.client.io.DocumentMetadataHandle;
29+
import com.marklogic.client.io.Format;
2730
import com.marklogic.client.io.JacksonDatabindHandle;
31+
import com.marklogic.client.io.StringHandle;
32+
import com.marklogic.client.util.RequestParameters;
2833

34+
import java.io.IOException;
2935
import java.text.SimpleDateFormat;
3036
import java.util.TimeZone;
3137

3238
public class JobManager {
3339

3440
private JSONDocumentManager docMgr;
41+
private JobDeleteResource jobDeleteRunner = null;
3542

3643
private static final String ISO_8601_FORMAT = "yyyy-MM-dd'T'HH:mm:ss.SSSXXX";
3744
private static SimpleDateFormat simpleDateFormat8601;
@@ -57,6 +64,7 @@ public class JobManager {
5764

5865
public JobManager(DatabaseClient jobClient) {
5966
this.docMgr = jobClient.newJSONDocumentManager();
67+
this.jobDeleteRunner = new JobDeleteResource(jobClient);
6068
}
6169

6270
public void saveJob(Job job) {
@@ -72,4 +80,51 @@ public void saveJob(Job job, Transaction transaction) {
7280
writeSet.add("/jobs/" + job.getJobId() + ".json", metadataHandle, contentHandle);
7381
docMgr.write(writeSet, transaction);
7482
}
83+
84+
public JobDeleteResponse deleteJobs(String jobIds) {
85+
return this.jobDeleteRunner.deleteJobs(jobIds);
86+
}
87+
88+
public class JobDeleteResource extends ResourceManager {
89+
private static final String DELETE_SERVICE = "delete-jobs";
90+
91+
private DatabaseClient srcClient;
92+
93+
public JobDeleteResource(DatabaseClient srcClient) {
94+
super();
95+
this.srcClient = srcClient;
96+
this.srcClient.init(DELETE_SERVICE, this);
97+
}
98+
99+
/**
100+
*
101+
* @param jobIds comma-separated list of jobIds to delete.
102+
* @return comma-separated list of jobIds that were successfully deleted
103+
*/
104+
public JobDeleteResponse deleteJobs(String jobIds) {
105+
JobDeleteResponse resp = null;
106+
try {
107+
RequestParameters params = new RequestParameters();
108+
params.add("jobIds", jobIds);
109+
110+
ResourceServices services = this.getServices();
111+
ResourceServices.ServiceResultIterator resultItr =
112+
services.post(params, new StringHandle("{}").withFormat(Format.JSON));
113+
if (resultItr == null || ! resultItr.hasNext()) {
114+
resp = new JobDeleteResponse();
115+
}
116+
else {
117+
ResourceServices.ServiceResult res = resultItr.next();
118+
StringHandle handle = new StringHandle();
119+
ObjectMapper objectMapper = new ObjectMapper();
120+
resp = objectMapper.readValue(res.getContent(handle).get(), JobDeleteResponse.class);
121+
}
122+
} catch (IOException e) {
123+
e.printStackTrace();
124+
throw new RuntimeException(e);
125+
}
126+
return resp;
127+
}
128+
}
129+
75130
}

‎marklogic-data-hub/src/main/resources/ml-modules/root/com.marklogic.hub/lib/job-lib.xqy‎

Lines changed: 82 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,82 @@
1+
xquery version "1.0-ml";
2+
3+
module namespace job = "http://marklogic.com/data-hub/job-lib";
4+
5+
import module namespace config = "http://marklogic.com/data-hub/config"
6+
at "/com.marklogic.hub/lib/config.xqy";
7+
8+
declare option xdmp:mapping "false";
9+
10+
declare variable $SUCCESS-KEY := "success";
11+
declare variable $FAILED-KEY := "failed";
12+
13+
(:
14+
: Given a list of job-ids, delete those jobs and any traces associated with them.
15+
:
16+
: ML 9.0-1.1 is the only DHF-supported version that does not support the <update> option
17+
: for xdmp:invoke-function. (<transaction-mode> has been deprecated.)
18+
:)
19+
declare function job:delete-jobs-and-traces($job-ids as xs:string*)
20+
{
21+
let $job-results := job:delete-jobs($job-ids)
22+
let $options :=
23+
<options xmlns="xdmp:eval">
24+
{
25+
if (xdmp:version() = "9.0-1.1") then
26+
<transaction-mode>update-auto-commit</transaction-mode>
27+
else
28+
<update>true</update>
29+
}
30+
<isolation>different-transaction</isolation>
31+
<database>{xdmp:database($config:TRACE-DATABASE)}</database>
32+
</options>
33+
let $deleted-traces :=
34+
xdmp:invoke-function(
35+
function() {
36+
job:delete-traces($job-ids)
37+
},
38+
$options
39+
)
40+
return
41+
document {
42+
object-node {
43+
"totalCount": fn:count(map:get($job-results, $SUCCESS-KEY)),
44+
"errorCount": fn:count(map:get($job-results, $FAILED-KEY)),
45+
"deletedJobs": json:to-array(map:get($job-results, $SUCCESS-KEY)),
46+
"deletedTraces": json:to-array($deleted-traces),
47+
"failedJobs": json:to-array(map:get($job-results, $FAILED-KEY)),
48+
"failedTraces": json:array(),
49+
"errors": json:array()
50+
}
51+
}
52+
};
53+
54+
declare private function job:delete-jobs($job-ids as xs:string*) as map:map
55+
{
56+
let $results := map:map()
57+
let $_ :=
58+
for $id in $job-ids
59+
let $uri := cts:uris((), (), cts:json-property-value-query("jobId", $id))
60+
return
61+
if (fn:doc-available($uri)) then (
62+
xdmp:document-delete($uri),
63+
map:put($results, $SUCCESS-KEY, (map:get($results, $SUCCESS-KEY), $id))
64+
)
65+
else
66+
map:put($results, $FAILED-KEY, (map:get($results, $FAILED-KEY), $id))
67+
return $results
68+
};
69+
70+
(:
71+
: tracing document includes:
72+
: <jobId>be3ae963-2cc7-4bcb-b71a-b25e585b7df5</jobId>
73+
: <traceId>11614584719163504615</traceId>
74+
:)
75+
declare private function job:delete-traces($job-ids as xs:string*) as xs:string*
76+
{
77+
for $id in $job-ids
78+
for $trace in /trace[jobId = $id]/traceId
79+
let $_ := xdmp:document-delete(xdmp:node-uri($trace))
80+
return $trace/fn:string()
81+
};
82+

‎marklogic-data-hub/src/main/resources/ml-modules/services/delete-jobs.xqy‎

Lines changed: 65 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,65 @@
1+
(:
2+
Copyright 2012-2016 MarkLogic Corporation
3+
4+
Licensed under the Apache License, Version 2.0 (the "License");
5+
you may not use this file except in compliance with the License.
6+
You may obtain a copy of the License at
7+
8+
http://www.apache.org/licenses/LICENSE-2.0
9+
10+
Unless required by applicable law or agreed to in writing, software
11+
distributed under the License is distributed on an "AS IS" BASIS,
12+
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
13+
See the License for the specific language governing permissions and
14+
limitations under the License.
15+
:)
16+
xquery version "1.0-ml";
17+
18+
module namespace service = "http://marklogic.com/rest-api/resource/delete-jobs";
19+
20+
import module namespace job = "http://marklogic.com/data-hub/job-lib"
21+
at "/com.marklogic.hub/lib/job-lib.xqy";
22+
23+
import module namespace perf = "http://marklogic.com/data-hub/perflog-lib"
24+
at "/com.marklogic.hub/lib/perflog-lib.xqy";
25+
26+
declare namespace rapi = "http://marklogic.com/rest-api";
27+
28+
declare option xdmp:mapping "false";
29+
30+
(:
31+
: REST API extension to delete jobs and associated traces.
32+
: Note: it shouldn't be necessary to use xdmp:invoke-function below, but a bug
33+
: in ML 8.0-7 (fixed in 8.0-7.1) messes up the transaction mode.
34+
:
35+
: Also, ML 9.0-1.1 is the only DHF-supported version that does not support the
36+
: <update> option for xdmp:invoke-function. (<transaction-mode> has been deprecated.)
37+
:)
38+
declare %rapi:transaction-mode("update") function service:post(
39+
$context as map:map,
40+
$params as map:map,
41+
$input as document-node()*
42+
) as document-node()?
43+
{
44+
let $options :=
45+
<options xmlns="xdmp:eval">
46+
{
47+
if (xdmp:version() = "9.0-1.1") then
48+
<transaction-mode>update-auto-commit</transaction-mode>
49+
else
50+
<update>true</update>
51+
}
52+
<isolation>same-statement</isolation>
53+
</options>
54+
return
55+
perf:log('/v1/resources/delete-jobs:post', function() {
56+
xdmp:invoke-function(
57+
function() {
58+
let $job-ids := fn:tokenize(map:get($params, "jobIds"), ",")
59+
return
60+
job:delete-jobs-and-traces($job-ids)
61+
},
62+
$options
63+
)
64+
})
65+
};

‎marklogic-data-hub/src/test/java/com/marklogic/hub/DataHubInstallTest.java‎

Lines changed: 7 additions & 7 deletions
Original file line numberDiff line numberDiff line change
@@ -35,8 +35,8 @@ public void testInstallHubModules() throws IOException {
3535
assertTrue(getModulesFile("/com.marklogic.hub/lib/config.xqy").startsWith(getResource("data-hub-test/core-modules/config.xqy")));
3636
int totalCount = getDocCount(HubConfig.DEFAULT_MODULES_DB_NAME, null);
3737
int hubModulesCount = getDocCount(HubConfig.DEFAULT_MODULES_DB_NAME, "hub-core-module");
38-
assertTrue(totalCount + " is not correct", 81 == totalCount || 61 == totalCount);
39-
assertTrue(hubModulesCount + " is not correct", 41 == hubModulesCount || 21 == hubModulesCount);
38+
assertTrue(totalCount + " is not correct", 84 == totalCount || 64 == totalCount);
39+
assertTrue(hubModulesCount + " is not correct", 42 == hubModulesCount || 22 == hubModulesCount);
4040

4141
assertTrue("trace options not installed", getModulesFile("/Default/data-hub-TRACING/rest-api/options/traces.xml").length() > 0);
4242
assertTrue("trace options not installed", getModulesFile("/Default/data-hub-JOBS/rest-api/options/jobs.xml").length() > 0);
@@ -59,12 +59,12 @@ public void testInstallUserModules() throws IOException, ParserConfigurationExce
5959
DataHub dataHub = new DataHub(hubConfig);
6060

6161
int totalCount = getDocCount(HubConfig.DEFAULT_MODULES_DB_NAME, null);
62-
assertTrue(totalCount + " is not correct", 81 == totalCount || 61 == totalCount);
62+
assertTrue(totalCount + " is not correct", 84 == totalCount || 64 == totalCount);
6363

6464
dataHub.installUserModules(true);
6565

6666
totalCount = getDocCount(HubConfig.DEFAULT_MODULES_DB_NAME, null);
67-
assertTrue(totalCount + " is not correct", 81 == totalCount || 101 == totalCount);
67+
assertTrue(totalCount + " is not correct", 84 == totalCount || 104 == totalCount);
6868

6969
assertEquals(
7070
getResource("data-hub-test/plugins/entities/test-entity/harmonize/final/collector.xqy"),
@@ -186,17 +186,17 @@ public void testClearUserModules() throws URISyntaxException {
186186
dataHub.clearUserModules();
187187

188188
int totalCount = getDocCount(HubConfig.DEFAULT_MODULES_DB_NAME, null);
189-
assertTrue(totalCount + " is not correct", 81 == totalCount || 61 == totalCount);
189+
assertTrue(totalCount + " is not correct", 84 == totalCount || 64 == totalCount);
190190

191191
dataHub.installUserModules(true);
192192

193193
totalCount = getDocCount(HubConfig.DEFAULT_MODULES_DB_NAME, null);
194-
assertTrue(totalCount + " is not correct", 81 == totalCount || 101 == totalCount);
194+
assertTrue(totalCount + " is not correct", 84 == totalCount || 104 == totalCount);
195195

196196
dataHub.clearUserModules();
197197

198198
totalCount = getDocCount(HubConfig.DEFAULT_MODULES_DB_NAME, null);
199-
assertTrue(totalCount + " is not correct", 81 == totalCount || 61 == totalCount);
199+
assertTrue(totalCount + " is not correct", 84 == totalCount || 64 == totalCount);
200200

201201
}
202202
}

0 commit comments

Comments
 (0)