Skip to content

Commit 32c6a17

Browse files
rjrudinrjrudin
committed
#227 Can now modify all aspects of QueryBatcherTemplate
Also using "where" instead of "source" for properties
1 parent 54f0076 commit 32c6a17

File tree

10 files changed

+137
-79
lines changed

10 files changed

+137
-79
lines changed

‎build.gradle‎

Lines changed: 3 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -72,11 +72,11 @@ pluginBundle {
7272
displayName = 'ml-gradle for MarkLogic'
7373
description = 'Gradle plugin for configuring and deploying applications to MarkLogic'
7474
tags = ['marklogic']
75-
version = "2.8.0"
75+
version = project.version
7676
}
7777
}
7878

7979
mavenCoordinates {
80-
version = "2.8.0"
80+
version = project.version
8181
}
82-
}
82+
}

‎gradle.properties‎

Lines changed: 2 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -1,5 +1,5 @@
11
group=com.marklogic
2-
version=3.0-alpha1
3-
mlAppDeployerDependency=com.marklogic:ml-app-deployer:3.0-alpha1
2+
version=3.0-alpha3
3+
mlAppDeployerDependency=com.marklogic:ml-app-deployer:3.0-alpha3
44
mlcpUtilDependency=com.marklogic:mlcp-util:0.3.0
55

‎src/main/groovy/com/marklogic/gradle/MarkLogicPlugin.groovy‎

Lines changed: 12 additions & 11 deletions
Original file line numberDiff line numberDiff line change
@@ -128,26 +128,27 @@ class MarkLogicPlugin implements Plugin<Project> {
128128
project.task("mlSetContentUpdatesAllowed", type: SetContentUpdatesAllowedTask, group: dbGroup, description: "Sets updated-allowed on each primary forest for the content database; must set the mode via e.g. -Pmode=flash-backup")
129129

130130
String dmGroup = "ml-Gradle Data Movement"
131-
131+
String dmGroupMessage = "; can also set the properties threadCount, batchSize, applyConsistentSnapshot, awaitCompletion, stopJob, " +
132+
"jobName, and logBatches to configure how the Data Movement QueryBatcher operates.";
132133
project.task("mlAddCollections", type: AddCollectionsTask, group: dmGroup, description: "Add all documents, either in a comma-separated list of " +
133-
"collection names specified by the 'sourceCollections' property or matching a URI pattern specified by the 'uriPattern' property, " +
134-
"to a comma-separated list of collection names specified by the 'collections' property")
134+
"collection names specified by the 'whereCollections' property or matching a URI pattern specified by the 'whereUriPattern' property, " +
135+
"to a comma-separated list of collection names specified by the 'collections' property" + dmGroupMessage)
135136
project.task("mlDeleteCollections", type: DeleteCollectionsTask, group: dmGroup, description: "Delete all documents in a comma-separated list of " +
136-
"collection names specified by the 'collections' property")
137+
"collection names specified by the 'collections' property" + dmGroupMessage)
137138
project.task("mlRemoveCollections", type: RemoveCollectionsTask, group: dmGroup, description: "Remove all documents, either in a comma-separated list of " +
138-
"collection names specified by the 'sourceCollections' property or matching a URI pattern specified by the 'uriPattern' property, " +
139+
"collection names specified by the 'whereCollections' property or matching a URI pattern specified by the 'whereUriPattern' property, " +
139140
"from a comma-separated list of collection names specified by the 'collections' property; " +
140-
"if the values of 'sourceCollections' and 'collections' are the same, you only need to specify the 'collections' property")
141+
"if the values of 'whereCollections' and 'collections' are the same, you only need to specify the 'collections' property" + dmGroupMessage)
141142
project.task("mlSetCollections", type: SetCollectionsTask, group: dmGroup, description: "Set collections on all documents, either in a comma-separated list of " +
142-
"collection names specified by the 'sourceCollections' property or matching a URI pattern specified by the 'uriPattern' property, " +
143-
"to a comma-separated list of collection names specified by the 'collections' property")
143+
"collection names specified by the 'whereCollections' property or matching a URI pattern specified by the 'whereUriPattern' property, " +
144+
"to a comma-separated list of collection names specified by the 'collections' property" + dmGroupMessage)
144145

145146
project.task("mlAddPermissions", type: AddPermissionsTask, group: dmGroup, description: "Add permissions, specified as a comma-separated list of roles and capabilities via the 'permissions' property, " +
146-
"to all documents either in the set of collection names specified by the 'collections' property or with URIs matching the pattern specified by the 'uriPattern' property")
147+
"to all documents either in the set of collection names specified by the 'collections' property or with URIs matching the pattern specified by the 'whereUriPattern' property" + dmGroupMessage)
147148
project.task("mlRemovePermissions", type: RemovePermissionsTask, group: dmGroup, description: "Remove permissions, specified as a comma-separated list of roles and capabilities via the 'permissions' property, " +
148-
"from all documents either in the set of collection names specified by the 'collections' property or with URIs matching the pattern specified by the 'uriPattern' property")
149+
"from all documents either in the set of collection names specified by the 'collections' property or with URIs matching the pattern specified by the 'whereUriPattern' property" + dmGroupMessage)
149150
project.task("mlSetPermissions", type: AddPermissionsTask, group: dmGroup, description: "Set permissions, specified as a comma-separated list of roles and capabilities via the 'permissions' property, " +
150-
"on all documents in the set of collection names specified by the 'collections' property or with URIs matching the pattern specified by the 'uriPattern' property")
151+
"on all documents in the set of collection names specified by the 'collections' property or with URIs matching the pattern specified by the 'whereUriPattern' property" + dmGroupMessage)
151152

152153
String devGroup = "ml-gradle Development"
153154
project.task("mlCreateResource", type: CreateResourceTask, group: devGroup, description: "Create a new resource extension in the modules services directory; use -PresourceName and -PresourceType to set the resource name and type (either xqy or sjs)")

‎src/main/groovy/com/marklogic/gradle/task/datamovement/AddCollectionsTask.groovy‎

Lines changed: 8 additions & 11 deletions
Original file line numberDiff line numberDiff line change
@@ -1,9 +1,7 @@
11
package com.marklogic.gradle.task.datamovement
22

33
import com.marklogic.client.datamovement.QueryBatchListener
4-
import com.marklogic.client.ext.datamovement.CollectionsQueryBatcherBuilder
54
import com.marklogic.client.ext.datamovement.QueryBatcherBuilder
6-
import com.marklogic.client.ext.datamovement.UriPatternQueryBatcherBuilder
75
import com.marklogic.client.ext.datamovement.listener.AddCollectionsListener
86
import org.gradle.api.tasks.TaskAction
97

@@ -12,7 +10,7 @@ class AddCollectionsTask extends DataMovementTask {
1210
@TaskAction
1311
void addCollections() {
1412
if (
15-
(!project.hasProperty("sourceCollections") && !project.hasProperty("pattern")) ||
13+
(!project.hasProperty("whereCollections") && !project.hasProperty("whereUriPattern")) ||
1614
!project.hasProperty("collections")
1715
) {
1816
println "Invalid inputs; " + getDescription()
@@ -24,14 +22,13 @@ class AddCollectionsTask extends DataMovementTask {
2422
QueryBatcherBuilder builder = null
2523

2624
String message = " to collections " + Arrays.asList(collections);
27-
if (project.hasProperty("collections")) {
28-
String[] sourceCollections = getProject().property("sourceCollections").split(",")
29-
message = "documents in collections " + Arrays.asList(sourceCollections) + message
30-
builder = new CollectionsQueryBatcherBuilder(sourceCollections)
31-
} else if (project.hasProperty("uriPattern")) {
32-
String pattern = getProject().property("pattern")
33-
message = "documents matching URI pattern " + pattern + message
34-
builder = new UriPatternQueryBatcherBuilder(pattern)
25+
26+
if (hasWhereCollectionsProperty()) {
27+
builder = constructBuilderFromWhereCollections()
28+
message = "documents in collections " + Arrays.asList(this.whereCollections) + message
29+
} else if (hasWhereUriPatternProperty()) {
30+
builder = constructBuilderFromWhereUriPattern()
31+
message = "documents matching URI pattern " + this.whereUriPattern + message
3532
}
3633

3734
println "Adding " + message

‎src/main/groovy/com/marklogic/gradle/task/datamovement/AddPermissionsTask.groovy‎

Lines changed: 8 additions & 11 deletions
Original file line numberDiff line numberDiff line change
@@ -1,17 +1,15 @@
11
package com.marklogic.gradle.task.datamovement
22

33
import com.marklogic.client.datamovement.QueryBatchListener
4-
import com.marklogic.client.ext.datamovement.CollectionsQueryBatcherBuilder
54
import com.marklogic.client.ext.datamovement.QueryBatcherBuilder
6-
import com.marklogic.client.ext.datamovement.UriPatternQueryBatcherBuilder
75
import com.marklogic.client.ext.datamovement.listener.AddPermissionsListener
86
import org.gradle.api.tasks.TaskAction
97

108
class AddPermissionsTask extends DataMovementTask {
119

1210
@TaskAction
1311
void addPermissions() {
14-
if ((!project.hasProperty("collections") && !project.hasProperty("uriPattern")) || !project.hasProperty("permissions")) {
12+
if ((!project.hasProperty("whereCollections") && !project.hasProperty("whereUriPattern")) || !project.hasProperty("permissions")) {
1513
println "Invalid input; " + getDescription()
1614
return;
1715
}
@@ -21,14 +19,13 @@ class AddPermissionsTask extends DataMovementTask {
2119
QueryBatcherBuilder builder = null
2220

2321
String message = "permissions " + Arrays.asList(permissions) + " to documents "
24-
if (project.hasProperty("collections")) {
25-
String[] collections = getProject().property("collections").split(",")
26-
message += "in collections " + Arrays.asList(collections)
27-
builder = new CollectionsQueryBatcherBuilder(collections)
28-
} else if (project.hasProperty("uriPattern")) {
29-
String pattern = getProject().property("uriPattern")
30-
message += "matching URI pattern " + pattern
31-
builder = new UriPatternQueryBatcherBuilder(pattern)
22+
23+
if (hasWhereCollectionsProperty()) {
24+
builder = constructBuilderFromWhereCollections()
25+
message += "in collections " + Arrays.asList(this.whereCollections)
26+
} else if (hasWhereUriPatternProperty()) {
27+
builder = constructBuilderFromWhereUriPattern()
28+
message += "matching URI pattern " + this.whereUriPattern
3229
}
3330

3431
println "Adding " + message

‎src/main/groovy/com/marklogic/gradle/task/datamovement/DataMovementTask.groovy‎

Lines changed: 75 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -1,17 +1,41 @@
11
package com.marklogic.gradle.task.datamovement
22

33
import com.marklogic.client.DatabaseClient
4+
import com.marklogic.client.datamovement.QueryBatch
45
import com.marklogic.client.datamovement.QueryBatchListener
6+
import com.marklogic.client.ext.datamovement.CollectionsQueryBatcherBuilder
57
import com.marklogic.client.ext.datamovement.QueryBatcherBuilder
68
import com.marklogic.client.ext.datamovement.QueryBatcherTemplate
9+
import com.marklogic.client.ext.datamovement.UriPatternQueryBatcherBuilder
710
import com.marklogic.gradle.task.MarkLogicTask
811

912
class DataMovementTask extends MarkLogicTask {
1013

14+
String whereUriPattern
15+
String[] whereCollections
16+
17+
boolean hasWhereCollectionsProperty() {
18+
return project.hasProperty("whereCollections")
19+
}
20+
21+
boolean hasWhereUriPatternProperty() {
22+
return project.hasProperty("whereUriPattern")
23+
}
24+
25+
QueryBatcherBuilder constructBuilderFromWhereCollections() {
26+
this.whereCollections = getProject().property("whereCollections").split(",")
27+
return new CollectionsQueryBatcherBuilder(this.whereCollections)
28+
}
29+
30+
QueryBatcherBuilder constructBuilderFromWhereUriPattern() {
31+
this.whereUriPattern = getProject().property("whereUriPattern")
32+
return new UriPatternQueryBatcherBuilder(this.whereUriPattern)
33+
}
34+
1135
void applyOnCollections(QueryBatchListener listener, String... collections) {
1236
DatabaseClient client = newClient()
1337
try {
14-
new QueryBatcherTemplate(client).applyOnCollections(listener, collections);
38+
newQueryBatcherTemplate(client).applyOnCollections(listener, collections);
1539
} finally {
1640
client.release()
1741
}
@@ -20,7 +44,7 @@ class DataMovementTask extends MarkLogicTask {
2044
void applyOnUriPattern(QueryBatchListener listener, String uriPattern) {
2145
DatabaseClient client = newClient()
2246
try {
23-
new QueryBatcherTemplate(client).applyOnUriPattern(listener, uriPattern);
47+
newQueryBatcherTemplate(client).applyOnUriPattern(listener, uriPattern);
2448
} finally {
2549
client.release()
2650
}
@@ -29,9 +53,57 @@ class DataMovementTask extends MarkLogicTask {
2953
void applyWithQueryBatcherBuilder(QueryBatchListener listener, QueryBatcherBuilder builder) {
3054
DatabaseClient client = newClient()
3155
try {
32-
new QueryBatcherTemplate(client).apply(listener, builder)
56+
newQueryBatcherTemplate(client).apply(listener, builder)
3357
} finally {
3458
client.release()
3559
}
3660
}
61+
62+
/**
63+
* Supports the following properties:
64+
* - threadCount
65+
* - batchSize
66+
* - applyConsistentSnapshot
67+
* - awaitCompletion
68+
* - stopJob
69+
* - jobName
70+
*
71+
* Can override this method in a subclass to further configure the QueryBatcherTemplate that's returned.
72+
*
73+
* @param client
74+
* @return
75+
*/
76+
QueryBatcherTemplate newQueryBatcherTemplate(DatabaseClient client) {
77+
QueryBatcherTemplate t = new QueryBatcherTemplate(client)
78+
if (project.hasProperty("threadCount")) {
79+
t.setThreadCount(Integer.parseInt(project.property("threadCount")))
80+
}
81+
if (project.hasProperty("batchSize")) {
82+
t.setBatchSize(Integer.parseInt(project.property("batchSize")))
83+
}
84+
if (project.hasProperty("applyConsistentSnapshot")) {
85+
t.setApplyConsistentSnapshot(Boolean.parseBoolean(project.property("applyConsistentSnapshot")))
86+
}
87+
if (project.hasProperty("awaitCompletion")) {
88+
t.setAwaitCompletion(Boolean.parseBoolean(project.property("awaitCompletion")))
89+
}
90+
if (project.hasProperty("stopJob")) {
91+
t.setStopJob(Boolean.parseBoolean(project.property("stopJob")))
92+
}
93+
if (project.hasProperty("jobName")) {
94+
t.setJobName(project.property("jobName"))
95+
}
96+
97+
if (project.hasProperty("logBatches")) {
98+
t.addUrisReadyListeners(new QueryBatchListener() {
99+
@Override
100+
void processEvent(QueryBatch batch) {
101+
println String.format("Processed batch number [%d]; job results so far: [%d]", batch.getJobBatchNumber(),
102+
batch.getJobResultsSoFar())
103+
}
104+
})
105+
}
106+
107+
return t
108+
}
37109
}

‎src/main/groovy/com/marklogic/gradle/task/datamovement/RemoveCollectionsTask.groovy‎

Lines changed: 7 additions & 11 deletions
Original file line numberDiff line numberDiff line change
@@ -1,9 +1,7 @@
11
package com.marklogic.gradle.task.datamovement
22

33
import com.marklogic.client.datamovement.QueryBatchListener
4-
import com.marklogic.client.ext.datamovement.CollectionsQueryBatcherBuilder
54
import com.marklogic.client.ext.datamovement.QueryBatcherBuilder
6-
import com.marklogic.client.ext.datamovement.UriPatternQueryBatcherBuilder
75
import com.marklogic.client.ext.datamovement.listener.RemoveCollectionsListener
86
import org.gradle.api.tasks.TaskAction
97

@@ -26,18 +24,16 @@ class RemoveCollectionsTask extends DataMovementTask {
2624
QueryBatchListener listener = new RemoveCollectionsListener(collections)
2725
QueryBatcherBuilder builder = null
2826

29-
if (project.hasProperty("uriPattern")) {
30-
String pattern = project.property("uriPattern")
31-
message = "documents matching URI pattern " + pattern + message
32-
builder = new UriPatternQueryBatcherBuilder(pattern)
27+
if (hasWhereUriPatternProperty()) {
28+
builder = constructBuilderFromWhereUriPattern()
29+
message = "documents matching URI pattern " + this.whereUriPattern + message
3330
}
3431
else {
35-
String[] sourceCollections = collections
36-
if (project.hasProperty("sourceCollections")) {
37-
sourceCollections = getProject().property("sourceCollections").split(",")
32+
this.whereCollections = collections
33+
if (hasWhereCollectionsProperty()) {
34+
builder = constructBuilderFromWhereCollections()
3835
}
39-
message = "documents in collections " + Arrays.asList(sourceCollections) + message
40-
builder = new CollectionsQueryBatcherBuilder(sourceCollections)
36+
message = "documents in collections " + Arrays.asList(this.whereCollections) + message
4137
}
4238

4339
println "Removing " + message

‎src/main/groovy/com/marklogic/gradle/task/datamovement/RemovePermissionsTask.groovy‎

Lines changed: 7 additions & 8 deletions
Original file line numberDiff line numberDiff line change
@@ -21,14 +21,13 @@ class RemovePermissionsTask extends DataMovementTask {
2121
QueryBatcherBuilder builder = null
2222

2323
String message = "permissions " + Arrays.asList(permissions) + " from documents "
24-
if (project.hasProperty("collections")) {
25-
String[] collections = getProject().property("collections").split(",")
26-
message += "in collections " + Arrays.asList(collections)
27-
builder = new CollectionsQueryBatcherBuilder(collections)
28-
} else if (project.hasProperty("uriPattern")) {
29-
String pattern = getProject().property("uriPattern")
30-
message += "matching URI pattern " + pattern
31-
builder = new UriPatternQueryBatcherBuilder(pattern)
24+
25+
if (hasWhereCollectionsProperty()) {
26+
builder = constructBuilderFromWhereCollections()
27+
message += "in collections " + Arrays.asList(this.whereCollections)
28+
} else if (hasWhereUriPatternProperty()) {
29+
builder = constructBuilderFromWhereUriPattern()
30+
message += "matching URI pattern " + this.whereUriPattern
3231
}
3332

3433
println "Removing " + message

‎src/main/groovy/com/marklogic/gradle/task/datamovement/SetCollectionsTask.groovy‎

Lines changed: 8 additions & 11 deletions
Original file line numberDiff line numberDiff line change
@@ -1,9 +1,7 @@
11
package com.marklogic.gradle.task.datamovement
22

33
import com.marklogic.client.datamovement.QueryBatchListener
4-
import com.marklogic.client.ext.datamovement.CollectionsQueryBatcherBuilder
54
import com.marklogic.client.ext.datamovement.QueryBatcherBuilder
6-
import com.marklogic.client.ext.datamovement.UriPatternQueryBatcherBuilder
75
import com.marklogic.client.ext.datamovement.listener.SetCollectionsListener
86
import org.gradle.api.tasks.TaskAction
97

@@ -12,7 +10,7 @@ class SetCollectionsTask extends DataMovementTask {
1210
@TaskAction
1311
void setCollections() {
1412
if (
15-
(!project.hasProperty("sourceCollections") && !project.hasProperty("pattern")) ||
13+
(!project.hasProperty("whereCollections") && !project.hasProperty("pattern")) ||
1614
!project.hasProperty("collections")
1715
) {
1816
println "Invalid inputs; " + getDescription()
@@ -24,14 +22,13 @@ class SetCollectionsTask extends DataMovementTask {
2422
QueryBatcherBuilder builder = null
2523

2624
String message = " collections " + Arrays.asList(collections);
27-
if (project.hasProperty("collections")) {
28-
String[] sourceCollections = getProject().property("sourceCollections").split(",")
29-
message += " on documents in collections " + Arrays.asList(sourceCollections)
30-
builder = new CollectionsQueryBatcherBuilder(sourceCollections)
31-
} else if (project.hasProperty("uriPattern")) {
32-
String pattern = getProject().property("pattern")
33-
message += " on documents matching URI pattern " + pattern
34-
builder = new UriPatternQueryBatcherBuilder(pattern)
25+
26+
if (hasWhereCollectionsProperty()) {
27+
builder = constructBuilderFromWhereCollections()
28+
message += " on documents in collections " + Arrays.asList(this.whereCollections)
29+
} else if (hasWhereUriPatternProperty()) {
30+
builder = constructBuilderFromWhereUriPattern()
31+
message += " on documents matching URI pattern " + this.whereUriPattern
3532
}
3633

3734
println "Setting " + message

‎src/main/groovy/com/marklogic/gradle/task/datamovement/SetPermissionsTask.groovy‎

Lines changed: 7 additions & 8 deletions
Original file line numberDiff line numberDiff line change
@@ -21,14 +21,13 @@ class SetPermissionsTask extends DataMovementTask {
2121
QueryBatcherBuilder builder = null
2222

2323
String message = "permissions " + Arrays.asList(permissions) + " on documents "
24-
if (project.hasProperty("collections")) {
25-
String[] collections = getProject().property("collections").split(",")
26-
message += "in collections " + Arrays.asList(collections)
27-
builder = new CollectionsQueryBatcherBuilder(collections)
28-
} else if (project.hasProperty("uriPattern")) {
29-
String pattern = getProject().property("uriPattern")
30-
message += "matching URI pattern " + pattern
31-
builder = new UriPatternQueryBatcherBuilder(pattern)
24+
25+
if (hasWhereCollectionsProperty()) {
26+
builder = constructBuilderFromWhereCollections()
27+
message += "in collections " + Arrays.asList(this.whereCollections)
28+
} else if (hasWhereUriPatternProperty()) {
29+
builder = constructBuilderFromWhereUriPattern()
30+
message += "matching URI pattern " + this.whereUriPattern
3231
}
3332

3433
println "Setting " + message

0 commit comments

Comments
 (0)