Skip to content

Commit f6c97ba

Browse files
rjrudinrjrudin
committed
#527 Can now manage partitions and partition queries
1 parent 9b91e6c commit f6c97ba

File tree

14 files changed

+211
-2
lines changed

14 files changed

+211
-2
lines changed

‎examples/partition-project/.gitignore‎

Lines changed: 6 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,6 @@
1+
/bin
2+
.classpath
3+
.project
4+
/build
5+
.gradle
6+
.settings

‎examples/partition-project/README.md‎

Lines changed: 38 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,38 @@
1+
This project shows a simple example of deploying partitions. As noted in the docs for
2+
[configuring query partitions](https://docs.marklogic.com/guide/admin/tiered-storage#id_31779), this example has the
3+
following configured on the content database:
4+
5+
- The assignment policy is set to "range" with a partition key of "myDate"
6+
- A range index on "myDate"
7+
- Locking is set to "strict"
8+
- And a schemas database is associated with the content database as well (GET calls to the Manage API for partitions fail otherwise)
9+
10+
Alternatively, partition queries can be deployed by using an assignment policy of "query" and adding files to
11+
./src/main/ml-config/databases/partition-example-content/partition-queries based on the
12+
[partition query docs](http://docs.marklogic.com/REST/POST/manage/v2/databases/[id-or-name]/partition-queries).
13+
14+
To try this example out, first deploy the application:
15+
16+
gradle mlDeploy
17+
18+
This will result in 3 forests - one "default" forest with no partition assigned, and then two forests based on
19+
partitions - "myDate-2011" and "myDate-2012". Note also that the "mlAddHostNameTokens" property is set so that the
20+
partition files can refer to a host name dynamically via the "mlHostName1" token as opposed to being hardcoded to a
21+
specific host name. See these [ml-app-deployer docs](https://github.com/marklogic-community/ml-app-deployer/wiki/Scheduled-Tasks#referring-to-host-names-in-scheduled-task-files) for more information.
22+
23+
You can then insert documents via qconsole (or any other ML interface) to test out the range assignments - e.g.
24+
25+
```
26+
declareUpdate();
27+
xdmp.documentInsert("test1.json", {"myDate":"2011-01-01"});
28+
xdmp.documentInsert("test2.json", {"myDate":"2012-01-01"})
29+
```
30+
31+
You can then use the ML Admin GUI to verify that both myDate-* forests have 1 document each.
32+
33+
Additionally, ml-gradle has tasks for taking partitions online and offline - e.g.
34+
35+
```
36+
gradle mlTakePartitionOffline -Pdatabase=partition-example-content -Ppartition=myDate-2011
37+
gradle mlTakePartitionOnline -Pdatabase=partition-example-content -Ppartition=myDate-2011
38+
```

‎examples/partition-project/build.gradle‎

Lines changed: 17 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,17 @@
1+
buildscript {
2+
repositories {
3+
mavenLocal()
4+
jcenter()
5+
}
6+
dependencies {
7+
classpath "com.marklogic:ml-gradle:${mlGradleVersion}"
8+
}
9+
}
10+
11+
apply plugin: "java"
12+
apply plugin: "com.marklogic.ml-gradle"
13+
14+
repositories {
15+
mavenLocal()
16+
jcenter()
17+
}

‎examples/partition-project/gradle.properties‎

Lines changed: 9 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,9 @@
1+
mlGradleVersion=3.17.0-develop
2+
3+
mlHost=localhost
4+
mlAppName=partition-example
5+
mlRestPort=8018
6+
mlUsername=admin
7+
mlPassword=admin
8+
mlContentForestsPerHost=1
9+
mlAddHostNameTokens=true

‎examples/partition-project/src/main/ml-config/databases/content-database.json‎

Lines changed: 26 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,26 @@
1+
{
2+
"database-name": "%%DATABASE%%",
3+
"schema-database": "%%SCHEMAS_DATABASE%%",
4+
"locking": "strict",
5+
"range-element-index": [
6+
{
7+
"scalar-type": "date",
8+
"namespace-uri": "",
9+
"localname": "myDate",
10+
"collation": "",
11+
"range-value-positions": false,
12+
"invalid-values": "reject"
13+
}
14+
],
15+
"assignment-policy": {
16+
"assignment-policy-name": "range",
17+
"partition-key": {
18+
"element-reference": {
19+
"namespace-uri": "",
20+
"localname": "myDate",
21+
"scalar-type": "date"
22+
}
23+
},
24+
"lower-bound-included": true
25+
}
26+
}

‎examples/partition-project/src/main/ml-config/databases/partition-example-content/partitions/partition1.xml‎

Lines changed: 17 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,17 @@
1+
<partition xmlns="http://marklogic.com/manage">
2+
<partition-name>myDate-2011</partition-name>
3+
<lower-bound>2011-01-01</lower-bound>
4+
<upper-bound>2011-12-31</upper-bound>
5+
<forests-per-host>1</forests-per-host>
6+
<hosts>
7+
<host>mlHostName1</host>
8+
</hosts>
9+
<!--
10+
<data-directory></data-directory>
11+
<large-data-directory></large-data-directory>
12+
<fast-data-directory></fast-data-directory>
13+
<options>
14+
<option>failover=none</option>
15+
</options>
16+
-->
17+
</partition>

‎examples/partition-project/src/main/ml-config/databases/partition-example-content/partitions/partition2.json‎

Lines changed: 12 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,12 @@
1+
{
2+
"partition-name": "myDate-2012",
3+
"lower-bound": "2012-01-01",
4+
"upper-bound": "2012-12-31",
5+
"forests-per-host": 1,
6+
"host": [
7+
"mlHostName1"
8+
],
9+
"option": [
10+
"failover=none"
11+
]
12+
}

‎examples/partition-project/src/main/ml-config/databases/schemas-database.json‎

Lines changed: 3 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,3 @@
1+
{
2+
"database-name": "%%SCHEMAS_DATABASE%%"
3+
}

‎gradle.properties‎

Lines changed: 2 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -1,8 +1,8 @@
11
group=com.marklogic
2-
version=3.16.4
2+
version=3.17.0-develop
33
javadocsDir=../gh-pages-marklogic-java/javadocs
44

5-
mlAppDeployerDependency=com.marklogic:ml-app-deployer:3.16.3
5+
mlAppDeployerDependency=com.marklogic:ml-app-deployer:3.17.0-develop
66
mlcpUtilDependency=com.marklogic:mlcp-util:0.9.0
77
mlDataMovementDependency=com.marklogic:marklogic-data-movement-components:1.2.0
88
mlUnitTestDependency=com.marklogic:marklogic-unit-test-client:1.0.beta

‎src/main/groovy/com/marklogic/gradle/MarkLogicPlugin.groovy‎

Lines changed: 12 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -38,6 +38,10 @@ import com.marklogic.gradle.task.plugins.InstallPluginsTask
3838
import com.marklogic.gradle.task.plugins.UninstallPluginsTask
3939
import com.marklogic.gradle.task.qconsole.ExportWorkspacesTask
4040
import com.marklogic.gradle.task.qconsole.ImportWorkspacesTask
41+
import com.marklogic.gradle.task.rebalancer.DeployPartitionQueriesTask
42+
import com.marklogic.gradle.task.rebalancer.DeployPartitionsTask
43+
import com.marklogic.gradle.task.rebalancer.TakePartitionOfflineTask
44+
import com.marklogic.gradle.task.rebalancer.TakePartitionOnlineTask
4145
import com.marklogic.gradle.task.restapis.DeployRestApisTask
4246
import com.marklogic.gradle.task.roxy.RoxyMigrateBuildStepsTask
4347
import com.marklogic.gradle.task.roxy.RoxyMigrateFilesTask
@@ -240,6 +244,14 @@ class MarkLogicPlugin implements Plugin<Project> {
240244
project.task("mlImportWorkspaces", type: ImportWorkspacesTask, group: qconsoleGroup, description: "Import workspaces into qconsole")
241245
project.task("mlExportWorkspaces", type: ExportWorkspacesTask, group: qconsoleGroup, description: "Export workspaces from qconsole")
242246

247+
String rebalancerGroup = "ml-gradle Rebalancer"
248+
project.task("mlDeployPartitions", type: DeployPartitionsTask, group: rebalancerGroup, description: "Deploy database-specific partitions")
249+
project.task("mlDeployPartitionQueries", type: DeployPartitionQueriesTask, group: rebalancerGroup, description: "Deploy database-specific partition queries")
250+
project.task("mlTakePartitionOffline", type: TakePartitionOfflineTask, group: rebalancerGroup,
251+
description: "Take a partition offline. Use -Pdatabase=dbName and -Ppartition=partitionName to specify the database and partition names.")
252+
project.task("mlTakePartitionOnline", type: TakePartitionOnlineTask, group: rebalancerGroup,
253+
description: "Take a partition online. Use -Pdatabase=dbName and -Ppartition=partitionName to specify the database and partition names.")
254+
243255
String restApisGroup = "ml-gradle REST API"
244256
project.task("mlDeployRestApis", type: DeployRestApisTask, group: restApisGroup, description: "Deploy the REST API instances defined by a resource file or the mlRestPort/mlTestRestPort properties")
245257

0 commit comments

Comments
 (0)