Skip to content
This repository was archived by the owner on Sep 16, 2024. It is now read-only.

Commit d546775

Browse files
rjrudinrjrudin
committed
#424 Moved replica data directory construction to ForestBuilder
Also added an escape hatch in AppConfig so that a custom ReplicaBuilderStrategy can be provided. Made ReplicaBuilderStrategy into an interface as well.
1 parent 7f10d41 commit d546775

File tree

10 files changed

+201
-111
lines changed

10 files changed

+201
-111
lines changed

‎src/main/java/com/marklogic/appdeployer/AppConfig.java‎

Lines changed: 11 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -1,6 +1,7 @@
11
package com.marklogic.appdeployer;
22

33
import com.marklogic.appdeployer.command.forests.ForestNamingStrategy;
4+
import com.marklogic.appdeployer.command.forests.ReplicaBuilderStrategy;
45
import com.marklogic.client.DatabaseClient;
56
import com.marklogic.client.DatabaseClientFactory;
67
import com.marklogic.client.DatabaseClientFactory.SSLHostnameVerifier;
@@ -184,8 +185,12 @@ public class AppConfig {
184185
private String replicaForestLargeDataDirectory;
185186
private String replicaForestFastDataDirectory;
186187

188+
// Allows for customizing how forests are named per database
187189
private Map<String, ForestNamingStrategy> forestNamingStrategies = new HashMap<>();
188190

191+
// Allows for customizing how replicas are built for all databases
192+
private ReplicaBuilderStrategy replicaBuilderStrategy;
193+
189194
// Path to use for DeployFlexrepCommand
190195
private String flexrepPath;
191196

@@ -1274,7 +1279,11 @@ public void setDeployPrivilegesWithCma(boolean deployPrivilegesWithCma) {
12741279
this.deployPrivilegesWithCma = deployPrivilegesWithCma;
12751280
}
12761281

1277-
public File getProjectDir() {
1278-
return projectDir;
1282+
public ReplicaBuilderStrategy getReplicaBuilderStrategy() {
1283+
return replicaBuilderStrategy;
1284+
}
1285+
1286+
public void setReplicaBuilderStrategy(ReplicaBuilderStrategy replicaBuilderStrategy) {
1287+
this.replicaBuilderStrategy = replicaBuilderStrategy;
12791288
}
12801289
}

‎src/main/java/com/marklogic/appdeployer/command/forests/AbstractReplicaBuilderStrategy.java‎

Lines changed: 57 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,57 @@
1+
package com.marklogic.appdeployer.command.forests;
2+
3+
import com.marklogic.appdeployer.AppConfig;
4+
import com.marklogic.mgmt.api.forest.ForestReplica;
5+
6+
import java.util.Map;
7+
8+
public abstract class AbstractReplicaBuilderStrategy implements ReplicaBuilderStrategy {
9+
10+
/**
11+
* Configures the fast and large data directories for a replica based on what's in AppConfig for the given
12+
* database.
13+
*
14+
* @param replica
15+
* @param databaseName
16+
* @param appConfig
17+
*/
18+
protected void configureReplica(ForestReplica replica, String databaseName, AppConfig appConfig) {
19+
// First set to the database-agnostic forest directories
20+
replica.setFastDataDirectory(appConfig.getForestFastDataDirectory());
21+
replica.setLargeDataDirectory(appConfig.getForestLargeDataDirectory());
22+
23+
// Now set to the database-specific forest directories if set
24+
if (databaseName != null) {
25+
Map<String, String> map = appConfig.getDatabaseFastDataDirectories();
26+
if (map != null && map.containsKey(databaseName)) {
27+
replica.setFastDataDirectory(map.get(databaseName));
28+
}
29+
map = appConfig.getDatabaseLargeDataDirectories();
30+
if (map != null && map.containsKey(databaseName)) {
31+
replica.setLargeDataDirectory(map.get(databaseName));
32+
}
33+
}
34+
35+
// Now set to the replica forest directories if set
36+
if (appConfig.getReplicaForestFastDataDirectory() != null) {
37+
replica.setFastDataDirectory(appConfig.getReplicaForestFastDataDirectory());
38+
}
39+
if (appConfig.getReplicaForestLargeDataDirectory() != null) {
40+
replica.setLargeDataDirectory(appConfig.getReplicaForestLargeDataDirectory());
41+
}
42+
43+
// And now set to the database-specific replica forest directories if set
44+
if (databaseName != null) {
45+
Map<String, String> map = appConfig.getDatabaseReplicaFastDataDirectories();
46+
if (map != null && map.containsKey(databaseName)) {
47+
replica.setFastDataDirectory(map.get(databaseName));
48+
}
49+
map = appConfig.getDatabaseReplicaLargeDataDirectories();
50+
if (map != null && map.containsKey(databaseName)) {
51+
replica.setLargeDataDirectory(map.get(databaseName));
52+
}
53+
}
54+
}
55+
56+
57+
}

‎src/main/java/com/marklogic/appdeployer/command/forests/ConfigureForestReplicasCommand.java‎

Lines changed: 2 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -134,7 +134,8 @@ protected void configureDatabaseReplicaForests(String databaseName, int replicaC
134134

135135
ForestBuilder forestBuilder = new ForestBuilder();
136136
ForestPlan forestPlan = new ForestPlan(databaseName, selectedHostNames).withReplicaCount(replicaCount);
137-
forestBuilder.addReplicasToForests(Arrays.asList(forest), forestPlan, context.getAppConfig());
137+
List<String> dataDirectories = forestBuilder.determineDataDirectories(databaseName, context.getAppConfig());
138+
forestBuilder.addReplicasToForests(Arrays.asList(forest), forestPlan, context.getAppConfig(), dataDirectories);
138139

139140
Forest forestWithOnlyReplicas = new Forest();
140141
forestWithOnlyReplicas.setForestReplica(forest.getForestReplica());

‎src/main/java/com/marklogic/appdeployer/command/forests/DeployForestsCommand.java‎

Lines changed: 7 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -44,6 +44,8 @@ public class DeployForestsCommand extends AbstractCommand {
4444
private boolean createForestsOnEachHost = true;
4545
private HostCalculator hostCalculator;
4646

47+
private ForestBuilder forestBuilder = new ForestBuilder();
48+
4749
/**
4850
* This was added back in 3.8.2 to preserve backwards compatibility, as it was removed in 3.7.0. If you use this
4951
* constructor, be sure to call setDatabaseName.
@@ -128,7 +130,7 @@ public List<Forest> buildForests(CommandContext context, boolean includeReplicas
128130
}
129131
}
130132

131-
return new ForestBuilder().buildForests(forestPlan, context.getAppConfig());
133+
return forestBuilder.buildForests(forestPlan, context.getAppConfig());
132134
}
133135

134136
protected String buildForestTemplate(CommandContext context, ForestManager forestManager) {
@@ -231,4 +233,8 @@ public void setHostCalculator(HostCalculator hostCalculator) {
231233
public void setDatabaseName(String databaseName) {
232234
this.databaseName = databaseName;
233235
}
236+
237+
public void setForestBuilder(ForestBuilder forestBuilder) {
238+
this.forestBuilder = forestBuilder;
239+
}
234240
}

‎src/main/java/com/marklogic/appdeployer/command/forests/DistributedReplicaBuilderStrategy.java‎

Lines changed: 5 additions & 16 deletions
Original file line numberDiff line numberDiff line change
@@ -7,30 +7,19 @@
77
import java.util.ArrayList;
88
import java.util.HashMap;
99
import java.util.List;
10-
import java.util.Map;
1110

12-
public class DistributedReplicaBuilderStrategy extends ReplicaBuilderStrategy {
11+
public class DistributedReplicaBuilderStrategy extends AbstractReplicaBuilderStrategy {
1312

1413
/**
1514
* Distributes the replicas throughout the cluster.
1615
*/
1716
public void buildReplicas(List<Forest> forests, ForestPlan forestPlan, AppConfig appConfig,
18-
List<String> dataDirectories, ForestNamingStrategy fns)
17+
List<String> replicaDataDirectories, ForestNamingStrategy fns)
1918
{
2019
final String databaseName = forestPlan.getDatabaseName();
2120
final List<String> hostNames = forestPlan.getHostNames();
2221
final int replicaCount = forestPlan.getReplicaCount();
2322

24-
if (appConfig.getReplicaForestDataDirectory() != null) {
25-
dataDirectories = new ArrayList<>();
26-
dataDirectories.add(appConfig.getReplicaForestDataDirectory());
27-
}
28-
Map<String, String> replicaDataDirectoryMap = appConfig.getDatabaseReplicaDataDirectories();
29-
if (replicaDataDirectoryMap != null && replicaDataDirectoryMap.containsKey(databaseName)) {
30-
dataDirectories = new ArrayList<>();
31-
dataDirectories.add(replicaDataDirectoryMap.get(databaseName));
32-
}
33-
3423
HashMap<String, List<Forest>> hostToForests = new HashMap<String, List<Forest>>();
3524

3625
for (Forest f : forests) {
@@ -62,7 +51,7 @@ public void buildReplicas(List<Forest> forests, ForestPlan forestPlan, AppConfig
6251

6352
for (Forest currForest : hostToForests.get(host)) {
6453
List<ForestReplica> replicas = new ArrayList<>();
65-
int dataDirectoryPointer = dataDirectories.indexOf(currForest.getDataDirectory());
54+
int dataDirectoryPointer = replicaDataDirectories.indexOf(currForest.getDataDirectory());
6655

6756
for (int i = 1; i <= replicaCount; i++) {
6857
ForestReplica replica = new ForestReplica();
@@ -77,11 +66,11 @@ public void buildReplicas(List<Forest> forests, ForestPlan forestPlan, AppConfig
7766
replica.setHost(availableHosts.get(replicaHostPointer));
7867

7968
dataDirectoryPointer++;
80-
if (dataDirectoryPointer == dataDirectories.size()) {
69+
if (dataDirectoryPointer == replicaDataDirectories.size()) {
8170
dataDirectoryPointer = 0;
8271
}
8372

84-
final String dataDir = dataDirectories.get(dataDirectoryPointer);
73+
final String dataDir = replicaDataDirectories.get(dataDirectoryPointer);
8574
if (dataDir != null && dataDir.trim().length() > 0) {
8675
replica.setDataDirectory(dataDir);
8776
}

‎src/main/java/com/marklogic/appdeployer/command/forests/ForestBuilder.java‎

Lines changed: 52 additions & 10 deletions
Original file line numberDiff line numberDiff line change
@@ -4,7 +4,6 @@
44
import com.marklogic.client.ext.helper.LoggingObject;
55
import com.marklogic.mgmt.api.API;
66
import com.marklogic.mgmt.api.forest.Forest;
7-
import com.marklogic.mgmt.api.forest.ForestReplica;
87
import com.marklogic.mgmt.mapper.DefaultResourceMapper;
98
import com.marklogic.mgmt.mapper.ResourceMapper;
109

@@ -30,11 +29,6 @@ public ForestBuilder(ForestNamingStrategy forestNamingStrategy) {
3029
this.replicaBuilderStrategy = new DistributedReplicaBuilderStrategy();
3130
}
3231

33-
public ForestBuilder(ForestNamingStrategy forestNamingStrategy, ReplicaBuilderStrategy replicaBuilderStrategy) {
34-
this.forestNamingStrategy = forestNamingStrategy;
35-
this.replicaBuilderStrategy = replicaBuilderStrategy;
36-
}
37-
3832
/**
3933
* Builds a list of Forest objects based on the given ForestPlan and AppConfig. If replicaCount on the ForestPlan
4034
* is greater than zero, then ForestReplica objects are added to each Forest as well.
@@ -104,7 +98,7 @@ public List<Forest> buildForests(ForestPlan forestPlan, AppConfig appConfig) {
10498
}
10599

106100
if (forestPlan.getReplicaCount() > 0) {
107-
addReplicasToForests(forests, forestPlan, appConfig);
101+
addReplicasToForests(forests, forestPlan, appConfig, dataDirectories);
108102
}
109103

110104
return forests;
@@ -118,7 +112,7 @@ public List<Forest> buildForests(ForestPlan forestPlan, AppConfig appConfig) {
118112
* @param forestPlan
119113
* @param appConfig
120114
*/
121-
public void addReplicasToForests(List<Forest> forests, ForestPlan forestPlan, AppConfig appConfig) {
115+
public void addReplicasToForests(List<Forest> forests, ForestPlan forestPlan, AppConfig appConfig, List<String> dataDirectories) {
122116
final String databaseName = forestPlan.getDatabaseName();
123117
final List<String> hostNames = forestPlan.getHostNames();
124118
final int replicaCount = forestPlan.getReplicaCount();
@@ -129,10 +123,21 @@ public void addReplicasToForests(List<Forest> forests, ForestPlan forestPlan, Ap
129123
replicaCount, databaseName, hostNames));
130124
}
131125

132-
List<String> dataDirectories = determineDataDirectories(databaseName, appConfig);
126+
// Determine if there are replica-specific data directories. If not, use the primary ones.
127+
List<String> replicaDataDirectories = determineReplicaDataDirectories(forestPlan, appConfig);
128+
if (replicaDataDirectories == null) {
129+
replicaDataDirectories = dataDirectories;
130+
}
133131

134-
replicaBuilderStrategy.buildReplicas(forests, forestPlan, appConfig, dataDirectories, determineForestNamingStrategy(databaseName, appConfig));
132+
ReplicaBuilderStrategy strategyToUse = replicaBuilderStrategy;
133+
if (appConfig.getReplicaBuilderStrategy() != null) {
134+
if (logger.isInfoEnabled()) {
135+
logger.info("Using ReplicaBuilderStrategy defined in AppConfig");
136+
}
137+
strategyToUse = appConfig.getReplicaBuilderStrategy();
138+
}
135139

140+
strategyToUse.buildReplicas(forests, forestPlan, appConfig, replicaDataDirectories, determineForestNamingStrategy(databaseName, appConfig));
136141
}
137142

138143
protected Forest newForest(ForestPlan forestPlan) {
@@ -177,6 +182,31 @@ protected List<String> determineDataDirectories(String databaseName, AppConfig a
177182
return dataDirectories;
178183
}
179184

185+
/**
186+
* Determines if there are replica-specific data directories for the database associated with the ForestPlan. If
187+
* not, then null will be returned.
188+
*
189+
* @param forestPlan
190+
* @param appConfig
191+
* @return
192+
*/
193+
protected List<String> determineReplicaDataDirectories(ForestPlan forestPlan, AppConfig appConfig) {
194+
List<String> replicaDataDirectories = null;
195+
if (appConfig.getReplicaForestDataDirectory() != null) {
196+
replicaDataDirectories = new ArrayList<>();
197+
replicaDataDirectories.add(appConfig.getReplicaForestDataDirectory());
198+
}
199+
200+
Map<String, String> replicaDataDirectoryMap = appConfig.getDatabaseReplicaDataDirectories();
201+
final String databaseName = forestPlan.getDatabaseName();
202+
if (replicaDataDirectoryMap != null && replicaDataDirectoryMap.containsKey(databaseName)) {
203+
replicaDataDirectories = new ArrayList<>();
204+
replicaDataDirectories.add(replicaDataDirectoryMap.get(databaseName));
205+
}
206+
207+
return replicaDataDirectories;
208+
}
209+
180210
/**
181211
* TODO This a little wonky in that it's using appConfig.getForestCounts, which is currently understood to be the
182212
* number of forests per host, not per data directory. Probably need a new property in appConfig to make this more
@@ -223,4 +253,16 @@ protected ForestNamingStrategy determineForestNamingStrategy(String databaseName
223253
}
224254
return fns != null ? fns : this.forestNamingStrategy;
225255
}
256+
257+
public void setReplicaBuilderStrategy(ReplicaBuilderStrategy replicaBuilderStrategy) {
258+
this.replicaBuilderStrategy = replicaBuilderStrategy;
259+
}
260+
261+
public ReplicaBuilderStrategy getReplicaBuilderStrategy() {
262+
return replicaBuilderStrategy;
263+
}
264+
265+
public ForestNamingStrategy getForestNamingStrategy() {
266+
return forestNamingStrategy;
267+
}
226268
}

‎src/main/java/com/marklogic/appdeployer/command/forests/GroupedReplicaBuilderStrategy.java‎

Lines changed: 11 additions & 21 deletions
Original file line numberDiff line numberDiff line change
@@ -6,35 +6,25 @@
66

77
import java.util.ArrayList;
88
import java.util.List;
9-
import java.util.Map;
109

11-
public class GroupedReplicaBuilderStrategy extends ReplicaBuilderStrategy {
10+
/**
11+
* This is the original ReplicaBuilderStrategy. The first replica for forest n on host h will be on host
12+
* (h+1)%hostCount. The distributed strategy is preferred so that replicas for host h are distributed across all of the
13+
* hosts instead of all being on one other host.
14+
*/
15+
@Deprecated
16+
public class GroupedReplicaBuilderStrategy extends AbstractReplicaBuilderStrategy {
1217

13-
/**
14-
* This is the original ReplicaBuilderStrategy. The first replica for forest n on host h will be on host
15-
* (h+1)%hostCount.
16-
*/
1718
public void buildReplicas(List<Forest> forests, ForestPlan forestPlan, AppConfig appConfig,
18-
List<String> dataDirectories, ForestNamingStrategy fns)
19-
{
19+
List<String> replicaDataDirectories, ForestNamingStrategy fns) {
2020
final String databaseName = forestPlan.getDatabaseName();
2121
final List<String> hostNames = forestPlan.getHostNames();
2222
final int replicaCount = forestPlan.getReplicaCount();
2323

24-
if (appConfig.getReplicaForestDataDirectory() != null) {
25-
dataDirectories = new ArrayList<>();
26-
dataDirectories.add(appConfig.getReplicaForestDataDirectory());
27-
}
28-
Map<String, String> replicaDataDirectoryMap = appConfig.getDatabaseReplicaDataDirectories();
29-
if (replicaDataDirectoryMap != null && replicaDataDirectoryMap.containsKey(databaseName)) {
30-
dataDirectories = new ArrayList<>();
31-
dataDirectories.add(replicaDataDirectoryMap.get(databaseName));
32-
}
33-
3424
for (Forest f : forests) {
3525
List<ForestReplica> replicas = new ArrayList<>();
3626
int hostPointer = hostNames.indexOf(f.getHost());
37-
int dataDirectoryPointer = dataDirectories.indexOf(f.getDataDirectory());
27+
int dataDirectoryPointer = replicaDataDirectories.indexOf(f.getDataDirectory());
3828
for (int i = 1; i <= replicaCount; i++) {
3929
ForestReplica replica = new ForestReplica();
4030
replica.setReplicaName(fns.getReplicaName(databaseName, f.getForestName(), i, appConfig));
@@ -46,13 +36,13 @@ public void buildReplicas(List<Forest> forests, ForestPlan forestPlan, AppConfig
4636
}
4737

4838
dataDirectoryPointer++;
49-
if (dataDirectoryPointer == dataDirectories.size()) {
39+
if (dataDirectoryPointer == replicaDataDirectories.size()) {
5040
dataDirectoryPointer = 0;
5141
}
5242

5343
replica.setHost(hostNames.get(hostPointer));
5444

55-
final String dataDir = dataDirectories.get(dataDirectoryPointer);
45+
final String dataDir = replicaDataDirectories.get(dataDirectoryPointer);
5646
if (dataDir != null && dataDir.trim().length() > 0) {
5747
replica.setDataDirectory(dataDir);
5848
}

0 commit comments

Comments
 (0)