Skip to content
This repository was archived by the owner on Sep 16, 2024. It is now read-only.

Commit 530ed42

Browse files
rjrudinrjrudin
committed
#372 Forests are now created properly when hosts are added
Or when forests are in general not already evenly distributed across each data directory and host. asdf
1 parent 4351c15 commit 530ed42

File tree

6 files changed

+198
-67
lines changed

6 files changed

+198
-67
lines changed

‎src/main/java/com/marklogic/appdeployer/command/forests/DeployForestsCommand.java‎

Lines changed: 18 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -4,9 +4,12 @@
44
import com.marklogic.appdeployer.command.AbstractCommand;
55
import com.marklogic.appdeployer.command.CommandContext;
66
import com.marklogic.appdeployer.command.SortOrderConstants;
7+
import com.marklogic.mgmt.api.API;
78
import com.marklogic.mgmt.api.configuration.Configuration;
89
import com.marklogic.mgmt.api.configuration.Configurations;
910
import com.marklogic.mgmt.api.forest.Forest;
11+
import com.marklogic.mgmt.mapper.DefaultResourceMapper;
12+
import com.marklogic.mgmt.mapper.ResourceMapper;
1013
import com.marklogic.mgmt.resource.databases.DatabaseManager;
1114
import com.marklogic.mgmt.resource.forests.ForestManager;
1215
import com.marklogic.mgmt.resource.hosts.DefaultHostNameProvider;
@@ -113,12 +116,12 @@ public List<Forest> buildForests(CommandContext context, boolean includeReplicas
113116
List<String> hostNames = new HostManager(context.getManageClient()).getHostNames();
114117
hostNames = determineHostNamesForForest(context, hostNames);
115118

116-
int countOfExistingForests = new DatabaseManager(context.getManageClient()).getPrimaryForestIds(this.databaseName).size();
117-
119+
// Need to know what primary forests exist already in case more need to be added, or a new host has been added
120+
List<Forest> forests = getExistingPrimaryForests(context, this.databaseName);
118121
ForestPlan forestPlan = new ForestPlan(this.databaseName, hostNames)
119122
.withTemplate(template)
120123
.withForestsPerDataDirectory(this.forestsPerHost)
121-
.withExistingForestsPerDataDirectory(countOfExistingForests);
124+
.withExistingForests(forests);
122125

123126
if (includeReplicas) {
124127
Map<String, Integer> map = context.getAppConfig().getDatabaseNamesAndReplicaCounts();
@@ -133,6 +136,18 @@ public List<Forest> buildForests(CommandContext context, boolean includeReplicas
133136
return forestBuilder.buildForests(forestPlan, context.getAppConfig());
134137
}
135138

139+
protected List<Forest> getExistingPrimaryForests(CommandContext context, String databaseName) {
140+
List<String> forestIds = new DatabaseManager(context.getManageClient()).getPrimaryForestIds(databaseName);
141+
ForestManager forestMgr = new ForestManager(context.getManageClient());
142+
ResourceMapper mapper = new DefaultResourceMapper(new API(context.getManageClient()));
143+
List<Forest> forests = new ArrayList<>();
144+
for (String forestId : forestIds) {
145+
String json = forestMgr.getPropertiesAsJson(forestId);
146+
forests.add(mapper.readResource(json, Forest.class));
147+
}
148+
return forests;
149+
}
150+
136151
protected String buildForestTemplate(CommandContext context, ForestManager forestManager) {
137152
String payload = null;
138153
if (forestFilename != null) {

‎src/main/java/com/marklogic/appdeployer/command/forests/ForestBuilder.java‎

Lines changed: 82 additions & 48 deletions
Original file line numberDiff line numberDiff line change
@@ -8,6 +8,7 @@
88
import com.marklogic.mgmt.mapper.ResourceMapper;
99

1010
import java.util.ArrayList;
11+
import java.util.LinkedHashMap;
1112
import java.util.List;
1213
import java.util.Map;
1314

@@ -19,6 +20,7 @@ public class ForestBuilder extends LoggingObject {
1920

2021
private ForestNamingStrategy forestNamingStrategy;
2122
private ReplicaBuilderStrategy replicaBuilderStrategy;
23+
private ResourceMapper resourceMapper;
2224

2325
public ForestBuilder() {
2426
this(new DefaultForestNamingStrategy());
@@ -27,6 +29,7 @@ public ForestBuilder() {
2729
public ForestBuilder(ForestNamingStrategy forestNamingStrategy) {
2830
this.forestNamingStrategy = forestNamingStrategy;
2931
this.replicaBuilderStrategy = new DistributedReplicaBuilderStrategy();
32+
this.resourceMapper = new DefaultResourceMapper(new API(null));
3033
}
3134

3235
/**
@@ -42,66 +45,100 @@ public ForestBuilder(ForestNamingStrategy forestNamingStrategy) {
4245
*/
4346
public List<Forest> buildForests(ForestPlan forestPlan, AppConfig appConfig) {
4447
final String databaseName = forestPlan.getDatabaseName();
45-
final List<String> hostNames = forestPlan.getHostNames();
4648

47-
List<String> dataDirectories = determineDataDirectories(databaseName, appConfig);
49+
// Find out what forests we have already, keyed on host and then data directory
50+
Map<String, Map<String, List<Forest>>> existingForestsMap = existingForestsMap(forestPlan);
4851

49-
int forestsPerDataDirectory = determineForestsPerDataDirectory(forestPlan, appConfig);
52+
final int forestsPerDataDirectory = determineForestsPerDataDirectory(forestPlan, appConfig);
53+
final List<String> dataDirectories = determineDataDirectories(databaseName, appConfig);
5054

51-
// Number of hosts * number of data directories * number of forests per data directory
52-
int numberToBuild = hostNames.size() * dataDirectories.size() * forestsPerDataDirectory;
55+
List<Forest> forestsToBuild = new ArrayList<>();
5356

54-
// So now we have numberToBuild - we want to iterate over each host, and over each data directory
55-
int forestCounter = (hostNames.size() * dataDirectories.size() * forestPlan.getExistingForestsPerDataDirectory()) + 1;
57+
// For naming any new forests we build, start with the current count and bump up as we build each forest
58+
int forestCounter = forestPlan.getExistingForests().size();
5659

57-
forestsPerDataDirectory -= forestPlan.getExistingForestsPerDataDirectory();
60+
/**
61+
* Now loop over each host, and for each host, loop over each data directory. See how many forests exist already
62+
* in that data directory. Build new forests as needed so each data directory has the correct amount.
63+
*/
64+
for (String hostName : forestPlan.getHostNames()) {
65+
Map<String, List<Forest>> hostMap = existingForestsMap.get(hostName);
66+
for (String dataDirectory : dataDirectories) {
67+
int forestsToCreate = forestsPerDataDirectory;
68+
if (hostMap != null && hostMap.containsKey(dataDirectory)) {
69+
forestsToCreate -= hostMap.get(dataDirectory).size();
70+
}
71+
for (int i = 0; i < forestsToCreate; i++) {
72+
forestCounter++;
73+
Forest forest = newForest(forestPlan);
74+
forest.setForestName(getForestName(databaseName, forestCounter, appConfig));
75+
forest.setHost(hostName);
76+
forest.setDatabase(databaseName);
77+
78+
if (dataDirectory != null && dataDirectory.trim().length() > 0) {
79+
forest.setDataDirectory(dataDirectory);
80+
}
5881

59-
List<Forest> forests = new ArrayList<>();
82+
// First see if we have any database-agnostic forest directories
83+
if (appConfig.getForestFastDataDirectory() != null) {
84+
forest.setFastDataDirectory(appConfig.getForestFastDataDirectory());
85+
}
86+
if (appConfig.getForestLargeDataDirectory() != null) {
87+
forest.setLargeDataDirectory(appConfig.getForestLargeDataDirectory());
88+
}
6089

61-
for (String hostName : hostNames) {
62-
for (String dataDirectory : dataDirectories) {
63-
for (int i = 0; i < forestsPerDataDirectory; i++) {
64-
if (forestCounter <= numberToBuild) {
65-
Forest forest = newForest(forestPlan);
66-
forest.setForestName(getForestName(databaseName, forestCounter, appConfig));
67-
forest.setHost(hostName);
68-
forest.setDatabase(databaseName);
69-
70-
if (dataDirectory != null && dataDirectory.trim().length() > 0) {
71-
forest.setDataDirectory(dataDirectory);
72-
}
73-
74-
// First see if we have any database-agnostic forest directories
75-
if (appConfig.getForestFastDataDirectory() != null) {
76-
forest.setFastDataDirectory(appConfig.getForestFastDataDirectory());
77-
}
78-
if (appConfig.getForestLargeDataDirectory() != null) {
79-
forest.setLargeDataDirectory(appConfig.getForestLargeDataDirectory());
80-
}
81-
82-
// Now check for database-specific forest directories
83-
Map<String, String> map = appConfig.getDatabaseFastDataDirectories();
84-
if (map != null && map.containsKey(databaseName)) {
85-
forest.setFastDataDirectory(map.get(databaseName));
86-
}
87-
map = appConfig.getDatabaseLargeDataDirectories();
88-
if (map != null && map.containsKey(databaseName)) {
89-
forest.setLargeDataDirectory(map.get(databaseName));
90-
}
91-
92-
forests.add(forest);
93-
94-
forestCounter++;
90+
// Now check for database-specific forest directories
91+
Map<String, String> map = appConfig.getDatabaseFastDataDirectories();
92+
if (map != null && map.containsKey(databaseName)) {
93+
forest.setFastDataDirectory(map.get(databaseName));
94+
}
95+
map = appConfig.getDatabaseLargeDataDirectories();
96+
if (map != null && map.containsKey(databaseName)) {
97+
forest.setLargeDataDirectory(map.get(databaseName));
9598
}
99+
100+
forestsToBuild.add(forest);
96101
}
97102
}
98103
}
99104

100105
if (forestPlan.getReplicaCount() > 0) {
101-
addReplicasToForests(forests, forestPlan, appConfig, dataDirectories);
106+
addReplicasToForests(forestsToBuild, forestPlan, appConfig, dataDirectories);
102107
}
103108

104-
return forests;
109+
return forestsToBuild;
110+
}
111+
112+
/**
113+
* Returns a map with keys of host names, where each value is a map whose keys are data directory paths bound to
114+
* a list of forests that already exist at each data directory path.
115+
*
116+
* @param forestPlan
117+
* @return
118+
*/
119+
protected Map<String, Map<String, List<Forest>>> existingForestsMap(ForestPlan forestPlan) {
120+
Map<String, Map<String, List<Forest>>> existingForestsMap = new LinkedHashMap<>();
121+
for (Forest f : forestPlan.getExistingForests()) {
122+
String host = f.getHost();
123+
String dataDirectory = f.getDataDirectory();
124+
if (dataDirectory == null) {
125+
dataDirectory = "";
126+
}
127+
128+
Map<String, List<Forest>> dataDirectoryMap = existingForestsMap.get(host);
129+
if (dataDirectoryMap == null) {
130+
dataDirectoryMap = new LinkedHashMap<>();
131+
existingForestsMap.put(host, dataDirectoryMap);
132+
}
133+
134+
List<Forest> list = dataDirectoryMap.get(dataDirectory);
135+
if (list == null) {
136+
list = new ArrayList<>();
137+
dataDirectoryMap.put(dataDirectory, list);
138+
}
139+
list.add(f);
140+
}
141+
return existingForestsMap;
105142
}
106143

107144
/**
@@ -146,7 +183,6 @@ protected Forest newForest(ForestPlan forestPlan) {
146183
return new Forest();
147184
}
148185
try {
149-
ResourceMapper resourceMapper = new DefaultResourceMapper(new API(null));
150186
return resourceMapper.readResource(template, Forest.class);
151187
} catch (Exception ex) {
152188
logger.warn("Unable to construct a new Forest using template: " + template, ex);
@@ -229,7 +265,6 @@ protected int determineForestsPerDataDirectory(ForestPlan forestPlan, AppConfig
229265
}
230266

231267
/**
232-
*
233268
* @param databaseName
234269
* @param forestNumber
235270
* @param appConfig
@@ -240,7 +275,6 @@ protected String getForestName(String databaseName, int forestNumber, AppConfig
240275
}
241276

242277
/**
243-
*
244278
* @param databaseName
245279
* @param appConfig
246280
* @return

‎src/main/java/com/marklogic/appdeployer/command/forests/ForestPlan.java‎

Lines changed: 8 additions & 7 deletions
Original file line numberDiff line numberDiff line change
@@ -2,6 +2,7 @@
22

33
import com.marklogic.mgmt.api.forest.Forest;
44

5+
import java.util.ArrayList;
56
import java.util.Arrays;
67
import java.util.List;
78

@@ -11,7 +12,7 @@ public class ForestPlan {
1112
private List<String> hostNames;
1213
private String template;
1314
private int forestsPerDataDirectory = 1;
14-
private int existingForestsPerDataDirectory = 0;
15+
private List<Forest> existingForests = new ArrayList<>();
1516
private int replicaCount = 0;
1617

1718
public ForestPlan(String databaseName, String... hostNames) {
@@ -33,8 +34,8 @@ public ForestPlan withForestsPerDataDirectory(int count) {
3334
return this;
3435
}
3536

36-
public ForestPlan withExistingForestsPerDataDirectory(int count) {
37-
this.existingForestsPerDataDirectory = count;
37+
public ForestPlan withExistingForests(List<Forest> existingForests) {
38+
this.existingForests = existingForests;
3839
return this;
3940
}
4041

@@ -55,15 +56,15 @@ public int getForestsPerDataDirectory() {
5556
return forestsPerDataDirectory;
5657
}
5758

58-
public int getExistingForestsPerDataDirectory() {
59-
return existingForestsPerDataDirectory;
60-
}
61-
6259
public int getReplicaCount() {
6360
return replicaCount;
6461
}
6562

6663
public String getTemplate() {
6764
return template;
6865
}
66+
67+
public List<Forest> getExistingForests() {
68+
return existingForests;
69+
}
6970
}

‎src/test/java/com/marklogic/appdeployer/command/forests/BuildForestReplicaTest.java‎

Lines changed: 29 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -12,12 +12,14 @@
1212

1313
public class BuildForestReplicaTest extends Assert {
1414

15+
private ForestBuilder builder = new ForestBuilder();
16+
1517
@Test
1618
public void customNamingStrategyWithDistributedStrategy() {
1719
AppConfig appConfig = newAppConfig("mlForestsPerHost", "my-database,2");
1820
addCustomNamingStrategy(appConfig);
1921

20-
List<Forest> forests = new ForestBuilder().buildForests(
22+
List<Forest> forests = builder.buildForests(
2123
new ForestPlan("my-database", "host1", "host2", "host3").withReplicaCount(2), appConfig);
2224

2325
Forest f1 = forests.get(0);
@@ -53,6 +55,30 @@ public void customNamingStrategyWithDistributedStrategy() {
5355
assertEquals("host2", f5.getForestReplica().get(1).getHost());
5456
}
5557

58+
@Test
59+
public void hostIsAdded() {
60+
AppConfig appConfig = newAppConfig();
61+
List<Forest> forests = builder.buildForests(new ForestPlan("testdb", "host1", "host2", "host3")
62+
.withForestsPerDataDirectory(1).withReplicaCount(1), appConfig);
63+
assertEquals(3, forests.size());
64+
assertEquals("testdb-1-replica-1", forests.get(0).getForestReplica().get(0).getReplicaName());
65+
assertEquals("host2", forests.get(0).getForestReplica().get(0).getHost());
66+
assertEquals("testdb-2-replica-1", forests.get(1).getForestReplica().get(0).getReplicaName());
67+
assertEquals("host3", forests.get(1).getForestReplica().get(0).getHost());
68+
assertEquals("testdb-3-replica-1", forests.get(2).getForestReplica().get(0).getReplicaName());
69+
assertEquals("host1", forests.get(2).getForestReplica().get(0).getHost());
70+
71+
forests = builder.buildForests(new ForestPlan("testdb", "host1", "host2", "host3", "host4")
72+
.withForestsPerDataDirectory(1).withExistingForests(forests).withReplicaCount(1), appConfig);
73+
assertEquals(1, forests.size());
74+
assertEquals("testdb-4-replica-1", forests.get(0).getForestReplica().get(0).getReplicaName());
75+
assertEquals(
76+
"When adding a new host and creating replicas, the replicas will naturally be uneven because the existing " +
77+
"forests on hosts 1, 2, and 3 won't have their replicas moved to 4 automatically, and thus host 4 won't " +
78+
"have any replicas on it. And for the new replicas, we expect those to start being created on the first host.",
79+
"host1", forests.get(0).getForestReplica().get(0).getHost());
80+
}
81+
5682
/**
5783
* This shows how replicas for host 1 all end up on host 2. The distributed strategy above is preferred, and
5884
* the grouped one is deprecated, but this test exists to show how the grouped one differs while it still exists.
@@ -64,7 +90,7 @@ public void customNamingStrategyWithGroupedStrategy() {
6490
addCustomNamingStrategy(appConfig);
6591
appConfig.setReplicaBuilderStrategy(new GroupedReplicaBuilderStrategy());
6692

67-
List<Forest> forests = new ForestBuilder().buildForests(
93+
List<Forest> forests = builder.buildForests(
6894
new ForestPlan("my-database", "host1", "host2", "host3").withReplicaCount(2), appConfig);
6995

7096
Forest f1 = forests.get(0);
@@ -171,7 +197,7 @@ public String getReplicaName(String databaseName, String forestName, int forestR
171197
private ForestReplica buildForestReplica(String... propertyNamesAndValues) {
172198
AppConfig config = newAppConfig(propertyNamesAndValues);
173199

174-
Forest forest = new ForestBuilder().buildForests(
200+
Forest forest = builder.buildForests(
175201
new ForestPlan("my-database", "host1", "host2").withReplicaCount(1), config).get(0);
176202
ForestReplica replica = forest.getForestReplica().get(0);
177203
assertEquals("my-database-1-replica-1", replica.getReplicaName());

0 commit comments

Comments
 (0)