Skip to content
This repository was archived by the owner on Sep 16, 2024. It is now read-only.

Commit ef25739

Browse files
rjrudinrjrudin
committed
#264 Can now specify multiple data directories
Reworked how forests are created too
1 parent 5d735ac commit ef25739

34 files changed

+1060
-762
lines changed

‎gradle.properties‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -1,6 +1,6 @@
11
group=com.marklogic
22
javadocsDir=../gh-pages-marklogic-java/javadocs
3-
version=3.6.3
3+
version=3.7.0
44
mlJavaclientUtilVersion=3.6.0
55
mlJunitVersion=3.1.0
66

‎src/main/java/com/marklogic/appdeployer/AppConfig.java‎

Lines changed: 19 additions & 19 deletions
Original file line numberDiff line numberDiff line change
@@ -148,22 +148,22 @@ public class AppConfig {
148148
private Integer contentForestsPerHost;
149149

150150
// Comma-delimited string used for configuring forest replicas
151-
private String databaseNamesAndReplicaCounts;
151+
private Map<String, Integer> databaseNamesAndReplicaCounts;
152152

153153
// Comma-delimited string of database names that should only have forests (most likely just one) created on one host
154154
private Set<String> databasesWithForestsOnOneHost;
155155

156-
private Map<String, Set<String>> databaseHosts;
157-
private Map<String, Set<String>> databaseGroups;
156+
private Map<String, List<String>> databaseHosts;
157+
private Map<String, List<String>> databaseGroups;
158158
private Map<String, String> hostGroups;
159-
159+
160160
// Data/fast/large directories for default forests
161161
private String forestDataDirectory;
162162
private String forestFastDataDirectory;
163163
private String forestLargeDataDirectory;
164164

165165
// Comma-delimited string of database names and data directories
166-
private Map<String, String> databaseDataDirectories;
166+
private Map<String, List<String>> databaseDataDirectories;
167167
private Map<String, String> databaseFastDataDirectories;
168168
private Map<String, String> databaseLargeDataDirectories;
169169
private Map<String, String> databaseReplicaDataDirectories;
@@ -619,14 +619,6 @@ public void setContentForestsPerHost(Integer contentForestsPerHost) {
619619
this.contentForestsPerHost = contentForestsPerHost;
620620
}
621621

622-
public String getDatabaseNamesAndReplicaCounts() {
623-
return databaseNamesAndReplicaCounts;
624-
}
625-
626-
public void setDatabaseNamesAndReplicaCounts(String databaseNamesAndReplicaCounts) {
627-
this.databaseNamesAndReplicaCounts = databaseNamesAndReplicaCounts;
628-
}
629-
630622
public String getModulePermissions() {
631623
return modulePermissions;
632624
}
@@ -1004,11 +996,11 @@ public void setModulesLoaderThreadCount(int modulesLoaderThreadCount) {
1004996
this.modulesLoaderThreadCount = modulesLoaderThreadCount;
1005997
}
1006998

1007-
public Map<String, String> getDatabaseDataDirectories() {
999+
public Map<String, List<String>> getDatabaseDataDirectories() {
10081000
return databaseDataDirectories;
10091001
}
10101002

1011-
public void setDatabaseDataDirectories(Map<String, String> databaseDataDirectories) {
1003+
public void setDatabaseDataDirectories(Map<String, List<String>> databaseDataDirectories) {
10121004
this.databaseDataDirectories = databaseDataDirectories;
10131005
}
10141006

@@ -1100,11 +1092,11 @@ public void setDatabasesWithForestsOnOneHost(Set<String> databasesWithForestsOnO
11001092
this.databasesWithForestsOnOneHost = databasesWithForestsOnOneHost;
11011093
}
11021094

1103-
public Map<String, Set<String>> getDatabaseHosts() {
1095+
public Map<String, List<String>> getDatabaseHosts() {
11041096
return databaseHosts;
11051097
}
11061098

1107-
public void setDatabaseHosts(Map<String, Set<String>> databaseHosts) {
1099+
public void setDatabaseHosts(Map<String, List<String>> databaseHosts) {
11081100
this.databaseHosts = databaseHosts;
11091101
}
11101102

@@ -1130,11 +1122,11 @@ public String[] getIncludeProperties() {
11301122
return this.includeProperties;
11311123
}
11321124

1133-
public Map<String, Set<String>> getDatabaseGroups() {
1125+
public Map<String, List<String>> getDatabaseGroups() {
11341126
return databaseGroups;
11351127
}
11361128

1137-
public void setDatabaseGroups(Map<String, Set<String>> databaseGroups) {
1129+
public void setDatabaseGroups(Map<String, List<String>> databaseGroups) {
11381130
this.databaseGroups = databaseGroups;
11391131
}
11401132

@@ -1193,4 +1185,12 @@ public String getCpfDatabaseName() {
11931185
public void setCpfDatabaseName(String cpfDatabaseName) {
11941186
this.cpfDatabaseName = cpfDatabaseName;
11951187
}
1188+
1189+
public Map<String, Integer> getDatabaseNamesAndReplicaCounts() {
1190+
return databaseNamesAndReplicaCounts;
1191+
}
1192+
1193+
public void setDatabaseNamesAndReplicaCounts(Map<String, Integer> databaseNamesAndReplicaCounts) {
1194+
this.databaseNamesAndReplicaCounts = databaseNamesAndReplicaCounts;
1195+
}
11961196
}

‎src/main/java/com/marklogic/appdeployer/DefaultAppConfigFactory.java‎

Lines changed: 12 additions & 7 deletions
Original file line numberDiff line numberDiff line change
@@ -280,7 +280,12 @@ public void initialize() {
280280
*/
281281
propertyConsumerMap.put("mlDatabaseNamesAndReplicaCounts", (config, prop) -> {
282282
logger.info("Database names and replica counts: " + prop);
283-
config.setDatabaseNamesAndReplicaCounts(prop);
283+
String[] tokens = prop.split(",");
284+
Map<String, Integer> map = new HashMap<>();
285+
for (int i = 0; i < tokens.length; i += 2) {
286+
map.put(tokens[i], Integer.parseInt(tokens[i + 1]));
287+
}
288+
config.setDatabaseNamesAndReplicaCounts(map);
284289
});
285290

286291
propertyConsumerMap.put("mlDatabasesWithForestsOnOneHost", (config, prop) -> {
@@ -295,7 +300,7 @@ public void initialize() {
295300

296301
propertyConsumerMap.put("mlDatabaseGroups", (config, prop) -> {
297302
logger.info("Databases and the groups containing the hosts that their forests will be created on: " + prop);
298-
config.setDatabaseGroups(buildSetMapFromDelimitedString(prop));
303+
config.setDatabaseGroups(buildMapOfListsFromDelimitedString(prop));
299304
});
300305

301306
propertyConsumerMap.put("mlHostGroups", (config, prop) -> {
@@ -305,7 +310,7 @@ public void initialize() {
305310

306311
propertyConsumerMap.put("mlDatabaseHosts", (config, prop) -> {
307312
logger.info("Databases and the hosts that their forests will be created on: " + prop);
308-
config.setDatabaseHosts(buildSetMapFromDelimitedString(prop));
313+
config.setDatabaseHosts(buildMapOfListsFromDelimitedString(prop));
309314
});
310315

311316
propertyConsumerMap.put("mlForestDataDirectory", (config, prop) -> {
@@ -340,7 +345,7 @@ public void initialize() {
340345

341346
propertyConsumerMap.put("mlDatabaseDataDirectories", (config, prop) -> {
342347
logger.info("Databases and forest data directories: " + prop);
343-
config.setDatabaseDataDirectories(buildMapFromCommaDelimitedString(prop));
348+
config.setDatabaseDataDirectories(buildMapOfListsFromDelimitedString(prop));
344349
});
345350

346351
propertyConsumerMap.put("mlDatabaseFastDataDirectories", (config, prop) -> {
@@ -638,13 +643,13 @@ protected Map<String, String> buildMapFromCommaDelimitedString(String str) {
638643
return map;
639644
}
640645

641-
protected Map<String, Set<String>> buildSetMapFromDelimitedString(String str) {
646+
protected Map<String, List<String>> buildMapOfListsFromDelimitedString(String str) {
642647
String[] tokens = str.split(",");
643-
Map<String, Set<String>> map = new LinkedHashMap<>();
648+
Map<String, List<String>> map = new LinkedHashMap<>();
644649
for (int i = 0; i < tokens.length; i += 2) {
645650
String dbName = tokens[i];
646651
String[] hostNames = tokens[i + 1].split("\\|");
647-
Set<String> names = new LinkedHashSet<>();
652+
List<String> names = new ArrayList<>();
648653
for (String name : hostNames) {
649654
names.add(name);
650655
}

‎src/main/java/com/marklogic/appdeployer/command/databases/DeployContentDatabasesCommand.java‎

Lines changed: 22 additions & 7 deletions
Original file line numberDiff line numberDiff line change
@@ -5,6 +5,7 @@
55
import com.marklogic.appdeployer.ConfigDir;
66
import com.marklogic.appdeployer.command.CommandContext;
77
import com.marklogic.appdeployer.command.SortOrderConstants;
8+
import com.marklogic.appdeployer.command.forests.DeployForestsCommand;
89
import com.marklogic.mgmt.SaveReceipt;
910
import com.marklogic.mgmt.resource.databases.DatabaseManager;
1011
import com.marklogic.rest.util.JsonNodeUtil;
@@ -39,13 +40,10 @@ public DeployContentDatabasesCommand(int forestsPerHost) {
3940
*/
4041
@Override
4142
public void execute(CommandContext context) {
42-
Integer count = context.getAppConfig().getContentForestsPerHost();
43-
if (count != null) {
44-
this.setForestsPerHost(count);
45-
}
46-
43+
// Construct the "main" content database first
4744
super.execute(context);
4845

46+
// And now check to see if we need to build a "test" content database
4947
AppConfig appConfig = context.getAppConfig();
5048
if (appConfig.isTestPortSet()) {
5149
if (logger.isInfoEnabled()) {
@@ -58,7 +56,7 @@ public void execute(CommandContext context) {
5856
String json = payloadTokenReplacer.replaceTokens(payload, appConfig, true);
5957
SaveReceipt receipt = dbMgr.save(json);
6058
if (shouldCreateForests(context, payload)) {
61-
buildDeployForestsCommand(payload, receipt, context).execute(context);
59+
buildDeployForestsCommand(receipt.getResourceId(), context).execute(context);
6260
}
6361
} else {
6462
logger.warn(format("Could not determine what payload to use for creating/updating test content database '%s'",
@@ -67,7 +65,24 @@ public void execute(CommandContext context) {
6765
}
6866
}
6967

70-
/**
68+
/**
69+
* Method is overridden to apply the content-database-specific property in AppConfig for number of forests per host.
70+
*
71+
* @param databaseName
72+
* @param context
73+
* @return
74+
*/
75+
@Override
76+
public DeployForestsCommand buildDeployForestsCommand(String databaseName, CommandContext context) {
77+
Integer count = context.getAppConfig().getContentForestsPerHost();
78+
if (count != null) {
79+
this.setForestsPerHost(count);
80+
}
81+
82+
return super.buildDeployForestsCommand(databaseName, context);
83+
}
84+
85+
/**
7186
* Just because there's not a content database file doesn't mean that one wasn't created via the command for
7287
* creating a REST API server. If the REST API server command didn't delete the content database, we'd still want
7388
* this command to attempt to do so in the event that no content database files exist.

‎src/main/java/com/marklogic/appdeployer/command/databases/DeployDatabaseCommand.java‎

Lines changed: 7 additions & 77 deletions
Original file line numberDiff line numberDiff line change
@@ -117,7 +117,7 @@ public void execute(CommandContext context) {
117117

118118
databaseName = receipt.getResourceId();
119119
if (shouldCreateForests(context, payload)) {
120-
buildDeployForestsCommand(payload, receipt, context).execute(context);
120+
buildDeployForestsCommand(databaseName, context).execute(context);
121121
}
122122

123123
if(!isSubDatabase()){
@@ -324,91 +324,21 @@ protected boolean customForestsExist(CommandContext context, String dbName) {
324324
}
325325

326326
/**
327-
* Allows for how an instance of DeployForestsCommand is built to be overridden by a subclass.
327+
* Initializes an instance of DeployForestsCommand. Public so that it can be accessed by a client that wishes to call
328+
* buildForests on the command.
328329
*
329-
* @param dbPayload
330-
* Needed so we can look up forest counts based on the database name
331-
* @param receipt
330+
* @param databaseName
332331
* @param context
333332
* @return
334333
*/
335-
protected DeployForestsCommand buildDeployForestsCommand(String dbPayload, SaveReceipt receipt,
336-
CommandContext context) {
337-
final String databaseName = receipt.getResourceId();
338-
339-
DeployForestsCommand c = new DeployForestsCommand();
334+
public DeployForestsCommand buildDeployForestsCommand(String databaseName, CommandContext context) {
335+
DeployForestsCommand c = new DeployForestsCommand(databaseName);
336+
c.setForestsPerHost(getForestsPerHost());
340337
c.setCreateForestsOnEachHost(createForestsOnEachHost);
341-
c.setForestsPerHost(determineForestCountPerHost(dbPayload, context));
342338
c.setForestFilename(forestFilename);
343-
c.setDatabaseName(databaseName);
344-
345-
Forest forest = buildForest(context.getAppConfig());
346-
forest.setObjectMapper(ObjectMapperFactory.getObjectMapper());
347-
c.setForestPayload(forest.getJson());
348339
return c;
349340
}
350341

351-
protected Forest buildForest(AppConfig config) {
352-
Forest forest = new Forest();
353-
forest.setForestName("%%FOREST_NAME%%");
354-
forest.setHost("%%FOREST_HOST%%");
355-
forest.setDatabase("%%FOREST_DATABASE%%");
356-
357-
// First see if we have any database-agnostic forest directories
358-
if (config.getForestDataDirectory() != null) {
359-
forest.setDataDirectory(config.getForestDataDirectory());
360-
}
361-
if (config.getForestFastDataDirectory() != null) {
362-
forest.setFastDataDirectory(config.getForestFastDataDirectory());
363-
}
364-
if (config.getForestLargeDataDirectory() != null) {
365-
forest.setLargeDataDirectory(config.getForestLargeDataDirectory());
366-
}
367-
368-
// Now check for database-specific forest directories
369-
Map<String, String> map = config.getDatabaseDataDirectories();
370-
if (map != null && map.containsKey(databaseName)) {
371-
forest.setDataDirectory(map.get(databaseName));
372-
}
373-
map = config.getDatabaseFastDataDirectories();
374-
if (map != null && map.containsKey(databaseName)) {
375-
forest.setFastDataDirectory(map.get(databaseName));
376-
}
377-
map = config.getDatabaseLargeDataDirectories();
378-
if (map != null && map.containsKey(databaseName)) {
379-
forest.setLargeDataDirectory(map.get(databaseName));
380-
}
381-
382-
return forest;
383-
}
384-
385-
/**
386-
* Checks the forestCounts map in AppConfig to see if the client has specified a number of forests per host for this
387-
* database.
388-
*
389-
* @param dbPayload
390-
* @param context
391-
* @return
392-
*/
393-
protected int determineForestCountPerHost(String dbPayload, CommandContext context) {
394-
int forestCount = forestsPerHost;
395-
if (dbPayload != null) {
396-
try {
397-
String dbName = new PayloadParser().getPayloadFieldValue(dbPayload, "database-name");
398-
Map<String, Integer> forestCounts = context.getAppConfig().getForestCounts();
399-
if (forestCounts != null && forestCounts.containsKey(dbName)) {
400-
Integer i = forestCounts.get(dbName);
401-
if (i != null) {
402-
forestCount = i;
403-
}
404-
}
405-
} catch (Exception ex) {
406-
logger.warn("Unable to determine forest counts, cause: " + ex.getMessage(), ex);
407-
}
408-
}
409-
return forestCount;
410-
}
411-
412342
protected String buildDefaultDatabasePayload(CommandContext context) {
413343
return format("{\"database-name\": \"%s\"}", databaseName);
414344
}

0 commit comments

Comments
 (0)