Skip to content

Commit 8488c32

Browse files
paxtonharepaxtonhare
committed
Merge pull request #83 from paxtonhare/56_staging_and_final_dbs
fixed #56 - created a staging and final database
2 parents 6e95190 + c7a30a8 commit 8488c32

29 files changed

+542
-186
lines changed

‎data-hub/build.gradle‎

Lines changed: 21 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -119,10 +119,29 @@ publishing {
119119
ext {
120120
mlAppConfig {
121121
modulePaths = ["data-hub/src/main/resources/ml-modules"]
122-
mlConfigDir = "data-hub/src/main/resources/ml-config"
122+
// mlConfigDir = "data-hub/src/main/resources/ml-config"
123123
}
124124

125+
// don't deploy the content database
126+
cmd = mlAppDeployer.getCommand("DeployContentDatabasesCommand")
127+
index = mlAppDeployer.commands.indexOf(cmd)
128+
mlAppDeployer.commands.remove(cmd)
129+
mlDatabaseCommands.remove(cmd)
130+
125131
def stagingDbCommand = new com.marklogic.appdeployer.command.databases.DeployDatabaseCommand("staging-database.json")
126-
mlAppDeployer.commands.add(stagingDbCommand)
132+
stagingDbCommand.setForestsPerHost(3);
133+
mlAppDeployer.commands.add(index, stagingDbCommand)
127134
mlDatabaseCommands.add(stagingDbCommand)
135+
136+
def finalDbCommand = new com.marklogic.appdeployer.command.databases.DeployDatabaseCommand("final-database.json")
137+
finalDbCommand.setForestsPerHost(3);
138+
mlAppDeployer.commands.add(index + 1, finalDbCommand)
139+
mlDatabaseCommands.add(finalDbCommand)
140+
141+
println(mlAppDeployer.commands.toString())
142+
println("")
143+
println("")
144+
println(mlDatabaseCommands.toString())
145+
146+
128147
}

‎data-hub/gradle.properties‎

Lines changed: 3 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -1,7 +1,8 @@
11
mlAppDeployerDependency=com.marklogic:ml-app-deployer:+
2-
mlConfigDir=data-hub/src/main/ml-config
2+
mlConfigDir=data-hub/src/main/resources/ml-config
33
publishUrl=file:../data-hub/releases
4-
mlRestPort=8010
4+
mlStagingRestPort=8010
5+
mlFinalRestPort=8011
56
mlUsername=admin
67
mlPassword=admin
78
mlHost=localhost

‎data-hub/src/main/java/com/marklogic/hub/DataHub.java‎

Lines changed: 82 additions & 32 deletions
Original file line numberDiff line numberDiff line change
@@ -31,12 +31,16 @@
3131
import org.springframework.core.io.Resource;
3232
import org.springframework.web.client.ResourceAccessException;
3333

34+
import com.fasterxml.jackson.core.JsonProcessingException;
35+
import com.fasterxml.jackson.core.util.DefaultPrettyPrinter;
36+
import com.fasterxml.jackson.databind.ObjectMapper;
37+
import com.fasterxml.jackson.databind.node.ObjectNode;
3438
import com.google.gson.Gson;
3539
import com.marklogic.appdeployer.AppConfig;
3640
import com.marklogic.appdeployer.ConfigDir;
3741
import com.marklogic.appdeployer.command.Command;
3842
import com.marklogic.appdeployer.command.appservers.UpdateRestApiServersCommand;
39-
import com.marklogic.appdeployer.command.databases.DeployContentDatabasesCommand;
43+
//import com.marklogic.appdeployer.command.databases.DeployDatabaseCommand;
4044
import com.marklogic.appdeployer.command.databases.DeploySchemasDatabaseCommand;
4145
import com.marklogic.appdeployer.command.databases.DeployTriggersDatabaseCommand;
4246
import com.marklogic.appdeployer.command.modules.AssetModulesFinder;
@@ -47,60 +51,101 @@
4751
import com.marklogic.client.modulesloader.Modules;
4852
import com.marklogic.client.modulesloader.ModulesFinder;
4953
import com.marklogic.client.modulesloader.ModulesManager;
50-
import com.marklogic.client.modulesloader.impl.DefaultModulesLoader;
54+
import com.marklogic.hub.commands.DeployHubDatabaseCommand;
55+
import com.marklogic.hub.commands.DeployModulesDatabaseCommand;
56+
import com.marklogic.hub.commands.DeployRestApiCommand;
5157
import com.marklogic.hub.commands.LoadModulesCommand;
5258
import com.marklogic.hub.util.GsonUtil;
5359
import com.marklogic.mgmt.ManageClient;
5460
import com.marklogic.mgmt.ManageConfig;
5561
import com.marklogic.mgmt.admin.AdminConfig;
5662
import com.marklogic.mgmt.admin.AdminManager;
5763
import com.marklogic.mgmt.appservers.ServerManager;
64+
import com.marklogic.mgmt.databases.DatabaseManager;
65+
import com.marklogic.mgmt.restapis.RestApiManager;
66+
import com.marklogic.rest.util.Fragment;
5867

5968
public class DataHub {
6069

6170
static final private Logger LOGGER = LoggerFactory.getLogger(DataHub.class);
62-
71+
static final private String STAGING_NAME = "data-hub-in-a-box-STAGING";
72+
static final private String FINAL_NAME = "data-hub-in-a-box-FINAL";
73+
static final private String MODULES_DB_NAME = "data-hub-in-a-box-modules";
6374
private ManageConfig config;
6475
private ManageClient client;
6576
public static String HUB_NAME = "data-hub-in-a-box";
6677
public static int FORESTS_PER_HOST = 4;
6778
private String host;
68-
private int restPort;
79+
private int stagingRestPort;
80+
private int finalRestPort;
6981
private String username;
7082
private String password;
7183

7284
private File assetInstallTimeFile;
7385

74-
private final static int DEFAULT_REST_PORT = 8010;
86+
private final static int DEFAULT_STAGING_REST_PORT = 8010;
87+
private final static int DEFAULT_FINAL_REST_PORT = 8011;
7588

7689
public DataHub(HubConfig config) {
7790
this(config.getHost(), config.getAdminUsername(), config.getAdminPassword());
7891
}
7992

8093
public DataHub(String host, String username, String password) {
81-
this(host, DEFAULT_REST_PORT, username, password);
94+
this(host, DEFAULT_STAGING_REST_PORT, DEFAULT_FINAL_REST_PORT, username, password);
8295
}
8396

84-
public DataHub(String host, int restPort, String username, String password) {
97+
public DataHub(String host, int stagingRestPort, int finalRestPort, String username, String password) {
8598
config = new ManageConfig(host, 8002, username, password);
8699
client = new ManageClient(config);
87100
this.host = host;
88-
this.restPort = restPort;
101+
this.stagingRestPort = stagingRestPort;
102+
this.finalRestPort = finalRestPort;
89103
this.username = username;
90104
this.password = password;
91105
}
92-
106+
93107
public void setAssetInstallTimeFile(File assetInstallTimeFile) {
94108
this.assetInstallTimeFile = assetInstallTimeFile;
95109
}
96-
110+
97111
/**
98112
* Determines if the data hub is installed in MarkLogic
99113
* @return true if installed, false otherwise
100114
*/
101115
public boolean isInstalled() {
102116
ServerManager sm = new ServerManager(client);
103-
return sm.exists("data-hub-in-a-box");
117+
DatabaseManager dm = new DatabaseManager(client);
118+
boolean stagingAppServerExists = sm.exists(STAGING_NAME);
119+
boolean finalAppServerExists = sm.exists(FINAL_NAME);
120+
boolean appserversOk = (stagingAppServerExists && finalAppServerExists);
121+
122+
boolean stagingDbExists = dm.exists(STAGING_NAME);
123+
boolean finalDbExists = dm.exists(FINAL_NAME);
124+
125+
boolean stagingForestsExist = false;
126+
boolean finalForestsExist = false;
127+
128+
boolean stagingIndexesOn = false;
129+
boolean finalIndexesOn = false;
130+
131+
if (stagingDbExists) {
132+
Fragment f = dm.getPropertiesAsXml(STAGING_NAME);
133+
stagingIndexesOn = Boolean.parseBoolean(f.getElementValue("//m:triple-index"));
134+
stagingIndexesOn = stagingIndexesOn && Boolean.parseBoolean(f.getElementValue("//m:collection-lexicon"));
135+
stagingForestsExist = (dm.getForestIds(STAGING_NAME).size() == FORESTS_PER_HOST);
136+
}
137+
138+
if (finalDbExists) {
139+
Fragment f = dm.getPropertiesAsXml(FINAL_NAME);
140+
finalIndexesOn = Boolean.parseBoolean(f.getElementValue("//m:triple-index"));
141+
finalIndexesOn = finalIndexesOn && Boolean.parseBoolean(f.getElementValue("//m:collection-lexicon"));
142+
finalForestsExist = (dm.getForestIds(FINAL_NAME).size() == FORESTS_PER_HOST);
143+
}
144+
boolean dbsOk = (stagingDbExists && stagingIndexesOn &&
145+
finalDbExists && finalIndexesOn);
146+
boolean forestsOk = (stagingForestsExist && finalForestsExist);
147+
148+
return (appserversOk && dbsOk && forestsOk);
104149
}
105150

106151
/**
@@ -129,13 +174,14 @@ public void validateServer() throws ServerValidationException {
129174
private AppConfig getAppConfig() throws IOException {
130175
AppConfig config = new AppConfig();
131176
config.setHost(host);
132-
config.setRestPort(restPort);
177+
config.setRestPort(stagingRestPort);
133178
config.setName(HUB_NAME);
134179
config.setRestAdminUsername(username);
135180
config.setRestAdminPassword(password);
136181
List<String> paths = new ArrayList<String>();
137182
paths.add(new ClassPathResource("ml-modules").getPath());
138-
config.setConfigDir(new ConfigDir(new File(new ClassPathResource("ml-config").getPath())));
183+
String configPath = new ClassPathResource("ml-config").getPath();
184+
config.setConfigDir(new ConfigDir(new File(configPath)));
139185
config.setModulePaths(paths);
140186
return config;
141187
}
@@ -164,7 +210,7 @@ public void install() throws IOException {
164210
public Map<File, Date> installUserModules(String pathToUserModules) throws IOException {
165211
AppConfig config = new AppConfig();
166212
config.setHost(host);
167-
config.setRestPort(restPort);
213+
config.setRestPort(stagingRestPort);
168214
config.setName(HUB_NAME);
169215
config.setRestAdminUsername(username);
170216
config.setRestAdminPassword(password);
@@ -188,17 +234,17 @@ public Map<File, Date> installUserModules(String pathToUserModules) throws IOExc
188234
paths[i] = dirs.get(i).getFile().getAbsolutePath();
189235
}
190236
Set<File> loadedFiles = loader.loadAssetsViaREST(paths);
191-
237+
192238
if (assetInstallTimeFile != null) {
193239
Gson gson = GsonUtil.createGson();
194-
240+
195241
String json = gson.toJson(modulesManager.getLastInstallInfo());
196242
try {
197243
FileUtils.write(assetInstallTimeFile, json);
198244
} catch (IOException e) {
199245
LOGGER.error("Cannot write asset install info.", e);
200246
}
201-
247+
202248
return modulesManager.getLastInstallInfo();
203249
}
204250
else {
@@ -221,27 +267,31 @@ private List<Command> getCommands(AppConfig config) {
221267

222268
// Databases
223269
List<Command> dbCommands = new ArrayList<Command>();
224-
DeployContentDatabasesCommand dcdc = new DeployContentDatabasesCommand();
225-
dcdc.setForestsPerHost(FORESTS_PER_HOST);
226-
dbCommands.add(dcdc);
270+
DeployHubDatabaseCommand staging = new DeployHubDatabaseCommand(STAGING_NAME);
271+
staging.setForestsPerHost(FORESTS_PER_HOST);
272+
dbCommands.add(staging);
273+
274+
DeployHubDatabaseCommand finalDb = new DeployHubDatabaseCommand(FINAL_NAME);
275+
finalDb.setForestsPerHost(FORESTS_PER_HOST);
276+
dbCommands.add(finalDb);
277+
278+
dbCommands.add(new DeployModulesDatabaseCommand(MODULES_DB_NAME));
227279
dbCommands.add(new DeployTriggersDatabaseCommand());
228280
dbCommands.add(new DeploySchemasDatabaseCommand());
229281
commands.addAll(dbCommands);
230282

231-
// REST API instance creation
232-
commands.add(new DeployRestApiServersCommand());
233-
234-
// App servers
235-
List<Command> serverCommands = new ArrayList<Command>();
236-
serverCommands.add(new UpdateRestApiServersCommand());
237-
commands.addAll(serverCommands);
283+
// App Servers
284+
commands.add(new DeployRestApiCommand(STAGING_NAME, stagingRestPort));
285+
commands.add(new DeployRestApiCommand(FINAL_NAME, finalRestPort));
238286

239287
// Modules
240288
commands.add(new LoadModulesCommand());
241289

242290
return commands;
243291
}
244292

293+
294+
245295
/**
246296
* Uninstalls the data hub configuration and server-side modules from MarkLogic
247297
* @throws IOException
@@ -253,14 +303,14 @@ public void uninstall() throws IOException {
253303
deployer.setCommands(getCommands(config));
254304
deployer.undeploy(config);
255305
}
256-
306+
257307
private class DataHubModuleManager implements ModulesManager {
258308
private Map<File, Date> lastInstallInfo = new HashMap<>();
259-
309+
260310
public Map<File, Date> getLastInstallInfo() {
261311
return lastInstallInfo;
262312
}
263-
313+
264314
@Override
265315
public void initialize() {
266316
LOGGER.debug("initializing DataHubModuleManager");
@@ -275,7 +325,7 @@ public boolean hasFileBeenModifiedSinceLastInstalled(File file) {
275325
LOGGER.warn("Cannot get canonical path of {}. Using absolute path instead.", file);
276326
lastInstallDate = lastInstallInfo.get(new File(file.getAbsolutePath()));
277327
}
278-
328+
279329
// a file has been modified if it has not been previously installed (new file)
280330
// or when its modified time is after the last install time
281331
return lastInstallDate == null || file.lastModified() > lastInstallDate.getTime();
@@ -291,6 +341,6 @@ public void saveLastInstalledTimestamp(File file, Date date) {
291341
lastInstallInfo.put(new File(file.getAbsolutePath()), date);
292342
}
293343
}
294-
344+
295345
}
296346
}

‎data-hub/src/main/java/com/marklogic/hub/DefaultHubConfigFactory.java‎

Lines changed: 9 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -47,10 +47,16 @@ public HubConfig newHubConfig() {
4747
c.setHost(prop);
4848
}
4949

50-
prop = getProperty("mlPort");
50+
prop = getProperty("mlStagingPort");
5151
if (prop != null) {
52-
logger.info("App REST port: " + prop);
53-
c.setPort(Integer.parseInt(prop));
52+
logger.info("Staging App REST port: " + prop);
53+
c.setStagingPort(Integer.parseInt(prop));
54+
}
55+
56+
prop = getProperty("mlFinalPort");
57+
if (prop != null) {
58+
logger.info("Final App REST port: " + prop);
59+
c.setFinalPort(Integer.parseInt(prop));
5460
}
5561

5662
prop = getProperty("mlAdminUsername");

‎data-hub/src/main/java/com/marklogic/hub/FlowManager.java‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -223,7 +223,7 @@ public void runFlow(Flow flow, int batchSize, JobExecutionListener listener) {
223223

224224
public void runInputFlow(Flow flow, HubConfig config) {
225225
try {
226-
Mlcp mlcp = new Mlcp(config.getHost(), config.getPort(), config.getAdminUsername(), config.getAdminPassword());
226+
Mlcp mlcp = new Mlcp(config.getHost(), config.getStagingPort(), config.getAdminUsername(), config.getAdminPassword());
227227
SourceOptions sourceOptions = new SourceOptions(flow.getDomainName(), flow.getName(), FlowType.INPUT.toString());
228228
mlcp.addSourceDirectory(config.getModulesPath(), sourceOptions);
229229
mlcp.loadContent();

‎data-hub/src/main/java/com/marklogic/hub/HubConfig.java‎

Lines changed: 20 additions & 7 deletions
Original file line numberDiff line numberDiff line change
@@ -20,15 +20,17 @@ public class HubConfig {
2020
public static final String DEFAULT_USERNAME = "admin";
2121
public static final String DEFAULT_PASSWORD = "admin";
2222
public static final String DEFAULT_HOST = "localhost";
23-
public static final Integer DEFAULT_PORT = 8010;
23+
public static final Integer DEFAULT_STAGING_PORT = 8010;
24+
public static final Integer DEFAULT_FINAL_PORT = 8011;
2425
public static final String DEFAULT_APP_NAME = "my-data-hub";
2526
public final static String DEFAULT_MODULES_PATH = "src/data-hub";
2627

2728
private String name = DEFAULT_APP_NAME;
2829
private String adminUsername = DEFAULT_USERNAME;
2930
private String adminPassword = DEFAULT_PASSWORD;
3031
private String host = DEFAULT_HOST;
31-
private Integer port = DEFAULT_PORT;
32+
private Integer stagingPort = DEFAULT_STAGING_PORT;
33+
private Integer finalPort = DEFAULT_FINAL_PORT;
3234
private String modulesPath;
3335

3436
public HubConfig() {
@@ -85,14 +87,25 @@ public void setAdminPassword(String password) {
8587
}
8688

8789
/**
88-
* @return the port of the REST API server used for loading modules
90+
* @return the port of the Staging REST API server used for loading modules
8991
*/
90-
public Integer getPort() {
91-
return port;
92+
public Integer getStagingPort() {
93+
return stagingPort;
9294
}
9395

94-
public void setPort(Integer port) {
95-
this.port = port;
96+
public void setStagingPort(Integer port) {
97+
this.stagingPort = port;
98+
}
99+
100+
/**
101+
* @return the port of the Final REST API server used for loading modules
102+
*/
103+
public Integer getFinalPort() {
104+
return finalPort;
105+
}
106+
107+
public void setFinalPort(Integer port) {
108+
this.finalPort = port;
96109
}
97110

98111
public String getModulesPath() {

0 commit comments

Comments
 (0)