Skip to content

Commit 181ecb4

Browse files
paxtonharepaxtonhare
committed
enhance performance of hub install
1 parent 6db9add commit 181ecb4

File tree

5 files changed

+71
-38
lines changed

5 files changed

+71
-38
lines changed

‎marklogic-data-hub/src/main/java/com/marklogic/hub/DataHub.java‎

Lines changed: 40 additions & 31 deletions
Original file line numberDiff line numberDiff line change
@@ -40,8 +40,6 @@
4040
import com.marklogic.appdeployer.command.Command;
4141
import com.marklogic.appdeployer.command.modules.AllButAssetsModulesFinder;
4242
import com.marklogic.appdeployer.command.modules.AssetModulesFinder;
43-
import com.marklogic.appdeployer.command.security.DeployRolesCommand;
44-
import com.marklogic.appdeployer.command.security.DeployUsersCommand;
4543
import com.marklogic.appdeployer.impl.SimpleAppDeployer;
4644
import com.marklogic.client.DatabaseClient;
4745
import com.marklogic.client.DatabaseClientFactory;
@@ -66,6 +64,7 @@
6664
import com.marklogic.mgmt.appservers.ServerManager;
6765
import com.marklogic.mgmt.databases.DatabaseManager;
6866
import com.marklogic.rest.util.Fragment;
67+
import com.marklogic.rest.util.ResourcesFragment;
6968

7069
public class DataHub {
7170

@@ -106,14 +105,17 @@ public void setAssetInstallTimeFile(File assetInstallTimeFile) {
106105
public boolean isInstalled() {
107106
ServerManager sm = new ServerManager(client);
108107
DatabaseManager dm = new DatabaseManager(client);
109-
boolean stagingAppServerExists = sm.exists(hubConfig.stagingHttpName);
110-
boolean finalAppServerExists = sm.exists(hubConfig.finalHttpName);
111-
boolean tracingAppServerExists = sm.exists(hubConfig.tracingHttpName);
108+
109+
ResourcesFragment srf = sm.getAsXml();
110+
boolean stagingAppServerExists = srf.resourceExists(hubConfig.stagingHttpName);
111+
boolean finalAppServerExists = srf.resourceExists(hubConfig.finalHttpName);
112+
boolean tracingAppServerExists = srf.resourceExists(hubConfig.tracingHttpName);
112113
boolean appserversOk = (stagingAppServerExists && finalAppServerExists && tracingAppServerExists);
113114

114-
boolean stagingDbExists = dm.exists(hubConfig.stagingDbName);
115-
boolean finalDbExists = dm.exists(hubConfig.finalDbName);
116-
boolean tracingDbExists = dm.exists(hubConfig.stagingDbName);
115+
ResourcesFragment drf = dm.getAsXml();
116+
boolean stagingDbExists = drf.resourceExists(hubConfig.stagingDbName);
117+
boolean finalDbExists = drf.resourceExists(hubConfig.finalDbName);
118+
boolean tracingDbExists = drf.resourceExists(hubConfig.stagingDbName);
117119

118120
boolean stagingForestsExist = false;
119121
boolean finalForestsExist = false;
@@ -127,20 +129,20 @@ public boolean isInstalled() {
127129
Fragment f = dm.getPropertiesAsXml(hubConfig.stagingDbName);
128130
stagingIndexesOn = Boolean.parseBoolean(f.getElementValue("//m:triple-index"));
129131
stagingIndexesOn = stagingIndexesOn && Boolean.parseBoolean(f.getElementValue("//m:collection-lexicon"));
130-
stagingForestsExist = (dm.getForestIds(hubConfig.stagingDbName).size() == hubConfig.stagingForestsPerHost);
132+
stagingForestsExist = (f.getElements("//m:forest").size() == hubConfig.stagingForestsPerHost);
131133
}
132134

133135
if (finalDbExists) {
134136
Fragment f = dm.getPropertiesAsXml(hubConfig.finalDbName);
135137
finalIndexesOn = Boolean.parseBoolean(f.getElementValue("//m:triple-index"));
136138
finalIndexesOn = finalIndexesOn && Boolean.parseBoolean(f.getElementValue("//m:collection-lexicon"));
137-
finalForestsExist = (dm.getForestIds(hubConfig.finalDbName).size() == hubConfig.finalForestsPerHost);
139+
finalForestsExist = (f.getElements("//m:forest").size() == hubConfig.finalForestsPerHost);
138140
}
139141

140142
if (tracingDbExists) {
141143
tracingIndexesOn = true;
142-
int forests = dm.getForestIds(hubConfig.tracingDbName).size();
143-
tracingForestsExist = (forests == hubConfig.tracingForestsPerHost);
144+
Fragment f = dm.getPropertiesAsXml(hubConfig.tracingDbName);
145+
tracingForestsExist = (f.getElements("//m:forest").size() == hubConfig.tracingForestsPerHost);
144146
}
145147

146148
boolean dbsOk = (stagingDbExists && stagingIndexesOn &&
@@ -157,11 +159,7 @@ public boolean isInstalled() {
157159
*/
158160
public void validateServer() throws ServerValidationException {
159161
try {
160-
AdminConfig adminConfig = new AdminConfig();
161-
adminConfig.setHost(hubConfig.host);
162-
adminConfig.setUsername(hubConfig.adminUsername);
163-
adminConfig.setPassword(hubConfig.adminPassword);
164-
AdminManager am = new AdminManager(adminConfig);
162+
AdminManager am = getAdminManager();
165163
String versionString = am.getServerVersion();
166164
int major = Integer.parseInt(versionString.substring(0, 1));
167165
int minor = Integer.parseInt(versionString.substring(2, 3) + versionString.substring(4, 5));
@@ -204,16 +202,21 @@ private AppConfig getAppConfig() throws IOException {
204202
* @throws IOException
205203
*/
206204
public void install() throws IOException {
205+
long startTime = System.nanoTime();
207206
LOGGER.debug("Installing the Data Hub into MarkLogic");
207+
208208
// clean up any lingering cache for deployed modules
209209
PropertiesModuleManager moduleManager = new PropertiesModuleManager(this.assetInstallTimeFile);
210210
moduleManager.deletePropertiesFile();
211211

212-
AdminManager manager = new AdminManager();
213212
AppConfig config = getAppConfig();
214-
SimpleAppDeployer deployer = new SimpleAppDeployer(client, manager);
213+
SimpleAppDeployer deployer = new SimpleAppDeployer(client, getAdminManager());
215214
deployer.setCommands(getCommands(config));
215+
216216
deployer.deploy(config);
217+
long endTime = System.nanoTime();
218+
long duration = (endTime - startTime);
219+
LOGGER.info("Install took: " + (duration / 1000000000) + " seconds");
217220
}
218221

219222
private DatabaseClient getDatabaseClient(int port) {
@@ -293,12 +296,6 @@ public JsonNode validateUserModules() {
293296
private List<Command> getCommands(AppConfig config) {
294297
List<Command> commands = new ArrayList<Command>();
295298

296-
// Security
297-
List<Command> securityCommands = new ArrayList<Command>();
298-
securityCommands.add(new DeployRolesCommand());
299-
securityCommands.add(new DeployUsersCommand());
300-
commands.addAll(securityCommands);
301-
302299
// Databases
303300
List<Command> dbCommands = new ArrayList<Command>();
304301
DeployHubDatabaseCommand staging = new DeployHubDatabaseCommand(hubConfig.stagingDbName);
@@ -317,9 +314,9 @@ private List<Command> getCommands(AppConfig config) {
317314
commands.addAll(dbCommands);
318315

319316
// App Servers
320-
commands.add(new DeployRestApiCommand(hubConfig.stagingHttpName, hubConfig.stagingPort));
321-
commands.add(new DeployRestApiCommand(hubConfig.finalHttpName, hubConfig.finalPort));
322-
commands.add(new DeployRestApiCommand(hubConfig.tracingHttpName, hubConfig.tracePort));
317+
commands.add(new DeployRestApiCommand(hubConfig.stagingHttpName, hubConfig.stagingPort, hubConfig.stagingDbName, hubConfig.stagingForestsPerHost));
318+
commands.add(new DeployRestApiCommand(hubConfig.finalHttpName, hubConfig.finalPort, hubConfig.finalDbName, hubConfig.finalForestsPerHost));
319+
commands.add(new DeployRestApiCommand(hubConfig.tracingHttpName, hubConfig.tracePort, hubConfig.tracingDbName, hubConfig.tracingForestsPerHost));
323320

324321
commands.add(new UpdateRestApiServersCommand(hubConfig.stagingHttpName));
325322
commands.add(new UpdateRestApiServersCommand(hubConfig.finalHttpName));
@@ -331,23 +328,35 @@ private List<Command> getCommands(AppConfig config) {
331328
return commands;
332329
}
333330

334-
331+
private AdminManager getAdminManager() {
332+
AdminConfig adminConfig = new AdminConfig();
333+
adminConfig.setHost(hubConfig.host);
334+
adminConfig.setUsername(hubConfig.adminUsername);
335+
adminConfig.setPassword(hubConfig.adminPassword);
336+
AdminManager manager = new AdminManager(adminConfig);
337+
return manager;
338+
}
335339

336340
/**
337341
* Uninstalls the data hub configuration and server-side modules from MarkLogic
338342
* @throws IOException
339343
*/
340344
public void uninstall() throws IOException {
345+
long startTime = System.nanoTime();
341346
LOGGER.debug("Uninstalling the Data Hub from MarkLogic");
342-
AdminManager manager = new AdminManager();
343347
AppConfig config = getAppConfig();
344-
SimpleAppDeployer deployer = new SimpleAppDeployer(client, manager);
348+
AdminManager adminManager = getAdminManager();
349+
adminManager.setWaitForRestartCheckInterval(250);
350+
SimpleAppDeployer deployer = new SimpleAppDeployer(client, adminManager);
345351
deployer.setCommands(getCommands(config));
346352
deployer.undeploy(config);
347353

348354
// clean up any lingering cache for deployed modules
349355
PropertiesModuleManager moduleManager = new PropertiesModuleManager(this.assetInstallTimeFile);
350356
moduleManager.deletePropertiesFile();
357+
long endTime = System.nanoTime();
358+
long duration = (endTime - startTime);
359+
LOGGER.info("Uninstall took: " + (duration / 1000000000) + " seconds");
351360
}
352361

353362
class EntitiesValidator extends ResourceManager {

‎marklogic-data-hub/src/main/java/com/marklogic/hub/commands/DeployHubDatabaseCommand.java‎

Lines changed: 5 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -14,7 +14,11 @@ public DeployHubDatabaseCommand(String databaseName) {
1414
super();
1515
this.setDatabaseName(databaseName);
1616
this.setCreateDatabaseWithoutFile(true);
17-
this.setExecuteSortOrder(SortOrderConstants.DEPLOY_CONTENT_DATABASES);
17+
this.setExecuteSortOrder(SortOrderConstants.DEPLOY_OTHER_SERVERS);
18+
}
19+
20+
@Override
21+
public void undo(CommandContext context) {
1822
}
1923

2024
@Override

‎marklogic-data-hub/src/main/java/com/marklogic/hub/commands/DeployModulesDatabaseCommand.java‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -9,7 +9,7 @@ public DeployModulesDatabaseCommand(String databaseName) {
99
super();
1010
this.setDatabaseName(databaseName);
1111
this.setCreateDatabaseWithoutFile(true);
12-
this.setExecuteSortOrder(SortOrderConstants.DEPLOY_OTHER_DATABASES);
12+
this.setExecuteSortOrder(SortOrderConstants.DEPLOY_OTHER_SERVERS);
1313
}
1414

1515
@Override

‎marklogic-data-hub/src/main/java/com/marklogic/hub/commands/DeployRestApiCommand.java‎

Lines changed: 8 additions & 4 deletions
Original file line numberDiff line numberDiff line change
@@ -20,13 +20,17 @@ public class DeployRestApiCommand extends AbstractCommand implements UndoableCom
2020

2121
private String name;
2222
private Integer port;
23+
private String databaseName;
24+
private int forestsPerHost;
2325
private boolean deleteModulesDatabase = false;
24-
private boolean deleteContentDatabase = false;
26+
private boolean deleteContentDatabase = true;
2527

26-
public DeployRestApiCommand(String name, int port) {
28+
public DeployRestApiCommand(String name, int port, String databaseName, int forestsPerHost) {
2729
super();
2830
this.name = name;
2931
this.port = port;
32+
this.databaseName = databaseName;
33+
this.forestsPerHost = forestsPerHost;
3034
setExecuteSortOrder(SortOrderConstants.DEPLOY_REST_API_SERVERS);
3135
}
3236

@@ -65,11 +69,11 @@ private String buildDefaultRestApiJson(AppConfig config) {
6569
ObjectNode n = node.putObject("rest-api");
6670
n.put("name", name);
6771
n.put("group", config.getGroupName());
68-
n.put("database", name);
72+
n.put("database", databaseName);
6973
n.put("modules-database", config.getModulesDatabaseName());
7074
n.put("port", "%%PORT%%");
7175
n.put("xdbc-enabled", true);
72-
n.put("forests-per-host", config.getContentForestsPerHost());
76+
n.put("forests-per-host", forestsPerHost);
7377
n.put("error-format", "json");
7478

7579
try {

‎marklogic-data-hub/src/main/java/com/marklogic/hub/commands/LoadModulesCommand.java‎

Lines changed: 17 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -29,6 +29,7 @@
2929
import com.marklogic.xcc.DocumentFormat;
3030
import com.marklogic.xcc.SecurityOptions;
3131
import com.marklogic.xcc.Session;
32+
import com.marklogic.xcc.Session.TransactionMode;
3233
import com.marklogic.xcc.exceptions.RequestException;
3334

3435
public class LoadModulesCommand extends AbstractCommand {
@@ -104,6 +105,7 @@ protected void initializeActiveSession(CommandContext context) {
104105
ContentSource cs = ContentSourceFactory.newContentSource(config.getHost(), port, config.getRestAdminUsername(), config.getRestAdminPassword(), config.getModulesDatabaseName(),
105106
securityOptions);
106107
activeSession = cs.newSession();
108+
activeSession.setTransactionMode(TransactionMode.UPDATE);
107109
}
108110

109111
@Override
@@ -136,20 +138,34 @@ public void execute(CommandContext context) {
136138

137139
loadFile(path, r.getInputStream(), appConfig);
138140
}
141+
activeSession.commit();
139142

143+
logger.info("Loading Service Extensions");
144+
long startTime = System.nanoTime();
140145
resources = findResources("classpath:/ml-modules/services", "/**/*.xq*");
141146
for (Resource r : resources) {
142147
ExtensionMetadataAndParams emap = extensionMetadataProvider.provideExtensionMetadataAndParams(r);
143148
this.modulesLoader.installService(r, emap.metadata, emap.methods.toArray(new MethodParameters[] {}));
144149
}
145150

151+
long endTime = System.nanoTime();
152+
long duration = (endTime - startTime);
153+
logger.info("Service Extensions took: " + (duration / 1000000000) + " seconds");
154+
155+
logger.info("Loading Rest Transforms");
156+
startTime = System.nanoTime();
146157
resources = findResources("classpath:/ml-modules/transforms", "/**/*.xq*");
147158
for (Resource r : resources) {
148159
ExtensionMetadataAndParams emap = extensionMetadataProvider.provideExtensionMetadataAndParams(r);
149160
this.modulesLoader.installTransform(r, emap.metadata);
150161
}
162+
endTime = System.nanoTime();
163+
duration = (endTime - startTime);
164+
logger.info("Rest Transforms took: " + (duration / 1000000000) + " seconds");
165+
166+
logger.info("Finished Loading Modules");
151167
}
152-
catch (IOException e) {
168+
catch (IOException | RequestException e) {
153169
e.printStackTrace();
154170
}
155171
}

0 commit comments

Comments
 (0)