Skip to content

Commit 76f88b9

Browse files
paxtonharepaxtonhare
committed
fixed #199
- created ML stuff (http server, db, forests) - updated all routes (gradle, quick start) to create tracing - enabled tracing from xquery - updated DataHub code to allow arbitrary changing of http/db names - added config.xqy that can tell xqy code the name of the staging, final, tracing databases. This file gets dynamically written with appropriate values on hub deploy
1 parent c55829d commit 76f88b9

File tree

32 files changed

+873
-320
lines changed

32 files changed

+873
-320
lines changed

‎examples/gradle-advanced/build.gradle‎

Lines changed: 7 additions & 4 deletions
Original file line numberDiff line numberDiff line change
@@ -31,19 +31,19 @@ ext {
3131
modulesDatabaseName = mlModulesDbName
3232
triggersDatabaseName = mlTriggersDbName
3333
schemasDatabaseName = mlSchemasDbName
34-
restPort = Integer.parseInt(mlStagingAppserverPort)
34+
restPort = Integer.parseInt(mlStagingPort)
3535

3636
// Configure custom tokens for our json files
3737
customTokens.put("%%STAGING_SERVER_NAME%%", mlStagingAppserverName)
38-
customTokens.put("%%STAGING_SERVER_PORT%%", mlStagingAppserverPort)
38+
customTokens.put("%%STAGING_SERVER_PORT%%", mlStagingPort)
3939
customTokens.put("%%STAGING_DB_NAME%%", mlStagingDbName)
4040

4141
customTokens.put("%%FINAL_SERVER_NAME%%", mlFinalAppserverName)
42-
customTokens.put("%%FINAL_SERVER_PORT%%", mlFinalAppserverPort)
42+
customTokens.put("%%FINAL_SERVER_PORT%%", mlFinalPort)
4343
customTokens.put("%%FINAL_DB_NAME%%", mlFinalDbName)
4444

4545
customTokens.put("%%TRACE_SERVER_NAME%%", mlTraceAppserverName)
46-
customTokens.put("%%TRACE_SERVER_PORT%%", mlTraceAppserverPort)
46+
customTokens.put("%%TRACE_SERVER_PORT%%", mlTracePort)
4747
customTokens.put("%%TRACE_DB_NAME%%", mlTraceDbName)
4848

4949
customTokens.put("%%MODULES_DB_NAME%%", mlModulesDbName)
@@ -62,16 +62,19 @@ ext {
6262

6363
// install the staging database
6464
def stagingDbCommand = new com.marklogic.appdeployer.command.databases.DeployDatabaseCommand("staging-database.json")
65+
stagingDbCommand.setForestsPerHost(Integer.parseInt(mlStagingForestsPerHost));
6566
mlAppDeployer.commands.add(stagingDbCommand)
6667
mlDatabaseCommands.add(stagingDbCommand)
6768

6869
// install the final database
6970
def finalDbCommand = new com.marklogic.appdeployer.command.databases.DeployDatabaseCommand("final-database.json")
71+
finalDbCommand.setForestsPerHost(Integer.parseInt(mlFinalForestsPerHost));
7072
mlAppDeployer.commands.add(finalDbCommand)
7173
mlDatabaseCommands.add(finalDbCommand)
7274

7375
// install the trace database
7476
def traceDbCommand = new com.marklogic.appdeployer.command.databases.DeployDatabaseCommand("trace-database.json")
77+
traceDbCommand.setForestsPerHost(Integer.parseInt(mlTraceForestsPerHost));
7578
mlAppDeployer.commands.add(traceDbCommand)
7679
mlDatabaseCommands.add(traceDbCommand)
7780

‎examples/gradle-advanced/gradle.properties‎

Lines changed: 6 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -2,16 +2,19 @@ mlHost=localhost
22
mlAppName=gradle-advanced-example
33

44
mlStagingAppserverName=data-hub-STAGING
5-
mlStagingAppserverPort=8100
5+
mlStagingPort=8100
66
mlStagingDbName=data-hub-STAGING
7+
mlStagingForestsPerHost=4
78

89
mlFinalAppserverName=data-hub-FINAL
9-
mlFinalAppserverPort=8101
10+
mlFinalPort=8101
1011
mlFinalDbName=data-hub-FINAL
12+
mlFinalForestsPerHost=4
1113

1214
mlTraceAppserverName=data-hub-TRACING
13-
mlTraceAppserverPort=8102
15+
mlTracePort=8102
1416
mlTraceDbName=data-hub-TRACING
17+
mlTraceForestsPerHost=1
1518

1619
mlModulesDbName=data-hub-MODULES
1720
mlTriggersDbName=data-hub-TRIGGERS

‎marklogic-data-hub/src/main/java/com/marklogic/hub/DataHub.java‎

Lines changed: 78 additions & 69 deletions
Original file line numberDiff line numberDiff line change
@@ -26,6 +26,7 @@
2626
import java.util.ArrayList;
2727
import java.util.HashSet;
2828
import java.util.List;
29+
import java.util.Map;
2930
import java.util.Set;
3031

3132
import org.slf4j.Logger;
@@ -70,44 +71,29 @@
7071
public class DataHub {
7172

7273
static final private Logger LOGGER = LoggerFactory.getLogger(DataHub.class);
73-
static final public String STAGING_NAME = "data-hub-STAGING";
74-
static final public String FINAL_NAME = "data-hub-FINAL";
75-
static final public String TRACING_NAME = "data-hub-TRACING";
76-
static final public String MODULES_DB_NAME = "data-hub-modules";
74+
7775
private ManageConfig config;
7876
private ManageClient client;
79-
public static String HUB_NAME = "data-hub";
80-
public static int FORESTS_PER_HOST = 4;
81-
private String host;
82-
private int stagingRestPort;
83-
private int finalRestPort;
84-
private int tracingRestPort;
85-
private String username;
86-
private String password;
8777

8878
private File assetInstallTimeFile = new File("./assetInstallTime.properties");
79+
private HubConfig hubConfig;
8980

90-
private final static int DEFAULT_STAGING_REST_PORT = 8010;
91-
private final static int DEFAULT_FINAL_REST_PORT = 8011;
92-
private final static int DEFAULT_TRACING_REST_PORT = 8012;
93-
94-
public DataHub(HubConfig config) {
95-
this(config.getHost(), config.getStagingPort(), config.getFinalPort(), config.getTracePort(), config.getAdminUsername(), config.getAdminPassword());
81+
public DataHub(HubConfig hubConfig) {
82+
init(hubConfig);
9683
}
9784

9885
public DataHub(String host, String username, String password) {
99-
this(host, DEFAULT_STAGING_REST_PORT, DEFAULT_FINAL_REST_PORT, DEFAULT_TRACING_REST_PORT, username, password);
86+
hubConfig = new HubConfig();
87+
hubConfig.host = host;
88+
hubConfig.adminUsername = username;
89+
hubConfig.adminPassword = password;
90+
init(hubConfig);
10091
}
10192

102-
public DataHub(String host, int stagingRestPort, int finalRestPort, int tracingRestPort, String username, String password) {
103-
config = new ManageConfig(host, 8002, username, password);
93+
private void init(HubConfig hubConfig) {
94+
this.hubConfig = hubConfig;
95+
config = new ManageConfig(hubConfig.host, 8002, hubConfig.adminUsername, hubConfig.adminPassword);
10496
client = new ManageClient(config);
105-
this.host = host;
106-
this.stagingRestPort = stagingRestPort;
107-
this.finalRestPort = finalRestPort;
108-
this.tracingRestPort = tracingRestPort;
109-
this.username = username;
110-
this.password = password;
11197
}
11298

11399
public void setAssetInstallTimeFile(File assetInstallTimeFile) {
@@ -121,35 +107,47 @@ public void setAssetInstallTimeFile(File assetInstallTimeFile) {
121107
public boolean isInstalled() {
122108
ServerManager sm = new ServerManager(client);
123109
DatabaseManager dm = new DatabaseManager(client);
124-
boolean stagingAppServerExists = sm.exists(STAGING_NAME);
125-
boolean finalAppServerExists = sm.exists(FINAL_NAME);
126-
boolean appserversOk = (stagingAppServerExists && finalAppServerExists);
110+
boolean stagingAppServerExists = sm.exists(hubConfig.stagingHttpName);
111+
boolean finalAppServerExists = sm.exists(hubConfig.finalHttpName);
112+
boolean tracingAppServerExists = sm.exists(hubConfig.tracingHttpName);
113+
boolean appserversOk = (stagingAppServerExists && finalAppServerExists && tracingAppServerExists);
127114

128-
boolean stagingDbExists = dm.exists(STAGING_NAME);
129-
boolean finalDbExists = dm.exists(FINAL_NAME);
115+
boolean stagingDbExists = dm.exists(hubConfig.stagingDbName);
116+
boolean finalDbExists = dm.exists(hubConfig.finalDbName);
117+
boolean tracingDbExists = dm.exists(hubConfig.stagingDbName);
130118

131119
boolean stagingForestsExist = false;
132120
boolean finalForestsExist = false;
121+
boolean tracingForestsExist = false;
133122

134123
boolean stagingIndexesOn = false;
135124
boolean finalIndexesOn = false;
125+
boolean tracingIndexesOn = false;
136126

137127
if (stagingDbExists) {
138-
Fragment f = dm.getPropertiesAsXml(STAGING_NAME);
128+
Fragment f = dm.getPropertiesAsXml(hubConfig.stagingDbName);
139129
stagingIndexesOn = Boolean.parseBoolean(f.getElementValue("//m:triple-index"));
140130
stagingIndexesOn = stagingIndexesOn && Boolean.parseBoolean(f.getElementValue("//m:collection-lexicon"));
141-
stagingForestsExist = (dm.getForestIds(STAGING_NAME).size() == FORESTS_PER_HOST);
131+
stagingForestsExist = (dm.getForestIds(hubConfig.stagingDbName).size() == hubConfig.stagingForestsPerHost);
142132
}
143133

144134
if (finalDbExists) {
145-
Fragment f = dm.getPropertiesAsXml(FINAL_NAME);
135+
Fragment f = dm.getPropertiesAsXml(hubConfig.finalDbName);
146136
finalIndexesOn = Boolean.parseBoolean(f.getElementValue("//m:triple-index"));
147137
finalIndexesOn = finalIndexesOn && Boolean.parseBoolean(f.getElementValue("//m:collection-lexicon"));
148-
finalForestsExist = (dm.getForestIds(FINAL_NAME).size() == FORESTS_PER_HOST);
138+
finalForestsExist = (dm.getForestIds(hubConfig.finalDbName).size() == hubConfig.finalForestsPerHost);
139+
}
140+
141+
if (tracingDbExists) {
142+
tracingIndexesOn = true;
143+
int forests = dm.getForestIds(hubConfig.tracingDbName).size();
144+
tracingForestsExist = (forests == hubConfig.tracingForestsPerHost);
149145
}
146+
150147
boolean dbsOk = (stagingDbExists && stagingIndexesOn &&
151-
finalDbExists && finalIndexesOn);
152-
boolean forestsOk = (stagingForestsExist && finalForestsExist);
148+
finalDbExists && finalIndexesOn &&
149+
tracingDbExists && tracingIndexesOn);
150+
boolean forestsOk = (stagingForestsExist && finalForestsExist && tracingForestsExist);
153151

154152
return (appserversOk && dbsOk && forestsOk);
155153
}
@@ -161,9 +159,9 @@ public boolean isInstalled() {
161159
public void validateServer() throws ServerValidationException {
162160
try {
163161
AdminConfig adminConfig = new AdminConfig();
164-
adminConfig.setHost(host);
165-
adminConfig.setUsername(username);
166-
adminConfig.setPassword(password);
162+
adminConfig.setHost(hubConfig.host);
163+
adminConfig.setUsername(hubConfig.adminUsername);
164+
adminConfig.setPassword(hubConfig.adminPassword);
167165
AdminManager am = new AdminManager(adminConfig);
168166
String versionString = am.getServerVersion();
169167
int major = Integer.parseInt(versionString.substring(0, 1));
@@ -179,16 +177,26 @@ public void validateServer() throws ServerValidationException {
179177

180178
private AppConfig getAppConfig() throws IOException {
181179
AppConfig config = new AppConfig();
182-
config.setHost(host);
183-
config.setRestPort(stagingRestPort);
184-
config.setName(HUB_NAME);
185-
config.setRestAdminUsername(username);
186-
config.setRestAdminPassword(password);
180+
config.setHost(hubConfig.host);
181+
config.setRestPort(hubConfig.stagingPort);
182+
config.setName(hubConfig.name);
183+
config.setRestAdminUsername(hubConfig.adminUsername);
184+
config.setRestAdminPassword(hubConfig.adminPassword);
185+
config.setModulesDatabaseName(hubConfig.modulesDbName);
186+
187187
List<String> paths = new ArrayList<String>();
188188
paths.add(new ClassPathResource("ml-modules").getPath());
189+
189190
String configPath = new ClassPathResource("ml-config").getPath();
190191
config.setConfigDir(new ConfigDir(new File(configPath)));
191192
config.setModulePaths(paths);
193+
194+
Map<String, String> customTokens = config.getCustomTokens();
195+
customTokens.put("%%STAGING_DATABASE%%", hubConfig.stagingDbName);
196+
customTokens.put("%%FINAL_DATABASE%%", hubConfig.finalDbName);
197+
customTokens.put("%%TRACING_DATABASE%%", hubConfig.tracingDbName);
198+
customTokens.put("%%MODULES_DATABASE%%", hubConfig.modulesDbName);
199+
192200
return config;
193201
}
194202

@@ -197,6 +205,7 @@ private AppConfig getAppConfig() throws IOException {
197205
* @throws IOException
198206
*/
199207
public void install() throws IOException {
208+
LOGGER.debug("Installing the Data Hub into MarkLogic");
200209
// clean up any lingering cache for deployed modules
201210
PropertiesModuleManager moduleManager = new PropertiesModuleManager(this.assetInstallTimeFile);
202211
moduleManager.deletePropertiesFile();
@@ -210,11 +219,11 @@ public void install() throws IOException {
210219

211220
private DatabaseClient getDatabaseClient(int port) {
212221
AppConfig config = new AppConfig();
213-
config.setHost(host);
214-
config.setName(HUB_NAME);
215-
config.setRestAdminUsername(username);
216-
config.setRestAdminPassword(password);
217-
DatabaseClient client = DatabaseClientFactory.newClient(host, port, username, password,
222+
config.setHost(hubConfig.host);
223+
config.setName(hubConfig.name);
224+
config.setRestAdminUsername(hubConfig.adminUsername);
225+
config.setRestAdminPassword(hubConfig.adminPassword);
226+
DatabaseClient client = DatabaseClientFactory.newClient(hubConfig.host, port, hubConfig.adminUsername, hubConfig.adminPassword,
218227
config.getRestAuthentication(), config.getRestSslContext(), config.getRestSslHostnameVerifier());
219228
return client;
220229
}
@@ -229,15 +238,12 @@ private DatabaseClient getDatabaseClient(int port) {
229238
* @throws IOException
230239
*/
231240
public Set<File> installUserModules(String pathToUserModules) throws IOException {
232-
AppConfig config = new AppConfig();
233-
config.setHost(host);
234-
config.setRestPort(finalRestPort);
235-
config.setName(HUB_NAME);
236-
config.setRestAdminUsername(username);
237-
config.setRestAdminPassword(password);
241+
LOGGER.debug("Installing user modules into MarkLogic");
242+
243+
AppConfig config = getAppConfig();
238244

239-
DatabaseClient stagingClient = getDatabaseClient(stagingRestPort);
240-
DatabaseClient finalClient = getDatabaseClient(finalRestPort);
245+
DatabaseClient stagingClient = getDatabaseClient(hubConfig.stagingPort);
246+
DatabaseClient finalClient = getDatabaseClient(hubConfig.finalPort);
241247

242248

243249
Set<File> loadedFiles = new HashSet<File>();
@@ -279,7 +285,8 @@ else if (isConformanceDir) {
279285
}
280286

281287
public JsonNode validateUserModules() {
282-
DatabaseClient client = getDatabaseClient(stagingRestPort);
288+
LOGGER.debug("validating user modules");
289+
DatabaseClient client = getDatabaseClient(hubConfig.stagingPort);
283290
EntitiesValidator ev = new EntitiesValidator(client);
284291
return ev.validate();
285292
}
@@ -295,26 +302,27 @@ private List<Command> getCommands(AppConfig config) {
295302

296303
// Databases
297304
List<Command> dbCommands = new ArrayList<Command>();
298-
DeployHubDatabaseCommand staging = new DeployHubDatabaseCommand(STAGING_NAME);
299-
staging.setForestsPerHost(FORESTS_PER_HOST);
305+
DeployHubDatabaseCommand staging = new DeployHubDatabaseCommand(hubConfig.stagingDbName);
306+
staging.setForestsPerHost(hubConfig.stagingForestsPerHost);
300307
dbCommands.add(staging);
301308

302-
DeployHubDatabaseCommand finalDb = new DeployHubDatabaseCommand(FINAL_NAME);
303-
finalDb.setForestsPerHost(FORESTS_PER_HOST);
309+
DeployHubDatabaseCommand finalDb = new DeployHubDatabaseCommand(hubConfig.finalDbName);
310+
finalDb.setForestsPerHost(hubConfig.finalForestsPerHost);
304311
dbCommands.add(finalDb);
305312

306-
DeployHubDatabaseCommand tracingDb = new DeployHubDatabaseCommand(TRACING_NAME);
313+
DeployHubDatabaseCommand tracingDb = new DeployHubDatabaseCommand(hubConfig.tracingDbName);
314+
tracingDb.setForestsPerHost(hubConfig.tracingForestsPerHost);
307315
dbCommands.add(tracingDb);
308316

309-
dbCommands.add(new DeployModulesDatabaseCommand(MODULES_DB_NAME));
317+
dbCommands.add(new DeployModulesDatabaseCommand(hubConfig.modulesDbName));
310318
dbCommands.add(new DeployTriggersDatabaseCommand());
311319
dbCommands.add(new DeploySchemasDatabaseCommand());
312320
commands.addAll(dbCommands);
313321

314322
// App Servers
315-
commands.add(new DeployRestApiCommand(STAGING_NAME, stagingRestPort));
316-
commands.add(new DeployRestApiCommand(FINAL_NAME, finalRestPort));
317-
commands.add(new DeployRestApiCommand(TRACING_NAME, tracingRestPort));
323+
commands.add(new DeployRestApiCommand(hubConfig.stagingHttpName, hubConfig.stagingPort));
324+
commands.add(new DeployRestApiCommand(hubConfig.finalHttpName, hubConfig.finalPort));
325+
commands.add(new DeployRestApiCommand(hubConfig.tracingHttpName, hubConfig.tracePort));
318326

319327
// Modules
320328
commands.add(new LoadModulesCommand());
@@ -329,6 +337,7 @@ private List<Command> getCommands(AppConfig config) {
329337
* @throws IOException
330338
*/
331339
public void uninstall() throws IOException {
340+
LOGGER.debug("Uninstalling the Data Hub from MarkLogic");
332341
AdminManager manager = new AdminManager();
333342
AppConfig config = getAppConfig();
334343
SimpleAppDeployer deployer = new SimpleAppDeployer(client, manager);

0 commit comments

Comments
 (0)