Skip to content

Commit 3ed1257

Browse files
rahulvudutalaSanjeevani19
rahulvudutala
authored and
Sanjeevani19
committed
DHFPROD-9901: Install Datahub on Marklogic Cloud
1 parent 73185d3 commit 3ed1257

File tree

5 files changed

+128
-4
lines changed

5 files changed

+128
-4
lines changed

‎marklogic-data-hub-api/src/main/java/com/marklogic/hub/HubClient.java‎

Lines changed: 2 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -30,6 +30,8 @@ static HubClient withHubClientConfig(HubClientConfig hubClientConfig) {
3030
*/
3131
String getUsername();
3232

33+
String getMlAuthentication();
34+
3335
DatabaseClient getStagingClient();
3436

3537
DatabaseClient getStagingClient(String databaseName);

‎marklogic-data-hub-api/src/main/java/com/marklogic/hub/HubClientConfig.java‎

Lines changed: 95 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -43,9 +43,12 @@ public class HubClientConfig {
4343
private String host;
4444
private String username;
4545
private String password;
46+
private String cloudApiKey;
47+
private String mlAuthentication;
4648

4749
private String stagingDbName;
4850
private Integer stagingPort;
51+
private String stagingBasePath;
4952
private String stagingAuthMethod;
5053
private Boolean stagingSimpleSsl;
5154
private SSLContext stagingSslContext;
@@ -57,6 +60,7 @@ public class HubClientConfig {
5760

5861
private String finalDbName;
5962
private Integer finalPort;
63+
private String finalBasePath;
6064
private String finalAuthMethod;
6165
private Boolean finalSimpleSsl;
6266
private SSLContext finalSslContext;
@@ -68,6 +72,7 @@ public class HubClientConfig {
6872

6973
private String jobDbName;
7074
private Integer jobPort;
75+
private String jobBasePath;
7176
private String jobAuthMethod;
7277
private Boolean jobSimpleSsl;
7378
private SSLContext jobSslContext;
@@ -154,6 +159,7 @@ public void applyProperties(Function<String, String> propertySource, ManageConfi
154159
if (propertyConsumerMap == null) {
155160
initializePropertyConsumerMap();
156161
}
162+
157163
for (Map.Entry<String, Consumer<String>> propertyEntry : propertyConsumerMap.entrySet()) {
158164
String value = propertySource.apply(propertyEntry.getKey());
159165
if (value != null) {
@@ -197,6 +203,12 @@ public DatabaseClient newStagingClient(String dbName) {
197203
config.setCertPassword(stagingCertPassword);
198204
config.setExternalName(stagingExternalName);
199205
config.setTrustManager(stagingTrustManager);
206+
if(mlAuthentication.equals("cloud")) {
207+
config.setSecurityContextType(SecurityContextType.CLOUD);
208+
config.setCloudApiKey(cloudApiKey);
209+
config.setBasePath(stagingBasePath);
210+
config.setPort(443);
211+
}
200212
if (isHostLoadBalancer) {
201213
config.setConnectionType(DatabaseClient.ConnectionType.GATEWAY);
202214
}
@@ -215,6 +227,12 @@ public DatabaseClient newFinalClient(String dbName) {
215227
config.setCertPassword(finalCertPassword);
216228
config.setExternalName(finalExternalName);
217229
config.setTrustManager(finalTrustManager);
230+
if(mlAuthentication.equals("cloud")) {
231+
config.setSecurityContextType(SecurityContextType.CLOUD);
232+
config.setCloudApiKey(cloudApiKey);
233+
config.setBasePath(finalBasePath);
234+
config.setPort(443);
235+
}
218236
if (isHostLoadBalancer) {
219237
config.setConnectionType(DatabaseClient.ConnectionType.GATEWAY);
220238
}
@@ -223,13 +241,20 @@ public DatabaseClient newFinalClient(String dbName) {
223241

224242
public DatabaseClient newJobDbClient() {
225243
DatabaseClientConfig config = new DatabaseClientConfig(host, jobPort, username, password);
244+
226245
config.setSecurityContextType(SecurityContextType.valueOf(jobAuthMethod.toUpperCase()));
227246
config.setSslHostnameVerifier(jobSslHostnameVerifier);
228247
config.setSslContext(jobSslContext);
229248
config.setCertFile(jobCertFile);
230249
config.setCertPassword(jobCertPassword);
231250
config.setExternalName(jobExternalName);
232251
config.setTrustManager(jobTrustManager);
252+
if(mlAuthentication.equals("cloud")) {
253+
config.setSecurityContextType(SecurityContextType.CLOUD);
254+
config.setCloudApiKey(cloudApiKey);
255+
config.setBasePath(jobBasePath);
256+
config.setPort(443);
257+
}
233258
if (isHostLoadBalancer) {
234259
config.setConnectionType(DatabaseClient.ConnectionType.GATEWAY);
235260
}
@@ -259,9 +284,11 @@ public void applyDefaultPropertyValues() {
259284
host = "localhost";
260285
isHostLoadBalancer = false;
261286
manageConfig = null;
287+
mlAuthentication = "local";
262288

263289
stagingDbName = "data-hub-STAGING";
264290
stagingPort = 8010;
291+
stagingBasePath = "";
265292
stagingAuthMethod = "digest";
266293
stagingSimpleSsl = false;
267294
stagingSslContext = null;
@@ -273,6 +300,7 @@ public void applyDefaultPropertyValues() {
273300

274301
finalDbName = "data-hub-FINAL";
275302
finalPort = 8011;
303+
finalBasePath = "";
276304
finalAuthMethod = "digest";
277305
finalSimpleSsl = false;
278306
finalSslContext = null;
@@ -284,6 +312,7 @@ public void applyDefaultPropertyValues() {
284312

285313
jobDbName = "data-hub-JOBS";
286314
jobPort = 8013;
315+
jobBasePath = "";
287316
jobAuthMethod = "digest";
288317
jobSimpleSsl = false;
289318
jobSslContext = null;
@@ -347,7 +376,13 @@ protected void initializePropertyConsumerMap() {
347376

348377
propertyConsumerMap.put("mlUsername", prop -> username = prop);
349378
propertyConsumerMap.put("mlPassword", prop -> password = prop);
350-
379+
propertyConsumerMap.put("mlCloudApiKey", prop -> cloudApiKey = prop);
380+
propertyConsumerMap.put("mlAuthentication", prop -> {
381+
if(prop.equals("cloud")) {
382+
initializeSslConfigForCloud();
383+
}
384+
mlAuthentication = prop;
385+
});
351386
propertyConsumerMap.put("mlDHFVersion", prop -> logger.warn("mlDHFVersion no longer has any impact " +
352387
"starting in version 5.3.0. You may safely remove this from your properties file."));
353388

@@ -359,6 +394,7 @@ protected void initializePropertyConsumerMap() {
359394

360395
propertyConsumerMap.put("mlStagingDbName", prop -> stagingDbName = prop);
361396
propertyConsumerMap.put("mlStagingPort", prop -> stagingPort = Integer.parseInt(prop));
397+
propertyConsumerMap.put("mlStagingBasePath", prop -> stagingBasePath = prop);
362398
propertyConsumerMap.put("mlStagingAuth", prop -> stagingAuthMethod = prop);
363399
propertyConsumerMap.put("mlStagingSimpleSsl", prop -> stagingSimpleSsl = Boolean.parseBoolean(prop));
364400
propertyConsumerMap.put("mlStagingCertFile", prop -> stagingCertFile = prop);
@@ -367,6 +403,7 @@ protected void initializePropertyConsumerMap() {
367403

368404
propertyConsumerMap.put("mlFinalDbName", prop -> finalDbName = prop);
369405
propertyConsumerMap.put("mlFinalPort", prop -> finalPort = Integer.parseInt(prop));
406+
propertyConsumerMap.put("mlFinalBasePath", prop -> finalBasePath = prop);
370407
propertyConsumerMap.put("mlFinalAuth", prop -> finalAuthMethod = prop);
371408
propertyConsumerMap.put("mlFinalSimpleSsl", prop -> finalSimpleSsl = Boolean.parseBoolean(prop));
372409
propertyConsumerMap.put("mlFinalCertFile", prop -> finalCertFile = prop);
@@ -375,6 +412,7 @@ protected void initializePropertyConsumerMap() {
375412

376413
propertyConsumerMap.put("mlJobDbName", prop -> jobDbName = prop);
377414
propertyConsumerMap.put("mlJobPort", prop -> jobPort = Integer.parseInt(prop));
415+
propertyConsumerMap.put("mlJobBasePath", prop -> jobBasePath = prop);
378416
propertyConsumerMap.put("mlJobAuth", prop -> jobAuthMethod = prop);
379417
propertyConsumerMap.put("mlJobSimpleSsl", prop -> jobSimpleSsl = Boolean.parseBoolean(prop));
380418
propertyConsumerMap.put("mlJobCertFile", prop -> jobCertFile = prop);
@@ -429,6 +467,22 @@ private void instantiateSslObjects() {
429467
}
430468
}
431469

470+
private void initializeSslConfigForCloud() {
471+
SSLContext sslContext = SimpleX509TrustManager.newSSLContext();
472+
stagingSslContext = sslContext;
473+
finalSslContext = sslContext;
474+
jobSslContext = sslContext;
475+
476+
SimpleX509TrustManager trustManager = new SimpleX509TrustManager();
477+
stagingTrustManager = trustManager;
478+
finalTrustManager = trustManager;
479+
jobTrustManager = trustManager;
480+
481+
stagingSslHostnameVerifier = DatabaseClientFactory.SSLHostnameVerifier.ANY;
482+
finalSslHostnameVerifier = DatabaseClientFactory.SSLHostnameVerifier.ANY;
483+
jobSslHostnameVerifier = DatabaseClientFactory.SSLHostnameVerifier.ANY;
484+
}
485+
432486
public void setConfiguredDatabaseClientFactory(ConfiguredDatabaseClientFactory configuredDatabaseClientFactory) {
433487
this.configuredDatabaseClientFactory = configuredDatabaseClientFactory;
434488
}
@@ -458,6 +512,22 @@ public void setPassword(String password) {
458512
this.password = password;
459513
}
460514

515+
public String getCloudApiKey() {
516+
return cloudApiKey;
517+
}
518+
519+
public void setCloudApiKey(String cloudApiKey) {
520+
this.cloudApiKey = cloudApiKey;
521+
}
522+
523+
public String getMlAuthentication() {
524+
return mlAuthentication;
525+
}
526+
527+
public void setMlAuthentication(String mlAuthentication) {
528+
this.mlAuthentication = mlAuthentication;
529+
}
530+
461531
public Integer getStagingPort() {
462532
return stagingPort;
463533
}
@@ -466,6 +536,14 @@ public void setStagingPort(Integer stagingPort) {
466536
this.stagingPort = stagingPort;
467537
}
468538

539+
public String getStagingBasePath() {
540+
return stagingBasePath;
541+
}
542+
543+
public void setStagingBasePath(String stagingBasePath) {
544+
this.stagingBasePath = stagingBasePath;
545+
}
546+
469547
public String getStagingAuthMethod() {
470548
return stagingAuthMethod;
471549
}
@@ -542,6 +620,14 @@ public void setFinalPort(Integer finalPort) {
542620
this.finalPort = finalPort;
543621
}
544622

623+
public String getFinalBasePath() {
624+
return finalBasePath;
625+
}
626+
627+
public void setFinalBasePath(String finalBasePath) {
628+
this.finalBasePath = finalBasePath;
629+
}
630+
545631
public String getFinalAuthMethod() {
546632
return finalAuthMethod;
547633
}
@@ -618,6 +704,14 @@ public void setJobPort(Integer jobPort) {
618704
this.jobPort = jobPort;
619705
}
620706

707+
public String getJobBasePath() {
708+
return jobBasePath;
709+
}
710+
711+
public void setJobBasePath(String jobBasePath) {
712+
this.jobBasePath = jobBasePath;
713+
}
714+
621715
public String getJobAuthMethod() {
622716
return jobAuthMethod;
623717
}

‎marklogic-data-hub-api/src/main/java/com/marklogic/hub/impl/HubClientImpl.java‎

Lines changed: 7 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -27,6 +27,7 @@ public class HubClientImpl implements HubClient {
2727

2828
private final HubClientConfig hubClientConfig;
2929
private String username;
30+
private String mlAuthentication;
3031
private DatabaseClient stagingClient;
3132
private DatabaseClient finalClient;
3233
private DatabaseClient jobsClient;
@@ -37,6 +38,7 @@ public class HubClientImpl implements HubClient {
3738
public HubClientImpl(HubClientConfig hubClientConfig) {
3839
this.hubClientConfig = hubClientConfig;
3940
username = hubClientConfig.getUsername();
41+
mlAuthentication = hubClientConfig.getMlAuthentication();
4042
stagingClient = hubClientConfig.newStagingClient(null);
4143
finalClient = hubClientConfig.newFinalClient(null);
4244
jobsClient = hubClientConfig.newJobDbClient();
@@ -50,6 +52,11 @@ public String getUsername() {
5052
return username;
5153
}
5254

55+
@Override
56+
public String getMlAuthentication() {
57+
return mlAuthentication;
58+
}
59+
5360
@Override
5461
public String getDbName(DatabaseKind kind) {
5562
return databaseNames.get(kind);

‎marklogic-data-hub/src/main/java/com/marklogic/hub/impl/HubConfigImpl.java‎

Lines changed: 10 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -1145,20 +1145,24 @@ public Path getStepDefinitionsDir() {
11451145
protected void addDhfPropertiesToCustomTokens(AppConfig appConfig) {
11461146
Map<String, String> customTokens = appConfig.getCustomTokens();
11471147
customTokens.put("%%mlHost%%", appConfig.getHost());
1148+
customTokens.put("%%mlAuthentication%%", getMlAuthentication());
11481149
customTokens.put("%%mlStagingAppserverName%%", stagingHttpName);
11491150
customTokens.put("%%mlStagingPort%%", getStagingPort().toString());
1151+
customTokens.put("%%mlStagingBasePath%%", getStagingBasePath());
11501152
customTokens.put("%%mlStagingDbName%%", getStagingDbName());
11511153
customTokens.put("%%mlStagingForestsPerHost%%", stagingForestsPerHost.toString());
11521154
customTokens.put("%%mlStagingAuth%%", getStagingAuthMethod());
11531155

11541156
customTokens.put("%%mlFinalAppserverName%%", finalHttpName);
11551157
customTokens.put("%%mlFinalPort%%", getFinalPort().toString());
1158+
customTokens.put("%%mlFinalBasePath%%", getFinalBasePath());
11561159
customTokens.put("%%mlFinalDbName%%", getFinalDbName());
11571160
customTokens.put("%%mlFinalForestsPerHost%%", finalForestsPerHost.toString());
11581161
customTokens.put("%%mlFinalAuth%%", getFinalAuthMethod());
11591162

11601163
customTokens.put("%%mlJobAppserverName%%", jobHttpName);
11611164
customTokens.put("%%mlJobPort%%", getJobPort().toString());
1165+
customTokens.put("%%mlJobBasePath%%", getJobBasePath());
11621166
customTokens.put("%%mlJobDbName%%", getJobDbName());
11631167
customTokens.put("%%mlJobForestsPerHost%%", jobForestsPerHost.toString());
11641168
customTokens.put("%%mlJobAuth%%", getJobAuthMethod());
@@ -1327,13 +1331,17 @@ protected void initializeModulePaths(AppConfig config) {
13271331
*/
13281332
private void applyFinalConnectionSettingsToMlGradleDefaultRestSettings(AppConfig config) {
13291333
if (getFinalAuthMethod() != null) {
1330-
config.setRestSecurityContextType(SecurityContextType.valueOf(getFinalAuthMethod().toUpperCase()));
1334+
String authMethod = getMlAuthentication().equalsIgnoreCase("cloud") ? getMlAuthentication() : getFinalAuthMethod();
1335+
config.setRestSecurityContextType(SecurityContextType.valueOf(authMethod.toUpperCase()));
13311336
}
13321337
if (Boolean.TRUE.equals(isProvisionedEnvironment)) {
13331338
config.setRestConnectionType(DatabaseClient.ConnectionType.GATEWAY);
13341339
config.setAppServicesConnectionType(DatabaseClient.ConnectionType.GATEWAY);
13351340
}
1336-
config.setRestPort(getFinalPort());
1341+
Integer finalPort = getMlAuthentication().equalsIgnoreCase("cloud") ? 443 : getFinalPort();
1342+
config.setRestPort(finalPort);
1343+
config.setCloudApiKey(getCloudApiKey());
1344+
config.setRestBasePath(getFinalBasePath());
13371345
config.setRestCertFile(getFinalCertFile());
13381346
config.setRestCertPassword(getFinalCertPassword());
13391347
config.setRestExternalName(getFinalExternalName());

‎marklogic-data-hub/src/main/java/com/marklogic/hub/step/impl/SourceQueryCollector.java‎

Lines changed: 14 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -25,6 +25,7 @@
2525
import okhttp3.Request;
2626
import okhttp3.Response;
2727
import org.apache.commons.io.IOUtils;
28+
import org.apache.commons.lang3.StringUtils;
2829

2930
import java.io.BufferedReader;
3031
import java.io.File;
@@ -51,13 +52,25 @@ public SourceQueryCollector(HubClient hubClient, String sourceDatabase) {
5152

5253
public DiskQueue<String> run(String flow, String step, Map<String, Object> options) {
5354
final DatabaseClient stagingClient = hubClient.getStagingClient();
55+
String basePath = "";
56+
if(hubClient.getMlAuthentication().equalsIgnoreCase("cloud")) {
57+
basePath = stagingClient.getBasePath();
58+
if(StringUtils.isNotEmpty(basePath) && !basePath.startsWith("/")) {
59+
basePath = ("/").concat(basePath);
60+
}
61+
62+
if(StringUtils.isNotEmpty(basePath) && basePath.endsWith("/")) {
63+
basePath = basePath.substring(0, basePath.length() - 1);
64+
}
65+
}
5466

5567
try {
5668
String uriString = String.format(
57-
"%s://%s:%d/v1/internal/hubcollector5?flow-name=%s&database=%s&step=%s",
69+
"%s://%s:%d%s/v1/internal/hubcollector5?flow-name=%s&database=%s&step=%s",
5870
stagingClient.getSecurityContext().getSSLContext() != null ? "https" : "http",
5971
stagingClient.getHost(),
6072
stagingClient.getPort(),
73+
basePath,
6174
URLEncoder.encode(flow, "UTF-8"),
6275
URLEncoder.encode(this.sourceDatabase, "UTF-8"),
6376
URLEncoder.encode(step, "UTF-8")

0 commit comments

Comments
 (0)