Skip to content

Commit f8a3d5a

Browse files
author
Charles Greer
authored
Merge pull request #1350 from marklogic/test-isolation-DHFPROD-1229
Work in Progress - Test isolation dhfprod 1229
2 parents c5e0b8c + 9d8e3ce commit f8a3d5a

File tree

61 files changed

+3608
-3227
lines changed

Some content is hidden

Large Commits have some content hidden by default. Use the searchbox below for content that may be hidden.

61 files changed

+3608
-3227
lines changed

‎examples/disconnected-project/gradle.properties‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -17,7 +17,7 @@ mlDHFVersion=4.0.1
1717
mlHost=localhost
1818

1919
# If you are working with a load balancer please specify it here
20-
# mlIsHostLoadBalancer=false
20+
# mlIsHostLoadBalancer=true/false
2121

2222
# Your MarkLogic Username and Password
2323
mlUsername=

‎examples/healthcare/build.gradle‎

Lines changed: 11 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,11 @@
1+
plugins {
2+
// this plugin lets you create properties files
3+
// for multiple environments... like dev, qa, prod
4+
id 'net.saliman.properties' version '1.4.6'
5+
6+
// this is the data hub framework gradle plugin
7+
// it includes ml-gradle. This plugin is what lets you
8+
// run DHF (Data Hub Framework) tasks from the
9+
// command line
10+
id 'com.marklogic.ml-data-hub' version '4.0.0'
11+
}

‎marklogic-data-hub/build.gradle‎

Lines changed: 54 additions & 22 deletions
Original file line numberDiff line numberDiff line change
@@ -25,7 +25,8 @@ ext.junitPlatformVersion = '1.0.0-RC3'
2525
ext.junitJupiterVersion = '5.0.0-RC3'
2626

2727
dependencies {
28-
compile 'com.marklogic:marklogic-client-api:4.1.0'
28+
compile files('tmp-lib/marklogic-client-api-4.1.0.jar')
29+
// compile 'com.marklogic:marklogic-client-api:4.1.0'
2930
compile 'com.marklogic:mlcp-util:0.9.0'
3031
compile 'com.marklogic:ml-app-deployer:3.8.3'
3132
compile 'com.marklogic:marklogic-data-movement-components:1.0'
@@ -52,7 +53,7 @@ dependencies {
5253
testCompile 'ch.qos.logback:logback-classic:1.1.11'
5354
testCompile 'org.slf4j:log4j-over-slf4j:1.7.13'
5455
testCompile("com.marklogic:mlcp-util:0.3.0")
55-
testCompile("com.marklogic:mlcp:9.0.3") {
56+
testCompile("com.marklogic:mlcp:9.0.7") {
5657
exclude group: 'org.apache.avro', module: 'avro-tools'
5758
}
5859
}
@@ -254,16 +255,60 @@ boolean certAuth = Boolean.parseBoolean(System.properties['certauth'])
254255
def props = new Properties()
255256
propFile.withInputStream { props.load(it) }
256257

258+
task bootstrap(type: JavaExec) {
259+
dependsOn 'setupSSL'
260+
classpath = sourceSets.test.runtimeClasspath
261+
main = 'com.marklogic.hub.util.Installer'
262+
}
263+
264+
task teardown(type: JavaExec) {
265+
classpath = sourceSets.test.runtimeClasspath
266+
main = 'com.marklogic.hub.util.UnInstaller'
267+
}
268+
257269
test {
258270
minHeapSize = "128m"
259271
maxHeapSize = "256m"
260-
261-
doFirst{
272+
classpath = project.sourceSets.test.runtimeClasspath
273+
}
274+
275+
task testAcceptance(type: Test) {
276+
useJUnit {
277+
include 'com/marklogic/hub/**'
278+
if (!(certAuth || sslRun)){
279+
include 'com/marklogic/hub/PiiE2E.class'
280+
}
281+
}
282+
}
283+
284+
task testIntegration(type: Test) {
285+
useJUnit {
286+
include 'com/marklogic/hub_integration/**'
287+
}
288+
}
289+
290+
task testBootstrap(type: Test) {
291+
useJUnit {
292+
exclude 'com/marklogic/bootstrap/**'
293+
}
294+
}
295+
296+
task cleanupTask{
297+
dependsOn 'teardown'
298+
doLast{
299+
if(sslRun || certAuth){
300+
propFile.write(fileContents)
301+
}
302+
}
303+
}
304+
305+
task setupSSL{
306+
doFirst{
262307
if(sslRun || certAuth){
263308
com.marklogic.mgmt.ManageClient manageClient = new ManageClient(new com.marklogic.mgmt.ManageConfig(mlHost, 8002, mlSecurityUsername, mlSecurityPassword))
264309
com.marklogic.mgmt.resource.hosts.HostManager hostManager = new HostManager(manageClient)
265310
def bootStrapHost = hostManager.getHostNames().get(0)
266-
if(! bootStrapHost.toLowerCase().contains("marklogic.com")){
311+
if(! bootStrapHost.toLowerCase().contains("marklogic.com")){
267312
throw new GradleException("The test with current options will run only in marklogic.com domain")
268313
}
269314
if(sslRun) {
@@ -289,22 +334,9 @@ test {
289334
}
290335
}
291336
}
292-
}
293-
294-
// include just the top-level tests. they include setup/teardown
295-
include 'com/marklogic/hub/CoreTestSuite.class'
296-
include 'com/marklogic/hub/EndToEndFlowTests.class'
297-
if (!(certAuth || sslRun)){
298-
include 'com/marklogic/hub/PiiE2E.class'
299337
}
300-
include 'com/marklogic/hub/ScaffoldingE2E.class'
301-
include 'com/marklogic/hub/MappingE2E.class'
302-
}
303-
task restoreFile{
304-
doFirst{
305-
if(sslRun || certAuth){
306-
propFile.write(fileContents)
307-
}
308-
}
309338
}
310-
test.finalizedBy(restoreFile)
339+
340+
test.dependsOn(bootstrap)
341+
//Uncomment this line if hub has to be uninstalled after test
342+
//test.finalizedBy(cleanupTask)

‎marklogic-data-hub/gradle.properties‎

Lines changed: 2 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -16,6 +16,8 @@ mlDHFVersion=4.0.0
1616
mlHost=localhost
1717
mlAppName=data-hub
1818

19+
mlIsHostLoadBalancer=false
20+
1921
#mlUsername=data-hub-user
2022
#mlPassword=eO3^)&X|j_O8m^k_2=xi
2123
mlUsername=hub-admin-user

‎marklogic-data-hub/src/main/java/com/marklogic/hub/HubConfig.java‎

Lines changed: 9 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -303,10 +303,16 @@ static HubConfig create(String projectDir) {
303303
void setHubUserName(String hubUserName);
304304

305305
/**
306-
* Gets a string array of hosts
307-
* @return String array of hosts
306+
* Gets a string of load balancer host
307+
* @return String of load balancer host
308308
*/
309-
String[] getLoadBalancerHosts();
309+
String getLoadBalancerHost();
310+
311+
/**
312+
* Signifies if the host is a load balancer host.
313+
* @return a Boolean.
314+
*/
315+
Boolean getIsHostLoadBalancer();
310316

311317
/**
312318
* Returns the path for the custom forests definition

‎marklogic-data-hub/src/main/java/com/marklogic/hub/deploy/commands/DeployHubAmpsCommand.java‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -43,7 +43,7 @@ public DeployHubAmpsCommand(HubConfig hubConfig) {
4343

4444
/**
4545
* Installs the amps for DHF via CMA endpoint
46-
* @param context
46+
* @param context The command context for execution.
4747
*/
4848
@Override
4949
public void execute(CommandContext context) {

‎marklogic-data-hub/src/main/java/com/marklogic/hub/deploy/commands/LoadUserStagingModulesCommand.java‎

Lines changed: 10 additions & 6 deletions
Original file line numberDiff line numberDiff line change
@@ -167,8 +167,10 @@ public void execute(CommandContext context) {
167167
//for now we'll use two different document managers
168168
JSONDocumentManager finalEntityDocMgr = finalClient.newJSONDocumentManager();
169169
JSONDocumentManager stagingEntityDocMgr = stagingClient.newJSONDocumentManager();
170-
JSONDocumentManager mappingDocMgr = finalClient.newJSONDocumentManager();
171-
DocumentWriteSet mappingDocumentWriteSet = mappingDocMgr.newWriteSet();
170+
JSONDocumentManager finalMappingDocMgr = finalClient.newJSONDocumentManager();
171+
JSONDocumentManager stagingMappingDocMgr = stagingClient.newJSONDocumentManager();
172+
DocumentWriteSet finalMappingDocumentWriteSet = finalMappingDocMgr.newWriteSet();
173+
DocumentWriteSet stagingMappingDocumentWriteSet = stagingMappingDocMgr.newWriteSet();
172174

173175
AllButAssetsModulesFinder allButAssetsModulesFinder = new AllButAssetsModulesFinder();
174176

@@ -217,7 +219,7 @@ else if (isEntityDir(dir, startPath.toAbsolutePath())) {
217219
finalEntityDocMgr.write("/entities/" + r.getFilename(), meta, handle);
218220

219221
// Uncomment to send entity model to staging db as well
220-
// stagingEntityDocMgr.write("/entities/" + r.getFilename(), meta, handle);
222+
stagingEntityDocMgr.write("/entities/" + r.getFilename(), meta, handle);
221223
modulesManager.saveLastLoadedTimestamp(r.getFile(), new Date());
222224
}
223225
}
@@ -260,7 +262,8 @@ public FileVisitResult preVisitDirectory(Path dir, BasicFileAttributes attrs) th
260262
InputStream inputStream = r.getInputStream();
261263
StringHandle handle = new StringHandle(IOUtils.toString(inputStream));
262264
inputStream.close();
263-
mappingDocumentWriteSet.add("/mappings/" + r.getFile().getParentFile().getName() + "/" + r.getFilename(), meta, handle);
265+
finalMappingDocumentWriteSet.add("/mappings/" + r.getFile().getParentFile().getName() + "/" + r.getFilename(), meta, handle);
266+
stagingMappingDocumentWriteSet.add("/mappings/" + r.getFile().getParentFile().getName() + "/" + r.getFilename(), meta, handle);
264267
modulesManager.saveLastLoadedTimestamp(r.getFile(), new Date());
265268
}
266269
}
@@ -289,8 +292,9 @@ public FileVisitResult visitFile(Path file, BasicFileAttributes attrs)
289292
documentManager.write(documentWriteSet);
290293
}
291294

292-
if (mappingDocumentWriteSet.size() > 0) {
293-
mappingDocMgr.write(mappingDocumentWriteSet);
295+
if (stagingMappingDocumentWriteSet.size() > 0) {
296+
finalMappingDocMgr.write(finalMappingDocumentWriteSet);
297+
stagingMappingDocMgr.write(stagingMappingDocumentWriteSet);
294298
}
295299
}
296300
threadPoolTaskExecutor.shutdown();

‎marklogic-data-hub/src/main/java/com/marklogic/hub/flow/impl/FlowRunnerImpl.java‎

Lines changed: 1 addition & 11 deletions
Original file line numberDiff line numberDiff line change
@@ -212,7 +212,7 @@ public JobTicket run() {
212212

213213
ConcurrentHashMap<DatabaseClient, FlowResource> databaseClientMap = new ConcurrentHashMap<>();
214214

215-
QueryBatcher tempQueryBatcher = dataMovementManager.newQueryBatcher(uris.iterator())
215+
QueryBatcher queryBatcher = dataMovementManager.newQueryBatcher(uris.iterator())
216216
.withBatchSize(batchSize)
217217
.withThreadCount(threadCount)
218218
.withJobId(jobId)
@@ -289,16 +289,6 @@ public JobTicket run() {
289289
});
290290

291291

292-
if (hubConfig.getLoadBalancerHosts() != null && hubConfig.getLoadBalancerHosts().length > 0){
293-
tempQueryBatcher = tempQueryBatcher.withForestConfig(
294-
new FilteredForestConfiguration(
295-
dataMovementManager.readForestConfig()
296-
).withWhiteList(hubConfig.getLoadBalancerHosts())
297-
);
298-
}
299-
QueryBatcher queryBatcher = tempQueryBatcher;
300-
301-
302292
JobTicket jobTicket = dataMovementManager.startJob(queryBatcher);
303293
ticketWrapper.put("jobTicket", jobTicket);
304294
jobManager.saveJob(job.withStatus(JobStatus.RUNNING_HARMONIZE));

‎marklogic-data-hub/src/main/java/com/marklogic/hub/impl/DataHubImpl.java‎

Lines changed: 87 additions & 5 deletions
Original file line numberDiff line numberDiff line change
@@ -18,13 +18,15 @@
1818
import com.fasterxml.jackson.databind.JsonNode;
1919
import com.marklogic.appdeployer.AppConfig;
2020
import com.marklogic.appdeployer.command.Command;
21+
import com.marklogic.appdeployer.command.CommandContext;
2122
import com.marklogic.appdeployer.command.CommandMapBuilder;
2223
import com.marklogic.appdeployer.command.appservers.DeployOtherServersCommand;
2324
import com.marklogic.appdeployer.command.forests.DeployCustomForestsCommand;
2425
import com.marklogic.appdeployer.command.modules.LoadModulesCommand;
2526
import com.marklogic.appdeployer.command.security.*;
2627
import com.marklogic.appdeployer.impl.SimpleAppDeployer;
2728
import com.marklogic.client.FailedRequestException;
29+
import com.marklogic.client.MarkLogicIOException;
2830
import com.marklogic.client.admin.QueryOptionsManager;
2931
import com.marklogic.client.admin.ResourceExtensionsManager;
3032
import com.marklogic.client.admin.ServerConfigurationManager;
@@ -46,16 +48,20 @@
4648
import com.marklogic.rest.util.ResourcesFragment;
4749
import org.apache.commons.io.FileUtils;
4850
import org.apache.commons.lang3.StringUtils;
51+
import org.apache.http.conn.HttpHostConnectException;
4952
import org.slf4j.Logger;
5053
import org.slf4j.LoggerFactory;
5154
import org.springframework.core.io.Resource;
5255
import org.springframework.core.io.support.PathMatchingResourcePatternResolver;
5356
import org.springframework.core.io.support.ResourcePatternResolver;
5457
import org.springframework.http.HttpStatus;
5558
import org.springframework.web.client.HttpClientErrorException;
59+
import org.springframework.web.client.HttpServerErrorException;
60+
import org.springframework.web.client.ResourceAccessException;
5661

5762
import java.io.File;
5863
import java.io.IOException;
64+
import java.net.ConnectException;
5965
import java.nio.file.Paths;
6066
import java.text.SimpleDateFormat;
6167
import java.util.*;
@@ -133,6 +139,24 @@ public InstallInfo isInstalled() {
133139
if (e.getStatusCode() == HttpStatus.UNAUTHORIZED) {
134140
throw new DataHubSecurityNotInstalledException();
135141
}
142+
} catch (HttpServerErrorException e) {
143+
// this result comes from a disabled manage client.
144+
// DHF gets this response from DHS, which has no public
145+
// manage client. So we'll assume a provisioned DH exists in this case.
146+
// TODO I don't know what TODO
147+
if (e.getStatusCode() == HttpStatus.SERVICE_UNAVAILABLE) {
148+
return assumedProvisionedInstallInfo(installInfo);
149+
}
150+
else {
151+
throw new DataHubConfigurationException(e);
152+
}
153+
} catch ( MarkLogicIOException e) {
154+
// If server is off, this happens.
155+
// we don't know about install state, so don't assume it's not?
156+
return assumedProvisionedInstallInfo(installInfo);
157+
} catch (ResourceAccessException e) {
158+
// in ALB scenario we actually get a connection exception, not gateway
159+
return assumedProvisionedInstallInfo(installInfo);
136160
}
137161

138162
installInfo.setAppServerExistent(DatabaseKind.STAGING, srf.resourceExists(hubConfig.getHttpName(DatabaseKind.STAGING)));
@@ -172,6 +196,34 @@ public InstallInfo isInstalled() {
172196
return installInfo;
173197
}
174198

199+
// this InstallInfo is used as a dummy to return DHS provisioned information
200+
private InstallInfo assumedProvisionedInstallInfo(InstallInfo installInfo) {
201+
installInfo.setAppServerExistent(DatabaseKind.STAGING, true);
202+
installInfo.setAppServerExistent(DatabaseKind.FINAL, true);
203+
installInfo.setAppServerExistent(DatabaseKind.JOB, true);
204+
205+
installInfo.setDbExistent(DatabaseKind.STAGING, true);
206+
installInfo.setDbExistent(DatabaseKind.FINAL, true);
207+
installInfo.setDbExistent(DatabaseKind.JOB, true);
208+
209+
installInfo.setDbExistent(DatabaseKind.MODULES, true);
210+
installInfo.setDbExistent(DatabaseKind.STAGING_SCHEMAS, true);
211+
installInfo.setDbExistent(DatabaseKind.STAGING_TRIGGERS, true);
212+
213+
installInfo.setTripleIndexOn(DatabaseKind.STAGING, true);
214+
installInfo.setCollectionLexiconOn(DatabaseKind.STAGING, true);
215+
installInfo.setForestsExistent(DatabaseKind.STAGING, true);
216+
217+
installInfo.setTripleIndexOn(DatabaseKind.FINAL, true);
218+
installInfo.setCollectionLexiconOn(DatabaseKind.FINAL, true);
219+
installInfo.setForestsExistent(DatabaseKind.FINAL, true);
220+
221+
installInfo.setForestsExistent(DatabaseKind.JOB, true);
222+
223+
return installInfo;
224+
225+
}
226+
175227
@Override
176228
public boolean isServerVersionValid(String versionString) {
177229
try {
@@ -406,6 +458,24 @@ public HashMap<String, Boolean> runPreInstallCheck(Versions versions) {
406458
return response;
407459
}
408460

461+
/*
462+
* just installs the hub modules, for more granular management of upgrade
463+
*/
464+
private void hubInstallModules() {
465+
AppConfig stagingConfig = hubConfig.getStagingAppConfig();
466+
CommandContext stagingContext = new CommandContext(stagingConfig, null, null);
467+
new LoadHubModulesCommand(hubConfig).execute(stagingContext);
468+
}
469+
470+
/*
471+
* just installs the user modules, for more granular management of upgrade
472+
*/
473+
private void loadUserModules() {
474+
AppConfig stagingConfig = hubConfig.getStagingAppConfig();
475+
CommandContext stagingContext = new CommandContext(stagingConfig, null, null);
476+
new LoadUserStagingModulesCommand(hubConfig).execute(stagingContext);
477+
}
478+
409479
/**
410480
* Installs the data hub configuration and server-side config files into MarkLogic
411481
*/
@@ -425,10 +495,20 @@ public void install(HubDeployStatusListener listener) {
425495

426496
logger.warn("Installing the Data Hub into MarkLogic");
427497

428-
AppConfig roleConfig = hubConfig.getStagingAppConfig();
429-
SimpleAppDeployer roleDeployer = new SimpleAppDeployer(getManageClient(), getAdminManager());
430-
roleDeployer.setCommands(getSecurityCommandList());
431-
roleDeployer.deploy(roleConfig);
498+
// in AWS setting this fails...
499+
// for now putting in try/catch
500+
try {
501+
AppConfig roleConfig = hubConfig.getStagingAppConfig();
502+
SimpleAppDeployer roleDeployer = new SimpleAppDeployer(getManageClient(), getAdminManager());
503+
roleDeployer.setCommands(getSecurityCommandList());
504+
roleDeployer.deploy(roleConfig);
505+
} catch (HttpServerErrorException e) {
506+
if (e.getStatusCode() == HttpStatus.SERVICE_UNAVAILABLE) {
507+
logger.warn("No manage client for security installs. Assuming DHS provisioning already threre");
508+
} else {
509+
throw new DataHubConfigurationException(e);
510+
}
511+
}
432512

433513
AppConfig finalConfig = hubConfig.getFinalAppConfig();
434514
HubAppDeployer finalDeployer = new HubAppDeployer(getManageClient(), getAdminManager(), listener, hubConfig.newStagingClient());
@@ -782,7 +862,9 @@ public boolean upgradeHub(List<String> updatedFlows) throws CantUpgradeException
782862
// install hub modules into MarkLogic
783863
runInDatabase("cts:uris(\"\", (), cts:and-not-query(cts:collection-query(\"hub-core-module\"), cts:document-query((\"/com.marklogic.hub/config.sjs\", \"/com.marklogic.hub/config.xqy\")))) ! xdmp:document-delete(.)", hubConfig.getDbName(DatabaseKind.MODULES));
784864

785-
this.install();
865+
// should be install user modules and hub modules, not install
866+
this.hubInstallModules();
867+
this.loadUserModules();
786868
}
787869

788870
//if none of this has thrown an exception, we're clear and can set the result to true

0 commit comments

Comments
 (0)