Skip to content

Commit 1b49755

Browse files
rjrudinSameeraPriyathamTadikonda
rjrudin
authored and
SameeraPriyathamTadikonda
committed
DHFPROD-5037: DH fields/indexes are now retained during hubDeploy
1 parent 9ee7d4f commit 1b49755

File tree

6 files changed

+203
-10
lines changed

6 files changed

+203
-10
lines changed

‎marklogic-data-hub/src/main/java/com/marklogic/hub/deploy/commands/DeployDatabaseFieldCommand.java‎

Lines changed: 40 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -19,12 +19,15 @@
1919
import com.marklogic.appdeployer.command.AbstractResourceCommand;
2020
import com.marklogic.appdeployer.command.CommandContext;
2121
import com.marklogic.appdeployer.command.SortOrderConstants;
22+
import com.marklogic.mgmt.ManageClient;
2223
import com.marklogic.mgmt.resource.ResourceManager;
2324
import com.marklogic.mgmt.resource.databases.DatabaseManager;
2425
import com.marklogic.rest.util.Fragment;
2526
import org.apache.commons.lang3.StringUtils;
2627
import org.jdom2.Element;
2728
import org.jdom2.Namespace;
29+
import org.jdom2.output.XMLOutputter;
30+
import org.springframework.http.ResponseEntity;
2831

2932
import java.io.File;
3033
import java.util.ArrayList;
@@ -52,7 +55,7 @@ protected File[] getResourceDirs(CommandContext context) {
5255

5356
@Override
5457
protected ResourceManager getResourceManager(CommandContext commandContext) {
55-
return new DatabaseManager(commandContext.getManageClient());
58+
return new HubDatabaseManager(commandContext.getManageClient());
5659
}
5760

5861
@Override
@@ -137,4 +140,40 @@ protected void addExistingRangePathIndexes(Fragment newProps, Fragment existingP
137140
}
138141
}
139142
}
143+
144+
public static class HubDatabaseManager extends DatabaseManager {
145+
146+
public HubDatabaseManager(ManageClient manageClient) {
147+
super(manageClient);
148+
}
149+
150+
protected String getResourceName() {
151+
return "database";
152+
}
153+
154+
@Override
155+
protected ResponseEntity<String> putPayload(ManageClient client, String path, String payload) {
156+
if (!payloadParser.isJsonPayload(payload)) {
157+
payload = removeDatabaseNameFromXmlPayload(payload);
158+
}
159+
return super.putPayload(client, path, payload);
160+
}
161+
162+
/**
163+
* The database-name is not required, and for a data-hub-developer, the inclusion of it will cause a
164+
* Forbidden error from the Manage API.
165+
*
166+
* @param payload
167+
* @return
168+
*/
169+
protected String removeDatabaseNameFromXmlPayload(String payload) {
170+
Fragment frag = new Fragment(payload);
171+
List<Element> elements = frag.getElements("/node()/m:database-name");
172+
if (elements != null) {
173+
// Should only be one of these, of course
174+
elements.forEach(el -> el.detach());
175+
}
176+
return new XMLOutputter().outputString(frag.getInternalDoc());
177+
}
178+
}
140179
}

‎marklogic-data-hub/src/main/java/com/marklogic/hub/dhs/DhsDeployer.java‎

Lines changed: 9 additions & 4 deletions
Original file line numberDiff line numberDiff line change
@@ -1,6 +1,7 @@
11
package com.marklogic.hub.dhs;
22

33
import com.marklogic.appdeployer.AppConfig;
4+
import com.marklogic.appdeployer.CmaConfig;
45
import com.marklogic.appdeployer.ConfigDir;
56
import com.marklogic.appdeployer.command.Command;
67
import com.marklogic.appdeployer.command.alert.DeployAlertActionsCommand;
@@ -21,11 +22,8 @@
2122
import com.marklogic.hub.DatabaseKind;
2223
import com.marklogic.hub.HubConfig;
2324
import com.marklogic.hub.deploy.HubAppDeployer;
24-
import com.marklogic.hub.deploy.commands.GenerateFunctionMetadataCommand;
25+
import com.marklogic.hub.deploy.commands.*;
2526
import com.marklogic.hub.dhs.installer.deploy.DeployHubQueryRolesetsCommand;
26-
import com.marklogic.hub.deploy.commands.HubDeployDatabaseCommandFactory;
27-
import com.marklogic.hub.deploy.commands.LoadUserArtifactsCommand;
28-
import com.marklogic.hub.deploy.commands.LoadUserModulesCommand;
2927
import com.marklogic.hub.impl.HubConfigImpl;
3028

3129
import java.util.ArrayList;
@@ -97,6 +95,10 @@ protected void prepareAppConfigForDeployingToDhs(HubConfig hubConfig) {
9795
logger.info("Setting security context type for App-Services to: " + authMethod);
9896
appConfig.setAppServicesSecurityContextType(SecurityContextType.valueOf(authMethod.toUpperCase()));
9997
}
98+
99+
// As part of the fix for DHFPROD-5073, disabling all CMA usage, as data-hub-developer/operator are not allowed
100+
// to use CMA
101+
appConfig.setCmaConfig(new CmaConfig(false));
100102
}
101103

102104
/**
@@ -172,6 +174,9 @@ protected List<Command> buildCommandsForDeveloper(HubConfig hubConfig) {
172174
deployOtherDatabasesCommand.setResourceFilenamesIncludePattern(buildPatternForDatabasesToUpdateIndexesFor());
173175
commands.add(deployOtherDatabasesCommand);
174176

177+
// Per DHFPROD-5073, need this so that OOTB DH fields/indexes can be restored in case they're removed by the user
178+
commands.add(new DeployDatabaseFieldCommand());
179+
175180
commands.add(new DeployAlertConfigsCommand());
176181
commands.add(new DeployAlertActionsCommand());
177182
commands.add(new DeployAlertRulesCommand());

‎marklogic-data-hub/src/test/java/com/marklogic/hub/deploy/commands/RemoveDatabaseNameFromDatabaseFieldsPayloadTest.java‎

Lines changed: 26 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,26 @@
1+
package com.marklogic.hub.deploy.commands;
2+
3+
import com.marklogic.rest.util.Fragment;
4+
import org.junit.jupiter.api.Test;
5+
6+
import static org.junit.jupiter.api.Assertions.*;
7+
8+
public class RemoveDatabaseNameFromDatabaseFieldsPayloadTest {
9+
10+
@Test
11+
void test() {
12+
String xml = "<database-properties xmlns=\"http://marklogic.com/manage\">\n" +
13+
" <database-name>data-hub-JOBS</database-name>\n" +
14+
" <triple-index>true</triple-index>\n" +
15+
"</database-properties>";
16+
17+
Fragment frag = new Fragment(xml);
18+
assertTrue(frag.elementExists("/node()/m:database-name"));
19+
assertEquals("true", frag.getElementValue("/node()/m:triple-index"));
20+
21+
xml = new DeployDatabaseFieldCommand.HubDatabaseManager(null).removeDatabaseNameFromXmlPayload(xml);
22+
frag = new Fragment(xml);
23+
assertFalse(frag.elementExists("/node()/m:database-name"));
24+
assertEquals("true", frag.getElementValue("/node()/m:triple-index"));
25+
}
26+
}

‎marklogic-data-hub/src/test/java/com/marklogic/hub/dhs/DeployAsDeveloperTest.java‎

Lines changed: 14 additions & 5 deletions
Original file line numberDiff line numberDiff line change
@@ -1,6 +1,7 @@
11
package com.marklogic.hub.dhs;
22

33
import com.marklogic.appdeployer.AppConfig;
4+
import com.marklogic.appdeployer.CmaConfig;
45
import com.marklogic.appdeployer.ConfigDir;
56
import com.marklogic.appdeployer.command.Command;
67
import com.marklogic.appdeployer.command.ResourceFilenameFilter;
@@ -19,6 +20,7 @@
1920
import com.marklogic.hub.DatabaseKind;
2021
import com.marklogic.hub.HubConfig;
2122
import com.marklogic.hub.HubTestBase;
23+
import com.marklogic.hub.deploy.commands.DeployDatabaseFieldCommand;
2224
import com.marklogic.hub.deploy.commands.GenerateFunctionMetadataCommand;
2325
import com.marklogic.hub.deploy.commands.LoadUserArtifactsCommand;
2426
import com.marklogic.hub.deploy.commands.LoadUserModulesCommand;
@@ -60,6 +62,9 @@ public void prepareAppConfig() {
6062
appConfig.getConfigDirs().add(new ConfigDir(new File("my-dhs-config")));
6163
assertEquals(3, appConfig.getConfigDirs().size(), "Should have 3, including ml-config which is added by default");
6264

65+
// Enable all CMA usage so we can verify that it's disabled below
66+
appConfig.setCmaConfig(new CmaConfig(true));
67+
6368
hubConfig.setAppConfig(appConfig);
6469

6570
assertFalse(hubConfig.getIsProvisionedEnvironment());
@@ -82,6 +87,13 @@ public void prepareAppConfig() {
8287
assertEquals("ml-config", appConfig.getConfigDirs().get(0).getBaseDir().getName());
8388
assertEquals("my-dhs-config", appConfig.getConfigDirs().get(1).getBaseDir().getName(), "A user is still " +
8489
"permitted to deploy their own resources from multiple configuration directories");
90+
91+
CmaConfig cmaConfig = appConfig.getCmaConfig();
92+
assertFalse(cmaConfig.isDeployDatabases());
93+
assertFalse(cmaConfig.isDeployPrivileges());
94+
assertFalse(cmaConfig.isDeployProtectedPaths());
95+
assertFalse(cmaConfig.isDeployQueryRolesets());
96+
assertFalse(cmaConfig.isDeployRoles());
8597
}
8698

8799
@Test
@@ -177,6 +189,7 @@ public void buildCommandList() {
177189
int index = 0;
178190
assertTrue(commands.get(index++) instanceof DeployHubQueryRolesetsCommand);
179191
assertTrue(commands.get(index++) instanceof DeployOtherDatabasesCommand);
192+
assertTrue(commands.get(index++) instanceof DeployDatabaseFieldCommand);
180193
assertTrue(commands.get(index++) instanceof LoadSchemasCommand);
181194
assertTrue(commands.get(index++) instanceof LoadUserModulesCommand);
182195
assertTrue(commands.get(index++) instanceof DeployTriggersCommand);
@@ -191,11 +204,7 @@ public void buildCommandList() {
191204
assertTrue(commands.get(index++) instanceof GenerateFunctionMetadataCommand);
192205
assertTrue(commands.get(index++) instanceof DeployProtectedPathsCommand);
193206

194-
assertEquals(15, commands.size(),
195-
"As of ML 10.0-3, the granular privilege for indexes doesn't seem to work with XML payloads. " +
196-
"Bug https://bugtrack.marklogic.com/54231 has been created to track that. Thus, " +
197-
"DeployDatabaseFieldCommand cannot be included and ml-config/database-fields/final-database.xml " +
198-
"cannot be processed.");
207+
assertEquals(16, commands.size(), "Per DHFPROD-5037, DeployDatabaseFieldsCommand now exists");
199208

200209
DeployOtherDatabasesCommand dodc = (DeployOtherDatabasesCommand) commands.get(1);
201210
ResourceFilenameFilter filter = (ResourceFilenameFilter) dodc.getResourceFilenameFilter();

‎marklogic-data-hub/src/test/java/com/marklogic/hub/dhs/DeployCustomUserFieldsTest.java‎

Lines changed: 82 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,82 @@
1+
package com.marklogic.hub.dhs;
2+
3+
import com.fasterxml.jackson.databind.node.ObjectNode;
4+
import com.marklogic.appdeployer.command.CommandContext;
5+
import com.marklogic.hub.ApplicationConfig;
6+
import com.marklogic.hub.DatabaseKind;
7+
import com.marklogic.hub.HubTestBase;
8+
import com.marklogic.hub.deploy.commands.DeployDatabaseFieldCommand;
9+
import com.marklogic.mgmt.resource.databases.DatabaseManager;
10+
import com.marklogic.mgmt.util.ObjectMapperFactory;
11+
import org.apache.commons.io.FileUtils;
12+
import org.junit.jupiter.api.AfterEach;
13+
import org.junit.jupiter.api.Test;
14+
import org.junit.jupiter.api.extension.ExtendWith;
15+
import org.springframework.core.io.ClassPathResource;
16+
import org.springframework.test.context.ContextConfiguration;
17+
import org.springframework.test.context.junit.jupiter.SpringExtension;
18+
19+
import java.io.File;
20+
import java.io.IOException;
21+
22+
import static org.junit.jupiter.api.Assertions.assertEquals;
23+
24+
@ExtendWith(SpringExtension.class)
25+
@ContextConfiguration(classes = ApplicationConfig.class)
26+
public class DeployCustomUserFieldsTest extends HubTestBase {
27+
28+
@AfterEach
29+
void afterEach() {
30+
// dhsDeployer mucks around with appConfig, so gotta reset everything
31+
adminHubConfig.resetAppConfigs();
32+
adminHubConfig.resetHubConfigs();
33+
adminHubConfig.refreshProject();
34+
35+
getDataHubAdminConfig();
36+
37+
// Wipe out all fields/indexes
38+
String json = "{\n" +
39+
" \"database-name\" : \"data-hub-FINAL\",\n" +
40+
" \"field\" : [ ],\n" +
41+
" \"range-field-index\" : [ ],\n" +
42+
" \"range-path-index\" : [ ]\n" +
43+
"}";
44+
new DatabaseManager(adminHubConfig.getManageClient()).save(json);
45+
46+
// Then run the command to deploy the OOTB fields/indexes
47+
new DeployDatabaseFieldCommand().execute(new CommandContext(adminHubConfig.getAppConfig(), adminHubConfig.getManageClient(), null));
48+
}
49+
50+
@Test
51+
void test() throws IOException {
52+
File configDir = new File(new ClassPathResource("test-projects/user-fields").getFile(), "ml-config");
53+
if (configDir.exists()) {
54+
FileUtils.copyDirectory(configDir, adminHubConfig.getHubProject().getUserConfigDir().toFile());
55+
}
56+
57+
// Get the initial counts of fields/indexes for later comparisons
58+
runAsDataHubDeveloper();
59+
ObjectNode db = readFinalDatabase();
60+
61+
// Not verifying rangePathIndex because the version of ml-app-deployer used doesn't have that as a property.
62+
// But the fix for this on develop, slated for 5.3.0, is verifying it, and the fixed to DeployDatabaseFieldCommand
63+
// is the same.
64+
final int initialFieldCount = db.get("field").size();
65+
final int initialFieldIndexCount = db.get("range-field-index").size();
66+
final int initialPathIndexCount = db.get("range-path-index").size();
67+
68+
// Deploy as a data-hub-developer
69+
new DhsDeployer().deployAsDeveloper(adminHubConfig);
70+
71+
// Verify that the existing DH fields/indexes still exist, and we have the user fields/indexes too (1 of each)
72+
db = readFinalDatabase();
73+
assertEquals(initialFieldCount + 1, db.get("field").size());
74+
assertEquals(initialFieldIndexCount + 1, db.get("range-field-index").size());
75+
assertEquals(initialPathIndexCount + 1, db.get("range-path-index").size());
76+
}
77+
78+
private ObjectNode readFinalDatabase() throws IOException {
79+
String json = new DatabaseManager(adminHubConfig.getManageClient()).getPropertiesAsJson(adminHubConfig.getDbName(DatabaseKind.FINAL));
80+
return (ObjectNode) ObjectMapperFactory.getObjectMapper().readTree(json);
81+
}
82+
}

‎marklogic-data-hub/src/test/resources/test-projects/user-fields/ml-config/databases/final-database.json‎

Lines changed: 32 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,32 @@
1+
{
2+
"database-name": "%%mlFinalDbName%%",
3+
"schema-database": "%%mlFinalSchemasDbName%%",
4+
"triggers-database": "%%mlFinalTriggersDbName%%",
5+
"triple-index": true,
6+
"collection-lexicon": true,
7+
"uri-lexicon": true,
8+
"field": [
9+
{
10+
"field-name": "exampleField",
11+
"include-root": false
12+
}
13+
],
14+
"range-field-index": [
15+
{
16+
"scalar-type": "string",
17+
"field-name": "exampleField",
18+
"invalid-values": "reject",
19+
"range-value-positions": false,
20+
"collation": "http://marklogic.com/collation/"
21+
}
22+
],
23+
"range-path-index": [
24+
{
25+
"scalar-type": "int",
26+
"path-expression": "/examplePathIndex",
27+
"collation": "",
28+
"range-value-positions": false,
29+
"invalid-values": "reject"
30+
}
31+
]
32+
}

0 commit comments

Comments
 (0)