DHFPROD-4558: data-hub-developer can now deploy query rolesets

Author: rjrudin

marklogic-data-hub/src/main/java/com/marklogic/hub/dhs/DhsDeployer.java

@@ -9,6 +9,7 @@
 import com.marklogic.appdeployer.command.databases.DeployOtherDatabasesCommand;
 import com.marklogic.appdeployer.command.schemas.LoadSchemasCommand;
 import com.marklogic.appdeployer.command.security.DeployPrivilegesCommand;
+import com.marklogic.appdeployer.command.security.DeployProtectedPathsCommand;
 import com.marklogic.appdeployer.command.security.DeployRolesCommand;
 import com.marklogic.appdeployer.command.tasks.DeployScheduledTasksCommand;
 import com.marklogic.appdeployer.command.temporal.DeployTemporalAxesCommand;
@@ -19,6 +20,7 @@
 import com.marklogic.hub.DatabaseKind;
 import com.marklogic.hub.HubConfig;
 import com.marklogic.hub.deploy.HubAppDeployer;
+import com.marklogic.hub.dhs.installer.deploy.DeployHubQueryRolesetsCommand;
 import com.marklogic.hub.deploy.commands.HubDeployDatabaseCommandFactory;
 import com.marklogic.hub.deploy.commands.LoadUserArtifactsCommand;
 import com.marklogic.hub.deploy.commands.LoadUserModulesCommand;
@@ -183,6 +185,9 @@ protected List<Command> buildCommandsForDeveloper(HubConfig hubConfig) {
         commands.add(new LoadSchemasCommand());
         commands.add(new DeployScheduledTasksCommand());
 
+        commands.add(new DeployProtectedPathsCommand());
+        commands.add(new DeployHubQueryRolesetsCommand());
+
         return commands;
     }
 

marklogic-data-hub/src/main/java/com/marklogic/hub/dhs/installer/deploy/DeployHubQueryRolesetsCommand.java

@@ -0,0 +1,52 @@
+package com.marklogic.hub.dhs.installer.deploy;
+
+import com.fasterxml.jackson.databind.JsonNode;
+import com.marklogic.appdeployer.command.CommandContext;
+import com.marklogic.appdeployer.command.security.DeployQueryRolesetsCommand;
+import com.marklogic.mgmt.SaveReceipt;
+import com.marklogic.mgmt.resource.ResourceManager;
+import com.marklogic.mgmt.util.ObjectMapperFactory;
+import org.springframework.web.client.HttpClientErrorException;
+
+/**
+ * This command is used instead of DeployQueryRolesetsCommand for when a data-hub-developer user is deploying
+ * query rolesets. See the comments in the test class for this class to understand why it exists.
+ */
+public class DeployHubQueryRolesetsCommand extends DeployQueryRolesetsCommand {
+
+    @Override
+    public boolean cmaShouldBeUsed(CommandContext context) {
+        return false;
+    }
+
+    @Override
+    protected SaveReceipt saveResource(ResourceManager mgr, CommandContext context, String payload) {
+        SaveReceipt receipt = null;
+        try {
+            receipt = super.saveResource(mgr, context, payload);
+        } catch (HttpClientErrorException ex) {
+            if (isPermissionedDeniedException(ex)) {
+                logger.info("Received SEC-PERMDENIED error when deploying query roleset; this can be safely ignored if the " +
+                    "query roleset already exists in MarkLogic.");
+            } else {
+                throw ex;
+            }
+        }
+        return receipt;
+    }
+
+    protected boolean isPermissionedDeniedException(HttpClientErrorException ex) {
+        try {
+            JsonNode error = ObjectMapperFactory.getObjectMapper().readTree(ex.getResponseBodyAsString());
+            if (error.has("errorResponse")) {
+                JsonNode errorResponse = error.get("errorResponse");
+                if (errorResponse.has("messageCode")) {
+                    return "SEC-PERMDENIED".equals(errorResponse.get("messageCode").asText());
+                }
+            }
+        } catch (Exception e) {
+            logger.warn("Unexpected error when trying to parse error for deploying query rolesets: " + e.getMessage());
+        }
+        return false;
+    }
+}

marklogic-data-hub/src/main/resources/hub-internal-config/security/roles/data-hub-developer.json

@@ -13,6 +13,11 @@
     "data-hub-entity-model-writer"
   ],
   "privilege": [
+    {
+      "privilege-name": "add-query-rolesets",
+      "action": "http://marklogic.com/xdmp/privileges/add-query-rolesets",
+      "kind": "execute"
+    },
     {
       "privilege-name": "any-uri",
       "action": "http://marklogic.com/xdmp/privileges/any-uri",
@@ -38,6 +43,11 @@
       "action": "http://marklogic.com/xdmp/privileges/remove-path",
       "kind": "execute"
     },
+    {
+      "privilege-name": "remove-query-rolesets",
+      "action": "http://marklogic.com/xdmp/privileges/remove-query-rolesets",
+      "kind": "execute"
+    },
     {
       "privilege-name": "rest-admin",
       "action": "http://marklogic.com/xdmp/privileges/rest-admin",

marklogic-data-hub/src/test/java/com/marklogic/hub/dhs/DeployAsDeveloperTest.java

@@ -9,6 +9,7 @@
 import com.marklogic.appdeployer.command.alert.DeployAlertRulesCommand;
 import com.marklogic.appdeployer.command.databases.DeployOtherDatabasesCommand;
 import com.marklogic.appdeployer.command.schemas.LoadSchemasCommand;
+import com.marklogic.appdeployer.command.security.DeployProtectedPathsCommand;
 import com.marklogic.appdeployer.command.tasks.DeployScheduledTasksCommand;
 import com.marklogic.appdeployer.command.temporal.DeployTemporalAxesCommand;
 import com.marklogic.appdeployer.command.temporal.DeployTemporalCollectionsCommand;
@@ -17,6 +18,7 @@
 import com.marklogic.hub.DatabaseKind;
 import com.marklogic.hub.HubConfig;
 import com.marklogic.hub.HubTestBase;
+import com.marklogic.hub.dhs.installer.deploy.DeployHubQueryRolesetsCommand;
 import com.marklogic.hub.deploy.commands.LoadUserArtifactsCommand;
 import com.marklogic.hub.deploy.commands.LoadUserModulesCommand;
 import com.marklogic.hub.impl.HubConfigImpl;
@@ -169,8 +171,9 @@ public void copyStagingSslConfigToAppServices() {
     public void buildCommandList() {
         List<Command> commands = new DhsDeployer().buildCommandsForDeveloper(hubConfig);
         Collections.sort(commands, Comparator.comparing(Command::getExecuteSortOrder));
-        System.out.println(commands);
         int index = 0;
+        assertTrue(commands.get(index++) instanceof DeployProtectedPathsCommand);
+        assertTrue(commands.get(index++) instanceof DeployHubQueryRolesetsCommand);
         assertTrue(commands.get(index++) instanceof DeployOtherDatabasesCommand);
         assertTrue(commands.get(index++) instanceof LoadSchemasCommand);
         assertTrue(commands.get(index++) instanceof LoadUserModulesCommand);
@@ -182,13 +185,13 @@ public void buildCommandList() {
         assertTrue(commands.get(index++) instanceof DeployAlertConfigsCommand);
         assertTrue(commands.get(index++) instanceof DeployAlertActionsCommand);
         assertTrue(commands.get(index++) instanceof DeployAlertRulesCommand);
-        assertEquals(11, commands.size(),
+        assertEquals(13, commands.size(),
             "As of ML 10.0-3, the granular privilege for indexes doesn't seem to work with XML payloads. " +
                 "Bug https://bugtrack.marklogic.com/54231 has been created to track that. Thus, " +
                 "DeployDatabaseFieldCommand cannot be included and ml-config/database-fields/final-database.xml " +
                 "cannot be processed.");
 
-        DeployOtherDatabasesCommand dodc = (DeployOtherDatabasesCommand) commands.get(0);
+        DeployOtherDatabasesCommand dodc = (DeployOtherDatabasesCommand) commands.get(2);
         ResourceFilenameFilter filter = (ResourceFilenameFilter) dodc.getResourceFilenameFilter();
         assertEquals("(staging|final|job)-database.json", filter.getIncludePattern().pattern(),
             "DHS users aren't allowed to create their own databases, so the command for deploying databases is restricted " +

marklogic-data-hub/src/test/java/com/marklogic/hub/dhs/installer/deploy/DeployHubQueryRolesetsCommandTest.java

@@ -0,0 +1,56 @@
+package com.marklogic.hub.dhs.installer.deploy;
+
+import com.marklogic.appdeployer.AppConfig;
+import com.marklogic.appdeployer.command.CommandContext;
+import com.marklogic.hub.security.AbstractSecurityTest;
+import com.marklogic.mgmt.SaveReceipt;
+import com.marklogic.mgmt.resource.security.QueryRolesetManager;
+import org.junit.jupiter.api.Test;
+
+import static org.junit.jupiter.api.Assertions.*;
+
+public class DeployHubQueryRolesetsCommandTest extends AbstractSecurityTest {
+
+    @Override
+    protected String getRoleName() {
+        return "data-hub-developer";
+    }
+
+    @Test
+    void test() {
+        DeployHubQueryRolesetsCommand command = new DeployHubQueryRolesetsCommand();
+        assertFalse(command.cmaShouldBeUsed(null), "We don't want to use CMA for query rolesets, because " +
+            "at least in ML 10.0-3, when QRs are submitted via CMA by a user without the security role and " +
+            "one or more QRs already exist, an exception will be thrown");
+
+
+        QueryRolesetManager mgr = new QueryRolesetManager(userWithRoleBeingTestedClient);
+        final String payload = "{\n" +
+            "\t\"role-name\": [\n" +
+            "\t\t\"view-admin\",\n" +
+            "\t\t\"flexrep-user\"\n" +
+            "\t]\n" +
+            "}\n";
+
+        SaveReceipt receipt = command.saveResource(mgr, new CommandContext(new AppConfig(), userWithRoleBeingTestedClient, null), payload);
+        final String rolesetPath = receipt.getResponse().getHeaders().getLocation().toString();
+        try {
+            assertEquals(201, receipt.getResponse().getStatusCodeValue(), "The query roleset should have been created " +
+                "successfully because data-hub-developer has the add-query-rolesets privilege");
+
+            receipt = command.saveResource(mgr, new CommandContext(new AppConfig(), userWithRoleBeingTestedClient, null), payload);
+            assertNull(receipt, "The receipt object will be null because the Manage API threw an exception, since a " +
+                "user without the security role can't call POST again on a query roleset, and the data-hub-developer " +
+                "user doesn't have the security role. And a PUT call can't be made because the GET endpoint for a " +
+                "roleset doesn't support passing in the role names that constitute a roleset, so there's no way to " +
+                "figure out the roleset ID which would be required by the PUT call." +
+                "" +
+                "So instead of an exception being thrown, check the logging to verify that a message was logged " +
+                "indicating that the SEC-PERMDENIED exception can be safely ignored if the query roleset has already " +
+                "been deployed. This is the best we can do in DHF based on ML 10.0-3.");
+        } finally {
+            // The lack of an exception from this verifies that the roleset was deleted successfully
+            userWithRoleBeingTestedClient.delete(rolesetPath);
+        }
+    }
+}

marklogic-data-hub/src/test/java/com/marklogic/hub/impl/DeployProtectedPathsWhenUpdatingIndexesTest.java

@@ -33,8 +33,7 @@ public void teardown() {
 
     @Test
     public void test() {
-        // Initializing this to ensure that updateIndexes can be called
-        getDataHubAdminConfig();
+        runAsDataHubDeveloper();
 
         givenAProtectedPathFile();
         dataHub.updateIndexes();

marklogic-data-hub/src/test/java/com/marklogic/hub/security/DataHubDeveloperTest.java

@@ -3,16 +3,19 @@
 import com.fasterxml.jackson.databind.JsonNode;
 import com.fasterxml.jackson.databind.node.ObjectNode;
 import com.marklogic.hub.HubConfig;
+import com.marklogic.mgmt.SaveReceipt;
 import com.marklogic.mgmt.api.database.Database;
 import com.marklogic.mgmt.api.database.GeospatialElementIndex;
 import com.marklogic.mgmt.api.security.protectedpath.Permission;
 import com.marklogic.mgmt.api.security.protectedpath.ProtectedPath;
+import com.marklogic.mgmt.api.security.queryroleset.QueryRoleset;
 import com.marklogic.mgmt.api.task.Task;
 import com.marklogic.mgmt.api.trigger.*;
 import com.marklogic.mgmt.resource.alert.AlertActionManager;
 import com.marklogic.mgmt.resource.alert.AlertConfigManager;
 import com.marklogic.mgmt.resource.alert.AlertRuleManager;
 import com.marklogic.mgmt.resource.security.ProtectedPathManager;
+import com.marklogic.mgmt.resource.security.QueryRolesetManager;
 import com.marklogic.mgmt.resource.tasks.TaskManager;
 import com.marklogic.mgmt.resource.temporal.TemporalAxesManager;
 import com.marklogic.mgmt.resource.temporal.TemporalCollectionManager;
@@ -24,6 +27,7 @@
 import java.io.IOException;
 import java.net.URLEncoder;
 import java.util.Arrays;
+import java.util.List;
 
 import static org.junit.jupiter.api.Assertions.*;
 
@@ -106,7 +110,31 @@ public void task10CreateProtectedPaths() throws Exception {
         } finally {
             userWithRoleBeingTestedClient.delete("/manage/v2/protected-paths/" + URLEncoder.encode(pathExpression, "UTF-8") + "?force=true");
             assertFalse(adminProtectedPathManager.exists(pathExpression),
-                "The remove-path privilege should allow the user to delete a protected path");
+                "The remove-path privilege should allow the user to delete a protected path. Note that in ML 10.0-3, the unprotect-path " +
+                    "privilege is not needed to perform this operation. The Admin UI requires first unprotecting a protected path, but " +
+                    "the Manage API is able to delete a protected path without unprotecting it first.");
+        }
+    }
+
+    @Test
+    void task10CreateQueryRolesets() {
+        final QueryRolesetManager mgr = new QueryRolesetManager(userWithRoleBeingTestedClient);
+        final List<String> originalRolesetIds = mgr.getAsXml().getListItemIdRefs();
+
+        QueryRoleset qr = new QueryRoleset(null);
+        qr.setRoleName(Arrays.asList("harmonized-reader", "harmonized-updater"));
+        qr.setApi(userWithRoleBeingTestedApi);
+        SaveReceipt receipt = mgr.save(qr.getJson());
+        final String rolesetPath = receipt.getResponse().getHeaders().getLocation().toString();
+
+        final List<String> newRolesetIds = mgr.getAsXml().getListItemIdRefs();
+        try {
+            assertEquals(originalRolesetIds.size() + 1, newRolesetIds.size(),
+                "Expected one more roleset to exist");
+        } finally {
+            userWithRoleBeingTestedClient.delete(rolesetPath);
+            final List<String> rolesetIdsAfterDeletion = mgr.getAsXml().getListItemIdRefs();
+            assertEquals(originalRolesetIds.size(), rolesetIdsAfterDeletion.size());
         }
     }