DHFPROD-4123: DHF installer no longer assumes group or server names

The group and server names can now be defined via the command line. The server names are used for copying REST options around. Group names are also used for copying REST options around, and also for granular scheduled task privileges.

Author: rjrudin

marklogic-data-hub/src/main/java/com/marklogic/hub/deploy/commands/CreateGranularPrivilegesCommand.java

@@ -13,6 +13,7 @@
 import com.marklogic.mgmt.resource.security.PrivilegeManager;
 
 import java.util.ArrayList;
+import java.util.Arrays;
 import java.util.List;
 
 /**
@@ -21,11 +22,17 @@
 public class CreateGranularPrivilegesCommand implements Command, UndoableCommand {
 
     private HubConfig hubConfig;
+    private List<String> groupNames;
 
     public CreateGranularPrivilegesCommand(HubConfig hubConfig) {
         this.hubConfig = hubConfig;
     }
 
+    public CreateGranularPrivilegesCommand(HubConfig hubConfig, List<String> groupNames) {
+        this.hubConfig = hubConfig;
+        this.groupNames = groupNames;
+    }
+
     /**
      * It is anticipated that these privileges can always be added at the end of a deployment because none of the steps
      * before that should depend on the privileges being added.
@@ -78,29 +85,7 @@ public void execute(CommandContext context) {
         p.setAction("http://marklogic.com/xdmp/privileges/admin/database/alerts/$$database-id(" + finalDbName + ")");
         mgr.save(p.getJson());
 
-        if (hubConfig.getIsProvisionedEnvironment()) {
-            p.setPrivilegeName("admin-group-scheduled-task-Analyzer");
-            p.setAction("http://marklogic.com/xdmp/privileges/admin/group/scheduled-task/$$group-id(Analyzer)");
-            mgr.save(p.getJson());
-
-            p.setPrivilegeName("admin-group-scheduled-task-Curator");
-            p.setAction("http://marklogic.com/xdmp/privileges/admin/group/scheduled-task/$$group-id(Curator)");
-            mgr.save(p.getJson());
-
-            p.setPrivilegeName("admin-group-scheduled-task-Evaluator");
-            p.setAction("http://marklogic.com/xdmp/privileges/admin/group/scheduled-task/$$group-id(Evaluator)");
-            mgr.save(p.getJson());
-
-            p.setPrivilegeName("admin-group-scheduled-task-Operator");
-            p.setAction("http://marklogic.com/xdmp/privileges/admin/group/scheduled-task/$$group-id(Operator)");
-            mgr.save(p.getJson());
-        } else {
-            String groupName = hubConfig.getAppConfig().getGroupName();
-
-            p.setPrivilegeName("admin-group-scheduled-task-" + groupName);
-            p.setAction("http://marklogic.com/xdmp/privileges/admin/group/scheduled-task/$$group-id(" + groupName + ")");
-            mgr.save(p.getJson());
-        }
+        buildScheduledTaskPrivileges().forEach(privilege -> mgr.save(privilege.getJson()));
     }
 
     @Override
@@ -128,15 +113,9 @@ public void undo(CommandContext context) {
         mgr.deleteAtPath("/manage/v2/privileges/admin-database-alerts-" + finalDbName + "?kind=execute");
         mgr.deleteAtPath("/manage/v2/privileges/admin-database-alerts-" + stagingDbName + "?kind=execute");
 
-        if (hubConfig.getIsProvisionedEnvironment()) {
-            mgr.deleteAtPath("/manage/v2/privileges/admin-group-scheduled-task-Analyzer" + "?kind=execute");
-            mgr.deleteAtPath("/manage/v2/privileges/admin-group-scheduled-task-Curator" + "?kind=execute");
-            mgr.deleteAtPath("/manage/v2/privileges/admin-group-scheduled-task-Evaluator" + "?kind=execute");
-            mgr.deleteAtPath("/manage/v2/privileges/admin-group-scheduled-task-Operator" + "?kind=execute");
-        } else {
-            String groupName = hubConfig.getAppConfig().getGroupName();
+        getGroupNamesForScheduledTaskPrivileges().forEach(groupName -> {
             mgr.deleteAtPath("/manage/v2/privileges/admin-group-scheduled-task-" + groupName + "?kind=execute");
-        }
+        });
     }
 
     /**
@@ -183,4 +162,32 @@ protected Privilege buildPrivilege(ManageClient client, String name, String acti
         p.addRole(role);
         return p;
     }
+
+    /**
+     * Builds a list of privileges to create based on the groupNames configured on this object. If none have been
+     * configured, then the groupName in AppConfig is used.
+     *
+     * @return
+     */
+    protected List<Privilege> buildScheduledTaskPrivileges() {
+        List<Privilege> privileges = new ArrayList<>();
+        getGroupNamesForScheduledTaskPrivileges().forEach(groupName -> {
+            Privilege p = new Privilege(null, "admin-group-scheduled-task-" + groupName);
+            p.setKind("execute");
+            p.setAction("http://marklogic.com/xdmp/privileges/admin/group/scheduled-task/$$group-id(" + groupName + ")");
+            p.addRole("data-hub-developer");
+            privileges.add(p);
+        });
+        return privileges;
+    }
+
+    protected List<String> getGroupNamesForScheduledTaskPrivileges() {
+        return (groupNames != null && !groupNames.isEmpty()) ?
+            groupNames :
+            Arrays.asList(hubConfig.getAppConfig().getGroupName());
+    }
+
+    public List<String> getGroupNames() {
+        return groupNames;
+    }
 }

marklogic-data-hub/src/main/java/com/marklogic/hub/dhs/installer/Options.java

@@ -45,6 +45,20 @@ public class Options {
     )
     private boolean disableSsl;
 
+    @Parameter(
+        names = {"--groups"},
+        description = "Comma-delimited list of group names that REST options should be copied to, and for which granular " +
+            "privileges should be created for scheduled tasks"
+    )
+    private String groupNames = "Evaluator,Curator,Analyzer,Operator";
+
+    @Parameter(
+        names = {"--servers"},
+        description = "Comma-delimited list of server names. Search options are expected to be loaded into the first server, and then they " +
+            "will be copied to the appropriate locations for each of the other servers combined with each of the groups defined by --groups."
+    )
+    private String serverNames = "data-hub-STAGING,data-hub-FINAL,data-hub-ANALYTICS,data-hub-ANALYTICS-REST,data-hub-OPERATION,data-hub-OPERATION-REST";
+
     @DynamicParameter(
         names = "-P",
         description = "Use this argument to include any property supported by DHF; e.g. -PmlHost=somehost"
@@ -98,4 +112,20 @@ public boolean isDisableSsl() {
     public void setDisableSsl(boolean disableSsl) {
         this.disableSsl = disableSsl;
     }
+
+    public String getGroupNames() {
+        return groupNames;
+    }
+
+    public void setGroupNames(String groupNames) {
+        this.groupNames = groupNames;
+    }
+
+    public String getServerNames() {
+        return serverNames;
+    }
+
+    public void setServerNames(String serverNames) {
+        this.serverNames = serverNames;
+    }
 }

marklogic-data-hub/src/main/java/com/marklogic/hub/dhs/installer/command/InstallIntoDhsCommand.java

@@ -7,6 +7,7 @@
 import com.marklogic.appdeployer.command.security.DeployPrivilegesCommand;
 import com.marklogic.appdeployer.command.security.DeployRolesCommand;
 import com.marklogic.appdeployer.command.triggers.DeployTriggersCommand;
+import com.marklogic.hub.DatabaseKind;
 import com.marklogic.hub.deploy.HubAppDeployer;
 import com.marklogic.hub.deploy.commands.*;
 import com.marklogic.hub.dhs.installer.Options;
@@ -34,7 +35,7 @@ public void run(ApplicationContext context, Options options) {
 
         String groupName = "Evaluator";
         modifyHubConfigForDhs(groupName);
-        deployer.setCommands(buildCommandsForDhs());
+        deployer.setCommands(buildCommandsForDhs(options));
         deployer.deploy(hubConfig.getAppConfig());
 
         // Update the servers in the Curator group
@@ -50,10 +51,12 @@ public void run(ApplicationContext context, Options options) {
      * In the spirit of whitelisting, we'll only setup the commands that we know we need for installing DHF.
      * We may need a more broad set of commands for user files.
      */
-    protected List<Command> buildCommandsForDhs() {
+    protected List<Command> buildCommandsForDhs(Options options) {
         DeployOtherDatabasesCommand dbCommand = new DeployOtherDatabasesCommand();
         dbCommand.setDeployDatabaseCommandFactory(new HubDeployDatabaseCommandFactory(hubConfig));
 
+        List<String> groupNames = Arrays.asList(options.getGroupNames().split(","));
+
         List<Command> commands = new ArrayList<>();
         commands.add(new DeployPrivilegesCommand());
 
@@ -82,10 +85,12 @@ protected List<Command> buildCommandsForDhs() {
         // another DatabaseClient, which the Versions class will do.
         commands.add(new GenerateFunctionMetadataCommand(hubConfig, true));
 
-        commands.add(new CopyQueryOptionsCommand(hubConfig));
+        commands.add(new CopyQueryOptionsCommand(hubConfig, groupNames,
+            Arrays.asList(options.getServerNames().split(",")), hubConfig.getDbName(DatabaseKind.JOB)
+        ));
         commands.add(new UpdateDhsModulesPermissionsCommand(hubConfig));
 
-        commands.add(new CreateGranularPrivilegesCommand(hubConfig));
+        commands.add(new CreateGranularPrivilegesCommand(hubConfig, groupNames));
 
         return commands;
     }

marklogic-data-hub/src/main/java/com/marklogic/hub/dhs/installer/deploy/CopyQueryOptionsCommand.java

@@ -6,61 +6,99 @@
 import com.marklogic.hub.HubConfig;
 import com.marklogic.hub.deploy.commands.LoadUserModulesCommand;
 
+import java.util.List;
+import java.util.stream.Collectors;
+
 public class CopyQueryOptionsCommand extends AbstractCommand {
 
     private HubConfig hubConfig;
+    private List<String> groupNames;
+    private List<String> serverNames;
+    private String jobDatabaseName;
+
+    private String scriptResponse;
 
-    public CopyQueryOptionsCommand(HubConfig hubConfig) {
+    public CopyQueryOptionsCommand(HubConfig hubConfig, List<String> groupNames, List<String> serverNames, String jobDatabaseName) {
         this.hubConfig = hubConfig;
+        this.groupNames = groupNames;
+        this.serverNames = serverNames;
+        this.jobDatabaseName = jobDatabaseName;
 
         // Need to run this after the contents of ml-modules-final is loaded
         setExecuteSortOrder(new LoadUserModulesCommand().getExecuteSortOrder() + 1);
     }
 
     @Override
     public void execute(CommandContext context) {
-        String script = "declareUpdate();\n" +
-            "[\n" +
-            "  '/Evaluator/data-hub-JOBS/rest-api/options/jobs.xml',\n" +
-            "  '/Evaluator/data-hub-JOBS/rest-api/options/traces.xml',\n" +
-            "  '/Evaluator/data-hub-STAGING/rest-api/options/default.xml',\n" +
-            "  '/Evaluator/data-hub-STAGING/rest-api/options/exp-default.xml'\n" +
-            "].forEach(uri => {\n" +
-            "  if (fn.docAvailable(uri)) { console.log('Reproducing: ' + uri); xdmp.documentInsert(uri.replace('Evaluator', 'Curator'), cts.doc(uri), \n" +
-            "    xdmp.documentGetPermissions(uri), xdmp.documentGetCollections(uri));}\n" +
-            "});\n" +
-
-            "\n" +
-            "const stagingUri = '/Evaluator/data-hub-STAGING/rest-api/options/default.xml';\n" +
-            "if (fn.docAvailable(stagingUri)) { " +
-            "  xdmp.documentInsert('/Evaluator/data-hub-FINAL/rest-api/options/default.xml', cts.doc(stagingUri), xdmp.documentGetPermissions(stagingUri), xdmp.documentGetCollections(stagingUri));\n" +
-            "  xdmp.documentInsert('/Curator/data-hub-FINAL/rest-api/options/default.xml', cts.doc(stagingUri), xdmp.documentGetPermissions(stagingUri), xdmp.documentGetCollections(stagingUri));" +
-            "}\n\n" +
-
-            "const explorerUri = '/Evaluator/data-hub-STAGING/rest-api/options/exp-default.xml';\n" +
-            "if (fn.docAvailable(explorerUri)) { " +
-            "  xdmp.documentInsert('/Evaluator/data-hub-FINAL/rest-api/options/exp-default.xml', cts.doc(explorerUri), xdmp.documentGetPermissions(explorerUri), xdmp.documentGetCollections(explorerUri));\n" +
-            "  xdmp.documentInsert('/Curator/data-hub-FINAL/rest-api/options/exp-default.xml', cts.doc(explorerUri), xdmp.documentGetPermissions(explorerUri), xdmp.documentGetCollections(explorerUri));" +
-            "}\n\n" +
-
-            "[\n" +
-            "  '/Analyzer/data-hub-ANALYTICS/rest-api/options/default.xml',\n" +
-            "  '/Analyzer/data-hub-ANALYTICS-REST/rest-api/options/default.xml',\n" +
-            "  '/Operator/data-hub-OPERATION/rest-api/options/default.xml',\n" +
-            "  '/Operator/data-hub-OPERATION-REST/rest-api/options/default.xml',\n" +
-            "  '/Evaluator/data-hub-ANALYTICS/rest-api/options/default.xml',\n" +
-            "  '/Evaluator/data-hub-OPERATION/rest-api/options/default.xml'\n" +
-            "].forEach(uri => {\n" +
-            "  if (fn.docAvailable(stagingUri)) {console.log('Inserting: ' + uri); xdmp.documentInsert(uri, cts.doc(stagingUri), xdmp.documentGetPermissions(stagingUri), \n" +
-            "    xdmp.documentGetCollections(stagingUri));}\n" +
-            "})";
+        final String script = buildScript();
 
         DatabaseClient client = hubConfig.newModulesDbClient();
         try {
-            client.newServerEval().javascript(script).eval().close();
+            scriptResponse = client.newServerEval().javascript(script)
+                .addVariable("stagingServer", serverNames.get(0))
+                .addVariable("jobServer", jobDatabaseName)
+                .addVariable("groups", groupNames.stream().collect(Collectors.joining(",")))
+                .addVariable("servers", serverNames.subList(1, serverNames.size()).stream().collect(Collectors.joining(",")))
+                .evalAs(String.class);
         } finally {
             client.release();
         }
 
+        logger.info("Copied search options to the following URIs in the modules database:\n" + scriptResponse);
+    }
+
+    protected String buildScript() {
+        return "declareUpdate();\n" +
+            "var stagingServer;\n" +
+            "var jobServer;\n" +
+            "var groups;\n" +
+            "var servers;\n" +
+            "\n" +
+            "groups = groups.split(',');\n" +
+            "servers = servers.split(',');\n" +
+            "const firstGroup = groups[0];\n" +
+            "\n" +
+            "// This array is returned for testing purposes\n" +
+            "const createdUris = [];\n" +
+            "\n" +
+            "let stagingUris = cts.uriMatch('/' + firstGroup + '/' + stagingServer + '/rest-api/options/*').toArray().forEach(uri => {\n" +
+            "  const doc = cts.doc(uri);\n" +
+            "  const perms = xdmp.documentGetPermissions(uri);\n" +
+            "  const collections = xdmp.documentGetCollections(uri);\n" +
+            "  // Copy options from first group to other groups for stagingServer\n" +
+            "  groups.slice(1).map(group => {\n" +
+            "    let newUri = uri.toString().replace('/' + firstGroup + '/', '/' + group + '/');\n" +
+            "    xdmp.documentInsert(newUri, doc, perms, collections);\n" +
+            "    createdUris.push(newUri);\n" +
+            "  });\n" +
+            "  \n" +
+            "  // Copy options to every group for all other servers\n" +
+            "  groups.forEach(group => {\n" +
+            "    servers.forEach(server => {\n" +
+            "      let newUri = uri.toString().replace('/' + firstGroup + '/', '/' + group + '/');\n" +
+            "      newUri = newUri.replace('/' + stagingServer + '/', '/' + server + '/');\n" +
+            "      xdmp.documentInsert(newUri, doc, perms, collections);\n" +
+            "      createdUris.push(newUri);\n" +
+            "    });\n" +
+            "  });\n" +
+            "});\n" +
+            "\n" +
+            "// Copy jobs options to job servers in other groups\n" +
+            "cts.uriMatch('/' + firstGroup + '/' + jobServer + '/rest-api/options/*').toArray().map(uri => {\n" +
+            "  const doc = cts.doc(uri);\n" +
+            "  const perms = xdmp.documentGetPermissions(uri);\n" +
+            "  const collections = xdmp.documentGetCollections(uri);\n" +
+            "  groups.slice(1).forEach(group => {\n" +
+            "    let newUri = uri.toString().replace('/' + firstGroup + '/', '/' + group + '/');\n" +
+            "    xdmp.documentInsert(newUri, doc, perms, collections);\n" +
+            "    createdUris.push(newUri);\n" +
+            "  });\n" +
+            "});\n" +
+            "\n" +
+            "createdUris;";
+    }
+
+    public String getScriptResponse() {
+        return scriptResponse;
     }
 }

marklogic-data-hub/src/test/java/com/marklogic/bootstrap/Installer.java

@@ -51,6 +51,13 @@ public void bootstrapHub() {
             dataHubOperator.setPassword("password");
             dataHubOperator.addRole("data-hub-operator");
             dataHubOperator.save();
+
+            User testAdmin = new User(new API(adminHubConfig.getManageClient()), "test-admin-for-data-hub-tests");
+            testAdmin.setDescription("This user is intended to be used by DHF tests that require admin or " +
+                "admin-like capabilities, such as being able to deploy a DHF application");
+            testAdmin.setPassword("password");
+            testAdmin.addRole("admin");
+            testAdmin.save();
         }
 
         if (getDataHubAdminConfig().getIsProvisionedEnvironment()) {

marklogic-data-hub/src/test/java/com/marklogic/hub/HubTestBase.java

@@ -459,6 +459,10 @@ protected HubConfigImpl runAsDataHubOperator() {
         return runAsFlowOperator();
     }
 
+    protected HubConfigImpl runAsAdmin() {
+        return runAsUser("test-admin-for-data-hub-tests", "password");
+    }
+    
     protected HubConfigImpl runAsUser(String mlUsername, String mlPassword) {
         adminHubConfig.setMlUsername(mlUsername);
         adminHubConfig.setMlPassword(mlPassword);