broker: add shutdown force option

Problem: we need a way for the shutdown command to pass a
"force" option to the shutdown script.

Add a {"force"?b} parameter to the shutdown request payload.
If specified, the shutdown script is called with the --force option.

Author: garlick

src/broker/runat.c

@@ -576,6 +576,28 @@ int runat_push_command_line (struct runat *r,
     return rc;
 }
 
+int runat_add_arg (struct runat *r, const char *name, const char *arg)
+{
+    struct runat_entry *entry;
+    struct runat_command *command;
+
+    if (!r || !name || !arg) {
+        errno = EINVAL;
+        return -1;
+    }
+    if (!(entry = zhashx_lookup (r->entries, name))) {
+        errno = ENOENT;
+        return -1;
+    }
+    command = zlist_first (entry->commands);
+    while (command) {
+        if (flux_cmd_argv_append (command->cmd, arg) < 0)
+            return -1;
+        command = zlist_next (entry->commands);
+    }
+    return 0;
+}
+
 int runat_get_exit_code (struct runat *r, const char *name, int *rc)
 {
     struct runat_entry *entry;

src/broker/runat.h

@@ -61,6 +61,10 @@ int runat_push_command_line (struct runat *r,
                              const char *cmdline,
                              int flags);
 
+/* Add an argument to all commands registered under 'name'.
+ */
+int runat_add_arg (struct runat *r, const char *name, const char *arg);
+
 /* Get exit code of completed command list.
  * If multiple commands fail, the exit code is that of the first failure.
  */

src/broker/shutdown.c

@@ -19,6 +19,7 @@
 #include "src/common/libutil/stdlog.h"
 
 #include "shutdown.h"
+#include "runat.h"
 #include "state_machine.h"
 
 #include "broker.h"
@@ -154,14 +155,24 @@ static void start_cb (flux_t *h,
     struct shutdown *shutdown = arg;
     flux_error_t error;
     const char *errmsg = NULL;
+    int force = 0;
 
-    if (flux_request_decode (msg, NULL, NULL) < 0)
+    if (flux_request_unpack (msg,
+                             NULL,
+                             "{s?b}",
+                             "force", &force) < 0)
         goto error;
     if (shutdown->request) {
         errno = EINVAL;
         errmsg = "shutdown is already in progress";
         goto error;
     }
+    if (force) {
+        if (runat_add_arg (shutdown->ctx->runat, "shutdown", "-f") < 0) {
+            errmsg = "error setting shutdown -f option";
+            goto error;
+        }
+    }
     if (state_machine_shutdown (shutdown->ctx->state_machine, &error) < 0) {
         errmsg = error.text;
         goto error;

src/cmd/builtin/shutdown.c

@@ -119,6 +119,7 @@ static int subcmd (optparse_t *p, int ac, char *av[])
     int loglevel = quiet ? LOG_WARNING
                  : verbose == 0 ? LOG_INFO : LOG_DEBUG;
     const char *target = NULL;
+    int force = 0;
 
     log_init ("flux-shutdown");
 
@@ -161,15 +162,19 @@ static int subcmd (optparse_t *p, int ac, char *av[])
         log_msg ("shutdown will dump KVS (this may take some time)");
     }
 
+    if (optparse_hasopt (p, "force"))
+        force = 1;
+
     /* N.B. set nodeid=FLUX_NODEID_ANY so we get immediate error from
      * broker if run on rank > 0.
      */
     if (!(f = flux_rpc_pack (h,
                              "shutdown.start",
                              FLUX_NODEID_ANY,
                              flags,
-                             "{s:i}",
-                             "loglevel", loglevel)))
+                             "{s:i s:b}",
+                             "loglevel", loglevel,
+                             "force", force)))
         log_err_exit ("could not send shutdown.start request");
 
     if ((flags & FLUX_RPC_STREAMING))
@@ -209,6 +214,9 @@ static struct optparse_option opts[] = {
     { .name = "no", .key = 'n', .has_arg = 0,
       .usage = "Answer no to any yes/no questions",
     },
+    { .name = "force", .key = 'f', .has_arg = 0,
+      .usage = "Force shutdown even if jobs are running",
+    },
     OPTPARSE_TABLE_END
 };