broker: run shutdown script not cleanup commands

Problem: the cleanup commands pushed into broker memory in rc1
are not easily maintained or extended.

Create a new "shutdown" script that lives next to rc1 and rc3.
Run this script instead of the "cleanup" commands when the broker
CLEANUP state is entered.  Upon completion of the shutdown script,
the broker transitions to the SHUTDOWN state.

Author: garlick

etc/Makefile.am

@@ -10,7 +10,8 @@ tmpfiles_DATA = flux.conf
 
 dist_fluxconf_SCRIPTS = \
         rc1 \
-        rc3
+        rc3 \
+        shutdown
 
 dist_fluxrc1_SCRIPTS = \
         rc1.d/02-cron

etc/rc1

@@ -109,14 +109,3 @@ if test $RANK -eq 0 -a "${FLUX_SCHED_MODULE}" != "none" \
                     -a -z "$(lookup_sched_module)"; then
     flux module load ${FLUX_SCHED_MODULE:-sched-simple}
 fi
-
-if test $RANK -eq 0; then
-    if test -z "${FLUX_DISABLE_JOB_CLEANUP}"; then
-	flux admin cleanup-push <<-EOT
-	flux jobtap remove perilog 2>/dev/null || true
-	flux queue stop --quiet --all --nocheckpoint
-	flux cancel --user=all --quiet --states RUN
-	flux queue idle --quiet
-	EOT
-    fi
-fi

etc/shutdown

@@ -0,0 +1,6 @@
+#!/bin/sh
+
+flux jobtap remove perilog 2>/dev/null
+flux queue stop --quiet --all --nocheckpoint
+flux cancel --user=all --quiet --states RUN
+flux queue idle --quiet

src/broker/broker.c

@@ -594,7 +594,11 @@ static void init_attrs_rc_paths (attr_t *attrs)
                   flux_conf_builtin_get ("rc1_path", FLUX_CONF_AUTO),
                   0) < 0)
         log_err_exit ("attr_add rc1_path");
-
+    if (attr_add (attrs,
+                  "broker.shutdown_path",
+                  flux_conf_builtin_get ("shutdown_path", FLUX_CONF_AUTO),
+                  0) < 0)
+        log_err_exit ("attr_add shutdown_path");
     if (attr_add (attrs,
                   "broker.rc3_path",
                   flux_conf_builtin_get ("rc3_path", FLUX_CONF_AUTO),
@@ -723,7 +727,7 @@ static int create_runat_rc2 (struct runat *r, const char *argz, size_t argz_len)
 
 static int create_runat_phases (broker_ctx_t *ctx)
 {
-    const char *rc1, *rc3, *local_uri;
+    const char *rc1, *rc3, *shutdown, *local_uri;
     bool rc2_none = false;
 
     if (attr_get (ctx->attrs, "local-uri", &local_uri, NULL) < 0) {
@@ -734,6 +738,10 @@ static int create_runat_phases (broker_ctx_t *ctx)
         log_err ("broker.rc1_path is not set");
         return -1;
     }
+    if (attr_get (ctx->attrs, "broker.shutdown_path", &shutdown, NULL) < 0) {
+        log_err ("broker.shutdown_path is not set");
+        return -1;
+    }
     if (attr_get (ctx->attrs, "broker.rc3_path", &rc3, NULL) < 0) {
         log_err ("broker.rc3_path is not set");
         return -1;
@@ -768,6 +776,18 @@ static int create_runat_phases (broker_ctx_t *ctx)
         }
     }
 
+    /* shutdown - clean up in preparation for instance shutdown
+     */
+    if (ctx->rank == 0 && shutdown && strlen (shutdown) > 0) {
+        if (runat_push_shell_command (ctx->runat,
+                                      "shutdown",
+                                      shutdown,
+                                      RUNAT_FLAG_LOG_STDIO) < 0) {
+            log_err ("runat_push_shell_command shutdown");
+            return -1;
+        }
+    }
+
     /* rc3 - finalization
      */
     if (rc3 && strlen (rc3) > 0) {

src/broker/state_machine.c

@@ -358,11 +358,16 @@ static void action_run (struct state_machine *s)
 #endif
 }
 
+/* In the cleanup state, we run the shutdown script.  When the shutdown
+ * script is complete, we enter shutdown state.
+ */
 static void action_cleanup (struct state_machine *s)
 {
-    if (runat_is_defined (s->ctx->runat, "cleanup")) {
-        if (runat_start (s->ctx->runat, "cleanup", runat_completion_cb, s) < 0) {
-            flux_log_error (s->ctx->h, "runat_start cleanup");
+    if (runat_is_defined (s->ctx->runat, "shutdown")) {
+        if (runat_start (s->ctx->runat,
+                         "shutdown",
+                         runat_completion_cb, s) < 0) {
+            flux_log_error (s->ctx->h, "runat_start shutdown");
             state_machine_post (s, "cleanup-fail");
         }
     }
@@ -525,8 +530,8 @@ void state_machine_kill (struct state_machine *s, int signum)
                 state_machine_post (s, "shutdown");
             break;
         case STATE_CLEANUP:
-            if (runat_abort (s->ctx->runat, "cleanup") < 0)
-                flux_log_error (h, "runat_abort cleanup (signal %d)", signum);
+            if (runat_abort (s->ctx->runat, "shutdown") < 0)
+                flux_log_error (h, "runat_abort shutdown (signal %d)", signum);
             break;
         case STATE_FINALIZE:
             (void)runat_abort (s->ctx->runat, "rc3");
@@ -603,7 +608,7 @@ static void runat_completion_cb (struct runat *r, const char *name, void *arg)
             s->ctx->exit_rc = rc;
         state_machine_post (s, rc == 0 ? "rc2-success" : "rc2-fail");
     }
-    else if (streq (name, "cleanup")) {
+    else if (streq (name, "shutdown")) {
         if (s->ctx->exit_rc == 0 && rc != 0)
             s->ctx->exit_rc = rc;
         state_machine_post (s, rc == 0 ? "cleanup-success" : "cleanup-fail");