job-exec: use Type=notify and timer with sdexec

Problem: jobs remain in R state when the flux-shell exits with
unkillable processes.

Run imp-shell units with Type=notify and the new sdexec stop timer.
Disable systemd's stop timer by setting TimeoutStopUsec=infinity.
This assumes the IMP has been modified to call sd_notify(3) at
appropriate transitions.

The stop timer, which is enabled by default with a timeout of 30s
and signal of SIGUSR1, may be configured or disabled via the TOML
[exec] table.

Fixes #6656

Author: garlick

src/modules/job-exec/exec.c

@@ -519,6 +519,7 @@ static int parse_service_option (json_t *jobspec,
     return 0;
 }
 
+
 static struct bulk_exec_ops exec_ops = {
     .on_start =     start_cb,
     .on_exit =      exit_cb,
@@ -604,14 +605,27 @@ static int exec_init (struct jobinfo *job)
             goto err;
         }
         /* The systemd user instance running as user flux is not privileged
-         * to signal guest processes, therefore only signal the IMP and
-         * never use SIGKILL.  See flux-framework/flux-core#6399
+         * to signal guest processes, therefore:
+         * - Set the KillMode=process so only the IMP is signaled
+         * - Use Type=notify in conjunction with IMP calling sd_notify(3) so
+         *   the unit transitions to deactivating when the shell exits.
+         * - Set TimeoutStopUsec=infinity to disable systemd's stop timeout.
+         * - Enable sdexec's stop timeout which is armed at deactivating,
+         *   delivers SIGUSR1 (proxy for SIGKILL) after 30s, then abandons
+         *   the unit and terminates the exec RPC after another 30s.
          */
         if (streq (service, "sdexec")) {
             if (flux_cmd_setopt (cmd, "SDEXEC_PROP_KillMode", "process") < 0
+                || flux_cmd_setopt (cmd, "SDEXEC_PROP_Type", "notify") < 0
+                || flux_cmd_setopt (cmd,
+                                    "SDEXEC_PROP_TimeoutStopUSec",
+                                    "infinity") < 0
+                || flux_cmd_setopt (cmd,
+                                    "SDEXEC_STOP_TIMER_SIGNAL",
+                                    config_get_sdexec_stop_timer_signal ()) < 0
                 || flux_cmd_setopt (cmd,
-                                    "SDEXEC_PROP_SendSIGKILL",
-                                    "off") < 0) {
+                                    "SDEXEC_STOP_TIMER_SEC",
+                                    config_get_sdexec_stop_timer_sec ()) < 0) {
                 flux_log_error (job->h,
                                 "Unable to set multiuser sdexec options");
                 return -1;

src/modules/job-exec/exec_config.c

@@ -36,6 +36,8 @@ struct exec_config {
     const char *exec_service;
     int exec_service_override;
     json_t *sdexec_properties;
+    int sdexec_stop_timer_sec;
+    int sdexec_stop_timer_signal;
     double default_barrier_timeout;
 };
 
@@ -107,6 +109,20 @@ json_t *config_get_sdexec_properties (void)
     return exec_conf.sdexec_properties;
 }
 
+const char *config_get_sdexec_stop_timer_sec (void)
+{
+    static char buf[32];
+    snprintf (buf, sizeof (buf), "%d", exec_conf.sdexec_stop_timer_sec);
+    return buf;
+}
+
+const char *config_get_sdexec_stop_timer_signal  (void)
+{
+    static char buf[32];
+    snprintf (buf, sizeof (buf), "%d", exec_conf.sdexec_stop_timer_signal);
+    return buf;
+}
+
 double config_get_default_barrier_timeout (void)
 {
     return exec_conf.default_barrier_timeout;
@@ -116,15 +132,19 @@ int config_get_stats (json_t **config_stats)
 {
     json_t *o = NULL;
 
-    if (!(o = json_pack ("{s:s? s:s? s:s? s:s? s:i s:f}",
+    if (!(o = json_pack ("{s:s? s:s? s:s? s:s? s:i s:f s:i s:i}",
                          "default_cwd", default_cwd,
                          "default_job_shell", exec_conf.default_job_shell,
                          "flux_imp_path", exec_conf.flux_imp_path,
                          "exec_service", exec_conf.exec_service,
                          "exec_service_override",
                          exec_conf.exec_service_override,
                          "default_barrier_timeout",
-                         exec_conf.default_barrier_timeout))) {
+                         exec_conf.default_barrier_timeout,
+                         "sdexec_stop_timer_sec",
+                         exec_conf.sdexec_stop_timer_sec,
+                         "sdexec_stop_timer_signal",
+                         exec_conf.sdexec_stop_timer_signal))) {
         errno = ENOMEM;
         return -1;
     }
@@ -153,6 +173,8 @@ static void exec_config_init (struct exec_config *ec)
     ec->exec_service = "rexec";
     ec->exec_service_override = 0;
     ec->sdexec_properties = NULL;
+    ec->sdexec_stop_timer_sec = 30;
+    ec->sdexec_stop_timer_signal = 10; // SIGUSR1
     ec->default_barrier_timeout = 1800.;
 }
 
@@ -249,6 +271,22 @@ int config_setup (flux_t *h,
         }
     }
 
+    /*  Check configuration for exec.stop-timer-* */
+    if (flux_conf_unpack (conf,
+                          &err,
+                          "{s?{s?i s?i}}",
+                          "exec",
+                            "sdexec-stop-timer-sec",
+                              &tmpconf.sdexec_stop_timer_sec,
+                            "sdexec-stop-timer-signal",
+                              &tmpconf.sdexec_stop_timer_signal) < 0) {
+        errprintf (errp,
+                   "error reading config values exec.sdexec-stop-timer-sec: %s"
+                   " or exec.sdexec-stop-timer-signal",
+                   err.text);
+        return -1;
+    }
+
     /*  Check configuration for exec.barrier-timeout */
     if (flux_conf_unpack (conf,
                           &err,

src/modules/job-exec/exec_config.h

@@ -38,6 +38,10 @@ double config_get_default_barrier_timeout (void);
 
 int config_get_stats (json_t **config_stats);
 
+const char *config_get_sdexec_stop_timer_sec (void);
+
+const char *config_get_sdexec_stop_timer_signal (void);
+
 int config_setup (flux_t *h,
                   const flux_conf_t *conf,
                   int argc,