Fix: sbd-integration: sync pacemakerd with sbd

daemons/pacemakerd/pacemakerd.c

@@ -40,8 +40,25 @@ static bool global_keep_tracking = false;
 #define PCMK_PROCESS_CHECK_INTERVAL 5
 
 static crm_trigger_t *shutdown_trigger = NULL;
+static crm_trigger_t *startup_trigger = NULL;
 static const char *pid_file = PCMK_RUN_DIR "/pacemaker.pid";
 
+/* state we report when asked via pacemakerd-api status-ping */
+static const char *pacemakerd_state = XML_PING_ATTR_PACEMAKERDSTATE_INIT;
+static gboolean running_with_sbd = FALSE; /* local copy */
+/* When contacted via pacemakerd-api by a client having sbd in
+ * the name we assume it is sbd-daemon which wants to know
+ * if pacemakerd shutdown gracefully.
+ * Thus when everything is shutdown properly pacemakerd
+ * waits till it has reported the graceful completion of
+ * shutdown to sbd and just when sbd-client closes the
+ * connection we can assume that the report has arrived
+ * properly so that pacemakerd can finally exit.
+ * Following two variables are used to track that handshake.
+ */
+static unsigned int shutdown_complete_state_reported_to = 0;
+static gboolean shutdown_complete_state_reported_client_closed = FALSE;
+
 typedef struct pcmk_child_s {
     pid_t pid;
     long flag;
@@ -374,21 +391,20 @@ escalate_shutdown(gpointer data)
 static gboolean
 pcmk_shutdown_worker(gpointer user_data)
 {
-    static int phase = 0;
+    static int phase = SIZEOF(pcmk_children);
     static time_t next_log = 0;
-    static int max = SIZEOF(pcmk_children);
 
     int lpc = 0;
 
-    if (phase == 0) {
+    if (phase == SIZEOF(pcmk_children)) {
         crm_notice("Shutting down Pacemaker");
-        phase = max;
+        pacemakerd_state = XML_PING_ATTR_PACEMAKERDSTATE_SHUTTINGDOWN;
     }
 
     for (; phase > 0; phase--) {
         /* Don't stop anything with start_seq < 1 */
 
-        for (lpc = max - 1; lpc >= 0; lpc--) {
+        for (lpc = SIZEOF(pcmk_children) - 1; lpc >= 0; lpc--) {
             pcmk_child_t *child = &(pcmk_children[lpc]);
 
             if (phase != child->start_seq) {
@@ -436,6 +452,11 @@ pcmk_shutdown_worker(gpointer user_data)
     }
 
     crm_notice("Shutdown complete");
+    pacemakerd_state = XML_PING_ATTR_PACEMAKERDSTATE_SHUTDOWNCOMPLETE;
+    if (!fatal_error && running_with_sbd &&
+        !shutdown_complete_state_reported_client_closed) {
+        return TRUE;
+    }
 
     {
         const char *delay = pcmk__env_option("shutdown_delay");
@@ -489,6 +510,51 @@ pcmk_ipc_accept(qb_ipcs_connection_t * c, uid_t uid, gid_t gid)
     return 0;
 }
 
+static void
+pcmk_handle_ping_request(pcmk__client_t *c, xmlNode *msg, uint32_t id)
+{
+    const char *value = NULL;
+    xmlNode *ping = NULL;
+    xmlNode *reply = NULL;
+    time_t pinged = time(NULL);
+    const char *from = crm_element_value(msg, F_CRM_SYS_FROM);
+
+    /* Pinged for status */
+    crm_trace("Pinged from %s.%s",
+              crm_str(crm_element_value(msg, F_CRM_ORIGIN)),
+              from?from:"unknown");
+    ping = create_xml_node(NULL, XML_CRM_TAG_PING);
+    value = crm_element_value(msg, F_CRM_SYS_TO);
+    crm_xml_add(ping, XML_PING_ATTR_SYSFROM, value);
+    crm_xml_add(ping, XML_PING_ATTR_PACEMAKERDSTATE, pacemakerd_state);
+    crm_xml_add_ll(ping, XML_ATTR_TSTAMP, (long long) pinged);
+    crm_xml_add(ping, XML_PING_ATTR_STATUS, "ok");
+    reply = create_reply(msg, ping);
+    free_xml(ping);
+    if (reply) {
+        if (pcmk__ipc_send_xml(c, id, reply, crm_ipc_server_event) !=
+                pcmk_rc_ok) {
+            crm_err("Failed sending ping-reply");
+        }
+        free_xml(reply);
+    } else {
+        crm_err("Failed building ping-reply");
+    }
+    /* just proceed state on sbd pinging us */
+    if (from && strstr(from, "sbd")) {
+        if (crm_str_eq(pacemakerd_state,
+                       XML_PING_ATTR_PACEMAKERDSTATE_SHUTDOWNCOMPLETE,
+                       TRUE)) {
+            shutdown_complete_state_reported_to = c->pid;
+        } else if (crm_str_eq(pacemakerd_state,
+                              XML_PING_ATTR_PACEMAKERDSTATE_WAITPING,
+                              TRUE)) {
+            pacemakerd_state = XML_PING_ATTR_PACEMAKERDSTATE_STARTINGDAEMONS;
+            mainloop_set_trigger(startup_trigger);
+        }
+    }
+}
+
 /* Exit code means? */
 static int32_t
 pcmk_ipc_dispatch(qb_ipcs_connection_t * qbc, void *data, size_t size)
@@ -514,6 +580,9 @@ pcmk_ipc_dispatch(qb_ipcs_connection_t * qbc, void *data, size_t size)
         crm_trace("Ignoring IPC request to purge node "
                   "because peer cache is not used");
 
+    } else if (crm_str_eq(task, CRM_OP_PING, TRUE)) {
+        pcmk_handle_ping_request(c, msg, id);
+
     } else {
         crm_debug("Unrecognized IPC command '%s' sent to pacemakerd",
                   crm_str(task));
@@ -533,6 +602,12 @@ pcmk_ipc_closed(qb_ipcs_connection_t * c)
         return 0;
     }
     crm_trace("Connection %p", c);
+    if (shutdown_complete_state_reported_to == client->pid) {
+        shutdown_complete_state_reported_client_closed = TRUE;
+        if (shutdown_trigger) {
+            mainloop_set_trigger(shutdown_trigger);
+        }
+    }
     pcmk__free_client(client);
     return 0;
 }
@@ -924,8 +999,8 @@ find_and_track_existing_processes(void)
     return pcmk_rc_ok;
 }
 
-static void
-init_children_processes(void)
+static gboolean
+init_children_processes(void *user_data)
 {
     int start_seq = 1, lpc = 0;
     static int max = SIZEOF(pcmk_children);
@@ -951,6 +1026,8 @@ init_children_processes(void)
      * This may be useful for the daemons to know
      */
     setenv("PCMK_respawned", "true", 1);
+    pacemakerd_state = XML_PING_ATTR_PACEMAKERDSTATE_RUNNING;
+    return TRUE;
 }
 
 static void
@@ -1154,6 +1231,7 @@ main(int argc, char **argv)
 
     if(pcmk_locate_sbd() > 0) {
         setenv("PCMK_watchdog", "true", 1);
+        running_with_sbd = TRUE;
     } else {
         setenv("PCMK_watchdog", "false", 1);
     }
@@ -1170,7 +1248,13 @@ main(int argc, char **argv)
     mainloop_add_signal(SIGTERM, pcmk_shutdown);
     mainloop_add_signal(SIGINT, pcmk_shutdown);
 
-    init_children_processes();
+    if (running_with_sbd) {
+        pacemakerd_state = XML_PING_ATTR_PACEMAKERDSTATE_WAITPING;
+        startup_trigger = mainloop_add_trigger(G_PRIORITY_HIGH, init_children_processes, NULL);
+    } else {
+        pacemakerd_state = XML_PING_ATTR_PACEMAKERDSTATE_STARTINGDAEMONS;
+        init_children_processes(NULL);
+    }
 
     crm_notice("Pacemaker daemon successfully started and accepting connections");
     g_main_loop_run(mainloop);

include/crm/common/Makefile.am

@@ -12,7 +12,7 @@ MAINTAINERCLEANFILES = Makefile.in
 headerdir=$(pkgincludedir)/crm/common
 
 header_HEADERS = xml.h ipc.h util.h iso8601.h mainloop.h logging.h results.h \
-		 nvpair.h acl.h ipc_controld.h
+		 nvpair.h acl.h ipc_controld.h ipc_pacemakerd.h
 noinst_HEADERS = internal.h alerts_internal.h \
 		 iso8601_internal.h remote_internal.h xml_internal.h \
 		 ipc_internal.h output.h cmdline_internal.h curses_internal.h \

include/crm/common/ipc_pacemakerd.h

@@ -0,0 +1,71 @@
+/*
+ * Copyright 2020 the Pacemaker project contributors
+ *
+ * The version control history for this file may have further details.
+ *
+ * This source code is licensed under the GNU Lesser General Public License
+ * version 2.1 or later (LGPLv2.1+) WITHOUT ANY WARRANTY.
+ */
+
+#ifndef PCMK__IPC_PACEMAKERD__H
+#  define PCMK__IPC_PACEMAKERD__H
+
+#ifdef __cplusplus
+extern "C" {
+#endif
+
+/**
+ * \file
+ * \brief IPC commands for Pacemakerd
+ *
+ * \ingroup core
+ */
+
+#include <sys/types.h>       // time_t
+#include <crm/common/ipc.h>  // pcmk_ipc_api_t
+
+enum pcmk_pacemakerd_state {
+    pcmk_pacemakerd_state_invalid = -1,
+    pcmk_pacemakerd_state_init = 0,
+    pcmk_pacemakerd_state_starting_daemons,
+    pcmk_pacemakerd_state_wait_for_ping,
+    pcmk_pacemakerd_state_running,
+    pcmk_pacemakerd_state_shutting_down,
+    pcmk_pacemakerd_state_shutdown_complete,
+    pcmk_pacemakerd_state_max = pcmk_pacemakerd_state_shutdown_complete,
+};
+
+//! Possible types of pacemakerd replies
+enum pcmk_pacemakerd_api_reply {
+    pcmk_pacemakerd_reply_unknown,
+    pcmk_pacemakerd_reply_ping,
+};
+
+/*!
+ * Pacemakerd reply passed to event callback
+ */
+typedef struct {
+    enum pcmk_pacemakerd_api_reply reply_type;
+
+    union {
+        // pcmk_pacemakerd_reply_ping
+        struct {
+            const char *sys_from;
+            enum pcmk_pacemakerd_state state;
+            time_t last_good;
+            int status;
+        } ping;
+    } data;
+} pcmk_pacemakerd_api_reply_t;
+
+int pcmk_pacemakerd_api_ping(pcmk_ipc_api_t *api, const char *ipc_name);
+enum pcmk_pacemakerd_state
+    pcmk_pacemakerd_api_daemon_state_text2enum(const char *state);
+const char
+    *pcmk_pacemakerd_api_daemon_state_enum2text(enum pcmk_pacemakerd_state state);
+
+#ifdef __cplusplus
+}
+#endif
+
+#endif // PCMK__IPC_PACEMAKERD__H

include/crm/msg_xml.h

@@ -123,6 +123,13 @@ extern "C" {
 #  define XML_PING_ATTR_STATUS		"result"
 #  define XML_PING_ATTR_SYSFROM		"crm_subsystem"
 #  define XML_PING_ATTR_CRMDSTATE   "crmd_state"
+#  define XML_PING_ATTR_PACEMAKERDSTATE "pacemakerd_state"
+#  define XML_PING_ATTR_PACEMAKERDSTATE_INIT "init"
+#  define XML_PING_ATTR_PACEMAKERDSTATE_STARTINGDAEMONS "starting_daemons"
+#  define XML_PING_ATTR_PACEMAKERDSTATE_WAITPING "wait_for_ping"
+#  define XML_PING_ATTR_PACEMAKERDSTATE_RUNNING "running"
+#  define XML_PING_ATTR_PACEMAKERDSTATE_SHUTTINGDOWN "shutting_down"
+#  define XML_PING_ATTR_PACEMAKERDSTATE_SHUTDOWNCOMPLETE "shutdown_complete"
 
 #  define XML_TAG_FRAGMENT		"cib_fragment"
 

lib/common/Makefile.am

@@ -50,6 +50,7 @@ libcrmcommon_la_SOURCES	+= io.c
 libcrmcommon_la_SOURCES	+= ipc_client.c
 libcrmcommon_la_SOURCES	+= ipc_common.c
 libcrmcommon_la_SOURCES	+= ipc_controld.c
+libcrmcommon_la_SOURCES	+= ipc_pacemakerd.c
 libcrmcommon_la_SOURCES	+= ipc_server.c
 libcrmcommon_la_SOURCES	+= iso8601.c
 libcrmcommon_la_SOURCES	+= logging.c

lib/common/crmcommon_private.h

@@ -175,6 +175,7 @@ typedef struct pcmk__ipc_methods_s {
 struct pcmk_ipc_api_s {
     enum pcmk_ipc_server server;          // Daemon this IPC API instance is for
     enum pcmk_ipc_dispatch dispatch_type; // How replies should be dispatched
+    size_t ipc_size_max;                  // maximum IPC buffer size
     crm_ipc_t *ipc;                       // IPC connection
     mainloop_io_t *mainloop_io;     // If using mainloop, I/O source for IPC
     bool free_on_disconnect;        // Whether disconnect should free object
@@ -209,4 +210,7 @@ bool pcmk__valid_ipc_header(const pcmk__ipc_header_t *header);
 G_GNUC_INTERNAL
 pcmk__ipc_methods_t *pcmk__controld_api_methods(void);
 
+G_GNUC_INTERNAL
+pcmk__ipc_methods_t *pcmk__pacemakerd_api_methods(void);
+
 #endif  // CRMCOMMON_PRIVATE__H

lib/common/ipc_client.c

@@ -41,13 +41,11 @@
  *
  * \note The caller is responsible for freeing *api using pcmk_free_ipc_api().
  * \note This is intended to supersede crm_ipc_new() but currently only
- *       supports the controller IPC API.
+ *       supports the controller & pacemakerd IPC API.
  */
 int
 pcmk_new_ipc_api(pcmk_ipc_api_t **api, enum pcmk_ipc_server server)
 {
-    size_t max_size = 0;
-
     if (api == NULL) {
         return EINVAL;
     }
@@ -64,13 +62,15 @@ pcmk_new_ipc_api(pcmk_ipc_api_t **api, enum pcmk_ipc_server server)
         return EOPNOTSUPP;
     }
 
+    (*api)->ipc_size_max = 0;
+
     // Set server methods and max_size (if not default)
     switch (server) {
         case pcmk_ipc_attrd:
             break;
 
         case pcmk_ipc_based:
-            max_size = 512 * 1024; // 512KB
+            (*api)->ipc_size_max = 512 * 1024; // 512KB
             break;
 
         case pcmk_ipc_controld:
@@ -84,11 +84,12 @@ pcmk_new_ipc_api(pcmk_ipc_api_t **api, enum pcmk_ipc_server server)
             break;
 
         case pcmk_ipc_pacemakerd:
+            (*api)->cmds = pcmk__pacemakerd_api_methods();
             break;
 
         case pcmk_ipc_schedulerd:
             // @TODO max_size could vary by client, maybe take as argument?
-            max_size = 5 * 1024 * 1024; // 5MB
+            (*api)->ipc_size_max = 5 * 1024 * 1024; // 5MB
             break;
     }
     if ((*api)->cmds == NULL) {
@@ -97,7 +98,8 @@ pcmk_new_ipc_api(pcmk_ipc_api_t **api, enum pcmk_ipc_server server)
         return ENOMEM;
     }
 
-    (*api)->ipc = crm_ipc_new(pcmk_ipc_name(*api, false), max_size);
+    (*api)->ipc = crm_ipc_new(pcmk_ipc_name(*api, false),
+                              (*api)->ipc_size_max);
     if ((*api)->ipc == NULL) {
         pcmk_free_ipc_api(*api);
         *api = NULL;
@@ -258,7 +260,7 @@ pcmk_ipc_name(pcmk_ipc_api_t *api, bool for_log)
             return for_log? "fencer" : NULL /* "stonith-ng" */;
 
         case pcmk_ipc_pacemakerd:
-            return for_log? "launcher" : NULL /* CRM_SYSTEM_MCP */;
+            return for_log? "launcher" : CRM_SYSTEM_MCP;
 
         case pcmk_ipc_schedulerd:
             return for_log? "scheduler" : NULL /* CRM_SYSTEM_PENGINE */;
@@ -391,7 +393,7 @@ pcmk_dispatch_ipc(pcmk_ipc_api_t *api)
     if (api == NULL) {
         return;
     }
-    while (crm_ipc_ready(api->ipc)) {
+    while (crm_ipc_ready(api->ipc) > 0) {
         if (crm_ipc_read(api->ipc) > 0) {
             dispatch_ipc_data(crm_ipc_buffer(api->ipc), 0, api);
         }
@@ -451,11 +453,20 @@ pcmk_connect_ipc(pcmk_ipc_api_t *api, enum pcmk_ipc_dispatch dispatch_type)
 {
     int rc = pcmk_rc_ok;
 
-    if ((api == NULL) || (api->ipc == NULL)) {
+    if (api == NULL) {
         crm_err("Cannot connect to uninitialized API object");
         return EINVAL;
     }
 
+    if (api->ipc == NULL) {
+        api->ipc = crm_ipc_new(pcmk_ipc_name(api, false),
+                                  api->ipc_size_max);
+        if (api->ipc == NULL) {
+            crm_err("Failed to re-create IPC API");
+            return ENOMEM;
+        }
+    }
+
     if (crm_ipc_connected(api->ipc)) {
         crm_trace("Already connected to %s IPC API", pcmk_ipc_name(api, true));
         return pcmk_rc_ok;