Fix: sbd-integration: sync pacemakerd with sbd

Make pacemakerd wait to be pinged by sbd before starting
sub-daemons. Pings further reply health-state and timestamp
of last successful check. On shutdown bring down all the
sub-daemons and wait to be polled for state by sbd before
finally exiting pacemakerd.
Add new api as not to make the xml-structure an external interface.

Author: wenningerk

daemons/pacemakerd/pacemakerd.c

@@ -40,8 +40,14 @@ static bool global_keep_tracking = false;
 #define PCMK_PROCESS_CHECK_INTERVAL 5
 
 static crm_trigger_t *shutdown_trigger = NULL;
+static crm_trigger_t *startup_trigger = NULL;
 static const char *pid_file = PCMK_RUN_DIR "/pacemaker.pid";
 
+static const char *pacemakerd_state = XML_PING_ATTR_PACEMAKERDSTATE_INIT;
+static gboolean running_with_sbd = FALSE;
+static uint shutdown_complete_state_reported_to = 0;
+static gboolean shutdown_complete_state_reported_client_closed = FALSE;
+
 typedef struct pcmk_child_s {
     pid_t pid;
     long flag;
@@ -374,21 +380,20 @@ escalate_shutdown(gpointer data)
 static gboolean
 pcmk_shutdown_worker(gpointer user_data)
 {
-    static int phase = 0;
+    static int phase = SIZEOF(pcmk_children);
     static time_t next_log = 0;
-    static int max = SIZEOF(pcmk_children);
 
     int lpc = 0;
 
-    if (phase == 0) {
+    if (phase == SIZEOF(pcmk_children)) {
         crm_notice("Shutting down Pacemaker");
-        phase = max;
+        pacemakerd_state = XML_PING_ATTR_PACEMAKERDSTATE_SHUTTINGDOWN;
     }
 
     for (; phase > 0; phase--) {
         /* Don't stop anything with start_seq < 1 */
 
-        for (lpc = max - 1; lpc >= 0; lpc--) {
+        for (lpc = SIZEOF(pcmk_children) - 1; lpc >= 0; lpc--) {
             pcmk_child_t *child = &(pcmk_children[lpc]);
 
             if (phase != child->start_seq) {
@@ -436,6 +441,11 @@ pcmk_shutdown_worker(gpointer user_data)
     }
 
     crm_notice("Shutdown complete");
+    pacemakerd_state = XML_PING_ATTR_PACEMAKERDSTATE_SHUTDOWNCOMPLETE;
+    if (!fatal_error && running_with_sbd &&
+        !shutdown_complete_state_reported_client_closed) {
+        return TRUE;
+    }
 
     {
         const char *delay = pcmk__env_option("shutdown_delay");
@@ -489,6 +499,50 @@ pcmk_ipc_accept(qb_ipcs_connection_t * c, uid_t uid, gid_t gid)
     return 0;
 }
 
+static void
+pcmk_handle_ping_request(pcmk__client_t *c, xmlNode *msg, uint32_t id)
+{
+    const char *value = NULL;
+    xmlNode *ping = NULL;
+    xmlNode *reply = NULL;
+    time_t pinged = time(NULL);
+    const char *from = crm_element_value(msg, F_CRM_SYS_FROM);
+
+    /* Pinged for status */
+    crm_trace("Pinged from %s.%s",
+              crm_element_value(msg, F_CRM_ORIGIN),
+              from?from:"unknown");
+    ping = create_xml_node(NULL, XML_CRM_TAG_PING);
+    value = crm_element_value(msg, F_CRM_SYS_TO);
+    crm_xml_add(ping, XML_PING_ATTR_SYSFROM, value);
+    crm_xml_add(ping, XML_PING_ATTR_PACEMAKERDSTATE, pacemakerd_state);
+    crm_xml_add_ll(ping, XML_ATTR_TSTAMP, (long long) pinged);
+    crm_xml_add(ping, XML_PING_ATTR_STATUS, "ok");
+    reply = create_reply(msg, ping);
+    free_xml(ping);
+    if (reply) {
+        if (pcmk__ipc_send_xml(c, id, reply, crm_ipc_server_event) <= 0) {
+            crm_err("Failed sending ping-reply");
+        }
+        free_xml(reply);
+    } else {
+        crm_err("Failed building ping-reply");
+    }
+    /* just proceed state on sbd pinging us */
+    if (from && strstr(from, "sbd")) {
+        if (crm_str_eq(pacemakerd_state,
+                       XML_PING_ATTR_PACEMAKERDSTATE_SHUTDOWNCOMPLETE,
+                       TRUE)) {
+            shutdown_complete_state_reported_to = c->pid;
+        } else if (crm_str_eq(pacemakerd_state,
+                              XML_PING_ATTR_PACEMAKERDSTATE_WAITPING,
+                              TRUE)) {
+            pacemakerd_state = XML_PING_ATTR_PACEMAKERDSTATE_STARTINGDAEMONS;
+            mainloop_set_trigger(startup_trigger);
+        }
+    }
+}
+
 /* Exit code means? */
 static int32_t
 pcmk_ipc_dispatch(qb_ipcs_connection_t * qbc, void *data, size_t size)
@@ -514,6 +568,9 @@ pcmk_ipc_dispatch(qb_ipcs_connection_t * qbc, void *data, size_t size)
         crm_trace("Ignoring IPC request to purge node "
                   "because peer cache is not used");
 
+    } else if (crm_str_eq(task, CRM_OP_PING, TRUE)) {
+        pcmk_handle_ping_request(c, msg, id);
+
     } else {
         crm_debug("Unrecognized IPC command '%s' sent to pacemakerd",
                   crm_str(task));
@@ -533,6 +590,12 @@ pcmk_ipc_closed(qb_ipcs_connection_t * c)
         return 0;
     }
     crm_trace("Connection %p", c);
+    if (shutdown_complete_state_reported_to == client->pid) {
+        shutdown_complete_state_reported_client_closed = TRUE;
+        if (shutdown_trigger) {
+            mainloop_set_trigger(shutdown_trigger);
+        }
+    }
     pcmk__free_client(client);
     return 0;
 }
@@ -924,8 +987,8 @@ find_and_track_existing_processes(void)
     return pcmk_rc_ok;
 }
 
-static void
-init_children_processes(void)
+static gboolean
+init_children_processes(void *user_data)
 {
     int start_seq = 1, lpc = 0;
     static int max = SIZEOF(pcmk_children);
@@ -951,6 +1014,8 @@ init_children_processes(void)
      * This may be useful for the daemons to know
      */
     setenv("PCMK_respawned", "true", 1);
+    pacemakerd_state = XML_PING_ATTR_PACEMAKERDSTATE_RUNNING;
+    return TRUE;
 }
 
 static void
@@ -1154,6 +1219,7 @@ main(int argc, char **argv)
 
     if(pcmk_locate_sbd() > 0) {
         setenv("PCMK_watchdog", "true", 1);
+        running_with_sbd = TRUE;
     } else {
         setenv("PCMK_watchdog", "false", 1);
     }
@@ -1170,7 +1236,13 @@ main(int argc, char **argv)
     mainloop_add_signal(SIGTERM, pcmk_shutdown);
     mainloop_add_signal(SIGINT, pcmk_shutdown);
 
-    init_children_processes();
+    if (running_with_sbd) {
+        pacemakerd_state = XML_PING_ATTR_PACEMAKERDSTATE_WAITPING;
+        startup_trigger = mainloop_add_trigger(G_PRIORITY_HIGH, init_children_processes, NULL);
+    } else {
+        pacemakerd_state = XML_PING_ATTR_PACEMAKERDSTATE_STARTINGDAEMONS;
+        init_children_processes(NULL);
+    }
 
     crm_notice("Pacemaker daemon successfully started and accepting connections");
     g_main_loop_run(mainloop);

include/crm/common/Makefile.am

@@ -12,7 +12,7 @@ MAINTAINERCLEANFILES = Makefile.in
 headerdir=$(pkgincludedir)/crm/common
 
 header_HEADERS = xml.h ipc.h util.h iso8601.h mainloop.h logging.h results.h \
-		 nvpair.h acl.h ipc_controld.h
+		 nvpair.h acl.h ipc_controld.h ipc_pacemakerd.h
 noinst_HEADERS = internal.h alerts_internal.h \
 		 iso8601_internal.h remote_internal.h xml_internal.h \
 		 ipc_internal.h output.h cmdline_internal.h curses_internal.h \

include/crm/common/ipc_pacemakerd.h

@@ -0,0 +1,74 @@
+/*
+ * Copyright 2020 the Pacemaker project contributors
+ *
+ * The version control history for this file may have further details.
+ *
+ * This source code is licensed under the GNU Lesser General Public License
+ * version 2.1 or later (LGPLv2.1+) WITHOUT ANY WARRANTY.
+ */
+
+#ifndef PCMK__IPC_PACEMAKERD__H
+#  define PCMK__IPC_PACEMAKERD__H
+
+#ifdef __cplusplus
+extern "C" {
+#endif
+
+/**
+ * \file
+ * \brief IPC commands for Pacemakerd
+ *
+ * \ingroup core
+ */
+
+#include <stdbool.h>                    // bool
+#include <libxml/tree.h>                // xmlNode
+#include <crm/common/ipc.h>             // pcmk_ipc_api_t
+
+enum pcmk_pacemakerd_state {
+    pcmk_pacemakerd_state_invalid = -1,
+    pcmk_pacemakerd_state_init = 0,
+    pcmk_pacemakerd_state_starting_daemons,
+    pcmk_pacemakerd_state_wait_for_ping,
+    pcmk_pacemakerd_state_running,
+    pcmk_pacemakerd_state_shutting_down,
+    pcmk_pacemakerd_state_shutdown_complete,
+    pcmk_pacemakerd_state_max = pcmk_pacemakerd_state_shutdown_complete,
+};
+
+//! Possible types of pacemakerd replies
+enum pcmk_pacemakerd_api_reply {
+    pcmk_pacemakerd_reply_unknown,
+    pcmk_pacemakerd_reply_ping,
+};
+
+/*!
+ * Pacemakerd reply passed to event callback
+ */
+typedef struct {
+    enum pcmk_pacemakerd_api_reply reply_type;
+
+    union {
+        // pcmk_pacemakerd_reply_ping
+        struct {
+            const char *sys_from;
+            enum pcmk_pacemakerd_state state;
+            time_t last_good;
+            int status;
+        } ping;
+    } data;
+} pcmk_pacemakerd_api_reply_t;
+
+int pcmk_pacemakerd_api_ping(pcmk_ipc_api_t *api, const char *ipc_name);
+unsigned int
+    pcmk_pacemakerd_api_replies_expected(pcmk_ipc_api_t *api);
+enum pcmk_pacemakerd_state
+    pcmk_pacemakerd_api_pacemakerd_state_text2enum(const char *state);
+const char
+    *pcmk_pacemakerd_api_pacemakerd_state_enum2text(enum pcmk_pacemakerd_state state);
+
+#ifdef __cplusplus
+}
+#endif
+
+#endif // PCMK__IPC_PACEMAKERD__H

include/crm/msg_xml.h

@@ -123,6 +123,13 @@ extern "C" {
 #  define XML_PING_ATTR_STATUS		"result"
 #  define XML_PING_ATTR_SYSFROM		"crm_subsystem"
 #  define XML_PING_ATTR_CRMDSTATE   "crmd_state"
+#  define XML_PING_ATTR_PACEMAKERDSTATE "pacemakerd_state"
+#  define XML_PING_ATTR_PACEMAKERDSTATE_INIT "init"
+#  define XML_PING_ATTR_PACEMAKERDSTATE_STARTINGDAEMONS "starting_daemons"
+#  define XML_PING_ATTR_PACEMAKERDSTATE_WAITPING "wait_for_ping"
+#  define XML_PING_ATTR_PACEMAKERDSTATE_RUNNING "running"
+#  define XML_PING_ATTR_PACEMAKERDSTATE_SHUTTINGDOWN "shutting_down"
+#  define XML_PING_ATTR_PACEMAKERDSTATE_SHUTDOWNCOMPLETE "shutdown_complete"
 
 #  define XML_TAG_FRAGMENT		"cib_fragment"
 

lib/common/Makefile.am

@@ -50,6 +50,7 @@ libcrmcommon_la_SOURCES	+= io.c
 libcrmcommon_la_SOURCES	+= ipc_client.c
 libcrmcommon_la_SOURCES	+= ipc_common.c
 libcrmcommon_la_SOURCES	+= ipc_controld.c
+libcrmcommon_la_SOURCES	+= ipc_pacemakerd.c
 libcrmcommon_la_SOURCES	+= ipc_server.c
 libcrmcommon_la_SOURCES	+= iso8601.c
 libcrmcommon_la_SOURCES	+= logging.c

lib/common/crmcommon_private.h

@@ -209,4 +209,7 @@ bool pcmk__valid_ipc_header(const pcmk__ipc_header_t *header);
 G_GNUC_INTERNAL
 pcmk__ipc_methods_t *pcmk__controld_api_methods(void);
 
+G_GNUC_INTERNAL
+pcmk__ipc_methods_t *pcmk__pacemakerd_api_methods(void);
+
 #endif  // CRMCOMMON_PRIVATE__H

lib/common/ipc_client.c

@@ -84,6 +84,7 @@ pcmk_new_ipc_api(pcmk_ipc_api_t **api, enum pcmk_ipc_server server)
             break;
 
         case pcmk_ipc_pacemakerd:
+            (*api)->cmds = pcmk__pacemakerd_api_methods();
             break;
 
         case pcmk_ipc_schedulerd:
@@ -258,7 +259,7 @@ pcmk_ipc_name(pcmk_ipc_api_t *api, bool for_log)
             return for_log? "fencer" : NULL /* "stonith-ng" */;
 
         case pcmk_ipc_pacemakerd:
-            return for_log? "launcher" : NULL /* CRM_SYSTEM_MCP */;
+            return for_log? "launcher" : CRM_SYSTEM_MCP;
 
         case pcmk_ipc_schedulerd:
             return for_log? "scheduler" : NULL /* CRM_SYSTEM_PENGINE */;