systemd: send MAINPID updates on re-exec

Author: pajod

gunicorn/arbiter.py

@@ -256,7 +256,10 @@ def handle_hup(self):
         - Gracefully shutdown the old worker processes
         """
         self.log.info("Hang up: %s", self.master_name)
+        systemd.sd_notify("RELOADING=1\nSTATUS=Gunicorn arbiter reloading..", self.log)
         self.reload()
+        # possibly premature, newly launched workers might have failed
+        systemd.sd_notify("READY=1\nSTATUS=Gunicorn arbiter reloaded", self.log)
 
     def handle_term(self):
         "SIGTERM handling"
@@ -332,6 +335,8 @@ def maybe_promote_master(self):
                 self.pidfile.rename(self.cfg.pidfile)
             # reset proctitle
             util._setproctitle("master [%s]" % self.proc_name)
+            # MAINPID does not change here, it was already set on fork
+            systemd.sd_notify("READY=1\nMAINPID=%d\nSTATUS=Gunicorn arbiter promoted" % (os.getpid(), ), self.log)
 
     def wakeup(self):
         """\
@@ -437,6 +442,9 @@ def reexec(self):
         os.chdir(self.START_CTX['cwd'])
 
         # exec the process using the original environment
+        self.log.debug("exe=%r argv=%r" % (self.START_CTX[0], self.START_CTX['args']))
+        # let systemd know are are in control
+        systemd.sd_notify("READY=1\nMAINPID=%d\nSTATUS=Gunicorn arbiter re-exec" % (master_pid, ), self.log)
         os.execve(self.START_CTX[0], self.START_CTX['args'], environ)
 
     def reload(self):
@@ -527,6 +535,8 @@ def reap_workers(self):
                 if self.reexec_pid == wpid:
                     self.reexec_pid = 0
                     self.log.info("Master exited before promotion.")
+                    # let systemd know we are (back) in control
+                    systemd.sd_notify("READY=1\nMAINPID=%d\nSTATUS=Gunicorn arbiter re-exec aborted" % (os.getpid(), ), self.log)
                     continue
                 else:
                     worker = self.WORKERS.pop(wpid, None)

gunicorn/systemd.py

@@ -4,6 +4,7 @@
 
 import os
 import socket
+import time
 
 SD_LISTEN_FDS_START = 3
 
@@ -66,6 +67,13 @@ def sd_notify(state, logger, unset_environment=False):
         if addr[0] == '@':
             addr = '\0' + addr[1:]
         sock.connect(addr)
+        if state[-1] != '\n':
+            state += "\n"
+        # needed for notify-reload, but no harm in sending unconditionally
+        # nsec = 10**-9, usec = 10**-6
+        monotonic_usecs = time.clock_gettime_ns(time.CLOCK_MONOTONIC) // 1000
+        state += "MONOTONIC_USEC=%d\n" % (monotonic_usecs, )
+        logger.debug("sd_notify: %r" % (state, ))
         sock.sendall(state.encode('utf-8'))
     except Exception:
         logger.debug("Exception while invoking sd_notify()", exc_info=True)

tests/test_arbiter.py

@@ -72,17 +72,20 @@ def test_arbiter_stop_does_not_unlink_when_using_reuse_port(close_sockets):
 @mock.patch('os.getpid')
 @mock.patch('os.fork')
 @mock.patch('os.execve')
-def test_arbiter_reexec_passing_systemd_sockets(execve, fork, getpid):
+@mock.patch('gunicorn.systemd.sd_notify')
+def test_arbiter_reexec_passing_systemd_sockets(sd_notify, execve, fork, getpid):
     arbiter = gunicorn.arbiter.Arbiter(DummyApplication())
     arbiter.LISTENERS = [mock.Mock(), mock.Mock()]
     arbiter.systemd = True
     fork.return_value = 0
-    getpid.side_effect = [2, 3]
+    sd_notify.return_value = None
+    getpid.side_effect = [2, 3, 3]  # 2 getpid calls in new master
     arbiter.reexec()
     environ = execve.call_args[0][2]
     assert environ['GUNICORN_PID'] == '2'
     assert environ['LISTEN_FDS'] == '2'
     assert environ['LISTEN_PID'] == '3'
+    sd_notify.assert_called_once()
 
 
 @mock.patch('os.getpid')