PYTHON-5594 Make process shutdown more robust (#322)

Author: blink1073

mongo_orchestration/daemon.py

@@ -22,7 +22,7 @@
 import sys
 import time
 
-from signal import SIGTERM
+from signal import SIGTERM, SIGKILL
 
 DEVNULL = open(os.devnull, 'r+b')
 
@@ -174,8 +174,15 @@ def stop(self):
             # Try killing the daemon process
             try:
                 os.kill(pid, SIGTERM)
+                t0 = time.time()
                 while is_unix_process_running(pid):
                     time.sleep(0.25)
+                    # After 5 seconds resend a SIGTERM to the process.
+                    if time.time() - t0 > 5:
+                        os.kill(pid, SIGTERM)
+                    # After 10 seconds send a SIGKILL to the process.
+                    if time.time() - t0 > 10:
+                        os.kill(pid, SIGKILL)
             except OSError as err:
                 if err.errno == errno.ESRCH:
                     if os.path.exists(self.pidfile):
@@ -209,4 +216,3 @@ def is_unix_process_running(pid):
         else:
             raise err
     return True
-

tests/test_launch.py

@@ -26,12 +26,6 @@ def run(cmd, **kwargs):
     if proc.returncode != 0:
         raise RuntimeError('Process failed!')
 
-
-def stop():
-    pid = Path("server.pid").read_text().strip()
-    os.kill(int(pid), signal.SIGTERM)
-    Path("server.pid").unlink()
-
 class TestLaunch(unittest.TestCase):
 
     def test_launch_single(self):
@@ -43,9 +37,7 @@ def test_launch_single(self):
         proc.send('q\n')
         proc.wait()
         self.assertEqual(proc.exitstatus, 0)
-        # TODO: https://jira.mongodb.org/browse/PYTHON-5594
-        # run('mongo-orchestration stop')
-        stop()
+        run('mongo-orchestration stop')
 
     def test_launch_replica_set(self):
         if os.name != 'posix':
@@ -56,9 +48,7 @@ def test_launch_replica_set(self):
         proc.send('q\n')
         proc.wait()
         self.assertEqual(proc.exitstatus, 0)
-        # TODO: https://jira.mongodb.org/browse/PYTHON-5594
-        # run('mongo-orchestration stop')
-        stop()
+        run('mongo-orchestration stop')
 
 
     def test_launch_sharded(self):
@@ -70,6 +60,4 @@ def test_launch_sharded(self):
         proc.send('q\n')
         proc.wait()
         self.assertEqual(proc.exitstatus, 0)
-        # TODO: https://jira.mongodb.org/browse/PYTHON-5594
-        # run('mongo-orchestration stop')
-        stop()
+        run('mongo-orchestration stop')