systemd: try to clean up any remaining processes in the cgroup

It might be some leftover processes that runs in the background, e.g.
ssh [mux] processes.

Author: lilydjwg

lilac2/systemd.py

@@ -214,18 +214,26 @@ def poll_rusage(
     subprocess.run(['systemctl', '--user', 'stop', '--quiet', name])
     if cgroup:
       # if we actually got the cgroup (i.e. service was started when we looked)
-      wait_cgroup_disappear(cgroup)
+      wait_cgroup_disappear(cgroup, name)
 
     p = subprocess.run(['systemctl', '--user', 'is-failed', '--quiet', name])
     if p.returncode == 0:
       subprocess.run(['systemctl', '--user', 'reset-failed', '--quiet', name])
   return RUsage(nsec / 1_000_000_000, mem_max), timedout
 
-def wait_cgroup_disappear(cgroup: str) -> None:
+def wait_cgroup_disappear(cgroup: str, name: str) -> None:
   d = f'/sys/fs/cgroup/{cgroup}'
   if not os.path.exists(d):
     return
 
+  count = 0
   while os.path.exists(d):
     logger.warning('waiting %s to disappear...', cgroup)
     time.sleep(1)
+    count += 1
+    if count == 10:
+      logger.warning('killing %s.', name)
+      subprocess.run(['systemctl', '--user', 'kill', name])
+    elif count == 30:
+      logger.warning('killing %s with SIGKILL.', name)
+      subprocess.run(['systemctl', '--user', 'kill', '--signal=KILL', name])