issue #482: ci: add stray process checks to all jobs

List of interesting processes can probably expand more over time.

Author: dw

.ci/ansible_tests.py

@@ -20,11 +20,17 @@ def pause_if_interactive():
             signal.pause()
 
 
+interesting = ci_lib.get_interesting_procs()
+
+
 with ci_lib.Fold('unit_tests'):
     os.environ['SKIP_MITOGEN'] = '1'
     ci_lib.run('./run_tests -v')
 
 
+ci_lib.check_stray_processes(interesting)
+
+
 with ci_lib.Fold('docker_setup'):
     containers = ci_lib.make_containers()
     ci_lib.start_containers(containers)
@@ -75,4 +81,7 @@ def pause_if_interactive():
         pause_if_interactive()
         raise
 
+
+ci_lib.check_stray_processes(interesting, containers)
+
 pause_if_interactive()

.ci/ci_lib.py

@@ -215,6 +215,44 @@ def make_containers(name_prefix='', port_offset=0):
     return lst
 
 
+INTERESTING_COMMS = ('python', 'ssh', 'sudo', 'su', 'doas')
+
+
+def proc_is_docker(pid):
+    try:
+        fp = open('/proc/%s/cgroup' % (pid,), 'rb')
+    except IOError:
+        return False
+
+    try:
+        return 'docker' in fp.read()
+    finally:
+        fp.close()
+
+
+def get_interesting_procs(container_name=None):
+    args = ['ps', '-a', '-x', '-oppid=', '-opid=', '-ocomm=', '-ocommand=']
+    if container_name is not None:
+        args = ['docker', 'exec', container_name] + args
+
+    out = []
+    for line in subprocess__check_output(args).splitlines():
+        ppid, pid, comm, rest = line.split(None, 3)
+        if (
+            (
+                any(comm.startswith(s) for s in INTERESTING_COMMS) or
+                'mitogen:' in rest
+            ) and
+            (
+                container_name is not None or
+                (not proc_is_docker(pid))
+            )
+        ):
+            out.append((int(pid), line))
+
+    return sorted(out)
+
+
 def start_containers(containers):
     if os.environ.get('KEEP'):
         return
@@ -236,9 +274,44 @@ def start_containers(containers):
         ]
         for container in containers
     ])
+
+    for container in containers:
+        container['interesting'] = get_interesting_procs(container['name'])
+
     return containers
 
 
+def verify_procs(hostname, old, new):
+    oldpids = set(pid for pid, _ in old)
+    if any(pid not in oldpids for pid, _ in new):
+        print('%r had stray processes running:' % (hostname,))
+        for pid, line in new:
+            if pid not in oldpids:
+                print('New process:', line)
+
+        print()
+        return False
+
+    return True
+
+
+def check_stray_processes(old, containers=None):
+    ok = True
+
+    new = get_interesting_procs()
+    if old is not None:
+        ok &= verify_procs('test host machine', old, new)
+
+    for container in containers or ():
+        ok &= verify_procs(
+            container['name'],
+            container['interesting'],
+            get_interesting_procs(container['name'])
+        )
+
+    assert ok, 'stray processes were found'
+
+
 def dump_file(path):
     print()
     print('--- %s ---' % (path,))

.ci/debops_common_tests.py

@@ -68,9 +68,15 @@
     os.environ['ANSIBLE_HOST_KEY_CHECKING'] = 'False'
 
 
+interesting = ci_lib.get_interesting_procs()
+
 with ci_lib.Fold('first_run'):
     ci_lib.run('debops common %s', ' '.join(sys.argv[1:]))
 
+ci_lib.check_stray_processes(interesting, containers)
+
 
 with ci_lib.Fold('second_run'):
     ci_lib.run('debops common %s', ' '.join(sys.argv[1:]))
+
+ci_lib.check_stray_processes(interesting, containers)

.ci/mitogen_install.py

@@ -14,4 +14,5 @@
         'docker pull %s' % (ci_lib.image_for_distro(ci_lib.DISTRO),),
     ])
 
+
 ci_lib.run_batches(batches)

.ci/mitogen_tests.py

@@ -14,4 +14,6 @@
 if not ci_lib.have_docker():
     os.environ['SKIP_DOCKER_TESTS'] = '1'
 
+interesting = ci_lib.get_interesting_procs()
 ci_lib.run('./run_tests -v')
+ci_lib.check_stray_processes(interesting)