Merge remote-tracking branch 'origin/v028' into stable

* origin/v028: (383 commits)
  Bump version for release.
  docs: update Changelog for 0.2.8.
  issue #627: add test and tweak Reaper behaviour.
  docs: lots more changelog concision
  docs: changelog concision
  docs: more changelog tweaks
  docs: reorder chapters
  docs: versionless <title>
  docs: update supported Ansible version, mention unsupported features
  docs: changelog fixes/tweaks
  issue #590: update Changelog.
  issue #621: send ADD_ROUTE earlier and add test for early logging.
  issue #590: whoops, import missing test modules
  issue #590: rework ParentEnumerationMethod to recursively handle bad modules
  issue #627: reduce the default pool size in a child to 2.
  tests: add a few extra service tests.
  docs: some more hyperlink joy
  docs: more hyperlinks
  docs: add domainrefs plugin to make link aliases everywhere \o/
  docs: link IS_DEAD in changelog
  docs: tweaks to better explain changelog race
  issue #533: update routing to account for DEL_ROUTE propagation race
  tests: use defer_sync() Rather than defer() + ancient sync_with_broker()
  tests: one case from doas_test was invoking su
  tests: hide memory-mapped files from lsof output
  issue #615: remove meaningless test
  issue #625: ignore SIGINT within MuxProcess
  issue #625: use exec() instead of subprocess in mitogen_ansible_playbook
  issue #615: regression test
  issue #615: update Changelog.
  issue #615: ensure 4GB max_message_size is configured for task workers.
  issue #615: update Changelog.
  issue #615: route a dead message to recipients when no reply is expected
  issue #615: fetch_file() might be called with AnsibleUnicode.
  issue #615: redirect 'fetch' action to 'mitogen_fetch'.
  issue #615: extricate slurp brainwrong from mitogen_fetch
  issue #615: ansible: import Ansible fetch.py action plug-in
  issue #533: include object identity of Stream in repr()
  docs: lots more changelog
  issue #595: add buildah to docs and changelog.
  docs: a few more internals.rst additions
  ci: update to Ansible 2.8.3
  tests: another random string changed in 2.8.3
  tests: fix sudo_flags_failure for Ansible 2.8.3
  ci: fix procps command line format warning
  Whoops, merge together lgtm.yml and .lgtm.yml
  issue #440: log Python version during bootstrap.
  docs: update changelog
  issue #558: disable test on OSX to cope with boundless mediocrity
  issue #558, #582: preserve remote tmpdir if caller did not supply one
  issue #613: must await 'exit' and 'disconnect' in wait=False test
  Import LGTM config to disable some stuff
  Fix up another handful of LGTM errors.
  tests: work around AnsibleModule.run_command() race.
  docs: mention another __main__ safeguard
  docs: tweaks
  formatting error
  docs: make Sphinx install soft fail on Python 2.
  issue #598: allow disabling preempt in terraform
  issue #598: update Changelog.
  issue #605: update Changelog.
  issue #605: ansible: share a sem_t instead of a pthread_mutex_t
  issue #613: add tests for all the weird shutdown methods
  Add mitogen.core.now() and use it everywhere; closes #614.
  docs: move decorator docs into core.py and use autodecorator
  preamble_size: make it work on Python 3.
  docs: upgrade Sphinx to 2.1.2, require Python 3 to build docs.
  docs: fix Sphinx warnings, add LogHandler, more docstrings
  docs: tidy up some Changelog text
  issue #615: fix up FileService tests for new logic
  issue #615: another Py3x fix.
  issue #615: Py3x fix.
  issue #615: update Changelog.
  issue #615: use FileService for target->controll file transfers
  issue #482: another Py3 fix
  ci: try removing exclude: to make Azure jobs work again
  compat: fix Py2.4 SyntaxError
  issue #482: remove 'ssh' from checked processes
  ci: Py3 fix
  issue #279: add one more test for max_message_size
  issue #482: ci: add stray process checks to all jobs
  tests: fix format string error
  core: MitogenProtocol.is_privileged was not set in children
  issue #482: tests: fail DockerMixin tests if stray processes exist
  docs: update Changelog.
  issue #586: update Changelog.
  docs: update Changelog.
  [security] core: undirectional routing wasn't respected in some cases
  docs: tidy up Select.all()
  issue #612: update Changelog.
  master: fix TypeError
  pkgutil: fix Python3 compatibility
  parent: use protocol for getting remote_id
  docs: merge signals.rst into internals.rst
  os_fork: do not attempt to cork the active thread.
  parent: fix get_log_level() for split out loggers.
  issue #547: fix service_test failures.
  issue #547: update Changelog.
  issue #547: core/service: race/deadlock-free service pool init
  docs: update Changelog.
  ...

Author: dw

.ci/ansible_tests.py

@@ -3,6 +3,7 @@
 
 import glob
 import os
+import signal
 import sys
 
 import ci_lib
@@ -13,11 +14,23 @@
 HOSTS_DIR = os.path.join(ci_lib.TMP, 'hosts')
 
 
+def pause_if_interactive():
+    if os.path.exists('/tmp/interactive'):
+        while True:
+            signal.pause()
+
+
+interesting = ci_lib.get_interesting_procs()
+
+
 with ci_lib.Fold('unit_tests'):
     os.environ['SKIP_MITOGEN'] = '1'
     ci_lib.run('./run_tests -v')
 
 
+ci_lib.check_stray_processes(interesting)
+
+
 with ci_lib.Fold('docker_setup'):
     containers = ci_lib.make_containers()
     ci_lib.start_containers(containers)
@@ -56,8 +69,19 @@
         run("sudo apt-get update")
         run("sudo apt-get install -y sshpass")
 
+    run("bash -c 'sudo ln -vfs /usr/lib/python2.7/plat-x86_64-linux-gnu/_sysconfigdata_nd.py /usr/lib/python2.7 || true'")
+    run("bash -c 'sudo ln -vfs /usr/lib/python2.7/plat-x86_64-linux-gnu/_sysconfigdata_nd.py $VIRTUAL_ENV/lib/python2.7 || true'")
 
 with ci_lib.Fold('ansible'):
     playbook = os.environ.get('PLAYBOOK', 'all.yml')
-    run('./run_ansible_playbook.py %s -i "%s" %s',
-        playbook, HOSTS_DIR, ' '.join(sys.argv[1:]))
+    try:
+        run('./run_ansible_playbook.py %s -i "%s" %s',
+            playbook, HOSTS_DIR, ' '.join(sys.argv[1:]))
+    except:
+        pause_if_interactive()
+        raise
+
+
+ci_lib.check_stray_processes(interesting, containers)
+
+pause_if_interactive()

.ci/azure-pipelines-steps.yml

@@ -5,16 +5,27 @@ parameters:
   sign: false
 
 steps:
-- task: UsePythonVersion@0
-  inputs:
-    versionSpec: '$(python.version)'
-    architecture: 'x64'
+- script: "PYTHONVERSION=$(python.version) .ci/prep_azure.py"
+  displayName: "Run prep_azure.py"
 
-- script: .ci/prep_azure.py
-  displayName: "Install requirements."
+# The VSTS-shipped Pythons available via UsePythonVErsion are pure garbage,
+# broken symlinks, incorrect permissions and missing codecs. So we use the
+# deadsnakes PPA to get sane Pythons, and setup a virtualenv to install our
+# stuff into. The virtualenv can probably be removed again, but this was a
+# hard-fought battle and for now I am tired of this crap.
+- script: |
+    sudo ln -fs /usr/bin/python$(python.version) /usr/bin/python
+    /usr/bin/python -m pip install -U virtualenv setuptools wheel
+    /usr/bin/python -m virtualenv /tmp/venv -p /usr/bin/python$(python.version)
+    echo "##vso[task.prependpath]/tmp/venv/bin"
+
+  displayName: activate venv
+
+- script: .ci/spawn_reverse_shell.py
+  displayName: "Spawn reverse shell"
 
 - script: .ci/$(MODE)_install.py
-  displayName: "Install requirements."
+  displayName: "Run $(MODE)_install.py"
 
 - script: .ci/$(MODE)_tests.py
-  displayName: Run tests.
+  displayName: "Run $(MODE)_tests.py"

.ci/azure-pipelines.yml

@@ -15,6 +15,9 @@ jobs:
       Mito27_27:
         python.version: '2.7'
         MODE: mitogen
+      Ans280_27:
+        python.version: '2.7'
+        MODE: localhost_ansible
 
 
 - job: Linux
@@ -87,3 +90,13 @@ jobs:
         #VER: 2.6.2
         #DISTROS: debian
         #STRATEGY: linear
+
+      Ansible_280_27:
+        python.version: '2.7'
+        MODE: ansible
+        VER: 2.8.0
+
+      Ansible_280_35:
+        python.version: '3.5'
+        MODE: ansible
+        VER: 2.8.0

.ci/ci_lib.py

@@ -57,8 +57,10 @@ def have_docker():
 
 # -----------------
 
-# Force stdout FD 1 to be a pipe, so tools like pip don't spam progress bars.
+# Force line buffering on stdout.
+sys.stdout = os.fdopen(1, 'w', 1)
 
+# Force stdout FD 1 to be a pipe, so tools like pip don't spam progress bars.
 if 'TRAVIS_HOME' in os.environ:
     proc = subprocess.Popen(
         args=['stdbuf', '-oL', 'cat'],
@@ -86,8 +88,13 @@ def _argv(s, *args):
 def run(s, *args, **kwargs):
     argv = ['/usr/bin/time', '--'] + _argv(s, *args)
     print('Running: %s' % (argv,))
-    ret = subprocess.check_call(argv, **kwargs)
-    print('Finished running: %s' % (argv,))
+    try:
+        ret = subprocess.check_call(argv, **kwargs)
+        print('Finished running: %s' % (argv,))
+    except Exception:
+        print('Exception occurred while running: %s' % (argv,))
+        raise
+
     return ret
 
 
@@ -208,6 +215,46 @@ def make_containers(name_prefix='', port_offset=0):
     return lst
 
 
+# ssh removed from here because 'linear' strategy relies on processes that hang
+# around after the Ansible run completes
+INTERESTING_COMMS = ('python', 'sudo', 'su', 'doas')
+
+
+def proc_is_docker(pid):
+    try:
+        fp = open('/proc/%s/cgroup' % (pid,), 'r')
+    except IOError:
+        return False
+
+    try:
+        return 'docker' in fp.read()
+    finally:
+        fp.close()
+
+
+def get_interesting_procs(container_name=None):
+    args = ['ps', 'ax', '-oppid=', '-opid=', '-ocomm=', '-ocommand=']
+    if container_name is not None:
+        args = ['docker', 'exec', container_name] + args
+
+    out = []
+    for line in subprocess__check_output(args).decode().splitlines():
+        ppid, pid, comm, rest = line.split(None, 3)
+        if (
+            (
+                any(comm.startswith(s) for s in INTERESTING_COMMS) or
+                'mitogen:' in rest
+            ) and
+            (
+                container_name is not None or
+                (not proc_is_docker(pid))
+            )
+        ):
+            out.append((int(pid), line))
+
+    return sorted(out)
+
+
 def start_containers(containers):
     if os.environ.get('KEEP'):
         return
@@ -217,6 +264,7 @@ def start_containers(containers):
             "docker rm -f %(name)s || true" % container,
             "docker run "
                 "--rm "
+                "--cpuset-cpus 0,1 "
                 "--detach "
                 "--privileged "
                 "--cap-add=SYS_PTRACE "
@@ -228,9 +276,44 @@ def start_containers(containers):
         ]
         for container in containers
     ])
+
+    for container in containers:
+        container['interesting'] = get_interesting_procs(container['name'])
+
     return containers
 
 
+def verify_procs(hostname, old, new):
+    oldpids = set(pid for pid, _ in old)
+    if any(pid not in oldpids for pid, _ in new):
+        print('%r had stray processes running:' % (hostname,))
+        for pid, line in new:
+            if pid not in oldpids:
+                print('New process:', line)
+
+        print()
+        return False
+
+    return True
+
+
+def check_stray_processes(old, containers=None):
+    ok = True
+
+    new = get_interesting_procs()
+    if old is not None:
+        ok &= verify_procs('test host machine', old, new)
+
+    for container in containers or ():
+        ok &= verify_procs(
+            container['name'],
+            container['interesting'],
+            get_interesting_procs(container['name'])
+        )
+
+    assert ok, 'stray processes were found'
+
+
 def dump_file(path):
     print()
     print('--- %s ---' % (path,))

.ci/debops_common_tests.py

@@ -3,6 +3,7 @@
 from __future__ import print_function
 import os
 import shutil
+import sys
 
 import ci_lib
 
@@ -67,9 +68,15 @@
     os.environ['ANSIBLE_HOST_KEY_CHECKING'] = 'False'
 
 
+interesting = ci_lib.get_interesting_procs()
+
 with ci_lib.Fold('first_run'):
-    ci_lib.run('debops common')
+    ci_lib.run('debops common %s', ' '.join(sys.argv[1:]))
+
+ci_lib.check_stray_processes(interesting, containers)
 
 
 with ci_lib.Fold('second_run'):
-    ci_lib.run('debops common')
+    ci_lib.run('debops common %s', ' '.join(sys.argv[1:]))
+
+ci_lib.check_stray_processes(interesting, containers)

.ci/localhost_ansible_install.py

@@ -0,0 +1,16 @@
+#!/usr/bin/env python
+
+import ci_lib
+
+batches = [
+    [
+        # Must be installed separately, as PyNACL indirect requirement causes
+        # newer version to be installed if done in a single pip run.
+        'pip install "pycparser<2.19" "idna<2.7"',
+        'pip install '
+            '-r tests/requirements.txt '
+            '-r tests/ansible/requirements.txt',
+    ]
+]
+
+ci_lib.run_batches(batches)

.ci/localhost_ansible_tests.py

@@ -0,0 +1,54 @@
+#!/usr/bin/env python
+# Run tests/ansible/all.yml under Ansible and Ansible-Mitogen
+
+import glob
+import os
+import shutil
+import sys
+
+import ci_lib
+from ci_lib import run
+
+
+TESTS_DIR = os.path.join(ci_lib.GIT_ROOT, 'tests/ansible')
+IMAGE_PREP_DIR = os.path.join(ci_lib.GIT_ROOT, 'tests/image_prep')
+HOSTS_DIR = os.path.join(TESTS_DIR, 'hosts')
+KEY_PATH = os.path.join(TESTS_DIR, '../data/docker/mitogen__has_sudo_pubkey.key')
+
+
+with ci_lib.Fold('unit_tests'):
+    os.environ['SKIP_MITOGEN'] = '1'
+    ci_lib.run('./run_tests -v')
+
+
+with ci_lib.Fold('job_setup'):
+    # Don't set -U as that will upgrade Paramiko to a non-2.6 compatible version.
+    run("pip install -q virtualenv ansible==%s", ci_lib.ANSIBLE_VERSION)
+
+    os.chmod(KEY_PATH, int('0600', 8))
+    if not ci_lib.exists_in_path('sshpass'):
+        run("brew install http://git.io/sshpass.rb")
+
+
+with ci_lib.Fold('machine_prep'):
+    ssh_dir = os.path.expanduser('~/.ssh')
+    if not os.path.exists(ssh_dir):
+        os.makedirs(ssh_dir, int('0700', 8))
+
+    key_path = os.path.expanduser('~/.ssh/id_rsa')
+    shutil.copy(KEY_PATH, key_path)
+
+    auth_path = os.path.expanduser('~/.ssh/authorized_keys')
+    os.system('ssh-keygen -y -f %s >> %s' % (key_path, auth_path))
+    os.chmod(auth_path, int('0600', 8))
+
+    if os.path.expanduser('~mitogen__user1') == '~mitogen__user1':
+        os.chdir(IMAGE_PREP_DIR)
+        run("ansible-playbook -c local -i localhost, _user_accounts.yml")
+
+
+with ci_lib.Fold('ansible'):
+    os.chdir(TESTS_DIR)
+    playbook = os.environ.get('PLAYBOOK', 'all.yml')
+    run('./run_ansible_playbook.py %s -l target %s',
+        playbook, ' '.join(sys.argv[1:]))

.ci/mitogen_install.py

@@ -14,4 +14,5 @@
         'docker pull %s' % (ci_lib.image_for_distro(ci_lib.DISTRO),),
     ])
 
+
 ci_lib.run_batches(batches)

.ci/mitogen_tests.py

@@ -14,4 +14,6 @@
 if not ci_lib.have_docker():
     os.environ['SKIP_DOCKER_TESTS'] = '1'
 
+interesting = ci_lib.get_interesting_procs()
 ci_lib.run('./run_tests -v')
+ci_lib.check_stray_processes(interesting)