Merge remote-tracking branch 'origin/ci280'

* origin/ci280:
  ci: Another round of fixes for random Ansible UI breakage in 2.7/2.8
  ci: work around various broken aspects of Travis VM image
  Use virtualenv Python for stub connections to workaround problem
  ci: Ansible 2.8 requires Python 2.7.
  tests: add 2.8 format async error timeout message
  ansible: prevent tempfile.mkstemp() leaks.
  update gitignore again
  ci: try bumping more Travis jobs to Ansible 2.8.
  add .*.pid to gitignore
  tests: allow running without hdrhistograms library.
  issue #578: update Changelog.
  travis: exclude docs-master from CI
  issue #589: ensure real FileService/PushFileService are in the docs
  issue #589: ensure real FileService/PushFileService are in the docs
  docs: add new contributor entry
  issue #589: remove outdated/incomplete examples
  issue #589: split services example out and make it run.

Author: dw

.ci/ansible_tests.py

@@ -63,6 +63,8 @@ def pause_if_interactive():
         run("sudo apt-get update")
         run("sudo apt-get install -y sshpass")
 
+    run("bash -c 'sudo ln -vfs /usr/lib/python2.7/plat-x86_64-linux-gnu/_sysconfigdata_nd.py /usr/lib/python2.7 || true'")
+    run("bash -c 'sudo ln -vfs /usr/lib/python2.7/plat-x86_64-linux-gnu/_sysconfigdata_nd.py $VIRTUAL_ENV/lib/python2.7 || true'")
 
 with ci_lib.Fold('ansible'):
     playbook = os.environ.get('PLAYBOOK', 'all.yml')

.gitignore

@@ -14,3 +14,4 @@ htmlcov/
 *.egg-info
 __pycache__/
 extra
+**/.*.pid

.travis.yml

@@ -6,12 +6,17 @@ notifications:
 
 language: python
 
+branches:
+  except:
+  - docs-master
+
 cache:
 - pip
 - directories:
   - /home/travis/virtualenv
 
 install:
+- grep -Erl git-lfs\|couchdb /etc/apt | sudo xargs rm -v
 - .ci/${MODE}_install.py
 
 script:
@@ -43,9 +48,9 @@ matrix:
     # 2.4.6.0; 2.7 -> 2.7
     - python: "2.7"
       env: MODE=debops_common VER=2.4.6.0
-    # 2.5.7; 3.6 -> 2.7
+    # 2.8.0; 3.6 -> 2.7
     - python: "3.6"
-      env: MODE=debops_common VER=2.6.2
+      env: MODE=debops_common VER=2.8.0
 
     # ansible_mitogen tests.
 
@@ -56,15 +61,16 @@ matrix:
     # 2.6 -> {debian, centos6, centos7}
     - python: "2.6"
       env: MODE=ansible VER=2.4.6.0
-    - python: "2.6"
-      env: MODE=ansible VER=2.6.2
+    # 2.7 -> {debian, centos6, centos7}
+    - python: "2.7"
+      env: MODE=ansible VER=2.8.0
 
     # 3.6 -> {debian, centos6, centos7}
     - python: "3.6"
       env: MODE=ansible VER=2.4.6.0
     - python: "3.6"
-      env: MODE=ansible VER=2.6.2
+      env: MODE=ansible VER=2.8.0
 
     # Sanity check against vanilla Ansible. One job suffices.
     - python: "2.7"
-      env: MODE=ansible VER=2.6.2 DISTROS=debian STRATEGY=linear
+      env: MODE=ansible VER=2.8.0 DISTROS=debian STRATEGY=linear

ansible_mitogen/runner.py

@@ -112,6 +112,45 @@ def shlex_split(s, comments=False):
                 for token in shlex.split(str(s), comments=comments)]
 
 
+class TempFileWatcher(object):
+    """
+    Since Ansible 2.7.0, lineinfile leaks file descriptors returned by
+    :func:`tempfile.mkstemp` (ansible/ansible#57327). Handle this and all
+    similar cases by recording descriptors produced by mkstemp during module
+    execution, and cleaning up any leaked descriptors on completion.
+    """
+    def __init__(self):
+        self._real_mkstemp = tempfile.mkstemp
+        # (fd, st.st_dev, st.st_ino)
+        self._fd_dev_inode = []
+        tempfile.mkstemp = self._wrap_mkstemp
+
+    def _wrap_mkstemp(self, *args, **kwargs):
+        fd, path = self._real_mkstemp(*args, **kwargs)
+        st = os.fstat(fd)
+        self._fd_dev_inode.append((fd, st.st_dev, st.st_ino))
+        return fd, path
+
+    def revert(self):
+        tempfile.mkstemp = self._real_mkstemp
+        for tup in self._fd_dev_inode:
+            self._revert_one(*tup)
+
+    def _revert_one(self, fd, st_dev, st_ino):
+        try:
+            st = os.fstat(fd)
+        except OSError:
+            # FD no longer exists.
+            return
+
+        if not (st.st_dev == st_dev and st.st_ino == st_ino):
+            # FD reused.
+            return
+
+        LOG.info("a tempfile.mkstemp() FD was leaked during the last task")
+        os.close(fd)
+
+
 class EnvironmentFileWatcher(object):
     """
     Usually Ansible edits to /etc/environment and ~/.pam_environment are
@@ -803,6 +842,7 @@ def setup(self):
         # module, but this has never been a bug report. Instead act like an
         # interpreter that had its script piped on stdin.
         self._argv = TemporaryArgv([''])
+        self._temp_watcher = TempFileWatcher()
         self._importer = ModuleUtilsImporter(
             context=self.service_context,
             module_utils=self.module_map['custom'],
@@ -818,6 +858,7 @@ def _revert_excepthook(self):
 
     def revert(self):
         self.atexit_wrapper.revert()
+        self._temp_watcher.revert()
         self._argv.revert()
         self._stdio.revert()
         self._revert_excepthook()

docs/changelog.rst

@@ -34,6 +34,9 @@ Enhancements
 Fixes
 ^^^^^
 
+* `#578 <https://github.com/dw/mitogen/issues/578>`_: the extension could crash
+  while rendering an error message, due to an incorrect format string.
+
 * `#590 <https://github.com/dw/mitogen/issues/590>`_: the importer can handle
   modules that replace themselves in :mod:`sys.modules` during import.
 
@@ -59,8 +62,10 @@ bug reports, testing, features and fixes in this release contributed by
 `Anton Markelov <https://github.com/strangeman>`_,
 `Nigel Metheringham <https://github.com/nigelm>`_,
 `Orion Poplawski <https://github.com/opoplawski>`_,
-`Ulrich Schreiner <https://github.com/ulrichSchreiner>`_, and
-`Yuki Nishida <https://github.com/yuki-nishida-exa>`_.
+`Szabó Dániel Ernő <https://github.com/r3ap3rpy>`_,
+`Ulrich Schreiner <https://github.com/ulrichSchreiner>`_,
+`Yuki Nishida <https://github.com/yuki-nishida-exa>`_, and
+`@ghp-rr <https://github.com/ghp-rr>`_.
 
 
 v0.2.7 (2019-05-19)

docs/contributors.rst

@@ -88,6 +88,9 @@ sponsorship and outstanding future-thinking of its early adopters.
     <h3>Private Sponsors</h3>
 
     <ul style="line-height: 120% !important;">
+    <li><a href="https://skunkwerks.at/">SkunkWerks</a> &mdash;
+        <em>Mitogen on FreeBSD runs like a kid in a candy store: fast &amp;
+        sweet.</em></li>
     <li>Donald Clark Jackson &mdash;
         <em>Mitogen is an exciting project, and I am happy to support its
         development.</em></li>

docs/services.rst

@@ -61,55 +61,7 @@ Pool
 Example
 -------
 
-.. code-block:: python
-
-    import mitogen
-    import mitogen.service
-
-
-    class FileService(mitogen.service.Service):
-        """
-        Simple file server, for demonstration purposes only! Use of this in
-        real code would be a security vulnerability as it would permit children
-        to read arbitrary files from the master's disk.
-        """
-        handle = 500
-        required_args = {
-            'path': str
-        }
-
-        def dispatch(self, args, msg):
-            with open(args['path'], 'r') as fp:
-                return fp.read()
-
-
-    def download_file(context, path):
-        s = mitogen.service.call(context, FileService.handle, {
-            'path': path
-        })
-
-        with open(path, 'w') as fp:
-            fp.write(s)
-
-
-    @mitogen.core.takes_econtext
-    def download_some_files(paths, econtext):
-        for path in paths:
-            download_file(econtext.master, path)
-
-
-    @mitogen.main()
-    def main(router):
-        pool = mitogen.service.Pool(router, size=1, services=[
-            FileService(router),
-        ])
-
-        remote = router.ssh(hostname='k3')
-        remote.call(download_some_files, [
-            '/etc/passwd',
-            '/etc/hosts',
-        ])
-        pool.stop()
+.. literalinclude:: ../examples/service/self_contained.py
 
 
 Reference
@@ -134,3 +86,12 @@ Reference
 .. autoclass:: mitogen.service.Pool
     :members:
 
+
+Built-in Services
+-----------------
+
+.. autoclass:: mitogen.service.FileService
+    :members:
+
+.. autoclass:: mitogen.service.PushFileService
+    :members:

examples/service/client.py

@@ -0,0 +1,52 @@
+import mitogen
+import mitogen.service
+
+
+class FileService(mitogen.service.Service):
+    """
+    Simple file server, for demonstration purposes only! Use of this in
+    real code would be a security vulnerability as it would permit children
+    to read any file from the master's disk.
+    """
+
+    @mitogen.service.expose(policy=mitogen.service.AllowAny())
+    @mitogen.service.arg_spec(spec={
+        'path': str
+    })
+    def read_file(self, path):
+        with open(path, 'rb') as fp:
+            return fp.read()
+
+
+def download_file(source_context, path):
+    s = source_context.call_service(
+        service_name=FileService,  # may also be string 'pkg.mod.FileService'
+        method_name='read_file',
+        path=path,
+    )
+
+    with open(path, 'w') as fp:
+        fp.write(s)
+
+
+def download_some_files(source_context, paths):
+    for path in paths:
+        download_file(source_context, path)
+
+
+@mitogen.main()
+def main(router):
+    pool = mitogen.service.Pool(router, services=[
+        FileService(router),
+    ])
+
+    remote = router.ssh(hostname='k3')
+    remote.call(download_some_files,
+        source_context=router.myself(),
+        paths=[
+            '/etc/passwd',
+            '/etc/hosts',
+        ]
+    )
+    pool.stop()
+