Merge remote-tracking branch 'origin/dmw'

* origin/dmw:
  issue #605: update Changelog.
  issue #605: ansible: share a sem_t instead of a pthread_mutex_t
  issue #613: add tests for all the weird shutdown methods
  Add mitogen.core.now() and use it everywhere; closes #614.
  docs: move decorator docs into core.py and use autodecorator
  preamble_size: make it work on Python 3.
  docs: upgrade Sphinx to 2.1.2, require Python 3 to build docs.
  docs: fix Sphinx warnings, add LogHandler, more docstrings
  docs: tidy up some Changelog text

Author: dw

ansible_mitogen/affinity.py

@@ -92,37 +92,37 @@
     _libc = ctypes.CDLL(None, use_errno=True)
     _strerror = _libc.strerror
     _strerror.restype = ctypes.c_char_p
-    _pthread_mutex_init = _libc.pthread_mutex_init
-    _pthread_mutex_lock = _libc.pthread_mutex_lock
-    _pthread_mutex_unlock = _libc.pthread_mutex_unlock
+    _sem_init = _libc.sem_init
+    _sem_wait = _libc.sem_wait
+    _sem_post = _libc.sem_post
     _sched_setaffinity = _libc.sched_setaffinity
 except (OSError, AttributeError):
     _libc = None
     _strerror = None
-    _pthread_mutex_init = None
-    _pthread_mutex_lock = None
-    _pthread_mutex_unlock = None
+    _sem_init = None
+    _sem_wait = None
+    _sem_post = None
     _sched_setaffinity = None
 
 
-class pthread_mutex_t(ctypes.Structure):
+class sem_t(ctypes.Structure):
     """
-    Wrap pthread_mutex_t to allow storing a lock in shared memory.
+    Wrap sem_t to allow storing a lock in shared memory.
     """
     _fields_ = [
-        ('data', ctypes.c_uint8 * 512),
+        ('data', ctypes.c_uint8 * 128),
     ]
 
     def init(self):
-        if _pthread_mutex_init(self.data, 0):
+        if _sem_init(self.data, 1, 1):
             raise Exception(_strerror(ctypes.get_errno()))
 
     def acquire(self):
-        if _pthread_mutex_lock(self.data):
+        if _sem_wait(self.data):
             raise Exception(_strerror(ctypes.get_errno()))
 
     def release(self):
-        if _pthread_mutex_unlock(self.data):
+        if _sem_post(self.data):
             raise Exception(_strerror(ctypes.get_errno()))
 
 
@@ -133,7 +133,7 @@ class State(ctypes.Structure):
     the context of the new child process.
     """
     _fields_ = [
-        ('lock', pthread_mutex_t),
+        ('lock', sem_t),
         ('counter', ctypes.c_uint8),
     ]
 

docs/api.rst

@@ -2,11 +2,6 @@
 API Reference
 *************
 
-.. toctree::
-    :hidden:
-
-    signals
-
 
 Package Layout
 ==============
@@ -31,29 +26,10 @@ mitogen.core
 .. automodule:: mitogen.core
 
 .. currentmodule:: mitogen.core
-.. decorator:: takes_econtext
-
-    Decorator that marks a function or class method to automatically receive a
-    kwarg named `econtext`, referencing the
-    :class:`mitogen.core.ExternalContext` active in the context in which the
-    function is being invoked in. The decorator is only meaningful when the
-    function is invoked via :data:`CALL_FUNCTION
-    <mitogen.core.CALL_FUNCTION>`.
-
-    When the function is invoked directly, `econtext` must still be passed to
-    it explicitly.
+.. autodecorator:: takes_econtext
 
 .. currentmodule:: mitogen.core
-.. decorator:: takes_router
-
-    Decorator that marks a function or class method to automatically receive a
-    kwarg named `router`, referencing the :class:`mitogen.core.Router`
-    active in the context in which the function is being invoked in. The
-    decorator is only meaningful when the function is invoked via
-    :data:`CALL_FUNCTION <mitogen.core.CALL_FUNCTION>`.
-
-    When the function is invoked directly, `router` must still be passed to it
-    explicitly.
+.. autodecorator:: takes_router
 
 
 mitogen.master
@@ -627,6 +603,14 @@ Fork Safety
 Utility Functions
 =================
 
+.. currentmodule:: mitogen.core
+.. function:: now
+
+   A reference to :func:`time.time` on Python 2, or :func:`time.monotonic` on
+   Python >3.3. We prefer :func:`time.monotonic` when available to ensure
+   timers are not impacted by system clock changes.
+
+
 .. module:: mitogen.utils
 
 A random assortment of utility functions useful on masters and children.

docs/changelog.rst

@@ -22,7 +22,7 @@ To avail of fixes in an unreleased version, please download a ZIP file
 `directly from GitHub <https://github.com/dw/mitogen/>`_.
 
 Enhancements
-^^^^^^^^^^^^
+~~~~~~~~~~~~
 
 * `#556 <https://github.com/dw/mitogen/issues/556>`_,
   `#587 <https://github.com/dw/mitogen/issues/587>`_: Ansible 2.8 is partially
@@ -61,7 +61,7 @@ Enhancements
 
 
 Mitogen for Ansible
-^^^^^^^^^^^^^^^^^^^
+~~~~~~~~~~~~~~~~~~~
 
 * `#363 <https://github.com/dw/mitogen/issues/363>`_: fix an obscure race
   matching *Permission denied* errors from some versions of ``su`` running on
@@ -107,20 +107,21 @@ Mitogen for Ansible
   server has been increased from `15*3` seconds to `30*10` seconds.
 
 * `#600 <https://github.com/dw/mitogen/issues/600>`_: functionality to reflect
-  changes to ``/etc/environment`` in the running interpreter did not account
-  for Unicode file contents. Now the file may contain data in any single byte
-  encoding.
+  changes to ``/etc/environment`` did not account for Unicode file contents.
+  The file may now use any single byte encoding.
 
 * `#602 <https://github.com/dw/mitogen/issues/602>`_: connection configuration
   is more accurately inferred for `meta: reset_connection`, the `synchronize`
-  module, and for any other action plug-ins that establish new connections of
-  their own.
+  module, and for any action plug-ins that establish additional connections.
+
+* `#605 <https://github.com/dw/mitogen/issues/605>`_: fix a deadlock managing a
+  shared counter used for load balancing.
 
 * `#615 <https://github.com/dw/mitogen/issues/615>`_: streaming file transfer
-  is implemented for the ``fetch`` and any other action that transfers files
-  from the target to the controller. Previously the file would be sent as a
-  single message, requiring the file to fit in RAM and be smaller than internal
-  limits on the size of a single message.
+  is implemented for ``fetch`` and other actions that transfer files from the
+  target to the controller. Previously the file was sent in one message,
+  requiring it to fit in RAM and be smaller than the internal message size
+  limit.
 
 * `7ae926b3 <https://github.com/dw/mitogen/commit/7ae926b3>`_: the
   ``lineinfile`` module began leaking writable temporary file descriptors since
@@ -188,21 +189,20 @@ Core Library
   :meth:`empty` method of :class:`mitogen.core.Latch`,
   :class:`mitogen.core.Receiver` and :class:`mitogen.select.Select` has been
   replaced by a more general :meth:`size` method. :meth:`empty` will be removed
-  in Mitogen 0.3
+  in 0.3
 
 * `ecc570cb <https://github.com/dw/mitogen/commit/ecc570cb>`_: previously
-  :meth:`mitogen.select.Select.add` would enqueue a single wake event when
-  adding an existing receiver, latch or subselect that contained multiple
-  buffered items, causing future :meth:`get` calls to block or fail even though
-  data existed that could be returned.
+  :meth:`mitogen.select.Select.add` would enqueue one wake event when adding an
+  existing receiver, latch or subselect that contained multiple buffered items,
+  causing :meth:`get` calls to block or fail even though data existed to return.
 
-* `5924af15 <https://github.com/dw/mitogen/commit/5924af15>`_: *[security]* the
-  unidirectional routing mode, in which contexts may only communicate with
-  parents and never siblings (so a program cannot accidentally bridge
-  air-gapped networks) was not inherited when a child context was initiated
-  directly from an existing child. This did not effect the Ansible extension,
-  since the controller initiates any new context used for routing, only forked
-  tasks are initiated by children.
+* `5924af15 <https://github.com/dw/mitogen/commit/5924af15>`_: *[security]*
+  unidirectional routing, where contexts may optionally only communicate with
+  parents and never siblings (so that air-gapped networks cannot be
+  unintentionally bridged) was not inherited when a child was initiated
+  directly from an another child. This did not effect Ansible, since the
+  controller initiates any new child used for routing, only forked tasks are
+  initiated by children.
 
 
 Thanks!
@@ -491,7 +491,7 @@ Enhancements
   `#491 <https://github.com/dw/mitogen/issues/491>`_,
   `#493 <https://github.com/dw/mitogen/issues/493>`_: the interface employed
   for in-process queues changed from `kqueue
-  <https://www.freebsd.org/cgi/man.cgi?query=kqueue&sektion=2>`_ / `epoll
+  <https://www.freebsd.org/cgi/man.cgi?query=kqueue>`_ / `epoll
   <http://man7.org/linux/man-pages/man7/epoll.7.html>`_ to `poll()
   <http://man7.org/linux/man-pages/man2/poll.2.html>`_, which requires no setup
   or teardown, yielding a 38% latency reduction for inter-thread communication.
@@ -1037,7 +1037,7 @@ bug reports, testing, features and fixes in this release contributed by
 `Josh Smift <https://github.com/jbscare>`_,
 `Luca Nunzi <https://github.com/0xlc>`_,
 `Orion Poplawski <https://github.com/opoplawski>`_,
-`Peter V. Saveliev <https://github.com/svinota>`_,
+`Peter V. Saveliev <https://github.com/svinota/>`_,
 `Pierre-Henry Muller <https://github.com/pierrehenrymuller>`_,
 `Pierre-Louis Bonicoli <https://github.com/jesteria>`_,
 `Prateek Jain <https://github.com/prateekj201>`_,
@@ -1095,7 +1095,7 @@ Core Library
 
 * `#300 <https://github.com/dw/mitogen/issues/300>`_: the broker could crash on
   OS X during shutdown due to scheduled `kqueue
-  <https://www.freebsd.org/cgi/man.cgi?query=kevent>`_ filter changes for
+  <https://www.freebsd.org/cgi/man.cgi?query=kqueue>`_ filter changes for
   descriptors that were closed before the IO loop resumes. As a temporary
   workaround, kqueue's bulk change feature is not used.
 

docs/conf.py

@@ -7,7 +7,7 @@
 
 author = u'Network Genomics'
 copyright = u'2019, Network Genomics'
-exclude_patterns = ['_build']
+exclude_patterns = ['_build', '.venv']
 extensions = ['sphinx.ext.autodoc', 'sphinx.ext.intersphinx', 'sphinxcontrib.programoutput']
 html_show_copyright = False
 html_show_sourcelink = False

docs/internals.rst

@@ -7,11 +7,6 @@ Internal API Reference
    Internal APIs are subject to rapid change even across minor releases. This
    page exists to help users modify and extend the library.
 
-.. toctree::
-    :hidden:
-
-    signals
-
 
 Constants
 =========
@@ -50,6 +45,10 @@ Logging
 
 See also :class:`mitogen.core.IoLoggerProtocol`.
 
+.. currentmodule:: mitogen.core
+.. autoclass:: LogHandler
+   :members:
+
 .. currentmodule:: mitogen.master
 .. autoclass:: LogForwarder
    :members:
@@ -270,6 +269,8 @@ Helpers
 .. autofunction:: minimize_source
 
 
+.. _signals:
+
 Signals
 =======
 
@@ -312,10 +313,19 @@ These signals are used internally by Mitogen.
       - ``disconnect``
       - Fired on the Broker thread when disconnection is detected.
 
+    * - :py:class:`mitogen.core.Stream`
+      - ``shutdown``
+      - Fired on the Broker thread when broker shutdown begins.
+
     * - :py:class:`mitogen.core.Context`
       - ``disconnect``
       - Fired on the Broker thread during shutdown (???)
 
+    * - :py:class:`mitogen.parent.Process`
+      - ``exit``
+      - Fired when :class:`mitogen.parent.Reaper` detects subprocess has fully
+        exitted.
+
     * - :py:class:`mitogen.core.Broker`
       - ``shutdown``
       - Fired after Broker.shutdown() is called.

docs/requirements.txt

@@ -1,3 +1,3 @@
-Sphinx==1.7.1
-sphinxcontrib-programoutput==0.11
+Sphinx==2.1.2
+sphinxcontrib-programoutput==0.14
 alabaster==0.7.10