Merge pull request #1065 from moreati/issue957

ansible_mitogen: Fix "filedescriptor out of range in select()" in WorkerProcess

Author: moreati

ansible_mitogen/process.py

@@ -281,11 +281,11 @@ def get_cpu_count(default=None):
 
 class Broker(mitogen.master.Broker):
     """
-    WorkerProcess maintains at most 2 file descriptors, therefore does not need
+    WorkerProcess maintains fewer file descriptors, therefore does not need
     the exuberant syscall expense of EpollPoller, so override it and restore
     the poll() poller.
     """
-    poller_class = mitogen.core.Poller
+    poller_class = mitogen.parent.POLLER_LIGHTWEIGHT
 
 
 class Binding(object):

docs/changelog.rst

@@ -22,6 +22,8 @@ Unreleased
 ----------
 
 * :gh:issue:`952` Fix Ansible `--ask-become-pass`, add test coverage
+* :gh:issue:`957` Fix Ansible exception when executing against 10s of hosts
+  "ValueError: filedescriptor out of range in select()"
 
 
 v0.3.7 (2024-04-08)

docs/contributors.rst

@@ -126,11 +126,13 @@ sponsorship and outstanding future-thinking of its early adopters.
     <li>jgadling</li>
     <li>John F Wall &mdash; <em>Making Ansible Great with Massive Parallelism</em></li>
     <li>KennethC</li>
+    <li><a href="https://github.com/lberruti">Luca Berruti</li>
     <li>Lewis Bellwood &mdash; <em>Happy to be apart of a great project.</em></li>
     <li>luto</li>
     <li><a href="https://mayeu.me/">Mayeu a.k.a Matthieu Maury</a></li>
     <li><a href="https://twitter.com/nathanhruby">@nathanhruby</a></li>
     <li><a href="https://github.com/opoplawski">Orion Poplawski</a></li>
+    <li><a href="https://github.com/philfry">Philippe Kueck</a></li>
     <li><a href="http://pageflows.com/">Ramy</a></li>
     <li>Scott Vokes</li>
     <li><a href="https://twitter.com/sirtux">Tom Eichhorn</a></li>

mitogen/core.py

@@ -2523,8 +2523,7 @@ class Poller(object):
     """
     A poller manages OS file descriptors the user is waiting to become
     available for IO. The :meth:`poll` method blocks the calling thread
-    until one or more become ready. The default implementation is based on
-    :func:`select.poll`.
+    until one or more become ready.
 
     Each descriptor has an associated `data` element, which is unique for each
     readiness type, and defaults to being the same as the file descriptor. The
@@ -2546,19 +2545,13 @@ class Poller(object):
     a resource leak.
 
     Pollers may only be used by one thread at a time.
+
+    This implementation uses :func:`select.select` for wider platform support.
+    That is considered an implementation detail. Previous versions have used
+    :func:`select.poll`. Future versions may decide at runtime.
     """
     SUPPORTED = True
 
-    # This changed from select() to poll() in Mitogen 0.2.4. Since poll() has
-    # no upper FD limit, it is suitable for use with Latch, which must handle
-    # FDs larger than select's limit during many-host runs. We want this
-    # because poll() requires no setup and teardown: just a single system call,
-    # which is important because Latch.get() creates a Poller on each
-    # invocation. In a microbenchmark, poll() vs. epoll_ctl() is 30% faster in
-    # this scenario. If select() must return in future, it is important
-    # Latch.poller_class is set from parent.py to point to the industrial
-    # strength poller for the OS, otherwise Latch will fail randomly.
-
     #: Increments on every poll(). Used to version _rfds and _wfds.
     _generation = 1
 
@@ -2681,11 +2674,10 @@ class Latch(object):
 
     See :ref:`waking-sleeping-threads` for further discussion.
     """
-    #: The :class:`Poller` implementation to use for waiting. Since the poller
-    #: will be very short-lived, we prefer :class:`mitogen.parent.PollPoller`
-    #: if it is available, or :class:`mitogen.core.Poller` otherwise, since
-    #: these implementations require no system calls to create, configure or
-    #: destroy.
+    #: The :class:`Poller` implementation to use. Instances are short lived so
+    #: prefer :class:`mitogen.parent.PollPoller` if it's available, otherwise
+    #: :class:`mitogen.core.Poller`. They don't need syscalls to create,
+    #: configure, or destroy. Replaced during import of :mod:`mitogen.parent`.
     poller_class = Poller
 
     #: If not :data:`None`, a function invoked as `notify(latch)` after a

mitogen/parent.py

@@ -745,8 +745,7 @@ def _upgrade_broker(broker):
     broker.timers = TimerList()
     LOG.debug('upgraded %r with %r (new: %d readers, %d writers; '
               'old: %d readers, %d writers)', old, new,
-              len(new.readers), len(new.writers),
-              len(old.readers), len(old.writers))
+              len(new._rfds), len(new._wfds), len(old._rfds), len(old._wfds))
 
 
 @mitogen.core.takes_econtext
@@ -902,22 +901,18 @@ def __repr__(self):
 class PollPoller(mitogen.core.Poller):
     """
     Poller based on the POSIX :linux:man2:`poll` interface. Not available on
-    some versions of OS X, otherwise it is the preferred poller for small FD
-    counts, as there is no setup/teardown/configuration system call overhead.
+    some Python/OS X combinations. Otherwise the preferred poller for small
+    FD counts; or if many pollers are created, used once, then closed.
+    There there is no setup/teardown/configuration system call overhead.
     """
     SUPPORTED = hasattr(select, 'poll')
-    _repr = 'PollPoller()'
+    _readmask = SUPPORTED and select.POLLIN | select.POLLHUP
 
     def __init__(self):
         super(PollPoller, self).__init__()
         self._pollobj = select.poll()
 
     # TODO: no proof we dont need writemask too
-    _readmask = (
-        getattr(select, 'POLLIN', 0) |
-        getattr(select, 'POLLHUP', 0)
-    )
-
     def _update(self, fd):
         mask = (((fd in self._rfds) and self._readmask) |
                 ((fd in self._wfds) and select.POLLOUT))
@@ -952,7 +947,6 @@ class KqueuePoller(mitogen.core.Poller):
     Poller based on the FreeBSD/Darwin :freebsd:man2:`kqueue` interface.
     """
     SUPPORTED = hasattr(select, 'kqueue')
-    _repr = 'KqueuePoller()'
 
     def __init__(self):
         super(KqueuePoller, self).__init__()
@@ -1030,7 +1024,7 @@ class EpollPoller(mitogen.core.Poller):
     Poller based on the Linux :linux:man7:`epoll` interface.
     """
     SUPPORTED = hasattr(select, 'epoll')
-    _repr = 'EpollPoller()'
+    _inmask = SUPPORTED and select.EPOLLIN | select.EPOLLHUP
 
     def __init__(self):
         super(EpollPoller, self).__init__()
@@ -1077,9 +1071,6 @@ def stop_transmit(self, fd):
         self._wfds.pop(fd, None)
         self._control(fd)
 
-    _inmask = (getattr(select, 'EPOLLIN', 0) |
-               getattr(select, 'EPOLLHUP', 0))
-
     def _poll(self, timeout):
         the_timeout = -1
         if timeout is not None:
@@ -1100,18 +1091,14 @@ def _poll(self, timeout):
                     yield data
 
 
-# 2.4 and 2.5 only had select.select() and select.poll().
-for _klass in mitogen.core.Poller, PollPoller, KqueuePoller, EpollPoller:
-    if _klass.SUPPORTED:
-        PREFERRED_POLLER = _klass
+POLLERS = (EpollPoller, KqueuePoller, PollPoller, mitogen.core.Poller)
+PREFERRED_POLLER = next(cls for cls in POLLERS if cls.SUPPORTED)
+
 
 # For processes that start many threads or connections, it's possible Latch
 # will also get high-numbered FDs, and so select() becomes useless there too.
-# So swap in our favourite poller.
-if PollPoller.SUPPORTED:
-    mitogen.core.Latch.poller_class = PollPoller
-else:
-    mitogen.core.Latch.poller_class = PREFERRED_POLLER
+POLLER_LIGHTWEIGHT = PollPoller.SUPPORTED and PollPoller or PREFERRED_POLLER
+mitogen.core.Latch.poller_class = POLLER_LIGHTWEIGHT
 
 
 class LineLoggingProtocolMixin(object):