Merge pull request #1459 from moreati/release-0.3.42

Release 0.3.42

Author: moreati

ansible_mitogen/connection.py

@@ -256,6 +256,22 @@ def _connect_lxd(spec):
     }
 
 
+def _connect_incus(spec):
+    """
+    Return ContextService arguments for an Incus container connection.
+    """
+    return {
+        'method': 'incus',
+        'kwargs': {
+            'container': spec.remote_addr(),
+            'python_path': spec.python_path(),
+            'incus_path': spec.mitogen_incus_path(),
+            'connect_timeout': spec.timeout(),
+            'remote_name': get_remote_name(spec),
+        }
+    }
+
+
 def _connect_machinectl(spec):
     """
     Return ContextService arguments for a machinectl connection.
@@ -415,6 +431,7 @@ def _connect_mitogen_doas(spec):
     'kubectl': _connect_kubectl,
     'jail': _connect_jail,
     'local': _connect_local,
+    'incus': _connect_incus,
     'lxc': _connect_lxc,
     'lxd': _connect_lxd,
     'machinectl': _connect_machinectl,

ansible_mitogen/plugins/connection/mitogen_incus.py

@@ -0,0 +1,21 @@
+# SPDX-FileCopyrightText: 2019 David Wilson
+# SPDX-FileCopyrightText: 2026 Mitogen authors <https://github.com/mitogen-hq>
+# SPDX-License-Identifier: BSD-3-Clause
+# !mitogen: minify_safe
+
+from __future__ import absolute_import, division, print_function
+__metaclass__ = type
+
+import os
+import sys
+
+try:
+    import ansible_mitogen
+except ImportError:
+    sys.path.insert(0, os.path.abspath(os.path.join(__file__, '../../../..')))
+
+import ansible_mitogen.connection
+
+
+class Connection(ansible_mitogen.connection.Connection):
+    transport = 'incus'

ansible_mitogen/process.py

@@ -185,6 +185,7 @@ def _setup_responder(responder):
     certain packages, and to generate custom responses for certain modules.
     """
     responder.whitelist_prefix('ansible')
+    responder.whitelist_prefix('ansible_collections')
     responder.whitelist_prefix('ansible_mitogen')
 
     # Ansible 2.3 is compatible with Python 2.4 targets, however

ansible_mitogen/transport_config.py

@@ -349,6 +349,12 @@ def mitogen_kubectl_path(self):
         The path to the "kubectl" program for the 'docker' transport.
         """
 
+    @abc.abstractmethod
+    def mitogen_incus_path(self):
+        """
+        The path to the "incus" program for the 'incus' transport.
+        """
+
     @abc.abstractmethod
     def mitogen_lxc_path(self):
         """
@@ -584,6 +590,9 @@ def mitogen_docker_path(self):
     def mitogen_kubectl_path(self):
         return self._connection.get_task_var('mitogen_kubectl_path')
 
+    def mitogen_incus_path(self):
+        return self._connection.get_task_var('mitogen_incus_path')
+
     def mitogen_lxc_path(self):
         return self._connection.get_task_var('mitogen_lxc_path')
 
@@ -832,6 +841,9 @@ def mitogen_docker_path(self):
     def mitogen_kubectl_path(self):
         return self._host_vars.get('mitogen_kubectl_path')
 
+    def mitogen_incus_path(self):
+        return self._host_vars.get('mitogen_incus_path')
+
     def mitogen_lxc_path(self):
         return self._host_vars.get('mitogen_lxc_path')
 

docs/changelog.rst

@@ -18,6 +18,17 @@ To avail of fixes in an unreleased version, please download a ZIP file
 `directly from GitHub <https://github.com/mitogen-hq/mitogen/>`_.
 
 
+v0.3.42 (2026-02-20)
+--------------------
+
+* :gh:issue:`1451` :mod:`mitogen`: Refactor module whitelist & blacklist with
+  module overrides and blocks. Improve error messages for denied modules.
+* :gh:issue:`1394` :mod:`ansible_mitogen`, :mod:`mitogen`: Add Incus connection
+  support, allowing users who migrated from LXD to Incus to use ansible-mitogen
+* :gh:issue:`1456` :mod:`ansible_mitogen`: Fix Ansible collections
+  :exc:`!mitogen.core.ModuleDeniedByOverridesError`
+
+
 v0.3.41 (2026-02-10)
 --------------------
 

mitogen/__init__.py

@@ -35,7 +35,7 @@
 
 
 #: Library version as a tuple.
-__version__ = (0, 3, 41)
+__version__ = (0, 3, 42)
 
 
 #: This is :data:`False` in slave contexts. Previously it was used to prevent

mitogen/core.py

@@ -72,6 +72,7 @@ def _path_importer_cache(cls, path):
 import os
 import pstats
 import pty
+import re
 import signal
 import socket
 import struct
@@ -360,6 +361,18 @@ class LatchError(Error):
     pass
 
 
+class ModuleDeniedByOverridesError(ModuleNotFoundError):
+    fmt = "Mitogen won't serve %s, it's not in the overrides list"
+
+
+class ModuleDeniedByBlocksError(ModuleNotFoundError):
+    fmt = "Mitogen won't serve %s, it's in the blocks list"
+
+
+class ModuleUnsuitableError(ModuleNotFoundError):
+    fmt = "Mitogen won't serve %s, it's e.g. binary, legacy, part of stdlib"
+
+
 class Blob(BytesType):
     """
     A serializable bytes subclass whose content is summarized in repr() output,
@@ -499,6 +512,11 @@ def has_parent_authority(msg, _stream=None):
     return _has_parent_authority(msg.auth_id)
 
 
+def module_lineage(fullname):
+    "Return an iterator of a module's parent fullnames and its own"
+    return (fullname[:m.start()] for m in re.finditer(r'\.|\Z', fullname))
+
+
 def _signals(obj, signal):
     return (
         obj.__dict__
@@ -564,21 +582,6 @@ def takes_router(func):
     return func
 
 
-def is_blacklisted_import(importer, fullname):
-    """
-    Return :data:`True` if `fullname` is part of a blacklisted package, or if
-    any packages have been whitelisted and `fullname` is not part of one.
-
-    NB:
-      - The default whitelist is `['']` which matches any module name.
-      - If a package is on both lists, then it is treated as blacklisted.
-      - If any package is whitelisted, then all non-whitelisted packages are
-        treated as blacklisted.
-    """
-    return ((not any(fullname.startswith(s) for s in importer.whitelist)) or
-                (any(fullname.startswith(s) for s in importer.blacklist)))
-
-
 def set_cloexec(fd):
     """
     Set the file descriptor `fd` to automatically close on :func:`os.execve`.
@@ -1302,6 +1305,46 @@ def __repr__(self):
         )
 
 
+class ImportPolicy(object):
+    """
+    Policy deciding which module prefixes :class:`Importer` will request from
+    :class:`mitogen.master.ModuleResponder` and which requests will be served
+    or denied.
+
+    :param overrides:
+        Prefixes always requested, ignoring local versions. If ``overrides``
+        has entries, then it's also used as an allow list by the responder -
+        any request for a prefix that's not overriden will be denied.
+
+    :param blocks:
+        Prefixes always denied by the responder, only local versions can be
+        used.
+    """
+    def __init__(self, overrides=(), blocks=()):
+        self.overrides = set(overrides)
+        self.blocks = set(blocks)
+        self._always = set(Importer.ALWAYS_BLACKLIST)
+
+    def denied(self, fullname):
+        fullnames = frozenset(module_lineage(fullname))
+        if self.overrides and not self.overrides.intersection(fullnames):
+            return ModuleDeniedByOverridesError
+        if self.blocks.intersection(fullnames): return ModuleDeniedByBlocksError
+        if self._always.intersection(fullnames): return ModuleUnsuitableError
+        return False
+
+    def denied_raise(self, fullname):
+        denial = self.denied(fullname)
+        if denial: raise denial(denial.fmt % (fullname,))
+
+    def overriden(self, fullname):
+        return bool(self.overrides.intersection(module_lineage(fullname)))
+
+    def __repr__(self):
+        args = (type(self).__name__, self.overrides, self.blocks)
+        return '%s(overrides=%r, blocks=%r)' % args
+
+
 class Importer(object):
     """
     Import protocol implementation that fetches modules from the parent
@@ -1361,18 +1404,12 @@ class Importer(object):
     if sys.version_info >= (3, 0):
         ALWAYS_BLACKLIST += ['cStringIO']
 
-    def __init__(self, router, context, core_src, whitelist=(), blacklist=()):
+    def __init__(self, router, context, core_src, policy):
         self._log = logging.getLogger('mitogen.importer')
         self._context = context
         self._present = {'mitogen': self.MITOGEN_PKG_CONTENT}
         self._lock = threading.Lock()
-        self.whitelist = list(whitelist) or ['']
-        self.blacklist = list(blacklist) + self.ALWAYS_BLACKLIST
-
-        # Preserve copies of the original server-supplied whitelist/blacklist
-        # for later use by children.
-        self.master_whitelist = self.whitelist[:]
-        self.master_blacklist = self.blacklist[:]
+        self.policy = policy
 
         # Presence of an entry in this map indicates in-flight GET_MODULE.
         self._callbacks = {}
@@ -1465,11 +1502,8 @@ def find_module(self, fullname, path=None):
                 self._log.debug('%s has no submodule %s', pkgname, suffix)
                 return None
 
-            # #114: explicitly whitelisted prefixes override any
-            # system-installed package.
-            if self.whitelist != ['']:
-                if any(fullname.startswith(s) for s in self.whitelist):
-                    return self
+            if self.policy.overriden(fullname):
+                return self
 
             try:
                 self.builtin_find_module(fullname)
@@ -1515,11 +1549,9 @@ def find_spec(self, fullname, path, target=None):
                       fullname, pkgname, pkg_loader)
             return None
 
-        # #114: whitelisted prefixes override any system-installed package.
-        if self.whitelist != ['']:
-            if any(s and fullname.startswith(s) for s in self.whitelist):
-                log.debug('Handling %s. It is whitelisted', fullname)
-                return importlib.machinery.ModuleSpec(fullname, loader=self)
+        if self.policy.overriden(fullname):
+            log.debug('Handling %s. It is overriden', fullname)
+            return importlib.machinery.ModuleSpec(fullname, loader=self)
 
         if fullname == '__main__':
             log.debug('Handling %s. A special case', fullname)
@@ -1540,10 +1572,6 @@ def find_spec(self, fullname, path, target=None):
         log.debug('Handling %s. Unavailable locally', fullname)
         return importlib.machinery.ModuleSpec(fullname, loader=self)
 
-    blacklisted_msg = (
-        'A %r request would be refused by the Mitogen master. The module is '
-        'on the deny list (blacklist) or not on the allow list (whitelist).'
-    )
     pkg_resources_msg = (
         'pkg_resources is prohibited from importing __main__, as it causes '
         'problems in applications whose main module is not designed to be '
@@ -1556,8 +1584,7 @@ def find_spec(self, fullname, path, target=None):
     )
 
     def _refuse_imports(self, fullname):
-        if is_blacklisted_import(self, fullname):
-            raise ModuleNotFoundError(self.blacklisted_msg % (fullname,))
+        self.policy.denied_raise(fullname)
 
         f = sys._getframe(2)
         requestee = f.f_globals['__name__']
@@ -4175,12 +4202,15 @@ def _setup_importer(self):
             else:
                 core_src = None
 
+            policy = ImportPolicy(
+                self.config['import_overrides'],
+                self.config['import_blocks'],
+            )
             importer = Importer(
                 self.router,
                 self.parent,
                 core_src,
-                self.config.get('whitelist', ()),
-                self.config.get('blacklist', ()),
+                policy,
             )
 
         self.importer = importer

mitogen/incus.py

@@ -0,0 +1,50 @@
+# SPDX-FileCopyrightText: 2019 David Wilson
+# SPDX-FileCopyrightText: 2026 Mitogen authors <https://github.com/mitogen-hq>
+# SPDX-License-Identifier: BSD-3-Clause
+# !mitogen: minify_safe
+
+import mitogen.parent
+
+
+class Options(mitogen.parent.Options):
+    container = None
+    incus_path = 'incus'
+    python_path = 'python'
+
+    def __init__(self, container, incus_path=None, **kwargs):
+        super(Options, self).__init__(**kwargs)
+        self.container = container
+        if incus_path:
+            self.incus_path = incus_path
+
+
+class Connection(mitogen.parent.Connection):
+    options_class = Options
+
+    child_is_immediate_subprocess = False
+    create_child_args = {
+        # If incus finds any of stdin, stdout, stderr connected to a TTY, to
+        # prevent input injection it creates a proxy pty, forcing all IO to be
+        # buffered in <4KiB chunks. So ensure stderr is also routed to the
+        # socketpair.
+        'merge_stdio': True
+    }
+
+    eof_error_hint = (
+        'Note: many versions of Incus do not report program execution failure '
+        'meaningfully. Please check the host logs (/var/log) for more '
+        'information.'
+    )
+
+    def _get_name(self):
+        return u'incus.' + self.options.container
+
+    def get_boot_command(self):
+        bits = [
+            self.options.incus_path,
+            'exec',
+            '--mode=non-interactive',
+            self.options.container,
+            '--',
+        ]
+        return bits + super(Connection, self).get_boot_command()