Merge remote-tracking branch 'origin/dmw'

Author: dw

.ci/ci_lib.py

@@ -264,7 +264,7 @@ def start_containers(containers):
             "docker rm -f %(name)s || true" % container,
             "docker run "
                 "--rm "
-                "--cpuset-cpus 0,1 "
+                # "--cpuset-cpus 0,1 "
                 "--detach "
                 "--privileged "
                 "--cap-add=SYS_PTRACE "

.ci/mitogen_py24_install.py

@@ -7,7 +7,7 @@
         'docker pull %s' % (ci_lib.image_for_distro(ci_lib.DISTRO),),
     ],
     [
-        'sudo tar -C / -jxvf tests/data/ubuntu-python-2.4.6.tar.bz2',
+        'curl https://dw.github.io/mitogen/binaries/ubuntu-python-2.4.6.tar.bz2 | sudo tar -C / -jxv',
     ]
 ]
 

ansible_mitogen/affinity.py

@@ -265,9 +265,19 @@ def _mask_to_bytes(self, mask):
             mask >>= 64
         return mitogen.core.b('').join(chunks)
 
+    def _get_thread_ids(self):
+        try:
+            ents = os.listdir('/proc/self/task')
+        except OSError:
+            LOG.debug('cannot fetch thread IDs for current process')
+            return [os.getpid()]
+
+        return [int(s) for s in ents if s.isdigit()]
+
     def _set_cpu_mask(self, mask):
         s = self._mask_to_bytes(mask)
-        _sched_setaffinity(os.getpid(), len(s), s)
+        for tid in self._get_thread_ids():
+            _sched_setaffinity(tid, len(s), s)
 
 
 if _sched_setaffinity is not None:

ansible_mitogen/connection.py

@@ -791,7 +791,9 @@ def _connect(self):
 
         inventory_name, stack = self._build_stack()
         worker_model = ansible_mitogen.process.get_worker_model()
-        self.binding = worker_model.get_binding(inventory_name)
+        self.binding = worker_model.get_binding(
+            mitogen.utils.cast(inventory_name)
+        )
         self._connect_stack(stack)
 
     def _put_connection(self):

ansible_mitogen/loaders.py

@@ -55,3 +55,8 @@
     from ansible.plugins import module_utils_loader
     from ansible.plugins import shell_loader
     from ansible.plugins import strategy_loader
+
+
+# These are original, unwrapped implementations
+action_loader__get = action_loader.get
+connection_loader__get = connection_loader.get

ansible_mitogen/logging.py

@@ -107,8 +107,9 @@ def setup():
     l_mitogen = logging.getLogger('mitogen')
     l_mitogen_io = logging.getLogger('mitogen.io')
     l_ansible_mitogen = logging.getLogger('ansible_mitogen')
+    l_operon = logging.getLogger('operon')
 
-    for logger in l_mitogen, l_mitogen_io, l_ansible_mitogen:
+    for logger in l_mitogen, l_mitogen_io, l_ansible_mitogen, l_operon:
         logger.handlers = [Handler(display.vvv)]
         logger.propagate = False
 

ansible_mitogen/mixins.py

@@ -55,6 +55,11 @@
 import ansible_mitogen.target
 from ansible.module_utils._text import to_text
 
+try:
+    from ansible.utils.unsafe_proxy import wrap_var
+except ImportError:
+    from ansible.vars.unsafe_proxy import wrap_var
+
 
 LOG = logging.getLogger(__name__)
 
@@ -306,7 +311,7 @@ def get_task_timeout_secs(self):
         except AttributeError:
             return getattr(self._task, 'async')
 
-    def _temp_file_gibberish(self, module_args, wrap_async):
+    def _set_temp_file_args(self, module_args, wrap_async):
         # Ansible>2.5 module_utils reuses the action's temporary directory if
         # one exists. Older versions error if this key is present.
         if ansible.__version__ > '2.5':
@@ -343,7 +348,7 @@ def _execute_module(self, module_name=None, module_args=None, tmp=None,
         self._update_module_args(module_name, module_args, task_vars)
         env = {}
         self._compute_environment_string(env)
-        self._temp_file_gibberish(module_args, wrap_async)
+        self._set_temp_file_args(module_args, wrap_async)
 
         self._connection._connect()
         result = ansible_mitogen.planner.invoke(
@@ -365,7 +370,7 @@ def _execute_module(self, module_name=None, module_args=None, tmp=None,
             # on _execute_module().
             self._remove_tmp_path(tmp)
 
-        return result
+        return wrap_var(result)
 
     def _postprocess_response(self, result):
         """

ansible_mitogen/planner.py

@@ -45,6 +45,7 @@
 from ansible.executor import module_common
 import ansible.errors
 import ansible.module_utils
+import ansible.release
 import mitogen.core
 import mitogen.select
 
@@ -58,6 +59,8 @@
 NO_INTERPRETER_MSG = 'module (%s) is missing interpreter line'
 NO_MODULE_MSG = 'The module %s was not found in configured module paths.'
 
+_planner_by_path = {}
+
 
 class Invocation(object):
     """
@@ -92,7 +95,12 @@ def __init__(self, action, connection, module_name, module_args,
         self.module_path = None
         #: Initially ``None``, but set by :func:`invoke`. The raw source or
         #: binary contents of the module.
-        self.module_source = None
+        self._module_source = None
+
+    def get_module_source(self):
+        if self._module_source is None:
+            self._module_source = read_file(self.module_path)
+        return self._module_source
 
     def __repr__(self):
         return 'Invocation(module_name=%s)' % (self.module_name,)
@@ -107,7 +115,8 @@ class Planner(object):
     def __init__(self, invocation):
         self._inv = invocation
 
-    def detect(self):
+    @classmethod
+    def detect(cls, path, source):
         """
         Return true if the supplied `invocation` matches the module type
         implemented by this planner.
@@ -171,8 +180,9 @@ class BinaryPlanner(Planner):
     """
     runner_name = 'BinaryRunner'
 
-    def detect(self):
-        return module_common._is_binary(self._inv.module_source)
+    @classmethod
+    def detect(cls, path, source):
+        return module_common._is_binary(source)
 
     def get_push_files(self):
         return [mitogen.core.to_text(self._inv.module_path)]
@@ -218,7 +228,7 @@ def _rewrite_interpreter(self, path):
 
     def _get_interpreter(self):
         path, arg = ansible_mitogen.parsing.parse_hashbang(
-            self._inv.module_source
+            self._inv.get_module_source()
         )
         if path is None:
             raise ansible.errors.AnsibleError(NO_INTERPRETER_MSG % (
@@ -247,8 +257,9 @@ class JsonArgsPlanner(ScriptPlanner):
     """
     runner_name = 'JsonArgsRunner'
 
-    def detect(self):
-        return module_common.REPLACER_JSONARGS in self._inv.module_source
+    @classmethod
+    def detect(cls, path, source):
+        return module_common.REPLACER_JSONARGS in source
 
 
 class WantJsonPlanner(ScriptPlanner):
@@ -265,8 +276,9 @@ class WantJsonPlanner(ScriptPlanner):
     """
     runner_name = 'WantJsonRunner'
 
-    def detect(self):
-        return b'WANT_JSON' in self._inv.module_source
+    @classmethod
+    def detect(cls, path, source):
+        return b'WANT_JSON' in source
 
 
 class NewStylePlanner(ScriptPlanner):
@@ -278,8 +290,9 @@ class NewStylePlanner(ScriptPlanner):
     runner_name = 'NewStyleRunner'
     marker = b'from ansible.module_utils.'
 
-    def detect(self):
-        return self.marker in self._inv.module_source
+    @classmethod
+    def detect(cls, path, source):
+        return cls.marker in source
 
     def _get_interpreter(self):
         return None, None
@@ -323,7 +336,6 @@ def get_search_path(self):
             for path in ansible_mitogen.loaders.module_utils_loader._get_paths(
                 subdirs=False
             )
-            if os.path.isdir(path)
         )
 
     _module_map = None
@@ -347,6 +359,10 @@ def get_module_map(self):
     def get_kwargs(self):
         return super(NewStylePlanner, self).get_kwargs(
             module_map=self.get_module_map(),
+            py_module_name=py_modname_from_path(
+                self._inv.module_name,
+                self._inv.module_path,
+            ),
         )
 
 
@@ -376,14 +392,16 @@ class ReplacerPlanner(NewStylePlanner):
     """
     runner_name = 'ReplacerRunner'
 
-    def detect(self):
-        return module_common.REPLACER in self._inv.module_source
+    @classmethod
+    def detect(cls, path, source):
+        return module_common.REPLACER in source
 
 
 class OldStylePlanner(ScriptPlanner):
     runner_name = 'OldStyleRunner'
 
-    def detect(self):
+    @classmethod
+    def detect(cls, path, source):
         # Everything else.
         return True
 
@@ -398,14 +416,54 @@ def detect(self):
 ]
 
 
-def get_module_data(name):
-    path = ansible_mitogen.loaders.module_loader.find_plugin(name, '')
-    if path is None:
-        raise ansible.errors.AnsibleError(NO_MODULE_MSG % (name,))
+try:
+    _get_ansible_module_fqn = module_common._get_ansible_module_fqn
+except AttributeError:
+    _get_ansible_module_fqn = None
+
 
-    with open(path, 'rb') as fp:
-        source = fp.read()
-    return mitogen.core.to_text(path), source
+def py_modname_from_path(name, path):
+    """
+    Fetch the logical name of a new-style module as it might appear in
+    :data:`sys.modules` of the target's Python interpreter.
+
+    * For Ansible <2.7, this is an unpackaged module named like
+      "ansible_module_%s".
+
+    * For Ansible <2.9, this is an unpackaged module named like
+      "ansible.modules.%s"
+
+    * Since Ansible 2.9, modules appearing within a package have the original
+      package hierarchy approximated on the target, enabling relative imports
+      to function correctly. For example, "ansible.modules.system.setup".
+    """
+    # 2.9+
+    if _get_ansible_module_fqn:
+        try:
+            return _get_ansible_module_fqn(path)
+        except ValueError:
+            pass
+
+    if ansible.__version__ < '2.7':
+        return 'ansible_module_' + name
+
+    return 'ansible.modules.' + name
+
+
+def read_file(path):
+    fd = os.open(path, os.O_RDONLY)
+    try:
+        bits = []
+        chunk = True
+        while True:
+            chunk = os.read(fd, 65536)
+            if not chunk:
+                break
+            bits.append(chunk)
+    finally:
+        os.close(fd)
+
+    return mitogen.core.b('').join(bits)
 
 
 def _propagate_deps(invocation, planner, context):
@@ -466,14 +524,12 @@ def _invoke_isolated_task(invocation, planner):
         context.shutdown()
 
 
-def _get_planner(invocation):
+def _get_planner(name, path, source):
     for klass in _planners:
-        planner = klass(invocation)
-        if planner.detect():
-            LOG.debug('%r accepted %r (filename %r)', planner,
-                      invocation.module_name, invocation.module_path)
-            return planner
-        LOG.debug('%r rejected %r', planner, invocation.module_name)
+        if klass.detect(path, source):
+            LOG.debug('%r accepted %r (filename %r)', klass, name, path)
+            return klass
+        LOG.debug('%r rejected %r', klass, name)
     raise ansible.errors.AnsibleError(NO_METHOD_MSG + repr(invocation))
 
 
@@ -488,10 +544,24 @@ def invoke(invocation):
     :raises ansible.errors.AnsibleError:
         Unrecognized/unsupported module type.
     """
-    (invocation.module_path,
-     invocation.module_source) = get_module_data(invocation.module_name)
-    planner = _get_planner(invocation)
+    path = ansible_mitogen.loaders.module_loader.find_plugin(
+        invocation.module_name,
+        '',
+    )
+    if path is None:
+        raise ansible.errors.AnsibleError(NO_MODULE_MSG % (
+            invocation.module_name,
+        ))
+
+    invocation.module_path = mitogen.core.to_text(path)
+    if invocation.module_path not in _planner_by_path:
+        _planner_by_path[invocation.module_path] = _get_planner(
+            invocation.module_name,
+            invocation.module_path,
+            invocation.get_module_source()
+        )
 
+    planner = _planner_by_path[invocation.module_path](invocation)
     if invocation.wrap_async:
         response = _invoke_async_task(invocation, planner)
     elif planner.should_fork():

ansible_mitogen/plugins/connection/mitogen_kubectl.py

@@ -31,11 +31,6 @@
 import os.path
 import sys
 
-try:
-    from ansible.plugins.connection import kubectl
-except ImportError:
-    kubectl = None
-
 from ansible.errors import AnsibleConnectionFailure
 from ansible.module_utils.six import iteritems
 
@@ -47,6 +42,19 @@
     del base_dir
 
 import ansible_mitogen.connection
+import ansible_mitogen.loaders
+
+
+_class = ansible_mitogen.loaders.connection_loader__get(
+    'kubectl',
+    class_only=True,
+)
+
+if _class:
+    kubectl = sys.modules[_class.__module__]
+    del _class
+else:
+    kubectl = None
 
 
 class Connection(ansible_mitogen.connection.Connection):