Merge remote-tracking branch 'origin/stable-029' into stable

Author: dw

.ci/ci_lib.py

@@ -264,7 +264,7 @@ def start_containers(containers):
             "docker rm -f %(name)s || true" % container,
             "docker run "
                 "--rm "
-                "--cpuset-cpus 0,1 "
+                # "--cpuset-cpus 0,1 "
                 "--detach "
                 "--privileged "
                 "--cap-add=SYS_PTRACE "

.ci/mitogen_py24_install.py

@@ -7,7 +7,7 @@
         'docker pull %s' % (ci_lib.image_for_distro(ci_lib.DISTRO),),
     ],
     [
-        'sudo tar -C / -jxvf tests/data/ubuntu-python-2.4.6.tar.bz2',
+        'curl https://dw.github.io/mitogen/binaries/ubuntu-python-2.4.6.tar.bz2 | sudo tar -C / -jxv',
     ]
 ]
 

ansible_mitogen/affinity.py

@@ -265,9 +265,19 @@ def _mask_to_bytes(self, mask):
             mask >>= 64
         return mitogen.core.b('').join(chunks)
 
+    def _get_thread_ids(self):
+        try:
+            ents = os.listdir('/proc/self/task')
+        except OSError:
+            LOG.debug('cannot fetch thread IDs for current process')
+            return [os.getpid()]
+
+        return [int(s) for s in ents if s.isdigit()]
+
     def _set_cpu_mask(self, mask):
         s = self._mask_to_bytes(mask)
-        _sched_setaffinity(os.getpid(), len(s), s)
+        for tid in self._get_thread_ids():
+            _sched_setaffinity(tid, len(s), s)
 
 
 if _sched_setaffinity is not None:

ansible_mitogen/connection.py

@@ -56,6 +56,12 @@
 
 LOG = logging.getLogger(__name__)
 
+task_vars_msg = (
+    'could not recover task_vars. This means some connection '
+    'settings may erroneously be reset to their defaults. '
+    'Please report a bug if you encounter this message.'
+)
+
 
 def get_remote_name(spec):
     """
@@ -486,15 +492,9 @@ class Connection(ansible.plugins.connection.ConnectionBase):
     # the case of the synchronize module.
     #
 
-    #: Set to the host name as it appears in inventory by on_action_run().
-    inventory_hostname = None
-
     #: Set to task_vars by on_action_run().
     _task_vars = None
 
-    #: Set to 'hostvars' by on_action_run()
-    host_vars = None
-
     #: Set by on_action_run()
     delegate_to_hostname = None
 
@@ -527,12 +527,10 @@ def on_action_run(self, task_vars, delegate_to_hostname, loader_basedir):
         :param str loader_basedir:
             Loader base directory; see :attr:`loader_basedir`.
         """
-        self.inventory_hostname = task_vars['inventory_hostname']
         self._task_vars = task_vars
-        self.host_vars = task_vars['hostvars']
         self.delegate_to_hostname = delegate_to_hostname
         self.loader_basedir = loader_basedir
-        self._mitogen_reset(mode='put')
+        self._put_connection()
 
     def _get_task_vars(self):
         """
@@ -552,8 +550,10 @@ def _get_task_vars(self):
         for new connections to be constructed in addition to the preconstructed
         connection passed into any running action.
         """
-        f = sys._getframe()
+        if self._task_vars is not None:
+            return self._task_vars
 
+        f = sys._getframe()
         while f:
             if f.f_code.co_name == 'run':
                 f_locals = f.f_locals
@@ -571,9 +571,23 @@ def _get_task_vars(self):
 
             f = f.f_back
 
-        LOG.warning('could not recover task_vars. This means some connection '
-                    'settings may erroneously be reset to their defaults. '
-                    'Please report a bug if you encounter this message.')
+        raise ansible.errors.AnsibleConnectionFailure(task_vars_msg)
+
+    def get_host_vars(self, inventory_hostname):
+        """
+        Fetch the HostVars for a host.
+
+        :returns:
+            Variables dictionary or :data:`None`.
+        :raises ansible.errors.AnsibleConnectionFailure:
+            Task vars unavailable.
+        """
+        task_vars = self._get_task_vars()
+        hostvars = task_vars.get('hostvars')
+        if hostvars:
+            return hostvars.get(inventory_hostname)
+
+        raise ansible.errors.AnsibleConnectionFailure(task_vars_msg)
 
     def get_task_var(self, key, default=None):
         """
@@ -586,17 +600,16 @@ def get_task_var(self, key, default=None):
         does not make sense to extract connection-related configuration for the
         delegated-to machine from them.
         """
-        task_vars = self._task_vars or self._get_task_vars()
-        if task_vars is not None:
-            if self.delegate_to_hostname is None:
+        task_vars = self._get_task_vars()
+        if self.delegate_to_hostname is None:
+            if key in task_vars:
+                return task_vars[key]
+        else:
+            delegated_vars = task_vars['ansible_delegated_vars']
+            if self.delegate_to_hostname in delegated_vars:
+                task_vars = delegated_vars[self.delegate_to_hostname]
                 if key in task_vars:
                     return task_vars[key]
-            else:
-                delegated_vars = task_vars['ansible_delegated_vars']
-                if self.delegate_to_hostname in delegated_vars:
-                    task_vars = delegated_vars[self.delegate_to_hostname]
-                    if key in task_vars:
-                        return task_vars[key]
 
         return default
 
@@ -628,15 +641,15 @@ def _spec_from_via(self, proxied_inventory_name, via_spec):
 
         # must use __contains__ to avoid a TypeError for a missing host on
         # Ansible 2.3.
-        if self.host_vars is None or inventory_name not in self.host_vars:
+        via_vars = self.get_host_vars(inventory_name)
+        if via_vars is None:
             raise ansible.errors.AnsibleConnectionFailure(
                 self.unknown_via_msg % (
                     via_spec,
                     proxied_inventory_name,
                 )
             )
 
-        via_vars = self.host_vars[inventory_name]
         return ansible_mitogen.transport_config.MitogenViaSpec(
             inventory_name=inventory_name,
             play_context=self._play_context,
@@ -712,7 +725,7 @@ def _build_stack(self):
             connection=self,
             play_context=self._play_context,
             transport=self.transport,
-            inventory_name=self.inventory_hostname,
+            inventory_name=self.get_task_var('inventory_hostname'),
         )
         stack = self._stack_from_spec(spec)
         return spec.inventory_name(), stack
@@ -778,18 +791,16 @@ def _connect(self):
 
         inventory_name, stack = self._build_stack()
         worker_model = ansible_mitogen.process.get_worker_model()
-        self.binding = worker_model.get_binding(inventory_name)
+        self.binding = worker_model.get_binding(
+            mitogen.utils.cast(inventory_name)
+        )
         self._connect_stack(stack)
 
-    def _mitogen_reset(self, mode):
+    def _put_connection(self):
         """
         Forget everything we know about the connected context. This function
         cannot be called _reset() since that name is used as a public API by
         Ansible 2.4 wait_for_connection plug-in.
-
-        :param str mode:
-            Name of ContextService method to use to discard the context, either
-            'put' or 'reset'.
         """
         if not self.context:
             return
@@ -798,7 +809,7 @@ def _mitogen_reset(self, mode):
         mitogen.service.call(
             call_context=self.binding.get_service_context(),
             service_name='ansible_mitogen.services.ContextService',
-            method_name=mode,
+            method_name='put',
             context=self.context
         )
 
@@ -813,24 +824,11 @@ def close(self):
         gracefully shut down, and wait for shutdown to complete. Safe to call
         multiple times.
         """
-        self._mitogen_reset(mode='put')
+        self._put_connection()
         if self.binding:
             self.binding.close()
             self.binding = None
 
-    def _reset_find_task_vars(self):
-        """
-        Monsterous hack: since "meta: reset_connection" does not run from an
-        action, we cannot capture task variables via :meth:`on_action_run`.
-        Instead walk the parent frames searching for the `all_vars` local from
-        StrategyBase._execute_meta(). If this fails, just leave task_vars
-        unset, likely causing a subtly wrong configuration to be selected.
-        """
-        frame = sys._getframe()
-        while frame and not self._task_vars:
-            self._task_vars = frame.f_locals.get('all_vars')
-            frame = frame.f_back
-
     reset_compat_msg = (
         'Mitogen only supports "reset_connection" on Ansible 2.5.6 or later'
     )
@@ -842,20 +840,31 @@ def reset(self):
         the 'disconnected' state, and informs ContextService the connection is
         bad somehow, and should be shut down and discarded.
         """
-        if self._task_vars is None:
-            self._reset_find_task_vars()
-
         if self._play_context.remote_addr is None:
             # <2.5.6 incorrectly populate PlayContext for reset_connection
             # https://github.com/ansible/ansible/issues/27520
             raise ansible.errors.AnsibleConnectionFailure(
                 self.reset_compat_msg
             )
 
-        self._connect()
-        self._mitogen_reset(mode='reset')
-        self.binding.close()
-        self.binding = None
+        # Clear out state in case we were ever connected.
+        self.close()
+
+        inventory_name, stack = self._build_stack()
+        if self._play_context.become:
+            stack = stack[:-1]
+
+        worker_model = ansible_mitogen.process.get_worker_model()
+        binding = worker_model.get_binding(inventory_name)
+        try:
+            mitogen.service.call(
+                call_context=binding.get_service_context(),
+                service_name='ansible_mitogen.services.ContextService',
+                method_name='reset',
+                stack=mitogen.utils.cast(list(stack)),
+            )
+        finally:
+            binding.close()
 
     # Compatibility with Ansible 2.4 wait_for_connection plug-in.
     _reset = reset

ansible_mitogen/loaders.py

@@ -55,3 +55,8 @@
     from ansible.plugins import module_utils_loader
     from ansible.plugins import shell_loader
     from ansible.plugins import strategy_loader
+
+
+# These are original, unwrapped implementations
+action_loader__get = action_loader.get
+connection_loader__get = connection_loader.get

ansible_mitogen/logging.py

@@ -107,8 +107,9 @@ def setup():
     l_mitogen = logging.getLogger('mitogen')
     l_mitogen_io = logging.getLogger('mitogen.io')
     l_ansible_mitogen = logging.getLogger('ansible_mitogen')
+    l_operon = logging.getLogger('operon')
 
-    for logger in l_mitogen, l_mitogen_io, l_ansible_mitogen:
+    for logger in l_mitogen, l_mitogen_io, l_ansible_mitogen, l_operon:
         logger.handlers = [Handler(display.vvv)]
         logger.propagate = False
 

ansible_mitogen/mixins.py

@@ -55,6 +55,11 @@
 import ansible_mitogen.target
 from ansible.module_utils._text import to_text
 
+try:
+    from ansible.utils.unsafe_proxy import wrap_var
+except ImportError:
+    from ansible.vars.unsafe_proxy import wrap_var
+
 
 LOG = logging.getLogger(__name__)
 
@@ -306,7 +311,7 @@ def get_task_timeout_secs(self):
         except AttributeError:
             return getattr(self._task, 'async')
 
-    def _temp_file_gibberish(self, module_args, wrap_async):
+    def _set_temp_file_args(self, module_args, wrap_async):
         # Ansible>2.5 module_utils reuses the action's temporary directory if
         # one exists. Older versions error if this key is present.
         if ansible.__version__ > '2.5':
@@ -343,7 +348,7 @@ def _execute_module(self, module_name=None, module_args=None, tmp=None,
         self._update_module_args(module_name, module_args, task_vars)
         env = {}
         self._compute_environment_string(env)
-        self._temp_file_gibberish(module_args, wrap_async)
+        self._set_temp_file_args(module_args, wrap_async)
 
         self._connection._connect()
         result = ansible_mitogen.planner.invoke(
@@ -365,7 +370,7 @@ def _execute_module(self, module_name=None, module_args=None, tmp=None,
             # on _execute_module().
             self._remove_tmp_path(tmp)
 
-        return result
+        return wrap_var(result)
 
     def _postprocess_response(self, result):
         """