interpreter discovery trials

ansible_mitogen/mixins.py

@@ -470,30 +470,27 @@ def _low_level_execute_command(self, cmd, sudoable=True, in_data=None,
         # chicken-and-egg issue, mitogen needs a python to run low_level_execute_command
         # which is required by Ansible's discover_interpreter function
         if self._mitogen_discovering_interpreter:
-            possible_pythons = [
-                '/usr/bin/python',
-                'python3',
-                'python3.7',
-                'python3.6',
-                'python3.5',
-                'python2.7',
-                'python2.6',
-                '/usr/libexec/platform-python',
-                '/usr/bin/python3',
-                'python'
-            ]
+            if self._connection.transport in {'ssh'}:
+                possible_pythons = [
+                    '$(for p in python3 python2 python; do command -v "$p" 2>/dev/null && break; done;)',
+                ]
+            else:
+                possible_pythons = ['python3' 'python2', 'python']
         else:
             # not used, just adding a filler value
             possible_pythons = ['python']
 
         for possible_python in possible_pythons:
             try:
+                LOG.debug('_low_level_execute_command(): trying possible_python=%r', possible_python)
                 self._mitogen_interpreter_candidate = possible_python
                 rc, stdout, stderr = self._connection.exec_command(
                     cmd, in_data, sudoable, mitogen_chdir=chdir,
                 )
+                LOG.debug('_low_level_execute_command(): got rc=%d, stdout=%r, stderr=%r', rc, stdout, stderr)
             # TODO: what exception is thrown?
-            except:
+            except BaseException as exc:
+                LOG.debug('%r._low_level_execute_command for possible_python=%r: %s, %r', self, possible_python, type(exc), exc)
                 # we've reached the last python attempted and failed
                 if possible_python == possible_pythons[-1]:
                     raise

ansible_mitogen/transport_config.py

@@ -73,11 +73,19 @@
 from ansible.module_utils.six import with_metaclass
 from ansible.module_utils.parsing.convert_bool import boolean
 
+import ansible_mitogen.utils
 import mitogen.core
 
 
 LOG = logging.getLogger(__name__)
 
+if ansible_mitogen.utils.ansible_version[:2] >= (2, 19):
+    _FALLBACK_INTERPRETER = ansible.executor.interpreter_discovery._FALLBACK_INTERPRETER
+elif ansible_mitogen.utils.ansible_version[:2] >= (2, 17):
+    _FALLBACK_INTERPRETER = u'/usr/bin/python3'
+else:
+    _FALLBACK_INTERPRETER = u'/usr/bin/python'
+
 
 def run_interpreter_discovery_if_necessary(s, task_vars, action, rediscover_python):
     """
@@ -107,7 +115,9 @@ def run_interpreter_discovery_if_necessary(s, task_vars, action, rediscover_pyth
             # blow away the discovered_interpreter_config cache and rediscover
             del task_vars['ansible_facts'][discovered_interpreter_config]
 
-        if discovered_interpreter_config not in task_vars['ansible_facts']:
+        try:
+            s = task_vars['ansible_facts'][discovered_interpreter_config]
+        except KeyError:
             action._mitogen_discovering_interpreter = True
             # fake pipelining so discover_interpreter can be happy
             action._connection.has_pipelining = True
@@ -121,8 +131,6 @@ def run_interpreter_discovery_if_necessary(s, task_vars, action, rediscover_pyth
             # cache discovered interpreter
             task_vars['ansible_facts'][discovered_interpreter_config] = s
             action._connection.has_pipelining = False
-        else:
-            s = task_vars['ansible_facts'][discovered_interpreter_config]
 
         # propagate discovered interpreter as fact
         action._discovered_interpreter_key = discovered_interpreter_config
@@ -144,9 +152,9 @@ def parse_python_path(s, task_vars, action, rediscover_python):
         s = 'auto'
 
     s = run_interpreter_discovery_if_necessary(s, task_vars, action, rediscover_python)
-    # if unable to determine python_path, fallback to '/usr/bin/python'
     if not s:
-        s = '/usr/bin/python'
+        s = _FALLBACK_INTERPRETER
+        # raise ValueError("Interpreter discovery failed, got: %r", s)
 
     return ansible.utils.shlex.shlex_split(s)
 
@@ -715,9 +723,6 @@ def port(self):
 
     def python_path(self, rediscover_python=False):
         s = self._host_vars.get('ansible_python_interpreter')
-        # #511, #536: executor/module_common.py::_get_shebang() hard-wires
-        # "/usr/bin/python" as the default interpreter path if no other
-        # interpreter is specified.
         return parse_python_path(
             s,
             task_vars=self._task_vars,

docs/index.rst

@@ -80,6 +80,10 @@ to your network topology**.
     :class: mitogen-right-150
 
 .. code::
+    import mitogen.master
+
+    broker = mitogen.master.Broker()
+    router = mitogen.master.Router(broker)
 
     bastion_host = router.ssh(
         hostname='jump-box.mycorp.com'
@@ -101,7 +105,9 @@ to your network topology**.
         container='billing0',
     )
 
-    internal_box.call(subprocess.check_call, ['./run-nightly-billing.py'])
+    internal_box.call(
+        subprocess.check_call, ['./nightly-billing.py'],
+    )
 
 The multiplexer also ensures the remote process is terminated if your Python
 program crashes, communication is lost, or the application code running in the

mitogen/parent.py

@@ -1462,7 +1462,7 @@ def get_python_argv(self):
             return self.options.python_path
         return [self.options.python_path]
 
-    def get_boot_command(self):
+    def _first_stage_base64(self):
         source = inspect.getsource(self._first_stage)
         source = textwrap.dedent('\n'.join(source.strip().split('\n')[2:]))
         source = source.replace('    ', ' ')
@@ -1472,17 +1472,23 @@ def get_boot_command(self):
                                 str(len(preamble_compressed)))
         compressed = zlib.compress(source.encode(), 9)
         encoded = binascii.b2a_base64(compressed).replace(b('\n'), b(''))
+        return encoded.decode('ascii')
 
+    def _bootstrap_argv(self):
         # Just enough to decode, decompress, and exec the first stage.
         # Priorities: wider compatibility, faster startup, shorter length.
         # `import os` here, instead of stage 1, to save a few bytes.
         # `sys.path=...` for https://github.com/python/cpython/issues/115911.
-        return self.get_python_argv() + [
+        return [
             '-c',
             'import sys;sys.path=[p for p in sys.path if p];import binascii,os,zlib;'
-            'exec(zlib.decompress(binascii.a2b_base64("%s")))' % (encoded.decode(),),
+            'exec(zlib.decompress(binascii.a2b_base64(sys.argv[1])))',
+            self._first_stage_base64(),
         ]
 
+    def get_boot_command(self):
+        return self.get_python_argv() + self._bootstrap_argv()
+
     def get_econtext_config(self):
         assert self.options.max_message_size is not None
         parent_ids = mitogen.parent_ids[:]
@@ -1518,7 +1524,7 @@ def _get_name(self):
 
     def start_child(self):
         args = self.get_boot_command()
-        LOG.debug('command line for %r: %s', self, Argv(args))
+        LOG.debug('command line for %r: %s', self, args)
         try:
             return self.create_child(args=args, **self.create_child_args)
         except OSError:

mitogen/ssh.py

@@ -296,10 +296,8 @@ def get_boot_command(self):
         if self.options.ssh_args:
             bits += self.options.ssh_args
         bits.append(self.options.hostname)
-        base = super(Connection, self).get_boot_command()
 
-        base_parts = []
-        for s in base:
-            val = s if s in self.SHLEX_IGNORE else shlex_quote(s).strip()
-            base_parts.append(val)
-        return bits + base_parts
+        # https://datatracker.ietf.org/doc/html/rfc4254#section-6.5
+        python_argv = self.get_python_argv()
+        bootstrap_argv = self._bootstrap_argv()
+        return bits + python_argv + [shlex_quote(s) for s in bootstrap_argv]

mitogen/sudo.py

@@ -32,6 +32,13 @@
 import logging
 import optparse
 import re
+import shlex
+
+try:
+    from shlex import quote as shlex_quote
+except ImportError:
+    from pipes import quote as shlex_quote
+
 
 import mitogen.core
 import mitogen.parent
@@ -256,8 +263,6 @@ def get_boot_command(self):
         # Note: sudo did not introduce long-format option processing until July
         # 2013, so even though we parse long-format options, supply short-form
         # to the sudo command.
-        boot_cmd = super(Connection, self).get_boot_command()
-
         bits = [self.options.sudo_path, '-u', self.options.username]
         if self.options.preserve_env:
             bits += ['-E']
@@ -270,25 +275,7 @@ def get_boot_command(self):
         if self.options.selinux_type:
             bits += ['-t', self.options.selinux_type]
 
-        # special handling for bash builtins
-        # TODO: more efficient way of doing this, at least
-        # it's only 1 iteration of boot_cmd to go through
-        source_found = False
-        for cmd in boot_cmd[:]:
-            # rip `source` from boot_cmd if it exists; sudo.py can't run this
-            # even with -i or -s options
-            # since we've already got our ssh command working we shouldn't
-            # need to source anymore
-            # couldn't figure out how to get this to work using sudo flags
-            if 'source' == cmd:
-                boot_cmd.remove(cmd)
-                source_found = True
-                continue
-            if source_found:
-                # remove words until we hit the python interpreter call
-                if not cmd.endswith('python'):
-                    boot_cmd.remove(cmd)
-                else:
-                    break
-
-        return bits + ['--'] + boot_cmd
+        python_argv = self.get_python_argv()
+        bootstrap_argv = self._bootstrap_argv()
+        boot_argv = python_argv + [shlex_quote(s) for s in bootstrap_argv]
+        return bits + ['--', 'sh', '-c', ' '.join(boot_argv)]

tests/data/stubs/stub-sudo.py

@@ -8,12 +8,14 @@
 os.environ['ORIGINAL_ARGV'] = json.dumps(sys.argv)
 os.environ['THIS_IS_STUB_SUDO'] = '1'
 
+rest_argv = sys.argv[sys.argv.index('--') + 1:]
+
 if os.environ.get('PREHISTORIC_SUDO'):
     # issue #481: old versions of sudo did in fact use execve, thus we must
     # have TTY handle preservation in core.py.
-    os.execv(sys.executable, sys.argv[sys.argv.index('--') + 1:])
+    os.execvp(rest_argv[0], rest_argv)
 else:
     # This must be a child process and not exec() since Mitogen replaces its
     # stderr descriptor, causing the last user of the slave PTY to close it,
     # resulting in the master side indicating EIO.
-    subprocess.check_call(sys.argv[sys.argv.index('--') + 1:])
+    subprocess.check_call(rest_argv)