Merge pull request #1127 from moreati/import-cleanups

Consolidate backward compatibility imports and polyfills

Author: moreati

ansible_mitogen/affinity.py

@@ -83,7 +83,6 @@
 import os
 import struct
 
-import mitogen.core
 import mitogen.parent
 
 
@@ -265,7 +264,7 @@ def _mask_to_bytes(self, mask):
         for x in range(16):
             chunks.append(struct.pack('<Q', mask & shiftmask))
             mask >>= 64
-        return mitogen.core.b('').join(chunks)
+        return b''.join(chunks)
 
     def _get_thread_ids(self):
         try:

ansible_mitogen/logging.py

@@ -32,15 +32,13 @@
 import logging
 import os
 
+import ansible.utils.display
+
 import mitogen.core
 import mitogen.utils
 
-try:
-    from __main__ import display
-except ImportError:
-    import ansible.utils.display
-    display = ansible.utils.display.Display()
 
+display = ansible.utils.display.Display()
 
 #: The process name set via :func:`set_process_name`.
 _process_name = None

ansible_mitogen/mixins.py

@@ -35,18 +35,16 @@
 import random
 import traceback
 
-try:
-    from shlex import quote as shlex_quote
-except ImportError:
-    from pipes import quote as shlex_quote
-
-from ansible.module_utils._text import to_bytes
-from ansible.parsing.utils.jsonify import jsonify
-
 import ansible
 import ansible.constants
 import ansible.plugins
 import ansible.plugins.action
+import ansible.utils.unsafe_proxy
+import ansible.vars.clean
+
+from ansible.module_utils.common.text.converters import to_bytes, to_text
+from ansible.module_utils.six.moves import shlex_quote
+from ansible.parsing.utils.jsonify import jsonify
 
 import mitogen.core
 import mitogen.select
@@ -57,24 +55,6 @@
 import ansible_mitogen.utils
 import ansible_mitogen.utils.unsafe
 
-from ansible.module_utils._text import to_text
-
-try:
-    from ansible.utils.unsafe_proxy import wrap_var
-except ImportError:
-    from ansible.vars.unsafe_proxy import wrap_var
-
-try:
-    # ansible 2.8 moved remove_internal_keys to the clean module
-    from ansible.vars.clean import remove_internal_keys
-except ImportError:
-    try:
-        from ansible.vars.manager import remove_internal_keys
-    except ImportError:
-        # ansible 2.3.3 has remove_internal_keys as a protected func on the action class
-        # we'll fallback to calling self._remove_internal_keys in this case
-        remove_internal_keys = lambda a: "Not found"
-
 
 LOG = logging.getLogger(__name__)
 
@@ -413,10 +393,7 @@ def _execute_module(self, module_name=None, module_args=None, tmp=None,
             self._remove_tmp_path(tmp)
 
         # prevents things like discovered_interpreter_* or ansible_discovered_interpreter_* from being set
-        # handle ansible 2.3.3 that has remove_internal_keys in a different place
-        check = remove_internal_keys(result)
-        if check == 'Not found':
-            self._remove_internal_keys(result)
+        ansible.vars.clean.remove_internal_keys(result)
 
         # taken from _execute_module of ansible 2.8.6
         # propagate interpreter discovery results back to the controller
@@ -440,7 +417,7 @@ def _execute_module(self, module_name=None, module_args=None, tmp=None,
                 result['deprecations'] = []
             result['deprecations'].extend(self._discovery_deprecation_warnings)
 
-        return wrap_var(result)
+        return ansible.utils.unsafe_proxy.wrap_var(result)
 
     def _postprocess_response(self, result):
         """

ansible_mitogen/planner.py

@@ -477,7 +477,7 @@ def read_file(path):
     finally:
         os.close(fd)
 
-    return mitogen.core.b('').join(bits)
+    return b''.join(bits)
 
 
 def _propagate_deps(invocation, planner, context):

ansible_mitogen/plugins/connection/mitogen_local.py

@@ -42,13 +42,7 @@
 import ansible_mitogen.connection
 import ansible_mitogen.process
 
-
-if sys.version_info > (3,):
-    viewkeys = dict.keys
-elif sys.version_info > (2, 7):
-    viewkeys = dict.viewkeys
-else:
-    viewkeys = lambda dct: set(dct)
+viewkeys = getattr(dict, 'viewkeys', dict.keys)
 
 
 def dict_diff(old, new):

ansible_mitogen/process.py

@@ -61,10 +61,9 @@
 import ansible
 import ansible.constants as C
 import ansible.errors
+
 import ansible_mitogen.logging
 import ansible_mitogen.services
-
-from mitogen.core import b
 import ansible_mitogen.affinity
 
 
@@ -639,7 +638,7 @@ def worker_main(self):
 
         try:
             # Let the parent know our listening socket is ready.
-            mitogen.core.io_op(self.model.child_sock.send, b('1'))
+            mitogen.core.io_op(self.model.child_sock.send, b'1')
             # Block until the socket is closed, which happens on parent exit.
             mitogen.core.io_op(self.model.child_sock.recv, 1)
         finally:

ansible_mitogen/runner.py

@@ -40,7 +40,9 @@
 __metaclass__ = type
 
 import atexit
+import ctypes
 import json
+import logging
 import os
 import re
 import shlex
@@ -50,19 +52,12 @@
 import traceback
 import types
 
+from ansible.module_utils.six.moves import shlex_quote
+
 import mitogen.core
 import ansible_mitogen.target  # TODO: circular import
-from mitogen.core import b
-from mitogen.core import bytes_partition
-from mitogen.core import str_rpartition
 from mitogen.core import to_text
 
-try:
-    import ctypes
-except ImportError:
-    # Python 2.4
-    ctypes = None
-
 try:
     # Python >= 3.4, PEP 451 ModuleSpec API
     import importlib.machinery
@@ -77,15 +72,6 @@
 except ImportError:
     from io import StringIO
 
-try:
-    from shlex import quote as shlex_quote
-except ImportError:
-    from pipes import quote as shlex_quote
-
-# Absolute imports for <2.5.
-logging = __import__('logging')
-
-
 # Prevent accidental import of an Ansible module from hanging on stdin read.
 import ansible.module_utils.basic
 ansible.module_utils.basic._ANSIBLE_ARGS = '{}'
@@ -95,15 +81,13 @@
 # explicit call to res_init() on each task invocation. BSD-alikes export it
 # directly, Linux #defines it as "__res_init".
 libc__res_init = None
-if ctypes:
-    libc = ctypes.CDLL(None)
-    for symbol in 'res_init', '__res_init':
-        try:
-            libc__res_init = getattr(libc, symbol)
-        except AttributeError:
-            pass
+libc = ctypes.CDLL(None)
+for symbol in 'res_init', '__res_init':
+    try:
+        libc__res_init = getattr(libc, symbol)
+    except AttributeError:
+        pass
 
-iteritems = getattr(dict, 'iteritems', dict.items)
 LOG = logging.getLogger(__name__)
 
 
@@ -217,13 +201,13 @@ def _parse(self, fp):
         for line in fp:
             # '   #export foo=some var  ' -> ['#export', 'foo=some var  ']
             bits = shlex_split_b(line)
-            if (not bits) or bits[0].startswith(b('#')):
+            if (not bits) or bits[0].startswith(b'#'):
                 continue
 
-            if bits[0] == b('export'):
+            if bits[0] == b'export':
                 bits.pop(0)
 
-            key, sep, value = bytes_partition(b(' ').join(bits), b('='))
+            key, sep, value = b' '.join(bits).partition(b'=')
             if key and sep:
                 yield key, value
 
@@ -596,7 +580,7 @@ def load_module(self, fullname):
             mod.__path__ = []
             mod.__package__ = str(fullname)
         else:
-            mod.__package__ = str(str_rpartition(to_text(fullname), '.')[0])
+            mod.__package__ = str(to_text(fullname).rpartition('.')[0])
         exec(code, mod.__dict__)
         self._loaded.add(fullname)
         return mod
@@ -611,7 +595,7 @@ class TemporaryEnvironment(object):
     def __init__(self, env=None):
         self.original = dict(os.environ)
         self.env = env or {}
-        for key, value in iteritems(self.env):
+        for key, value in mitogen.core.iteritems(self.env):
             key = mitogen.core.to_text(key)
             value = mitogen.core.to_text(value)
             if value is None:
@@ -819,7 +803,7 @@ def __init__(self, interpreter_fragment, is_python, **kwargs):
         self.interpreter_fragment = interpreter_fragment
         self.is_python = is_python
 
-    b_ENCODING_STRING = b('# -*- coding: utf-8 -*-')
+    b_ENCODING_STRING = b'# -*- coding: utf-8 -*-'
 
     def _get_program(self):
         return self._rewrite_source(
@@ -852,13 +836,13 @@ def _rewrite_source(self, s):
         # While Ansible rewrites the #! using ansible_*_interpreter, it is
         # never actually used to execute the script, instead it is a shell
         # fragment consumed by shell/__init__.py::build_module_command().
-        new = [b('#!') + utf8(self.interpreter_fragment)]
+        new = [b'#!' + utf8(self.interpreter_fragment)]
         if self.is_python:
             new.append(self.b_ENCODING_STRING)
 
-        _, _, rest = bytes_partition(s, b('\n'))
+        _, _, rest = s.partition(b'\n')
         new.append(rest)
-        return b('\n').join(new)
+        return b'\n'.join(new)
 
 
 class NewStyleRunner(ScriptRunner):
@@ -971,16 +955,15 @@ def _setup_args(self):
     # change the default encoding. This hack was removed from Ansible long ago,
     # but not before permeating into many third party modules.
     PREHISTORIC_HACK_RE = re.compile(
-        b(r'reload\s*\(\s*sys\s*\)\s*'
-          r'sys\s*\.\s*setdefaultencoding\([^)]+\)')
+        br'reload\s*\(\s*sys\s*\)\s*sys\s*\.\s*setdefaultencoding\([^)]+\)',
     )
 
     def _setup_program(self):
         source = ansible_mitogen.target.get_small_file(
             context=self.service_context,
             path=self.path,
         )
-        self.source = self.PREHISTORIC_HACK_RE.sub(b(''), source)
+        self.source = self.PREHISTORIC_HACK_RE.sub(b'', source)
 
     def _get_code(self):
         try:
@@ -998,7 +981,7 @@ def _get_code(self):
     if mitogen.core.PY3:
         main_module_name = '__main__'
     else:
-        main_module_name = b('__main__')
+        main_module_name = b'__main__'
 
     def _handle_magic_exception(self, mod, exc):
         """
@@ -1030,7 +1013,7 @@ def _get_module_package(self):
         approximation of the original package hierarchy, so that relative
         imports function correctly.
         """
-        pkg, sep, modname = str_rpartition(self.py_module_name, '.')
+        pkg, sep, _ = self.py_module_name.rpartition('.')
         if not sep:
             return None
         if mitogen.core.PY3:
@@ -1073,7 +1056,7 @@ def _run(self):
 
 
 class JsonArgsRunner(ScriptRunner):
-    JSON_ARGS = b('<<INCLUDE_ANSIBLE_MODULE_JSON_ARGS>>')
+    JSON_ARGS = b'<<INCLUDE_ANSIBLE_MODULE_JSON_ARGS>>'
 
     def _get_args_contents(self):
         return json.dumps(self.args).encode()

ansible_mitogen/services.py

@@ -50,6 +50,8 @@
 
 import ansible.constants
 
+from ansible.module_utils.six import reraise
+
 import mitogen.core
 import mitogen.service
 import ansible_mitogen.loaders
@@ -66,20 +68,6 @@
 ansible_mitogen.loaders.shell_loader.get('sh')
 
 
-if sys.version_info[0] == 3:
-    def reraise(tp, value, tb):
-        if value is None:
-            value = tp()
-        if value.__traceback__ is not tb:
-            raise value.with_traceback(tb)
-        raise value
-else:
-    exec(
-        "def reraise(tp, value, tb=None):\n"
-        "    raise tp, value, tb\n"
-     )
-
-
 def _get_candidate_temp_dirs():
     try:
         # >=2.5