Merge pull request #870 from moreati/ansible4

Support for Ansible 3 & 4

Author: moreati

.ci/azure-pipelines-steps.yml

@@ -13,7 +13,7 @@ steps:
 - script: python -mpip install tox
   displayName: Install tooling
 
-- script: tox -e $(tox.env)
+- script: tox -e "$(tox.env)"
   displayName: "Run tests"
   env:
     AWS_ACCESS_KEY_ID: $(AWS_ACCESS_KEY_ID)

.ci/azure-pipelines.yml

@@ -25,14 +25,14 @@ jobs:
         tox.env: py27-mode_mitogen
 
       # TODO: test python3, python3 tests are broken
-      Ans210_27:
+      Local_Py27:
         python.version: '2.7'
-        tox.env: py27-mode_localhost-ansible2.10
+        tox.env: py27-mode_localhost-ansible{2.10,3,4}
 
       # NOTE: this hangs when ran in Ubuntu 18.04
-      Vanilla_210_27:
+      Vanilla_Py27:
         python.version: '2.7'
-        tox.env: py27-mode_localhost-ansible2.10
+        tox.env: py27-mode_localhost-ansible{2.10,3,4}
         STRATEGY: linear
         ANSIBLE_SKIP_TAGS: resource_intensive
 
@@ -45,22 +45,17 @@ jobs:
   - template: azure-pipelines-steps.yml
   strategy:
     matrix:
-      Mito27Debian_27:
+      Mito_Py27:
         python.version: '2.7'
-        tox.env: py27-mode_mitogen-distro_debian9
+        tox.env: py27-mode_mitogen-distro_{centos6,centos7,centos8,debian9,debian10,debian11,ubuntu1604,ubuntu1804,ubuntu2004}
 
-      Mito36CentOS6_26:
+      Mito_Py36:
         python.version: '3.6'
-        tox.env: py36-mode_mitogen-distro_centos6
+        tox.env: py36-mode_mitogen-distro_{centos6,centos7,centos8,debian9,debian10,debian11,ubuntu1604,ubuntu1804,ubuntu2004}
 
-      Mito39Debian_27:
+      Mito_Py39:
         python.version: '3.9'
-        tox.env: py39-mode_mitogen-distro_debian9
-
-      #Py26CentOS7:
-        #python.version: '2.7'
-        #MODE: mitogen
-        #DISTRO: centos6
+        tox.env: py39-mode_mitogen-distro_{centos6,centos7,centos8,debian9,debian10,debian11,ubuntu1604,ubuntu1804,ubuntu2004}
 
       #DebOps_2460_27_27:
         #python.version: '2.7'
@@ -99,14 +94,14 @@ jobs:
         #DISTROS: debian
         #STRATEGY: linear
 
-      Ansible_210_27:
+      Ansible_Py27:
         python.version: '2.7'
-        tox.env: py27-mode_ansible-ansible2.10
+        tox.env: py27-mode_ansible-ansible{2.10,3,4}
 
-      Ansible_210_36:
+      Ansible_Py36:
         python.version: '3.6'
-        tox.env: py36-mode_ansible-ansible2.10
+        tox.env: py36-mode_ansible-ansible{2.10,3,4}
 
-      Ansible_210_39:
+      Ansible_Py39:
         python.version: '3.9'
-        tox.env: py39-mode_ansible-ansible2.10
+        tox.env: py39-mode_ansible-ansible{2.10,3,4}

.ci/ci_lib.py

@@ -22,6 +22,14 @@
     )
 )
 
+_print = print
+def print(*args, **kwargs):
+    file = kwargs.get('file', sys.stdout)
+    flush = kwargs.pop('flush', False)
+    _print(*args, **kwargs)
+    if flush:
+        file.flush()
+
 
 #
 # check_output() monkeypatch cutpasted from testlib.py
@@ -71,24 +79,22 @@ def _argv(s, *args):
 
 
 def run(s, *args, **kwargs):
-    """ Run a command, with arguments, and print timing information
+    """ Run a command, with arguments
 
     >>> rc = run('echo "%s %s"', 'foo', 'bar')
-    Running: ['/usr/bin/time', '--', 'echo', 'foo bar']
+    Running: ['echo', 'foo bar']
     foo bar
-    0.00user 0.00system 0:00.00elapsed ?%CPU (0avgtext+0avgdata 1964maxresident)k
-    0inputs+0outputs (0major+71minor)pagefaults 0swaps
-    Finished running: ['/usr/bin/time', '--', 'echo', 'foo bar']
+    Finished running: ['echo', 'foo bar']
     >>> rc
     0
     """
-    argv = ['/usr/bin/time', '--'] + _argv(s, *args)
-    print('Running: %s' % (argv,))
+    argv = _argv(s, *args)
+    print('Running: %s' % (argv,), flush=True)
     try:
         ret = subprocess.check_call(argv, **kwargs)
-        print('Finished running: %s' % (argv,))
+        print('Finished running: %s' % (argv,), flush=True)
     except Exception:
-        print('Exception occurred while running: %s' % (argv,))
+        print('Exception occurred while running: %s' % (argv,), file=sys.stderr, flush=True)
         raise
 
     return ret
@@ -155,7 +161,7 @@ def get_output(s, *args, **kwargs):
     'foo bar\n'
     """
     argv = _argv(s, *args)
-    print('Running: %s' % (argv,))
+    print('Running: %s' % (argv,), flush=True)
     return subprocess.check_output(argv, **kwargs)
 
 
@@ -368,12 +374,10 @@ def start_containers(containers):
 def verify_procs(hostname, old, new):
     oldpids = set(pid for pid, _ in old)
     if any(pid not in oldpids for pid, _ in new):
-        print('%r had stray processes running:' % (hostname,))
+        print('%r had stray processes running:' % (hostname,), file=sys.stderr, flush=True)
         for pid, line in new:
             if pid not in oldpids:
-                print('New process:', line)
-
-        print()
+                print('New process:', line, flush=True)
         return False
 
     return True
@@ -397,13 +401,10 @@ def check_stray_processes(old, containers=None):
 
 
 def dump_file(path):
-    print()
-    print('--- %s ---' % (path,))
-    print()
+    print('--- %s ---' % (path,), flush=True)
     with open(path, 'r') as fp:
-        print(fp.read().rstrip())
-    print('---')
-    print()
+        print(fp.read().rstrip(), flush=True)
+    print('---', flush=True)
 
 
 # SSH passes these through to the container when run interactively, causing

.ci/debops_common_tests.py

@@ -1,8 +1,6 @@
 #!/usr/bin/env python
 
-from __future__ import print_function
 import os
-import shutil
 import sys
 
 import ci_lib
@@ -60,11 +58,7 @@
             for container in containers
         )
 
-    print()
-    print(' echo --- ansible/inventory/hosts: ---')
-    ci_lib.run('cat ansible/inventory/hosts')
-    print('---')
-    print()
+    ci_lib.dump_file('ansible/inventory/hosts')
 
     # Now we have real host key checking, we need to turn it off
     os.environ['ANSIBLE_HOST_KEY_CHECKING'] = 'False'

.ci/localhost_ansible_tests.py

@@ -33,15 +33,19 @@
 
 with ci_lib.Fold('machine_prep'):
     # generate a new ssh key for localhost ssh
-    os.system("ssh-keygen -P '' -m pem -f ~/.ssh/id_rsa")
-    os.system("cat ~/.ssh/id_rsa.pub >> ~/.ssh/authorized_keys")
+    if not os.path.exists(os.path.expanduser("~/.ssh/id_rsa")):
+        os.system("ssh-keygen -P '' -m pem -f ~/.ssh/id_rsa")
+        os.system("cat ~/.ssh/id_rsa.pub >> ~/.ssh/authorized_keys")
+
     # also generate it for the sudo user
-    os.system("sudo ssh-keygen -P '' -m pem -f /var/root/.ssh/id_rsa")
-    os.system("sudo cat /var/root/.ssh/id_rsa.pub | sudo tee -a /var/root/.ssh/authorized_keys")
+    if os.system("sudo [ -f /var/root/.ssh/id_rsa ]") != 0:
+        os.system("sudo ssh-keygen -P '' -m pem -f /var/root/.ssh/id_rsa")
+        os.system("sudo cat /var/root/.ssh/id_rsa.pub | sudo tee -a /var/root/.ssh/authorized_keys")
+
     os.chmod(os.path.expanduser('~/.ssh'), int('0700', 8))
     os.chmod(os.path.expanduser('~/.ssh/authorized_keys'), int('0600', 8))
     # run chmod through sudo since it's owned by root
-    os.system('sudo chmod 600 /var/root/.ssh')
+    os.system('sudo chmod 700 /var/root/.ssh')
     os.system('sudo chmod 600 /var/root/.ssh/authorized_keys')
 
     if os.path.expanduser('~mitogen__user1') == '~mitogen__user1':

ansible_mitogen/loaders.py

@@ -45,7 +45,7 @@
 import ansible
 
 ANSIBLE_VERSION_MIN = (2, 10)
-ANSIBLE_VERSION_MAX = (2, 10)
+ANSIBLE_VERSION_MAX = (2, 11)
 
 NEW_VERSION_MSG = (
     "Your Ansible version (%s) is too recent. The most recent version\n"
@@ -79,7 +79,7 @@ def assert_supported_release():
 
     if v[:2] > ANSIBLE_VERSION_MAX:
         raise ansible.errors.AnsibleError(
-            NEW_VERSION_MSG % (ansible.__version__, ANSIBLE_VERSION_MAX)
+            NEW_VERSION_MSG % (v, ANSIBLE_VERSION_MAX)
         )
 
 

ansible_mitogen/transport_config.py

@@ -451,7 +451,7 @@ def private_key_file(self):
         return self._play_context.private_key_file
 
     def ssh_executable(self):
-        return self._play_context.ssh_executable
+        return C.config.get_config_value("ssh_executable", plugin_type="connection", plugin_name="ssh", variables=self._task_vars.get("vars", {}))
 
     def timeout(self):
         return self._play_context.timeout
@@ -467,9 +467,9 @@ def ssh_args(self):
         return [
             mitogen.core.to_text(term)
             for s in (
-                getattr(self._play_context, 'ssh_args', ''),
-                getattr(self._play_context, 'ssh_common_args', ''),
-                getattr(self._play_context, 'ssh_extra_args', '')
+                C.config.get_config_value("ssh_args", plugin_type="connection", plugin_name="ssh", variables=self._task_vars.get("vars", {})),
+                C.config.get_config_value("ssh_common_args", plugin_type="connection", plugin_name="ssh", variables=self._task_vars.get("vars", {})),
+                C.config.get_config_value("ssh_extra_args", plugin_type="connection", plugin_name="ssh", variables=self._task_vars.get("vars", {}))
             )
             for term in ansible.utils.shlex.shlex_split(s or '')
         ]
@@ -679,10 +679,7 @@ def private_key_file(self):
         )
 
     def ssh_executable(self):
-        return (
-            self._host_vars.get('ansible_ssh_executable') or
-            C.ANSIBLE_SSH_EXECUTABLE
-        )
+        return C.config.get_config_value("ssh_executable", plugin_type="connection", plugin_name="ssh", variables=self._task_vars.get("vars", {}))
 
     def timeout(self):
         # TODO: must come from PlayContext too.
@@ -699,22 +696,9 @@ def ssh_args(self):
         return [
             mitogen.core.to_text(term)
             for s in (
-                (
-                    self._host_vars.get('ansible_ssh_args') or
-                    getattr(C, 'ANSIBLE_SSH_ARGS', None) or
-                    os.environ.get('ANSIBLE_SSH_ARGS')
-                    # TODO: ini entry. older versions.
-                ),
-                (
-                    self._host_vars.get('ansible_ssh_common_args') or
-                    os.environ.get('ANSIBLE_SSH_COMMON_ARGS')
-                    # TODO: ini entry.
-                ),
-                (
-                    self._host_vars.get('ansible_ssh_extra_args') or
-                    os.environ.get('ANSIBLE_SSH_EXTRA_ARGS')
-                    # TODO: ini entry.
-                ),
+                C.config.get_config_value("ssh_args", plugin_type="connection", plugin_name="ssh", variables=self._task_vars.get("vars", {})),
+                C.config.get_config_value("ssh_common_args", plugin_type="connection", plugin_name="ssh", variables=self._task_vars.get("vars", {})),
+                C.config.get_config_value("ssh_extra_args", plugin_type="connection", plugin_name="ssh", variables=self._task_vars.get("vars", {}))
             )
             for term in ansible.utils.shlex.shlex_split(s)
             if s

docs/ansible_detailed.rst

@@ -145,9 +145,10 @@ Testimonials
 Noteworthy Differences
 ----------------------
 
-* Ansible 2.3-2.9 are supported along with Python 2.6, 2.7, 3.6 and 3.7. Verify
-  your installation is running one of these versions by checking ``ansible
-  --version`` output.
+* Mitogen 0.2.x supports Ansible 2.3-2.9; with Python 2.6, 2.7, or 3.6.
+  Mitogen 0.3.1+ supports Ansible 2.10, 3, and 4; with Python 2.7, or 3.6-3.9.
+  Verify your installation is running one of these versions by checking
+  ``ansible --version`` output.
 
 * The ``raw`` action executes as a regular Mitogen connection, which requires
   Python on the target, precluding its use for installing Python. This will be

docs/changelog.rst

@@ -21,6 +21,7 @@ To avail of fixes in an unreleased version, please download a ZIP file
 v0.3.1.dev0 (unreleased)
 ------------------------
 
+* :gh:issue:`834` Support for Ansible 3 and 4 (ansible-core 2.11)
 * :gh:issue:`869` Continuous Integration tests are now run with Tox
 * :gh:issue:`869` Continuous Integration tests now cover CentOS 6 & 8, Debian 9 & 11, Ubuntu 16.04 & 20.04
 * :gh:issue:`860` Add initial support for podman connection (w/o Ansible support yet)

setup.py

@@ -64,6 +64,7 @@ def long_description():
     zip_safe = False,
     classifiers = [
         'Environment :: Console',
+        'Frameworks :: Ansible',
         'Intended Audience :: System Administrators',
         'License :: OSI Approved :: BSD License',
         'Operating System :: MacOS :: MacOS X',