Backport 0.3.x CI fixes and unofficial tox tests

Author: moreati

.ci/ansible_tests.py

@@ -37,9 +37,6 @@ def pause_if_interactive():
 
 
 with ci_lib.Fold('job_setup'):
-    # Don't set -U as that will upgrade Paramiko to a non-2.6 compatible version.
-    run("pip install -q ansible==%s", ci_lib.ANSIBLE_VERSION)
-
     os.chdir(TESTS_DIR)
     os.chmod('../data/docker/mitogen__has_sudo_pubkey.key', int('0600', 7))
 
@@ -69,8 +66,6 @@ def pause_if_interactive():
         run("sudo apt-get update")
         run("sudo apt-get install -y sshpass")
 
-    run("bash -c 'sudo ln -vfs /usr/lib/python2.7/plat-x86_64-linux-gnu/_sysconfigdata_nd.py /usr/lib/python2.7 || true'")
-    run("bash -c 'sudo ln -vfs /usr/lib/python2.7/plat-x86_64-linux-gnu/_sysconfigdata_nd.py $VIRTUAL_ENV/lib/python2.7 || true'")
 
 with ci_lib.Fold('ansible'):
     playbook = os.environ.get('PLAYBOOK', 'all.yml')

.ci/localhost_ansible_tests.py

@@ -20,9 +20,6 @@
 
 
 with ci_lib.Fold('job_setup'):
-    # Don't set -U as that will upgrade Paramiko to a non-2.6 compatible version.
-    run("pip install -q virtualenv ansible==%s", ci_lib.ANSIBLE_VERSION)
-
     os.chmod(KEY_PATH, int('0600', 8))
     if not ci_lib.exists_in_path('sshpass'):
         run("brew install http://git.io/sshpass.rb")

tests/ansible/integration/playbook_semantics/delegate_to.yml

@@ -51,17 +51,23 @@
       shell: whoami > /tmp/delegate_to.yml.txt
       delegate_to: localhost
       become: true
+      tags:
+        - requires_local_sudo
 
     - name: "delegate_to, sudo"
       assert:
         that: "lookup('file', '/tmp/delegate_to.yml.txt') == 'root'"
+      tags:
+        - requires_local_sudo
 
     - name: "delegate_to, sudo"
       file:
         path: /tmp/delegate_to.yml.txt
         state: absent
       delegate_to: localhost
       become: true
+      tags:
+        - requires_local_sudo
 
 
     #
@@ -71,14 +77,20 @@
       shell: whoami > /tmp/delegate_to.yml.txt
       connection: local
       become: true
+      tags:
+        - requires_local_sudo
 
     - name: "connection:local, sudo"
       assert:
         that: "lookup('file', '/tmp/delegate_to.yml.txt') == 'root'"
+      tags:
+        - requires_local_sudo
 
     - name: "connection:local, sudo"
       file:
         path: /tmp/delegate_to.yml.txt
         state: absent
       connection: local
       become: true
+      tags:
+        - requires_local_sudo

tests/ansible/regression/issue_152__local_action_wrong_interpreter.yml

@@ -15,7 +15,7 @@
         content: |
           #!/bin/bash
           export CUSTOM_INTERPRETER=1
-          exec python2.7 "$@"
+          exec python "$@"
 
     - custom_python_detect_environment:
       vars:

tests/ansible/requirements.txt

@@ -1,4 +1,4 @@
 paramiko==2.3.2  # Last 2.6-compat version.
 hdrhistogram==0.6.1
 PyYAML==3.11; python_version < '2.7'
-PyYAML==3.13; python_version >= '2.7'
+PyYAML==5.3.1; python_version >= '2.7'  # Latest release (Jan 2021)

tests/requirements.txt

@@ -3,7 +3,7 @@ coverage==4.5.1
 Django==1.6.11  # Last version supporting 2.6.
 mock==2.0.0
 pytz==2018.5
-cffi==1.11.2  # Random pin to try and fix pyparser==2.18 not having effect
+cffi==1.14.3  # Random pin to try and fix pyparser==2.18 not having effect
 pycparser==2.18  # Last version supporting 2.6.
 faulthandler==3.1; python_version < '3.3'  # used by testlib
 pytest-catchlog==1.2.2

tox.ini

@@ -1,44 +1,89 @@
+# This file is a local convenience. It is not a substitute for the full CI
+# suite, and does not cover the full range of Python versions for Mitogen.
+
+# I use this on Ubuntu 20.04, with the following additions
+#
+#   sudo add-apt-repository ppa:deadsnakes/ppa
+#   sudo apt update
+#   sudo apt install python3.5 python3.6 python3.7 python3.9 tox libsasl2-dev libldap2-dev libssl-dev ssh-pass
+
+# Last version to support each python version
+#
+#            tox      vir'env  pip      ansible  coverage
+# ========== ======== ======== ======== ======== ========
+# python2.4  1.4      1.8      1.1      ???
+# python2.5  1.6.1    1.9.1    1.3.1    ???
+# python2.6  2.9.1    15.2.0   9.0.3    2.6.20   4.5.4
+
 [tox]
 envlist =
     init,
-    py26,
-    py27,
-    py35,
-    py36,
-    py37,
+    py{27,36,39}-mode_ansible,
+    py{27,36,39}-mode_mitogen,
+    py{27,36,39}-mode_mitogen-distro_centos7,
     report,
+requires =
+    tox-factor
 
 [testenv]
-usedevelop = True
-deps =
-    -r{toxinidir}/dev_requirements.txt
-    -r{toxinidir}/tests/ansible/requirements.txt
-
+basepython =
+    py26: python2.6
+    py27: python2.7
+    py36: python3.6
+    py37: python3.7
+    py38: python3.8
+    py39: python3.9
+commands_pre =
+    mode_ansible: {toxinidir}/.ci/ansible_install.py
+    mode_debops_common: {toxinidir}/.ci/debops_common_install.py
+    mode_mitogen: {toxinidir}/.ci/mitogen_install.py
 commands =
-    {posargs:bash run_tests}
-whitelist_externals =
-    bash
+    mode_ansible: {toxinidir}/.ci/ansible_tests.py
+    mode_debops_common: {toxinidir}/.ci/debops_common_tests.py
+    mode_mitogen: {toxinidir}/.ci/mitogen_tests.py
+passenv =
+    ANSIBLE_*
+    HOME
 setenv =
+    ANSIBLE_SKIP_TAGS = requires_local_sudo
     NOCOVERAGE_ERASE = 1
     NOCOVERAGE_REPORT = 1
+    ansible2.3: VER=2.3.3.0
+    ansible2.4: VER=2.4.6.0
+    ansible2.8: VER=2.8.3
+    ansible2.9: VER=2.9.6
+    ansible2.10: VER=2.10.0
+    distro_centos5: DISTRO=centos5
+    distro_centos6: DISTRO=centos6
+    distro_centos7: DISTRO=centos7
+    distro_debian: DISTRO=debian
+    distro_debianpy3: DISTRO=debian-py3
+    distros_centos5: DISTROS=centos5
+    distros_debian: DISTROS=debian
+    mode_ansible: MODE=ansible
+    mode_debops_common: MODE=debops_common
+    mode_mitogen: MODE=mitogen
+    strategy_linear: STRATEGY=linear
 
 [testenv:init]
+basepython = python3
 commands =
     coverage erase
 deps =
-    coverage
+    coverage==4.5.4
 
 [testenv:report]
+basepython = python3
 commands =
     coverage html
     echo "coverage report is at file://{toxinidir}/htmlcov/index.html"
 deps =
-    coverage
+    coverage==4.5.4
 whitelist_externals =
     echo
 
 [testenv:docs]
-basepython = python
+basepython = python3
 changedir = docs
 commands =
     sphinx-build -W -b html -d {envtmpdir}/doctrees . {envtmpdir}/html